walk.go 8.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307
  1. /* Copyright 2018 The Bazel Authors. All rights reserved.
  2. Licensed under the Apache License, Version 2.0 (the "License");
  3. you may not use this file except in compliance with the License.
  4. You may obtain a copy of the License at
  5. http://www.apache.org/licenses/LICENSE-2.0
  6. Unless required by applicable law or agreed to in writing, software
  7. distributed under the License is distributed on an "AS IS" BASIS,
  8. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  9. See the License for the specific language governing permissions and
  10. limitations under the License.
  11. */
  12. package walk
  13. import (
  14. "io/ioutil"
  15. "log"
  16. "os"
  17. "path"
  18. "path/filepath"
  19. "strings"
  20. "github.com/bazelbuild/bazel-gazelle/internal/config"
  21. "github.com/bazelbuild/bazel-gazelle/internal/pathtools"
  22. "github.com/bazelbuild/bazel-gazelle/internal/rule"
  23. )
  24. // Mode determines which directories Walk visits and which directories
  25. // should be updated.
  26. type Mode int
  27. const (
  28. // In VisitAllUpdateSubdirsMode, Walk visits every directory in the
  29. // repository. The directories given to Walk and their subdirectories are
  30. // updated.
  31. VisitAllUpdateSubdirsMode Mode = iota
  32. // In VisitAllUpdateDirsMode, Walk visits every directory in the repository.
  33. // Only the directories given to Walk are updated (not their subdirectories).
  34. VisitAllUpdateDirsMode
  35. // In UpdateDirsMode, Walk only visits and updates directories given to Walk.
  36. // Build files in parent directories are read in order to produce a complete
  37. // configuration, but the callback is not called for parent directories.
  38. UpdateDirsMode
  39. )
  40. // WalkFunc is a callback called by Walk in each visited directory.
  41. //
  42. // dir is the absolute file system path to the directory being visited.
  43. //
  44. // rel is the relative slash-separated path to the directory from the
  45. // repository root. Will be "" for the repository root directory itself.
  46. //
  47. // c is the configuration for the current directory. This may have been
  48. // modified by directives in the directory's build file.
  49. //
  50. // update is true when the build file may be updated.
  51. //
  52. // f is the existing build file in the directory. Will be nil if there
  53. // was no file.
  54. //
  55. // subdirs is a list of base names of subdirectories within dir, not
  56. // including excluded files.
  57. //
  58. // regularFiles is a list of base names of regular files within dir, not
  59. // including excluded files.
  60. //
  61. // genFiles is a list of names of generated files, found by reading
  62. // "out" and "outs" attributes of rules in f.
  63. type WalkFunc func(dir, rel string, c *config.Config, update bool, f *rule.File, subdirs, regularFiles, genFiles []string)
  64. // Walk traverses the directory tree rooted at c.RepoRoot in depth-first order.
  65. //
  66. // Walk calls the Configure method on each configuration extension in cexts
  67. // in each directory in pre-order, whether a build file is present in the
  68. // directory or not. cexts must contain a walk.Configurer.
  69. //
  70. // Walk calls the callback wf in post-order.
  71. func Walk(c *config.Config, cexts []config.Configurer, dirs []string, mode Mode, wf WalkFunc) {
  72. knownDirectives := make(map[string]bool)
  73. for _, cext := range cexts {
  74. for _, d := range cext.KnownDirectives() {
  75. knownDirectives[d] = true
  76. }
  77. }
  78. symlinks := symlinkResolver{visited: []string{c.RepoRoot}}
  79. updateRels := buildUpdateRelMap(c.RepoRoot, dirs)
  80. var visit func(*config.Config, string, string, bool)
  81. visit = func(c *config.Config, dir, rel string, updateParent bool) {
  82. haveError := false
  83. // TODO: OPT: ReadDir stats all the files, which is slow. We just care about
  84. // names and modes, so we should use something like
  85. // golang.org/x/tools/internal/fastwalk to speed this up.
  86. files, err := ioutil.ReadDir(dir)
  87. if err != nil {
  88. log.Print(err)
  89. return
  90. }
  91. f, err := loadBuildFile(c, rel, dir, files)
  92. if err != nil {
  93. log.Print(err)
  94. haveError = true
  95. }
  96. c = configure(cexts, knownDirectives, c, rel, f)
  97. wc := getWalkConfig(c)
  98. var subdirs, regularFiles []string
  99. for _, fi := range files {
  100. base := fi.Name()
  101. switch {
  102. case base == "" || base[0] == '.' || base[0] == '_' || wc.isExcluded(rel, base):
  103. continue
  104. case fi.IsDir() || fi.Mode()&os.ModeSymlink != 0 && symlinks.follow(c, dir, rel, base):
  105. subdirs = append(subdirs, base)
  106. default:
  107. regularFiles = append(regularFiles, base)
  108. }
  109. }
  110. shouldUpdate := shouldUpdate(rel, mode, updateParent, updateRels)
  111. for _, sub := range subdirs {
  112. if subRel := path.Join(rel, sub); shouldVisit(subRel, mode, updateRels) {
  113. visit(c, filepath.Join(dir, sub), subRel, shouldUpdate)
  114. }
  115. }
  116. update := !haveError && !wc.ignore && shouldUpdate
  117. if shouldCall(rel, mode, updateRels) {
  118. genFiles := findGenFiles(wc, f)
  119. wf(dir, rel, c, update, f, subdirs, regularFiles, genFiles)
  120. }
  121. }
  122. visit(c, c.RepoRoot, "", false)
  123. }
  124. // buildUpdateRelMap builds a table of prefixes, used to determine which
  125. // directories to update and visit.
  126. //
  127. // root and dirs must be absolute, canonical file paths. Each entry in dirs
  128. // must be a subdirectory of root. The caller is responsible for checking this.
  129. //
  130. // buildUpdateRelMap returns a map from slash-separated paths relative to the
  131. // root directory ("" for the root itself) to a boolean indicating whether
  132. // the directory should be updated.
  133. func buildUpdateRelMap(root string, dirs []string) map[string]bool {
  134. relMap := make(map[string]bool)
  135. for _, dir := range dirs {
  136. rel, _ := filepath.Rel(root, dir)
  137. rel = filepath.ToSlash(rel)
  138. if rel == "." {
  139. rel = ""
  140. }
  141. i := 0
  142. for {
  143. next := strings.IndexByte(rel[i:], '/') + i
  144. if next-i < 0 {
  145. relMap[rel] = true
  146. break
  147. }
  148. prefix := rel[:next]
  149. relMap[prefix] = relMap[prefix] // set to false if not present
  150. i = next + 1
  151. }
  152. }
  153. return relMap
  154. }
  155. // shouldCall returns true if Walk should call the callback in the
  156. // directory rel.
  157. func shouldCall(rel string, mode Mode, updateRels map[string]bool) bool {
  158. return mode != UpdateDirsMode || updateRels[rel]
  159. }
  160. // shouldUpdate returns true if Walk should pass true to the callback's update
  161. // parameter in the directory rel. This indicates the build file should be
  162. // updated.
  163. func shouldUpdate(rel string, mode Mode, updateParent bool, updateRels map[string]bool) bool {
  164. return mode == VisitAllUpdateSubdirsMode && updateParent || updateRels[rel]
  165. }
  166. // shouldVisit returns true if Walk should visit the subdirectory rel.
  167. func shouldVisit(rel string, mode Mode, updateRels map[string]bool) bool {
  168. if mode != UpdateDirsMode {
  169. return true
  170. }
  171. _, ok := updateRels[rel]
  172. return ok
  173. }
  174. func loadBuildFile(c *config.Config, pkg, dir string, files []os.FileInfo) (*rule.File, error) {
  175. var err error
  176. readDir := dir
  177. readFiles := files
  178. if c.ReadBuildFilesDir != "" {
  179. readDir = filepath.Join(c.ReadBuildFilesDir, filepath.FromSlash(pkg))
  180. readFiles, err = ioutil.ReadDir(readDir)
  181. if err != nil {
  182. return nil, err
  183. }
  184. }
  185. path := rule.MatchBuildFileName(readDir, c.ValidBuildFileNames, readFiles)
  186. if path == "" {
  187. return nil, nil
  188. }
  189. return rule.LoadFile(path, pkg)
  190. }
  191. func configure(cexts []config.Configurer, knownDirectives map[string]bool, c *config.Config, rel string, f *rule.File) *config.Config {
  192. if rel != "" {
  193. c = c.Clone()
  194. }
  195. if f != nil {
  196. for _, d := range f.Directives {
  197. if !knownDirectives[d.Key] {
  198. log.Printf("%s: unknown directive: gazelle:%s", f.Path, d.Key)
  199. }
  200. }
  201. }
  202. for _, cext := range cexts {
  203. cext.Configure(c, rel, f)
  204. }
  205. return c
  206. }
  207. func findGenFiles(wc *walkConfig, f *rule.File) []string {
  208. if f == nil {
  209. return nil
  210. }
  211. var strs []string
  212. for _, r := range f.Rules {
  213. for _, key := range []string{"out", "outs"} {
  214. if s := r.AttrString(key); s != "" {
  215. strs = append(strs, s)
  216. } else if ss := r.AttrStrings(key); len(ss) > 0 {
  217. strs = append(strs, ss...)
  218. }
  219. }
  220. }
  221. var genFiles []string
  222. for _, s := range strs {
  223. if !wc.isExcluded(f.Pkg, s) {
  224. genFiles = append(genFiles, s)
  225. }
  226. }
  227. return genFiles
  228. }
  229. type symlinkResolver struct {
  230. visited []string
  231. }
  232. // Decide if symlink dir/base should be followed.
  233. func (r *symlinkResolver) follow(c *config.Config, dir, rel, base string) bool {
  234. if dir == c.RepoRoot && strings.HasPrefix(base, "bazel-") {
  235. // Links such as bazel-<workspace>, bazel-out, bazel-genfiles are created by
  236. // Bazel to point to internal build directories.
  237. return false
  238. }
  239. // See if the user has explicitly directed us to follow the link.
  240. wc := getWalkConfig(c)
  241. linkRel := path.Join(rel, base)
  242. for _, follow := range wc.follow {
  243. if linkRel == follow {
  244. return true
  245. }
  246. }
  247. // See if the symlink points to a tree that has been already visited.
  248. fullpath := filepath.Join(dir, base)
  249. dest, err := filepath.EvalSymlinks(fullpath)
  250. if err != nil {
  251. return false
  252. }
  253. if !filepath.IsAbs(dest) {
  254. dest, err = filepath.Abs(filepath.Join(dir, dest))
  255. if err != nil {
  256. return false
  257. }
  258. }
  259. for _, p := range r.visited {
  260. if pathtools.HasPrefix(dest, p) || pathtools.HasPrefix(p, dest) {
  261. return false
  262. }
  263. }
  264. r.visited = append(r.visited, dest)
  265. stat, err := os.Stat(fullpath)
  266. if err != nil {
  267. return false
  268. }
  269. return stat.IsDir()
  270. }