buildozer.go 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002
  1. /*
  2. Copyright 2016 Google Inc. All Rights Reserved.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. */
  13. // Buildozer is a tool for programatically editing BUILD files.
  14. package edit
  15. import (
  16. "bufio"
  17. "bytes"
  18. "errors"
  19. "fmt"
  20. "io"
  21. "io/ioutil"
  22. "log"
  23. "os"
  24. "os/exec"
  25. "path"
  26. "path/filepath"
  27. "regexp"
  28. "runtime"
  29. "strconv"
  30. "strings"
  31. apipb "github.com/bazelbuild/buildtools/api_proto"
  32. "github.com/bazelbuild/buildtools/build"
  33. "github.com/bazelbuild/buildtools/file"
  34. "github.com/golang/protobuf/proto"
  35. )
  36. // Options represents choices about how buildozer should behave.
  37. type Options struct {
  38. Stdout bool // write changed BUILD file to stdout
  39. Buildifier string // path to buildifier binary
  40. Parallelism int // number of cores to use for concurrent actions
  41. NumIO int // number of concurrent actions
  42. CommandsFile string // file name to read commands from, use '-' for stdin (format:|-separated command line arguments to buildozer, excluding flags
  43. KeepGoing bool // apply all commands, even if there are failures
  44. FilterRuleTypes []string // list of rule types to change, empty means all
  45. PreferEOLComments bool // when adding a new comment, put it on the same line if possible
  46. RootDir string // If present, use this folder rather than $PWD to find the root dir
  47. Quiet bool // suppress informational messages.
  48. EditVariables bool // for attributes that simply assign a variable (e.g. hdrs = LIB_HDRS), edit the build variable instead of appending to the attribute.
  49. IsPrintingProto bool // output serialized devtools.buildozer.Output protos instead of human-readable strings
  50. }
  51. // NewOpts returns a new Options struct with some defaults set.
  52. func NewOpts() *Options {
  53. return &Options{NumIO: 200, PreferEOLComments: true}
  54. }
  55. // Usage is a user-overriden func to print the program usage.
  56. var Usage = func() {}
  57. var fileModified = false // set to true when a file has been fixed
  58. const stdinPackageName = "-" // the special package name to represent stdin
  59. // CmdEnvironment stores the information the commands below have access to.
  60. type CmdEnvironment struct {
  61. File *build.File // the AST
  62. Rule *build.Rule // the rule to modify
  63. Vars map[string]*build.BinaryExpr // global variables set in the build file
  64. Pkg string // the full package name
  65. Args []string // the command-line arguments
  66. output *apipb.Output_Record // output proto, stores whatever a command wants to print
  67. }
  68. // The cmdXXX functions implement the various commands.
  69. func cmdAdd(opts *Options, env CmdEnvironment) (*build.File, error) {
  70. attr := env.Args[0]
  71. for _, val := range env.Args[1:] {
  72. if IsIntList(attr) {
  73. AddValueToListAttribute(env.Rule, attr, env.Pkg, &build.LiteralExpr{Token: val}, &env.Vars)
  74. continue
  75. }
  76. strVal := &build.StringExpr{Value: ShortenLabel(val, env.Pkg)}
  77. AddValueToListAttribute(env.Rule, attr, env.Pkg, strVal, &env.Vars)
  78. }
  79. return env.File, nil
  80. }
  81. func cmdComment(opts *Options, env CmdEnvironment) (*build.File, error) {
  82. // The comment string is always the last argument in the list.
  83. str := env.Args[len(env.Args)-1]
  84. str = strings.Replace(str, "\\n", "\n", -1)
  85. // Multiline comments should go on a separate line.
  86. fullLine := !opts.PreferEOLComments || strings.Contains(str, "\n")
  87. str = strings.Replace("# "+str, "\n", "\n# ", -1)
  88. comment := []build.Comment{{Token: str}}
  89. // The comment might be attached to a rule, an attribute, or a value in a list,
  90. // depending on how many arguments are passed.
  91. switch len(env.Args) {
  92. case 1: // Attach to a rule
  93. env.Rule.Call.Comments.Before = comment
  94. case 2: // Attach to an attribute
  95. if attr := env.Rule.AttrDefn(env.Args[0]); attr != nil {
  96. if fullLine {
  97. attr.X.Comment().Before = comment
  98. } else {
  99. attr.Y.Comment().Suffix = comment
  100. }
  101. }
  102. case 3: // Attach to a specific value in a list
  103. if attr := env.Rule.Attr(env.Args[0]); attr != nil {
  104. if expr := ListFind(attr, env.Args[1], env.Pkg); expr != nil {
  105. if fullLine {
  106. expr.Comments.Before = comment
  107. } else {
  108. expr.Comments.Suffix = comment
  109. }
  110. }
  111. }
  112. default:
  113. panic("cmdComment")
  114. }
  115. return env.File, nil
  116. }
  117. // commentsText concatenates comments into a single line.
  118. func commentsText(comments []build.Comment) string {
  119. var segments []string
  120. for _, comment := range comments {
  121. token := comment.Token
  122. if strings.HasPrefix(token, "#") {
  123. token = token[1:]
  124. }
  125. segments = append(segments, strings.TrimSpace(token))
  126. }
  127. return strings.Replace(strings.Join(segments, " "), "\n", " ", -1)
  128. }
  129. func cmdPrintComment(opts *Options, env CmdEnvironment) (*build.File, error) {
  130. attrError := func() error {
  131. return fmt.Errorf("rule \"//%s:%s\" has no attribute \"%s\"", env.Pkg, env.Rule.Name(), env.Args[0])
  132. }
  133. switch len(env.Args) {
  134. case 0: // Print rule comment.
  135. env.output.Fields = []*apipb.Output_Record_Field{
  136. {Value: &apipb.Output_Record_Field_Text{commentsText(env.Rule.Call.Comments.Before)}},
  137. }
  138. case 1: // Print attribute comment.
  139. attr := env.Rule.AttrDefn(env.Args[0])
  140. if attr == nil {
  141. return nil, attrError()
  142. }
  143. comments := append(attr.Before, attr.Suffix...)
  144. env.output.Fields = []*apipb.Output_Record_Field{
  145. {Value: &apipb.Output_Record_Field_Text{commentsText(comments)}},
  146. }
  147. case 2: // Print comment of a specific value in a list.
  148. attr := env.Rule.Attr(env.Args[0])
  149. if attr == nil {
  150. return nil, attrError()
  151. }
  152. value := env.Args[1]
  153. expr := ListFind(attr, value, env.Pkg)
  154. if expr == nil {
  155. return nil, fmt.Errorf("attribute \"%s\" has no value \"%s\"", env.Args[0], value)
  156. }
  157. comments := append(expr.Comments.Before, expr.Comments.Suffix...)
  158. env.output.Fields = []*apipb.Output_Record_Field{
  159. {Value: &apipb.Output_Record_Field_Text{commentsText(comments)}},
  160. }
  161. default:
  162. panic("cmdPrintComment")
  163. }
  164. return nil, nil
  165. }
  166. func cmdDelete(opts *Options, env CmdEnvironment) (*build.File, error) {
  167. return DeleteRule(env.File, env.Rule), nil
  168. }
  169. func cmdMove(opts *Options, env CmdEnvironment) (*build.File, error) {
  170. oldAttr := env.Args[0]
  171. newAttr := env.Args[1]
  172. if len(env.Args) == 3 && env.Args[2] == "*" {
  173. if err := MoveAllListAttributeValues(env.Rule, oldAttr, newAttr, env.Pkg, &env.Vars); err != nil {
  174. return nil, err
  175. }
  176. return env.File, nil
  177. }
  178. fixed := false
  179. for _, val := range env.Args[2:] {
  180. if deleted := ListAttributeDelete(env.Rule, oldAttr, val, env.Pkg); deleted != nil {
  181. AddValueToListAttribute(env.Rule, newAttr, env.Pkg, deleted, &env.Vars)
  182. fixed = true
  183. }
  184. }
  185. if fixed {
  186. return env.File, nil
  187. }
  188. return nil, nil
  189. }
  190. func cmdNew(opts *Options, env CmdEnvironment) (*build.File, error) {
  191. kind := env.Args[0]
  192. name := env.Args[1]
  193. addAtEOF, insertionIndex, err := findInsertionIndex(env)
  194. if err != nil {
  195. return nil, err
  196. }
  197. if FindRuleByName(env.File, name) != nil {
  198. return nil, fmt.Errorf("rule '%s' already exists", name)
  199. }
  200. call := &build.CallExpr{X: &build.LiteralExpr{Token: kind}}
  201. rule := &build.Rule{call, ""}
  202. rule.SetAttr("name", &build.StringExpr{Value: name})
  203. if addAtEOF {
  204. env.File.Stmt = InsertAfterLastOfSameKind(env.File.Stmt, rule.Call)
  205. } else {
  206. env.File.Stmt = InsertAfter(insertionIndex, env.File.Stmt, call)
  207. }
  208. return env.File, nil
  209. }
  210. // findInsertionIndex is used by cmdNew to find the place at which to insert the new rule.
  211. func findInsertionIndex(env CmdEnvironment) (bool, int, error) {
  212. if len(env.Args) < 4 {
  213. return true, 0, nil
  214. }
  215. relativeToRuleName := env.Args[3]
  216. ruleIdx, _ := IndexOfRuleByName(env.File, relativeToRuleName)
  217. if ruleIdx == -1 {
  218. return true, 0, nil
  219. }
  220. switch env.Args[2] {
  221. case "before":
  222. return false, ruleIdx - 1, nil
  223. case "after":
  224. return false, ruleIdx, nil
  225. default:
  226. return true, 0, fmt.Errorf("Unknown relative operator '%s'; allowed: 'before', 'after'", env.Args[1])
  227. }
  228. }
  229. func cmdNewLoad(opts *Options, env CmdEnvironment) (*build.File, error) {
  230. env.File.Stmt = InsertLoad(env.File.Stmt, env.Args)
  231. return env.File, nil
  232. }
  233. func cmdPrint(opts *Options, env CmdEnvironment) (*build.File, error) {
  234. format := env.Args
  235. if len(format) == 0 {
  236. format = []string{"name", "kind"}
  237. }
  238. fields := make([]*apipb.Output_Record_Field, len(format))
  239. for i, str := range format {
  240. value := env.Rule.Attr(str)
  241. if str == "kind" {
  242. fields[i] = &apipb.Output_Record_Field{Value: &apipb.Output_Record_Field_Text{env.Rule.Kind()}}
  243. } else if str == "name" {
  244. fields[i] = &apipb.Output_Record_Field{Value: &apipb.Output_Record_Field_Text{env.Rule.Name()}}
  245. } else if str == "label" {
  246. if env.Rule.Name() != "" {
  247. fields[i] = &apipb.Output_Record_Field{Value: &apipb.Output_Record_Field_Text{fmt.Sprintf("//%s:%s", env.Pkg, env.Rule.Name())}}
  248. } else {
  249. return nil, nil
  250. }
  251. } else if str == "rule" {
  252. fields[i] = &apipb.Output_Record_Field{
  253. Value: &apipb.Output_Record_Field_Text{build.FormatString(env.Rule.Call)},
  254. }
  255. } else if str == "startline" {
  256. fields[i] = &apipb.Output_Record_Field{Value: &apipb.Output_Record_Field_Number{int32(env.Rule.Call.ListStart.Line)}}
  257. } else if str == "endline" {
  258. fields[i] = &apipb.Output_Record_Field{Value: &apipb.Output_Record_Field_Number{int32(env.Rule.Call.End.Pos.Line)}}
  259. } else if value == nil {
  260. fmt.Fprintf(os.Stderr, "rule \"//%s:%s\" has no attribute \"%s\"\n",
  261. env.Pkg, env.Rule.Name(), str)
  262. fields[i] = &apipb.Output_Record_Field{Value: &apipb.Output_Record_Field_Error{Error: apipb.Output_Record_Field_MISSING}}
  263. } else if lit, ok := value.(*build.LiteralExpr); ok {
  264. fields[i] = &apipb.Output_Record_Field{Value: &apipb.Output_Record_Field_Text{lit.Token}}
  265. } else if string, ok := value.(*build.StringExpr); ok {
  266. fields[i] = &apipb.Output_Record_Field{
  267. Value: &apipb.Output_Record_Field_Text{string.Value},
  268. QuoteWhenPrinting: true,
  269. }
  270. } else if strList := env.Rule.AttrStrings(str); strList != nil {
  271. fields[i] = &apipb.Output_Record_Field{Value: &apipb.Output_Record_Field_List{List: &apipb.RepeatedString{Strings: strList}}}
  272. } else {
  273. // Some other Expr we haven't listed above. Just print it.
  274. fields[i] = &apipb.Output_Record_Field{Value: &apipb.Output_Record_Field_Text{build.FormatString(value)}}
  275. }
  276. }
  277. env.output.Fields = fields
  278. return nil, nil
  279. }
  280. func attrKeysForPattern(rule *build.Rule, pattern string) []string {
  281. if pattern == "*" {
  282. return rule.AttrKeys()
  283. }
  284. return []string{pattern}
  285. }
  286. func cmdRemove(opts *Options, env CmdEnvironment) (*build.File, error) {
  287. if len(env.Args) == 1 { // Remove the attribute
  288. if env.Rule.DelAttr(env.Args[0]) != nil {
  289. return env.File, nil
  290. }
  291. } else { // Remove values in the attribute.
  292. fixed := false
  293. for _, key := range attrKeysForPattern(env.Rule, env.Args[0]) {
  294. for _, val := range env.Args[1:] {
  295. ListAttributeDelete(env.Rule, key, val, env.Pkg)
  296. fixed = true
  297. }
  298. }
  299. if fixed {
  300. return env.File, nil
  301. }
  302. }
  303. return nil, nil
  304. }
  305. func cmdRename(opts *Options, env CmdEnvironment) (*build.File, error) {
  306. oldAttr := env.Args[0]
  307. newAttr := env.Args[1]
  308. if err := RenameAttribute(env.Rule, oldAttr, newAttr); err != nil {
  309. return nil, err
  310. }
  311. return env.File, nil
  312. }
  313. func cmdReplace(opts *Options, env CmdEnvironment) (*build.File, error) {
  314. oldV := env.Args[1]
  315. newV := env.Args[2]
  316. for _, key := range attrKeysForPattern(env.Rule, env.Args[0]) {
  317. attr := env.Rule.Attr(key)
  318. if e, ok := attr.(*build.StringExpr); ok {
  319. if LabelsEqual(e.Value, oldV, env.Pkg) {
  320. env.Rule.SetAttr(key, getAttrValueExpr(key, []string{newV}))
  321. }
  322. } else {
  323. ListReplace(attr, oldV, newV, env.Pkg)
  324. }
  325. }
  326. return env.File, nil
  327. }
  328. func cmdSubstitute(opts *Options, env CmdEnvironment) (*build.File, error) {
  329. oldRegexp, err := regexp.Compile(env.Args[1])
  330. if err != nil {
  331. return nil, err
  332. }
  333. newTemplate := env.Args[2]
  334. for _, key := range attrKeysForPattern(env.Rule, env.Args[0]) {
  335. attr := env.Rule.Attr(key)
  336. e, ok := attr.(*build.StringExpr)
  337. if !ok {
  338. ListSubstitute(attr, oldRegexp, newTemplate)
  339. continue
  340. }
  341. if newValue, ok := stringSubstitute(e.Value, oldRegexp, newTemplate); ok {
  342. env.Rule.SetAttr(key, getAttrValueExpr(key, []string{newValue}))
  343. }
  344. }
  345. return env.File, nil
  346. }
  347. func cmdSet(opts *Options, env CmdEnvironment) (*build.File, error) {
  348. attr := env.Args[0]
  349. args := env.Args[1:]
  350. if attr == "kind" {
  351. env.Rule.SetKind(args[0])
  352. } else {
  353. env.Rule.SetAttr(attr, getAttrValueExpr(attr, args))
  354. }
  355. return env.File, nil
  356. }
  357. func cmdSetIfAbsent(opts *Options, env CmdEnvironment) (*build.File, error) {
  358. attr := env.Args[0]
  359. args := env.Args[1:]
  360. if attr == "kind" {
  361. return nil, fmt.Errorf("setting 'kind' is not allowed for set_if_absent. Got %s", env.Args)
  362. }
  363. if env.Rule.Attr(attr) == nil {
  364. env.Rule.SetAttr(attr, getAttrValueExpr(attr, args))
  365. }
  366. return env.File, nil
  367. }
  368. func getAttrValueExpr(attr string, args []string) build.Expr {
  369. switch {
  370. case attr == "kind":
  371. return nil
  372. case IsIntList(attr):
  373. var list []build.Expr
  374. for _, i := range args {
  375. list = append(list, &build.LiteralExpr{Token: i})
  376. }
  377. return &build.ListExpr{List: list}
  378. case IsList(attr) && !(len(args) == 1 && strings.HasPrefix(args[0], "glob(")):
  379. var list []build.Expr
  380. for _, i := range args {
  381. list = append(list, &build.StringExpr{Value: i})
  382. }
  383. return &build.ListExpr{List: list}
  384. case IsString(attr):
  385. return &build.StringExpr{Value: args[0]}
  386. default:
  387. return &build.LiteralExpr{Token: args[0]}
  388. }
  389. }
  390. func cmdCopy(opts *Options, env CmdEnvironment) (*build.File, error) {
  391. attrName := env.Args[0]
  392. from := env.Args[1]
  393. return copyAttributeBetweenRules(env, attrName, from)
  394. }
  395. func cmdCopyNoOverwrite(opts *Options, env CmdEnvironment) (*build.File, error) {
  396. attrName := env.Args[0]
  397. from := env.Args[1]
  398. if env.Rule.Attr(attrName) != nil {
  399. return env.File, nil
  400. }
  401. return copyAttributeBetweenRules(env, attrName, from)
  402. }
  403. func copyAttributeBetweenRules(env CmdEnvironment, attrName string, from string) (*build.File, error) {
  404. fromRule := FindRuleByName(env.File, from)
  405. if fromRule == nil {
  406. return nil, fmt.Errorf("could not find rule '%s'", from)
  407. }
  408. attr := fromRule.Attr(attrName)
  409. if attr == nil {
  410. return nil, fmt.Errorf("rule '%s' does not have attribute '%s'", from, attrName)
  411. }
  412. ast, err := build.Parse("" /* filename */, []byte(build.FormatString(attr)))
  413. if err != nil {
  414. return nil, fmt.Errorf("could not parse attribute value %v", build.FormatString(attr))
  415. }
  416. env.Rule.SetAttr(attrName, ast.Stmt[0])
  417. return env.File, nil
  418. }
  419. func cmdFix(opts *Options, env CmdEnvironment) (*build.File, error) {
  420. // Fix the whole file
  421. if env.Rule.Kind() == "package" {
  422. return FixFile(env.File, env.Pkg, env.Args), nil
  423. }
  424. // Fix a specific rule
  425. return FixRule(env.File, env.Pkg, env.Rule, env.Args), nil
  426. }
  427. // CommandInfo provides a command function and info on incoming arguments.
  428. type CommandInfo struct {
  429. Fn func(*Options, CmdEnvironment) (*build.File, error)
  430. MinArg int
  431. MaxArg int
  432. Template string
  433. }
  434. // AllCommands associates the command names with their function and number
  435. // of arguments.
  436. var AllCommands = map[string]CommandInfo{
  437. "add": {cmdAdd, 2, -1, "<attr> <value(s)>"},
  438. "new_load": {cmdNewLoad, 1, -1, "<path> <symbol(s)>"},
  439. "comment": {cmdComment, 1, 3, "<attr>? <value>? <comment>"},
  440. "print_comment": {cmdPrintComment, 0, 2, "<attr>? <value>?"},
  441. "delete": {cmdDelete, 0, 0, ""},
  442. "fix": {cmdFix, 0, -1, "<fix(es)>?"},
  443. "move": {cmdMove, 3, -1, "<old_attr> <new_attr> <value(s)>"},
  444. "new": {cmdNew, 2, 4, "<rule_kind> <rule_name> [(before|after) <relative_rule_name>]"},
  445. "print": {cmdPrint, 0, -1, "<attribute(s)>"},
  446. "remove": {cmdRemove, 1, -1, "<attr> <value(s)>"},
  447. "rename": {cmdRename, 2, 2, "<old_attr> <new_attr>"},
  448. "replace": {cmdReplace, 3, 3, "<attr> <old_value> <new_value>"},
  449. "substitute": {cmdSubstitute, 3, 3, "<attr> <old_regexp> <new_template>"},
  450. "set": {cmdSet, 2, -1, "<attr> <value(s)>"},
  451. "set_if_absent": {cmdSetIfAbsent, 2, -1, "<attr> <value(s)>"},
  452. "copy": {cmdCopy, 2, 2, "<attr> <from_rule>"},
  453. "copy_no_overwrite": {cmdCopyNoOverwrite, 2, 2, "<attr> <from_rule>"},
  454. }
  455. func expandTargets(f *build.File, rule string) ([]*build.Rule, error) {
  456. if r := FindRuleByName(f, rule); r != nil {
  457. return []*build.Rule{r}, nil
  458. } else if r := FindExportedFile(f, rule); r != nil {
  459. return []*build.Rule{r}, nil
  460. } else if rule == "all" || rule == "*" {
  461. // "all" is a valid name, it is a wildcard only if no such rule is found.
  462. return f.Rules(""), nil
  463. } else if strings.HasPrefix(rule, "%") {
  464. // "%java_library" will match all java_library functions in the package
  465. // "%<LINENUM>" will match the rule which begins at LINENUM.
  466. // This is for convenience, "%" is not a valid character in bazel targets.
  467. kind := rule[1:]
  468. if linenum, err := strconv.Atoi(kind); err == nil {
  469. if r := f.RuleAt(linenum); r != nil {
  470. return []*build.Rule{r}, nil
  471. }
  472. } else {
  473. return f.Rules(kind), nil
  474. }
  475. }
  476. return nil, fmt.Errorf("rule '%s' not found", rule)
  477. }
  478. func filterRules(opts *Options, rules []*build.Rule) (result []*build.Rule) {
  479. if len(opts.FilterRuleTypes) == 0 {
  480. return rules
  481. }
  482. for _, rule := range rules {
  483. acceptableType := false
  484. for _, filterType := range opts.FilterRuleTypes {
  485. if rule.Kind() == filterType {
  486. acceptableType = true
  487. break
  488. }
  489. }
  490. if acceptableType || rule.Kind() == "package" {
  491. result = append(result, rule)
  492. }
  493. }
  494. return
  495. }
  496. // command contains a list of tokens that describe a buildozer command.
  497. type command struct {
  498. tokens []string
  499. }
  500. // checkCommandUsage checks the number of argument of a command.
  501. // It prints an error and usage when it is not valid.
  502. func checkCommandUsage(name string, cmd CommandInfo, count int) {
  503. if count >= cmd.MinArg && (cmd.MaxArg == -1 || count <= cmd.MaxArg) {
  504. return
  505. }
  506. if count < cmd.MinArg {
  507. fmt.Fprintf(os.Stderr, "Too few arguments for command '%s', expected at least %d.\n",
  508. name, cmd.MinArg)
  509. } else {
  510. fmt.Fprintf(os.Stderr, "Too many arguments for command '%s', expected at most %d.\n",
  511. name, cmd.MaxArg)
  512. }
  513. Usage()
  514. }
  515. // Match text that only contains spaces if they're escaped with '\'.
  516. var spaceRegex = regexp.MustCompile(`(\\ |[^ ])+`)
  517. // SplitOnSpaces behaves like strings.Fields, except that spaces can be escaped.
  518. // " some dummy\\ string" -> ["some", "dummy string"]
  519. func SplitOnSpaces(input string) []string {
  520. result := spaceRegex.FindAllString(input, -1)
  521. for i, s := range result {
  522. result[i] = strings.Replace(s, `\ `, " ", -1)
  523. }
  524. return result
  525. }
  526. // parseCommands parses commands and targets they should be applied on from
  527. // a list of arguments.
  528. // Each argument can be either:
  529. // - a command (as defined by AllCommands) and its parameters, separated by
  530. // whitespace
  531. // - a target all commands that are parsed during one call to parseCommands
  532. // should be applied on
  533. func parseCommands(args []string) (commands []command, targets []string) {
  534. for _, arg := range args {
  535. commandTokens := SplitOnSpaces(arg)
  536. cmd, found := AllCommands[commandTokens[0]]
  537. if found {
  538. checkCommandUsage(commandTokens[0], cmd, len(commandTokens)-1)
  539. commands = append(commands, command{commandTokens})
  540. } else {
  541. targets = append(targets, arg)
  542. }
  543. }
  544. return
  545. }
  546. // commandsForTarget contains commands to be executed on the given target.
  547. type commandsForTarget struct {
  548. target string
  549. commands []command
  550. }
  551. // commandsForFile contains the file name and all commands that should be
  552. // applied on that file, indexed by their target.
  553. type commandsForFile struct {
  554. file string
  555. commands []commandsForTarget
  556. }
  557. // commandError returns an error that formats 'err' in the context of the
  558. // commands to be executed on the given target.
  559. func commandError(commands []command, target string, err error) error {
  560. return fmt.Errorf("error while executing commands %s on target %s: %s", commands, target, err)
  561. }
  562. // rewriteResult contains the outcome of applying fixes to a single file.
  563. type rewriteResult struct {
  564. file string
  565. errs []error
  566. modified bool
  567. records []*apipb.Output_Record
  568. }
  569. // getGlobalVariables returns the global variable assignments in the provided list of expressions.
  570. // That is, for each variable assignment of the form
  571. // a = v
  572. // vars["a"] will contain the BinaryExpr whose Y value is the assignment "a = v".
  573. func getGlobalVariables(exprs []build.Expr) (vars map[string]*build.BinaryExpr) {
  574. vars = make(map[string]*build.BinaryExpr)
  575. for _, expr := range exprs {
  576. if binExpr, ok := expr.(*build.BinaryExpr); ok {
  577. if binExpr.Op != "=" {
  578. continue
  579. }
  580. if lhs, ok := binExpr.X.(*build.LiteralExpr); ok {
  581. vars[lhs.Token] = binExpr
  582. }
  583. }
  584. }
  585. return vars
  586. }
  587. // When checking the filesystem, we need to look for any of the
  588. // possible buildFileNames. For historical reasons, the
  589. // parts of the tool that generate paths that we may want to examine
  590. // continue to assume that build files are all named "BUILD".
  591. var buildFileNames = [...]string{"BUILD.bazel", "BUILD", "BUCK"}
  592. var buildFileNamesSet = map[string]bool{
  593. "BUILD.bazel": true,
  594. "BUILD": true,
  595. "BUCK": true,
  596. }
  597. // rewrite parses the BUILD file for the given file, transforms the AST,
  598. // and write the changes back in the file (or on stdout).
  599. func rewrite(opts *Options, commandsForFile commandsForFile) *rewriteResult {
  600. name := commandsForFile.file
  601. var data []byte
  602. var err error
  603. var fi os.FileInfo
  604. records := []*apipb.Output_Record{}
  605. if name == stdinPackageName { // read on stdin
  606. data, err = ioutil.ReadAll(os.Stdin)
  607. if err != nil {
  608. return &rewriteResult{file: name, errs: []error{err}}
  609. }
  610. } else {
  611. origName := name
  612. for _, suffix := range buildFileNames {
  613. if strings.HasSuffix(name, "/"+suffix) {
  614. name = strings.TrimSuffix(name, suffix)
  615. break
  616. }
  617. }
  618. for _, suffix := range buildFileNames {
  619. name = name + suffix
  620. data, fi, err = file.ReadFile(name)
  621. if err == nil {
  622. break
  623. }
  624. name = strings.TrimSuffix(name, suffix)
  625. }
  626. if err != nil {
  627. data, fi, err = file.ReadFile(name)
  628. }
  629. if err != nil {
  630. err = errors.New("file not found or not readable")
  631. return &rewriteResult{file: origName, errs: []error{err}}
  632. }
  633. }
  634. f, err := build.Parse(name, data)
  635. if err != nil {
  636. return &rewriteResult{file: name, errs: []error{err}}
  637. }
  638. vars := map[string]*build.BinaryExpr{}
  639. if opts.EditVariables {
  640. vars = getGlobalVariables(f.Stmt)
  641. }
  642. var errs []error
  643. changed := false
  644. for _, commands := range commandsForFile.commands {
  645. target := commands.target
  646. commands := commands.commands
  647. _, absPkg, rule := InterpretLabelForWorkspaceLocation(opts.RootDir, target)
  648. _, pkg, _ := ParseLabel(target)
  649. if pkg == stdinPackageName { // Special-case: This is already absolute
  650. absPkg = stdinPackageName
  651. }
  652. targets, err := expandTargets(f, rule)
  653. if err != nil {
  654. cerr := commandError(commands, target, err)
  655. errs = append(errs, cerr)
  656. if !opts.KeepGoing {
  657. return &rewriteResult{file: name, errs: errs, records: records}
  658. }
  659. }
  660. targets = filterRules(opts, targets)
  661. for _, cmd := range commands {
  662. for _, r := range targets {
  663. cmdInfo := AllCommands[cmd.tokens[0]]
  664. record := &apipb.Output_Record{}
  665. newf, err := cmdInfo.Fn(opts, CmdEnvironment{f, r, vars, absPkg, cmd.tokens[1:], record})
  666. if len(record.Fields) != 0 {
  667. records = append(records, record)
  668. }
  669. if err != nil {
  670. cerr := commandError([]command{cmd}, target, err)
  671. if opts.KeepGoing {
  672. errs = append(errs, cerr)
  673. } else {
  674. return &rewriteResult{file: name, errs: []error{cerr}, records: records}
  675. }
  676. }
  677. if newf != nil {
  678. changed = true
  679. f = newf
  680. }
  681. }
  682. }
  683. }
  684. if !changed {
  685. return &rewriteResult{file: name, errs: errs, records: records}
  686. }
  687. f = RemoveEmptyPackage(f)
  688. ndata, err := runBuildifier(opts, f)
  689. if err != nil {
  690. return &rewriteResult{file: name, errs: []error{fmt.Errorf("running buildifier: %v", err)}, records: records}
  691. }
  692. if opts.Stdout || name == stdinPackageName {
  693. os.Stdout.Write(ndata)
  694. return &rewriteResult{file: name, errs: errs, records: records}
  695. }
  696. if bytes.Equal(data, ndata) {
  697. return &rewriteResult{file: name, errs: errs, records: records}
  698. }
  699. if err := EditFile(fi, name); err != nil {
  700. return &rewriteResult{file: name, errs: []error{err}, records: records}
  701. }
  702. if err := file.WriteFile(name, ndata); err != nil {
  703. return &rewriteResult{file: name, errs: []error{err}, records: records}
  704. }
  705. fileModified = true
  706. return &rewriteResult{file: name, errs: errs, modified: true, records: records}
  707. }
  708. // EditFile is a function that does any prework needed before editing a file.
  709. // e.g. "checking out for write" from a locking source control repo.
  710. var EditFile = func(fi os.FileInfo, name string) error {
  711. return nil
  712. }
  713. // runBuildifier formats the build file f.
  714. // Runs opts.Buildifier if it's non-empty, otherwise uses built-in formatter.
  715. // opts.Buildifier is useful to force consistency with other tools that call Buildifier.
  716. func runBuildifier(opts *Options, f *build.File) ([]byte, error) {
  717. if opts.Buildifier == "" {
  718. build.Rewrite(f, nil)
  719. return build.Format(f), nil
  720. }
  721. cmd := exec.Command(opts.Buildifier)
  722. data := build.Format(f)
  723. cmd.Stdin = bytes.NewBuffer(data)
  724. stdout := bytes.NewBuffer(nil)
  725. stderr := bytes.NewBuffer(nil)
  726. cmd.Stdout = stdout
  727. cmd.Stderr = stderr
  728. err := cmd.Run()
  729. if stderr.Len() > 0 {
  730. return nil, fmt.Errorf("%s", stderr.Bytes())
  731. }
  732. if err != nil {
  733. return nil, err
  734. }
  735. return stdout.Bytes(), nil
  736. }
  737. // Given a target, whose package may contain a trailing "/...", returns all
  738. // extisting BUILD file paths which match the package.
  739. func targetExpressionToBuildFiles(opts *Options, target string) []string {
  740. file, _, _ := InterpretLabelForWorkspaceLocation(opts.RootDir, target)
  741. if opts.RootDir == "" {
  742. var err error
  743. if file, err = filepath.Abs(file); err != nil {
  744. fmt.Printf("Cannot make path absolute: %s\n", err.Error())
  745. os.Exit(1)
  746. }
  747. }
  748. if !strings.HasSuffix(file, "/.../BUILD") {
  749. return []string{file}
  750. }
  751. var buildFiles []string
  752. searchDirs := []string{strings.TrimSuffix(file, "/.../BUILD")}
  753. for len(searchDirs) != 0 {
  754. lastIndex := len(searchDirs) - 1
  755. dir := searchDirs[lastIndex]
  756. searchDirs = searchDirs[:lastIndex]
  757. dirFiles, err := ioutil.ReadDir(dir)
  758. if err != nil {
  759. continue
  760. }
  761. for _, dirFile := range dirFiles {
  762. if dirFile.IsDir() {
  763. searchDirs = append(searchDirs, path.Join(dir, dirFile.Name()))
  764. } else if _, ok := buildFileNamesSet[dirFile.Name()]; ok {
  765. buildFiles = append(buildFiles, path.Join(dir, dirFile.Name()))
  766. }
  767. }
  768. }
  769. return buildFiles
  770. }
  771. // appendCommands adds the given commands to be applied to each of the given targets
  772. // via the commandMap.
  773. func appendCommands(opts *Options, commandMap map[string][]commandsForTarget, args []string) {
  774. commands, targets := parseCommands(args)
  775. for _, target := range targets {
  776. if strings.HasSuffix(target, "/BUILD") {
  777. target = strings.TrimSuffix(target, "/BUILD") + ":__pkg__"
  778. }
  779. var buildFiles []string
  780. _, pkg, _ := ParseLabel(target)
  781. if pkg == stdinPackageName {
  782. buildFiles = []string{stdinPackageName}
  783. } else {
  784. buildFiles = targetExpressionToBuildFiles(opts, target)
  785. }
  786. for _, file := range buildFiles {
  787. commandMap[file] = append(commandMap[file], commandsForTarget{target, commands})
  788. }
  789. }
  790. }
  791. func appendCommandsFromFile(opts *Options, commandsByFile map[string][]commandsForTarget, fileName string) {
  792. var reader io.Reader
  793. if opts.CommandsFile == stdinPackageName {
  794. reader = os.Stdin
  795. } else {
  796. rc := file.OpenReadFile(opts.CommandsFile)
  797. reader = rc
  798. defer rc.Close()
  799. }
  800. scanner := bufio.NewScanner(reader)
  801. for scanner.Scan() {
  802. line := scanner.Text()
  803. if line == "" {
  804. continue
  805. }
  806. args := strings.Split(line, "|")
  807. appendCommands(opts, commandsByFile, args)
  808. }
  809. if err := scanner.Err(); err != nil {
  810. fmt.Fprintf(os.Stderr, "Error while reading commands file: %v", scanner.Err())
  811. }
  812. }
  813. func printRecord(writer io.Writer, record *apipb.Output_Record) {
  814. fields := record.Fields
  815. line := make([]string, len(fields))
  816. for i, field := range fields {
  817. switch value := field.Value.(type) {
  818. case *apipb.Output_Record_Field_Text:
  819. if field.QuoteWhenPrinting && strings.ContainsRune(value.Text, ' ') {
  820. line[i] = fmt.Sprintf("%q", value.Text)
  821. } else {
  822. line[i] = value.Text
  823. }
  824. break
  825. case *apipb.Output_Record_Field_Number:
  826. line[i] = strconv.Itoa(int(value.Number))
  827. break
  828. case *apipb.Output_Record_Field_Error:
  829. switch value.Error {
  830. case apipb.Output_Record_Field_UNKNOWN:
  831. line[i] = "(unknown)"
  832. break
  833. case apipb.Output_Record_Field_MISSING:
  834. line[i] = "(missing)"
  835. break
  836. }
  837. break
  838. case *apipb.Output_Record_Field_List:
  839. line[i] = fmt.Sprintf("[%s]", strings.Join(value.List.Strings, " "))
  840. break
  841. }
  842. }
  843. fmt.Fprint(writer, strings.Join(line, " ")+"\n")
  844. }
  845. // Buildozer loops over all arguments on the command line fixing BUILD files.
  846. func Buildozer(opts *Options, args []string) int {
  847. commandsByFile := make(map[string][]commandsForTarget)
  848. if opts.CommandsFile != "" {
  849. appendCommandsFromFile(opts, commandsByFile, opts.CommandsFile)
  850. } else {
  851. if len(args) == 0 {
  852. Usage()
  853. }
  854. appendCommands(opts, commandsByFile, args)
  855. }
  856. numFiles := len(commandsByFile)
  857. if opts.Parallelism > 0 {
  858. runtime.GOMAXPROCS(opts.Parallelism)
  859. }
  860. results := make(chan *rewriteResult, numFiles)
  861. data := make(chan commandsForFile)
  862. for i := 0; i < opts.NumIO; i++ {
  863. go func(results chan *rewriteResult, data chan commandsForFile) {
  864. for commandsForFile := range data {
  865. results <- rewrite(opts, commandsForFile)
  866. }
  867. }(results, data)
  868. }
  869. for file, commands := range commandsByFile {
  870. data <- commandsForFile{file, commands}
  871. }
  872. close(data)
  873. records := []*apipb.Output_Record{}
  874. hasErrors := false
  875. for i := 0; i < numFiles; i++ {
  876. fileResults := <-results
  877. if fileResults == nil {
  878. continue
  879. }
  880. hasErrors = hasErrors || len(fileResults.errs) > 0
  881. for _, err := range fileResults.errs {
  882. fmt.Fprintf(os.Stderr, "%s: %s\n", fileResults.file, err)
  883. }
  884. if fileResults.modified && !opts.Quiet {
  885. fmt.Fprintf(os.Stderr, "fixed %s\n", fileResults.file)
  886. }
  887. if fileResults.records != nil {
  888. records = append(records, fileResults.records...)
  889. }
  890. }
  891. if opts.IsPrintingProto {
  892. data, err := proto.Marshal(&apipb.Output{Records: records})
  893. if err != nil {
  894. log.Fatal("marshaling error: ", err)
  895. }
  896. fmt.Fprintf(os.Stdout, "%s", data)
  897. } else {
  898. for _, record := range records {
  899. printRecord(os.Stdout, record)
  900. }
  901. }
  902. if hasErrors {
  903. return 2
  904. }
  905. if !fileModified && !opts.Stdout {
  906. return 3
  907. }
  908. return 0
  909. }