rewrite.go 26 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003
  1. /*
  2. Copyright 2016 Google Inc. All Rights Reserved.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. */
  13. // Rewriting of high-level (not purely syntactic) BUILD constructs.
  14. package build
  15. import (
  16. "github.com/bazelbuild/buildtools/tables"
  17. "path"
  18. "path/filepath"
  19. "regexp"
  20. "sort"
  21. "strings"
  22. )
  23. // For debugging: flag to disable certain rewrites.
  24. var DisableRewrites []string
  25. // disabled reports whether the named rewrite is disabled.
  26. func disabled(name string) bool {
  27. for _, x := range DisableRewrites {
  28. if name == x {
  29. return true
  30. }
  31. }
  32. return false
  33. }
  34. // For debugging: allow sorting of these lists even with sorting otherwise disabled.
  35. var AllowSort []string
  36. // allowedSort reports whether sorting is allowed in the named context.
  37. func allowedSort(name string) bool {
  38. for _, x := range AllowSort {
  39. if name == x {
  40. return true
  41. }
  42. }
  43. return false
  44. }
  45. // Rewrite applies the high-level Buildifier rewrites to f, modifying it in place.
  46. // If info is non-nil, Rewrite updates it with information about the rewrite.
  47. func Rewrite(f *File, info *RewriteInfo) {
  48. // Allocate an info so that helpers can assume it's there.
  49. if info == nil {
  50. info = new(RewriteInfo)
  51. }
  52. for _, r := range rewrites {
  53. if !disabled(r.name) {
  54. if f.Type&r.scope != 0 {
  55. r.fn(f, info)
  56. }
  57. }
  58. }
  59. }
  60. // RewriteInfo collects information about what Rewrite did.
  61. type RewriteInfo struct {
  62. EditLabel int // number of label strings edited
  63. NameCall int // number of calls with argument names added
  64. SortCall int // number of call argument lists sorted
  65. SortStringList int // number of string lists sorted
  66. UnsafeSort int // number of unsafe string lists sorted
  67. SortLoad int // number of load argument lists sorted
  68. FormatDocstrings int // number of reindented docstrings
  69. ReorderArguments int // number of reordered function call arguments
  70. EditOctal int // number of edited octals
  71. Log []string // log entries - may change
  72. }
  73. // Stats returns a map with statistics about applied rewrites
  74. func (info *RewriteInfo) Stats() map[string]int {
  75. return map[string]int{
  76. "label": info.EditLabel,
  77. "callname": info.NameCall,
  78. "callsort": info.SortCall,
  79. "listsort": info.SortStringList,
  80. "unsafesort": info.UnsafeSort,
  81. "sortload": info.SortLoad,
  82. "formatdocstrings": info.FormatDocstrings,
  83. "reorderarguments": info.ReorderArguments,
  84. "editoctal": info.EditOctal,
  85. }
  86. }
  87. // Each rewrite function can be either applied for BUILD files, other files (such as .bzl),
  88. // or all files.
  89. const (
  90. scopeDefault = TypeDefault | TypeBzl // .bzl and generic Starlark files
  91. scopeBuild = TypeBuild | TypeWorkspace // BUILD and WORKSPACE files
  92. scopeBoth = scopeDefault | scopeBuild
  93. )
  94. // rewrites is the list of all Buildifier rewrites, in the order in which they are applied.
  95. // The order here matters: for example, label canonicalization must happen
  96. // before sorting lists of strings.
  97. var rewrites = []struct {
  98. name string
  99. fn func(*File, *RewriteInfo)
  100. scope FileType
  101. }{
  102. {"callsort", sortCallArgs, scopeBuild},
  103. {"label", fixLabels, scopeBuild},
  104. {"listsort", sortStringLists, scopeBoth},
  105. {"multiplus", fixMultilinePlus, scopeBuild},
  106. {"loadsort", sortAllLoadArgs, scopeBoth},
  107. {"formatdocstrings", formatDocstrings, scopeBoth},
  108. {"reorderarguments", reorderArguments, scopeBoth},
  109. {"editoctal", editOctals, scopeBoth},
  110. }
  111. // DisableLoadSortForBuildFiles disables the loadsort transformation for BUILD files.
  112. // This is a temporary function for backward compatibility, can be called if there's plenty of
  113. // already formatted BUILD files that shouldn't be changed by the transformation.
  114. func DisableLoadSortForBuildFiles() {
  115. for i := range rewrites {
  116. if rewrites[i].name == "loadsort" {
  117. rewrites[i].scope = scopeDefault
  118. break
  119. }
  120. }
  121. }
  122. // leaveAlone reports whether any of the nodes on the stack are marked
  123. // with a comment containing "buildifier: leave-alone".
  124. func leaveAlone(stk []Expr, final Expr) bool {
  125. for _, x := range stk {
  126. if leaveAlone1(x) {
  127. return true
  128. }
  129. }
  130. if final != nil && leaveAlone1(final) {
  131. return true
  132. }
  133. return false
  134. }
  135. // hasComment reports whether x is marked with a comment that
  136. // after being converted to lower case, contains the specified text.
  137. func hasComment(x Expr, text string) bool {
  138. for _, com := range x.Comment().Before {
  139. if strings.Contains(strings.ToLower(com.Token), text) {
  140. return true
  141. }
  142. }
  143. return false
  144. }
  145. // leaveAlone1 reports whether x is marked with a comment containing
  146. // "buildifier: leave-alone", case-insensitive.
  147. func leaveAlone1(x Expr) bool {
  148. return hasComment(x, "buildifier: leave-alone")
  149. }
  150. // doNotSort reports whether x is marked with a comment containing
  151. // "do not sort", case-insensitive.
  152. func doNotSort(x Expr) bool {
  153. return hasComment(x, "do not sort")
  154. }
  155. // keepSorted reports whether x is marked with a comment containing
  156. // "keep sorted", case-insensitive.
  157. func keepSorted(x Expr) bool {
  158. return hasComment(x, "keep sorted")
  159. }
  160. // fixLabels rewrites labels into a canonical form.
  161. //
  162. // First, it joins labels written as string addition, turning
  163. // "//x" + ":y" (usually split across multiple lines) into "//x:y".
  164. //
  165. // Second, it removes redundant target qualifiers, turning labels like
  166. // "//third_party/m4:m4" into "//third_party/m4" as well as ones like
  167. // "@foo//:foo" into "@foo".
  168. //
  169. func fixLabels(f *File, info *RewriteInfo) {
  170. joinLabel := func(p *Expr) {
  171. add, ok := (*p).(*BinaryExpr)
  172. if !ok || add.Op != "+" {
  173. return
  174. }
  175. str1, ok := add.X.(*StringExpr)
  176. if !ok || !strings.HasPrefix(str1.Value, "//") || strings.Contains(str1.Value, " ") {
  177. return
  178. }
  179. str2, ok := add.Y.(*StringExpr)
  180. if !ok || strings.Contains(str2.Value, " ") {
  181. return
  182. }
  183. info.EditLabel++
  184. str1.Value += str2.Value
  185. // Deleting nodes add and str2.
  186. // Merge comments from add, str1, and str2 and save in str1.
  187. com1 := add.Comment()
  188. com2 := str1.Comment()
  189. com3 := str2.Comment()
  190. com1.Before = append(com1.Before, com2.Before...)
  191. com1.Before = append(com1.Before, com3.Before...)
  192. com1.Suffix = append(com1.Suffix, com2.Suffix...)
  193. com1.Suffix = append(com1.Suffix, com3.Suffix...)
  194. *str1.Comment() = *com1
  195. *p = str1
  196. }
  197. labelPrefix := "//"
  198. if tables.StripLabelLeadingSlashes {
  199. labelPrefix = ""
  200. }
  201. // labelRE matches label strings, e.g. @r//x/y/z:abc
  202. // where $1 is @r//x/y/z, $2 is @r//, $3 is r, $4 is z, $5 is abc.
  203. labelRE := regexp.MustCompile(`^(((?:@(\w+))?//|` + labelPrefix + `)(?:.+/)?([^:]*))(?::([^:]+))?$`)
  204. shortenLabel := func(v Expr) {
  205. str, ok := v.(*StringExpr)
  206. if !ok {
  207. return
  208. }
  209. editPerformed := false
  210. if tables.StripLabelLeadingSlashes && strings.HasPrefix(str.Value, "//") {
  211. if filepath.Dir(f.Path) == "." || !strings.HasPrefix(str.Value, "//:") {
  212. editPerformed = true
  213. str.Value = str.Value[2:]
  214. }
  215. }
  216. if tables.ShortenAbsoluteLabelsToRelative {
  217. thisPackage := labelPrefix + filepath.Dir(f.Path)
  218. // filepath.Dir on Windows uses backslashes as separators, while labels always have slashes.
  219. if filepath.Separator != '/' {
  220. thisPackage = strings.Replace(thisPackage, string(filepath.Separator), "/", -1)
  221. }
  222. if str.Value == thisPackage {
  223. editPerformed = true
  224. str.Value = ":" + path.Base(str.Value)
  225. } else if strings.HasPrefix(str.Value, thisPackage+":") {
  226. editPerformed = true
  227. str.Value = str.Value[len(thisPackage):]
  228. }
  229. }
  230. m := labelRE.FindStringSubmatch(str.Value)
  231. if m == nil {
  232. return
  233. }
  234. if m[4] != "" && m[4] == m[5] { // e.g. //foo:foo
  235. editPerformed = true
  236. str.Value = m[1]
  237. } else if m[3] != "" && m[4] == "" && m[3] == m[5] { // e.g. @foo//:foo
  238. editPerformed = true
  239. str.Value = "@" + m[3]
  240. }
  241. if editPerformed {
  242. info.EditLabel++
  243. }
  244. }
  245. Walk(f, func(v Expr, stk []Expr) {
  246. switch v := v.(type) {
  247. case *CallExpr:
  248. if leaveAlone(stk, v) {
  249. return
  250. }
  251. for i := range v.List {
  252. if leaveAlone1(v.List[i]) {
  253. continue
  254. }
  255. as, ok := v.List[i].(*AssignExpr)
  256. if !ok {
  257. continue
  258. }
  259. key, ok := as.LHS.(*Ident)
  260. if !ok || !tables.IsLabelArg[key.Name] || tables.LabelBlacklist[callName(v)+"."+key.Name] {
  261. continue
  262. }
  263. if leaveAlone1(as.RHS) {
  264. continue
  265. }
  266. if list, ok := as.RHS.(*ListExpr); ok {
  267. for i := range list.List {
  268. if leaveAlone1(list.List[i]) {
  269. continue
  270. }
  271. joinLabel(&list.List[i])
  272. shortenLabel(list.List[i])
  273. }
  274. }
  275. if set, ok := as.RHS.(*SetExpr); ok {
  276. for i := range set.List {
  277. if leaveAlone1(set.List[i]) {
  278. continue
  279. }
  280. joinLabel(&set.List[i])
  281. shortenLabel(set.List[i])
  282. }
  283. } else {
  284. joinLabel(&as.RHS)
  285. shortenLabel(as.RHS)
  286. }
  287. }
  288. }
  289. })
  290. }
  291. // callName returns the name of the rule being called by call.
  292. // If the call is not to a literal rule name, callName returns "".
  293. func callName(call *CallExpr) string {
  294. rule, ok := call.X.(*Ident)
  295. if !ok {
  296. return ""
  297. }
  298. return rule.Name
  299. }
  300. // sortCallArgs sorts lists of named arguments to a call.
  301. func sortCallArgs(f *File, info *RewriteInfo) {
  302. Walk(f, func(v Expr, stk []Expr) {
  303. call, ok := v.(*CallExpr)
  304. if !ok {
  305. return
  306. }
  307. if leaveAlone(stk, call) {
  308. return
  309. }
  310. rule := callName(call)
  311. if rule == "" {
  312. return
  313. }
  314. // Find the tail of the argument list with named arguments.
  315. start := len(call.List)
  316. for start > 0 && argName(call.List[start-1]) != "" {
  317. start--
  318. }
  319. // Record information about each arg into a sortable list.
  320. var args namedArgs
  321. for i, x := range call.List[start:] {
  322. name := argName(x)
  323. args = append(args, namedArg{ruleNamePriority(rule, name), name, i, x})
  324. }
  325. // Sort the list and put the args back in the new order.
  326. if sort.IsSorted(args) {
  327. return
  328. }
  329. info.SortCall++
  330. sort.Sort(args)
  331. for i, x := range args {
  332. call.List[start+i] = x.expr
  333. }
  334. })
  335. }
  336. // ruleNamePriority maps a rule argument name to its sorting priority.
  337. // It could use the auto-generated per-rule tables but for now it just
  338. // falls back to the original list.
  339. func ruleNamePriority(rule, arg string) int {
  340. ruleArg := rule + "." + arg
  341. if val, ok := tables.NamePriority[ruleArg]; ok {
  342. return val
  343. }
  344. return tables.NamePriority[arg]
  345. /*
  346. list := ruleArgOrder[rule]
  347. if len(list) == 0 {
  348. return tables.NamePriority[arg]
  349. }
  350. for i, x := range list {
  351. if x == arg {
  352. return i
  353. }
  354. }
  355. return len(list)
  356. */
  357. }
  358. // If x is of the form key=value, argName returns the string key.
  359. // Otherwise argName returns "".
  360. func argName(x Expr) string {
  361. if as, ok := x.(*AssignExpr); ok {
  362. if id, ok := as.LHS.(*Ident); ok {
  363. return id.Name
  364. }
  365. }
  366. return ""
  367. }
  368. // A namedArg records information needed for sorting
  369. // a named call argument into its proper position.
  370. type namedArg struct {
  371. priority int // kind of name; first sort key
  372. name string // name; second sort key
  373. index int // original index; final sort key
  374. expr Expr // name=value argument
  375. }
  376. // namedArgs is a slice of namedArg that implements sort.Interface
  377. type namedArgs []namedArg
  378. func (x namedArgs) Len() int { return len(x) }
  379. func (x namedArgs) Swap(i, j int) { x[i], x[j] = x[j], x[i] }
  380. func (x namedArgs) Less(i, j int) bool {
  381. p := x[i]
  382. q := x[j]
  383. if p.priority != q.priority {
  384. return p.priority < q.priority
  385. }
  386. if p.name != q.name {
  387. return p.name < q.name
  388. }
  389. return p.index < q.index
  390. }
  391. // sortStringLists sorts lists of string literals used as specific rule arguments.
  392. func sortStringLists(f *File, info *RewriteInfo) {
  393. Walk(f, func(v Expr, stk []Expr) {
  394. switch v := v.(type) {
  395. case *CallExpr:
  396. if leaveAlone(stk, v) {
  397. return
  398. }
  399. rule := callName(v)
  400. for _, arg := range v.List {
  401. if leaveAlone1(arg) {
  402. continue
  403. }
  404. as, ok := arg.(*AssignExpr)
  405. if !ok || leaveAlone1(as) || doNotSort(as) {
  406. continue
  407. }
  408. key, ok := as.LHS.(*Ident)
  409. if !ok {
  410. continue
  411. }
  412. context := rule + "." + key.Name
  413. if !tables.IsSortableListArg[key.Name] || tables.SortableBlacklist[context] || f.Type == TypeDefault || f.Type == TypeBzl {
  414. continue
  415. }
  416. if disabled("unsafesort") && !tables.SortableWhitelist[context] && !allowedSort(context) {
  417. continue
  418. }
  419. sortStringList(as.RHS, info, context)
  420. }
  421. case *AssignExpr:
  422. if disabled("unsafesort") {
  423. return
  424. }
  425. // "keep sorted" comment on x = list forces sorting of list.
  426. as := v
  427. if keepSorted(as) {
  428. sortStringList(as.RHS, info, "?")
  429. }
  430. case *KeyValueExpr:
  431. if disabled("unsafesort") {
  432. return
  433. }
  434. // "keep sorted" before key: list also forces sorting of list.
  435. if keepSorted(v) {
  436. sortStringList(v.Value, info, "?")
  437. }
  438. case *ListExpr:
  439. if disabled("unsafesort") {
  440. return
  441. }
  442. // "keep sorted" comment above first list element also forces sorting of list.
  443. if len(v.List) > 0 && (keepSorted(v) || keepSorted(v.List[0])) {
  444. sortStringList(v, info, "?")
  445. }
  446. }
  447. })
  448. }
  449. // SortStringList sorts x, a list of strings.
  450. func SortStringList(x Expr) {
  451. sortStringList(x, nil, "")
  452. }
  453. // sortStringList sorts x, a list of strings.
  454. // The list is broken by non-strings and by blank lines and comments into chunks.
  455. // Each chunk is sorted in place.
  456. func sortStringList(x Expr, info *RewriteInfo, context string) {
  457. list, ok := x.(*ListExpr)
  458. if !ok || len(list.List) < 2 || doNotSort(list.List[0]) {
  459. return
  460. }
  461. forceSort := keepSorted(list) || keepSorted(list.List[0])
  462. // TODO(bazel-team): Decide how to recognize lists that cannot
  463. // be sorted. Avoiding all lists with comments avoids sorting
  464. // lists that say explicitly, in some form or another, why they
  465. // cannot be sorted. For example, many cc_test rules require
  466. // certain order in their deps attributes.
  467. if !forceSort {
  468. if line, _ := hasComments(list); line {
  469. return
  470. }
  471. }
  472. // Sort chunks of the list with no intervening blank lines or comments.
  473. for i := 0; i < len(list.List); {
  474. if _, ok := list.List[i].(*StringExpr); !ok {
  475. i++
  476. continue
  477. }
  478. j := i + 1
  479. for ; j < len(list.List); j++ {
  480. if str, ok := list.List[j].(*StringExpr); !ok || len(str.Before) > 0 {
  481. break
  482. }
  483. }
  484. var chunk []stringSortKey
  485. for index, x := range list.List[i:j] {
  486. chunk = append(chunk, makeSortKey(index, x.(*StringExpr)))
  487. }
  488. if !sort.IsSorted(byStringExpr(chunk)) || !isUniq(chunk) {
  489. if info != nil {
  490. info.SortStringList++
  491. if !tables.SortableWhitelist[context] {
  492. info.UnsafeSort++
  493. info.Log = append(info.Log, "sort:"+context)
  494. }
  495. }
  496. before := chunk[0].x.Comment().Before
  497. chunk[0].x.Comment().Before = nil
  498. sort.Sort(byStringExpr(chunk))
  499. chunk = uniq(chunk)
  500. chunk[0].x.Comment().Before = before
  501. for offset, key := range chunk {
  502. list.List[i+offset] = key.x
  503. }
  504. list.List = append(list.List[:(i+len(chunk))], list.List[j:]...)
  505. }
  506. i = j
  507. }
  508. }
  509. // uniq removes duplicates from a list, which must already be sorted.
  510. // It edits the list in place.
  511. func uniq(sortedList []stringSortKey) []stringSortKey {
  512. out := sortedList[:0]
  513. for _, sk := range sortedList {
  514. if len(out) == 0 || sk.value != out[len(out)-1].value {
  515. out = append(out, sk)
  516. }
  517. }
  518. return out
  519. }
  520. // isUniq reports whether the sorted list only contains unique elements.
  521. func isUniq(list []stringSortKey) bool {
  522. for i := range list {
  523. if i+1 < len(list) && list[i].value == list[i+1].value {
  524. return false
  525. }
  526. }
  527. return true
  528. }
  529. // If stk describes a call argument like rule(arg=...), callArgName
  530. // returns the name of that argument, formatted as "rule.arg".
  531. func callArgName(stk []Expr) string {
  532. n := len(stk)
  533. if n < 2 {
  534. return ""
  535. }
  536. arg := argName(stk[n-1])
  537. if arg == "" {
  538. return ""
  539. }
  540. call, ok := stk[n-2].(*CallExpr)
  541. if !ok {
  542. return ""
  543. }
  544. rule, ok := call.X.(*Ident)
  545. if !ok {
  546. return ""
  547. }
  548. return rule.Name + "." + arg
  549. }
  550. // A stringSortKey records information about a single string literal to be
  551. // sorted. The strings are first grouped into four phases: most strings,
  552. // strings beginning with ":", strings beginning with "//", and strings
  553. // beginning with "@". The next significant part of the comparison is the list
  554. // of elements in the value, where elements are split at `.' and `:'. Finally
  555. // we compare by value and break ties by original index.
  556. type stringSortKey struct {
  557. phase int
  558. split []string
  559. value string
  560. original int
  561. x Expr
  562. }
  563. func makeSortKey(index int, x *StringExpr) stringSortKey {
  564. key := stringSortKey{
  565. value: x.Value,
  566. original: index,
  567. x: x,
  568. }
  569. switch {
  570. case strings.HasPrefix(x.Value, ":"):
  571. key.phase = 1
  572. case strings.HasPrefix(x.Value, "//") || (tables.StripLabelLeadingSlashes && !strings.HasPrefix(x.Value, "@")):
  573. key.phase = 2
  574. case strings.HasPrefix(x.Value, "@"):
  575. key.phase = 3
  576. }
  577. key.split = strings.Split(strings.Replace(x.Value, ":", ".", -1), ".")
  578. return key
  579. }
  580. // byStringExpr implements sort.Interface for a list of stringSortKey.
  581. type byStringExpr []stringSortKey
  582. func (x byStringExpr) Len() int { return len(x) }
  583. func (x byStringExpr) Swap(i, j int) { x[i], x[j] = x[j], x[i] }
  584. func (x byStringExpr) Less(i, j int) bool {
  585. xi := x[i]
  586. xj := x[j]
  587. if xi.phase != xj.phase {
  588. return xi.phase < xj.phase
  589. }
  590. for k := 0; k < len(xi.split) && k < len(xj.split); k++ {
  591. if xi.split[k] != xj.split[k] {
  592. return xi.split[k] < xj.split[k]
  593. }
  594. }
  595. if len(xi.split) != len(xj.split) {
  596. return len(xi.split) < len(xj.split)
  597. }
  598. if xi.value != xj.value {
  599. return xi.value < xj.value
  600. }
  601. return xi.original < xj.original
  602. }
  603. // fixMultilinePlus turns
  604. //
  605. // ... +
  606. // [ ... ]
  607. //
  608. // ... +
  609. // call(...)
  610. //
  611. // into
  612. // ... + [
  613. // ...
  614. // ]
  615. //
  616. // ... + call(
  617. // ...
  618. // )
  619. //
  620. // which typically works better with our aggressively compact formatting.
  621. func fixMultilinePlus(f *File, info *RewriteInfo) {
  622. // List manipulation helpers.
  623. // As a special case, we treat f([...]) as a list, mainly
  624. // for glob.
  625. // isList reports whether x is a list.
  626. var isList func(x Expr) bool
  627. isList = func(x Expr) bool {
  628. switch x := x.(type) {
  629. case *ListExpr:
  630. return true
  631. case *CallExpr:
  632. if len(x.List) == 1 {
  633. return isList(x.List[0])
  634. }
  635. }
  636. return false
  637. }
  638. // isMultiLine reports whether x is a multiline list.
  639. var isMultiLine func(Expr) bool
  640. isMultiLine = func(x Expr) bool {
  641. switch x := x.(type) {
  642. case *ListExpr:
  643. return x.ForceMultiLine || len(x.List) > 1
  644. case *CallExpr:
  645. if x.ForceMultiLine || len(x.List) > 1 && !x.ForceCompact {
  646. return true
  647. }
  648. if len(x.List) == 1 {
  649. return isMultiLine(x.List[0])
  650. }
  651. }
  652. return false
  653. }
  654. // forceMultiLine tries to force the list x to use a multiline form.
  655. // It reports whether it was successful.
  656. var forceMultiLine func(Expr) bool
  657. forceMultiLine = func(x Expr) bool {
  658. switch x := x.(type) {
  659. case *ListExpr:
  660. // Already multi line?
  661. if x.ForceMultiLine {
  662. return true
  663. }
  664. // If this is a list containing a list, force the
  665. // inner list to be multiline instead.
  666. if len(x.List) == 1 && forceMultiLine(x.List[0]) {
  667. return true
  668. }
  669. x.ForceMultiLine = true
  670. return true
  671. case *CallExpr:
  672. if len(x.List) == 1 {
  673. return forceMultiLine(x.List[0])
  674. }
  675. }
  676. return false
  677. }
  678. skip := map[Expr]bool{}
  679. Walk(f, func(v Expr, stk []Expr) {
  680. if skip[v] {
  681. return
  682. }
  683. bin, ok := v.(*BinaryExpr)
  684. if !ok || bin.Op != "+" {
  685. return
  686. }
  687. // Found a +.
  688. // w + x + y + z parses as ((w + x) + y) + z,
  689. // so chase down the left side to make a list of
  690. // all the things being added together, separated
  691. // by the BinaryExprs that join them.
  692. // Mark them as "skip" so that when Walk recurses
  693. // into the subexpressions, we won't reprocess them.
  694. var all []Expr
  695. for {
  696. all = append(all, bin.Y, bin)
  697. bin1, ok := bin.X.(*BinaryExpr)
  698. if !ok || bin1.Op != "+" {
  699. break
  700. }
  701. bin = bin1
  702. skip[bin] = true
  703. }
  704. all = append(all, bin.X)
  705. // Because the outermost expression was the
  706. // rightmost one, the list is backward. Reverse it.
  707. for i, j := 0, len(all)-1; i < j; i, j = i+1, j-1 {
  708. all[i], all[j] = all[j], all[i]
  709. }
  710. // The 'all' slice is alternating addends and BinaryExpr +'s:
  711. // w, +, x, +, y, +, z
  712. // If there are no lists involved, don't rewrite anything.
  713. haveList := false
  714. for i := 0; i < len(all); i += 2 {
  715. if isList(all[i]) {
  716. haveList = true
  717. break
  718. }
  719. }
  720. if !haveList {
  721. return
  722. }
  723. // Okay, there are lists.
  724. // Consider each + next to a line break.
  725. for i := 1; i < len(all); i += 2 {
  726. bin := all[i].(*BinaryExpr)
  727. if !bin.LineBreak {
  728. continue
  729. }
  730. // We're going to break the line after the +.
  731. // If it is followed by a list, force that to be
  732. // multiline instead.
  733. if forceMultiLine(all[i+1]) {
  734. bin.LineBreak = false
  735. continue
  736. }
  737. // If the previous list was multiline already,
  738. // don't bother with the line break after
  739. // the +.
  740. if isMultiLine(all[i-1]) {
  741. bin.LineBreak = false
  742. continue
  743. }
  744. }
  745. })
  746. }
  747. // sortAllLoadArgs sorts all load arguments in the file
  748. func sortAllLoadArgs(f *File, info *RewriteInfo) {
  749. Walk(f, func(v Expr, stk []Expr) {
  750. if load, ok := v.(*LoadStmt); ok {
  751. if SortLoadArgs(load) {
  752. info.SortLoad++
  753. }
  754. }
  755. })
  756. }
  757. // hasComments reports whether any comments are associated with
  758. // the list or its elements.
  759. func hasComments(list *ListExpr) (line, suffix bool) {
  760. com := list.Comment()
  761. if len(com.Before) > 0 || len(com.After) > 0 || len(list.End.Before) > 0 {
  762. line = true
  763. }
  764. if len(com.Suffix) > 0 {
  765. suffix = true
  766. }
  767. for _, elem := range list.List {
  768. com := elem.Comment()
  769. if len(com.Before) > 0 {
  770. line = true
  771. }
  772. if len(com.Suffix) > 0 {
  773. suffix = true
  774. }
  775. }
  776. return
  777. }
  778. // A wrapper for a LoadStmt's From and To slices for consistent sorting of their contents.
  779. // It's assumed that the following slices have the same length. The contents are sorted by
  780. // the `To` attribute, but all items with equal "From" and "To" parts are placed before the items
  781. // with different parts.
  782. type loadArgs struct {
  783. From []*Ident
  784. To []*Ident
  785. modified bool
  786. }
  787. func (args loadArgs) Len() int {
  788. return len(args.From)
  789. }
  790. func (args loadArgs) Swap(i, j int) {
  791. args.From[i], args.From[j] = args.From[j], args.From[i]
  792. args.To[i], args.To[j] = args.To[j], args.To[i]
  793. args.modified = true
  794. }
  795. func (args loadArgs) Less(i, j int) bool {
  796. // Arguments with equal "from" and "to" parts are prioritized
  797. equalI := args.From[i].Name == args.To[i].Name
  798. equalJ := args.From[j].Name == args.To[j].Name
  799. if equalI != equalJ {
  800. // If equalI and !equalJ, return true, otherwise false.
  801. // Equivalently, return equalI.
  802. return equalI
  803. }
  804. return args.To[i].Name < args.To[j].Name
  805. }
  806. // SortLoadArgs sorts a load statement arguments (lexicographically, but positional first)
  807. func SortLoadArgs(load *LoadStmt) bool {
  808. args := loadArgs{From: load.From, To: load.To}
  809. sort.Sort(args)
  810. return args.modified
  811. }
  812. // formatDocstrings fixes the indentation and trailing whitespace of docstrings
  813. func formatDocstrings(f *File, info *RewriteInfo) {
  814. Walk(f, func(v Expr, stk []Expr) {
  815. def, ok := v.(*DefStmt)
  816. if !ok || len(def.Body) == 0 {
  817. return
  818. }
  819. docstring, ok := def.Body[0].(*StringExpr)
  820. if !ok || !docstring.TripleQuote {
  821. return
  822. }
  823. oldIndentation := docstring.Start.LineRune - 1 // LineRune starts with 1
  824. newIndentation := nestedIndentation * len(stk)
  825. // Operate on Token, not Value, because their line breaks can be different if a line ends with
  826. // a backslash.
  827. updatedToken := formatString(docstring.Token, oldIndentation, newIndentation)
  828. if updatedToken != docstring.Token {
  829. docstring.Token = updatedToken
  830. // Update the value to keep it consistent with Token
  831. docstring.Value, _, _ = Unquote(updatedToken)
  832. info.FormatDocstrings++
  833. }
  834. })
  835. }
  836. // formatString modifies a string value of a docstring to match the new indentation level and
  837. // to remove trailing whitespace from its lines.
  838. func formatString(value string, oldIndentation, newIndentation int) string {
  839. difference := newIndentation - oldIndentation
  840. lines := strings.Split(value, "\n")
  841. for i, line := range lines {
  842. if i == 0 {
  843. // The first line shouldn't be touched because it starts right after ''' or """
  844. continue
  845. }
  846. if difference > 0 {
  847. line = strings.Repeat(" ", difference) + line
  848. } else {
  849. for i, rune := range line {
  850. if i == -difference || rune != ' ' {
  851. line = line[i:]
  852. break
  853. }
  854. }
  855. }
  856. if i != len(lines)-1 {
  857. // Remove trailing space from the line unless it's the last line that's responsible
  858. // for the indentation of the closing `"""`
  859. line = strings.TrimRight(line, " ")
  860. }
  861. lines[i] = line
  862. }
  863. return strings.Join(lines, "\n")
  864. }
  865. // argumentType returns an integer by which funcall arguments can be sorted:
  866. // 1 for positional, 2 for named, 3 for *args, 4 for **kwargs
  867. func argumentType(expr Expr) int {
  868. switch expr := expr.(type) {
  869. case *UnaryExpr:
  870. switch expr.Op {
  871. case "**":
  872. return 4
  873. case "*":
  874. return 3
  875. }
  876. case *AssignExpr:
  877. return 2
  878. }
  879. return 1
  880. }
  881. // reorderArguments fixes the order of arguments of a function call
  882. // (positional, named, *args, **kwargs)
  883. func reorderArguments(f *File, info *RewriteInfo) {
  884. Walk(f, func(expr Expr, stack []Expr) {
  885. call, ok := expr.(*CallExpr)
  886. if !ok {
  887. return
  888. }
  889. compare := func(i, j int) bool {
  890. return argumentType(call.List[i]) < argumentType(call.List[j])
  891. }
  892. if !sort.SliceIsSorted(call.List, compare) {
  893. sort.SliceStable(call.List, compare)
  894. info.ReorderArguments++
  895. }
  896. })
  897. }
  898. // editOctals inserts 'o' into octal numbers to make it more obvious they are octal
  899. // 0123 -> 0o123
  900. func editOctals(f *File, info *RewriteInfo) {
  901. Walk(f, func(expr Expr, stack []Expr) {
  902. l, ok := expr.(*LiteralExpr)
  903. if !ok {
  904. return
  905. }
  906. if len(l.Token) > 1 && l.Token[0] == '0' && l.Token[1] >= '0' && l.Token[1] <= '9' {
  907. l.Token = "0o" + l.Token[1:]
  908. info.EditOctal++
  909. }
  910. })
  911. }