mk2rbc.go 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622
  1. // Copyright 2021 Google LLC
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. // The application to convert product configuration makefiles to Starlark.
  15. // Converts either given list of files (and optionally the dependent files
  16. // of the same kind), or all all product configuration makefiles in the
  17. // given source tree.
  18. // Previous version of a converted file can be backed up.
  19. // Optionally prints detailed statistics at the end.
  20. package main
  21. import (
  22. "bufio"
  23. "flag"
  24. "fmt"
  25. "io/ioutil"
  26. "os"
  27. "os/exec"
  28. "path/filepath"
  29. "regexp"
  30. "runtime/debug"
  31. "runtime/pprof"
  32. "sort"
  33. "strings"
  34. "time"
  35. "android/soong/androidmk/parser"
  36. "android/soong/mk2rbc"
  37. )
  38. var (
  39. // TODO(asmundak): remove this option once there is a consensus on suffix
  40. suffix = flag.String("suffix", ".rbc", "generated files' suffix")
  41. dryRun = flag.Bool("dry_run", false, "dry run")
  42. recurse = flag.Bool("convert_dependents", false, "convert all dependent files")
  43. mode = flag.String("mode", "", `"backup" to back up existing files, "write" to overwrite them`)
  44. errstat = flag.Bool("error_stat", false, "print error statistics")
  45. traceVar = flag.String("trace", "", "comma-separated list of variables to trace")
  46. // TODO(asmundak): this option is for debugging
  47. allInSource = flag.Bool("all", false, "convert all product config makefiles in the tree under //")
  48. outputTop = flag.String("outdir", "", "write output files into this directory hierarchy")
  49. launcher = flag.String("launcher", "", "generated launcher path.")
  50. boardlauncher = flag.String("boardlauncher", "", "generated board configuration launcher path.")
  51. printProductConfigMap = flag.Bool("print_product_config_map", false, "print product config map and exit")
  52. cpuProfile = flag.String("cpu_profile", "", "write cpu profile to file")
  53. traceCalls = flag.Bool("trace_calls", false, "trace function calls")
  54. inputVariables = flag.String("input_variables", "", "starlark file containing product config and global variables")
  55. makefileList = flag.String("makefile_list", "", "path to a list of all makefiles in the source tree, generated by soong's finder. If not provided, mk2rbc will find the makefiles itself (more slowly than if this flag was provided)")
  56. )
  57. func init() {
  58. // Simplistic flag aliasing: works, but the usage string is ugly and
  59. // both flag and its alias can be present on the command line
  60. flagAlias := func(target string, alias string) {
  61. if f := flag.Lookup(target); f != nil {
  62. flag.Var(f.Value, alias, "alias for --"+f.Name)
  63. return
  64. }
  65. quit("cannot alias unknown flag " + target)
  66. }
  67. flagAlias("suffix", "s")
  68. flagAlias("dry_run", "n")
  69. flagAlias("convert_dependents", "r")
  70. flagAlias("error_stat", "e")
  71. }
  72. var backupSuffix string
  73. var tracedVariables []string
  74. var errorLogger = errorSink{data: make(map[string]datum)}
  75. var makefileFinder mk2rbc.MakefileFinder
  76. func main() {
  77. flag.Usage = func() {
  78. cmd := filepath.Base(os.Args[0])
  79. fmt.Fprintf(flag.CommandLine.Output(),
  80. "Usage: %[1]s flags file...\n", cmd)
  81. flag.PrintDefaults()
  82. }
  83. flag.Parse()
  84. if _, err := os.Stat("build/soong/mk2rbc"); err != nil {
  85. quit("Must be run from the root of the android tree. (build/soong/mk2rbc does not exist)")
  86. }
  87. // Delouse
  88. if *suffix == ".mk" {
  89. quit("cannot use .mk as generated file suffix")
  90. }
  91. if *suffix == "" {
  92. quit("suffix cannot be empty")
  93. }
  94. if *outputTop != "" {
  95. if err := os.MkdirAll(*outputTop, os.ModeDir+os.ModePerm); err != nil {
  96. quit(err)
  97. }
  98. s, err := filepath.Abs(*outputTop)
  99. if err != nil {
  100. quit(err)
  101. }
  102. *outputTop = s
  103. }
  104. if *allInSource && len(flag.Args()) > 0 {
  105. quit("file list cannot be specified when -all is present")
  106. }
  107. if *allInSource && *launcher != "" {
  108. quit("--all and --launcher are mutually exclusive")
  109. }
  110. // Flag-driven adjustments
  111. if (*suffix)[0] != '.' {
  112. *suffix = "." + *suffix
  113. }
  114. if *mode == "backup" {
  115. backupSuffix = time.Now().Format("20060102150405")
  116. }
  117. if *traceVar != "" {
  118. tracedVariables = strings.Split(*traceVar, ",")
  119. }
  120. if *cpuProfile != "" {
  121. f, err := os.Create(*cpuProfile)
  122. if err != nil {
  123. quit(err)
  124. }
  125. pprof.StartCPUProfile(f)
  126. defer pprof.StopCPUProfile()
  127. }
  128. if *makefileList != "" {
  129. makefileFinder = &FileListMakefileFinder{
  130. cachedMakefiles: nil,
  131. filePath: *makefileList,
  132. }
  133. } else {
  134. makefileFinder = &FindCommandMakefileFinder{}
  135. }
  136. // Find out global variables
  137. getConfigVariables()
  138. getSoongVariables()
  139. if *printProductConfigMap {
  140. productConfigMap := buildProductConfigMap()
  141. var products []string
  142. for p := range productConfigMap {
  143. products = append(products, p)
  144. }
  145. sort.Strings(products)
  146. for _, p := range products {
  147. fmt.Println(p, productConfigMap[p])
  148. }
  149. os.Exit(0)
  150. }
  151. // Convert!
  152. files := flag.Args()
  153. if *allInSource {
  154. productConfigMap := buildProductConfigMap()
  155. for _, path := range productConfigMap {
  156. files = append(files, path)
  157. }
  158. }
  159. ok := true
  160. for _, mkFile := range files {
  161. ok = convertOne(mkFile, []string{}) && ok
  162. }
  163. if *launcher != "" {
  164. if len(files) != 1 {
  165. quit(fmt.Errorf("a launcher can be generated only for a single product"))
  166. }
  167. if *inputVariables == "" {
  168. quit(fmt.Errorf("the product launcher requires an input variables file"))
  169. }
  170. if !convertOne(*inputVariables, []string{}) {
  171. quit(fmt.Errorf("the product launcher input variables file failed to convert"))
  172. }
  173. err := writeGenerated(*launcher, mk2rbc.Launcher(outputFilePath(files[0]), outputFilePath(*inputVariables),
  174. mk2rbc.MakePath2ModuleName(files[0])))
  175. if err != nil {
  176. fmt.Fprintf(os.Stderr, "%s: %s", files[0], err)
  177. ok = false
  178. }
  179. }
  180. if *boardlauncher != "" {
  181. if len(files) != 1 {
  182. quit(fmt.Errorf("a launcher can be generated only for a single product"))
  183. }
  184. if *inputVariables == "" {
  185. quit(fmt.Errorf("the board launcher requires an input variables file"))
  186. }
  187. if !convertOne(*inputVariables, []string{}) {
  188. quit(fmt.Errorf("the board launcher input variables file failed to convert"))
  189. }
  190. err := writeGenerated(*boardlauncher, mk2rbc.BoardLauncher(
  191. outputFilePath(files[0]), outputFilePath(*inputVariables)))
  192. if err != nil {
  193. fmt.Fprintf(os.Stderr, "%s: %s", files[0], err)
  194. ok = false
  195. }
  196. }
  197. if *errstat {
  198. errorLogger.printStatistics()
  199. printStats()
  200. }
  201. if !ok {
  202. os.Exit(1)
  203. }
  204. }
  205. func quit(s interface{}) {
  206. fmt.Fprintln(os.Stderr, s)
  207. os.Exit(2)
  208. }
  209. func buildProductConfigMap() map[string]string {
  210. const androidProductsMk = "AndroidProducts.mk"
  211. // Build the list of AndroidProducts.mk files: it's
  212. // build/make/target/product/AndroidProducts.mk + device/**/AndroidProducts.mk plus + vendor/**/AndroidProducts.mk
  213. targetAndroidProductsFile := filepath.Join("build", "make", "target", "product", androidProductsMk)
  214. if _, err := os.Stat(targetAndroidProductsFile); err != nil {
  215. fmt.Fprintf(os.Stderr, "%s: %s\n", targetAndroidProductsFile, err)
  216. }
  217. productConfigMap := make(map[string]string)
  218. if err := mk2rbc.UpdateProductConfigMap(productConfigMap, targetAndroidProductsFile); err != nil {
  219. fmt.Fprintf(os.Stderr, "%s: %s\n", targetAndroidProductsFile, err)
  220. }
  221. for _, t := range []string{"device", "vendor"} {
  222. _ = filepath.WalkDir(t,
  223. func(path string, d os.DirEntry, err error) error {
  224. if err != nil || d.IsDir() || filepath.Base(path) != androidProductsMk {
  225. return nil
  226. }
  227. if err2 := mk2rbc.UpdateProductConfigMap(productConfigMap, path); err2 != nil {
  228. fmt.Fprintf(os.Stderr, "%s: %s\n", path, err)
  229. // Keep going, we want to find all such errors in a single run
  230. }
  231. return nil
  232. })
  233. }
  234. return productConfigMap
  235. }
  236. func getConfigVariables() {
  237. path := filepath.Join("build", "make", "core", "product.mk")
  238. if err := mk2rbc.FindConfigVariables(path, mk2rbc.KnownVariables); err != nil {
  239. quit(err)
  240. }
  241. }
  242. // Implements mkparser.Scope, to be used by mkparser.Value.Value()
  243. type fileNameScope struct {
  244. mk2rbc.ScopeBase
  245. }
  246. func (s fileNameScope) Get(name string) string {
  247. if name != "BUILD_SYSTEM" {
  248. return fmt.Sprintf("$(%s)", name)
  249. }
  250. return filepath.Join("build", "make", "core")
  251. }
  252. func getSoongVariables() {
  253. path := filepath.Join("build", "make", "core", "soong_config.mk")
  254. err := mk2rbc.FindSoongVariables(path, fileNameScope{}, mk2rbc.KnownVariables)
  255. if err != nil {
  256. quit(err)
  257. }
  258. }
  259. var converted = make(map[string]*mk2rbc.StarlarkScript)
  260. //goland:noinspection RegExpRepeatedSpace
  261. var cpNormalizer = regexp.MustCompile(
  262. "# Copyright \\(C\\) 20.. The Android Open Source Project")
  263. const cpNormalizedCopyright = "# Copyright (C) 20xx The Android Open Source Project"
  264. const copyright = `#
  265. # Copyright (C) 20xx The Android Open Source Project
  266. #
  267. # Licensed under the Apache License, Version 2.0 (the "License");
  268. # you may not use this file except in compliance with the License.
  269. # You may obtain a copy of the License at
  270. #
  271. # http://www.apache.org/licenses/LICENSE-2.0
  272. #
  273. # Unless required by applicable law or agreed to in writing, software
  274. # distributed under the License is distributed on an "AS IS" BASIS,
  275. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  276. # See the License for the specific language governing permissions and
  277. # limitations under the License.
  278. #
  279. `
  280. // Convert a single file.
  281. // Write the result either to the same directory, to the same place in
  282. // the output hierarchy, or to the stdout.
  283. // Optionally, recursively convert the files this one includes by
  284. // $(call inherit-product) or an include statement.
  285. func convertOne(mkFile string, loadStack []string) (ok bool) {
  286. if v, ok := converted[mkFile]; ok {
  287. if v == nil {
  288. fmt.Fprintf(os.Stderr, "Cycle in load graph:\n%s\n%s\n\n", strings.Join(loadStack, "\n"), mkFile)
  289. return false
  290. }
  291. return true
  292. }
  293. converted[mkFile] = nil
  294. defer func() {
  295. if r := recover(); r != nil {
  296. ok = false
  297. fmt.Fprintf(os.Stderr, "%s: panic while converting: %s\n%s\n", mkFile, r, debug.Stack())
  298. }
  299. }()
  300. mk2starRequest := mk2rbc.Request{
  301. MkFile: mkFile,
  302. Reader: nil,
  303. OutputDir: *outputTop,
  304. OutputSuffix: *suffix,
  305. TracedVariables: tracedVariables,
  306. TraceCalls: *traceCalls,
  307. SourceFS: os.DirFS("."),
  308. MakefileFinder: makefileFinder,
  309. ErrorLogger: errorLogger,
  310. }
  311. ss, err := mk2rbc.Convert(mk2starRequest)
  312. if err != nil {
  313. fmt.Fprintln(os.Stderr, mkFile, ": ", err)
  314. return false
  315. }
  316. script := ss.String()
  317. outputPath := outputFilePath(mkFile)
  318. if *dryRun {
  319. fmt.Printf("==== %s ====\n", outputPath)
  320. // Print generated script after removing the copyright header
  321. outText := cpNormalizer.ReplaceAllString(script, cpNormalizedCopyright)
  322. fmt.Println(strings.TrimPrefix(outText, copyright))
  323. } else {
  324. if err := maybeBackup(outputPath); err != nil {
  325. fmt.Fprintln(os.Stderr, err)
  326. return false
  327. }
  328. if err := writeGenerated(outputPath, script); err != nil {
  329. fmt.Fprintln(os.Stderr, err)
  330. return false
  331. }
  332. }
  333. loadStack = append(loadStack, mkFile)
  334. ok = true
  335. if *recurse {
  336. for _, sub := range ss.SubConfigFiles() {
  337. // File may be absent if it is a conditional load
  338. if _, err := os.Stat(sub); os.IsNotExist(err) {
  339. continue
  340. }
  341. ok = convertOne(sub, loadStack) && ok
  342. }
  343. }
  344. converted[mkFile] = ss
  345. return ok
  346. }
  347. // Optionally saves the previous version of the generated file
  348. func maybeBackup(filename string) error {
  349. stat, err := os.Stat(filename)
  350. if os.IsNotExist(err) {
  351. return nil
  352. }
  353. if !stat.Mode().IsRegular() {
  354. return fmt.Errorf("%s exists and is not a regular file", filename)
  355. }
  356. switch *mode {
  357. case "backup":
  358. return os.Rename(filename, filename+backupSuffix)
  359. case "write":
  360. return os.Remove(filename)
  361. default:
  362. return fmt.Errorf("%s already exists, use --mode option", filename)
  363. }
  364. }
  365. func outputFilePath(mkFile string) string {
  366. path := strings.TrimSuffix(mkFile, filepath.Ext(mkFile)) + *suffix
  367. if *outputTop != "" {
  368. path = filepath.Join(*outputTop, path)
  369. }
  370. return path
  371. }
  372. func writeGenerated(path string, contents string) error {
  373. if err := os.MkdirAll(filepath.Dir(path), os.ModeDir|os.ModePerm); err != nil {
  374. return err
  375. }
  376. if err := ioutil.WriteFile(path, []byte(contents), 0644); err != nil {
  377. return err
  378. }
  379. return nil
  380. }
  381. func printStats() {
  382. var sortedFiles []string
  383. for p := range converted {
  384. sortedFiles = append(sortedFiles, p)
  385. }
  386. sort.Strings(sortedFiles)
  387. nOk, nPartial, nFailed := 0, 0, 0
  388. for _, f := range sortedFiles {
  389. if converted[f] == nil {
  390. nFailed++
  391. } else if converted[f].HasErrors() {
  392. nPartial++
  393. } else {
  394. nOk++
  395. }
  396. }
  397. if nPartial > 0 {
  398. fmt.Fprintf(os.Stderr, "Conversion was partially successful for:\n")
  399. for _, f := range sortedFiles {
  400. if ss := converted[f]; ss != nil && ss.HasErrors() {
  401. fmt.Fprintln(os.Stderr, " ", f)
  402. }
  403. }
  404. }
  405. if nFailed > 0 {
  406. fmt.Fprintf(os.Stderr, "Conversion failed for files:\n")
  407. for _, f := range sortedFiles {
  408. if converted[f] == nil {
  409. fmt.Fprintln(os.Stderr, " ", f)
  410. }
  411. }
  412. }
  413. }
  414. type datum struct {
  415. count int
  416. formattingArgs []string
  417. }
  418. type errorSink struct {
  419. data map[string]datum
  420. }
  421. func (ebt errorSink) NewError(el mk2rbc.ErrorLocation, node parser.Node, message string, args ...interface{}) {
  422. fmt.Fprint(os.Stderr, el, ": ")
  423. fmt.Fprintf(os.Stderr, message, args...)
  424. fmt.Fprintln(os.Stderr)
  425. if !*errstat {
  426. return
  427. }
  428. v, exists := ebt.data[message]
  429. if exists {
  430. v.count++
  431. } else {
  432. v = datum{1, nil}
  433. }
  434. if strings.Contains(message, "%s") {
  435. var newArg1 string
  436. if len(args) == 0 {
  437. panic(fmt.Errorf(`%s has %%s but args are missing`, message))
  438. }
  439. newArg1 = fmt.Sprint(args[0])
  440. if message == "unsupported line" {
  441. newArg1 = node.Dump()
  442. } else if message == "unsupported directive %s" {
  443. if newArg1 == "include" || newArg1 == "-include" {
  444. newArg1 = node.Dump()
  445. }
  446. }
  447. v.formattingArgs = append(v.formattingArgs, newArg1)
  448. }
  449. ebt.data[message] = v
  450. }
  451. func (ebt errorSink) printStatistics() {
  452. if len(ebt.data) > 0 {
  453. fmt.Fprintln(os.Stderr, "Error counts:")
  454. }
  455. for message, data := range ebt.data {
  456. if len(data.formattingArgs) == 0 {
  457. fmt.Fprintf(os.Stderr, "%4d %s\n", data.count, message)
  458. continue
  459. }
  460. itemsByFreq, count := stringsWithFreq(data.formattingArgs, 30)
  461. fmt.Fprintf(os.Stderr, "%4d %s [%d unique items]:\n", data.count, message, count)
  462. fmt.Fprintln(os.Stderr, " ", itemsByFreq)
  463. }
  464. }
  465. func stringsWithFreq(items []string, topN int) (string, int) {
  466. freq := make(map[string]int)
  467. for _, item := range items {
  468. freq[strings.TrimPrefix(strings.TrimSuffix(item, "]"), "[")]++
  469. }
  470. var sorted []string
  471. for item := range freq {
  472. sorted = append(sorted, item)
  473. }
  474. sort.Slice(sorted, func(i int, j int) bool {
  475. return freq[sorted[i]] > freq[sorted[j]]
  476. })
  477. sep := ""
  478. res := ""
  479. for i, item := range sorted {
  480. if i >= topN {
  481. res += " ..."
  482. break
  483. }
  484. count := freq[item]
  485. if count > 1 {
  486. res += fmt.Sprintf("%s%s(%d)", sep, item, count)
  487. } else {
  488. res += fmt.Sprintf("%s%s", sep, item)
  489. }
  490. sep = ", "
  491. }
  492. return res, len(sorted)
  493. }
  494. // FindCommandMakefileFinder is an implementation of mk2rbc.MakefileFinder that
  495. // runs the unix find command to find all the makefiles in the source tree.
  496. type FindCommandMakefileFinder struct {
  497. cachedRoot string
  498. cachedMakefiles []string
  499. }
  500. func (l *FindCommandMakefileFinder) Find(root string) []string {
  501. if l.cachedMakefiles != nil && l.cachedRoot == root {
  502. return l.cachedMakefiles
  503. }
  504. // Return all *.mk files but not in hidden directories.
  505. // NOTE(asmundak): as it turns out, even the WalkDir (which is an _optimized_ directory tree walker)
  506. // is about twice slower than running `find` command (14s vs 6s on the internal Android source tree).
  507. common_args := []string{"!", "-type", "d", "-name", "*.mk", "!", "-path", "*/.*/*"}
  508. if root != "" {
  509. common_args = append([]string{root}, common_args...)
  510. }
  511. cmd := exec.Command("/usr/bin/find", common_args...)
  512. stdout, err := cmd.StdoutPipe()
  513. if err == nil {
  514. err = cmd.Start()
  515. }
  516. if err != nil {
  517. panic(fmt.Errorf("cannot get the output from %s: %s", cmd, err))
  518. }
  519. scanner := bufio.NewScanner(stdout)
  520. result := make([]string, 0)
  521. for scanner.Scan() {
  522. result = append(result, strings.TrimPrefix(scanner.Text(), "./"))
  523. }
  524. stdout.Close()
  525. err = scanner.Err()
  526. if err != nil {
  527. panic(fmt.Errorf("cannot get the output from %s: %s", cmd, err))
  528. }
  529. l.cachedRoot = root
  530. l.cachedMakefiles = result
  531. return l.cachedMakefiles
  532. }
  533. // FileListMakefileFinder is an implementation of mk2rbc.MakefileFinder that
  534. // reads a file containing the list of makefiles in the android source tree.
  535. // This file is generated by soong's finder, so that it can be computed while
  536. // soong is already walking the source tree looking for other files. If the root
  537. // to find makefiles under is not the root of the android source tree, it will
  538. // fall back to using FindCommandMakefileFinder.
  539. type FileListMakefileFinder struct {
  540. FindCommandMakefileFinder
  541. cachedMakefiles []string
  542. filePath string
  543. }
  544. func (l *FileListMakefileFinder) Find(root string) []string {
  545. root, err1 := filepath.Abs(root)
  546. wd, err2 := os.Getwd()
  547. if root != wd || err1 != nil || err2 != nil {
  548. return l.FindCommandMakefileFinder.Find(root)
  549. }
  550. if l.cachedMakefiles != nil {
  551. return l.cachedMakefiles
  552. }
  553. file, err := os.Open(l.filePath)
  554. if err != nil {
  555. panic(fmt.Errorf("Cannot read makefile list: %s\n", err))
  556. }
  557. defer file.Close()
  558. result := make([]string, 0)
  559. scanner := bufio.NewScanner(file)
  560. for scanner.Scan() {
  561. line := scanner.Text()
  562. if len(line) > 0 {
  563. result = append(result, line)
  564. }
  565. }
  566. if err = scanner.Err(); err != nil {
  567. panic(fmt.Errorf("Cannot read makefile list: %s\n", err))
  568. }
  569. l.cachedMakefiles = result
  570. return l.cachedMakefiles
  571. }