mk2rbc.go 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631
  1. // Copyright 2021 Google LLC
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. // The application to convert product configuration makefiles to Starlark.
  15. // Converts either given list of files (and optionally the dependent files
  16. // of the same kind), or all all product configuration makefiles in the
  17. // given source tree.
  18. // Previous version of a converted file can be backed up.
  19. // Optionally prints detailed statistics at the end.
  20. package main
  21. import (
  22. "bufio"
  23. "flag"
  24. "fmt"
  25. "io/ioutil"
  26. "os"
  27. "os/exec"
  28. "path/filepath"
  29. "regexp"
  30. "runtime/debug"
  31. "runtime/pprof"
  32. "sort"
  33. "strings"
  34. "time"
  35. "android/soong/androidmk/parser"
  36. "android/soong/mk2rbc"
  37. )
  38. var (
  39. // TODO(asmundak): remove this option once there is a consensus on suffix
  40. suffix = flag.String("suffix", ".rbc", "generated files' suffix")
  41. dryRun = flag.Bool("dry_run", false, "dry run")
  42. recurse = flag.Bool("convert_dependents", false, "convert all dependent files")
  43. mode = flag.String("mode", "", `"backup" to back up existing files, "write" to overwrite them`)
  44. errstat = flag.Bool("error_stat", false, "print error statistics")
  45. traceVar = flag.String("trace", "", "comma-separated list of variables to trace")
  46. // TODO(asmundak): this option is for debugging
  47. allInSource = flag.Bool("all", false, "convert all product config makefiles in the tree under //")
  48. outputTop = flag.String("outdir", "", "write output files into this directory hierarchy")
  49. launcher = flag.String("launcher", "", "generated launcher path.")
  50. boardlauncher = flag.String("boardlauncher", "", "generated board configuration launcher path.")
  51. printProductConfigMap = flag.Bool("print_product_config_map", false, "print product config map and exit")
  52. cpuProfile = flag.String("cpu_profile", "", "write cpu profile to file")
  53. traceCalls = flag.Bool("trace_calls", false, "trace function calls")
  54. inputVariables = flag.String("input_variables", "", "starlark file containing product config and global variables")
  55. makefileList = flag.String("makefile_list", "", "path to a list of all makefiles in the source tree, generated by soong's finder. If not provided, mk2rbc will find the makefiles itself (more slowly than if this flag was provided)")
  56. )
  57. func init() {
  58. // Simplistic flag aliasing: works, but the usage string is ugly and
  59. // both flag and its alias can be present on the command line
  60. flagAlias := func(target string, alias string) {
  61. if f := flag.Lookup(target); f != nil {
  62. flag.Var(f.Value, alias, "alias for --"+f.Name)
  63. return
  64. }
  65. quit("cannot alias unknown flag " + target)
  66. }
  67. flagAlias("suffix", "s")
  68. flagAlias("dry_run", "n")
  69. flagAlias("convert_dependents", "r")
  70. flagAlias("error_stat", "e")
  71. }
  72. var backupSuffix string
  73. var tracedVariables []string
  74. var errorLogger = errorSink{data: make(map[string]datum)}
  75. var makefileFinder mk2rbc.MakefileFinder
  76. func main() {
  77. flag.Usage = func() {
  78. cmd := filepath.Base(os.Args[0])
  79. fmt.Fprintf(flag.CommandLine.Output(),
  80. "Usage: %[1]s flags file...\n", cmd)
  81. flag.PrintDefaults()
  82. }
  83. flag.Parse()
  84. if _, err := os.Stat("build/soong/mk2rbc"); err != nil {
  85. quit("Must be run from the root of the android tree. (build/soong/mk2rbc does not exist)")
  86. }
  87. // Delouse
  88. if *suffix == ".mk" {
  89. quit("cannot use .mk as generated file suffix")
  90. }
  91. if *suffix == "" {
  92. quit("suffix cannot be empty")
  93. }
  94. if *outputTop != "" {
  95. if err := os.MkdirAll(*outputTop, os.ModeDir+os.ModePerm); err != nil {
  96. quit(err)
  97. }
  98. s, err := filepath.Abs(*outputTop)
  99. if err != nil {
  100. quit(err)
  101. }
  102. *outputTop = s
  103. }
  104. if *allInSource && len(flag.Args()) > 0 {
  105. quit("file list cannot be specified when -all is present")
  106. }
  107. if *allInSource && *launcher != "" {
  108. quit("--all and --launcher are mutually exclusive")
  109. }
  110. // Flag-driven adjustments
  111. if (*suffix)[0] != '.' {
  112. *suffix = "." + *suffix
  113. }
  114. if *mode == "backup" {
  115. backupSuffix = time.Now().Format("20060102150405")
  116. }
  117. if *traceVar != "" {
  118. tracedVariables = strings.Split(*traceVar, ",")
  119. }
  120. if *cpuProfile != "" {
  121. f, err := os.Create(*cpuProfile)
  122. if err != nil {
  123. quit(err)
  124. }
  125. pprof.StartCPUProfile(f)
  126. defer pprof.StopCPUProfile()
  127. }
  128. if *makefileList != "" {
  129. makefileFinder = &FileListMakefileFinder{
  130. cachedMakefiles: nil,
  131. filePath: *makefileList,
  132. }
  133. } else {
  134. makefileFinder = &FindCommandMakefileFinder{}
  135. }
  136. // Find out global variables
  137. getConfigVariables()
  138. getSoongVariables()
  139. if *printProductConfigMap {
  140. productConfigMap := buildProductConfigMap()
  141. var products []string
  142. for p := range productConfigMap {
  143. products = append(products, p)
  144. }
  145. sort.Strings(products)
  146. for _, p := range products {
  147. fmt.Println(p, productConfigMap[p])
  148. }
  149. os.Exit(0)
  150. }
  151. // Convert!
  152. files := flag.Args()
  153. if *allInSource {
  154. productConfigMap := buildProductConfigMap()
  155. for _, path := range productConfigMap {
  156. files = append(files, path)
  157. }
  158. }
  159. ok := true
  160. for _, mkFile := range files {
  161. ok = convertOne(mkFile, []string{}) && ok
  162. }
  163. if *launcher != "" {
  164. if len(files) != 1 {
  165. quit(fmt.Errorf("a launcher can be generated only for a single product"))
  166. }
  167. if *inputVariables == "" {
  168. quit(fmt.Errorf("the product launcher requires an input variables file"))
  169. }
  170. if !convertOne(*inputVariables, []string{}) {
  171. quit(fmt.Errorf("the product launcher input variables file failed to convert"))
  172. }
  173. err := writeGenerated(*launcher, mk2rbc.Launcher(outputModulePath(files[0]), outputModulePath(*inputVariables),
  174. mk2rbc.MakePath2ModuleName(files[0])))
  175. if err != nil {
  176. fmt.Fprintf(os.Stderr, "%s: %s", files[0], err)
  177. ok = false
  178. }
  179. }
  180. if *boardlauncher != "" {
  181. if len(files) != 1 {
  182. quit(fmt.Errorf("a launcher can be generated only for a single product"))
  183. }
  184. if *inputVariables == "" {
  185. quit(fmt.Errorf("the board launcher requires an input variables file"))
  186. }
  187. if !convertOne(*inputVariables, []string{}) {
  188. quit(fmt.Errorf("the board launcher input variables file failed to convert"))
  189. }
  190. err := writeGenerated(*boardlauncher, mk2rbc.BoardLauncher(
  191. outputModulePath(files[0]), outputModulePath(*inputVariables)))
  192. if err != nil {
  193. fmt.Fprintf(os.Stderr, "%s: %s", files[0], err)
  194. ok = false
  195. }
  196. }
  197. if *errstat {
  198. errorLogger.printStatistics()
  199. printStats()
  200. }
  201. if !ok {
  202. os.Exit(1)
  203. }
  204. }
  205. func quit(s interface{}) {
  206. fmt.Fprintln(os.Stderr, s)
  207. os.Exit(2)
  208. }
  209. func buildProductConfigMap() map[string]string {
  210. const androidProductsMk = "AndroidProducts.mk"
  211. // Build the list of AndroidProducts.mk files: it's
  212. // build/make/target/product/AndroidProducts.mk + device/**/AndroidProducts.mk plus + vendor/**/AndroidProducts.mk
  213. targetAndroidProductsFile := filepath.Join("build", "make", "target", "product", androidProductsMk)
  214. if _, err := os.Stat(targetAndroidProductsFile); err != nil {
  215. fmt.Fprintf(os.Stderr, "%s: %s\n", targetAndroidProductsFile, err)
  216. }
  217. productConfigMap := make(map[string]string)
  218. if err := mk2rbc.UpdateProductConfigMap(productConfigMap, targetAndroidProductsFile); err != nil {
  219. fmt.Fprintf(os.Stderr, "%s: %s\n", targetAndroidProductsFile, err)
  220. }
  221. for _, t := range []string{"device", "vendor"} {
  222. _ = filepath.WalkDir(t,
  223. func(path string, d os.DirEntry, err error) error {
  224. if err != nil || d.IsDir() || filepath.Base(path) != androidProductsMk {
  225. return nil
  226. }
  227. if err2 := mk2rbc.UpdateProductConfigMap(productConfigMap, path); err2 != nil {
  228. fmt.Fprintf(os.Stderr, "%s: %s\n", path, err)
  229. // Keep going, we want to find all such errors in a single run
  230. }
  231. return nil
  232. })
  233. }
  234. return productConfigMap
  235. }
  236. func getConfigVariables() {
  237. path := filepath.Join("build", "make", "core", "product.mk")
  238. if err := mk2rbc.FindConfigVariables(path, mk2rbc.KnownVariables); err != nil {
  239. quit(err)
  240. }
  241. }
  242. // Implements mkparser.Scope, to be used by mkparser.Value.Value()
  243. type fileNameScope struct {
  244. mk2rbc.ScopeBase
  245. }
  246. func (s fileNameScope) Get(name string) string {
  247. if name != "BUILD_SYSTEM" {
  248. return fmt.Sprintf("$(%s)", name)
  249. }
  250. return filepath.Join("build", "make", "core")
  251. }
  252. func getSoongVariables() {
  253. path := filepath.Join("build", "make", "core", "soong_config.mk")
  254. err := mk2rbc.FindSoongVariables(path, fileNameScope{}, mk2rbc.KnownVariables)
  255. if err != nil {
  256. quit(err)
  257. }
  258. }
  259. var converted = make(map[string]*mk2rbc.StarlarkScript)
  260. //goland:noinspection RegExpRepeatedSpace
  261. var cpNormalizer = regexp.MustCompile(
  262. "# Copyright \\(C\\) 20.. The Android Open Source Project")
  263. const cpNormalizedCopyright = "# Copyright (C) 20xx The Android Open Source Project"
  264. const copyright = `#
  265. # Copyright (C) 20xx The Android Open Source Project
  266. #
  267. # Licensed under the Apache License, Version 2.0 (the "License");
  268. # you may not use this file except in compliance with the License.
  269. # You may obtain a copy of the License at
  270. #
  271. # http://www.apache.org/licenses/LICENSE-2.0
  272. #
  273. # Unless required by applicable law or agreed to in writing, software
  274. # distributed under the License is distributed on an "AS IS" BASIS,
  275. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  276. # See the License for the specific language governing permissions and
  277. # limitations under the License.
  278. #
  279. `
  280. // Convert a single file.
  281. // Write the result either to the same directory, to the same place in
  282. // the output hierarchy, or to the stdout.
  283. // Optionally, recursively convert the files this one includes by
  284. // $(call inherit-product) or an include statement.
  285. func convertOne(mkFile string, loadStack []string) (ok bool) {
  286. if v, ok := converted[mkFile]; ok {
  287. if v == nil {
  288. fmt.Fprintf(os.Stderr, "Cycle in load graph:\n%s\n%s\n\n", strings.Join(loadStack, "\n"), mkFile)
  289. return false
  290. }
  291. return true
  292. }
  293. converted[mkFile] = nil
  294. defer func() {
  295. if r := recover(); r != nil {
  296. ok = false
  297. fmt.Fprintf(os.Stderr, "%s: panic while converting: %s\n%s\n", mkFile, r, debug.Stack())
  298. }
  299. }()
  300. mk2starRequest := mk2rbc.Request{
  301. MkFile: mkFile,
  302. Reader: nil,
  303. OutputDir: *outputTop,
  304. OutputSuffix: *suffix,
  305. TracedVariables: tracedVariables,
  306. TraceCalls: *traceCalls,
  307. SourceFS: os.DirFS("."),
  308. MakefileFinder: makefileFinder,
  309. ErrorLogger: errorLogger,
  310. }
  311. ss, err := mk2rbc.Convert(mk2starRequest)
  312. if err != nil {
  313. fmt.Fprintln(os.Stderr, mkFile, ": ", err)
  314. return false
  315. }
  316. script := ss.String()
  317. outputPath := outputFilePath(mkFile)
  318. if *dryRun {
  319. fmt.Printf("==== %s ====\n", outputPath)
  320. // Print generated script after removing the copyright header
  321. outText := cpNormalizer.ReplaceAllString(script, cpNormalizedCopyright)
  322. fmt.Println(strings.TrimPrefix(outText, copyright))
  323. } else {
  324. if err := maybeBackup(outputPath); err != nil {
  325. fmt.Fprintln(os.Stderr, err)
  326. return false
  327. }
  328. if err := writeGenerated(outputPath, script); err != nil {
  329. fmt.Fprintln(os.Stderr, err)
  330. return false
  331. }
  332. }
  333. loadStack = append(loadStack, mkFile)
  334. ok = true
  335. if *recurse {
  336. for _, sub := range ss.SubConfigFiles() {
  337. // File may be absent if it is a conditional load
  338. if _, err := os.Stat(sub); os.IsNotExist(err) {
  339. continue
  340. }
  341. ok = convertOne(sub, loadStack) && ok
  342. }
  343. }
  344. converted[mkFile] = ss
  345. return ok
  346. }
  347. // Optionally saves the previous version of the generated file
  348. func maybeBackup(filename string) error {
  349. stat, err := os.Stat(filename)
  350. if os.IsNotExist(err) {
  351. return nil
  352. }
  353. if !stat.Mode().IsRegular() {
  354. return fmt.Errorf("%s exists and is not a regular file", filename)
  355. }
  356. switch *mode {
  357. case "backup":
  358. return os.Rename(filename, filename+backupSuffix)
  359. case "write":
  360. return os.Remove(filename)
  361. default:
  362. return fmt.Errorf("%s already exists, use --mode option", filename)
  363. }
  364. }
  365. func outputFilePath(mkFile string) string {
  366. path := strings.TrimSuffix(mkFile, filepath.Ext(mkFile)) + *suffix
  367. if *outputTop != "" {
  368. path = filepath.Join(*outputTop, path)
  369. }
  370. return path
  371. }
  372. func outputModulePath(mkFile string) string {
  373. path := outputFilePath(mkFile)
  374. path, err := mk2rbc.RelativeToCwd(path)
  375. if err != nil {
  376. panic(err)
  377. }
  378. return "//" + path
  379. }
  380. func writeGenerated(path string, contents string) error {
  381. if err := os.MkdirAll(filepath.Dir(path), os.ModeDir|os.ModePerm); err != nil {
  382. return err
  383. }
  384. if err := ioutil.WriteFile(path, []byte(contents), 0644); err != nil {
  385. return err
  386. }
  387. return nil
  388. }
  389. func printStats() {
  390. var sortedFiles []string
  391. for p := range converted {
  392. sortedFiles = append(sortedFiles, p)
  393. }
  394. sort.Strings(sortedFiles)
  395. nOk, nPartial, nFailed := 0, 0, 0
  396. for _, f := range sortedFiles {
  397. if converted[f] == nil {
  398. nFailed++
  399. } else if converted[f].HasErrors() {
  400. nPartial++
  401. } else {
  402. nOk++
  403. }
  404. }
  405. if nPartial > 0 {
  406. fmt.Fprintf(os.Stderr, "Conversion was partially successful for:\n")
  407. for _, f := range sortedFiles {
  408. if ss := converted[f]; ss != nil && ss.HasErrors() {
  409. fmt.Fprintln(os.Stderr, " ", f)
  410. }
  411. }
  412. }
  413. if nFailed > 0 {
  414. fmt.Fprintf(os.Stderr, "Conversion failed for files:\n")
  415. for _, f := range sortedFiles {
  416. if converted[f] == nil {
  417. fmt.Fprintln(os.Stderr, " ", f)
  418. }
  419. }
  420. }
  421. }
  422. type datum struct {
  423. count int
  424. formattingArgs []string
  425. }
  426. type errorSink struct {
  427. data map[string]datum
  428. }
  429. func (ebt errorSink) NewError(el mk2rbc.ErrorLocation, node parser.Node, message string, args ...interface{}) {
  430. fmt.Fprint(os.Stderr, el, ": ")
  431. fmt.Fprintf(os.Stderr, message, args...)
  432. fmt.Fprintln(os.Stderr)
  433. if !*errstat {
  434. return
  435. }
  436. v, exists := ebt.data[message]
  437. if exists {
  438. v.count++
  439. } else {
  440. v = datum{1, nil}
  441. }
  442. if strings.Contains(message, "%s") {
  443. var newArg1 string
  444. if len(args) == 0 {
  445. panic(fmt.Errorf(`%s has %%s but args are missing`, message))
  446. }
  447. newArg1 = fmt.Sprint(args[0])
  448. if message == "unsupported line" {
  449. newArg1 = node.Dump()
  450. } else if message == "unsupported directive %s" {
  451. if newArg1 == "include" || newArg1 == "-include" {
  452. newArg1 = node.Dump()
  453. }
  454. }
  455. v.formattingArgs = append(v.formattingArgs, newArg1)
  456. }
  457. ebt.data[message] = v
  458. }
  459. func (ebt errorSink) printStatistics() {
  460. if len(ebt.data) > 0 {
  461. fmt.Fprintln(os.Stderr, "Error counts:")
  462. }
  463. for message, data := range ebt.data {
  464. if len(data.formattingArgs) == 0 {
  465. fmt.Fprintf(os.Stderr, "%4d %s\n", data.count, message)
  466. continue
  467. }
  468. itemsByFreq, count := stringsWithFreq(data.formattingArgs, 30)
  469. fmt.Fprintf(os.Stderr, "%4d %s [%d unique items]:\n", data.count, message, count)
  470. fmt.Fprintln(os.Stderr, " ", itemsByFreq)
  471. }
  472. }
  473. func stringsWithFreq(items []string, topN int) (string, int) {
  474. freq := make(map[string]int)
  475. for _, item := range items {
  476. freq[strings.TrimPrefix(strings.TrimSuffix(item, "]"), "[")]++
  477. }
  478. var sorted []string
  479. for item := range freq {
  480. sorted = append(sorted, item)
  481. }
  482. sort.Slice(sorted, func(i int, j int) bool {
  483. return freq[sorted[i]] > freq[sorted[j]]
  484. })
  485. sep := ""
  486. res := ""
  487. for i, item := range sorted {
  488. if i >= topN {
  489. res += " ..."
  490. break
  491. }
  492. count := freq[item]
  493. if count > 1 {
  494. res += fmt.Sprintf("%s%s(%d)", sep, item, count)
  495. } else {
  496. res += fmt.Sprintf("%s%s", sep, item)
  497. }
  498. sep = ", "
  499. }
  500. return res, len(sorted)
  501. }
  502. // FindCommandMakefileFinder is an implementation of mk2rbc.MakefileFinder that
  503. // runs the unix find command to find all the makefiles in the source tree.
  504. type FindCommandMakefileFinder struct {
  505. cachedRoot string
  506. cachedMakefiles []string
  507. }
  508. func (l *FindCommandMakefileFinder) Find(root string) []string {
  509. if l.cachedMakefiles != nil && l.cachedRoot == root {
  510. return l.cachedMakefiles
  511. }
  512. // Return all *.mk files but not in hidden directories.
  513. // NOTE(asmundak): as it turns out, even the WalkDir (which is an _optimized_ directory tree walker)
  514. // is about twice slower than running `find` command (14s vs 6s on the internal Android source tree).
  515. common_args := []string{"!", "-type", "d", "-name", "*.mk", "!", "-path", "*/.*/*"}
  516. if root != "" {
  517. common_args = append([]string{root}, common_args...)
  518. }
  519. cmd := exec.Command("/usr/bin/find", common_args...)
  520. stdout, err := cmd.StdoutPipe()
  521. if err == nil {
  522. err = cmd.Start()
  523. }
  524. if err != nil {
  525. panic(fmt.Errorf("cannot get the output from %s: %s", cmd, err))
  526. }
  527. scanner := bufio.NewScanner(stdout)
  528. result := make([]string, 0)
  529. for scanner.Scan() {
  530. result = append(result, strings.TrimPrefix(scanner.Text(), "./"))
  531. }
  532. stdout.Close()
  533. err = scanner.Err()
  534. if err != nil {
  535. panic(fmt.Errorf("cannot get the output from %s: %s", cmd, err))
  536. }
  537. l.cachedRoot = root
  538. l.cachedMakefiles = result
  539. return l.cachedMakefiles
  540. }
  541. // FileListMakefileFinder is an implementation of mk2rbc.MakefileFinder that
  542. // reads a file containing the list of makefiles in the android source tree.
  543. // This file is generated by soong's finder, so that it can be computed while
  544. // soong is already walking the source tree looking for other files. If the root
  545. // to find makefiles under is not the root of the android source tree, it will
  546. // fall back to using FindCommandMakefileFinder.
  547. type FileListMakefileFinder struct {
  548. FindCommandMakefileFinder
  549. cachedMakefiles []string
  550. filePath string
  551. }
  552. func (l *FileListMakefileFinder) Find(root string) []string {
  553. root, err1 := filepath.Abs(root)
  554. wd, err2 := os.Getwd()
  555. if root != wd || err1 != nil || err2 != nil {
  556. return l.FindCommandMakefileFinder.Find(root)
  557. }
  558. if l.cachedMakefiles != nil {
  559. return l.cachedMakefiles
  560. }
  561. file, err := os.Open(l.filePath)
  562. if err != nil {
  563. panic(fmt.Errorf("Cannot read makefile list: %s\n", err))
  564. }
  565. defer file.Close()
  566. result := make([]string, 0)
  567. scanner := bufio.NewScanner(file)
  568. for scanner.Scan() {
  569. line := scanner.Text()
  570. if len(line) > 0 {
  571. result = append(result, line)
  572. }
  573. }
  574. if err = scanner.Err(); err != nil {
  575. panic(fmt.Errorf("Cannot read makefile list: %s\n", err))
  576. }
  577. l.cachedMakefiles = result
  578. return l.cachedMakefiles
  579. }