mk2rbc.go 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547
  1. // Copyright 2021 Google LLC
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. // The application to convert product configuration makefiles to Starlark.
  15. // Converts either given list of files (and optionally the dependent files
  16. // of the same kind), or all all product configuration makefiles in the
  17. // given source tree.
  18. // Previous version of a converted file can be backed up.
  19. // Optionally prints detailed statistics at the end.
  20. package main
  21. import (
  22. "bufio"
  23. "flag"
  24. "fmt"
  25. "io/ioutil"
  26. "os"
  27. "os/exec"
  28. "path/filepath"
  29. "regexp"
  30. "runtime/debug"
  31. "runtime/pprof"
  32. "sort"
  33. "strings"
  34. "time"
  35. "android/soong/androidmk/parser"
  36. "android/soong/mk2rbc"
  37. )
  38. var (
  39. rootDir = flag.String("root", ".", "the value of // for load paths")
  40. // TODO(asmundak): remove this option once there is a consensus on suffix
  41. suffix = flag.String("suffix", ".rbc", "generated files' suffix")
  42. dryRun = flag.Bool("dry_run", false, "dry run")
  43. recurse = flag.Bool("convert_dependents", false, "convert all dependent files")
  44. mode = flag.String("mode", "", `"backup" to back up existing files, "write" to overwrite them`)
  45. warn = flag.Bool("warnings", false, "warn about partially failed conversions")
  46. verbose = flag.Bool("v", false, "print summary")
  47. errstat = flag.Bool("error_stat", false, "print error statistics")
  48. traceVar = flag.String("trace", "", "comma-separated list of variables to trace")
  49. // TODO(asmundak): this option is for debugging
  50. allInSource = flag.Bool("all", false, "convert all product config makefiles in the tree under //")
  51. outputTop = flag.String("outdir", "", "write output files into this directory hierarchy")
  52. launcher = flag.String("launcher", "", "generated launcher path. If set, the non-flag argument is _product_name_")
  53. printProductConfigMap = flag.Bool("print_product_config_map", false, "print product config map and exit")
  54. cpuProfile = flag.String("cpu_profile", "", "write cpu profile to file")
  55. traceCalls = flag.Bool("trace_calls", false, "trace function calls")
  56. )
  57. func init() {
  58. // Poor man's flag aliasing: works, but the usage string is ugly and
  59. // both flag and its alias can be present on the command line
  60. flagAlias := func(target string, alias string) {
  61. if f := flag.Lookup(target); f != nil {
  62. flag.Var(f.Value, alias, "alias for --"+f.Name)
  63. return
  64. }
  65. quit("cannot alias unknown flag " + target)
  66. }
  67. flagAlias("suffix", "s")
  68. flagAlias("root", "d")
  69. flagAlias("dry_run", "n")
  70. flagAlias("convert_dependents", "r")
  71. flagAlias("warnings", "w")
  72. flagAlias("error_stat", "e")
  73. }
  74. var backupSuffix string
  75. var tracedVariables []string
  76. var errorLogger = errorsByType{data: make(map[string]datum)}
  77. var makefileFinder = &LinuxMakefileFinder{}
  78. func main() {
  79. flag.Usage = func() {
  80. cmd := filepath.Base(os.Args[0])
  81. fmt.Fprintf(flag.CommandLine.Output(),
  82. "Usage: %[1]s flags file...\n"+
  83. "or: %[1]s flags --launcher=PATH PRODUCT\n", cmd)
  84. flag.PrintDefaults()
  85. }
  86. flag.Parse()
  87. // Delouse
  88. if *suffix == ".mk" {
  89. quit("cannot use .mk as generated file suffix")
  90. }
  91. if *suffix == "" {
  92. quit("suffix cannot be empty")
  93. }
  94. if *outputTop != "" {
  95. if err := os.MkdirAll(*outputTop, os.ModeDir+os.ModePerm); err != nil {
  96. quit(err)
  97. }
  98. s, err := filepath.Abs(*outputTop)
  99. if err != nil {
  100. quit(err)
  101. }
  102. *outputTop = s
  103. }
  104. if *allInSource && len(flag.Args()) > 0 {
  105. quit("file list cannot be specified when -all is present")
  106. }
  107. if *allInSource && *launcher != "" {
  108. quit("--all and --launcher are mutually exclusive")
  109. }
  110. // Flag-driven adjustments
  111. if (*suffix)[0] != '.' {
  112. *suffix = "." + *suffix
  113. }
  114. if *mode == "backup" {
  115. backupSuffix = time.Now().Format("20060102150405")
  116. }
  117. if *traceVar != "" {
  118. tracedVariables = strings.Split(*traceVar, ",")
  119. }
  120. if *cpuProfile != "" {
  121. f, err := os.Create(*cpuProfile)
  122. if err != nil {
  123. quit(err)
  124. }
  125. pprof.StartCPUProfile(f)
  126. defer pprof.StopCPUProfile()
  127. }
  128. // Find out global variables
  129. getConfigVariables()
  130. getSoongVariables()
  131. if *printProductConfigMap {
  132. productConfigMap := buildProductConfigMap()
  133. var products []string
  134. for p := range productConfigMap {
  135. products = append(products, p)
  136. }
  137. sort.Strings(products)
  138. for _, p := range products {
  139. fmt.Println(p, productConfigMap[p])
  140. }
  141. os.Exit(0)
  142. }
  143. // Convert!
  144. ok := true
  145. if *launcher != "" {
  146. if len(flag.Args()) != 1 {
  147. quit(fmt.Errorf("a launcher can be generated only for a single product"))
  148. }
  149. product := flag.Args()[0]
  150. productConfigMap := buildProductConfigMap()
  151. path, found := productConfigMap[product]
  152. if !found {
  153. quit(fmt.Errorf("cannot generate configuration launcher for %s, it is not a known product",
  154. product))
  155. }
  156. ok = convertOne(path) && ok
  157. err := writeGenerated(*launcher, mk2rbc.Launcher(outputFilePath(path), mk2rbc.MakePath2ModuleName(path)))
  158. if err != nil {
  159. fmt.Fprintf(os.Stderr, "%s:%s", path, err)
  160. ok = false
  161. }
  162. } else {
  163. files := flag.Args()
  164. if *allInSource {
  165. productConfigMap := buildProductConfigMap()
  166. for _, path := range productConfigMap {
  167. files = append(files, path)
  168. }
  169. }
  170. for _, mkFile := range files {
  171. ok = convertOne(mkFile) && ok
  172. }
  173. }
  174. printStats()
  175. if *errstat {
  176. errorLogger.printStatistics()
  177. }
  178. if !ok {
  179. os.Exit(1)
  180. }
  181. }
  182. func quit(s interface{}) {
  183. fmt.Fprintln(os.Stderr, s)
  184. os.Exit(2)
  185. }
  186. func buildProductConfigMap() map[string]string {
  187. const androidProductsMk = "AndroidProducts.mk"
  188. // Build the list of AndroidProducts.mk files: it's
  189. // build/make/target/product/AndroidProducts.mk plus
  190. // device/**/AndroidProducts.mk
  191. targetAndroidProductsFile := filepath.Join(*rootDir, "build", "make", "target", "product", androidProductsMk)
  192. if _, err := os.Stat(targetAndroidProductsFile); err != nil {
  193. fmt.Fprintf(os.Stderr, "%s: %s\n(hint: %s is not a source tree root)\n",
  194. targetAndroidProductsFile, err, *rootDir)
  195. }
  196. productConfigMap := make(map[string]string)
  197. if err := mk2rbc.UpdateProductConfigMap(productConfigMap, targetAndroidProductsFile); err != nil {
  198. fmt.Fprintf(os.Stderr, "%s: %s\n", targetAndroidProductsFile, err)
  199. }
  200. _ = filepath.Walk(filepath.Join(*rootDir, "device"),
  201. func(path string, info os.FileInfo, err error) error {
  202. if info.IsDir() || filepath.Base(path) != androidProductsMk {
  203. return nil
  204. }
  205. if err2 := mk2rbc.UpdateProductConfigMap(productConfigMap, path); err2 != nil {
  206. fmt.Fprintf(os.Stderr, "%s: %s\n", path, err)
  207. // Keep going, we want to find all such errors in a single run
  208. }
  209. return nil
  210. })
  211. return productConfigMap
  212. }
  213. func getConfigVariables() {
  214. path := filepath.Join(*rootDir, "build", "make", "core", "product.mk")
  215. if err := mk2rbc.FindConfigVariables(path, mk2rbc.KnownVariables); err != nil {
  216. quit(fmt.Errorf("%s\n(check --root[=%s], it should point to the source root)",
  217. err, *rootDir))
  218. }
  219. }
  220. // Implements mkparser.Scope, to be used by mkparser.Value.Value()
  221. type fileNameScope struct {
  222. mk2rbc.ScopeBase
  223. }
  224. func (s fileNameScope) Get(name string) string {
  225. if name != "BUILD_SYSTEM" {
  226. return fmt.Sprintf("$(%s)", name)
  227. }
  228. return filepath.Join(*rootDir, "build", "make", "core")
  229. }
  230. func getSoongVariables() {
  231. path := filepath.Join(*rootDir, "build", "make", "core", "soong_config.mk")
  232. err := mk2rbc.FindSoongVariables(path, fileNameScope{}, mk2rbc.KnownVariables)
  233. if err != nil {
  234. quit(err)
  235. }
  236. }
  237. var converted = make(map[string]*mk2rbc.StarlarkScript)
  238. //goland:noinspection RegExpRepeatedSpace
  239. var cpNormalizer = regexp.MustCompile(
  240. "# Copyright \\(C\\) 20.. The Android Open Source Project")
  241. const cpNormalizedCopyright = "# Copyright (C) 20xx The Android Open Source Project"
  242. const copyright = `#
  243. # Copyright (C) 20xx The Android Open Source Project
  244. #
  245. # Licensed under the Apache License, Version 2.0 (the "License");
  246. # you may not use this file except in compliance with the License.
  247. # You may obtain a copy of the License at
  248. #
  249. # http://www.apache.org/licenses/LICENSE-2.0
  250. #
  251. # Unless required by applicable law or agreed to in writing, software
  252. # distributed under the License is distributed on an "AS IS" BASIS,
  253. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  254. # See the License for the specific language governing permissions and
  255. # limitations under the License.
  256. #
  257. `
  258. // Convert a single file.
  259. // Write the result either to the same directory, to the same place in
  260. // the output hierarchy, or to the stdout.
  261. // Optionally, recursively convert the files this one includes by
  262. // $(call inherit-product) or an include statement.
  263. func convertOne(mkFile string) (ok bool) {
  264. if v, ok := converted[mkFile]; ok {
  265. return v != nil
  266. }
  267. converted[mkFile] = nil
  268. defer func() {
  269. if r := recover(); r != nil {
  270. ok = false
  271. fmt.Fprintf(os.Stderr, "%s: panic while converting: %s\n%s\n", mkFile, r, debug.Stack())
  272. }
  273. }()
  274. mk2starRequest := mk2rbc.Request{
  275. MkFile: mkFile,
  276. Reader: nil,
  277. RootDir: *rootDir,
  278. OutputDir: *outputTop,
  279. OutputSuffix: *suffix,
  280. TracedVariables: tracedVariables,
  281. TraceCalls: *traceCalls,
  282. WarnPartialSuccess: *warn,
  283. SourceFS: os.DirFS(*rootDir),
  284. MakefileFinder: makefileFinder,
  285. }
  286. if *errstat {
  287. mk2starRequest.ErrorLogger = errorLogger
  288. }
  289. ss, err := mk2rbc.Convert(mk2starRequest)
  290. if err != nil {
  291. fmt.Fprintln(os.Stderr, mkFile, ": ", err)
  292. return false
  293. }
  294. script := ss.String()
  295. outputPath := outputFilePath(mkFile)
  296. if *dryRun {
  297. fmt.Printf("==== %s ====\n", outputPath)
  298. // Print generated script after removing the copyright header
  299. outText := cpNormalizer.ReplaceAllString(script, cpNormalizedCopyright)
  300. fmt.Println(strings.TrimPrefix(outText, copyright))
  301. } else {
  302. if err := maybeBackup(outputPath); err != nil {
  303. fmt.Fprintln(os.Stderr, err)
  304. return false
  305. }
  306. if err := writeGenerated(outputPath, script); err != nil {
  307. fmt.Fprintln(os.Stderr, err)
  308. return false
  309. }
  310. }
  311. ok = true
  312. if *recurse {
  313. for _, sub := range ss.SubConfigFiles() {
  314. // File may be absent if it is a conditional load
  315. if _, err := os.Stat(sub); os.IsNotExist(err) {
  316. continue
  317. }
  318. ok = convertOne(sub) && ok
  319. }
  320. }
  321. converted[mkFile] = ss
  322. return ok
  323. }
  324. // Optionally saves the previous version of the generated file
  325. func maybeBackup(filename string) error {
  326. stat, err := os.Stat(filename)
  327. if os.IsNotExist(err) {
  328. return nil
  329. }
  330. if !stat.Mode().IsRegular() {
  331. return fmt.Errorf("%s exists and is not a regular file", filename)
  332. }
  333. switch *mode {
  334. case "backup":
  335. return os.Rename(filename, filename+backupSuffix)
  336. case "write":
  337. return os.Remove(filename)
  338. default:
  339. return fmt.Errorf("%s already exists, use --mode option", filename)
  340. }
  341. }
  342. func outputFilePath(mkFile string) string {
  343. path := strings.TrimSuffix(mkFile, filepath.Ext(mkFile)) + *suffix
  344. if *outputTop != "" {
  345. path = filepath.Join(*outputTop, path)
  346. }
  347. return path
  348. }
  349. func writeGenerated(path string, contents string) error {
  350. if err := os.MkdirAll(filepath.Dir(path), os.ModeDir|os.ModePerm); err != nil {
  351. return err
  352. }
  353. if err := ioutil.WriteFile(path, []byte(contents), 0644); err != nil {
  354. return err
  355. }
  356. return nil
  357. }
  358. func printStats() {
  359. var sortedFiles []string
  360. if !*warn && !*verbose {
  361. return
  362. }
  363. for p := range converted {
  364. sortedFiles = append(sortedFiles, p)
  365. }
  366. sort.Strings(sortedFiles)
  367. nOk, nPartial, nFailed := 0, 0, 0
  368. for _, f := range sortedFiles {
  369. if converted[f] == nil {
  370. nFailed++
  371. } else if converted[f].HasErrors() {
  372. nPartial++
  373. } else {
  374. nOk++
  375. }
  376. }
  377. if *warn {
  378. if nPartial > 0 {
  379. fmt.Fprintf(os.Stderr, "Conversion was partially successful for:\n")
  380. for _, f := range sortedFiles {
  381. if ss := converted[f]; ss != nil && ss.HasErrors() {
  382. fmt.Fprintln(os.Stderr, " ", f)
  383. }
  384. }
  385. }
  386. if nFailed > 0 {
  387. fmt.Fprintf(os.Stderr, "Conversion failed for files:\n")
  388. for _, f := range sortedFiles {
  389. if converted[f] == nil {
  390. fmt.Fprintln(os.Stderr, " ", f)
  391. }
  392. }
  393. }
  394. }
  395. if *verbose {
  396. fmt.Fprintf(os.Stderr, "%-16s%5d\n", "Succeeded:", nOk)
  397. fmt.Fprintf(os.Stderr, "%-16s%5d\n", "Partial:", nPartial)
  398. fmt.Fprintf(os.Stderr, "%-16s%5d\n", "Failed:", nFailed)
  399. }
  400. }
  401. type datum struct {
  402. count int
  403. formattingArgs []string
  404. }
  405. type errorsByType struct {
  406. data map[string]datum
  407. }
  408. func (ebt errorsByType) NewError(message string, node parser.Node, args ...interface{}) {
  409. v, exists := ebt.data[message]
  410. if exists {
  411. v.count++
  412. } else {
  413. v = datum{1, nil}
  414. }
  415. if strings.Contains(message, "%s") {
  416. var newArg1 string
  417. if len(args) == 0 {
  418. panic(fmt.Errorf(`%s has %%s but args are missing`, message))
  419. }
  420. newArg1 = fmt.Sprint(args[0])
  421. if message == "unsupported line" {
  422. newArg1 = node.Dump()
  423. } else if message == "unsupported directive %s" {
  424. if newArg1 == "include" || newArg1 == "-include" {
  425. newArg1 = node.Dump()
  426. }
  427. }
  428. v.formattingArgs = append(v.formattingArgs, newArg1)
  429. }
  430. ebt.data[message] = v
  431. }
  432. func (ebt errorsByType) printStatistics() {
  433. if len(ebt.data) > 0 {
  434. fmt.Fprintln(os.Stderr, "Error counts:")
  435. }
  436. for message, data := range ebt.data {
  437. if len(data.formattingArgs) == 0 {
  438. fmt.Fprintf(os.Stderr, "%4d %s\n", data.count, message)
  439. continue
  440. }
  441. itemsByFreq, count := stringsWithFreq(data.formattingArgs, 30)
  442. fmt.Fprintf(os.Stderr, "%4d %s [%d unique items]:\n", data.count, message, count)
  443. fmt.Fprintln(os.Stderr, " ", itemsByFreq)
  444. }
  445. }
  446. func stringsWithFreq(items []string, topN int) (string, int) {
  447. freq := make(map[string]int)
  448. for _, item := range items {
  449. freq[strings.TrimPrefix(strings.TrimSuffix(item, "]"), "[")]++
  450. }
  451. var sorted []string
  452. for item := range freq {
  453. sorted = append(sorted, item)
  454. }
  455. sort.Slice(sorted, func(i int, j int) bool {
  456. return freq[sorted[i]] > freq[sorted[j]]
  457. })
  458. sep := ""
  459. res := ""
  460. for i, item := range sorted {
  461. if i >= topN {
  462. res += " ..."
  463. break
  464. }
  465. count := freq[item]
  466. if count > 1 {
  467. res += fmt.Sprintf("%s%s(%d)", sep, item, count)
  468. } else {
  469. res += fmt.Sprintf("%s%s", sep, item)
  470. }
  471. sep = ", "
  472. }
  473. return res, len(sorted)
  474. }
  475. type LinuxMakefileFinder struct {
  476. cachedRoot string
  477. cachedMakefiles []string
  478. }
  479. func (l *LinuxMakefileFinder) Find(root string) []string {
  480. if l.cachedMakefiles != nil && l.cachedRoot == root {
  481. return l.cachedMakefiles
  482. }
  483. l.cachedRoot = root
  484. l.cachedMakefiles = make([]string, 0)
  485. // Return all *.mk files but not in hidden directories.
  486. // NOTE(asmundak): as it turns out, even the WalkDir (which is an _optimized_ directory tree walker)
  487. // is about twice slower than running `find` command (14s vs 6s on the internal Android source tree).
  488. common_args := []string{"!", "-type", "d", "-name", "*.mk", "!", "-path", "*/.*/*"}
  489. if root != "" {
  490. common_args = append([]string{root}, common_args...)
  491. }
  492. cmd := exec.Command("/usr/bin/find", common_args...)
  493. stdout, err := cmd.StdoutPipe()
  494. if err == nil {
  495. err = cmd.Start()
  496. }
  497. if err != nil {
  498. panic(fmt.Errorf("cannot get the output from %s: %s", cmd, err))
  499. }
  500. scanner := bufio.NewScanner(stdout)
  501. for scanner.Scan() {
  502. l.cachedMakefiles = append(l.cachedMakefiles, strings.TrimPrefix(scanner.Text(), "./"))
  503. }
  504. stdout.Close()
  505. return l.cachedMakefiles
  506. }