123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622 |
- // Copyright 2021 Google LLC
- //
- // Licensed under the Apache License, Version 2.0 (the "License");
- // you may not use this file except in compliance with the License.
- // You may obtain a copy of the License at
- //
- // http://www.apache.org/licenses/LICENSE-2.0
- //
- // Unless required by applicable law or agreed to in writing, software
- // distributed under the License is distributed on an "AS IS" BASIS,
- // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- // See the License for the specific language governing permissions and
- // limitations under the License.
- // The application to convert product configuration makefiles to Starlark.
- // Converts either given list of files (and optionally the dependent files
- // of the same kind), or all all product configuration makefiles in the
- // given source tree.
- // Previous version of a converted file can be backed up.
- // Optionally prints detailed statistics at the end.
- package main
- import (
- "bufio"
- "flag"
- "fmt"
- "io/ioutil"
- "os"
- "os/exec"
- "path/filepath"
- "regexp"
- "runtime/debug"
- "runtime/pprof"
- "sort"
- "strings"
- "time"
- "android/soong/androidmk/parser"
- "android/soong/mk2rbc"
- )
- var (
- // TODO(asmundak): remove this option once there is a consensus on suffix
- suffix = flag.String("suffix", ".rbc", "generated files' suffix")
- dryRun = flag.Bool("dry_run", false, "dry run")
- recurse = flag.Bool("convert_dependents", false, "convert all dependent files")
- mode = flag.String("mode", "", `"backup" to back up existing files, "write" to overwrite them`)
- errstat = flag.Bool("error_stat", false, "print error statistics")
- traceVar = flag.String("trace", "", "comma-separated list of variables to trace")
- // TODO(asmundak): this option is for debugging
- allInSource = flag.Bool("all", false, "convert all product config makefiles in the tree under //")
- outputTop = flag.String("outdir", "", "write output files into this directory hierarchy")
- launcher = flag.String("launcher", "", "generated launcher path.")
- boardlauncher = flag.String("boardlauncher", "", "generated board configuration launcher path.")
- printProductConfigMap = flag.Bool("print_product_config_map", false, "print product config map and exit")
- cpuProfile = flag.String("cpu_profile", "", "write cpu profile to file")
- traceCalls = flag.Bool("trace_calls", false, "trace function calls")
- inputVariables = flag.String("input_variables", "", "starlark file containing product config and global variables")
- makefileList = flag.String("makefile_list", "", "path to a list of all makefiles in the source tree, generated by soong's finder. If not provided, mk2rbc will find the makefiles itself (more slowly than if this flag was provided)")
- )
- func init() {
- // Simplistic flag aliasing: works, but the usage string is ugly and
- // both flag and its alias can be present on the command line
- flagAlias := func(target string, alias string) {
- if f := flag.Lookup(target); f != nil {
- flag.Var(f.Value, alias, "alias for --"+f.Name)
- return
- }
- quit("cannot alias unknown flag " + target)
- }
- flagAlias("suffix", "s")
- flagAlias("dry_run", "n")
- flagAlias("convert_dependents", "r")
- flagAlias("error_stat", "e")
- }
- var backupSuffix string
- var tracedVariables []string
- var errorLogger = errorSink{data: make(map[string]datum)}
- var makefileFinder mk2rbc.MakefileFinder
- func main() {
- flag.Usage = func() {
- cmd := filepath.Base(os.Args[0])
- fmt.Fprintf(flag.CommandLine.Output(),
- "Usage: %[1]s flags file...\n", cmd)
- flag.PrintDefaults()
- }
- flag.Parse()
- if _, err := os.Stat("build/soong/mk2rbc"); err != nil {
- quit("Must be run from the root of the android tree. (build/soong/mk2rbc does not exist)")
- }
- // Delouse
- if *suffix == ".mk" {
- quit("cannot use .mk as generated file suffix")
- }
- if *suffix == "" {
- quit("suffix cannot be empty")
- }
- if *outputTop != "" {
- if err := os.MkdirAll(*outputTop, os.ModeDir+os.ModePerm); err != nil {
- quit(err)
- }
- s, err := filepath.Abs(*outputTop)
- if err != nil {
- quit(err)
- }
- *outputTop = s
- }
- if *allInSource && len(flag.Args()) > 0 {
- quit("file list cannot be specified when -all is present")
- }
- if *allInSource && *launcher != "" {
- quit("--all and --launcher are mutually exclusive")
- }
- // Flag-driven adjustments
- if (*suffix)[0] != '.' {
- *suffix = "." + *suffix
- }
- if *mode == "backup" {
- backupSuffix = time.Now().Format("20060102150405")
- }
- if *traceVar != "" {
- tracedVariables = strings.Split(*traceVar, ",")
- }
- if *cpuProfile != "" {
- f, err := os.Create(*cpuProfile)
- if err != nil {
- quit(err)
- }
- pprof.StartCPUProfile(f)
- defer pprof.StopCPUProfile()
- }
- if *makefileList != "" {
- makefileFinder = &FileListMakefileFinder{
- cachedMakefiles: nil,
- filePath: *makefileList,
- }
- } else {
- makefileFinder = &FindCommandMakefileFinder{}
- }
- // Find out global variables
- getConfigVariables()
- getSoongVariables()
- if *printProductConfigMap {
- productConfigMap := buildProductConfigMap()
- var products []string
- for p := range productConfigMap {
- products = append(products, p)
- }
- sort.Strings(products)
- for _, p := range products {
- fmt.Println(p, productConfigMap[p])
- }
- os.Exit(0)
- }
- // Convert!
- files := flag.Args()
- if *allInSource {
- productConfigMap := buildProductConfigMap()
- for _, path := range productConfigMap {
- files = append(files, path)
- }
- }
- ok := true
- for _, mkFile := range files {
- ok = convertOne(mkFile, []string{}) && ok
- }
- if *launcher != "" {
- if len(files) != 1 {
- quit(fmt.Errorf("a launcher can be generated only for a single product"))
- }
- if *inputVariables == "" {
- quit(fmt.Errorf("the product launcher requires an input variables file"))
- }
- if !convertOne(*inputVariables, []string{}) {
- quit(fmt.Errorf("the product launcher input variables file failed to convert"))
- }
- err := writeGenerated(*launcher, mk2rbc.Launcher(outputFilePath(files[0]), outputFilePath(*inputVariables),
- mk2rbc.MakePath2ModuleName(files[0])))
- if err != nil {
- fmt.Fprintf(os.Stderr, "%s: %s", files[0], err)
- ok = false
- }
- }
- if *boardlauncher != "" {
- if len(files) != 1 {
- quit(fmt.Errorf("a launcher can be generated only for a single product"))
- }
- if *inputVariables == "" {
- quit(fmt.Errorf("the board launcher requires an input variables file"))
- }
- if !convertOne(*inputVariables, []string{}) {
- quit(fmt.Errorf("the board launcher input variables file failed to convert"))
- }
- err := writeGenerated(*boardlauncher, mk2rbc.BoardLauncher(
- outputFilePath(files[0]), outputFilePath(*inputVariables)))
- if err != nil {
- fmt.Fprintf(os.Stderr, "%s: %s", files[0], err)
- ok = false
- }
- }
- if *errstat {
- errorLogger.printStatistics()
- printStats()
- }
- if !ok {
- os.Exit(1)
- }
- }
- func quit(s interface{}) {
- fmt.Fprintln(os.Stderr, s)
- os.Exit(2)
- }
- func buildProductConfigMap() map[string]string {
- const androidProductsMk = "AndroidProducts.mk"
- // Build the list of AndroidProducts.mk files: it's
- // build/make/target/product/AndroidProducts.mk + device/**/AndroidProducts.mk plus + vendor/**/AndroidProducts.mk
- targetAndroidProductsFile := filepath.Join("build", "make", "target", "product", androidProductsMk)
- if _, err := os.Stat(targetAndroidProductsFile); err != nil {
- fmt.Fprintf(os.Stderr, "%s: %s\n", targetAndroidProductsFile, err)
- }
- productConfigMap := make(map[string]string)
- if err := mk2rbc.UpdateProductConfigMap(productConfigMap, targetAndroidProductsFile); err != nil {
- fmt.Fprintf(os.Stderr, "%s: %s\n", targetAndroidProductsFile, err)
- }
- for _, t := range []string{"device", "vendor"} {
- _ = filepath.WalkDir(t,
- func(path string, d os.DirEntry, err error) error {
- if err != nil || d.IsDir() || filepath.Base(path) != androidProductsMk {
- return nil
- }
- if err2 := mk2rbc.UpdateProductConfigMap(productConfigMap, path); err2 != nil {
- fmt.Fprintf(os.Stderr, "%s: %s\n", path, err)
- // Keep going, we want to find all such errors in a single run
- }
- return nil
- })
- }
- return productConfigMap
- }
- func getConfigVariables() {
- path := filepath.Join("build", "make", "core", "product.mk")
- if err := mk2rbc.FindConfigVariables(path, mk2rbc.KnownVariables); err != nil {
- quit(err)
- }
- }
- // Implements mkparser.Scope, to be used by mkparser.Value.Value()
- type fileNameScope struct {
- mk2rbc.ScopeBase
- }
- func (s fileNameScope) Get(name string) string {
- if name != "BUILD_SYSTEM" {
- return fmt.Sprintf("$(%s)", name)
- }
- return filepath.Join("build", "make", "core")
- }
- func getSoongVariables() {
- path := filepath.Join("build", "make", "core", "soong_config.mk")
- err := mk2rbc.FindSoongVariables(path, fileNameScope{}, mk2rbc.KnownVariables)
- if err != nil {
- quit(err)
- }
- }
- var converted = make(map[string]*mk2rbc.StarlarkScript)
- //goland:noinspection RegExpRepeatedSpace
- var cpNormalizer = regexp.MustCompile(
- "# Copyright \\(C\\) 20.. The Android Open Source Project")
- const cpNormalizedCopyright = "# Copyright (C) 20xx The Android Open Source Project"
- const copyright = `#
- # Copyright (C) 20xx The Android Open Source Project
- #
- # Licensed under the Apache License, Version 2.0 (the "License");
- # you may not use this file except in compliance with the License.
- # You may obtain a copy of the License at
- #
- # http://www.apache.org/licenses/LICENSE-2.0
- #
- # Unless required by applicable law or agreed to in writing, software
- # distributed under the License is distributed on an "AS IS" BASIS,
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- # See the License for the specific language governing permissions and
- # limitations under the License.
- #
- `
- // Convert a single file.
- // Write the result either to the same directory, to the same place in
- // the output hierarchy, or to the stdout.
- // Optionally, recursively convert the files this one includes by
- // $(call inherit-product) or an include statement.
- func convertOne(mkFile string, loadStack []string) (ok bool) {
- if v, ok := converted[mkFile]; ok {
- if v == nil {
- fmt.Fprintf(os.Stderr, "Cycle in load graph:\n%s\n%s\n\n", strings.Join(loadStack, "\n"), mkFile)
- return false
- }
- return true
- }
- converted[mkFile] = nil
- defer func() {
- if r := recover(); r != nil {
- ok = false
- fmt.Fprintf(os.Stderr, "%s: panic while converting: %s\n%s\n", mkFile, r, debug.Stack())
- }
- }()
- mk2starRequest := mk2rbc.Request{
- MkFile: mkFile,
- Reader: nil,
- OutputDir: *outputTop,
- OutputSuffix: *suffix,
- TracedVariables: tracedVariables,
- TraceCalls: *traceCalls,
- SourceFS: os.DirFS("."),
- MakefileFinder: makefileFinder,
- ErrorLogger: errorLogger,
- }
- ss, err := mk2rbc.Convert(mk2starRequest)
- if err != nil {
- fmt.Fprintln(os.Stderr, mkFile, ": ", err)
- return false
- }
- script := ss.String()
- outputPath := outputFilePath(mkFile)
- if *dryRun {
- fmt.Printf("==== %s ====\n", outputPath)
- // Print generated script after removing the copyright header
- outText := cpNormalizer.ReplaceAllString(script, cpNormalizedCopyright)
- fmt.Println(strings.TrimPrefix(outText, copyright))
- } else {
- if err := maybeBackup(outputPath); err != nil {
- fmt.Fprintln(os.Stderr, err)
- return false
- }
- if err := writeGenerated(outputPath, script); err != nil {
- fmt.Fprintln(os.Stderr, err)
- return false
- }
- }
- loadStack = append(loadStack, mkFile)
- ok = true
- if *recurse {
- for _, sub := range ss.SubConfigFiles() {
- // File may be absent if it is a conditional load
- if _, err := os.Stat(sub); os.IsNotExist(err) {
- continue
- }
- ok = convertOne(sub, loadStack) && ok
- }
- }
- converted[mkFile] = ss
- return ok
- }
- // Optionally saves the previous version of the generated file
- func maybeBackup(filename string) error {
- stat, err := os.Stat(filename)
- if os.IsNotExist(err) {
- return nil
- }
- if !stat.Mode().IsRegular() {
- return fmt.Errorf("%s exists and is not a regular file", filename)
- }
- switch *mode {
- case "backup":
- return os.Rename(filename, filename+backupSuffix)
- case "write":
- return os.Remove(filename)
- default:
- return fmt.Errorf("%s already exists, use --mode option", filename)
- }
- }
- func outputFilePath(mkFile string) string {
- path := strings.TrimSuffix(mkFile, filepath.Ext(mkFile)) + *suffix
- if *outputTop != "" {
- path = filepath.Join(*outputTop, path)
- }
- return path
- }
- func writeGenerated(path string, contents string) error {
- if err := os.MkdirAll(filepath.Dir(path), os.ModeDir|os.ModePerm); err != nil {
- return err
- }
- if err := ioutil.WriteFile(path, []byte(contents), 0644); err != nil {
- return err
- }
- return nil
- }
- func printStats() {
- var sortedFiles []string
- for p := range converted {
- sortedFiles = append(sortedFiles, p)
- }
- sort.Strings(sortedFiles)
- nOk, nPartial, nFailed := 0, 0, 0
- for _, f := range sortedFiles {
- if converted[f] == nil {
- nFailed++
- } else if converted[f].HasErrors() {
- nPartial++
- } else {
- nOk++
- }
- }
- if nPartial > 0 {
- fmt.Fprintf(os.Stderr, "Conversion was partially successful for:\n")
- for _, f := range sortedFiles {
- if ss := converted[f]; ss != nil && ss.HasErrors() {
- fmt.Fprintln(os.Stderr, " ", f)
- }
- }
- }
- if nFailed > 0 {
- fmt.Fprintf(os.Stderr, "Conversion failed for files:\n")
- for _, f := range sortedFiles {
- if converted[f] == nil {
- fmt.Fprintln(os.Stderr, " ", f)
- }
- }
- }
- }
- type datum struct {
- count int
- formattingArgs []string
- }
- type errorSink struct {
- data map[string]datum
- }
- func (ebt errorSink) NewError(el mk2rbc.ErrorLocation, node parser.Node, message string, args ...interface{}) {
- fmt.Fprint(os.Stderr, el, ": ")
- fmt.Fprintf(os.Stderr, message, args...)
- fmt.Fprintln(os.Stderr)
- if !*errstat {
- return
- }
- v, exists := ebt.data[message]
- if exists {
- v.count++
- } else {
- v = datum{1, nil}
- }
- if strings.Contains(message, "%s") {
- var newArg1 string
- if len(args) == 0 {
- panic(fmt.Errorf(`%s has %%s but args are missing`, message))
- }
- newArg1 = fmt.Sprint(args[0])
- if message == "unsupported line" {
- newArg1 = node.Dump()
- } else if message == "unsupported directive %s" {
- if newArg1 == "include" || newArg1 == "-include" {
- newArg1 = node.Dump()
- }
- }
- v.formattingArgs = append(v.formattingArgs, newArg1)
- }
- ebt.data[message] = v
- }
- func (ebt errorSink) printStatistics() {
- if len(ebt.data) > 0 {
- fmt.Fprintln(os.Stderr, "Error counts:")
- }
- for message, data := range ebt.data {
- if len(data.formattingArgs) == 0 {
- fmt.Fprintf(os.Stderr, "%4d %s\n", data.count, message)
- continue
- }
- itemsByFreq, count := stringsWithFreq(data.formattingArgs, 30)
- fmt.Fprintf(os.Stderr, "%4d %s [%d unique items]:\n", data.count, message, count)
- fmt.Fprintln(os.Stderr, " ", itemsByFreq)
- }
- }
- func stringsWithFreq(items []string, topN int) (string, int) {
- freq := make(map[string]int)
- for _, item := range items {
- freq[strings.TrimPrefix(strings.TrimSuffix(item, "]"), "[")]++
- }
- var sorted []string
- for item := range freq {
- sorted = append(sorted, item)
- }
- sort.Slice(sorted, func(i int, j int) bool {
- return freq[sorted[i]] > freq[sorted[j]]
- })
- sep := ""
- res := ""
- for i, item := range sorted {
- if i >= topN {
- res += " ..."
- break
- }
- count := freq[item]
- if count > 1 {
- res += fmt.Sprintf("%s%s(%d)", sep, item, count)
- } else {
- res += fmt.Sprintf("%s%s", sep, item)
- }
- sep = ", "
- }
- return res, len(sorted)
- }
- // FindCommandMakefileFinder is an implementation of mk2rbc.MakefileFinder that
- // runs the unix find command to find all the makefiles in the source tree.
- type FindCommandMakefileFinder struct {
- cachedRoot string
- cachedMakefiles []string
- }
- func (l *FindCommandMakefileFinder) Find(root string) []string {
- if l.cachedMakefiles != nil && l.cachedRoot == root {
- return l.cachedMakefiles
- }
- // Return all *.mk files but not in hidden directories.
- // NOTE(asmundak): as it turns out, even the WalkDir (which is an _optimized_ directory tree walker)
- // is about twice slower than running `find` command (14s vs 6s on the internal Android source tree).
- common_args := []string{"!", "-type", "d", "-name", "*.mk", "!", "-path", "*/.*/*"}
- if root != "" {
- common_args = append([]string{root}, common_args...)
- }
- cmd := exec.Command("/usr/bin/find", common_args...)
- stdout, err := cmd.StdoutPipe()
- if err == nil {
- err = cmd.Start()
- }
- if err != nil {
- panic(fmt.Errorf("cannot get the output from %s: %s", cmd, err))
- }
- scanner := bufio.NewScanner(stdout)
- result := make([]string, 0)
- for scanner.Scan() {
- result = append(result, strings.TrimPrefix(scanner.Text(), "./"))
- }
- stdout.Close()
- err = scanner.Err()
- if err != nil {
- panic(fmt.Errorf("cannot get the output from %s: %s", cmd, err))
- }
- l.cachedRoot = root
- l.cachedMakefiles = result
- return l.cachedMakefiles
- }
- // FileListMakefileFinder is an implementation of mk2rbc.MakefileFinder that
- // reads a file containing the list of makefiles in the android source tree.
- // This file is generated by soong's finder, so that it can be computed while
- // soong is already walking the source tree looking for other files. If the root
- // to find makefiles under is not the root of the android source tree, it will
- // fall back to using FindCommandMakefileFinder.
- type FileListMakefileFinder struct {
- FindCommandMakefileFinder
- cachedMakefiles []string
- filePath string
- }
- func (l *FileListMakefileFinder) Find(root string) []string {
- root, err1 := filepath.Abs(root)
- wd, err2 := os.Getwd()
- if root != wd || err1 != nil || err2 != nil {
- return l.FindCommandMakefileFinder.Find(root)
- }
- if l.cachedMakefiles != nil {
- return l.cachedMakefiles
- }
- file, err := os.Open(l.filePath)
- if err != nil {
- panic(fmt.Errorf("Cannot read makefile list: %s\n", err))
- }
- defer file.Close()
- result := make([]string, 0)
- scanner := bufio.NewScanner(file)
- for scanner.Scan() {
- line := scanner.Text()
- if len(line) > 0 {
- result = append(result, line)
- }
- }
- if err = scanner.Err(); err != nil {
- panic(fmt.Errorf("Cannot read makefile list: %s\n", err))
- }
- l.cachedMakefiles = result
- return l.cachedMakefiles
- }
|