Преглед на файлове

Product config makefiles to Starlark converter

Test: treehugger; internal tests in mk2rbc_test.go
Bug: 172923994
Change-Id: I43120b9c181ef2b8d9453e743233811b0fec268b
Sasha Smundak преди 3 години
родител
ревизия
b051c4ede3
променени са 11 файла, в които са добавени 3885 реда и са изтрити 1 реда
  1. 9 0
      androidmk/parser/make_strings.go
  2. 2 1
      androidmk/parser/parser.go
  3. 39 0
      mk2rbc/Android.bp
  4. 14 0
      mk2rbc/TODO
  5. 498 0
      mk2rbc/cmd/mk2rbc.go
  6. 580 0
      mk2rbc/expr.go
  7. 1344 0
      mk2rbc/mk2rbc.go
  8. 857 0
      mk2rbc/mk2rbc_test.go
  9. 237 0
      mk2rbc/node.go
  10. 5 0
      mk2rbc/types.go
  11. 300 0
      mk2rbc/variable.go

+ 9 - 0
androidmk/parser/make_strings.go

@@ -278,6 +278,15 @@ func (ms *MakeString) ReplaceLiteral(input string, output string) {
 	}
 }
 
+// If MakeString is $(var) after trimming, returns var
+func (ms *MakeString) SingleVariable() (*MakeString, bool) {
+	if len(ms.Strings) != 2 || strings.TrimSpace(ms.Strings[0]) != "" ||
+		strings.TrimSpace(ms.Strings[1]) != "" {
+		return nil, false
+	}
+	return ms.Variables[0].Name, true
+}
+
 func splitAnyN(s, sep string, n int) []string {
 	ret := []string{}
 	for n == -1 || n > 1 {

+ 2 - 1
androidmk/parser/parser.go

@@ -216,13 +216,14 @@ func (p *parser) parseDirective() bool {
 		// Nothing
 	case "else":
 		p.ignoreSpaces()
-		if p.tok != '\n' {
+		if p.tok != '\n' && p.tok != '#' {
 			d = p.scanner.TokenText()
 			p.accept(scanner.Ident)
 			if d == "ifdef" || d == "ifndef" || d == "ifeq" || d == "ifneq" {
 				d = "el" + d
 				p.ignoreSpaces()
 				expression = p.parseExpression()
+				expression.TrimRightSpaces()
 			} else {
 				p.errorf("expected ifdef/ifndef/ifeq/ifneq, found %s", d)
 			}

+ 39 - 0
mk2rbc/Android.bp

@@ -0,0 +1,39 @@
+//
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+blueprint_go_binary {
+    name: "mk2rbc",
+    srcs: ["cmd/mk2rbc.go"],
+    deps: [
+        "mk2rbc-lib",
+        "androidmk-parser",
+    ],
+}
+
+bootstrap_go_package {
+    name: "mk2rbc-lib",
+    pkgPath: "android/soong/mk2rbc",
+    srcs: [
+        "android_products.go",
+        "config_variables.go",
+        "expr.go",
+        "mk2rbc.go",
+        "node.go",
+        "soong_variables.go",
+        "types.go",
+        "variable.go",
+    ],
+    deps: ["androidmk-parser"],
+}

+ 14 - 0
mk2rbc/TODO

@@ -0,0 +1,14 @@
+* Checking filter/filter-out results is incorrect if pattern contains '%'
+* Need heuristics to recognize that a variable is local. Propose to use lowercase.
+* Need heuristics for the local variable type. Propose '_list' suffix
+* Internal source tree has variables in the inherit-product macro argument. Handle it
+* Enumerate all environment variables that configuration files use.
+* Break mk2rbc.go into multiple files.
+* If variable's type is not yet known, try to divine it from the value assigned to it
+  (it may be a variable of the known type, or a function result)
+* ifneq (,$(VAR)) should translate to
+    if getattr(<>, "VAR", <default>):
+* Launcher file needs to have same suffix as the rest of the generated files
+* Implement $(shell) function
+* Write execution tests
+* Review all TODOs in mk2rbc.go

+ 498 - 0
mk2rbc/cmd/mk2rbc.go

@@ -0,0 +1,498 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// The application to convert product configuration makefiles to Starlark.
+// Converts either given list of files (and optionally the dependent files
+// of the same kind), or all all product configuration makefiles in the
+// given source tree.
+// Previous version of a converted file can be backed up.
+// Optionally prints detailed statistics at the end.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"regexp"
+	"runtime/debug"
+	"sort"
+	"strings"
+	"time"
+
+	"android/soong/androidmk/parser"
+	"android/soong/mk2rbc"
+)
+
+var (
+	rootDir = flag.String("root", ".", "the value of // for load paths")
+	// TODO(asmundak): remove this option once there is a consensus on suffix
+	suffix   = flag.String("suffix", ".rbc", "generated files' suffix")
+	dryRun   = flag.Bool("dry_run", false, "dry run")
+	recurse  = flag.Bool("convert_dependents", false, "convert all dependent files")
+	mode     = flag.String("mode", "", `"backup" to back up existing files, "write" to overwrite them`)
+	warn     = flag.Bool("warnings", false, "warn about partially failed conversions")
+	verbose  = flag.Bool("v", false, "print summary")
+	errstat  = flag.Bool("error_stat", false, "print error statistics")
+	traceVar = flag.String("trace", "", "comma-separated list of variables to trace")
+	// TODO(asmundak): this option is for debugging
+	allInSource           = flag.Bool("all", false, "convert all product config makefiles in the tree under //")
+	outputTop             = flag.String("outdir", "", "write output files into this directory hierarchy")
+	launcher              = flag.String("launcher", "", "generated launcher path. If set, the non-flag argument is _product_name_")
+	printProductConfigMap = flag.Bool("print_product_config_map", false, "print product config map and exit")
+	traceCalls            = flag.Bool("trace_calls", false, "trace function calls")
+)
+
+func init() {
+	// Poor man's flag aliasing: works, but the usage string is ugly and
+	// both flag and its alias can be present on the command line
+	flagAlias := func(target string, alias string) {
+		if f := flag.Lookup(target); f != nil {
+			flag.Var(f.Value, alias, "alias for --"+f.Name)
+			return
+		}
+		quit("cannot alias unknown flag " + target)
+	}
+	flagAlias("suffix", "s")
+	flagAlias("root", "d")
+	flagAlias("dry_run", "n")
+	flagAlias("convert_dependents", "r")
+	flagAlias("warnings", "w")
+	flagAlias("error_stat", "e")
+}
+
+var backupSuffix string
+var tracedVariables []string
+var errorLogger = errorsByType{data: make(map[string]datum)}
+
+func main() {
+	flag.Usage = func() {
+		cmd := filepath.Base(os.Args[0])
+		fmt.Fprintf(flag.CommandLine.Output(),
+			"Usage: %[1]s flags file...\n"+
+				"or:    %[1]s flags --launcher=PATH PRODUCT\n", cmd)
+		flag.PrintDefaults()
+	}
+	flag.Parse()
+
+	// Delouse
+	if *suffix == ".mk" {
+		quit("cannot use .mk as generated file suffix")
+	}
+	if *suffix == "" {
+		quit("suffix cannot be empty")
+	}
+	if *outputTop != "" {
+		if err := os.MkdirAll(*outputTop, os.ModeDir+os.ModePerm); err != nil {
+			quit(err)
+		}
+		s, err := filepath.Abs(*outputTop)
+		if err != nil {
+			quit(err)
+		}
+		*outputTop = s
+	}
+	if *allInSource && len(flag.Args()) > 0 {
+		quit("file list cannot be specified when -all is present")
+	}
+	if *allInSource && *launcher != "" {
+		quit("--all and --launcher are mutually exclusive")
+	}
+
+	// Flag-driven adjustments
+	if (*suffix)[0] != '.' {
+		*suffix = "." + *suffix
+	}
+	if *mode == "backup" {
+		backupSuffix = time.Now().Format("20060102150405")
+	}
+	if *traceVar != "" {
+		tracedVariables = strings.Split(*traceVar, ",")
+	}
+
+	// Find out global variables
+	getConfigVariables()
+	getSoongVariables()
+
+	if *printProductConfigMap {
+		productConfigMap := buildProductConfigMap()
+		var products []string
+		for p := range productConfigMap {
+			products = append(products, p)
+		}
+		sort.Strings(products)
+		for _, p := range products {
+			fmt.Println(p, productConfigMap[p])
+		}
+		os.Exit(0)
+	}
+	if len(flag.Args()) == 0 {
+		flag.Usage()
+	}
+	// Convert!
+	ok := true
+	if *launcher != "" {
+		if len(flag.Args()) != 1 {
+			quit(fmt.Errorf("a launcher can be generated only for a single product"))
+		}
+		product := flag.Args()[0]
+		productConfigMap := buildProductConfigMap()
+		path, found := productConfigMap[product]
+		if !found {
+			quit(fmt.Errorf("cannot generate configuration launcher for %s, it is not a known product",
+				product))
+		}
+		ok = convertOne(path) && ok
+		err := writeGenerated(*launcher, mk2rbc.Launcher(outputFilePath(path), mk2rbc.MakePath2ModuleName(path)))
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "%s:%s", path, err)
+			ok = false
+		}
+
+	} else {
+		files := flag.Args()
+		if *allInSource {
+			productConfigMap := buildProductConfigMap()
+			for _, path := range productConfigMap {
+				files = append(files, path)
+			}
+		}
+		for _, mkFile := range files {
+			ok = convertOne(mkFile) && ok
+		}
+	}
+
+	printStats()
+	if *errstat {
+		errorLogger.printStatistics()
+	}
+	if !ok {
+		os.Exit(1)
+	}
+}
+
+func quit(s interface{}) {
+	fmt.Fprintln(os.Stderr, s)
+	os.Exit(2)
+}
+
+func buildProductConfigMap() map[string]string {
+	const androidProductsMk = "AndroidProducts.mk"
+	// Build the list of AndroidProducts.mk files: it's
+	// build/make/target/product/AndroidProducts.mk plus
+	// device/**/AndroidProducts.mk
+	targetAndroidProductsFile := filepath.Join(*rootDir, "build", "make", "target", "product", androidProductsMk)
+	if _, err := os.Stat(targetAndroidProductsFile); err != nil {
+		fmt.Fprintf(os.Stderr, "%s: %s\n(hint: %s is not a source tree root)\n",
+			targetAndroidProductsFile, err, *rootDir)
+	}
+	productConfigMap := make(map[string]string)
+	if err := mk2rbc.UpdateProductConfigMap(productConfigMap, targetAndroidProductsFile); err != nil {
+		fmt.Fprintf(os.Stderr, "%s: %s\n", targetAndroidProductsFile, err)
+	}
+	_ = filepath.Walk(filepath.Join(*rootDir, "device"),
+		func(path string, info os.FileInfo, err error) error {
+			if info.IsDir() || filepath.Base(path) != androidProductsMk {
+				return nil
+			}
+			if err2 := mk2rbc.UpdateProductConfigMap(productConfigMap, path); err2 != nil {
+				fmt.Fprintf(os.Stderr, "%s: %s\n", path, err)
+				// Keep going, we want to find all such errors in a single run
+			}
+			return nil
+		})
+	return productConfigMap
+}
+
+func getConfigVariables() {
+	path := filepath.Join(*rootDir, "build", "make", "core", "product.mk")
+	if err := mk2rbc.FindConfigVariables(path, mk2rbc.KnownVariables); err != nil {
+		quit(fmt.Errorf("%s\n(check --root[=%s], it should point to the source root)",
+			err, *rootDir))
+	}
+}
+
+// Implements mkparser.Scope, to be used by mkparser.Value.Value()
+type fileNameScope struct {
+	mk2rbc.ScopeBase
+}
+
+func (s fileNameScope) Get(name string) string {
+	if name != "BUILD_SYSTEM" {
+		return fmt.Sprintf("$(%s)", name)
+	}
+	return filepath.Join(*rootDir, "build", "make", "core")
+}
+
+func getSoongVariables() {
+	path := filepath.Join(*rootDir, "build", "make", "core", "soong_config.mk")
+	err := mk2rbc.FindSoongVariables(path, fileNameScope{}, mk2rbc.KnownVariables)
+	if err != nil {
+		quit(err)
+	}
+}
+
+var converted = make(map[string]*mk2rbc.StarlarkScript)
+
+//goland:noinspection RegExpRepeatedSpace
+var cpNormalizer = regexp.MustCompile(
+	"#  Copyright \\(C\\) 20.. The Android Open Source Project")
+
+const cpNormalizedCopyright = "#  Copyright (C) 20xx The Android Open Source Project"
+const copyright = `#
+#  Copyright (C) 20xx The Android Open Source Project
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+`
+
+// Convert a single file.
+// Write the result either to the same directory, to the same place in
+// the output hierarchy, or to the stdout.
+// Optionally, recursively convert the files this one includes by
+// $(call inherit-product) or an include statement.
+func convertOne(mkFile string) (ok bool) {
+	if v, ok := converted[mkFile]; ok {
+		return v != nil
+	}
+	converted[mkFile] = nil
+	defer func() {
+		if r := recover(); r != nil {
+			ok = false
+			fmt.Fprintf(os.Stderr, "%s: panic while converting: %s\n%s\n", mkFile, r, debug.Stack())
+		}
+	}()
+
+	mk2starRequest := mk2rbc.Request{
+		MkFile:             mkFile,
+		Reader:             nil,
+		RootDir:            *rootDir,
+		OutputDir:          *outputTop,
+		OutputSuffix:       *suffix,
+		TracedVariables:    tracedVariables,
+		TraceCalls:         *traceCalls,
+		WarnPartialSuccess: *warn,
+	}
+	if *errstat {
+		mk2starRequest.ErrorLogger = errorLogger
+	}
+	ss, err := mk2rbc.Convert(mk2starRequest)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, mkFile, ": ", err)
+		return false
+	}
+	script := ss.String()
+	outputPath := outputFilePath(mkFile)
+
+	if *dryRun {
+		fmt.Printf("==== %s ====\n", outputPath)
+		// Print generated script after removing the copyright header
+		outText := cpNormalizer.ReplaceAllString(script, cpNormalizedCopyright)
+		fmt.Println(strings.TrimPrefix(outText, copyright))
+	} else {
+		if err := maybeBackup(outputPath); err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			return false
+		}
+		if err := writeGenerated(outputPath, script); err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			return false
+		}
+	}
+	ok = true
+	if *recurse {
+		for _, sub := range ss.SubConfigFiles() {
+			// File may be absent if it is a conditional load
+			if _, err := os.Stat(sub); os.IsNotExist(err) {
+				continue
+			}
+			ok = convertOne(sub) && ok
+		}
+	}
+	converted[mkFile] = ss
+	return ok
+}
+
+// Optionally saves the previous version of the generated file
+func maybeBackup(filename string) error {
+	stat, err := os.Stat(filename)
+	if os.IsNotExist(err) {
+		return nil
+	}
+	if !stat.Mode().IsRegular() {
+		return fmt.Errorf("%s exists and is not a regular file", filename)
+	}
+	switch *mode {
+	case "backup":
+		return os.Rename(filename, filename+backupSuffix)
+	case "write":
+		return os.Remove(filename)
+	default:
+		return fmt.Errorf("%s already exists, use --mode option", filename)
+	}
+}
+
+func outputFilePath(mkFile string) string {
+	path := strings.TrimSuffix(mkFile, filepath.Ext(mkFile)) + *suffix
+	if *outputTop != "" {
+		path = filepath.Join(*outputTop, path)
+	}
+	return path
+}
+
+func writeGenerated(path string, contents string) error {
+	if err := os.MkdirAll(filepath.Dir(path), os.ModeDir|os.ModePerm); err != nil {
+		return err
+	}
+	if err := ioutil.WriteFile(path, []byte(contents), 0644); err != nil {
+		return err
+	}
+	return nil
+}
+
+func printStats() {
+	var sortedFiles []string
+	if !*warn && !*verbose {
+		return
+	}
+	for p := range converted {
+		sortedFiles = append(sortedFiles, p)
+	}
+	sort.Strings(sortedFiles)
+
+	nOk, nPartial, nFailed := 0, 0, 0
+	for _, f := range sortedFiles {
+		if converted[f] == nil {
+			nFailed++
+		} else if converted[f].HasErrors() {
+			nPartial++
+		} else {
+			nOk++
+		}
+	}
+	if *warn {
+		if nPartial > 0 {
+			fmt.Fprintf(os.Stderr, "Conversion was partially successful for:\n")
+			for _, f := range sortedFiles {
+				if ss := converted[f]; ss != nil && ss.HasErrors() {
+					fmt.Fprintln(os.Stderr, "  ", f)
+				}
+			}
+		}
+
+		if nFailed > 0 {
+			fmt.Fprintf(os.Stderr, "Conversion failed for files:\n")
+			for _, f := range sortedFiles {
+				if converted[f] == nil {
+					fmt.Fprintln(os.Stderr, "  ", f)
+				}
+			}
+		}
+	}
+	if *verbose {
+		fmt.Fprintf(os.Stderr, "%-16s%5d\n", "Succeeded:", nOk)
+		fmt.Fprintf(os.Stderr, "%-16s%5d\n", "Partial:", nPartial)
+		fmt.Fprintf(os.Stderr, "%-16s%5d\n", "Failed:", nFailed)
+	}
+}
+
+type datum struct {
+	count          int
+	formattingArgs []string
+}
+
+type errorsByType struct {
+	data map[string]datum
+}
+
+func (ebt errorsByType) NewError(message string, node parser.Node, args ...interface{}) {
+	v, exists := ebt.data[message]
+	if exists {
+		v.count++
+	} else {
+		v = datum{1, nil}
+	}
+	if strings.Contains(message, "%s") {
+		var newArg1 string
+		if len(args) == 0 {
+			panic(fmt.Errorf(`%s has %%s but args are missing`, message))
+		}
+		newArg1 = fmt.Sprint(args[0])
+		if message == "unsupported line" {
+			newArg1 = node.Dump()
+		} else if message == "unsupported directive %s" {
+			if newArg1 == "include" || newArg1 == "-include" {
+				newArg1 = node.Dump()
+			}
+		}
+		v.formattingArgs = append(v.formattingArgs, newArg1)
+	}
+	ebt.data[message] = v
+}
+
+func (ebt errorsByType) printStatistics() {
+	if len(ebt.data) > 0 {
+		fmt.Fprintln(os.Stderr, "Error counts:")
+	}
+	for message, data := range ebt.data {
+		if len(data.formattingArgs) == 0 {
+			fmt.Fprintf(os.Stderr, "%4d %s\n", data.count, message)
+			continue
+		}
+		itemsByFreq, count := stringsWithFreq(data.formattingArgs, 30)
+		fmt.Fprintf(os.Stderr, "%4d %s [%d unique items]:\n", data.count, message, count)
+		fmt.Fprintln(os.Stderr, "      ", itemsByFreq)
+	}
+}
+
+func stringsWithFreq(items []string, topN int) (string, int) {
+	freq := make(map[string]int)
+	for _, item := range items {
+		freq[strings.TrimPrefix(strings.TrimSuffix(item, "]"), "[")]++
+	}
+	var sorted []string
+	for item := range freq {
+		sorted = append(sorted, item)
+	}
+	sort.Slice(sorted, func(i int, j int) bool {
+		return freq[sorted[i]] > freq[sorted[j]]
+	})
+	sep := ""
+	res := ""
+	for i, item := range sorted {
+		if i >= topN {
+			res += " ..."
+			break
+		}
+		count := freq[item]
+		if count > 1 {
+			res += fmt.Sprintf("%s%s(%d)", sep, item, count)
+		} else {
+			res += fmt.Sprintf("%s%s", sep, item)
+		}
+		sep = ", "
+	}
+	return res, len(sorted)
+}

+ 580 - 0
mk2rbc/expr.go

@@ -0,0 +1,580 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mk2rbc
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	mkparser "android/soong/androidmk/parser"
+)
+
+// Represents an expression in the Starlark code. An expression has
+// a type, and it can be evaluated.
+type starlarkExpr interface {
+	starlarkNode
+	typ() starlarkType
+	// Try to substitute variable values. Return substitution result
+	// and whether it is the same as the original expression.
+	eval(valueMap map[string]starlarkExpr) (res starlarkExpr, same bool)
+	// Emit the code to copy the expression, otherwise we will end up
+	// with source and target pointing to the same list.
+	emitListVarCopy(gctx *generationContext)
+}
+
+func maybeString(expr starlarkExpr) (string, bool) {
+	if x, ok := expr.(*stringLiteralExpr); ok {
+		return x.literal, true
+	}
+	return "", false
+}
+
+type stringLiteralExpr struct {
+	literal string
+}
+
+func (s *stringLiteralExpr) eval(_ map[string]starlarkExpr) (res starlarkExpr, same bool) {
+	res = s
+	same = true
+	return
+}
+
+func (s *stringLiteralExpr) emit(gctx *generationContext) {
+	gctx.writef("%q", s.literal)
+}
+
+func (_ *stringLiteralExpr) typ() starlarkType {
+	return starlarkTypeString
+}
+
+func (s *stringLiteralExpr) emitListVarCopy(gctx *generationContext) {
+	s.emit(gctx)
+}
+
+// Integer literal
+type intLiteralExpr struct {
+	literal int
+}
+
+func (s *intLiteralExpr) eval(_ map[string]starlarkExpr) (res starlarkExpr, same bool) {
+	res = s
+	same = true
+	return
+}
+
+func (s *intLiteralExpr) emit(gctx *generationContext) {
+	gctx.writef("%d", s.literal)
+}
+
+func (_ *intLiteralExpr) typ() starlarkType {
+	return starlarkTypeInt
+}
+
+func (s *intLiteralExpr) emitListVarCopy(gctx *generationContext) {
+	s.emit(gctx)
+}
+
+// interpolateExpr represents Starlark's interpolation operator <string> % list
+// we break <string> into a list of chunks, i.e., "first%second%third" % (X, Y)
+// will have chunks = ["first", "second", "third"] and args = [X, Y]
+type interpolateExpr struct {
+	chunks []string // string chunks, separated by '%'
+	args   []starlarkExpr
+}
+
+func (xi *interpolateExpr) emit(gctx *generationContext) {
+	if len(xi.chunks) != len(xi.args)+1 {
+		panic(fmt.Errorf("malformed interpolateExpr: #chunks(%d) != #args(%d)+1",
+			len(xi.chunks), len(xi.args)))
+	}
+	// Generate format as join of chunks, but first escape '%' in them
+	format := strings.ReplaceAll(xi.chunks[0], "%", "%%")
+	for _, chunk := range xi.chunks[1:] {
+		format += "%s" + strings.ReplaceAll(chunk, "%", "%%")
+	}
+	gctx.writef("%q %% ", format)
+	emitarg := func(arg starlarkExpr) {
+		if arg.typ() == starlarkTypeList {
+			gctx.write(`" ".join(`)
+			arg.emit(gctx)
+			gctx.write(`)`)
+		} else {
+			arg.emit(gctx)
+		}
+	}
+	if len(xi.args) == 1 {
+		emitarg(xi.args[0])
+	} else {
+		sep := "("
+		for _, arg := range xi.args {
+			gctx.write(sep)
+			emitarg(arg)
+			sep = ", "
+		}
+		gctx.write(")")
+	}
+}
+
+func (xi *interpolateExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, same bool) {
+	same = true
+	newChunks := []string{xi.chunks[0]}
+	var newArgs []starlarkExpr
+	for i, arg := range xi.args {
+		newArg, sameArg := arg.eval(valueMap)
+		same = same && sameArg
+		switch x := newArg.(type) {
+		case *stringLiteralExpr:
+			newChunks[len(newChunks)-1] += x.literal + xi.chunks[i+1]
+			same = false
+			continue
+		case *intLiteralExpr:
+			newChunks[len(newChunks)-1] += strconv.Itoa(x.literal) + xi.chunks[i+1]
+			same = false
+			continue
+		default:
+			newChunks = append(newChunks, xi.chunks[i+1])
+			newArgs = append(newArgs, newArg)
+		}
+	}
+	if same {
+		res = xi
+	} else if len(newChunks) == 1 {
+		res = &stringLiteralExpr{newChunks[0]}
+	} else {
+		res = &interpolateExpr{chunks: newChunks, args: newArgs}
+	}
+	return
+}
+
+func (_ *interpolateExpr) typ() starlarkType {
+	return starlarkTypeString
+}
+
+func (xi *interpolateExpr) emitListVarCopy(gctx *generationContext) {
+	xi.emit(gctx)
+}
+
+type variableRefExpr struct {
+	ref       variable
+	isDefined bool
+}
+
+func (v *variableRefExpr) eval(map[string]starlarkExpr) (res starlarkExpr, same bool) {
+	predefined, ok := v.ref.(*predefinedVariable)
+	if same = !ok; same {
+		res = v
+	} else {
+		res = predefined.value
+	}
+	return
+}
+
+func (v *variableRefExpr) emit(gctx *generationContext) {
+	v.ref.emitGet(gctx, v.isDefined)
+}
+
+func (v *variableRefExpr) typ() starlarkType {
+	return v.ref.valueType()
+}
+
+func (v *variableRefExpr) emitListVarCopy(gctx *generationContext) {
+	v.emit(gctx)
+	if v.typ() == starlarkTypeList {
+		gctx.write("[:]") // this will copy the list
+	}
+}
+
+type notExpr struct {
+	expr starlarkExpr
+}
+
+func (n *notExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, same bool) {
+	if x, same := n.expr.eval(valueMap); same {
+		res = n
+	} else {
+		res = &notExpr{expr: x}
+	}
+	return
+}
+
+func (n *notExpr) emit(ctx *generationContext) {
+	ctx.write("not ")
+	n.expr.emit(ctx)
+}
+
+func (_ *notExpr) typ() starlarkType {
+	return starlarkTypeBool
+}
+
+func (n *notExpr) emitListVarCopy(gctx *generationContext) {
+	n.emit(gctx)
+}
+
+type eqExpr struct {
+	left, right starlarkExpr
+	isEq        bool // if false, it's !=
+}
+
+func (eq *eqExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, same bool) {
+	xLeft, sameLeft := eq.left.eval(valueMap)
+	xRight, sameRight := eq.right.eval(valueMap)
+	if same = sameLeft && sameRight; same {
+		res = eq
+	} else {
+		res = &eqExpr{left: xLeft, right: xRight, isEq: eq.isEq}
+	}
+	return
+}
+
+func (eq *eqExpr) emit(gctx *generationContext) {
+	// Are we checking that a variable is empty?
+	var varRef *variableRefExpr
+	if s, ok := maybeString(eq.left); ok && s == "" {
+		varRef, ok = eq.right.(*variableRefExpr)
+	} else if s, ok := maybeString(eq.right); ok && s == "" {
+		varRef, ok = eq.left.(*variableRefExpr)
+	}
+	if varRef != nil {
+		// Yes.
+		if eq.isEq {
+			gctx.write("not ")
+		}
+		varRef.emit(gctx)
+		return
+	}
+
+	// General case
+	eq.left.emit(gctx)
+	if eq.isEq {
+		gctx.write(" == ")
+	} else {
+		gctx.write(" != ")
+	}
+	eq.right.emit(gctx)
+}
+
+func (_ *eqExpr) typ() starlarkType {
+	return starlarkTypeBool
+}
+
+func (eq *eqExpr) emitListVarCopy(gctx *generationContext) {
+	eq.emit(gctx)
+}
+
+// variableDefinedExpr corresponds to Make's ifdef VAR
+type variableDefinedExpr struct {
+	v variable
+}
+
+func (v *variableDefinedExpr) eval(_ map[string]starlarkExpr) (res starlarkExpr, same bool) {
+	res = v
+	same = true
+	return
+
+}
+
+func (v *variableDefinedExpr) emit(gctx *generationContext) {
+	if v.v != nil {
+		v.v.emitDefined(gctx)
+		return
+	}
+	gctx.writef("%s(%q)", cfnWarning, "TODO(VAR)")
+}
+
+func (_ *variableDefinedExpr) typ() starlarkType {
+	return starlarkTypeBool
+}
+
+func (v *variableDefinedExpr) emitListVarCopy(gctx *generationContext) {
+	v.emit(gctx)
+}
+
+type listExpr struct {
+	items []starlarkExpr
+}
+
+func (l *listExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, same bool) {
+	newItems := make([]starlarkExpr, len(l.items))
+	same = true
+	for i, item := range l.items {
+		var sameItem bool
+		newItems[i], sameItem = item.eval(valueMap)
+		same = same && sameItem
+	}
+	if same {
+		res = l
+	} else {
+		res = &listExpr{newItems}
+	}
+	return
+}
+
+func (l *listExpr) emit(gctx *generationContext) {
+	if !gctx.inAssignment || len(l.items) < 2 {
+		gctx.write("[")
+		sep := ""
+		for _, item := range l.items {
+			gctx.write(sep)
+			item.emit(gctx)
+			sep = ", "
+		}
+		gctx.write("]")
+		return
+	}
+
+	gctx.write("[")
+	gctx.indentLevel += 2
+
+	for _, item := range l.items {
+		gctx.newLine()
+		item.emit(gctx)
+		gctx.write(",")
+	}
+	gctx.indentLevel -= 2
+	gctx.newLine()
+	gctx.write("]")
+}
+
+func (_ *listExpr) typ() starlarkType {
+	return starlarkTypeList
+}
+
+func (l *listExpr) emitListVarCopy(gctx *generationContext) {
+	l.emit(gctx)
+}
+
+func newStringListExpr(items []string) *listExpr {
+	v := listExpr{}
+	for _, item := range items {
+		v.items = append(v.items, &stringLiteralExpr{item})
+	}
+	return &v
+}
+
+// concatExpr generates epxr1 + expr2 + ... + exprN in Starlark.
+type concatExpr struct {
+	items []starlarkExpr
+}
+
+func (c *concatExpr) emit(gctx *generationContext) {
+	if len(c.items) == 1 {
+		c.items[0].emit(gctx)
+		return
+	}
+
+	if !gctx.inAssignment {
+		c.items[0].emit(gctx)
+		for _, item := range c.items[1:] {
+			gctx.write(" + ")
+			item.emit(gctx)
+		}
+		return
+	}
+	gctx.write("(")
+	c.items[0].emit(gctx)
+	gctx.indentLevel += 2
+	for _, item := range c.items[1:] {
+		gctx.write(" +")
+		gctx.newLine()
+		item.emit(gctx)
+	}
+	gctx.write(")")
+	gctx.indentLevel -= 2
+}
+
+func (c *concatExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, same bool) {
+	same = true
+	xConcat := &concatExpr{items: make([]starlarkExpr, len(c.items))}
+	for i, item := range c.items {
+		var sameItem bool
+		xConcat.items[i], sameItem = item.eval(valueMap)
+		same = same && sameItem
+	}
+	if same {
+		res = c
+	} else {
+		res = xConcat
+	}
+	return
+}
+
+func (_ *concatExpr) typ() starlarkType {
+	return starlarkTypeList
+}
+
+func (c *concatExpr) emitListVarCopy(gctx *generationContext) {
+	c.emit(gctx)
+}
+
+// inExpr generates <expr> [not] in <list>
+type inExpr struct {
+	expr  starlarkExpr
+	list  starlarkExpr
+	isNot bool
+}
+
+func (i *inExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, same bool) {
+	x := &inExpr{isNot: i.isNot}
+	var sameExpr, sameList bool
+	x.expr, sameExpr = i.expr.eval(valueMap)
+	x.list, sameList = i.list.eval(valueMap)
+	if same = sameExpr && sameList; same {
+		res = i
+	} else {
+		res = x
+	}
+	return
+}
+
+func (i *inExpr) emit(gctx *generationContext) {
+	i.expr.emit(gctx)
+	if i.isNot {
+		gctx.write(" not in ")
+	} else {
+		gctx.write(" in ")
+	}
+	i.list.emit(gctx)
+}
+
+func (_ *inExpr) typ() starlarkType {
+	return starlarkTypeBool
+}
+
+func (i *inExpr) emitListVarCopy(gctx *generationContext) {
+	i.emit(gctx)
+}
+
+type indexExpr struct {
+	array starlarkExpr
+	index starlarkExpr
+}
+
+func (ix indexExpr) emit(gctx *generationContext) {
+	ix.array.emit(gctx)
+	gctx.write("[")
+	ix.index.emit(gctx)
+	gctx.write("]")
+}
+
+func (ix indexExpr) typ() starlarkType {
+	return starlarkTypeString
+}
+
+func (ix indexExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, same bool) {
+	newArray, isSameArray := ix.array.eval(valueMap)
+	newIndex, isSameIndex := ix.index.eval(valueMap)
+	if same = isSameArray && isSameIndex; same {
+		res = ix
+	} else {
+		res = &indexExpr{newArray, newIndex}
+	}
+	return
+}
+
+func (ix indexExpr) emitListVarCopy(gctx *generationContext) {
+	ix.emit(gctx)
+}
+
+type callExpr struct {
+	object     starlarkExpr // nil if static call
+	name       string
+	args       []starlarkExpr
+	returnType starlarkType
+}
+
+func (cx *callExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, same bool) {
+	newCallExpr := &callExpr{name: cx.name, args: make([]starlarkExpr, len(cx.args)),
+		returnType: cx.returnType}
+	if cx.object != nil {
+		newCallExpr.object, same = cx.object.eval(valueMap)
+	} else {
+		same = true
+	}
+	for i, args := range cx.args {
+		var s bool
+		newCallExpr.args[i], s = args.eval(valueMap)
+		same = same && s
+	}
+	if same {
+		res = cx
+	} else {
+		res = newCallExpr
+	}
+	return
+}
+
+func (cx *callExpr) emit(gctx *generationContext) {
+	if cx.object != nil {
+		gctx.write("(")
+		cx.object.emit(gctx)
+		gctx.write(")")
+		gctx.write(".", cx.name, "(")
+	} else {
+		kf, found := knownFunctions[cx.name]
+		if !found {
+			panic(fmt.Errorf("callExpr with unknown function %q", cx.name))
+		}
+		if kf.runtimeName[0] == '!' {
+			panic(fmt.Errorf("callExpr for %q should not be there", cx.name))
+		}
+		gctx.write(kf.runtimeName, "(")
+	}
+	sep := ""
+	for _, arg := range cx.args {
+		gctx.write(sep)
+		arg.emit(gctx)
+		sep = ", "
+	}
+	gctx.write(")")
+}
+
+func (cx *callExpr) typ() starlarkType {
+	return cx.returnType
+}
+
+func (cx *callExpr) emitListVarCopy(gctx *generationContext) {
+	cx.emit(gctx)
+}
+
+type badExpr struct {
+	node    mkparser.Node
+	message string
+}
+
+func (b *badExpr) eval(_ map[string]starlarkExpr) (res starlarkExpr, same bool) {
+	res = b
+	same = true
+	return
+}
+
+func (b *badExpr) emit(_ *generationContext) {
+	panic("implement me")
+}
+
+func (_ *badExpr) typ() starlarkType {
+	return starlarkTypeUnknown
+}
+
+func (b *badExpr) emitListVarCopy(gctx *generationContext) {
+	panic("implement me")
+}
+
+func maybeConvertToStringList(expr starlarkExpr) starlarkExpr {
+	if xString, ok := expr.(*stringLiteralExpr); ok {
+		return newStringListExpr(strings.Fields(xString.literal))
+	}
+	return expr
+}

+ 1344 - 0
mk2rbc/mk2rbc.go

@@ -0,0 +1,1344 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Convert makefile containing device configuration to Starlark file
+// The conversion can handle the following constructs in a makefile:
+//   * comments
+//   * simple variable assignments
+//   * $(call init-product,<file>)
+//   * $(call inherit-product-if-exists
+//   * if directives
+// All other constructs are carried over to the output starlark file as comments.
+//
+package mk2rbc
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+	"text/scanner"
+
+	mkparser "android/soong/androidmk/parser"
+)
+
+const (
+	baseUri = "//build/make/core:product_config.rbc"
+	// The name of the struct exported by the product_config.rbc
+	// that contains the functions and variables available to
+	// product configuration Starlark files.
+	baseName = "rblf"
+
+	// And here are the functions and variables:
+	cfnGetCfg          = baseName + ".cfg"
+	cfnMain            = baseName + ".product_configuration"
+	cfnPrintVars       = baseName + ".printvars"
+	cfnWarning         = baseName + ".warning"
+	cfnLocalAppend     = baseName + ".local_append"
+	cfnLocalSetDefault = baseName + ".local_set_default"
+	cfnInherit         = baseName + ".inherit"
+	cfnSetListDefault  = baseName + ".setdefault"
+)
+
+const (
+	// Phony makefile functions, they are eventually rewritten
+	// according to knownFunctions map
+	fileExistsPhony     = "$file_exists"
+	wildcardExistsPhony = "$wildcard_exists"
+)
+
+const (
+	callLoadAlways = "inherit-product"
+	callLoadIf     = "inherit-product-if-exists"
+)
+
+var knownFunctions = map[string]struct {
+	// The name of the runtime function this function call in makefiles maps to.
+	// If it starts with !, then this makefile function call is rewritten to
+	// something else.
+	runtimeName string
+	returnType  starlarkType
+}{
+	fileExistsPhony:                       {baseName + ".file_exists", starlarkTypeBool},
+	wildcardExistsPhony:                   {baseName + ".file_wildcard_exists", starlarkTypeBool},
+	"add-to-product-copy-files-if-exists": {baseName + ".copy_if_exists", starlarkTypeList},
+	"addprefix":                           {baseName + ".addprefix", starlarkTypeList},
+	"addsuffix":                           {baseName + ".addsuffix", starlarkTypeList},
+	"enforce-product-packages-exist":      {baseName + ".enforce_product_packages_exist", starlarkTypeVoid},
+	"error":                               {baseName + ".mkerror", starlarkTypeVoid},
+	"findstring":                          {"!findstring", starlarkTypeInt},
+	"find-copy-subdir-files":              {baseName + ".find_and_copy", starlarkTypeList},
+	"filter":                              {baseName + ".filter", starlarkTypeList},
+	"filter-out":                          {baseName + ".filter_out", starlarkTypeList},
+	"info":                                {baseName + ".mkinfo", starlarkTypeVoid},
+	"is-board-platform":                   {"!is-board-platform", starlarkTypeBool},
+	"is-board-platform-in-list":           {"!is-board-platform-in-list", starlarkTypeBool},
+	"is-product-in-list":                  {"!is-product-in-list", starlarkTypeBool},
+	"is-vendor-board-platform":            {"!is-vendor-board-platform", starlarkTypeBool},
+	callLoadAlways:                        {"!inherit-product", starlarkTypeVoid},
+	callLoadIf:                            {"!inherit-product-if-exists", starlarkTypeVoid},
+	"produce_copy_files":                  {baseName + ".produce_copy_files", starlarkTypeList},
+	"require-artifacts-in-path":           {baseName + ".require_artifacts_in_path", starlarkTypeVoid},
+	"require-artifacts-in-path-relaxed":   {baseName + ".require_artifacts_in_path_relaxed", starlarkTypeVoid},
+	// TODO(asmundak): remove it once all calls are removed from configuration makefiles. see b/183161002
+	"shell":    {baseName + ".shell", starlarkTypeString},
+	"strip":    {baseName + ".mkstrip", starlarkTypeString},
+	"subst":    {baseName + ".subst", starlarkTypeString},
+	"warning":  {baseName + ".mkwarning", starlarkTypeVoid},
+	"word":     {baseName + "!word", starlarkTypeString},
+	"wildcard": {baseName + ".expand_wildcard", starlarkTypeList},
+}
+
+var builtinFuncRex = regexp.MustCompile(
+	"^(addprefix|addsuffix|abspath|and|basename|call|dir|error|eval" +
+		"|flavor|foreach|file|filter|filter-out|findstring|firstword|guile" +
+		"|if|info|join|lastword|notdir|or|origin|patsubst|realpath" +
+		"|shell|sort|strip|subst|suffix|value|warning|word|wordlist|words" +
+		"|wildcard)")
+
+// Conversion request parameters
+type Request struct {
+	MkFile             string    // file to convert
+	Reader             io.Reader // if set, read input from this stream instead
+	RootDir            string    // root directory path used to resolve included files
+	OutputSuffix       string    // generated Starlark files suffix
+	OutputDir          string    // if set, root of the output hierarchy
+	ErrorLogger        ErrorMonitorCB
+	TracedVariables    []string // trace assignment to these variables
+	TraceCalls         bool
+	WarnPartialSuccess bool
+}
+
+// An error sink allowing to gather error statistics.
+// NewError is called on every error encountered during processing.
+type ErrorMonitorCB interface {
+	NewError(s string, node mkparser.Node, args ...interface{})
+}
+
+// Derives module name for a given file. It is base name
+// (file name without suffix), with some characters replaced to make it a Starlark identifier
+func moduleNameForFile(mkFile string) string {
+	base := strings.TrimSuffix(filepath.Base(mkFile), filepath.Ext(mkFile))
+	// TODO(asmundak): what else can be in the product file names?
+	return strings.ReplaceAll(base, "-", "_")
+}
+
+func cloneMakeString(mkString *mkparser.MakeString) *mkparser.MakeString {
+	r := &mkparser.MakeString{StringPos: mkString.StringPos}
+	r.Strings = append(r.Strings, mkString.Strings...)
+	r.Variables = append(r.Variables, mkString.Variables...)
+	return r
+}
+
+func isMakeControlFunc(s string) bool {
+	return s == "error" || s == "warning" || s == "info"
+}
+
+// Starlark output generation context
+type generationContext struct {
+	buf          strings.Builder
+	starScript   *StarlarkScript
+	indentLevel  int
+	inAssignment bool
+	tracedCount  int
+}
+
+func NewGenerateContext(ss *StarlarkScript) *generationContext {
+	return &generationContext{starScript: ss}
+}
+
+// emit returns generated script
+func (gctx *generationContext) emit() string {
+	ss := gctx.starScript
+
+	// The emitted code has the following layout:
+	//    <initial comments>
+	//    preamble, i.e.,
+	//      load statement for the runtime support
+	//      load statement for each unique submodule pulled in by this one
+	//    def init(g, handle):
+	//      cfg = rblf.cfg(handle)
+	//      <statements>
+	//      <warning if conversion was not clean>
+
+	iNode := len(ss.nodes)
+	for i, node := range ss.nodes {
+		if _, ok := node.(*commentNode); !ok {
+			iNode = i
+			break
+		}
+		node.emit(gctx)
+	}
+
+	gctx.emitPreamble()
+
+	gctx.newLine()
+	// The arguments passed to the init function are the global dictionary
+	// ('g') and the product configuration dictionary ('cfg')
+	gctx.write("def init(g, handle):")
+	gctx.indentLevel++
+	if gctx.starScript.traceCalls {
+		gctx.newLine()
+		gctx.writef(`print(">%s")`, gctx.starScript.mkFile)
+	}
+	gctx.newLine()
+	gctx.writef("cfg = %s(handle)", cfnGetCfg)
+	for _, node := range ss.nodes[iNode:] {
+		node.emit(gctx)
+	}
+
+	if ss.hasErrors && ss.warnPartialSuccess {
+		gctx.newLine()
+		gctx.writef("%s(%q, %q)", cfnWarning, filepath.Base(ss.mkFile), "partially successful conversion")
+	}
+	if gctx.starScript.traceCalls {
+		gctx.newLine()
+		gctx.writef(`print("<%s")`, gctx.starScript.mkFile)
+	}
+	gctx.indentLevel--
+	gctx.write("\n")
+	return gctx.buf.String()
+}
+
+func (gctx *generationContext) emitPreamble() {
+	gctx.newLine()
+	gctx.writef("load(%q, %q)", baseUri, baseName)
+	// Emit exactly one load statement for each URI.
+	loadedSubConfigs := make(map[string]string)
+	for _, sc := range gctx.starScript.inherited {
+		uri := sc.path
+		if m, ok := loadedSubConfigs[uri]; ok {
+			// No need to emit load statement, but fix module name.
+			sc.moduleLocalName = m
+			continue
+		}
+		if !sc.loadAlways {
+			uri += "|init"
+		}
+		gctx.newLine()
+		gctx.writef("load(%q, %s = \"init\")", uri, sc.entryName())
+		loadedSubConfigs[uri] = sc.moduleLocalName
+	}
+	gctx.write("\n")
+}
+
+func (gctx *generationContext) emitPass() {
+	gctx.newLine()
+	gctx.write("pass")
+}
+
+func (gctx *generationContext) write(ss ...string) {
+	for _, s := range ss {
+		gctx.buf.WriteString(s)
+	}
+}
+
+func (gctx *generationContext) writef(format string, args ...interface{}) {
+	gctx.write(fmt.Sprintf(format, args...))
+}
+
+func (gctx *generationContext) newLine() {
+	if gctx.buf.Len() == 0 {
+		return
+	}
+	gctx.write("\n")
+	gctx.writef("%*s", 2*gctx.indentLevel, "")
+}
+
+type knownVariable struct {
+	name      string
+	class     varClass
+	valueType starlarkType
+}
+
+type knownVariables map[string]knownVariable
+
+func (pcv knownVariables) NewVariable(name string, varClass varClass, valueType starlarkType) {
+	v, exists := pcv[name]
+	if !exists {
+		pcv[name] = knownVariable{name, varClass, valueType}
+		return
+	}
+	// Conflict resolution:
+	//    * config class trumps everything
+	//    * any type trumps unknown type
+	match := varClass == v.class
+	if !match {
+		if varClass == VarClassConfig {
+			v.class = VarClassConfig
+			match = true
+		} else if v.class == VarClassConfig {
+			match = true
+		}
+	}
+	if valueType != v.valueType {
+		if valueType != starlarkTypeUnknown {
+			if v.valueType == starlarkTypeUnknown {
+				v.valueType = valueType
+			} else {
+				match = false
+			}
+		}
+	}
+	if !match {
+		fmt.Fprintf(os.Stderr, "cannot redefine %s as %v/%v (already defined as %v/%v)\n",
+			name, varClass, valueType, v.class, v.valueType)
+	}
+}
+
+// All known product variables.
+var KnownVariables = make(knownVariables)
+
+func init() {
+	for _, kv := range []string{
+		// Kernel-related variables that we know are lists.
+		"BOARD_VENDOR_KERNEL_MODULES",
+		"BOARD_VENDOR_RAMDISK_KERNEL_MODULES",
+		"BOARD_VENDOR_RAMDISK_KERNEL_MODULES_LOAD",
+		"BOARD_RECOVERY_KERNEL_MODULES",
+		// Other variables we knwo are lists
+		"ART_APEX_JARS",
+	} {
+		KnownVariables.NewVariable(kv, VarClassSoong, starlarkTypeList)
+	}
+}
+
+type nodeReceiver interface {
+	newNode(node starlarkNode)
+}
+
+// Information about the generated Starlark script.
+type StarlarkScript struct {
+	mkFile             string
+	moduleName         string
+	mkPos              scanner.Position
+	nodes              []starlarkNode
+	inherited          []*inheritedModule
+	hasErrors          bool
+	topDir             string
+	traceCalls         bool // print enter/exit each init function
+	warnPartialSuccess bool
+}
+
+func (ss *StarlarkScript) newNode(node starlarkNode) {
+	ss.nodes = append(ss.nodes, node)
+}
+
+// varAssignmentScope points to the last assignment for each variable
+// in the current block. It is used during the parsing to chain
+// the assignments to a variable together.
+type varAssignmentScope struct {
+	outer *varAssignmentScope
+	vars  map[string]*assignmentNode
+}
+
+// parseContext holds the script we are generating and all the ephemeral data
+// needed during the parsing.
+type parseContext struct {
+	script           *StarlarkScript
+	nodes            []mkparser.Node // Makefile as parsed by mkparser
+	currentNodeIndex int             // Node in it we are processing
+	ifNestLevel      int
+	moduleNameCount  map[string]int // count of imported modules with given basename
+	fatalError       error
+	builtinMakeVars  map[string]starlarkExpr
+	outputSuffix     string
+	errorLogger      ErrorMonitorCB
+	tracedVariables  map[string]bool // variables to be traced in the generated script
+	variables        map[string]variable
+	varAssignments   *varAssignmentScope
+	receiver         nodeReceiver // receptacle for the generated starlarkNode's
+	receiverStack    []nodeReceiver
+	outputDir        string
+}
+
+func newParseContext(ss *StarlarkScript, nodes []mkparser.Node) *parseContext {
+	predefined := []struct{ name, value string }{
+		{"SRC_TARGET_DIR", filepath.Join("build", "make", "target")},
+		{"LOCAL_PATH", filepath.Dir(ss.mkFile)},
+		{"TOPDIR", ss.topDir},
+		// TODO(asmundak): maybe read it from build/make/core/envsetup.mk?
+		{"TARGET_COPY_OUT_SYSTEM", "system"},
+		{"TARGET_COPY_OUT_SYSTEM_OTHER", "system_other"},
+		{"TARGET_COPY_OUT_DATA", "data"},
+		{"TARGET_COPY_OUT_ASAN", filepath.Join("data", "asan")},
+		{"TARGET_COPY_OUT_OEM", "oem"},
+		{"TARGET_COPY_OUT_RAMDISK", "ramdisk"},
+		{"TARGET_COPY_OUT_DEBUG_RAMDISK", "debug_ramdisk"},
+		{"TARGET_COPY_OUT_VENDOR_DEBUG_RAMDISK", "vendor_debug_ramdisk"},
+		{"TARGET_COPY_OUT_TEST_HARNESS_RAMDISK", "test_harness_ramdisk"},
+		{"TARGET_COPY_OUT_ROOT", "root"},
+		{"TARGET_COPY_OUT_RECOVERY", "recovery"},
+		{"TARGET_COPY_OUT_VENDOR", "||VENDOR-PATH-PH||"},
+		{"TARGET_COPY_OUT_VENDOR_RAMDISK", "vendor_ramdisk"},
+		{"TARGET_COPY_OUT_PRODUCT", "||PRODUCT-PATH-PH||"},
+		{"TARGET_COPY_OUT_PRODUCT_SERVICES", "||PRODUCT-PATH-PH||"},
+		{"TARGET_COPY_OUT_SYSTEM_EXT", "||SYSTEM_EXT-PATH-PH||"},
+		{"TARGET_COPY_OUT_ODM", "||ODM-PATH-PH||"},
+		{"TARGET_COPY_OUT_VENDOR_DLKM", "||VENDOR_DLKM-PATH-PH||"},
+		{"TARGET_COPY_OUT_ODM_DLKM", "||ODM_DLKM-PATH-PH||"},
+		// TODO(asmundak): to process internal config files, we need the following variables:
+		//    BOARD_CONFIG_VENDOR_PATH
+		//    TARGET_VENDOR
+		//    target_base_product
+		//
+
+		// the following utility variables are set in build/make/common/core.mk:
+		{"empty", ""},
+		{"space", " "},
+		{"comma", ","},
+		{"newline", "\n"},
+		{"pound", "#"},
+		{"backslash", "\\"},
+	}
+	ctx := &parseContext{
+		script:           ss,
+		nodes:            nodes,
+		currentNodeIndex: 0,
+		ifNestLevel:      0,
+		moduleNameCount:  make(map[string]int),
+		builtinMakeVars:  map[string]starlarkExpr{},
+		variables:        make(map[string]variable),
+	}
+	ctx.pushVarAssignments()
+	for _, item := range predefined {
+		ctx.variables[item.name] = &predefinedVariable{
+			baseVariable: baseVariable{nam: item.name, typ: starlarkTypeString},
+			value:        &stringLiteralExpr{item.value},
+		}
+	}
+
+	return ctx
+}
+
+func (ctx *parseContext) lastAssignment(name string) *assignmentNode {
+	for va := ctx.varAssignments; va != nil; va = va.outer {
+		if v, ok := va.vars[name]; ok {
+			return v
+		}
+	}
+	return nil
+}
+
+func (ctx *parseContext) setLastAssignment(name string, asgn *assignmentNode) {
+	ctx.varAssignments.vars[name] = asgn
+}
+
+func (ctx *parseContext) pushVarAssignments() {
+	va := &varAssignmentScope{
+		outer: ctx.varAssignments,
+		vars:  make(map[string]*assignmentNode),
+	}
+	ctx.varAssignments = va
+}
+
+func (ctx *parseContext) popVarAssignments() {
+	ctx.varAssignments = ctx.varAssignments.outer
+}
+
+func (ctx *parseContext) pushReceiver(rcv nodeReceiver) {
+	ctx.receiverStack = append(ctx.receiverStack, ctx.receiver)
+	ctx.receiver = rcv
+}
+
+func (ctx *parseContext) popReceiver() {
+	last := len(ctx.receiverStack) - 1
+	if last < 0 {
+		panic(fmt.Errorf("popReceiver: receiver stack empty"))
+	}
+	ctx.receiver = ctx.receiverStack[last]
+	ctx.receiverStack = ctx.receiverStack[0:last]
+}
+
+func (ctx *parseContext) hasNodes() bool {
+	return ctx.currentNodeIndex < len(ctx.nodes)
+}
+
+func (ctx *parseContext) getNode() mkparser.Node {
+	if !ctx.hasNodes() {
+		return nil
+	}
+	node := ctx.nodes[ctx.currentNodeIndex]
+	ctx.currentNodeIndex++
+	return node
+}
+
+func (ctx *parseContext) backNode() {
+	if ctx.currentNodeIndex <= 0 {
+		panic("Cannot back off")
+	}
+	ctx.currentNodeIndex--
+}
+
+func (ctx *parseContext) handleAssignment(a *mkparser.Assignment) {
+	// Handle only simple variables
+	if !a.Name.Const() {
+		ctx.errorf(a, "Only simple variables are handled")
+		return
+	}
+	name := a.Name.Strings[0]
+	lhs := ctx.addVariable(name)
+	if lhs == nil {
+		ctx.errorf(a, "unknown variable %s", name)
+		return
+	}
+	_, isTraced := ctx.tracedVariables[name]
+	asgn := &assignmentNode{lhs: lhs, mkValue: a.Value, isTraced: isTraced}
+	if lhs.valueType() == starlarkTypeUnknown {
+		// Try to divine variable type from the RHS
+		asgn.value = ctx.parseMakeString(a, a.Value)
+		if xBad, ok := asgn.value.(*badExpr); ok {
+			ctx.wrapBadExpr(xBad)
+			return
+		}
+		inferred_type := asgn.value.typ()
+		if inferred_type != starlarkTypeUnknown {
+			if ogv, ok := lhs.(*otherGlobalVariable); ok {
+				ogv.typ = inferred_type
+			} else if pcv, ok := lhs.(*productConfigVariable); ok {
+				pcv.typ = inferred_type
+			} else {
+				panic(fmt.Errorf("cannot assign new type to a variable %s, its flavor is %T", lhs.name(), lhs))
+			}
+		}
+	}
+	if lhs.valueType() == starlarkTypeList {
+		xConcat := ctx.buildConcatExpr(a)
+		if xConcat == nil {
+			return
+		}
+		switch len(xConcat.items) {
+		case 0:
+			asgn.value = &listExpr{}
+		case 1:
+			asgn.value = xConcat.items[0]
+		default:
+			asgn.value = xConcat
+		}
+	} else {
+		asgn.value = ctx.parseMakeString(a, a.Value)
+		if xBad, ok := asgn.value.(*badExpr); ok {
+			ctx.wrapBadExpr(xBad)
+			return
+		}
+	}
+
+	// TODO(asmundak): move evaluation to a separate pass
+	asgn.value, _ = asgn.value.eval(ctx.builtinMakeVars)
+
+	asgn.previous = ctx.lastAssignment(name)
+	ctx.setLastAssignment(name, asgn)
+	switch a.Type {
+	case "=", ":=":
+		asgn.flavor = asgnSet
+	case "+=":
+		if asgn.previous == nil && !asgn.lhs.isPreset() {
+			asgn.flavor = asgnMaybeAppend
+		} else {
+			asgn.flavor = asgnAppend
+		}
+	case "?=":
+		asgn.flavor = asgnMaybeSet
+	default:
+		panic(fmt.Errorf("unexpected assignment type %s", a.Type))
+	}
+
+	ctx.receiver.newNode(asgn)
+}
+
+func (ctx *parseContext) buildConcatExpr(a *mkparser.Assignment) *concatExpr {
+	xConcat := &concatExpr{}
+	var xItemList *listExpr
+	addToItemList := func(x ...starlarkExpr) {
+		if xItemList == nil {
+			xItemList = &listExpr{[]starlarkExpr{}}
+		}
+		xItemList.items = append(xItemList.items, x...)
+	}
+	finishItemList := func() {
+		if xItemList != nil {
+			xConcat.items = append(xConcat.items, xItemList)
+			xItemList = nil
+		}
+	}
+
+	items := a.Value.Words()
+	for _, item := range items {
+		// A function call in RHS is supposed to return a list, all other item
+		// expressions return individual elements.
+		switch x := ctx.parseMakeString(a, item).(type) {
+		case *badExpr:
+			ctx.wrapBadExpr(x)
+			return nil
+		case *stringLiteralExpr:
+			addToItemList(maybeConvertToStringList(x).(*listExpr).items...)
+		default:
+			switch x.typ() {
+			case starlarkTypeList:
+				finishItemList()
+				xConcat.items = append(xConcat.items, x)
+			case starlarkTypeString:
+				finishItemList()
+				xConcat.items = append(xConcat.items, &callExpr{
+					object:     x,
+					name:       "split",
+					args:       nil,
+					returnType: starlarkTypeList,
+				})
+			default:
+				addToItemList(x)
+			}
+		}
+	}
+	if xItemList != nil {
+		xConcat.items = append(xConcat.items, xItemList)
+	}
+	return xConcat
+}
+
+func (ctx *parseContext) newInheritedModule(v mkparser.Node, pathExpr starlarkExpr, loadAlways bool) *inheritedModule {
+	var path string
+	x, _ := pathExpr.eval(ctx.builtinMakeVars)
+	s, ok := x.(*stringLiteralExpr)
+	if !ok {
+		ctx.errorf(v, "inherit-product/include argument is too complex")
+		return nil
+	}
+
+	path = s.literal
+	moduleName := moduleNameForFile(path)
+	moduleLocalName := "_" + moduleName
+	n, found := ctx.moduleNameCount[moduleName]
+	if found {
+		moduleLocalName += fmt.Sprintf("%d", n)
+	}
+	ctx.moduleNameCount[moduleName] = n + 1
+	ln := &inheritedModule{
+		path:            ctx.loadedModulePath(path),
+		originalPath:    path,
+		moduleName:      moduleName,
+		moduleLocalName: moduleLocalName,
+		loadAlways:      loadAlways,
+	}
+	ctx.script.inherited = append(ctx.script.inherited, ln)
+	return ln
+}
+
+func (ctx *parseContext) handleInheritModule(v mkparser.Node, pathExpr starlarkExpr, loadAlways bool) {
+	if im := ctx.newInheritedModule(v, pathExpr, loadAlways); im != nil {
+		ctx.receiver.newNode(&inheritNode{im})
+	}
+}
+
+func (ctx *parseContext) handleInclude(v mkparser.Node, pathExpr starlarkExpr, loadAlways bool) {
+	if ln := ctx.newInheritedModule(v, pathExpr, loadAlways); ln != nil {
+		ctx.receiver.newNode(&includeNode{ln})
+	}
+}
+
+func (ctx *parseContext) handleVariable(v *mkparser.Variable) {
+	// Handle:
+	//   $(call inherit-product,...)
+	//   $(call inherit-product-if-exists,...)
+	//   $(info xxx)
+	//   $(warning xxx)
+	//   $(error xxx)
+	expr := ctx.parseReference(v, v.Name)
+	switch x := expr.(type) {
+	case *callExpr:
+		if x.name == callLoadAlways || x.name == callLoadIf {
+			ctx.handleInheritModule(v, x.args[0], x.name == callLoadAlways)
+		} else if isMakeControlFunc(x.name) {
+			// File name is the first argument
+			args := []starlarkExpr{
+				&stringLiteralExpr{ctx.script.mkFile},
+				x.args[0],
+			}
+			ctx.receiver.newNode(&exprNode{
+				&callExpr{name: x.name, args: args, returnType: starlarkTypeUnknown},
+			})
+		} else {
+			ctx.receiver.newNode(&exprNode{expr})
+		}
+	case *badExpr:
+		ctx.wrapBadExpr(x)
+		return
+	default:
+		ctx.errorf(v, "cannot handle %s", v.Dump())
+		return
+	}
+}
+
+func (ctx *parseContext) handleDefine(directive *mkparser.Directive) {
+	tokens := strings.Fields(directive.Args.Strings[0])
+	ctx.errorf(directive, "define is not supported: %s", tokens[0])
+}
+
+func (ctx *parseContext) handleIfBlock(ifDirective *mkparser.Directive) {
+	ssSwitch := &switchNode{}
+	ctx.pushReceiver(ssSwitch)
+	for ctx.processBranch(ifDirective); ctx.hasNodes() && ctx.fatalError == nil; {
+		node := ctx.getNode()
+		switch x := node.(type) {
+		case *mkparser.Directive:
+			switch x.Name {
+			case "else", "elifdef", "elifndef", "elifeq", "elifneq":
+				ctx.processBranch(x)
+			case "endif":
+				ctx.popReceiver()
+				ctx.receiver.newNode(ssSwitch)
+				return
+			default:
+				ctx.errorf(node, "unexpected directive %s", x.Name)
+			}
+		default:
+			ctx.errorf(ifDirective, "unexpected statement")
+		}
+	}
+	if ctx.fatalError == nil {
+		ctx.fatalError = fmt.Errorf("no matching endif for %s", ifDirective.Dump())
+	}
+	ctx.popReceiver()
+}
+
+// processBranch processes a single branch (if/elseif/else) until the next directive
+// on the same level.
+func (ctx *parseContext) processBranch(check *mkparser.Directive) {
+	block := switchCase{gate: ctx.parseCondition(check)}
+	defer func() {
+		ctx.popVarAssignments()
+		ctx.ifNestLevel--
+
+	}()
+	ctx.pushVarAssignments()
+	ctx.ifNestLevel++
+
+	ctx.pushReceiver(&block)
+	for ctx.hasNodes() {
+		node := ctx.getNode()
+		if ctx.handleSimpleStatement(node) {
+			continue
+		}
+		switch d := node.(type) {
+		case *mkparser.Directive:
+			switch d.Name {
+			case "else", "elifdef", "elifndef", "elifeq", "elifneq", "endif":
+				ctx.popReceiver()
+				ctx.receiver.newNode(&block)
+				ctx.backNode()
+				return
+			case "ifdef", "ifndef", "ifeq", "ifneq":
+				ctx.handleIfBlock(d)
+			default:
+				ctx.errorf(d, "unexpected directive %s", d.Name)
+			}
+		default:
+			ctx.errorf(node, "unexpected statement")
+		}
+	}
+	ctx.fatalError = fmt.Errorf("no matching endif for %s", check.Dump())
+	ctx.popReceiver()
+}
+
+func (ctx *parseContext) newIfDefinedNode(check *mkparser.Directive) (starlarkExpr, bool) {
+	if !check.Args.Const() {
+		return ctx.newBadExpr(check, "ifdef variable ref too complex: %s", check.Args.Dump()), false
+	}
+	v := ctx.addVariable(check.Args.Strings[0])
+	return &variableDefinedExpr{v}, true
+}
+
+func (ctx *parseContext) parseCondition(check *mkparser.Directive) starlarkNode {
+	switch check.Name {
+	case "ifdef", "ifndef", "elifdef", "elifndef":
+		v, ok := ctx.newIfDefinedNode(check)
+		if ok && strings.HasSuffix(check.Name, "ndef") {
+			v = &notExpr{v}
+		}
+		return &ifNode{
+			isElif: strings.HasPrefix(check.Name, "elif"),
+			expr:   v,
+		}
+	case "ifeq", "ifneq", "elifeq", "elifneq":
+		return &ifNode{
+			isElif: strings.HasPrefix(check.Name, "elif"),
+			expr:   ctx.parseCompare(check),
+		}
+	case "else":
+		return &elseNode{}
+	default:
+		panic(fmt.Errorf("%s: unknown directive: %s", ctx.script.mkFile, check.Dump()))
+	}
+}
+
+func (ctx *parseContext) newBadExpr(node mkparser.Node, text string, args ...interface{}) starlarkExpr {
+	message := fmt.Sprintf(text, args...)
+	if ctx.errorLogger != nil {
+		ctx.errorLogger.NewError(text, node, args)
+	}
+	ctx.script.hasErrors = true
+	return &badExpr{node, message}
+}
+
+func (ctx *parseContext) parseCompare(cond *mkparser.Directive) starlarkExpr {
+	// Strip outer parentheses
+	mkArg := cloneMakeString(cond.Args)
+	mkArg.Strings[0] = strings.TrimLeft(mkArg.Strings[0], "( ")
+	n := len(mkArg.Strings)
+	mkArg.Strings[n-1] = strings.TrimRight(mkArg.Strings[n-1], ") ")
+	args := mkArg.Split(",")
+	// TODO(asmundak): handle the case where the arguments are in quotes and space-separated
+	if len(args) != 2 {
+		return ctx.newBadExpr(cond, "ifeq/ifneq len(args) != 2 %s", cond.Dump())
+	}
+	args[0].TrimRightSpaces()
+	args[1].TrimLeftSpaces()
+
+	isEq := !strings.HasSuffix(cond.Name, "neq")
+	switch xLeft := ctx.parseMakeString(cond, args[0]).(type) {
+	case *stringLiteralExpr, *variableRefExpr:
+		switch xRight := ctx.parseMakeString(cond, args[1]).(type) {
+		case *stringLiteralExpr, *variableRefExpr:
+			return &eqExpr{left: xLeft, right: xRight, isEq: isEq}
+		case *badExpr:
+			return xRight
+		default:
+			expr, ok := ctx.parseCheckFunctionCallResult(cond, xLeft, args[1])
+			if ok {
+				return expr
+			}
+			return ctx.newBadExpr(cond, "right operand is too complex: %s", args[1].Dump())
+		}
+	case *badExpr:
+		return xLeft
+	default:
+		switch xRight := ctx.parseMakeString(cond, args[1]).(type) {
+		case *stringLiteralExpr, *variableRefExpr:
+			expr, ok := ctx.parseCheckFunctionCallResult(cond, xRight, args[0])
+			if ok {
+				return expr
+			}
+			return ctx.newBadExpr(cond, "left operand is too complex: %s", args[0].Dump())
+		case *badExpr:
+			return xRight
+		default:
+			return ctx.newBadExpr(cond, "operands are too complex: (%s,%s)", args[0].Dump(), args[1].Dump())
+		}
+	}
+}
+
+func (ctx *parseContext) parseCheckFunctionCallResult(directive *mkparser.Directive, xValue starlarkExpr,
+	varArg *mkparser.MakeString) (starlarkExpr, bool) {
+	mkSingleVar, ok := varArg.SingleVariable()
+	if !ok {
+		return nil, false
+	}
+	expr := ctx.parseReference(directive, mkSingleVar)
+	negate := strings.HasSuffix(directive.Name, "neq")
+	checkIsSomethingFunction := func(xCall *callExpr) starlarkExpr {
+		s, ok := maybeString(xValue)
+		if !ok || s != "true" {
+			return ctx.newBadExpr(directive,
+				fmt.Sprintf("the result of %s can be compared only to 'true'", xCall.name))
+		}
+		if len(xCall.args) < 1 {
+			return ctx.newBadExpr(directive, "%s requires an argument", xCall.name)
+		}
+		return nil
+	}
+	switch x := expr.(type) {
+	case *callExpr:
+		switch x.name {
+		case "filter":
+			return ctx.parseCompareFilterFuncResult(directive, x, xValue, !negate), true
+		case "filter-out":
+			return ctx.parseCompareFilterFuncResult(directive, x, xValue, negate), true
+		case "wildcard":
+			return ctx.parseCompareWildcardFuncResult(directive, x, xValue, negate), true
+		case "findstring":
+			return ctx.parseCheckFindstringFuncResult(directive, x, xValue, negate), true
+		case "strip":
+			return ctx.parseCompareStripFuncResult(directive, x, xValue, negate), true
+		case "is-board-platform":
+			if xBad := checkIsSomethingFunction(x); xBad != nil {
+				return xBad, true
+			}
+			return &eqExpr{
+				left:  &variableRefExpr{ctx.addVariable("TARGET_BOARD_PLATFORM"), false},
+				right: x.args[0],
+				isEq:  !negate,
+			}, true
+		case "is-board-platform-in-list":
+			if xBad := checkIsSomethingFunction(x); xBad != nil {
+				return xBad, true
+			}
+			return &inExpr{
+				expr:  &variableRefExpr{ctx.addVariable("TARGET_BOARD_PLATFORM"), false},
+				list:  maybeConvertToStringList(x.args[0]),
+				isNot: negate,
+			}, true
+		case "is-product-in-list":
+			if xBad := checkIsSomethingFunction(x); xBad != nil {
+				return xBad, true
+			}
+			return &inExpr{
+				expr:  &variableRefExpr{ctx.addVariable("TARGET_PRODUCT"), true},
+				list:  maybeConvertToStringList(x.args[0]),
+				isNot: negate,
+			}, true
+		case "is-vendor-board-platform":
+			if xBad := checkIsSomethingFunction(x); xBad != nil {
+				return xBad, true
+			}
+			s, ok := maybeString(x.args[0])
+			if !ok {
+				return ctx.newBadExpr(directive, "cannot handle non-constant argument to is-vendor-board-platform"), true
+			}
+			return &inExpr{
+				expr:  &variableRefExpr{ctx.addVariable("TARGET_BOARD_PLATFORM"), false},
+				list:  &variableRefExpr{ctx.addVariable(s + "_BOARD_PLATFORMS"), true},
+				isNot: negate,
+			}, true
+		default:
+			return ctx.newBadExpr(directive, "Unknown function in ifeq: %s", x.name), true
+		}
+	case *badExpr:
+		return x, true
+	default:
+		return nil, false
+	}
+}
+
+func (ctx *parseContext) parseCompareFilterFuncResult(cond *mkparser.Directive,
+	filterFuncCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr {
+	// We handle:
+	// *  ifeq/ifneq (,$(filter v1 v2 ..., $(VAR)) becomes if VAR not in/in ["v1", "v2", ...]
+	// *  ifeq/ifneq (,$(filter $(VAR), v1 v2 ...) becomes if VAR not in/in ["v1", "v2", ...]
+	// *  ifeq/ifneq ($(VAR),$(filter $(VAR), v1 v2 ...) becomes if VAR in/not in ["v1", "v2"]
+	// TODO(Asmundak): check the last case works for filter-out, too.
+	xPattern := filterFuncCall.args[0]
+	xText := filterFuncCall.args[1]
+	var xInList *stringLiteralExpr
+	var xVar starlarkExpr
+	var ok bool
+	switch x := xValue.(type) {
+	case *stringLiteralExpr:
+		if x.literal != "" {
+			return ctx.newBadExpr(cond, "filter comparison to non-empty value: %s", xValue)
+		}
+		// Either pattern or text should be const, and the
+		// non-const one should be varRefExpr
+		if xInList, ok = xPattern.(*stringLiteralExpr); ok {
+			xVar = xText
+		} else if xInList, ok = xText.(*stringLiteralExpr); ok {
+			xVar = xPattern
+		}
+	case *variableRefExpr:
+		if v, ok := xPattern.(*variableRefExpr); ok {
+			if xInList, ok = xText.(*stringLiteralExpr); ok && v.ref.name() == x.ref.name() {
+				// ifeq/ifneq ($(VAR),$(filter $(VAR), v1 v2 ...), flip negate,
+				// it's the opposite to what is done when comparing to empty.
+				xVar = xPattern
+				negate = !negate
+			}
+		}
+	}
+	if xVar != nil && xInList != nil {
+		if _, ok := xVar.(*variableRefExpr); ok {
+			slExpr := newStringListExpr(strings.Fields(xInList.literal))
+			// Generate simpler code for the common cases:
+			if xVar.typ() == starlarkTypeList {
+				if len(slExpr.items) == 1 {
+					// Checking that a string belongs to list
+					return &inExpr{isNot: negate, list: xVar, expr: slExpr.items[0]}
+				} else {
+					// TODO(asmundak):
+					panic("TBD")
+				}
+			}
+			return &inExpr{isNot: negate, list: newStringListExpr(strings.Fields(xInList.literal)), expr: xVar}
+		}
+	}
+	return ctx.newBadExpr(cond, "filter arguments are too complex: %s", cond.Dump())
+}
+
+func (ctx *parseContext) parseCompareWildcardFuncResult(directive *mkparser.Directive,
+	xCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr {
+	if x, ok := xValue.(*stringLiteralExpr); !ok || x.literal != "" {
+		return ctx.newBadExpr(directive, "wildcard result can be compared only to empty: %s", xValue)
+	}
+	callFunc := wildcardExistsPhony
+	if s, ok := xCall.args[0].(*stringLiteralExpr); ok && !strings.ContainsAny(s.literal, "*?{[") {
+		callFunc = fileExistsPhony
+	}
+	var cc starlarkExpr = &callExpr{name: callFunc, args: xCall.args, returnType: starlarkTypeBool}
+	if !negate {
+		cc = &notExpr{cc}
+	}
+	return cc
+}
+
+func (ctx *parseContext) parseCheckFindstringFuncResult(directive *mkparser.Directive,
+	xCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr {
+	if x, ok := xValue.(*stringLiteralExpr); !ok || x.literal != "" {
+		return ctx.newBadExpr(directive, "findstring result can be compared only to empty: %s", xValue)
+	}
+	return &eqExpr{
+		left: &callExpr{
+			object:     xCall.args[1],
+			name:       "find",
+			args:       []starlarkExpr{xCall.args[0]},
+			returnType: starlarkTypeInt,
+		},
+		right: &intLiteralExpr{-1},
+		isEq:  !negate,
+	}
+}
+
+func (ctx *parseContext) parseCompareStripFuncResult(directive *mkparser.Directive,
+	xCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr {
+	if _, ok := xValue.(*stringLiteralExpr); !ok {
+		return ctx.newBadExpr(directive, "strip result can be compared only to string: %s", xValue)
+	}
+	return &eqExpr{
+		left: &callExpr{
+			name:       "strip",
+			args:       xCall.args,
+			returnType: starlarkTypeString,
+		},
+		right: xValue, isEq: !negate}
+}
+
+// parses $(...), returning an expression
+func (ctx *parseContext) parseReference(node mkparser.Node, ref *mkparser.MakeString) starlarkExpr {
+	ref.TrimLeftSpaces()
+	ref.TrimRightSpaces()
+	refDump := ref.Dump()
+
+	// Handle only the case where the first (or only) word is constant
+	words := ref.SplitN(" ", 2)
+	if !words[0].Const() {
+		return ctx.newBadExpr(node, "reference is too complex: %s", refDump)
+	}
+
+	// If it is a single word, it can be a simple variable
+	// reference or a function call
+	if len(words) == 1 {
+		if isMakeControlFunc(refDump) || refDump == "shell" {
+			return &callExpr{
+				name:       refDump,
+				args:       []starlarkExpr{&stringLiteralExpr{""}},
+				returnType: starlarkTypeUnknown,
+			}
+		}
+		if v := ctx.addVariable(refDump); v != nil {
+			return &variableRefExpr{v, ctx.lastAssignment(v.name()) != nil}
+		}
+		return ctx.newBadExpr(node, "unknown variable %s", refDump)
+	}
+
+	expr := &callExpr{name: words[0].Dump(), returnType: starlarkTypeUnknown}
+	args := words[1]
+	args.TrimLeftSpaces()
+	// Make control functions and shell need special treatment as everything
+	// after the name is a single text argument
+	if isMakeControlFunc(expr.name) || expr.name == "shell" {
+		x := ctx.parseMakeString(node, args)
+		if xBad, ok := x.(*badExpr); ok {
+			return xBad
+		}
+		expr.args = []starlarkExpr{x}
+		return expr
+	}
+	if expr.name == "call" {
+		words = args.SplitN(",", 2)
+		if words[0].Empty() || !words[0].Const() {
+			return ctx.newBadExpr(nil, "cannot handle %s", refDump)
+		}
+		expr.name = words[0].Dump()
+		if len(words) < 2 {
+			return expr
+		}
+		args = words[1]
+	}
+	if kf, found := knownFunctions[expr.name]; found {
+		expr.returnType = kf.returnType
+	} else {
+		return ctx.newBadExpr(node, "cannot handle invoking %s", expr.name)
+	}
+	switch expr.name {
+	case "word":
+		return ctx.parseWordFunc(node, args)
+	case "subst":
+		return ctx.parseSubstFunc(node, args)
+	default:
+		for _, arg := range args.Split(",") {
+			arg.TrimLeftSpaces()
+			arg.TrimRightSpaces()
+			x := ctx.parseMakeString(node, arg)
+			if xBad, ok := x.(*badExpr); ok {
+				return xBad
+			}
+			expr.args = append(expr.args, x)
+		}
+	}
+	return expr
+}
+
+func (ctx *parseContext) parseSubstFunc(node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
+	words := args.Split(",")
+	if len(words) != 3 {
+		return ctx.newBadExpr(node, "subst function should have 3 arguments")
+	}
+	if !words[0].Const() || !words[1].Const() {
+		return ctx.newBadExpr(node, "subst function's from and to arguments should be constant")
+	}
+	from := words[0].Strings[0]
+	to := words[1].Strings[0]
+	words[2].TrimLeftSpaces()
+	words[2].TrimRightSpaces()
+	obj := ctx.parseMakeString(node, words[2])
+	return &callExpr{
+		object:     obj,
+		name:       "replace",
+		args:       []starlarkExpr{&stringLiteralExpr{from}, &stringLiteralExpr{to}},
+		returnType: starlarkTypeString,
+	}
+}
+
+func (ctx *parseContext) parseWordFunc(node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
+	words := args.Split(",")
+	if len(words) != 2 {
+		return ctx.newBadExpr(node, "word function should have 2 arguments")
+	}
+	var index uint64 = 0
+	if words[0].Const() {
+		index, _ = strconv.ParseUint(strings.TrimSpace(words[0].Strings[0]), 10, 64)
+	}
+	if index < 1 {
+		return ctx.newBadExpr(node, "word index should be constant positive integer")
+	}
+	words[1].TrimLeftSpaces()
+	words[1].TrimRightSpaces()
+	array := ctx.parseMakeString(node, words[1])
+	if xBad, ok := array.(*badExpr); ok {
+		return xBad
+	}
+	if array.typ() != starlarkTypeList {
+		array = &callExpr{object: array, name: "split", returnType: starlarkTypeList}
+	}
+	return indexExpr{array, &intLiteralExpr{int(index - 1)}}
+}
+
+func (ctx *parseContext) parseMakeString(node mkparser.Node, mk *mkparser.MakeString) starlarkExpr {
+	if mk.Const() {
+		return &stringLiteralExpr{mk.Dump()}
+	}
+	if mkRef, ok := mk.SingleVariable(); ok {
+		return ctx.parseReference(node, mkRef)
+	}
+	// If we reached here, it's neither string literal nor a simple variable,
+	// we need a full-blown interpolation node that will generate
+	// "a%b%c" % (X, Y) for a$(X)b$(Y)c
+	xInterp := &interpolateExpr{args: make([]starlarkExpr, len(mk.Variables))}
+	for i, ref := range mk.Variables {
+		arg := ctx.parseReference(node, ref.Name)
+		if x, ok := arg.(*badExpr); ok {
+			return x
+		}
+		xInterp.args[i] = arg
+	}
+	xInterp.chunks = append(xInterp.chunks, mk.Strings...)
+	return xInterp
+}
+
+// Handles the statements whose treatment is the same in all contexts: comment,
+// assignment, variable (which is a macro call in reality) and all constructs that
+// do not handle in any context ('define directive and any unrecognized stuff).
+// Return true if we handled it.
+func (ctx *parseContext) handleSimpleStatement(node mkparser.Node) bool {
+	handled := true
+	switch x := node.(type) {
+	case *mkparser.Comment:
+		ctx.insertComment("#" + x.Comment)
+	case *mkparser.Assignment:
+		ctx.handleAssignment(x)
+	case *mkparser.Variable:
+		ctx.handleVariable(x)
+	case *mkparser.Directive:
+		switch x.Name {
+		case "define":
+			ctx.handleDefine(x)
+		case "include", "-include":
+			ctx.handleInclude(node, ctx.parseMakeString(node, x.Args), x.Name[0] != '-')
+		default:
+			handled = false
+		}
+	default:
+		ctx.errorf(x, "unsupported line %s", x.Dump())
+	}
+	return handled
+}
+
+func (ctx *parseContext) insertComment(s string) {
+	ctx.receiver.newNode(&commentNode{strings.TrimSpace(s)})
+}
+
+func (ctx *parseContext) carryAsComment(failedNode mkparser.Node) {
+	for _, line := range strings.Split(failedNode.Dump(), "\n") {
+		ctx.insertComment("# " + line)
+	}
+}
+
+// records that the given node failed to be converted and includes an explanatory message
+func (ctx *parseContext) errorf(failedNode mkparser.Node, message string, args ...interface{}) {
+	if ctx.errorLogger != nil {
+		ctx.errorLogger.NewError(message, failedNode, args...)
+	}
+	message = fmt.Sprintf(message, args...)
+	ctx.insertComment(fmt.Sprintf("# MK2RBC TRANSLATION ERROR: %s", message))
+	ctx.carryAsComment(failedNode)
+	ctx.script.hasErrors = true
+}
+
+func (ctx *parseContext) wrapBadExpr(xBad *badExpr) {
+	ctx.insertComment(fmt.Sprintf("# MK2RBC TRANSLATION ERROR: %s", xBad.message))
+	ctx.carryAsComment(xBad.node)
+}
+
+func (ctx *parseContext) loadedModulePath(path string) string {
+	// During the transition to Roboleaf some of the product configuration files
+	// will be converted and checked in while the others will be generated on the fly
+	// and run. The runner  (rbcrun application) accommodates this by allowing three
+	// different ways to specify the loaded file location:
+	//  1) load(":<file>",...) loads <file> from the same directory
+	//  2) load("//path/relative/to/source/root:<file>", ...) loads <file> source tree
+	//  3) load("/absolute/path/to/<file> absolute path
+	// If the file being generated and the file it wants to load are in the same directory,
+	// generate option 1.
+	// Otherwise, if output directory is not specified, generate 2)
+	// Finally, if output directory has been specified and the file being generated and
+	// the file it wants to load from are in the different directories, generate 2) or 3):
+	//  * if the file being loaded exists in the source tree, generate 2)
+	//  * otherwise, generate 3)
+	// Finally, figure out the loaded module path and name and create a node for it
+	loadedModuleDir := filepath.Dir(path)
+	base := filepath.Base(path)
+	loadedModuleName := strings.TrimSuffix(base, filepath.Ext(base)) + ctx.outputSuffix
+	if loadedModuleDir == filepath.Dir(ctx.script.mkFile) {
+		return ":" + loadedModuleName
+	}
+	if ctx.outputDir == "" {
+		return fmt.Sprintf("//%s:%s", loadedModuleDir, loadedModuleName)
+	}
+	if _, err := os.Stat(filepath.Join(loadedModuleDir, loadedModuleName)); err == nil {
+		return fmt.Sprintf("//%s:%s", loadedModuleDir, loadedModuleName)
+	}
+	return filepath.Join(ctx.outputDir, loadedModuleDir, loadedModuleName)
+}
+
+func (ss *StarlarkScript) String() string {
+	return NewGenerateContext(ss).emit()
+}
+
+func (ss *StarlarkScript) SubConfigFiles() []string {
+	var subs []string
+	for _, src := range ss.inherited {
+		subs = append(subs, src.originalPath)
+	}
+	return subs
+}
+
+func (ss *StarlarkScript) HasErrors() bool {
+	return ss.hasErrors
+}
+
+// Convert reads and parses a makefile. If successful, parsed tree
+// is returned and then can be passed to String() to get the generated
+// Starlark file.
+func Convert(req Request) (*StarlarkScript, error) {
+	reader := req.Reader
+	if reader == nil {
+		mkContents, err := ioutil.ReadFile(req.MkFile)
+		if err != nil {
+			return nil, err
+		}
+		reader = bytes.NewBuffer(mkContents)
+	}
+	parser := mkparser.NewParser(req.MkFile, reader)
+	nodes, errs := parser.Parse()
+	if len(errs) > 0 {
+		for _, e := range errs {
+			fmt.Fprintln(os.Stderr, "ERROR:", e)
+		}
+		return nil, fmt.Errorf("bad makefile %s", req.MkFile)
+	}
+	starScript := &StarlarkScript{
+		moduleName:         moduleNameForFile(req.MkFile),
+		mkFile:             req.MkFile,
+		topDir:             req.RootDir,
+		traceCalls:         req.TraceCalls,
+		warnPartialSuccess: req.WarnPartialSuccess,
+	}
+	ctx := newParseContext(starScript, nodes)
+	ctx.outputSuffix = req.OutputSuffix
+	ctx.outputDir = req.OutputDir
+	ctx.errorLogger = req.ErrorLogger
+	if len(req.TracedVariables) > 0 {
+		ctx.tracedVariables = make(map[string]bool)
+		for _, v := range req.TracedVariables {
+			ctx.tracedVariables[v] = true
+		}
+	}
+	ctx.pushReceiver(starScript)
+	for ctx.hasNodes() && ctx.fatalError == nil {
+		node := ctx.getNode()
+		if ctx.handleSimpleStatement(node) {
+			continue
+		}
+		switch x := node.(type) {
+		case *mkparser.Directive:
+			switch x.Name {
+			case "ifeq", "ifneq", "ifdef", "ifndef":
+				ctx.handleIfBlock(x)
+			default:
+				ctx.errorf(x, "unexpected directive %s", x.Name)
+			}
+		default:
+			ctx.errorf(x, "unsupported line")
+		}
+	}
+	if ctx.fatalError != nil {
+		return nil, ctx.fatalError
+	}
+	return starScript, nil
+}
+
+func Launcher(path, name string) string {
+	var buf bytes.Buffer
+	fmt.Fprintf(&buf, "load(%q, %q)\n", baseUri, baseName)
+	fmt.Fprintf(&buf, "load(%q, \"init\")\n", path)
+	fmt.Fprintf(&buf, "g, config = %s(%q, init)\n", cfnMain, name)
+	fmt.Fprintf(&buf, "%s(g, config)\n", cfnPrintVars)
+	return buf.String()
+}
+
+func MakePath2ModuleName(mkPath string) string {
+	return strings.TrimSuffix(mkPath, filepath.Ext(mkPath))
+}

+ 857 - 0
mk2rbc/mk2rbc_test.go

@@ -0,0 +1,857 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mk2rbc
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+)
+
+var testCases = []struct {
+	desc     string
+	mkname   string
+	in       string
+	expected string
+}{
+	{
+		desc:   "Comment",
+		mkname: "product.mk",
+		in: `
+# Comment
+# FOO= a\
+     b
+`,
+		expected: `# Comment
+# FOO= a
+#     b
+load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+`,
+	},
+	{
+		desc:   "Name conversion",
+		mkname: "path/bar-baz.mk",
+		in: `
+# Comment
+`,
+		expected: `# Comment
+load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+`,
+	},
+	{
+		desc:   "Item variable",
+		mkname: "pixel3.mk",
+		in: `
+PRODUCT_NAME := Pixel 3
+PRODUCT_MODEL :=
+local_var = foo
+`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  cfg["PRODUCT_NAME"] = "Pixel 3"
+  cfg["PRODUCT_MODEL"] = ""
+  _local_var = "foo"
+`,
+	},
+	{
+		desc:   "List variable",
+		mkname: "pixel4.mk",
+		in: `
+PRODUCT_PACKAGES = package1  package2
+PRODUCT_COPY_FILES += file2:target
+PRODUCT_PACKAGES += package3
+PRODUCT_COPY_FILES =
+`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  cfg["PRODUCT_PACKAGES"] = [
+      "package1",
+      "package2",
+  ]
+  rblf.setdefault(handle, "PRODUCT_COPY_FILES")
+  cfg["PRODUCT_COPY_FILES"] += ["file2:target"]
+  cfg["PRODUCT_PACKAGES"] += ["package3"]
+  cfg["PRODUCT_COPY_FILES"] = []
+`,
+	},
+	{
+		desc:   "Unknown function",
+		mkname: "product.mk",
+		in: `
+PRODUCT_NAME := $(call foo, bar)
+`,
+		expected: `# MK2RBC TRANSLATION ERROR: cannot handle invoking foo
+# PRODUCT_NAME := $(call foo, bar)
+load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  rblf.warning("product.mk", "partially successful conversion")
+`,
+	},
+	{
+		desc:   "Inherit configuration always",
+		mkname: "product.mk",
+		in: `
+ifdef PRODUCT_NAME
+$(call inherit-product, part.mk)
+else # Comment
+$(call inherit-product, $(LOCAL_PATH)/part.mk)
+endif
+`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+load(":part.star", _part_init = "init")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  if g.get("PRODUCT_NAME") != None:
+    rblf.inherit(handle, "part", _part_init)
+  else:
+    # Comment
+    rblf.inherit(handle, "./part", _part_init)
+`,
+	},
+	{
+		desc:   "Inherit configuration if it exists",
+		mkname: "product.mk",
+		in: `
+$(call inherit-product-if-exists, part.mk)
+`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+load(":part.star|init", _part_init = "init")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  if _part_init != None:
+    rblf.inherit(handle, "part", _part_init)
+`,
+	},
+
+	{
+		desc:   "Include configuration",
+		mkname: "product.mk",
+		in: `
+ifdef PRODUCT_NAME
+include part.mk
+else
+-include $(LOCAL_PATH)/part.mk)
+endif
+`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+load(":part.star", _part_init = "init")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  if g.get("PRODUCT_NAME") != None:
+    _part_init(g, handle)
+  else:
+    if _part_init != None:
+      _part_init(g, handle)
+`,
+	},
+
+	{
+		desc:   "Synonymous inherited configurations",
+		mkname: "path/product.mk",
+		in: `
+$(call inherit-product, foo/font.mk)
+$(call inherit-product, bar/font.mk)
+`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+load("//foo:font.star", _font_init = "init")
+load("//bar:font.star", _font1_init = "init")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  rblf.inherit(handle, "foo/font", _font_init)
+  rblf.inherit(handle, "bar/font", _font1_init)
+`,
+	},
+	{
+		desc:   "Directive define",
+		mkname: "product.mk",
+		in: `
+define some-macro
+    $(info foo)
+endef
+`,
+		expected: `# MK2RBC TRANSLATION ERROR: define is not supported: some-macro
+# define  some-macro
+#     $(info foo)
+# endef
+load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  rblf.warning("product.mk", "partially successful conversion")
+`,
+	},
+	{
+		desc:   "Ifdef",
+		mkname: "product.mk",
+		in: `
+ifdef  PRODUCT_NAME
+  PRODUCT_NAME = gizmo
+else
+endif
+`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  if g.get("PRODUCT_NAME") != None:
+    cfg["PRODUCT_NAME"] = "gizmo"
+  else:
+    pass
+`,
+	},
+	{
+		desc:   "Simple functions",
+		mkname: "product.mk",
+		in: `
+$(warning this is the warning)
+$(warning)
+$(info this is the info)
+$(error this is the error)
+PRODUCT_NAME:=$(shell echo *)
+`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  rblf.mkwarning("product.mk", "this is the warning")
+  rblf.mkwarning("product.mk", "")
+  rblf.mkinfo("product.mk", "this is the info")
+  rblf.mkerror("product.mk", "this is the error")
+  cfg["PRODUCT_NAME"] = rblf.shell("echo *")
+`,
+	},
+	{
+		desc:   "Empty if",
+		mkname: "product.mk",
+		in: `
+ifdef PRODUCT_NAME
+# Comment
+endif
+`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  if g.get("PRODUCT_NAME") != None:
+    # Comment
+    pass
+`,
+	},
+	{
+		desc:   "if/else/endif",
+		mkname: "product.mk",
+		in: `
+ifndef PRODUCT_NAME
+  PRODUCT_NAME=gizmo1
+else
+  PRODUCT_NAME=gizmo2
+endif
+`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  if not g.get("PRODUCT_NAME") != None:
+    cfg["PRODUCT_NAME"] = "gizmo1"
+  else:
+    cfg["PRODUCT_NAME"] = "gizmo2"
+`,
+	},
+	{
+		desc:   "else if",
+		mkname: "product.mk",
+		in: `
+ifdef  PRODUCT_NAME
+  PRODUCT_NAME = gizmo
+else ifndef PRODUCT_PACKAGES   # Comment
+endif
+	`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  if g.get("PRODUCT_NAME") != None:
+    cfg["PRODUCT_NAME"] = "gizmo"
+  elif not g.get("PRODUCT_PACKAGES") != None:
+    # Comment
+    pass
+`,
+	},
+	{
+		desc:   "ifeq / ifneq",
+		mkname: "product.mk",
+		in: `
+ifeq (aosp_arm, $(TARGET_PRODUCT))
+  PRODUCT_MODEL = pix2
+else
+  PRODUCT_MODEL = pix21
+endif
+ifneq (aosp_x86, $(TARGET_PRODUCT))
+  PRODUCT_MODEL = pix3
+endif
+`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  if "aosp_arm" == g["TARGET_PRODUCT"]:
+    cfg["PRODUCT_MODEL"] = "pix2"
+  else:
+    cfg["PRODUCT_MODEL"] = "pix21"
+  if "aosp_x86" != g["TARGET_PRODUCT"]:
+    cfg["PRODUCT_MODEL"] = "pix3"
+`,
+	},
+	{
+		desc:   "Check filter result",
+		mkname: "product.mk",
+		in: `
+ifeq (,$(filter userdebug eng, $(TARGET_BUILD_VARIANT)))
+endif
+ifneq (,$(filter userdebug,$(TARGET_BUILD_VARIANT))
+endif
+ifneq (,$(filter plaf,$(PLATFORM_LIST)))
+endif
+ifeq ($(TARGET_BUILD_VARIANT), $(filter $(TARGET_BUILD_VARIANT), userdebug eng))
+endif
+`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  if g["TARGET_BUILD_VARIANT"] not in ["userdebug", "eng"]:
+    pass
+  if g["TARGET_BUILD_VARIANT"] in ["userdebug"]:
+    pass
+  if "plaf" in g.get("PLATFORM_LIST", []):
+    pass
+  if g["TARGET_BUILD_VARIANT"] in ["userdebug", "eng"]:
+    pass
+`,
+	},
+	{
+		desc:   "Get filter result",
+		mkname: "product.mk",
+		in: `
+PRODUCT_LIST2=$(filter-out %/foo.ko,$(wildcard path/*.ko))
+`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  cfg["PRODUCT_LIST2"] = rblf.filter_out("%/foo.ko", rblf.expand_wildcard("path/*.ko"))
+`,
+	},
+	{
+		desc:   "filter $(VAR), values",
+		mkname: "product.mk",
+		in: `
+ifeq (,$(filter $(TARGET_PRODUCT), yukawa_gms yukawa_sei510_gms)
+  ifneq (,$(filter $(TARGET_PRODUCT), yukawa_gms)
+  endif
+endif
+
+`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  if g["TARGET_PRODUCT"] not in ["yukawa_gms", "yukawa_sei510_gms"]:
+    if g["TARGET_PRODUCT"] in ["yukawa_gms"]:
+      pass
+`,
+	},
+	{
+		desc:   "ifeq",
+		mkname: "product.mk",
+		in: `
+ifeq (aosp, $(TARGET_PRODUCT)) # Comment
+else ifneq (, $(TARGET_PRODUCT))
+endif
+`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  if "aosp" == g["TARGET_PRODUCT"]:
+    # Comment
+    pass
+  elif g["TARGET_PRODUCT"]:
+    pass
+`,
+	},
+	{
+		desc:   "Nested if",
+		mkname: "product.mk",
+		in: `
+ifdef PRODUCT_NAME
+  PRODUCT_PACKAGES = pack-if0
+  ifdef PRODUCT_MODEL
+    PRODUCT_PACKAGES = pack-if-if
+  else ifdef PRODUCT_NAME
+    PRODUCT_PACKAGES = pack-if-elif
+  else
+    PRODUCT_PACKAGES = pack-if-else
+  endif
+  PRODUCT_PACKAGES = pack-if
+else ifneq (,$(TARGET_PRODUCT))
+  PRODUCT_PACKAGES = pack-elif
+else
+  PRODUCT_PACKAGES = pack-else
+endif
+`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  if g.get("PRODUCT_NAME") != None:
+    cfg["PRODUCT_PACKAGES"] = ["pack-if0"]
+    if g.get("PRODUCT_MODEL") != None:
+      cfg["PRODUCT_PACKAGES"] = ["pack-if-if"]
+    elif g.get("PRODUCT_NAME") != None:
+      cfg["PRODUCT_PACKAGES"] = ["pack-if-elif"]
+    else:
+      cfg["PRODUCT_PACKAGES"] = ["pack-if-else"]
+    cfg["PRODUCT_PACKAGES"] = ["pack-if"]
+  elif g["TARGET_PRODUCT"]:
+    cfg["PRODUCT_PACKAGES"] = ["pack-elif"]
+  else:
+    cfg["PRODUCT_PACKAGES"] = ["pack-else"]
+`,
+	},
+	{
+		desc:   "Wildcard",
+		mkname: "product.mk",
+		in: `
+ifeq (,$(wildcard foo.mk))
+endif
+ifneq (,$(wildcard foo*.mk))
+endif
+`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  if not rblf.file_exists("foo.mk"):
+    pass
+  if rblf.file_wildcard_exists("foo*.mk"):
+    pass
+`,
+	},
+	{
+		desc:   "ifneq $(X),true",
+		mkname: "product.mk",
+		in: `
+ifneq ($(VARIABLE),true)
+endif
+`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  if g.get("VARIABLE", "") != "true":
+    pass
+`,
+	},
+	{
+		desc:   "Const neq",
+		mkname: "product.mk",
+		in: `
+ifneq (1,0)
+endif
+`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  if "1" != "0":
+    pass
+`,
+	},
+	{
+		desc:   "is-board calls",
+		mkname: "product.mk",
+		in: `
+ifeq ($(call is-board-platform-in-list,msm8998), true)
+else ifneq ($(call is-board-platform,copper),true)
+else ifneq ($(call is-vendor-board-platform,QCOM),true)
+else ifeq ($(call is-product-in-list, $(PLATFORM_LIST)), true)
+endif
+`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  if g.get("TARGET_BOARD_PLATFORM", "") in ["msm8998"]:
+    pass
+  elif g.get("TARGET_BOARD_PLATFORM", "") != "copper":
+    pass
+  elif g.get("TARGET_BOARD_PLATFORM", "") not in g["QCOM_BOARD_PLATFORMS"]:
+    pass
+  elif g["TARGET_PRODUCT"] in g.get("PLATFORM_LIST", []):
+    pass
+`,
+	},
+	{
+		desc:   "findstring call",
+		mkname: "product.mk",
+		in: `
+ifneq ($(findstring foo,$(PRODUCT_PACKAGES)),)
+endif
+`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  if (cfg.get("PRODUCT_PACKAGES", [])).find("foo") != -1:
+    pass
+`,
+	},
+	{
+		desc:   "rhs call",
+		mkname: "product.mk",
+		in: `
+PRODUCT_COPY_FILES = $(call add-to-product-copy-files-if-exists, path:distpath) \
+ $(call find-copy-subdir-files, *, fromdir, todir) $(wildcard foo.*)
+`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  cfg["PRODUCT_COPY_FILES"] = (rblf.copy_if_exists("path:distpath") +
+      rblf.find_and_copy("*", "fromdir", "todir") +
+      rblf.expand_wildcard("foo.*"))
+`,
+	},
+	{
+		desc:   "inferred type",
+		mkname: "product.mk",
+		in: `
+HIKEY_MODS := $(wildcard foo/*.ko)
+BOARD_VENDOR_KERNEL_MODULES += $(HIKEY_MODS)
+`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  g["HIKEY_MODS"] = rblf.expand_wildcard("foo/*.ko")
+  g.setdefault("BOARD_VENDOR_KERNEL_MODULES", [])
+  g["BOARD_VENDOR_KERNEL_MODULES"] += g["HIKEY_MODS"]
+`,
+	},
+	{
+		desc:   "list with vars",
+		mkname: "product.mk",
+		in: `
+PRODUCT_COPY_FILES += path1:$(TARGET_PRODUCT)/path1 $(PRODUCT_MODEL)/path2:$(TARGET_PRODUCT)/path2
+`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  rblf.setdefault(handle, "PRODUCT_COPY_FILES")
+  cfg["PRODUCT_COPY_FILES"] += (("path1:%s/path1" % g["TARGET_PRODUCT"]).split() +
+      ("%s/path2:%s/path2" % (cfg.get("PRODUCT_MODEL", ""), g["TARGET_PRODUCT"])).split())
+`,
+	},
+	{
+		desc:   "misc calls",
+		mkname: "product.mk",
+		in: `
+$(call enforce-product-packages-exist,)
+$(call enforce-product-packages-exist, foo)
+$(call require-artifacts-in-path, foo, bar)
+$(call require-artifacts-in-path-relaxed, foo, bar)
+`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  rblf.enforce_product_packages_exist("")
+  rblf.enforce_product_packages_exist("foo")
+  rblf.require_artifacts_in_path("foo", "bar")
+  rblf.require_artifacts_in_path_relaxed("foo", "bar")
+`,
+	},
+	{
+		desc:   "list with functions",
+		mkname: "product.mk",
+		in: `
+PRODUCT_COPY_FILES := $(call find-copy-subdir-files,*.kl,from1,to1) \
+ $(call find-copy-subdir-files,*.kc,from2,to2) \
+ foo bar
+`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  cfg["PRODUCT_COPY_FILES"] = (rblf.find_and_copy("*.kl", "from1", "to1") +
+      rblf.find_and_copy("*.kc", "from2", "to2") +
+      [
+          "foo",
+          "bar",
+      ])
+`,
+	},
+	{
+		desc:   "Text functions",
+		mkname: "product.mk",
+		in: `
+PRODUCT_COPY_FILES := $(addprefix pfx-,a b c)
+PRODUCT_COPY_FILES := $(addsuffix .sff, a b c)
+PRODUCT_NAME := $(word 1, $(subst ., ,$(TARGET_BOARD_PLATFORM)))
+
+`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  cfg["PRODUCT_COPY_FILES"] = rblf.addprefix("pfx-", "a b c")
+  cfg["PRODUCT_COPY_FILES"] = rblf.addsuffix(".sff", "a b c")
+  cfg["PRODUCT_NAME"] = ((g.get("TARGET_BOARD_PLATFORM", "")).replace(".", " ")).split()[0]
+`,
+	},
+	{
+		desc:   "assignment flavors",
+		mkname: "product.mk",
+		in: `
+PRODUCT_LIST1 := a
+PRODUCT_LIST2 += a
+PRODUCT_LIST1 += b
+PRODUCT_LIST2 += b
+PRODUCT_LIST3 ?= a
+PRODUCT_LIST1 = c
+PLATFORM_LIST += x
+PRODUCT_PACKAGES := $(PLATFORM_LIST)
+`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  cfg["PRODUCT_LIST1"] = ["a"]
+  rblf.setdefault(handle, "PRODUCT_LIST2")
+  cfg["PRODUCT_LIST2"] += ["a"]
+  cfg["PRODUCT_LIST1"] += ["b"]
+  cfg["PRODUCT_LIST2"] += ["b"]
+  if cfg.get("PRODUCT_LIST3") == None:
+    cfg["PRODUCT_LIST3"] = ["a"]
+  cfg["PRODUCT_LIST1"] = ["c"]
+  g.setdefault("PLATFORM_LIST", [])
+  g["PLATFORM_LIST"] += ["x"]
+  cfg["PRODUCT_PACKAGES"] = g["PLATFORM_LIST"][:]
+`,
+	},
+	{
+		desc:   "assigment flavors2",
+		mkname: "product.mk",
+		in: `
+PRODUCT_LIST1 = a
+ifeq (0,1)
+  PRODUCT_LIST1 += b
+  PRODUCT_LIST2 += b
+endif
+PRODUCT_LIST1 += c
+PRODUCT_LIST2 += c
+`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  cfg["PRODUCT_LIST1"] = ["a"]
+  if "0" == "1":
+    cfg["PRODUCT_LIST1"] += ["b"]
+    rblf.setdefault(handle, "PRODUCT_LIST2")
+    cfg["PRODUCT_LIST2"] += ["b"]
+  cfg["PRODUCT_LIST1"] += ["c"]
+  rblf.setdefault(handle, "PRODUCT_LIST2")
+  cfg["PRODUCT_LIST2"] += ["c"]
+`,
+	},
+	{
+		desc:   "string split",
+		mkname: "product.mk",
+		in: `
+PRODUCT_LIST1 = a
+local = b
+local += c
+FOO = d
+FOO += e
+PRODUCT_LIST1 += $(local)
+PRODUCT_LIST1 += $(FOO)
+`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  cfg["PRODUCT_LIST1"] = ["a"]
+  _local = "b"
+  _local += " " + "c"
+  g["FOO"] = "d"
+  g["FOO"] += " " + "e"
+  cfg["PRODUCT_LIST1"] += (_local).split()
+  cfg["PRODUCT_LIST1"] += (g["FOO"]).split()
+`,
+	},
+	{
+		desc:   "apex_jars",
+		mkname: "product.mk",
+		in: `
+PRODUCT_BOOT_JARS := $(ART_APEX_JARS) framework-minus-apex
+`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  cfg["PRODUCT_BOOT_JARS"] = (g.get("ART_APEX_JARS", []) +
+      ["framework-minus-apex"])
+`,
+	},
+	{
+		desc:   "strip function",
+		mkname: "product.mk",
+		in: `
+ifeq ($(filter hwaddress,$(PRODUCT_PACKAGES)),)
+   PRODUCT_PACKAGES := $(strip $(PRODUCT_PACKAGES) hwaddress)
+endif
+`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  if "hwaddress" not in cfg.get("PRODUCT_PACKAGES", []):
+    cfg["PRODUCT_PACKAGES"] = (rblf.mkstrip("%s hwaddress" % " ".join(cfg.get("PRODUCT_PACKAGES", [])))).split()
+`,
+	},
+	{
+		desc:   "strip func in condition",
+		mkname: "product.mk",
+		in: `
+ifneq ($(strip $(TARGET_VENDOR)),)
+endif
+`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  if rblf.mkstrip(g.get("TARGET_VENDOR", "")) != "":
+    pass
+`,
+	},
+	{
+		desc:   "ref after set",
+		mkname: "product.mk",
+		in: `
+PRODUCT_ADB_KEYS:=value
+FOO := $(PRODUCT_ADB_KEYS)
+ifneq (,$(PRODUCT_ADB_KEYS))
+endif
+`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  g["PRODUCT_ADB_KEYS"] = "value"
+  g["FOO"] = g["PRODUCT_ADB_KEYS"]
+  if g["PRODUCT_ADB_KEYS"]:
+    pass
+`,
+	},
+	{
+		desc:   "ref before set",
+		mkname: "product.mk",
+		in: `
+V1 := $(PRODUCT_ADB_KEYS)
+ifeq (,$(PRODUCT_ADB_KEYS))
+  V2 := $(PRODUCT_ADB_KEYS)
+  PRODUCT_ADB_KEYS:=foo
+  V3 := $(PRODUCT_ADB_KEYS)
+endif`,
+		expected: `load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+  cfg = rblf.cfg(handle)
+  g["V1"] = g.get("PRODUCT_ADB_KEYS", "")
+  if not g.get("PRODUCT_ADB_KEYS", ""):
+    g["V2"] = g.get("PRODUCT_ADB_KEYS", "")
+    g["PRODUCT_ADB_KEYS"] = "foo"
+    g["V3"] = g["PRODUCT_ADB_KEYS"]
+`,
+	},
+}
+
+var known_variables = []struct {
+	name  string
+	class varClass
+	starlarkType
+}{
+	{"PRODUCT_NAME", VarClassConfig, starlarkTypeString},
+	{"PRODUCT_MODEL", VarClassConfig, starlarkTypeString},
+	{"PRODUCT_PACKAGES", VarClassConfig, starlarkTypeList},
+	{"PRODUCT_BOOT_JARS", VarClassConfig, starlarkTypeList},
+	{"PRODUCT_COPY_FILES", VarClassConfig, starlarkTypeList},
+	{"PRODUCT_IS_64BIT", VarClassConfig, starlarkTypeString},
+	{"PRODUCT_LIST1", VarClassConfig, starlarkTypeList},
+	{"PRODUCT_LIST2", VarClassConfig, starlarkTypeList},
+	{"PRODUCT_LIST3", VarClassConfig, starlarkTypeList},
+	{"TARGET_PRODUCT", VarClassSoong, starlarkTypeString},
+	{"TARGET_BUILD_VARIANT", VarClassSoong, starlarkTypeString},
+	{"TARGET_BOARD_PLATFORM", VarClassSoong, starlarkTypeString},
+	{"QCOM_BOARD_PLATFORMS", VarClassSoong, starlarkTypeString},
+	{"PLATFORM_LIST", VarClassSoong, starlarkTypeList}, // TODO(asmundak): make it local instead of soong
+}
+
+func TestGood(t *testing.T) {
+	for _, v := range known_variables {
+		KnownVariables.NewVariable(v.name, v.class, v.starlarkType)
+	}
+	for _, test := range testCases {
+		t.Run(test.desc,
+			func(t *testing.T) {
+				ss, err := Convert(Request{
+					MkFile:             test.mkname,
+					Reader:             bytes.NewBufferString(test.in),
+					RootDir:            ".",
+					OutputSuffix:       ".star",
+					WarnPartialSuccess: true,
+				})
+				if err != nil {
+					t.Error(err)
+					return
+				}
+				got := ss.String()
+				if got != test.expected {
+					t.Errorf("%q failed\nExpected:\n%s\nActual:\n%s\n", test.desc,
+						strings.ReplaceAll(test.expected, "\n", "␤\n"),
+						strings.ReplaceAll(got, "\n", "␤\n"))
+				}
+			})
+	}
+}

+ 237 - 0
mk2rbc/node.go

@@ -0,0 +1,237 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mk2rbc
+
+import (
+	"fmt"
+	"strings"
+
+	mkparser "android/soong/androidmk/parser"
+)
+
+// A parsed node for which starlark code will be generated
+// by calling emit().
+type starlarkNode interface {
+	emit(ctx *generationContext)
+}
+
+// Types used to keep processed makefile data:
+type commentNode struct {
+	text string
+}
+
+func (c *commentNode) emit(gctx *generationContext) {
+	chunks := strings.Split(c.text, "\\\n")
+	gctx.newLine()
+	gctx.write(chunks[0]) // It has '#' at the beginning already.
+	for _, chunk := range chunks[1:] {
+		gctx.newLine()
+		gctx.write("#", chunk)
+	}
+}
+
+type inheritedModule struct {
+	path            string // Converted Starlark file path
+	originalPath    string // Makefile file path
+	moduleName      string
+	moduleLocalName string
+	loadAlways      bool
+}
+
+func (im inheritedModule) name() string {
+	return MakePath2ModuleName(im.originalPath)
+}
+
+func (im inheritedModule) entryName() string {
+	return im.moduleLocalName + "_init"
+}
+
+type inheritNode struct {
+	*inheritedModule
+}
+
+func (inn *inheritNode) emit(gctx *generationContext) {
+	// Unconditional case:
+	//    rblf.inherit(handle, <module>, module_init)
+	// Conditional case:
+	//    if <module>_init != None:
+	//      same as above
+	gctx.newLine()
+	if inn.loadAlways {
+		gctx.writef("%s(handle, %q, %s)", cfnInherit, inn.name(), inn.entryName())
+		return
+	}
+	gctx.writef("if %s != None:", inn.entryName())
+	gctx.indentLevel++
+	gctx.newLine()
+	gctx.writef("%s(handle, %q, %s)", cfnInherit, inn.name(), inn.entryName())
+	gctx.indentLevel--
+}
+
+type includeNode struct {
+	*inheritedModule
+}
+
+func (inn *includeNode) emit(gctx *generationContext) {
+	gctx.newLine()
+	if inn.loadAlways {
+		gctx.writef("%s(g, handle)", inn.entryName())
+		return
+	}
+	gctx.writef("if %s != None:", inn.entryName())
+	gctx.indentLevel++
+	gctx.newLine()
+	gctx.writef("%s(g, handle)", inn.entryName())
+	gctx.indentLevel--
+}
+
+type assignmentFlavor int
+
+const (
+	// Assignment flavors
+	asgnSet         assignmentFlavor = iota // := or =
+	asgnMaybeSet    assignmentFlavor = iota // ?= and variable may be unset
+	asgnAppend      assignmentFlavor = iota // += and variable has been set before
+	asgnMaybeAppend assignmentFlavor = iota // += and variable may be unset
+)
+
+type assignmentNode struct {
+	lhs      variable
+	value    starlarkExpr
+	mkValue  *mkparser.MakeString
+	flavor   assignmentFlavor
+	isTraced bool
+	previous *assignmentNode
+}
+
+func (asgn *assignmentNode) emit(gctx *generationContext) {
+	gctx.newLine()
+	gctx.inAssignment = true
+	asgn.lhs.emitSet(gctx, asgn)
+	gctx.inAssignment = false
+
+	if asgn.isTraced {
+		gctx.newLine()
+		gctx.tracedCount++
+		gctx.writef(`print("%s.%d: %s := ", `, gctx.starScript.mkFile, gctx.tracedCount, asgn.lhs.name())
+		asgn.lhs.emitGet(gctx, true)
+		gctx.writef(")")
+	}
+}
+
+type exprNode struct {
+	expr starlarkExpr
+}
+
+func (exn *exprNode) emit(gctx *generationContext) {
+	gctx.newLine()
+	exn.expr.emit(gctx)
+}
+
+type ifNode struct {
+	isElif bool // true if this is 'elif' statement
+	expr   starlarkExpr
+}
+
+func (in *ifNode) emit(gctx *generationContext) {
+	ifElif := "if "
+	if in.isElif {
+		ifElif = "elif "
+	}
+
+	gctx.newLine()
+	if bad, ok := in.expr.(*badExpr); ok {
+		gctx.write("# MK2STAR ERROR converting:")
+		gctx.newLine()
+		gctx.writef("#   %s", bad.node.Dump())
+		gctx.newLine()
+		gctx.writef("# %s", bad.message)
+		gctx.newLine()
+		// The init function emits a warning if the conversion was not
+		// fullly successful, so here we (arbitrarily) take the false path.
+		gctx.writef("%sFalse:", ifElif)
+		return
+	}
+	gctx.write(ifElif)
+	in.expr.emit(gctx)
+	gctx.write(":")
+}
+
+type elseNode struct{}
+
+func (br *elseNode) emit(gctx *generationContext) {
+	gctx.newLine()
+	gctx.write("else:")
+}
+
+// switchCase represents as single if/elseif/else branch. All the necessary
+// info about flavor (if/elseif/else) is supposed to be kept in `gate`.
+type switchCase struct {
+	gate  starlarkNode
+	nodes []starlarkNode
+}
+
+func (cb *switchCase) newNode(node starlarkNode) {
+	cb.nodes = append(cb.nodes, node)
+}
+
+func (cb *switchCase) emit(gctx *generationContext) {
+	cb.gate.emit(gctx)
+	gctx.indentLevel++
+	hasStatements := false
+	emitNode := func(node starlarkNode) {
+		if _, ok := node.(*commentNode); !ok {
+			hasStatements = true
+		}
+		node.emit(gctx)
+	}
+	if len(cb.nodes) > 0 {
+		emitNode(cb.nodes[0])
+		for _, node := range cb.nodes[1:] {
+			emitNode(node)
+		}
+		if !hasStatements {
+			gctx.emitPass()
+		}
+	} else {
+		gctx.emitPass()
+	}
+	gctx.indentLevel--
+}
+
+// A single complete if ... elseif ... else ... endif sequences
+type switchNode struct {
+	ssCases []*switchCase
+}
+
+func (ssw *switchNode) newNode(node starlarkNode) {
+	switch br := node.(type) {
+	case *switchCase:
+		ssw.ssCases = append(ssw.ssCases, br)
+	default:
+		panic(fmt.Errorf("expected switchCase node, got %t", br))
+	}
+}
+
+func (ssw *switchNode) emit(gctx *generationContext) {
+	if len(ssw.ssCases) == 0 {
+		gctx.emitPass()
+	} else {
+		ssw.ssCases[0].emit(gctx)
+		for _, ssCase := range ssw.ssCases[1:] {
+			ssCase.emit(gctx)
+		}
+	}
+}

+ 5 - 0
mk2rbc/types.go

@@ -18,6 +18,11 @@ package mk2rbc
 type starlarkType int
 
 const (
+	// Variable types. Initially we only know the types of the  product
+	// configuration variables that are lists, and the types of some
+	// hardwired variables. The remaining variables are first entered as
+	// having an unknown type and treated as strings, but sometimes we
+	//  can infer variable's type from the value assigned to it.
 	starlarkTypeUnknown starlarkType = iota
 	starlarkTypeList    starlarkType = iota
 	starlarkTypeString  starlarkType = iota

+ 300 - 0
mk2rbc/variable.go

@@ -0,0 +1,300 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mk2rbc
+
+import (
+	"fmt"
+	"os"
+	"strings"
+)
+
+type variable interface {
+	name() string
+	emitGet(gctx *generationContext, isDefined bool)
+	emitSet(gctx *generationContext, asgn *assignmentNode)
+	emitDefined(gctx *generationContext)
+	valueType() starlarkType
+	defaultValueString() string
+	isPreset() bool
+}
+
+type baseVariable struct {
+	nam    string
+	typ    starlarkType
+	preset bool // true if it has been initialized at startup
+}
+
+func (v baseVariable) name() string {
+	return v.nam
+}
+
+func (v baseVariable) valueType() starlarkType {
+	return v.typ
+}
+
+func (v baseVariable) isPreset() bool {
+	return v.preset
+}
+
+var defaultValuesByType = map[starlarkType]string{
+	starlarkTypeUnknown: `""`,
+	starlarkTypeList:    "[]",
+	starlarkTypeString:  `""`,
+	starlarkTypeInt:     "0",
+	starlarkTypeBool:    "False",
+	starlarkTypeVoid:    "None",
+}
+
+func (v baseVariable) defaultValueString() string {
+	if v, ok := defaultValuesByType[v.valueType()]; ok {
+		return v
+	}
+	panic(fmt.Errorf("%s has unknown type %q", v.name(), v.valueType()))
+}
+
+type productConfigVariable struct {
+	baseVariable
+}
+
+func (pcv productConfigVariable) emitSet(gctx *generationContext, asgn *assignmentNode) {
+	emitAssignment := func() {
+		pcv.emitGet(gctx, true)
+		gctx.write(" = ")
+		asgn.value.emitListVarCopy(gctx)
+	}
+	emitAppend := func() {
+		pcv.emitGet(gctx, true)
+		gctx.write(" += ")
+		if pcv.valueType() == starlarkTypeString {
+			gctx.writef(`" " + `)
+		}
+		asgn.value.emit(gctx)
+	}
+
+	switch asgn.flavor {
+	case asgnSet:
+		emitAssignment()
+	case asgnAppend:
+		emitAppend()
+	case asgnMaybeAppend:
+		// If we are not sure variable has been assigned before, emit setdefault
+		if pcv.typ == starlarkTypeList {
+			gctx.writef("%s(handle, %q)", cfnSetListDefault, pcv.name())
+		} else {
+			gctx.writef("cfg.setdefault(%q, %s)", pcv.name(), pcv.defaultValueString())
+		}
+		gctx.newLine()
+		emitAppend()
+	case asgnMaybeSet:
+		gctx.writef("if cfg.get(%q) == None:", pcv.nam)
+		gctx.indentLevel++
+		gctx.newLine()
+		emitAssignment()
+		gctx.indentLevel--
+	}
+}
+
+func (pcv productConfigVariable) emitGet(gctx *generationContext, isDefined bool) {
+	if isDefined || pcv.isPreset() {
+		gctx.writef("cfg[%q]", pcv.nam)
+	} else {
+		gctx.writef("cfg.get(%q, %s)", pcv.nam, pcv.defaultValueString())
+	}
+}
+
+func (pcv productConfigVariable) emitDefined(gctx *generationContext) {
+	gctx.writef("g.get(%q) != None", pcv.name())
+}
+
+type otherGlobalVariable struct {
+	baseVariable
+}
+
+func (scv otherGlobalVariable) emitSet(gctx *generationContext, asgn *assignmentNode) {
+	emitAssignment := func() {
+		scv.emitGet(gctx, true)
+		gctx.write(" = ")
+		asgn.value.emitListVarCopy(gctx)
+	}
+
+	emitAppend := func() {
+		scv.emitGet(gctx, true)
+		gctx.write(" += ")
+		if scv.valueType() == starlarkTypeString {
+			gctx.writef(`" " + `)
+		}
+		asgn.value.emit(gctx)
+	}
+
+	switch asgn.flavor {
+	case asgnSet:
+		emitAssignment()
+	case asgnAppend:
+		emitAppend()
+	case asgnMaybeAppend:
+		// If we are not sure variable has been assigned before, emit setdefault
+		gctx.writef("g.setdefault(%q, %s)", scv.name(), scv.defaultValueString())
+		gctx.newLine()
+		emitAppend()
+	case asgnMaybeSet:
+		gctx.writef("if g.get(%q) == None:", scv.nam)
+		gctx.indentLevel++
+		gctx.newLine()
+		emitAssignment()
+		gctx.indentLevel--
+	}
+}
+
+func (scv otherGlobalVariable) emitGet(gctx *generationContext, isDefined bool) {
+	if isDefined || scv.isPreset() {
+		gctx.writef("g[%q]", scv.nam)
+	} else {
+		gctx.writef("g.get(%q, %s)", scv.nam, scv.defaultValueString())
+	}
+}
+
+func (scv otherGlobalVariable) emitDefined(gctx *generationContext) {
+	gctx.writef("g.get(%q) != None", scv.name())
+}
+
+type localVariable struct {
+	baseVariable
+}
+
+func (lv localVariable) emitDefined(_ *generationContext) {
+	panic("implement me")
+}
+
+func (lv localVariable) String() string {
+	return "_" + lv.nam
+}
+
+func (lv localVariable) emitSet(gctx *generationContext, asgn *assignmentNode) {
+	switch asgn.flavor {
+	case asgnSet:
+		gctx.writef("%s = ", lv)
+		asgn.value.emitListVarCopy(gctx)
+	case asgnAppend:
+		lv.emitGet(gctx, false)
+		gctx.write(" += ")
+		if lv.valueType() == starlarkTypeString {
+			gctx.writef(`" " + `)
+		}
+		asgn.value.emit(gctx)
+	case asgnMaybeAppend:
+		gctx.writef("%s(%q, ", cfnLocalAppend, lv)
+		asgn.value.emit(gctx)
+		gctx.write(")")
+	case asgnMaybeSet:
+		gctx.writef("%s(%q, ", cfnLocalSetDefault, lv)
+		asgn.value.emit(gctx)
+		gctx.write(")")
+	}
+}
+
+func (lv localVariable) emitGet(gctx *generationContext, _ bool) {
+	gctx.writef("%s", lv)
+}
+
+type predefinedVariable struct {
+	baseVariable
+	value starlarkExpr
+}
+
+func (pv predefinedVariable) emitGet(gctx *generationContext, _ bool) {
+	pv.value.emit(gctx)
+}
+
+func (pv predefinedVariable) emitSet(_ *generationContext, asgn *assignmentNode) {
+	if expectedValue, ok1 := maybeString(pv.value); ok1 {
+		actualValue, ok2 := maybeString(asgn.value)
+		if ok2 {
+			if actualValue == expectedValue {
+				return
+			}
+			fmt.Fprintf(os.Stderr, "cannot set predefined variable %s to %q, its value should be %q",
+				pv.name(), actualValue, expectedValue)
+			return
+		}
+	}
+	panic(fmt.Errorf("cannot set predefined variable %s to %q", pv.name(), asgn.mkValue.Dump()))
+}
+
+func (pv predefinedVariable) emitDefined(gctx *generationContext) {
+	gctx.write("True")
+}
+
+var localProductConfigVariables = map[string]string{
+	"LOCAL_AUDIO_PRODUCT_PACKAGE":         "PRODUCT_PACKAGES",
+	"LOCAL_AUDIO_PRODUCT_COPY_FILES":      "PRODUCT_COPY_FILES",
+	"LOCAL_AUDIO_DEVICE_PACKAGE_OVERLAYS": "DEVICE_PACKAGE_OVERLAYS",
+	"LOCAL_DUMPSTATE_PRODUCT_PACKAGE":     "PRODUCT_PACKAGES",
+	"LOCAL_GATEKEEPER_PRODUCT_PACKAGE":    "PRODUCT_PACKAGES",
+	"LOCAL_HEALTH_PRODUCT_PACKAGE":        "PRODUCT_PACKAGES",
+	"LOCAL_SENSOR_PRODUCT_PACKAGE":        "PRODUCT_PACKAGES",
+	"LOCAL_KEYMASTER_PRODUCT_PACKAGE":     "PRODUCT_PACKAGES",
+	"LOCAL_KEYMINT_PRODUCT_PACKAGE":       "PRODUCT_PACKAGES",
+}
+
+var presetVariables = map[string]bool{
+	"BUILD_ID":                  true,
+	"HOST_ARCH":                 true,
+	"HOST_OS":                   true,
+	"HOST_BUILD_TYPE":           true,
+	"OUT_DIR":                   true,
+	"PLATFORM_VERSION_CODENAME": true,
+	"PLATFORM_VERSION":          true,
+	"TARGET_ARCH":               true,
+	"TARGET_ARCH_VARIANT":       true,
+	"TARGET_BUILD_TYPE":         true,
+	"TARGET_BUILD_VARIANT":      true,
+	"TARGET_PRODUCT":            true,
+}
+
+// addVariable returns a variable with a given name. A variable is
+// added if it does not exist yet.
+func (ctx *parseContext) addVariable(name string) variable {
+	v, found := ctx.variables[name]
+	if !found {
+		_, preset := presetVariables[name]
+		if vi, found := KnownVariables[name]; found {
+			switch vi.class {
+			case VarClassConfig:
+				v = &productConfigVariable{baseVariable{nam: name, typ: vi.valueType, preset: preset}}
+			case VarClassSoong:
+				v = &otherGlobalVariable{baseVariable{nam: name, typ: vi.valueType, preset: preset}}
+			}
+		} else if name == strings.ToLower(name) {
+			// Heuristics: if variable's name is all lowercase, consider it local
+			// string variable.
+			v = &localVariable{baseVariable{nam: name, typ: starlarkTypeString}}
+		} else {
+			vt := starlarkTypeUnknown
+			if strings.HasPrefix(name, "LOCAL_") {
+				// Heuristics: local variables that contribute to corresponding config variables
+				if cfgVarName, found := localProductConfigVariables[name]; found {
+					vi, found2 := KnownVariables[cfgVarName]
+					if !found2 {
+						panic(fmt.Errorf("unknown config variable %s for %s", cfgVarName, name))
+					}
+					vt = vi.valueType
+				}
+			}
+			v = &otherGlobalVariable{baseVariable{nam: name, typ: vt}}
+		}
+		ctx.variables[name] = v
+	}
+	return v
+}