mk2rbc.go 71 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148
  1. // Copyright 2021 Google LLC
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. // Convert makefile containing device configuration to Starlark file
  15. // The conversion can handle the following constructs in a makefile:
  16. // - comments
  17. // - simple variable assignments
  18. // - $(call init-product,<file>)
  19. // - $(call inherit-product-if-exists
  20. // - if directives
  21. //
  22. // All other constructs are carried over to the output starlark file as comments.
  23. package mk2rbc
  24. import (
  25. "bytes"
  26. "fmt"
  27. "io"
  28. "io/fs"
  29. "io/ioutil"
  30. "os"
  31. "path/filepath"
  32. "regexp"
  33. "sort"
  34. "strconv"
  35. "strings"
  36. "text/scanner"
  37. mkparser "android/soong/androidmk/parser"
  38. )
  39. const (
  40. annotationCommentPrefix = "RBC#"
  41. baseUri = "//build/make/core:product_config.rbc"
  42. // The name of the struct exported by the product_config.rbc
  43. // that contains the functions and variables available to
  44. // product configuration Starlark files.
  45. baseName = "rblf"
  46. soongNsPrefix = "SOONG_CONFIG_"
  47. // And here are the functions and variables:
  48. cfnGetCfg = baseName + ".cfg"
  49. cfnMain = baseName + ".product_configuration"
  50. cfnBoardMain = baseName + ".board_configuration"
  51. cfnPrintVars = baseName + ".printvars"
  52. cfnInherit = baseName + ".inherit"
  53. cfnSetListDefault = baseName + ".setdefault"
  54. )
  55. const (
  56. soongConfigAppend = "soong_config_append"
  57. soongConfigAssign = "soong_config_set"
  58. )
  59. var knownFunctions = map[string]interface {
  60. parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr
  61. }{
  62. "abspath": &simpleCallParser{name: baseName + ".abspath", returnType: starlarkTypeString},
  63. "add-product-dex-preopt-module-config": &simpleCallParser{name: baseName + ".add_product_dex_preopt_module_config", returnType: starlarkTypeString, addHandle: true},
  64. "add_soong_config_namespace": &simpleCallParser{name: baseName + ".soong_config_namespace", returnType: starlarkTypeVoid, addGlobals: true},
  65. "add_soong_config_var_value": &simpleCallParser{name: baseName + ".soong_config_set", returnType: starlarkTypeVoid, addGlobals: true},
  66. soongConfigAssign: &simpleCallParser{name: baseName + ".soong_config_set", returnType: starlarkTypeVoid, addGlobals: true},
  67. soongConfigAppend: &simpleCallParser{name: baseName + ".soong_config_append", returnType: starlarkTypeVoid, addGlobals: true},
  68. "soong_config_get": &simpleCallParser{name: baseName + ".soong_config_get", returnType: starlarkTypeString, addGlobals: true},
  69. "add-to-product-copy-files-if-exists": &simpleCallParser{name: baseName + ".copy_if_exists", returnType: starlarkTypeList},
  70. "addprefix": &simpleCallParser{name: baseName + ".addprefix", returnType: starlarkTypeList},
  71. "addsuffix": &simpleCallParser{name: baseName + ".addsuffix", returnType: starlarkTypeList},
  72. "copy-files": &simpleCallParser{name: baseName + ".copy_files", returnType: starlarkTypeList},
  73. "dir": &simpleCallParser{name: baseName + ".dir", returnType: starlarkTypeString},
  74. "dist-for-goals": &simpleCallParser{name: baseName + ".mkdist_for_goals", returnType: starlarkTypeVoid, addGlobals: true},
  75. "enforce-product-packages-exist": &simpleCallParser{name: baseName + ".enforce_product_packages_exist", returnType: starlarkTypeVoid, addHandle: true},
  76. "error": &makeControlFuncParser{name: baseName + ".mkerror"},
  77. "findstring": &simpleCallParser{name: baseName + ".findstring", returnType: starlarkTypeInt},
  78. "find-copy-subdir-files": &simpleCallParser{name: baseName + ".find_and_copy", returnType: starlarkTypeList},
  79. "filter": &simpleCallParser{name: baseName + ".filter", returnType: starlarkTypeList},
  80. "filter-out": &simpleCallParser{name: baseName + ".filter_out", returnType: starlarkTypeList},
  81. "firstword": &simpleCallParser{name: baseName + ".first_word", returnType: starlarkTypeString},
  82. "foreach": &foreachCallParser{},
  83. "if": &ifCallParser{},
  84. "info": &makeControlFuncParser{name: baseName + ".mkinfo"},
  85. "is-board-platform": &simpleCallParser{name: baseName + ".board_platform_is", returnType: starlarkTypeBool, addGlobals: true},
  86. "is-board-platform2": &simpleCallParser{name: baseName + ".board_platform_is", returnType: starlarkTypeBool, addGlobals: true},
  87. "is-board-platform-in-list": &simpleCallParser{name: baseName + ".board_platform_in", returnType: starlarkTypeBool, addGlobals: true},
  88. "is-board-platform-in-list2": &simpleCallParser{name: baseName + ".board_platform_in", returnType: starlarkTypeBool, addGlobals: true},
  89. "is-product-in-list": &isProductInListCallParser{},
  90. "is-vendor-board-platform": &isVendorBoardPlatformCallParser{},
  91. "is-vendor-board-qcom": &isVendorBoardQcomCallParser{},
  92. "lastword": &simpleCallParser{name: baseName + ".last_word", returnType: starlarkTypeString},
  93. "notdir": &simpleCallParser{name: baseName + ".notdir", returnType: starlarkTypeString},
  94. "math_max": &mathMaxOrMinCallParser{function: "max"},
  95. "math_min": &mathMaxOrMinCallParser{function: "min"},
  96. "math_gt_or_eq": &mathComparisonCallParser{op: ">="},
  97. "math_gt": &mathComparisonCallParser{op: ">"},
  98. "math_lt": &mathComparisonCallParser{op: "<"},
  99. "my-dir": &myDirCallParser{},
  100. "patsubst": &substCallParser{fname: "patsubst"},
  101. "product-copy-files-by-pattern": &simpleCallParser{name: baseName + ".product_copy_files_by_pattern", returnType: starlarkTypeList},
  102. "require-artifacts-in-path": &simpleCallParser{name: baseName + ".require_artifacts_in_path", returnType: starlarkTypeVoid, addHandle: true},
  103. "require-artifacts-in-path-relaxed": &simpleCallParser{name: baseName + ".require_artifacts_in_path_relaxed", returnType: starlarkTypeVoid, addHandle: true},
  104. // TODO(asmundak): remove it once all calls are removed from configuration makefiles. see b/183161002
  105. "shell": &shellCallParser{},
  106. "sort": &simpleCallParser{name: baseName + ".mksort", returnType: starlarkTypeList},
  107. "strip": &simpleCallParser{name: baseName + ".mkstrip", returnType: starlarkTypeString},
  108. "subst": &substCallParser{fname: "subst"},
  109. "warning": &makeControlFuncParser{name: baseName + ".mkwarning"},
  110. "word": &wordCallParser{},
  111. "words": &wordsCallParser{},
  112. "wildcard": &simpleCallParser{name: baseName + ".expand_wildcard", returnType: starlarkTypeList},
  113. }
  114. // The same as knownFunctions, but returns a []starlarkNode instead of a starlarkExpr
  115. var knownNodeFunctions = map[string]interface {
  116. parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) []starlarkNode
  117. }{
  118. "eval": &evalNodeParser{},
  119. "if": &ifCallNodeParser{},
  120. "inherit-product": &inheritProductCallParser{loadAlways: true},
  121. "inherit-product-if-exists": &inheritProductCallParser{loadAlways: false},
  122. "foreach": &foreachCallNodeParser{},
  123. }
  124. // These look like variables, but are actually functions, and would give
  125. // undefined variable errors if we converted them as variables. Instead,
  126. // emit an error instead of converting them.
  127. var unsupportedFunctions = map[string]bool{
  128. "local-generated-sources-dir": true,
  129. "local-intermediates-dir": true,
  130. }
  131. // These are functions that we don't implement conversions for, but
  132. // we allow seeing their definitions in the product config files.
  133. var ignoredDefines = map[string]bool{
  134. "find-word-in-list": true, // internal macro
  135. "get-vendor-board-platforms": true, // internal macro, used by is-board-platform, etc.
  136. "is-android-codename": true, // unused by product config
  137. "is-android-codename-in-list": true, // unused by product config
  138. "is-chipset-in-board-platform": true, // unused by product config
  139. "is-chipset-prefix-in-board-platform": true, // unused by product config
  140. "is-not-board-platform": true, // defined but never used
  141. "is-platform-sdk-version-at-least": true, // unused by product config
  142. "match-prefix": true, // internal macro
  143. "match-word": true, // internal macro
  144. "match-word-in-list": true, // internal macro
  145. "tb-modules": true, // defined in hardware/amlogic/tb_modules/tb_detect.mk, unused
  146. }
  147. var identifierFullMatchRegex = regexp.MustCompile("^[a-zA-Z_][a-zA-Z0-9_]*$")
  148. // Conversion request parameters
  149. type Request struct {
  150. MkFile string // file to convert
  151. Reader io.Reader // if set, read input from this stream instead
  152. OutputSuffix string // generated Starlark files suffix
  153. OutputDir string // if set, root of the output hierarchy
  154. ErrorLogger ErrorLogger
  155. TracedVariables []string // trace assignment to these variables
  156. TraceCalls bool
  157. SourceFS fs.FS
  158. MakefileFinder MakefileFinder
  159. }
  160. // ErrorLogger prints errors and gathers error statistics.
  161. // Its NewError function is called on every error encountered during the conversion.
  162. type ErrorLogger interface {
  163. NewError(el ErrorLocation, node mkparser.Node, text string, args ...interface{})
  164. }
  165. type ErrorLocation struct {
  166. MkFile string
  167. MkLine int
  168. }
  169. func (el ErrorLocation) String() string {
  170. return fmt.Sprintf("%s:%d", el.MkFile, el.MkLine)
  171. }
  172. // Derives module name for a given file. It is base name
  173. // (file name without suffix), with some characters replaced to make it a Starlark identifier
  174. func moduleNameForFile(mkFile string) string {
  175. base := strings.TrimSuffix(filepath.Base(mkFile), filepath.Ext(mkFile))
  176. // TODO(asmundak): what else can be in the product file names?
  177. return strings.NewReplacer("-", "_", ".", "_").Replace(base)
  178. }
  179. func cloneMakeString(mkString *mkparser.MakeString) *mkparser.MakeString {
  180. r := &mkparser.MakeString{StringPos: mkString.StringPos}
  181. r.Strings = append(r.Strings, mkString.Strings...)
  182. r.Variables = append(r.Variables, mkString.Variables...)
  183. return r
  184. }
  185. func isMakeControlFunc(s string) bool {
  186. return s == "error" || s == "warning" || s == "info"
  187. }
  188. // varAssignmentScope points to the last assignment for each variable
  189. // in the current block. It is used during the parsing to chain
  190. // the assignments to a variable together.
  191. type varAssignmentScope struct {
  192. outer *varAssignmentScope
  193. vars map[string]bool
  194. }
  195. // Starlark output generation context
  196. type generationContext struct {
  197. buf strings.Builder
  198. starScript *StarlarkScript
  199. indentLevel int
  200. inAssignment bool
  201. tracedCount int
  202. varAssignments *varAssignmentScope
  203. }
  204. func NewGenerateContext(ss *StarlarkScript) *generationContext {
  205. return &generationContext{
  206. starScript: ss,
  207. varAssignments: &varAssignmentScope{
  208. outer: nil,
  209. vars: make(map[string]bool),
  210. },
  211. }
  212. }
  213. func (gctx *generationContext) pushVariableAssignments() {
  214. va := &varAssignmentScope{
  215. outer: gctx.varAssignments,
  216. vars: make(map[string]bool),
  217. }
  218. gctx.varAssignments = va
  219. }
  220. func (gctx *generationContext) popVariableAssignments() {
  221. gctx.varAssignments = gctx.varAssignments.outer
  222. }
  223. func (gctx *generationContext) hasBeenAssigned(v variable) bool {
  224. for va := gctx.varAssignments; va != nil; va = va.outer {
  225. if _, ok := va.vars[v.name()]; ok {
  226. return true
  227. }
  228. }
  229. return false
  230. }
  231. func (gctx *generationContext) setHasBeenAssigned(v variable) {
  232. gctx.varAssignments.vars[v.name()] = true
  233. }
  234. // emit returns generated script
  235. func (gctx *generationContext) emit() string {
  236. ss := gctx.starScript
  237. // The emitted code has the following layout:
  238. // <initial comments>
  239. // preamble, i.e.,
  240. // load statement for the runtime support
  241. // load statement for each unique submodule pulled in by this one
  242. // def init(g, handle):
  243. // cfg = rblf.cfg(handle)
  244. // <statements>
  245. // <warning if conversion was not clean>
  246. iNode := len(ss.nodes)
  247. for i, node := range ss.nodes {
  248. if _, ok := node.(*commentNode); !ok {
  249. iNode = i
  250. break
  251. }
  252. node.emit(gctx)
  253. }
  254. gctx.emitPreamble()
  255. gctx.newLine()
  256. // The arguments passed to the init function are the global dictionary
  257. // ('g') and the product configuration dictionary ('cfg')
  258. gctx.write("def init(g, handle):")
  259. gctx.indentLevel++
  260. if gctx.starScript.traceCalls {
  261. gctx.newLine()
  262. gctx.writef(`print(">%s")`, gctx.starScript.mkFile)
  263. }
  264. gctx.newLine()
  265. gctx.writef("cfg = %s(handle)", cfnGetCfg)
  266. for _, node := range ss.nodes[iNode:] {
  267. node.emit(gctx)
  268. }
  269. if gctx.starScript.traceCalls {
  270. gctx.newLine()
  271. gctx.writef(`print("<%s")`, gctx.starScript.mkFile)
  272. }
  273. gctx.indentLevel--
  274. gctx.write("\n")
  275. return gctx.buf.String()
  276. }
  277. func (gctx *generationContext) emitPreamble() {
  278. gctx.newLine()
  279. gctx.writef("load(%q, %q)", baseUri, baseName)
  280. // Emit exactly one load statement for each URI.
  281. loadedSubConfigs := make(map[string]string)
  282. for _, mi := range gctx.starScript.inherited {
  283. uri := mi.path
  284. if m, ok := loadedSubConfigs[uri]; ok {
  285. // No need to emit load statement, but fix module name.
  286. mi.moduleLocalName = m
  287. continue
  288. }
  289. if mi.optional || mi.missing {
  290. uri += "|init"
  291. }
  292. gctx.newLine()
  293. gctx.writef("load(%q, %s = \"init\")", uri, mi.entryName())
  294. loadedSubConfigs[uri] = mi.moduleLocalName
  295. }
  296. gctx.write("\n")
  297. }
  298. func (gctx *generationContext) emitPass() {
  299. gctx.newLine()
  300. gctx.write("pass")
  301. }
  302. func (gctx *generationContext) write(ss ...string) {
  303. for _, s := range ss {
  304. gctx.buf.WriteString(s)
  305. }
  306. }
  307. func (gctx *generationContext) writef(format string, args ...interface{}) {
  308. gctx.write(fmt.Sprintf(format, args...))
  309. }
  310. func (gctx *generationContext) newLine() {
  311. if gctx.buf.Len() == 0 {
  312. return
  313. }
  314. gctx.write("\n")
  315. gctx.writef("%*s", 2*gctx.indentLevel, "")
  316. }
  317. func (gctx *generationContext) emitConversionError(el ErrorLocation, message string) {
  318. gctx.writef(`rblf.mk2rbc_error("%s", %q)`, el, message)
  319. }
  320. func (gctx *generationContext) emitLoadCheck(im inheritedModule) {
  321. if !im.needsLoadCheck() {
  322. return
  323. }
  324. gctx.newLine()
  325. gctx.writef("if not %s:", im.entryName())
  326. gctx.indentLevel++
  327. gctx.newLine()
  328. gctx.write(`rblf.mkerror("`, gctx.starScript.mkFile, `", "Cannot find %s" % (`)
  329. im.pathExpr().emit(gctx)
  330. gctx.write("))")
  331. gctx.indentLevel--
  332. }
  333. type knownVariable struct {
  334. name string
  335. class varClass
  336. valueType starlarkType
  337. }
  338. type knownVariables map[string]knownVariable
  339. func (pcv knownVariables) NewVariable(name string, varClass varClass, valueType starlarkType) {
  340. v, exists := pcv[name]
  341. if !exists {
  342. pcv[name] = knownVariable{name, varClass, valueType}
  343. return
  344. }
  345. // Conflict resolution:
  346. // * config class trumps everything
  347. // * any type trumps unknown type
  348. match := varClass == v.class
  349. if !match {
  350. if varClass == VarClassConfig {
  351. v.class = VarClassConfig
  352. match = true
  353. } else if v.class == VarClassConfig {
  354. match = true
  355. }
  356. }
  357. if valueType != v.valueType {
  358. if valueType != starlarkTypeUnknown {
  359. if v.valueType == starlarkTypeUnknown {
  360. v.valueType = valueType
  361. } else {
  362. match = false
  363. }
  364. }
  365. }
  366. if !match {
  367. fmt.Fprintf(os.Stderr, "cannot redefine %s as %v/%v (already defined as %v/%v)\n",
  368. name, varClass, valueType, v.class, v.valueType)
  369. }
  370. }
  371. // All known product variables.
  372. var KnownVariables = make(knownVariables)
  373. func init() {
  374. for _, kv := range []string{
  375. // Kernel-related variables that we know are lists.
  376. "BOARD_VENDOR_KERNEL_MODULES",
  377. "BOARD_VENDOR_RAMDISK_KERNEL_MODULES",
  378. "BOARD_VENDOR_RAMDISK_KERNEL_MODULES_LOAD",
  379. "BOARD_RECOVERY_KERNEL_MODULES",
  380. // Other variables we knwo are lists
  381. "ART_APEX_JARS",
  382. } {
  383. KnownVariables.NewVariable(kv, VarClassSoong, starlarkTypeList)
  384. }
  385. }
  386. // Information about the generated Starlark script.
  387. type StarlarkScript struct {
  388. mkFile string
  389. moduleName string
  390. mkPos scanner.Position
  391. nodes []starlarkNode
  392. inherited []*moduleInfo
  393. hasErrors bool
  394. traceCalls bool // print enter/exit each init function
  395. sourceFS fs.FS
  396. makefileFinder MakefileFinder
  397. nodeLocator func(pos mkparser.Pos) int
  398. }
  399. // parseContext holds the script we are generating and all the ephemeral data
  400. // needed during the parsing.
  401. type parseContext struct {
  402. script *StarlarkScript
  403. nodes []mkparser.Node // Makefile as parsed by mkparser
  404. currentNodeIndex int // Node in it we are processing
  405. ifNestLevel int
  406. moduleNameCount map[string]int // count of imported modules with given basename
  407. fatalError error
  408. outputSuffix string
  409. errorLogger ErrorLogger
  410. tracedVariables map[string]bool // variables to be traced in the generated script
  411. variables map[string]variable
  412. outputDir string
  413. dependentModules map[string]*moduleInfo
  414. soongNamespaces map[string]map[string]bool
  415. includeTops []string
  416. typeHints map[string]starlarkType
  417. atTopOfMakefile bool
  418. }
  419. func newParseContext(ss *StarlarkScript, nodes []mkparser.Node) *parseContext {
  420. predefined := []struct{ name, value string }{
  421. {"SRC_TARGET_DIR", filepath.Join("build", "make", "target")},
  422. {"LOCAL_PATH", filepath.Dir(ss.mkFile)},
  423. {"MAKEFILE_LIST", ss.mkFile},
  424. {"TOPDIR", ""}, // TOPDIR is just set to an empty string in cleanbuild.mk and core.mk
  425. // TODO(asmundak): maybe read it from build/make/core/envsetup.mk?
  426. {"TARGET_COPY_OUT_SYSTEM", "system"},
  427. {"TARGET_COPY_OUT_SYSTEM_OTHER", "system_other"},
  428. {"TARGET_COPY_OUT_DATA", "data"},
  429. {"TARGET_COPY_OUT_ASAN", filepath.Join("data", "asan")},
  430. {"TARGET_COPY_OUT_OEM", "oem"},
  431. {"TARGET_COPY_OUT_RAMDISK", "ramdisk"},
  432. {"TARGET_COPY_OUT_DEBUG_RAMDISK", "debug_ramdisk"},
  433. {"TARGET_COPY_OUT_VENDOR_DEBUG_RAMDISK", "vendor_debug_ramdisk"},
  434. {"TARGET_COPY_OUT_TEST_HARNESS_RAMDISK", "test_harness_ramdisk"},
  435. {"TARGET_COPY_OUT_ROOT", "root"},
  436. {"TARGET_COPY_OUT_RECOVERY", "recovery"},
  437. {"TARGET_COPY_OUT_VENDOR_RAMDISK", "vendor_ramdisk"},
  438. // TODO(asmundak): to process internal config files, we need the following variables:
  439. // TARGET_VENDOR
  440. // target_base_product
  441. //
  442. // the following utility variables are set in build/make/common/core.mk:
  443. {"empty", ""},
  444. {"space", " "},
  445. {"comma", ","},
  446. {"newline", "\n"},
  447. {"pound", "#"},
  448. {"backslash", "\\"},
  449. }
  450. ctx := &parseContext{
  451. script: ss,
  452. nodes: nodes,
  453. currentNodeIndex: 0,
  454. ifNestLevel: 0,
  455. moduleNameCount: make(map[string]int),
  456. variables: make(map[string]variable),
  457. dependentModules: make(map[string]*moduleInfo),
  458. soongNamespaces: make(map[string]map[string]bool),
  459. includeTops: []string{},
  460. typeHints: make(map[string]starlarkType),
  461. atTopOfMakefile: true,
  462. }
  463. for _, item := range predefined {
  464. ctx.variables[item.name] = &predefinedVariable{
  465. baseVariable: baseVariable{nam: item.name, typ: starlarkTypeString},
  466. value: &stringLiteralExpr{item.value},
  467. }
  468. }
  469. return ctx
  470. }
  471. func (ctx *parseContext) hasNodes() bool {
  472. return ctx.currentNodeIndex < len(ctx.nodes)
  473. }
  474. func (ctx *parseContext) getNode() mkparser.Node {
  475. if !ctx.hasNodes() {
  476. return nil
  477. }
  478. node := ctx.nodes[ctx.currentNodeIndex]
  479. ctx.currentNodeIndex++
  480. return node
  481. }
  482. func (ctx *parseContext) backNode() {
  483. if ctx.currentNodeIndex <= 0 {
  484. panic("Cannot back off")
  485. }
  486. ctx.currentNodeIndex--
  487. }
  488. func (ctx *parseContext) handleAssignment(a *mkparser.Assignment) []starlarkNode {
  489. // Handle only simple variables
  490. if !a.Name.Const() || a.Target != nil {
  491. return []starlarkNode{ctx.newBadNode(a, "Only simple variables are handled")}
  492. }
  493. name := a.Name.Strings[0]
  494. // The `override` directive
  495. // override FOO :=
  496. // is parsed as an assignment to a variable named `override FOO`.
  497. // There are very few places where `override` is used, just flag it.
  498. if strings.HasPrefix(name, "override ") {
  499. return []starlarkNode{ctx.newBadNode(a, "cannot handle override directive")}
  500. }
  501. if name == ".KATI_READONLY" {
  502. // Skip assignments to .KATI_READONLY. If it was in the output file, it
  503. // would be an error because it would be sorted before the definition of
  504. // the variable it's trying to make readonly.
  505. return []starlarkNode{}
  506. }
  507. // Soong configuration
  508. if strings.HasPrefix(name, soongNsPrefix) {
  509. return ctx.handleSoongNsAssignment(strings.TrimPrefix(name, soongNsPrefix), a)
  510. }
  511. lhs := ctx.addVariable(name)
  512. if lhs == nil {
  513. return []starlarkNode{ctx.newBadNode(a, "unknown variable %s", name)}
  514. }
  515. _, isTraced := ctx.tracedVariables[lhs.name()]
  516. asgn := &assignmentNode{lhs: lhs, mkValue: a.Value, isTraced: isTraced, location: ctx.errorLocation(a)}
  517. if lhs.valueType() == starlarkTypeUnknown {
  518. // Try to divine variable type from the RHS
  519. asgn.value = ctx.parseMakeString(a, a.Value)
  520. inferred_type := asgn.value.typ()
  521. if inferred_type != starlarkTypeUnknown {
  522. lhs.setValueType(inferred_type)
  523. }
  524. }
  525. if lhs.valueType() == starlarkTypeList {
  526. xConcat, xBad := ctx.buildConcatExpr(a)
  527. if xBad != nil {
  528. asgn.value = xBad
  529. } else {
  530. switch len(xConcat.items) {
  531. case 0:
  532. asgn.value = &listExpr{}
  533. case 1:
  534. asgn.value = xConcat.items[0]
  535. default:
  536. asgn.value = xConcat
  537. }
  538. }
  539. } else {
  540. asgn.value = ctx.parseMakeString(a, a.Value)
  541. }
  542. if asgn.lhs.valueType() == starlarkTypeString &&
  543. asgn.value.typ() != starlarkTypeUnknown &&
  544. asgn.value.typ() != starlarkTypeString {
  545. asgn.value = &toStringExpr{expr: asgn.value}
  546. }
  547. switch a.Type {
  548. case "=", ":=":
  549. asgn.flavor = asgnSet
  550. case "+=":
  551. asgn.flavor = asgnAppend
  552. case "?=":
  553. asgn.flavor = asgnMaybeSet
  554. default:
  555. panic(fmt.Errorf("unexpected assignment type %s", a.Type))
  556. }
  557. return []starlarkNode{asgn}
  558. }
  559. func (ctx *parseContext) handleSoongNsAssignment(name string, asgn *mkparser.Assignment) []starlarkNode {
  560. val := ctx.parseMakeString(asgn, asgn.Value)
  561. if xBad, ok := val.(*badExpr); ok {
  562. return []starlarkNode{&exprNode{expr: xBad}}
  563. }
  564. // Unfortunately, Soong namespaces can be set up by directly setting corresponding Make
  565. // variables instead of via add_soong_config_namespace + add_soong_config_var_value.
  566. // Try to divine the call from the assignment as follows:
  567. if name == "NAMESPACES" {
  568. // Upon seeng
  569. // SOONG_CONFIG_NAMESPACES += foo
  570. // remember that there is a namespace `foo` and act as we saw
  571. // $(call add_soong_config_namespace,foo)
  572. s, ok := maybeString(val)
  573. if !ok {
  574. return []starlarkNode{ctx.newBadNode(asgn, "cannot handle variables in SOONG_CONFIG_NAMESPACES assignment, please use add_soong_config_namespace instead")}
  575. }
  576. result := make([]starlarkNode, 0)
  577. for _, ns := range strings.Fields(s) {
  578. ctx.addSoongNamespace(ns)
  579. result = append(result, &exprNode{&callExpr{
  580. name: baseName + ".soong_config_namespace",
  581. args: []starlarkExpr{&globalsExpr{}, &stringLiteralExpr{ns}},
  582. returnType: starlarkTypeVoid,
  583. }})
  584. }
  585. return result
  586. } else {
  587. // Upon seeing
  588. // SOONG_CONFIG_x_y = v
  589. // find a namespace called `x` and act as if we encountered
  590. // $(call soong_config_set,x,y,v)
  591. // or check that `x_y` is a namespace, and then add the RHS of this assignment as variables in
  592. // it.
  593. // Emit an error in the ambiguous situation (namespaces `foo_bar` with a variable `baz`
  594. // and `foo` with a variable `bar_baz`.
  595. namespaceName := ""
  596. if ctx.hasSoongNamespace(name) {
  597. namespaceName = name
  598. }
  599. var varName string
  600. for pos, ch := range name {
  601. if !(ch == '_' && ctx.hasSoongNamespace(name[0:pos])) {
  602. continue
  603. }
  604. if namespaceName != "" {
  605. return []starlarkNode{ctx.newBadNode(asgn, "ambiguous soong namespace (may be either `%s` or `%s`)", namespaceName, name[0:pos])}
  606. }
  607. namespaceName = name[0:pos]
  608. varName = name[pos+1:]
  609. }
  610. if namespaceName == "" {
  611. return []starlarkNode{ctx.newBadNode(asgn, "cannot figure out Soong namespace, please use add_soong_config_var_value macro instead")}
  612. }
  613. if varName == "" {
  614. // Remember variables in this namespace
  615. s, ok := maybeString(val)
  616. if !ok {
  617. return []starlarkNode{ctx.newBadNode(asgn, "cannot handle variables in SOONG_CONFIG_ assignment, please use add_soong_config_var_value instead")}
  618. }
  619. ctx.updateSoongNamespace(asgn.Type != "+=", namespaceName, strings.Fields(s))
  620. return []starlarkNode{}
  621. }
  622. // Finally, handle assignment to a namespace variable
  623. if !ctx.hasNamespaceVar(namespaceName, varName) {
  624. return []starlarkNode{ctx.newBadNode(asgn, "no %s variable in %s namespace, please use add_soong_config_var_value instead", varName, namespaceName)}
  625. }
  626. fname := baseName + "." + soongConfigAssign
  627. if asgn.Type == "+=" {
  628. fname = baseName + "." + soongConfigAppend
  629. }
  630. return []starlarkNode{&exprNode{&callExpr{
  631. name: fname,
  632. args: []starlarkExpr{&globalsExpr{}, &stringLiteralExpr{namespaceName}, &stringLiteralExpr{varName}, val},
  633. returnType: starlarkTypeVoid,
  634. }}}
  635. }
  636. }
  637. func (ctx *parseContext) buildConcatExpr(a *mkparser.Assignment) (*concatExpr, *badExpr) {
  638. xConcat := &concatExpr{}
  639. var xItemList *listExpr
  640. addToItemList := func(x ...starlarkExpr) {
  641. if xItemList == nil {
  642. xItemList = &listExpr{[]starlarkExpr{}}
  643. }
  644. xItemList.items = append(xItemList.items, x...)
  645. }
  646. finishItemList := func() {
  647. if xItemList != nil {
  648. xConcat.items = append(xConcat.items, xItemList)
  649. xItemList = nil
  650. }
  651. }
  652. items := a.Value.Words()
  653. for _, item := range items {
  654. // A function call in RHS is supposed to return a list, all other item
  655. // expressions return individual elements.
  656. switch x := ctx.parseMakeString(a, item).(type) {
  657. case *badExpr:
  658. return nil, x
  659. case *stringLiteralExpr:
  660. addToItemList(maybeConvertToStringList(x).(*listExpr).items...)
  661. default:
  662. switch x.typ() {
  663. case starlarkTypeList:
  664. finishItemList()
  665. xConcat.items = append(xConcat.items, x)
  666. case starlarkTypeString:
  667. finishItemList()
  668. xConcat.items = append(xConcat.items, &callExpr{
  669. object: x,
  670. name: "split",
  671. args: nil,
  672. returnType: starlarkTypeList,
  673. })
  674. default:
  675. addToItemList(x)
  676. }
  677. }
  678. }
  679. if xItemList != nil {
  680. xConcat.items = append(xConcat.items, xItemList)
  681. }
  682. return xConcat, nil
  683. }
  684. func (ctx *parseContext) newDependentModule(path string, optional bool) *moduleInfo {
  685. modulePath := ctx.loadedModulePath(path)
  686. if mi, ok := ctx.dependentModules[modulePath]; ok {
  687. mi.optional = mi.optional && optional
  688. return mi
  689. }
  690. moduleName := moduleNameForFile(path)
  691. moduleLocalName := "_" + moduleName
  692. n, found := ctx.moduleNameCount[moduleName]
  693. if found {
  694. moduleLocalName += fmt.Sprintf("%d", n)
  695. }
  696. ctx.moduleNameCount[moduleName] = n + 1
  697. _, err := fs.Stat(ctx.script.sourceFS, path)
  698. mi := &moduleInfo{
  699. path: modulePath,
  700. originalPath: path,
  701. moduleLocalName: moduleLocalName,
  702. optional: optional,
  703. missing: err != nil,
  704. }
  705. ctx.dependentModules[modulePath] = mi
  706. ctx.script.inherited = append(ctx.script.inherited, mi)
  707. return mi
  708. }
  709. func (ctx *parseContext) handleSubConfig(
  710. v mkparser.Node, pathExpr starlarkExpr, loadAlways bool, processModule func(inheritedModule) starlarkNode) []starlarkNode {
  711. // Allow seeing $(sort $(wildcard realPathExpr)) or $(wildcard realPathExpr)
  712. // because those are functionally the same as not having the sort/wildcard calls.
  713. if ce, ok := pathExpr.(*callExpr); ok && ce.name == "rblf.mksort" && len(ce.args) == 1 {
  714. if ce2, ok2 := ce.args[0].(*callExpr); ok2 && ce2.name == "rblf.expand_wildcard" && len(ce2.args) == 1 {
  715. pathExpr = ce2.args[0]
  716. }
  717. } else if ce2, ok2 := pathExpr.(*callExpr); ok2 && ce2.name == "rblf.expand_wildcard" && len(ce2.args) == 1 {
  718. pathExpr = ce2.args[0]
  719. }
  720. // In a simple case, the name of a module to inherit/include is known statically.
  721. if path, ok := maybeString(pathExpr); ok {
  722. // Note that even if this directive loads a module unconditionally, a module may be
  723. // absent without causing any harm if this directive is inside an if/else block.
  724. moduleShouldExist := loadAlways && ctx.ifNestLevel == 0
  725. if strings.Contains(path, "*") {
  726. if paths, err := fs.Glob(ctx.script.sourceFS, path); err == nil {
  727. sort.Strings(paths)
  728. result := make([]starlarkNode, 0)
  729. for _, p := range paths {
  730. mi := ctx.newDependentModule(p, !moduleShouldExist)
  731. result = append(result, processModule(inheritedStaticModule{mi, loadAlways}))
  732. }
  733. return result
  734. } else {
  735. return []starlarkNode{ctx.newBadNode(v, "cannot glob wildcard argument")}
  736. }
  737. } else {
  738. mi := ctx.newDependentModule(path, !moduleShouldExist)
  739. return []starlarkNode{processModule(inheritedStaticModule{mi, loadAlways})}
  740. }
  741. }
  742. // If module path references variables (e.g., $(v1)/foo/$(v2)/device-config.mk), find all the paths in the
  743. // source tree that may be a match and the corresponding variable values. For instance, if the source tree
  744. // contains vendor1/foo/abc/dev.mk and vendor2/foo/def/dev.mk, the first one will be inherited when
  745. // (v1, v2) == ('vendor1', 'abc'), and the second one when (v1, v2) == ('vendor2', 'def').
  746. // We then emit the code that loads all of them, e.g.:
  747. // load("//vendor1/foo/abc:dev.rbc", _dev1_init="init")
  748. // load("//vendor2/foo/def/dev.rbc", _dev2_init="init")
  749. // And then inherit it as follows:
  750. // _e = {
  751. // "vendor1/foo/abc/dev.mk": ("vendor1/foo/abc/dev", _dev1_init),
  752. // "vendor2/foo/def/dev.mk": ("vendor2/foo/def/dev", _dev_init2) }.get("%s/foo/%s/dev.mk" % (v1, v2))
  753. // if _e:
  754. // rblf.inherit(handle, _e[0], _e[1])
  755. //
  756. var matchingPaths []string
  757. var needsWarning = false
  758. if interpolate, ok := pathExpr.(*interpolateExpr); ok {
  759. pathPattern := []string{interpolate.chunks[0]}
  760. for _, chunk := range interpolate.chunks[1:] {
  761. if chunk != "" {
  762. pathPattern = append(pathPattern, chunk)
  763. }
  764. }
  765. if len(pathPattern) == 1 {
  766. pathPattern = append(pathPattern, "")
  767. }
  768. matchingPaths = ctx.findMatchingPaths(pathPattern)
  769. needsWarning = pathPattern[0] == "" && len(ctx.includeTops) == 0
  770. } else if len(ctx.includeTops) > 0 {
  771. matchingPaths = append(matchingPaths, ctx.findMatchingPaths([]string{"", ""})...)
  772. } else {
  773. return []starlarkNode{ctx.newBadNode(v, "inherit-product/include argument is too complex")}
  774. }
  775. // Safeguard against $(call inherit-product,$(PRODUCT_PATH))
  776. const maxMatchingFiles = 150
  777. if len(matchingPaths) > maxMatchingFiles {
  778. return []starlarkNode{ctx.newBadNode(v, "there are >%d files matching the pattern, please rewrite it", maxMatchingFiles)}
  779. }
  780. res := inheritedDynamicModule{pathExpr, []*moduleInfo{}, loadAlways, ctx.errorLocation(v), needsWarning}
  781. for _, p := range matchingPaths {
  782. // A product configuration files discovered dynamically may attempt to inherit
  783. // from another one which does not exist in this source tree. Prevent load errors
  784. // by always loading the dynamic files as optional.
  785. res.candidateModules = append(res.candidateModules, ctx.newDependentModule(p, true))
  786. }
  787. return []starlarkNode{processModule(res)}
  788. }
  789. func (ctx *parseContext) findMatchingPaths(pattern []string) []string {
  790. files := ctx.script.makefileFinder.Find(".")
  791. if len(pattern) == 0 {
  792. return files
  793. }
  794. // Create regular expression from the pattern
  795. regexString := "^" + regexp.QuoteMeta(pattern[0])
  796. for _, s := range pattern[1:] {
  797. regexString += ".*" + regexp.QuoteMeta(s)
  798. }
  799. regexString += "$"
  800. rex := regexp.MustCompile(regexString)
  801. includeTopRegexString := ""
  802. if len(ctx.includeTops) > 0 {
  803. for i, top := range ctx.includeTops {
  804. if i > 0 {
  805. includeTopRegexString += "|"
  806. }
  807. includeTopRegexString += "^" + regexp.QuoteMeta(top)
  808. }
  809. } else {
  810. includeTopRegexString = ".*"
  811. }
  812. includeTopRegex := regexp.MustCompile(includeTopRegexString)
  813. // Now match
  814. var res []string
  815. for _, p := range files {
  816. if rex.MatchString(p) && includeTopRegex.MatchString(p) {
  817. res = append(res, p)
  818. }
  819. }
  820. return res
  821. }
  822. type inheritProductCallParser struct {
  823. loadAlways bool
  824. }
  825. func (p *inheritProductCallParser) parse(ctx *parseContext, v mkparser.Node, args *mkparser.MakeString) []starlarkNode {
  826. args.TrimLeftSpaces()
  827. args.TrimRightSpaces()
  828. pathExpr := ctx.parseMakeString(v, args)
  829. if _, ok := pathExpr.(*badExpr); ok {
  830. return []starlarkNode{ctx.newBadNode(v, "Unable to parse argument to inherit")}
  831. }
  832. return ctx.handleSubConfig(v, pathExpr, p.loadAlways, func(im inheritedModule) starlarkNode {
  833. return &inheritNode{im, p.loadAlways}
  834. })
  835. }
  836. func (ctx *parseContext) handleInclude(v *mkparser.Directive) []starlarkNode {
  837. loadAlways := v.Name[0] != '-'
  838. return ctx.handleSubConfig(v, ctx.parseMakeString(v, v.Args), loadAlways, func(im inheritedModule) starlarkNode {
  839. return &includeNode{im, loadAlways}
  840. })
  841. }
  842. func (ctx *parseContext) handleVariable(v *mkparser.Variable) []starlarkNode {
  843. // Handle:
  844. // $(call inherit-product,...)
  845. // $(call inherit-product-if-exists,...)
  846. // $(info xxx)
  847. // $(warning xxx)
  848. // $(error xxx)
  849. // $(call other-custom-functions,...)
  850. if name, args, ok := ctx.maybeParseFunctionCall(v, v.Name); ok {
  851. if kf, ok := knownNodeFunctions[name]; ok {
  852. return kf.parse(ctx, v, args)
  853. }
  854. }
  855. return []starlarkNode{&exprNode{expr: ctx.parseReference(v, v.Name)}}
  856. }
  857. func (ctx *parseContext) maybeHandleDefine(directive *mkparser.Directive) starlarkNode {
  858. macro_name := strings.Fields(directive.Args.Strings[0])[0]
  859. // Ignore the macros that we handle
  860. _, ignored := ignoredDefines[macro_name]
  861. _, known := knownFunctions[macro_name]
  862. if !ignored && !known {
  863. return ctx.newBadNode(directive, "define is not supported: %s", macro_name)
  864. }
  865. return nil
  866. }
  867. func (ctx *parseContext) handleIfBlock(ifDirective *mkparser.Directive) starlarkNode {
  868. ssSwitch := &switchNode{
  869. ssCases: []*switchCase{ctx.processBranch(ifDirective)},
  870. }
  871. for ctx.hasNodes() && ctx.fatalError == nil {
  872. node := ctx.getNode()
  873. switch x := node.(type) {
  874. case *mkparser.Directive:
  875. switch x.Name {
  876. case "else", "elifdef", "elifndef", "elifeq", "elifneq":
  877. ssSwitch.ssCases = append(ssSwitch.ssCases, ctx.processBranch(x))
  878. case "endif":
  879. return ssSwitch
  880. default:
  881. return ctx.newBadNode(node, "unexpected directive %s", x.Name)
  882. }
  883. default:
  884. return ctx.newBadNode(ifDirective, "unexpected statement")
  885. }
  886. }
  887. if ctx.fatalError == nil {
  888. ctx.fatalError = fmt.Errorf("no matching endif for %s", ifDirective.Dump())
  889. }
  890. return ctx.newBadNode(ifDirective, "no matching endif for %s", ifDirective.Dump())
  891. }
  892. // processBranch processes a single branch (if/elseif/else) until the next directive
  893. // on the same level.
  894. func (ctx *parseContext) processBranch(check *mkparser.Directive) *switchCase {
  895. block := &switchCase{gate: ctx.parseCondition(check)}
  896. defer func() {
  897. ctx.ifNestLevel--
  898. }()
  899. ctx.ifNestLevel++
  900. for ctx.hasNodes() {
  901. node := ctx.getNode()
  902. if d, ok := node.(*mkparser.Directive); ok {
  903. switch d.Name {
  904. case "else", "elifdef", "elifndef", "elifeq", "elifneq", "endif":
  905. ctx.backNode()
  906. return block
  907. }
  908. }
  909. block.nodes = append(block.nodes, ctx.handleSimpleStatement(node)...)
  910. }
  911. ctx.fatalError = fmt.Errorf("no matching endif for %s", check.Dump())
  912. return block
  913. }
  914. func (ctx *parseContext) parseCondition(check *mkparser.Directive) starlarkNode {
  915. switch check.Name {
  916. case "ifdef", "ifndef", "elifdef", "elifndef":
  917. if !check.Args.Const() {
  918. return ctx.newBadNode(check, "ifdef variable ref too complex: %s", check.Args.Dump())
  919. }
  920. v := NewVariableRefExpr(ctx.addVariable(check.Args.Strings[0]))
  921. if strings.HasSuffix(check.Name, "ndef") {
  922. v = &notExpr{v}
  923. }
  924. return &ifNode{
  925. isElif: strings.HasPrefix(check.Name, "elif"),
  926. expr: v,
  927. }
  928. case "ifeq", "ifneq", "elifeq", "elifneq":
  929. return &ifNode{
  930. isElif: strings.HasPrefix(check.Name, "elif"),
  931. expr: ctx.parseCompare(check),
  932. }
  933. case "else":
  934. return &elseNode{}
  935. default:
  936. panic(fmt.Errorf("%s: unknown directive: %s", ctx.script.mkFile, check.Dump()))
  937. }
  938. }
  939. func (ctx *parseContext) newBadExpr(node mkparser.Node, text string, args ...interface{}) starlarkExpr {
  940. if ctx.errorLogger != nil {
  941. ctx.errorLogger.NewError(ctx.errorLocation(node), node, text, args...)
  942. }
  943. ctx.script.hasErrors = true
  944. return &badExpr{errorLocation: ctx.errorLocation(node), message: fmt.Sprintf(text, args...)}
  945. }
  946. // records that the given node failed to be converted and includes an explanatory message
  947. func (ctx *parseContext) newBadNode(failedNode mkparser.Node, message string, args ...interface{}) starlarkNode {
  948. return &exprNode{ctx.newBadExpr(failedNode, message, args...)}
  949. }
  950. func (ctx *parseContext) parseCompare(cond *mkparser.Directive) starlarkExpr {
  951. // Strip outer parentheses
  952. mkArg := cloneMakeString(cond.Args)
  953. mkArg.Strings[0] = strings.TrimLeft(mkArg.Strings[0], "( ")
  954. n := len(mkArg.Strings)
  955. mkArg.Strings[n-1] = strings.TrimRight(mkArg.Strings[n-1], ") ")
  956. args := mkArg.Split(",")
  957. // TODO(asmundak): handle the case where the arguments are in quotes and space-separated
  958. if len(args) != 2 {
  959. return ctx.newBadExpr(cond, "ifeq/ifneq len(args) != 2 %s", cond.Dump())
  960. }
  961. args[0].TrimRightSpaces()
  962. args[1].TrimLeftSpaces()
  963. isEq := !strings.HasSuffix(cond.Name, "neq")
  964. xLeft := ctx.parseMakeString(cond, args[0])
  965. xRight := ctx.parseMakeString(cond, args[1])
  966. if bad, ok := xLeft.(*badExpr); ok {
  967. return bad
  968. }
  969. if bad, ok := xRight.(*badExpr); ok {
  970. return bad
  971. }
  972. if expr, ok := ctx.parseCompareSpecialCases(cond, xLeft, xRight); ok {
  973. return expr
  974. }
  975. var stringOperand string
  976. var otherOperand starlarkExpr
  977. if s, ok := maybeString(xLeft); ok {
  978. stringOperand = s
  979. otherOperand = xRight
  980. } else if s, ok := maybeString(xRight); ok {
  981. stringOperand = s
  982. otherOperand = xLeft
  983. }
  984. // If we've identified one of the operands as being a string literal, check
  985. // for some special cases we can do to simplify the resulting expression.
  986. if otherOperand != nil {
  987. if stringOperand == "" {
  988. if isEq {
  989. return negateExpr(otherOperand)
  990. } else {
  991. return otherOperand
  992. }
  993. }
  994. if stringOperand == "true" && otherOperand.typ() == starlarkTypeBool {
  995. if !isEq {
  996. return negateExpr(otherOperand)
  997. } else {
  998. return otherOperand
  999. }
  1000. }
  1001. if otherOperand.typ() == starlarkTypeList {
  1002. fields := strings.Fields(stringOperand)
  1003. elements := make([]starlarkExpr, len(fields))
  1004. for i, s := range fields {
  1005. elements[i] = &stringLiteralExpr{literal: s}
  1006. }
  1007. return &eqExpr{
  1008. left: otherOperand,
  1009. right: &listExpr{elements},
  1010. isEq: isEq,
  1011. }
  1012. }
  1013. if intOperand, err := strconv.Atoi(strings.TrimSpace(stringOperand)); err == nil && otherOperand.typ() == starlarkTypeInt {
  1014. return &eqExpr{
  1015. left: otherOperand,
  1016. right: &intLiteralExpr{literal: intOperand},
  1017. isEq: isEq,
  1018. }
  1019. }
  1020. }
  1021. return &eqExpr{left: xLeft, right: xRight, isEq: isEq}
  1022. }
  1023. // Given an if statement's directive and the left/right starlarkExprs,
  1024. // check if the starlarkExprs are one of a few hardcoded special cases
  1025. // that can be converted to a simpler equality expression than simply comparing
  1026. // the two.
  1027. func (ctx *parseContext) parseCompareSpecialCases(directive *mkparser.Directive, left starlarkExpr,
  1028. right starlarkExpr) (starlarkExpr, bool) {
  1029. isEq := !strings.HasSuffix(directive.Name, "neq")
  1030. // All the special cases require a call on one side and a
  1031. // string literal/variable on the other. Turn the left/right variables into
  1032. // call/value variables, and return false if that's not possible.
  1033. var value starlarkExpr = nil
  1034. call, ok := left.(*callExpr)
  1035. if ok {
  1036. switch right.(type) {
  1037. case *stringLiteralExpr, *variableRefExpr:
  1038. value = right
  1039. }
  1040. } else {
  1041. call, _ = right.(*callExpr)
  1042. switch left.(type) {
  1043. case *stringLiteralExpr, *variableRefExpr:
  1044. value = left
  1045. }
  1046. }
  1047. if call == nil || value == nil {
  1048. return nil, false
  1049. }
  1050. switch call.name {
  1051. case baseName + ".filter":
  1052. return ctx.parseCompareFilterFuncResult(directive, call, value, isEq)
  1053. case baseName + ".findstring":
  1054. return ctx.parseCheckFindstringFuncResult(directive, call, value, !isEq), true
  1055. case baseName + ".strip":
  1056. return ctx.parseCompareStripFuncResult(directive, call, value, !isEq), true
  1057. }
  1058. return nil, false
  1059. }
  1060. func (ctx *parseContext) parseCompareFilterFuncResult(cond *mkparser.Directive,
  1061. filterFuncCall *callExpr, xValue starlarkExpr, negate bool) (starlarkExpr, bool) {
  1062. // We handle:
  1063. // * ifeq/ifneq (,$(filter v1 v2 ..., EXPR) becomes if EXPR not in/in ["v1", "v2", ...]
  1064. // * ifeq/ifneq (,$(filter EXPR, v1 v2 ...) becomes if EXPR not in/in ["v1", "v2", ...]
  1065. if x, ok := xValue.(*stringLiteralExpr); !ok || x.literal != "" {
  1066. return nil, false
  1067. }
  1068. xPattern := filterFuncCall.args[0]
  1069. xText := filterFuncCall.args[1]
  1070. var xInList *stringLiteralExpr
  1071. var expr starlarkExpr
  1072. var ok bool
  1073. if xInList, ok = xPattern.(*stringLiteralExpr); ok && !strings.ContainsRune(xInList.literal, '%') && xText.typ() == starlarkTypeList {
  1074. expr = xText
  1075. } else if xInList, ok = xText.(*stringLiteralExpr); ok {
  1076. expr = xPattern
  1077. } else {
  1078. return nil, false
  1079. }
  1080. slExpr := newStringListExpr(strings.Fields(xInList.literal))
  1081. // Generate simpler code for the common cases:
  1082. if expr.typ() == starlarkTypeList {
  1083. if len(slExpr.items) == 1 {
  1084. // Checking that a string belongs to list
  1085. return &inExpr{isNot: negate, list: expr, expr: slExpr.items[0]}, true
  1086. } else {
  1087. return nil, false
  1088. }
  1089. } else if len(slExpr.items) == 1 {
  1090. return &eqExpr{left: expr, right: slExpr.items[0], isEq: !negate}, true
  1091. } else {
  1092. return &inExpr{isNot: negate, list: newStringListExpr(strings.Fields(xInList.literal)), expr: expr}, true
  1093. }
  1094. }
  1095. func (ctx *parseContext) parseCheckFindstringFuncResult(directive *mkparser.Directive,
  1096. xCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr {
  1097. if isEmptyString(xValue) {
  1098. return &eqExpr{
  1099. left: &callExpr{
  1100. object: xCall.args[1],
  1101. name: "find",
  1102. args: []starlarkExpr{xCall.args[0]},
  1103. returnType: starlarkTypeInt,
  1104. },
  1105. right: &intLiteralExpr{-1},
  1106. isEq: !negate,
  1107. }
  1108. } else if s, ok := maybeString(xValue); ok {
  1109. if s2, ok := maybeString(xCall.args[0]); ok && s == s2 {
  1110. return &eqExpr{
  1111. left: &callExpr{
  1112. object: xCall.args[1],
  1113. name: "find",
  1114. args: []starlarkExpr{xCall.args[0]},
  1115. returnType: starlarkTypeInt,
  1116. },
  1117. right: &intLiteralExpr{-1},
  1118. isEq: negate,
  1119. }
  1120. }
  1121. }
  1122. return ctx.newBadExpr(directive, "$(findstring) can only be compared to nothing or its first argument")
  1123. }
  1124. func (ctx *parseContext) parseCompareStripFuncResult(directive *mkparser.Directive,
  1125. xCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr {
  1126. if _, ok := xValue.(*stringLiteralExpr); !ok {
  1127. return ctx.newBadExpr(directive, "strip result can be compared only to string: %s", xValue)
  1128. }
  1129. return &eqExpr{
  1130. left: &callExpr{
  1131. name: "strip",
  1132. args: xCall.args,
  1133. returnType: starlarkTypeString,
  1134. },
  1135. right: xValue, isEq: !negate}
  1136. }
  1137. func (ctx *parseContext) maybeParseFunctionCall(node mkparser.Node, ref *mkparser.MakeString) (name string, args *mkparser.MakeString, ok bool) {
  1138. ref.TrimLeftSpaces()
  1139. ref.TrimRightSpaces()
  1140. words := ref.SplitN(" ", 2)
  1141. if !words[0].Const() {
  1142. return "", nil, false
  1143. }
  1144. name = words[0].Dump()
  1145. args = mkparser.SimpleMakeString("", words[0].Pos())
  1146. if len(words) >= 2 {
  1147. args = words[1]
  1148. }
  1149. args.TrimLeftSpaces()
  1150. if name == "call" {
  1151. words = args.SplitN(",", 2)
  1152. if words[0].Empty() || !words[0].Const() {
  1153. return "", nil, false
  1154. }
  1155. name = words[0].Dump()
  1156. if len(words) < 2 {
  1157. args = mkparser.SimpleMakeString("", words[0].Pos())
  1158. } else {
  1159. args = words[1]
  1160. }
  1161. }
  1162. ok = true
  1163. return
  1164. }
  1165. // parses $(...), returning an expression
  1166. func (ctx *parseContext) parseReference(node mkparser.Node, ref *mkparser.MakeString) starlarkExpr {
  1167. ref.TrimLeftSpaces()
  1168. ref.TrimRightSpaces()
  1169. refDump := ref.Dump()
  1170. // Handle only the case where the first (or only) word is constant
  1171. words := ref.SplitN(" ", 2)
  1172. if !words[0].Const() {
  1173. if len(words) == 1 {
  1174. expr := ctx.parseMakeString(node, ref)
  1175. return &callExpr{
  1176. object: &identifierExpr{"cfg"},
  1177. name: "get",
  1178. args: []starlarkExpr{
  1179. expr,
  1180. &callExpr{
  1181. object: &identifierExpr{"g"},
  1182. name: "get",
  1183. args: []starlarkExpr{
  1184. expr,
  1185. &stringLiteralExpr{literal: ""},
  1186. },
  1187. returnType: starlarkTypeUnknown,
  1188. },
  1189. },
  1190. returnType: starlarkTypeUnknown,
  1191. }
  1192. } else {
  1193. return ctx.newBadExpr(node, "reference is too complex: %s", refDump)
  1194. }
  1195. }
  1196. if name, _, ok := ctx.maybeParseFunctionCall(node, ref); ok {
  1197. if _, unsupported := unsupportedFunctions[name]; unsupported {
  1198. return ctx.newBadExpr(node, "%s is not supported", refDump)
  1199. }
  1200. }
  1201. // If it is a single word, it can be a simple variable
  1202. // reference or a function call
  1203. if len(words) == 1 && !isMakeControlFunc(refDump) && refDump != "shell" && refDump != "eval" {
  1204. if strings.HasPrefix(refDump, soongNsPrefix) {
  1205. // TODO (asmundak): if we find many, maybe handle them.
  1206. return ctx.newBadExpr(node, "SOONG_CONFIG_ variables cannot be referenced, use soong_config_get instead: %s", refDump)
  1207. }
  1208. // Handle substitution references: https://www.gnu.org/software/make/manual/html_node/Substitution-Refs.html
  1209. if strings.Contains(refDump, ":") {
  1210. parts := strings.SplitN(refDump, ":", 2)
  1211. substParts := strings.SplitN(parts[1], "=", 2)
  1212. if len(substParts) < 2 || strings.Count(substParts[0], "%") > 1 {
  1213. return ctx.newBadExpr(node, "Invalid substitution reference")
  1214. }
  1215. if !strings.Contains(substParts[0], "%") {
  1216. if strings.Contains(substParts[1], "%") {
  1217. return ctx.newBadExpr(node, "A substitution reference must have a %% in the \"before\" part of the substitution if it has one in the \"after\" part.")
  1218. }
  1219. substParts[0] = "%" + substParts[0]
  1220. substParts[1] = "%" + substParts[1]
  1221. }
  1222. v := ctx.addVariable(parts[0])
  1223. if v == nil {
  1224. return ctx.newBadExpr(node, "unknown variable %s", refDump)
  1225. }
  1226. return &callExpr{
  1227. name: baseName + ".mkpatsubst",
  1228. returnType: starlarkTypeString,
  1229. args: []starlarkExpr{
  1230. &stringLiteralExpr{literal: substParts[0]},
  1231. &stringLiteralExpr{literal: substParts[1]},
  1232. NewVariableRefExpr(v),
  1233. },
  1234. }
  1235. }
  1236. if v := ctx.addVariable(refDump); v != nil {
  1237. return NewVariableRefExpr(v)
  1238. }
  1239. return ctx.newBadExpr(node, "unknown variable %s", refDump)
  1240. }
  1241. if name, args, ok := ctx.maybeParseFunctionCall(node, ref); ok {
  1242. if kf, found := knownFunctions[name]; found {
  1243. return kf.parse(ctx, node, args)
  1244. } else {
  1245. return ctx.newBadExpr(node, "cannot handle invoking %s", name)
  1246. }
  1247. }
  1248. return ctx.newBadExpr(node, "cannot handle %s", refDump)
  1249. }
  1250. type simpleCallParser struct {
  1251. name string
  1252. returnType starlarkType
  1253. addGlobals bool
  1254. addHandle bool
  1255. }
  1256. func (p *simpleCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
  1257. expr := &callExpr{name: p.name, returnType: p.returnType}
  1258. if p.addGlobals {
  1259. expr.args = append(expr.args, &globalsExpr{})
  1260. }
  1261. if p.addHandle {
  1262. expr.args = append(expr.args, &identifierExpr{name: "handle"})
  1263. }
  1264. for _, arg := range args.Split(",") {
  1265. arg.TrimLeftSpaces()
  1266. arg.TrimRightSpaces()
  1267. x := ctx.parseMakeString(node, arg)
  1268. if xBad, ok := x.(*badExpr); ok {
  1269. return xBad
  1270. }
  1271. expr.args = append(expr.args, x)
  1272. }
  1273. return expr
  1274. }
  1275. type makeControlFuncParser struct {
  1276. name string
  1277. }
  1278. func (p *makeControlFuncParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
  1279. // Make control functions need special treatment as everything
  1280. // after the name is a single text argument
  1281. x := ctx.parseMakeString(node, args)
  1282. if xBad, ok := x.(*badExpr); ok {
  1283. return xBad
  1284. }
  1285. return &callExpr{
  1286. name: p.name,
  1287. args: []starlarkExpr{
  1288. &stringLiteralExpr{ctx.script.mkFile},
  1289. x,
  1290. },
  1291. returnType: starlarkTypeUnknown,
  1292. }
  1293. }
  1294. type shellCallParser struct{}
  1295. func (p *shellCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
  1296. // Shell functions need special treatment as everything
  1297. // after the name is a single text argument
  1298. x := ctx.parseMakeString(node, args)
  1299. if xBad, ok := x.(*badExpr); ok {
  1300. return xBad
  1301. }
  1302. return &callExpr{
  1303. name: baseName + ".shell",
  1304. args: []starlarkExpr{x},
  1305. returnType: starlarkTypeUnknown,
  1306. }
  1307. }
  1308. type myDirCallParser struct{}
  1309. func (p *myDirCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
  1310. if !args.Empty() {
  1311. return ctx.newBadExpr(node, "my-dir function cannot have any arguments passed to it.")
  1312. }
  1313. return &stringLiteralExpr{literal: filepath.Dir(ctx.script.mkFile)}
  1314. }
  1315. type isProductInListCallParser struct{}
  1316. func (p *isProductInListCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
  1317. if args.Empty() {
  1318. return ctx.newBadExpr(node, "is-product-in-list requires an argument")
  1319. }
  1320. return &inExpr{
  1321. expr: NewVariableRefExpr(ctx.addVariable("TARGET_PRODUCT")),
  1322. list: maybeConvertToStringList(ctx.parseMakeString(node, args)),
  1323. isNot: false,
  1324. }
  1325. }
  1326. type isVendorBoardPlatformCallParser struct{}
  1327. func (p *isVendorBoardPlatformCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
  1328. if args.Empty() || !identifierFullMatchRegex.MatchString(args.Dump()) {
  1329. return ctx.newBadExpr(node, "cannot handle non-constant argument to is-vendor-board-platform")
  1330. }
  1331. return &inExpr{
  1332. expr: NewVariableRefExpr(ctx.addVariable("TARGET_BOARD_PLATFORM")),
  1333. list: NewVariableRefExpr(ctx.addVariable(args.Dump() + "_BOARD_PLATFORMS")),
  1334. isNot: false,
  1335. }
  1336. }
  1337. type isVendorBoardQcomCallParser struct{}
  1338. func (p *isVendorBoardQcomCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
  1339. if !args.Empty() {
  1340. return ctx.newBadExpr(node, "is-vendor-board-qcom does not accept any arguments")
  1341. }
  1342. return &inExpr{
  1343. expr: NewVariableRefExpr(ctx.addVariable("TARGET_BOARD_PLATFORM")),
  1344. list: NewVariableRefExpr(ctx.addVariable("QCOM_BOARD_PLATFORMS")),
  1345. isNot: false,
  1346. }
  1347. }
  1348. type substCallParser struct {
  1349. fname string
  1350. }
  1351. func (p *substCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
  1352. words := args.Split(",")
  1353. if len(words) != 3 {
  1354. return ctx.newBadExpr(node, "%s function should have 3 arguments", p.fname)
  1355. }
  1356. from := ctx.parseMakeString(node, words[0])
  1357. if xBad, ok := from.(*badExpr); ok {
  1358. return xBad
  1359. }
  1360. to := ctx.parseMakeString(node, words[1])
  1361. if xBad, ok := to.(*badExpr); ok {
  1362. return xBad
  1363. }
  1364. words[2].TrimLeftSpaces()
  1365. words[2].TrimRightSpaces()
  1366. obj := ctx.parseMakeString(node, words[2])
  1367. typ := obj.typ()
  1368. if typ == starlarkTypeString && p.fname == "subst" {
  1369. // Optimization: if it's $(subst from, to, string), emit string.replace(from, to)
  1370. return &callExpr{
  1371. object: obj,
  1372. name: "replace",
  1373. args: []starlarkExpr{from, to},
  1374. returnType: typ,
  1375. }
  1376. }
  1377. return &callExpr{
  1378. name: baseName + ".mk" + p.fname,
  1379. args: []starlarkExpr{from, to, obj},
  1380. returnType: obj.typ(),
  1381. }
  1382. }
  1383. type ifCallParser struct{}
  1384. func (p *ifCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
  1385. words := args.Split(",")
  1386. if len(words) != 2 && len(words) != 3 {
  1387. return ctx.newBadExpr(node, "if function should have 2 or 3 arguments, found "+strconv.Itoa(len(words)))
  1388. }
  1389. condition := ctx.parseMakeString(node, words[0])
  1390. ifTrue := ctx.parseMakeString(node, words[1])
  1391. var ifFalse starlarkExpr
  1392. if len(words) == 3 {
  1393. ifFalse = ctx.parseMakeString(node, words[2])
  1394. } else {
  1395. switch ifTrue.typ() {
  1396. case starlarkTypeList:
  1397. ifFalse = &listExpr{items: []starlarkExpr{}}
  1398. case starlarkTypeInt:
  1399. ifFalse = &intLiteralExpr{literal: 0}
  1400. case starlarkTypeBool:
  1401. ifFalse = &boolLiteralExpr{literal: false}
  1402. default:
  1403. ifFalse = &stringLiteralExpr{literal: ""}
  1404. }
  1405. }
  1406. return &ifExpr{
  1407. condition,
  1408. ifTrue,
  1409. ifFalse,
  1410. }
  1411. }
  1412. type ifCallNodeParser struct{}
  1413. func (p *ifCallNodeParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) []starlarkNode {
  1414. words := args.Split(",")
  1415. if len(words) != 2 && len(words) != 3 {
  1416. return []starlarkNode{ctx.newBadNode(node, "if function should have 2 or 3 arguments, found "+strconv.Itoa(len(words)))}
  1417. }
  1418. ifn := &ifNode{expr: ctx.parseMakeString(node, words[0])}
  1419. cases := []*switchCase{
  1420. {
  1421. gate: ifn,
  1422. nodes: ctx.parseNodeMakeString(node, words[1]),
  1423. },
  1424. }
  1425. if len(words) == 3 {
  1426. cases = append(cases, &switchCase{
  1427. gate: &elseNode{},
  1428. nodes: ctx.parseNodeMakeString(node, words[2]),
  1429. })
  1430. }
  1431. if len(cases) == 2 {
  1432. if len(cases[1].nodes) == 0 {
  1433. // Remove else branch if it has no contents
  1434. cases = cases[:1]
  1435. } else if len(cases[0].nodes) == 0 {
  1436. // If the if branch has no contents but the else does,
  1437. // move them to the if and negate its condition
  1438. ifn.expr = negateExpr(ifn.expr)
  1439. cases[0].nodes = cases[1].nodes
  1440. cases = cases[:1]
  1441. }
  1442. }
  1443. return []starlarkNode{&switchNode{ssCases: cases}}
  1444. }
  1445. type foreachCallParser struct{}
  1446. func (p *foreachCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
  1447. words := args.Split(",")
  1448. if len(words) != 3 {
  1449. return ctx.newBadExpr(node, "foreach function should have 3 arguments, found "+strconv.Itoa(len(words)))
  1450. }
  1451. if !words[0].Const() || words[0].Empty() || !identifierFullMatchRegex.MatchString(words[0].Strings[0]) {
  1452. return ctx.newBadExpr(node, "first argument to foreach function must be a simple string identifier")
  1453. }
  1454. loopVarName := words[0].Strings[0]
  1455. list := ctx.parseMakeString(node, words[1])
  1456. action := ctx.parseMakeString(node, words[2]).transform(func(expr starlarkExpr) starlarkExpr {
  1457. if varRefExpr, ok := expr.(*variableRefExpr); ok && varRefExpr.ref.name() == loopVarName {
  1458. return &identifierExpr{loopVarName}
  1459. }
  1460. return nil
  1461. })
  1462. if list.typ() != starlarkTypeList {
  1463. list = &callExpr{
  1464. name: baseName + ".words",
  1465. returnType: starlarkTypeList,
  1466. args: []starlarkExpr{list},
  1467. }
  1468. }
  1469. var result starlarkExpr = &foreachExpr{
  1470. varName: loopVarName,
  1471. list: list,
  1472. action: action,
  1473. }
  1474. if action.typ() == starlarkTypeList {
  1475. result = &callExpr{
  1476. name: baseName + ".flatten_2d_list",
  1477. args: []starlarkExpr{result},
  1478. returnType: starlarkTypeList,
  1479. }
  1480. }
  1481. return result
  1482. }
  1483. func transformNode(node starlarkNode, transformer func(expr starlarkExpr) starlarkExpr) {
  1484. switch a := node.(type) {
  1485. case *ifNode:
  1486. a.expr = a.expr.transform(transformer)
  1487. case *switchCase:
  1488. transformNode(a.gate, transformer)
  1489. for _, n := range a.nodes {
  1490. transformNode(n, transformer)
  1491. }
  1492. case *switchNode:
  1493. for _, n := range a.ssCases {
  1494. transformNode(n, transformer)
  1495. }
  1496. case *exprNode:
  1497. a.expr = a.expr.transform(transformer)
  1498. case *assignmentNode:
  1499. a.value = a.value.transform(transformer)
  1500. case *foreachNode:
  1501. a.list = a.list.transform(transformer)
  1502. for _, n := range a.actions {
  1503. transformNode(n, transformer)
  1504. }
  1505. case *inheritNode:
  1506. if b, ok := a.module.(inheritedDynamicModule); ok {
  1507. b.path = b.path.transform(transformer)
  1508. a.module = b
  1509. }
  1510. case *includeNode:
  1511. if b, ok := a.module.(inheritedDynamicModule); ok {
  1512. b.path = b.path.transform(transformer)
  1513. a.module = b
  1514. }
  1515. }
  1516. }
  1517. type foreachCallNodeParser struct{}
  1518. func (p *foreachCallNodeParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) []starlarkNode {
  1519. words := args.Split(",")
  1520. if len(words) != 3 {
  1521. return []starlarkNode{ctx.newBadNode(node, "foreach function should have 3 arguments, found "+strconv.Itoa(len(words)))}
  1522. }
  1523. if !words[0].Const() || words[0].Empty() || !identifierFullMatchRegex.MatchString(words[0].Strings[0]) {
  1524. return []starlarkNode{ctx.newBadNode(node, "first argument to foreach function must be a simple string identifier")}
  1525. }
  1526. loopVarName := words[0].Strings[0]
  1527. list := ctx.parseMakeString(node, words[1])
  1528. if list.typ() != starlarkTypeList {
  1529. list = &callExpr{
  1530. name: baseName + ".words",
  1531. returnType: starlarkTypeList,
  1532. args: []starlarkExpr{list},
  1533. }
  1534. }
  1535. actions := ctx.parseNodeMakeString(node, words[2])
  1536. // TODO(colefaust): Replace transforming code with something more elegant
  1537. for _, action := range actions {
  1538. transformNode(action, func(expr starlarkExpr) starlarkExpr {
  1539. if varRefExpr, ok := expr.(*variableRefExpr); ok && varRefExpr.ref.name() == loopVarName {
  1540. return &identifierExpr{loopVarName}
  1541. }
  1542. return nil
  1543. })
  1544. }
  1545. return []starlarkNode{&foreachNode{
  1546. varName: loopVarName,
  1547. list: list,
  1548. actions: actions,
  1549. }}
  1550. }
  1551. type wordCallParser struct{}
  1552. func (p *wordCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
  1553. words := args.Split(",")
  1554. if len(words) != 2 {
  1555. return ctx.newBadExpr(node, "word function should have 2 arguments")
  1556. }
  1557. var index = 0
  1558. if words[0].Const() {
  1559. if i, err := strconv.Atoi(strings.TrimSpace(words[0].Strings[0])); err == nil {
  1560. index = i
  1561. }
  1562. }
  1563. if index < 1 {
  1564. return ctx.newBadExpr(node, "word index should be constant positive integer")
  1565. }
  1566. words[1].TrimLeftSpaces()
  1567. words[1].TrimRightSpaces()
  1568. array := ctx.parseMakeString(node, words[1])
  1569. if bad, ok := array.(*badExpr); ok {
  1570. return bad
  1571. }
  1572. if array.typ() != starlarkTypeList {
  1573. array = &callExpr{
  1574. name: baseName + ".words",
  1575. args: []starlarkExpr{array},
  1576. returnType: starlarkTypeList,
  1577. }
  1578. }
  1579. return &indexExpr{array, &intLiteralExpr{index - 1}}
  1580. }
  1581. type wordsCallParser struct{}
  1582. func (p *wordsCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
  1583. args.TrimLeftSpaces()
  1584. args.TrimRightSpaces()
  1585. array := ctx.parseMakeString(node, args)
  1586. if bad, ok := array.(*badExpr); ok {
  1587. return bad
  1588. }
  1589. if array.typ() != starlarkTypeList {
  1590. array = &callExpr{
  1591. name: baseName + ".words",
  1592. args: []starlarkExpr{array},
  1593. returnType: starlarkTypeList,
  1594. }
  1595. }
  1596. return &callExpr{
  1597. name: "len",
  1598. args: []starlarkExpr{array},
  1599. returnType: starlarkTypeInt,
  1600. }
  1601. }
  1602. func parseIntegerArguments(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString, expectedArgs int) ([]starlarkExpr, error) {
  1603. parsedArgs := make([]starlarkExpr, 0)
  1604. for _, arg := range args.Split(",") {
  1605. expr := ctx.parseMakeString(node, arg)
  1606. if expr.typ() == starlarkTypeList {
  1607. return nil, fmt.Errorf("argument to math argument has type list, which cannot be converted to int")
  1608. }
  1609. if s, ok := maybeString(expr); ok {
  1610. intVal, err := strconv.Atoi(strings.TrimSpace(s))
  1611. if err != nil {
  1612. return nil, err
  1613. }
  1614. expr = &intLiteralExpr{literal: intVal}
  1615. } else if expr.typ() != starlarkTypeInt {
  1616. expr = &callExpr{
  1617. name: "int",
  1618. args: []starlarkExpr{expr},
  1619. returnType: starlarkTypeInt,
  1620. }
  1621. }
  1622. parsedArgs = append(parsedArgs, expr)
  1623. }
  1624. if len(parsedArgs) != expectedArgs {
  1625. return nil, fmt.Errorf("function should have %d arguments", expectedArgs)
  1626. }
  1627. return parsedArgs, nil
  1628. }
  1629. type mathComparisonCallParser struct {
  1630. op string
  1631. }
  1632. func (p *mathComparisonCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
  1633. parsedArgs, err := parseIntegerArguments(ctx, node, args, 2)
  1634. if err != nil {
  1635. return ctx.newBadExpr(node, err.Error())
  1636. }
  1637. return &binaryOpExpr{
  1638. left: parsedArgs[0],
  1639. right: parsedArgs[1],
  1640. op: p.op,
  1641. returnType: starlarkTypeBool,
  1642. }
  1643. }
  1644. type mathMaxOrMinCallParser struct {
  1645. function string
  1646. }
  1647. func (p *mathMaxOrMinCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
  1648. parsedArgs, err := parseIntegerArguments(ctx, node, args, 2)
  1649. if err != nil {
  1650. return ctx.newBadExpr(node, err.Error())
  1651. }
  1652. return &callExpr{
  1653. object: nil,
  1654. name: p.function,
  1655. args: parsedArgs,
  1656. returnType: starlarkTypeInt,
  1657. }
  1658. }
  1659. type evalNodeParser struct{}
  1660. func (p *evalNodeParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) []starlarkNode {
  1661. parser := mkparser.NewParser("Eval expression", strings.NewReader(args.Dump()))
  1662. nodes, errs := parser.Parse()
  1663. if errs != nil {
  1664. return []starlarkNode{ctx.newBadNode(node, "Unable to parse eval statement")}
  1665. }
  1666. if len(nodes) == 0 {
  1667. return []starlarkNode{}
  1668. } else if len(nodes) == 1 {
  1669. switch n := nodes[0].(type) {
  1670. case *mkparser.Assignment:
  1671. if n.Name.Const() {
  1672. return ctx.handleAssignment(n)
  1673. }
  1674. case *mkparser.Comment:
  1675. return []starlarkNode{&commentNode{strings.TrimSpace("#" + n.Comment)}}
  1676. case *mkparser.Directive:
  1677. if n.Name == "include" || n.Name == "-include" {
  1678. return ctx.handleInclude(n)
  1679. }
  1680. case *mkparser.Variable:
  1681. // Technically inherit-product(-if-exists) don't need to be put inside
  1682. // an eval, but some makefiles do it, presumably because they copy+pasted
  1683. // from a $(eval include ...)
  1684. if name, _, ok := ctx.maybeParseFunctionCall(n, n.Name); ok {
  1685. if name == "inherit-product" || name == "inherit-product-if-exists" {
  1686. return ctx.handleVariable(n)
  1687. }
  1688. }
  1689. }
  1690. }
  1691. return []starlarkNode{ctx.newBadNode(node, "Eval expression too complex; only assignments, comments, includes, and inherit-products are supported")}
  1692. }
  1693. func (ctx *parseContext) parseMakeString(node mkparser.Node, mk *mkparser.MakeString) starlarkExpr {
  1694. if mk.Const() {
  1695. return &stringLiteralExpr{mk.Dump()}
  1696. }
  1697. if mkRef, ok := mk.SingleVariable(); ok {
  1698. return ctx.parseReference(node, mkRef)
  1699. }
  1700. // If we reached here, it's neither string literal nor a simple variable,
  1701. // we need a full-blown interpolation node that will generate
  1702. // "a%b%c" % (X, Y) for a$(X)b$(Y)c
  1703. parts := make([]starlarkExpr, len(mk.Variables)+len(mk.Strings))
  1704. for i := 0; i < len(parts); i++ {
  1705. if i%2 == 0 {
  1706. parts[i] = &stringLiteralExpr{literal: mk.Strings[i/2]}
  1707. } else {
  1708. parts[i] = ctx.parseReference(node, mk.Variables[i/2].Name)
  1709. if x, ok := parts[i].(*badExpr); ok {
  1710. return x
  1711. }
  1712. }
  1713. }
  1714. return NewInterpolateExpr(parts)
  1715. }
  1716. func (ctx *parseContext) parseNodeMakeString(node mkparser.Node, mk *mkparser.MakeString) []starlarkNode {
  1717. // Discard any constant values in the make string, as they would be top level
  1718. // string literals and do nothing.
  1719. result := make([]starlarkNode, 0, len(mk.Variables))
  1720. for i := range mk.Variables {
  1721. result = append(result, ctx.handleVariable(&mk.Variables[i])...)
  1722. }
  1723. return result
  1724. }
  1725. // Handles the statements whose treatment is the same in all contexts: comment,
  1726. // assignment, variable (which is a macro call in reality) and all constructs that
  1727. // do not handle in any context ('define directive and any unrecognized stuff).
  1728. func (ctx *parseContext) handleSimpleStatement(node mkparser.Node) []starlarkNode {
  1729. var result []starlarkNode
  1730. switch x := node.(type) {
  1731. case *mkparser.Comment:
  1732. if n, handled := ctx.maybeHandleAnnotation(x); handled && n != nil {
  1733. result = []starlarkNode{n}
  1734. } else if !handled {
  1735. result = []starlarkNode{&commentNode{strings.TrimSpace("#" + x.Comment)}}
  1736. }
  1737. case *mkparser.Assignment:
  1738. result = ctx.handleAssignment(x)
  1739. case *mkparser.Variable:
  1740. result = ctx.handleVariable(x)
  1741. case *mkparser.Directive:
  1742. switch x.Name {
  1743. case "define":
  1744. if res := ctx.maybeHandleDefine(x); res != nil {
  1745. result = []starlarkNode{res}
  1746. }
  1747. case "include", "-include":
  1748. result = ctx.handleInclude(x)
  1749. case "ifeq", "ifneq", "ifdef", "ifndef":
  1750. result = []starlarkNode{ctx.handleIfBlock(x)}
  1751. default:
  1752. result = []starlarkNode{ctx.newBadNode(x, "unexpected directive %s", x.Name)}
  1753. }
  1754. default:
  1755. result = []starlarkNode{ctx.newBadNode(x, "unsupported line %s", strings.ReplaceAll(x.Dump(), "\n", "\n#"))}
  1756. }
  1757. // Clear the includeTops after each non-comment statement
  1758. // so that include annotations placed on certain statements don't apply
  1759. // globally for the rest of the makefile was well.
  1760. if _, wasComment := node.(*mkparser.Comment); !wasComment {
  1761. ctx.atTopOfMakefile = false
  1762. ctx.includeTops = []string{}
  1763. }
  1764. if result == nil {
  1765. result = []starlarkNode{}
  1766. }
  1767. return result
  1768. }
  1769. // The types allowed in a type_hint
  1770. var typeHintMap = map[string]starlarkType{
  1771. "string": starlarkTypeString,
  1772. "list": starlarkTypeList,
  1773. }
  1774. // Processes annotation. An annotation is a comment that starts with #RBC# and provides
  1775. // a conversion hint -- say, where to look for the dynamically calculated inherit/include
  1776. // paths. Returns true if the comment was a successfully-handled annotation.
  1777. func (ctx *parseContext) maybeHandleAnnotation(cnode *mkparser.Comment) (starlarkNode, bool) {
  1778. maybeTrim := func(s, prefix string) (string, bool) {
  1779. if strings.HasPrefix(s, prefix) {
  1780. return strings.TrimSpace(strings.TrimPrefix(s, prefix)), true
  1781. }
  1782. return s, false
  1783. }
  1784. annotation, ok := maybeTrim(cnode.Comment, annotationCommentPrefix)
  1785. if !ok {
  1786. return nil, false
  1787. }
  1788. if p, ok := maybeTrim(annotation, "include_top"); ok {
  1789. // Don't allow duplicate include tops, because then we will generate
  1790. // invalid starlark code. (duplicate keys in the _entry dictionary)
  1791. for _, top := range ctx.includeTops {
  1792. if top == p {
  1793. return nil, true
  1794. }
  1795. }
  1796. ctx.includeTops = append(ctx.includeTops, p)
  1797. return nil, true
  1798. } else if p, ok := maybeTrim(annotation, "type_hint"); ok {
  1799. // Type hints must come at the beginning the file, to avoid confusion
  1800. // if a type hint was specified later and thus only takes effect for half
  1801. // of the file.
  1802. if !ctx.atTopOfMakefile {
  1803. return ctx.newBadNode(cnode, "type_hint annotations must come before the first Makefile statement"), true
  1804. }
  1805. parts := strings.Fields(p)
  1806. if len(parts) <= 1 {
  1807. return ctx.newBadNode(cnode, "Invalid type_hint annotation: %s. Must be a variable type followed by a list of variables of that type", p), true
  1808. }
  1809. var varType starlarkType
  1810. if varType, ok = typeHintMap[parts[0]]; !ok {
  1811. varType = starlarkTypeUnknown
  1812. }
  1813. if varType == starlarkTypeUnknown {
  1814. return ctx.newBadNode(cnode, "Invalid type_hint annotation. Only list/string types are accepted, found %s", parts[0]), true
  1815. }
  1816. for _, name := range parts[1:] {
  1817. // Don't allow duplicate type hints
  1818. if _, ok := ctx.typeHints[name]; ok {
  1819. return ctx.newBadNode(cnode, "Duplicate type hint for variable %s", name), true
  1820. }
  1821. ctx.typeHints[name] = varType
  1822. }
  1823. return nil, true
  1824. }
  1825. return ctx.newBadNode(cnode, "unsupported annotation %s", cnode.Comment), true
  1826. }
  1827. func (ctx *parseContext) loadedModulePath(path string) string {
  1828. // During the transition to Roboleaf some of the product configuration files
  1829. // will be converted and checked in while the others will be generated on the fly
  1830. // and run. The runner (rbcrun application) accommodates this by allowing three
  1831. // different ways to specify the loaded file location:
  1832. // 1) load(":<file>",...) loads <file> from the same directory
  1833. // 2) load("//path/relative/to/source/root:<file>", ...) loads <file> source tree
  1834. // 3) load("/absolute/path/to/<file> absolute path
  1835. // If the file being generated and the file it wants to load are in the same directory,
  1836. // generate option 1.
  1837. // Otherwise, if output directory is not specified, generate 2)
  1838. // Finally, if output directory has been specified and the file being generated and
  1839. // the file it wants to load from are in the different directories, generate 2) or 3):
  1840. // * if the file being loaded exists in the source tree, generate 2)
  1841. // * otherwise, generate 3)
  1842. // Finally, figure out the loaded module path and name and create a node for it
  1843. loadedModuleDir := filepath.Dir(path)
  1844. base := filepath.Base(path)
  1845. loadedModuleName := strings.TrimSuffix(base, filepath.Ext(base)) + ctx.outputSuffix
  1846. if loadedModuleDir == filepath.Dir(ctx.script.mkFile) {
  1847. return ":" + loadedModuleName
  1848. }
  1849. if ctx.outputDir == "" {
  1850. return fmt.Sprintf("//%s:%s", loadedModuleDir, loadedModuleName)
  1851. }
  1852. if _, err := os.Stat(filepath.Join(loadedModuleDir, loadedModuleName)); err == nil {
  1853. return fmt.Sprintf("//%s:%s", loadedModuleDir, loadedModuleName)
  1854. }
  1855. return filepath.Join(ctx.outputDir, loadedModuleDir, loadedModuleName)
  1856. }
  1857. func (ctx *parseContext) addSoongNamespace(ns string) {
  1858. if _, ok := ctx.soongNamespaces[ns]; ok {
  1859. return
  1860. }
  1861. ctx.soongNamespaces[ns] = make(map[string]bool)
  1862. }
  1863. func (ctx *parseContext) hasSoongNamespace(name string) bool {
  1864. _, ok := ctx.soongNamespaces[name]
  1865. return ok
  1866. }
  1867. func (ctx *parseContext) updateSoongNamespace(replace bool, namespaceName string, varNames []string) {
  1868. ctx.addSoongNamespace(namespaceName)
  1869. vars := ctx.soongNamespaces[namespaceName]
  1870. if replace {
  1871. vars = make(map[string]bool)
  1872. ctx.soongNamespaces[namespaceName] = vars
  1873. }
  1874. for _, v := range varNames {
  1875. vars[v] = true
  1876. }
  1877. }
  1878. func (ctx *parseContext) hasNamespaceVar(namespaceName string, varName string) bool {
  1879. vars, ok := ctx.soongNamespaces[namespaceName]
  1880. if ok {
  1881. _, ok = vars[varName]
  1882. }
  1883. return ok
  1884. }
  1885. func (ctx *parseContext) errorLocation(node mkparser.Node) ErrorLocation {
  1886. return ErrorLocation{ctx.script.mkFile, ctx.script.nodeLocator(node.Pos())}
  1887. }
  1888. func (ss *StarlarkScript) String() string {
  1889. return NewGenerateContext(ss).emit()
  1890. }
  1891. func (ss *StarlarkScript) SubConfigFiles() []string {
  1892. var subs []string
  1893. for _, src := range ss.inherited {
  1894. subs = append(subs, src.originalPath)
  1895. }
  1896. return subs
  1897. }
  1898. func (ss *StarlarkScript) HasErrors() bool {
  1899. return ss.hasErrors
  1900. }
  1901. // Convert reads and parses a makefile. If successful, parsed tree
  1902. // is returned and then can be passed to String() to get the generated
  1903. // Starlark file.
  1904. func Convert(req Request) (*StarlarkScript, error) {
  1905. reader := req.Reader
  1906. if reader == nil {
  1907. mkContents, err := ioutil.ReadFile(req.MkFile)
  1908. if err != nil {
  1909. return nil, err
  1910. }
  1911. reader = bytes.NewBuffer(mkContents)
  1912. }
  1913. parser := mkparser.NewParser(req.MkFile, reader)
  1914. nodes, errs := parser.Parse()
  1915. if len(errs) > 0 {
  1916. for _, e := range errs {
  1917. fmt.Fprintln(os.Stderr, "ERROR:", e)
  1918. }
  1919. return nil, fmt.Errorf("bad makefile %s", req.MkFile)
  1920. }
  1921. starScript := &StarlarkScript{
  1922. moduleName: moduleNameForFile(req.MkFile),
  1923. mkFile: req.MkFile,
  1924. traceCalls: req.TraceCalls,
  1925. sourceFS: req.SourceFS,
  1926. makefileFinder: req.MakefileFinder,
  1927. nodeLocator: func(pos mkparser.Pos) int { return parser.Unpack(pos).Line },
  1928. nodes: make([]starlarkNode, 0),
  1929. }
  1930. ctx := newParseContext(starScript, nodes)
  1931. ctx.outputSuffix = req.OutputSuffix
  1932. ctx.outputDir = req.OutputDir
  1933. ctx.errorLogger = req.ErrorLogger
  1934. if len(req.TracedVariables) > 0 {
  1935. ctx.tracedVariables = make(map[string]bool)
  1936. for _, v := range req.TracedVariables {
  1937. ctx.tracedVariables[v] = true
  1938. }
  1939. }
  1940. for ctx.hasNodes() && ctx.fatalError == nil {
  1941. starScript.nodes = append(starScript.nodes, ctx.handleSimpleStatement(ctx.getNode())...)
  1942. }
  1943. if ctx.fatalError != nil {
  1944. return nil, ctx.fatalError
  1945. }
  1946. return starScript, nil
  1947. }
  1948. func Launcher(mainModuleUri, inputVariablesUri, mainModuleName string) string {
  1949. var buf bytes.Buffer
  1950. fmt.Fprintf(&buf, "load(%q, %q)\n", baseUri, baseName)
  1951. fmt.Fprintf(&buf, "load(%q, input_variables_init = \"init\")\n", inputVariablesUri)
  1952. fmt.Fprintf(&buf, "load(%q, \"init\")\n", mainModuleUri)
  1953. fmt.Fprintf(&buf, "%s(%s(%q, init, input_variables_init))\n", cfnPrintVars, cfnMain, mainModuleName)
  1954. return buf.String()
  1955. }
  1956. func BoardLauncher(mainModuleUri string, inputVariablesUri string) string {
  1957. var buf bytes.Buffer
  1958. fmt.Fprintf(&buf, "load(%q, %q)\n", baseUri, baseName)
  1959. fmt.Fprintf(&buf, "load(%q, \"init\")\n", mainModuleUri)
  1960. fmt.Fprintf(&buf, "load(%q, input_variables_init = \"init\")\n", inputVariablesUri)
  1961. fmt.Fprintf(&buf, "%s(%s(init, input_variables_init))\n", cfnPrintVars, cfnBoardMain)
  1962. return buf.String()
  1963. }
  1964. func MakePath2ModuleName(mkPath string) string {
  1965. return strings.TrimSuffix(mkPath, filepath.Ext(mkPath))
  1966. }