mk2rbc.go 68 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052
  1. // Copyright 2021 Google LLC
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. // Convert makefile containing device configuration to Starlark file
  15. // The conversion can handle the following constructs in a makefile:
  16. // * comments
  17. // * simple variable assignments
  18. // * $(call init-product,<file>)
  19. // * $(call inherit-product-if-exists
  20. // * if directives
  21. // All other constructs are carried over to the output starlark file as comments.
  22. //
  23. package mk2rbc
  24. import (
  25. "bytes"
  26. "fmt"
  27. "io"
  28. "io/fs"
  29. "io/ioutil"
  30. "os"
  31. "path/filepath"
  32. "regexp"
  33. "strconv"
  34. "strings"
  35. "text/scanner"
  36. mkparser "android/soong/androidmk/parser"
  37. )
  38. const (
  39. annotationCommentPrefix = "RBC#"
  40. baseUri = "//build/make/core:product_config.rbc"
  41. // The name of the struct exported by the product_config.rbc
  42. // that contains the functions and variables available to
  43. // product configuration Starlark files.
  44. baseName = "rblf"
  45. soongNsPrefix = "SOONG_CONFIG_"
  46. // And here are the functions and variables:
  47. cfnGetCfg = baseName + ".cfg"
  48. cfnMain = baseName + ".product_configuration"
  49. cfnBoardMain = baseName + ".board_configuration"
  50. cfnPrintVars = baseName + ".printvars"
  51. cfnInherit = baseName + ".inherit"
  52. cfnSetListDefault = baseName + ".setdefault"
  53. )
  54. const (
  55. soongConfigAppend = "soong_config_append"
  56. soongConfigAssign = "soong_config_set"
  57. )
  58. var knownFunctions = map[string]interface {
  59. parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr
  60. }{
  61. "abspath": &simpleCallParser{name: baseName + ".abspath", returnType: starlarkTypeString},
  62. "add-product-dex-preopt-module-config": &simpleCallParser{name: baseName + ".add_product_dex_preopt_module_config", returnType: starlarkTypeString, addHandle: true},
  63. "add_soong_config_namespace": &simpleCallParser{name: baseName + ".soong_config_namespace", returnType: starlarkTypeVoid, addGlobals: true},
  64. "add_soong_config_var_value": &simpleCallParser{name: baseName + ".soong_config_set", returnType: starlarkTypeVoid, addGlobals: true},
  65. soongConfigAssign: &simpleCallParser{name: baseName + ".soong_config_set", returnType: starlarkTypeVoid, addGlobals: true},
  66. soongConfigAppend: &simpleCallParser{name: baseName + ".soong_config_append", returnType: starlarkTypeVoid, addGlobals: true},
  67. "soong_config_get": &simpleCallParser{name: baseName + ".soong_config_get", returnType: starlarkTypeString, addGlobals: true},
  68. "add-to-product-copy-files-if-exists": &simpleCallParser{name: baseName + ".copy_if_exists", returnType: starlarkTypeList},
  69. "addprefix": &simpleCallParser{name: baseName + ".addprefix", returnType: starlarkTypeList},
  70. "addsuffix": &simpleCallParser{name: baseName + ".addsuffix", returnType: starlarkTypeList},
  71. "copy-files": &simpleCallParser{name: baseName + ".copy_files", returnType: starlarkTypeList},
  72. "dir": &simpleCallParser{name: baseName + ".dir", returnType: starlarkTypeString},
  73. "dist-for-goals": &simpleCallParser{name: baseName + ".mkdist_for_goals", returnType: starlarkTypeVoid, addGlobals: true},
  74. "enforce-product-packages-exist": &simpleCallParser{name: baseName + ".enforce_product_packages_exist", returnType: starlarkTypeVoid},
  75. "error": &makeControlFuncParser{name: baseName + ".mkerror"},
  76. "findstring": &simpleCallParser{name: baseName + ".findstring", returnType: starlarkTypeInt},
  77. "find-copy-subdir-files": &simpleCallParser{name: baseName + ".find_and_copy", returnType: starlarkTypeList},
  78. "filter": &simpleCallParser{name: baseName + ".filter", returnType: starlarkTypeList},
  79. "filter-out": &simpleCallParser{name: baseName + ".filter_out", returnType: starlarkTypeList},
  80. "firstword": &firstOrLastwordCallParser{isLastWord: false},
  81. "foreach": &foreachCallParser{},
  82. "if": &ifCallParser{},
  83. "info": &makeControlFuncParser{name: baseName + ".mkinfo"},
  84. "is-board-platform": &simpleCallParser{name: baseName + ".board_platform_is", returnType: starlarkTypeBool, addGlobals: true},
  85. "is-board-platform2": &simpleCallParser{name: baseName + ".board_platform_is", returnType: starlarkTypeBool, addGlobals: true},
  86. "is-board-platform-in-list": &simpleCallParser{name: baseName + ".board_platform_in", returnType: starlarkTypeBool, addGlobals: true},
  87. "is-board-platform-in-list2": &simpleCallParser{name: baseName + ".board_platform_in", returnType: starlarkTypeBool, addGlobals: true},
  88. "is-product-in-list": &isProductInListCallParser{},
  89. "is-vendor-board-platform": &isVendorBoardPlatformCallParser{},
  90. "is-vendor-board-qcom": &isVendorBoardQcomCallParser{},
  91. "lastword": &firstOrLastwordCallParser{isLastWord: true},
  92. "notdir": &simpleCallParser{name: baseName + ".notdir", returnType: starlarkTypeString},
  93. "math_max": &mathMaxOrMinCallParser{function: "max"},
  94. "math_min": &mathMaxOrMinCallParser{function: "min"},
  95. "math_gt_or_eq": &mathComparisonCallParser{op: ">="},
  96. "math_gt": &mathComparisonCallParser{op: ">"},
  97. "math_lt": &mathComparisonCallParser{op: "<"},
  98. "my-dir": &myDirCallParser{},
  99. "patsubst": &substCallParser{fname: "patsubst"},
  100. "product-copy-files-by-pattern": &simpleCallParser{name: baseName + ".product_copy_files_by_pattern", returnType: starlarkTypeList},
  101. "require-artifacts-in-path": &simpleCallParser{name: baseName + ".require_artifacts_in_path", returnType: starlarkTypeVoid},
  102. "require-artifacts-in-path-relaxed": &simpleCallParser{name: baseName + ".require_artifacts_in_path_relaxed", returnType: starlarkTypeVoid},
  103. // TODO(asmundak): remove it once all calls are removed from configuration makefiles. see b/183161002
  104. "shell": &shellCallParser{},
  105. "strip": &simpleCallParser{name: baseName + ".mkstrip", returnType: starlarkTypeString},
  106. "subst": &substCallParser{fname: "subst"},
  107. "warning": &makeControlFuncParser{name: baseName + ".mkwarning"},
  108. "word": &wordCallParser{},
  109. "wildcard": &simpleCallParser{name: baseName + ".expand_wildcard", returnType: starlarkTypeList},
  110. }
  111. // The same as knownFunctions, but returns a []starlarkNode instead of a starlarkExpr
  112. var knownNodeFunctions = map[string]interface {
  113. parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) []starlarkNode
  114. }{
  115. "eval": &evalNodeParser{},
  116. "if": &ifCallNodeParser{},
  117. "inherit-product": &inheritProductCallParser{loadAlways: true},
  118. "inherit-product-if-exists": &inheritProductCallParser{loadAlways: false},
  119. "foreach": &foreachCallNodeParser{},
  120. }
  121. // These are functions that we don't implement conversions for, but
  122. // we allow seeing their definitions in the product config files.
  123. var ignoredDefines = map[string]bool{
  124. "find-word-in-list": true, // internal macro
  125. "get-vendor-board-platforms": true, // internal macro, used by is-board-platform, etc.
  126. "is-android-codename": true, // unused by product config
  127. "is-android-codename-in-list": true, // unused by product config
  128. "is-chipset-in-board-platform": true, // unused by product config
  129. "is-chipset-prefix-in-board-platform": true, // unused by product config
  130. "is-not-board-platform": true, // defined but never used
  131. "is-platform-sdk-version-at-least": true, // unused by product config
  132. "match-prefix": true, // internal macro
  133. "match-word": true, // internal macro
  134. "match-word-in-list": true, // internal macro
  135. "tb-modules": true, // defined in hardware/amlogic/tb_modules/tb_detect.mk, unused
  136. }
  137. var identifierFullMatchRegex = regexp.MustCompile("^[a-zA-Z_][a-zA-Z0-9_]*$")
  138. // Conversion request parameters
  139. type Request struct {
  140. MkFile string // file to convert
  141. Reader io.Reader // if set, read input from this stream instead
  142. OutputSuffix string // generated Starlark files suffix
  143. OutputDir string // if set, root of the output hierarchy
  144. ErrorLogger ErrorLogger
  145. TracedVariables []string // trace assignment to these variables
  146. TraceCalls bool
  147. SourceFS fs.FS
  148. MakefileFinder MakefileFinder
  149. }
  150. // ErrorLogger prints errors and gathers error statistics.
  151. // Its NewError function is called on every error encountered during the conversion.
  152. type ErrorLogger interface {
  153. NewError(el ErrorLocation, node mkparser.Node, text string, args ...interface{})
  154. }
  155. type ErrorLocation struct {
  156. MkFile string
  157. MkLine int
  158. }
  159. func (el ErrorLocation) String() string {
  160. return fmt.Sprintf("%s:%d", el.MkFile, el.MkLine)
  161. }
  162. // Derives module name for a given file. It is base name
  163. // (file name without suffix), with some characters replaced to make it a Starlark identifier
  164. func moduleNameForFile(mkFile string) string {
  165. base := strings.TrimSuffix(filepath.Base(mkFile), filepath.Ext(mkFile))
  166. // TODO(asmundak): what else can be in the product file names?
  167. return strings.NewReplacer("-", "_", ".", "_").Replace(base)
  168. }
  169. func cloneMakeString(mkString *mkparser.MakeString) *mkparser.MakeString {
  170. r := &mkparser.MakeString{StringPos: mkString.StringPos}
  171. r.Strings = append(r.Strings, mkString.Strings...)
  172. r.Variables = append(r.Variables, mkString.Variables...)
  173. return r
  174. }
  175. func isMakeControlFunc(s string) bool {
  176. return s == "error" || s == "warning" || s == "info"
  177. }
  178. // varAssignmentScope points to the last assignment for each variable
  179. // in the current block. It is used during the parsing to chain
  180. // the assignments to a variable together.
  181. type varAssignmentScope struct {
  182. outer *varAssignmentScope
  183. vars map[string]bool
  184. }
  185. // Starlark output generation context
  186. type generationContext struct {
  187. buf strings.Builder
  188. starScript *StarlarkScript
  189. indentLevel int
  190. inAssignment bool
  191. tracedCount int
  192. varAssignments *varAssignmentScope
  193. }
  194. func NewGenerateContext(ss *StarlarkScript) *generationContext {
  195. return &generationContext{
  196. starScript: ss,
  197. varAssignments: &varAssignmentScope{
  198. outer: nil,
  199. vars: make(map[string]bool),
  200. },
  201. }
  202. }
  203. func (gctx *generationContext) pushVariableAssignments() {
  204. va := &varAssignmentScope{
  205. outer: gctx.varAssignments,
  206. vars: make(map[string]bool),
  207. }
  208. gctx.varAssignments = va
  209. }
  210. func (gctx *generationContext) popVariableAssignments() {
  211. gctx.varAssignments = gctx.varAssignments.outer
  212. }
  213. func (gctx *generationContext) hasBeenAssigned(v variable) bool {
  214. for va := gctx.varAssignments; va != nil; va = va.outer {
  215. if _, ok := va.vars[v.name()]; ok {
  216. return true
  217. }
  218. }
  219. return false
  220. }
  221. func (gctx *generationContext) setHasBeenAssigned(v variable) {
  222. gctx.varAssignments.vars[v.name()] = true
  223. }
  224. // emit returns generated script
  225. func (gctx *generationContext) emit() string {
  226. ss := gctx.starScript
  227. // The emitted code has the following layout:
  228. // <initial comments>
  229. // preamble, i.e.,
  230. // load statement for the runtime support
  231. // load statement for each unique submodule pulled in by this one
  232. // def init(g, handle):
  233. // cfg = rblf.cfg(handle)
  234. // <statements>
  235. // <warning if conversion was not clean>
  236. iNode := len(ss.nodes)
  237. for i, node := range ss.nodes {
  238. if _, ok := node.(*commentNode); !ok {
  239. iNode = i
  240. break
  241. }
  242. node.emit(gctx)
  243. }
  244. gctx.emitPreamble()
  245. gctx.newLine()
  246. // The arguments passed to the init function are the global dictionary
  247. // ('g') and the product configuration dictionary ('cfg')
  248. gctx.write("def init(g, handle):")
  249. gctx.indentLevel++
  250. if gctx.starScript.traceCalls {
  251. gctx.newLine()
  252. gctx.writef(`print(">%s")`, gctx.starScript.mkFile)
  253. }
  254. gctx.newLine()
  255. gctx.writef("cfg = %s(handle)", cfnGetCfg)
  256. for _, node := range ss.nodes[iNode:] {
  257. node.emit(gctx)
  258. }
  259. if gctx.starScript.traceCalls {
  260. gctx.newLine()
  261. gctx.writef(`print("<%s")`, gctx.starScript.mkFile)
  262. }
  263. gctx.indentLevel--
  264. gctx.write("\n")
  265. return gctx.buf.String()
  266. }
  267. func (gctx *generationContext) emitPreamble() {
  268. gctx.newLine()
  269. gctx.writef("load(%q, %q)", baseUri, baseName)
  270. // Emit exactly one load statement for each URI.
  271. loadedSubConfigs := make(map[string]string)
  272. for _, mi := range gctx.starScript.inherited {
  273. uri := mi.path
  274. if m, ok := loadedSubConfigs[uri]; ok {
  275. // No need to emit load statement, but fix module name.
  276. mi.moduleLocalName = m
  277. continue
  278. }
  279. if mi.optional || mi.missing {
  280. uri += "|init"
  281. }
  282. gctx.newLine()
  283. gctx.writef("load(%q, %s = \"init\")", uri, mi.entryName())
  284. loadedSubConfigs[uri] = mi.moduleLocalName
  285. }
  286. gctx.write("\n")
  287. }
  288. func (gctx *generationContext) emitPass() {
  289. gctx.newLine()
  290. gctx.write("pass")
  291. }
  292. func (gctx *generationContext) write(ss ...string) {
  293. for _, s := range ss {
  294. gctx.buf.WriteString(s)
  295. }
  296. }
  297. func (gctx *generationContext) writef(format string, args ...interface{}) {
  298. gctx.write(fmt.Sprintf(format, args...))
  299. }
  300. func (gctx *generationContext) newLine() {
  301. if gctx.buf.Len() == 0 {
  302. return
  303. }
  304. gctx.write("\n")
  305. gctx.writef("%*s", 2*gctx.indentLevel, "")
  306. }
  307. func (gctx *generationContext) emitConversionError(el ErrorLocation, message string) {
  308. gctx.writef(`rblf.mk2rbc_error("%s", %q)`, el, message)
  309. }
  310. func (gctx *generationContext) emitLoadCheck(im inheritedModule) {
  311. if !im.needsLoadCheck() {
  312. return
  313. }
  314. gctx.newLine()
  315. gctx.writef("if not %s:", im.entryName())
  316. gctx.indentLevel++
  317. gctx.newLine()
  318. gctx.write(`rblf.mkerror("`, gctx.starScript.mkFile, `", "Cannot find %s" % (`)
  319. im.pathExpr().emit(gctx)
  320. gctx.write("))")
  321. gctx.indentLevel--
  322. }
  323. type knownVariable struct {
  324. name string
  325. class varClass
  326. valueType starlarkType
  327. }
  328. type knownVariables map[string]knownVariable
  329. func (pcv knownVariables) NewVariable(name string, varClass varClass, valueType starlarkType) {
  330. v, exists := pcv[name]
  331. if !exists {
  332. pcv[name] = knownVariable{name, varClass, valueType}
  333. return
  334. }
  335. // Conflict resolution:
  336. // * config class trumps everything
  337. // * any type trumps unknown type
  338. match := varClass == v.class
  339. if !match {
  340. if varClass == VarClassConfig {
  341. v.class = VarClassConfig
  342. match = true
  343. } else if v.class == VarClassConfig {
  344. match = true
  345. }
  346. }
  347. if valueType != v.valueType {
  348. if valueType != starlarkTypeUnknown {
  349. if v.valueType == starlarkTypeUnknown {
  350. v.valueType = valueType
  351. } else {
  352. match = false
  353. }
  354. }
  355. }
  356. if !match {
  357. fmt.Fprintf(os.Stderr, "cannot redefine %s as %v/%v (already defined as %v/%v)\n",
  358. name, varClass, valueType, v.class, v.valueType)
  359. }
  360. }
  361. // All known product variables.
  362. var KnownVariables = make(knownVariables)
  363. func init() {
  364. for _, kv := range []string{
  365. // Kernel-related variables that we know are lists.
  366. "BOARD_VENDOR_KERNEL_MODULES",
  367. "BOARD_VENDOR_RAMDISK_KERNEL_MODULES",
  368. "BOARD_VENDOR_RAMDISK_KERNEL_MODULES_LOAD",
  369. "BOARD_RECOVERY_KERNEL_MODULES",
  370. // Other variables we knwo are lists
  371. "ART_APEX_JARS",
  372. } {
  373. KnownVariables.NewVariable(kv, VarClassSoong, starlarkTypeList)
  374. }
  375. }
  376. // Information about the generated Starlark script.
  377. type StarlarkScript struct {
  378. mkFile string
  379. moduleName string
  380. mkPos scanner.Position
  381. nodes []starlarkNode
  382. inherited []*moduleInfo
  383. hasErrors bool
  384. traceCalls bool // print enter/exit each init function
  385. sourceFS fs.FS
  386. makefileFinder MakefileFinder
  387. nodeLocator func(pos mkparser.Pos) int
  388. }
  389. // parseContext holds the script we are generating and all the ephemeral data
  390. // needed during the parsing.
  391. type parseContext struct {
  392. script *StarlarkScript
  393. nodes []mkparser.Node // Makefile as parsed by mkparser
  394. currentNodeIndex int // Node in it we are processing
  395. ifNestLevel int
  396. moduleNameCount map[string]int // count of imported modules with given basename
  397. fatalError error
  398. outputSuffix string
  399. errorLogger ErrorLogger
  400. tracedVariables map[string]bool // variables to be traced in the generated script
  401. variables map[string]variable
  402. outputDir string
  403. dependentModules map[string]*moduleInfo
  404. soongNamespaces map[string]map[string]bool
  405. includeTops []string
  406. typeHints map[string]starlarkType
  407. atTopOfMakefile bool
  408. }
  409. func newParseContext(ss *StarlarkScript, nodes []mkparser.Node) *parseContext {
  410. predefined := []struct{ name, value string }{
  411. {"SRC_TARGET_DIR", filepath.Join("build", "make", "target")},
  412. {"LOCAL_PATH", filepath.Dir(ss.mkFile)},
  413. {"TOPDIR", ""}, // TOPDIR is just set to an empty string in cleanbuild.mk and core.mk
  414. // TODO(asmundak): maybe read it from build/make/core/envsetup.mk?
  415. {"TARGET_COPY_OUT_SYSTEM", "system"},
  416. {"TARGET_COPY_OUT_SYSTEM_OTHER", "system_other"},
  417. {"TARGET_COPY_OUT_DATA", "data"},
  418. {"TARGET_COPY_OUT_ASAN", filepath.Join("data", "asan")},
  419. {"TARGET_COPY_OUT_OEM", "oem"},
  420. {"TARGET_COPY_OUT_RAMDISK", "ramdisk"},
  421. {"TARGET_COPY_OUT_DEBUG_RAMDISK", "debug_ramdisk"},
  422. {"TARGET_COPY_OUT_VENDOR_DEBUG_RAMDISK", "vendor_debug_ramdisk"},
  423. {"TARGET_COPY_OUT_TEST_HARNESS_RAMDISK", "test_harness_ramdisk"},
  424. {"TARGET_COPY_OUT_ROOT", "root"},
  425. {"TARGET_COPY_OUT_RECOVERY", "recovery"},
  426. {"TARGET_COPY_OUT_VENDOR_RAMDISK", "vendor_ramdisk"},
  427. // TODO(asmundak): to process internal config files, we need the following variables:
  428. // TARGET_VENDOR
  429. // target_base_product
  430. //
  431. // the following utility variables are set in build/make/common/core.mk:
  432. {"empty", ""},
  433. {"space", " "},
  434. {"comma", ","},
  435. {"newline", "\n"},
  436. {"pound", "#"},
  437. {"backslash", "\\"},
  438. }
  439. ctx := &parseContext{
  440. script: ss,
  441. nodes: nodes,
  442. currentNodeIndex: 0,
  443. ifNestLevel: 0,
  444. moduleNameCount: make(map[string]int),
  445. variables: make(map[string]variable),
  446. dependentModules: make(map[string]*moduleInfo),
  447. soongNamespaces: make(map[string]map[string]bool),
  448. includeTops: []string{},
  449. typeHints: make(map[string]starlarkType),
  450. atTopOfMakefile: true,
  451. }
  452. for _, item := range predefined {
  453. ctx.variables[item.name] = &predefinedVariable{
  454. baseVariable: baseVariable{nam: item.name, typ: starlarkTypeString},
  455. value: &stringLiteralExpr{item.value},
  456. }
  457. }
  458. return ctx
  459. }
  460. func (ctx *parseContext) hasNodes() bool {
  461. return ctx.currentNodeIndex < len(ctx.nodes)
  462. }
  463. func (ctx *parseContext) getNode() mkparser.Node {
  464. if !ctx.hasNodes() {
  465. return nil
  466. }
  467. node := ctx.nodes[ctx.currentNodeIndex]
  468. ctx.currentNodeIndex++
  469. return node
  470. }
  471. func (ctx *parseContext) backNode() {
  472. if ctx.currentNodeIndex <= 0 {
  473. panic("Cannot back off")
  474. }
  475. ctx.currentNodeIndex--
  476. }
  477. func (ctx *parseContext) handleAssignment(a *mkparser.Assignment) []starlarkNode {
  478. // Handle only simple variables
  479. if !a.Name.Const() {
  480. return []starlarkNode{ctx.newBadNode(a, "Only simple variables are handled")}
  481. }
  482. name := a.Name.Strings[0]
  483. // The `override` directive
  484. // override FOO :=
  485. // is parsed as an assignment to a variable named `override FOO`.
  486. // There are very few places where `override` is used, just flag it.
  487. if strings.HasPrefix(name, "override ") {
  488. return []starlarkNode{ctx.newBadNode(a, "cannot handle override directive")}
  489. }
  490. // Soong configuration
  491. if strings.HasPrefix(name, soongNsPrefix) {
  492. return ctx.handleSoongNsAssignment(strings.TrimPrefix(name, soongNsPrefix), a)
  493. }
  494. lhs := ctx.addVariable(name)
  495. if lhs == nil {
  496. return []starlarkNode{ctx.newBadNode(a, "unknown variable %s", name)}
  497. }
  498. _, isTraced := ctx.tracedVariables[lhs.name()]
  499. asgn := &assignmentNode{lhs: lhs, mkValue: a.Value, isTraced: isTraced, location: ctx.errorLocation(a)}
  500. if lhs.valueType() == starlarkTypeUnknown {
  501. // Try to divine variable type from the RHS
  502. asgn.value = ctx.parseMakeString(a, a.Value)
  503. if xBad, ok := asgn.value.(*badExpr); ok {
  504. return []starlarkNode{&exprNode{xBad}}
  505. }
  506. inferred_type := asgn.value.typ()
  507. if inferred_type != starlarkTypeUnknown {
  508. lhs.setValueType(inferred_type)
  509. }
  510. }
  511. if lhs.valueType() == starlarkTypeList {
  512. xConcat, xBad := ctx.buildConcatExpr(a)
  513. if xBad != nil {
  514. return []starlarkNode{&exprNode{expr: xBad}}
  515. }
  516. switch len(xConcat.items) {
  517. case 0:
  518. asgn.value = &listExpr{}
  519. case 1:
  520. asgn.value = xConcat.items[0]
  521. default:
  522. asgn.value = xConcat
  523. }
  524. } else {
  525. asgn.value = ctx.parseMakeString(a, a.Value)
  526. if xBad, ok := asgn.value.(*badExpr); ok {
  527. return []starlarkNode{&exprNode{expr: xBad}}
  528. }
  529. }
  530. if asgn.lhs.valueType() == starlarkTypeString &&
  531. asgn.value.typ() != starlarkTypeUnknown &&
  532. asgn.value.typ() != starlarkTypeString {
  533. asgn.value = &toStringExpr{expr: asgn.value}
  534. }
  535. switch a.Type {
  536. case "=", ":=":
  537. asgn.flavor = asgnSet
  538. case "+=":
  539. asgn.flavor = asgnAppend
  540. case "?=":
  541. asgn.flavor = asgnMaybeSet
  542. default:
  543. panic(fmt.Errorf("unexpected assignment type %s", a.Type))
  544. }
  545. return []starlarkNode{asgn}
  546. }
  547. func (ctx *parseContext) handleSoongNsAssignment(name string, asgn *mkparser.Assignment) []starlarkNode {
  548. val := ctx.parseMakeString(asgn, asgn.Value)
  549. if xBad, ok := val.(*badExpr); ok {
  550. return []starlarkNode{&exprNode{expr: xBad}}
  551. }
  552. // Unfortunately, Soong namespaces can be set up by directly setting corresponding Make
  553. // variables instead of via add_soong_config_namespace + add_soong_config_var_value.
  554. // Try to divine the call from the assignment as follows:
  555. if name == "NAMESPACES" {
  556. // Upon seeng
  557. // SOONG_CONFIG_NAMESPACES += foo
  558. // remember that there is a namespace `foo` and act as we saw
  559. // $(call add_soong_config_namespace,foo)
  560. s, ok := maybeString(val)
  561. if !ok {
  562. return []starlarkNode{ctx.newBadNode(asgn, "cannot handle variables in SOONG_CONFIG_NAMESPACES assignment, please use add_soong_config_namespace instead")}
  563. }
  564. result := make([]starlarkNode, 0)
  565. for _, ns := range strings.Fields(s) {
  566. ctx.addSoongNamespace(ns)
  567. result = append(result, &exprNode{&callExpr{
  568. name: baseName + ".soong_config_namespace",
  569. args: []starlarkExpr{&globalsExpr{}, &stringLiteralExpr{ns}},
  570. returnType: starlarkTypeVoid,
  571. }})
  572. }
  573. return result
  574. } else {
  575. // Upon seeing
  576. // SOONG_CONFIG_x_y = v
  577. // find a namespace called `x` and act as if we encountered
  578. // $(call soong_config_set,x,y,v)
  579. // or check that `x_y` is a namespace, and then add the RHS of this assignment as variables in
  580. // it.
  581. // Emit an error in the ambiguous situation (namespaces `foo_bar` with a variable `baz`
  582. // and `foo` with a variable `bar_baz`.
  583. namespaceName := ""
  584. if ctx.hasSoongNamespace(name) {
  585. namespaceName = name
  586. }
  587. var varName string
  588. for pos, ch := range name {
  589. if !(ch == '_' && ctx.hasSoongNamespace(name[0:pos])) {
  590. continue
  591. }
  592. if namespaceName != "" {
  593. return []starlarkNode{ctx.newBadNode(asgn, "ambiguous soong namespace (may be either `%s` or `%s`)", namespaceName, name[0:pos])}
  594. }
  595. namespaceName = name[0:pos]
  596. varName = name[pos+1:]
  597. }
  598. if namespaceName == "" {
  599. return []starlarkNode{ctx.newBadNode(asgn, "cannot figure out Soong namespace, please use add_soong_config_var_value macro instead")}
  600. }
  601. if varName == "" {
  602. // Remember variables in this namespace
  603. s, ok := maybeString(val)
  604. if !ok {
  605. return []starlarkNode{ctx.newBadNode(asgn, "cannot handle variables in SOONG_CONFIG_ assignment, please use add_soong_config_var_value instead")}
  606. }
  607. ctx.updateSoongNamespace(asgn.Type != "+=", namespaceName, strings.Fields(s))
  608. return []starlarkNode{}
  609. }
  610. // Finally, handle assignment to a namespace variable
  611. if !ctx.hasNamespaceVar(namespaceName, varName) {
  612. return []starlarkNode{ctx.newBadNode(asgn, "no %s variable in %s namespace, please use add_soong_config_var_value instead", varName, namespaceName)}
  613. }
  614. fname := baseName + "." + soongConfigAssign
  615. if asgn.Type == "+=" {
  616. fname = baseName + "." + soongConfigAppend
  617. }
  618. return []starlarkNode{&exprNode{&callExpr{
  619. name: fname,
  620. args: []starlarkExpr{&globalsExpr{}, &stringLiteralExpr{namespaceName}, &stringLiteralExpr{varName}, val},
  621. returnType: starlarkTypeVoid,
  622. }}}
  623. }
  624. }
  625. func (ctx *parseContext) buildConcatExpr(a *mkparser.Assignment) (*concatExpr, *badExpr) {
  626. xConcat := &concatExpr{}
  627. var xItemList *listExpr
  628. addToItemList := func(x ...starlarkExpr) {
  629. if xItemList == nil {
  630. xItemList = &listExpr{[]starlarkExpr{}}
  631. }
  632. xItemList.items = append(xItemList.items, x...)
  633. }
  634. finishItemList := func() {
  635. if xItemList != nil {
  636. xConcat.items = append(xConcat.items, xItemList)
  637. xItemList = nil
  638. }
  639. }
  640. items := a.Value.Words()
  641. for _, item := range items {
  642. // A function call in RHS is supposed to return a list, all other item
  643. // expressions return individual elements.
  644. switch x := ctx.parseMakeString(a, item).(type) {
  645. case *badExpr:
  646. return nil, x
  647. case *stringLiteralExpr:
  648. addToItemList(maybeConvertToStringList(x).(*listExpr).items...)
  649. default:
  650. switch x.typ() {
  651. case starlarkTypeList:
  652. finishItemList()
  653. xConcat.items = append(xConcat.items, x)
  654. case starlarkTypeString:
  655. finishItemList()
  656. xConcat.items = append(xConcat.items, &callExpr{
  657. object: x,
  658. name: "split",
  659. args: nil,
  660. returnType: starlarkTypeList,
  661. })
  662. default:
  663. addToItemList(x)
  664. }
  665. }
  666. }
  667. if xItemList != nil {
  668. xConcat.items = append(xConcat.items, xItemList)
  669. }
  670. return xConcat, nil
  671. }
  672. func (ctx *parseContext) newDependentModule(path string, optional bool) *moduleInfo {
  673. modulePath := ctx.loadedModulePath(path)
  674. if mi, ok := ctx.dependentModules[modulePath]; ok {
  675. mi.optional = mi.optional && optional
  676. return mi
  677. }
  678. moduleName := moduleNameForFile(path)
  679. moduleLocalName := "_" + moduleName
  680. n, found := ctx.moduleNameCount[moduleName]
  681. if found {
  682. moduleLocalName += fmt.Sprintf("%d", n)
  683. }
  684. ctx.moduleNameCount[moduleName] = n + 1
  685. _, err := fs.Stat(ctx.script.sourceFS, path)
  686. mi := &moduleInfo{
  687. path: modulePath,
  688. originalPath: path,
  689. moduleLocalName: moduleLocalName,
  690. optional: optional,
  691. missing: err != nil,
  692. }
  693. ctx.dependentModules[modulePath] = mi
  694. ctx.script.inherited = append(ctx.script.inherited, mi)
  695. return mi
  696. }
  697. func (ctx *parseContext) handleSubConfig(
  698. v mkparser.Node, pathExpr starlarkExpr, loadAlways bool, processModule func(inheritedModule) starlarkNode) []starlarkNode {
  699. // In a simple case, the name of a module to inherit/include is known statically.
  700. if path, ok := maybeString(pathExpr); ok {
  701. // Note that even if this directive loads a module unconditionally, a module may be
  702. // absent without causing any harm if this directive is inside an if/else block.
  703. moduleShouldExist := loadAlways && ctx.ifNestLevel == 0
  704. if strings.Contains(path, "*") {
  705. if paths, err := fs.Glob(ctx.script.sourceFS, path); err == nil {
  706. result := make([]starlarkNode, 0)
  707. for _, p := range paths {
  708. mi := ctx.newDependentModule(p, !moduleShouldExist)
  709. result = append(result, processModule(inheritedStaticModule{mi, loadAlways}))
  710. }
  711. return result
  712. } else {
  713. return []starlarkNode{ctx.newBadNode(v, "cannot glob wildcard argument")}
  714. }
  715. } else {
  716. mi := ctx.newDependentModule(path, !moduleShouldExist)
  717. return []starlarkNode{processModule(inheritedStaticModule{mi, loadAlways})}
  718. }
  719. }
  720. // If module path references variables (e.g., $(v1)/foo/$(v2)/device-config.mk), find all the paths in the
  721. // source tree that may be a match and the corresponding variable values. For instance, if the source tree
  722. // contains vendor1/foo/abc/dev.mk and vendor2/foo/def/dev.mk, the first one will be inherited when
  723. // (v1, v2) == ('vendor1', 'abc'), and the second one when (v1, v2) == ('vendor2', 'def').
  724. // We then emit the code that loads all of them, e.g.:
  725. // load("//vendor1/foo/abc:dev.rbc", _dev1_init="init")
  726. // load("//vendor2/foo/def/dev.rbc", _dev2_init="init")
  727. // And then inherit it as follows:
  728. // _e = {
  729. // "vendor1/foo/abc/dev.mk": ("vendor1/foo/abc/dev", _dev1_init),
  730. // "vendor2/foo/def/dev.mk": ("vendor2/foo/def/dev", _dev_init2) }.get("%s/foo/%s/dev.mk" % (v1, v2))
  731. // if _e:
  732. // rblf.inherit(handle, _e[0], _e[1])
  733. //
  734. var matchingPaths []string
  735. varPath, ok := pathExpr.(*interpolateExpr)
  736. if !ok {
  737. return []starlarkNode{ctx.newBadNode(v, "inherit-product/include argument is too complex")}
  738. }
  739. pathPattern := []string{varPath.chunks[0]}
  740. for _, chunk := range varPath.chunks[1:] {
  741. if chunk != "" {
  742. pathPattern = append(pathPattern, chunk)
  743. }
  744. }
  745. if pathPattern[0] == "" && len(ctx.includeTops) > 0 {
  746. // If pattern starts from the top. restrict it to the directories where
  747. // we know inherit-product uses dynamically calculated path.
  748. for _, p := range ctx.includeTops {
  749. pathPattern[0] = p
  750. matchingPaths = append(matchingPaths, ctx.findMatchingPaths(pathPattern)...)
  751. }
  752. } else {
  753. matchingPaths = ctx.findMatchingPaths(pathPattern)
  754. }
  755. // Safeguard against $(call inherit-product,$(PRODUCT_PATH))
  756. const maxMatchingFiles = 150
  757. if len(matchingPaths) > maxMatchingFiles {
  758. return []starlarkNode{ctx.newBadNode(v, "there are >%d files matching the pattern, please rewrite it", maxMatchingFiles)}
  759. }
  760. needsWarning := pathPattern[0] == "" && len(ctx.includeTops) == 0
  761. res := inheritedDynamicModule{*varPath, []*moduleInfo{}, loadAlways, ctx.errorLocation(v), needsWarning}
  762. for _, p := range matchingPaths {
  763. // A product configuration files discovered dynamically may attempt to inherit
  764. // from another one which does not exist in this source tree. Prevent load errors
  765. // by always loading the dynamic files as optional.
  766. res.candidateModules = append(res.candidateModules, ctx.newDependentModule(p, true))
  767. }
  768. return []starlarkNode{processModule(res)}
  769. }
  770. func (ctx *parseContext) findMatchingPaths(pattern []string) []string {
  771. files := ctx.script.makefileFinder.Find(".")
  772. if len(pattern) == 0 {
  773. return files
  774. }
  775. // Create regular expression from the pattern
  776. s_regexp := "^" + regexp.QuoteMeta(pattern[0])
  777. for _, s := range pattern[1:] {
  778. s_regexp += ".*" + regexp.QuoteMeta(s)
  779. }
  780. s_regexp += "$"
  781. rex := regexp.MustCompile(s_regexp)
  782. // Now match
  783. var res []string
  784. for _, p := range files {
  785. if rex.MatchString(p) {
  786. res = append(res, p)
  787. }
  788. }
  789. return res
  790. }
  791. type inheritProductCallParser struct {
  792. loadAlways bool
  793. }
  794. func (p *inheritProductCallParser) parse(ctx *parseContext, v mkparser.Node, args *mkparser.MakeString) []starlarkNode {
  795. args.TrimLeftSpaces()
  796. args.TrimRightSpaces()
  797. pathExpr := ctx.parseMakeString(v, args)
  798. if _, ok := pathExpr.(*badExpr); ok {
  799. return []starlarkNode{ctx.newBadNode(v, "Unable to parse argument to inherit")}
  800. }
  801. return ctx.handleSubConfig(v, pathExpr, p.loadAlways, func(im inheritedModule) starlarkNode {
  802. return &inheritNode{im, p.loadAlways}
  803. })
  804. }
  805. func (ctx *parseContext) handleInclude(v mkparser.Node, pathExpr starlarkExpr, loadAlways bool) []starlarkNode {
  806. return ctx.handleSubConfig(v, pathExpr, loadAlways, func(im inheritedModule) starlarkNode {
  807. return &includeNode{im, loadAlways}
  808. })
  809. }
  810. func (ctx *parseContext) handleVariable(v *mkparser.Variable) []starlarkNode {
  811. // Handle:
  812. // $(call inherit-product,...)
  813. // $(call inherit-product-if-exists,...)
  814. // $(info xxx)
  815. // $(warning xxx)
  816. // $(error xxx)
  817. // $(call other-custom-functions,...)
  818. if name, args, ok := ctx.maybeParseFunctionCall(v, v.Name); ok {
  819. if kf, ok := knownNodeFunctions[name]; ok {
  820. return kf.parse(ctx, v, args)
  821. }
  822. }
  823. return []starlarkNode{&exprNode{expr: ctx.parseReference(v, v.Name)}}
  824. }
  825. func (ctx *parseContext) maybeHandleDefine(directive *mkparser.Directive) starlarkNode {
  826. macro_name := strings.Fields(directive.Args.Strings[0])[0]
  827. // Ignore the macros that we handle
  828. _, ignored := ignoredDefines[macro_name]
  829. _, known := knownFunctions[macro_name]
  830. if !ignored && !known {
  831. return ctx.newBadNode(directive, "define is not supported: %s", macro_name)
  832. }
  833. return nil
  834. }
  835. func (ctx *parseContext) handleIfBlock(ifDirective *mkparser.Directive) starlarkNode {
  836. ssSwitch := &switchNode{
  837. ssCases: []*switchCase{ctx.processBranch(ifDirective)},
  838. }
  839. for ctx.hasNodes() && ctx.fatalError == nil {
  840. node := ctx.getNode()
  841. switch x := node.(type) {
  842. case *mkparser.Directive:
  843. switch x.Name {
  844. case "else", "elifdef", "elifndef", "elifeq", "elifneq":
  845. ssSwitch.ssCases = append(ssSwitch.ssCases, ctx.processBranch(x))
  846. case "endif":
  847. return ssSwitch
  848. default:
  849. return ctx.newBadNode(node, "unexpected directive %s", x.Name)
  850. }
  851. default:
  852. return ctx.newBadNode(ifDirective, "unexpected statement")
  853. }
  854. }
  855. if ctx.fatalError == nil {
  856. ctx.fatalError = fmt.Errorf("no matching endif for %s", ifDirective.Dump())
  857. }
  858. return ctx.newBadNode(ifDirective, "no matching endif for %s", ifDirective.Dump())
  859. }
  860. // processBranch processes a single branch (if/elseif/else) until the next directive
  861. // on the same level.
  862. func (ctx *parseContext) processBranch(check *mkparser.Directive) *switchCase {
  863. block := &switchCase{gate: ctx.parseCondition(check)}
  864. defer func() {
  865. ctx.ifNestLevel--
  866. }()
  867. ctx.ifNestLevel++
  868. for ctx.hasNodes() {
  869. node := ctx.getNode()
  870. if d, ok := node.(*mkparser.Directive); ok {
  871. switch d.Name {
  872. case "else", "elifdef", "elifndef", "elifeq", "elifneq", "endif":
  873. ctx.backNode()
  874. return block
  875. }
  876. }
  877. block.nodes = append(block.nodes, ctx.handleSimpleStatement(node)...)
  878. }
  879. ctx.fatalError = fmt.Errorf("no matching endif for %s", check.Dump())
  880. return block
  881. }
  882. func (ctx *parseContext) parseCondition(check *mkparser.Directive) starlarkNode {
  883. switch check.Name {
  884. case "ifdef", "ifndef", "elifdef", "elifndef":
  885. if !check.Args.Const() {
  886. return ctx.newBadNode(check, "ifdef variable ref too complex: %s", check.Args.Dump())
  887. }
  888. v := NewVariableRefExpr(ctx.addVariable(check.Args.Strings[0]))
  889. if strings.HasSuffix(check.Name, "ndef") {
  890. v = &notExpr{v}
  891. }
  892. return &ifNode{
  893. isElif: strings.HasPrefix(check.Name, "elif"),
  894. expr: v,
  895. }
  896. case "ifeq", "ifneq", "elifeq", "elifneq":
  897. return &ifNode{
  898. isElif: strings.HasPrefix(check.Name, "elif"),
  899. expr: ctx.parseCompare(check),
  900. }
  901. case "else":
  902. return &elseNode{}
  903. default:
  904. panic(fmt.Errorf("%s: unknown directive: %s", ctx.script.mkFile, check.Dump()))
  905. }
  906. }
  907. func (ctx *parseContext) newBadExpr(node mkparser.Node, text string, args ...interface{}) starlarkExpr {
  908. if ctx.errorLogger != nil {
  909. ctx.errorLogger.NewError(ctx.errorLocation(node), node, text, args...)
  910. }
  911. ctx.script.hasErrors = true
  912. return &badExpr{errorLocation: ctx.errorLocation(node), message: fmt.Sprintf(text, args...)}
  913. }
  914. // records that the given node failed to be converted and includes an explanatory message
  915. func (ctx *parseContext) newBadNode(failedNode mkparser.Node, message string, args ...interface{}) starlarkNode {
  916. return &exprNode{ctx.newBadExpr(failedNode, message, args...)}
  917. }
  918. func (ctx *parseContext) parseCompare(cond *mkparser.Directive) starlarkExpr {
  919. // Strip outer parentheses
  920. mkArg := cloneMakeString(cond.Args)
  921. mkArg.Strings[0] = strings.TrimLeft(mkArg.Strings[0], "( ")
  922. n := len(mkArg.Strings)
  923. mkArg.Strings[n-1] = strings.TrimRight(mkArg.Strings[n-1], ") ")
  924. args := mkArg.Split(",")
  925. // TODO(asmundak): handle the case where the arguments are in quotes and space-separated
  926. if len(args) != 2 {
  927. return ctx.newBadExpr(cond, "ifeq/ifneq len(args) != 2 %s", cond.Dump())
  928. }
  929. args[0].TrimRightSpaces()
  930. args[1].TrimLeftSpaces()
  931. isEq := !strings.HasSuffix(cond.Name, "neq")
  932. xLeft := ctx.parseMakeString(cond, args[0])
  933. xRight := ctx.parseMakeString(cond, args[1])
  934. if bad, ok := xLeft.(*badExpr); ok {
  935. return bad
  936. }
  937. if bad, ok := xRight.(*badExpr); ok {
  938. return bad
  939. }
  940. if expr, ok := ctx.parseCompareSpecialCases(cond, xLeft, xRight); ok {
  941. return expr
  942. }
  943. var stringOperand string
  944. var otherOperand starlarkExpr
  945. if s, ok := maybeString(xLeft); ok {
  946. stringOperand = s
  947. otherOperand = xRight
  948. } else if s, ok := maybeString(xRight); ok {
  949. stringOperand = s
  950. otherOperand = xLeft
  951. }
  952. // If we've identified one of the operands as being a string literal, check
  953. // for some special cases we can do to simplify the resulting expression.
  954. if otherOperand != nil {
  955. if stringOperand == "" {
  956. if isEq {
  957. return negateExpr(otherOperand)
  958. } else {
  959. return otherOperand
  960. }
  961. }
  962. if stringOperand == "true" && otherOperand.typ() == starlarkTypeBool {
  963. if !isEq {
  964. return negateExpr(otherOperand)
  965. } else {
  966. return otherOperand
  967. }
  968. }
  969. if intOperand, err := strconv.Atoi(strings.TrimSpace(stringOperand)); err == nil && otherOperand.typ() == starlarkTypeInt {
  970. return &eqExpr{
  971. left: otherOperand,
  972. right: &intLiteralExpr{literal: intOperand},
  973. isEq: isEq,
  974. }
  975. }
  976. }
  977. return &eqExpr{left: xLeft, right: xRight, isEq: isEq}
  978. }
  979. // Given an if statement's directive and the left/right starlarkExprs,
  980. // check if the starlarkExprs are one of a few hardcoded special cases
  981. // that can be converted to a simpler equality expression than simply comparing
  982. // the two.
  983. func (ctx *parseContext) parseCompareSpecialCases(directive *mkparser.Directive, left starlarkExpr,
  984. right starlarkExpr) (starlarkExpr, bool) {
  985. isEq := !strings.HasSuffix(directive.Name, "neq")
  986. // All the special cases require a call on one side and a
  987. // string literal/variable on the other. Turn the left/right variables into
  988. // call/value variables, and return false if that's not possible.
  989. var value starlarkExpr = nil
  990. call, ok := left.(*callExpr)
  991. if ok {
  992. switch right.(type) {
  993. case *stringLiteralExpr, *variableRefExpr:
  994. value = right
  995. }
  996. } else {
  997. call, _ = right.(*callExpr)
  998. switch left.(type) {
  999. case *stringLiteralExpr, *variableRefExpr:
  1000. value = left
  1001. }
  1002. }
  1003. if call == nil || value == nil {
  1004. return nil, false
  1005. }
  1006. switch call.name {
  1007. case baseName + ".filter":
  1008. return ctx.parseCompareFilterFuncResult(directive, call, value, isEq)
  1009. case baseName + ".expand_wildcard":
  1010. return ctx.parseCompareWildcardFuncResult(directive, call, value, !isEq), true
  1011. case baseName + ".findstring":
  1012. return ctx.parseCheckFindstringFuncResult(directive, call, value, !isEq), true
  1013. case baseName + ".strip":
  1014. return ctx.parseCompareStripFuncResult(directive, call, value, !isEq), true
  1015. }
  1016. return nil, false
  1017. }
  1018. func (ctx *parseContext) parseCompareFilterFuncResult(cond *mkparser.Directive,
  1019. filterFuncCall *callExpr, xValue starlarkExpr, negate bool) (starlarkExpr, bool) {
  1020. // We handle:
  1021. // * ifeq/ifneq (,$(filter v1 v2 ..., EXPR) becomes if EXPR not in/in ["v1", "v2", ...]
  1022. // * ifeq/ifneq (,$(filter EXPR, v1 v2 ...) becomes if EXPR not in/in ["v1", "v2", ...]
  1023. if x, ok := xValue.(*stringLiteralExpr); !ok || x.literal != "" {
  1024. return nil, false
  1025. }
  1026. xPattern := filterFuncCall.args[0]
  1027. xText := filterFuncCall.args[1]
  1028. var xInList *stringLiteralExpr
  1029. var expr starlarkExpr
  1030. var ok bool
  1031. if xInList, ok = xPattern.(*stringLiteralExpr); ok && !strings.ContainsRune(xInList.literal, '%') && xText.typ() == starlarkTypeList {
  1032. expr = xText
  1033. } else if xInList, ok = xText.(*stringLiteralExpr); ok {
  1034. expr = xPattern
  1035. } else {
  1036. return nil, false
  1037. }
  1038. slExpr := newStringListExpr(strings.Fields(xInList.literal))
  1039. // Generate simpler code for the common cases:
  1040. if expr.typ() == starlarkTypeList {
  1041. if len(slExpr.items) == 1 {
  1042. // Checking that a string belongs to list
  1043. return &inExpr{isNot: negate, list: expr, expr: slExpr.items[0]}, true
  1044. } else {
  1045. return nil, false
  1046. }
  1047. } else if len(slExpr.items) == 1 {
  1048. return &eqExpr{left: expr, right: slExpr.items[0], isEq: !negate}, true
  1049. } else {
  1050. return &inExpr{isNot: negate, list: newStringListExpr(strings.Fields(xInList.literal)), expr: expr}, true
  1051. }
  1052. }
  1053. func (ctx *parseContext) parseCompareWildcardFuncResult(directive *mkparser.Directive,
  1054. xCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr {
  1055. if !isEmptyString(xValue) {
  1056. return ctx.newBadExpr(directive, "wildcard result can be compared only to empty: %s", xValue)
  1057. }
  1058. callFunc := baseName + ".file_wildcard_exists"
  1059. if s, ok := xCall.args[0].(*stringLiteralExpr); ok && !strings.ContainsAny(s.literal, "*?{[") {
  1060. callFunc = baseName + ".file_exists"
  1061. }
  1062. var cc starlarkExpr = &callExpr{name: callFunc, args: xCall.args, returnType: starlarkTypeBool}
  1063. if !negate {
  1064. cc = &notExpr{cc}
  1065. }
  1066. return cc
  1067. }
  1068. func (ctx *parseContext) parseCheckFindstringFuncResult(directive *mkparser.Directive,
  1069. xCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr {
  1070. if isEmptyString(xValue) {
  1071. return &eqExpr{
  1072. left: &callExpr{
  1073. object: xCall.args[1],
  1074. name: "find",
  1075. args: []starlarkExpr{xCall.args[0]},
  1076. returnType: starlarkTypeInt,
  1077. },
  1078. right: &intLiteralExpr{-1},
  1079. isEq: !negate,
  1080. }
  1081. } else if s, ok := maybeString(xValue); ok {
  1082. if s2, ok := maybeString(xCall.args[0]); ok && s == s2 {
  1083. return &eqExpr{
  1084. left: &callExpr{
  1085. object: xCall.args[1],
  1086. name: "find",
  1087. args: []starlarkExpr{xCall.args[0]},
  1088. returnType: starlarkTypeInt,
  1089. },
  1090. right: &intLiteralExpr{-1},
  1091. isEq: negate,
  1092. }
  1093. }
  1094. }
  1095. return ctx.newBadExpr(directive, "$(findstring) can only be compared to nothing or its first argument")
  1096. }
  1097. func (ctx *parseContext) parseCompareStripFuncResult(directive *mkparser.Directive,
  1098. xCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr {
  1099. if _, ok := xValue.(*stringLiteralExpr); !ok {
  1100. return ctx.newBadExpr(directive, "strip result can be compared only to string: %s", xValue)
  1101. }
  1102. return &eqExpr{
  1103. left: &callExpr{
  1104. name: "strip",
  1105. args: xCall.args,
  1106. returnType: starlarkTypeString,
  1107. },
  1108. right: xValue, isEq: !negate}
  1109. }
  1110. func (ctx *parseContext) maybeParseFunctionCall(node mkparser.Node, ref *mkparser.MakeString) (name string, args *mkparser.MakeString, ok bool) {
  1111. ref.TrimLeftSpaces()
  1112. ref.TrimRightSpaces()
  1113. words := ref.SplitN(" ", 2)
  1114. if !words[0].Const() {
  1115. return "", nil, false
  1116. }
  1117. name = words[0].Dump()
  1118. args = mkparser.SimpleMakeString("", words[0].Pos())
  1119. if len(words) >= 2 {
  1120. args = words[1]
  1121. }
  1122. args.TrimLeftSpaces()
  1123. if name == "call" {
  1124. words = args.SplitN(",", 2)
  1125. if words[0].Empty() || !words[0].Const() {
  1126. return "", nil, false
  1127. }
  1128. name = words[0].Dump()
  1129. if len(words) < 2 {
  1130. args = &mkparser.MakeString{}
  1131. } else {
  1132. args = words[1]
  1133. }
  1134. }
  1135. ok = true
  1136. return
  1137. }
  1138. // parses $(...), returning an expression
  1139. func (ctx *parseContext) parseReference(node mkparser.Node, ref *mkparser.MakeString) starlarkExpr {
  1140. ref.TrimLeftSpaces()
  1141. ref.TrimRightSpaces()
  1142. refDump := ref.Dump()
  1143. // Handle only the case where the first (or only) word is constant
  1144. words := ref.SplitN(" ", 2)
  1145. if !words[0].Const() {
  1146. return ctx.newBadExpr(node, "reference is too complex: %s", refDump)
  1147. }
  1148. // If it is a single word, it can be a simple variable
  1149. // reference or a function call
  1150. if len(words) == 1 && !isMakeControlFunc(refDump) && refDump != "shell" && refDump != "eval" {
  1151. if strings.HasPrefix(refDump, soongNsPrefix) {
  1152. // TODO (asmundak): if we find many, maybe handle them.
  1153. return ctx.newBadExpr(node, "SOONG_CONFIG_ variables cannot be referenced, use soong_config_get instead: %s", refDump)
  1154. }
  1155. // Handle substitution references: https://www.gnu.org/software/make/manual/html_node/Substitution-Refs.html
  1156. if strings.Contains(refDump, ":") {
  1157. parts := strings.SplitN(refDump, ":", 2)
  1158. substParts := strings.SplitN(parts[1], "=", 2)
  1159. if len(substParts) < 2 || strings.Count(substParts[0], "%") > 1 {
  1160. return ctx.newBadExpr(node, "Invalid substitution reference")
  1161. }
  1162. if !strings.Contains(substParts[0], "%") {
  1163. if strings.Contains(substParts[1], "%") {
  1164. return ctx.newBadExpr(node, "A substitution reference must have a %% in the \"before\" part of the substitution if it has one in the \"after\" part.")
  1165. }
  1166. substParts[0] = "%" + substParts[0]
  1167. substParts[1] = "%" + substParts[1]
  1168. }
  1169. v := ctx.addVariable(parts[0])
  1170. if v == nil {
  1171. return ctx.newBadExpr(node, "unknown variable %s", refDump)
  1172. }
  1173. return &callExpr{
  1174. name: baseName + ".mkpatsubst",
  1175. returnType: starlarkTypeString,
  1176. args: []starlarkExpr{
  1177. &stringLiteralExpr{literal: substParts[0]},
  1178. &stringLiteralExpr{literal: substParts[1]},
  1179. NewVariableRefExpr(v),
  1180. },
  1181. }
  1182. }
  1183. if v := ctx.addVariable(refDump); v != nil {
  1184. return NewVariableRefExpr(v)
  1185. }
  1186. return ctx.newBadExpr(node, "unknown variable %s", refDump)
  1187. }
  1188. if name, args, ok := ctx.maybeParseFunctionCall(node, ref); ok {
  1189. if kf, found := knownFunctions[name]; found {
  1190. return kf.parse(ctx, node, args)
  1191. } else {
  1192. return ctx.newBadExpr(node, "cannot handle invoking %s", name)
  1193. }
  1194. } else {
  1195. return ctx.newBadExpr(node, "cannot handle %s", refDump)
  1196. }
  1197. }
  1198. type simpleCallParser struct {
  1199. name string
  1200. returnType starlarkType
  1201. addGlobals bool
  1202. addHandle bool
  1203. }
  1204. func (p *simpleCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
  1205. expr := &callExpr{name: p.name, returnType: p.returnType}
  1206. if p.addGlobals {
  1207. expr.args = append(expr.args, &globalsExpr{})
  1208. }
  1209. if p.addHandle {
  1210. expr.args = append(expr.args, &identifierExpr{name: "handle"})
  1211. }
  1212. for _, arg := range args.Split(",") {
  1213. arg.TrimLeftSpaces()
  1214. arg.TrimRightSpaces()
  1215. x := ctx.parseMakeString(node, arg)
  1216. if xBad, ok := x.(*badExpr); ok {
  1217. return xBad
  1218. }
  1219. expr.args = append(expr.args, x)
  1220. }
  1221. return expr
  1222. }
  1223. type makeControlFuncParser struct {
  1224. name string
  1225. }
  1226. func (p *makeControlFuncParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
  1227. // Make control functions need special treatment as everything
  1228. // after the name is a single text argument
  1229. x := ctx.parseMakeString(node, args)
  1230. if xBad, ok := x.(*badExpr); ok {
  1231. return xBad
  1232. }
  1233. return &callExpr{
  1234. name: p.name,
  1235. args: []starlarkExpr{
  1236. &stringLiteralExpr{ctx.script.mkFile},
  1237. x,
  1238. },
  1239. returnType: starlarkTypeUnknown,
  1240. }
  1241. }
  1242. type shellCallParser struct{}
  1243. func (p *shellCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
  1244. // Shell functions need special treatment as everything
  1245. // after the name is a single text argument
  1246. x := ctx.parseMakeString(node, args)
  1247. if xBad, ok := x.(*badExpr); ok {
  1248. return xBad
  1249. }
  1250. return &callExpr{
  1251. name: baseName + ".shell",
  1252. args: []starlarkExpr{x},
  1253. returnType: starlarkTypeUnknown,
  1254. }
  1255. }
  1256. type myDirCallParser struct{}
  1257. func (p *myDirCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
  1258. if !args.Empty() {
  1259. return ctx.newBadExpr(node, "my-dir function cannot have any arguments passed to it.")
  1260. }
  1261. return &stringLiteralExpr{literal: filepath.Dir(ctx.script.mkFile)}
  1262. }
  1263. type isProductInListCallParser struct{}
  1264. func (p *isProductInListCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
  1265. if args.Empty() {
  1266. return ctx.newBadExpr(node, "is-product-in-list requires an argument")
  1267. }
  1268. return &inExpr{
  1269. expr: NewVariableRefExpr(ctx.addVariable("TARGET_PRODUCT")),
  1270. list: maybeConvertToStringList(ctx.parseMakeString(node, args)),
  1271. isNot: false,
  1272. }
  1273. }
  1274. type isVendorBoardPlatformCallParser struct{}
  1275. func (p *isVendorBoardPlatformCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
  1276. if args.Empty() || !identifierFullMatchRegex.MatchString(args.Dump()) {
  1277. return ctx.newBadExpr(node, "cannot handle non-constant argument to is-vendor-board-platform")
  1278. }
  1279. return &inExpr{
  1280. expr: NewVariableRefExpr(ctx.addVariable("TARGET_BOARD_PLATFORM")),
  1281. list: NewVariableRefExpr(ctx.addVariable(args.Dump() + "_BOARD_PLATFORMS")),
  1282. isNot: false,
  1283. }
  1284. }
  1285. type isVendorBoardQcomCallParser struct{}
  1286. func (p *isVendorBoardQcomCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
  1287. if !args.Empty() {
  1288. return ctx.newBadExpr(node, "is-vendor-board-qcom does not accept any arguments")
  1289. }
  1290. return &inExpr{
  1291. expr: NewVariableRefExpr(ctx.addVariable("TARGET_BOARD_PLATFORM")),
  1292. list: NewVariableRefExpr(ctx.addVariable("QCOM_BOARD_PLATFORMS")),
  1293. isNot: false,
  1294. }
  1295. }
  1296. type substCallParser struct {
  1297. fname string
  1298. }
  1299. func (p *substCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
  1300. words := args.Split(",")
  1301. if len(words) != 3 {
  1302. return ctx.newBadExpr(node, "%s function should have 3 arguments", p.fname)
  1303. }
  1304. from := ctx.parseMakeString(node, words[0])
  1305. if xBad, ok := from.(*badExpr); ok {
  1306. return xBad
  1307. }
  1308. to := ctx.parseMakeString(node, words[1])
  1309. if xBad, ok := to.(*badExpr); ok {
  1310. return xBad
  1311. }
  1312. words[2].TrimLeftSpaces()
  1313. words[2].TrimRightSpaces()
  1314. obj := ctx.parseMakeString(node, words[2])
  1315. typ := obj.typ()
  1316. if typ == starlarkTypeString && p.fname == "subst" {
  1317. // Optimization: if it's $(subst from, to, string), emit string.replace(from, to)
  1318. return &callExpr{
  1319. object: obj,
  1320. name: "replace",
  1321. args: []starlarkExpr{from, to},
  1322. returnType: typ,
  1323. }
  1324. }
  1325. return &callExpr{
  1326. name: baseName + ".mk" + p.fname,
  1327. args: []starlarkExpr{from, to, obj},
  1328. returnType: obj.typ(),
  1329. }
  1330. }
  1331. type ifCallParser struct{}
  1332. func (p *ifCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
  1333. words := args.Split(",")
  1334. if len(words) != 2 && len(words) != 3 {
  1335. return ctx.newBadExpr(node, "if function should have 2 or 3 arguments, found "+strconv.Itoa(len(words)))
  1336. }
  1337. condition := ctx.parseMakeString(node, words[0])
  1338. ifTrue := ctx.parseMakeString(node, words[1])
  1339. var ifFalse starlarkExpr
  1340. if len(words) == 3 {
  1341. ifFalse = ctx.parseMakeString(node, words[2])
  1342. } else {
  1343. switch ifTrue.typ() {
  1344. case starlarkTypeList:
  1345. ifFalse = &listExpr{items: []starlarkExpr{}}
  1346. case starlarkTypeInt:
  1347. ifFalse = &intLiteralExpr{literal: 0}
  1348. case starlarkTypeBool:
  1349. ifFalse = &boolLiteralExpr{literal: false}
  1350. default:
  1351. ifFalse = &stringLiteralExpr{literal: ""}
  1352. }
  1353. }
  1354. return &ifExpr{
  1355. condition,
  1356. ifTrue,
  1357. ifFalse,
  1358. }
  1359. }
  1360. type ifCallNodeParser struct{}
  1361. func (p *ifCallNodeParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) []starlarkNode {
  1362. words := args.Split(",")
  1363. if len(words) != 2 && len(words) != 3 {
  1364. return []starlarkNode{ctx.newBadNode(node, "if function should have 2 or 3 arguments, found "+strconv.Itoa(len(words)))}
  1365. }
  1366. ifn := &ifNode{expr: ctx.parseMakeString(node, words[0])}
  1367. cases := []*switchCase{
  1368. {
  1369. gate: ifn,
  1370. nodes: ctx.parseNodeMakeString(node, words[1]),
  1371. },
  1372. }
  1373. if len(words) == 3 {
  1374. cases = append(cases, &switchCase{
  1375. gate: &elseNode{},
  1376. nodes: ctx.parseNodeMakeString(node, words[2]),
  1377. })
  1378. }
  1379. if len(cases) == 2 {
  1380. if len(cases[1].nodes) == 0 {
  1381. // Remove else branch if it has no contents
  1382. cases = cases[:1]
  1383. } else if len(cases[0].nodes) == 0 {
  1384. // If the if branch has no contents but the else does,
  1385. // move them to the if and negate its condition
  1386. ifn.expr = negateExpr(ifn.expr)
  1387. cases[0].nodes = cases[1].nodes
  1388. cases = cases[:1]
  1389. }
  1390. }
  1391. return []starlarkNode{&switchNode{ssCases: cases}}
  1392. }
  1393. type foreachCallParser struct{}
  1394. func (p *foreachCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
  1395. words := args.Split(",")
  1396. if len(words) != 3 {
  1397. return ctx.newBadExpr(node, "foreach function should have 3 arguments, found "+strconv.Itoa(len(words)))
  1398. }
  1399. if !words[0].Const() || words[0].Empty() || !identifierFullMatchRegex.MatchString(words[0].Strings[0]) {
  1400. return ctx.newBadExpr(node, "first argument to foreach function must be a simple string identifier")
  1401. }
  1402. loopVarName := words[0].Strings[0]
  1403. list := ctx.parseMakeString(node, words[1])
  1404. action := ctx.parseMakeString(node, words[2]).transform(func(expr starlarkExpr) starlarkExpr {
  1405. if varRefExpr, ok := expr.(*variableRefExpr); ok && varRefExpr.ref.name() == loopVarName {
  1406. return &identifierExpr{loopVarName}
  1407. }
  1408. return nil
  1409. })
  1410. if list.typ() != starlarkTypeList {
  1411. list = &callExpr{
  1412. name: baseName + ".words",
  1413. returnType: starlarkTypeList,
  1414. args: []starlarkExpr{list},
  1415. }
  1416. }
  1417. return &foreachExpr{
  1418. varName: loopVarName,
  1419. list: list,
  1420. action: action,
  1421. }
  1422. }
  1423. func transformNode(node starlarkNode, transformer func(expr starlarkExpr) starlarkExpr) {
  1424. switch a := node.(type) {
  1425. case *ifNode:
  1426. a.expr = a.expr.transform(transformer)
  1427. case *switchCase:
  1428. transformNode(a.gate, transformer)
  1429. for _, n := range a.nodes {
  1430. transformNode(n, transformer)
  1431. }
  1432. case *switchNode:
  1433. for _, n := range a.ssCases {
  1434. transformNode(n, transformer)
  1435. }
  1436. case *exprNode:
  1437. a.expr = a.expr.transform(transformer)
  1438. case *assignmentNode:
  1439. a.value = a.value.transform(transformer)
  1440. case *foreachNode:
  1441. a.list = a.list.transform(transformer)
  1442. for _, n := range a.actions {
  1443. transformNode(n, transformer)
  1444. }
  1445. }
  1446. }
  1447. type foreachCallNodeParser struct{}
  1448. func (p *foreachCallNodeParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) []starlarkNode {
  1449. words := args.Split(",")
  1450. if len(words) != 3 {
  1451. return []starlarkNode{ctx.newBadNode(node, "foreach function should have 3 arguments, found "+strconv.Itoa(len(words)))}
  1452. }
  1453. if !words[0].Const() || words[0].Empty() || !identifierFullMatchRegex.MatchString(words[0].Strings[0]) {
  1454. return []starlarkNode{ctx.newBadNode(node, "first argument to foreach function must be a simple string identifier")}
  1455. }
  1456. loopVarName := words[0].Strings[0]
  1457. list := ctx.parseMakeString(node, words[1])
  1458. if list.typ() != starlarkTypeList {
  1459. list = &callExpr{
  1460. name: baseName + ".words",
  1461. returnType: starlarkTypeList,
  1462. args: []starlarkExpr{list},
  1463. }
  1464. }
  1465. actions := ctx.parseNodeMakeString(node, words[2])
  1466. // TODO(colefaust): Replace transforming code with something more elegant
  1467. for _, action := range actions {
  1468. transformNode(action, func(expr starlarkExpr) starlarkExpr {
  1469. if varRefExpr, ok := expr.(*variableRefExpr); ok && varRefExpr.ref.name() == loopVarName {
  1470. return &identifierExpr{loopVarName}
  1471. }
  1472. return nil
  1473. })
  1474. }
  1475. return []starlarkNode{&foreachNode{
  1476. varName: loopVarName,
  1477. list: list,
  1478. actions: actions,
  1479. }}
  1480. }
  1481. type wordCallParser struct{}
  1482. func (p *wordCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
  1483. words := args.Split(",")
  1484. if len(words) != 2 {
  1485. return ctx.newBadExpr(node, "word function should have 2 arguments")
  1486. }
  1487. var index uint64 = 0
  1488. if words[0].Const() {
  1489. index, _ = strconv.ParseUint(strings.TrimSpace(words[0].Strings[0]), 10, 64)
  1490. }
  1491. if index < 1 {
  1492. return ctx.newBadExpr(node, "word index should be constant positive integer")
  1493. }
  1494. words[1].TrimLeftSpaces()
  1495. words[1].TrimRightSpaces()
  1496. array := ctx.parseMakeString(node, words[1])
  1497. if xBad, ok := array.(*badExpr); ok {
  1498. return xBad
  1499. }
  1500. if array.typ() != starlarkTypeList {
  1501. array = &callExpr{object: array, name: "split", returnType: starlarkTypeList}
  1502. }
  1503. return &indexExpr{array, &intLiteralExpr{int(index - 1)}}
  1504. }
  1505. type firstOrLastwordCallParser struct {
  1506. isLastWord bool
  1507. }
  1508. func (p *firstOrLastwordCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
  1509. arg := ctx.parseMakeString(node, args)
  1510. if bad, ok := arg.(*badExpr); ok {
  1511. return bad
  1512. }
  1513. index := &intLiteralExpr{0}
  1514. if p.isLastWord {
  1515. if v, ok := arg.(*variableRefExpr); ok && v.ref.name() == "MAKEFILE_LIST" {
  1516. return &stringLiteralExpr{ctx.script.mkFile}
  1517. }
  1518. index.literal = -1
  1519. }
  1520. if arg.typ() == starlarkTypeList {
  1521. return &indexExpr{arg, index}
  1522. }
  1523. return &indexExpr{&callExpr{object: arg, name: "split", returnType: starlarkTypeList}, index}
  1524. }
  1525. func parseIntegerArguments(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString, expectedArgs int) ([]starlarkExpr, error) {
  1526. parsedArgs := make([]starlarkExpr, 0)
  1527. for _, arg := range args.Split(",") {
  1528. expr := ctx.parseMakeString(node, arg)
  1529. if expr.typ() == starlarkTypeList {
  1530. return nil, fmt.Errorf("argument to math argument has type list, which cannot be converted to int")
  1531. }
  1532. if s, ok := maybeString(expr); ok {
  1533. intVal, err := strconv.Atoi(strings.TrimSpace(s))
  1534. if err != nil {
  1535. return nil, err
  1536. }
  1537. expr = &intLiteralExpr{literal: intVal}
  1538. } else if expr.typ() != starlarkTypeInt {
  1539. expr = &callExpr{
  1540. name: "int",
  1541. args: []starlarkExpr{expr},
  1542. returnType: starlarkTypeInt,
  1543. }
  1544. }
  1545. parsedArgs = append(parsedArgs, expr)
  1546. }
  1547. if len(parsedArgs) != expectedArgs {
  1548. return nil, fmt.Errorf("function should have %d arguments", expectedArgs)
  1549. }
  1550. return parsedArgs, nil
  1551. }
  1552. type mathComparisonCallParser struct {
  1553. op string
  1554. }
  1555. func (p *mathComparisonCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
  1556. parsedArgs, err := parseIntegerArguments(ctx, node, args, 2)
  1557. if err != nil {
  1558. return ctx.newBadExpr(node, err.Error())
  1559. }
  1560. return &binaryOpExpr{
  1561. left: parsedArgs[0],
  1562. right: parsedArgs[1],
  1563. op: p.op,
  1564. returnType: starlarkTypeBool,
  1565. }
  1566. }
  1567. type mathMaxOrMinCallParser struct {
  1568. function string
  1569. }
  1570. func (p *mathMaxOrMinCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
  1571. parsedArgs, err := parseIntegerArguments(ctx, node, args, 2)
  1572. if err != nil {
  1573. return ctx.newBadExpr(node, err.Error())
  1574. }
  1575. return &callExpr{
  1576. object: nil,
  1577. name: p.function,
  1578. args: parsedArgs,
  1579. returnType: starlarkTypeInt,
  1580. }
  1581. }
  1582. type evalNodeParser struct{}
  1583. func (p *evalNodeParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) []starlarkNode {
  1584. parser := mkparser.NewParser("Eval expression", strings.NewReader(args.Dump()))
  1585. nodes, errs := parser.Parse()
  1586. if errs != nil {
  1587. return []starlarkNode{ctx.newBadNode(node, "Unable to parse eval statement")}
  1588. }
  1589. if len(nodes) == 0 {
  1590. return []starlarkNode{}
  1591. } else if len(nodes) == 1 {
  1592. switch n := nodes[0].(type) {
  1593. case *mkparser.Assignment:
  1594. if n.Name.Const() {
  1595. return ctx.handleAssignment(n)
  1596. }
  1597. case *mkparser.Comment:
  1598. return []starlarkNode{&commentNode{strings.TrimSpace("#" + n.Comment)}}
  1599. }
  1600. }
  1601. return []starlarkNode{ctx.newBadNode(node, "Eval expression too complex; only assignments and comments are supported")}
  1602. }
  1603. func (ctx *parseContext) parseMakeString(node mkparser.Node, mk *mkparser.MakeString) starlarkExpr {
  1604. if mk.Const() {
  1605. return &stringLiteralExpr{mk.Dump()}
  1606. }
  1607. if mkRef, ok := mk.SingleVariable(); ok {
  1608. return ctx.parseReference(node, mkRef)
  1609. }
  1610. // If we reached here, it's neither string literal nor a simple variable,
  1611. // we need a full-blown interpolation node that will generate
  1612. // "a%b%c" % (X, Y) for a$(X)b$(Y)c
  1613. parts := make([]starlarkExpr, len(mk.Variables)+len(mk.Strings))
  1614. for i := 0; i < len(parts); i++ {
  1615. if i%2 == 0 {
  1616. parts[i] = &stringLiteralExpr{literal: mk.Strings[i/2]}
  1617. } else {
  1618. parts[i] = ctx.parseReference(node, mk.Variables[i/2].Name)
  1619. if x, ok := parts[i].(*badExpr); ok {
  1620. return x
  1621. }
  1622. }
  1623. }
  1624. return NewInterpolateExpr(parts)
  1625. }
  1626. func (ctx *parseContext) parseNodeMakeString(node mkparser.Node, mk *mkparser.MakeString) []starlarkNode {
  1627. // Discard any constant values in the make string, as they would be top level
  1628. // string literals and do nothing.
  1629. result := make([]starlarkNode, 0, len(mk.Variables))
  1630. for i := range mk.Variables {
  1631. result = append(result, ctx.handleVariable(&mk.Variables[i])...)
  1632. }
  1633. return result
  1634. }
  1635. // Handles the statements whose treatment is the same in all contexts: comment,
  1636. // assignment, variable (which is a macro call in reality) and all constructs that
  1637. // do not handle in any context ('define directive and any unrecognized stuff).
  1638. func (ctx *parseContext) handleSimpleStatement(node mkparser.Node) []starlarkNode {
  1639. var result []starlarkNode
  1640. switch x := node.(type) {
  1641. case *mkparser.Comment:
  1642. if n, handled := ctx.maybeHandleAnnotation(x); handled && n != nil {
  1643. result = []starlarkNode{n}
  1644. } else if !handled {
  1645. result = []starlarkNode{&commentNode{strings.TrimSpace("#" + x.Comment)}}
  1646. }
  1647. case *mkparser.Assignment:
  1648. result = ctx.handleAssignment(x)
  1649. case *mkparser.Variable:
  1650. result = ctx.handleVariable(x)
  1651. case *mkparser.Directive:
  1652. switch x.Name {
  1653. case "define":
  1654. if res := ctx.maybeHandleDefine(x); res != nil {
  1655. result = []starlarkNode{res}
  1656. }
  1657. case "include", "-include":
  1658. result = ctx.handleInclude(node, ctx.parseMakeString(node, x.Args), x.Name[0] != '-')
  1659. case "ifeq", "ifneq", "ifdef", "ifndef":
  1660. result = []starlarkNode{ctx.handleIfBlock(x)}
  1661. default:
  1662. result = []starlarkNode{ctx.newBadNode(x, "unexpected directive %s", x.Name)}
  1663. }
  1664. default:
  1665. result = []starlarkNode{ctx.newBadNode(x, "unsupported line %s", strings.ReplaceAll(x.Dump(), "\n", "\n#"))}
  1666. }
  1667. // Clear the includeTops after each non-comment statement
  1668. // so that include annotations placed on certain statements don't apply
  1669. // globally for the rest of the makefile was well.
  1670. if _, wasComment := node.(*mkparser.Comment); !wasComment {
  1671. ctx.atTopOfMakefile = false
  1672. ctx.includeTops = []string{}
  1673. }
  1674. if result == nil {
  1675. result = []starlarkNode{}
  1676. }
  1677. return result
  1678. }
  1679. // The types allowed in a type_hint
  1680. var typeHintMap = map[string]starlarkType{
  1681. "string": starlarkTypeString,
  1682. "list": starlarkTypeList,
  1683. }
  1684. // Processes annotation. An annotation is a comment that starts with #RBC# and provides
  1685. // a conversion hint -- say, where to look for the dynamically calculated inherit/include
  1686. // paths. Returns true if the comment was a successfully-handled annotation.
  1687. func (ctx *parseContext) maybeHandleAnnotation(cnode *mkparser.Comment) (starlarkNode, bool) {
  1688. maybeTrim := func(s, prefix string) (string, bool) {
  1689. if strings.HasPrefix(s, prefix) {
  1690. return strings.TrimSpace(strings.TrimPrefix(s, prefix)), true
  1691. }
  1692. return s, false
  1693. }
  1694. annotation, ok := maybeTrim(cnode.Comment, annotationCommentPrefix)
  1695. if !ok {
  1696. return nil, false
  1697. }
  1698. if p, ok := maybeTrim(annotation, "include_top"); ok {
  1699. // Don't allow duplicate include tops, because then we will generate
  1700. // invalid starlark code. (duplicate keys in the _entry dictionary)
  1701. for _, top := range ctx.includeTops {
  1702. if top == p {
  1703. return nil, true
  1704. }
  1705. }
  1706. ctx.includeTops = append(ctx.includeTops, p)
  1707. return nil, true
  1708. } else if p, ok := maybeTrim(annotation, "type_hint"); ok {
  1709. // Type hints must come at the beginning the file, to avoid confusion
  1710. // if a type hint was specified later and thus only takes effect for half
  1711. // of the file.
  1712. if !ctx.atTopOfMakefile {
  1713. return ctx.newBadNode(cnode, "type_hint annotations must come before the first Makefile statement"), true
  1714. }
  1715. parts := strings.Fields(p)
  1716. if len(parts) <= 1 {
  1717. return ctx.newBadNode(cnode, "Invalid type_hint annotation: %s. Must be a variable type followed by a list of variables of that type", p), true
  1718. }
  1719. var varType starlarkType
  1720. if varType, ok = typeHintMap[parts[0]]; !ok {
  1721. varType = starlarkTypeUnknown
  1722. }
  1723. if varType == starlarkTypeUnknown {
  1724. return ctx.newBadNode(cnode, "Invalid type_hint annotation. Only list/string types are accepted, found %s", parts[0]), true
  1725. }
  1726. for _, name := range parts[1:] {
  1727. // Don't allow duplicate type hints
  1728. if _, ok := ctx.typeHints[name]; ok {
  1729. return ctx.newBadNode(cnode, "Duplicate type hint for variable %s", name), true
  1730. }
  1731. ctx.typeHints[name] = varType
  1732. }
  1733. return nil, true
  1734. }
  1735. return ctx.newBadNode(cnode, "unsupported annotation %s", cnode.Comment), true
  1736. }
  1737. func (ctx *parseContext) loadedModulePath(path string) string {
  1738. // During the transition to Roboleaf some of the product configuration files
  1739. // will be converted and checked in while the others will be generated on the fly
  1740. // and run. The runner (rbcrun application) accommodates this by allowing three
  1741. // different ways to specify the loaded file location:
  1742. // 1) load(":<file>",...) loads <file> from the same directory
  1743. // 2) load("//path/relative/to/source/root:<file>", ...) loads <file> source tree
  1744. // 3) load("/absolute/path/to/<file> absolute path
  1745. // If the file being generated and the file it wants to load are in the same directory,
  1746. // generate option 1.
  1747. // Otherwise, if output directory is not specified, generate 2)
  1748. // Finally, if output directory has been specified and the file being generated and
  1749. // the file it wants to load from are in the different directories, generate 2) or 3):
  1750. // * if the file being loaded exists in the source tree, generate 2)
  1751. // * otherwise, generate 3)
  1752. // Finally, figure out the loaded module path and name and create a node for it
  1753. loadedModuleDir := filepath.Dir(path)
  1754. base := filepath.Base(path)
  1755. loadedModuleName := strings.TrimSuffix(base, filepath.Ext(base)) + ctx.outputSuffix
  1756. if loadedModuleDir == filepath.Dir(ctx.script.mkFile) {
  1757. return ":" + loadedModuleName
  1758. }
  1759. if ctx.outputDir == "" {
  1760. return fmt.Sprintf("//%s:%s", loadedModuleDir, loadedModuleName)
  1761. }
  1762. if _, err := os.Stat(filepath.Join(loadedModuleDir, loadedModuleName)); err == nil {
  1763. return fmt.Sprintf("//%s:%s", loadedModuleDir, loadedModuleName)
  1764. }
  1765. return filepath.Join(ctx.outputDir, loadedModuleDir, loadedModuleName)
  1766. }
  1767. func (ctx *parseContext) addSoongNamespace(ns string) {
  1768. if _, ok := ctx.soongNamespaces[ns]; ok {
  1769. return
  1770. }
  1771. ctx.soongNamespaces[ns] = make(map[string]bool)
  1772. }
  1773. func (ctx *parseContext) hasSoongNamespace(name string) bool {
  1774. _, ok := ctx.soongNamespaces[name]
  1775. return ok
  1776. }
  1777. func (ctx *parseContext) updateSoongNamespace(replace bool, namespaceName string, varNames []string) {
  1778. ctx.addSoongNamespace(namespaceName)
  1779. vars := ctx.soongNamespaces[namespaceName]
  1780. if replace {
  1781. vars = make(map[string]bool)
  1782. ctx.soongNamespaces[namespaceName] = vars
  1783. }
  1784. for _, v := range varNames {
  1785. vars[v] = true
  1786. }
  1787. }
  1788. func (ctx *parseContext) hasNamespaceVar(namespaceName string, varName string) bool {
  1789. vars, ok := ctx.soongNamespaces[namespaceName]
  1790. if ok {
  1791. _, ok = vars[varName]
  1792. }
  1793. return ok
  1794. }
  1795. func (ctx *parseContext) errorLocation(node mkparser.Node) ErrorLocation {
  1796. return ErrorLocation{ctx.script.mkFile, ctx.script.nodeLocator(node.Pos())}
  1797. }
  1798. func (ss *StarlarkScript) String() string {
  1799. return NewGenerateContext(ss).emit()
  1800. }
  1801. func (ss *StarlarkScript) SubConfigFiles() []string {
  1802. var subs []string
  1803. for _, src := range ss.inherited {
  1804. subs = append(subs, src.originalPath)
  1805. }
  1806. return subs
  1807. }
  1808. func (ss *StarlarkScript) HasErrors() bool {
  1809. return ss.hasErrors
  1810. }
  1811. // Convert reads and parses a makefile. If successful, parsed tree
  1812. // is returned and then can be passed to String() to get the generated
  1813. // Starlark file.
  1814. func Convert(req Request) (*StarlarkScript, error) {
  1815. reader := req.Reader
  1816. if reader == nil {
  1817. mkContents, err := ioutil.ReadFile(req.MkFile)
  1818. if err != nil {
  1819. return nil, err
  1820. }
  1821. reader = bytes.NewBuffer(mkContents)
  1822. }
  1823. parser := mkparser.NewParser(req.MkFile, reader)
  1824. nodes, errs := parser.Parse()
  1825. if len(errs) > 0 {
  1826. for _, e := range errs {
  1827. fmt.Fprintln(os.Stderr, "ERROR:", e)
  1828. }
  1829. return nil, fmt.Errorf("bad makefile %s", req.MkFile)
  1830. }
  1831. starScript := &StarlarkScript{
  1832. moduleName: moduleNameForFile(req.MkFile),
  1833. mkFile: req.MkFile,
  1834. traceCalls: req.TraceCalls,
  1835. sourceFS: req.SourceFS,
  1836. makefileFinder: req.MakefileFinder,
  1837. nodeLocator: func(pos mkparser.Pos) int { return parser.Unpack(pos).Line },
  1838. nodes: make([]starlarkNode, 0),
  1839. }
  1840. ctx := newParseContext(starScript, nodes)
  1841. ctx.outputSuffix = req.OutputSuffix
  1842. ctx.outputDir = req.OutputDir
  1843. ctx.errorLogger = req.ErrorLogger
  1844. if len(req.TracedVariables) > 0 {
  1845. ctx.tracedVariables = make(map[string]bool)
  1846. for _, v := range req.TracedVariables {
  1847. ctx.tracedVariables[v] = true
  1848. }
  1849. }
  1850. for ctx.hasNodes() && ctx.fatalError == nil {
  1851. starScript.nodes = append(starScript.nodes, ctx.handleSimpleStatement(ctx.getNode())...)
  1852. }
  1853. if ctx.fatalError != nil {
  1854. return nil, ctx.fatalError
  1855. }
  1856. return starScript, nil
  1857. }
  1858. func Launcher(mainModuleUri, inputVariablesUri, mainModuleName string) string {
  1859. var buf bytes.Buffer
  1860. fmt.Fprintf(&buf, "load(%q, %q)\n", baseUri, baseName)
  1861. fmt.Fprintf(&buf, "load(%q, input_variables_init = \"init\")\n", inputVariablesUri)
  1862. fmt.Fprintf(&buf, "load(%q, \"init\")\n", mainModuleUri)
  1863. fmt.Fprintf(&buf, "%s(%s(%q, init, input_variables_init))\n", cfnPrintVars, cfnMain, mainModuleName)
  1864. return buf.String()
  1865. }
  1866. func BoardLauncher(mainModuleUri string, inputVariablesUri string) string {
  1867. var buf bytes.Buffer
  1868. fmt.Fprintf(&buf, "load(%q, %q)\n", baseUri, baseName)
  1869. fmt.Fprintf(&buf, "load(%q, \"init\")\n", mainModuleUri)
  1870. fmt.Fprintf(&buf, "load(%q, input_variables_init = \"init\")\n", inputVariablesUri)
  1871. fmt.Fprintf(&buf, "%s(%s(init, input_variables_init))\n", cfnPrintVars, cfnBoardMain)
  1872. return buf.String()
  1873. }
  1874. func MakePath2ModuleName(mkPath string) string {
  1875. return strings.TrimSuffix(mkPath, filepath.Ext(mkPath))
  1876. }