mk2rbc.go 47 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480
  1. // Copyright 2021 Google LLC
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. // Convert makefile containing device configuration to Starlark file
  15. // The conversion can handle the following constructs in a makefile:
  16. // * comments
  17. // * simple variable assignments
  18. // * $(call init-product,<file>)
  19. // * $(call inherit-product-if-exists
  20. // * if directives
  21. // All other constructs are carried over to the output starlark file as comments.
  22. //
  23. package mk2rbc
  24. import (
  25. "bytes"
  26. "fmt"
  27. "io"
  28. "io/fs"
  29. "io/ioutil"
  30. "os"
  31. "path/filepath"
  32. "regexp"
  33. "strconv"
  34. "strings"
  35. "text/scanner"
  36. mkparser "android/soong/androidmk/parser"
  37. )
  38. const (
  39. baseUri = "//build/make/core:product_config.rbc"
  40. // The name of the struct exported by the product_config.rbc
  41. // that contains the functions and variables available to
  42. // product configuration Starlark files.
  43. baseName = "rblf"
  44. // And here are the functions and variables:
  45. cfnGetCfg = baseName + ".cfg"
  46. cfnMain = baseName + ".product_configuration"
  47. cfnPrintVars = baseName + ".printvars"
  48. cfnWarning = baseName + ".warning"
  49. cfnLocalAppend = baseName + ".local_append"
  50. cfnLocalSetDefault = baseName + ".local_set_default"
  51. cfnInherit = baseName + ".inherit"
  52. cfnSetListDefault = baseName + ".setdefault"
  53. )
  54. const (
  55. // Phony makefile functions, they are eventually rewritten
  56. // according to knownFunctions map
  57. fileExistsPhony = "$file_exists"
  58. wildcardExistsPhony = "$wildcard_exists"
  59. )
  60. const (
  61. callLoadAlways = "inherit-product"
  62. callLoadIf = "inherit-product-if-exists"
  63. )
  64. var knownFunctions = map[string]struct {
  65. // The name of the runtime function this function call in makefiles maps to.
  66. // If it starts with !, then this makefile function call is rewritten to
  67. // something else.
  68. runtimeName string
  69. returnType starlarkType
  70. }{
  71. fileExistsPhony: {baseName + ".file_exists", starlarkTypeBool},
  72. wildcardExistsPhony: {baseName + ".file_wildcard_exists", starlarkTypeBool},
  73. "add-to-product-copy-files-if-exists": {baseName + ".copy_if_exists", starlarkTypeList},
  74. "addprefix": {baseName + ".addprefix", starlarkTypeList},
  75. "addsuffix": {baseName + ".addsuffix", starlarkTypeList},
  76. "enforce-product-packages-exist": {baseName + ".enforce_product_packages_exist", starlarkTypeVoid},
  77. "error": {baseName + ".mkerror", starlarkTypeVoid},
  78. "findstring": {"!findstring", starlarkTypeInt},
  79. "find-copy-subdir-files": {baseName + ".find_and_copy", starlarkTypeList},
  80. "find-word-in-list": {"!find-word-in-list", starlarkTypeUnknown}, // internal macro
  81. "filter": {baseName + ".filter", starlarkTypeList},
  82. "filter-out": {baseName + ".filter_out", starlarkTypeList},
  83. "get-vendor-board-platforms": {"!get-vendor-board-platforms", starlarkTypeList}, // internal macro, used by is-board-platform, etc.
  84. "info": {baseName + ".mkinfo", starlarkTypeVoid},
  85. "is-android-codename": {"!is-android-codename", starlarkTypeBool}, // unused by product config
  86. "is-android-codename-in-list": {"!is-android-codename-in-list", starlarkTypeBool}, // unused by product config
  87. "is-board-platform": {"!is-board-platform", starlarkTypeBool},
  88. "is-board-platform-in-list": {"!is-board-platform-in-list", starlarkTypeBool},
  89. "is-chipset-in-board-platform": {"!is-chipset-in-board-platform", starlarkTypeUnknown}, // unused by product config
  90. "is-chipset-prefix-in-board-platform": {"!is-chipset-prefix-in-board-platform", starlarkTypeBool}, // unused by product config
  91. "is-not-board-platform": {"!is-not-board-platform", starlarkTypeBool}, // defined but never used
  92. "is-platform-sdk-version-at-least": {"!is-platform-sdk-version-at-least", starlarkTypeBool}, // unused by product config
  93. "is-product-in-list": {"!is-product-in-list", starlarkTypeBool},
  94. "is-vendor-board-platform": {"!is-vendor-board-platform", starlarkTypeBool},
  95. callLoadAlways: {"!inherit-product", starlarkTypeVoid},
  96. callLoadIf: {"!inherit-product-if-exists", starlarkTypeVoid},
  97. "match-prefix": {"!match-prefix", starlarkTypeUnknown}, // internal macro
  98. "match-word": {"!match-word", starlarkTypeUnknown}, // internal macro
  99. "match-word-in-list": {"!match-word-in-list", starlarkTypeUnknown}, // internal macro
  100. "my-dir": {"!my-dir", starlarkTypeString},
  101. "patsubst": {baseName + ".mkpatsubst", starlarkTypeString},
  102. "produce_copy_files": {baseName + ".produce_copy_files", starlarkTypeList},
  103. "require-artifacts-in-path": {baseName + ".require_artifacts_in_path", starlarkTypeVoid},
  104. "require-artifacts-in-path-relaxed": {baseName + ".require_artifacts_in_path_relaxed", starlarkTypeVoid},
  105. // TODO(asmundak): remove it once all calls are removed from configuration makefiles. see b/183161002
  106. "shell": {baseName + ".shell", starlarkTypeString},
  107. "strip": {baseName + ".mkstrip", starlarkTypeString},
  108. "tb-modules": {"!tb-modules", starlarkTypeUnknown}, // defined in hardware/amlogic/tb_modules/tb_detect.mk, unused
  109. "subst": {baseName + ".mksubst", starlarkTypeString},
  110. "warning": {baseName + ".mkwarning", starlarkTypeVoid},
  111. "word": {baseName + "!word", starlarkTypeString},
  112. "wildcard": {baseName + ".expand_wildcard", starlarkTypeList},
  113. }
  114. var builtinFuncRex = regexp.MustCompile(
  115. "^(addprefix|addsuffix|abspath|and|basename|call|dir|error|eval" +
  116. "|flavor|foreach|file|filter|filter-out|findstring|firstword|guile" +
  117. "|if|info|join|lastword|notdir|or|origin|patsubst|realpath" +
  118. "|shell|sort|strip|subst|suffix|value|warning|word|wordlist|words" +
  119. "|wildcard)")
  120. // Conversion request parameters
  121. type Request struct {
  122. MkFile string // file to convert
  123. Reader io.Reader // if set, read input from this stream instead
  124. RootDir string // root directory path used to resolve included files
  125. OutputSuffix string // generated Starlark files suffix
  126. OutputDir string // if set, root of the output hierarchy
  127. ErrorLogger ErrorMonitorCB
  128. TracedVariables []string // trace assignment to these variables
  129. TraceCalls bool
  130. WarnPartialSuccess bool
  131. SourceFS fs.FS
  132. MakefileFinder MakefileFinder
  133. }
  134. // An error sink allowing to gather error statistics.
  135. // NewError is called on every error encountered during processing.
  136. type ErrorMonitorCB interface {
  137. NewError(s string, node mkparser.Node, args ...interface{})
  138. }
  139. // Derives module name for a given file. It is base name
  140. // (file name without suffix), with some characters replaced to make it a Starlark identifier
  141. func moduleNameForFile(mkFile string) string {
  142. base := strings.TrimSuffix(filepath.Base(mkFile), filepath.Ext(mkFile))
  143. // TODO(asmundak): what else can be in the product file names?
  144. return strings.NewReplacer("-", "_", ".", "_").Replace(base)
  145. }
  146. func cloneMakeString(mkString *mkparser.MakeString) *mkparser.MakeString {
  147. r := &mkparser.MakeString{StringPos: mkString.StringPos}
  148. r.Strings = append(r.Strings, mkString.Strings...)
  149. r.Variables = append(r.Variables, mkString.Variables...)
  150. return r
  151. }
  152. func isMakeControlFunc(s string) bool {
  153. return s == "error" || s == "warning" || s == "info"
  154. }
  155. // Starlark output generation context
  156. type generationContext struct {
  157. buf strings.Builder
  158. starScript *StarlarkScript
  159. indentLevel int
  160. inAssignment bool
  161. tracedCount int
  162. }
  163. func NewGenerateContext(ss *StarlarkScript) *generationContext {
  164. return &generationContext{starScript: ss}
  165. }
  166. // emit returns generated script
  167. func (gctx *generationContext) emit() string {
  168. ss := gctx.starScript
  169. // The emitted code has the following layout:
  170. // <initial comments>
  171. // preamble, i.e.,
  172. // load statement for the runtime support
  173. // load statement for each unique submodule pulled in by this one
  174. // def init(g, handle):
  175. // cfg = rblf.cfg(handle)
  176. // <statements>
  177. // <warning if conversion was not clean>
  178. iNode := len(ss.nodes)
  179. for i, node := range ss.nodes {
  180. if _, ok := node.(*commentNode); !ok {
  181. iNode = i
  182. break
  183. }
  184. node.emit(gctx)
  185. }
  186. gctx.emitPreamble()
  187. gctx.newLine()
  188. // The arguments passed to the init function are the global dictionary
  189. // ('g') and the product configuration dictionary ('cfg')
  190. gctx.write("def init(g, handle):")
  191. gctx.indentLevel++
  192. if gctx.starScript.traceCalls {
  193. gctx.newLine()
  194. gctx.writef(`print(">%s")`, gctx.starScript.mkFile)
  195. }
  196. gctx.newLine()
  197. gctx.writef("cfg = %s(handle)", cfnGetCfg)
  198. for _, node := range ss.nodes[iNode:] {
  199. node.emit(gctx)
  200. }
  201. if ss.hasErrors && ss.warnPartialSuccess {
  202. gctx.newLine()
  203. gctx.writef("%s(%q, %q)", cfnWarning, filepath.Base(ss.mkFile), "partially successful conversion")
  204. }
  205. if gctx.starScript.traceCalls {
  206. gctx.newLine()
  207. gctx.writef(`print("<%s")`, gctx.starScript.mkFile)
  208. }
  209. gctx.indentLevel--
  210. gctx.write("\n")
  211. return gctx.buf.String()
  212. }
  213. func (gctx *generationContext) emitPreamble() {
  214. gctx.newLine()
  215. gctx.writef("load(%q, %q)", baseUri, baseName)
  216. // Emit exactly one load statement for each URI.
  217. loadedSubConfigs := make(map[string]string)
  218. for _, sc := range gctx.starScript.inherited {
  219. uri := sc.path
  220. if m, ok := loadedSubConfigs[uri]; ok {
  221. // No need to emit load statement, but fix module name.
  222. sc.moduleLocalName = m
  223. continue
  224. }
  225. if sc.optional {
  226. uri += "|init"
  227. }
  228. gctx.newLine()
  229. gctx.writef("load(%q, %s = \"init\")", uri, sc.entryName())
  230. loadedSubConfigs[uri] = sc.moduleLocalName
  231. }
  232. gctx.write("\n")
  233. }
  234. func (gctx *generationContext) emitPass() {
  235. gctx.newLine()
  236. gctx.write("pass")
  237. }
  238. func (gctx *generationContext) write(ss ...string) {
  239. for _, s := range ss {
  240. gctx.buf.WriteString(s)
  241. }
  242. }
  243. func (gctx *generationContext) writef(format string, args ...interface{}) {
  244. gctx.write(fmt.Sprintf(format, args...))
  245. }
  246. func (gctx *generationContext) newLine() {
  247. if gctx.buf.Len() == 0 {
  248. return
  249. }
  250. gctx.write("\n")
  251. gctx.writef("%*s", 2*gctx.indentLevel, "")
  252. }
  253. type knownVariable struct {
  254. name string
  255. class varClass
  256. valueType starlarkType
  257. }
  258. type knownVariables map[string]knownVariable
  259. func (pcv knownVariables) NewVariable(name string, varClass varClass, valueType starlarkType) {
  260. v, exists := pcv[name]
  261. if !exists {
  262. pcv[name] = knownVariable{name, varClass, valueType}
  263. return
  264. }
  265. // Conflict resolution:
  266. // * config class trumps everything
  267. // * any type trumps unknown type
  268. match := varClass == v.class
  269. if !match {
  270. if varClass == VarClassConfig {
  271. v.class = VarClassConfig
  272. match = true
  273. } else if v.class == VarClassConfig {
  274. match = true
  275. }
  276. }
  277. if valueType != v.valueType {
  278. if valueType != starlarkTypeUnknown {
  279. if v.valueType == starlarkTypeUnknown {
  280. v.valueType = valueType
  281. } else {
  282. match = false
  283. }
  284. }
  285. }
  286. if !match {
  287. fmt.Fprintf(os.Stderr, "cannot redefine %s as %v/%v (already defined as %v/%v)\n",
  288. name, varClass, valueType, v.class, v.valueType)
  289. }
  290. }
  291. // All known product variables.
  292. var KnownVariables = make(knownVariables)
  293. func init() {
  294. for _, kv := range []string{
  295. // Kernel-related variables that we know are lists.
  296. "BOARD_VENDOR_KERNEL_MODULES",
  297. "BOARD_VENDOR_RAMDISK_KERNEL_MODULES",
  298. "BOARD_VENDOR_RAMDISK_KERNEL_MODULES_LOAD",
  299. "BOARD_RECOVERY_KERNEL_MODULES",
  300. // Other variables we knwo are lists
  301. "ART_APEX_JARS",
  302. } {
  303. KnownVariables.NewVariable(kv, VarClassSoong, starlarkTypeList)
  304. }
  305. }
  306. type nodeReceiver interface {
  307. newNode(node starlarkNode)
  308. }
  309. // Information about the generated Starlark script.
  310. type StarlarkScript struct {
  311. mkFile string
  312. moduleName string
  313. mkPos scanner.Position
  314. nodes []starlarkNode
  315. inherited []*moduleInfo
  316. hasErrors bool
  317. topDir string
  318. traceCalls bool // print enter/exit each init function
  319. warnPartialSuccess bool
  320. sourceFS fs.FS
  321. makefileFinder MakefileFinder
  322. }
  323. func (ss *StarlarkScript) newNode(node starlarkNode) {
  324. ss.nodes = append(ss.nodes, node)
  325. }
  326. // varAssignmentScope points to the last assignment for each variable
  327. // in the current block. It is used during the parsing to chain
  328. // the assignments to a variable together.
  329. type varAssignmentScope struct {
  330. outer *varAssignmentScope
  331. vars map[string]*assignmentNode
  332. }
  333. // parseContext holds the script we are generating and all the ephemeral data
  334. // needed during the parsing.
  335. type parseContext struct {
  336. script *StarlarkScript
  337. nodes []mkparser.Node // Makefile as parsed by mkparser
  338. currentNodeIndex int // Node in it we are processing
  339. ifNestLevel int
  340. moduleNameCount map[string]int // count of imported modules with given basename
  341. fatalError error
  342. builtinMakeVars map[string]starlarkExpr
  343. outputSuffix string
  344. errorLogger ErrorMonitorCB
  345. tracedVariables map[string]bool // variables to be traced in the generated script
  346. variables map[string]variable
  347. varAssignments *varAssignmentScope
  348. receiver nodeReceiver // receptacle for the generated starlarkNode's
  349. receiverStack []nodeReceiver
  350. outputDir string
  351. dependentModules map[string]*moduleInfo
  352. }
  353. func newParseContext(ss *StarlarkScript, nodes []mkparser.Node) *parseContext {
  354. topdir, _ := filepath.Split(filepath.Join(ss.topDir, "foo"))
  355. predefined := []struct{ name, value string }{
  356. {"SRC_TARGET_DIR", filepath.Join("build", "make", "target")},
  357. {"LOCAL_PATH", filepath.Dir(ss.mkFile)},
  358. {"TOPDIR", topdir},
  359. // TODO(asmundak): maybe read it from build/make/core/envsetup.mk?
  360. {"TARGET_COPY_OUT_SYSTEM", "system"},
  361. {"TARGET_COPY_OUT_SYSTEM_OTHER", "system_other"},
  362. {"TARGET_COPY_OUT_DATA", "data"},
  363. {"TARGET_COPY_OUT_ASAN", filepath.Join("data", "asan")},
  364. {"TARGET_COPY_OUT_OEM", "oem"},
  365. {"TARGET_COPY_OUT_RAMDISK", "ramdisk"},
  366. {"TARGET_COPY_OUT_DEBUG_RAMDISK", "debug_ramdisk"},
  367. {"TARGET_COPY_OUT_VENDOR_DEBUG_RAMDISK", "vendor_debug_ramdisk"},
  368. {"TARGET_COPY_OUT_TEST_HARNESS_RAMDISK", "test_harness_ramdisk"},
  369. {"TARGET_COPY_OUT_ROOT", "root"},
  370. {"TARGET_COPY_OUT_RECOVERY", "recovery"},
  371. {"TARGET_COPY_OUT_VENDOR", "||VENDOR-PATH-PH||"},
  372. {"TARGET_COPY_OUT_VENDOR_RAMDISK", "vendor_ramdisk"},
  373. {"TARGET_COPY_OUT_PRODUCT", "||PRODUCT-PATH-PH||"},
  374. {"TARGET_COPY_OUT_PRODUCT_SERVICES", "||PRODUCT-PATH-PH||"},
  375. {"TARGET_COPY_OUT_SYSTEM_EXT", "||SYSTEM_EXT-PATH-PH||"},
  376. {"TARGET_COPY_OUT_ODM", "||ODM-PATH-PH||"},
  377. {"TARGET_COPY_OUT_VENDOR_DLKM", "||VENDOR_DLKM-PATH-PH||"},
  378. {"TARGET_COPY_OUT_ODM_DLKM", "||ODM_DLKM-PATH-PH||"},
  379. // TODO(asmundak): to process internal config files, we need the following variables:
  380. // BOARD_CONFIG_VENDOR_PATH
  381. // TARGET_VENDOR
  382. // target_base_product
  383. //
  384. // the following utility variables are set in build/make/common/core.mk:
  385. {"empty", ""},
  386. {"space", " "},
  387. {"comma", ","},
  388. {"newline", "\n"},
  389. {"pound", "#"},
  390. {"backslash", "\\"},
  391. }
  392. ctx := &parseContext{
  393. script: ss,
  394. nodes: nodes,
  395. currentNodeIndex: 0,
  396. ifNestLevel: 0,
  397. moduleNameCount: make(map[string]int),
  398. builtinMakeVars: map[string]starlarkExpr{},
  399. variables: make(map[string]variable),
  400. dependentModules: make(map[string]*moduleInfo),
  401. }
  402. ctx.pushVarAssignments()
  403. for _, item := range predefined {
  404. ctx.variables[item.name] = &predefinedVariable{
  405. baseVariable: baseVariable{nam: item.name, typ: starlarkTypeString},
  406. value: &stringLiteralExpr{item.value},
  407. }
  408. }
  409. return ctx
  410. }
  411. func (ctx *parseContext) lastAssignment(name string) *assignmentNode {
  412. for va := ctx.varAssignments; va != nil; va = va.outer {
  413. if v, ok := va.vars[name]; ok {
  414. return v
  415. }
  416. }
  417. return nil
  418. }
  419. func (ctx *parseContext) setLastAssignment(name string, asgn *assignmentNode) {
  420. ctx.varAssignments.vars[name] = asgn
  421. }
  422. func (ctx *parseContext) pushVarAssignments() {
  423. va := &varAssignmentScope{
  424. outer: ctx.varAssignments,
  425. vars: make(map[string]*assignmentNode),
  426. }
  427. ctx.varAssignments = va
  428. }
  429. func (ctx *parseContext) popVarAssignments() {
  430. ctx.varAssignments = ctx.varAssignments.outer
  431. }
  432. func (ctx *parseContext) pushReceiver(rcv nodeReceiver) {
  433. ctx.receiverStack = append(ctx.receiverStack, ctx.receiver)
  434. ctx.receiver = rcv
  435. }
  436. func (ctx *parseContext) popReceiver() {
  437. last := len(ctx.receiverStack) - 1
  438. if last < 0 {
  439. panic(fmt.Errorf("popReceiver: receiver stack empty"))
  440. }
  441. ctx.receiver = ctx.receiverStack[last]
  442. ctx.receiverStack = ctx.receiverStack[0:last]
  443. }
  444. func (ctx *parseContext) hasNodes() bool {
  445. return ctx.currentNodeIndex < len(ctx.nodes)
  446. }
  447. func (ctx *parseContext) getNode() mkparser.Node {
  448. if !ctx.hasNodes() {
  449. return nil
  450. }
  451. node := ctx.nodes[ctx.currentNodeIndex]
  452. ctx.currentNodeIndex++
  453. return node
  454. }
  455. func (ctx *parseContext) backNode() {
  456. if ctx.currentNodeIndex <= 0 {
  457. panic("Cannot back off")
  458. }
  459. ctx.currentNodeIndex--
  460. }
  461. func (ctx *parseContext) handleAssignment(a *mkparser.Assignment) {
  462. // Handle only simple variables
  463. if !a.Name.Const() {
  464. ctx.errorf(a, "Only simple variables are handled")
  465. return
  466. }
  467. name := a.Name.Strings[0]
  468. lhs := ctx.addVariable(name)
  469. if lhs == nil {
  470. ctx.errorf(a, "unknown variable %s", name)
  471. return
  472. }
  473. _, isTraced := ctx.tracedVariables[name]
  474. asgn := &assignmentNode{lhs: lhs, mkValue: a.Value, isTraced: isTraced}
  475. if lhs.valueType() == starlarkTypeUnknown {
  476. // Try to divine variable type from the RHS
  477. asgn.value = ctx.parseMakeString(a, a.Value)
  478. if xBad, ok := asgn.value.(*badExpr); ok {
  479. ctx.wrapBadExpr(xBad)
  480. return
  481. }
  482. inferred_type := asgn.value.typ()
  483. if inferred_type != starlarkTypeUnknown {
  484. lhs.setValueType(inferred_type)
  485. }
  486. }
  487. if lhs.valueType() == starlarkTypeList {
  488. xConcat := ctx.buildConcatExpr(a)
  489. if xConcat == nil {
  490. return
  491. }
  492. switch len(xConcat.items) {
  493. case 0:
  494. asgn.value = &listExpr{}
  495. case 1:
  496. asgn.value = xConcat.items[0]
  497. default:
  498. asgn.value = xConcat
  499. }
  500. } else {
  501. asgn.value = ctx.parseMakeString(a, a.Value)
  502. if xBad, ok := asgn.value.(*badExpr); ok {
  503. ctx.wrapBadExpr(xBad)
  504. return
  505. }
  506. }
  507. // TODO(asmundak): move evaluation to a separate pass
  508. asgn.value, _ = asgn.value.eval(ctx.builtinMakeVars)
  509. asgn.previous = ctx.lastAssignment(name)
  510. ctx.setLastAssignment(name, asgn)
  511. switch a.Type {
  512. case "=", ":=":
  513. asgn.flavor = asgnSet
  514. case "+=":
  515. if asgn.previous == nil && !asgn.lhs.isPreset() {
  516. asgn.flavor = asgnMaybeAppend
  517. } else {
  518. asgn.flavor = asgnAppend
  519. }
  520. case "?=":
  521. asgn.flavor = asgnMaybeSet
  522. default:
  523. panic(fmt.Errorf("unexpected assignment type %s", a.Type))
  524. }
  525. ctx.receiver.newNode(asgn)
  526. }
  527. func (ctx *parseContext) buildConcatExpr(a *mkparser.Assignment) *concatExpr {
  528. xConcat := &concatExpr{}
  529. var xItemList *listExpr
  530. addToItemList := func(x ...starlarkExpr) {
  531. if xItemList == nil {
  532. xItemList = &listExpr{[]starlarkExpr{}}
  533. }
  534. xItemList.items = append(xItemList.items, x...)
  535. }
  536. finishItemList := func() {
  537. if xItemList != nil {
  538. xConcat.items = append(xConcat.items, xItemList)
  539. xItemList = nil
  540. }
  541. }
  542. items := a.Value.Words()
  543. for _, item := range items {
  544. // A function call in RHS is supposed to return a list, all other item
  545. // expressions return individual elements.
  546. switch x := ctx.parseMakeString(a, item).(type) {
  547. case *badExpr:
  548. ctx.wrapBadExpr(x)
  549. return nil
  550. case *stringLiteralExpr:
  551. addToItemList(maybeConvertToStringList(x).(*listExpr).items...)
  552. default:
  553. switch x.typ() {
  554. case starlarkTypeList:
  555. finishItemList()
  556. xConcat.items = append(xConcat.items, x)
  557. case starlarkTypeString:
  558. finishItemList()
  559. xConcat.items = append(xConcat.items, &callExpr{
  560. object: x,
  561. name: "split",
  562. args: nil,
  563. returnType: starlarkTypeList,
  564. })
  565. default:
  566. addToItemList(x)
  567. }
  568. }
  569. }
  570. if xItemList != nil {
  571. xConcat.items = append(xConcat.items, xItemList)
  572. }
  573. return xConcat
  574. }
  575. func (ctx *parseContext) newDependentModule(path string, optional bool) *moduleInfo {
  576. modulePath := ctx.loadedModulePath(path)
  577. if mi, ok := ctx.dependentModules[modulePath]; ok {
  578. mi.optional = mi.optional || optional
  579. return mi
  580. }
  581. moduleName := moduleNameForFile(path)
  582. moduleLocalName := "_" + moduleName
  583. n, found := ctx.moduleNameCount[moduleName]
  584. if found {
  585. moduleLocalName += fmt.Sprintf("%d", n)
  586. }
  587. ctx.moduleNameCount[moduleName] = n + 1
  588. mi := &moduleInfo{
  589. path: modulePath,
  590. originalPath: path,
  591. moduleLocalName: moduleLocalName,
  592. optional: optional,
  593. }
  594. ctx.dependentModules[modulePath] = mi
  595. ctx.script.inherited = append(ctx.script.inherited, mi)
  596. return mi
  597. }
  598. func (ctx *parseContext) handleSubConfig(
  599. v mkparser.Node, pathExpr starlarkExpr, loadAlways bool, processModule func(inheritedModule)) {
  600. pathExpr, _ = pathExpr.eval(ctx.builtinMakeVars)
  601. // In a simple case, the name of a module to inherit/include is known statically.
  602. if path, ok := maybeString(pathExpr); ok {
  603. if strings.Contains(path, "*") {
  604. if paths, err := fs.Glob(ctx.script.sourceFS, path); err == nil {
  605. for _, p := range paths {
  606. processModule(inheritedStaticModule{ctx.newDependentModule(p, !loadAlways), loadAlways})
  607. }
  608. } else {
  609. ctx.errorf(v, "cannot glob wildcard argument")
  610. }
  611. } else {
  612. processModule(inheritedStaticModule{ctx.newDependentModule(path, !loadAlways), loadAlways})
  613. }
  614. return
  615. }
  616. // If module path references variables (e.g., $(v1)/foo/$(v2)/device-config.mk), find all the paths in the
  617. // source tree that may be a match and the corresponding variable values. For instance, if the source tree
  618. // contains vendor1/foo/abc/dev.mk and vendor2/foo/def/dev.mk, the first one will be inherited when
  619. // (v1, v2) == ('vendor1', 'abc'), and the second one when (v1, v2) == ('vendor2', 'def').
  620. // We then emit the code that loads all of them, e.g.:
  621. // load("//vendor1/foo/abc:dev.rbc", _dev1_init="init")
  622. // load("//vendor2/foo/def/dev.rbc", _dev2_init="init")
  623. // And then inherit it as follows:
  624. // _e = {
  625. // "vendor1/foo/abc/dev.mk": ("vendor1/foo/abc/dev", _dev1_init),
  626. // "vendor2/foo/def/dev.mk": ("vendor2/foo/def/dev", _dev_init2) }.get("%s/foo/%s/dev.mk" % (v1, v2))
  627. // if _e:
  628. // rblf.inherit(handle, _e[0], _e[1])
  629. //
  630. var matchingPaths []string
  631. varPath, ok := pathExpr.(*interpolateExpr)
  632. if !ok {
  633. ctx.errorf(v, "inherit-product/include argument is too complex")
  634. return
  635. }
  636. pathPattern := []string{varPath.chunks[0]}
  637. for _, chunk := range varPath.chunks[1:] {
  638. if chunk != "" {
  639. pathPattern = append(pathPattern, chunk)
  640. }
  641. }
  642. if pathPattern[0] != "" {
  643. matchingPaths = ctx.findMatchingPaths(pathPattern)
  644. } else {
  645. // Heuristics -- if pattern starts from top, restrict it to the directories where
  646. // we know inherit-product uses dynamically calculated path.
  647. for _, t := range []string{"vendor/qcom", "vendor/google_devices"} {
  648. pathPattern[0] = t
  649. matchingPaths = append(matchingPaths, ctx.findMatchingPaths(pathPattern)...)
  650. }
  651. }
  652. // Safeguard against $(call inherit-product,$(PRODUCT_PATH))
  653. const maxMatchingFiles = 100
  654. if len(matchingPaths) > maxMatchingFiles {
  655. ctx.errorf(v, "there are >%d files matching the pattern, please rewrite it", maxMatchingFiles)
  656. return
  657. }
  658. res := inheritedDynamicModule{*varPath, []*moduleInfo{}, loadAlways}
  659. for _, p := range matchingPaths {
  660. // A product configuration files discovered dynamically may attempt to inherit
  661. // from another one which does not exist in this source tree. Prevent load errors
  662. // by always loading the dynamic files as optional.
  663. res.candidateModules = append(res.candidateModules, ctx.newDependentModule(p, true))
  664. }
  665. processModule(res)
  666. }
  667. func (ctx *parseContext) findMatchingPaths(pattern []string) []string {
  668. files := ctx.script.makefileFinder.Find(ctx.script.topDir)
  669. if len(pattern) == 0 {
  670. return files
  671. }
  672. // Create regular expression from the pattern
  673. s_regexp := "^" + regexp.QuoteMeta(pattern[0])
  674. for _, s := range pattern[1:] {
  675. s_regexp += ".*" + regexp.QuoteMeta(s)
  676. }
  677. s_regexp += "$"
  678. rex := regexp.MustCompile(s_regexp)
  679. // Now match
  680. var res []string
  681. for _, p := range files {
  682. if rex.MatchString(p) {
  683. res = append(res, p)
  684. }
  685. }
  686. return res
  687. }
  688. func (ctx *parseContext) handleInheritModule(v mkparser.Node, pathExpr starlarkExpr, loadAlways bool) {
  689. ctx.handleSubConfig(v, pathExpr, loadAlways, func(im inheritedModule) {
  690. ctx.receiver.newNode(&inheritNode{im})
  691. })
  692. }
  693. func (ctx *parseContext) handleInclude(v mkparser.Node, pathExpr starlarkExpr, loadAlways bool) {
  694. ctx.handleSubConfig(v, pathExpr, loadAlways, func(im inheritedModule) {
  695. ctx.receiver.newNode(&includeNode{im})
  696. })
  697. }
  698. func (ctx *parseContext) handleVariable(v *mkparser.Variable) {
  699. // Handle:
  700. // $(call inherit-product,...)
  701. // $(call inherit-product-if-exists,...)
  702. // $(info xxx)
  703. // $(warning xxx)
  704. // $(error xxx)
  705. expr := ctx.parseReference(v, v.Name)
  706. switch x := expr.(type) {
  707. case *callExpr:
  708. if x.name == callLoadAlways || x.name == callLoadIf {
  709. ctx.handleInheritModule(v, x.args[0], x.name == callLoadAlways)
  710. } else if isMakeControlFunc(x.name) {
  711. // File name is the first argument
  712. args := []starlarkExpr{
  713. &stringLiteralExpr{ctx.script.mkFile},
  714. x.args[0],
  715. }
  716. ctx.receiver.newNode(&exprNode{
  717. &callExpr{name: x.name, args: args, returnType: starlarkTypeUnknown},
  718. })
  719. } else {
  720. ctx.receiver.newNode(&exprNode{expr})
  721. }
  722. case *badExpr:
  723. ctx.wrapBadExpr(x)
  724. return
  725. default:
  726. ctx.errorf(v, "cannot handle %s", v.Dump())
  727. return
  728. }
  729. }
  730. func (ctx *parseContext) handleDefine(directive *mkparser.Directive) {
  731. macro_name := strings.Fields(directive.Args.Strings[0])[0]
  732. // Ignore the macros that we handle
  733. if _, ok := knownFunctions[macro_name]; !ok {
  734. ctx.errorf(directive, "define is not supported: %s", macro_name)
  735. }
  736. }
  737. func (ctx *parseContext) handleIfBlock(ifDirective *mkparser.Directive) {
  738. ssSwitch := &switchNode{}
  739. ctx.pushReceiver(ssSwitch)
  740. for ctx.processBranch(ifDirective); ctx.hasNodes() && ctx.fatalError == nil; {
  741. node := ctx.getNode()
  742. switch x := node.(type) {
  743. case *mkparser.Directive:
  744. switch x.Name {
  745. case "else", "elifdef", "elifndef", "elifeq", "elifneq":
  746. ctx.processBranch(x)
  747. case "endif":
  748. ctx.popReceiver()
  749. ctx.receiver.newNode(ssSwitch)
  750. return
  751. default:
  752. ctx.errorf(node, "unexpected directive %s", x.Name)
  753. }
  754. default:
  755. ctx.errorf(ifDirective, "unexpected statement")
  756. }
  757. }
  758. if ctx.fatalError == nil {
  759. ctx.fatalError = fmt.Errorf("no matching endif for %s", ifDirective.Dump())
  760. }
  761. ctx.popReceiver()
  762. }
  763. // processBranch processes a single branch (if/elseif/else) until the next directive
  764. // on the same level.
  765. func (ctx *parseContext) processBranch(check *mkparser.Directive) {
  766. block := switchCase{gate: ctx.parseCondition(check)}
  767. defer func() {
  768. ctx.popVarAssignments()
  769. ctx.ifNestLevel--
  770. }()
  771. ctx.pushVarAssignments()
  772. ctx.ifNestLevel++
  773. ctx.pushReceiver(&block)
  774. for ctx.hasNodes() {
  775. node := ctx.getNode()
  776. if ctx.handleSimpleStatement(node) {
  777. continue
  778. }
  779. switch d := node.(type) {
  780. case *mkparser.Directive:
  781. switch d.Name {
  782. case "else", "elifdef", "elifndef", "elifeq", "elifneq", "endif":
  783. ctx.popReceiver()
  784. ctx.receiver.newNode(&block)
  785. ctx.backNode()
  786. return
  787. case "ifdef", "ifndef", "ifeq", "ifneq":
  788. ctx.handleIfBlock(d)
  789. default:
  790. ctx.errorf(d, "unexpected directive %s", d.Name)
  791. }
  792. default:
  793. ctx.errorf(node, "unexpected statement")
  794. }
  795. }
  796. ctx.fatalError = fmt.Errorf("no matching endif for %s", check.Dump())
  797. ctx.popReceiver()
  798. }
  799. func (ctx *parseContext) newIfDefinedNode(check *mkparser.Directive) (starlarkExpr, bool) {
  800. if !check.Args.Const() {
  801. return ctx.newBadExpr(check, "ifdef variable ref too complex: %s", check.Args.Dump()), false
  802. }
  803. v := ctx.addVariable(check.Args.Strings[0])
  804. return &variableDefinedExpr{v}, true
  805. }
  806. func (ctx *parseContext) parseCondition(check *mkparser.Directive) starlarkNode {
  807. switch check.Name {
  808. case "ifdef", "ifndef", "elifdef", "elifndef":
  809. v, ok := ctx.newIfDefinedNode(check)
  810. if ok && strings.HasSuffix(check.Name, "ndef") {
  811. v = &notExpr{v}
  812. }
  813. return &ifNode{
  814. isElif: strings.HasPrefix(check.Name, "elif"),
  815. expr: v,
  816. }
  817. case "ifeq", "ifneq", "elifeq", "elifneq":
  818. return &ifNode{
  819. isElif: strings.HasPrefix(check.Name, "elif"),
  820. expr: ctx.parseCompare(check),
  821. }
  822. case "else":
  823. return &elseNode{}
  824. default:
  825. panic(fmt.Errorf("%s: unknown directive: %s", ctx.script.mkFile, check.Dump()))
  826. }
  827. }
  828. func (ctx *parseContext) newBadExpr(node mkparser.Node, text string, args ...interface{}) starlarkExpr {
  829. message := fmt.Sprintf(text, args...)
  830. if ctx.errorLogger != nil {
  831. ctx.errorLogger.NewError(text, node, args)
  832. }
  833. ctx.script.hasErrors = true
  834. return &badExpr{node, message}
  835. }
  836. func (ctx *parseContext) parseCompare(cond *mkparser.Directive) starlarkExpr {
  837. // Strip outer parentheses
  838. mkArg := cloneMakeString(cond.Args)
  839. mkArg.Strings[0] = strings.TrimLeft(mkArg.Strings[0], "( ")
  840. n := len(mkArg.Strings)
  841. mkArg.Strings[n-1] = strings.TrimRight(mkArg.Strings[n-1], ") ")
  842. args := mkArg.Split(",")
  843. // TODO(asmundak): handle the case where the arguments are in quotes and space-separated
  844. if len(args) != 2 {
  845. return ctx.newBadExpr(cond, "ifeq/ifneq len(args) != 2 %s", cond.Dump())
  846. }
  847. args[0].TrimRightSpaces()
  848. args[1].TrimLeftSpaces()
  849. isEq := !strings.HasSuffix(cond.Name, "neq")
  850. switch xLeft := ctx.parseMakeString(cond, args[0]).(type) {
  851. case *stringLiteralExpr, *variableRefExpr:
  852. switch xRight := ctx.parseMakeString(cond, args[1]).(type) {
  853. case *stringLiteralExpr, *variableRefExpr:
  854. return &eqExpr{left: xLeft, right: xRight, isEq: isEq}
  855. case *badExpr:
  856. return xRight
  857. default:
  858. expr, ok := ctx.parseCheckFunctionCallResult(cond, xLeft, args[1])
  859. if ok {
  860. return expr
  861. }
  862. return ctx.newBadExpr(cond, "right operand is too complex: %s", args[1].Dump())
  863. }
  864. case *badExpr:
  865. return xLeft
  866. default:
  867. switch xRight := ctx.parseMakeString(cond, args[1]).(type) {
  868. case *stringLiteralExpr, *variableRefExpr:
  869. expr, ok := ctx.parseCheckFunctionCallResult(cond, xRight, args[0])
  870. if ok {
  871. return expr
  872. }
  873. return ctx.newBadExpr(cond, "left operand is too complex: %s", args[0].Dump())
  874. case *badExpr:
  875. return xRight
  876. default:
  877. return ctx.newBadExpr(cond, "operands are too complex: (%s,%s)", args[0].Dump(), args[1].Dump())
  878. }
  879. }
  880. }
  881. func (ctx *parseContext) parseCheckFunctionCallResult(directive *mkparser.Directive, xValue starlarkExpr,
  882. varArg *mkparser.MakeString) (starlarkExpr, bool) {
  883. mkSingleVar, ok := varArg.SingleVariable()
  884. if !ok {
  885. return nil, false
  886. }
  887. expr := ctx.parseReference(directive, mkSingleVar)
  888. negate := strings.HasSuffix(directive.Name, "neq")
  889. checkIsSomethingFunction := func(xCall *callExpr) starlarkExpr {
  890. s, ok := maybeString(xValue)
  891. if !ok || s != "true" {
  892. return ctx.newBadExpr(directive,
  893. fmt.Sprintf("the result of %s can be compared only to 'true'", xCall.name))
  894. }
  895. if len(xCall.args) < 1 {
  896. return ctx.newBadExpr(directive, "%s requires an argument", xCall.name)
  897. }
  898. return nil
  899. }
  900. switch x := expr.(type) {
  901. case *callExpr:
  902. switch x.name {
  903. case "filter":
  904. return ctx.parseCompareFilterFuncResult(directive, x, xValue, !negate), true
  905. case "filter-out":
  906. return ctx.parseCompareFilterFuncResult(directive, x, xValue, negate), true
  907. case "wildcard":
  908. return ctx.parseCompareWildcardFuncResult(directive, x, xValue, negate), true
  909. case "findstring":
  910. return ctx.parseCheckFindstringFuncResult(directive, x, xValue, negate), true
  911. case "strip":
  912. return ctx.parseCompareStripFuncResult(directive, x, xValue, negate), true
  913. case "is-board-platform":
  914. if xBad := checkIsSomethingFunction(x); xBad != nil {
  915. return xBad, true
  916. }
  917. return &eqExpr{
  918. left: &variableRefExpr{ctx.addVariable("TARGET_BOARD_PLATFORM"), false},
  919. right: x.args[0],
  920. isEq: !negate,
  921. }, true
  922. case "is-board-platform-in-list":
  923. if xBad := checkIsSomethingFunction(x); xBad != nil {
  924. return xBad, true
  925. }
  926. return &inExpr{
  927. expr: &variableRefExpr{ctx.addVariable("TARGET_BOARD_PLATFORM"), false},
  928. list: maybeConvertToStringList(x.args[0]),
  929. isNot: negate,
  930. }, true
  931. case "is-product-in-list":
  932. if xBad := checkIsSomethingFunction(x); xBad != nil {
  933. return xBad, true
  934. }
  935. return &inExpr{
  936. expr: &variableRefExpr{ctx.addVariable("TARGET_PRODUCT"), true},
  937. list: maybeConvertToStringList(x.args[0]),
  938. isNot: negate,
  939. }, true
  940. case "is-vendor-board-platform":
  941. if xBad := checkIsSomethingFunction(x); xBad != nil {
  942. return xBad, true
  943. }
  944. s, ok := maybeString(x.args[0])
  945. if !ok {
  946. return ctx.newBadExpr(directive, "cannot handle non-constant argument to is-vendor-board-platform"), true
  947. }
  948. return &inExpr{
  949. expr: &variableRefExpr{ctx.addVariable("TARGET_BOARD_PLATFORM"), false},
  950. list: &variableRefExpr{ctx.addVariable(s + "_BOARD_PLATFORMS"), true},
  951. isNot: negate,
  952. }, true
  953. default:
  954. return ctx.newBadExpr(directive, "Unknown function in ifeq: %s", x.name), true
  955. }
  956. case *badExpr:
  957. return x, true
  958. default:
  959. return nil, false
  960. }
  961. }
  962. func (ctx *parseContext) parseCompareFilterFuncResult(cond *mkparser.Directive,
  963. filterFuncCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr {
  964. // We handle:
  965. // * ifeq/ifneq (,$(filter v1 v2 ..., EXPR) becomes if EXPR not in/in ["v1", "v2", ...]
  966. // * ifeq/ifneq (,$(filter EXPR, v1 v2 ...) becomes if EXPR not in/in ["v1", "v2", ...]
  967. // * ifeq/ifneq ($(VAR),$(filter $(VAR), v1 v2 ...) becomes if VAR in/not in ["v1", "v2"]
  968. // TODO(Asmundak): check the last case works for filter-out, too.
  969. xPattern := filterFuncCall.args[0]
  970. xText := filterFuncCall.args[1]
  971. var xInList *stringLiteralExpr
  972. var expr starlarkExpr
  973. var ok bool
  974. switch x := xValue.(type) {
  975. case *stringLiteralExpr:
  976. if x.literal != "" {
  977. return ctx.newBadExpr(cond, "filter comparison to non-empty value: %s", xValue)
  978. }
  979. // Either pattern or text should be const, and the
  980. // non-const one should be varRefExpr
  981. if xInList, ok = xPattern.(*stringLiteralExpr); ok {
  982. expr = xText
  983. } else if xInList, ok = xText.(*stringLiteralExpr); ok {
  984. expr = xPattern
  985. } else {
  986. return &callExpr{
  987. object: nil,
  988. name: filterFuncCall.name,
  989. args: filterFuncCall.args,
  990. returnType: starlarkTypeBool,
  991. }
  992. }
  993. case *variableRefExpr:
  994. if v, ok := xPattern.(*variableRefExpr); ok {
  995. if xInList, ok = xText.(*stringLiteralExpr); ok && v.ref.name() == x.ref.name() {
  996. // ifeq/ifneq ($(VAR),$(filter $(VAR), v1 v2 ...), flip negate,
  997. // it's the opposite to what is done when comparing to empty.
  998. expr = xPattern
  999. negate = !negate
  1000. }
  1001. }
  1002. }
  1003. if expr != nil && xInList != nil {
  1004. slExpr := newStringListExpr(strings.Fields(xInList.literal))
  1005. // Generate simpler code for the common cases:
  1006. if expr.typ() == starlarkTypeList {
  1007. if len(slExpr.items) == 1 {
  1008. // Checking that a string belongs to list
  1009. return &inExpr{isNot: negate, list: expr, expr: slExpr.items[0]}
  1010. } else {
  1011. // TODO(asmundak):
  1012. panic("TBD")
  1013. }
  1014. } else if len(slExpr.items) == 1 {
  1015. return &eqExpr{left: expr, right: slExpr.items[0], isEq: !negate}
  1016. } else {
  1017. return &inExpr{isNot: negate, list: newStringListExpr(strings.Fields(xInList.literal)), expr: expr}
  1018. }
  1019. }
  1020. return ctx.newBadExpr(cond, "filter arguments are too complex: %s", cond.Dump())
  1021. }
  1022. func (ctx *parseContext) parseCompareWildcardFuncResult(directive *mkparser.Directive,
  1023. xCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr {
  1024. if !isEmptyString(xValue) {
  1025. return ctx.newBadExpr(directive, "wildcard result can be compared only to empty: %s", xValue)
  1026. }
  1027. callFunc := wildcardExistsPhony
  1028. if s, ok := xCall.args[0].(*stringLiteralExpr); ok && !strings.ContainsAny(s.literal, "*?{[") {
  1029. callFunc = fileExistsPhony
  1030. }
  1031. var cc starlarkExpr = &callExpr{name: callFunc, args: xCall.args, returnType: starlarkTypeBool}
  1032. if !negate {
  1033. cc = &notExpr{cc}
  1034. }
  1035. return cc
  1036. }
  1037. func (ctx *parseContext) parseCheckFindstringFuncResult(directive *mkparser.Directive,
  1038. xCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr {
  1039. if isEmptyString(xValue) {
  1040. return &eqExpr{
  1041. left: &callExpr{
  1042. object: xCall.args[1],
  1043. name: "find",
  1044. args: []starlarkExpr{xCall.args[0]},
  1045. returnType: starlarkTypeInt,
  1046. },
  1047. right: &intLiteralExpr{-1},
  1048. isEq: !negate,
  1049. }
  1050. }
  1051. return ctx.newBadExpr(directive, "findstring result can be compared only to empty: %s", xValue)
  1052. }
  1053. func (ctx *parseContext) parseCompareStripFuncResult(directive *mkparser.Directive,
  1054. xCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr {
  1055. if _, ok := xValue.(*stringLiteralExpr); !ok {
  1056. return ctx.newBadExpr(directive, "strip result can be compared only to string: %s", xValue)
  1057. }
  1058. return &eqExpr{
  1059. left: &callExpr{
  1060. name: "strip",
  1061. args: xCall.args,
  1062. returnType: starlarkTypeString,
  1063. },
  1064. right: xValue, isEq: !negate}
  1065. }
  1066. // parses $(...), returning an expression
  1067. func (ctx *parseContext) parseReference(node mkparser.Node, ref *mkparser.MakeString) starlarkExpr {
  1068. ref.TrimLeftSpaces()
  1069. ref.TrimRightSpaces()
  1070. refDump := ref.Dump()
  1071. // Handle only the case where the first (or only) word is constant
  1072. words := ref.SplitN(" ", 2)
  1073. if !words[0].Const() {
  1074. return ctx.newBadExpr(node, "reference is too complex: %s", refDump)
  1075. }
  1076. // If it is a single word, it can be a simple variable
  1077. // reference or a function call
  1078. if len(words) == 1 {
  1079. if isMakeControlFunc(refDump) || refDump == "shell" {
  1080. return &callExpr{
  1081. name: refDump,
  1082. args: []starlarkExpr{&stringLiteralExpr{""}},
  1083. returnType: starlarkTypeUnknown,
  1084. }
  1085. }
  1086. if v := ctx.addVariable(refDump); v != nil {
  1087. return &variableRefExpr{v, ctx.lastAssignment(v.name()) != nil}
  1088. }
  1089. return ctx.newBadExpr(node, "unknown variable %s", refDump)
  1090. }
  1091. expr := &callExpr{name: words[0].Dump(), returnType: starlarkTypeUnknown}
  1092. args := words[1]
  1093. args.TrimLeftSpaces()
  1094. // Make control functions and shell need special treatment as everything
  1095. // after the name is a single text argument
  1096. if isMakeControlFunc(expr.name) || expr.name == "shell" {
  1097. x := ctx.parseMakeString(node, args)
  1098. if xBad, ok := x.(*badExpr); ok {
  1099. return xBad
  1100. }
  1101. expr.args = []starlarkExpr{x}
  1102. return expr
  1103. }
  1104. if expr.name == "call" {
  1105. words = args.SplitN(",", 2)
  1106. if words[0].Empty() || !words[0].Const() {
  1107. return ctx.newBadExpr(node, "cannot handle %s", refDump)
  1108. }
  1109. expr.name = words[0].Dump()
  1110. if len(words) < 2 {
  1111. args = &mkparser.MakeString{}
  1112. } else {
  1113. args = words[1]
  1114. }
  1115. }
  1116. if kf, found := knownFunctions[expr.name]; found {
  1117. expr.returnType = kf.returnType
  1118. } else {
  1119. return ctx.newBadExpr(node, "cannot handle invoking %s", expr.name)
  1120. }
  1121. switch expr.name {
  1122. case "word":
  1123. return ctx.parseWordFunc(node, args)
  1124. case "my-dir":
  1125. return &variableRefExpr{ctx.addVariable("LOCAL_PATH"), true}
  1126. case "subst", "patsubst":
  1127. return ctx.parseSubstFunc(node, expr.name, args)
  1128. default:
  1129. for _, arg := range args.Split(",") {
  1130. arg.TrimLeftSpaces()
  1131. arg.TrimRightSpaces()
  1132. x := ctx.parseMakeString(node, arg)
  1133. if xBad, ok := x.(*badExpr); ok {
  1134. return xBad
  1135. }
  1136. expr.args = append(expr.args, x)
  1137. }
  1138. }
  1139. return expr
  1140. }
  1141. func (ctx *parseContext) parseSubstFunc(node mkparser.Node, fname string, args *mkparser.MakeString) starlarkExpr {
  1142. words := args.Split(",")
  1143. if len(words) != 3 {
  1144. return ctx.newBadExpr(node, "%s function should have 3 arguments", fname)
  1145. }
  1146. if !words[0].Const() || !words[1].Const() {
  1147. return ctx.newBadExpr(node, "%s function's from and to arguments should be constant", fname)
  1148. }
  1149. from := words[0].Strings[0]
  1150. to := words[1].Strings[0]
  1151. words[2].TrimLeftSpaces()
  1152. words[2].TrimRightSpaces()
  1153. obj := ctx.parseMakeString(node, words[2])
  1154. typ := obj.typ()
  1155. if typ == starlarkTypeString && fname == "subst" {
  1156. // Optimization: if it's $(subst from, to, string), emit string.replace(from, to)
  1157. return &callExpr{
  1158. object: obj,
  1159. name: "replace",
  1160. args: []starlarkExpr{&stringLiteralExpr{from}, &stringLiteralExpr{to}},
  1161. returnType: typ,
  1162. }
  1163. }
  1164. return &callExpr{
  1165. name: fname,
  1166. args: []starlarkExpr{&stringLiteralExpr{from}, &stringLiteralExpr{to}, obj},
  1167. returnType: obj.typ(),
  1168. }
  1169. }
  1170. func (ctx *parseContext) parseWordFunc(node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
  1171. words := args.Split(",")
  1172. if len(words) != 2 {
  1173. return ctx.newBadExpr(node, "word function should have 2 arguments")
  1174. }
  1175. var index uint64 = 0
  1176. if words[0].Const() {
  1177. index, _ = strconv.ParseUint(strings.TrimSpace(words[0].Strings[0]), 10, 64)
  1178. }
  1179. if index < 1 {
  1180. return ctx.newBadExpr(node, "word index should be constant positive integer")
  1181. }
  1182. words[1].TrimLeftSpaces()
  1183. words[1].TrimRightSpaces()
  1184. array := ctx.parseMakeString(node, words[1])
  1185. if xBad, ok := array.(*badExpr); ok {
  1186. return xBad
  1187. }
  1188. if array.typ() != starlarkTypeList {
  1189. array = &callExpr{object: array, name: "split", returnType: starlarkTypeList}
  1190. }
  1191. return indexExpr{array, &intLiteralExpr{int(index - 1)}}
  1192. }
  1193. func (ctx *parseContext) parseMakeString(node mkparser.Node, mk *mkparser.MakeString) starlarkExpr {
  1194. if mk.Const() {
  1195. return &stringLiteralExpr{mk.Dump()}
  1196. }
  1197. if mkRef, ok := mk.SingleVariable(); ok {
  1198. return ctx.parseReference(node, mkRef)
  1199. }
  1200. // If we reached here, it's neither string literal nor a simple variable,
  1201. // we need a full-blown interpolation node that will generate
  1202. // "a%b%c" % (X, Y) for a$(X)b$(Y)c
  1203. xInterp := &interpolateExpr{args: make([]starlarkExpr, len(mk.Variables))}
  1204. for i, ref := range mk.Variables {
  1205. arg := ctx.parseReference(node, ref.Name)
  1206. if x, ok := arg.(*badExpr); ok {
  1207. return x
  1208. }
  1209. xInterp.args[i] = arg
  1210. }
  1211. xInterp.chunks = append(xInterp.chunks, mk.Strings...)
  1212. return xInterp
  1213. }
  1214. // Handles the statements whose treatment is the same in all contexts: comment,
  1215. // assignment, variable (which is a macro call in reality) and all constructs that
  1216. // do not handle in any context ('define directive and any unrecognized stuff).
  1217. // Return true if we handled it.
  1218. func (ctx *parseContext) handleSimpleStatement(node mkparser.Node) bool {
  1219. handled := true
  1220. switch x := node.(type) {
  1221. case *mkparser.Comment:
  1222. ctx.insertComment("#" + x.Comment)
  1223. case *mkparser.Assignment:
  1224. ctx.handleAssignment(x)
  1225. case *mkparser.Variable:
  1226. ctx.handleVariable(x)
  1227. case *mkparser.Directive:
  1228. switch x.Name {
  1229. case "define":
  1230. ctx.handleDefine(x)
  1231. case "include", "-include":
  1232. ctx.handleInclude(node, ctx.parseMakeString(node, x.Args), x.Name[0] != '-')
  1233. default:
  1234. handled = false
  1235. }
  1236. default:
  1237. ctx.errorf(x, "unsupported line %s", x.Dump())
  1238. }
  1239. return handled
  1240. }
  1241. func (ctx *parseContext) insertComment(s string) {
  1242. ctx.receiver.newNode(&commentNode{strings.TrimSpace(s)})
  1243. }
  1244. func (ctx *parseContext) carryAsComment(failedNode mkparser.Node) {
  1245. for _, line := range strings.Split(failedNode.Dump(), "\n") {
  1246. ctx.insertComment("# " + line)
  1247. }
  1248. }
  1249. // records that the given node failed to be converted and includes an explanatory message
  1250. func (ctx *parseContext) errorf(failedNode mkparser.Node, message string, args ...interface{}) {
  1251. if ctx.errorLogger != nil {
  1252. ctx.errorLogger.NewError(message, failedNode, args...)
  1253. }
  1254. message = fmt.Sprintf(message, args...)
  1255. ctx.insertComment(fmt.Sprintf("# MK2RBC TRANSLATION ERROR: %s", message))
  1256. ctx.carryAsComment(failedNode)
  1257. ctx.script.hasErrors = true
  1258. }
  1259. func (ctx *parseContext) wrapBadExpr(xBad *badExpr) {
  1260. ctx.insertComment(fmt.Sprintf("# MK2RBC TRANSLATION ERROR: %s", xBad.message))
  1261. ctx.carryAsComment(xBad.node)
  1262. }
  1263. func (ctx *parseContext) loadedModulePath(path string) string {
  1264. // During the transition to Roboleaf some of the product configuration files
  1265. // will be converted and checked in while the others will be generated on the fly
  1266. // and run. The runner (rbcrun application) accommodates this by allowing three
  1267. // different ways to specify the loaded file location:
  1268. // 1) load(":<file>",...) loads <file> from the same directory
  1269. // 2) load("//path/relative/to/source/root:<file>", ...) loads <file> source tree
  1270. // 3) load("/absolute/path/to/<file> absolute path
  1271. // If the file being generated and the file it wants to load are in the same directory,
  1272. // generate option 1.
  1273. // Otherwise, if output directory is not specified, generate 2)
  1274. // Finally, if output directory has been specified and the file being generated and
  1275. // the file it wants to load from are in the different directories, generate 2) or 3):
  1276. // * if the file being loaded exists in the source tree, generate 2)
  1277. // * otherwise, generate 3)
  1278. // Finally, figure out the loaded module path and name and create a node for it
  1279. loadedModuleDir := filepath.Dir(path)
  1280. base := filepath.Base(path)
  1281. loadedModuleName := strings.TrimSuffix(base, filepath.Ext(base)) + ctx.outputSuffix
  1282. if loadedModuleDir == filepath.Dir(ctx.script.mkFile) {
  1283. return ":" + loadedModuleName
  1284. }
  1285. if ctx.outputDir == "" {
  1286. return fmt.Sprintf("//%s:%s", loadedModuleDir, loadedModuleName)
  1287. }
  1288. if _, err := os.Stat(filepath.Join(loadedModuleDir, loadedModuleName)); err == nil {
  1289. return fmt.Sprintf("//%s:%s", loadedModuleDir, loadedModuleName)
  1290. }
  1291. return filepath.Join(ctx.outputDir, loadedModuleDir, loadedModuleName)
  1292. }
  1293. func (ss *StarlarkScript) String() string {
  1294. return NewGenerateContext(ss).emit()
  1295. }
  1296. func (ss *StarlarkScript) SubConfigFiles() []string {
  1297. var subs []string
  1298. for _, src := range ss.inherited {
  1299. subs = append(subs, src.originalPath)
  1300. }
  1301. return subs
  1302. }
  1303. func (ss *StarlarkScript) HasErrors() bool {
  1304. return ss.hasErrors
  1305. }
  1306. // Convert reads and parses a makefile. If successful, parsed tree
  1307. // is returned and then can be passed to String() to get the generated
  1308. // Starlark file.
  1309. func Convert(req Request) (*StarlarkScript, error) {
  1310. reader := req.Reader
  1311. if reader == nil {
  1312. mkContents, err := ioutil.ReadFile(req.MkFile)
  1313. if err != nil {
  1314. return nil, err
  1315. }
  1316. reader = bytes.NewBuffer(mkContents)
  1317. }
  1318. parser := mkparser.NewParser(req.MkFile, reader)
  1319. nodes, errs := parser.Parse()
  1320. if len(errs) > 0 {
  1321. for _, e := range errs {
  1322. fmt.Fprintln(os.Stderr, "ERROR:", e)
  1323. }
  1324. return nil, fmt.Errorf("bad makefile %s", req.MkFile)
  1325. }
  1326. starScript := &StarlarkScript{
  1327. moduleName: moduleNameForFile(req.MkFile),
  1328. mkFile: req.MkFile,
  1329. topDir: req.RootDir,
  1330. traceCalls: req.TraceCalls,
  1331. warnPartialSuccess: req.WarnPartialSuccess,
  1332. sourceFS: req.SourceFS,
  1333. makefileFinder: req.MakefileFinder,
  1334. }
  1335. ctx := newParseContext(starScript, nodes)
  1336. ctx.outputSuffix = req.OutputSuffix
  1337. ctx.outputDir = req.OutputDir
  1338. ctx.errorLogger = req.ErrorLogger
  1339. if len(req.TracedVariables) > 0 {
  1340. ctx.tracedVariables = make(map[string]bool)
  1341. for _, v := range req.TracedVariables {
  1342. ctx.tracedVariables[v] = true
  1343. }
  1344. }
  1345. ctx.pushReceiver(starScript)
  1346. for ctx.hasNodes() && ctx.fatalError == nil {
  1347. node := ctx.getNode()
  1348. if ctx.handleSimpleStatement(node) {
  1349. continue
  1350. }
  1351. switch x := node.(type) {
  1352. case *mkparser.Directive:
  1353. switch x.Name {
  1354. case "ifeq", "ifneq", "ifdef", "ifndef":
  1355. ctx.handleIfBlock(x)
  1356. default:
  1357. ctx.errorf(x, "unexpected directive %s", x.Name)
  1358. }
  1359. default:
  1360. ctx.errorf(x, "unsupported line")
  1361. }
  1362. }
  1363. if ctx.fatalError != nil {
  1364. return nil, ctx.fatalError
  1365. }
  1366. return starScript, nil
  1367. }
  1368. func Launcher(path, name string) string {
  1369. var buf bytes.Buffer
  1370. fmt.Fprintf(&buf, "load(%q, %q)\n", baseUri, baseName)
  1371. fmt.Fprintf(&buf, "load(%q, \"init\")\n", path)
  1372. fmt.Fprintf(&buf, "g, config = %s(%q, init)\n", cfnMain, name)
  1373. fmt.Fprintf(&buf, "%s(g, config)\n", cfnPrintVars)
  1374. return buf.String()
  1375. }
  1376. func MakePath2ModuleName(mkPath string) string {
  1377. return strings.TrimSuffix(mkPath, filepath.Ext(mkPath))
  1378. }