config_bp2build.go 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517
  1. // Copyright 2021 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package android
  15. import (
  16. "fmt"
  17. "reflect"
  18. "regexp"
  19. "sort"
  20. "strings"
  21. "android/soong/bazel"
  22. "android/soong/starlark_fmt"
  23. "github.com/google/blueprint"
  24. )
  25. // BazelVarExporter is a collection of configuration variables that can be exported for use in Bazel rules
  26. type BazelVarExporter interface {
  27. // asBazel expands strings of configuration variables into their concrete values
  28. asBazel(Config, ExportedStringVariables, ExportedStringListVariables, ExportedConfigDependingVariables) []bazelConstant
  29. }
  30. // ExportedVariables is a collection of interdependent configuration variables
  31. type ExportedVariables struct {
  32. // Maps containing toolchain variables that are independent of the
  33. // environment variables of the build.
  34. exportedStringVars ExportedStringVariables
  35. exportedStringListVars ExportedStringListVariables
  36. exportedStringListDictVars ExportedStringListDictVariables
  37. exportedVariableReferenceDictVars ExportedVariableReferenceDictVariables
  38. /// Maps containing variables that are dependent on the build config.
  39. exportedConfigDependingVars ExportedConfigDependingVariables
  40. pctx PackageContext
  41. }
  42. // NewExportedVariables creats an empty ExportedVariables struct with non-nil maps
  43. func NewExportedVariables(pctx PackageContext) ExportedVariables {
  44. return ExportedVariables{
  45. exportedStringVars: ExportedStringVariables{},
  46. exportedStringListVars: ExportedStringListVariables{},
  47. exportedStringListDictVars: ExportedStringListDictVariables{},
  48. exportedVariableReferenceDictVars: ExportedVariableReferenceDictVariables{},
  49. exportedConfigDependingVars: ExportedConfigDependingVariables{},
  50. pctx: pctx,
  51. }
  52. }
  53. func (ev ExportedVariables) asBazel(config Config,
  54. stringVars ExportedStringVariables, stringListVars ExportedStringListVariables, cfgDepVars ExportedConfigDependingVariables) []bazelConstant {
  55. ret := []bazelConstant{}
  56. ret = append(ret, ev.exportedStringVars.asBazel(config, stringVars, stringListVars, cfgDepVars)...)
  57. ret = append(ret, ev.exportedStringListVars.asBazel(config, stringVars, stringListVars, cfgDepVars)...)
  58. ret = append(ret, ev.exportedStringListDictVars.asBazel(config, stringVars, stringListVars, cfgDepVars)...)
  59. // Note: ExportedVariableReferenceDictVars collections can only contain references to other variables and must be printed last
  60. ret = append(ret, ev.exportedVariableReferenceDictVars.asBazel(config, stringVars, stringListVars, cfgDepVars)...)
  61. ret = append(ret, ev.exportedConfigDependingVars.asBazel(config, stringVars, stringListVars, cfgDepVars)...)
  62. return ret
  63. }
  64. // ExportStringStaticVariable declares a static string variable and exports it to
  65. // Bazel's toolchain.
  66. func (ev ExportedVariables) ExportStringStaticVariable(name string, value string) {
  67. ev.pctx.StaticVariable(name, value)
  68. ev.exportedStringVars.set(name, value)
  69. }
  70. // ExportStringListStaticVariable declares a static variable and exports it to
  71. // Bazel's toolchain.
  72. func (ev ExportedVariables) ExportStringListStaticVariable(name string, value []string) {
  73. ev.pctx.StaticVariable(name, strings.Join(value, " "))
  74. ev.exportedStringListVars.set(name, value)
  75. }
  76. // ExportVariableConfigMethod declares a variable whose value is evaluated at
  77. // runtime via a function with access to the Config and exports it to Bazel's
  78. // toolchain.
  79. func (ev ExportedVariables) ExportVariableConfigMethod(name string, method interface{}) blueprint.Variable {
  80. ev.exportedConfigDependingVars.set(name, method)
  81. return ev.pctx.VariableConfigMethod(name, method)
  82. }
  83. // ExportSourcePathVariable declares a static "source path" variable and exports
  84. // it to Bazel's toolchain.
  85. func (ev ExportedVariables) ExportSourcePathVariable(name string, value string) {
  86. ev.pctx.SourcePathVariable(name, value)
  87. ev.exportedStringVars.set(name, value)
  88. }
  89. // ExportVariableFuncVariable declares a variable whose value is evaluated at
  90. // runtime via a function and exports it to Bazel's toolchain.
  91. func (ev ExportedVariables) ExportVariableFuncVariable(name string, f func() string) {
  92. ev.exportedConfigDependingVars.set(name, func(config Config) string {
  93. return f()
  94. })
  95. ev.pctx.VariableFunc(name, func(PackageVarContext) string {
  96. return f()
  97. })
  98. }
  99. // ExportString only exports a variable to Bazel, but does not declare it in Soong
  100. func (ev ExportedVariables) ExportString(name string, value string) {
  101. ev.exportedStringVars.set(name, value)
  102. }
  103. // ExportStringList only exports a variable to Bazel, but does not declare it in Soong
  104. func (ev ExportedVariables) ExportStringList(name string, value []string) {
  105. ev.exportedStringListVars.set(name, value)
  106. }
  107. // ExportStringListDict only exports a variable to Bazel, but does not declare it in Soong
  108. func (ev ExportedVariables) ExportStringListDict(name string, value map[string][]string) {
  109. ev.exportedStringListDictVars.set(name, value)
  110. }
  111. // ExportVariableReferenceDict only exports a variable to Bazel, but does not declare it in Soong
  112. func (ev ExportedVariables) ExportVariableReferenceDict(name string, value map[string]string) {
  113. ev.exportedVariableReferenceDictVars.set(name, value)
  114. }
  115. // ExportedConfigDependingVariables is a mapping of variable names to functions
  116. // of type func(config Config) string which return the runtime-evaluated string
  117. // value of a particular variable
  118. type ExportedConfigDependingVariables map[string]interface{}
  119. func (m ExportedConfigDependingVariables) set(k string, v interface{}) {
  120. m[k] = v
  121. }
  122. func (m ExportedConfigDependingVariables) asBazel(config Config,
  123. stringVars ExportedStringVariables, stringListVars ExportedStringListVariables, cfgDepVars ExportedConfigDependingVariables) []bazelConstant {
  124. ret := make([]bazelConstant, 0, len(m))
  125. for variable, unevaluatedVar := range m {
  126. evalFunc := reflect.ValueOf(unevaluatedVar)
  127. validateVariableMethod(variable, evalFunc)
  128. evaluatedResult := evalFunc.Call([]reflect.Value{reflect.ValueOf(config)})
  129. evaluatedValue := evaluatedResult[0].Interface().(string)
  130. expandedVars, err := expandVar(config, evaluatedValue, stringVars, stringListVars, cfgDepVars)
  131. if err != nil {
  132. panic(fmt.Errorf("error expanding config variable %s: %s", variable, err))
  133. }
  134. if len(expandedVars) > 1 {
  135. ret = append(ret, bazelConstant{
  136. variableName: variable,
  137. internalDefinition: starlark_fmt.PrintStringList(expandedVars, 0),
  138. })
  139. } else {
  140. ret = append(ret, bazelConstant{
  141. variableName: variable,
  142. internalDefinition: fmt.Sprintf(`"%s"`, validateCharacters(expandedVars[0])),
  143. })
  144. }
  145. }
  146. return ret
  147. }
  148. // Ensure that string s has no invalid characters to be generated into the bzl file.
  149. func validateCharacters(s string) string {
  150. for _, c := range []string{`\n`, `"`, `\`} {
  151. if strings.Contains(s, c) {
  152. panic(fmt.Errorf("%s contains illegal character %s", s, c))
  153. }
  154. }
  155. return s
  156. }
  157. type bazelConstant struct {
  158. variableName string
  159. internalDefinition string
  160. sortLast bool
  161. }
  162. // ExportedStringVariables is a mapping of variable names to string values
  163. type ExportedStringVariables map[string]string
  164. func (m ExportedStringVariables) set(k string, v string) {
  165. m[k] = v
  166. }
  167. func (m ExportedStringVariables) asBazel(config Config,
  168. stringVars ExportedStringVariables, stringListVars ExportedStringListVariables, cfgDepVars ExportedConfigDependingVariables) []bazelConstant {
  169. ret := make([]bazelConstant, 0, len(m))
  170. for k, variableValue := range m {
  171. expandedVar, err := expandVar(config, variableValue, stringVars, stringListVars, cfgDepVars)
  172. if err != nil {
  173. panic(fmt.Errorf("error expanding config variable %s: %s", k, err))
  174. }
  175. if len(expandedVar) > 1 {
  176. panic(fmt.Errorf("%s expands to more than one string value: %s", variableValue, expandedVar))
  177. }
  178. ret = append(ret, bazelConstant{
  179. variableName: k,
  180. internalDefinition: fmt.Sprintf(`"%s"`, validateCharacters(expandedVar[0])),
  181. })
  182. }
  183. return ret
  184. }
  185. // ExportedStringListVariables is a mapping of variable names to a list of strings
  186. type ExportedStringListVariables map[string][]string
  187. func (m ExportedStringListVariables) set(k string, v []string) {
  188. m[k] = v
  189. }
  190. func (m ExportedStringListVariables) asBazel(config Config,
  191. stringScope ExportedStringVariables, stringListScope ExportedStringListVariables,
  192. exportedVars ExportedConfigDependingVariables) []bazelConstant {
  193. ret := make([]bazelConstant, 0, len(m))
  194. // For each exported variable, recursively expand elements in the variableValue
  195. // list to ensure that interpolated variables are expanded according to their values
  196. // in the variable scope.
  197. for k, variableValue := range m {
  198. var expandedVars []string
  199. for _, v := range variableValue {
  200. expandedVar, err := expandVar(config, v, stringScope, stringListScope, exportedVars)
  201. if err != nil {
  202. panic(fmt.Errorf("Error expanding config variable %s=%s: %s", k, v, err))
  203. }
  204. expandedVars = append(expandedVars, expandedVar...)
  205. }
  206. // Assign the list as a bzl-private variable; this variable will be exported
  207. // out through a constants struct later.
  208. ret = append(ret, bazelConstant{
  209. variableName: k,
  210. internalDefinition: starlark_fmt.PrintStringList(expandedVars, 0),
  211. })
  212. }
  213. return ret
  214. }
  215. // ExportedStringListDictVariables is a mapping from variable names to a
  216. // dictionary which maps keys to lists of strings
  217. type ExportedStringListDictVariables map[string]map[string][]string
  218. func (m ExportedStringListDictVariables) set(k string, v map[string][]string) {
  219. m[k] = v
  220. }
  221. // Since dictionaries are not supported in Ninja, we do not expand variables for dictionaries
  222. func (m ExportedStringListDictVariables) asBazel(_ Config, _ ExportedStringVariables,
  223. _ ExportedStringListVariables, _ ExportedConfigDependingVariables) []bazelConstant {
  224. ret := make([]bazelConstant, 0, len(m))
  225. for k, dict := range m {
  226. ret = append(ret, bazelConstant{
  227. variableName: k,
  228. internalDefinition: starlark_fmt.PrintStringListDict(dict, 0),
  229. })
  230. }
  231. return ret
  232. }
  233. // ExportedVariableReferenceDictVariables is a mapping from variable names to a
  234. // dictionary which references previously defined variables. This is used to
  235. // create a Starlark output such as:
  236. //
  237. // string_var1 = "string1
  238. // var_ref_dict_var1 = {
  239. // "key1": string_var1
  240. // }
  241. //
  242. // This type of variable collection must be expanded last so that it recognizes
  243. // previously defined variables.
  244. type ExportedVariableReferenceDictVariables map[string]map[string]string
  245. func (m ExportedVariableReferenceDictVariables) set(k string, v map[string]string) {
  246. m[k] = v
  247. }
  248. func (m ExportedVariableReferenceDictVariables) asBazel(_ Config, _ ExportedStringVariables,
  249. _ ExportedStringListVariables, _ ExportedConfigDependingVariables) []bazelConstant {
  250. ret := make([]bazelConstant, 0, len(m))
  251. for n, dict := range m {
  252. for k, v := range dict {
  253. matches, err := variableReference(v)
  254. if err != nil {
  255. panic(err)
  256. } else if !matches.matches {
  257. panic(fmt.Errorf("Expected a variable reference, got %q", v))
  258. } else if len(matches.fullVariableReference) != len(v) {
  259. panic(fmt.Errorf("Expected only a variable reference, got %q", v))
  260. }
  261. dict[k] = "_" + matches.variable
  262. }
  263. ret = append(ret, bazelConstant{
  264. variableName: n,
  265. internalDefinition: starlark_fmt.PrintDict(dict, 0),
  266. sortLast: true,
  267. })
  268. }
  269. return ret
  270. }
  271. // BazelToolchainVars expands an ExportedVariables collection and returns a string
  272. // of formatted Starlark variable definitions
  273. func BazelToolchainVars(config Config, exportedVars ExportedVariables) string {
  274. results := exportedVars.asBazel(
  275. config,
  276. exportedVars.exportedStringVars,
  277. exportedVars.exportedStringListVars,
  278. exportedVars.exportedConfigDependingVars,
  279. )
  280. sort.Slice(results, func(i, j int) bool {
  281. if results[i].sortLast != results[j].sortLast {
  282. return !results[i].sortLast
  283. }
  284. return results[i].variableName < results[j].variableName
  285. })
  286. definitions := make([]string, 0, len(results))
  287. constants := make([]string, 0, len(results))
  288. for _, b := range results {
  289. definitions = append(definitions,
  290. fmt.Sprintf("_%s = %s", b.variableName, b.internalDefinition))
  291. constants = append(constants,
  292. fmt.Sprintf("%[1]s%[2]s = _%[2]s,", starlark_fmt.Indention(1), b.variableName))
  293. }
  294. // Build the exported constants struct.
  295. ret := bazel.GeneratedBazelFileWarning
  296. ret += "\n\n"
  297. ret += strings.Join(definitions, "\n\n")
  298. ret += "\n\n"
  299. ret += "constants = struct(\n"
  300. ret += strings.Join(constants, "\n")
  301. ret += "\n)"
  302. return ret
  303. }
  304. type match struct {
  305. matches bool
  306. fullVariableReference string
  307. variable string
  308. }
  309. func variableReference(input string) (match, error) {
  310. // e.g. "${ExternalCflags}"
  311. r := regexp.MustCompile(`\${(?:config\.)?([a-zA-Z0-9_]+)}`)
  312. matches := r.FindStringSubmatch(input)
  313. if len(matches) == 0 {
  314. return match{}, nil
  315. }
  316. if len(matches) != 2 {
  317. return match{}, fmt.Errorf("Expected to only match 1 subexpression in %s, got %d", input, len(matches)-1)
  318. }
  319. return match{
  320. matches: true,
  321. fullVariableReference: matches[0],
  322. // Index 1 of FindStringSubmatch contains the subexpression match
  323. // (variable name) of the capture group.
  324. variable: matches[1],
  325. }, nil
  326. }
  327. // expandVar recursively expand interpolated variables in the exportedVars scope.
  328. //
  329. // We're using a string slice to track the seen variables to avoid
  330. // stackoverflow errors with infinite recursion. it's simpler to use a
  331. // string slice than to handle a pass-by-referenced map, which would make it
  332. // quite complex to track depth-first interpolations. It's also unlikely the
  333. // interpolation stacks are deep (n > 1).
  334. func expandVar(config Config, toExpand string, stringScope ExportedStringVariables,
  335. stringListScope ExportedStringListVariables, exportedVars ExportedConfigDependingVariables) ([]string, error) {
  336. // Internal recursive function.
  337. var expandVarInternal func(string, map[string]bool) (string, error)
  338. expandVarInternal = func(toExpand string, seenVars map[string]bool) (string, error) {
  339. var ret string
  340. remainingString := toExpand
  341. for len(remainingString) > 0 {
  342. matches, err := variableReference(remainingString)
  343. if err != nil {
  344. panic(err)
  345. }
  346. if !matches.matches {
  347. return ret + remainingString, nil
  348. }
  349. matchIndex := strings.Index(remainingString, matches.fullVariableReference)
  350. ret += remainingString[:matchIndex]
  351. remainingString = remainingString[matchIndex+len(matches.fullVariableReference):]
  352. variable := matches.variable
  353. // toExpand contains a variable.
  354. if _, ok := seenVars[variable]; ok {
  355. return ret, fmt.Errorf(
  356. "Unbounded recursive interpolation of variable: %s", variable)
  357. }
  358. // A map is passed-by-reference. Create a new map for
  359. // this scope to prevent variables seen in one depth-first expansion
  360. // to be also treated as "seen" in other depth-first traversals.
  361. newSeenVars := map[string]bool{}
  362. for k := range seenVars {
  363. newSeenVars[k] = true
  364. }
  365. newSeenVars[variable] = true
  366. if unexpandedVars, ok := stringListScope[variable]; ok {
  367. expandedVars := []string{}
  368. for _, unexpandedVar := range unexpandedVars {
  369. expandedVar, err := expandVarInternal(unexpandedVar, newSeenVars)
  370. if err != nil {
  371. return ret, err
  372. }
  373. expandedVars = append(expandedVars, expandedVar)
  374. }
  375. ret += strings.Join(expandedVars, " ")
  376. } else if unexpandedVar, ok := stringScope[variable]; ok {
  377. expandedVar, err := expandVarInternal(unexpandedVar, newSeenVars)
  378. if err != nil {
  379. return ret, err
  380. }
  381. ret += expandedVar
  382. } else if unevaluatedVar, ok := exportedVars[variable]; ok {
  383. evalFunc := reflect.ValueOf(unevaluatedVar)
  384. validateVariableMethod(variable, evalFunc)
  385. evaluatedResult := evalFunc.Call([]reflect.Value{reflect.ValueOf(config)})
  386. evaluatedValue := evaluatedResult[0].Interface().(string)
  387. expandedVar, err := expandVarInternal(evaluatedValue, newSeenVars)
  388. if err != nil {
  389. return ret, err
  390. }
  391. ret += expandedVar
  392. } else {
  393. return "", fmt.Errorf("Unbound config variable %s", variable)
  394. }
  395. }
  396. return ret, nil
  397. }
  398. var ret []string
  399. stringFields := splitStringKeepingQuotedSubstring(toExpand, ' ')
  400. for _, v := range stringFields {
  401. val, err := expandVarInternal(v, map[string]bool{})
  402. if err != nil {
  403. return ret, err
  404. }
  405. ret = append(ret, val)
  406. }
  407. return ret, nil
  408. }
  409. // splitStringKeepingQuotedSubstring splits a string on a provided separator,
  410. // but it will not split substrings inside unescaped double quotes. If the double
  411. // quotes are escaped, then the returned string will only include the quote, and
  412. // not the escape.
  413. func splitStringKeepingQuotedSubstring(s string, delimiter byte) []string {
  414. var ret []string
  415. quote := byte('"')
  416. var substring []byte
  417. quoted := false
  418. escaped := false
  419. for i := range s {
  420. if !quoted && s[i] == delimiter {
  421. ret = append(ret, string(substring))
  422. substring = []byte{}
  423. continue
  424. }
  425. characterIsEscape := i < len(s)-1 && s[i] == '\\' && s[i+1] == quote
  426. if characterIsEscape {
  427. escaped = true
  428. continue
  429. }
  430. if s[i] == quote {
  431. if !escaped {
  432. quoted = !quoted
  433. }
  434. escaped = false
  435. }
  436. substring = append(substring, s[i])
  437. }
  438. ret = append(ret, string(substring))
  439. return ret
  440. }
  441. func validateVariableMethod(name string, methodValue reflect.Value) {
  442. methodType := methodValue.Type()
  443. if methodType.Kind() != reflect.Func {
  444. panic(fmt.Errorf("method given for variable %s is not a function",
  445. name))
  446. }
  447. if n := methodType.NumIn(); n != 1 {
  448. panic(fmt.Errorf("method for variable %s has %d inputs (should be 1)",
  449. name, n))
  450. }
  451. if n := methodType.NumOut(); n != 1 {
  452. panic(fmt.Errorf("method for variable %s has %d outputs (should be 1)",
  453. name, n))
  454. }
  455. if kind := methodType.Out(0).Kind(); kind != reflect.String {
  456. panic(fmt.Errorf("method for variable %s does not return a string",
  457. name))
  458. }
  459. }