bazel_handler.go 52 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522
  1. // Copyright 2020 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package android
  15. import (
  16. "bytes"
  17. "fmt"
  18. "os"
  19. "os/exec"
  20. "path"
  21. "path/filepath"
  22. "runtime"
  23. "sort"
  24. "strings"
  25. "sync"
  26. "android/soong/android/allowlists"
  27. "android/soong/bazel/cquery"
  28. "android/soong/shared"
  29. "android/soong/starlark_fmt"
  30. "github.com/google/blueprint"
  31. "github.com/google/blueprint/metrics"
  32. "android/soong/bazel"
  33. )
  34. var (
  35. _ = pctx.HostBinToolVariable("bazelBuildRunfilesTool", "build-runfiles")
  36. buildRunfilesRule = pctx.AndroidStaticRule("bazelBuildRunfiles", blueprint.RuleParams{
  37. Command: "${bazelBuildRunfilesTool} ${in} ${outDir}",
  38. Depfile: "",
  39. Description: "",
  40. CommandDeps: []string{"${bazelBuildRunfilesTool}"},
  41. }, "outDir")
  42. allowedBazelEnvironmentVars = []string{
  43. // clang-tidy
  44. "ALLOW_LOCAL_TIDY_TRUE",
  45. "DEFAULT_TIDY_HEADER_DIRS",
  46. "TIDY_TIMEOUT",
  47. "WITH_TIDY",
  48. "WITH_TIDY_FLAGS",
  49. "TIDY_EXTERNAL_VENDOR",
  50. "SKIP_ABI_CHECKS",
  51. "UNSAFE_DISABLE_APEX_ALLOWED_DEPS_CHECK",
  52. "AUTO_ZERO_INITIALIZE",
  53. "AUTO_PATTERN_INITIALIZE",
  54. "AUTO_UNINITIALIZE",
  55. "USE_CCACHE",
  56. "LLVM_NEXT",
  57. "ALLOW_UNKNOWN_WARNING_OPTION",
  58. // Overrides the version in the apex_manifest.json. The version is unique for
  59. // each branch (internal, aosp, mainline releases, dessert releases). This
  60. // enables modules built on an older branch to be installed against a newer
  61. // device for development purposes.
  62. "OVERRIDE_APEX_MANIFEST_DEFAULT_VERSION",
  63. }
  64. )
  65. func init() {
  66. RegisterMixedBuildsMutator(InitRegistrationContext)
  67. }
  68. func RegisterMixedBuildsMutator(ctx RegistrationContext) {
  69. ctx.FinalDepsMutators(func(ctx RegisterMutatorsContext) {
  70. ctx.BottomUp("mixed_builds_prep", mixedBuildsPrepareMutator).Parallel()
  71. })
  72. }
  73. func mixedBuildsPrepareMutator(ctx BottomUpMutatorContext) {
  74. if m := ctx.Module(); m.Enabled() {
  75. if mixedBuildMod, ok := m.(MixedBuildBuildable); ok {
  76. if mixedBuildMod.IsMixedBuildSupported(ctx) && MixedBuildsEnabled(ctx) {
  77. mixedBuildMod.QueueBazelCall(ctx)
  78. }
  79. }
  80. }
  81. }
  82. type cqueryRequest interface {
  83. // Name returns a string name for this request type. Such request type names must be unique,
  84. // and must only consist of alphanumeric characters.
  85. Name() string
  86. // StarlarkFunctionBody returns a starlark function body to process this request type.
  87. // The returned string is the body of a Starlark function which obtains
  88. // all request-relevant information about a target and returns a string containing
  89. // this information.
  90. // The function should have the following properties:
  91. // - The arguments are `target` (a configured target) and `id_string` (the label + configuration).
  92. // - The return value must be a string.
  93. // - The function body should not be indented outside of its own scope.
  94. StarlarkFunctionBody() string
  95. }
  96. // Portion of cquery map key to describe target configuration.
  97. type configKey struct {
  98. arch string
  99. osType OsType
  100. apexKey ApexConfigKey
  101. }
  102. type ApexConfigKey struct {
  103. WithinApex bool
  104. ApexSdkVersion string
  105. }
  106. func (c ApexConfigKey) String() string {
  107. return fmt.Sprintf("%s_%s", withinApexToString(c.WithinApex), c.ApexSdkVersion)
  108. }
  109. func withinApexToString(withinApex bool) string {
  110. if withinApex {
  111. return "within_apex"
  112. }
  113. return ""
  114. }
  115. func (c configKey) String() string {
  116. return fmt.Sprintf("%s::%s::%s", c.arch, c.osType, c.apexKey)
  117. }
  118. // Map key to describe bazel cquery requests.
  119. type cqueryKey struct {
  120. label string
  121. requestType cqueryRequest
  122. configKey configKey
  123. }
  124. func makeCqueryKey(label string, cqueryRequest cqueryRequest, cfgKey configKey) cqueryKey {
  125. if strings.HasPrefix(label, "//") {
  126. // Normalize Bazel labels to specify main repository explicitly.
  127. label = "@" + label
  128. }
  129. return cqueryKey{label, cqueryRequest, cfgKey}
  130. }
  131. func (c cqueryKey) String() string {
  132. return fmt.Sprintf("cquery(%s,%s,%s)", c.label, c.requestType.Name(), c.configKey)
  133. }
  134. type invokeBazelContext interface {
  135. GetEventHandler() *metrics.EventHandler
  136. }
  137. // BazelContext is a context object useful for interacting with Bazel during
  138. // the course of a build. Use of Bazel to evaluate part of the build graph
  139. // is referred to as a "mixed build". (Some modules are managed by Soong,
  140. // some are managed by Bazel). To facilitate interop between these build
  141. // subgraphs, Soong may make requests to Bazel and evaluate their responses
  142. // so that Soong modules may accurately depend on Bazel targets.
  143. type BazelContext interface {
  144. // Add a cquery request to the bazel request queue. All queued requests
  145. // will be sent to Bazel on a subsequent invocation of InvokeBazel.
  146. QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey)
  147. // ** Cquery Results Retrieval Functions
  148. // The below functions pertain to retrieving cquery results from a prior
  149. // InvokeBazel function call and parsing the results.
  150. // Returns result files built by building the given bazel target label.
  151. GetOutputFiles(label string, cfgKey configKey) ([]string, error)
  152. // Returns the results of GetOutputFiles and GetCcObjectFiles in a single query (in that order).
  153. GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error)
  154. // Returns the executable binary resultant from building together the python sources
  155. // TODO(b/232976601): Remove.
  156. GetPythonBinary(label string, cfgKey configKey) (string, error)
  157. // Returns the results of the GetApexInfo query (including output files)
  158. GetApexInfo(label string, cfgkey configKey) (cquery.ApexInfo, error)
  159. // Returns the results of the GetCcUnstrippedInfo query
  160. GetCcUnstrippedInfo(label string, cfgkey configKey) (cquery.CcUnstrippedInfo, error)
  161. // ** end Cquery Results Retrieval Functions
  162. // Issues commands to Bazel to receive results for all cquery requests
  163. // queued in the BazelContext. The ctx argument is optional and is only
  164. // used for performance data collection
  165. InvokeBazel(config Config, ctx invokeBazelContext) error
  166. // Returns true if Bazel handling is enabled for the module with the given name.
  167. // Note that this only implies "bazel mixed build" allowlisting. The caller
  168. // should independently verify the module is eligible for Bazel handling
  169. // (for example, that it is MixedBuildBuildable).
  170. IsModuleNameAllowed(moduleName string, withinApex bool) bool
  171. IsModuleDclaAllowed(moduleName string) bool
  172. // Returns the bazel output base (the root directory for all bazel intermediate outputs).
  173. OutputBase() string
  174. // Returns build statements which should get registered to reflect Bazel's outputs.
  175. BuildStatementsToRegister() []*bazel.BuildStatement
  176. // Returns the depsets defined in Bazel's aquery response.
  177. AqueryDepsets() []bazel.AqueryDepset
  178. }
  179. type bazelRunner interface {
  180. createBazelCommand(config Config, paths *bazelPaths, runName bazel.RunName, command bazelCommand, extraFlags ...string) *exec.Cmd
  181. issueBazelCommand(bazelCmd *exec.Cmd, eventHandler *metrics.EventHandler) (output string, errorMessage string, error error)
  182. }
  183. type bazelPaths struct {
  184. homeDir string
  185. bazelPath string
  186. outputBase string
  187. workspaceDir string
  188. soongOutDir string
  189. metricsDir string
  190. bazelDepsFile string
  191. }
  192. // A context object which tracks queued requests that need to be made to Bazel,
  193. // and their results after the requests have been made.
  194. type mixedBuildBazelContext struct {
  195. bazelRunner
  196. paths *bazelPaths
  197. // cquery requests that have not yet been issued to Bazel. This list is maintained
  198. // in a sorted state, and is guaranteed to have no duplicates.
  199. requests []cqueryKey
  200. requestMutex sync.Mutex // requests can be written in parallel
  201. results map[cqueryKey]string // Results of cquery requests after Bazel invocations
  202. // Build statements which should get registered to reflect Bazel's outputs.
  203. buildStatements []*bazel.BuildStatement
  204. // Depsets which should be used for Bazel's build statements.
  205. depsets []bazel.AqueryDepset
  206. // Per-module allowlist/denylist functionality to control whether analysis of
  207. // modules are handled by Bazel. For modules which do not have a Bazel definition
  208. // (or do not sufficiently support bazel handling via MixedBuildBuildable),
  209. // this allowlist will have no effect, even if the module is explicitly allowlisted here.
  210. // Per-module denylist to opt modules out of bazel handling.
  211. bazelDisabledModules map[string]bool
  212. // Per-module allowlist to opt modules in to bazel handling.
  213. bazelEnabledModules map[string]bool
  214. // DCLA modules are enabled when used in apex.
  215. bazelDclaEnabledModules map[string]bool
  216. // If true, modules are bazel-enabled by default, unless present in bazelDisabledModules.
  217. modulesDefaultToBazel bool
  218. targetProduct string
  219. targetBuildVariant string
  220. }
  221. var _ BazelContext = &mixedBuildBazelContext{}
  222. // A bazel context to use when Bazel is disabled.
  223. type noopBazelContext struct{}
  224. var _ BazelContext = noopBazelContext{}
  225. // A bazel context to use for tests.
  226. type MockBazelContext struct {
  227. OutputBaseDir string
  228. LabelToOutputFiles map[string][]string
  229. LabelToCcInfo map[string]cquery.CcInfo
  230. LabelToPythonBinary map[string]string
  231. LabelToApexInfo map[string]cquery.ApexInfo
  232. LabelToCcBinary map[string]cquery.CcUnstrippedInfo
  233. BazelRequests map[string]bool
  234. }
  235. func (m MockBazelContext) QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey) {
  236. key := BuildMockBazelContextRequestKey(label, requestType, cfgKey.arch, cfgKey.osType, cfgKey.apexKey)
  237. if m.BazelRequests == nil {
  238. m.BazelRequests = make(map[string]bool)
  239. }
  240. m.BazelRequests[key] = true
  241. }
  242. func (m MockBazelContext) GetOutputFiles(label string, _ configKey) ([]string, error) {
  243. result, ok := m.LabelToOutputFiles[label]
  244. if !ok {
  245. return []string{}, fmt.Errorf("no target with label %q in LabelToOutputFiles", label)
  246. }
  247. return result, nil
  248. }
  249. func (m MockBazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error) {
  250. result, ok := m.LabelToCcInfo[label]
  251. if !ok {
  252. key := BuildMockBazelContextResultKey(label, cfgKey.arch, cfgKey.osType, cfgKey.apexKey)
  253. result, ok = m.LabelToCcInfo[key]
  254. if !ok {
  255. return cquery.CcInfo{}, fmt.Errorf("no target with label %q in LabelToCcInfo", label)
  256. }
  257. }
  258. return result, nil
  259. }
  260. func (m MockBazelContext) GetPythonBinary(label string, _ configKey) (string, error) {
  261. result, ok := m.LabelToPythonBinary[label]
  262. if !ok {
  263. return "", fmt.Errorf("no target with label %q in LabelToPythonBinary", label)
  264. }
  265. return result, nil
  266. }
  267. func (m MockBazelContext) GetApexInfo(label string, _ configKey) (cquery.ApexInfo, error) {
  268. result, ok := m.LabelToApexInfo[label]
  269. if !ok {
  270. return cquery.ApexInfo{}, fmt.Errorf("no target with label %q in LabelToApexInfo", label)
  271. }
  272. return result, nil
  273. }
  274. func (m MockBazelContext) GetCcUnstrippedInfo(label string, _ configKey) (cquery.CcUnstrippedInfo, error) {
  275. result, ok := m.LabelToCcBinary[label]
  276. if !ok {
  277. return cquery.CcUnstrippedInfo{}, fmt.Errorf("no target with label %q in LabelToCcBinary", label)
  278. }
  279. return result, nil
  280. }
  281. func (m MockBazelContext) InvokeBazel(_ Config, _ invokeBazelContext) error {
  282. panic("unimplemented")
  283. }
  284. func (m MockBazelContext) IsModuleNameAllowed(_ string, _ bool) bool {
  285. return true
  286. }
  287. func (m MockBazelContext) IsModuleDclaAllowed(_ string) bool {
  288. return true
  289. }
  290. func (m MockBazelContext) OutputBase() string { return m.OutputBaseDir }
  291. func (m MockBazelContext) BuildStatementsToRegister() []*bazel.BuildStatement {
  292. return []*bazel.BuildStatement{}
  293. }
  294. func (m MockBazelContext) AqueryDepsets() []bazel.AqueryDepset {
  295. return []bazel.AqueryDepset{}
  296. }
  297. var _ BazelContext = MockBazelContext{}
  298. func BuildMockBazelContextRequestKey(label string, request cqueryRequest, arch string, osType OsType, apexKey ApexConfigKey) string {
  299. cfgKey := configKey{
  300. arch: arch,
  301. osType: osType,
  302. apexKey: apexKey,
  303. }
  304. return strings.Join([]string{label, request.Name(), cfgKey.String()}, "_")
  305. }
  306. func BuildMockBazelContextResultKey(label string, arch string, osType OsType, apexKey ApexConfigKey) string {
  307. cfgKey := configKey{
  308. arch: arch,
  309. osType: osType,
  310. apexKey: apexKey,
  311. }
  312. return strings.Join([]string{label, cfgKey.String()}, "_")
  313. }
  314. func (bazelCtx *mixedBuildBazelContext) QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey) {
  315. key := makeCqueryKey(label, requestType, cfgKey)
  316. bazelCtx.requestMutex.Lock()
  317. defer bazelCtx.requestMutex.Unlock()
  318. // Insert key into requests, maintaining the sort, and only if it's not duplicate.
  319. keyString := key.String()
  320. foundEqual := false
  321. notLessThanKeyString := func(i int) bool {
  322. s := bazelCtx.requests[i].String()
  323. v := strings.Compare(s, keyString)
  324. if v == 0 {
  325. foundEqual = true
  326. }
  327. return v >= 0
  328. }
  329. targetIndex := sort.Search(len(bazelCtx.requests), notLessThanKeyString)
  330. if foundEqual {
  331. return
  332. }
  333. if targetIndex == len(bazelCtx.requests) {
  334. bazelCtx.requests = append(bazelCtx.requests, key)
  335. } else {
  336. bazelCtx.requests = append(bazelCtx.requests[:targetIndex+1], bazelCtx.requests[targetIndex:]...)
  337. bazelCtx.requests[targetIndex] = key
  338. }
  339. }
  340. func (bazelCtx *mixedBuildBazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, error) {
  341. key := makeCqueryKey(label, cquery.GetOutputFiles, cfgKey)
  342. if rawString, ok := bazelCtx.results[key]; ok {
  343. bazelOutput := strings.TrimSpace(rawString)
  344. return cquery.GetOutputFiles.ParseResult(bazelOutput), nil
  345. }
  346. return nil, fmt.Errorf("no bazel response found for %v", key)
  347. }
  348. func (bazelCtx *mixedBuildBazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error) {
  349. key := makeCqueryKey(label, cquery.GetCcInfo, cfgKey)
  350. if rawString, ok := bazelCtx.results[key]; ok {
  351. bazelOutput := strings.TrimSpace(rawString)
  352. return cquery.GetCcInfo.ParseResult(bazelOutput)
  353. }
  354. return cquery.CcInfo{}, fmt.Errorf("no bazel response found for %v", key)
  355. }
  356. func (bazelCtx *mixedBuildBazelContext) GetPythonBinary(label string, cfgKey configKey) (string, error) {
  357. key := makeCqueryKey(label, cquery.GetPythonBinary, cfgKey)
  358. if rawString, ok := bazelCtx.results[key]; ok {
  359. bazelOutput := strings.TrimSpace(rawString)
  360. return cquery.GetPythonBinary.ParseResult(bazelOutput), nil
  361. }
  362. return "", fmt.Errorf("no bazel response found for %v", key)
  363. }
  364. func (bazelCtx *mixedBuildBazelContext) GetApexInfo(label string, cfgKey configKey) (cquery.ApexInfo, error) {
  365. key := makeCqueryKey(label, cquery.GetApexInfo, cfgKey)
  366. if rawString, ok := bazelCtx.results[key]; ok {
  367. return cquery.GetApexInfo.ParseResult(strings.TrimSpace(rawString))
  368. }
  369. return cquery.ApexInfo{}, fmt.Errorf("no bazel response found for %v", key)
  370. }
  371. func (bazelCtx *mixedBuildBazelContext) GetCcUnstrippedInfo(label string, cfgKey configKey) (cquery.CcUnstrippedInfo, error) {
  372. key := makeCqueryKey(label, cquery.GetCcUnstrippedInfo, cfgKey)
  373. if rawString, ok := bazelCtx.results[key]; ok {
  374. return cquery.GetCcUnstrippedInfo.ParseResult(strings.TrimSpace(rawString))
  375. }
  376. return cquery.CcUnstrippedInfo{}, fmt.Errorf("no bazel response for %s", key)
  377. }
  378. func (n noopBazelContext) QueueBazelRequest(_ string, _ cqueryRequest, _ configKey) {
  379. panic("unimplemented")
  380. }
  381. func (n noopBazelContext) GetOutputFiles(_ string, _ configKey) ([]string, error) {
  382. panic("unimplemented")
  383. }
  384. func (n noopBazelContext) GetCcInfo(_ string, _ configKey) (cquery.CcInfo, error) {
  385. panic("unimplemented")
  386. }
  387. func (n noopBazelContext) GetPythonBinary(_ string, _ configKey) (string, error) {
  388. panic("unimplemented")
  389. }
  390. func (n noopBazelContext) GetApexInfo(_ string, _ configKey) (cquery.ApexInfo, error) {
  391. panic("unimplemented")
  392. }
  393. func (n noopBazelContext) GetCcUnstrippedInfo(_ string, _ configKey) (cquery.CcUnstrippedInfo, error) {
  394. //TODO implement me
  395. panic("implement me")
  396. }
  397. func (n noopBazelContext) InvokeBazel(_ Config, _ invokeBazelContext) error {
  398. panic("unimplemented")
  399. }
  400. func (m noopBazelContext) OutputBase() string {
  401. return ""
  402. }
  403. func (n noopBazelContext) IsModuleNameAllowed(_ string, _ bool) bool {
  404. return false
  405. }
  406. func (n noopBazelContext) IsModuleDclaAllowed(_ string) bool {
  407. return false
  408. }
  409. func (m noopBazelContext) BuildStatementsToRegister() []*bazel.BuildStatement {
  410. return []*bazel.BuildStatement{}
  411. }
  412. func (m noopBazelContext) AqueryDepsets() []bazel.AqueryDepset {
  413. return []bazel.AqueryDepset{}
  414. }
  415. func addToStringSet(set map[string]bool, items []string) {
  416. for _, item := range items {
  417. set[item] = true
  418. }
  419. }
  420. func GetBazelEnabledAndDisabledModules(buildMode SoongBuildMode, forceEnabled map[string]struct{}) (map[string]bool, map[string]bool) {
  421. disabledModules := map[string]bool{}
  422. enabledModules := map[string]bool{}
  423. switch buildMode {
  424. case BazelProdMode:
  425. addToStringSet(enabledModules, allowlists.ProdMixedBuildsEnabledList)
  426. for enabledAdHocModule := range forceEnabled {
  427. enabledModules[enabledAdHocModule] = true
  428. }
  429. case BazelStagingMode:
  430. // Staging mode includes all prod modules plus all staging modules.
  431. addToStringSet(enabledModules, allowlists.ProdMixedBuildsEnabledList)
  432. addToStringSet(enabledModules, allowlists.StagingMixedBuildsEnabledList)
  433. for enabledAdHocModule := range forceEnabled {
  434. enabledModules[enabledAdHocModule] = true
  435. }
  436. case BazelDevMode:
  437. addToStringSet(disabledModules, allowlists.MixedBuildsDisabledList)
  438. default:
  439. panic("Expected BazelProdMode, BazelStagingMode, or BazelDevMode")
  440. }
  441. return enabledModules, disabledModules
  442. }
  443. func GetBazelEnabledModules(buildMode SoongBuildMode) []string {
  444. enabledModules, disabledModules := GetBazelEnabledAndDisabledModules(buildMode, nil)
  445. enabledList := make([]string, 0, len(enabledModules))
  446. for module := range enabledModules {
  447. if !disabledModules[module] {
  448. enabledList = append(enabledList, module)
  449. }
  450. }
  451. sort.Strings(enabledList)
  452. return enabledList
  453. }
  454. func NewBazelContext(c *config) (BazelContext, error) {
  455. if c.BuildMode != BazelProdMode && c.BuildMode != BazelStagingMode && c.BuildMode != BazelDevMode {
  456. return noopBazelContext{}, nil
  457. }
  458. enabledModules, disabledModules := GetBazelEnabledAndDisabledModules(c.BuildMode, c.BazelModulesForceEnabledByFlag())
  459. paths := bazelPaths{
  460. soongOutDir: c.soongOutDir,
  461. }
  462. var missing []string
  463. vars := []struct {
  464. name string
  465. ptr *string
  466. // True if the environment variable needs to be tracked so that changes to the variable
  467. // cause the ninja file to be regenerated, false otherwise. False should only be set for
  468. // environment variables that have no effect on the generated ninja file.
  469. track bool
  470. }{
  471. {"BAZEL_HOME", &paths.homeDir, true},
  472. {"BAZEL_PATH", &paths.bazelPath, true},
  473. {"BAZEL_OUTPUT_BASE", &paths.outputBase, true},
  474. {"BAZEL_WORKSPACE", &paths.workspaceDir, true},
  475. {"BAZEL_METRICS_DIR", &paths.metricsDir, false},
  476. {"BAZEL_DEPS_FILE", &paths.bazelDepsFile, true},
  477. }
  478. for _, v := range vars {
  479. if v.track {
  480. if s := c.Getenv(v.name); len(s) > 1 {
  481. *v.ptr = s
  482. continue
  483. }
  484. } else if s, ok := c.env[v.name]; ok {
  485. *v.ptr = s
  486. } else {
  487. missing = append(missing, v.name)
  488. }
  489. }
  490. if len(missing) > 0 {
  491. return nil, fmt.Errorf("missing required env vars to use bazel: %s", missing)
  492. }
  493. targetBuildVariant := "user"
  494. if c.Eng() {
  495. targetBuildVariant = "eng"
  496. } else if c.Debuggable() {
  497. targetBuildVariant = "userdebug"
  498. }
  499. targetProduct := "unknown"
  500. if c.HasDeviceProduct() {
  501. targetProduct = c.DeviceProduct()
  502. }
  503. dclaMixedBuildsEnabledList := []string{}
  504. if c.BuildMode == BazelProdMode {
  505. dclaMixedBuildsEnabledList = allowlists.ProdDclaMixedBuildsEnabledList
  506. } else if c.BuildMode == BazelStagingMode {
  507. dclaMixedBuildsEnabledList = append(allowlists.ProdDclaMixedBuildsEnabledList,
  508. allowlists.StagingDclaMixedBuildsEnabledList...)
  509. }
  510. dclaEnabledModules := map[string]bool{}
  511. addToStringSet(dclaEnabledModules, dclaMixedBuildsEnabledList)
  512. return &mixedBuildBazelContext{
  513. bazelRunner: &builtinBazelRunner{c.UseBazelProxy, absolutePath(c.outDir)},
  514. paths: &paths,
  515. modulesDefaultToBazel: c.BuildMode == BazelDevMode,
  516. bazelEnabledModules: enabledModules,
  517. bazelDisabledModules: disabledModules,
  518. bazelDclaEnabledModules: dclaEnabledModules,
  519. targetProduct: targetProduct,
  520. targetBuildVariant: targetBuildVariant,
  521. }, nil
  522. }
  523. func (p *bazelPaths) BazelMetricsDir() string {
  524. return p.metricsDir
  525. }
  526. func (context *mixedBuildBazelContext) IsModuleNameAllowed(moduleName string, withinApex bool) bool {
  527. if context.bazelDisabledModules[moduleName] {
  528. return false
  529. }
  530. if context.bazelEnabledModules[moduleName] {
  531. return true
  532. }
  533. if withinApex && context.IsModuleDclaAllowed(moduleName) {
  534. return true
  535. }
  536. return context.modulesDefaultToBazel
  537. }
  538. func (context *mixedBuildBazelContext) IsModuleDclaAllowed(moduleName string) bool {
  539. return context.bazelDclaEnabledModules[moduleName]
  540. }
  541. func pwdPrefix() string {
  542. // Darwin doesn't have /proc
  543. if runtime.GOOS != "darwin" {
  544. return "PWD=/proc/self/cwd"
  545. }
  546. return ""
  547. }
  548. type bazelCommand struct {
  549. command string
  550. // query or label
  551. expression string
  552. }
  553. type mockBazelRunner struct {
  554. bazelCommandResults map[bazelCommand]string
  555. // use *exec.Cmd as a key to get the bazelCommand, the map will be used in issueBazelCommand()
  556. // Register createBazelCommand() invocations. Later, an
  557. // issueBazelCommand() invocation can be mapped to the *exec.Cmd instance
  558. // and then to the expected result via bazelCommandResults
  559. tokens map[*exec.Cmd]bazelCommand
  560. commands []bazelCommand
  561. extraFlags []string
  562. }
  563. func (r *mockBazelRunner) createBazelCommand(_ Config, _ *bazelPaths, _ bazel.RunName,
  564. command bazelCommand, extraFlags ...string) *exec.Cmd {
  565. r.commands = append(r.commands, command)
  566. r.extraFlags = append(r.extraFlags, strings.Join(extraFlags, " "))
  567. cmd := &exec.Cmd{}
  568. if r.tokens == nil {
  569. r.tokens = make(map[*exec.Cmd]bazelCommand)
  570. }
  571. r.tokens[cmd] = command
  572. return cmd
  573. }
  574. func (r *mockBazelRunner) issueBazelCommand(bazelCmd *exec.Cmd, _ *metrics.EventHandler) (string, string, error) {
  575. if command, ok := r.tokens[bazelCmd]; ok {
  576. return r.bazelCommandResults[command], "", nil
  577. }
  578. return "", "", nil
  579. }
  580. type builtinBazelRunner struct {
  581. useBazelProxy bool
  582. outDir string
  583. }
  584. // Issues the given bazel command with given build label and additional flags.
  585. // Returns (stdout, stderr, error). The first and second return values are strings
  586. // containing the stdout and stderr of the run command, and an error is returned if
  587. // the invocation returned an error code.
  588. func (r *builtinBazelRunner) issueBazelCommand(bazelCmd *exec.Cmd, eventHandler *metrics.EventHandler) (string, string, error) {
  589. if r.useBazelProxy {
  590. eventHandler.Begin("client_proxy")
  591. defer eventHandler.End("client_proxy")
  592. proxyClient := bazel.NewProxyClient(r.outDir)
  593. // Omit the arg containing the Bazel binary, as that is handled by the proxy
  594. // server.
  595. bazelFlags := bazelCmd.Args[1:]
  596. // TODO(b/270989498): Refactor these functions to not take exec.Cmd, as its
  597. // not actually executed for client proxying.
  598. resp, err := proxyClient.IssueCommand(bazel.CmdRequest{bazelFlags, bazelCmd.Env})
  599. if err != nil {
  600. return "", "", err
  601. }
  602. if len(resp.ErrorString) > 0 {
  603. return "", "", fmt.Errorf(resp.ErrorString)
  604. }
  605. return resp.Stdout, resp.Stderr, nil
  606. } else {
  607. eventHandler.Begin("bazel command")
  608. defer eventHandler.End("bazel command")
  609. stderr := &bytes.Buffer{}
  610. bazelCmd.Stderr = stderr
  611. if output, err := bazelCmd.Output(); err != nil {
  612. return "", string(stderr.Bytes()),
  613. fmt.Errorf("bazel command failed: %s\n---command---\n%s\n---env---\n%s\n---stderr---\n%s---",
  614. err, bazelCmd, strings.Join(bazelCmd.Env, "\n"), stderr)
  615. } else {
  616. return string(output), string(stderr.Bytes()), nil
  617. }
  618. }
  619. }
  620. func (r *builtinBazelRunner) createBazelCommand(config Config, paths *bazelPaths, runName bazel.RunName, command bazelCommand,
  621. extraFlags ...string) *exec.Cmd {
  622. cmdFlags := []string{
  623. "--output_base=" + absolutePath(paths.outputBase),
  624. command.command,
  625. command.expression,
  626. // TODO(asmundak): is it needed in every build?
  627. "--profile=" + shared.BazelMetricsFilename(paths, runName),
  628. // Set default platforms to canonicalized values for mixed builds requests.
  629. // If these are set in the bazelrc, they will have values that are
  630. // non-canonicalized to @sourceroot labels, and thus be invalid when
  631. // referenced from the buildroot.
  632. //
  633. // The actual platform values here may be overridden by configuration
  634. // transitions from the buildroot.
  635. fmt.Sprintf("--extra_toolchains=%s", "//prebuilts/clang/host/linux-x86:all"),
  636. // We don't need to set --host_platforms because it's set in bazelrc files
  637. // that the bazel shell script wrapper passes
  638. // Explicitly disable downloading rules (such as canonical C++ and Java rules) from the network.
  639. "--experimental_repository_disable_download",
  640. // Suppress noise
  641. "--ui_event_filters=-INFO",
  642. "--noshow_progress",
  643. "--norun_validations",
  644. }
  645. cmdFlags = append(cmdFlags, extraFlags...)
  646. bazelCmd := exec.Command(paths.bazelPath, cmdFlags...)
  647. bazelCmd.Dir = absolutePath(paths.syntheticWorkspaceDir())
  648. extraEnv := []string{
  649. "HOME=" + paths.homeDir,
  650. pwdPrefix(),
  651. "BUILD_DIR=" + absolutePath(paths.soongOutDir),
  652. // Make OUT_DIR absolute here so build/bazel/bin/bazel uses the correct
  653. // OUT_DIR at <root>/out, instead of <root>/out/soong/workspace/out.
  654. "OUT_DIR=" + absolutePath(paths.outDir()),
  655. // Disables local host detection of gcc; toolchain information is defined
  656. // explicitly in BUILD files.
  657. "BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1",
  658. }
  659. for _, envvar := range allowedBazelEnvironmentVars {
  660. val := config.Getenv(envvar)
  661. if val == "" {
  662. continue
  663. }
  664. extraEnv = append(extraEnv, fmt.Sprintf("%s=%s", envvar, val))
  665. }
  666. bazelCmd.Env = append(os.Environ(), extraEnv...)
  667. return bazelCmd
  668. }
  669. func printableCqueryCommand(bazelCmd *exec.Cmd) string {
  670. outputString := strings.Join(bazelCmd.Env, " ") + " \"" + strings.Join(bazelCmd.Args, "\" \"") + "\""
  671. return outputString
  672. }
  673. func (context *mixedBuildBazelContext) mainBzlFileContents() []byte {
  674. // TODO(cparsons): Define configuration transitions programmatically based
  675. // on available archs.
  676. contents := `
  677. #####################################################
  678. # This file is generated by soong_build. Do not edit.
  679. #####################################################
  680. def _config_node_transition_impl(settings, attr):
  681. if attr.os == "android" and attr.arch == "target":
  682. target = "{PRODUCT}-{VARIANT}"
  683. else:
  684. target = "{PRODUCT}-{VARIANT}_%s_%s" % (attr.os, attr.arch)
  685. apex_name = ""
  686. if attr.within_apex:
  687. # //build/bazel/rules/apex:apex_name has to be set to a non_empty value,
  688. # otherwise //build/bazel/rules/apex:non_apex will be true and the
  689. # "-D__ANDROID_APEX__" compiler flag will be missing. Apex_name is used
  690. # in some validation on bazel side which don't really apply in mixed
  691. # build because soong will do the work, so we just set it to a fixed
  692. # value here.
  693. apex_name = "dcla_apex"
  694. outputs = {
  695. "//command_line_option:platforms": "@soong_injection//product_config_platforms/products/{PRODUCT}-{VARIANT}:%s" % target,
  696. "@//build/bazel/rules/apex:within_apex": attr.within_apex,
  697. "@//build/bazel/rules/apex:min_sdk_version": attr.apex_sdk_version,
  698. "@//build/bazel/rules/apex:apex_name": apex_name,
  699. }
  700. return outputs
  701. _config_node_transition = transition(
  702. implementation = _config_node_transition_impl,
  703. inputs = [],
  704. outputs = [
  705. "//command_line_option:platforms",
  706. "@//build/bazel/rules/apex:within_apex",
  707. "@//build/bazel/rules/apex:min_sdk_version",
  708. "@//build/bazel/rules/apex:apex_name",
  709. ],
  710. )
  711. def _passthrough_rule_impl(ctx):
  712. return [DefaultInfo(files = depset(ctx.files.deps))]
  713. config_node = rule(
  714. implementation = _passthrough_rule_impl,
  715. attrs = {
  716. "arch" : attr.string(mandatory = True),
  717. "os" : attr.string(mandatory = True),
  718. "within_apex" : attr.bool(default = False),
  719. "apex_sdk_version" : attr.string(mandatory = True),
  720. "deps" : attr.label_list(cfg = _config_node_transition, allow_files = True),
  721. "_allowlist_function_transition": attr.label(default = "@bazel_tools//tools/allowlists/function_transition_allowlist"),
  722. },
  723. )
  724. # Rule representing the root of the build, to depend on all Bazel targets that
  725. # are required for the build. Building this target will build the entire Bazel
  726. # build tree.
  727. mixed_build_root = rule(
  728. implementation = _passthrough_rule_impl,
  729. attrs = {
  730. "deps" : attr.label_list(),
  731. },
  732. )
  733. def _phony_root_impl(ctx):
  734. return []
  735. # Rule to depend on other targets but build nothing.
  736. # This is useful as follows: building a target of this rule will generate
  737. # symlink forests for all dependencies of the target, without executing any
  738. # actions of the build.
  739. phony_root = rule(
  740. implementation = _phony_root_impl,
  741. attrs = {"deps" : attr.label_list()},
  742. )
  743. `
  744. productReplacer := strings.NewReplacer(
  745. "{PRODUCT}", context.targetProduct,
  746. "{VARIANT}", context.targetBuildVariant)
  747. return []byte(productReplacer.Replace(contents))
  748. }
  749. func (context *mixedBuildBazelContext) mainBuildFileContents() []byte {
  750. // TODO(cparsons): Map label to attribute programmatically; don't use hard-coded
  751. // architecture mapping.
  752. formatString := `
  753. # This file is generated by soong_build. Do not edit.
  754. load(":main.bzl", "config_node", "mixed_build_root", "phony_root")
  755. %s
  756. mixed_build_root(name = "buildroot",
  757. deps = [%s],
  758. testonly = True, # Unblocks testonly deps.
  759. )
  760. phony_root(name = "phonyroot",
  761. deps = [":buildroot"],
  762. testonly = True, # Unblocks testonly deps.
  763. )
  764. `
  765. configNodeFormatString := `
  766. config_node(name = "%s",
  767. arch = "%s",
  768. os = "%s",
  769. within_apex = %s,
  770. apex_sdk_version = "%s",
  771. deps = [%s],
  772. testonly = True, # Unblocks testonly deps.
  773. )
  774. `
  775. configNodesSection := ""
  776. labelsByConfig := map[string][]string{}
  777. for _, val := range context.requests {
  778. labelString := fmt.Sprintf("\"@%s\"", val.label)
  779. configString := getConfigString(val)
  780. labelsByConfig[configString] = append(labelsByConfig[configString], labelString)
  781. }
  782. // Configs need to be sorted to maintain determinism of the BUILD file.
  783. sortedConfigs := make([]string, 0, len(labelsByConfig))
  784. for val := range labelsByConfig {
  785. sortedConfigs = append(sortedConfigs, val)
  786. }
  787. sort.Slice(sortedConfigs, func(i, j int) bool { return sortedConfigs[i] < sortedConfigs[j] })
  788. allLabels := []string{}
  789. for _, configString := range sortedConfigs {
  790. labels := labelsByConfig[configString]
  791. configTokens := strings.Split(configString, "|")
  792. if len(configTokens) < 2 {
  793. panic(fmt.Errorf("Unexpected config string format: %s", configString))
  794. }
  795. archString := configTokens[0]
  796. osString := configTokens[1]
  797. withinApex := "False"
  798. apexSdkVerString := ""
  799. targetString := fmt.Sprintf("%s_%s", osString, archString)
  800. if len(configTokens) > 2 {
  801. targetString += "_" + configTokens[2]
  802. if configTokens[2] == withinApexToString(true) {
  803. withinApex = "True"
  804. }
  805. }
  806. if len(configTokens) > 3 {
  807. targetString += "_" + configTokens[3]
  808. apexSdkVerString = configTokens[3]
  809. }
  810. allLabels = append(allLabels, fmt.Sprintf("\":%s\"", targetString))
  811. labelsString := strings.Join(labels, ",\n ")
  812. configNodesSection += fmt.Sprintf(configNodeFormatString, targetString, archString, osString, withinApex, apexSdkVerString,
  813. labelsString)
  814. }
  815. return []byte(fmt.Sprintf(formatString, configNodesSection, strings.Join(allLabels, ",\n ")))
  816. }
  817. func indent(original string) string {
  818. result := ""
  819. for _, line := range strings.Split(original, "\n") {
  820. result += " " + line + "\n"
  821. }
  822. return result
  823. }
  824. // Returns the file contents of the buildroot.cquery file that should be used for the cquery
  825. // expression in order to obtain information about buildroot and its dependencies.
  826. // The contents of this file depend on the mixedBuildBazelContext's requests; requests are enumerated
  827. // and grouped by their request type. The data retrieved for each label depends on its
  828. // request type.
  829. func (context *mixedBuildBazelContext) cqueryStarlarkFileContents() []byte {
  830. requestTypeToCqueryIdEntries := map[cqueryRequest][]string{}
  831. for _, val := range context.requests {
  832. cqueryId := getCqueryId(val)
  833. mapEntryString := fmt.Sprintf("%q : True", cqueryId)
  834. requestTypeToCqueryIdEntries[val.requestType] =
  835. append(requestTypeToCqueryIdEntries[val.requestType], mapEntryString)
  836. }
  837. labelRegistrationMapSection := ""
  838. functionDefSection := ""
  839. mainSwitchSection := ""
  840. mapDeclarationFormatString := `
  841. %s = {
  842. %s
  843. }
  844. `
  845. functionDefFormatString := `
  846. def %s(target, id_string):
  847. %s
  848. `
  849. mainSwitchSectionFormatString := `
  850. if id_string in %s:
  851. return id_string + ">>" + %s(target, id_string)
  852. `
  853. for requestType := range requestTypeToCqueryIdEntries {
  854. labelMapName := requestType.Name() + "_Labels"
  855. functionName := requestType.Name() + "_Fn"
  856. labelRegistrationMapSection += fmt.Sprintf(mapDeclarationFormatString,
  857. labelMapName,
  858. strings.Join(requestTypeToCqueryIdEntries[requestType], ",\n "))
  859. functionDefSection += fmt.Sprintf(functionDefFormatString,
  860. functionName,
  861. indent(requestType.StarlarkFunctionBody()))
  862. mainSwitchSection += fmt.Sprintf(mainSwitchSectionFormatString,
  863. labelMapName, functionName)
  864. }
  865. formatString := `
  866. # This file is generated by soong_build. Do not edit.
  867. # a drop-in replacement for json.encode(), not available in cquery environment
  868. # TODO(cparsons): bring json module in and remove this function
  869. def json_encode(input):
  870. # Avoiding recursion by limiting
  871. # - a dict to contain anything except a dict
  872. # - a list to contain only primitives
  873. def encode_primitive(p):
  874. t = type(p)
  875. if t == "string" or t == "int":
  876. return repr(p)
  877. fail("unsupported value '%s' of type '%s'" % (p, type(p)))
  878. def encode_list(list):
  879. return "[%s]" % ", ".join([encode_primitive(item) for item in list])
  880. def encode_list_or_primitive(v):
  881. return encode_list(v) if type(v) == "list" else encode_primitive(v)
  882. if type(input) == "dict":
  883. # TODO(juu): the result is read line by line so can't use '\n' yet
  884. kv_pairs = [("%s: %s" % (encode_primitive(k), encode_list_or_primitive(v))) for (k, v) in input.items()]
  885. return "{ %s }" % ", ".join(kv_pairs)
  886. else:
  887. return encode_list_or_primitive(input)
  888. {LABEL_REGISTRATION_MAP_SECTION}
  889. {FUNCTION_DEF_SECTION}
  890. def get_arch(target):
  891. # TODO(b/199363072): filegroups and file targets aren't associated with any
  892. # specific platform architecture in mixed builds. This is consistent with how
  893. # Soong treats filegroups, but it may not be the case with manually-written
  894. # filegroup BUILD targets.
  895. buildoptions = build_options(target)
  896. if buildoptions == None:
  897. # File targets do not have buildoptions. File targets aren't associated with
  898. # any specific platform architecture in mixed builds, so use the host.
  899. return "x86_64|linux"
  900. platforms = buildoptions["//command_line_option:platforms"]
  901. if len(platforms) != 1:
  902. # An individual configured target should have only one platform architecture.
  903. # Note that it's fine for there to be multiple architectures for the same label,
  904. # but each is its own configured target.
  905. fail("expected exactly 1 platform for " + str(target.label) + " but got " + str(platforms))
  906. platform_name = platforms[0].name
  907. if platform_name == "host":
  908. return "HOST"
  909. if not platform_name.startswith("{TARGET_PRODUCT}-{TARGET_BUILD_VARIANT}"):
  910. fail("expected platform name of the form '{TARGET_PRODUCT}-{TARGET_BUILD_VARIANT}_android_<arch>' or '{TARGET_PRODUCT}-{TARGET_BUILD_VARIANT}_linux_<arch>', but was " + str(platforms))
  911. platform_name = platform_name.removeprefix("{TARGET_PRODUCT}-{TARGET_BUILD_VARIANT}").removeprefix("_")
  912. config_key = ""
  913. if not platform_name:
  914. config_key = "target|android"
  915. elif platform_name.startswith("android_"):
  916. config_key = platform_name.removeprefix("android_") + "|android"
  917. elif platform_name.startswith("linux_"):
  918. config_key = platform_name.removeprefix("linux_") + "|linux"
  919. else:
  920. fail("expected platform name of the form '{TARGET_PRODUCT}-{TARGET_BUILD_VARIANT}_android_<arch>' or '{TARGET_PRODUCT}-{TARGET_BUILD_VARIANT}_linux_<arch>', but was " + str(platforms))
  921. within_apex = buildoptions.get("//build/bazel/rules/apex:within_apex")
  922. apex_sdk_version = buildoptions.get("//build/bazel/rules/apex:min_sdk_version")
  923. if within_apex:
  924. config_key += "|within_apex"
  925. if apex_sdk_version != None and len(apex_sdk_version) > 0:
  926. config_key += "|" + apex_sdk_version
  927. return config_key
  928. def format(target):
  929. id_string = str(target.label) + "|" + get_arch(target)
  930. # TODO(b/248106697): Remove once Bazel is updated to always normalize labels.
  931. if id_string.startswith("//"):
  932. id_string = "@" + id_string
  933. {MAIN_SWITCH_SECTION}
  934. # This target was not requested via cquery, and thus must be a dependency
  935. # of a requested target.
  936. return id_string + ">>NONE"
  937. `
  938. replacer := strings.NewReplacer(
  939. "{TARGET_PRODUCT}", context.targetProduct,
  940. "{TARGET_BUILD_VARIANT}", context.targetBuildVariant,
  941. "{LABEL_REGISTRATION_MAP_SECTION}", labelRegistrationMapSection,
  942. "{FUNCTION_DEF_SECTION}", functionDefSection,
  943. "{MAIN_SWITCH_SECTION}", mainSwitchSection)
  944. return []byte(replacer.Replace(formatString))
  945. }
  946. // Returns a path containing build-related metadata required for interfacing
  947. // with Bazel. Example: out/soong/bazel.
  948. func (p *bazelPaths) intermediatesDir() string {
  949. return filepath.Join(p.soongOutDir, "bazel")
  950. }
  951. // Returns the path where the contents of the @soong_injection repository live.
  952. // It is used by Soong to tell Bazel things it cannot over the command line.
  953. func (p *bazelPaths) injectedFilesDir() string {
  954. return filepath.Join(p.soongOutDir, bazel.SoongInjectionDirName)
  955. }
  956. // Returns the path of the synthetic Bazel workspace that contains a symlink
  957. // forest composed the whole source tree and BUILD files generated by bp2build.
  958. func (p *bazelPaths) syntheticWorkspaceDir() string {
  959. return filepath.Join(p.soongOutDir, "workspace")
  960. }
  961. // Returns the path to the top level out dir ($OUT_DIR).
  962. func (p *bazelPaths) outDir() string {
  963. return filepath.Dir(p.soongOutDir)
  964. }
  965. const buildrootLabel = "@soong_injection//mixed_builds:buildroot"
  966. var (
  967. cqueryCmd = bazelCommand{"cquery", fmt.Sprintf("deps(%s, 2)", buildrootLabel)}
  968. aqueryCmd = bazelCommand{"aquery", fmt.Sprintf("deps(%s)", buildrootLabel)}
  969. buildCmd = bazelCommand{"build", "@soong_injection//mixed_builds:phonyroot"}
  970. )
  971. // Issues commands to Bazel to receive results for all cquery requests
  972. // queued in the BazelContext.
  973. func (context *mixedBuildBazelContext) InvokeBazel(config Config, ctx invokeBazelContext) error {
  974. eventHandler := ctx.GetEventHandler()
  975. eventHandler.Begin("bazel")
  976. defer eventHandler.End("bazel")
  977. if metricsDir := context.paths.BazelMetricsDir(); metricsDir != "" {
  978. if err := os.MkdirAll(metricsDir, 0777); err != nil {
  979. return err
  980. }
  981. }
  982. context.results = make(map[cqueryKey]string)
  983. if err := context.runCquery(config, ctx); err != nil {
  984. return err
  985. }
  986. if err := context.runAquery(config, ctx); err != nil {
  987. return err
  988. }
  989. if err := context.generateBazelSymlinks(config, ctx); err != nil {
  990. return err
  991. }
  992. // Clear requests.
  993. context.requests = []cqueryKey{}
  994. return nil
  995. }
  996. func (context *mixedBuildBazelContext) runCquery(config Config, ctx invokeBazelContext) error {
  997. eventHandler := ctx.GetEventHandler()
  998. eventHandler.Begin("cquery")
  999. defer eventHandler.End("cquery")
  1000. soongInjectionPath := absolutePath(context.paths.injectedFilesDir())
  1001. mixedBuildsPath := filepath.Join(soongInjectionPath, "mixed_builds")
  1002. if _, err := os.Stat(mixedBuildsPath); os.IsNotExist(err) {
  1003. err = os.MkdirAll(mixedBuildsPath, 0777)
  1004. if err != nil {
  1005. return err
  1006. }
  1007. }
  1008. if err := writeFileBytesIfChanged(filepath.Join(soongInjectionPath, "WORKSPACE.bazel"), []byte{}, 0666); err != nil {
  1009. return err
  1010. }
  1011. if err := writeFileBytesIfChanged(filepath.Join(mixedBuildsPath, "main.bzl"), context.mainBzlFileContents(), 0666); err != nil {
  1012. return err
  1013. }
  1014. if err := writeFileBytesIfChanged(filepath.Join(mixedBuildsPath, "BUILD.bazel"), context.mainBuildFileContents(), 0666); err != nil {
  1015. return err
  1016. }
  1017. cqueryFileRelpath := filepath.Join(context.paths.injectedFilesDir(), "buildroot.cquery")
  1018. if err := writeFileBytesIfChanged(absolutePath(cqueryFileRelpath), context.cqueryStarlarkFileContents(), 0666); err != nil {
  1019. return err
  1020. }
  1021. extraFlags := []string{"--output=starlark", "--starlark:file=" + absolutePath(cqueryFileRelpath)}
  1022. if Bool(config.productVariables.ClangCoverage) {
  1023. extraFlags = append(extraFlags, "--collect_code_coverage")
  1024. }
  1025. cqueryCommandWithFlag := context.createBazelCommand(config, context.paths, bazel.CqueryBuildRootRunName, cqueryCmd, extraFlags...)
  1026. cqueryOutput, cqueryErrorMessage, cqueryErr := context.issueBazelCommand(cqueryCommandWithFlag, eventHandler)
  1027. if cqueryErr != nil {
  1028. return cqueryErr
  1029. }
  1030. cqueryCommandPrint := fmt.Sprintf("cquery command line:\n %s \n\n\n", printableCqueryCommand(cqueryCommandWithFlag))
  1031. if err := os.WriteFile(filepath.Join(soongInjectionPath, "cquery.out"), []byte(cqueryCommandPrint+cqueryOutput), 0666); err != nil {
  1032. return err
  1033. }
  1034. cqueryResults := map[string]string{}
  1035. for _, outputLine := range strings.Split(cqueryOutput, "\n") {
  1036. if strings.Contains(outputLine, ">>") {
  1037. splitLine := strings.SplitN(outputLine, ">>", 2)
  1038. cqueryResults[splitLine[0]] = splitLine[1]
  1039. }
  1040. }
  1041. for _, val := range context.requests {
  1042. if cqueryResult, ok := cqueryResults[getCqueryId(val)]; ok {
  1043. context.results[val] = cqueryResult
  1044. } else {
  1045. return fmt.Errorf("missing result for bazel target %s. query output: [%s], cquery err: [%s]",
  1046. getCqueryId(val), cqueryOutput, cqueryErrorMessage)
  1047. }
  1048. }
  1049. return nil
  1050. }
  1051. func writeFileBytesIfChanged(path string, contents []byte, perm os.FileMode) error {
  1052. oldContents, err := os.ReadFile(path)
  1053. if err != nil || !bytes.Equal(contents, oldContents) {
  1054. err = os.WriteFile(path, contents, perm)
  1055. }
  1056. return nil
  1057. }
  1058. func (context *mixedBuildBazelContext) runAquery(config Config, ctx invokeBazelContext) error {
  1059. eventHandler := ctx.GetEventHandler()
  1060. eventHandler.Begin("aquery")
  1061. defer eventHandler.End("aquery")
  1062. // Issue an aquery command to retrieve action information about the bazel build tree.
  1063. //
  1064. // Use jsonproto instead of proto; actual proto parsing would require a dependency on Bazel's
  1065. // proto sources, which would add a number of unnecessary dependencies.
  1066. extraFlags := []string{"--output=proto", "--include_file_write_contents"}
  1067. if Bool(config.productVariables.ClangCoverage) {
  1068. extraFlags = append(extraFlags, "--collect_code_coverage")
  1069. paths := make([]string, 0, 2)
  1070. if p := config.productVariables.NativeCoveragePaths; len(p) > 0 {
  1071. for i := range p {
  1072. // TODO(b/259404593) convert path wildcard to regex values
  1073. if p[i] == "*" {
  1074. p[i] = ".*"
  1075. }
  1076. }
  1077. paths = append(paths, JoinWithPrefixAndSeparator(p, "+", ","))
  1078. }
  1079. if p := config.productVariables.NativeCoverageExcludePaths; len(p) > 0 {
  1080. paths = append(paths, JoinWithPrefixAndSeparator(p, "-", ","))
  1081. }
  1082. if len(paths) > 0 {
  1083. extraFlags = append(extraFlags, "--instrumentation_filter="+strings.Join(paths, ","))
  1084. }
  1085. }
  1086. aqueryOutput, _, err := context.issueBazelCommand(context.createBazelCommand(config, context.paths, bazel.AqueryBuildRootRunName, aqueryCmd,
  1087. extraFlags...), eventHandler)
  1088. if err != nil {
  1089. return err
  1090. }
  1091. context.buildStatements, context.depsets, err = bazel.AqueryBuildStatements([]byte(aqueryOutput), eventHandler)
  1092. return err
  1093. }
  1094. func (context *mixedBuildBazelContext) generateBazelSymlinks(config Config, ctx invokeBazelContext) error {
  1095. eventHandler := ctx.GetEventHandler()
  1096. eventHandler.Begin("symlinks")
  1097. defer eventHandler.End("symlinks")
  1098. // Issue a build command of the phony root to generate symlink forests for dependencies of the
  1099. // Bazel build. This is necessary because aquery invocations do not generate this symlink forest,
  1100. // but some of symlinks may be required to resolve source dependencies of the build.
  1101. _, _, err := context.issueBazelCommand(context.createBazelCommand(config, context.paths, bazel.BazelBuildPhonyRootRunName, buildCmd), eventHandler)
  1102. return err
  1103. }
  1104. func (context *mixedBuildBazelContext) BuildStatementsToRegister() []*bazel.BuildStatement {
  1105. return context.buildStatements
  1106. }
  1107. func (context *mixedBuildBazelContext) AqueryDepsets() []bazel.AqueryDepset {
  1108. return context.depsets
  1109. }
  1110. func (context *mixedBuildBazelContext) OutputBase() string {
  1111. return context.paths.outputBase
  1112. }
  1113. // Singleton used for registering BUILD file ninja dependencies (needed
  1114. // for correctness of builds which use Bazel.
  1115. func BazelSingleton() Singleton {
  1116. return &bazelSingleton{}
  1117. }
  1118. type bazelSingleton struct{}
  1119. func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) {
  1120. // bazelSingleton is a no-op if mixed-soong-bazel-builds are disabled.
  1121. if !ctx.Config().IsMixedBuildsEnabled() {
  1122. return
  1123. }
  1124. // Add ninja file dependencies for files which all bazel invocations require.
  1125. bazelBuildList := absolutePath(filepath.Join(
  1126. filepath.Dir(ctx.Config().moduleListFile), "bazel.list"))
  1127. ctx.AddNinjaFileDeps(bazelBuildList)
  1128. data, err := os.ReadFile(bazelBuildList)
  1129. if err != nil {
  1130. ctx.Errorf(err.Error())
  1131. }
  1132. files := strings.Split(strings.TrimSpace(string(data)), "\n")
  1133. for _, file := range files {
  1134. ctx.AddNinjaFileDeps(file)
  1135. }
  1136. for _, depset := range ctx.Config().BazelContext.AqueryDepsets() {
  1137. var outputs []Path
  1138. var orderOnlies []Path
  1139. for _, depsetDepHash := range depset.TransitiveDepSetHashes {
  1140. otherDepsetName := bazelDepsetName(depsetDepHash)
  1141. outputs = append(outputs, PathForPhony(ctx, otherDepsetName))
  1142. }
  1143. for _, artifactPath := range depset.DirectArtifacts {
  1144. pathInBazelOut := PathForBazelOut(ctx, artifactPath)
  1145. if artifactPath == "bazel-out/volatile-status.txt" {
  1146. // See https://bazel.build/docs/user-manual#workspace-status
  1147. orderOnlies = append(orderOnlies, pathInBazelOut)
  1148. } else {
  1149. outputs = append(outputs, pathInBazelOut)
  1150. }
  1151. }
  1152. thisDepsetName := bazelDepsetName(depset.ContentHash)
  1153. ctx.Build(pctx, BuildParams{
  1154. Rule: blueprint.Phony,
  1155. Outputs: []WritablePath{PathForPhony(ctx, thisDepsetName)},
  1156. Implicits: outputs,
  1157. OrderOnly: orderOnlies,
  1158. })
  1159. }
  1160. executionRoot := path.Join(ctx.Config().BazelContext.OutputBase(), "execroot", "__main__")
  1161. bazelOutDir := path.Join(executionRoot, "bazel-out")
  1162. for index, buildStatement := range ctx.Config().BazelContext.BuildStatementsToRegister() {
  1163. // nil build statements are a valid case where we do not create an action because it is
  1164. // unnecessary or handled by other processing
  1165. if buildStatement == nil {
  1166. continue
  1167. }
  1168. if len(buildStatement.Command) > 0 {
  1169. rule := NewRuleBuilder(pctx, ctx)
  1170. createCommand(rule.Command(), buildStatement, executionRoot, bazelOutDir, ctx)
  1171. desc := fmt.Sprintf("%s: %s", buildStatement.Mnemonic, buildStatement.OutputPaths)
  1172. rule.Build(fmt.Sprintf("bazel %d", index), desc)
  1173. continue
  1174. }
  1175. // Certain actions returned by aquery (for instance FileWrite) do not contain a command
  1176. // and thus require special treatment. If BuildStatement were an interface implementing
  1177. // buildRule(ctx) function, the code here would just call it.
  1178. // Unfortunately, the BuildStatement is defined in
  1179. // the 'bazel' package, which cannot depend on 'android' package where ctx is defined,
  1180. // because this would cause circular dependency. So, until we move aquery processing
  1181. // to the 'android' package, we need to handle special cases here.
  1182. switch buildStatement.Mnemonic {
  1183. case "FileWrite", "SourceSymlinkManifest":
  1184. out := PathForBazelOut(ctx, buildStatement.OutputPaths[0])
  1185. WriteFileRuleVerbatim(ctx, out, buildStatement.FileContents)
  1186. case "SymlinkTree":
  1187. // build-runfiles arguments are the manifest file and the target directory
  1188. // where it creates the symlink tree according to this manifest (and then
  1189. // writes the MANIFEST file to it).
  1190. outManifest := PathForBazelOut(ctx, buildStatement.OutputPaths[0])
  1191. outManifestPath := outManifest.String()
  1192. if !strings.HasSuffix(outManifestPath, "MANIFEST") {
  1193. panic("the base name of the symlink tree action should be MANIFEST, got " + outManifestPath)
  1194. }
  1195. outDir := filepath.Dir(outManifestPath)
  1196. ctx.Build(pctx, BuildParams{
  1197. Rule: buildRunfilesRule,
  1198. Output: outManifest,
  1199. Inputs: []Path{PathForBazelOut(ctx, buildStatement.InputPaths[0])},
  1200. Description: "symlink tree for " + outDir,
  1201. Args: map[string]string{
  1202. "outDir": outDir,
  1203. },
  1204. })
  1205. default:
  1206. panic(fmt.Sprintf("unhandled build statement: %v", buildStatement))
  1207. }
  1208. }
  1209. }
  1210. // Register bazel-owned build statements (obtained from the aquery invocation).
  1211. func createCommand(cmd *RuleBuilderCommand, buildStatement *bazel.BuildStatement, executionRoot string, bazelOutDir string, ctx BuilderContext) {
  1212. // executionRoot is the action cwd.
  1213. cmd.Text(fmt.Sprintf("cd '%s' &&", executionRoot))
  1214. // Remove old outputs, as some actions might not rerun if the outputs are detected.
  1215. if len(buildStatement.OutputPaths) > 0 {
  1216. cmd.Text("rm -rf") // -r because outputs can be Bazel dir/tree artifacts.
  1217. for _, outputPath := range buildStatement.OutputPaths {
  1218. cmd.Text(fmt.Sprintf("'%s'", outputPath))
  1219. }
  1220. cmd.Text("&&")
  1221. }
  1222. for _, pair := range buildStatement.Env {
  1223. // Set per-action env variables, if any.
  1224. cmd.Flag(pair.Key + "=" + pair.Value)
  1225. }
  1226. // The actual Bazel action.
  1227. if len(buildStatement.Command) > 16*1024 {
  1228. commandFile := PathForBazelOut(ctx, buildStatement.OutputPaths[0]+".sh")
  1229. WriteFileRule(ctx, commandFile, buildStatement.Command)
  1230. cmd.Text("bash").Text(buildStatement.OutputPaths[0] + ".sh").Implicit(commandFile)
  1231. } else {
  1232. cmd.Text(buildStatement.Command)
  1233. }
  1234. for _, outputPath := range buildStatement.OutputPaths {
  1235. cmd.ImplicitOutput(PathForBazelOut(ctx, outputPath))
  1236. }
  1237. for _, inputPath := range buildStatement.InputPaths {
  1238. cmd.Implicit(PathForBazelOut(ctx, inputPath))
  1239. }
  1240. for _, inputDepsetHash := range buildStatement.InputDepsetHashes {
  1241. otherDepsetName := bazelDepsetName(inputDepsetHash)
  1242. cmd.Implicit(PathForPhony(ctx, otherDepsetName))
  1243. }
  1244. if depfile := buildStatement.Depfile; depfile != nil {
  1245. // The paths in depfile are relative to `executionRoot`.
  1246. // Hence, they need to be corrected by replacing "bazel-out"
  1247. // with the full `bazelOutDir`.
  1248. // Otherwise, implicit outputs and implicit inputs under "bazel-out/"
  1249. // would be deemed missing.
  1250. // (Note: The regexp uses a capture group because the version of sed
  1251. // does not support a look-behind pattern.)
  1252. replacement := fmt.Sprintf(`&& sed -i'' -E 's@(^|\s|")bazel-out/@\1%s/@g' '%s'`,
  1253. bazelOutDir, *depfile)
  1254. cmd.Text(replacement)
  1255. cmd.ImplicitDepFile(PathForBazelOut(ctx, *depfile))
  1256. }
  1257. for _, symlinkPath := range buildStatement.SymlinkPaths {
  1258. cmd.ImplicitSymlinkOutput(PathForBazelOut(ctx, symlinkPath))
  1259. }
  1260. }
  1261. func getCqueryId(key cqueryKey) string {
  1262. return key.label + "|" + getConfigString(key)
  1263. }
  1264. func getConfigString(key cqueryKey) string {
  1265. arch := key.configKey.arch
  1266. if len(arch) == 0 || arch == "common" {
  1267. if key.configKey.osType.Class == Device {
  1268. // For the generic Android, the expected result is "target|android", which
  1269. // corresponds to the product_variable_config named "android_target" in
  1270. // build/bazel/platforms/BUILD.bazel.
  1271. arch = "target"
  1272. } else {
  1273. // Use host platform, which is currently hardcoded to be x86_64.
  1274. arch = "x86_64"
  1275. }
  1276. }
  1277. osName := key.configKey.osType.Name
  1278. if len(osName) == 0 || osName == "common_os" || osName == "linux_glibc" || osName == "linux_musl" {
  1279. // Use host OS, which is currently hardcoded to be linux.
  1280. osName = "linux"
  1281. }
  1282. keyString := arch + "|" + osName
  1283. if key.configKey.apexKey.WithinApex {
  1284. keyString += "|" + withinApexToString(key.configKey.apexKey.WithinApex)
  1285. }
  1286. if len(key.configKey.apexKey.ApexSdkVersion) > 0 {
  1287. keyString += "|" + key.configKey.apexKey.ApexSdkVersion
  1288. }
  1289. return keyString
  1290. }
  1291. func GetConfigKey(ctx BaseModuleContext) configKey {
  1292. return configKey{
  1293. // use string because Arch is not a valid key in go
  1294. arch: ctx.Arch().String(),
  1295. osType: ctx.Os(),
  1296. }
  1297. }
  1298. func GetConfigKeyApexVariant(ctx BaseModuleContext, apexKey *ApexConfigKey) configKey {
  1299. configKey := GetConfigKey(ctx)
  1300. if apexKey != nil {
  1301. configKey.apexKey = ApexConfigKey{
  1302. WithinApex: apexKey.WithinApex,
  1303. ApexSdkVersion: apexKey.ApexSdkVersion,
  1304. }
  1305. }
  1306. return configKey
  1307. }
  1308. func bazelDepsetName(contentHash string) string {
  1309. return fmt.Sprintf("bazel_depset_%s", contentHash)
  1310. }
  1311. func EnvironmentVarsFile(config Config) string {
  1312. return fmt.Sprintf(bazel.GeneratedBazelFileWarning+`
  1313. _env = %s
  1314. env = _env
  1315. `,
  1316. starlark_fmt.PrintStringList(allowedBazelEnvironmentVars, 0),
  1317. )
  1318. }