bazel_handler.go 49 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419
  1. // Copyright 2020 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package android
  15. import (
  16. "bytes"
  17. "fmt"
  18. "os"
  19. "path"
  20. "path/filepath"
  21. "runtime"
  22. "sort"
  23. "strings"
  24. "sync"
  25. "android/soong/android/allowlists"
  26. "android/soong/bazel/cquery"
  27. "android/soong/shared"
  28. "android/soong/starlark_import"
  29. "github.com/google/blueprint"
  30. "github.com/google/blueprint/metrics"
  31. "android/soong/bazel"
  32. )
  33. var (
  34. _ = pctx.HostBinToolVariable("bazelBuildRunfilesTool", "build-runfiles")
  35. buildRunfilesRule = pctx.AndroidStaticRule("bazelBuildRunfiles", blueprint.RuleParams{
  36. Command: "${bazelBuildRunfilesTool} ${in} ${outDir}",
  37. Depfile: "",
  38. Description: "",
  39. CommandDeps: []string{"${bazelBuildRunfilesTool}"},
  40. }, "outDir")
  41. )
  42. func registerMixedBuildsMutator(ctx RegisterMutatorsContext) {
  43. ctx.BottomUp("mixed_builds_prep", mixedBuildsPrepareMutator).Parallel()
  44. }
  45. func RegisterMixedBuildsMutator(ctx RegistrationContext) {
  46. ctx.FinalDepsMutators(registerMixedBuildsMutator)
  47. }
  48. func mixedBuildsPrepareMutator(ctx BottomUpMutatorContext) {
  49. if m := ctx.Module(); m.Enabled() {
  50. if mixedBuildMod, ok := m.(MixedBuildBuildable); ok {
  51. mixedBuildEnabled := MixedBuildsEnabled(ctx)
  52. queueMixedBuild := mixedBuildMod.IsMixedBuildSupported(ctx) && mixedBuildEnabled == MixedBuildEnabled
  53. if queueMixedBuild {
  54. mixedBuildMod.QueueBazelCall(ctx)
  55. }
  56. }
  57. }
  58. }
  59. type cqueryRequest interface {
  60. // Name returns a string name for this request type. Such request type names must be unique,
  61. // and must only consist of alphanumeric characters.
  62. Name() string
  63. // StarlarkFunctionBody returns a starlark function body to process this request type.
  64. // The returned string is the body of a Starlark function which obtains
  65. // all request-relevant information about a target and returns a string containing
  66. // this information.
  67. // The function should have the following properties:
  68. // - The arguments are `target` (a configured target) and `id_string` (the label + configuration).
  69. // - The return value must be a string.
  70. // - The function body should not be indented outside of its own scope.
  71. StarlarkFunctionBody() string
  72. }
  73. // Portion of cquery map key to describe target configuration.
  74. type configKey struct {
  75. arch string
  76. osType OsType
  77. apexKey ApexConfigKey
  78. }
  79. type ApexConfigKey struct {
  80. WithinApex bool
  81. ApexSdkVersion string
  82. }
  83. func (c ApexConfigKey) String() string {
  84. return fmt.Sprintf("%s_%s", withinApexToString(c.WithinApex), c.ApexSdkVersion)
  85. }
  86. func withinApexToString(withinApex bool) string {
  87. if withinApex {
  88. return "within_apex"
  89. }
  90. return ""
  91. }
  92. func (c configKey) String() string {
  93. return fmt.Sprintf("%s::%s::%s", c.arch, c.osType, c.apexKey)
  94. }
  95. // Map key to describe bazel cquery requests.
  96. type cqueryKey struct {
  97. label string
  98. requestType cqueryRequest
  99. configKey configKey
  100. }
  101. func makeCqueryKey(label string, cqueryRequest cqueryRequest, cfgKey configKey) cqueryKey {
  102. if strings.HasPrefix(label, "//") {
  103. // Normalize Bazel labels to specify main repository explicitly.
  104. label = "@" + label
  105. }
  106. return cqueryKey{label, cqueryRequest, cfgKey}
  107. }
  108. func (c cqueryKey) String() string {
  109. return fmt.Sprintf("cquery(%s,%s,%s)", c.label, c.requestType.Name(), c.configKey)
  110. }
  111. type invokeBazelContext interface {
  112. GetEventHandler() *metrics.EventHandler
  113. }
  114. // BazelContext is a context object useful for interacting with Bazel during
  115. // the course of a build. Use of Bazel to evaluate part of the build graph
  116. // is referred to as a "mixed build". (Some modules are managed by Soong,
  117. // some are managed by Bazel). To facilitate interop between these build
  118. // subgraphs, Soong may make requests to Bazel and evaluate their responses
  119. // so that Soong modules may accurately depend on Bazel targets.
  120. type BazelContext interface {
  121. // Add a cquery request to the bazel request queue. All queued requests
  122. // will be sent to Bazel on a subsequent invocation of InvokeBazel.
  123. QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey)
  124. // ** Cquery Results Retrieval Functions
  125. // The below functions pertain to retrieving cquery results from a prior
  126. // InvokeBazel function call and parsing the results.
  127. // Returns result files built by building the given bazel target label.
  128. GetOutputFiles(label string, cfgKey configKey) ([]string, error)
  129. // Returns the results of GetOutputFiles and GetCcObjectFiles in a single query (in that order).
  130. GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error)
  131. // Returns the results of the GetApexInfo query (including output files)
  132. GetApexInfo(label string, cfgkey configKey) (cquery.ApexInfo, error)
  133. // Returns the results of the GetCcUnstrippedInfo query
  134. GetCcUnstrippedInfo(label string, cfgkey configKey) (cquery.CcUnstrippedInfo, error)
  135. // Returns the results of the GetPrebuiltFileInfo query
  136. GetPrebuiltFileInfo(label string, cfgKey configKey) (cquery.PrebuiltFileInfo, error)
  137. // ** end Cquery Results Retrieval Functions
  138. // Issues commands to Bazel to receive results for all cquery requests
  139. // queued in the BazelContext. The ctx argument is optional and is only
  140. // used for performance data collection
  141. InvokeBazel(config Config, ctx invokeBazelContext) error
  142. // Returns true if Bazel handling is enabled for the module with the given name.
  143. // Note that this only implies "bazel mixed build" allowlisting. The caller
  144. // should independently verify the module is eligible for Bazel handling
  145. // (for example, that it is MixedBuildBuildable).
  146. IsModuleNameAllowed(moduleName string, withinApex bool) bool
  147. IsModuleDclaAllowed(moduleName string) bool
  148. // Returns the bazel output base (the root directory for all bazel intermediate outputs).
  149. OutputBase() string
  150. // Returns build statements which should get registered to reflect Bazel's outputs.
  151. BuildStatementsToRegister() []*bazel.BuildStatement
  152. // Returns the depsets defined in Bazel's aquery response.
  153. AqueryDepsets() []bazel.AqueryDepset
  154. }
  155. type bazelRunner interface {
  156. issueBazelCommand(cmdRequest bazel.CmdRequest, paths *bazelPaths, eventHandler *metrics.EventHandler) (output string, errorMessage string, error error)
  157. }
  158. type bazelPaths struct {
  159. homeDir string
  160. bazelPath string
  161. outputBase string
  162. workspaceDir string
  163. soongOutDir string
  164. metricsDir string
  165. bazelDepsFile string
  166. }
  167. // A context object which tracks queued requests that need to be made to Bazel,
  168. // and their results after the requests have been made.
  169. type mixedBuildBazelContext struct {
  170. bazelRunner
  171. paths *bazelPaths
  172. // cquery requests that have not yet been issued to Bazel. This list is maintained
  173. // in a sorted state, and is guaranteed to have no duplicates.
  174. requests []cqueryKey
  175. requestMutex sync.Mutex // requests can be written in parallel
  176. results map[cqueryKey]string // Results of cquery requests after Bazel invocations
  177. // Build statements which should get registered to reflect Bazel's outputs.
  178. buildStatements []*bazel.BuildStatement
  179. // Depsets which should be used for Bazel's build statements.
  180. depsets []bazel.AqueryDepset
  181. // Per-module allowlist/denylist functionality to control whether analysis of
  182. // modules are handled by Bazel. For modules which do not have a Bazel definition
  183. // (or do not sufficiently support bazel handling via MixedBuildBuildable),
  184. // this allowlist will have no effect, even if the module is explicitly allowlisted here.
  185. // Per-module denylist to opt modules out of bazel handling.
  186. bazelDisabledModules map[string]bool
  187. // Per-module allowlist to opt modules in to bazel handling.
  188. bazelEnabledModules map[string]bool
  189. // DCLA modules are enabled when used in apex.
  190. bazelDclaEnabledModules map[string]bool
  191. // If true, modules are bazel-enabled by default, unless present in bazelDisabledModules.
  192. modulesDefaultToBazel bool
  193. targetProduct string
  194. targetBuildVariant string
  195. }
  196. var _ BazelContext = &mixedBuildBazelContext{}
  197. // A bazel context to use when Bazel is disabled.
  198. type noopBazelContext struct{}
  199. var _ BazelContext = noopBazelContext{}
  200. // A bazel context to use for tests.
  201. type MockBazelContext struct {
  202. OutputBaseDir string
  203. LabelToOutputFiles map[string][]string
  204. LabelToCcInfo map[string]cquery.CcInfo
  205. LabelToPythonBinary map[string]string
  206. LabelToApexInfo map[string]cquery.ApexInfo
  207. LabelToCcBinary map[string]cquery.CcUnstrippedInfo
  208. LabelToPrebuiltFileInfo map[string]cquery.PrebuiltFileInfo
  209. BazelRequests map[string]bool
  210. }
  211. func (m MockBazelContext) QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey) {
  212. key := BuildMockBazelContextRequestKey(label, requestType, cfgKey.arch, cfgKey.osType, cfgKey.apexKey)
  213. if m.BazelRequests == nil {
  214. m.BazelRequests = make(map[string]bool)
  215. }
  216. m.BazelRequests[key] = true
  217. }
  218. func (m MockBazelContext) GetOutputFiles(label string, _ configKey) ([]string, error) {
  219. result, ok := m.LabelToOutputFiles[label]
  220. if !ok {
  221. return []string{}, fmt.Errorf("no target with label %q in LabelToOutputFiles", label)
  222. }
  223. return result, nil
  224. }
  225. func (m MockBazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error) {
  226. result, ok := m.LabelToCcInfo[label]
  227. if !ok {
  228. key := BuildMockBazelContextResultKey(label, cfgKey.arch, cfgKey.osType, cfgKey.apexKey)
  229. result, ok = m.LabelToCcInfo[key]
  230. if !ok {
  231. return cquery.CcInfo{}, fmt.Errorf("no target with label %q in LabelToCcInfo", label)
  232. }
  233. }
  234. return result, nil
  235. }
  236. func (m MockBazelContext) GetApexInfo(label string, _ configKey) (cquery.ApexInfo, error) {
  237. result, ok := m.LabelToApexInfo[label]
  238. if !ok {
  239. return cquery.ApexInfo{}, fmt.Errorf("no target with label %q in LabelToApexInfo", label)
  240. }
  241. return result, nil
  242. }
  243. func (m MockBazelContext) GetCcUnstrippedInfo(label string, _ configKey) (cquery.CcUnstrippedInfo, error) {
  244. result, ok := m.LabelToCcBinary[label]
  245. if !ok {
  246. return cquery.CcUnstrippedInfo{}, fmt.Errorf("no target with label %q in LabelToCcBinary", label)
  247. }
  248. return result, nil
  249. }
  250. func (m MockBazelContext) GetPrebuiltFileInfo(label string, _ configKey) (cquery.PrebuiltFileInfo, error) {
  251. result, ok := m.LabelToPrebuiltFileInfo[label]
  252. if !ok {
  253. return cquery.PrebuiltFileInfo{}, fmt.Errorf("no target with label %q in LabelToPrebuiltFileInfo", label)
  254. }
  255. return result, nil
  256. }
  257. func (m MockBazelContext) InvokeBazel(_ Config, _ invokeBazelContext) error {
  258. panic("unimplemented")
  259. }
  260. func (m MockBazelContext) IsModuleNameAllowed(_ string, _ bool) bool {
  261. return true
  262. }
  263. func (m MockBazelContext) IsModuleDclaAllowed(_ string) bool {
  264. return true
  265. }
  266. func (m MockBazelContext) OutputBase() string { return m.OutputBaseDir }
  267. func (m MockBazelContext) BuildStatementsToRegister() []*bazel.BuildStatement {
  268. return []*bazel.BuildStatement{}
  269. }
  270. func (m MockBazelContext) AqueryDepsets() []bazel.AqueryDepset {
  271. return []bazel.AqueryDepset{}
  272. }
  273. var _ BazelContext = MockBazelContext{}
  274. func BuildMockBazelContextRequestKey(label string, request cqueryRequest, arch string, osType OsType, apexKey ApexConfigKey) string {
  275. cfgKey := configKey{
  276. arch: arch,
  277. osType: osType,
  278. apexKey: apexKey,
  279. }
  280. return strings.Join([]string{label, request.Name(), cfgKey.String()}, "_")
  281. }
  282. func BuildMockBazelContextResultKey(label string, arch string, osType OsType, apexKey ApexConfigKey) string {
  283. cfgKey := configKey{
  284. arch: arch,
  285. osType: osType,
  286. apexKey: apexKey,
  287. }
  288. return strings.Join([]string{label, cfgKey.String()}, "_")
  289. }
  290. func (bazelCtx *mixedBuildBazelContext) QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey) {
  291. key := makeCqueryKey(label, requestType, cfgKey)
  292. bazelCtx.requestMutex.Lock()
  293. defer bazelCtx.requestMutex.Unlock()
  294. // Insert key into requests, maintaining the sort, and only if it's not duplicate.
  295. keyString := key.String()
  296. foundEqual := false
  297. notLessThanKeyString := func(i int) bool {
  298. s := bazelCtx.requests[i].String()
  299. v := strings.Compare(s, keyString)
  300. if v == 0 {
  301. foundEqual = true
  302. }
  303. return v >= 0
  304. }
  305. targetIndex := sort.Search(len(bazelCtx.requests), notLessThanKeyString)
  306. if foundEqual {
  307. return
  308. }
  309. if targetIndex == len(bazelCtx.requests) {
  310. bazelCtx.requests = append(bazelCtx.requests, key)
  311. } else {
  312. bazelCtx.requests = append(bazelCtx.requests[:targetIndex+1], bazelCtx.requests[targetIndex:]...)
  313. bazelCtx.requests[targetIndex] = key
  314. }
  315. }
  316. func (bazelCtx *mixedBuildBazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, error) {
  317. key := makeCqueryKey(label, cquery.GetOutputFiles, cfgKey)
  318. if rawString, ok := bazelCtx.results[key]; ok {
  319. bazelOutput := strings.TrimSpace(rawString)
  320. return cquery.GetOutputFiles.ParseResult(bazelOutput), nil
  321. }
  322. return nil, fmt.Errorf("no bazel response found for %v", key)
  323. }
  324. func (bazelCtx *mixedBuildBazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error) {
  325. key := makeCqueryKey(label, cquery.GetCcInfo, cfgKey)
  326. if rawString, ok := bazelCtx.results[key]; ok {
  327. bazelOutput := strings.TrimSpace(rawString)
  328. return cquery.GetCcInfo.ParseResult(bazelOutput)
  329. }
  330. return cquery.CcInfo{}, fmt.Errorf("no bazel response found for %v", key)
  331. }
  332. func (bazelCtx *mixedBuildBazelContext) GetApexInfo(label string, cfgKey configKey) (cquery.ApexInfo, error) {
  333. key := makeCqueryKey(label, cquery.GetApexInfo, cfgKey)
  334. if rawString, ok := bazelCtx.results[key]; ok {
  335. return cquery.GetApexInfo.ParseResult(strings.TrimSpace(rawString))
  336. }
  337. return cquery.ApexInfo{}, fmt.Errorf("no bazel response found for %v", key)
  338. }
  339. func (bazelCtx *mixedBuildBazelContext) GetCcUnstrippedInfo(label string, cfgKey configKey) (cquery.CcUnstrippedInfo, error) {
  340. key := makeCqueryKey(label, cquery.GetCcUnstrippedInfo, cfgKey)
  341. if rawString, ok := bazelCtx.results[key]; ok {
  342. return cquery.GetCcUnstrippedInfo.ParseResult(strings.TrimSpace(rawString))
  343. }
  344. return cquery.CcUnstrippedInfo{}, fmt.Errorf("no bazel response for %s", key)
  345. }
  346. func (bazelCtx *mixedBuildBazelContext) GetPrebuiltFileInfo(label string, cfgKey configKey) (cquery.PrebuiltFileInfo, error) {
  347. key := makeCqueryKey(label, cquery.GetPrebuiltFileInfo, cfgKey)
  348. if rawString, ok := bazelCtx.results[key]; ok {
  349. return cquery.GetPrebuiltFileInfo.ParseResult(strings.TrimSpace(rawString))
  350. }
  351. return cquery.PrebuiltFileInfo{}, fmt.Errorf("no bazel response for %s", key)
  352. }
  353. func (n noopBazelContext) QueueBazelRequest(_ string, _ cqueryRequest, _ configKey) {
  354. panic("unimplemented")
  355. }
  356. func (n noopBazelContext) GetOutputFiles(_ string, _ configKey) ([]string, error) {
  357. panic("unimplemented")
  358. }
  359. func (n noopBazelContext) GetCcInfo(_ string, _ configKey) (cquery.CcInfo, error) {
  360. panic("unimplemented")
  361. }
  362. func (n noopBazelContext) GetApexInfo(_ string, _ configKey) (cquery.ApexInfo, error) {
  363. panic("unimplemented")
  364. }
  365. func (n noopBazelContext) GetCcUnstrippedInfo(_ string, _ configKey) (cquery.CcUnstrippedInfo, error) {
  366. //TODO implement me
  367. panic("implement me")
  368. }
  369. func (n noopBazelContext) GetPrebuiltFileInfo(_ string, _ configKey) (cquery.PrebuiltFileInfo, error) {
  370. panic("implement me")
  371. }
  372. func (n noopBazelContext) InvokeBazel(_ Config, _ invokeBazelContext) error {
  373. panic("unimplemented")
  374. }
  375. func (m noopBazelContext) OutputBase() string {
  376. return ""
  377. }
  378. func (n noopBazelContext) IsModuleNameAllowed(_ string, _ bool) bool {
  379. return false
  380. }
  381. func (n noopBazelContext) IsModuleDclaAllowed(_ string) bool {
  382. return false
  383. }
  384. func (m noopBazelContext) BuildStatementsToRegister() []*bazel.BuildStatement {
  385. return []*bazel.BuildStatement{}
  386. }
  387. func (m noopBazelContext) AqueryDepsets() []bazel.AqueryDepset {
  388. return []bazel.AqueryDepset{}
  389. }
  390. func AddToStringSet(set map[string]bool, items []string) {
  391. for _, item := range items {
  392. set[item] = true
  393. }
  394. }
  395. func GetBazelEnabledAndDisabledModules(buildMode SoongBuildMode, forceEnabled map[string]struct{}) (map[string]bool, map[string]bool) {
  396. disabledModules := map[string]bool{}
  397. enabledModules := map[string]bool{}
  398. switch buildMode {
  399. case BazelProdMode:
  400. AddToStringSet(enabledModules, allowlists.ProdMixedBuildsEnabledList)
  401. for enabledAdHocModule := range forceEnabled {
  402. enabledModules[enabledAdHocModule] = true
  403. }
  404. case BazelStagingMode:
  405. // Staging mode includes all prod modules plus all staging modules.
  406. AddToStringSet(enabledModules, allowlists.ProdMixedBuildsEnabledList)
  407. AddToStringSet(enabledModules, allowlists.StagingMixedBuildsEnabledList)
  408. for enabledAdHocModule := range forceEnabled {
  409. enabledModules[enabledAdHocModule] = true
  410. }
  411. case BazelDevMode:
  412. AddToStringSet(disabledModules, allowlists.MixedBuildsDisabledList)
  413. default:
  414. panic("Expected BazelProdMode, BazelStagingMode, or BazelDevMode")
  415. }
  416. return enabledModules, disabledModules
  417. }
  418. func GetBazelEnabledModules(buildMode SoongBuildMode) []string {
  419. enabledModules, disabledModules := GetBazelEnabledAndDisabledModules(buildMode, nil)
  420. enabledList := make([]string, 0, len(enabledModules))
  421. for module := range enabledModules {
  422. if !disabledModules[module] {
  423. enabledList = append(enabledList, module)
  424. }
  425. }
  426. sort.Strings(enabledList)
  427. return enabledList
  428. }
  429. func NewBazelContext(c *config) (BazelContext, error) {
  430. if c.BuildMode != BazelProdMode && c.BuildMode != BazelStagingMode && c.BuildMode != BazelDevMode {
  431. return noopBazelContext{}, nil
  432. }
  433. enabledModules, disabledModules := GetBazelEnabledAndDisabledModules(c.BuildMode, c.BazelModulesForceEnabledByFlag())
  434. paths := bazelPaths{
  435. soongOutDir: c.soongOutDir,
  436. }
  437. var missing []string
  438. vars := []struct {
  439. name string
  440. ptr *string
  441. // True if the environment variable needs to be tracked so that changes to the variable
  442. // cause the ninja file to be regenerated, false otherwise. False should only be set for
  443. // environment variables that have no effect on the generated ninja file.
  444. track bool
  445. }{
  446. {"BAZEL_HOME", &paths.homeDir, true},
  447. {"BAZEL_PATH", &paths.bazelPath, true},
  448. {"BAZEL_OUTPUT_BASE", &paths.outputBase, true},
  449. {"BAZEL_WORKSPACE", &paths.workspaceDir, true},
  450. {"BAZEL_METRICS_DIR", &paths.metricsDir, false},
  451. {"BAZEL_DEPS_FILE", &paths.bazelDepsFile, true},
  452. }
  453. for _, v := range vars {
  454. if v.track {
  455. if s := c.Getenv(v.name); len(s) > 1 {
  456. *v.ptr = s
  457. continue
  458. }
  459. } else if s, ok := c.env[v.name]; ok {
  460. *v.ptr = s
  461. } else {
  462. missing = append(missing, v.name)
  463. }
  464. }
  465. if len(missing) > 0 {
  466. return nil, fmt.Errorf("missing required env vars to use bazel: %s", missing)
  467. }
  468. targetBuildVariant := "user"
  469. if c.Eng() {
  470. targetBuildVariant = "eng"
  471. } else if c.Debuggable() {
  472. targetBuildVariant = "userdebug"
  473. }
  474. targetProduct := "unknown"
  475. if c.HasDeviceProduct() {
  476. targetProduct = c.DeviceProduct()
  477. }
  478. dclaMixedBuildsEnabledList := []string{}
  479. if c.BuildMode == BazelProdMode {
  480. dclaMixedBuildsEnabledList = allowlists.ProdDclaMixedBuildsEnabledList
  481. } else if c.BuildMode == BazelStagingMode {
  482. dclaMixedBuildsEnabledList = append(allowlists.ProdDclaMixedBuildsEnabledList,
  483. allowlists.StagingDclaMixedBuildsEnabledList...)
  484. }
  485. dclaEnabledModules := map[string]bool{}
  486. AddToStringSet(dclaEnabledModules, dclaMixedBuildsEnabledList)
  487. return &mixedBuildBazelContext{
  488. bazelRunner: &builtinBazelRunner{c.UseBazelProxy, absolutePath(c.outDir)},
  489. paths: &paths,
  490. modulesDefaultToBazel: c.BuildMode == BazelDevMode,
  491. bazelEnabledModules: enabledModules,
  492. bazelDisabledModules: disabledModules,
  493. bazelDclaEnabledModules: dclaEnabledModules,
  494. targetProduct: targetProduct,
  495. targetBuildVariant: targetBuildVariant,
  496. }, nil
  497. }
  498. func (p *bazelPaths) BazelMetricsDir() string {
  499. return p.metricsDir
  500. }
  501. func (context *mixedBuildBazelContext) IsModuleNameAllowed(moduleName string, withinApex bool) bool {
  502. if context.bazelDisabledModules[moduleName] {
  503. return false
  504. }
  505. if context.bazelEnabledModules[moduleName] {
  506. return true
  507. }
  508. if withinApex && context.IsModuleDclaAllowed(moduleName) {
  509. return true
  510. }
  511. return context.modulesDefaultToBazel
  512. }
  513. func (context *mixedBuildBazelContext) IsModuleDclaAllowed(moduleName string) bool {
  514. return context.bazelDclaEnabledModules[moduleName]
  515. }
  516. func pwdPrefix() string {
  517. // Darwin doesn't have /proc
  518. if runtime.GOOS != "darwin" {
  519. return "PWD=/proc/self/cwd"
  520. }
  521. return ""
  522. }
  523. type bazelCommand struct {
  524. command string
  525. // query or label
  526. expression string
  527. }
  528. type builtinBazelRunner struct {
  529. useBazelProxy bool
  530. outDir string
  531. }
  532. // Issues the given bazel command with given build label and additional flags.
  533. // Returns (stdout, stderr, error). The first and second return values are strings
  534. // containing the stdout and stderr of the run command, and an error is returned if
  535. // the invocation returned an error code.
  536. func (r *builtinBazelRunner) issueBazelCommand(cmdRequest bazel.CmdRequest, paths *bazelPaths, eventHandler *metrics.EventHandler) (string, string, error) {
  537. if r.useBazelProxy {
  538. eventHandler.Begin("client_proxy")
  539. defer eventHandler.End("client_proxy")
  540. proxyClient := bazel.NewProxyClient(r.outDir)
  541. resp, err := proxyClient.IssueCommand(cmdRequest)
  542. if err != nil {
  543. return "", "", err
  544. }
  545. if len(resp.ErrorString) > 0 {
  546. return "", "", fmt.Errorf(resp.ErrorString)
  547. }
  548. return resp.Stdout, resp.Stderr, nil
  549. } else {
  550. eventHandler.Begin("bazel command")
  551. defer eventHandler.End("bazel command")
  552. stdout, stderr, err := bazel.ExecBazel(paths.bazelPath, absolutePath(paths.syntheticWorkspaceDir()), cmdRequest)
  553. return string(stdout), string(stderr), err
  554. }
  555. }
  556. func (context *mixedBuildBazelContext) createBazelCommand(config Config, runName bazel.RunName, command bazelCommand,
  557. extraFlags ...string) bazel.CmdRequest {
  558. if runtime.GOOS != "linux" && runtime.GOOS != "darwin" {
  559. panic("Unknown GOOS: " + runtime.GOOS)
  560. }
  561. cmdFlags := []string{
  562. "--output_base=" + absolutePath(context.paths.outputBase),
  563. command.command,
  564. command.expression,
  565. "--profile=" + shared.BazelMetricsFilename(context.paths, runName),
  566. "--host_platform=@soong_injection//product_config_platforms:mixed_builds_product-" + context.targetBuildVariant + "_" + runtime.GOOS + "_x86_64",
  567. // Don't specify --platforms, because on some products/branches (like kernel-build-tools)
  568. // the main platform for mixed_builds_product-variant doesn't exist because an arch isn't
  569. // specified in product config. The derivative platforms that config_node transitions into
  570. // will still work.
  571. // Suppress noise
  572. "--ui_event_filters=-INFO",
  573. "--noshow_progress",
  574. "--norun_validations",
  575. }
  576. cmdFlags = append(cmdFlags, extraFlags...)
  577. extraEnv := []string{
  578. "HOME=" + context.paths.homeDir,
  579. pwdPrefix(),
  580. "BUILD_DIR=" + absolutePath(context.paths.soongOutDir),
  581. // Make OUT_DIR absolute here so build/bazel/bin/bazel uses the correct
  582. // OUT_DIR at <root>/out, instead of <root>/out/soong/workspace/out.
  583. "OUT_DIR=" + absolutePath(context.paths.outDir()),
  584. // Disables local host detection of gcc; toolchain information is defined
  585. // explicitly in BUILD files.
  586. "BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1",
  587. }
  588. capturedEnvVars, err := starlark_import.GetStarlarkValue[[]string]("captured_env_vars")
  589. if err != nil {
  590. panic(err)
  591. }
  592. for _, envvar := range capturedEnvVars {
  593. val := config.Getenv(envvar)
  594. if val == "" {
  595. continue
  596. }
  597. extraEnv = append(extraEnv, fmt.Sprintf("%s=%s", envvar, val))
  598. }
  599. envVars := append(os.Environ(), extraEnv...)
  600. return bazel.CmdRequest{cmdFlags, envVars}
  601. }
  602. func (context *mixedBuildBazelContext) printableCqueryCommand(bazelCmd bazel.CmdRequest) string {
  603. args := append([]string{context.paths.bazelPath}, bazelCmd.Argv...)
  604. outputString := strings.Join(bazelCmd.Env, " ") + " \"" + strings.Join(args, "\" \"") + "\""
  605. return outputString
  606. }
  607. func (context *mixedBuildBazelContext) mainBzlFileContents() []byte {
  608. // TODO(cparsons): Define configuration transitions programmatically based
  609. // on available archs.
  610. contents := `
  611. #####################################################
  612. # This file is generated by soong_build. Do not edit.
  613. #####################################################
  614. def _config_node_transition_impl(settings, attr):
  615. if attr.os == "android" and attr.arch == "target":
  616. target = "mixed_builds_product-{VARIANT}"
  617. else:
  618. target = "mixed_builds_product-{VARIANT}_%s_%s" % (attr.os, attr.arch)
  619. apex_name = ""
  620. if attr.within_apex:
  621. # //build/bazel/rules/apex:apex_name has to be set to a non_empty value,
  622. # otherwise //build/bazel/rules/apex:non_apex will be true and the
  623. # "-D__ANDROID_APEX__" compiler flag will be missing. Apex_name is used
  624. # in some validation on bazel side which don't really apply in mixed
  625. # build because soong will do the work, so we just set it to a fixed
  626. # value here.
  627. apex_name = "dcla_apex"
  628. outputs = {
  629. "//command_line_option:platforms": "@soong_injection//product_config_platforms:%s" % target,
  630. "@//build/bazel/rules/apex:within_apex": attr.within_apex,
  631. "@//build/bazel/rules/apex:min_sdk_version": attr.apex_sdk_version,
  632. "@//build/bazel/rules/apex:apex_name": apex_name,
  633. }
  634. return outputs
  635. _config_node_transition = transition(
  636. implementation = _config_node_transition_impl,
  637. inputs = [],
  638. outputs = [
  639. "//command_line_option:platforms",
  640. "@//build/bazel/rules/apex:within_apex",
  641. "@//build/bazel/rules/apex:min_sdk_version",
  642. "@//build/bazel/rules/apex:apex_name",
  643. ],
  644. )
  645. def _passthrough_rule_impl(ctx):
  646. return [DefaultInfo(files = depset(ctx.files.deps))]
  647. config_node = rule(
  648. implementation = _passthrough_rule_impl,
  649. attrs = {
  650. "arch" : attr.string(mandatory = True),
  651. "os" : attr.string(mandatory = True),
  652. "within_apex" : attr.bool(default = False),
  653. "apex_sdk_version" : attr.string(mandatory = True),
  654. "deps" : attr.label_list(cfg = _config_node_transition, allow_files = True),
  655. "_allowlist_function_transition": attr.label(default = "@bazel_tools//tools/allowlists/function_transition_allowlist"),
  656. },
  657. )
  658. # Rule representing the root of the build, to depend on all Bazel targets that
  659. # are required for the build. Building this target will build the entire Bazel
  660. # build tree.
  661. mixed_build_root = rule(
  662. implementation = _passthrough_rule_impl,
  663. attrs = {
  664. "deps" : attr.label_list(),
  665. },
  666. )
  667. def _phony_root_impl(ctx):
  668. return []
  669. # Rule to depend on other targets but build nothing.
  670. # This is useful as follows: building a target of this rule will generate
  671. # symlink forests for all dependencies of the target, without executing any
  672. # actions of the build.
  673. phony_root = rule(
  674. implementation = _phony_root_impl,
  675. attrs = {"deps" : attr.label_list()},
  676. )
  677. `
  678. productReplacer := strings.NewReplacer(
  679. "{PRODUCT}", context.targetProduct,
  680. "{VARIANT}", context.targetBuildVariant)
  681. return []byte(productReplacer.Replace(contents))
  682. }
  683. func (context *mixedBuildBazelContext) mainBuildFileContents() []byte {
  684. // TODO(cparsons): Map label to attribute programmatically; don't use hard-coded
  685. // architecture mapping.
  686. formatString := `
  687. # This file is generated by soong_build. Do not edit.
  688. load(":main.bzl", "config_node", "mixed_build_root", "phony_root")
  689. %s
  690. mixed_build_root(name = "buildroot",
  691. deps = [%s],
  692. testonly = True, # Unblocks testonly deps.
  693. )
  694. phony_root(name = "phonyroot",
  695. deps = [":buildroot"],
  696. testonly = True, # Unblocks testonly deps.
  697. )
  698. `
  699. configNodeFormatString := `
  700. config_node(name = "%s",
  701. arch = "%s",
  702. os = "%s",
  703. within_apex = %s,
  704. apex_sdk_version = "%s",
  705. deps = [%s],
  706. testonly = True, # Unblocks testonly deps.
  707. )
  708. `
  709. configNodesSection := ""
  710. labelsByConfig := map[string][]string{}
  711. for _, val := range context.requests {
  712. labelString := fmt.Sprintf("\"@%s\"", val.label)
  713. configString := getConfigString(val)
  714. labelsByConfig[configString] = append(labelsByConfig[configString], labelString)
  715. }
  716. // Configs need to be sorted to maintain determinism of the BUILD file.
  717. sortedConfigs := make([]string, 0, len(labelsByConfig))
  718. for val := range labelsByConfig {
  719. sortedConfigs = append(sortedConfigs, val)
  720. }
  721. sort.Slice(sortedConfigs, func(i, j int) bool { return sortedConfigs[i] < sortedConfigs[j] })
  722. allLabels := []string{}
  723. for _, configString := range sortedConfigs {
  724. labels := labelsByConfig[configString]
  725. configTokens := strings.Split(configString, "|")
  726. if len(configTokens) < 2 {
  727. panic(fmt.Errorf("Unexpected config string format: %s", configString))
  728. }
  729. archString := configTokens[0]
  730. osString := configTokens[1]
  731. withinApex := "False"
  732. apexSdkVerString := ""
  733. targetString := fmt.Sprintf("%s_%s", osString, archString)
  734. if len(configTokens) > 2 {
  735. targetString += "_" + configTokens[2]
  736. if configTokens[2] == withinApexToString(true) {
  737. withinApex = "True"
  738. }
  739. }
  740. if len(configTokens) > 3 {
  741. targetString += "_" + configTokens[3]
  742. apexSdkVerString = configTokens[3]
  743. }
  744. allLabels = append(allLabels, fmt.Sprintf("\":%s\"", targetString))
  745. labelsString := strings.Join(labels, ",\n ")
  746. configNodesSection += fmt.Sprintf(configNodeFormatString, targetString, archString, osString, withinApex, apexSdkVerString,
  747. labelsString)
  748. }
  749. return []byte(fmt.Sprintf(formatString, configNodesSection, strings.Join(allLabels, ",\n ")))
  750. }
  751. func indent(original string) string {
  752. result := ""
  753. for _, line := range strings.Split(original, "\n") {
  754. result += " " + line + "\n"
  755. }
  756. return result
  757. }
  758. // Returns the file contents of the buildroot.cquery file that should be used for the cquery
  759. // expression in order to obtain information about buildroot and its dependencies.
  760. // The contents of this file depend on the mixedBuildBazelContext's requests; requests are enumerated
  761. // and grouped by their request type. The data retrieved for each label depends on its
  762. // request type.
  763. func (context *mixedBuildBazelContext) cqueryStarlarkFileContents() []byte {
  764. requestTypeToCqueryIdEntries := map[cqueryRequest][]string{}
  765. requestTypes := []cqueryRequest{}
  766. for _, val := range context.requests {
  767. cqueryId := getCqueryId(val)
  768. mapEntryString := fmt.Sprintf("%q : True", cqueryId)
  769. if _, seenKey := requestTypeToCqueryIdEntries[val.requestType]; !seenKey {
  770. requestTypes = append(requestTypes, val.requestType)
  771. }
  772. requestTypeToCqueryIdEntries[val.requestType] =
  773. append(requestTypeToCqueryIdEntries[val.requestType], mapEntryString)
  774. }
  775. labelRegistrationMapSection := ""
  776. functionDefSection := ""
  777. mainSwitchSection := ""
  778. mapDeclarationFormatString := `
  779. %s = {
  780. %s
  781. }
  782. `
  783. functionDefFormatString := `
  784. def %s(target, id_string):
  785. %s
  786. `
  787. mainSwitchSectionFormatString := `
  788. if id_string in %s:
  789. return id_string + ">>" + %s(target, id_string)
  790. `
  791. for _, requestType := range requestTypes {
  792. labelMapName := requestType.Name() + "_Labels"
  793. functionName := requestType.Name() + "_Fn"
  794. labelRegistrationMapSection += fmt.Sprintf(mapDeclarationFormatString,
  795. labelMapName,
  796. strings.Join(requestTypeToCqueryIdEntries[requestType], ",\n "))
  797. functionDefSection += fmt.Sprintf(functionDefFormatString,
  798. functionName,
  799. indent(requestType.StarlarkFunctionBody()))
  800. mainSwitchSection += fmt.Sprintf(mainSwitchSectionFormatString,
  801. labelMapName, functionName)
  802. }
  803. formatString := `
  804. # This file is generated by soong_build. Do not edit.
  805. {LABEL_REGISTRATION_MAP_SECTION}
  806. {FUNCTION_DEF_SECTION}
  807. def get_arch(target):
  808. # TODO(b/199363072): filegroups and file targets aren't associated with any
  809. # specific platform architecture in mixed builds. This is consistent with how
  810. # Soong treats filegroups, but it may not be the case with manually-written
  811. # filegroup BUILD targets.
  812. buildoptions = build_options(target)
  813. if buildoptions == None:
  814. # File targets do not have buildoptions. File targets aren't associated with
  815. # any specific platform architecture in mixed builds, so use the host.
  816. return "x86_64|linux"
  817. platforms = buildoptions["//command_line_option:platforms"]
  818. if len(platforms) != 1:
  819. # An individual configured target should have only one platform architecture.
  820. # Note that it's fine for there to be multiple architectures for the same label,
  821. # but each is its own configured target.
  822. fail("expected exactly 1 platform for " + str(target.label) + " but got " + str(platforms))
  823. platform_name = platforms[0].name
  824. if platform_name == "host":
  825. return "HOST"
  826. if not platform_name.startswith("mixed_builds_product-{TARGET_BUILD_VARIANT}"):
  827. fail("expected platform name of the form 'mixed_builds_product-{TARGET_BUILD_VARIANT}_android_<arch>' or 'mixed_builds_product-{TARGET_BUILD_VARIANT}_linux_<arch>', but was " + str(platforms))
  828. platform_name = platform_name.removeprefix("mixed_builds_product-{TARGET_BUILD_VARIANT}").removeprefix("_")
  829. config_key = ""
  830. if not platform_name:
  831. config_key = "target|android"
  832. elif platform_name.startswith("android_"):
  833. config_key = platform_name.removeprefix("android_") + "|android"
  834. elif platform_name.startswith("linux_"):
  835. config_key = platform_name.removeprefix("linux_") + "|linux"
  836. else:
  837. fail("expected platform name of the form 'mixed_builds_product-{TARGET_BUILD_VARIANT}_android_<arch>' or 'mixed_builds_product-{TARGET_BUILD_VARIANT}_linux_<arch>', but was " + str(platforms))
  838. within_apex = buildoptions.get("//build/bazel/rules/apex:within_apex")
  839. apex_sdk_version = buildoptions.get("//build/bazel/rules/apex:min_sdk_version")
  840. if within_apex:
  841. config_key += "|within_apex"
  842. if apex_sdk_version != None and len(apex_sdk_version) > 0:
  843. config_key += "|" + apex_sdk_version
  844. return config_key
  845. def format(target):
  846. id_string = str(target.label) + "|" + get_arch(target)
  847. # TODO(b/248106697): Remove once Bazel is updated to always normalize labels.
  848. if id_string.startswith("//"):
  849. id_string = "@" + id_string
  850. {MAIN_SWITCH_SECTION}
  851. # This target was not requested via cquery, and thus must be a dependency
  852. # of a requested target.
  853. return id_string + ">>NONE"
  854. `
  855. replacer := strings.NewReplacer(
  856. "{TARGET_PRODUCT}", context.targetProduct,
  857. "{TARGET_BUILD_VARIANT}", context.targetBuildVariant,
  858. "{LABEL_REGISTRATION_MAP_SECTION}", labelRegistrationMapSection,
  859. "{FUNCTION_DEF_SECTION}", functionDefSection,
  860. "{MAIN_SWITCH_SECTION}", mainSwitchSection)
  861. return []byte(replacer.Replace(formatString))
  862. }
  863. // Returns a path containing build-related metadata required for interfacing
  864. // with Bazel. Example: out/soong/bazel.
  865. func (p *bazelPaths) intermediatesDir() string {
  866. return filepath.Join(p.soongOutDir, "bazel")
  867. }
  868. // Returns the path where the contents of the @soong_injection repository live.
  869. // It is used by Soong to tell Bazel things it cannot over the command line.
  870. func (p *bazelPaths) injectedFilesDir() string {
  871. return filepath.Join(p.soongOutDir, bazel.SoongInjectionDirName)
  872. }
  873. // Returns the path of the synthetic Bazel workspace that contains a symlink
  874. // forest composed the whole source tree and BUILD files generated by bp2build.
  875. func (p *bazelPaths) syntheticWorkspaceDir() string {
  876. return filepath.Join(p.soongOutDir, "workspace")
  877. }
  878. // Returns the path to the top level out dir ($OUT_DIR).
  879. func (p *bazelPaths) outDir() string {
  880. return filepath.Dir(p.soongOutDir)
  881. }
  882. const buildrootLabel = "@soong_injection//mixed_builds:buildroot"
  883. var (
  884. cqueryCmd = bazelCommand{"cquery", fmt.Sprintf("deps(%s, 2)", buildrootLabel)}
  885. aqueryCmd = bazelCommand{"aquery", fmt.Sprintf("deps(%s)", buildrootLabel)}
  886. buildCmd = bazelCommand{"build", "@soong_injection//mixed_builds:phonyroot"}
  887. allBazelCommands = []bazelCommand{aqueryCmd, cqueryCmd, buildCmd}
  888. )
  889. // Issues commands to Bazel to receive results for all cquery requests
  890. // queued in the BazelContext.
  891. func (context *mixedBuildBazelContext) InvokeBazel(config Config, ctx invokeBazelContext) error {
  892. eventHandler := ctx.GetEventHandler()
  893. eventHandler.Begin("bazel")
  894. defer eventHandler.End("bazel")
  895. if metricsDir := context.paths.BazelMetricsDir(); metricsDir != "" {
  896. if err := os.MkdirAll(metricsDir, 0777); err != nil {
  897. return err
  898. }
  899. }
  900. context.results = make(map[cqueryKey]string)
  901. if err := context.runCquery(config, ctx); err != nil {
  902. return err
  903. }
  904. if err := context.runAquery(config, ctx); err != nil {
  905. return err
  906. }
  907. if err := context.generateBazelSymlinks(config, ctx); err != nil {
  908. return err
  909. }
  910. // Clear requests.
  911. context.requests = []cqueryKey{}
  912. return nil
  913. }
  914. func (context *mixedBuildBazelContext) runCquery(config Config, ctx invokeBazelContext) error {
  915. eventHandler := ctx.GetEventHandler()
  916. eventHandler.Begin("cquery")
  917. defer eventHandler.End("cquery")
  918. soongInjectionPath := absolutePath(context.paths.injectedFilesDir())
  919. mixedBuildsPath := filepath.Join(soongInjectionPath, "mixed_builds")
  920. if _, err := os.Stat(mixedBuildsPath); os.IsNotExist(err) {
  921. err = os.MkdirAll(mixedBuildsPath, 0777)
  922. if err != nil {
  923. return err
  924. }
  925. }
  926. if err := writeFileBytesIfChanged(filepath.Join(soongInjectionPath, "WORKSPACE.bazel"), []byte{}, 0666); err != nil {
  927. return err
  928. }
  929. if err := writeFileBytesIfChanged(filepath.Join(mixedBuildsPath, "main.bzl"), context.mainBzlFileContents(), 0666); err != nil {
  930. return err
  931. }
  932. if err := writeFileBytesIfChanged(filepath.Join(mixedBuildsPath, "BUILD.bazel"), context.mainBuildFileContents(), 0666); err != nil {
  933. return err
  934. }
  935. cqueryFileRelpath := filepath.Join(context.paths.injectedFilesDir(), "buildroot.cquery")
  936. if err := writeFileBytesIfChanged(absolutePath(cqueryFileRelpath), context.cqueryStarlarkFileContents(), 0666); err != nil {
  937. return err
  938. }
  939. extraFlags := []string{"--output=starlark", "--starlark:file=" + absolutePath(cqueryFileRelpath)}
  940. if Bool(config.productVariables.ClangCoverage) {
  941. extraFlags = append(extraFlags, "--collect_code_coverage")
  942. }
  943. cqueryCmdRequest := context.createBazelCommand(config, bazel.CqueryBuildRootRunName, cqueryCmd, extraFlags...)
  944. cqueryOutput, cqueryErrorMessage, cqueryErr := context.issueBazelCommand(cqueryCmdRequest, context.paths, eventHandler)
  945. if cqueryErr != nil {
  946. return cqueryErr
  947. }
  948. cqueryCommandPrint := fmt.Sprintf("cquery command line:\n %s \n\n\n", context.printableCqueryCommand(cqueryCmdRequest))
  949. if err := os.WriteFile(filepath.Join(soongInjectionPath, "cquery.out"), []byte(cqueryCommandPrint+cqueryOutput), 0666); err != nil {
  950. return err
  951. }
  952. cqueryResults := map[string]string{}
  953. for _, outputLine := range strings.Split(cqueryOutput, "\n") {
  954. if strings.Contains(outputLine, ">>") {
  955. splitLine := strings.SplitN(outputLine, ">>", 2)
  956. cqueryResults[splitLine[0]] = splitLine[1]
  957. }
  958. }
  959. for _, val := range context.requests {
  960. if cqueryResult, ok := cqueryResults[getCqueryId(val)]; ok {
  961. context.results[val] = cqueryResult
  962. } else {
  963. return fmt.Errorf("missing result for bazel target %s. query output: [%s], cquery err: [%s]",
  964. getCqueryId(val), cqueryOutput, cqueryErrorMessage)
  965. }
  966. }
  967. return nil
  968. }
  969. func writeFileBytesIfChanged(path string, contents []byte, perm os.FileMode) error {
  970. oldContents, err := os.ReadFile(path)
  971. if err != nil || !bytes.Equal(contents, oldContents) {
  972. err = os.WriteFile(path, contents, perm)
  973. }
  974. return nil
  975. }
  976. func (context *mixedBuildBazelContext) runAquery(config Config, ctx invokeBazelContext) error {
  977. eventHandler := ctx.GetEventHandler()
  978. eventHandler.Begin("aquery")
  979. defer eventHandler.End("aquery")
  980. // Issue an aquery command to retrieve action information about the bazel build tree.
  981. //
  982. // Use jsonproto instead of proto; actual proto parsing would require a dependency on Bazel's
  983. // proto sources, which would add a number of unnecessary dependencies.
  984. extraFlags := []string{"--output=proto", "--include_file_write_contents"}
  985. if Bool(config.productVariables.ClangCoverage) {
  986. extraFlags = append(extraFlags, "--collect_code_coverage")
  987. paths := make([]string, 0, 2)
  988. if p := config.productVariables.NativeCoveragePaths; len(p) > 0 {
  989. for i := range p {
  990. // TODO(b/259404593) convert path wildcard to regex values
  991. if p[i] == "*" {
  992. p[i] = ".*"
  993. }
  994. }
  995. paths = append(paths, JoinWithPrefixAndSeparator(p, "+", ","))
  996. }
  997. if p := config.productVariables.NativeCoverageExcludePaths; len(p) > 0 {
  998. paths = append(paths, JoinWithPrefixAndSeparator(p, "-", ","))
  999. }
  1000. if len(paths) > 0 {
  1001. extraFlags = append(extraFlags, "--instrumentation_filter="+strings.Join(paths, ","))
  1002. }
  1003. }
  1004. aqueryOutput, _, err := context.issueBazelCommand(context.createBazelCommand(config, bazel.AqueryBuildRootRunName, aqueryCmd,
  1005. extraFlags...), context.paths, eventHandler)
  1006. if err != nil {
  1007. return err
  1008. }
  1009. context.buildStatements, context.depsets, err = bazel.AqueryBuildStatements([]byte(aqueryOutput), eventHandler)
  1010. return err
  1011. }
  1012. func (context *mixedBuildBazelContext) generateBazelSymlinks(config Config, ctx invokeBazelContext) error {
  1013. eventHandler := ctx.GetEventHandler()
  1014. eventHandler.Begin("symlinks")
  1015. defer eventHandler.End("symlinks")
  1016. // Issue a build command of the phony root to generate symlink forests for dependencies of the
  1017. // Bazel build. This is necessary because aquery invocations do not generate this symlink forest,
  1018. // but some of symlinks may be required to resolve source dependencies of the build.
  1019. _, _, err := context.issueBazelCommand(context.createBazelCommand(config, bazel.BazelBuildPhonyRootRunName, buildCmd), context.paths, eventHandler)
  1020. return err
  1021. }
  1022. func (context *mixedBuildBazelContext) BuildStatementsToRegister() []*bazel.BuildStatement {
  1023. return context.buildStatements
  1024. }
  1025. func (context *mixedBuildBazelContext) AqueryDepsets() []bazel.AqueryDepset {
  1026. return context.depsets
  1027. }
  1028. func (context *mixedBuildBazelContext) OutputBase() string {
  1029. return context.paths.outputBase
  1030. }
  1031. // Singleton used for registering BUILD file ninja dependencies (needed
  1032. // for correctness of builds which use Bazel.
  1033. func BazelSingleton() Singleton {
  1034. return &bazelSingleton{}
  1035. }
  1036. type bazelSingleton struct{}
  1037. func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) {
  1038. // bazelSingleton is a no-op if mixed-soong-bazel-builds are disabled.
  1039. if !ctx.Config().IsMixedBuildsEnabled() {
  1040. return
  1041. }
  1042. // Add ninja file dependencies for files which all bazel invocations require.
  1043. bazelBuildList := absolutePath(filepath.Join(
  1044. filepath.Dir(ctx.Config().moduleListFile), "bazel.list"))
  1045. ctx.AddNinjaFileDeps(bazelBuildList)
  1046. data, err := os.ReadFile(bazelBuildList)
  1047. if err != nil {
  1048. ctx.Errorf(err.Error())
  1049. }
  1050. files := strings.Split(strings.TrimSpace(string(data)), "\n")
  1051. for _, file := range files {
  1052. ctx.AddNinjaFileDeps(file)
  1053. }
  1054. for _, depset := range ctx.Config().BazelContext.AqueryDepsets() {
  1055. var outputs []Path
  1056. var orderOnlies []Path
  1057. for _, depsetDepHash := range depset.TransitiveDepSetHashes {
  1058. otherDepsetName := bazelDepsetName(depsetDepHash)
  1059. outputs = append(outputs, PathForPhony(ctx, otherDepsetName))
  1060. }
  1061. for _, artifactPath := range depset.DirectArtifacts {
  1062. pathInBazelOut := PathForBazelOut(ctx, artifactPath)
  1063. if artifactPath == "bazel-out/volatile-status.txt" {
  1064. // See https://bazel.build/docs/user-manual#workspace-status
  1065. orderOnlies = append(orderOnlies, pathInBazelOut)
  1066. } else {
  1067. outputs = append(outputs, pathInBazelOut)
  1068. }
  1069. }
  1070. thisDepsetName := bazelDepsetName(depset.ContentHash)
  1071. ctx.Build(pctx, BuildParams{
  1072. Rule: blueprint.Phony,
  1073. Outputs: []WritablePath{PathForPhony(ctx, thisDepsetName)},
  1074. Implicits: outputs,
  1075. OrderOnly: orderOnlies,
  1076. })
  1077. }
  1078. executionRoot := path.Join(ctx.Config().BazelContext.OutputBase(), "execroot", "__main__")
  1079. bazelOutDir := path.Join(executionRoot, "bazel-out")
  1080. for index, buildStatement := range ctx.Config().BazelContext.BuildStatementsToRegister() {
  1081. // nil build statements are a valid case where we do not create an action because it is
  1082. // unnecessary or handled by other processing
  1083. if buildStatement == nil {
  1084. continue
  1085. }
  1086. if len(buildStatement.Command) > 0 {
  1087. rule := NewRuleBuilder(pctx, ctx)
  1088. createCommand(rule.Command(), buildStatement, executionRoot, bazelOutDir, ctx)
  1089. desc := fmt.Sprintf("%s: %s", buildStatement.Mnemonic, buildStatement.OutputPaths)
  1090. rule.Build(fmt.Sprintf("bazel %d", index), desc)
  1091. continue
  1092. }
  1093. // Certain actions returned by aquery (for instance FileWrite) do not contain a command
  1094. // and thus require special treatment. If BuildStatement were an interface implementing
  1095. // buildRule(ctx) function, the code here would just call it.
  1096. // Unfortunately, the BuildStatement is defined in
  1097. // the 'bazel' package, which cannot depend on 'android' package where ctx is defined,
  1098. // because this would cause circular dependency. So, until we move aquery processing
  1099. // to the 'android' package, we need to handle special cases here.
  1100. switch buildStatement.Mnemonic {
  1101. case "FileWrite", "SourceSymlinkManifest":
  1102. out := PathForBazelOut(ctx, buildStatement.OutputPaths[0])
  1103. WriteFileRuleVerbatim(ctx, out, buildStatement.FileContents)
  1104. case "SymlinkTree":
  1105. // build-runfiles arguments are the manifest file and the target directory
  1106. // where it creates the symlink tree according to this manifest (and then
  1107. // writes the MANIFEST file to it).
  1108. outManifest := PathForBazelOut(ctx, buildStatement.OutputPaths[0])
  1109. outManifestPath := outManifest.String()
  1110. if !strings.HasSuffix(outManifestPath, "MANIFEST") {
  1111. panic("the base name of the symlink tree action should be MANIFEST, got " + outManifestPath)
  1112. }
  1113. outDir := filepath.Dir(outManifestPath)
  1114. ctx.Build(pctx, BuildParams{
  1115. Rule: buildRunfilesRule,
  1116. Output: outManifest,
  1117. Inputs: []Path{PathForBazelOut(ctx, buildStatement.InputPaths[0])},
  1118. Description: "symlink tree for " + outDir,
  1119. Args: map[string]string{
  1120. "outDir": outDir,
  1121. },
  1122. })
  1123. default:
  1124. panic(fmt.Sprintf("unhandled build statement: %v", buildStatement))
  1125. }
  1126. }
  1127. }
  1128. // Register bazel-owned build statements (obtained from the aquery invocation).
  1129. func createCommand(cmd *RuleBuilderCommand, buildStatement *bazel.BuildStatement, executionRoot string, bazelOutDir string, ctx BuilderContext) {
  1130. // executionRoot is the action cwd.
  1131. cmd.Text(fmt.Sprintf("cd '%s' &&", executionRoot))
  1132. // Remove old outputs, as some actions might not rerun if the outputs are detected.
  1133. if len(buildStatement.OutputPaths) > 0 {
  1134. cmd.Text("rm -rf") // -r because outputs can be Bazel dir/tree artifacts.
  1135. for _, outputPath := range buildStatement.OutputPaths {
  1136. cmd.Text(fmt.Sprintf("'%s'", outputPath))
  1137. }
  1138. cmd.Text("&&")
  1139. }
  1140. for _, pair := range buildStatement.Env {
  1141. // Set per-action env variables, if any.
  1142. cmd.Flag(pair.Key + "=" + pair.Value)
  1143. }
  1144. // The actual Bazel action.
  1145. if len(buildStatement.Command) > 16*1024 {
  1146. commandFile := PathForBazelOut(ctx, buildStatement.OutputPaths[0]+".sh")
  1147. WriteFileRule(ctx, commandFile, buildStatement.Command)
  1148. cmd.Text("bash").Text(buildStatement.OutputPaths[0] + ".sh").Implicit(commandFile)
  1149. } else {
  1150. cmd.Text(buildStatement.Command)
  1151. }
  1152. for _, outputPath := range buildStatement.OutputPaths {
  1153. cmd.ImplicitOutput(PathForBazelOut(ctx, outputPath))
  1154. }
  1155. for _, inputPath := range buildStatement.InputPaths {
  1156. cmd.Implicit(PathForBazelOut(ctx, inputPath))
  1157. }
  1158. for _, inputDepsetHash := range buildStatement.InputDepsetHashes {
  1159. otherDepsetName := bazelDepsetName(inputDepsetHash)
  1160. cmd.Implicit(PathForPhony(ctx, otherDepsetName))
  1161. }
  1162. if depfile := buildStatement.Depfile; depfile != nil {
  1163. // The paths in depfile are relative to `executionRoot`.
  1164. // Hence, they need to be corrected by replacing "bazel-out"
  1165. // with the full `bazelOutDir`.
  1166. // Otherwise, implicit outputs and implicit inputs under "bazel-out/"
  1167. // would be deemed missing.
  1168. // (Note: The regexp uses a capture group because the version of sed
  1169. // does not support a look-behind pattern.)
  1170. replacement := fmt.Sprintf(`&& sed -i'' -E 's@(^|\s|")bazel-out/@\1%s/@g' '%s'`,
  1171. bazelOutDir, *depfile)
  1172. cmd.Text(replacement)
  1173. cmd.ImplicitDepFile(PathForBazelOut(ctx, *depfile))
  1174. }
  1175. for _, symlinkPath := range buildStatement.SymlinkPaths {
  1176. cmd.ImplicitSymlinkOutput(PathForBazelOut(ctx, symlinkPath))
  1177. }
  1178. }
  1179. func getCqueryId(key cqueryKey) string {
  1180. return key.label + "|" + getConfigString(key)
  1181. }
  1182. func getConfigString(key cqueryKey) string {
  1183. arch := key.configKey.arch
  1184. if len(arch) == 0 || arch == "common" {
  1185. if key.configKey.osType.Class == Device {
  1186. // For the generic Android, the expected result is "target|android", which
  1187. // corresponds to the product_variable_config named "android_target" in
  1188. // build/bazel/platforms/BUILD.bazel.
  1189. arch = "target"
  1190. } else {
  1191. // Use host platform, which is currently hardcoded to be x86_64.
  1192. arch = "x86_64"
  1193. }
  1194. }
  1195. osName := key.configKey.osType.Name
  1196. if len(osName) == 0 || osName == "common_os" || osName == "linux_glibc" || osName == "linux_musl" {
  1197. // Use host OS, which is currently hardcoded to be linux.
  1198. osName = "linux"
  1199. }
  1200. keyString := arch + "|" + osName
  1201. if key.configKey.apexKey.WithinApex {
  1202. keyString += "|" + withinApexToString(key.configKey.apexKey.WithinApex)
  1203. }
  1204. if len(key.configKey.apexKey.ApexSdkVersion) > 0 {
  1205. keyString += "|" + key.configKey.apexKey.ApexSdkVersion
  1206. }
  1207. return keyString
  1208. }
  1209. func GetConfigKey(ctx BaseModuleContext) configKey {
  1210. return configKey{
  1211. // use string because Arch is not a valid key in go
  1212. arch: ctx.Arch().String(),
  1213. osType: ctx.Os(),
  1214. }
  1215. }
  1216. func GetConfigKeyApexVariant(ctx BaseModuleContext, apexKey *ApexConfigKey) configKey {
  1217. configKey := GetConfigKey(ctx)
  1218. if apexKey != nil {
  1219. configKey.apexKey = ApexConfigKey{
  1220. WithinApex: apexKey.WithinApex,
  1221. ApexSdkVersion: apexKey.ApexSdkVersion,
  1222. }
  1223. }
  1224. return configKey
  1225. }
  1226. func bazelDepsetName(contentHash string) string {
  1227. return fmt.Sprintf("bazel_depset_%s", contentHash)
  1228. }