bazel_handler.go 49 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422
  1. // Copyright 2020 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package android
  15. import (
  16. "bytes"
  17. "fmt"
  18. "os"
  19. "path"
  20. "path/filepath"
  21. "runtime"
  22. "sort"
  23. "strings"
  24. "sync"
  25. "android/soong/android/allowlists"
  26. "android/soong/bazel/cquery"
  27. "android/soong/shared"
  28. "android/soong/starlark_import"
  29. "github.com/google/blueprint"
  30. "github.com/google/blueprint/metrics"
  31. "android/soong/bazel"
  32. )
  33. var (
  34. _ = pctx.HostBinToolVariable("bazelBuildRunfilesTool", "build-runfiles")
  35. buildRunfilesRule = pctx.AndroidStaticRule("bazelBuildRunfiles", blueprint.RuleParams{
  36. Command: "${bazelBuildRunfilesTool} ${in} ${outDir}",
  37. Depfile: "",
  38. Description: "",
  39. CommandDeps: []string{"${bazelBuildRunfilesTool}"},
  40. }, "outDir")
  41. )
  42. func registerMixedBuildsMutator(ctx RegisterMutatorsContext) {
  43. ctx.BottomUp("mixed_builds_prep", mixedBuildsPrepareMutator).Parallel()
  44. }
  45. func RegisterMixedBuildsMutator(ctx RegistrationContext) {
  46. ctx.FinalDepsMutators(registerMixedBuildsMutator)
  47. }
  48. func mixedBuildsPrepareMutator(ctx BottomUpMutatorContext) {
  49. if m := ctx.Module(); m.Enabled() {
  50. if mixedBuildMod, ok := m.(MixedBuildBuildable); ok {
  51. mixedBuildEnabled := MixedBuildsEnabled(ctx)
  52. queueMixedBuild := mixedBuildMod.IsMixedBuildSupported(ctx) && mixedBuildEnabled == MixedBuildEnabled
  53. if queueMixedBuild {
  54. mixedBuildMod.QueueBazelCall(ctx)
  55. }
  56. }
  57. }
  58. }
  59. type cqueryRequest interface {
  60. // Name returns a string name for this request type. Such request type names must be unique,
  61. // and must only consist of alphanumeric characters.
  62. Name() string
  63. // StarlarkFunctionBody returns a starlark function body to process this request type.
  64. // The returned string is the body of a Starlark function which obtains
  65. // all request-relevant information about a target and returns a string containing
  66. // this information.
  67. // The function should have the following properties:
  68. // - The arguments are `target` (a configured target) and `id_string` (the label + configuration).
  69. // - The return value must be a string.
  70. // - The function body should not be indented outside of its own scope.
  71. StarlarkFunctionBody() string
  72. }
  73. // Portion of cquery map key to describe target configuration.
  74. type configKey struct {
  75. arch string
  76. osType OsType
  77. apexKey ApexConfigKey
  78. }
  79. type ApexConfigKey struct {
  80. WithinApex bool
  81. ApexSdkVersion string
  82. ApiDomain string
  83. }
  84. func (c ApexConfigKey) String() string {
  85. return fmt.Sprintf("%s_%s_%s", withinApexToString(c.WithinApex), c.ApexSdkVersion, c.ApiDomain)
  86. }
  87. func withinApexToString(withinApex bool) string {
  88. if withinApex {
  89. return "within_apex"
  90. }
  91. return ""
  92. }
  93. func (c configKey) String() string {
  94. return fmt.Sprintf("%s::%s::%s", c.arch, c.osType, c.apexKey)
  95. }
  96. // Map key to describe bazel cquery requests.
  97. type cqueryKey struct {
  98. label string
  99. requestType cqueryRequest
  100. configKey configKey
  101. }
  102. func makeCqueryKey(label string, cqueryRequest cqueryRequest, cfgKey configKey) cqueryKey {
  103. if strings.HasPrefix(label, "//") {
  104. // Normalize Bazel labels to specify main repository explicitly.
  105. label = "@" + label
  106. }
  107. return cqueryKey{label, cqueryRequest, cfgKey}
  108. }
  109. func (c cqueryKey) String() string {
  110. return fmt.Sprintf("cquery(%s,%s,%s)", c.label, c.requestType.Name(), c.configKey)
  111. }
  112. type invokeBazelContext interface {
  113. GetEventHandler() *metrics.EventHandler
  114. }
  115. // BazelContext is a context object useful for interacting with Bazel during
  116. // the course of a build. Use of Bazel to evaluate part of the build graph
  117. // is referred to as a "mixed build". (Some modules are managed by Soong,
  118. // some are managed by Bazel). To facilitate interop between these build
  119. // subgraphs, Soong may make requests to Bazel and evaluate their responses
  120. // so that Soong modules may accurately depend on Bazel targets.
  121. type BazelContext interface {
  122. // Add a cquery request to the bazel request queue. All queued requests
  123. // will be sent to Bazel on a subsequent invocation of InvokeBazel.
  124. QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey)
  125. // ** Cquery Results Retrieval Functions
  126. // The below functions pertain to retrieving cquery results from a prior
  127. // InvokeBazel function call and parsing the results.
  128. // Returns result files built by building the given bazel target label.
  129. GetOutputFiles(label string, cfgKey configKey) ([]string, error)
  130. // Returns the results of GetOutputFiles and GetCcObjectFiles in a single query (in that order).
  131. GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error)
  132. // Returns the results of the GetApexInfo query (including output files)
  133. GetApexInfo(label string, cfgkey configKey) (cquery.ApexInfo, error)
  134. // Returns the results of the GetCcUnstrippedInfo query
  135. GetCcUnstrippedInfo(label string, cfgkey configKey) (cquery.CcUnstrippedInfo, error)
  136. // Returns the results of the GetPrebuiltFileInfo query
  137. GetPrebuiltFileInfo(label string, cfgKey configKey) (cquery.PrebuiltFileInfo, error)
  138. // ** end Cquery Results Retrieval Functions
  139. // Issues commands to Bazel to receive results for all cquery requests
  140. // queued in the BazelContext. The ctx argument is optional and is only
  141. // used for performance data collection
  142. InvokeBazel(config Config, ctx invokeBazelContext) error
  143. // Returns true if Bazel handling is enabled for the module with the given name.
  144. // Note that this only implies "bazel mixed build" allowlisting. The caller
  145. // should independently verify the module is eligible for Bazel handling
  146. // (for example, that it is MixedBuildBuildable).
  147. IsModuleNameAllowed(moduleName string, withinApex bool) bool
  148. // Returns the bazel output base (the root directory for all bazel intermediate outputs).
  149. OutputBase() string
  150. // Returns build statements which should get registered to reflect Bazel's outputs.
  151. BuildStatementsToRegister() []*bazel.BuildStatement
  152. // Returns the depsets defined in Bazel's aquery response.
  153. AqueryDepsets() []bazel.AqueryDepset
  154. }
  155. type bazelRunner interface {
  156. issueBazelCommand(cmdRequest bazel.CmdRequest, paths *bazelPaths, eventHandler *metrics.EventHandler) (output string, errorMessage string, error error)
  157. }
  158. type bazelPaths struct {
  159. homeDir string
  160. bazelPath string
  161. outputBase string
  162. workspaceDir string
  163. soongOutDir string
  164. metricsDir string
  165. bazelDepsFile string
  166. }
  167. // A context object which tracks queued requests that need to be made to Bazel,
  168. // and their results after the requests have been made.
  169. type mixedBuildBazelContext struct {
  170. bazelRunner
  171. paths *bazelPaths
  172. // cquery requests that have not yet been issued to Bazel. This list is maintained
  173. // in a sorted state, and is guaranteed to have no duplicates.
  174. requests []cqueryKey
  175. requestMutex sync.Mutex // requests can be written in parallel
  176. results map[cqueryKey]string // Results of cquery requests after Bazel invocations
  177. // Build statements which should get registered to reflect Bazel's outputs.
  178. buildStatements []*bazel.BuildStatement
  179. // Depsets which should be used for Bazel's build statements.
  180. depsets []bazel.AqueryDepset
  181. // Per-module allowlist/denylist functionality to control whether analysis of
  182. // modules are handled by Bazel. For modules which do not have a Bazel definition
  183. // (or do not sufficiently support bazel handling via MixedBuildBuildable),
  184. // this allowlist will have no effect, even if the module is explicitly allowlisted here.
  185. // Per-module denylist to opt modules out of bazel handling.
  186. bazelDisabledModules map[string]bool
  187. // Per-module allowlist to opt modules in to bazel handling.
  188. bazelEnabledModules map[string]bool
  189. // DCLA modules are enabled when used in apex.
  190. bazelDclaEnabledModules map[string]bool
  191. targetProduct string
  192. targetBuildVariant string
  193. }
  194. var _ BazelContext = &mixedBuildBazelContext{}
  195. // A bazel context to use when Bazel is disabled.
  196. type noopBazelContext struct{}
  197. var _ BazelContext = noopBazelContext{}
  198. // A bazel context to use for tests.
  199. type MockBazelContext struct {
  200. OutputBaseDir string
  201. LabelToOutputFiles map[string][]string
  202. LabelToCcInfo map[string]cquery.CcInfo
  203. LabelToPythonBinary map[string]string
  204. LabelToApexInfo map[string]cquery.ApexInfo
  205. LabelToCcBinary map[string]cquery.CcUnstrippedInfo
  206. LabelToPrebuiltFileInfo map[string]cquery.PrebuiltFileInfo
  207. BazelRequests map[string]bool
  208. }
  209. func (m MockBazelContext) QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey) {
  210. key := BuildMockBazelContextRequestKey(label, requestType, cfgKey.arch, cfgKey.osType, cfgKey.apexKey)
  211. if m.BazelRequests == nil {
  212. m.BazelRequests = make(map[string]bool)
  213. }
  214. m.BazelRequests[key] = true
  215. }
  216. func (m MockBazelContext) GetOutputFiles(label string, _ configKey) ([]string, error) {
  217. result, ok := m.LabelToOutputFiles[label]
  218. if !ok {
  219. return []string{}, fmt.Errorf("no target with label %q in LabelToOutputFiles", label)
  220. }
  221. return result, nil
  222. }
  223. func (m MockBazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error) {
  224. result, ok := m.LabelToCcInfo[label]
  225. if !ok {
  226. key := BuildMockBazelContextResultKey(label, cfgKey.arch, cfgKey.osType, cfgKey.apexKey)
  227. result, ok = m.LabelToCcInfo[key]
  228. if !ok {
  229. return cquery.CcInfo{}, fmt.Errorf("no target with label %q in LabelToCcInfo", label)
  230. }
  231. }
  232. return result, nil
  233. }
  234. func (m MockBazelContext) GetApexInfo(label string, _ configKey) (cquery.ApexInfo, error) {
  235. result, ok := m.LabelToApexInfo[label]
  236. if !ok {
  237. return cquery.ApexInfo{}, fmt.Errorf("no target with label %q in LabelToApexInfo", label)
  238. }
  239. return result, nil
  240. }
  241. func (m MockBazelContext) GetCcUnstrippedInfo(label string, _ configKey) (cquery.CcUnstrippedInfo, error) {
  242. result, ok := m.LabelToCcBinary[label]
  243. if !ok {
  244. return cquery.CcUnstrippedInfo{}, fmt.Errorf("no target with label %q in LabelToCcBinary", label)
  245. }
  246. return result, nil
  247. }
  248. func (m MockBazelContext) GetPrebuiltFileInfo(label string, _ configKey) (cquery.PrebuiltFileInfo, error) {
  249. result, ok := m.LabelToPrebuiltFileInfo[label]
  250. if !ok {
  251. return cquery.PrebuiltFileInfo{}, fmt.Errorf("no target with label %q in LabelToPrebuiltFileInfo", label)
  252. }
  253. return result, nil
  254. }
  255. func (m MockBazelContext) InvokeBazel(_ Config, _ invokeBazelContext) error {
  256. panic("unimplemented")
  257. }
  258. func (m MockBazelContext) IsModuleNameAllowed(_ string, _ bool) bool {
  259. return true
  260. }
  261. func (m MockBazelContext) OutputBase() string { return m.OutputBaseDir }
  262. func (m MockBazelContext) BuildStatementsToRegister() []*bazel.BuildStatement {
  263. return []*bazel.BuildStatement{}
  264. }
  265. func (m MockBazelContext) AqueryDepsets() []bazel.AqueryDepset {
  266. return []bazel.AqueryDepset{}
  267. }
  268. var _ BazelContext = MockBazelContext{}
  269. func BuildMockBazelContextRequestKey(label string, request cqueryRequest, arch string, osType OsType, apexKey ApexConfigKey) string {
  270. cfgKey := configKey{
  271. arch: arch,
  272. osType: osType,
  273. apexKey: apexKey,
  274. }
  275. return strings.Join([]string{label, request.Name(), cfgKey.String()}, "_")
  276. }
  277. func BuildMockBazelContextResultKey(label string, arch string, osType OsType, apexKey ApexConfigKey) string {
  278. cfgKey := configKey{
  279. arch: arch,
  280. osType: osType,
  281. apexKey: apexKey,
  282. }
  283. return strings.Join([]string{label, cfgKey.String()}, "_")
  284. }
  285. func (bazelCtx *mixedBuildBazelContext) QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey) {
  286. key := makeCqueryKey(label, requestType, cfgKey)
  287. bazelCtx.requestMutex.Lock()
  288. defer bazelCtx.requestMutex.Unlock()
  289. // Insert key into requests, maintaining the sort, and only if it's not duplicate.
  290. keyString := key.String()
  291. foundEqual := false
  292. notLessThanKeyString := func(i int) bool {
  293. s := bazelCtx.requests[i].String()
  294. v := strings.Compare(s, keyString)
  295. if v == 0 {
  296. foundEqual = true
  297. }
  298. return v >= 0
  299. }
  300. targetIndex := sort.Search(len(bazelCtx.requests), notLessThanKeyString)
  301. if foundEqual {
  302. return
  303. }
  304. if targetIndex == len(bazelCtx.requests) {
  305. bazelCtx.requests = append(bazelCtx.requests, key)
  306. } else {
  307. bazelCtx.requests = append(bazelCtx.requests[:targetIndex+1], bazelCtx.requests[targetIndex:]...)
  308. bazelCtx.requests[targetIndex] = key
  309. }
  310. }
  311. func (bazelCtx *mixedBuildBazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, error) {
  312. key := makeCqueryKey(label, cquery.GetOutputFiles, cfgKey)
  313. if rawString, ok := bazelCtx.results[key]; ok {
  314. bazelOutput := strings.TrimSpace(rawString)
  315. return cquery.GetOutputFiles.ParseResult(bazelOutput), nil
  316. }
  317. return nil, fmt.Errorf("no bazel response found for %v", key)
  318. }
  319. func (bazelCtx *mixedBuildBazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error) {
  320. key := makeCqueryKey(label, cquery.GetCcInfo, cfgKey)
  321. if rawString, ok := bazelCtx.results[key]; ok {
  322. bazelOutput := strings.TrimSpace(rawString)
  323. return cquery.GetCcInfo.ParseResult(bazelOutput)
  324. }
  325. return cquery.CcInfo{}, fmt.Errorf("no bazel response found for %v", key)
  326. }
  327. func (bazelCtx *mixedBuildBazelContext) GetApexInfo(label string, cfgKey configKey) (cquery.ApexInfo, error) {
  328. key := makeCqueryKey(label, cquery.GetApexInfo, cfgKey)
  329. if rawString, ok := bazelCtx.results[key]; ok {
  330. return cquery.GetApexInfo.ParseResult(strings.TrimSpace(rawString))
  331. }
  332. return cquery.ApexInfo{}, fmt.Errorf("no bazel response found for %v", key)
  333. }
  334. func (bazelCtx *mixedBuildBazelContext) GetCcUnstrippedInfo(label string, cfgKey configKey) (cquery.CcUnstrippedInfo, error) {
  335. key := makeCqueryKey(label, cquery.GetCcUnstrippedInfo, cfgKey)
  336. if rawString, ok := bazelCtx.results[key]; ok {
  337. return cquery.GetCcUnstrippedInfo.ParseResult(strings.TrimSpace(rawString))
  338. }
  339. return cquery.CcUnstrippedInfo{}, fmt.Errorf("no bazel response for %s", key)
  340. }
  341. func (bazelCtx *mixedBuildBazelContext) GetPrebuiltFileInfo(label string, cfgKey configKey) (cquery.PrebuiltFileInfo, error) {
  342. key := makeCqueryKey(label, cquery.GetPrebuiltFileInfo, cfgKey)
  343. if rawString, ok := bazelCtx.results[key]; ok {
  344. return cquery.GetPrebuiltFileInfo.ParseResult(strings.TrimSpace(rawString))
  345. }
  346. return cquery.PrebuiltFileInfo{}, fmt.Errorf("no bazel response for %s", key)
  347. }
  348. func (n noopBazelContext) QueueBazelRequest(_ string, _ cqueryRequest, _ configKey) {
  349. panic("unimplemented")
  350. }
  351. func (n noopBazelContext) GetOutputFiles(_ string, _ configKey) ([]string, error) {
  352. panic("unimplemented")
  353. }
  354. func (n noopBazelContext) GetCcInfo(_ string, _ configKey) (cquery.CcInfo, error) {
  355. panic("unimplemented")
  356. }
  357. func (n noopBazelContext) GetApexInfo(_ string, _ configKey) (cquery.ApexInfo, error) {
  358. panic("unimplemented")
  359. }
  360. func (n noopBazelContext) GetCcUnstrippedInfo(_ string, _ configKey) (cquery.CcUnstrippedInfo, error) {
  361. //TODO implement me
  362. panic("implement me")
  363. }
  364. func (n noopBazelContext) GetPrebuiltFileInfo(_ string, _ configKey) (cquery.PrebuiltFileInfo, error) {
  365. panic("implement me")
  366. }
  367. func (n noopBazelContext) InvokeBazel(_ Config, _ invokeBazelContext) error {
  368. panic("unimplemented")
  369. }
  370. func (m noopBazelContext) OutputBase() string {
  371. return ""
  372. }
  373. func (n noopBazelContext) IsModuleNameAllowed(_ string, _ bool) bool {
  374. return false
  375. }
  376. func (m noopBazelContext) BuildStatementsToRegister() []*bazel.BuildStatement {
  377. return []*bazel.BuildStatement{}
  378. }
  379. func (m noopBazelContext) AqueryDepsets() []bazel.AqueryDepset {
  380. return []bazel.AqueryDepset{}
  381. }
  382. func AddToStringSet(set map[string]bool, items []string) {
  383. for _, item := range items {
  384. set[item] = true
  385. }
  386. }
  387. func GetBazelEnabledAndDisabledModules(buildMode SoongBuildMode, forceEnabled map[string]struct{}) (map[string]bool, map[string]bool) {
  388. disabledModules := map[string]bool{}
  389. enabledModules := map[string]bool{}
  390. switch buildMode {
  391. case BazelProdMode:
  392. AddToStringSet(enabledModules, allowlists.ProdMixedBuildsEnabledList)
  393. for enabledAdHocModule := range forceEnabled {
  394. enabledModules[enabledAdHocModule] = true
  395. }
  396. case BazelStagingMode:
  397. // Staging mode includes all prod modules plus all staging modules.
  398. AddToStringSet(enabledModules, allowlists.ProdMixedBuildsEnabledList)
  399. AddToStringSet(enabledModules, allowlists.StagingMixedBuildsEnabledList)
  400. for enabledAdHocModule := range forceEnabled {
  401. enabledModules[enabledAdHocModule] = true
  402. }
  403. default:
  404. panic("Expected BazelProdMode or BazelStagingMode")
  405. }
  406. return enabledModules, disabledModules
  407. }
  408. func GetBazelEnabledModules(buildMode SoongBuildMode) []string {
  409. enabledModules, disabledModules := GetBazelEnabledAndDisabledModules(buildMode, nil)
  410. enabledList := make([]string, 0, len(enabledModules))
  411. for module := range enabledModules {
  412. if !disabledModules[module] {
  413. enabledList = append(enabledList, module)
  414. }
  415. }
  416. sort.Strings(enabledList)
  417. return enabledList
  418. }
  419. func NewBazelContext(c *config) (BazelContext, error) {
  420. if c.BuildMode != BazelProdMode && c.BuildMode != BazelStagingMode {
  421. return noopBazelContext{}, nil
  422. }
  423. enabledModules, disabledModules := GetBazelEnabledAndDisabledModules(c.BuildMode, c.BazelModulesForceEnabledByFlag())
  424. paths := bazelPaths{
  425. soongOutDir: c.soongOutDir,
  426. }
  427. var missing []string
  428. vars := []struct {
  429. name string
  430. ptr *string
  431. // True if the environment variable needs to be tracked so that changes to the variable
  432. // cause the ninja file to be regenerated, false otherwise. False should only be set for
  433. // environment variables that have no effect on the generated ninja file.
  434. track bool
  435. }{
  436. {"BAZEL_HOME", &paths.homeDir, true},
  437. {"BAZEL_PATH", &paths.bazelPath, true},
  438. {"BAZEL_OUTPUT_BASE", &paths.outputBase, true},
  439. {"BAZEL_WORKSPACE", &paths.workspaceDir, true},
  440. {"BAZEL_METRICS_DIR", &paths.metricsDir, false},
  441. {"BAZEL_DEPS_FILE", &paths.bazelDepsFile, true},
  442. }
  443. for _, v := range vars {
  444. if v.track {
  445. if s := c.Getenv(v.name); len(s) > 1 {
  446. *v.ptr = s
  447. continue
  448. }
  449. } else if s, ok := c.env[v.name]; ok {
  450. *v.ptr = s
  451. } else {
  452. missing = append(missing, v.name)
  453. }
  454. }
  455. if len(missing) > 0 {
  456. return nil, fmt.Errorf("missing required env vars to use bazel: %s", missing)
  457. }
  458. targetBuildVariant := "user"
  459. if c.Eng() {
  460. targetBuildVariant = "eng"
  461. } else if c.Debuggable() {
  462. targetBuildVariant = "userdebug"
  463. }
  464. targetProduct := "unknown"
  465. if c.HasDeviceProduct() {
  466. targetProduct = c.DeviceProduct()
  467. }
  468. dclaMixedBuildsEnabledList := []string{}
  469. if c.BuildMode == BazelProdMode {
  470. dclaMixedBuildsEnabledList = allowlists.ProdDclaMixedBuildsEnabledList
  471. } else if c.BuildMode == BazelStagingMode {
  472. dclaMixedBuildsEnabledList = append(allowlists.ProdDclaMixedBuildsEnabledList,
  473. allowlists.StagingDclaMixedBuildsEnabledList...)
  474. }
  475. dclaEnabledModules := map[string]bool{}
  476. AddToStringSet(dclaEnabledModules, dclaMixedBuildsEnabledList)
  477. return &mixedBuildBazelContext{
  478. bazelRunner: &builtinBazelRunner{c.UseBazelProxy, absolutePath(c.outDir)},
  479. paths: &paths,
  480. bazelEnabledModules: enabledModules,
  481. bazelDisabledModules: disabledModules,
  482. bazelDclaEnabledModules: dclaEnabledModules,
  483. targetProduct: targetProduct,
  484. targetBuildVariant: targetBuildVariant,
  485. }, nil
  486. }
  487. func (p *bazelPaths) BazelMetricsDir() string {
  488. return p.metricsDir
  489. }
  490. func (context *mixedBuildBazelContext) IsModuleNameAllowed(moduleName string, withinApex bool) bool {
  491. if context.bazelDisabledModules[moduleName] {
  492. return false
  493. }
  494. if context.bazelEnabledModules[moduleName] {
  495. return true
  496. }
  497. if withinApex && context.bazelDclaEnabledModules[moduleName] {
  498. return true
  499. }
  500. return false
  501. }
  502. func pwdPrefix() string {
  503. // Darwin doesn't have /proc
  504. if runtime.GOOS != "darwin" {
  505. return "PWD=/proc/self/cwd"
  506. }
  507. return ""
  508. }
  509. type bazelCommand struct {
  510. command string
  511. // query or label
  512. expression string
  513. }
  514. type builtinBazelRunner struct {
  515. useBazelProxy bool
  516. outDir string
  517. }
  518. // Issues the given bazel command with given build label and additional flags.
  519. // Returns (stdout, stderr, error). The first and second return values are strings
  520. // containing the stdout and stderr of the run command, and an error is returned if
  521. // the invocation returned an error code.
  522. func (r *builtinBazelRunner) issueBazelCommand(cmdRequest bazel.CmdRequest, paths *bazelPaths, eventHandler *metrics.EventHandler) (string, string, error) {
  523. if r.useBazelProxy {
  524. eventHandler.Begin("client_proxy")
  525. defer eventHandler.End("client_proxy")
  526. proxyClient := bazel.NewProxyClient(r.outDir)
  527. resp, err := proxyClient.IssueCommand(cmdRequest)
  528. if err != nil {
  529. return "", "", err
  530. }
  531. if len(resp.ErrorString) > 0 {
  532. return "", "", fmt.Errorf(resp.ErrorString)
  533. }
  534. return resp.Stdout, resp.Stderr, nil
  535. } else {
  536. eventHandler.Begin("bazel command")
  537. defer eventHandler.End("bazel command")
  538. stdout, stderr, err := bazel.ExecBazel(paths.bazelPath, absolutePath(paths.syntheticWorkspaceDir()), cmdRequest)
  539. return string(stdout), string(stderr), err
  540. }
  541. }
  542. func (context *mixedBuildBazelContext) createBazelCommand(config Config, runName bazel.RunName, command bazelCommand,
  543. extraFlags ...string) bazel.CmdRequest {
  544. if runtime.GOOS != "linux" && runtime.GOOS != "darwin" {
  545. panic("Unknown GOOS: " + runtime.GOOS)
  546. }
  547. cmdFlags := []string{
  548. "--output_base=" + absolutePath(context.paths.outputBase),
  549. command.command,
  550. command.expression,
  551. "--profile=" + shared.BazelMetricsFilename(context.paths, runName),
  552. "--host_platform=@soong_injection//product_config_platforms:mixed_builds_product-" + context.targetBuildVariant + "_" + runtime.GOOS + "_x86_64",
  553. // Don't specify --platforms, because on some products/branches (like kernel-build-tools)
  554. // the main platform for mixed_builds_product-variant doesn't exist because an arch isn't
  555. // specified in product config. The derivative platforms that config_node transitions into
  556. // will still work.
  557. // Suppress noise
  558. "--ui_event_filters=-INFO",
  559. "--noshow_progress",
  560. "--norun_validations",
  561. }
  562. cmdFlags = append(cmdFlags, extraFlags...)
  563. extraEnv := []string{
  564. "HOME=" + context.paths.homeDir,
  565. pwdPrefix(),
  566. "BUILD_DIR=" + absolutePath(context.paths.soongOutDir),
  567. // Make OUT_DIR absolute here so build/bazel/bin/bazel uses the correct
  568. // OUT_DIR at <root>/out, instead of <root>/out/soong/workspace/out.
  569. "OUT_DIR=" + absolutePath(context.paths.outDir()),
  570. // Disables local host detection of gcc; toolchain information is defined
  571. // explicitly in BUILD files.
  572. "BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1",
  573. }
  574. capturedEnvVars, err := starlark_import.GetStarlarkValue[[]string]("captured_env_vars")
  575. if err != nil {
  576. panic(err)
  577. }
  578. for _, envvar := range capturedEnvVars {
  579. val := config.Getenv(envvar)
  580. if val == "" {
  581. continue
  582. }
  583. extraEnv = append(extraEnv, fmt.Sprintf("%s=%s", envvar, val))
  584. }
  585. envVars := append(os.Environ(), extraEnv...)
  586. return bazel.CmdRequest{cmdFlags, envVars}
  587. }
  588. func (context *mixedBuildBazelContext) printableCqueryCommand(bazelCmd bazel.CmdRequest) string {
  589. args := append([]string{context.paths.bazelPath}, bazelCmd.Argv...)
  590. outputString := strings.Join(bazelCmd.Env, " ") + " \"" + strings.Join(args, "\" \"") + "\""
  591. return outputString
  592. }
  593. func (context *mixedBuildBazelContext) mainBzlFileContents() []byte {
  594. // TODO(cparsons): Define configuration transitions programmatically based
  595. // on available archs.
  596. contents := `
  597. #####################################################
  598. # This file is generated by soong_build. Do not edit.
  599. #####################################################
  600. def _config_node_transition_impl(settings, attr):
  601. if attr.os == "android" and attr.arch == "target":
  602. target = "mixed_builds_product-{VARIANT}"
  603. else:
  604. target = "mixed_builds_product-{VARIANT}_%s_%s" % (attr.os, attr.arch)
  605. apex_name = ""
  606. if attr.within_apex:
  607. # //build/bazel/rules/apex:apex_name has to be set to a non_empty value,
  608. # otherwise //build/bazel/rules/apex:non_apex will be true and the
  609. # "-D__ANDROID_APEX__" compiler flag will be missing. Apex_name is used
  610. # in some validation on bazel side which don't really apply in mixed
  611. # build because soong will do the work, so we just set it to a fixed
  612. # value here.
  613. apex_name = "dcla_apex"
  614. outputs = {
  615. "//command_line_option:platforms": "@soong_injection//product_config_platforms:%s" % target,
  616. "@//build/bazel/rules/apex:within_apex": attr.within_apex,
  617. "@//build/bazel/rules/apex:min_sdk_version": attr.apex_sdk_version,
  618. "@//build/bazel/rules/apex:apex_name": apex_name,
  619. "@//build/bazel/rules/apex:api_domain": attr.api_domain,
  620. }
  621. return outputs
  622. _config_node_transition = transition(
  623. implementation = _config_node_transition_impl,
  624. inputs = [],
  625. outputs = [
  626. "//command_line_option:platforms",
  627. "@//build/bazel/rules/apex:within_apex",
  628. "@//build/bazel/rules/apex:min_sdk_version",
  629. "@//build/bazel/rules/apex:apex_name",
  630. "@//build/bazel/rules/apex:api_domain",
  631. ],
  632. )
  633. def _passthrough_rule_impl(ctx):
  634. return [DefaultInfo(files = depset(ctx.files.deps))]
  635. config_node = rule(
  636. implementation = _passthrough_rule_impl,
  637. attrs = {
  638. "arch" : attr.string(mandatory = True),
  639. "os" : attr.string(mandatory = True),
  640. "within_apex" : attr.bool(default = False),
  641. "apex_sdk_version" : attr.string(mandatory = True),
  642. "api_domain" : attr.string(mandatory = True),
  643. "deps" : attr.label_list(cfg = _config_node_transition, allow_files = True),
  644. "_allowlist_function_transition": attr.label(default = "@bazel_tools//tools/allowlists/function_transition_allowlist"),
  645. },
  646. )
  647. # Rule representing the root of the build, to depend on all Bazel targets that
  648. # are required for the build. Building this target will build the entire Bazel
  649. # build tree.
  650. mixed_build_root = rule(
  651. implementation = _passthrough_rule_impl,
  652. attrs = {
  653. "deps" : attr.label_list(),
  654. },
  655. )
  656. def _phony_root_impl(ctx):
  657. return []
  658. # Rule to depend on other targets but build nothing.
  659. # This is useful as follows: building a target of this rule will generate
  660. # symlink forests for all dependencies of the target, without executing any
  661. # actions of the build.
  662. phony_root = rule(
  663. implementation = _phony_root_impl,
  664. attrs = {"deps" : attr.label_list()},
  665. )
  666. `
  667. productReplacer := strings.NewReplacer(
  668. "{PRODUCT}", context.targetProduct,
  669. "{VARIANT}", context.targetBuildVariant)
  670. return []byte(productReplacer.Replace(contents))
  671. }
  672. func (context *mixedBuildBazelContext) mainBuildFileContents() []byte {
  673. // TODO(cparsons): Map label to attribute programmatically; don't use hard-coded
  674. // architecture mapping.
  675. formatString := `
  676. # This file is generated by soong_build. Do not edit.
  677. load(":main.bzl", "config_node", "mixed_build_root", "phony_root")
  678. %s
  679. mixed_build_root(name = "buildroot",
  680. deps = [%s],
  681. testonly = True, # Unblocks testonly deps.
  682. )
  683. phony_root(name = "phonyroot",
  684. deps = [":buildroot"],
  685. testonly = True, # Unblocks testonly deps.
  686. )
  687. `
  688. configNodeFormatString := `
  689. config_node(name = "%s",
  690. arch = "%s",
  691. os = "%s",
  692. within_apex = %s,
  693. apex_sdk_version = "%s",
  694. api_domain = "%s",
  695. deps = [%s],
  696. testonly = True, # Unblocks testonly deps.
  697. )
  698. `
  699. configNodesSection := ""
  700. labelsByConfig := map[string][]string{}
  701. for _, val := range context.requests {
  702. labelString := fmt.Sprintf("\"@%s\"", val.label)
  703. configString := getConfigString(val)
  704. labelsByConfig[configString] = append(labelsByConfig[configString], labelString)
  705. }
  706. // Configs need to be sorted to maintain determinism of the BUILD file.
  707. sortedConfigs := make([]string, 0, len(labelsByConfig))
  708. for val := range labelsByConfig {
  709. sortedConfigs = append(sortedConfigs, val)
  710. }
  711. sort.Slice(sortedConfigs, func(i, j int) bool { return sortedConfigs[i] < sortedConfigs[j] })
  712. allLabels := []string{}
  713. for _, configString := range sortedConfigs {
  714. labels := labelsByConfig[configString]
  715. configTokens := strings.Split(configString, "|")
  716. if len(configTokens) < 2 {
  717. panic(fmt.Errorf("Unexpected config string format: %s", configString))
  718. }
  719. archString := configTokens[0]
  720. osString := configTokens[1]
  721. withinApex := "False"
  722. apexSdkVerString := ""
  723. apiDomainString := ""
  724. if osString == "android" {
  725. // api domains are meaningful only for device variants
  726. apiDomainString = "system"
  727. }
  728. targetString := fmt.Sprintf("%s_%s", osString, archString)
  729. if len(configTokens) > 2 {
  730. targetString += "_" + configTokens[2]
  731. if configTokens[2] == withinApexToString(true) {
  732. withinApex = "True"
  733. }
  734. }
  735. if len(configTokens) > 3 {
  736. targetString += "_" + configTokens[3]
  737. apexSdkVerString = configTokens[3]
  738. }
  739. if len(configTokens) > 4 {
  740. apiDomainString = configTokens[4]
  741. targetString += "_" + apiDomainString
  742. }
  743. allLabels = append(allLabels, fmt.Sprintf("\":%s\"", targetString))
  744. labelsString := strings.Join(labels, ",\n ")
  745. configNodesSection += fmt.Sprintf(configNodeFormatString, targetString, archString, osString, withinApex, apexSdkVerString, apiDomainString,
  746. labelsString)
  747. }
  748. return []byte(fmt.Sprintf(formatString, configNodesSection, strings.Join(allLabels, ",\n ")))
  749. }
  750. func indent(original string) string {
  751. result := ""
  752. for _, line := range strings.Split(original, "\n") {
  753. result += " " + line + "\n"
  754. }
  755. return result
  756. }
  757. // Returns the file contents of the buildroot.cquery file that should be used for the cquery
  758. // expression in order to obtain information about buildroot and its dependencies.
  759. // The contents of this file depend on the mixedBuildBazelContext's requests; requests are enumerated
  760. // and grouped by their request type. The data retrieved for each label depends on its
  761. // request type.
  762. func (context *mixedBuildBazelContext) cqueryStarlarkFileContents() []byte {
  763. requestTypeToCqueryIdEntries := map[cqueryRequest][]string{}
  764. requestTypes := []cqueryRequest{}
  765. for _, val := range context.requests {
  766. cqueryId := getCqueryId(val)
  767. mapEntryString := fmt.Sprintf("%q : True", cqueryId)
  768. if _, seenKey := requestTypeToCqueryIdEntries[val.requestType]; !seenKey {
  769. requestTypes = append(requestTypes, val.requestType)
  770. }
  771. requestTypeToCqueryIdEntries[val.requestType] =
  772. append(requestTypeToCqueryIdEntries[val.requestType], mapEntryString)
  773. }
  774. labelRegistrationMapSection := ""
  775. functionDefSection := ""
  776. mainSwitchSection := ""
  777. mapDeclarationFormatString := `
  778. %s = {
  779. %s
  780. }
  781. `
  782. functionDefFormatString := `
  783. def %s(target, id_string):
  784. %s
  785. `
  786. mainSwitchSectionFormatString := `
  787. if id_string in %s:
  788. return id_string + ">>" + %s(target, id_string)
  789. `
  790. for _, requestType := range requestTypes {
  791. labelMapName := requestType.Name() + "_Labels"
  792. functionName := requestType.Name() + "_Fn"
  793. labelRegistrationMapSection += fmt.Sprintf(mapDeclarationFormatString,
  794. labelMapName,
  795. strings.Join(requestTypeToCqueryIdEntries[requestType], ",\n "))
  796. functionDefSection += fmt.Sprintf(functionDefFormatString,
  797. functionName,
  798. indent(requestType.StarlarkFunctionBody()))
  799. mainSwitchSection += fmt.Sprintf(mainSwitchSectionFormatString,
  800. labelMapName, functionName)
  801. }
  802. formatString := `
  803. # This file is generated by soong_build. Do not edit.
  804. {LABEL_REGISTRATION_MAP_SECTION}
  805. {FUNCTION_DEF_SECTION}
  806. def get_arch(target):
  807. # TODO(b/199363072): filegroups and file targets aren't associated with any
  808. # specific platform architecture in mixed builds. This is consistent with how
  809. # Soong treats filegroups, but it may not be the case with manually-written
  810. # filegroup BUILD targets.
  811. buildoptions = build_options(target)
  812. if buildoptions == None:
  813. # File targets do not have buildoptions. File targets aren't associated with
  814. # any specific platform architecture in mixed builds, so use the host.
  815. return "x86_64|linux"
  816. platforms = buildoptions["//command_line_option:platforms"]
  817. if len(platforms) != 1:
  818. # An individual configured target should have only one platform architecture.
  819. # Note that it's fine for there to be multiple architectures for the same label,
  820. # but each is its own configured target.
  821. fail("expected exactly 1 platform for " + str(target.label) + " but got " + str(platforms))
  822. platform_name = platforms[0].name
  823. if platform_name == "host":
  824. return "HOST"
  825. if not platform_name.startswith("mixed_builds_product-{TARGET_BUILD_VARIANT}"):
  826. fail("expected platform name of the form 'mixed_builds_product-{TARGET_BUILD_VARIANT}_android_<arch>' or 'mixed_builds_product-{TARGET_BUILD_VARIANT}_linux_<arch>', but was " + str(platforms))
  827. platform_name = platform_name.removeprefix("mixed_builds_product-{TARGET_BUILD_VARIANT}").removeprefix("_")
  828. config_key = ""
  829. if not platform_name:
  830. config_key = "target|android"
  831. elif platform_name.startswith("android_"):
  832. config_key = platform_name.removeprefix("android_") + "|android"
  833. elif platform_name.startswith("linux_"):
  834. config_key = platform_name.removeprefix("linux_") + "|linux"
  835. else:
  836. fail("expected platform name of the form 'mixed_builds_product-{TARGET_BUILD_VARIANT}_android_<arch>' or 'mixed_builds_product-{TARGET_BUILD_VARIANT}_linux_<arch>', but was " + str(platforms))
  837. within_apex = buildoptions.get("//build/bazel/rules/apex:within_apex")
  838. apex_sdk_version = buildoptions.get("//build/bazel/rules/apex:min_sdk_version")
  839. api_domain = buildoptions.get("//build/bazel/rules/apex:api_domain")
  840. if within_apex:
  841. config_key += "|within_apex"
  842. if apex_sdk_version != None and len(apex_sdk_version) > 0:
  843. config_key += "|" + apex_sdk_version
  844. if api_domain != None and len(api_domain) > 0:
  845. config_key += "|" + api_domain
  846. return config_key
  847. def format(target):
  848. id_string = str(target.label) + "|" + get_arch(target)
  849. # TODO(b/248106697): Remove once Bazel is updated to always normalize labels.
  850. if id_string.startswith("//"):
  851. id_string = "@" + id_string
  852. {MAIN_SWITCH_SECTION}
  853. # This target was not requested via cquery, and thus must be a dependency
  854. # of a requested target.
  855. return id_string + ">>NONE"
  856. `
  857. replacer := strings.NewReplacer(
  858. "{TARGET_PRODUCT}", context.targetProduct,
  859. "{TARGET_BUILD_VARIANT}", context.targetBuildVariant,
  860. "{LABEL_REGISTRATION_MAP_SECTION}", labelRegistrationMapSection,
  861. "{FUNCTION_DEF_SECTION}", functionDefSection,
  862. "{MAIN_SWITCH_SECTION}", mainSwitchSection)
  863. return []byte(replacer.Replace(formatString))
  864. }
  865. // Returns a path containing build-related metadata required for interfacing
  866. // with Bazel. Example: out/soong/bazel.
  867. func (p *bazelPaths) intermediatesDir() string {
  868. return filepath.Join(p.soongOutDir, "bazel")
  869. }
  870. // Returns the path where the contents of the @soong_injection repository live.
  871. // It is used by Soong to tell Bazel things it cannot over the command line.
  872. func (p *bazelPaths) injectedFilesDir() string {
  873. return filepath.Join(p.soongOutDir, bazel.SoongInjectionDirName)
  874. }
  875. // Returns the path of the synthetic Bazel workspace that contains a symlink
  876. // forest composed the whole source tree and BUILD files generated by bp2build.
  877. func (p *bazelPaths) syntheticWorkspaceDir() string {
  878. return filepath.Join(p.soongOutDir, "workspace")
  879. }
  880. // Returns the path to the top level out dir ($OUT_DIR).
  881. func (p *bazelPaths) outDir() string {
  882. return filepath.Dir(p.soongOutDir)
  883. }
  884. const buildrootLabel = "@soong_injection//mixed_builds:buildroot"
  885. var (
  886. cqueryCmd = bazelCommand{"cquery", fmt.Sprintf("deps(%s, 2)", buildrootLabel)}
  887. aqueryCmd = bazelCommand{"aquery", fmt.Sprintf("deps(%s)", buildrootLabel)}
  888. buildCmd = bazelCommand{"build", "@soong_injection//mixed_builds:phonyroot"}
  889. allBazelCommands = []bazelCommand{aqueryCmd, cqueryCmd, buildCmd}
  890. )
  891. // Issues commands to Bazel to receive results for all cquery requests
  892. // queued in the BazelContext.
  893. func (context *mixedBuildBazelContext) InvokeBazel(config Config, ctx invokeBazelContext) error {
  894. eventHandler := ctx.GetEventHandler()
  895. eventHandler.Begin("bazel")
  896. defer eventHandler.End("bazel")
  897. if metricsDir := context.paths.BazelMetricsDir(); metricsDir != "" {
  898. if err := os.MkdirAll(metricsDir, 0777); err != nil {
  899. return err
  900. }
  901. }
  902. context.results = make(map[cqueryKey]string)
  903. if err := context.runCquery(config, ctx); err != nil {
  904. return err
  905. }
  906. if err := context.runAquery(config, ctx); err != nil {
  907. return err
  908. }
  909. if err := context.generateBazelSymlinks(config, ctx); err != nil {
  910. return err
  911. }
  912. // Clear requests.
  913. context.requests = []cqueryKey{}
  914. return nil
  915. }
  916. func (context *mixedBuildBazelContext) runCquery(config Config, ctx invokeBazelContext) error {
  917. eventHandler := ctx.GetEventHandler()
  918. eventHandler.Begin("cquery")
  919. defer eventHandler.End("cquery")
  920. soongInjectionPath := absolutePath(context.paths.injectedFilesDir())
  921. mixedBuildsPath := filepath.Join(soongInjectionPath, "mixed_builds")
  922. if _, err := os.Stat(mixedBuildsPath); os.IsNotExist(err) {
  923. err = os.MkdirAll(mixedBuildsPath, 0777)
  924. if err != nil {
  925. return err
  926. }
  927. }
  928. if err := writeFileBytesIfChanged(filepath.Join(soongInjectionPath, "WORKSPACE.bazel"), []byte{}, 0666); err != nil {
  929. return err
  930. }
  931. if err := writeFileBytesIfChanged(filepath.Join(mixedBuildsPath, "main.bzl"), context.mainBzlFileContents(), 0666); err != nil {
  932. return err
  933. }
  934. if err := writeFileBytesIfChanged(filepath.Join(mixedBuildsPath, "BUILD.bazel"), context.mainBuildFileContents(), 0666); err != nil {
  935. return err
  936. }
  937. cqueryFileRelpath := filepath.Join(context.paths.injectedFilesDir(), "buildroot.cquery")
  938. if err := writeFileBytesIfChanged(absolutePath(cqueryFileRelpath), context.cqueryStarlarkFileContents(), 0666); err != nil {
  939. return err
  940. }
  941. extraFlags := []string{"--output=starlark", "--starlark:file=" + absolutePath(cqueryFileRelpath)}
  942. if Bool(config.productVariables.ClangCoverage) {
  943. extraFlags = append(extraFlags, "--collect_code_coverage")
  944. }
  945. cqueryCmdRequest := context.createBazelCommand(config, bazel.CqueryBuildRootRunName, cqueryCmd, extraFlags...)
  946. cqueryOutput, cqueryErrorMessage, cqueryErr := context.issueBazelCommand(cqueryCmdRequest, context.paths, eventHandler)
  947. if cqueryErr != nil {
  948. return cqueryErr
  949. }
  950. cqueryCommandPrint := fmt.Sprintf("cquery command line:\n %s \n\n\n", context.printableCqueryCommand(cqueryCmdRequest))
  951. if err := os.WriteFile(filepath.Join(soongInjectionPath, "cquery.out"), []byte(cqueryCommandPrint+cqueryOutput), 0666); err != nil {
  952. return err
  953. }
  954. cqueryResults := map[string]string{}
  955. for _, outputLine := range strings.Split(cqueryOutput, "\n") {
  956. if strings.Contains(outputLine, ">>") {
  957. splitLine := strings.SplitN(outputLine, ">>", 2)
  958. cqueryResults[splitLine[0]] = splitLine[1]
  959. }
  960. }
  961. for _, val := range context.requests {
  962. if cqueryResult, ok := cqueryResults[getCqueryId(val)]; ok {
  963. context.results[val] = cqueryResult
  964. } else {
  965. return fmt.Errorf("missing result for bazel target %s. query output: [%s], cquery err: [%s]",
  966. getCqueryId(val), cqueryOutput, cqueryErrorMessage)
  967. }
  968. }
  969. return nil
  970. }
  971. func writeFileBytesIfChanged(path string, contents []byte, perm os.FileMode) error {
  972. oldContents, err := os.ReadFile(path)
  973. if err != nil || !bytes.Equal(contents, oldContents) {
  974. err = os.WriteFile(path, contents, perm)
  975. }
  976. return nil
  977. }
  978. func (context *mixedBuildBazelContext) runAquery(config Config, ctx invokeBazelContext) error {
  979. eventHandler := ctx.GetEventHandler()
  980. eventHandler.Begin("aquery")
  981. defer eventHandler.End("aquery")
  982. // Issue an aquery command to retrieve action information about the bazel build tree.
  983. //
  984. // Use jsonproto instead of proto; actual proto parsing would require a dependency on Bazel's
  985. // proto sources, which would add a number of unnecessary dependencies.
  986. extraFlags := []string{"--output=proto", "--include_file_write_contents"}
  987. if Bool(config.productVariables.ClangCoverage) {
  988. extraFlags = append(extraFlags, "--collect_code_coverage")
  989. paths := make([]string, 0, 2)
  990. if p := config.productVariables.NativeCoveragePaths; len(p) > 0 {
  991. for i := range p {
  992. // TODO(b/259404593) convert path wildcard to regex values
  993. if p[i] == "*" {
  994. p[i] = ".*"
  995. }
  996. }
  997. paths = append(paths, JoinWithPrefixAndSeparator(p, "+", ","))
  998. }
  999. if p := config.productVariables.NativeCoverageExcludePaths; len(p) > 0 {
  1000. paths = append(paths, JoinWithPrefixAndSeparator(p, "-", ","))
  1001. }
  1002. if len(paths) > 0 {
  1003. extraFlags = append(extraFlags, "--instrumentation_filter="+strings.Join(paths, ","))
  1004. }
  1005. }
  1006. aqueryOutput, _, err := context.issueBazelCommand(context.createBazelCommand(config, bazel.AqueryBuildRootRunName, aqueryCmd,
  1007. extraFlags...), context.paths, eventHandler)
  1008. if err != nil {
  1009. return err
  1010. }
  1011. context.buildStatements, context.depsets, err = bazel.AqueryBuildStatements([]byte(aqueryOutput), eventHandler)
  1012. return err
  1013. }
  1014. func (context *mixedBuildBazelContext) generateBazelSymlinks(config Config, ctx invokeBazelContext) error {
  1015. eventHandler := ctx.GetEventHandler()
  1016. eventHandler.Begin("symlinks")
  1017. defer eventHandler.End("symlinks")
  1018. // Issue a build command of the phony root to generate symlink forests for dependencies of the
  1019. // Bazel build. This is necessary because aquery invocations do not generate this symlink forest,
  1020. // but some of symlinks may be required to resolve source dependencies of the build.
  1021. _, _, err := context.issueBazelCommand(context.createBazelCommand(config, bazel.BazelBuildPhonyRootRunName, buildCmd), context.paths, eventHandler)
  1022. return err
  1023. }
  1024. func (context *mixedBuildBazelContext) BuildStatementsToRegister() []*bazel.BuildStatement {
  1025. return context.buildStatements
  1026. }
  1027. func (context *mixedBuildBazelContext) AqueryDepsets() []bazel.AqueryDepset {
  1028. return context.depsets
  1029. }
  1030. func (context *mixedBuildBazelContext) OutputBase() string {
  1031. return context.paths.outputBase
  1032. }
  1033. // Singleton used for registering BUILD file ninja dependencies (needed
  1034. // for correctness of builds which use Bazel.
  1035. func BazelSingleton() Singleton {
  1036. return &bazelSingleton{}
  1037. }
  1038. type bazelSingleton struct{}
  1039. func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) {
  1040. // bazelSingleton is a no-op if mixed-soong-bazel-builds are disabled.
  1041. if !ctx.Config().IsMixedBuildsEnabled() {
  1042. return
  1043. }
  1044. // Add ninja file dependencies for files which all bazel invocations require.
  1045. bazelBuildList := absolutePath(filepath.Join(
  1046. filepath.Dir(ctx.Config().moduleListFile), "bazel.list"))
  1047. ctx.AddNinjaFileDeps(bazelBuildList)
  1048. data, err := os.ReadFile(bazelBuildList)
  1049. if err != nil {
  1050. ctx.Errorf(err.Error())
  1051. }
  1052. files := strings.Split(strings.TrimSpace(string(data)), "\n")
  1053. for _, file := range files {
  1054. ctx.AddNinjaFileDeps(file)
  1055. }
  1056. for _, depset := range ctx.Config().BazelContext.AqueryDepsets() {
  1057. var outputs []Path
  1058. var orderOnlies []Path
  1059. for _, depsetDepHash := range depset.TransitiveDepSetHashes {
  1060. otherDepsetName := bazelDepsetName(depsetDepHash)
  1061. outputs = append(outputs, PathForPhony(ctx, otherDepsetName))
  1062. }
  1063. for _, artifactPath := range depset.DirectArtifacts {
  1064. pathInBazelOut := PathForBazelOut(ctx, artifactPath)
  1065. if artifactPath == "bazel-out/volatile-status.txt" {
  1066. // See https://bazel.build/docs/user-manual#workspace-status
  1067. orderOnlies = append(orderOnlies, pathInBazelOut)
  1068. } else {
  1069. outputs = append(outputs, pathInBazelOut)
  1070. }
  1071. }
  1072. thisDepsetName := bazelDepsetName(depset.ContentHash)
  1073. ctx.Build(pctx, BuildParams{
  1074. Rule: blueprint.Phony,
  1075. Outputs: []WritablePath{PathForPhony(ctx, thisDepsetName)},
  1076. Implicits: outputs,
  1077. OrderOnly: orderOnlies,
  1078. })
  1079. }
  1080. executionRoot := path.Join(ctx.Config().BazelContext.OutputBase(), "execroot", "__main__")
  1081. bazelOutDir := path.Join(executionRoot, "bazel-out")
  1082. for index, buildStatement := range ctx.Config().BazelContext.BuildStatementsToRegister() {
  1083. // nil build statements are a valid case where we do not create an action because it is
  1084. // unnecessary or handled by other processing
  1085. if buildStatement == nil {
  1086. continue
  1087. }
  1088. if len(buildStatement.Command) > 0 {
  1089. rule := NewRuleBuilder(pctx, ctx)
  1090. createCommand(rule.Command(), buildStatement, executionRoot, bazelOutDir, ctx)
  1091. desc := fmt.Sprintf("%s: %s", buildStatement.Mnemonic, buildStatement.OutputPaths)
  1092. rule.Build(fmt.Sprintf("bazel %d", index), desc)
  1093. continue
  1094. }
  1095. // Certain actions returned by aquery (for instance FileWrite) do not contain a command
  1096. // and thus require special treatment. If BuildStatement were an interface implementing
  1097. // buildRule(ctx) function, the code here would just call it.
  1098. // Unfortunately, the BuildStatement is defined in
  1099. // the 'bazel' package, which cannot depend on 'android' package where ctx is defined,
  1100. // because this would cause circular dependency. So, until we move aquery processing
  1101. // to the 'android' package, we need to handle special cases here.
  1102. switch buildStatement.Mnemonic {
  1103. case "FileWrite", "SourceSymlinkManifest":
  1104. out := PathForBazelOut(ctx, buildStatement.OutputPaths[0])
  1105. WriteFileRuleVerbatim(ctx, out, buildStatement.FileContents)
  1106. case "SymlinkTree":
  1107. // build-runfiles arguments are the manifest file and the target directory
  1108. // where it creates the symlink tree according to this manifest (and then
  1109. // writes the MANIFEST file to it).
  1110. outManifest := PathForBazelOut(ctx, buildStatement.OutputPaths[0])
  1111. outManifestPath := outManifest.String()
  1112. if !strings.HasSuffix(outManifestPath, "MANIFEST") {
  1113. panic("the base name of the symlink tree action should be MANIFEST, got " + outManifestPath)
  1114. }
  1115. outDir := filepath.Dir(outManifestPath)
  1116. ctx.Build(pctx, BuildParams{
  1117. Rule: buildRunfilesRule,
  1118. Output: outManifest,
  1119. Inputs: []Path{PathForBazelOut(ctx, buildStatement.InputPaths[0])},
  1120. Description: "symlink tree for " + outDir,
  1121. Args: map[string]string{
  1122. "outDir": outDir,
  1123. },
  1124. })
  1125. default:
  1126. panic(fmt.Sprintf("unhandled build statement: %v", buildStatement))
  1127. }
  1128. }
  1129. }
  1130. // Register bazel-owned build statements (obtained from the aquery invocation).
  1131. func createCommand(cmd *RuleBuilderCommand, buildStatement *bazel.BuildStatement, executionRoot string, bazelOutDir string, ctx BuilderContext) {
  1132. // executionRoot is the action cwd.
  1133. cmd.Text(fmt.Sprintf("cd '%s' &&", executionRoot))
  1134. // Remove old outputs, as some actions might not rerun if the outputs are detected.
  1135. if len(buildStatement.OutputPaths) > 0 {
  1136. cmd.Text("rm -rf") // -r because outputs can be Bazel dir/tree artifacts.
  1137. for _, outputPath := range buildStatement.OutputPaths {
  1138. cmd.Text(fmt.Sprintf("'%s'", outputPath))
  1139. }
  1140. cmd.Text("&&")
  1141. }
  1142. for _, pair := range buildStatement.Env {
  1143. // Set per-action env variables, if any.
  1144. cmd.Flag(pair.Key + "=" + pair.Value)
  1145. }
  1146. // The actual Bazel action.
  1147. if len(buildStatement.Command) > 16*1024 {
  1148. commandFile := PathForBazelOut(ctx, buildStatement.OutputPaths[0]+".sh")
  1149. WriteFileRule(ctx, commandFile, buildStatement.Command)
  1150. cmd.Text("bash").Text(buildStatement.OutputPaths[0] + ".sh").Implicit(commandFile)
  1151. } else {
  1152. cmd.Text(buildStatement.Command)
  1153. }
  1154. for _, outputPath := range buildStatement.OutputPaths {
  1155. cmd.ImplicitOutput(PathForBazelOut(ctx, outputPath))
  1156. }
  1157. for _, inputPath := range buildStatement.InputPaths {
  1158. cmd.Implicit(PathForBazelOut(ctx, inputPath))
  1159. }
  1160. for _, inputDepsetHash := range buildStatement.InputDepsetHashes {
  1161. otherDepsetName := bazelDepsetName(inputDepsetHash)
  1162. cmd.Implicit(PathForPhony(ctx, otherDepsetName))
  1163. }
  1164. if depfile := buildStatement.Depfile; depfile != nil {
  1165. // The paths in depfile are relative to `executionRoot`.
  1166. // Hence, they need to be corrected by replacing "bazel-out"
  1167. // with the full `bazelOutDir`.
  1168. // Otherwise, implicit outputs and implicit inputs under "bazel-out/"
  1169. // would be deemed missing.
  1170. // (Note: The regexp uses a capture group because the version of sed
  1171. // does not support a look-behind pattern.)
  1172. replacement := fmt.Sprintf(`&& sed -i'' -E 's@(^|\s|")bazel-out/@\1%s/@g' '%s'`,
  1173. bazelOutDir, *depfile)
  1174. cmd.Text(replacement)
  1175. cmd.ImplicitDepFile(PathForBazelOut(ctx, *depfile))
  1176. }
  1177. for _, symlinkPath := range buildStatement.SymlinkPaths {
  1178. cmd.ImplicitSymlinkOutput(PathForBazelOut(ctx, symlinkPath))
  1179. }
  1180. }
  1181. func getCqueryId(key cqueryKey) string {
  1182. return key.label + "|" + getConfigString(key)
  1183. }
  1184. func getConfigString(key cqueryKey) string {
  1185. arch := key.configKey.arch
  1186. if len(arch) == 0 || arch == "common" {
  1187. if key.configKey.osType.Class == Device {
  1188. // For the generic Android, the expected result is "target|android", which
  1189. // corresponds to the product_variable_config named "android_target" in
  1190. // build/bazel/platforms/BUILD.bazel.
  1191. arch = "target"
  1192. } else {
  1193. // Use host platform, which is currently hardcoded to be x86_64.
  1194. arch = "x86_64"
  1195. }
  1196. }
  1197. osName := key.configKey.osType.Name
  1198. if len(osName) == 0 || osName == "common_os" || osName == "linux_glibc" || osName == "linux_musl" {
  1199. // Use host OS, which is currently hardcoded to be linux.
  1200. osName = "linux"
  1201. }
  1202. keyString := arch + "|" + osName
  1203. if key.configKey.apexKey.WithinApex {
  1204. keyString += "|" + withinApexToString(key.configKey.apexKey.WithinApex)
  1205. }
  1206. if len(key.configKey.apexKey.ApexSdkVersion) > 0 {
  1207. keyString += "|" + key.configKey.apexKey.ApexSdkVersion
  1208. }
  1209. if len(key.configKey.apexKey.ApiDomain) > 0 {
  1210. keyString += "|" + key.configKey.apexKey.ApiDomain
  1211. }
  1212. return keyString
  1213. }
  1214. func GetConfigKey(ctx BaseModuleContext) configKey {
  1215. return configKey{
  1216. // use string because Arch is not a valid key in go
  1217. arch: ctx.Arch().String(),
  1218. osType: ctx.Os(),
  1219. }
  1220. }
  1221. func GetConfigKeyApexVariant(ctx BaseModuleContext, apexKey *ApexConfigKey) configKey {
  1222. configKey := GetConfigKey(ctx)
  1223. if apexKey != nil {
  1224. configKey.apexKey = ApexConfigKey{
  1225. WithinApex: apexKey.WithinApex,
  1226. ApexSdkVersion: apexKey.ApexSdkVersion,
  1227. ApiDomain: apexKey.ApiDomain,
  1228. }
  1229. }
  1230. return configKey
  1231. }
  1232. func bazelDepsetName(contentHash string) string {
  1233. return fmt.Sprintf("bazel_depset_%s", contentHash)
  1234. }