bazel_handler.go 50 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436
  1. // Copyright 2020 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package android
  15. import (
  16. "bytes"
  17. "fmt"
  18. "os"
  19. "path"
  20. "path/filepath"
  21. "runtime"
  22. "sort"
  23. "strings"
  24. "sync"
  25. "android/soong/android/allowlists"
  26. "android/soong/bazel/cquery"
  27. "android/soong/shared"
  28. "android/soong/starlark_import"
  29. "github.com/google/blueprint"
  30. "github.com/google/blueprint/metrics"
  31. "android/soong/bazel"
  32. )
  33. var (
  34. _ = pctx.HostBinToolVariable("bazelBuildRunfilesTool", "build-runfiles")
  35. buildRunfilesRule = pctx.AndroidStaticRule("bazelBuildRunfiles", blueprint.RuleParams{
  36. Command: "${bazelBuildRunfilesTool} ${in} ${outDir}",
  37. Depfile: "",
  38. Description: "",
  39. CommandDeps: []string{"${bazelBuildRunfilesTool}"},
  40. }, "outDir")
  41. )
  42. func registerMixedBuildsMutator(ctx RegisterMutatorsContext) {
  43. ctx.BottomUp("mixed_builds_prep", mixedBuildsPrepareMutator).Parallel()
  44. }
  45. func RegisterMixedBuildsMutator(ctx RegistrationContext) {
  46. ctx.FinalDepsMutators(registerMixedBuildsMutator)
  47. }
  48. func mixedBuildsPrepareMutator(ctx BottomUpMutatorContext) {
  49. if m := ctx.Module(); m.Enabled() {
  50. if mixedBuildMod, ok := m.(MixedBuildBuildable); ok {
  51. mixedBuildEnabled := MixedBuildsEnabled(ctx)
  52. queueMixedBuild := mixedBuildMod.IsMixedBuildSupported(ctx) && mixedBuildEnabled == MixedBuildEnabled
  53. if queueMixedBuild {
  54. mixedBuildMod.QueueBazelCall(ctx)
  55. }
  56. }
  57. }
  58. }
  59. type cqueryRequest interface {
  60. // Name returns a string name for this request type. Such request type names must be unique,
  61. // and must only consist of alphanumeric characters.
  62. Name() string
  63. // StarlarkFunctionBody returns a starlark function body to process this request type.
  64. // The returned string is the body of a Starlark function which obtains
  65. // all request-relevant information about a target and returns a string containing
  66. // this information.
  67. // The function should have the following properties:
  68. // - The arguments are `target` (a configured target) and `id_string` (the label + configuration).
  69. // - The return value must be a string.
  70. // - The function body should not be indented outside of its own scope.
  71. StarlarkFunctionBody() string
  72. }
  73. // Portion of cquery map key to describe target configuration.
  74. type configKey struct {
  75. arch string
  76. osType OsType
  77. apexKey ApexConfigKey
  78. }
  79. type ApexConfigKey struct {
  80. WithinApex bool
  81. ApexSdkVersion string
  82. ApiDomain string
  83. }
  84. func (c ApexConfigKey) String() string {
  85. return fmt.Sprintf("%s_%s_%s", withinApexToString(c.WithinApex), c.ApexSdkVersion, c.ApiDomain)
  86. }
  87. func withinApexToString(withinApex bool) string {
  88. if withinApex {
  89. return "within_apex"
  90. }
  91. return ""
  92. }
  93. func (c configKey) String() string {
  94. return fmt.Sprintf("%s::%s::%s", c.arch, c.osType, c.apexKey)
  95. }
  96. // Map key to describe bazel cquery requests.
  97. type cqueryKey struct {
  98. label string
  99. requestType cqueryRequest
  100. configKey configKey
  101. }
  102. func makeCqueryKey(label string, cqueryRequest cqueryRequest, cfgKey configKey) cqueryKey {
  103. if strings.HasPrefix(label, "//") {
  104. // Normalize Bazel labels to specify main repository explicitly.
  105. label = "@" + label
  106. }
  107. return cqueryKey{label, cqueryRequest, cfgKey}
  108. }
  109. func (c cqueryKey) String() string {
  110. return fmt.Sprintf("cquery(%s,%s,%s)", c.label, c.requestType.Name(), c.configKey)
  111. }
  112. type invokeBazelContext interface {
  113. GetEventHandler() *metrics.EventHandler
  114. }
  115. // BazelContext is a context object useful for interacting with Bazel during
  116. // the course of a build. Use of Bazel to evaluate part of the build graph
  117. // is referred to as a "mixed build". (Some modules are managed by Soong,
  118. // some are managed by Bazel). To facilitate interop between these build
  119. // subgraphs, Soong may make requests to Bazel and evaluate their responses
  120. // so that Soong modules may accurately depend on Bazel targets.
  121. type BazelContext interface {
  122. // Add a cquery request to the bazel request queue. All queued requests
  123. // will be sent to Bazel on a subsequent invocation of InvokeBazel.
  124. QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey)
  125. // ** Cquery Results Retrieval Functions
  126. // The below functions pertain to retrieving cquery results from a prior
  127. // InvokeBazel function call and parsing the results.
  128. // Returns result files built by building the given bazel target label.
  129. GetOutputFiles(label string, cfgKey configKey) ([]string, error)
  130. // Returns the results of GetOutputFiles and GetCcObjectFiles in a single query (in that order).
  131. GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error)
  132. // Returns the results of the GetApexInfo query (including output files)
  133. GetApexInfo(label string, cfgkey configKey) (cquery.ApexInfo, error)
  134. // Returns the results of the GetCcUnstrippedInfo query
  135. GetCcUnstrippedInfo(label string, cfgkey configKey) (cquery.CcUnstrippedInfo, error)
  136. // Returns the results of the GetPrebuiltFileInfo query
  137. GetPrebuiltFileInfo(label string, cfgKey configKey) (cquery.PrebuiltFileInfo, error)
  138. // ** end Cquery Results Retrieval Functions
  139. // Issues commands to Bazel to receive results for all cquery requests
  140. // queued in the BazelContext. The ctx argument is optional and is only
  141. // used for performance data collection
  142. InvokeBazel(config Config, ctx invokeBazelContext) error
  143. // Returns true if Bazel handling is enabled for the module with the given name.
  144. // Note that this only implies "bazel mixed build" allowlisting. The caller
  145. // should independently verify the module is eligible for Bazel handling
  146. // (for example, that it is MixedBuildBuildable).
  147. IsModuleNameAllowed(moduleName string, withinApex bool) bool
  148. IsModuleDclaAllowed(moduleName string) bool
  149. // Returns the bazel output base (the root directory for all bazel intermediate outputs).
  150. OutputBase() string
  151. // Returns build statements which should get registered to reflect Bazel's outputs.
  152. BuildStatementsToRegister() []*bazel.BuildStatement
  153. // Returns the depsets defined in Bazel's aquery response.
  154. AqueryDepsets() []bazel.AqueryDepset
  155. }
  156. type bazelRunner interface {
  157. issueBazelCommand(cmdRequest bazel.CmdRequest, paths *bazelPaths, eventHandler *metrics.EventHandler) (output string, errorMessage string, error error)
  158. }
  159. type bazelPaths struct {
  160. homeDir string
  161. bazelPath string
  162. outputBase string
  163. workspaceDir string
  164. soongOutDir string
  165. metricsDir string
  166. bazelDepsFile string
  167. }
  168. // A context object which tracks queued requests that need to be made to Bazel,
  169. // and their results after the requests have been made.
  170. type mixedBuildBazelContext struct {
  171. bazelRunner
  172. paths *bazelPaths
  173. // cquery requests that have not yet been issued to Bazel. This list is maintained
  174. // in a sorted state, and is guaranteed to have no duplicates.
  175. requests []cqueryKey
  176. requestMutex sync.Mutex // requests can be written in parallel
  177. results map[cqueryKey]string // Results of cquery requests after Bazel invocations
  178. // Build statements which should get registered to reflect Bazel's outputs.
  179. buildStatements []*bazel.BuildStatement
  180. // Depsets which should be used for Bazel's build statements.
  181. depsets []bazel.AqueryDepset
  182. // Per-module allowlist/denylist functionality to control whether analysis of
  183. // modules are handled by Bazel. For modules which do not have a Bazel definition
  184. // (or do not sufficiently support bazel handling via MixedBuildBuildable),
  185. // this allowlist will have no effect, even if the module is explicitly allowlisted here.
  186. // Per-module denylist to opt modules out of bazel handling.
  187. bazelDisabledModules map[string]bool
  188. // Per-module allowlist to opt modules in to bazel handling.
  189. bazelEnabledModules map[string]bool
  190. // DCLA modules are enabled when used in apex.
  191. bazelDclaEnabledModules map[string]bool
  192. targetProduct string
  193. targetBuildVariant string
  194. }
  195. var _ BazelContext = &mixedBuildBazelContext{}
  196. // A bazel context to use when Bazel is disabled.
  197. type noopBazelContext struct{}
  198. var _ BazelContext = noopBazelContext{}
  199. // A bazel context to use for tests.
  200. type MockBazelContext struct {
  201. OutputBaseDir string
  202. LabelToOutputFiles map[string][]string
  203. LabelToCcInfo map[string]cquery.CcInfo
  204. LabelToPythonBinary map[string]string
  205. LabelToApexInfo map[string]cquery.ApexInfo
  206. LabelToCcBinary map[string]cquery.CcUnstrippedInfo
  207. LabelToPrebuiltFileInfo map[string]cquery.PrebuiltFileInfo
  208. BazelRequests map[string]bool
  209. }
  210. func (m MockBazelContext) QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey) {
  211. key := BuildMockBazelContextRequestKey(label, requestType, cfgKey.arch, cfgKey.osType, cfgKey.apexKey)
  212. if m.BazelRequests == nil {
  213. m.BazelRequests = make(map[string]bool)
  214. }
  215. m.BazelRequests[key] = true
  216. }
  217. func (m MockBazelContext) GetOutputFiles(label string, _ configKey) ([]string, error) {
  218. result, ok := m.LabelToOutputFiles[label]
  219. if !ok {
  220. return []string{}, fmt.Errorf("no target with label %q in LabelToOutputFiles", label)
  221. }
  222. return result, nil
  223. }
  224. func (m MockBazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error) {
  225. result, ok := m.LabelToCcInfo[label]
  226. if !ok {
  227. key := BuildMockBazelContextResultKey(label, cfgKey.arch, cfgKey.osType, cfgKey.apexKey)
  228. result, ok = m.LabelToCcInfo[key]
  229. if !ok {
  230. return cquery.CcInfo{}, fmt.Errorf("no target with label %q in LabelToCcInfo", label)
  231. }
  232. }
  233. return result, nil
  234. }
  235. func (m MockBazelContext) GetApexInfo(label string, _ configKey) (cquery.ApexInfo, error) {
  236. result, ok := m.LabelToApexInfo[label]
  237. if !ok {
  238. return cquery.ApexInfo{}, fmt.Errorf("no target with label %q in LabelToApexInfo", label)
  239. }
  240. return result, nil
  241. }
  242. func (m MockBazelContext) GetCcUnstrippedInfo(label string, _ configKey) (cquery.CcUnstrippedInfo, error) {
  243. result, ok := m.LabelToCcBinary[label]
  244. if !ok {
  245. return cquery.CcUnstrippedInfo{}, fmt.Errorf("no target with label %q in LabelToCcBinary", label)
  246. }
  247. return result, nil
  248. }
  249. func (m MockBazelContext) GetPrebuiltFileInfo(label string, _ configKey) (cquery.PrebuiltFileInfo, error) {
  250. result, ok := m.LabelToPrebuiltFileInfo[label]
  251. if !ok {
  252. return cquery.PrebuiltFileInfo{}, fmt.Errorf("no target with label %q in LabelToPrebuiltFileInfo", label)
  253. }
  254. return result, nil
  255. }
  256. func (m MockBazelContext) InvokeBazel(_ Config, _ invokeBazelContext) error {
  257. panic("unimplemented")
  258. }
  259. func (m MockBazelContext) IsModuleNameAllowed(_ string, _ bool) bool {
  260. return true
  261. }
  262. func (m MockBazelContext) IsModuleDclaAllowed(_ string) bool {
  263. return true
  264. }
  265. func (m MockBazelContext) OutputBase() string { return m.OutputBaseDir }
  266. func (m MockBazelContext) BuildStatementsToRegister() []*bazel.BuildStatement {
  267. return []*bazel.BuildStatement{}
  268. }
  269. func (m MockBazelContext) AqueryDepsets() []bazel.AqueryDepset {
  270. return []bazel.AqueryDepset{}
  271. }
  272. var _ BazelContext = MockBazelContext{}
  273. func BuildMockBazelContextRequestKey(label string, request cqueryRequest, arch string, osType OsType, apexKey ApexConfigKey) string {
  274. cfgKey := configKey{
  275. arch: arch,
  276. osType: osType,
  277. apexKey: apexKey,
  278. }
  279. return strings.Join([]string{label, request.Name(), cfgKey.String()}, "_")
  280. }
  281. func BuildMockBazelContextResultKey(label string, arch string, osType OsType, apexKey ApexConfigKey) string {
  282. cfgKey := configKey{
  283. arch: arch,
  284. osType: osType,
  285. apexKey: apexKey,
  286. }
  287. return strings.Join([]string{label, cfgKey.String()}, "_")
  288. }
  289. func (bazelCtx *mixedBuildBazelContext) QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey) {
  290. key := makeCqueryKey(label, requestType, cfgKey)
  291. bazelCtx.requestMutex.Lock()
  292. defer bazelCtx.requestMutex.Unlock()
  293. // Insert key into requests, maintaining the sort, and only if it's not duplicate.
  294. keyString := key.String()
  295. foundEqual := false
  296. notLessThanKeyString := func(i int) bool {
  297. s := bazelCtx.requests[i].String()
  298. v := strings.Compare(s, keyString)
  299. if v == 0 {
  300. foundEqual = true
  301. }
  302. return v >= 0
  303. }
  304. targetIndex := sort.Search(len(bazelCtx.requests), notLessThanKeyString)
  305. if foundEqual {
  306. return
  307. }
  308. if targetIndex == len(bazelCtx.requests) {
  309. bazelCtx.requests = append(bazelCtx.requests, key)
  310. } else {
  311. bazelCtx.requests = append(bazelCtx.requests[:targetIndex+1], bazelCtx.requests[targetIndex:]...)
  312. bazelCtx.requests[targetIndex] = key
  313. }
  314. }
  315. func (bazelCtx *mixedBuildBazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, error) {
  316. key := makeCqueryKey(label, cquery.GetOutputFiles, cfgKey)
  317. if rawString, ok := bazelCtx.results[key]; ok {
  318. bazelOutput := strings.TrimSpace(rawString)
  319. return cquery.GetOutputFiles.ParseResult(bazelOutput), nil
  320. }
  321. return nil, fmt.Errorf("no bazel response found for %v", key)
  322. }
  323. func (bazelCtx *mixedBuildBazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error) {
  324. key := makeCqueryKey(label, cquery.GetCcInfo, cfgKey)
  325. if rawString, ok := bazelCtx.results[key]; ok {
  326. bazelOutput := strings.TrimSpace(rawString)
  327. return cquery.GetCcInfo.ParseResult(bazelOutput)
  328. }
  329. return cquery.CcInfo{}, fmt.Errorf("no bazel response found for %v", key)
  330. }
  331. func (bazelCtx *mixedBuildBazelContext) GetApexInfo(label string, cfgKey configKey) (cquery.ApexInfo, error) {
  332. key := makeCqueryKey(label, cquery.GetApexInfo, cfgKey)
  333. if rawString, ok := bazelCtx.results[key]; ok {
  334. return cquery.GetApexInfo.ParseResult(strings.TrimSpace(rawString))
  335. }
  336. return cquery.ApexInfo{}, fmt.Errorf("no bazel response found for %v", key)
  337. }
  338. func (bazelCtx *mixedBuildBazelContext) GetCcUnstrippedInfo(label string, cfgKey configKey) (cquery.CcUnstrippedInfo, error) {
  339. key := makeCqueryKey(label, cquery.GetCcUnstrippedInfo, cfgKey)
  340. if rawString, ok := bazelCtx.results[key]; ok {
  341. return cquery.GetCcUnstrippedInfo.ParseResult(strings.TrimSpace(rawString))
  342. }
  343. return cquery.CcUnstrippedInfo{}, fmt.Errorf("no bazel response for %s", key)
  344. }
  345. func (bazelCtx *mixedBuildBazelContext) GetPrebuiltFileInfo(label string, cfgKey configKey) (cquery.PrebuiltFileInfo, error) {
  346. key := makeCqueryKey(label, cquery.GetPrebuiltFileInfo, cfgKey)
  347. if rawString, ok := bazelCtx.results[key]; ok {
  348. return cquery.GetPrebuiltFileInfo.ParseResult(strings.TrimSpace(rawString))
  349. }
  350. return cquery.PrebuiltFileInfo{}, fmt.Errorf("no bazel response for %s", key)
  351. }
  352. func (n noopBazelContext) QueueBazelRequest(_ string, _ cqueryRequest, _ configKey) {
  353. panic("unimplemented")
  354. }
  355. func (n noopBazelContext) GetOutputFiles(_ string, _ configKey) ([]string, error) {
  356. panic("unimplemented")
  357. }
  358. func (n noopBazelContext) GetCcInfo(_ string, _ configKey) (cquery.CcInfo, error) {
  359. panic("unimplemented")
  360. }
  361. func (n noopBazelContext) GetApexInfo(_ string, _ configKey) (cquery.ApexInfo, error) {
  362. panic("unimplemented")
  363. }
  364. func (n noopBazelContext) GetCcUnstrippedInfo(_ string, _ configKey) (cquery.CcUnstrippedInfo, error) {
  365. //TODO implement me
  366. panic("implement me")
  367. }
  368. func (n noopBazelContext) GetPrebuiltFileInfo(_ string, _ configKey) (cquery.PrebuiltFileInfo, error) {
  369. panic("implement me")
  370. }
  371. func (n noopBazelContext) InvokeBazel(_ Config, _ invokeBazelContext) error {
  372. panic("unimplemented")
  373. }
  374. func (m noopBazelContext) OutputBase() string {
  375. return ""
  376. }
  377. func (n noopBazelContext) IsModuleNameAllowed(_ string, _ bool) bool {
  378. return false
  379. }
  380. func (n noopBazelContext) IsModuleDclaAllowed(_ string) bool {
  381. return false
  382. }
  383. func (m noopBazelContext) BuildStatementsToRegister() []*bazel.BuildStatement {
  384. return []*bazel.BuildStatement{}
  385. }
  386. func (m noopBazelContext) AqueryDepsets() []bazel.AqueryDepset {
  387. return []bazel.AqueryDepset{}
  388. }
  389. func AddToStringSet(set map[string]bool, items []string) {
  390. for _, item := range items {
  391. set[item] = true
  392. }
  393. }
  394. func GetBazelEnabledAndDisabledModules(buildMode SoongBuildMode, forceEnabled map[string]struct{}) (map[string]bool, map[string]bool) {
  395. disabledModules := map[string]bool{}
  396. enabledModules := map[string]bool{}
  397. switch buildMode {
  398. case BazelProdMode:
  399. AddToStringSet(enabledModules, allowlists.ProdMixedBuildsEnabledList)
  400. for enabledAdHocModule := range forceEnabled {
  401. enabledModules[enabledAdHocModule] = true
  402. }
  403. case BazelStagingMode:
  404. // Staging mode includes all prod modules plus all staging modules.
  405. AddToStringSet(enabledModules, allowlists.ProdMixedBuildsEnabledList)
  406. AddToStringSet(enabledModules, allowlists.StagingMixedBuildsEnabledList)
  407. for enabledAdHocModule := range forceEnabled {
  408. enabledModules[enabledAdHocModule] = true
  409. }
  410. default:
  411. panic("Expected BazelProdMode or BazelStagingMode")
  412. }
  413. return enabledModules, disabledModules
  414. }
  415. func GetBazelEnabledModules(buildMode SoongBuildMode) []string {
  416. enabledModules, disabledModules := GetBazelEnabledAndDisabledModules(buildMode, nil)
  417. enabledList := make([]string, 0, len(enabledModules))
  418. for module := range enabledModules {
  419. if !disabledModules[module] {
  420. enabledList = append(enabledList, module)
  421. }
  422. }
  423. sort.Strings(enabledList)
  424. return enabledList
  425. }
  426. func NewBazelContext(c *config) (BazelContext, error) {
  427. if c.BuildMode != BazelProdMode && c.BuildMode != BazelStagingMode {
  428. return noopBazelContext{}, nil
  429. }
  430. enabledModules, disabledModules := GetBazelEnabledAndDisabledModules(c.BuildMode, c.BazelModulesForceEnabledByFlag())
  431. paths := bazelPaths{
  432. soongOutDir: c.soongOutDir,
  433. }
  434. var missing []string
  435. vars := []struct {
  436. name string
  437. ptr *string
  438. // True if the environment variable needs to be tracked so that changes to the variable
  439. // cause the ninja file to be regenerated, false otherwise. False should only be set for
  440. // environment variables that have no effect on the generated ninja file.
  441. track bool
  442. }{
  443. {"BAZEL_HOME", &paths.homeDir, true},
  444. {"BAZEL_PATH", &paths.bazelPath, true},
  445. {"BAZEL_OUTPUT_BASE", &paths.outputBase, true},
  446. {"BAZEL_WORKSPACE", &paths.workspaceDir, true},
  447. {"BAZEL_METRICS_DIR", &paths.metricsDir, false},
  448. {"BAZEL_DEPS_FILE", &paths.bazelDepsFile, true},
  449. }
  450. for _, v := range vars {
  451. if v.track {
  452. if s := c.Getenv(v.name); len(s) > 1 {
  453. *v.ptr = s
  454. continue
  455. }
  456. } else if s, ok := c.env[v.name]; ok {
  457. *v.ptr = s
  458. } else {
  459. missing = append(missing, v.name)
  460. }
  461. }
  462. if len(missing) > 0 {
  463. return nil, fmt.Errorf("missing required env vars to use bazel: %s", missing)
  464. }
  465. targetBuildVariant := "user"
  466. if c.Eng() {
  467. targetBuildVariant = "eng"
  468. } else if c.Debuggable() {
  469. targetBuildVariant = "userdebug"
  470. }
  471. targetProduct := "unknown"
  472. if c.HasDeviceProduct() {
  473. targetProduct = c.DeviceProduct()
  474. }
  475. dclaMixedBuildsEnabledList := []string{}
  476. if c.BuildMode == BazelProdMode {
  477. dclaMixedBuildsEnabledList = allowlists.ProdDclaMixedBuildsEnabledList
  478. } else if c.BuildMode == BazelStagingMode {
  479. dclaMixedBuildsEnabledList = append(allowlists.ProdDclaMixedBuildsEnabledList,
  480. allowlists.StagingDclaMixedBuildsEnabledList...)
  481. }
  482. dclaEnabledModules := map[string]bool{}
  483. AddToStringSet(dclaEnabledModules, dclaMixedBuildsEnabledList)
  484. return &mixedBuildBazelContext{
  485. bazelRunner: &builtinBazelRunner{c.UseBazelProxy, absolutePath(c.outDir)},
  486. paths: &paths,
  487. bazelEnabledModules: enabledModules,
  488. bazelDisabledModules: disabledModules,
  489. bazelDclaEnabledModules: dclaEnabledModules,
  490. targetProduct: targetProduct,
  491. targetBuildVariant: targetBuildVariant,
  492. }, nil
  493. }
  494. func (p *bazelPaths) BazelMetricsDir() string {
  495. return p.metricsDir
  496. }
  497. func (context *mixedBuildBazelContext) IsModuleNameAllowed(moduleName string, withinApex bool) bool {
  498. if context.bazelDisabledModules[moduleName] {
  499. return false
  500. }
  501. if context.bazelEnabledModules[moduleName] {
  502. return true
  503. }
  504. if withinApex && context.IsModuleDclaAllowed(moduleName) {
  505. return true
  506. }
  507. return false
  508. }
  509. func (context *mixedBuildBazelContext) IsModuleDclaAllowed(moduleName string) bool {
  510. return context.bazelDclaEnabledModules[moduleName]
  511. }
  512. func pwdPrefix() string {
  513. // Darwin doesn't have /proc
  514. if runtime.GOOS != "darwin" {
  515. return "PWD=/proc/self/cwd"
  516. }
  517. return ""
  518. }
  519. type bazelCommand struct {
  520. command string
  521. // query or label
  522. expression string
  523. }
  524. type builtinBazelRunner struct {
  525. useBazelProxy bool
  526. outDir string
  527. }
  528. // Issues the given bazel command with given build label and additional flags.
  529. // Returns (stdout, stderr, error). The first and second return values are strings
  530. // containing the stdout and stderr of the run command, and an error is returned if
  531. // the invocation returned an error code.
  532. func (r *builtinBazelRunner) issueBazelCommand(cmdRequest bazel.CmdRequest, paths *bazelPaths, eventHandler *metrics.EventHandler) (string, string, error) {
  533. if r.useBazelProxy {
  534. eventHandler.Begin("client_proxy")
  535. defer eventHandler.End("client_proxy")
  536. proxyClient := bazel.NewProxyClient(r.outDir)
  537. resp, err := proxyClient.IssueCommand(cmdRequest)
  538. if err != nil {
  539. return "", "", err
  540. }
  541. if len(resp.ErrorString) > 0 {
  542. return "", "", fmt.Errorf(resp.ErrorString)
  543. }
  544. return resp.Stdout, resp.Stderr, nil
  545. } else {
  546. eventHandler.Begin("bazel command")
  547. defer eventHandler.End("bazel command")
  548. stdout, stderr, err := bazel.ExecBazel(paths.bazelPath, absolutePath(paths.syntheticWorkspaceDir()), cmdRequest)
  549. return string(stdout), string(stderr), err
  550. }
  551. }
  552. func (context *mixedBuildBazelContext) createBazelCommand(config Config, runName bazel.RunName, command bazelCommand,
  553. extraFlags ...string) bazel.CmdRequest {
  554. if runtime.GOOS != "linux" && runtime.GOOS != "darwin" {
  555. panic("Unknown GOOS: " + runtime.GOOS)
  556. }
  557. cmdFlags := []string{
  558. "--output_base=" + absolutePath(context.paths.outputBase),
  559. command.command,
  560. command.expression,
  561. "--profile=" + shared.BazelMetricsFilename(context.paths, runName),
  562. "--host_platform=@soong_injection//product_config_platforms:mixed_builds_product-" + context.targetBuildVariant + "_" + runtime.GOOS + "_x86_64",
  563. // Don't specify --platforms, because on some products/branches (like kernel-build-tools)
  564. // the main platform for mixed_builds_product-variant doesn't exist because an arch isn't
  565. // specified in product config. The derivative platforms that config_node transitions into
  566. // will still work.
  567. // Suppress noise
  568. "--ui_event_filters=-INFO",
  569. "--noshow_progress",
  570. "--norun_validations",
  571. }
  572. cmdFlags = append(cmdFlags, extraFlags...)
  573. extraEnv := []string{
  574. "HOME=" + context.paths.homeDir,
  575. pwdPrefix(),
  576. "BUILD_DIR=" + absolutePath(context.paths.soongOutDir),
  577. // Make OUT_DIR absolute here so build/bazel/bin/bazel uses the correct
  578. // OUT_DIR at <root>/out, instead of <root>/out/soong/workspace/out.
  579. "OUT_DIR=" + absolutePath(context.paths.outDir()),
  580. // Disables local host detection of gcc; toolchain information is defined
  581. // explicitly in BUILD files.
  582. "BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1",
  583. }
  584. capturedEnvVars, err := starlark_import.GetStarlarkValue[[]string]("captured_env_vars")
  585. if err != nil {
  586. panic(err)
  587. }
  588. for _, envvar := range capturedEnvVars {
  589. val := config.Getenv(envvar)
  590. if val == "" {
  591. continue
  592. }
  593. extraEnv = append(extraEnv, fmt.Sprintf("%s=%s", envvar, val))
  594. }
  595. envVars := append(os.Environ(), extraEnv...)
  596. return bazel.CmdRequest{cmdFlags, envVars}
  597. }
  598. func (context *mixedBuildBazelContext) printableCqueryCommand(bazelCmd bazel.CmdRequest) string {
  599. args := append([]string{context.paths.bazelPath}, bazelCmd.Argv...)
  600. outputString := strings.Join(bazelCmd.Env, " ") + " \"" + strings.Join(args, "\" \"") + "\""
  601. return outputString
  602. }
  603. func (context *mixedBuildBazelContext) mainBzlFileContents() []byte {
  604. // TODO(cparsons): Define configuration transitions programmatically based
  605. // on available archs.
  606. contents := `
  607. #####################################################
  608. # This file is generated by soong_build. Do not edit.
  609. #####################################################
  610. def _config_node_transition_impl(settings, attr):
  611. if attr.os == "android" and attr.arch == "target":
  612. target = "mixed_builds_product-{VARIANT}"
  613. else:
  614. target = "mixed_builds_product-{VARIANT}_%s_%s" % (attr.os, attr.arch)
  615. apex_name = ""
  616. if attr.within_apex:
  617. # //build/bazel/rules/apex:apex_name has to be set to a non_empty value,
  618. # otherwise //build/bazel/rules/apex:non_apex will be true and the
  619. # "-D__ANDROID_APEX__" compiler flag will be missing. Apex_name is used
  620. # in some validation on bazel side which don't really apply in mixed
  621. # build because soong will do the work, so we just set it to a fixed
  622. # value here.
  623. apex_name = "dcla_apex"
  624. outputs = {
  625. "//command_line_option:platforms": "@soong_injection//product_config_platforms:%s" % target,
  626. "@//build/bazel/rules/apex:within_apex": attr.within_apex,
  627. "@//build/bazel/rules/apex:min_sdk_version": attr.apex_sdk_version,
  628. "@//build/bazel/rules/apex:apex_name": apex_name,
  629. "@//build/bazel/rules/apex:api_domain": attr.api_domain,
  630. }
  631. return outputs
  632. _config_node_transition = transition(
  633. implementation = _config_node_transition_impl,
  634. inputs = [],
  635. outputs = [
  636. "//command_line_option:platforms",
  637. "@//build/bazel/rules/apex:within_apex",
  638. "@//build/bazel/rules/apex:min_sdk_version",
  639. "@//build/bazel/rules/apex:apex_name",
  640. "@//build/bazel/rules/apex:api_domain",
  641. ],
  642. )
  643. def _passthrough_rule_impl(ctx):
  644. return [DefaultInfo(files = depset(ctx.files.deps))]
  645. config_node = rule(
  646. implementation = _passthrough_rule_impl,
  647. attrs = {
  648. "arch" : attr.string(mandatory = True),
  649. "os" : attr.string(mandatory = True),
  650. "within_apex" : attr.bool(default = False),
  651. "apex_sdk_version" : attr.string(mandatory = True),
  652. "api_domain" : attr.string(mandatory = True),
  653. "deps" : attr.label_list(cfg = _config_node_transition, allow_files = True),
  654. "_allowlist_function_transition": attr.label(default = "@bazel_tools//tools/allowlists/function_transition_allowlist"),
  655. },
  656. )
  657. # Rule representing the root of the build, to depend on all Bazel targets that
  658. # are required for the build. Building this target will build the entire Bazel
  659. # build tree.
  660. mixed_build_root = rule(
  661. implementation = _passthrough_rule_impl,
  662. attrs = {
  663. "deps" : attr.label_list(),
  664. },
  665. )
  666. def _phony_root_impl(ctx):
  667. return []
  668. # Rule to depend on other targets but build nothing.
  669. # This is useful as follows: building a target of this rule will generate
  670. # symlink forests for all dependencies of the target, without executing any
  671. # actions of the build.
  672. phony_root = rule(
  673. implementation = _phony_root_impl,
  674. attrs = {"deps" : attr.label_list()},
  675. )
  676. `
  677. productReplacer := strings.NewReplacer(
  678. "{PRODUCT}", context.targetProduct,
  679. "{VARIANT}", context.targetBuildVariant)
  680. return []byte(productReplacer.Replace(contents))
  681. }
  682. func (context *mixedBuildBazelContext) mainBuildFileContents() []byte {
  683. // TODO(cparsons): Map label to attribute programmatically; don't use hard-coded
  684. // architecture mapping.
  685. formatString := `
  686. # This file is generated by soong_build. Do not edit.
  687. load(":main.bzl", "config_node", "mixed_build_root", "phony_root")
  688. %s
  689. mixed_build_root(name = "buildroot",
  690. deps = [%s],
  691. testonly = True, # Unblocks testonly deps.
  692. )
  693. phony_root(name = "phonyroot",
  694. deps = [":buildroot"],
  695. testonly = True, # Unblocks testonly deps.
  696. )
  697. `
  698. configNodeFormatString := `
  699. config_node(name = "%s",
  700. arch = "%s",
  701. os = "%s",
  702. within_apex = %s,
  703. apex_sdk_version = "%s",
  704. api_domain = "%s",
  705. deps = [%s],
  706. testonly = True, # Unblocks testonly deps.
  707. )
  708. `
  709. configNodesSection := ""
  710. labelsByConfig := map[string][]string{}
  711. for _, val := range context.requests {
  712. labelString := fmt.Sprintf("\"@%s\"", val.label)
  713. configString := getConfigString(val)
  714. labelsByConfig[configString] = append(labelsByConfig[configString], labelString)
  715. }
  716. // Configs need to be sorted to maintain determinism of the BUILD file.
  717. sortedConfigs := make([]string, 0, len(labelsByConfig))
  718. for val := range labelsByConfig {
  719. sortedConfigs = append(sortedConfigs, val)
  720. }
  721. sort.Slice(sortedConfigs, func(i, j int) bool { return sortedConfigs[i] < sortedConfigs[j] })
  722. allLabels := []string{}
  723. for _, configString := range sortedConfigs {
  724. labels := labelsByConfig[configString]
  725. configTokens := strings.Split(configString, "|")
  726. if len(configTokens) < 2 {
  727. panic(fmt.Errorf("Unexpected config string format: %s", configString))
  728. }
  729. archString := configTokens[0]
  730. osString := configTokens[1]
  731. withinApex := "False"
  732. apexSdkVerString := ""
  733. apiDomainString := ""
  734. if osString == "android" {
  735. // api domains are meaningful only for device variants
  736. apiDomainString = "system"
  737. }
  738. targetString := fmt.Sprintf("%s_%s", osString, archString)
  739. if len(configTokens) > 2 {
  740. targetString += "_" + configTokens[2]
  741. if configTokens[2] == withinApexToString(true) {
  742. withinApex = "True"
  743. }
  744. }
  745. if len(configTokens) > 3 {
  746. targetString += "_" + configTokens[3]
  747. apexSdkVerString = configTokens[3]
  748. }
  749. if len(configTokens) > 4 {
  750. apiDomainString = configTokens[4]
  751. targetString += "_" + apiDomainString
  752. }
  753. allLabels = append(allLabels, fmt.Sprintf("\":%s\"", targetString))
  754. labelsString := strings.Join(labels, ",\n ")
  755. configNodesSection += fmt.Sprintf(configNodeFormatString, targetString, archString, osString, withinApex, apexSdkVerString, apiDomainString,
  756. labelsString)
  757. }
  758. return []byte(fmt.Sprintf(formatString, configNodesSection, strings.Join(allLabels, ",\n ")))
  759. }
  760. func indent(original string) string {
  761. result := ""
  762. for _, line := range strings.Split(original, "\n") {
  763. result += " " + line + "\n"
  764. }
  765. return result
  766. }
  767. // Returns the file contents of the buildroot.cquery file that should be used for the cquery
  768. // expression in order to obtain information about buildroot and its dependencies.
  769. // The contents of this file depend on the mixedBuildBazelContext's requests; requests are enumerated
  770. // and grouped by their request type. The data retrieved for each label depends on its
  771. // request type.
  772. func (context *mixedBuildBazelContext) cqueryStarlarkFileContents() []byte {
  773. requestTypeToCqueryIdEntries := map[cqueryRequest][]string{}
  774. requestTypes := []cqueryRequest{}
  775. for _, val := range context.requests {
  776. cqueryId := getCqueryId(val)
  777. mapEntryString := fmt.Sprintf("%q : True", cqueryId)
  778. if _, seenKey := requestTypeToCqueryIdEntries[val.requestType]; !seenKey {
  779. requestTypes = append(requestTypes, val.requestType)
  780. }
  781. requestTypeToCqueryIdEntries[val.requestType] =
  782. append(requestTypeToCqueryIdEntries[val.requestType], mapEntryString)
  783. }
  784. labelRegistrationMapSection := ""
  785. functionDefSection := ""
  786. mainSwitchSection := ""
  787. mapDeclarationFormatString := `
  788. %s = {
  789. %s
  790. }
  791. `
  792. functionDefFormatString := `
  793. def %s(target, id_string):
  794. %s
  795. `
  796. mainSwitchSectionFormatString := `
  797. if id_string in %s:
  798. return id_string + ">>" + %s(target, id_string)
  799. `
  800. for _, requestType := range requestTypes {
  801. labelMapName := requestType.Name() + "_Labels"
  802. functionName := requestType.Name() + "_Fn"
  803. labelRegistrationMapSection += fmt.Sprintf(mapDeclarationFormatString,
  804. labelMapName,
  805. strings.Join(requestTypeToCqueryIdEntries[requestType], ",\n "))
  806. functionDefSection += fmt.Sprintf(functionDefFormatString,
  807. functionName,
  808. indent(requestType.StarlarkFunctionBody()))
  809. mainSwitchSection += fmt.Sprintf(mainSwitchSectionFormatString,
  810. labelMapName, functionName)
  811. }
  812. formatString := `
  813. # This file is generated by soong_build. Do not edit.
  814. {LABEL_REGISTRATION_MAP_SECTION}
  815. {FUNCTION_DEF_SECTION}
  816. def get_arch(target):
  817. # TODO(b/199363072): filegroups and file targets aren't associated with any
  818. # specific platform architecture in mixed builds. This is consistent with how
  819. # Soong treats filegroups, but it may not be the case with manually-written
  820. # filegroup BUILD targets.
  821. buildoptions = build_options(target)
  822. if buildoptions == None:
  823. # File targets do not have buildoptions. File targets aren't associated with
  824. # any specific platform architecture in mixed builds, so use the host.
  825. return "x86_64|linux"
  826. platforms = buildoptions["//command_line_option:platforms"]
  827. if len(platforms) != 1:
  828. # An individual configured target should have only one platform architecture.
  829. # Note that it's fine for there to be multiple architectures for the same label,
  830. # but each is its own configured target.
  831. fail("expected exactly 1 platform for " + str(target.label) + " but got " + str(platforms))
  832. platform_name = platforms[0].name
  833. if platform_name == "host":
  834. return "HOST"
  835. if not platform_name.startswith("mixed_builds_product-{TARGET_BUILD_VARIANT}"):
  836. fail("expected platform name of the form 'mixed_builds_product-{TARGET_BUILD_VARIANT}_android_<arch>' or 'mixed_builds_product-{TARGET_BUILD_VARIANT}_linux_<arch>', but was " + str(platforms))
  837. platform_name = platform_name.removeprefix("mixed_builds_product-{TARGET_BUILD_VARIANT}").removeprefix("_")
  838. config_key = ""
  839. if not platform_name:
  840. config_key = "target|android"
  841. elif platform_name.startswith("android_"):
  842. config_key = platform_name.removeprefix("android_") + "|android"
  843. elif platform_name.startswith("linux_"):
  844. config_key = platform_name.removeprefix("linux_") + "|linux"
  845. else:
  846. fail("expected platform name of the form 'mixed_builds_product-{TARGET_BUILD_VARIANT}_android_<arch>' or 'mixed_builds_product-{TARGET_BUILD_VARIANT}_linux_<arch>', but was " + str(platforms))
  847. within_apex = buildoptions.get("//build/bazel/rules/apex:within_apex")
  848. apex_sdk_version = buildoptions.get("//build/bazel/rules/apex:min_sdk_version")
  849. api_domain = buildoptions.get("//build/bazel/rules/apex:api_domain")
  850. if within_apex:
  851. config_key += "|within_apex"
  852. if apex_sdk_version != None and len(apex_sdk_version) > 0:
  853. config_key += "|" + apex_sdk_version
  854. if api_domain != None and len(api_domain) > 0:
  855. config_key += "|" + api_domain
  856. return config_key
  857. def format(target):
  858. id_string = str(target.label) + "|" + get_arch(target)
  859. # TODO(b/248106697): Remove once Bazel is updated to always normalize labels.
  860. if id_string.startswith("//"):
  861. id_string = "@" + id_string
  862. {MAIN_SWITCH_SECTION}
  863. # This target was not requested via cquery, and thus must be a dependency
  864. # of a requested target.
  865. return id_string + ">>NONE"
  866. `
  867. replacer := strings.NewReplacer(
  868. "{TARGET_PRODUCT}", context.targetProduct,
  869. "{TARGET_BUILD_VARIANT}", context.targetBuildVariant,
  870. "{LABEL_REGISTRATION_MAP_SECTION}", labelRegistrationMapSection,
  871. "{FUNCTION_DEF_SECTION}", functionDefSection,
  872. "{MAIN_SWITCH_SECTION}", mainSwitchSection)
  873. return []byte(replacer.Replace(formatString))
  874. }
  875. // Returns a path containing build-related metadata required for interfacing
  876. // with Bazel. Example: out/soong/bazel.
  877. func (p *bazelPaths) intermediatesDir() string {
  878. return filepath.Join(p.soongOutDir, "bazel")
  879. }
  880. // Returns the path where the contents of the @soong_injection repository live.
  881. // It is used by Soong to tell Bazel things it cannot over the command line.
  882. func (p *bazelPaths) injectedFilesDir() string {
  883. return filepath.Join(p.soongOutDir, bazel.SoongInjectionDirName)
  884. }
  885. // Returns the path of the synthetic Bazel workspace that contains a symlink
  886. // forest composed the whole source tree and BUILD files generated by bp2build.
  887. func (p *bazelPaths) syntheticWorkspaceDir() string {
  888. return filepath.Join(p.soongOutDir, "workspace")
  889. }
  890. // Returns the path to the top level out dir ($OUT_DIR).
  891. func (p *bazelPaths) outDir() string {
  892. return filepath.Dir(p.soongOutDir)
  893. }
  894. const buildrootLabel = "@soong_injection//mixed_builds:buildroot"
  895. var (
  896. cqueryCmd = bazelCommand{"cquery", fmt.Sprintf("deps(%s, 2)", buildrootLabel)}
  897. aqueryCmd = bazelCommand{"aquery", fmt.Sprintf("deps(%s)", buildrootLabel)}
  898. buildCmd = bazelCommand{"build", "@soong_injection//mixed_builds:phonyroot"}
  899. allBazelCommands = []bazelCommand{aqueryCmd, cqueryCmd, buildCmd}
  900. )
  901. // Issues commands to Bazel to receive results for all cquery requests
  902. // queued in the BazelContext.
  903. func (context *mixedBuildBazelContext) InvokeBazel(config Config, ctx invokeBazelContext) error {
  904. eventHandler := ctx.GetEventHandler()
  905. eventHandler.Begin("bazel")
  906. defer eventHandler.End("bazel")
  907. if metricsDir := context.paths.BazelMetricsDir(); metricsDir != "" {
  908. if err := os.MkdirAll(metricsDir, 0777); err != nil {
  909. return err
  910. }
  911. }
  912. context.results = make(map[cqueryKey]string)
  913. if err := context.runCquery(config, ctx); err != nil {
  914. return err
  915. }
  916. if err := context.runAquery(config, ctx); err != nil {
  917. return err
  918. }
  919. if err := context.generateBazelSymlinks(config, ctx); err != nil {
  920. return err
  921. }
  922. // Clear requests.
  923. context.requests = []cqueryKey{}
  924. return nil
  925. }
  926. func (context *mixedBuildBazelContext) runCquery(config Config, ctx invokeBazelContext) error {
  927. eventHandler := ctx.GetEventHandler()
  928. eventHandler.Begin("cquery")
  929. defer eventHandler.End("cquery")
  930. soongInjectionPath := absolutePath(context.paths.injectedFilesDir())
  931. mixedBuildsPath := filepath.Join(soongInjectionPath, "mixed_builds")
  932. if _, err := os.Stat(mixedBuildsPath); os.IsNotExist(err) {
  933. err = os.MkdirAll(mixedBuildsPath, 0777)
  934. if err != nil {
  935. return err
  936. }
  937. }
  938. if err := writeFileBytesIfChanged(filepath.Join(soongInjectionPath, "WORKSPACE.bazel"), []byte{}, 0666); err != nil {
  939. return err
  940. }
  941. if err := writeFileBytesIfChanged(filepath.Join(mixedBuildsPath, "main.bzl"), context.mainBzlFileContents(), 0666); err != nil {
  942. return err
  943. }
  944. if err := writeFileBytesIfChanged(filepath.Join(mixedBuildsPath, "BUILD.bazel"), context.mainBuildFileContents(), 0666); err != nil {
  945. return err
  946. }
  947. cqueryFileRelpath := filepath.Join(context.paths.injectedFilesDir(), "buildroot.cquery")
  948. if err := writeFileBytesIfChanged(absolutePath(cqueryFileRelpath), context.cqueryStarlarkFileContents(), 0666); err != nil {
  949. return err
  950. }
  951. extraFlags := []string{"--output=starlark", "--starlark:file=" + absolutePath(cqueryFileRelpath)}
  952. if Bool(config.productVariables.ClangCoverage) {
  953. extraFlags = append(extraFlags, "--collect_code_coverage")
  954. }
  955. cqueryCmdRequest := context.createBazelCommand(config, bazel.CqueryBuildRootRunName, cqueryCmd, extraFlags...)
  956. cqueryOutput, cqueryErrorMessage, cqueryErr := context.issueBazelCommand(cqueryCmdRequest, context.paths, eventHandler)
  957. if cqueryErr != nil {
  958. return cqueryErr
  959. }
  960. cqueryCommandPrint := fmt.Sprintf("cquery command line:\n %s \n\n\n", context.printableCqueryCommand(cqueryCmdRequest))
  961. if err := os.WriteFile(filepath.Join(soongInjectionPath, "cquery.out"), []byte(cqueryCommandPrint+cqueryOutput), 0666); err != nil {
  962. return err
  963. }
  964. cqueryResults := map[string]string{}
  965. for _, outputLine := range strings.Split(cqueryOutput, "\n") {
  966. if strings.Contains(outputLine, ">>") {
  967. splitLine := strings.SplitN(outputLine, ">>", 2)
  968. cqueryResults[splitLine[0]] = splitLine[1]
  969. }
  970. }
  971. for _, val := range context.requests {
  972. if cqueryResult, ok := cqueryResults[getCqueryId(val)]; ok {
  973. context.results[val] = cqueryResult
  974. } else {
  975. return fmt.Errorf("missing result for bazel target %s. query output: [%s], cquery err: [%s]",
  976. getCqueryId(val), cqueryOutput, cqueryErrorMessage)
  977. }
  978. }
  979. return nil
  980. }
  981. func writeFileBytesIfChanged(path string, contents []byte, perm os.FileMode) error {
  982. oldContents, err := os.ReadFile(path)
  983. if err != nil || !bytes.Equal(contents, oldContents) {
  984. err = os.WriteFile(path, contents, perm)
  985. }
  986. return nil
  987. }
  988. func (context *mixedBuildBazelContext) runAquery(config Config, ctx invokeBazelContext) error {
  989. eventHandler := ctx.GetEventHandler()
  990. eventHandler.Begin("aquery")
  991. defer eventHandler.End("aquery")
  992. // Issue an aquery command to retrieve action information about the bazel build tree.
  993. //
  994. // Use jsonproto instead of proto; actual proto parsing would require a dependency on Bazel's
  995. // proto sources, which would add a number of unnecessary dependencies.
  996. extraFlags := []string{"--output=proto", "--include_file_write_contents"}
  997. if Bool(config.productVariables.ClangCoverage) {
  998. extraFlags = append(extraFlags, "--collect_code_coverage")
  999. paths := make([]string, 0, 2)
  1000. if p := config.productVariables.NativeCoveragePaths; len(p) > 0 {
  1001. for i := range p {
  1002. // TODO(b/259404593) convert path wildcard to regex values
  1003. if p[i] == "*" {
  1004. p[i] = ".*"
  1005. }
  1006. }
  1007. paths = append(paths, JoinWithPrefixAndSeparator(p, "+", ","))
  1008. }
  1009. if p := config.productVariables.NativeCoverageExcludePaths; len(p) > 0 {
  1010. paths = append(paths, JoinWithPrefixAndSeparator(p, "-", ","))
  1011. }
  1012. if len(paths) > 0 {
  1013. extraFlags = append(extraFlags, "--instrumentation_filter="+strings.Join(paths, ","))
  1014. }
  1015. }
  1016. aqueryOutput, _, err := context.issueBazelCommand(context.createBazelCommand(config, bazel.AqueryBuildRootRunName, aqueryCmd,
  1017. extraFlags...), context.paths, eventHandler)
  1018. if err != nil {
  1019. return err
  1020. }
  1021. context.buildStatements, context.depsets, err = bazel.AqueryBuildStatements([]byte(aqueryOutput), eventHandler)
  1022. return err
  1023. }
  1024. func (context *mixedBuildBazelContext) generateBazelSymlinks(config Config, ctx invokeBazelContext) error {
  1025. eventHandler := ctx.GetEventHandler()
  1026. eventHandler.Begin("symlinks")
  1027. defer eventHandler.End("symlinks")
  1028. // Issue a build command of the phony root to generate symlink forests for dependencies of the
  1029. // Bazel build. This is necessary because aquery invocations do not generate this symlink forest,
  1030. // but some of symlinks may be required to resolve source dependencies of the build.
  1031. _, _, err := context.issueBazelCommand(context.createBazelCommand(config, bazel.BazelBuildPhonyRootRunName, buildCmd), context.paths, eventHandler)
  1032. return err
  1033. }
  1034. func (context *mixedBuildBazelContext) BuildStatementsToRegister() []*bazel.BuildStatement {
  1035. return context.buildStatements
  1036. }
  1037. func (context *mixedBuildBazelContext) AqueryDepsets() []bazel.AqueryDepset {
  1038. return context.depsets
  1039. }
  1040. func (context *mixedBuildBazelContext) OutputBase() string {
  1041. return context.paths.outputBase
  1042. }
  1043. // Singleton used for registering BUILD file ninja dependencies (needed
  1044. // for correctness of builds which use Bazel.
  1045. func BazelSingleton() Singleton {
  1046. return &bazelSingleton{}
  1047. }
  1048. type bazelSingleton struct{}
  1049. func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) {
  1050. // bazelSingleton is a no-op if mixed-soong-bazel-builds are disabled.
  1051. if !ctx.Config().IsMixedBuildsEnabled() {
  1052. return
  1053. }
  1054. // Add ninja file dependencies for files which all bazel invocations require.
  1055. bazelBuildList := absolutePath(filepath.Join(
  1056. filepath.Dir(ctx.Config().moduleListFile), "bazel.list"))
  1057. ctx.AddNinjaFileDeps(bazelBuildList)
  1058. data, err := os.ReadFile(bazelBuildList)
  1059. if err != nil {
  1060. ctx.Errorf(err.Error())
  1061. }
  1062. files := strings.Split(strings.TrimSpace(string(data)), "\n")
  1063. for _, file := range files {
  1064. ctx.AddNinjaFileDeps(file)
  1065. }
  1066. for _, depset := range ctx.Config().BazelContext.AqueryDepsets() {
  1067. var outputs []Path
  1068. var orderOnlies []Path
  1069. for _, depsetDepHash := range depset.TransitiveDepSetHashes {
  1070. otherDepsetName := bazelDepsetName(depsetDepHash)
  1071. outputs = append(outputs, PathForPhony(ctx, otherDepsetName))
  1072. }
  1073. for _, artifactPath := range depset.DirectArtifacts {
  1074. pathInBazelOut := PathForBazelOut(ctx, artifactPath)
  1075. if artifactPath == "bazel-out/volatile-status.txt" {
  1076. // See https://bazel.build/docs/user-manual#workspace-status
  1077. orderOnlies = append(orderOnlies, pathInBazelOut)
  1078. } else {
  1079. outputs = append(outputs, pathInBazelOut)
  1080. }
  1081. }
  1082. thisDepsetName := bazelDepsetName(depset.ContentHash)
  1083. ctx.Build(pctx, BuildParams{
  1084. Rule: blueprint.Phony,
  1085. Outputs: []WritablePath{PathForPhony(ctx, thisDepsetName)},
  1086. Implicits: outputs,
  1087. OrderOnly: orderOnlies,
  1088. })
  1089. }
  1090. executionRoot := path.Join(ctx.Config().BazelContext.OutputBase(), "execroot", "__main__")
  1091. bazelOutDir := path.Join(executionRoot, "bazel-out")
  1092. for index, buildStatement := range ctx.Config().BazelContext.BuildStatementsToRegister() {
  1093. // nil build statements are a valid case where we do not create an action because it is
  1094. // unnecessary or handled by other processing
  1095. if buildStatement == nil {
  1096. continue
  1097. }
  1098. if len(buildStatement.Command) > 0 {
  1099. rule := NewRuleBuilder(pctx, ctx)
  1100. createCommand(rule.Command(), buildStatement, executionRoot, bazelOutDir, ctx)
  1101. desc := fmt.Sprintf("%s: %s", buildStatement.Mnemonic, buildStatement.OutputPaths)
  1102. rule.Build(fmt.Sprintf("bazel %d", index), desc)
  1103. continue
  1104. }
  1105. // Certain actions returned by aquery (for instance FileWrite) do not contain a command
  1106. // and thus require special treatment. If BuildStatement were an interface implementing
  1107. // buildRule(ctx) function, the code here would just call it.
  1108. // Unfortunately, the BuildStatement is defined in
  1109. // the 'bazel' package, which cannot depend on 'android' package where ctx is defined,
  1110. // because this would cause circular dependency. So, until we move aquery processing
  1111. // to the 'android' package, we need to handle special cases here.
  1112. switch buildStatement.Mnemonic {
  1113. case "FileWrite", "SourceSymlinkManifest":
  1114. out := PathForBazelOut(ctx, buildStatement.OutputPaths[0])
  1115. WriteFileRuleVerbatim(ctx, out, buildStatement.FileContents)
  1116. case "SymlinkTree":
  1117. // build-runfiles arguments are the manifest file and the target directory
  1118. // where it creates the symlink tree according to this manifest (and then
  1119. // writes the MANIFEST file to it).
  1120. outManifest := PathForBazelOut(ctx, buildStatement.OutputPaths[0])
  1121. outManifestPath := outManifest.String()
  1122. if !strings.HasSuffix(outManifestPath, "MANIFEST") {
  1123. panic("the base name of the symlink tree action should be MANIFEST, got " + outManifestPath)
  1124. }
  1125. outDir := filepath.Dir(outManifestPath)
  1126. ctx.Build(pctx, BuildParams{
  1127. Rule: buildRunfilesRule,
  1128. Output: outManifest,
  1129. Inputs: []Path{PathForBazelOut(ctx, buildStatement.InputPaths[0])},
  1130. Description: "symlink tree for " + outDir,
  1131. Args: map[string]string{
  1132. "outDir": outDir,
  1133. },
  1134. })
  1135. default:
  1136. panic(fmt.Sprintf("unhandled build statement: %v", buildStatement))
  1137. }
  1138. }
  1139. }
  1140. // Register bazel-owned build statements (obtained from the aquery invocation).
  1141. func createCommand(cmd *RuleBuilderCommand, buildStatement *bazel.BuildStatement, executionRoot string, bazelOutDir string, ctx BuilderContext) {
  1142. // executionRoot is the action cwd.
  1143. cmd.Text(fmt.Sprintf("cd '%s' &&", executionRoot))
  1144. // Remove old outputs, as some actions might not rerun if the outputs are detected.
  1145. if len(buildStatement.OutputPaths) > 0 {
  1146. cmd.Text("rm -rf") // -r because outputs can be Bazel dir/tree artifacts.
  1147. for _, outputPath := range buildStatement.OutputPaths {
  1148. cmd.Text(fmt.Sprintf("'%s'", outputPath))
  1149. }
  1150. cmd.Text("&&")
  1151. }
  1152. for _, pair := range buildStatement.Env {
  1153. // Set per-action env variables, if any.
  1154. cmd.Flag(pair.Key + "=" + pair.Value)
  1155. }
  1156. // The actual Bazel action.
  1157. if len(buildStatement.Command) > 16*1024 {
  1158. commandFile := PathForBazelOut(ctx, buildStatement.OutputPaths[0]+".sh")
  1159. WriteFileRule(ctx, commandFile, buildStatement.Command)
  1160. cmd.Text("bash").Text(buildStatement.OutputPaths[0] + ".sh").Implicit(commandFile)
  1161. } else {
  1162. cmd.Text(buildStatement.Command)
  1163. }
  1164. for _, outputPath := range buildStatement.OutputPaths {
  1165. cmd.ImplicitOutput(PathForBazelOut(ctx, outputPath))
  1166. }
  1167. for _, inputPath := range buildStatement.InputPaths {
  1168. cmd.Implicit(PathForBazelOut(ctx, inputPath))
  1169. }
  1170. for _, inputDepsetHash := range buildStatement.InputDepsetHashes {
  1171. otherDepsetName := bazelDepsetName(inputDepsetHash)
  1172. cmd.Implicit(PathForPhony(ctx, otherDepsetName))
  1173. }
  1174. if depfile := buildStatement.Depfile; depfile != nil {
  1175. // The paths in depfile are relative to `executionRoot`.
  1176. // Hence, they need to be corrected by replacing "bazel-out"
  1177. // with the full `bazelOutDir`.
  1178. // Otherwise, implicit outputs and implicit inputs under "bazel-out/"
  1179. // would be deemed missing.
  1180. // (Note: The regexp uses a capture group because the version of sed
  1181. // does not support a look-behind pattern.)
  1182. replacement := fmt.Sprintf(`&& sed -i'' -E 's@(^|\s|")bazel-out/@\1%s/@g' '%s'`,
  1183. bazelOutDir, *depfile)
  1184. cmd.Text(replacement)
  1185. cmd.ImplicitDepFile(PathForBazelOut(ctx, *depfile))
  1186. }
  1187. for _, symlinkPath := range buildStatement.SymlinkPaths {
  1188. cmd.ImplicitSymlinkOutput(PathForBazelOut(ctx, symlinkPath))
  1189. }
  1190. }
  1191. func getCqueryId(key cqueryKey) string {
  1192. return key.label + "|" + getConfigString(key)
  1193. }
  1194. func getConfigString(key cqueryKey) string {
  1195. arch := key.configKey.arch
  1196. if len(arch) == 0 || arch == "common" {
  1197. if key.configKey.osType.Class == Device {
  1198. // For the generic Android, the expected result is "target|android", which
  1199. // corresponds to the product_variable_config named "android_target" in
  1200. // build/bazel/platforms/BUILD.bazel.
  1201. arch = "target"
  1202. } else {
  1203. // Use host platform, which is currently hardcoded to be x86_64.
  1204. arch = "x86_64"
  1205. }
  1206. }
  1207. osName := key.configKey.osType.Name
  1208. if len(osName) == 0 || osName == "common_os" || osName == "linux_glibc" || osName == "linux_musl" {
  1209. // Use host OS, which is currently hardcoded to be linux.
  1210. osName = "linux"
  1211. }
  1212. keyString := arch + "|" + osName
  1213. if key.configKey.apexKey.WithinApex {
  1214. keyString += "|" + withinApexToString(key.configKey.apexKey.WithinApex)
  1215. }
  1216. if len(key.configKey.apexKey.ApexSdkVersion) > 0 {
  1217. keyString += "|" + key.configKey.apexKey.ApexSdkVersion
  1218. }
  1219. if len(key.configKey.apexKey.ApiDomain) > 0 {
  1220. keyString += "|" + key.configKey.apexKey.ApiDomain
  1221. }
  1222. return keyString
  1223. }
  1224. func GetConfigKey(ctx BaseModuleContext) configKey {
  1225. return configKey{
  1226. // use string because Arch is not a valid key in go
  1227. arch: ctx.Arch().String(),
  1228. osType: ctx.Os(),
  1229. }
  1230. }
  1231. func GetConfigKeyApexVariant(ctx BaseModuleContext, apexKey *ApexConfigKey) configKey {
  1232. configKey := GetConfigKey(ctx)
  1233. if apexKey != nil {
  1234. configKey.apexKey = ApexConfigKey{
  1235. WithinApex: apexKey.WithinApex,
  1236. ApexSdkVersion: apexKey.ApexSdkVersion,
  1237. ApiDomain: apexKey.ApiDomain,
  1238. }
  1239. }
  1240. return configKey
  1241. }
  1242. func bazelDepsetName(contentHash string) string {
  1243. return fmt.Sprintf("bazel_depset_%s", contentHash)
  1244. }