main.go 36 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897
  1. // Copyright 2015 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package main
  15. import (
  16. "bytes"
  17. "errors"
  18. "flag"
  19. "fmt"
  20. "os"
  21. "path/filepath"
  22. "regexp"
  23. "strings"
  24. "time"
  25. "android/soong/android"
  26. "android/soong/android/allowlists"
  27. "android/soong/bazel"
  28. "android/soong/bp2build"
  29. "android/soong/shared"
  30. "android/soong/ui/metrics/bp2build_metrics_proto"
  31. "github.com/google/blueprint"
  32. "github.com/google/blueprint/bootstrap"
  33. "github.com/google/blueprint/deptools"
  34. "github.com/google/blueprint/metrics"
  35. androidProtobuf "google.golang.org/protobuf/android"
  36. )
  37. var (
  38. topDir string
  39. availableEnvFile string
  40. usedEnvFile string
  41. globFile string
  42. globListDir string
  43. delveListen string
  44. delvePath string
  45. cmdlineArgs android.CmdArgs
  46. )
  47. func init() {
  48. // Flags that make sense in every mode
  49. flag.StringVar(&topDir, "top", "", "Top directory of the Android source tree")
  50. flag.StringVar(&cmdlineArgs.SoongOutDir, "soong_out", "", "Soong output directory (usually $TOP/out/soong)")
  51. flag.StringVar(&availableEnvFile, "available_env", "", "File containing available environment variables")
  52. flag.StringVar(&usedEnvFile, "used_env", "", "File containing used environment variables")
  53. flag.StringVar(&globFile, "globFile", "build-globs.ninja", "the Ninja file of globs to output")
  54. flag.StringVar(&globListDir, "globListDir", "", "the directory containing the glob list files")
  55. flag.StringVar(&cmdlineArgs.OutDir, "out", "", "the ninja builddir directory")
  56. flag.StringVar(&cmdlineArgs.ModuleListFile, "l", "", "file that lists filepaths to parse")
  57. // Debug flags
  58. flag.StringVar(&delveListen, "delve_listen", "", "Delve port to listen on for debugging")
  59. flag.StringVar(&delvePath, "delve_path", "", "Path to Delve. Only used if --delve_listen is set")
  60. flag.StringVar(&cmdlineArgs.Cpuprofile, "cpuprofile", "", "write cpu profile to file")
  61. flag.StringVar(&cmdlineArgs.TraceFile, "trace", "", "write trace to file")
  62. flag.StringVar(&cmdlineArgs.Memprofile, "memprofile", "", "write memory profile to file")
  63. flag.BoolVar(&cmdlineArgs.NoGC, "nogc", false, "turn off GC for debugging")
  64. // Flags representing various modes soong_build can run in
  65. flag.StringVar(&cmdlineArgs.ModuleGraphFile, "module_graph_file", "", "JSON module graph file to output")
  66. flag.StringVar(&cmdlineArgs.ModuleActionsFile, "module_actions_file", "", "JSON file to output inputs/outputs of actions of modules")
  67. flag.StringVar(&cmdlineArgs.DocFile, "soong_docs", "", "build documentation file to output")
  68. flag.StringVar(&cmdlineArgs.BazelQueryViewDir, "bazel_queryview_dir", "", "path to the bazel queryview directory relative to --top")
  69. flag.StringVar(&cmdlineArgs.BazelApiBp2buildDir, "bazel_api_bp2build_dir", "", "path to the bazel api_bp2build directory relative to --top")
  70. flag.StringVar(&cmdlineArgs.Bp2buildMarker, "bp2build_marker", "", "If set, run bp2build, touch the specified marker file then exit")
  71. flag.StringVar(&cmdlineArgs.SymlinkForestMarker, "symlink_forest_marker", "", "If set, create the bp2build symlink forest, touch the specified marker file, then exit")
  72. flag.StringVar(&cmdlineArgs.OutFile, "o", "build.ninja", "the Ninja file to output")
  73. flag.StringVar(&cmdlineArgs.SoongVariables, "soong_variables", "soong.variables", "the file contains all build variables")
  74. flag.StringVar(&cmdlineArgs.BazelForceEnabledModules, "bazel-force-enabled-modules", "", "additional modules to build with Bazel. Comma-delimited")
  75. flag.BoolVar(&cmdlineArgs.EmptyNinjaFile, "empty-ninja-file", false, "write out a 0-byte ninja file")
  76. flag.BoolVar(&cmdlineArgs.MultitreeBuild, "multitree-build", false, "this is a multitree build")
  77. flag.BoolVar(&cmdlineArgs.BazelMode, "bazel-mode", false, "use bazel for analysis of certain modules")
  78. flag.BoolVar(&cmdlineArgs.BazelModeStaging, "bazel-mode-staging", false, "use bazel for analysis of certain near-ready modules")
  79. flag.BoolVar(&cmdlineArgs.BazelModeDev, "bazel-mode-dev", false, "use bazel for analysis of a large number of modules (less stable)")
  80. flag.BoolVar(&cmdlineArgs.UseBazelProxy, "use-bazel-proxy", false, "communicate with bazel using unix socket proxy instead of spawning subprocesses")
  81. flag.BoolVar(&cmdlineArgs.BuildFromTextStub, "build-from-text-stub", false, "build Java stubs from API text files instead of source files")
  82. flag.BoolVar(&cmdlineArgs.EnsureAllowlistIntegrity, "ensure-allowlist-integrity", false, "verify that allowlisted modules are mixed-built")
  83. // Flags that probably shouldn't be flags of soong_build, but we haven't found
  84. // the time to remove them yet
  85. flag.BoolVar(&cmdlineArgs.RunGoTests, "t", false, "build and run go tests during bootstrap")
  86. // Disable deterministic randomization in the protobuf package, so incremental
  87. // builds with unrelated Soong changes don't trigger large rebuilds (since we
  88. // write out text protos in command lines, and command line changes trigger
  89. // rebuilds).
  90. androidProtobuf.DisableRand()
  91. }
  92. func newNameResolver(config android.Config) *android.NameResolver {
  93. return android.NewNameResolver(config)
  94. }
  95. func newContext(configuration android.Config) *android.Context {
  96. ctx := android.NewContext(configuration)
  97. ctx.SetNameInterface(newNameResolver(configuration))
  98. ctx.SetAllowMissingDependencies(configuration.AllowMissingDependencies())
  99. ctx.AddIncludeTags(configuration.IncludeTags()...)
  100. ctx.AddSourceRootDirs(configuration.SourceRootDirs()...)
  101. return ctx
  102. }
  103. // Bazel-enabled mode. Attaches a mutator to queue Bazel requests, adds a
  104. // BeforePrepareBuildActionsHook to invoke Bazel, and then uses Bazel metadata
  105. // for modules that should be handled by Bazel.
  106. func runMixedModeBuild(ctx *android.Context, extraNinjaDeps []string) string {
  107. ctx.EventHandler.Begin("mixed_build")
  108. defer ctx.EventHandler.End("mixed_build")
  109. bazelHook := func() error {
  110. return ctx.Config().BazelContext.InvokeBazel(ctx.Config(), ctx)
  111. }
  112. ctx.SetBeforePrepareBuildActionsHook(bazelHook)
  113. ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs.Args, bootstrap.DoEverything, ctx.Context, ctx.Config())
  114. ninjaDeps = append(ninjaDeps, extraNinjaDeps...)
  115. bazelPaths, err := readFileLines(ctx.Config().Getenv("BAZEL_DEPS_FILE"))
  116. if err != nil {
  117. panic("Bazel deps file not found: " + err.Error())
  118. }
  119. ninjaDeps = append(ninjaDeps, bazelPaths...)
  120. ninjaDeps = append(ninjaDeps, writeBuildGlobsNinjaFile(ctx)...)
  121. writeDepFile(cmdlineArgs.OutFile, ctx.EventHandler, ninjaDeps)
  122. if needToWriteNinjaHint(ctx) {
  123. writeNinjaHint(ctx)
  124. }
  125. return cmdlineArgs.OutFile
  126. }
  127. func needToWriteNinjaHint(ctx *android.Context) bool {
  128. switch ctx.Config().GetenvWithDefault("SOONG_GENERATES_NINJA_HINT", "") {
  129. case "always":
  130. return true
  131. case "depend":
  132. if _, err := os.Stat(filepath.Join(ctx.Config().OutDir(), ".ninja_log")); errors.Is(err, os.ErrNotExist) {
  133. return true
  134. }
  135. }
  136. return false
  137. }
  138. // Run the code-generation phase to convert BazelTargetModules to BUILD files.
  139. func runQueryView(queryviewDir, queryviewMarker string, ctx *android.Context) {
  140. ctx.EventHandler.Begin("queryview")
  141. defer ctx.EventHandler.End("queryview")
  142. codegenContext := bp2build.NewCodegenContext(ctx.Config(), ctx, bp2build.QueryView, topDir)
  143. err := createBazelWorkspace(codegenContext, shared.JoinPath(topDir, queryviewDir), false)
  144. maybeQuit(err, "")
  145. touch(shared.JoinPath(topDir, queryviewMarker))
  146. }
  147. // Run the code-generation phase to convert API contributions to BUILD files.
  148. // Return marker file for the new synthetic workspace
  149. func runApiBp2build(ctx *android.Context, extraNinjaDeps []string) string {
  150. ctx.EventHandler.Begin("api_bp2build")
  151. defer ctx.EventHandler.End("api_bp2build")
  152. // api_bp2build does not run the typical pipeline of soong mutators.
  153. // Hoevever, it still runs the defaults mutator which can create dependencies.
  154. // These dependencies might not always exist (e.g. in tests)
  155. ctx.SetAllowMissingDependencies(ctx.Config().AllowMissingDependencies())
  156. ctx.RegisterForApiBazelConversion()
  157. // Register the Android.bp files in the tree
  158. // Add them to the workspace's .d file
  159. ctx.SetModuleListFile(cmdlineArgs.ModuleListFile)
  160. if paths, err := ctx.ListModulePaths("."); err == nil {
  161. extraNinjaDeps = append(extraNinjaDeps, paths...)
  162. } else {
  163. panic(err)
  164. }
  165. // Run the loading and analysis phase
  166. ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs.Args,
  167. bootstrap.StopBeforePrepareBuildActions,
  168. ctx.Context,
  169. ctx.Config())
  170. ninjaDeps = append(ninjaDeps, extraNinjaDeps...)
  171. // Add the globbed dependencies
  172. ninjaDeps = append(ninjaDeps, writeBuildGlobsNinjaFile(ctx)...)
  173. // Run codegen to generate BUILD files
  174. codegenContext := bp2build.NewCodegenContext(ctx.Config(), ctx, bp2build.ApiBp2build, topDir)
  175. absoluteApiBp2buildDir := shared.JoinPath(topDir, cmdlineArgs.BazelApiBp2buildDir)
  176. // Always generate bp2build_all_srcs filegroups in api_bp2build.
  177. // This is necessary to force each Android.bp file to create an equivalent BUILD file
  178. // and prevent package boundray issues.
  179. // e.g.
  180. // Source
  181. // f/b/Android.bp
  182. // java_library{
  183. // name: "foo",
  184. // api: "api/current.txt",
  185. // }
  186. //
  187. // f/b/api/Android.bp <- will cause package boundary issues
  188. //
  189. // Gen
  190. // f/b/BUILD
  191. // java_contribution{
  192. // name: "foo.contribution",
  193. // api: "//f/b/api:current.txt",
  194. // }
  195. //
  196. // If we don't generate f/b/api/BUILD, foo.contribution will be unbuildable.
  197. err := createBazelWorkspace(codegenContext, absoluteApiBp2buildDir, true)
  198. maybeQuit(err, "")
  199. ninjaDeps = append(ninjaDeps, codegenContext.AdditionalNinjaDeps()...)
  200. // Create soong_injection repository
  201. soongInjectionFiles, err := bp2build.CreateSoongInjectionDirFiles(codegenContext, bp2build.CreateCodegenMetrics())
  202. maybeQuit(err, "")
  203. absoluteSoongInjectionDir := shared.JoinPath(topDir, ctx.Config().SoongOutDir(), bazel.SoongInjectionDirName)
  204. for _, file := range soongInjectionFiles {
  205. // The API targets in api_bp2build workspace do not have any dependency on api_bp2build.
  206. // But we need to create these files to prevent errors during Bazel analysis.
  207. // These need to be created in Read-Write mode.
  208. // This is because the subsequent step (bp2build in api domain analysis) creates them in Read-Write mode
  209. // to allow users to edit/experiment in the synthetic workspace.
  210. writeReadWriteFile(absoluteSoongInjectionDir, file)
  211. }
  212. workspace := shared.JoinPath(ctx.Config().SoongOutDir(), "api_bp2build")
  213. // Create the symlink forest
  214. symlinkDeps, _, _ := bp2build.PlantSymlinkForest(
  215. ctx.Config().IsEnvTrue("BP2BUILD_VERBOSE"),
  216. topDir,
  217. workspace,
  218. cmdlineArgs.BazelApiBp2buildDir,
  219. apiBuildFileExcludes(ctx))
  220. ninjaDeps = append(ninjaDeps, symlinkDeps...)
  221. workspaceMarkerFile := workspace + ".marker"
  222. writeDepFile(workspaceMarkerFile, ctx.EventHandler, ninjaDeps)
  223. touch(shared.JoinPath(topDir, workspaceMarkerFile))
  224. return workspaceMarkerFile
  225. }
  226. // With some exceptions, api_bp2build does not have any dependencies on the checked-in BUILD files
  227. // Exclude them from the generated workspace to prevent unrelated errors during the loading phase
  228. func apiBuildFileExcludes(ctx *android.Context) []string {
  229. ret := bazelArtifacts()
  230. srcs, err := getExistingBazelRelatedFiles(topDir)
  231. maybeQuit(err, "Error determining existing Bazel-related files")
  232. for _, src := range srcs {
  233. // Exclude all src BUILD files
  234. if src != "WORKSPACE" &&
  235. src != "BUILD" &&
  236. src != "BUILD.bazel" &&
  237. !strings.HasPrefix(src, "build/bazel") &&
  238. !strings.HasPrefix(src, "external/bazel-skylib") &&
  239. !strings.HasPrefix(src, "prebuilts/clang") {
  240. ret = append(ret, src)
  241. }
  242. }
  243. // Android.bp files for api surfaces are mounted to out/, but out/ should not be a
  244. // dep for api_bp2build. Otherwise, api_bp2build will be run every single time
  245. ret = append(ret, ctx.Config().OutDir())
  246. return ret
  247. }
  248. func writeNinjaHint(ctx *android.Context) error {
  249. ctx.BeginEvent("ninja_hint")
  250. defer ctx.EndEvent("ninja_hint")
  251. // The current predictor focuses on reducing false negatives.
  252. // If there are too many false positives (e.g., most modules are marked as positive),
  253. // real long-running jobs cannot run early.
  254. // Therefore, the model should be adjusted in this case.
  255. // The model should also be adjusted if there are critical false negatives.
  256. predicate := func(j *blueprint.JsonModule) (prioritized bool, weight int) {
  257. prioritized = false
  258. weight = 0
  259. for prefix, w := range allowlists.HugeModuleTypePrefixMap {
  260. if strings.HasPrefix(j.Type, prefix) {
  261. prioritized = true
  262. weight = w
  263. return
  264. }
  265. }
  266. dep_count := len(j.Deps)
  267. src_count := 0
  268. for _, a := range j.Module["Actions"].([]blueprint.JSONAction) {
  269. src_count += len(a.Inputs)
  270. }
  271. input_size := dep_count + src_count
  272. // Current threshold is an arbitrary value which only consider recall rather than accuracy.
  273. if input_size > allowlists.INPUT_SIZE_THRESHOLD {
  274. prioritized = true
  275. weight += ((input_size) / allowlists.INPUT_SIZE_THRESHOLD) * allowlists.DEFAULT_PRIORITIZED_WEIGHT
  276. // To prevent some modules from having too large a priority value.
  277. if weight > allowlists.HIGH_PRIORITIZED_WEIGHT {
  278. weight = allowlists.HIGH_PRIORITIZED_WEIGHT
  279. }
  280. }
  281. return
  282. }
  283. outputsMap := ctx.Context.GetWeightedOutputsFromPredicate(predicate)
  284. var outputBuilder strings.Builder
  285. for output, weight := range outputsMap {
  286. outputBuilder.WriteString(fmt.Sprintf("%s,%d\n", output, weight))
  287. }
  288. weightListFile := filepath.Join(topDir, ctx.Config().OutDir(), ".ninja_weight_list")
  289. err := os.WriteFile(weightListFile, []byte(outputBuilder.String()), 0644)
  290. if err != nil {
  291. return fmt.Errorf("could not write ninja weight list file %s", err)
  292. }
  293. return nil
  294. }
  295. func writeMetrics(configuration android.Config, eventHandler *metrics.EventHandler, metricsDir string) {
  296. if len(metricsDir) < 1 {
  297. fmt.Fprintf(os.Stderr, "\nMissing required env var for generating soong metrics: LOG_DIR\n")
  298. os.Exit(1)
  299. }
  300. metricsFile := filepath.Join(metricsDir, "soong_build_metrics.pb")
  301. err := android.WriteMetrics(configuration, eventHandler, metricsFile)
  302. maybeQuit(err, "error writing soong_build metrics %s", metricsFile)
  303. }
  304. // Errors out if any modules expected to be mixed_built were not, unless
  305. // the modules did not exist.
  306. func checkForAllowlistIntegrityError(configuration android.Config, isStagingMode bool) error {
  307. modules := findMisconfiguredModules(configuration, isStagingMode)
  308. if len(modules) == 0 {
  309. return nil
  310. }
  311. return fmt.Errorf("Error: expected the following modules to be mixed_built: %s", modules)
  312. }
  313. // Returns true if the given module has all of the following true:
  314. // 1. Is allowlisted to be built with Bazel.
  315. // 2. Has a variant which is *not* built with Bazel.
  316. // 3. Has no variant which is built with Bazel.
  317. //
  318. // This indicates the allowlisting of this variant had no effect.
  319. // TODO(b/280457637): Return true for nonexistent modules.
  320. func isAllowlistMisconfiguredForModule(module string, mixedBuildsEnabled map[string]struct{}, mixedBuildsDisabled map[string]struct{}) bool {
  321. _, enabled := mixedBuildsEnabled[module]
  322. if enabled {
  323. return false
  324. }
  325. _, disabled := mixedBuildsDisabled[module]
  326. return disabled
  327. }
  328. // Returns the list of modules that should have been mixed_built (per the
  329. // allowlists and cmdline flags) but were not.
  330. // Note: nonexistent modules are excluded from the list. See b/280457637
  331. func findMisconfiguredModules(configuration android.Config, isStagingMode bool) []string {
  332. retval := []string{}
  333. forceEnabledModules := configuration.BazelModulesForceEnabledByFlag()
  334. mixedBuildsEnabled := configuration.GetMixedBuildsEnabledModules()
  335. mixedBuildsDisabled := configuration.GetMixedBuildsDisabledModules()
  336. for _, module := range allowlists.ProdMixedBuildsEnabledList {
  337. if isAllowlistMisconfiguredForModule(module, mixedBuildsEnabled, mixedBuildsDisabled) {
  338. retval = append(retval, module)
  339. }
  340. }
  341. if isStagingMode {
  342. for _, module := range allowlists.StagingMixedBuildsEnabledList {
  343. if isAllowlistMisconfiguredForModule(module, mixedBuildsEnabled, mixedBuildsDisabled) {
  344. retval = append(retval, module)
  345. }
  346. }
  347. }
  348. for module, _ := range forceEnabledModules {
  349. if isAllowlistMisconfiguredForModule(module, mixedBuildsEnabled, mixedBuildsDisabled) {
  350. retval = append(retval, module)
  351. }
  352. }
  353. return retval
  354. }
  355. func writeJsonModuleGraphAndActions(ctx *android.Context, cmdArgs android.CmdArgs) {
  356. graphFile, graphErr := os.Create(shared.JoinPath(topDir, cmdArgs.ModuleGraphFile))
  357. maybeQuit(graphErr, "graph err")
  358. defer graphFile.Close()
  359. actionsFile, actionsErr := os.Create(shared.JoinPath(topDir, cmdArgs.ModuleActionsFile))
  360. maybeQuit(actionsErr, "actions err")
  361. defer actionsFile.Close()
  362. ctx.Context.PrintJSONGraphAndActions(graphFile, actionsFile)
  363. }
  364. func writeBuildGlobsNinjaFile(ctx *android.Context) []string {
  365. ctx.EventHandler.Begin("globs_ninja_file")
  366. defer ctx.EventHandler.End("globs_ninja_file")
  367. globDir := bootstrap.GlobDirectory(ctx.Config().SoongOutDir(), globListDir)
  368. bootstrap.WriteBuildGlobsNinjaFile(&bootstrap.GlobSingleton{
  369. GlobLister: ctx.Globs,
  370. GlobFile: globFile,
  371. GlobDir: globDir,
  372. SrcDir: ctx.SrcDir(),
  373. }, ctx.Config())
  374. return bootstrap.GlobFileListFiles(globDir)
  375. }
  376. func writeDepFile(outputFile string, eventHandler *metrics.EventHandler, ninjaDeps []string) {
  377. eventHandler.Begin("ninja_deps")
  378. defer eventHandler.End("ninja_deps")
  379. depFile := shared.JoinPath(topDir, outputFile+".d")
  380. err := deptools.WriteDepFile(depFile, outputFile, ninjaDeps)
  381. maybeQuit(err, "error writing depfile '%s'", depFile)
  382. }
  383. // runSoongOnlyBuild runs the standard Soong build in a number of different modes.
  384. func runSoongOnlyBuild(ctx *android.Context, extraNinjaDeps []string) string {
  385. ctx.EventHandler.Begin("soong_build")
  386. defer ctx.EventHandler.End("soong_build")
  387. var stopBefore bootstrap.StopBefore
  388. switch ctx.Config().BuildMode {
  389. case android.GenerateModuleGraph:
  390. stopBefore = bootstrap.StopBeforeWriteNinja
  391. case android.GenerateQueryView, android.GenerateDocFile:
  392. stopBefore = bootstrap.StopBeforePrepareBuildActions
  393. default:
  394. stopBefore = bootstrap.DoEverything
  395. }
  396. ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs.Args, stopBefore, ctx.Context, ctx.Config())
  397. ninjaDeps = append(ninjaDeps, extraNinjaDeps...)
  398. globListFiles := writeBuildGlobsNinjaFile(ctx)
  399. ninjaDeps = append(ninjaDeps, globListFiles...)
  400. // Convert the Soong module graph into Bazel BUILD files.
  401. switch ctx.Config().BuildMode {
  402. case android.GenerateQueryView:
  403. queryviewMarkerFile := cmdlineArgs.BazelQueryViewDir + ".marker"
  404. runQueryView(cmdlineArgs.BazelQueryViewDir, queryviewMarkerFile, ctx)
  405. writeDepFile(queryviewMarkerFile, ctx.EventHandler, ninjaDeps)
  406. return queryviewMarkerFile
  407. case android.GenerateModuleGraph:
  408. writeJsonModuleGraphAndActions(ctx, cmdlineArgs)
  409. writeDepFile(cmdlineArgs.ModuleGraphFile, ctx.EventHandler, ninjaDeps)
  410. return cmdlineArgs.ModuleGraphFile
  411. case android.GenerateDocFile:
  412. // TODO: we could make writeDocs() return the list of documentation files
  413. // written and add them to the .d file. Then soong_docs would be re-run
  414. // whenever one is deleted.
  415. err := writeDocs(ctx, shared.JoinPath(topDir, cmdlineArgs.DocFile))
  416. maybeQuit(err, "error building Soong documentation")
  417. writeDepFile(cmdlineArgs.DocFile, ctx.EventHandler, ninjaDeps)
  418. return cmdlineArgs.DocFile
  419. default:
  420. // The actual output (build.ninja) was written in the RunBlueprint() call
  421. // above
  422. writeDepFile(cmdlineArgs.OutFile, ctx.EventHandler, ninjaDeps)
  423. if needToWriteNinjaHint(ctx) {
  424. writeNinjaHint(ctx)
  425. }
  426. return cmdlineArgs.OutFile
  427. }
  428. }
  429. // soong_ui dumps the available environment variables to
  430. // soong.environment.available . Then soong_build itself is run with an empty
  431. // environment so that the only way environment variables can be accessed is
  432. // using Config, which tracks access to them.
  433. // At the end of the build, a file called soong.environment.used is written
  434. // containing the current value of all used environment variables. The next
  435. // time soong_ui is run, it checks whether any environment variables that was
  436. // used had changed and if so, it deletes soong.environment.used to cause a
  437. // rebuild.
  438. //
  439. // The dependency of build.ninja on soong.environment.used is declared in
  440. // build.ninja.d
  441. func parseAvailableEnv() map[string]string {
  442. if availableEnvFile == "" {
  443. fmt.Fprintf(os.Stderr, "--available_env not set\n")
  444. os.Exit(1)
  445. }
  446. result, err := shared.EnvFromFile(shared.JoinPath(topDir, availableEnvFile))
  447. maybeQuit(err, "error reading available environment file '%s'", availableEnvFile)
  448. return result
  449. }
  450. func main() {
  451. flag.Parse()
  452. shared.ReexecWithDelveMaybe(delveListen, delvePath)
  453. android.InitSandbox(topDir)
  454. availableEnv := parseAvailableEnv()
  455. configuration, err := android.NewConfig(cmdlineArgs, availableEnv)
  456. maybeQuit(err, "")
  457. if configuration.Getenv("ALLOW_MISSING_DEPENDENCIES") == "true" {
  458. configuration.SetAllowMissingDependencies()
  459. }
  460. extraNinjaDeps := []string{configuration.ProductVariablesFileName, usedEnvFile}
  461. if shared.IsDebugging() {
  462. // Add a non-existent file to the dependencies so that soong_build will rerun when the debugger is
  463. // enabled even if it completed successfully.
  464. extraNinjaDeps = append(extraNinjaDeps, filepath.Join(configuration.SoongOutDir(), "always_rerun_for_delve"))
  465. }
  466. // Bypass configuration.Getenv, as LOG_DIR does not need to be dependency tracked. By definition, it will
  467. // change between every CI build, so tracking it would require re-running Soong for every build.
  468. metricsDir := availableEnv["LOG_DIR"]
  469. ctx := newContext(configuration)
  470. var finalOutputFile string
  471. writeSymlink := false
  472. // Run Soong for a specific activity, like bp2build, queryview
  473. // or the actual Soong build for the build.ninja file.
  474. switch configuration.BuildMode {
  475. case android.SymlinkForest:
  476. finalOutputFile = runSymlinkForestCreation(ctx, extraNinjaDeps, metricsDir)
  477. case android.Bp2build:
  478. // Run the alternate pipeline of bp2build mutators and singleton to convert
  479. // Blueprint to BUILD files before everything else.
  480. finalOutputFile = runBp2Build(ctx, extraNinjaDeps, metricsDir)
  481. case android.ApiBp2build:
  482. finalOutputFile = runApiBp2build(ctx, extraNinjaDeps)
  483. writeMetrics(configuration, ctx.EventHandler, metricsDir)
  484. default:
  485. ctx.Register()
  486. isMixedBuildsEnabled := configuration.IsMixedBuildsEnabled()
  487. if isMixedBuildsEnabled {
  488. finalOutputFile = runMixedModeBuild(ctx, extraNinjaDeps)
  489. if cmdlineArgs.EnsureAllowlistIntegrity {
  490. if err := checkForAllowlistIntegrityError(configuration, cmdlineArgs.BazelModeStaging); err != nil {
  491. maybeQuit(err, "")
  492. }
  493. }
  494. writeSymlink = true
  495. } else {
  496. finalOutputFile = runSoongOnlyBuild(ctx, extraNinjaDeps)
  497. if configuration.BuildMode == android.AnalysisNoBazel {
  498. writeSymlink = true
  499. }
  500. }
  501. writeMetrics(configuration, ctx.EventHandler, metricsDir)
  502. }
  503. // Register this environment variablesas being an implicit dependencies of
  504. // soong_build. Changes to this environment variable will result in
  505. // retriggering soong_build.
  506. configuration.Getenv("USE_BAZEL_VERSION")
  507. writeUsedEnvironmentFile(configuration)
  508. // Touch the output file so that it's the newest file created by soong_build.
  509. // This is necessary because, if soong_build generated any files which
  510. // are ninja inputs to the main output file, then ninja would superfluously
  511. // rebuild this output file on the next build invocation.
  512. touch(shared.JoinPath(topDir, finalOutputFile))
  513. // TODO(b/277029044): Remove this function once build.<product>.ninja lands
  514. if writeSymlink {
  515. writeBuildNinjaSymlink(configuration, finalOutputFile)
  516. }
  517. }
  518. // TODO(b/277029044): Remove this function once build.<product>.ninja lands
  519. func writeBuildNinjaSymlink(config android.Config, source string) {
  520. targetPath := shared.JoinPath(topDir, config.SoongOutDir(), "build.ninja")
  521. sourcePath := shared.JoinPath(topDir, source)
  522. if targetPath == sourcePath {
  523. return
  524. }
  525. os.Remove(targetPath)
  526. os.Symlink(sourcePath, targetPath)
  527. }
  528. func writeUsedEnvironmentFile(configuration android.Config) {
  529. if usedEnvFile == "" {
  530. return
  531. }
  532. path := shared.JoinPath(topDir, usedEnvFile)
  533. data, err := shared.EnvFileContents(configuration.EnvDeps())
  534. maybeQuit(err, "error writing used environment file '%s'\n", usedEnvFile)
  535. if preexistingData, err := os.ReadFile(path); err != nil {
  536. if !os.IsNotExist(err) {
  537. maybeQuit(err, "error reading used environment file '%s'", usedEnvFile)
  538. }
  539. } else if bytes.Equal(preexistingData, data) {
  540. // used environment file is unchanged
  541. return
  542. }
  543. err = os.WriteFile(path, data, 0666)
  544. maybeQuit(err, "error writing used environment file '%s'", usedEnvFile)
  545. }
  546. func touch(path string) {
  547. f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)
  548. maybeQuit(err, "Error touching '%s'", path)
  549. err = f.Close()
  550. maybeQuit(err, "Error touching '%s'", path)
  551. currentTime := time.Now().Local()
  552. err = os.Chtimes(path, currentTime, currentTime)
  553. maybeQuit(err, "error touching '%s'", path)
  554. }
  555. // Read the bazel.list file that the Soong Finder already dumped earlier (hopefully)
  556. // It contains the locations of BUILD files, BUILD.bazel files, etc. in the source dir
  557. func getExistingBazelRelatedFiles(topDir string) ([]string, error) {
  558. bazelFinderFile := filepath.Join(filepath.Dir(cmdlineArgs.ModuleListFile), "bazel.list")
  559. if !filepath.IsAbs(bazelFinderFile) {
  560. // Assume this was a relative path under topDir
  561. bazelFinderFile = filepath.Join(topDir, bazelFinderFile)
  562. }
  563. return readFileLines(bazelFinderFile)
  564. }
  565. func bazelArtifacts() []string {
  566. return []string{
  567. "bazel-bin",
  568. "bazel-genfiles",
  569. "bazel-out",
  570. "bazel-testlogs",
  571. "bazel-workspace",
  572. "bazel-" + filepath.Base(topDir),
  573. }
  574. }
  575. // This could in theory easily be separated into a binary that generically
  576. // merges two directories into a symlink tree. The main obstacle is that this
  577. // function currently depends on both Bazel-specific knowledge (the existence
  578. // of bazel-* symlinks) and configuration (the set of BUILD.bazel files that
  579. // should and should not be kept)
  580. //
  581. // Ideally, bp2build would write a file that contains instructions to the
  582. // symlink tree creation binary. Then the latter would not need to depend on
  583. // the very heavy-weight machinery of soong_build .
  584. func runSymlinkForestCreation(ctx *android.Context, extraNinjaDeps []string, metricsDir string) string {
  585. var ninjaDeps []string
  586. var mkdirCount, symlinkCount uint64
  587. ctx.EventHandler.Do("symlink_forest", func() {
  588. ninjaDeps = append(ninjaDeps, extraNinjaDeps...)
  589. verbose := ctx.Config().IsEnvTrue("BP2BUILD_VERBOSE")
  590. // PlantSymlinkForest() returns all the directories that were readdir()'ed.
  591. // Such a directory SHOULD be added to `ninjaDeps` so that a child directory
  592. // or file created/deleted under it would trigger an update of the symlink forest.
  593. generatedRoot := shared.JoinPath(ctx.Config().SoongOutDir(), "bp2build")
  594. workspaceRoot := shared.JoinPath(ctx.Config().SoongOutDir(), "workspace")
  595. var symlinkForestDeps []string
  596. ctx.EventHandler.Do("plant", func() {
  597. symlinkForestDeps, mkdirCount, symlinkCount = bp2build.PlantSymlinkForest(
  598. verbose, topDir, workspaceRoot, generatedRoot, excludedFromSymlinkForest(ctx, verbose))
  599. })
  600. ninjaDeps = append(ninjaDeps, symlinkForestDeps...)
  601. })
  602. writeDepFile(cmdlineArgs.SymlinkForestMarker, ctx.EventHandler, ninjaDeps)
  603. touch(shared.JoinPath(topDir, cmdlineArgs.SymlinkForestMarker))
  604. codegenMetrics := bp2build.ReadCodegenMetrics(metricsDir)
  605. if codegenMetrics == nil {
  606. m := bp2build.CreateCodegenMetrics()
  607. codegenMetrics = &m
  608. } else {
  609. //TODO (usta) we cannot determine if we loaded a stale file, i.e. from an unrelated prior
  610. //invocation of codegen. We should simply use a separate .pb file
  611. }
  612. codegenMetrics.SetSymlinkCount(symlinkCount)
  613. codegenMetrics.SetMkDirCount(mkdirCount)
  614. writeBp2BuildMetrics(codegenMetrics, ctx.EventHandler, metricsDir)
  615. return cmdlineArgs.SymlinkForestMarker
  616. }
  617. func excludedFromSymlinkForest(ctx *android.Context, verbose bool) []string {
  618. excluded := bazelArtifacts()
  619. if cmdlineArgs.OutDir[0] != '/' {
  620. excluded = append(excluded, cmdlineArgs.OutDir)
  621. }
  622. // Find BUILD files in the srcDir which are not in the allowlist
  623. // (android.Bp2BuildConversionAllowlist#ShouldKeepExistingBuildFileForDir)
  624. // and return their paths so they can be left out of the Bazel workspace dir (i.e. ignored)
  625. existingBazelFiles, err := getExistingBazelRelatedFiles(topDir)
  626. maybeQuit(err, "Error determining existing Bazel-related files")
  627. for _, path := range existingBazelFiles {
  628. fullPath := shared.JoinPath(topDir, path)
  629. fileInfo, err2 := os.Stat(fullPath)
  630. if err2 != nil {
  631. // Warn about error, but continue trying to check files
  632. fmt.Fprintf(os.Stderr, "WARNING: Error accessing path '%s', err: %s\n", fullPath, err2)
  633. continue
  634. }
  635. // Exclude only files named 'BUILD' or 'BUILD.bazel' and unless forcibly kept
  636. if fileInfo.IsDir() ||
  637. (fileInfo.Name() != "BUILD" && fileInfo.Name() != "BUILD.bazel") ||
  638. ctx.Config().Bp2buildPackageConfig.ShouldKeepExistingBuildFileForDir(filepath.Dir(path)) {
  639. // Don't ignore this existing build file
  640. continue
  641. }
  642. if verbose {
  643. fmt.Fprintf(os.Stderr, "Ignoring existing BUILD file: %s\n", path)
  644. }
  645. excluded = append(excluded, path)
  646. }
  647. // Temporarily exclude stuff to make `bazel build //external/...` (and `bazel build //frameworks/...`) work
  648. excluded = append(excluded,
  649. // FIXME: 'autotest_lib' is a symlink back to external/autotest, and this causes an infinite
  650. // symlink expansion error for Bazel
  651. "external/autotest/venv/autotest_lib",
  652. "external/autotest/autotest_lib",
  653. "external/autotest/client/autotest_lib/client",
  654. // FIXME: The external/google-fruit/extras/bazel_root/third_party/fruit dir is poison
  655. // It contains several symlinks back to real source dirs, and those source dirs contain
  656. // BUILD files we want to ignore
  657. "external/google-fruit/extras/bazel_root/third_party/fruit",
  658. // FIXME: 'frameworks/compile/slang' has a filegroup error due to an escaping issue
  659. "frameworks/compile/slang",
  660. // FIXME(b/260809113): 'prebuilts/clang/host/linux-x86/clang-dev' is a tool-generated symlink
  661. // directory that contains a BUILD file. The bazel files finder code doesn't traverse into symlink dirs,
  662. // and hence is not aware of this BUILD file and exclude it accordingly during symlink forest generation
  663. // when checking against keepExistingBuildFiles allowlist.
  664. //
  665. // This is necessary because globs in //prebuilts/clang/host/linux-x86/BUILD
  666. // currently assume no subpackages (keepExistingBuildFile is not recursive for that directory).
  667. //
  668. // This is a bandaid until we the symlink forest logic can intelligently exclude BUILD files found in
  669. // source symlink dirs according to the keepExistingBuildFile allowlist.
  670. "prebuilts/clang/host/linux-x86/clang-dev",
  671. )
  672. return excluded
  673. }
  674. // buildTargetsByPackage parses Bazel BUILD.bazel and BUILD files under
  675. // the workspace, and returns a map containing names of Bazel targets defined in
  676. // these BUILD files.
  677. // For example, maps "//foo/bar" to ["baz", "qux"] if `//foo/bar:{baz,qux}` exist.
  678. func buildTargetsByPackage(ctx *android.Context) map[string][]string {
  679. existingBazelFiles, err := getExistingBazelRelatedFiles(topDir)
  680. maybeQuit(err, "Error determining existing Bazel-related files")
  681. result := map[string][]string{}
  682. // Search for instances of `name = "$NAME"` (with arbitrary spacing).
  683. targetNameRegex := regexp.MustCompile(`(?m)^\s*name\s*=\s*\"([^\"]+)\"`)
  684. for _, path := range existingBazelFiles {
  685. if !ctx.Config().Bp2buildPackageConfig.ShouldKeepExistingBuildFileForDir(filepath.Dir(path)) {
  686. continue
  687. }
  688. fullPath := shared.JoinPath(topDir, path)
  689. sourceDir := filepath.Dir(path)
  690. fileInfo, err := os.Stat(fullPath)
  691. maybeQuit(err, "Error accessing Bazel file '%s'", fullPath)
  692. if !fileInfo.IsDir() &&
  693. (fileInfo.Name() == "BUILD" || fileInfo.Name() == "BUILD.bazel") {
  694. // Process this BUILD file.
  695. buildFileContent, err := os.ReadFile(fullPath)
  696. maybeQuit(err, "Error reading Bazel file '%s'", fullPath)
  697. matches := targetNameRegex.FindAllStringSubmatch(string(buildFileContent), -1)
  698. for _, match := range matches {
  699. result[sourceDir] = append(result[sourceDir], match[1])
  700. }
  701. }
  702. }
  703. return result
  704. }
  705. // Run Soong in the bp2build mode. This creates a standalone context that registers
  706. // an alternate pipeline of mutators and singletons specifically for generating
  707. // Bazel BUILD files instead of Ninja files.
  708. func runBp2Build(ctx *android.Context, extraNinjaDeps []string, metricsDir string) string {
  709. var codegenMetrics *bp2build.CodegenMetrics
  710. ctx.EventHandler.Do("bp2build", func() {
  711. ctx.EventHandler.Do("read_build", func() {
  712. ctx.Config().SetBazelBuildFileTargets(buildTargetsByPackage(ctx))
  713. })
  714. // Propagate "allow misssing dependencies" bit. This is normally set in
  715. // newContext(), but we create ctx without calling that method.
  716. ctx.SetAllowMissingDependencies(ctx.Config().AllowMissingDependencies())
  717. ctx.SetNameInterface(newNameResolver(ctx.Config()))
  718. ctx.RegisterForBazelConversion()
  719. ctx.SetModuleListFile(cmdlineArgs.ModuleListFile)
  720. // Skip cloning modules during bp2build's blueprint run. Some mutators set
  721. // bp2build-related module values which should be preserved during codegen.
  722. ctx.SkipCloneModulesAfterMutators = true
  723. var ninjaDeps []string
  724. ninjaDeps = append(ninjaDeps, extraNinjaDeps...)
  725. // Run the loading and analysis pipeline to prepare the graph of regular
  726. // Modules parsed from Android.bp files, and the BazelTargetModules mapped
  727. // from the regular Modules.
  728. ctx.EventHandler.Do("bootstrap", func() {
  729. blueprintArgs := cmdlineArgs
  730. bootstrapDeps := bootstrap.RunBlueprint(blueprintArgs.Args,
  731. bootstrap.StopBeforePrepareBuildActions, ctx.Context, ctx.Config())
  732. ninjaDeps = append(ninjaDeps, bootstrapDeps...)
  733. })
  734. globListFiles := writeBuildGlobsNinjaFile(ctx)
  735. ninjaDeps = append(ninjaDeps, globListFiles...)
  736. // Run the code-generation phase to convert BazelTargetModules to BUILD files
  737. // and print conversion codegenMetrics to the user.
  738. codegenContext := bp2build.NewCodegenContext(ctx.Config(), ctx, bp2build.Bp2Build, topDir)
  739. ctx.EventHandler.Do("codegen", func() {
  740. codegenMetrics = bp2build.Codegen(codegenContext)
  741. })
  742. ninjaDeps = append(ninjaDeps, codegenContext.AdditionalNinjaDeps()...)
  743. writeDepFile(cmdlineArgs.Bp2buildMarker, ctx.EventHandler, ninjaDeps)
  744. touch(shared.JoinPath(topDir, cmdlineArgs.Bp2buildMarker))
  745. })
  746. // Only report metrics when in bp2build mode. The metrics aren't relevant
  747. // for queryview, since that's a total repo-wide conversion and there's a
  748. // 1:1 mapping for each module.
  749. if ctx.Config().IsEnvTrue("BP2BUILD_VERBOSE") {
  750. codegenMetrics.Print()
  751. }
  752. writeBp2BuildMetrics(codegenMetrics, ctx.EventHandler, metricsDir)
  753. return cmdlineArgs.Bp2buildMarker
  754. }
  755. // Write Bp2Build metrics into $LOG_DIR
  756. func writeBp2BuildMetrics(codegenMetrics *bp2build.CodegenMetrics, eventHandler *metrics.EventHandler, metricsDir string) {
  757. for _, event := range eventHandler.CompletedEvents() {
  758. codegenMetrics.AddEvent(&bp2build_metrics_proto.Event{
  759. Name: event.Id,
  760. StartTime: uint64(event.Start.UnixNano()),
  761. RealTime: event.RuntimeNanoseconds(),
  762. })
  763. }
  764. if len(metricsDir) < 1 {
  765. fmt.Fprintf(os.Stderr, "\nMissing required env var for generating bp2build metrics: LOG_DIR\n")
  766. os.Exit(1)
  767. }
  768. codegenMetrics.Write(metricsDir)
  769. }
  770. func readFileLines(path string) ([]string, error) {
  771. data, err := os.ReadFile(path)
  772. if err == nil {
  773. return strings.Split(strings.TrimSpace(string(data)), "\n"), nil
  774. }
  775. return nil, err
  776. }
  777. func maybeQuit(err error, format string, args ...interface{}) {
  778. if err == nil {
  779. return
  780. }
  781. if format != "" {
  782. fmt.Fprintln(os.Stderr, fmt.Sprintf(format, args...)+": "+err.Error())
  783. } else {
  784. fmt.Fprintln(os.Stderr, err)
  785. }
  786. os.Exit(1)
  787. }