main.go 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654
  1. // Copyright 2015 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package main
  15. import (
  16. "bytes"
  17. "flag"
  18. "fmt"
  19. "os"
  20. "path/filepath"
  21. "strings"
  22. "time"
  23. "android/soong/android"
  24. "android/soong/bazel"
  25. "android/soong/bp2build"
  26. "android/soong/shared"
  27. "android/soong/ui/metrics/bp2build_metrics_proto"
  28. "github.com/google/blueprint/bootstrap"
  29. "github.com/google/blueprint/deptools"
  30. "github.com/google/blueprint/metrics"
  31. androidProtobuf "google.golang.org/protobuf/android"
  32. )
  33. var (
  34. topDir string
  35. availableEnvFile string
  36. usedEnvFile string
  37. globFile string
  38. globListDir string
  39. delveListen string
  40. delvePath string
  41. cmdlineArgs android.CmdArgs
  42. )
  43. func init() {
  44. // Flags that make sense in every mode
  45. flag.StringVar(&topDir, "top", "", "Top directory of the Android source tree")
  46. flag.StringVar(&cmdlineArgs.SoongOutDir, "soong_out", "", "Soong output directory (usually $TOP/out/soong)")
  47. flag.StringVar(&availableEnvFile, "available_env", "", "File containing available environment variables")
  48. flag.StringVar(&usedEnvFile, "used_env", "", "File containing used environment variables")
  49. flag.StringVar(&globFile, "globFile", "build-globs.ninja", "the Ninja file of globs to output")
  50. flag.StringVar(&globListDir, "globListDir", "", "the directory containing the glob list files")
  51. flag.StringVar(&cmdlineArgs.OutDir, "out", "", "the ninja builddir directory")
  52. flag.StringVar(&cmdlineArgs.ModuleListFile, "l", "", "file that lists filepaths to parse")
  53. // Debug flags
  54. flag.StringVar(&delveListen, "delve_listen", "", "Delve port to listen on for debugging")
  55. flag.StringVar(&delvePath, "delve_path", "", "Path to Delve. Only used if --delve_listen is set")
  56. flag.StringVar(&cmdlineArgs.Cpuprofile, "cpuprofile", "", "write cpu profile to file")
  57. flag.StringVar(&cmdlineArgs.TraceFile, "trace", "", "write trace to file")
  58. flag.StringVar(&cmdlineArgs.Memprofile, "memprofile", "", "write memory profile to file")
  59. flag.BoolVar(&cmdlineArgs.NoGC, "nogc", false, "turn off GC for debugging")
  60. // Flags representing various modes soong_build can run in
  61. flag.StringVar(&cmdlineArgs.ModuleGraphFile, "module_graph_file", "", "JSON module graph file to output")
  62. flag.StringVar(&cmdlineArgs.ModuleActionsFile, "module_actions_file", "", "JSON file to output inputs/outputs of actions of modules")
  63. flag.StringVar(&cmdlineArgs.DocFile, "soong_docs", "", "build documentation file to output")
  64. flag.StringVar(&cmdlineArgs.BazelQueryViewDir, "bazel_queryview_dir", "", "path to the bazel queryview directory relative to --top")
  65. flag.StringVar(&cmdlineArgs.BazelApiBp2buildDir, "bazel_api_bp2build_dir", "", "path to the bazel api_bp2build directory relative to --top")
  66. flag.StringVar(&cmdlineArgs.Bp2buildMarker, "bp2build_marker", "", "If set, run bp2build, touch the specified marker file then exit")
  67. flag.StringVar(&cmdlineArgs.SymlinkForestMarker, "symlink_forest_marker", "", "If set, create the bp2build symlink forest, touch the specified marker file, then exit")
  68. flag.StringVar(&cmdlineArgs.OutFile, "o", "build.ninja", "the Ninja file to output")
  69. flag.StringVar(&cmdlineArgs.BazelForceEnabledModules, "bazel-force-enabled-modules", "", "additional modules to build with Bazel. Comma-delimited")
  70. flag.BoolVar(&cmdlineArgs.EmptyNinjaFile, "empty-ninja-file", false, "write out a 0-byte ninja file")
  71. flag.BoolVar(&cmdlineArgs.BazelMode, "bazel-mode", false, "use bazel for analysis of certain modules")
  72. flag.BoolVar(&cmdlineArgs.BazelModeStaging, "bazel-mode-staging", false, "use bazel for analysis of certain near-ready modules")
  73. flag.BoolVar(&cmdlineArgs.BazelModeDev, "bazel-mode-dev", false, "use bazel for analysis of a large number of modules (less stable)")
  74. flag.BoolVar(&cmdlineArgs.UseBazelProxy, "use-bazel-proxy", false, "communicate with bazel using unix socket proxy instead of spawning subprocesses")
  75. // Flags that probably shouldn't be flags of soong_build, but we haven't found
  76. // the time to remove them yet
  77. flag.BoolVar(&cmdlineArgs.RunGoTests, "t", false, "build and run go tests during bootstrap")
  78. // Disable deterministic randomization in the protobuf package, so incremental
  79. // builds with unrelated Soong changes don't trigger large rebuilds (since we
  80. // write out text protos in command lines, and command line changes trigger
  81. // rebuilds).
  82. androidProtobuf.DisableRand()
  83. }
  84. func newNameResolver(config android.Config) *android.NameResolver {
  85. return android.NewNameResolver(config)
  86. }
  87. func newContext(configuration android.Config) *android.Context {
  88. ctx := android.NewContext(configuration)
  89. ctx.SetNameInterface(newNameResolver(configuration))
  90. ctx.SetAllowMissingDependencies(configuration.AllowMissingDependencies())
  91. ctx.AddIncludeTags(configuration.IncludeTags()...)
  92. return ctx
  93. }
  94. // Bazel-enabled mode. Attaches a mutator to queue Bazel requests, adds a
  95. // BeforePrepareBuildActionsHook to invoke Bazel, and then uses Bazel metadata
  96. // for modules that should be handled by Bazel.
  97. func runMixedModeBuild(ctx *android.Context, extraNinjaDeps []string) string {
  98. ctx.EventHandler.Begin("mixed_build")
  99. defer ctx.EventHandler.End("mixed_build")
  100. bazelHook := func() error {
  101. return ctx.Config().BazelContext.InvokeBazel(ctx.Config(), ctx)
  102. }
  103. ctx.SetBeforePrepareBuildActionsHook(bazelHook)
  104. ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs.Args, bootstrap.DoEverything, ctx.Context, ctx.Config())
  105. ninjaDeps = append(ninjaDeps, extraNinjaDeps...)
  106. bazelPaths, err := readFileLines(ctx.Config().Getenv("BAZEL_DEPS_FILE"))
  107. if err != nil {
  108. panic("Bazel deps file not found: " + err.Error())
  109. }
  110. ninjaDeps = append(ninjaDeps, bazelPaths...)
  111. ninjaDeps = append(ninjaDeps, writeBuildGlobsNinjaFile(ctx)...)
  112. writeDepFile(cmdlineArgs.OutFile, ctx.EventHandler, ninjaDeps)
  113. return cmdlineArgs.OutFile
  114. }
  115. // Run the code-generation phase to convert BazelTargetModules to BUILD files.
  116. func runQueryView(queryviewDir, queryviewMarker string, ctx *android.Context) {
  117. ctx.EventHandler.Begin("queryview")
  118. defer ctx.EventHandler.End("queryview")
  119. codegenContext := bp2build.NewCodegenContext(ctx.Config(), ctx, bp2build.QueryView, topDir)
  120. err := createBazelWorkspace(codegenContext, shared.JoinPath(topDir, queryviewDir))
  121. maybeQuit(err, "")
  122. touch(shared.JoinPath(topDir, queryviewMarker))
  123. }
  124. // Run the code-generation phase to convert API contributions to BUILD files.
  125. // Return marker file for the new synthetic workspace
  126. func runApiBp2build(ctx *android.Context, extraNinjaDeps []string) string {
  127. ctx.EventHandler.Begin("api_bp2build")
  128. defer ctx.EventHandler.End("api_bp2build")
  129. // api_bp2build does not run the typical pipeline of soong mutators.
  130. // Hoevever, it still runs the defaults mutator which can create dependencies.
  131. // These dependencies might not always exist (e.g. in tests)
  132. ctx.SetAllowMissingDependencies(ctx.Config().AllowMissingDependencies())
  133. ctx.RegisterForApiBazelConversion()
  134. // Register the Android.bp files in the tree
  135. // Add them to the workspace's .d file
  136. ctx.SetModuleListFile(cmdlineArgs.ModuleListFile)
  137. if paths, err := ctx.ListModulePaths("."); err == nil {
  138. extraNinjaDeps = append(extraNinjaDeps, paths...)
  139. } else {
  140. panic(err)
  141. }
  142. // Run the loading and analysis phase
  143. ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs.Args,
  144. bootstrap.StopBeforePrepareBuildActions,
  145. ctx.Context,
  146. ctx.Config())
  147. ninjaDeps = append(ninjaDeps, extraNinjaDeps...)
  148. // Add the globbed dependencies
  149. ninjaDeps = append(ninjaDeps, writeBuildGlobsNinjaFile(ctx)...)
  150. // Run codegen to generate BUILD files
  151. codegenContext := bp2build.NewCodegenContext(ctx.Config(), ctx, bp2build.ApiBp2build, topDir)
  152. absoluteApiBp2buildDir := shared.JoinPath(topDir, cmdlineArgs.BazelApiBp2buildDir)
  153. err := createBazelWorkspace(codegenContext, absoluteApiBp2buildDir)
  154. maybeQuit(err, "")
  155. ninjaDeps = append(ninjaDeps, codegenContext.AdditionalNinjaDeps()...)
  156. // Create soong_injection repository
  157. soongInjectionFiles, err := bp2build.CreateSoongInjectionDirFiles(codegenContext, bp2build.CreateCodegenMetrics())
  158. maybeQuit(err, "")
  159. absoluteSoongInjectionDir := shared.JoinPath(topDir, ctx.Config().SoongOutDir(), bazel.SoongInjectionDirName)
  160. for _, file := range soongInjectionFiles {
  161. // The API targets in api_bp2build workspace do not have any dependency on api_bp2build.
  162. // But we need to create these files to prevent errors during Bazel analysis.
  163. // These need to be created in Read-Write mode.
  164. // This is because the subsequent step (bp2build in api domain analysis) creates them in Read-Write mode
  165. // to allow users to edit/experiment in the synthetic workspace.
  166. writeReadWriteFile(absoluteSoongInjectionDir, file)
  167. }
  168. workspace := shared.JoinPath(ctx.Config().SoongOutDir(), "api_bp2build")
  169. // Create the symlink forest
  170. symlinkDeps, _, _ := bp2build.PlantSymlinkForest(
  171. ctx.Config().IsEnvTrue("BP2BUILD_VERBOSE"),
  172. topDir,
  173. workspace,
  174. cmdlineArgs.BazelApiBp2buildDir,
  175. apiBuildFileExcludes(ctx))
  176. ninjaDeps = append(ninjaDeps, symlinkDeps...)
  177. workspaceMarkerFile := workspace + ".marker"
  178. writeDepFile(workspaceMarkerFile, ctx.EventHandler, ninjaDeps)
  179. touch(shared.JoinPath(topDir, workspaceMarkerFile))
  180. return workspaceMarkerFile
  181. }
  182. // With some exceptions, api_bp2build does not have any dependencies on the checked-in BUILD files
  183. // Exclude them from the generated workspace to prevent unrelated errors during the loading phase
  184. func apiBuildFileExcludes(ctx *android.Context) []string {
  185. ret := bazelArtifacts()
  186. srcs, err := getExistingBazelRelatedFiles(topDir)
  187. maybeQuit(err, "Error determining existing Bazel-related files")
  188. for _, src := range srcs {
  189. // Exclude all src BUILD files
  190. if src != "WORKSPACE" &&
  191. src != "BUILD" &&
  192. src != "BUILD.bazel" &&
  193. !strings.HasPrefix(src, "build/bazel") &&
  194. !strings.HasPrefix(src, "external/bazel-skylib") &&
  195. !strings.HasPrefix(src, "prebuilts/clang") {
  196. ret = append(ret, src)
  197. }
  198. }
  199. // Android.bp files for api surfaces are mounted to out/, but out/ should not be a
  200. // dep for api_bp2build. Otherwise, api_bp2build will be run every single time
  201. ret = append(ret, ctx.Config().OutDir())
  202. return ret
  203. }
  204. func writeMetrics(configuration android.Config, eventHandler *metrics.EventHandler, metricsDir string) {
  205. if len(metricsDir) < 1 {
  206. fmt.Fprintf(os.Stderr, "\nMissing required env var for generating soong metrics: LOG_DIR\n")
  207. os.Exit(1)
  208. }
  209. metricsFile := filepath.Join(metricsDir, "soong_build_metrics.pb")
  210. err := android.WriteMetrics(configuration, eventHandler, metricsFile)
  211. maybeQuit(err, "error writing soong_build metrics %s", metricsFile)
  212. }
  213. func writeJsonModuleGraphAndActions(ctx *android.Context, cmdArgs android.CmdArgs) {
  214. graphFile, graphErr := os.Create(shared.JoinPath(topDir, cmdArgs.ModuleGraphFile))
  215. maybeQuit(graphErr, "graph err")
  216. defer graphFile.Close()
  217. actionsFile, actionsErr := os.Create(shared.JoinPath(topDir, cmdArgs.ModuleActionsFile))
  218. maybeQuit(actionsErr, "actions err")
  219. defer actionsFile.Close()
  220. ctx.Context.PrintJSONGraphAndActions(graphFile, actionsFile)
  221. }
  222. func writeBuildGlobsNinjaFile(ctx *android.Context) []string {
  223. ctx.EventHandler.Begin("globs_ninja_file")
  224. defer ctx.EventHandler.End("globs_ninja_file")
  225. globDir := bootstrap.GlobDirectory(ctx.Config().SoongOutDir(), globListDir)
  226. bootstrap.WriteBuildGlobsNinjaFile(&bootstrap.GlobSingleton{
  227. GlobLister: ctx.Globs,
  228. GlobFile: globFile,
  229. GlobDir: globDir,
  230. SrcDir: ctx.SrcDir(),
  231. }, ctx.Config())
  232. return bootstrap.GlobFileListFiles(globDir)
  233. }
  234. func writeDepFile(outputFile string, eventHandler *metrics.EventHandler, ninjaDeps []string) {
  235. eventHandler.Begin("ninja_deps")
  236. defer eventHandler.End("ninja_deps")
  237. depFile := shared.JoinPath(topDir, outputFile+".d")
  238. err := deptools.WriteDepFile(depFile, outputFile, ninjaDeps)
  239. maybeQuit(err, "error writing depfile '%s'", depFile)
  240. }
  241. // runSoongOnlyBuild runs the standard Soong build in a number of different modes.
  242. func runSoongOnlyBuild(ctx *android.Context, extraNinjaDeps []string) string {
  243. ctx.EventHandler.Begin("soong_build")
  244. defer ctx.EventHandler.End("soong_build")
  245. var stopBefore bootstrap.StopBefore
  246. switch ctx.Config().BuildMode {
  247. case android.GenerateModuleGraph:
  248. stopBefore = bootstrap.StopBeforeWriteNinja
  249. case android.GenerateQueryView, android.GenerateDocFile:
  250. stopBefore = bootstrap.StopBeforePrepareBuildActions
  251. default:
  252. stopBefore = bootstrap.DoEverything
  253. }
  254. ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs.Args, stopBefore, ctx.Context, ctx.Config())
  255. ninjaDeps = append(ninjaDeps, extraNinjaDeps...)
  256. globListFiles := writeBuildGlobsNinjaFile(ctx)
  257. ninjaDeps = append(ninjaDeps, globListFiles...)
  258. // Convert the Soong module graph into Bazel BUILD files.
  259. switch ctx.Config().BuildMode {
  260. case android.GenerateQueryView:
  261. queryviewMarkerFile := cmdlineArgs.BazelQueryViewDir + ".marker"
  262. runQueryView(cmdlineArgs.BazelQueryViewDir, queryviewMarkerFile, ctx)
  263. writeDepFile(queryviewMarkerFile, ctx.EventHandler, ninjaDeps)
  264. return queryviewMarkerFile
  265. case android.GenerateModuleGraph:
  266. writeJsonModuleGraphAndActions(ctx, cmdlineArgs)
  267. writeDepFile(cmdlineArgs.ModuleGraphFile, ctx.EventHandler, ninjaDeps)
  268. return cmdlineArgs.ModuleGraphFile
  269. case android.GenerateDocFile:
  270. // TODO: we could make writeDocs() return the list of documentation files
  271. // written and add them to the .d file. Then soong_docs would be re-run
  272. // whenever one is deleted.
  273. err := writeDocs(ctx, shared.JoinPath(topDir, cmdlineArgs.DocFile))
  274. maybeQuit(err, "error building Soong documentation")
  275. writeDepFile(cmdlineArgs.DocFile, ctx.EventHandler, ninjaDeps)
  276. return cmdlineArgs.DocFile
  277. default:
  278. // The actual output (build.ninja) was written in the RunBlueprint() call
  279. // above
  280. writeDepFile(cmdlineArgs.OutFile, ctx.EventHandler, ninjaDeps)
  281. return cmdlineArgs.OutFile
  282. }
  283. }
  284. // soong_ui dumps the available environment variables to
  285. // soong.environment.available . Then soong_build itself is run with an empty
  286. // environment so that the only way environment variables can be accessed is
  287. // using Config, which tracks access to them.
  288. // At the end of the build, a file called soong.environment.used is written
  289. // containing the current value of all used environment variables. The next
  290. // time soong_ui is run, it checks whether any environment variables that was
  291. // used had changed and if so, it deletes soong.environment.used to cause a
  292. // rebuild.
  293. //
  294. // The dependency of build.ninja on soong.environment.used is declared in
  295. // build.ninja.d
  296. func parseAvailableEnv() map[string]string {
  297. if availableEnvFile == "" {
  298. fmt.Fprintf(os.Stderr, "--available_env not set\n")
  299. os.Exit(1)
  300. }
  301. result, err := shared.EnvFromFile(shared.JoinPath(topDir, availableEnvFile))
  302. maybeQuit(err, "error reading available environment file '%s'", availableEnvFile)
  303. return result
  304. }
  305. func main() {
  306. flag.Parse()
  307. shared.ReexecWithDelveMaybe(delveListen, delvePath)
  308. android.InitSandbox(topDir)
  309. availableEnv := parseAvailableEnv()
  310. configuration, err := android.NewConfig(cmdlineArgs, availableEnv)
  311. maybeQuit(err, "")
  312. if configuration.Getenv("ALLOW_MISSING_DEPENDENCIES") == "true" {
  313. configuration.SetAllowMissingDependencies()
  314. }
  315. extraNinjaDeps := []string{configuration.ProductVariablesFileName, usedEnvFile}
  316. if shared.IsDebugging() {
  317. // Add a non-existent file to the dependencies so that soong_build will rerun when the debugger is
  318. // enabled even if it completed successfully.
  319. extraNinjaDeps = append(extraNinjaDeps, filepath.Join(configuration.SoongOutDir(), "always_rerun_for_delve"))
  320. }
  321. // Bypass configuration.Getenv, as LOG_DIR does not need to be dependency tracked. By definition, it will
  322. // change between every CI build, so tracking it would require re-running Soong for every build.
  323. metricsDir := availableEnv["LOG_DIR"]
  324. ctx := newContext(configuration)
  325. var finalOutputFile string
  326. // Run Soong for a specific activity, like bp2build, queryview
  327. // or the actual Soong build for the build.ninja file.
  328. switch configuration.BuildMode {
  329. case android.SymlinkForest:
  330. finalOutputFile = runSymlinkForestCreation(ctx, extraNinjaDeps, metricsDir)
  331. case android.Bp2build:
  332. // Run the alternate pipeline of bp2build mutators and singleton to convert
  333. // Blueprint to BUILD files before everything else.
  334. finalOutputFile = runBp2Build(ctx, extraNinjaDeps, metricsDir)
  335. case android.ApiBp2build:
  336. finalOutputFile = runApiBp2build(ctx, extraNinjaDeps)
  337. writeMetrics(configuration, ctx.EventHandler, metricsDir)
  338. default:
  339. ctx.Register()
  340. if configuration.IsMixedBuildsEnabled() {
  341. finalOutputFile = runMixedModeBuild(ctx, extraNinjaDeps)
  342. } else {
  343. finalOutputFile = runSoongOnlyBuild(ctx, extraNinjaDeps)
  344. }
  345. writeMetrics(configuration, ctx.EventHandler, metricsDir)
  346. }
  347. writeUsedEnvironmentFile(configuration, finalOutputFile)
  348. }
  349. func writeUsedEnvironmentFile(configuration android.Config, finalOutputFile string) {
  350. if usedEnvFile == "" {
  351. return
  352. }
  353. path := shared.JoinPath(topDir, usedEnvFile)
  354. data, err := shared.EnvFileContents(configuration.EnvDeps())
  355. maybeQuit(err, "error writing used environment file '%s'\n", usedEnvFile)
  356. if preexistingData, err := os.ReadFile(path); err != nil {
  357. if !os.IsNotExist(err) {
  358. maybeQuit(err, "error reading used environment file '%s'", usedEnvFile)
  359. }
  360. } else if bytes.Equal(preexistingData, data) {
  361. // used environment file is unchanged
  362. return
  363. }
  364. err = os.WriteFile(path, data, 0666)
  365. maybeQuit(err, "error writing used environment file '%s'", usedEnvFile)
  366. // Touch the output file so that it's not older than the file we just
  367. // wrote. We can't write the environment file earlier because one an access
  368. // new environment variables while writing it.
  369. touch(shared.JoinPath(topDir, finalOutputFile))
  370. }
  371. func touch(path string) {
  372. f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)
  373. maybeQuit(err, "Error touching '%s'", path)
  374. err = f.Close()
  375. maybeQuit(err, "Error touching '%s'", path)
  376. currentTime := time.Now().Local()
  377. err = os.Chtimes(path, currentTime, currentTime)
  378. maybeQuit(err, "error touching '%s'", path)
  379. }
  380. // Read the bazel.list file that the Soong Finder already dumped earlier (hopefully)
  381. // It contains the locations of BUILD files, BUILD.bazel files, etc. in the source dir
  382. func getExistingBazelRelatedFiles(topDir string) ([]string, error) {
  383. bazelFinderFile := filepath.Join(filepath.Dir(cmdlineArgs.ModuleListFile), "bazel.list")
  384. if !filepath.IsAbs(bazelFinderFile) {
  385. // Assume this was a relative path under topDir
  386. bazelFinderFile = filepath.Join(topDir, bazelFinderFile)
  387. }
  388. return readFileLines(bazelFinderFile)
  389. }
  390. func bazelArtifacts() []string {
  391. return []string{
  392. "bazel-bin",
  393. "bazel-genfiles",
  394. "bazel-out",
  395. "bazel-testlogs",
  396. "bazel-workspace",
  397. "bazel-" + filepath.Base(topDir),
  398. }
  399. }
  400. // This could in theory easily be separated into a binary that generically
  401. // merges two directories into a symlink tree. The main obstacle is that this
  402. // function currently depends on both Bazel-specific knowledge (the existence
  403. // of bazel-* symlinks) and configuration (the set of BUILD.bazel files that
  404. // should and should not be kept)
  405. //
  406. // Ideally, bp2build would write a file that contains instructions to the
  407. // symlink tree creation binary. Then the latter would not need to depend on
  408. // the very heavy-weight machinery of soong_build .
  409. func runSymlinkForestCreation(ctx *android.Context, extraNinjaDeps []string, metricsDir string) string {
  410. var ninjaDeps []string
  411. var mkdirCount, symlinkCount uint64
  412. ctx.EventHandler.Do("symlink_forest", func() {
  413. ninjaDeps = append(ninjaDeps, extraNinjaDeps...)
  414. verbose := ctx.Config().IsEnvTrue("BP2BUILD_VERBOSE")
  415. // PlantSymlinkForest() returns all the directories that were readdir()'ed.
  416. // Such a directory SHOULD be added to `ninjaDeps` so that a child directory
  417. // or file created/deleted under it would trigger an update of the symlink forest.
  418. generatedRoot := shared.JoinPath(ctx.Config().SoongOutDir(), "bp2build")
  419. workspaceRoot := shared.JoinPath(ctx.Config().SoongOutDir(), "workspace")
  420. var symlinkForestDeps []string
  421. ctx.EventHandler.Do("plant", func() {
  422. symlinkForestDeps, mkdirCount, symlinkCount = bp2build.PlantSymlinkForest(
  423. verbose, topDir, workspaceRoot, generatedRoot, excludedFromSymlinkForest(ctx, verbose))
  424. })
  425. ninjaDeps = append(ninjaDeps, symlinkForestDeps...)
  426. })
  427. writeDepFile(cmdlineArgs.SymlinkForestMarker, ctx.EventHandler, ninjaDeps)
  428. touch(shared.JoinPath(topDir, cmdlineArgs.SymlinkForestMarker))
  429. codegenMetrics := bp2build.ReadCodegenMetrics(metricsDir)
  430. if codegenMetrics == nil {
  431. m := bp2build.CreateCodegenMetrics()
  432. codegenMetrics = &m
  433. } else {
  434. //TODO (usta) we cannot determine if we loaded a stale file, i.e. from an unrelated prior
  435. //invocation of codegen. We should simply use a separate .pb file
  436. }
  437. codegenMetrics.SetSymlinkCount(symlinkCount)
  438. codegenMetrics.SetMkDirCount(mkdirCount)
  439. writeBp2BuildMetrics(codegenMetrics, ctx.EventHandler, metricsDir)
  440. return cmdlineArgs.SymlinkForestMarker
  441. }
  442. func excludedFromSymlinkForest(ctx *android.Context, verbose bool) []string {
  443. excluded := bazelArtifacts()
  444. if cmdlineArgs.OutDir[0] != '/' {
  445. excluded = append(excluded, cmdlineArgs.OutDir)
  446. }
  447. // Find BUILD files in the srcDir which are not in the allowlist
  448. // (android.Bp2BuildConversionAllowlist#ShouldKeepExistingBuildFileForDir)
  449. // and return their paths so they can be left out of the Bazel workspace dir (i.e. ignored)
  450. existingBazelFiles, err := getExistingBazelRelatedFiles(topDir)
  451. maybeQuit(err, "Error determining existing Bazel-related files")
  452. for _, path := range existingBazelFiles {
  453. fullPath := shared.JoinPath(topDir, path)
  454. fileInfo, err2 := os.Stat(fullPath)
  455. if err2 != nil {
  456. // Warn about error, but continue trying to check files
  457. fmt.Fprintf(os.Stderr, "WARNING: Error accessing path '%s', err: %s\n", fullPath, err2)
  458. continue
  459. }
  460. // Exclude only files named 'BUILD' or 'BUILD.bazel' and unless forcibly kept
  461. if fileInfo.IsDir() ||
  462. (fileInfo.Name() != "BUILD" && fileInfo.Name() != "BUILD.bazel") ||
  463. ctx.Config().Bp2buildPackageConfig.ShouldKeepExistingBuildFileForDir(filepath.Dir(path)) {
  464. // Don't ignore this existing build file
  465. continue
  466. }
  467. if verbose {
  468. fmt.Fprintf(os.Stderr, "Ignoring existing BUILD file: %s\n", path)
  469. }
  470. excluded = append(excluded, path)
  471. }
  472. // Temporarily exclude stuff to make `bazel build //external/...` (and `bazel build //frameworks/...`) work
  473. excluded = append(excluded,
  474. // FIXME: 'autotest_lib' is a symlink back to external/autotest, and this causes an infinite
  475. // symlink expansion error for Bazel
  476. "external/autotest/venv/autotest_lib",
  477. "external/autotest/autotest_lib",
  478. "external/autotest/client/autotest_lib/client",
  479. // FIXME: The external/google-fruit/extras/bazel_root/third_party/fruit dir is poison
  480. // It contains several symlinks back to real source dirs, and those source dirs contain
  481. // BUILD files we want to ignore
  482. "external/google-fruit/extras/bazel_root/third_party/fruit",
  483. // FIXME: 'frameworks/compile/slang' has a filegroup error due to an escaping issue
  484. "frameworks/compile/slang",
  485. // FIXME(b/260809113): 'prebuilts/clang/host/linux-x86/clang-dev' is a tool-generated symlink
  486. // directory that contains a BUILD file. The bazel files finder code doesn't traverse into symlink dirs,
  487. // and hence is not aware of this BUILD file and exclude it accordingly during symlink forest generation
  488. // when checking against keepExistingBuildFiles allowlist.
  489. //
  490. // This is necessary because globs in //prebuilts/clang/host/linux-x86/BUILD
  491. // currently assume no subpackages (keepExistingBuildFile is not recursive for that directory).
  492. //
  493. // This is a bandaid until we the symlink forest logic can intelligently exclude BUILD files found in
  494. // source symlink dirs according to the keepExistingBuildFile allowlist.
  495. "prebuilts/clang/host/linux-x86/clang-dev",
  496. )
  497. return excluded
  498. }
  499. // Run Soong in the bp2build mode. This creates a standalone context that registers
  500. // an alternate pipeline of mutators and singletons specifically for generating
  501. // Bazel BUILD files instead of Ninja files.
  502. func runBp2Build(ctx *android.Context, extraNinjaDeps []string, metricsDir string) string {
  503. var codegenMetrics *bp2build.CodegenMetrics
  504. ctx.EventHandler.Do("bp2build", func() {
  505. // Propagate "allow misssing dependencies" bit. This is normally set in
  506. // newContext(), but we create ctx without calling that method.
  507. ctx.SetAllowMissingDependencies(ctx.Config().AllowMissingDependencies())
  508. ctx.SetNameInterface(newNameResolver(ctx.Config()))
  509. ctx.RegisterForBazelConversion()
  510. ctx.SetModuleListFile(cmdlineArgs.ModuleListFile)
  511. var ninjaDeps []string
  512. ninjaDeps = append(ninjaDeps, extraNinjaDeps...)
  513. // Run the loading and analysis pipeline to prepare the graph of regular
  514. // Modules parsed from Android.bp files, and the BazelTargetModules mapped
  515. // from the regular Modules.
  516. ctx.EventHandler.Do("bootstrap", func() {
  517. blueprintArgs := cmdlineArgs
  518. bootstrapDeps := bootstrap.RunBlueprint(blueprintArgs.Args,
  519. bootstrap.StopBeforePrepareBuildActions, ctx.Context, ctx.Config())
  520. ninjaDeps = append(ninjaDeps, bootstrapDeps...)
  521. })
  522. globListFiles := writeBuildGlobsNinjaFile(ctx)
  523. ninjaDeps = append(ninjaDeps, globListFiles...)
  524. // Run the code-generation phase to convert BazelTargetModules to BUILD files
  525. // and print conversion codegenMetrics to the user.
  526. codegenContext := bp2build.NewCodegenContext(ctx.Config(), ctx, bp2build.Bp2Build, topDir)
  527. ctx.EventHandler.Do("codegen", func() {
  528. codegenMetrics = bp2build.Codegen(codegenContext)
  529. })
  530. ninjaDeps = append(ninjaDeps, codegenContext.AdditionalNinjaDeps()...)
  531. writeDepFile(cmdlineArgs.Bp2buildMarker, ctx.EventHandler, ninjaDeps)
  532. touch(shared.JoinPath(topDir, cmdlineArgs.Bp2buildMarker))
  533. })
  534. // Only report metrics when in bp2build mode. The metrics aren't relevant
  535. // for queryview, since that's a total repo-wide conversion and there's a
  536. // 1:1 mapping for each module.
  537. if ctx.Config().IsEnvTrue("BP2BUILD_VERBOSE") {
  538. codegenMetrics.Print()
  539. }
  540. writeBp2BuildMetrics(codegenMetrics, ctx.EventHandler, metricsDir)
  541. return cmdlineArgs.Bp2buildMarker
  542. }
  543. // Write Bp2Build metrics into $LOG_DIR
  544. func writeBp2BuildMetrics(codegenMetrics *bp2build.CodegenMetrics, eventHandler *metrics.EventHandler, metricsDir string) {
  545. for _, event := range eventHandler.CompletedEvents() {
  546. codegenMetrics.AddEvent(&bp2build_metrics_proto.Event{
  547. Name: event.Id,
  548. StartTime: uint64(event.Start.UnixNano()),
  549. RealTime: event.RuntimeNanoseconds(),
  550. })
  551. }
  552. if len(metricsDir) < 1 {
  553. fmt.Fprintf(os.Stderr, "\nMissing required env var for generating bp2build metrics: LOG_DIR\n")
  554. os.Exit(1)
  555. }
  556. codegenMetrics.Write(metricsDir)
  557. }
  558. func readFileLines(path string) ([]string, error) {
  559. data, err := os.ReadFile(path)
  560. if err == nil {
  561. return strings.Split(strings.TrimSpace(string(data)), "\n"), nil
  562. }
  563. return nil, err
  564. }
  565. func maybeQuit(err error, format string, args ...interface{}) {
  566. if err == nil {
  567. return
  568. }
  569. if format != "" {
  570. fmt.Fprintln(os.Stderr, fmt.Sprintf(format, args...)+": "+err.Error())
  571. } else {
  572. fmt.Fprintln(os.Stderr, err)
  573. }
  574. os.Exit(1)
  575. }