bazel.go 9.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257
  1. // Copyright 2020 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package build
  15. import (
  16. "bytes"
  17. "fmt"
  18. "io/ioutil"
  19. "os"
  20. "path/filepath"
  21. "strings"
  22. "android/soong/bazel"
  23. "android/soong/shared"
  24. "android/soong/ui/metrics"
  25. )
  26. func getBazelInfo(ctx Context, config Config, bazelExecutable string, bazelEnv map[string]string, query string) string {
  27. infoCmd := Command(ctx, config, "bazel", bazelExecutable)
  28. if extraStartupArgs, ok := infoCmd.Environment.Get("BAZEL_STARTUP_ARGS"); ok {
  29. infoCmd.Args = append(infoCmd.Args, strings.Fields(extraStartupArgs)...)
  30. }
  31. // Obtain the output directory path in the execution root.
  32. infoCmd.Args = append(infoCmd.Args,
  33. "info",
  34. query,
  35. )
  36. for k, v := range bazelEnv {
  37. infoCmd.Environment.Set(k, v)
  38. }
  39. infoCmd.Dir = filepath.Join(config.OutDir(), "..")
  40. queryResult := strings.TrimSpace(string(infoCmd.OutputOrFatal()))
  41. return queryResult
  42. }
  43. // Main entry point to construct the Bazel build command line, environment
  44. // variables and post-processing steps (e.g. converge output directories)
  45. func runBazel(ctx Context, config Config) {
  46. ctx.BeginTrace(metrics.RunBazel, "bazel")
  47. defer ctx.EndTrace()
  48. // "droid" is the default ninja target.
  49. // TODO(b/160568333): stop hardcoding 'droid' to support building any
  50. // Ninja target.
  51. outputGroups := "droid"
  52. if len(config.ninjaArgs) > 0 {
  53. // At this stage, the residue slice of args passed to ninja
  54. // are the ninja targets to build, which can correspond directly
  55. // to ninja_build's output_groups.
  56. outputGroups = strings.Join(config.ninjaArgs, ",")
  57. }
  58. // Environment variables are the primary mechanism to pass information from
  59. // soong_ui configuration or context to Bazel.
  60. bazelEnv := make(map[string]string)
  61. // Use *_NINJA variables to pass the root-relative path of the combined,
  62. // kati-generated, soong-generated, and packaging Ninja files to Bazel.
  63. // Bazel reads these from the lunch() repository rule.
  64. bazelEnv["COMBINED_NINJA"] = config.CombinedNinjaFile()
  65. bazelEnv["KATI_NINJA"] = config.KatiBuildNinjaFile()
  66. bazelEnv["PACKAGE_NINJA"] = config.KatiPackageNinjaFile()
  67. bazelEnv["SOONG_NINJA"] = config.SoongNinjaFile()
  68. // NOTE: When Bazel is used, config.DistDir() is rigged to return a fake distdir under config.OutDir()
  69. // This is to ensure that Bazel can actually write there. See config.go for more details.
  70. bazelEnv["DIST_DIR"] = config.DistDir()
  71. bazelEnv["SHELL"] = "/bin/bash"
  72. // `build/bazel/bin/bazel` is the default entry point for executing Bazel in the AOSP
  73. // source tree.
  74. bazelExecutable := filepath.Join("build", "bazel", "bin", "bazel")
  75. cmd := Command(ctx, config, "bazel", bazelExecutable)
  76. // Append custom startup flags to the Bazel command. Startup flags affect
  77. // the Bazel server itself, and any changes to these flags would incur a
  78. // restart of the server, losing much of the in-memory incrementality.
  79. if extraStartupArgs, ok := cmd.Environment.Get("BAZEL_STARTUP_ARGS"); ok {
  80. cmd.Args = append(cmd.Args, strings.Fields(extraStartupArgs)...)
  81. }
  82. // Start constructing the `build` command.
  83. actionName := bazel.BazelNinjaExecRunName
  84. cmd.Args = append(cmd.Args,
  85. "build",
  86. // Use output_groups to select the set of outputs to produce from a
  87. // ninja_build target.
  88. "--output_groups="+outputGroups,
  89. // Generate a performance profile
  90. "--profile="+filepath.Join(shared.BazelMetricsFilename(config, actionName)),
  91. "--slim_profile=true",
  92. )
  93. if config.UseRBE() {
  94. for _, envVar := range []string{
  95. // RBE client
  96. "RBE_compare",
  97. "RBE_exec_strategy",
  98. "RBE_invocation_id",
  99. "RBE_log_dir",
  100. "RBE_num_retries_if_mismatched",
  101. "RBE_platform",
  102. "RBE_remote_accept_cache",
  103. "RBE_remote_update_cache",
  104. "RBE_server_address",
  105. // TODO: remove old FLAG_ variables.
  106. "FLAG_compare",
  107. "FLAG_exec_root",
  108. "FLAG_exec_strategy",
  109. "FLAG_invocation_id",
  110. "FLAG_log_dir",
  111. "FLAG_platform",
  112. "FLAG_remote_accept_cache",
  113. "FLAG_remote_update_cache",
  114. "FLAG_server_address",
  115. } {
  116. cmd.Args = append(cmd.Args,
  117. "--action_env="+envVar)
  118. }
  119. // We need to calculate --RBE_exec_root ourselves
  120. ctx.Println("Getting Bazel execution_root...")
  121. cmd.Args = append(cmd.Args, "--action_env=RBE_exec_root="+getBazelInfo(ctx, config, bazelExecutable, bazelEnv, "execution_root"))
  122. }
  123. // Ensure that the PATH environment variable value used in the action
  124. // environment is the restricted set computed from soong_ui, and not a
  125. // user-provided one, for hermeticity reasons.
  126. if pathEnvValue, ok := config.environ.Get("PATH"); ok {
  127. cmd.Environment.Set("PATH", pathEnvValue)
  128. cmd.Args = append(cmd.Args, "--action_env=PATH="+pathEnvValue)
  129. }
  130. // Allow Bazel actions to see the SHELL variable (passed to Bazel above)
  131. cmd.Args = append(cmd.Args, "--action_env=SHELL")
  132. // Append custom build flags to the Bazel command. Changes to these flags
  133. // may invalidate Bazel's analysis cache.
  134. // These should be appended as the final args, so that they take precedence.
  135. if extraBuildArgs, ok := cmd.Environment.Get("BAZEL_BUILD_ARGS"); ok {
  136. cmd.Args = append(cmd.Args, strings.Fields(extraBuildArgs)...)
  137. }
  138. // Append the label of the default ninja_build target.
  139. cmd.Args = append(cmd.Args,
  140. "//:"+config.TargetProduct()+"-"+config.TargetBuildVariant(),
  141. )
  142. // Execute the command at the root of the directory.
  143. cmd.Dir = filepath.Join(config.OutDir(), "..")
  144. for k, v := range bazelEnv {
  145. cmd.Environment.Set(k, v)
  146. }
  147. // Make a human-readable version of the bazelEnv map
  148. bazelEnvStringBuffer := new(bytes.Buffer)
  149. for k, v := range bazelEnv {
  150. fmt.Fprintf(bazelEnvStringBuffer, "%s=%s ", k, v)
  151. }
  152. // Print the implicit command line
  153. ctx.Println("Bazel implicit command line: " + strings.Join(cmd.Environment.Environ(), " ") + " " + cmd.Cmd.String() + "\n")
  154. // Print the explicit command line too
  155. ctx.Println("Bazel explicit command line: " + bazelEnvStringBuffer.String() + cmd.Cmd.String() + "\n")
  156. // Execute the build command.
  157. cmd.RunAndStreamOrFatal()
  158. // Post-processing steps start here. Once the Bazel build completes, the
  159. // output files are still stored in the execution root, not in $OUT_DIR.
  160. // Ensure that the $OUT_DIR contains the expected set of files by symlinking
  161. // the files from the execution root's output direction into $OUT_DIR.
  162. ctx.Println("Getting Bazel output_path...")
  163. outputBasePath := getBazelInfo(ctx, config, bazelExecutable, bazelEnv, "output_path")
  164. // TODO: Don't hardcode out/ as the bazel output directory. This is
  165. // currently hardcoded as ninja_build.output_root.
  166. bazelNinjaBuildOutputRoot := filepath.Join(outputBasePath, "..", "out")
  167. ctx.Println("Populating output directory...")
  168. populateOutdir(ctx, config, bazelNinjaBuildOutputRoot, ".")
  169. }
  170. // For all files F recursively under rootPath/relativePath, creates symlinks
  171. // such that OutDir/F resolves to rootPath/F via symlinks.
  172. // NOTE: For distdir paths we rename files instead of creating symlinks, so that the distdir is independent.
  173. func populateOutdir(ctx Context, config Config, rootPath string, relativePath string) {
  174. destDir := filepath.Join(rootPath, relativePath)
  175. os.MkdirAll(destDir, 0755)
  176. files, err := ioutil.ReadDir(destDir)
  177. if err != nil {
  178. ctx.Fatal(err)
  179. }
  180. for _, f := range files {
  181. // The original Bazel file path
  182. destPath := filepath.Join(destDir, f.Name())
  183. // The desired Soong file path
  184. srcPath := filepath.Join(config.OutDir(), relativePath, f.Name())
  185. destLstatResult, destLstatErr := os.Lstat(destPath)
  186. if destLstatErr != nil {
  187. ctx.Fatalf("Unable to Lstat dest %s: %s", destPath, destLstatErr)
  188. }
  189. srcLstatResult, srcLstatErr := os.Lstat(srcPath)
  190. if srcLstatErr == nil {
  191. if srcLstatResult.IsDir() && destLstatResult.IsDir() {
  192. // src and dest are both existing dirs - recurse on the dest dir contents...
  193. populateOutdir(ctx, config, rootPath, filepath.Join(relativePath, f.Name()))
  194. } else {
  195. // Ignore other pre-existing src files (could be pre-existing files, directories, symlinks, ...)
  196. // This can arise for files which are generated under OutDir outside of soong_build, such as .bootstrap files.
  197. // FIXME: This might cause a problem later e.g. if a symlink in the build graph changes...
  198. }
  199. } else {
  200. if !os.IsNotExist(srcLstatErr) {
  201. ctx.Fatalf("Unable to Lstat src %s: %s", srcPath, srcLstatErr)
  202. }
  203. if strings.Contains(destDir, config.DistDir()) {
  204. // We need to make a "real" file/dir instead of making a symlink (because the distdir can't have symlinks)
  205. // Rename instead of copy in order to save disk space.
  206. if err := os.Rename(destPath, srcPath); err != nil {
  207. ctx.Fatalf("Unable to rename %s -> %s due to error %s", srcPath, destPath, err)
  208. }
  209. } else {
  210. // src does not exist, so try to create a src -> dest symlink (i.e. a Soong path -> Bazel path symlink)
  211. if err := os.Symlink(destPath, srcPath); err != nil {
  212. ctx.Fatalf("Unable to create symlink %s -> %s due to error %s", srcPath, destPath, err)
  213. }
  214. }
  215. }
  216. }
  217. }