config.go 51 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794
  1. // Copyright 2017 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package build
  15. import (
  16. "context"
  17. "encoding/json"
  18. "errors"
  19. "fmt"
  20. "io/ioutil"
  21. "math/rand"
  22. "os"
  23. "os/exec"
  24. "path/filepath"
  25. "runtime"
  26. "strconv"
  27. "strings"
  28. "syscall"
  29. "time"
  30. "android/soong/shared"
  31. "google.golang.org/protobuf/proto"
  32. smpb "android/soong/ui/metrics/metrics_proto"
  33. )
  34. const (
  35. envConfigDir = "vendor/google/tools/soong_config"
  36. jsonSuffix = "json"
  37. configFetcher = "vendor/google/tools/soong/expconfigfetcher"
  38. envConfigFetchTimeout = 20 * time.Second
  39. )
  40. var (
  41. rbeRandPrefix int
  42. googleProdCredsExistCache bool
  43. )
  44. func init() {
  45. rand.Seed(time.Now().UnixNano())
  46. rbeRandPrefix = rand.Intn(1000)
  47. }
  48. type Config struct{ *configImpl }
  49. type configImpl struct {
  50. // Some targets that are implemented in soong_build
  51. // (bp2build, json-module-graph) are not here and have their own bits below.
  52. arguments []string
  53. goma bool
  54. environ *Environment
  55. distDir string
  56. buildDateTime string
  57. logsPrefix string
  58. // From the arguments
  59. parallel int
  60. keepGoing int
  61. verbose bool
  62. checkbuild bool
  63. dist bool
  64. jsonModuleGraph bool
  65. apiBp2build bool // Generate BUILD files for Soong modules that contribute APIs
  66. bp2build bool
  67. queryview bool
  68. reportMkMetrics bool // Collect and report mk2bp migration progress metrics.
  69. soongDocs bool
  70. multitreeBuild bool // This is a multitree build.
  71. skipConfig bool
  72. skipKati bool
  73. skipKatiNinja bool
  74. skipSoong bool
  75. skipNinja bool
  76. skipSoongTests bool
  77. searchApiDir bool // Scan the Android.bp files generated in out/api_surfaces
  78. skipMetricsUpload bool
  79. buildStartedTime int64 // For metrics-upload-only - manually specify a build-started time
  80. buildFromTextStub bool
  81. ensureAllowlistIntegrity bool // For CI builds - make sure modules are mixed-built
  82. bazelExitCode int32 // For b-runs - necessary for updating NonZeroExit
  83. // From the product config
  84. katiArgs []string
  85. ninjaArgs []string
  86. katiSuffix string
  87. targetDevice string
  88. targetDeviceDir string
  89. sandboxConfig *SandboxConfig
  90. // Autodetected
  91. totalRAM uint64
  92. brokenDupRules bool
  93. brokenUsesNetwork bool
  94. brokenNinjaEnvVars []string
  95. pathReplaced bool
  96. bazelProdMode bool
  97. bazelStagingMode bool
  98. // Set by multiproduct_kati
  99. emptyNinjaFile bool
  100. metricsUploader string
  101. bazelForceEnabledModules string
  102. includeTags []string
  103. sourceRootDirs []string
  104. // Data source to write ninja weight list
  105. ninjaWeightListSource NinjaWeightListSource
  106. }
  107. type NinjaWeightListSource uint
  108. const (
  109. // ninja doesn't use weight list.
  110. NOT_USED NinjaWeightListSource = iota
  111. // ninja uses weight list based on previous builds by ninja log
  112. NINJA_LOG
  113. // ninja thinks every task has the same weight.
  114. EVENLY_DISTRIBUTED
  115. // ninja uses an external custom weight list
  116. EXTERNAL_FILE
  117. // ninja uses a prioritized module list from Soong
  118. HINT_FROM_SOONG
  119. // If ninja log exists, use NINJA_LOG, if not, use HINT_FROM_SOONG instead.
  120. // We can assume it is an incremental build if ninja log exists.
  121. DEFAULT
  122. )
  123. const srcDirFileCheck = "build/soong/root.bp"
  124. var buildFiles = []string{"Android.mk", "Android.bp"}
  125. type BuildAction uint
  126. const (
  127. // Builds all of the modules and their dependencies of a specified directory, relative to the root
  128. // directory of the source tree.
  129. BUILD_MODULES_IN_A_DIRECTORY BuildAction = iota
  130. // Builds all of the modules and their dependencies of a list of specified directories. All specified
  131. // directories are relative to the root directory of the source tree.
  132. BUILD_MODULES_IN_DIRECTORIES
  133. // Build a list of specified modules. If none was specified, simply build the whole source tree.
  134. BUILD_MODULES
  135. )
  136. // checkTopDir validates that the current directory is at the root directory of the source tree.
  137. func checkTopDir(ctx Context) {
  138. if _, err := os.Stat(srcDirFileCheck); err != nil {
  139. if os.IsNotExist(err) {
  140. ctx.Fatalf("Current working directory must be the source tree. %q not found.", srcDirFileCheck)
  141. }
  142. ctx.Fatalln("Error verifying tree state:", err)
  143. }
  144. }
  145. // fetchEnvConfig optionally fetches a configuration file that can then subsequently be
  146. // loaded into Soong environment to control certain aspects of build behavior (e.g., enabling RBE).
  147. // If a configuration file already exists on disk, the fetch is run in the background
  148. // so as to NOT block the rest of the build execution.
  149. func fetchEnvConfig(ctx Context, config *configImpl, envConfigName string) error {
  150. configName := envConfigName + "." + jsonSuffix
  151. expConfigFetcher := &smpb.ExpConfigFetcher{Filename: &configName}
  152. defer func() {
  153. ctx.Metrics.ExpConfigFetcher(expConfigFetcher)
  154. }()
  155. if !config.GoogleProdCredsExist() {
  156. status := smpb.ExpConfigFetcher_MISSING_GCERT
  157. expConfigFetcher.Status = &status
  158. return nil
  159. }
  160. s, err := os.Stat(configFetcher)
  161. if err != nil {
  162. if os.IsNotExist(err) {
  163. return nil
  164. }
  165. return err
  166. }
  167. if s.Mode()&0111 == 0 {
  168. status := smpb.ExpConfigFetcher_ERROR
  169. expConfigFetcher.Status = &status
  170. return fmt.Errorf("configuration fetcher binary %v is not executable: %v", configFetcher, s.Mode())
  171. }
  172. configExists := false
  173. outConfigFilePath := filepath.Join(config.OutDir(), configName)
  174. if _, err := os.Stat(outConfigFilePath); err == nil {
  175. configExists = true
  176. }
  177. tCtx, cancel := context.WithTimeout(ctx, envConfigFetchTimeout)
  178. fetchStart := time.Now()
  179. cmd := exec.CommandContext(tCtx, configFetcher, "-output_config_dir", config.OutDir(),
  180. "-output_config_name", configName)
  181. if err := cmd.Start(); err != nil {
  182. status := smpb.ExpConfigFetcher_ERROR
  183. expConfigFetcher.Status = &status
  184. return err
  185. }
  186. fetchCfg := func() error {
  187. if err := cmd.Wait(); err != nil {
  188. status := smpb.ExpConfigFetcher_ERROR
  189. expConfigFetcher.Status = &status
  190. return err
  191. }
  192. fetchEnd := time.Now()
  193. expConfigFetcher.Micros = proto.Uint64(uint64(fetchEnd.Sub(fetchStart).Microseconds()))
  194. expConfigFetcher.Filename = proto.String(outConfigFilePath)
  195. if _, err := os.Stat(outConfigFilePath); err != nil {
  196. status := smpb.ExpConfigFetcher_NO_CONFIG
  197. expConfigFetcher.Status = &status
  198. return err
  199. }
  200. status := smpb.ExpConfigFetcher_CONFIG
  201. expConfigFetcher.Status = &status
  202. return nil
  203. }
  204. // If a config file does not exist, wait for the config file to be fetched. Otherwise
  205. // fetch the config file in the background and return immediately.
  206. if !configExists {
  207. defer cancel()
  208. return fetchCfg()
  209. }
  210. go func() {
  211. defer cancel()
  212. if err := fetchCfg(); err != nil {
  213. ctx.Verbosef("Failed to fetch config file %v: %v\n", configName, err)
  214. }
  215. }()
  216. return nil
  217. }
  218. func loadEnvConfig(ctx Context, config *configImpl, bc string) error {
  219. if bc == "" {
  220. return nil
  221. }
  222. configDirs := []string{
  223. config.OutDir(),
  224. os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG_DIR"),
  225. envConfigDir,
  226. }
  227. for _, dir := range configDirs {
  228. cfgFile := filepath.Join(os.Getenv("TOP"), dir, fmt.Sprintf("%s.%s", bc, jsonSuffix))
  229. envVarsJSON, err := ioutil.ReadFile(cfgFile)
  230. if err != nil {
  231. continue
  232. }
  233. ctx.Verbosef("Loading config file %v\n", cfgFile)
  234. var envVars map[string]map[string]string
  235. if err := json.Unmarshal(envVarsJSON, &envVars); err != nil {
  236. fmt.Fprintf(os.Stderr, "Env vars config file %s did not parse correctly: %s", cfgFile, err.Error())
  237. continue
  238. }
  239. for k, v := range envVars["env"] {
  240. if os.Getenv(k) != "" {
  241. continue
  242. }
  243. config.environ.Set(k, v)
  244. }
  245. ctx.Verbosef("Finished loading config file %v\n", cfgFile)
  246. break
  247. }
  248. return nil
  249. }
  250. func defaultBazelProdMode(cfg *configImpl) bool {
  251. // Environment flag to disable Bazel for users which experience
  252. // broken bazel-handled builds, or significant performance regressions.
  253. if cfg.IsBazelMixedBuildForceDisabled() {
  254. return false
  255. }
  256. // Darwin-host builds are currently untested with Bazel.
  257. if runtime.GOOS == "darwin" {
  258. return false
  259. }
  260. return true
  261. }
  262. func UploadOnlyConfig(ctx Context, args ...string) Config {
  263. ret := &configImpl{
  264. environ: OsEnvironment(),
  265. sandboxConfig: &SandboxConfig{},
  266. }
  267. ret.parseArgs(ctx, args)
  268. srcDir := absPath(ctx, ".")
  269. bc := os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG")
  270. if err := loadEnvConfig(ctx, ret, bc); err != nil {
  271. ctx.Fatalln("Failed to parse env config files: %v", err)
  272. }
  273. ret.metricsUploader = GetMetricsUploader(srcDir, ret.environ)
  274. return Config{ret}
  275. }
  276. func NewConfig(ctx Context, args ...string) Config {
  277. ret := &configImpl{
  278. environ: OsEnvironment(),
  279. sandboxConfig: &SandboxConfig{},
  280. ninjaWeightListSource: DEFAULT,
  281. }
  282. // Default matching ninja
  283. ret.parallel = runtime.NumCPU() + 2
  284. ret.keepGoing = 1
  285. ret.totalRAM = detectTotalRAM(ctx)
  286. ret.parseArgs(ctx, args)
  287. if ret.ninjaWeightListSource == HINT_FROM_SOONG {
  288. ret.environ.Set("SOONG_GENERATES_NINJA_HINT", "always")
  289. } else if ret.ninjaWeightListSource == DEFAULT {
  290. defaultNinjaWeightListSource := NINJA_LOG
  291. if _, err := os.Stat(filepath.Join(ret.OutDir(), ninjaLogFileName)); errors.Is(err, os.ErrNotExist) {
  292. ctx.Verboseln("$OUT/.ninja_log doesn't exist, use HINT_FROM_SOONG instead")
  293. defaultNinjaWeightListSource = HINT_FROM_SOONG
  294. } else {
  295. ctx.Verboseln("$OUT/.ninja_log exist, use NINJA_LOG")
  296. }
  297. ret.ninjaWeightListSource = defaultNinjaWeightListSource
  298. // soong_build generates ninja hint depending on ninja log existence.
  299. // Set it "depend" to avoid soong re-run due to env variable change.
  300. ret.environ.Set("SOONG_GENERATES_NINJA_HINT", "depend")
  301. }
  302. // Make sure OUT_DIR is set appropriately
  303. if outDir, ok := ret.environ.Get("OUT_DIR"); ok {
  304. ret.environ.Set("OUT_DIR", filepath.Clean(outDir))
  305. } else {
  306. outDir := "out"
  307. if baseDir, ok := ret.environ.Get("OUT_DIR_COMMON_BASE"); ok {
  308. if wd, err := os.Getwd(); err != nil {
  309. ctx.Fatalln("Failed to get working directory:", err)
  310. } else {
  311. outDir = filepath.Join(baseDir, filepath.Base(wd))
  312. }
  313. }
  314. ret.environ.Set("OUT_DIR", outDir)
  315. }
  316. // loadEnvConfig needs to know what the OUT_DIR is, so it should
  317. // be called after we determine the appropriate out directory.
  318. bc := os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG")
  319. if bc != "" {
  320. if err := fetchEnvConfig(ctx, ret, bc); err != nil {
  321. ctx.Verbosef("Failed to fetch config file: %v\n", err)
  322. }
  323. if err := loadEnvConfig(ctx, ret, bc); err != nil {
  324. ctx.Fatalln("Failed to parse env config files: %v", err)
  325. }
  326. }
  327. if distDir, ok := ret.environ.Get("DIST_DIR"); ok {
  328. ret.distDir = filepath.Clean(distDir)
  329. } else {
  330. ret.distDir = filepath.Join(ret.OutDir(), "dist")
  331. }
  332. if srcDirIsWritable, ok := ret.environ.Get("BUILD_BROKEN_SRC_DIR_IS_WRITABLE"); ok {
  333. ret.sandboxConfig.SetSrcDirIsRO(srcDirIsWritable == "false")
  334. }
  335. ret.environ.Unset(
  336. // We're already using it
  337. "USE_SOONG_UI",
  338. // We should never use GOROOT/GOPATH from the shell environment
  339. "GOROOT",
  340. "GOPATH",
  341. // These should only come from Soong, not the environment.
  342. "CLANG",
  343. "CLANG_CXX",
  344. "CCC_CC",
  345. "CCC_CXX",
  346. // Used by the goma compiler wrapper, but should only be set by
  347. // gomacc
  348. "GOMACC_PATH",
  349. // We handle this above
  350. "OUT_DIR_COMMON_BASE",
  351. // This is handled above too, and set for individual commands later
  352. "DIST_DIR",
  353. // Variables that have caused problems in the past
  354. "BASH_ENV",
  355. "CDPATH",
  356. "DISPLAY",
  357. "GREP_OPTIONS",
  358. "JAVAC",
  359. "NDK_ROOT",
  360. "POSIXLY_CORRECT",
  361. // Drop make flags
  362. "MAKEFLAGS",
  363. "MAKELEVEL",
  364. "MFLAGS",
  365. // Set in envsetup.sh, reset in makefiles
  366. "ANDROID_JAVA_TOOLCHAIN",
  367. // Set by envsetup.sh, but shouldn't be used inside the build because envsetup.sh is optional
  368. "ANDROID_BUILD_TOP",
  369. "ANDROID_HOST_OUT",
  370. "ANDROID_PRODUCT_OUT",
  371. "ANDROID_HOST_OUT_TESTCASES",
  372. "ANDROID_TARGET_OUT_TESTCASES",
  373. "ANDROID_TOOLCHAIN",
  374. "ANDROID_TOOLCHAIN_2ND_ARCH",
  375. "ANDROID_DEV_SCRIPTS",
  376. "ANDROID_EMULATOR_PREBUILTS",
  377. "ANDROID_PRE_BUILD_PATHS",
  378. )
  379. if ret.UseGoma() || ret.ForceUseGoma() {
  380. ctx.Println("Goma for Android has been deprecated and replaced with RBE. See go/rbe_for_android for instructions on how to use RBE.")
  381. ctx.Fatalln("USE_GOMA / FORCE_USE_GOMA flag is no longer supported.")
  382. }
  383. // Tell python not to spam the source tree with .pyc files.
  384. ret.environ.Set("PYTHONDONTWRITEBYTECODE", "1")
  385. tmpDir := absPath(ctx, ret.TempDir())
  386. ret.environ.Set("TMPDIR", tmpDir)
  387. // Always set ASAN_SYMBOLIZER_PATH so that ASAN-based tools can symbolize any crashes
  388. symbolizerPath := filepath.Join("prebuilts/clang/host", ret.HostPrebuiltTag(),
  389. "llvm-binutils-stable/llvm-symbolizer")
  390. ret.environ.Set("ASAN_SYMBOLIZER_PATH", absPath(ctx, symbolizerPath))
  391. // Precondition: the current directory is the top of the source tree
  392. checkTopDir(ctx)
  393. srcDir := absPath(ctx, ".")
  394. if strings.ContainsRune(srcDir, ' ') {
  395. ctx.Println("You are building in a directory whose absolute path contains a space character:")
  396. ctx.Println()
  397. ctx.Printf("%q\n", srcDir)
  398. ctx.Println()
  399. ctx.Fatalln("Directory names containing spaces are not supported")
  400. }
  401. ret.metricsUploader = GetMetricsUploader(srcDir, ret.environ)
  402. if outDir := ret.OutDir(); strings.ContainsRune(outDir, ' ') {
  403. ctx.Println("The absolute path of your output directory ($OUT_DIR) contains a space character:")
  404. ctx.Println()
  405. ctx.Printf("%q\n", outDir)
  406. ctx.Println()
  407. ctx.Fatalln("Directory names containing spaces are not supported")
  408. }
  409. if distDir := ret.RealDistDir(); strings.ContainsRune(distDir, ' ') {
  410. ctx.Println("The absolute path of your dist directory ($DIST_DIR) contains a space character:")
  411. ctx.Println()
  412. ctx.Printf("%q\n", distDir)
  413. ctx.Println()
  414. ctx.Fatalln("Directory names containing spaces are not supported")
  415. }
  416. // Configure Java-related variables, including adding it to $PATH
  417. java8Home := filepath.Join("prebuilts/jdk/jdk8", ret.HostPrebuiltTag())
  418. java9Home := filepath.Join("prebuilts/jdk/jdk9", ret.HostPrebuiltTag())
  419. java11Home := filepath.Join("prebuilts/jdk/jdk11", ret.HostPrebuiltTag())
  420. java17Home := filepath.Join("prebuilts/jdk/jdk17", ret.HostPrebuiltTag())
  421. javaHome := func() string {
  422. if override, ok := ret.environ.Get("OVERRIDE_ANDROID_JAVA_HOME"); ok {
  423. return override
  424. }
  425. if toolchain11, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN"); ok && toolchain11 != "true" {
  426. ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN is no longer supported. An OpenJDK 11 toolchain is now the global default.")
  427. }
  428. if toolchain17, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK17_TOOLCHAIN"); ok && toolchain17 != "true" {
  429. ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK17_TOOLCHAIN is no longer supported. An OpenJDK 17 toolchain is now the global default.")
  430. }
  431. return java17Home
  432. }()
  433. absJavaHome := absPath(ctx, javaHome)
  434. ret.configureLocale(ctx)
  435. newPath := []string{filepath.Join(absJavaHome, "bin")}
  436. if path, ok := ret.environ.Get("PATH"); ok && path != "" {
  437. newPath = append(newPath, path)
  438. }
  439. ret.environ.Unset("OVERRIDE_ANDROID_JAVA_HOME")
  440. ret.environ.Set("JAVA_HOME", absJavaHome)
  441. ret.environ.Set("ANDROID_JAVA_HOME", javaHome)
  442. ret.environ.Set("ANDROID_JAVA8_HOME", java8Home)
  443. ret.environ.Set("ANDROID_JAVA9_HOME", java9Home)
  444. ret.environ.Set("ANDROID_JAVA11_HOME", java11Home)
  445. ret.environ.Set("PATH", strings.Join(newPath, string(filepath.ListSeparator)))
  446. if ret.MultitreeBuild() {
  447. ret.environ.Set("MULTITREE_BUILD", "true")
  448. }
  449. outDir := ret.OutDir()
  450. buildDateTimeFile := filepath.Join(outDir, "build_date.txt")
  451. if buildDateTime, ok := ret.environ.Get("BUILD_DATETIME"); ok && buildDateTime != "" {
  452. ret.buildDateTime = buildDateTime
  453. } else {
  454. ret.buildDateTime = strconv.FormatInt(time.Now().Unix(), 10)
  455. }
  456. ret.environ.Set("BUILD_DATETIME_FILE", buildDateTimeFile)
  457. if ret.UseRBE() {
  458. for k, v := range getRBEVars(ctx, Config{ret}) {
  459. ret.environ.Set(k, v)
  460. }
  461. }
  462. if ret.BuildFromTextStub() {
  463. // TODO(b/271443071): support hidden api check for from-text stub build
  464. ret.environ.Set("UNSAFE_DISABLE_HIDDENAPI_FLAGS", "true")
  465. }
  466. bpd := ret.BazelMetricsDir()
  467. if err := os.RemoveAll(bpd); err != nil {
  468. ctx.Fatalf("Unable to remove bazel profile directory %q: %v", bpd, err)
  469. }
  470. c := Config{ret}
  471. storeConfigMetrics(ctx, c)
  472. return c
  473. }
  474. // NewBuildActionConfig returns a build configuration based on the build action. The arguments are
  475. // processed based on the build action and extracts any arguments that belongs to the build action.
  476. func NewBuildActionConfig(action BuildAction, dir string, ctx Context, args ...string) Config {
  477. return NewConfig(ctx, getConfigArgs(action, dir, ctx, args)...)
  478. }
  479. // storeConfigMetrics selects a set of configuration information and store in
  480. // the metrics system for further analysis.
  481. func storeConfigMetrics(ctx Context, config Config) {
  482. if ctx.Metrics == nil {
  483. return
  484. }
  485. ctx.Metrics.BuildConfig(buildConfig(config))
  486. s := &smpb.SystemResourceInfo{
  487. TotalPhysicalMemory: proto.Uint64(config.TotalRAM()),
  488. AvailableCpus: proto.Int32(int32(runtime.NumCPU())),
  489. }
  490. ctx.Metrics.SystemResourceInfo(s)
  491. }
  492. func getNinjaWeightListSourceInMetric(s NinjaWeightListSource) *smpb.BuildConfig_NinjaWeightListSource {
  493. switch s {
  494. case NINJA_LOG:
  495. return smpb.BuildConfig_NINJA_LOG.Enum()
  496. case EVENLY_DISTRIBUTED:
  497. return smpb.BuildConfig_EVENLY_DISTRIBUTED.Enum()
  498. case EXTERNAL_FILE:
  499. return smpb.BuildConfig_EXTERNAL_FILE.Enum()
  500. case HINT_FROM_SOONG:
  501. return smpb.BuildConfig_HINT_FROM_SOONG.Enum()
  502. default:
  503. return smpb.BuildConfig_NOT_USED.Enum()
  504. }
  505. }
  506. func buildConfig(config Config) *smpb.BuildConfig {
  507. c := &smpb.BuildConfig{
  508. ForceUseGoma: proto.Bool(config.ForceUseGoma()),
  509. UseGoma: proto.Bool(config.UseGoma()),
  510. UseRbe: proto.Bool(config.UseRBE()),
  511. BazelMixedBuild: proto.Bool(config.BazelBuildEnabled()),
  512. ForceDisableBazelMixedBuild: proto.Bool(config.IsBazelMixedBuildForceDisabled()),
  513. NinjaWeightListSource: getNinjaWeightListSourceInMetric(config.NinjaWeightListSource()),
  514. }
  515. c.Targets = append(c.Targets, config.arguments...)
  516. return c
  517. }
  518. // getConfigArgs processes the command arguments based on the build action and creates a set of new
  519. // arguments to be accepted by Config.
  520. func getConfigArgs(action BuildAction, dir string, ctx Context, args []string) []string {
  521. // The next block of code verifies that the current directory is the root directory of the source
  522. // tree. It then finds the relative path of dir based on the root directory of the source tree
  523. // and verify that dir is inside of the source tree.
  524. checkTopDir(ctx)
  525. topDir, err := os.Getwd()
  526. if err != nil {
  527. ctx.Fatalf("Error retrieving top directory: %v", err)
  528. }
  529. dir, err = filepath.EvalSymlinks(dir)
  530. if err != nil {
  531. ctx.Fatalf("Unable to evaluate symlink of %s: %v", dir, err)
  532. }
  533. dir, err = filepath.Abs(dir)
  534. if err != nil {
  535. ctx.Fatalf("Unable to find absolute path %s: %v", dir, err)
  536. }
  537. relDir, err := filepath.Rel(topDir, dir)
  538. if err != nil {
  539. ctx.Fatalf("Unable to find relative path %s of %s: %v", relDir, topDir, err)
  540. }
  541. // If there are ".." in the path, it's not in the source tree.
  542. if strings.Contains(relDir, "..") {
  543. ctx.Fatalf("Directory %s is not under the source tree %s", dir, topDir)
  544. }
  545. configArgs := args[:]
  546. // If the arguments contains GET-INSTALL-PATH, change the target name prefix from MODULES-IN- to
  547. // GET-INSTALL-PATH-IN- to extract the installation path instead of building the modules.
  548. targetNamePrefix := "MODULES-IN-"
  549. if inList("GET-INSTALL-PATH", configArgs) {
  550. targetNamePrefix = "GET-INSTALL-PATH-IN-"
  551. configArgs = removeFromList("GET-INSTALL-PATH", configArgs)
  552. }
  553. var targets []string
  554. switch action {
  555. case BUILD_MODULES:
  556. // No additional processing is required when building a list of specific modules or all modules.
  557. case BUILD_MODULES_IN_A_DIRECTORY:
  558. // If dir is the root source tree, all the modules are built of the source tree are built so
  559. // no need to find the build file.
  560. if topDir == dir {
  561. break
  562. }
  563. buildFile := findBuildFile(ctx, relDir)
  564. if buildFile == "" {
  565. ctx.Fatalf("Build file not found for %s directory", relDir)
  566. }
  567. targets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)}
  568. case BUILD_MODULES_IN_DIRECTORIES:
  569. newConfigArgs, dirs := splitArgs(configArgs)
  570. configArgs = newConfigArgs
  571. targets = getTargetsFromDirs(ctx, relDir, dirs, targetNamePrefix)
  572. }
  573. // Tidy only override all other specified targets.
  574. tidyOnly := os.Getenv("WITH_TIDY_ONLY")
  575. if tidyOnly == "true" || tidyOnly == "1" {
  576. configArgs = append(configArgs, "tidy_only")
  577. } else {
  578. configArgs = append(configArgs, targets...)
  579. }
  580. return configArgs
  581. }
  582. // convertToTarget replaces "/" to "-" in dir and pre-append the targetNamePrefix to the target name.
  583. func convertToTarget(dir string, targetNamePrefix string) string {
  584. return targetNamePrefix + strings.ReplaceAll(dir, "/", "-")
  585. }
  586. // hasBuildFile returns true if dir contains an Android build file.
  587. func hasBuildFile(ctx Context, dir string) bool {
  588. for _, buildFile := range buildFiles {
  589. _, err := os.Stat(filepath.Join(dir, buildFile))
  590. if err == nil {
  591. return true
  592. }
  593. if !os.IsNotExist(err) {
  594. ctx.Fatalf("Error retrieving the build file stats: %v", err)
  595. }
  596. }
  597. return false
  598. }
  599. // findBuildFile finds a build file (makefile or blueprint file) by looking if there is a build file
  600. // in the current and any sub directory of dir. If a build file is not found, traverse the path
  601. // up by one directory and repeat again until either a build file is found or reached to the root
  602. // source tree. The returned filename of build file is "Android.mk". If one was not found, a blank
  603. // string is returned.
  604. func findBuildFile(ctx Context, dir string) string {
  605. // If the string is empty or ".", assume it is top directory of the source tree.
  606. if dir == "" || dir == "." {
  607. return ""
  608. }
  609. found := false
  610. for buildDir := dir; buildDir != "."; buildDir = filepath.Dir(buildDir) {
  611. err := filepath.Walk(buildDir, func(path string, info os.FileInfo, err error) error {
  612. if err != nil {
  613. return err
  614. }
  615. if found {
  616. return filepath.SkipDir
  617. }
  618. if info.IsDir() {
  619. return nil
  620. }
  621. for _, buildFile := range buildFiles {
  622. if info.Name() == buildFile {
  623. found = true
  624. return filepath.SkipDir
  625. }
  626. }
  627. return nil
  628. })
  629. if err != nil {
  630. ctx.Fatalf("Error finding Android build file: %v", err)
  631. }
  632. if found {
  633. return filepath.Join(buildDir, "Android.mk")
  634. }
  635. }
  636. return ""
  637. }
  638. // splitArgs iterates over the arguments list and splits into two lists: arguments and directories.
  639. func splitArgs(args []string) (newArgs []string, dirs []string) {
  640. specialArgs := map[string]bool{
  641. "showcommands": true,
  642. "snod": true,
  643. "dist": true,
  644. "checkbuild": true,
  645. }
  646. newArgs = []string{}
  647. dirs = []string{}
  648. for _, arg := range args {
  649. // It's a dash argument if it starts with "-" or it's a key=value pair, it's not a directory.
  650. if strings.IndexRune(arg, '-') == 0 || strings.IndexRune(arg, '=') != -1 {
  651. newArgs = append(newArgs, arg)
  652. continue
  653. }
  654. if _, ok := specialArgs[arg]; ok {
  655. newArgs = append(newArgs, arg)
  656. continue
  657. }
  658. dirs = append(dirs, arg)
  659. }
  660. return newArgs, dirs
  661. }
  662. // getTargetsFromDirs iterates over the dirs list and creates a list of targets to build. If a
  663. // directory from the dirs list does not exist, a fatal error is raised. relDir is related to the
  664. // source root tree where the build action command was invoked. Each directory is validated if the
  665. // build file can be found and follows the format "dir1:target1,target2,...". Target is optional.
  666. func getTargetsFromDirs(ctx Context, relDir string, dirs []string, targetNamePrefix string) (targets []string) {
  667. for _, dir := range dirs {
  668. // The directory may have specified specific modules to build. ":" is the separator to separate
  669. // the directory and the list of modules.
  670. s := strings.Split(dir, ":")
  671. l := len(s)
  672. if l > 2 { // more than one ":" was specified.
  673. ctx.Fatalf("%s not in proper directory:target1,target2,... format (\":\" was specified more than once)", dir)
  674. }
  675. dir = filepath.Join(relDir, s[0])
  676. if _, err := os.Stat(dir); err != nil {
  677. ctx.Fatalf("couldn't find directory %s", dir)
  678. }
  679. // Verify that if there are any targets specified after ":". Each target is separated by ",".
  680. var newTargets []string
  681. if l == 2 && s[1] != "" {
  682. newTargets = strings.Split(s[1], ",")
  683. if inList("", newTargets) {
  684. ctx.Fatalf("%s not in proper directory:target1,target2,... format", dir)
  685. }
  686. }
  687. // If there are specified targets to build in dir, an android build file must exist for the one
  688. // shot build. For the non-targets case, find the appropriate build file and build all the
  689. // modules in dir (or the closest one in the dir path).
  690. if len(newTargets) > 0 {
  691. if !hasBuildFile(ctx, dir) {
  692. ctx.Fatalf("Couldn't locate a build file from %s directory", dir)
  693. }
  694. } else {
  695. buildFile := findBuildFile(ctx, dir)
  696. if buildFile == "" {
  697. ctx.Fatalf("Build file not found for %s directory", dir)
  698. }
  699. newTargets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)}
  700. }
  701. targets = append(targets, newTargets...)
  702. }
  703. return targets
  704. }
  705. func (c *configImpl) parseArgs(ctx Context, args []string) {
  706. for i := 0; i < len(args); i++ {
  707. arg := strings.TrimSpace(args[i])
  708. if arg == "showcommands" {
  709. c.verbose = true
  710. } else if arg == "--empty-ninja-file" {
  711. c.emptyNinjaFile = true
  712. } else if arg == "--skip-ninja" {
  713. c.skipNinja = true
  714. } else if arg == "--skip-make" {
  715. // TODO(ccross): deprecate this, it has confusing behaviors. It doesn't run kati,
  716. // but it does run a Kati ninja file if the .kati_enabled marker file was created
  717. // by a previous build.
  718. c.skipConfig = true
  719. c.skipKati = true
  720. } else if arg == "--soong-only" {
  721. c.skipKati = true
  722. c.skipKatiNinja = true
  723. } else if arg == "--config-only" {
  724. c.skipKati = true
  725. c.skipKatiNinja = true
  726. c.skipSoong = true
  727. } else if arg == "--skip-config" {
  728. c.skipConfig = true
  729. } else if arg == "--skip-soong-tests" {
  730. c.skipSoongTests = true
  731. } else if arg == "--skip-metrics-upload" {
  732. c.skipMetricsUpload = true
  733. } else if arg == "--mk-metrics" {
  734. c.reportMkMetrics = true
  735. } else if arg == "--multitree-build" {
  736. c.multitreeBuild = true
  737. } else if arg == "--bazel-mode" {
  738. c.bazelProdMode = true
  739. } else if arg == "--bazel-mode-staging" {
  740. c.bazelStagingMode = true
  741. } else if arg == "--search-api-dir" {
  742. c.searchApiDir = true
  743. } else if strings.HasPrefix(arg, "--ninja_weight_source=") {
  744. source := strings.TrimPrefix(arg, "--ninja_weight_source=")
  745. if source == "ninja_log" {
  746. c.ninjaWeightListSource = NINJA_LOG
  747. } else if source == "evenly_distributed" {
  748. c.ninjaWeightListSource = EVENLY_DISTRIBUTED
  749. } else if source == "not_used" {
  750. c.ninjaWeightListSource = NOT_USED
  751. } else if source == "soong" {
  752. c.ninjaWeightListSource = HINT_FROM_SOONG
  753. } else if strings.HasPrefix(source, "file,") {
  754. c.ninjaWeightListSource = EXTERNAL_FILE
  755. filePath := strings.TrimPrefix(source, "file,")
  756. err := validateNinjaWeightList(filePath)
  757. if err != nil {
  758. ctx.Fatalf("Malformed weight list from %s: %s", filePath, err)
  759. }
  760. _, err = copyFile(filePath, filepath.Join(c.OutDir(), ".ninja_weight_list"))
  761. if err != nil {
  762. ctx.Fatalf("Error to copy ninja weight list from %s: %s", filePath, err)
  763. }
  764. } else {
  765. ctx.Fatalf("unknown option for ninja_weight_source: %s", source)
  766. }
  767. } else if arg == "--build-from-text-stub" {
  768. c.buildFromTextStub = true
  769. } else if strings.HasPrefix(arg, "--build-command=") {
  770. buildCmd := strings.TrimPrefix(arg, "--build-command=")
  771. // remove quotations
  772. buildCmd = strings.TrimPrefix(buildCmd, "\"")
  773. buildCmd = strings.TrimSuffix(buildCmd, "\"")
  774. ctx.Metrics.SetBuildCommand([]string{buildCmd})
  775. } else if strings.HasPrefix(arg, "--bazel-force-enabled-modules=") {
  776. c.bazelForceEnabledModules = strings.TrimPrefix(arg, "--bazel-force-enabled-modules=")
  777. } else if strings.HasPrefix(arg, "--build-started-time-unix-millis=") {
  778. buildTimeStr := strings.TrimPrefix(arg, "--build-started-time-unix-millis=")
  779. val, err := strconv.ParseInt(buildTimeStr, 10, 64)
  780. if err == nil {
  781. c.buildStartedTime = val
  782. } else {
  783. ctx.Fatalf("Error parsing build-time-started-unix-millis", err)
  784. }
  785. } else if arg == "--ensure-allowlist-integrity" {
  786. c.ensureAllowlistIntegrity = true
  787. } else if strings.HasPrefix(arg, "--bazel-exit-code=") {
  788. bazelExitCodeStr := strings.TrimPrefix(arg, "--bazel-exit-code=")
  789. val, err := strconv.Atoi(bazelExitCodeStr)
  790. if err == nil {
  791. c.bazelExitCode = int32(val)
  792. } else {
  793. ctx.Fatalf("Error parsing bazel-exit-code", err)
  794. }
  795. } else if len(arg) > 0 && arg[0] == '-' {
  796. parseArgNum := func(def int) int {
  797. if len(arg) > 2 {
  798. p, err := strconv.ParseUint(arg[2:], 10, 31)
  799. if err != nil {
  800. ctx.Fatalf("Failed to parse %q: %v", arg, err)
  801. }
  802. return int(p)
  803. } else if i+1 < len(args) {
  804. p, err := strconv.ParseUint(args[i+1], 10, 31)
  805. if err == nil {
  806. i++
  807. return int(p)
  808. }
  809. }
  810. return def
  811. }
  812. if len(arg) > 1 && arg[1] == 'j' {
  813. c.parallel = parseArgNum(c.parallel)
  814. } else if len(arg) > 1 && arg[1] == 'k' {
  815. c.keepGoing = parseArgNum(0)
  816. } else {
  817. ctx.Fatalln("Unknown option:", arg)
  818. }
  819. } else if k, v, ok := decodeKeyValue(arg); ok && len(k) > 0 {
  820. if k == "OUT_DIR" {
  821. ctx.Fatalln("OUT_DIR may only be set in the environment, not as a command line option.")
  822. }
  823. c.environ.Set(k, v)
  824. } else if arg == "dist" {
  825. c.dist = true
  826. } else if arg == "json-module-graph" {
  827. c.jsonModuleGraph = true
  828. } else if arg == "bp2build" {
  829. c.bp2build = true
  830. } else if arg == "api_bp2build" {
  831. c.apiBp2build = true
  832. } else if arg == "queryview" {
  833. c.queryview = true
  834. } else if arg == "soong_docs" {
  835. c.soongDocs = true
  836. } else {
  837. if arg == "checkbuild" {
  838. c.checkbuild = true
  839. }
  840. c.arguments = append(c.arguments, arg)
  841. }
  842. }
  843. if (!c.bazelProdMode) && (!c.bazelStagingMode) {
  844. c.bazelProdMode = defaultBazelProdMode(c)
  845. }
  846. }
  847. func validateNinjaWeightList(weightListFilePath string) (err error) {
  848. data, err := os.ReadFile(weightListFilePath)
  849. if err != nil {
  850. return
  851. }
  852. lines := strings.Split(strings.TrimSpace(string(data)), "\n")
  853. for _, line := range lines {
  854. fields := strings.Split(line, ",")
  855. if len(fields) != 2 {
  856. return fmt.Errorf("wrong format, each line should have two fields, but '%s'", line)
  857. }
  858. _, err = strconv.Atoi(fields[1])
  859. if err != nil {
  860. return
  861. }
  862. }
  863. return
  864. }
  865. func (c *configImpl) configureLocale(ctx Context) {
  866. cmd := Command(ctx, Config{c}, "locale", "locale", "-a")
  867. output, err := cmd.Output()
  868. var locales []string
  869. if err == nil {
  870. locales = strings.Split(string(output), "\n")
  871. } else {
  872. // If we're unable to list the locales, let's assume en_US.UTF-8
  873. locales = []string{"en_US.UTF-8"}
  874. ctx.Verbosef("Failed to list locales (%q), falling back to %q", err, locales)
  875. }
  876. // gettext uses LANGUAGE, which is passed directly through
  877. // For LANG and LC_*, only preserve the evaluated version of
  878. // LC_MESSAGES
  879. userLang := ""
  880. if lc_all, ok := c.environ.Get("LC_ALL"); ok {
  881. userLang = lc_all
  882. } else if lc_messages, ok := c.environ.Get("LC_MESSAGES"); ok {
  883. userLang = lc_messages
  884. } else if lang, ok := c.environ.Get("LANG"); ok {
  885. userLang = lang
  886. }
  887. c.environ.UnsetWithPrefix("LC_")
  888. if userLang != "" {
  889. c.environ.Set("LC_MESSAGES", userLang)
  890. }
  891. // The for LANG, use C.UTF-8 if it exists (Debian currently, proposed
  892. // for others)
  893. if inList("C.UTF-8", locales) {
  894. c.environ.Set("LANG", "C.UTF-8")
  895. } else if inList("C.utf8", locales) {
  896. // These normalize to the same thing
  897. c.environ.Set("LANG", "C.UTF-8")
  898. } else if inList("en_US.UTF-8", locales) {
  899. c.environ.Set("LANG", "en_US.UTF-8")
  900. } else if inList("en_US.utf8", locales) {
  901. // These normalize to the same thing
  902. c.environ.Set("LANG", "en_US.UTF-8")
  903. } else {
  904. ctx.Fatalln("System doesn't support either C.UTF-8 or en_US.UTF-8")
  905. }
  906. }
  907. func (c *configImpl) Environment() *Environment {
  908. return c.environ
  909. }
  910. func (c *configImpl) Arguments() []string {
  911. return c.arguments
  912. }
  913. func (c *configImpl) SoongBuildInvocationNeeded() bool {
  914. if len(c.Arguments()) > 0 {
  915. // Explicit targets requested that are not special targets like b2pbuild
  916. // or the JSON module graph
  917. return true
  918. }
  919. if !c.JsonModuleGraph() && !c.Bp2Build() && !c.Queryview() && !c.SoongDocs() && !c.ApiBp2build() {
  920. // Command line was empty, the default Ninja target is built
  921. return true
  922. }
  923. // bp2build + dist may be used to dist bp2build logs but does not require SoongBuildInvocation
  924. if c.Dist() && !c.Bp2Build() {
  925. return true
  926. }
  927. // build.ninja doesn't need to be generated
  928. return false
  929. }
  930. func (c *configImpl) OutDir() string {
  931. if outDir, ok := c.environ.Get("OUT_DIR"); ok {
  932. return outDir
  933. }
  934. return "out"
  935. }
  936. func (c *configImpl) DistDir() string {
  937. return c.distDir
  938. }
  939. func (c *configImpl) RealDistDir() string {
  940. return c.distDir
  941. }
  942. func (c *configImpl) NinjaArgs() []string {
  943. if c.skipKati {
  944. return c.arguments
  945. }
  946. return c.ninjaArgs
  947. }
  948. func (c *configImpl) BazelOutDir() string {
  949. return filepath.Join(c.OutDir(), "bazel")
  950. }
  951. func (c *configImpl) bazelOutputBase() string {
  952. return filepath.Join(c.BazelOutDir(), "output")
  953. }
  954. func (c *configImpl) SoongOutDir() string {
  955. return filepath.Join(c.OutDir(), "soong")
  956. }
  957. func (c *configImpl) ApiSurfacesOutDir() string {
  958. return filepath.Join(c.OutDir(), "api_surfaces")
  959. }
  960. func (c *configImpl) PrebuiltOS() string {
  961. switch runtime.GOOS {
  962. case "linux":
  963. return "linux-x86"
  964. case "darwin":
  965. return "darwin-x86"
  966. default:
  967. panic("Unknown GOOS")
  968. }
  969. }
  970. func (c *configImpl) HostToolDir() string {
  971. if c.SkipKatiNinja() {
  972. return filepath.Join(c.SoongOutDir(), "host", c.PrebuiltOS(), "bin")
  973. } else {
  974. return filepath.Join(c.OutDir(), "host", c.PrebuiltOS(), "bin")
  975. }
  976. }
  977. func (c *configImpl) NamedGlobFile(name string) string {
  978. return shared.JoinPath(c.SoongOutDir(), "globs-"+name+".ninja")
  979. }
  980. func (c *configImpl) UsedEnvFile(tag string) string {
  981. if v, ok := c.environ.Get("TARGET_PRODUCT"); ok {
  982. return shared.JoinPath(c.SoongOutDir(), usedEnvFile+"."+v+"."+tag)
  983. }
  984. return shared.JoinPath(c.SoongOutDir(), usedEnvFile+"."+tag)
  985. }
  986. func (c *configImpl) Bp2BuildFilesMarkerFile() string {
  987. return shared.JoinPath(c.SoongOutDir(), "bp2build_files_marker")
  988. }
  989. func (c *configImpl) Bp2BuildWorkspaceMarkerFile() string {
  990. return shared.JoinPath(c.SoongOutDir(), "bp2build_workspace_marker")
  991. }
  992. func (c *configImpl) SoongDocsHtml() string {
  993. return shared.JoinPath(c.SoongOutDir(), "docs/soong_build.html")
  994. }
  995. func (c *configImpl) QueryviewMarkerFile() string {
  996. return shared.JoinPath(c.SoongOutDir(), "queryview.marker")
  997. }
  998. func (c *configImpl) ApiBp2buildMarkerFile() string {
  999. return shared.JoinPath(c.SoongOutDir(), "api_bp2build.marker")
  1000. }
  1001. func (c *configImpl) ModuleGraphFile() string {
  1002. return shared.JoinPath(c.SoongOutDir(), "module-graph.json")
  1003. }
  1004. func (c *configImpl) ModuleActionsFile() string {
  1005. return shared.JoinPath(c.SoongOutDir(), "module-actions.json")
  1006. }
  1007. func (c *configImpl) TempDir() string {
  1008. return shared.TempDirForOutDir(c.SoongOutDir())
  1009. }
  1010. func (c *configImpl) FileListDir() string {
  1011. return filepath.Join(c.OutDir(), ".module_paths")
  1012. }
  1013. func (c *configImpl) KatiSuffix() string {
  1014. if c.katiSuffix != "" {
  1015. return c.katiSuffix
  1016. }
  1017. panic("SetKatiSuffix has not been called")
  1018. }
  1019. // Checkbuild returns true if "checkbuild" was one of the build goals, which means that the
  1020. // user is interested in additional checks at the expense of build time.
  1021. func (c *configImpl) Checkbuild() bool {
  1022. return c.checkbuild
  1023. }
  1024. func (c *configImpl) Dist() bool {
  1025. return c.dist
  1026. }
  1027. func (c *configImpl) JsonModuleGraph() bool {
  1028. return c.jsonModuleGraph
  1029. }
  1030. func (c *configImpl) Bp2Build() bool {
  1031. return c.bp2build
  1032. }
  1033. func (c *configImpl) ApiBp2build() bool {
  1034. return c.apiBp2build
  1035. }
  1036. func (c *configImpl) Queryview() bool {
  1037. return c.queryview
  1038. }
  1039. func (c *configImpl) SoongDocs() bool {
  1040. return c.soongDocs
  1041. }
  1042. func (c *configImpl) IsVerbose() bool {
  1043. return c.verbose
  1044. }
  1045. func (c *configImpl) MultitreeBuild() bool {
  1046. return c.multitreeBuild
  1047. }
  1048. func (c *configImpl) NinjaWeightListSource() NinjaWeightListSource {
  1049. return c.ninjaWeightListSource
  1050. }
  1051. func (c *configImpl) SkipKati() bool {
  1052. return c.skipKati
  1053. }
  1054. func (c *configImpl) SkipKatiNinja() bool {
  1055. return c.skipKatiNinja
  1056. }
  1057. func (c *configImpl) SkipSoong() bool {
  1058. return c.skipSoong
  1059. }
  1060. func (c *configImpl) SkipNinja() bool {
  1061. return c.skipNinja
  1062. }
  1063. func (c *configImpl) SetSkipNinja(v bool) {
  1064. c.skipNinja = v
  1065. }
  1066. func (c *configImpl) SkipConfig() bool {
  1067. return c.skipConfig
  1068. }
  1069. func (c *configImpl) BuildFromTextStub() bool {
  1070. return c.buildFromTextStub
  1071. }
  1072. func (c *configImpl) TargetProduct() string {
  1073. if v, ok := c.environ.Get("TARGET_PRODUCT"); ok {
  1074. return v
  1075. }
  1076. panic("TARGET_PRODUCT is not defined")
  1077. }
  1078. func (c *configImpl) TargetProductOrErr() (string, error) {
  1079. if v, ok := c.environ.Get("TARGET_PRODUCT"); ok {
  1080. return v, nil
  1081. }
  1082. return "", fmt.Errorf("TARGET_PRODUCT is not defined")
  1083. }
  1084. func (c *configImpl) TargetDevice() string {
  1085. return c.targetDevice
  1086. }
  1087. func (c *configImpl) SetTargetDevice(device string) {
  1088. c.targetDevice = device
  1089. }
  1090. func (c *configImpl) TargetBuildVariant() string {
  1091. if v, ok := c.environ.Get("TARGET_BUILD_VARIANT"); ok {
  1092. return v
  1093. }
  1094. panic("TARGET_BUILD_VARIANT is not defined")
  1095. }
  1096. func (c *configImpl) KatiArgs() []string {
  1097. return c.katiArgs
  1098. }
  1099. func (c *configImpl) Parallel() int {
  1100. return c.parallel
  1101. }
  1102. func (c *configImpl) GetSourceRootDirs() []string {
  1103. return c.sourceRootDirs
  1104. }
  1105. func (c *configImpl) SetSourceRootDirs(i []string) {
  1106. c.sourceRootDirs = i
  1107. }
  1108. func (c *configImpl) GetIncludeTags() []string {
  1109. return c.includeTags
  1110. }
  1111. func (c *configImpl) SetIncludeTags(i []string) {
  1112. c.includeTags = i
  1113. }
  1114. func (c *configImpl) GetLogsPrefix() string {
  1115. return c.logsPrefix
  1116. }
  1117. func (c *configImpl) SetLogsPrefix(prefix string) {
  1118. c.logsPrefix = prefix
  1119. }
  1120. func (c *configImpl) HighmemParallel() int {
  1121. if i, ok := c.environ.GetInt("NINJA_HIGHMEM_NUM_JOBS"); ok {
  1122. return i
  1123. }
  1124. const minMemPerHighmemProcess = 8 * 1024 * 1024 * 1024
  1125. parallel := c.Parallel()
  1126. if c.UseRemoteBuild() {
  1127. // Ninja doesn't support nested pools, and when remote builds are enabled the total ninja parallelism
  1128. // is set very high (i.e. 500). Using a large value here would cause the total number of running jobs
  1129. // to be the sum of the sizes of the local and highmem pools, which will cause extra CPU contention.
  1130. // Return 1/16th of the size of the local pool, rounding up.
  1131. return (parallel + 15) / 16
  1132. } else if c.totalRAM == 0 {
  1133. // Couldn't detect the total RAM, don't restrict highmem processes.
  1134. return parallel
  1135. } else if c.totalRAM <= 16*1024*1024*1024 {
  1136. // Less than 16GB of ram, restrict to 1 highmem processes
  1137. return 1
  1138. } else if c.totalRAM <= 32*1024*1024*1024 {
  1139. // Less than 32GB of ram, restrict to 2 highmem processes
  1140. return 2
  1141. } else if p := int(c.totalRAM / minMemPerHighmemProcess); p < parallel {
  1142. // If less than 8GB total RAM per process, reduce the number of highmem processes
  1143. return p
  1144. }
  1145. // No restriction on highmem processes
  1146. return parallel
  1147. }
  1148. func (c *configImpl) TotalRAM() uint64 {
  1149. return c.totalRAM
  1150. }
  1151. // ForceUseGoma determines whether we should override Goma deprecation
  1152. // and use Goma for the current build or not.
  1153. func (c *configImpl) ForceUseGoma() bool {
  1154. if v, ok := c.environ.Get("FORCE_USE_GOMA"); ok {
  1155. v = strings.TrimSpace(v)
  1156. if v != "" && v != "false" {
  1157. return true
  1158. }
  1159. }
  1160. return false
  1161. }
  1162. func (c *configImpl) UseGoma() bool {
  1163. if v, ok := c.environ.Get("USE_GOMA"); ok {
  1164. v = strings.TrimSpace(v)
  1165. if v != "" && v != "false" {
  1166. return true
  1167. }
  1168. }
  1169. return false
  1170. }
  1171. func (c *configImpl) StartGoma() bool {
  1172. if !c.UseGoma() {
  1173. return false
  1174. }
  1175. if v, ok := c.environ.Get("NOSTART_GOMA"); ok {
  1176. v = strings.TrimSpace(v)
  1177. if v != "" && v != "false" {
  1178. return false
  1179. }
  1180. }
  1181. return true
  1182. }
  1183. func (c *configImpl) UseRBE() bool {
  1184. if v, ok := c.Environment().Get("USE_RBE"); ok {
  1185. v = strings.TrimSpace(v)
  1186. if v != "" && v != "false" {
  1187. return true
  1188. }
  1189. }
  1190. return false
  1191. }
  1192. func (c *configImpl) BazelBuildEnabled() bool {
  1193. return c.bazelProdMode || c.bazelStagingMode
  1194. }
  1195. func (c *configImpl) StartRBE() bool {
  1196. if !c.UseRBE() {
  1197. return false
  1198. }
  1199. if v, ok := c.environ.Get("NOSTART_RBE"); ok {
  1200. v = strings.TrimSpace(v)
  1201. if v != "" && v != "false" {
  1202. return false
  1203. }
  1204. }
  1205. return true
  1206. }
  1207. func (c *configImpl) rbeProxyLogsDir() string {
  1208. for _, f := range []string{"RBE_proxy_log_dir", "FLAG_output_dir"} {
  1209. if v, ok := c.environ.Get(f); ok {
  1210. return v
  1211. }
  1212. }
  1213. buildTmpDir := shared.TempDirForOutDir(c.SoongOutDir())
  1214. return filepath.Join(buildTmpDir, "rbe")
  1215. }
  1216. func (c *configImpl) rbeCacheDir() string {
  1217. for _, f := range []string{"RBE_cache_dir", "FLAG_cache_dir"} {
  1218. if v, ok := c.environ.Get(f); ok {
  1219. return v
  1220. }
  1221. }
  1222. return shared.JoinPath(c.SoongOutDir(), "rbe")
  1223. }
  1224. func (c *configImpl) shouldCleanupRBELogsDir() bool {
  1225. // Perform a log directory cleanup only when the log directory
  1226. // is auto created by the build rather than user-specified.
  1227. for _, f := range []string{"RBE_proxy_log_dir", "FLAG_output_dir"} {
  1228. if _, ok := c.environ.Get(f); ok {
  1229. return false
  1230. }
  1231. }
  1232. return true
  1233. }
  1234. func (c *configImpl) rbeExecRoot() string {
  1235. for _, f := range []string{"RBE_exec_root", "FLAG_exec_root"} {
  1236. if v, ok := c.environ.Get(f); ok {
  1237. return v
  1238. }
  1239. }
  1240. wd, err := os.Getwd()
  1241. if err != nil {
  1242. return ""
  1243. }
  1244. return wd
  1245. }
  1246. func (c *configImpl) rbeDir() string {
  1247. if v, ok := c.environ.Get("RBE_DIR"); ok {
  1248. return v
  1249. }
  1250. return "prebuilts/remoteexecution-client/live/"
  1251. }
  1252. func (c *configImpl) rbeReproxy() string {
  1253. for _, f := range []string{"RBE_re_proxy", "FLAG_re_proxy"} {
  1254. if v, ok := c.environ.Get(f); ok {
  1255. return v
  1256. }
  1257. }
  1258. return filepath.Join(c.rbeDir(), "reproxy")
  1259. }
  1260. func (c *configImpl) rbeAuth() (string, string) {
  1261. credFlags := []string{
  1262. "use_application_default_credentials",
  1263. "use_gce_credentials",
  1264. "credential_file",
  1265. "use_google_prod_creds",
  1266. }
  1267. for _, cf := range credFlags {
  1268. for _, f := range []string{"RBE_" + cf, "FLAG_" + cf} {
  1269. if v, ok := c.environ.Get(f); ok {
  1270. v = strings.TrimSpace(v)
  1271. if v != "" && v != "false" && v != "0" {
  1272. return "RBE_" + cf, v
  1273. }
  1274. }
  1275. }
  1276. }
  1277. return "RBE_use_application_default_credentials", "true"
  1278. }
  1279. func (c *configImpl) rbeSockAddr(dir string) (string, error) {
  1280. maxNameLen := len(syscall.RawSockaddrUnix{}.Path)
  1281. base := fmt.Sprintf("reproxy_%v.sock", rbeRandPrefix)
  1282. name := filepath.Join(dir, base)
  1283. if len(name) < maxNameLen {
  1284. return name, nil
  1285. }
  1286. name = filepath.Join("/tmp", base)
  1287. if len(name) < maxNameLen {
  1288. return name, nil
  1289. }
  1290. return "", fmt.Errorf("cannot generate a proxy socket address shorter than the limit of %v", maxNameLen)
  1291. }
  1292. // IsGooglerEnvironment returns true if the current build is running
  1293. // on a Google developer machine and false otherwise.
  1294. func (c *configImpl) IsGooglerEnvironment() bool {
  1295. cf := "ANDROID_BUILD_ENVIRONMENT_CONFIG"
  1296. if v, ok := c.environ.Get(cf); ok {
  1297. return v == "googler"
  1298. }
  1299. return false
  1300. }
  1301. // GoogleProdCredsExist determine whether credentials exist on the
  1302. // Googler machine to use remote execution.
  1303. func (c *configImpl) GoogleProdCredsExist() bool {
  1304. if googleProdCredsExistCache {
  1305. return googleProdCredsExistCache
  1306. }
  1307. if _, err := exec.Command("/usr/bin/gcertstatus").Output(); err != nil {
  1308. return false
  1309. }
  1310. googleProdCredsExistCache = true
  1311. return true
  1312. }
  1313. // UseRemoteBuild indicates whether to use a remote build acceleration system
  1314. // to speed up the build.
  1315. func (c *configImpl) UseRemoteBuild() bool {
  1316. return c.UseGoma() || c.UseRBE()
  1317. }
  1318. // StubbyExists checks whether the stubby binary exists on the machine running
  1319. // the build.
  1320. func (c *configImpl) StubbyExists() bool {
  1321. if _, err := exec.LookPath("stubby"); err != nil {
  1322. return false
  1323. }
  1324. return true
  1325. }
  1326. // RemoteParallel controls how many remote jobs (i.e., commands which contain
  1327. // gomacc) are run in parallel. Note the parallelism of all other jobs is
  1328. // still limited by Parallel()
  1329. func (c *configImpl) RemoteParallel() int {
  1330. if !c.UseRemoteBuild() {
  1331. return 0
  1332. }
  1333. if i, ok := c.environ.GetInt("NINJA_REMOTE_NUM_JOBS"); ok {
  1334. return i
  1335. }
  1336. return 500
  1337. }
  1338. func (c *configImpl) SetKatiArgs(args []string) {
  1339. c.katiArgs = args
  1340. }
  1341. func (c *configImpl) SetNinjaArgs(args []string) {
  1342. c.ninjaArgs = args
  1343. }
  1344. func (c *configImpl) SetKatiSuffix(suffix string) {
  1345. c.katiSuffix = suffix
  1346. }
  1347. func (c *configImpl) LastKatiSuffixFile() string {
  1348. return filepath.Join(c.OutDir(), "last_kati_suffix")
  1349. }
  1350. func (c *configImpl) HasKatiSuffix() bool {
  1351. return c.katiSuffix != ""
  1352. }
  1353. func (c *configImpl) KatiEnvFile() string {
  1354. return filepath.Join(c.OutDir(), "env"+c.KatiSuffix()+".sh")
  1355. }
  1356. func (c *configImpl) KatiBuildNinjaFile() string {
  1357. return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiBuildSuffix+".ninja")
  1358. }
  1359. func (c *configImpl) KatiPackageNinjaFile() string {
  1360. return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiPackageSuffix+".ninja")
  1361. }
  1362. func (c *configImpl) SoongVarsFile() string {
  1363. targetProduct, err := c.TargetProductOrErr()
  1364. if err != nil {
  1365. return filepath.Join(c.SoongOutDir(), "soong.variables")
  1366. } else {
  1367. return filepath.Join(c.SoongOutDir(), "soong."+targetProduct+".variables")
  1368. }
  1369. }
  1370. func (c *configImpl) SoongNinjaFile() string {
  1371. targetProduct, err := c.TargetProductOrErr()
  1372. if err != nil {
  1373. return filepath.Join(c.SoongOutDir(), "build.ninja")
  1374. } else {
  1375. return filepath.Join(c.SoongOutDir(), "build."+targetProduct+".ninja")
  1376. }
  1377. }
  1378. func (c *configImpl) CombinedNinjaFile() string {
  1379. if c.katiSuffix == "" {
  1380. return filepath.Join(c.OutDir(), "combined.ninja")
  1381. }
  1382. return filepath.Join(c.OutDir(), "combined"+c.KatiSuffix()+".ninja")
  1383. }
  1384. func (c *configImpl) SoongAndroidMk() string {
  1385. return filepath.Join(c.SoongOutDir(), "Android-"+c.TargetProduct()+".mk")
  1386. }
  1387. func (c *configImpl) SoongMakeVarsMk() string {
  1388. return filepath.Join(c.SoongOutDir(), "make_vars-"+c.TargetProduct()+".mk")
  1389. }
  1390. func (c *configImpl) ProductOut() string {
  1391. return filepath.Join(c.OutDir(), "target", "product", c.TargetDevice())
  1392. }
  1393. func (c *configImpl) DevicePreviousProductConfig() string {
  1394. return filepath.Join(c.ProductOut(), "previous_build_config.mk")
  1395. }
  1396. func (c *configImpl) KatiPackageMkDir() string {
  1397. return filepath.Join(c.ProductOut(), "obj", "CONFIG", "kati_packaging")
  1398. }
  1399. func (c *configImpl) hostOutRoot() string {
  1400. return filepath.Join(c.OutDir(), "host")
  1401. }
  1402. func (c *configImpl) HostOut() string {
  1403. return filepath.Join(c.hostOutRoot(), c.HostPrebuiltTag())
  1404. }
  1405. // This probably needs to be multi-valued, so not exporting it for now
  1406. func (c *configImpl) hostCrossOut() string {
  1407. if runtime.GOOS == "linux" {
  1408. return filepath.Join(c.hostOutRoot(), "windows-x86")
  1409. } else {
  1410. return ""
  1411. }
  1412. }
  1413. func (c *configImpl) HostPrebuiltTag() string {
  1414. if runtime.GOOS == "linux" {
  1415. return "linux-x86"
  1416. } else if runtime.GOOS == "darwin" {
  1417. return "darwin-x86"
  1418. } else {
  1419. panic("Unsupported OS")
  1420. }
  1421. }
  1422. func (c *configImpl) PrebuiltBuildTool(name string) string {
  1423. if v, ok := c.environ.Get("SANITIZE_HOST"); ok {
  1424. if sanitize := strings.Fields(v); inList("address", sanitize) {
  1425. asan := filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "asan/bin", name)
  1426. if _, err := os.Stat(asan); err == nil {
  1427. return asan
  1428. }
  1429. }
  1430. }
  1431. return filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "bin", name)
  1432. }
  1433. func (c *configImpl) SetBuildBrokenDupRules(val bool) {
  1434. c.brokenDupRules = val
  1435. }
  1436. func (c *configImpl) BuildBrokenDupRules() bool {
  1437. return c.brokenDupRules
  1438. }
  1439. func (c *configImpl) SetBuildBrokenUsesNetwork(val bool) {
  1440. c.brokenUsesNetwork = val
  1441. }
  1442. func (c *configImpl) BuildBrokenUsesNetwork() bool {
  1443. return c.brokenUsesNetwork
  1444. }
  1445. func (c *configImpl) SetBuildBrokenNinjaUsesEnvVars(val []string) {
  1446. c.brokenNinjaEnvVars = val
  1447. }
  1448. func (c *configImpl) BuildBrokenNinjaUsesEnvVars() []string {
  1449. return c.brokenNinjaEnvVars
  1450. }
  1451. func (c *configImpl) SetTargetDeviceDir(dir string) {
  1452. c.targetDeviceDir = dir
  1453. }
  1454. func (c *configImpl) TargetDeviceDir() string {
  1455. return c.targetDeviceDir
  1456. }
  1457. func (c *configImpl) BuildDateTime() string {
  1458. return c.buildDateTime
  1459. }
  1460. func (c *configImpl) MetricsUploaderApp() string {
  1461. return c.metricsUploader
  1462. }
  1463. // LogsDir returns the absolute path to the logs directory where build log and
  1464. // metrics files are located. By default, the logs directory is the out
  1465. // directory. If the argument dist is specified, the logs directory
  1466. // is <dist_dir>/logs.
  1467. func (c *configImpl) LogsDir() string {
  1468. dir := c.OutDir()
  1469. if c.Dist() {
  1470. // Always write logs to the real dist dir, even if Bazel is using a rigged dist dir for other files
  1471. dir = filepath.Join(c.RealDistDir(), "logs")
  1472. }
  1473. absDir, err := filepath.Abs(dir)
  1474. if err != nil {
  1475. fmt.Fprintf(os.Stderr, "\nError making log dir '%s' absolute: %s\n", dir, err.Error())
  1476. os.Exit(1)
  1477. }
  1478. return absDir
  1479. }
  1480. // BazelMetricsDir returns the <logs dir>/bazel_metrics directory
  1481. // where the bazel profiles are located.
  1482. func (c *configImpl) BazelMetricsDir() string {
  1483. return filepath.Join(c.LogsDir(), "bazel_metrics")
  1484. }
  1485. // MkFileMetrics returns the file path for make-related metrics.
  1486. func (c *configImpl) MkMetrics() string {
  1487. return filepath.Join(c.LogsDir(), "mk_metrics.pb")
  1488. }
  1489. func (c *configImpl) SetEmptyNinjaFile(v bool) {
  1490. c.emptyNinjaFile = v
  1491. }
  1492. func (c *configImpl) EmptyNinjaFile() bool {
  1493. return c.emptyNinjaFile
  1494. }
  1495. func (c *configImpl) IsBazelMixedBuildForceDisabled() bool {
  1496. return c.Environment().IsEnvTrue("BUILD_BROKEN_DISABLE_BAZEL")
  1497. }
  1498. func (c *configImpl) IsPersistentBazelEnabled() bool {
  1499. return c.Environment().IsEnvTrue("USE_PERSISTENT_BAZEL")
  1500. }
  1501. // GetBazeliskBazelVersion returns the Bazel version to use for this build,
  1502. // or the empty string if the current canonical prod Bazel should be used.
  1503. // This environment variable should only be set to debug the build system.
  1504. // The Bazel version, if set, will be passed to Bazelisk, and Bazelisk will
  1505. // handle downloading and invoking the correct Bazel binary.
  1506. func (c *configImpl) GetBazeliskBazelVersion() string {
  1507. value, _ := c.Environment().Get("USE_BAZEL_VERSION")
  1508. return value
  1509. }
  1510. func (c *configImpl) BazelModulesForceEnabledByFlag() string {
  1511. return c.bazelForceEnabledModules
  1512. }
  1513. func (c *configImpl) SkipMetricsUpload() bool {
  1514. return c.skipMetricsUpload
  1515. }
  1516. func (c *configImpl) EnsureAllowlistIntegrity() bool {
  1517. return c.ensureAllowlistIntegrity
  1518. }
  1519. // Returns a Time object if one was passed via a command-line flag.
  1520. // Otherwise returns the passed default.
  1521. func (c *configImpl) BuildStartedTimeOrDefault(defaultTime time.Time) time.Time {
  1522. if c.buildStartedTime == 0 {
  1523. return defaultTime
  1524. }
  1525. return time.UnixMilli(c.buildStartedTime)
  1526. }
  1527. func (c *configImpl) BazelExitCode() int32 {
  1528. return c.bazelExitCode
  1529. }
  1530. func GetMetricsUploader(topDir string, env *Environment) string {
  1531. if p, ok := env.Get("METRICS_UPLOADER"); ok {
  1532. metricsUploader := filepath.Join(topDir, p)
  1533. if _, err := os.Stat(metricsUploader); err == nil {
  1534. return metricsUploader
  1535. }
  1536. }
  1537. return ""
  1538. }