config.go 49 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759
  1. // Copyright 2017 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package build
  15. import (
  16. "context"
  17. "encoding/json"
  18. "fmt"
  19. "io/ioutil"
  20. "math/rand"
  21. "os"
  22. "os/exec"
  23. "path/filepath"
  24. "runtime"
  25. "strconv"
  26. "strings"
  27. "syscall"
  28. "time"
  29. "android/soong/shared"
  30. "google.golang.org/protobuf/proto"
  31. smpb "android/soong/ui/metrics/metrics_proto"
  32. )
  33. const (
  34. envConfigDir = "vendor/google/tools/soong_config"
  35. jsonSuffix = "json"
  36. configFetcher = "vendor/google/tools/soong/expconfigfetcher"
  37. envConfigFetchTimeout = 10 * time.Second
  38. )
  39. var (
  40. rbeRandPrefix int
  41. googleProdCredsExistCache bool
  42. )
  43. func init() {
  44. rand.Seed(time.Now().UnixNano())
  45. rbeRandPrefix = rand.Intn(1000)
  46. }
  47. type Config struct{ *configImpl }
  48. type configImpl struct {
  49. // Some targets that are implemented in soong_build
  50. // (bp2build, json-module-graph) are not here and have their own bits below.
  51. arguments []string
  52. goma bool
  53. environ *Environment
  54. distDir string
  55. buildDateTime string
  56. logsPrefix string
  57. // From the arguments
  58. parallel int
  59. keepGoing int
  60. verbose bool
  61. checkbuild bool
  62. dist bool
  63. jsonModuleGraph bool
  64. apiBp2build bool // Generate BUILD files for Soong modules that contribute APIs
  65. bp2build bool
  66. queryview bool
  67. reportMkMetrics bool // Collect and report mk2bp migration progress metrics.
  68. soongDocs bool
  69. multitreeBuild bool // This is a multitree build.
  70. skipConfig bool
  71. skipKati bool
  72. skipKatiNinja bool
  73. skipSoong bool
  74. skipNinja bool
  75. skipSoongTests bool
  76. searchApiDir bool // Scan the Android.bp files generated in out/api_surfaces
  77. skipMetricsUpload bool
  78. buildStartedTime int64 // For metrics-upload-only - manually specify a build-started time
  79. buildFromTextStub bool
  80. ensureAllowlistIntegrity bool // For CI builds - make sure modules are mixed-built
  81. bazelExitCode int32 // For b-runs - necessary for updating NonZeroExit
  82. // From the product config
  83. katiArgs []string
  84. ninjaArgs []string
  85. katiSuffix string
  86. targetDevice string
  87. targetDeviceDir string
  88. sandboxConfig *SandboxConfig
  89. // Autodetected
  90. totalRAM uint64
  91. brokenDupRules bool
  92. brokenUsesNetwork bool
  93. brokenNinjaEnvVars []string
  94. pathReplaced bool
  95. bazelProdMode bool
  96. bazelDevMode bool
  97. bazelStagingMode bool
  98. // Set by multiproduct_kati
  99. emptyNinjaFile bool
  100. metricsUploader string
  101. bazelForceEnabledModules string
  102. includeTags []string
  103. sourceRootDirs []string
  104. // Data source to write ninja weight list
  105. ninjaWeightListSource NinjaWeightListSource
  106. }
  107. type NinjaWeightListSource uint
  108. const (
  109. // ninja doesn't use weight list.
  110. NOT_USED NinjaWeightListSource = iota
  111. // ninja uses weight list based on previous builds by ninja log
  112. NINJA_LOG
  113. // ninja thinks every task has the same weight.
  114. EVENLY_DISTRIBUTED
  115. // ninja uses an external custom weight list
  116. EXTERNAL_FILE
  117. // ninja uses a prioritized module list from Soong
  118. HINT_FROM_SOONG
  119. )
  120. const srcDirFileCheck = "build/soong/root.bp"
  121. var buildFiles = []string{"Android.mk", "Android.bp"}
  122. type BuildAction uint
  123. const (
  124. // Builds all of the modules and their dependencies of a specified directory, relative to the root
  125. // directory of the source tree.
  126. BUILD_MODULES_IN_A_DIRECTORY BuildAction = iota
  127. // Builds all of the modules and their dependencies of a list of specified directories. All specified
  128. // directories are relative to the root directory of the source tree.
  129. BUILD_MODULES_IN_DIRECTORIES
  130. // Build a list of specified modules. If none was specified, simply build the whole source tree.
  131. BUILD_MODULES
  132. )
  133. // checkTopDir validates that the current directory is at the root directory of the source tree.
  134. func checkTopDir(ctx Context) {
  135. if _, err := os.Stat(srcDirFileCheck); err != nil {
  136. if os.IsNotExist(err) {
  137. ctx.Fatalf("Current working directory must be the source tree. %q not found.", srcDirFileCheck)
  138. }
  139. ctx.Fatalln("Error verifying tree state:", err)
  140. }
  141. }
  142. // fetchEnvConfig optionally fetches a configuration file that can then subsequently be
  143. // loaded into Soong environment to control certain aspects of build behavior (e.g., enabling RBE).
  144. // If a configuration file already exists on disk, the fetch is run in the background
  145. // so as to NOT block the rest of the build execution.
  146. func fetchEnvConfig(ctx Context, config *configImpl, envConfigName string) error {
  147. configName := envConfigName + "." + jsonSuffix
  148. expConfigFetcher := &smpb.ExpConfigFetcher{Filename: &configName}
  149. defer func() {
  150. ctx.Metrics.ExpConfigFetcher(expConfigFetcher)
  151. }()
  152. if !config.GoogleProdCredsExist() {
  153. status := smpb.ExpConfigFetcher_MISSING_GCERT
  154. expConfigFetcher.Status = &status
  155. return nil
  156. }
  157. s, err := os.Stat(configFetcher)
  158. if err != nil {
  159. if os.IsNotExist(err) {
  160. return nil
  161. }
  162. return err
  163. }
  164. if s.Mode()&0111 == 0 {
  165. status := smpb.ExpConfigFetcher_ERROR
  166. expConfigFetcher.Status = &status
  167. return fmt.Errorf("configuration fetcher binary %v is not executable: %v", configFetcher, s.Mode())
  168. }
  169. configExists := false
  170. outConfigFilePath := filepath.Join(config.OutDir(), configName)
  171. if _, err := os.Stat(outConfigFilePath); err == nil {
  172. configExists = true
  173. }
  174. tCtx, cancel := context.WithTimeout(ctx, envConfigFetchTimeout)
  175. fetchStart := time.Now()
  176. cmd := exec.CommandContext(tCtx, configFetcher, "-output_config_dir", config.OutDir(),
  177. "-output_config_name", configName)
  178. if err := cmd.Start(); err != nil {
  179. status := smpb.ExpConfigFetcher_ERROR
  180. expConfigFetcher.Status = &status
  181. return err
  182. }
  183. fetchCfg := func() error {
  184. if err := cmd.Wait(); err != nil {
  185. status := smpb.ExpConfigFetcher_ERROR
  186. expConfigFetcher.Status = &status
  187. return err
  188. }
  189. fetchEnd := time.Now()
  190. expConfigFetcher.Micros = proto.Uint64(uint64(fetchEnd.Sub(fetchStart).Microseconds()))
  191. expConfigFetcher.Filename = proto.String(outConfigFilePath)
  192. if _, err := os.Stat(outConfigFilePath); err != nil {
  193. status := smpb.ExpConfigFetcher_NO_CONFIG
  194. expConfigFetcher.Status = &status
  195. return err
  196. }
  197. status := smpb.ExpConfigFetcher_CONFIG
  198. expConfigFetcher.Status = &status
  199. return nil
  200. }
  201. // If a config file does not exist, wait for the config file to be fetched. Otherwise
  202. // fetch the config file in the background and return immediately.
  203. if !configExists {
  204. defer cancel()
  205. return fetchCfg()
  206. }
  207. go func() {
  208. defer cancel()
  209. if err := fetchCfg(); err != nil {
  210. ctx.Verbosef("Failed to fetch config file %v: %v\n", configName, err)
  211. }
  212. }()
  213. return nil
  214. }
  215. func loadEnvConfig(ctx Context, config *configImpl, bc string) error {
  216. if bc == "" {
  217. return nil
  218. }
  219. configDirs := []string{
  220. config.OutDir(),
  221. os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG_DIR"),
  222. envConfigDir,
  223. }
  224. for _, dir := range configDirs {
  225. cfgFile := filepath.Join(os.Getenv("TOP"), dir, fmt.Sprintf("%s.%s", bc, jsonSuffix))
  226. envVarsJSON, err := ioutil.ReadFile(cfgFile)
  227. if err != nil {
  228. continue
  229. }
  230. ctx.Verbosef("Loading config file %v\n", cfgFile)
  231. var envVars map[string]map[string]string
  232. if err := json.Unmarshal(envVarsJSON, &envVars); err != nil {
  233. fmt.Fprintf(os.Stderr, "Env vars config file %s did not parse correctly: %s", cfgFile, err.Error())
  234. continue
  235. }
  236. for k, v := range envVars["env"] {
  237. if os.Getenv(k) != "" {
  238. continue
  239. }
  240. config.environ.Set(k, v)
  241. }
  242. ctx.Verbosef("Finished loading config file %v\n", cfgFile)
  243. break
  244. }
  245. return nil
  246. }
  247. func defaultBazelProdMode(cfg *configImpl) bool {
  248. // Environment flag to disable Bazel for users which experience
  249. // broken bazel-handled builds, or significant performance regressions.
  250. if cfg.IsBazelMixedBuildForceDisabled() {
  251. return false
  252. }
  253. // Darwin-host builds are currently untested with Bazel.
  254. if runtime.GOOS == "darwin" {
  255. return false
  256. }
  257. return true
  258. }
  259. func UploadOnlyConfig(ctx Context, args ...string) Config {
  260. ret := &configImpl{
  261. environ: OsEnvironment(),
  262. sandboxConfig: &SandboxConfig{},
  263. }
  264. ret.parseArgs(ctx, args)
  265. srcDir := absPath(ctx, ".")
  266. bc := os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG")
  267. if err := loadEnvConfig(ctx, ret, bc); err != nil {
  268. ctx.Fatalln("Failed to parse env config files: %v", err)
  269. }
  270. ret.metricsUploader = GetMetricsUploader(srcDir, ret.environ)
  271. return Config{ret}
  272. }
  273. func NewConfig(ctx Context, args ...string) Config {
  274. ret := &configImpl{
  275. environ: OsEnvironment(),
  276. sandboxConfig: &SandboxConfig{},
  277. }
  278. // Default matching ninja
  279. ret.parallel = runtime.NumCPU() + 2
  280. ret.keepGoing = 1
  281. ret.totalRAM = detectTotalRAM(ctx)
  282. ret.parseArgs(ctx, args)
  283. if ret.ninjaWeightListSource == HINT_FROM_SOONG {
  284. ret.environ.Set("SOONG_GENERATES_NINJA_HINT", "true")
  285. }
  286. // Make sure OUT_DIR is set appropriately
  287. if outDir, ok := ret.environ.Get("OUT_DIR"); ok {
  288. ret.environ.Set("OUT_DIR", filepath.Clean(outDir))
  289. } else {
  290. outDir := "out"
  291. if baseDir, ok := ret.environ.Get("OUT_DIR_COMMON_BASE"); ok {
  292. if wd, err := os.Getwd(); err != nil {
  293. ctx.Fatalln("Failed to get working directory:", err)
  294. } else {
  295. outDir = filepath.Join(baseDir, filepath.Base(wd))
  296. }
  297. }
  298. ret.environ.Set("OUT_DIR", outDir)
  299. }
  300. // loadEnvConfig needs to know what the OUT_DIR is, so it should
  301. // be called after we determine the appropriate out directory.
  302. bc := os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG")
  303. if bc != "" {
  304. if err := fetchEnvConfig(ctx, ret, bc); err != nil {
  305. ctx.Verbosef("Failed to fetch config file: %v\n", err)
  306. }
  307. if err := loadEnvConfig(ctx, ret, bc); err != nil {
  308. ctx.Fatalln("Failed to parse env config files: %v", err)
  309. }
  310. }
  311. if distDir, ok := ret.environ.Get("DIST_DIR"); ok {
  312. ret.distDir = filepath.Clean(distDir)
  313. } else {
  314. ret.distDir = filepath.Join(ret.OutDir(), "dist")
  315. }
  316. if srcDirIsWritable, ok := ret.environ.Get("BUILD_BROKEN_SRC_DIR_IS_WRITABLE"); ok {
  317. ret.sandboxConfig.SetSrcDirIsRO(srcDirIsWritable == "false")
  318. }
  319. ret.environ.Unset(
  320. // We're already using it
  321. "USE_SOONG_UI",
  322. // We should never use GOROOT/GOPATH from the shell environment
  323. "GOROOT",
  324. "GOPATH",
  325. // These should only come from Soong, not the environment.
  326. "CLANG",
  327. "CLANG_CXX",
  328. "CCC_CC",
  329. "CCC_CXX",
  330. // Used by the goma compiler wrapper, but should only be set by
  331. // gomacc
  332. "GOMACC_PATH",
  333. // We handle this above
  334. "OUT_DIR_COMMON_BASE",
  335. // This is handled above too, and set for individual commands later
  336. "DIST_DIR",
  337. // Variables that have caused problems in the past
  338. "BASH_ENV",
  339. "CDPATH",
  340. "DISPLAY",
  341. "GREP_OPTIONS",
  342. "JAVAC",
  343. "NDK_ROOT",
  344. "POSIXLY_CORRECT",
  345. // Drop make flags
  346. "MAKEFLAGS",
  347. "MAKELEVEL",
  348. "MFLAGS",
  349. // Set in envsetup.sh, reset in makefiles
  350. "ANDROID_JAVA_TOOLCHAIN",
  351. // Set by envsetup.sh, but shouldn't be used inside the build because envsetup.sh is optional
  352. "ANDROID_BUILD_TOP",
  353. "ANDROID_HOST_OUT",
  354. "ANDROID_PRODUCT_OUT",
  355. "ANDROID_HOST_OUT_TESTCASES",
  356. "ANDROID_TARGET_OUT_TESTCASES",
  357. "ANDROID_TOOLCHAIN",
  358. "ANDROID_TOOLCHAIN_2ND_ARCH",
  359. "ANDROID_DEV_SCRIPTS",
  360. "ANDROID_EMULATOR_PREBUILTS",
  361. "ANDROID_PRE_BUILD_PATHS",
  362. )
  363. if ret.UseGoma() || ret.ForceUseGoma() {
  364. ctx.Println("Goma for Android has been deprecated and replaced with RBE. See go/rbe_for_android for instructions on how to use RBE.")
  365. ctx.Fatalln("USE_GOMA / FORCE_USE_GOMA flag is no longer supported.")
  366. }
  367. // Tell python not to spam the source tree with .pyc files.
  368. ret.environ.Set("PYTHONDONTWRITEBYTECODE", "1")
  369. tmpDir := absPath(ctx, ret.TempDir())
  370. ret.environ.Set("TMPDIR", tmpDir)
  371. // Always set ASAN_SYMBOLIZER_PATH so that ASAN-based tools can symbolize any crashes
  372. symbolizerPath := filepath.Join("prebuilts/clang/host", ret.HostPrebuiltTag(),
  373. "llvm-binutils-stable/llvm-symbolizer")
  374. ret.environ.Set("ASAN_SYMBOLIZER_PATH", absPath(ctx, symbolizerPath))
  375. // Precondition: the current directory is the top of the source tree
  376. checkTopDir(ctx)
  377. srcDir := absPath(ctx, ".")
  378. if strings.ContainsRune(srcDir, ' ') {
  379. ctx.Println("You are building in a directory whose absolute path contains a space character:")
  380. ctx.Println()
  381. ctx.Printf("%q\n", srcDir)
  382. ctx.Println()
  383. ctx.Fatalln("Directory names containing spaces are not supported")
  384. }
  385. ret.metricsUploader = GetMetricsUploader(srcDir, ret.environ)
  386. if outDir := ret.OutDir(); strings.ContainsRune(outDir, ' ') {
  387. ctx.Println("The absolute path of your output directory ($OUT_DIR) contains a space character:")
  388. ctx.Println()
  389. ctx.Printf("%q\n", outDir)
  390. ctx.Println()
  391. ctx.Fatalln("Directory names containing spaces are not supported")
  392. }
  393. if distDir := ret.RealDistDir(); strings.ContainsRune(distDir, ' ') {
  394. ctx.Println("The absolute path of your dist directory ($DIST_DIR) contains a space character:")
  395. ctx.Println()
  396. ctx.Printf("%q\n", distDir)
  397. ctx.Println()
  398. ctx.Fatalln("Directory names containing spaces are not supported")
  399. }
  400. // Configure Java-related variables, including adding it to $PATH
  401. java8Home := filepath.Join("prebuilts/jdk/jdk8", ret.HostPrebuiltTag())
  402. java9Home := filepath.Join("prebuilts/jdk/jdk9", ret.HostPrebuiltTag())
  403. java11Home := filepath.Join("prebuilts/jdk/jdk11", ret.HostPrebuiltTag())
  404. java17Home := filepath.Join("prebuilts/jdk/jdk17", ret.HostPrebuiltTag())
  405. javaHome := func() string {
  406. if override, ok := ret.environ.Get("OVERRIDE_ANDROID_JAVA_HOME"); ok {
  407. return override
  408. }
  409. if toolchain11, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN"); ok && toolchain11 != "true" {
  410. ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN is no longer supported. An OpenJDK 11 toolchain is now the global default.")
  411. }
  412. if toolchain17, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK17_TOOLCHAIN"); ok && toolchain17 != "true" {
  413. ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK17_TOOLCHAIN is no longer supported. An OpenJDK 17 toolchain is now the global default.")
  414. }
  415. return java17Home
  416. }()
  417. absJavaHome := absPath(ctx, javaHome)
  418. ret.configureLocale(ctx)
  419. newPath := []string{filepath.Join(absJavaHome, "bin")}
  420. if path, ok := ret.environ.Get("PATH"); ok && path != "" {
  421. newPath = append(newPath, path)
  422. }
  423. ret.environ.Unset("OVERRIDE_ANDROID_JAVA_HOME")
  424. ret.environ.Set("JAVA_HOME", absJavaHome)
  425. ret.environ.Set("ANDROID_JAVA_HOME", javaHome)
  426. ret.environ.Set("ANDROID_JAVA8_HOME", java8Home)
  427. ret.environ.Set("ANDROID_JAVA9_HOME", java9Home)
  428. ret.environ.Set("ANDROID_JAVA11_HOME", java11Home)
  429. ret.environ.Set("PATH", strings.Join(newPath, string(filepath.ListSeparator)))
  430. if ret.MultitreeBuild() {
  431. ret.environ.Set("MULTITREE_BUILD", "true")
  432. }
  433. outDir := ret.OutDir()
  434. buildDateTimeFile := filepath.Join(outDir, "build_date.txt")
  435. if buildDateTime, ok := ret.environ.Get("BUILD_DATETIME"); ok && buildDateTime != "" {
  436. ret.buildDateTime = buildDateTime
  437. } else {
  438. ret.buildDateTime = strconv.FormatInt(time.Now().Unix(), 10)
  439. }
  440. ret.environ.Set("BUILD_DATETIME_FILE", buildDateTimeFile)
  441. if ret.UseRBE() {
  442. for k, v := range getRBEVars(ctx, Config{ret}) {
  443. ret.environ.Set(k, v)
  444. }
  445. }
  446. if ret.BuildFromTextStub() {
  447. // TODO(b/271443071): support hidden api check for from-text stub build
  448. ret.environ.Set("UNSAFE_DISABLE_HIDDENAPI_FLAGS", "true")
  449. }
  450. bpd := ret.BazelMetricsDir()
  451. if err := os.RemoveAll(bpd); err != nil {
  452. ctx.Fatalf("Unable to remove bazel profile directory %q: %v", bpd, err)
  453. }
  454. c := Config{ret}
  455. storeConfigMetrics(ctx, c)
  456. return c
  457. }
  458. // NewBuildActionConfig returns a build configuration based on the build action. The arguments are
  459. // processed based on the build action and extracts any arguments that belongs to the build action.
  460. func NewBuildActionConfig(action BuildAction, dir string, ctx Context, args ...string) Config {
  461. return NewConfig(ctx, getConfigArgs(action, dir, ctx, args)...)
  462. }
  463. // storeConfigMetrics selects a set of configuration information and store in
  464. // the metrics system for further analysis.
  465. func storeConfigMetrics(ctx Context, config Config) {
  466. if ctx.Metrics == nil {
  467. return
  468. }
  469. ctx.Metrics.BuildConfig(buildConfig(config))
  470. s := &smpb.SystemResourceInfo{
  471. TotalPhysicalMemory: proto.Uint64(config.TotalRAM()),
  472. AvailableCpus: proto.Int32(int32(runtime.NumCPU())),
  473. }
  474. ctx.Metrics.SystemResourceInfo(s)
  475. }
  476. func getNinjaWeightListSourceInMetric(s NinjaWeightListSource) *smpb.BuildConfig_NinjaWeightListSource {
  477. switch s {
  478. case NINJA_LOG:
  479. return smpb.BuildConfig_NINJA_LOG.Enum()
  480. case EVENLY_DISTRIBUTED:
  481. return smpb.BuildConfig_EVENLY_DISTRIBUTED.Enum()
  482. case EXTERNAL_FILE:
  483. return smpb.BuildConfig_EXTERNAL_FILE.Enum()
  484. case HINT_FROM_SOONG:
  485. return smpb.BuildConfig_HINT_FROM_SOONG.Enum()
  486. default:
  487. return smpb.BuildConfig_NOT_USED.Enum()
  488. }
  489. }
  490. func buildConfig(config Config) *smpb.BuildConfig {
  491. c := &smpb.BuildConfig{
  492. ForceUseGoma: proto.Bool(config.ForceUseGoma()),
  493. UseGoma: proto.Bool(config.UseGoma()),
  494. UseRbe: proto.Bool(config.UseRBE()),
  495. BazelMixedBuild: proto.Bool(config.BazelBuildEnabled()),
  496. ForceDisableBazelMixedBuild: proto.Bool(config.IsBazelMixedBuildForceDisabled()),
  497. NinjaWeightListSource: getNinjaWeightListSourceInMetric(config.NinjaWeightListSource()),
  498. }
  499. c.Targets = append(c.Targets, config.arguments...)
  500. return c
  501. }
  502. // getConfigArgs processes the command arguments based on the build action and creates a set of new
  503. // arguments to be accepted by Config.
  504. func getConfigArgs(action BuildAction, dir string, ctx Context, args []string) []string {
  505. // The next block of code verifies that the current directory is the root directory of the source
  506. // tree. It then finds the relative path of dir based on the root directory of the source tree
  507. // and verify that dir is inside of the source tree.
  508. checkTopDir(ctx)
  509. topDir, err := os.Getwd()
  510. if err != nil {
  511. ctx.Fatalf("Error retrieving top directory: %v", err)
  512. }
  513. dir, err = filepath.EvalSymlinks(dir)
  514. if err != nil {
  515. ctx.Fatalf("Unable to evaluate symlink of %s: %v", dir, err)
  516. }
  517. dir, err = filepath.Abs(dir)
  518. if err != nil {
  519. ctx.Fatalf("Unable to find absolute path %s: %v", dir, err)
  520. }
  521. relDir, err := filepath.Rel(topDir, dir)
  522. if err != nil {
  523. ctx.Fatalf("Unable to find relative path %s of %s: %v", relDir, topDir, err)
  524. }
  525. // If there are ".." in the path, it's not in the source tree.
  526. if strings.Contains(relDir, "..") {
  527. ctx.Fatalf("Directory %s is not under the source tree %s", dir, topDir)
  528. }
  529. configArgs := args[:]
  530. // If the arguments contains GET-INSTALL-PATH, change the target name prefix from MODULES-IN- to
  531. // GET-INSTALL-PATH-IN- to extract the installation path instead of building the modules.
  532. targetNamePrefix := "MODULES-IN-"
  533. if inList("GET-INSTALL-PATH", configArgs) {
  534. targetNamePrefix = "GET-INSTALL-PATH-IN-"
  535. configArgs = removeFromList("GET-INSTALL-PATH", configArgs)
  536. }
  537. var targets []string
  538. switch action {
  539. case BUILD_MODULES:
  540. // No additional processing is required when building a list of specific modules or all modules.
  541. case BUILD_MODULES_IN_A_DIRECTORY:
  542. // If dir is the root source tree, all the modules are built of the source tree are built so
  543. // no need to find the build file.
  544. if topDir == dir {
  545. break
  546. }
  547. buildFile := findBuildFile(ctx, relDir)
  548. if buildFile == "" {
  549. ctx.Fatalf("Build file not found for %s directory", relDir)
  550. }
  551. targets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)}
  552. case BUILD_MODULES_IN_DIRECTORIES:
  553. newConfigArgs, dirs := splitArgs(configArgs)
  554. configArgs = newConfigArgs
  555. targets = getTargetsFromDirs(ctx, relDir, dirs, targetNamePrefix)
  556. }
  557. // Tidy only override all other specified targets.
  558. tidyOnly := os.Getenv("WITH_TIDY_ONLY")
  559. if tidyOnly == "true" || tidyOnly == "1" {
  560. configArgs = append(configArgs, "tidy_only")
  561. } else {
  562. configArgs = append(configArgs, targets...)
  563. }
  564. return configArgs
  565. }
  566. // convertToTarget replaces "/" to "-" in dir and pre-append the targetNamePrefix to the target name.
  567. func convertToTarget(dir string, targetNamePrefix string) string {
  568. return targetNamePrefix + strings.ReplaceAll(dir, "/", "-")
  569. }
  570. // hasBuildFile returns true if dir contains an Android build file.
  571. func hasBuildFile(ctx Context, dir string) bool {
  572. for _, buildFile := range buildFiles {
  573. _, err := os.Stat(filepath.Join(dir, buildFile))
  574. if err == nil {
  575. return true
  576. }
  577. if !os.IsNotExist(err) {
  578. ctx.Fatalf("Error retrieving the build file stats: %v", err)
  579. }
  580. }
  581. return false
  582. }
  583. // findBuildFile finds a build file (makefile or blueprint file) by looking if there is a build file
  584. // in the current and any sub directory of dir. If a build file is not found, traverse the path
  585. // up by one directory and repeat again until either a build file is found or reached to the root
  586. // source tree. The returned filename of build file is "Android.mk". If one was not found, a blank
  587. // string is returned.
  588. func findBuildFile(ctx Context, dir string) string {
  589. // If the string is empty or ".", assume it is top directory of the source tree.
  590. if dir == "" || dir == "." {
  591. return ""
  592. }
  593. found := false
  594. for buildDir := dir; buildDir != "."; buildDir = filepath.Dir(buildDir) {
  595. err := filepath.Walk(buildDir, func(path string, info os.FileInfo, err error) error {
  596. if err != nil {
  597. return err
  598. }
  599. if found {
  600. return filepath.SkipDir
  601. }
  602. if info.IsDir() {
  603. return nil
  604. }
  605. for _, buildFile := range buildFiles {
  606. if info.Name() == buildFile {
  607. found = true
  608. return filepath.SkipDir
  609. }
  610. }
  611. return nil
  612. })
  613. if err != nil {
  614. ctx.Fatalf("Error finding Android build file: %v", err)
  615. }
  616. if found {
  617. return filepath.Join(buildDir, "Android.mk")
  618. }
  619. }
  620. return ""
  621. }
  622. // splitArgs iterates over the arguments list and splits into two lists: arguments and directories.
  623. func splitArgs(args []string) (newArgs []string, dirs []string) {
  624. specialArgs := map[string]bool{
  625. "showcommands": true,
  626. "snod": true,
  627. "dist": true,
  628. "checkbuild": true,
  629. }
  630. newArgs = []string{}
  631. dirs = []string{}
  632. for _, arg := range args {
  633. // It's a dash argument if it starts with "-" or it's a key=value pair, it's not a directory.
  634. if strings.IndexRune(arg, '-') == 0 || strings.IndexRune(arg, '=') != -1 {
  635. newArgs = append(newArgs, arg)
  636. continue
  637. }
  638. if _, ok := specialArgs[arg]; ok {
  639. newArgs = append(newArgs, arg)
  640. continue
  641. }
  642. dirs = append(dirs, arg)
  643. }
  644. return newArgs, dirs
  645. }
  646. // getTargetsFromDirs iterates over the dirs list and creates a list of targets to build. If a
  647. // directory from the dirs list does not exist, a fatal error is raised. relDir is related to the
  648. // source root tree where the build action command was invoked. Each directory is validated if the
  649. // build file can be found and follows the format "dir1:target1,target2,...". Target is optional.
  650. func getTargetsFromDirs(ctx Context, relDir string, dirs []string, targetNamePrefix string) (targets []string) {
  651. for _, dir := range dirs {
  652. // The directory may have specified specific modules to build. ":" is the separator to separate
  653. // the directory and the list of modules.
  654. s := strings.Split(dir, ":")
  655. l := len(s)
  656. if l > 2 { // more than one ":" was specified.
  657. ctx.Fatalf("%s not in proper directory:target1,target2,... format (\":\" was specified more than once)", dir)
  658. }
  659. dir = filepath.Join(relDir, s[0])
  660. if _, err := os.Stat(dir); err != nil {
  661. ctx.Fatalf("couldn't find directory %s", dir)
  662. }
  663. // Verify that if there are any targets specified after ":". Each target is separated by ",".
  664. var newTargets []string
  665. if l == 2 && s[1] != "" {
  666. newTargets = strings.Split(s[1], ",")
  667. if inList("", newTargets) {
  668. ctx.Fatalf("%s not in proper directory:target1,target2,... format", dir)
  669. }
  670. }
  671. // If there are specified targets to build in dir, an android build file must exist for the one
  672. // shot build. For the non-targets case, find the appropriate build file and build all the
  673. // modules in dir (or the closest one in the dir path).
  674. if len(newTargets) > 0 {
  675. if !hasBuildFile(ctx, dir) {
  676. ctx.Fatalf("Couldn't locate a build file from %s directory", dir)
  677. }
  678. } else {
  679. buildFile := findBuildFile(ctx, dir)
  680. if buildFile == "" {
  681. ctx.Fatalf("Build file not found for %s directory", dir)
  682. }
  683. newTargets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)}
  684. }
  685. targets = append(targets, newTargets...)
  686. }
  687. return targets
  688. }
  689. func (c *configImpl) parseArgs(ctx Context, args []string) {
  690. for i := 0; i < len(args); i++ {
  691. arg := strings.TrimSpace(args[i])
  692. if arg == "showcommands" {
  693. c.verbose = true
  694. } else if arg == "--empty-ninja-file" {
  695. c.emptyNinjaFile = true
  696. } else if arg == "--skip-ninja" {
  697. c.skipNinja = true
  698. } else if arg == "--skip-make" {
  699. // TODO(ccross): deprecate this, it has confusing behaviors. It doesn't run kati,
  700. // but it does run a Kati ninja file if the .kati_enabled marker file was created
  701. // by a previous build.
  702. c.skipConfig = true
  703. c.skipKati = true
  704. } else if arg == "--soong-only" {
  705. c.skipKati = true
  706. c.skipKatiNinja = true
  707. } else if arg == "--config-only" {
  708. c.skipKati = true
  709. c.skipKatiNinja = true
  710. c.skipSoong = true
  711. } else if arg == "--skip-config" {
  712. c.skipConfig = true
  713. } else if arg == "--skip-soong-tests" {
  714. c.skipSoongTests = true
  715. } else if arg == "--skip-metrics-upload" {
  716. c.skipMetricsUpload = true
  717. } else if arg == "--mk-metrics" {
  718. c.reportMkMetrics = true
  719. } else if arg == "--multitree-build" {
  720. c.multitreeBuild = true
  721. } else if arg == "--bazel-mode" {
  722. c.bazelProdMode = true
  723. } else if arg == "--bazel-mode-dev" {
  724. c.bazelDevMode = true
  725. } else if arg == "--bazel-mode-staging" {
  726. c.bazelStagingMode = true
  727. } else if arg == "--search-api-dir" {
  728. c.searchApiDir = true
  729. } else if strings.HasPrefix(arg, "--ninja_weight_source=") {
  730. source := strings.TrimPrefix(arg, "--ninja_weight_source=")
  731. if source == "ninja_log" {
  732. c.ninjaWeightListSource = NINJA_LOG
  733. } else if source == "evenly_distributed" {
  734. c.ninjaWeightListSource = EVENLY_DISTRIBUTED
  735. } else if source == "not_used" {
  736. c.ninjaWeightListSource = NOT_USED
  737. } else if source == "soong" {
  738. c.ninjaWeightListSource = HINT_FROM_SOONG
  739. } else if strings.HasPrefix(source, "file,") {
  740. c.ninjaWeightListSource = EXTERNAL_FILE
  741. filePath := strings.TrimPrefix(source, "file,")
  742. err := validateNinjaWeightList(filePath)
  743. if err != nil {
  744. ctx.Fatalf("Malformed weight list from %s: %s", filePath, err)
  745. }
  746. _, err = copyFile(filePath, filepath.Join(c.OutDir(), ".ninja_weight_list"))
  747. if err != nil {
  748. ctx.Fatalf("Error to copy ninja weight list from %s: %s", filePath, err)
  749. }
  750. } else {
  751. ctx.Fatalf("unknown option for ninja_weight_source: %s", source)
  752. }
  753. } else if arg == "--build-from-text-stub" {
  754. c.buildFromTextStub = true
  755. } else if strings.HasPrefix(arg, "--build-command=") {
  756. buildCmd := strings.TrimPrefix(arg, "--build-command=")
  757. // remove quotations
  758. buildCmd = strings.TrimPrefix(buildCmd, "\"")
  759. buildCmd = strings.TrimSuffix(buildCmd, "\"")
  760. ctx.Metrics.SetBuildCommand([]string{buildCmd})
  761. } else if strings.HasPrefix(arg, "--bazel-force-enabled-modules=") {
  762. c.bazelForceEnabledModules = strings.TrimPrefix(arg, "--bazel-force-enabled-modules=")
  763. } else if strings.HasPrefix(arg, "--build-started-time-unix-millis=") {
  764. buildTimeStr := strings.TrimPrefix(arg, "--build-started-time-unix-millis=")
  765. val, err := strconv.ParseInt(buildTimeStr, 10, 64)
  766. if err == nil {
  767. c.buildStartedTime = val
  768. } else {
  769. ctx.Fatalf("Error parsing build-time-started-unix-millis", err)
  770. }
  771. } else if arg == "--ensure-allowlist-integrity" {
  772. c.ensureAllowlistIntegrity = true
  773. } else if strings.HasPrefix(arg, "--bazel-exit-code=") {
  774. bazelExitCodeStr := strings.TrimPrefix(arg, "--bazel-exit-code=")
  775. val, err := strconv.Atoi(bazelExitCodeStr)
  776. if err == nil {
  777. c.bazelExitCode = int32(val)
  778. } else {
  779. ctx.Fatalf("Error parsing bazel-exit-code", err)
  780. }
  781. } else if len(arg) > 0 && arg[0] == '-' {
  782. parseArgNum := func(def int) int {
  783. if len(arg) > 2 {
  784. p, err := strconv.ParseUint(arg[2:], 10, 31)
  785. if err != nil {
  786. ctx.Fatalf("Failed to parse %q: %v", arg, err)
  787. }
  788. return int(p)
  789. } else if i+1 < len(args) {
  790. p, err := strconv.ParseUint(args[i+1], 10, 31)
  791. if err == nil {
  792. i++
  793. return int(p)
  794. }
  795. }
  796. return def
  797. }
  798. if len(arg) > 1 && arg[1] == 'j' {
  799. c.parallel = parseArgNum(c.parallel)
  800. } else if len(arg) > 1 && arg[1] == 'k' {
  801. c.keepGoing = parseArgNum(0)
  802. } else {
  803. ctx.Fatalln("Unknown option:", arg)
  804. }
  805. } else if k, v, ok := decodeKeyValue(arg); ok && len(k) > 0 {
  806. if k == "OUT_DIR" {
  807. ctx.Fatalln("OUT_DIR may only be set in the environment, not as a command line option.")
  808. }
  809. c.environ.Set(k, v)
  810. } else if arg == "dist" {
  811. c.dist = true
  812. } else if arg == "json-module-graph" {
  813. c.jsonModuleGraph = true
  814. } else if arg == "bp2build" {
  815. c.bp2build = true
  816. } else if arg == "api_bp2build" {
  817. c.apiBp2build = true
  818. } else if arg == "queryview" {
  819. c.queryview = true
  820. } else if arg == "soong_docs" {
  821. c.soongDocs = true
  822. } else {
  823. if arg == "checkbuild" {
  824. c.checkbuild = true
  825. }
  826. c.arguments = append(c.arguments, arg)
  827. }
  828. }
  829. if (!c.bazelProdMode) && (!c.bazelDevMode) && (!c.bazelStagingMode) {
  830. c.bazelProdMode = defaultBazelProdMode(c)
  831. }
  832. }
  833. func validateNinjaWeightList(weightListFilePath string) (err error) {
  834. data, err := os.ReadFile(weightListFilePath)
  835. if err != nil {
  836. return
  837. }
  838. lines := strings.Split(strings.TrimSpace(string(data)), "\n")
  839. for _, line := range lines {
  840. fields := strings.Split(line, ",")
  841. if len(fields) != 2 {
  842. return fmt.Errorf("wrong format, each line should have two fields, but '%s'", line)
  843. }
  844. _, err = strconv.Atoi(fields[1])
  845. if err != nil {
  846. return
  847. }
  848. }
  849. return
  850. }
  851. func (c *configImpl) configureLocale(ctx Context) {
  852. cmd := Command(ctx, Config{c}, "locale", "locale", "-a")
  853. output, err := cmd.Output()
  854. var locales []string
  855. if err == nil {
  856. locales = strings.Split(string(output), "\n")
  857. } else {
  858. // If we're unable to list the locales, let's assume en_US.UTF-8
  859. locales = []string{"en_US.UTF-8"}
  860. ctx.Verbosef("Failed to list locales (%q), falling back to %q", err, locales)
  861. }
  862. // gettext uses LANGUAGE, which is passed directly through
  863. // For LANG and LC_*, only preserve the evaluated version of
  864. // LC_MESSAGES
  865. userLang := ""
  866. if lc_all, ok := c.environ.Get("LC_ALL"); ok {
  867. userLang = lc_all
  868. } else if lc_messages, ok := c.environ.Get("LC_MESSAGES"); ok {
  869. userLang = lc_messages
  870. } else if lang, ok := c.environ.Get("LANG"); ok {
  871. userLang = lang
  872. }
  873. c.environ.UnsetWithPrefix("LC_")
  874. if userLang != "" {
  875. c.environ.Set("LC_MESSAGES", userLang)
  876. }
  877. // The for LANG, use C.UTF-8 if it exists (Debian currently, proposed
  878. // for others)
  879. if inList("C.UTF-8", locales) {
  880. c.environ.Set("LANG", "C.UTF-8")
  881. } else if inList("C.utf8", locales) {
  882. // These normalize to the same thing
  883. c.environ.Set("LANG", "C.UTF-8")
  884. } else if inList("en_US.UTF-8", locales) {
  885. c.environ.Set("LANG", "en_US.UTF-8")
  886. } else if inList("en_US.utf8", locales) {
  887. // These normalize to the same thing
  888. c.environ.Set("LANG", "en_US.UTF-8")
  889. } else {
  890. ctx.Fatalln("System doesn't support either C.UTF-8 or en_US.UTF-8")
  891. }
  892. }
  893. func (c *configImpl) Environment() *Environment {
  894. return c.environ
  895. }
  896. func (c *configImpl) Arguments() []string {
  897. return c.arguments
  898. }
  899. func (c *configImpl) SoongBuildInvocationNeeded() bool {
  900. if len(c.Arguments()) > 0 {
  901. // Explicit targets requested that are not special targets like b2pbuild
  902. // or the JSON module graph
  903. return true
  904. }
  905. if !c.JsonModuleGraph() && !c.Bp2Build() && !c.Queryview() && !c.SoongDocs() && !c.ApiBp2build() {
  906. // Command line was empty, the default Ninja target is built
  907. return true
  908. }
  909. // bp2build + dist may be used to dist bp2build logs but does not require SoongBuildInvocation
  910. if c.Dist() && !c.Bp2Build() {
  911. return true
  912. }
  913. // build.ninja doesn't need to be generated
  914. return false
  915. }
  916. func (c *configImpl) OutDir() string {
  917. if outDir, ok := c.environ.Get("OUT_DIR"); ok {
  918. return outDir
  919. }
  920. return "out"
  921. }
  922. func (c *configImpl) DistDir() string {
  923. return c.distDir
  924. }
  925. func (c *configImpl) RealDistDir() string {
  926. return c.distDir
  927. }
  928. func (c *configImpl) NinjaArgs() []string {
  929. if c.skipKati {
  930. return c.arguments
  931. }
  932. return c.ninjaArgs
  933. }
  934. func (c *configImpl) BazelOutDir() string {
  935. return filepath.Join(c.OutDir(), "bazel")
  936. }
  937. func (c *configImpl) bazelOutputBase() string {
  938. return filepath.Join(c.BazelOutDir(), "output")
  939. }
  940. func (c *configImpl) SoongOutDir() string {
  941. return filepath.Join(c.OutDir(), "soong")
  942. }
  943. func (c *configImpl) ApiSurfacesOutDir() string {
  944. return filepath.Join(c.OutDir(), "api_surfaces")
  945. }
  946. func (c *configImpl) PrebuiltOS() string {
  947. switch runtime.GOOS {
  948. case "linux":
  949. return "linux-x86"
  950. case "darwin":
  951. return "darwin-x86"
  952. default:
  953. panic("Unknown GOOS")
  954. }
  955. }
  956. func (c *configImpl) HostToolDir() string {
  957. if c.SkipKatiNinja() {
  958. return filepath.Join(c.SoongOutDir(), "host", c.PrebuiltOS(), "bin")
  959. } else {
  960. return filepath.Join(c.OutDir(), "host", c.PrebuiltOS(), "bin")
  961. }
  962. }
  963. func (c *configImpl) NamedGlobFile(name string) string {
  964. return shared.JoinPath(c.SoongOutDir(), "globs-"+name+".ninja")
  965. }
  966. func (c *configImpl) UsedEnvFile(tag string) string {
  967. return shared.JoinPath(c.SoongOutDir(), usedEnvFile+"."+tag)
  968. }
  969. func (c *configImpl) Bp2BuildFilesMarkerFile() string {
  970. return shared.JoinPath(c.SoongOutDir(), "bp2build_files_marker")
  971. }
  972. func (c *configImpl) Bp2BuildWorkspaceMarkerFile() string {
  973. return shared.JoinPath(c.SoongOutDir(), "bp2build_workspace_marker")
  974. }
  975. func (c *configImpl) SoongDocsHtml() string {
  976. return shared.JoinPath(c.SoongOutDir(), "docs/soong_build.html")
  977. }
  978. func (c *configImpl) QueryviewMarkerFile() string {
  979. return shared.JoinPath(c.SoongOutDir(), "queryview.marker")
  980. }
  981. func (c *configImpl) ApiBp2buildMarkerFile() string {
  982. return shared.JoinPath(c.SoongOutDir(), "api_bp2build.marker")
  983. }
  984. func (c *configImpl) ModuleGraphFile() string {
  985. return shared.JoinPath(c.SoongOutDir(), "module-graph.json")
  986. }
  987. func (c *configImpl) ModuleActionsFile() string {
  988. return shared.JoinPath(c.SoongOutDir(), "module-actions.json")
  989. }
  990. func (c *configImpl) TempDir() string {
  991. return shared.TempDirForOutDir(c.SoongOutDir())
  992. }
  993. func (c *configImpl) FileListDir() string {
  994. return filepath.Join(c.OutDir(), ".module_paths")
  995. }
  996. func (c *configImpl) KatiSuffix() string {
  997. if c.katiSuffix != "" {
  998. return c.katiSuffix
  999. }
  1000. panic("SetKatiSuffix has not been called")
  1001. }
  1002. // Checkbuild returns true if "checkbuild" was one of the build goals, which means that the
  1003. // user is interested in additional checks at the expense of build time.
  1004. func (c *configImpl) Checkbuild() bool {
  1005. return c.checkbuild
  1006. }
  1007. func (c *configImpl) Dist() bool {
  1008. return c.dist
  1009. }
  1010. func (c *configImpl) JsonModuleGraph() bool {
  1011. return c.jsonModuleGraph
  1012. }
  1013. func (c *configImpl) Bp2Build() bool {
  1014. return c.bp2build
  1015. }
  1016. func (c *configImpl) ApiBp2build() bool {
  1017. return c.apiBp2build
  1018. }
  1019. func (c *configImpl) Queryview() bool {
  1020. return c.queryview
  1021. }
  1022. func (c *configImpl) SoongDocs() bool {
  1023. return c.soongDocs
  1024. }
  1025. func (c *configImpl) IsVerbose() bool {
  1026. return c.verbose
  1027. }
  1028. func (c *configImpl) MultitreeBuild() bool {
  1029. return c.multitreeBuild
  1030. }
  1031. func (c *configImpl) NinjaWeightListSource() NinjaWeightListSource {
  1032. return c.ninjaWeightListSource
  1033. }
  1034. func (c *configImpl) SkipKati() bool {
  1035. return c.skipKati
  1036. }
  1037. func (c *configImpl) SkipKatiNinja() bool {
  1038. return c.skipKatiNinja
  1039. }
  1040. func (c *configImpl) SkipSoong() bool {
  1041. return c.skipSoong
  1042. }
  1043. func (c *configImpl) SkipNinja() bool {
  1044. return c.skipNinja
  1045. }
  1046. func (c *configImpl) SetSkipNinja(v bool) {
  1047. c.skipNinja = v
  1048. }
  1049. func (c *configImpl) SkipConfig() bool {
  1050. return c.skipConfig
  1051. }
  1052. func (c *configImpl) BuildFromTextStub() bool {
  1053. return c.buildFromTextStub
  1054. }
  1055. func (c *configImpl) TargetProduct() string {
  1056. if v, ok := c.environ.Get("TARGET_PRODUCT"); ok {
  1057. return v
  1058. }
  1059. panic("TARGET_PRODUCT is not defined")
  1060. }
  1061. func (c *configImpl) TargetDevice() string {
  1062. return c.targetDevice
  1063. }
  1064. func (c *configImpl) SetTargetDevice(device string) {
  1065. c.targetDevice = device
  1066. }
  1067. func (c *configImpl) TargetBuildVariant() string {
  1068. if v, ok := c.environ.Get("TARGET_BUILD_VARIANT"); ok {
  1069. return v
  1070. }
  1071. panic("TARGET_BUILD_VARIANT is not defined")
  1072. }
  1073. func (c *configImpl) KatiArgs() []string {
  1074. return c.katiArgs
  1075. }
  1076. func (c *configImpl) Parallel() int {
  1077. return c.parallel
  1078. }
  1079. func (c *configImpl) GetSourceRootDirs() []string {
  1080. return c.sourceRootDirs
  1081. }
  1082. func (c *configImpl) SetSourceRootDirs(i []string) {
  1083. c.sourceRootDirs = i
  1084. }
  1085. func (c *configImpl) GetIncludeTags() []string {
  1086. return c.includeTags
  1087. }
  1088. func (c *configImpl) SetIncludeTags(i []string) {
  1089. c.includeTags = i
  1090. }
  1091. func (c *configImpl) GetLogsPrefix() string {
  1092. return c.logsPrefix
  1093. }
  1094. func (c *configImpl) SetLogsPrefix(prefix string) {
  1095. c.logsPrefix = prefix
  1096. }
  1097. func (c *configImpl) HighmemParallel() int {
  1098. if i, ok := c.environ.GetInt("NINJA_HIGHMEM_NUM_JOBS"); ok {
  1099. return i
  1100. }
  1101. const minMemPerHighmemProcess = 8 * 1024 * 1024 * 1024
  1102. parallel := c.Parallel()
  1103. if c.UseRemoteBuild() {
  1104. // Ninja doesn't support nested pools, and when remote builds are enabled the total ninja parallelism
  1105. // is set very high (i.e. 500). Using a large value here would cause the total number of running jobs
  1106. // to be the sum of the sizes of the local and highmem pools, which will cause extra CPU contention.
  1107. // Return 1/16th of the size of the local pool, rounding up.
  1108. return (parallel + 15) / 16
  1109. } else if c.totalRAM == 0 {
  1110. // Couldn't detect the total RAM, don't restrict highmem processes.
  1111. return parallel
  1112. } else if c.totalRAM <= 16*1024*1024*1024 {
  1113. // Less than 16GB of ram, restrict to 1 highmem processes
  1114. return 1
  1115. } else if c.totalRAM <= 32*1024*1024*1024 {
  1116. // Less than 32GB of ram, restrict to 2 highmem processes
  1117. return 2
  1118. } else if p := int(c.totalRAM / minMemPerHighmemProcess); p < parallel {
  1119. // If less than 8GB total RAM per process, reduce the number of highmem processes
  1120. return p
  1121. }
  1122. // No restriction on highmem processes
  1123. return parallel
  1124. }
  1125. func (c *configImpl) TotalRAM() uint64 {
  1126. return c.totalRAM
  1127. }
  1128. // ForceUseGoma determines whether we should override Goma deprecation
  1129. // and use Goma for the current build or not.
  1130. func (c *configImpl) ForceUseGoma() bool {
  1131. if v, ok := c.environ.Get("FORCE_USE_GOMA"); ok {
  1132. v = strings.TrimSpace(v)
  1133. if v != "" && v != "false" {
  1134. return true
  1135. }
  1136. }
  1137. return false
  1138. }
  1139. func (c *configImpl) UseGoma() bool {
  1140. if v, ok := c.environ.Get("USE_GOMA"); ok {
  1141. v = strings.TrimSpace(v)
  1142. if v != "" && v != "false" {
  1143. return true
  1144. }
  1145. }
  1146. return false
  1147. }
  1148. func (c *configImpl) StartGoma() bool {
  1149. if !c.UseGoma() {
  1150. return false
  1151. }
  1152. if v, ok := c.environ.Get("NOSTART_GOMA"); ok {
  1153. v = strings.TrimSpace(v)
  1154. if v != "" && v != "false" {
  1155. return false
  1156. }
  1157. }
  1158. return true
  1159. }
  1160. func (c *configImpl) UseRBE() bool {
  1161. if v, ok := c.Environment().Get("USE_RBE"); ok {
  1162. v = strings.TrimSpace(v)
  1163. if v != "" && v != "false" {
  1164. return true
  1165. }
  1166. }
  1167. return false
  1168. }
  1169. func (c *configImpl) BazelBuildEnabled() bool {
  1170. return c.bazelProdMode || c.bazelDevMode || c.bazelStagingMode
  1171. }
  1172. func (c *configImpl) StartRBE() bool {
  1173. if !c.UseRBE() {
  1174. return false
  1175. }
  1176. if v, ok := c.environ.Get("NOSTART_RBE"); ok {
  1177. v = strings.TrimSpace(v)
  1178. if v != "" && v != "false" {
  1179. return false
  1180. }
  1181. }
  1182. return true
  1183. }
  1184. func (c *configImpl) rbeProxyLogsDir() string {
  1185. for _, f := range []string{"RBE_proxy_log_dir", "FLAG_output_dir"} {
  1186. if v, ok := c.environ.Get(f); ok {
  1187. return v
  1188. }
  1189. }
  1190. buildTmpDir := shared.TempDirForOutDir(c.SoongOutDir())
  1191. return filepath.Join(buildTmpDir, "rbe")
  1192. }
  1193. func (c *configImpl) rbeCacheDir() string {
  1194. for _, f := range []string{"RBE_cache_dir", "FLAG_cache_dir"} {
  1195. if v, ok := c.environ.Get(f); ok {
  1196. return v
  1197. }
  1198. }
  1199. return shared.JoinPath(c.SoongOutDir(), "rbe")
  1200. }
  1201. func (c *configImpl) shouldCleanupRBELogsDir() bool {
  1202. // Perform a log directory cleanup only when the log directory
  1203. // is auto created by the build rather than user-specified.
  1204. for _, f := range []string{"RBE_proxy_log_dir", "FLAG_output_dir"} {
  1205. if _, ok := c.environ.Get(f); ok {
  1206. return false
  1207. }
  1208. }
  1209. return true
  1210. }
  1211. func (c *configImpl) rbeExecRoot() string {
  1212. for _, f := range []string{"RBE_exec_root", "FLAG_exec_root"} {
  1213. if v, ok := c.environ.Get(f); ok {
  1214. return v
  1215. }
  1216. }
  1217. wd, err := os.Getwd()
  1218. if err != nil {
  1219. return ""
  1220. }
  1221. return wd
  1222. }
  1223. func (c *configImpl) rbeDir() string {
  1224. if v, ok := c.environ.Get("RBE_DIR"); ok {
  1225. return v
  1226. }
  1227. return "prebuilts/remoteexecution-client/live/"
  1228. }
  1229. func (c *configImpl) rbeReproxy() string {
  1230. for _, f := range []string{"RBE_re_proxy", "FLAG_re_proxy"} {
  1231. if v, ok := c.environ.Get(f); ok {
  1232. return v
  1233. }
  1234. }
  1235. return filepath.Join(c.rbeDir(), "reproxy")
  1236. }
  1237. func (c *configImpl) rbeAuth() (string, string) {
  1238. credFlags := []string{
  1239. "use_application_default_credentials",
  1240. "use_gce_credentials",
  1241. "credential_file",
  1242. "use_google_prod_creds",
  1243. }
  1244. for _, cf := range credFlags {
  1245. for _, f := range []string{"RBE_" + cf, "FLAG_" + cf} {
  1246. if v, ok := c.environ.Get(f); ok {
  1247. v = strings.TrimSpace(v)
  1248. if v != "" && v != "false" && v != "0" {
  1249. return "RBE_" + cf, v
  1250. }
  1251. }
  1252. }
  1253. }
  1254. return "RBE_use_application_default_credentials", "true"
  1255. }
  1256. func (c *configImpl) rbeSockAddr(dir string) (string, error) {
  1257. maxNameLen := len(syscall.RawSockaddrUnix{}.Path)
  1258. base := fmt.Sprintf("reproxy_%v.sock", rbeRandPrefix)
  1259. name := filepath.Join(dir, base)
  1260. if len(name) < maxNameLen {
  1261. return name, nil
  1262. }
  1263. name = filepath.Join("/tmp", base)
  1264. if len(name) < maxNameLen {
  1265. return name, nil
  1266. }
  1267. return "", fmt.Errorf("cannot generate a proxy socket address shorter than the limit of %v", maxNameLen)
  1268. }
  1269. // IsGooglerEnvironment returns true if the current build is running
  1270. // on a Google developer machine and false otherwise.
  1271. func (c *configImpl) IsGooglerEnvironment() bool {
  1272. cf := "ANDROID_BUILD_ENVIRONMENT_CONFIG"
  1273. if v, ok := c.environ.Get(cf); ok {
  1274. return v == "googler"
  1275. }
  1276. return false
  1277. }
  1278. // GoogleProdCredsExist determine whether credentials exist on the
  1279. // Googler machine to use remote execution.
  1280. func (c *configImpl) GoogleProdCredsExist() bool {
  1281. if googleProdCredsExistCache {
  1282. return googleProdCredsExistCache
  1283. }
  1284. if _, err := exec.Command("/usr/bin/prodcertstatus", "--simple_output", "--nocheck_loas").Output(); err != nil {
  1285. return false
  1286. }
  1287. googleProdCredsExistCache = true
  1288. return true
  1289. }
  1290. // UseRemoteBuild indicates whether to use a remote build acceleration system
  1291. // to speed up the build.
  1292. func (c *configImpl) UseRemoteBuild() bool {
  1293. return c.UseGoma() || c.UseRBE()
  1294. }
  1295. // StubbyExists checks whether the stubby binary exists on the machine running
  1296. // the build.
  1297. func (c *configImpl) StubbyExists() bool {
  1298. if _, err := exec.LookPath("stubby"); err != nil {
  1299. return false
  1300. }
  1301. return true
  1302. }
  1303. // RemoteParallel controls how many remote jobs (i.e., commands which contain
  1304. // gomacc) are run in parallel. Note the parallelism of all other jobs is
  1305. // still limited by Parallel()
  1306. func (c *configImpl) RemoteParallel() int {
  1307. if !c.UseRemoteBuild() {
  1308. return 0
  1309. }
  1310. if i, ok := c.environ.GetInt("NINJA_REMOTE_NUM_JOBS"); ok {
  1311. return i
  1312. }
  1313. return 500
  1314. }
  1315. func (c *configImpl) SetKatiArgs(args []string) {
  1316. c.katiArgs = args
  1317. }
  1318. func (c *configImpl) SetNinjaArgs(args []string) {
  1319. c.ninjaArgs = args
  1320. }
  1321. func (c *configImpl) SetKatiSuffix(suffix string) {
  1322. c.katiSuffix = suffix
  1323. }
  1324. func (c *configImpl) LastKatiSuffixFile() string {
  1325. return filepath.Join(c.OutDir(), "last_kati_suffix")
  1326. }
  1327. func (c *configImpl) HasKatiSuffix() bool {
  1328. return c.katiSuffix != ""
  1329. }
  1330. func (c *configImpl) KatiEnvFile() string {
  1331. return filepath.Join(c.OutDir(), "env"+c.KatiSuffix()+".sh")
  1332. }
  1333. func (c *configImpl) KatiBuildNinjaFile() string {
  1334. return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiBuildSuffix+".ninja")
  1335. }
  1336. func (c *configImpl) KatiPackageNinjaFile() string {
  1337. return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiPackageSuffix+".ninja")
  1338. }
  1339. func (c *configImpl) SoongVarsFile() string {
  1340. return filepath.Join(c.SoongOutDir(), "soong.variables")
  1341. }
  1342. func (c *configImpl) SoongNinjaFile() string {
  1343. return filepath.Join(c.SoongOutDir(), "build.ninja")
  1344. }
  1345. func (c *configImpl) CombinedNinjaFile() string {
  1346. if c.katiSuffix == "" {
  1347. return filepath.Join(c.OutDir(), "combined.ninja")
  1348. }
  1349. return filepath.Join(c.OutDir(), "combined"+c.KatiSuffix()+".ninja")
  1350. }
  1351. func (c *configImpl) SoongAndroidMk() string {
  1352. return filepath.Join(c.SoongOutDir(), "Android-"+c.TargetProduct()+".mk")
  1353. }
  1354. func (c *configImpl) SoongMakeVarsMk() string {
  1355. return filepath.Join(c.SoongOutDir(), "make_vars-"+c.TargetProduct()+".mk")
  1356. }
  1357. func (c *configImpl) ProductOut() string {
  1358. return filepath.Join(c.OutDir(), "target", "product", c.TargetDevice())
  1359. }
  1360. func (c *configImpl) DevicePreviousProductConfig() string {
  1361. return filepath.Join(c.ProductOut(), "previous_build_config.mk")
  1362. }
  1363. func (c *configImpl) KatiPackageMkDir() string {
  1364. return filepath.Join(c.ProductOut(), "obj", "CONFIG", "kati_packaging")
  1365. }
  1366. func (c *configImpl) hostOutRoot() string {
  1367. return filepath.Join(c.OutDir(), "host")
  1368. }
  1369. func (c *configImpl) HostOut() string {
  1370. return filepath.Join(c.hostOutRoot(), c.HostPrebuiltTag())
  1371. }
  1372. // This probably needs to be multi-valued, so not exporting it for now
  1373. func (c *configImpl) hostCrossOut() string {
  1374. if runtime.GOOS == "linux" {
  1375. return filepath.Join(c.hostOutRoot(), "windows-x86")
  1376. } else {
  1377. return ""
  1378. }
  1379. }
  1380. func (c *configImpl) HostPrebuiltTag() string {
  1381. if runtime.GOOS == "linux" {
  1382. return "linux-x86"
  1383. } else if runtime.GOOS == "darwin" {
  1384. return "darwin-x86"
  1385. } else {
  1386. panic("Unsupported OS")
  1387. }
  1388. }
  1389. func (c *configImpl) PrebuiltBuildTool(name string) string {
  1390. if v, ok := c.environ.Get("SANITIZE_HOST"); ok {
  1391. if sanitize := strings.Fields(v); inList("address", sanitize) {
  1392. asan := filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "asan/bin", name)
  1393. if _, err := os.Stat(asan); err == nil {
  1394. return asan
  1395. }
  1396. }
  1397. }
  1398. return filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "bin", name)
  1399. }
  1400. func (c *configImpl) SetBuildBrokenDupRules(val bool) {
  1401. c.brokenDupRules = val
  1402. }
  1403. func (c *configImpl) BuildBrokenDupRules() bool {
  1404. return c.brokenDupRules
  1405. }
  1406. func (c *configImpl) SetBuildBrokenUsesNetwork(val bool) {
  1407. c.brokenUsesNetwork = val
  1408. }
  1409. func (c *configImpl) BuildBrokenUsesNetwork() bool {
  1410. return c.brokenUsesNetwork
  1411. }
  1412. func (c *configImpl) SetBuildBrokenNinjaUsesEnvVars(val []string) {
  1413. c.brokenNinjaEnvVars = val
  1414. }
  1415. func (c *configImpl) BuildBrokenNinjaUsesEnvVars() []string {
  1416. return c.brokenNinjaEnvVars
  1417. }
  1418. func (c *configImpl) SetTargetDeviceDir(dir string) {
  1419. c.targetDeviceDir = dir
  1420. }
  1421. func (c *configImpl) TargetDeviceDir() string {
  1422. return c.targetDeviceDir
  1423. }
  1424. func (c *configImpl) BuildDateTime() string {
  1425. return c.buildDateTime
  1426. }
  1427. func (c *configImpl) MetricsUploaderApp() string {
  1428. return c.metricsUploader
  1429. }
  1430. // LogsDir returns the absolute path to the logs directory where build log and
  1431. // metrics files are located. By default, the logs directory is the out
  1432. // directory. If the argument dist is specified, the logs directory
  1433. // is <dist_dir>/logs.
  1434. func (c *configImpl) LogsDir() string {
  1435. dir := c.OutDir()
  1436. if c.Dist() {
  1437. // Always write logs to the real dist dir, even if Bazel is using a rigged dist dir for other files
  1438. dir = filepath.Join(c.RealDistDir(), "logs")
  1439. }
  1440. absDir, err := filepath.Abs(dir)
  1441. if err != nil {
  1442. fmt.Fprintf(os.Stderr, "\nError making log dir '%s' absolute: %s\n", dir, err.Error())
  1443. os.Exit(1)
  1444. }
  1445. return absDir
  1446. }
  1447. // BazelMetricsDir returns the <logs dir>/bazel_metrics directory
  1448. // where the bazel profiles are located.
  1449. func (c *configImpl) BazelMetricsDir() string {
  1450. return filepath.Join(c.LogsDir(), "bazel_metrics")
  1451. }
  1452. // MkFileMetrics returns the file path for make-related metrics.
  1453. func (c *configImpl) MkMetrics() string {
  1454. return filepath.Join(c.LogsDir(), "mk_metrics.pb")
  1455. }
  1456. func (c *configImpl) SetEmptyNinjaFile(v bool) {
  1457. c.emptyNinjaFile = v
  1458. }
  1459. func (c *configImpl) EmptyNinjaFile() bool {
  1460. return c.emptyNinjaFile
  1461. }
  1462. func (c *configImpl) IsBazelMixedBuildForceDisabled() bool {
  1463. return c.Environment().IsEnvTrue("BUILD_BROKEN_DISABLE_BAZEL")
  1464. }
  1465. func (c *configImpl) IsPersistentBazelEnabled() bool {
  1466. return c.Environment().IsEnvTrue("USE_PERSISTENT_BAZEL")
  1467. }
  1468. // GetBazeliskBazelVersion returns the Bazel version to use for this build,
  1469. // or the empty string if the current canonical prod Bazel should be used.
  1470. // This environment variable should only be set to debug the build system.
  1471. // The Bazel version, if set, will be passed to Bazelisk, and Bazelisk will
  1472. // handle downloading and invoking the correct Bazel binary.
  1473. func (c *configImpl) GetBazeliskBazelVersion() string {
  1474. value, _ := c.Environment().Get("USE_BAZEL_VERSION")
  1475. return value
  1476. }
  1477. func (c *configImpl) BazelModulesForceEnabledByFlag() string {
  1478. return c.bazelForceEnabledModules
  1479. }
  1480. func (c *configImpl) SkipMetricsUpload() bool {
  1481. return c.skipMetricsUpload
  1482. }
  1483. func (c *configImpl) EnsureAllowlistIntegrity() bool {
  1484. return c.ensureAllowlistIntegrity
  1485. }
  1486. // Returns a Time object if one was passed via a command-line flag.
  1487. // Otherwise returns the passed default.
  1488. func (c *configImpl) BuildStartedTimeOrDefault(defaultTime time.Time) time.Time {
  1489. if c.buildStartedTime == 0 {
  1490. return defaultTime
  1491. }
  1492. return time.UnixMilli(c.buildStartedTime)
  1493. }
  1494. func (c *configImpl) BazelExitCode() int32 {
  1495. return c.bazelExitCode
  1496. }
  1497. func GetMetricsUploader(topDir string, env *Environment) string {
  1498. if p, ok := env.Get("METRICS_UPLOADER"); ok {
  1499. metricsUploader := filepath.Join(topDir, p)
  1500. if _, err := os.Stat(metricsUploader); err == nil {
  1501. return metricsUploader
  1502. }
  1503. }
  1504. return ""
  1505. }