config.go 49 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738
  1. // Copyright 2017 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package build
  15. import (
  16. "context"
  17. "encoding/json"
  18. "fmt"
  19. "io/ioutil"
  20. "math/rand"
  21. "os"
  22. "os/exec"
  23. "path/filepath"
  24. "runtime"
  25. "strconv"
  26. "strings"
  27. "syscall"
  28. "time"
  29. "android/soong/shared"
  30. "google.golang.org/protobuf/proto"
  31. smpb "android/soong/ui/metrics/metrics_proto"
  32. )
  33. const (
  34. envConfigDir = "vendor/google/tools/soong_config"
  35. jsonSuffix = "json"
  36. configFetcher = "vendor/google/tools/soong/expconfigfetcher"
  37. envConfigFetchTimeout = 10 * time.Second
  38. )
  39. var (
  40. rbeRandPrefix int
  41. googleProdCredsExistCache bool
  42. )
  43. func init() {
  44. rand.Seed(time.Now().UnixNano())
  45. rbeRandPrefix = rand.Intn(1000)
  46. }
  47. type Config struct{ *configImpl }
  48. type configImpl struct {
  49. // Some targets that are implemented in soong_build
  50. // (bp2build, json-module-graph) are not here and have their own bits below.
  51. arguments []string
  52. goma bool
  53. environ *Environment
  54. distDir string
  55. buildDateTime string
  56. logsPrefix string
  57. // From the arguments
  58. parallel int
  59. keepGoing int
  60. verbose bool
  61. checkbuild bool
  62. dist bool
  63. jsonModuleGraph bool
  64. apiBp2build bool // Generate BUILD files for Soong modules that contribute APIs
  65. bp2build bool
  66. queryview bool
  67. reportMkMetrics bool // Collect and report mk2bp migration progress metrics.
  68. soongDocs bool
  69. multitreeBuild bool // This is a multitree build.
  70. skipConfig bool
  71. skipKati bool
  72. skipKatiNinja bool
  73. skipSoong bool
  74. skipNinja bool
  75. skipSoongTests bool
  76. searchApiDir bool // Scan the Android.bp files generated in out/api_surfaces
  77. skipMetricsUpload bool
  78. buildStartedTime int64 // For metrics-upload-only - manually specify a build-started time
  79. buildFromTextStub bool
  80. ensureAllowlistIntegrity bool // For CI builds - make sure modules are mixed-built
  81. // From the product config
  82. katiArgs []string
  83. ninjaArgs []string
  84. katiSuffix string
  85. targetDevice string
  86. targetDeviceDir string
  87. sandboxConfig *SandboxConfig
  88. // Autodetected
  89. totalRAM uint64
  90. brokenDupRules bool
  91. brokenUsesNetwork bool
  92. brokenNinjaEnvVars []string
  93. pathReplaced bool
  94. bazelProdMode bool
  95. bazelDevMode bool
  96. bazelStagingMode bool
  97. // Set by multiproduct_kati
  98. emptyNinjaFile bool
  99. metricsUploader string
  100. bazelForceEnabledModules string
  101. includeTags []string
  102. sourceRootDirs []string
  103. // Data source to write ninja weight list
  104. ninjaWeightListSource NinjaWeightListSource
  105. }
  106. type NinjaWeightListSource uint
  107. const (
  108. // ninja doesn't use weight list.
  109. NOT_USED NinjaWeightListSource = iota
  110. // ninja uses weight list based on previous builds by ninja log
  111. NINJA_LOG
  112. // ninja thinks every task has the same weight.
  113. EVENLY_DISTRIBUTED
  114. // ninja uses an external custom weight list
  115. EXTERNAL_FILE
  116. // ninja uses a prioritized module list from Soong
  117. HINT_FROM_SOONG
  118. )
  119. const srcDirFileCheck = "build/soong/root.bp"
  120. var buildFiles = []string{"Android.mk", "Android.bp"}
  121. type BuildAction uint
  122. const (
  123. // Builds all of the modules and their dependencies of a specified directory, relative to the root
  124. // directory of the source tree.
  125. BUILD_MODULES_IN_A_DIRECTORY BuildAction = iota
  126. // Builds all of the modules and their dependencies of a list of specified directories. All specified
  127. // directories are relative to the root directory of the source tree.
  128. BUILD_MODULES_IN_DIRECTORIES
  129. // Build a list of specified modules. If none was specified, simply build the whole source tree.
  130. BUILD_MODULES
  131. )
  132. // checkTopDir validates that the current directory is at the root directory of the source tree.
  133. func checkTopDir(ctx Context) {
  134. if _, err := os.Stat(srcDirFileCheck); err != nil {
  135. if os.IsNotExist(err) {
  136. ctx.Fatalf("Current working directory must be the source tree. %q not found.", srcDirFileCheck)
  137. }
  138. ctx.Fatalln("Error verifying tree state:", err)
  139. }
  140. }
  141. // fetchEnvConfig optionally fetches a configuration file that can then subsequently be
  142. // loaded into Soong environment to control certain aspects of build behavior (e.g., enabling RBE).
  143. // If a configuration file already exists on disk, the fetch is run in the background
  144. // so as to NOT block the rest of the build execution.
  145. func fetchEnvConfig(ctx Context, config *configImpl, envConfigName string) error {
  146. configName := envConfigName + "." + jsonSuffix
  147. expConfigFetcher := &smpb.ExpConfigFetcher{Filename: &configName}
  148. defer func() {
  149. ctx.Metrics.ExpConfigFetcher(expConfigFetcher)
  150. }()
  151. if !config.GoogleProdCredsExist() {
  152. status := smpb.ExpConfigFetcher_MISSING_GCERT
  153. expConfigFetcher.Status = &status
  154. return nil
  155. }
  156. s, err := os.Stat(configFetcher)
  157. if err != nil {
  158. if os.IsNotExist(err) {
  159. return nil
  160. }
  161. return err
  162. }
  163. if s.Mode()&0111 == 0 {
  164. status := smpb.ExpConfigFetcher_ERROR
  165. expConfigFetcher.Status = &status
  166. return fmt.Errorf("configuration fetcher binary %v is not executable: %v", configFetcher, s.Mode())
  167. }
  168. configExists := false
  169. outConfigFilePath := filepath.Join(config.OutDir(), configName)
  170. if _, err := os.Stat(outConfigFilePath); err == nil {
  171. configExists = true
  172. }
  173. tCtx, cancel := context.WithTimeout(ctx, envConfigFetchTimeout)
  174. fetchStart := time.Now()
  175. cmd := exec.CommandContext(tCtx, configFetcher, "-output_config_dir", config.OutDir(),
  176. "-output_config_name", configName)
  177. if err := cmd.Start(); err != nil {
  178. status := smpb.ExpConfigFetcher_ERROR
  179. expConfigFetcher.Status = &status
  180. return err
  181. }
  182. fetchCfg := func() error {
  183. if err := cmd.Wait(); err != nil {
  184. status := smpb.ExpConfigFetcher_ERROR
  185. expConfigFetcher.Status = &status
  186. return err
  187. }
  188. fetchEnd := time.Now()
  189. expConfigFetcher.Micros = proto.Uint64(uint64(fetchEnd.Sub(fetchStart).Microseconds()))
  190. expConfigFetcher.Filename = proto.String(outConfigFilePath)
  191. if _, err := os.Stat(outConfigFilePath); err != nil {
  192. status := smpb.ExpConfigFetcher_NO_CONFIG
  193. expConfigFetcher.Status = &status
  194. return err
  195. }
  196. status := smpb.ExpConfigFetcher_CONFIG
  197. expConfigFetcher.Status = &status
  198. return nil
  199. }
  200. // If a config file does not exist, wait for the config file to be fetched. Otherwise
  201. // fetch the config file in the background and return immediately.
  202. if !configExists {
  203. defer cancel()
  204. return fetchCfg()
  205. }
  206. go func() {
  207. defer cancel()
  208. if err := fetchCfg(); err != nil {
  209. ctx.Verbosef("Failed to fetch config file %v: %v\n", configName, err)
  210. }
  211. }()
  212. return nil
  213. }
  214. func loadEnvConfig(ctx Context, config *configImpl, bc string) error {
  215. if bc == "" {
  216. return nil
  217. }
  218. configDirs := []string{
  219. config.OutDir(),
  220. os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG_DIR"),
  221. envConfigDir,
  222. }
  223. for _, dir := range configDirs {
  224. cfgFile := filepath.Join(os.Getenv("TOP"), dir, fmt.Sprintf("%s.%s", bc, jsonSuffix))
  225. envVarsJSON, err := ioutil.ReadFile(cfgFile)
  226. if err != nil {
  227. continue
  228. }
  229. ctx.Verbosef("Loading config file %v\n", cfgFile)
  230. var envVars map[string]map[string]string
  231. if err := json.Unmarshal(envVarsJSON, &envVars); err != nil {
  232. fmt.Fprintf(os.Stderr, "Env vars config file %s did not parse correctly: %s", cfgFile, err.Error())
  233. continue
  234. }
  235. for k, v := range envVars["env"] {
  236. if os.Getenv(k) != "" {
  237. continue
  238. }
  239. config.environ.Set(k, v)
  240. }
  241. ctx.Verbosef("Finished loading config file %v\n", cfgFile)
  242. break
  243. }
  244. return nil
  245. }
  246. func defaultBazelProdMode(cfg *configImpl) bool {
  247. // Environment flag to disable Bazel for users which experience
  248. // broken bazel-handled builds, or significant performance regressions.
  249. if cfg.IsBazelMixedBuildForceDisabled() {
  250. return false
  251. }
  252. // Darwin-host builds are currently untested with Bazel.
  253. if runtime.GOOS == "darwin" {
  254. return false
  255. }
  256. return true
  257. }
  258. func UploadOnlyConfig(ctx Context, _ ...string) Config {
  259. ret := &configImpl{
  260. environ: OsEnvironment(),
  261. sandboxConfig: &SandboxConfig{},
  262. }
  263. srcDir := absPath(ctx, ".")
  264. bc := os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG")
  265. if err := loadEnvConfig(ctx, ret, bc); err != nil {
  266. ctx.Fatalln("Failed to parse env config files: %v", err)
  267. }
  268. ret.metricsUploader = GetMetricsUploader(srcDir, ret.environ)
  269. return Config{ret}
  270. }
  271. func NewConfig(ctx Context, args ...string) Config {
  272. ret := &configImpl{
  273. environ: OsEnvironment(),
  274. sandboxConfig: &SandboxConfig{},
  275. }
  276. // Default matching ninja
  277. ret.parallel = runtime.NumCPU() + 2
  278. ret.keepGoing = 1
  279. ret.totalRAM = detectTotalRAM(ctx)
  280. ret.parseArgs(ctx, args)
  281. if ret.ninjaWeightListSource == HINT_FROM_SOONG {
  282. ret.environ.Set("SOONG_GENERATES_NINJA_HINT", "true")
  283. }
  284. // Make sure OUT_DIR is set appropriately
  285. if outDir, ok := ret.environ.Get("OUT_DIR"); ok {
  286. ret.environ.Set("OUT_DIR", filepath.Clean(outDir))
  287. } else {
  288. outDir := "out"
  289. if baseDir, ok := ret.environ.Get("OUT_DIR_COMMON_BASE"); ok {
  290. if wd, err := os.Getwd(); err != nil {
  291. ctx.Fatalln("Failed to get working directory:", err)
  292. } else {
  293. outDir = filepath.Join(baseDir, filepath.Base(wd))
  294. }
  295. }
  296. ret.environ.Set("OUT_DIR", outDir)
  297. }
  298. // loadEnvConfig needs to know what the OUT_DIR is, so it should
  299. // be called after we determine the appropriate out directory.
  300. bc := os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG")
  301. if bc != "" {
  302. if err := fetchEnvConfig(ctx, ret, bc); err != nil {
  303. ctx.Verbosef("Failed to fetch config file: %v\n", err)
  304. }
  305. if err := loadEnvConfig(ctx, ret, bc); err != nil {
  306. ctx.Fatalln("Failed to parse env config files: %v", err)
  307. }
  308. }
  309. if distDir, ok := ret.environ.Get("DIST_DIR"); ok {
  310. ret.distDir = filepath.Clean(distDir)
  311. } else {
  312. ret.distDir = filepath.Join(ret.OutDir(), "dist")
  313. }
  314. if srcDirIsWritable, ok := ret.environ.Get("BUILD_BROKEN_SRC_DIR_IS_WRITABLE"); ok {
  315. ret.sandboxConfig.SetSrcDirIsRO(srcDirIsWritable == "false")
  316. }
  317. ret.environ.Unset(
  318. // We're already using it
  319. "USE_SOONG_UI",
  320. // We should never use GOROOT/GOPATH from the shell environment
  321. "GOROOT",
  322. "GOPATH",
  323. // These should only come from Soong, not the environment.
  324. "CLANG",
  325. "CLANG_CXX",
  326. "CCC_CC",
  327. "CCC_CXX",
  328. // Used by the goma compiler wrapper, but should only be set by
  329. // gomacc
  330. "GOMACC_PATH",
  331. // We handle this above
  332. "OUT_DIR_COMMON_BASE",
  333. // This is handled above too, and set for individual commands later
  334. "DIST_DIR",
  335. // Variables that have caused problems in the past
  336. "BASH_ENV",
  337. "CDPATH",
  338. "DISPLAY",
  339. "GREP_OPTIONS",
  340. "JAVAC",
  341. "NDK_ROOT",
  342. "POSIXLY_CORRECT",
  343. // Drop make flags
  344. "MAKEFLAGS",
  345. "MAKELEVEL",
  346. "MFLAGS",
  347. // Set in envsetup.sh, reset in makefiles
  348. "ANDROID_JAVA_TOOLCHAIN",
  349. // Set by envsetup.sh, but shouldn't be used inside the build because envsetup.sh is optional
  350. "ANDROID_BUILD_TOP",
  351. "ANDROID_HOST_OUT",
  352. "ANDROID_PRODUCT_OUT",
  353. "ANDROID_HOST_OUT_TESTCASES",
  354. "ANDROID_TARGET_OUT_TESTCASES",
  355. "ANDROID_TOOLCHAIN",
  356. "ANDROID_TOOLCHAIN_2ND_ARCH",
  357. "ANDROID_DEV_SCRIPTS",
  358. "ANDROID_EMULATOR_PREBUILTS",
  359. "ANDROID_PRE_BUILD_PATHS",
  360. )
  361. if ret.UseGoma() || ret.ForceUseGoma() {
  362. ctx.Println("Goma for Android has been deprecated and replaced with RBE. See go/rbe_for_android for instructions on how to use RBE.")
  363. ctx.Fatalln("USE_GOMA / FORCE_USE_GOMA flag is no longer supported.")
  364. }
  365. // Tell python not to spam the source tree with .pyc files.
  366. ret.environ.Set("PYTHONDONTWRITEBYTECODE", "1")
  367. tmpDir := absPath(ctx, ret.TempDir())
  368. ret.environ.Set("TMPDIR", tmpDir)
  369. // Always set ASAN_SYMBOLIZER_PATH so that ASAN-based tools can symbolize any crashes
  370. symbolizerPath := filepath.Join("prebuilts/clang/host", ret.HostPrebuiltTag(),
  371. "llvm-binutils-stable/llvm-symbolizer")
  372. ret.environ.Set("ASAN_SYMBOLIZER_PATH", absPath(ctx, symbolizerPath))
  373. // Precondition: the current directory is the top of the source tree
  374. checkTopDir(ctx)
  375. srcDir := absPath(ctx, ".")
  376. if strings.ContainsRune(srcDir, ' ') {
  377. ctx.Println("You are building in a directory whose absolute path contains a space character:")
  378. ctx.Println()
  379. ctx.Printf("%q\n", srcDir)
  380. ctx.Println()
  381. ctx.Fatalln("Directory names containing spaces are not supported")
  382. }
  383. ret.metricsUploader = GetMetricsUploader(srcDir, ret.environ)
  384. if outDir := ret.OutDir(); strings.ContainsRune(outDir, ' ') {
  385. ctx.Println("The absolute path of your output directory ($OUT_DIR) contains a space character:")
  386. ctx.Println()
  387. ctx.Printf("%q\n", outDir)
  388. ctx.Println()
  389. ctx.Fatalln("Directory names containing spaces are not supported")
  390. }
  391. if distDir := ret.RealDistDir(); strings.ContainsRune(distDir, ' ') {
  392. ctx.Println("The absolute path of your dist directory ($DIST_DIR) contains a space character:")
  393. ctx.Println()
  394. ctx.Printf("%q\n", distDir)
  395. ctx.Println()
  396. ctx.Fatalln("Directory names containing spaces are not supported")
  397. }
  398. // Configure Java-related variables, including adding it to $PATH
  399. java8Home := filepath.Join("prebuilts/jdk/jdk8", ret.HostPrebuiltTag())
  400. java9Home := filepath.Join("prebuilts/jdk/jdk9", ret.HostPrebuiltTag())
  401. java11Home := filepath.Join("prebuilts/jdk/jdk11", ret.HostPrebuiltTag())
  402. java17Home := filepath.Join("prebuilts/jdk/jdk17", ret.HostPrebuiltTag())
  403. javaHome := func() string {
  404. if override, ok := ret.environ.Get("OVERRIDE_ANDROID_JAVA_HOME"); ok {
  405. return override
  406. }
  407. if toolchain11, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN"); ok && toolchain11 != "true" {
  408. ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN is no longer supported. An OpenJDK 11 toolchain is now the global default.")
  409. }
  410. if toolchain17, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK17_TOOLCHAIN"); ok && toolchain17 != "true" {
  411. ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK17_TOOLCHAIN is no longer supported. An OpenJDK 17 toolchain is now the global default.")
  412. }
  413. return java17Home
  414. }()
  415. absJavaHome := absPath(ctx, javaHome)
  416. ret.configureLocale(ctx)
  417. newPath := []string{filepath.Join(absJavaHome, "bin")}
  418. if path, ok := ret.environ.Get("PATH"); ok && path != "" {
  419. newPath = append(newPath, path)
  420. }
  421. ret.environ.Unset("OVERRIDE_ANDROID_JAVA_HOME")
  422. ret.environ.Set("JAVA_HOME", absJavaHome)
  423. ret.environ.Set("ANDROID_JAVA_HOME", javaHome)
  424. ret.environ.Set("ANDROID_JAVA8_HOME", java8Home)
  425. ret.environ.Set("ANDROID_JAVA9_HOME", java9Home)
  426. ret.environ.Set("ANDROID_JAVA11_HOME", java11Home)
  427. ret.environ.Set("PATH", strings.Join(newPath, string(filepath.ListSeparator)))
  428. if ret.MultitreeBuild() {
  429. ret.environ.Set("MULTITREE_BUILD", "true")
  430. }
  431. outDir := ret.OutDir()
  432. buildDateTimeFile := filepath.Join(outDir, "build_date.txt")
  433. if buildDateTime, ok := ret.environ.Get("BUILD_DATETIME"); ok && buildDateTime != "" {
  434. ret.buildDateTime = buildDateTime
  435. } else {
  436. ret.buildDateTime = strconv.FormatInt(time.Now().Unix(), 10)
  437. }
  438. ret.environ.Set("BUILD_DATETIME_FILE", buildDateTimeFile)
  439. if ret.UseRBE() {
  440. for k, v := range getRBEVars(ctx, Config{ret}) {
  441. ret.environ.Set(k, v)
  442. }
  443. }
  444. if ret.BuildFromTextStub() {
  445. // TODO(b/271443071): support hidden api check for from-text stub build
  446. ret.environ.Set("UNSAFE_DISABLE_HIDDENAPI_FLAGS", "true")
  447. }
  448. bpd := ret.BazelMetricsDir()
  449. if err := os.RemoveAll(bpd); err != nil {
  450. ctx.Fatalf("Unable to remove bazel profile directory %q: %v", bpd, err)
  451. }
  452. c := Config{ret}
  453. storeConfigMetrics(ctx, c)
  454. return c
  455. }
  456. // NewBuildActionConfig returns a build configuration based on the build action. The arguments are
  457. // processed based on the build action and extracts any arguments that belongs to the build action.
  458. func NewBuildActionConfig(action BuildAction, dir string, ctx Context, args ...string) Config {
  459. return NewConfig(ctx, getConfigArgs(action, dir, ctx, args)...)
  460. }
  461. // storeConfigMetrics selects a set of configuration information and store in
  462. // the metrics system for further analysis.
  463. func storeConfigMetrics(ctx Context, config Config) {
  464. if ctx.Metrics == nil {
  465. return
  466. }
  467. ctx.Metrics.BuildConfig(buildConfig(config))
  468. s := &smpb.SystemResourceInfo{
  469. TotalPhysicalMemory: proto.Uint64(config.TotalRAM()),
  470. AvailableCpus: proto.Int32(int32(runtime.NumCPU())),
  471. }
  472. ctx.Metrics.SystemResourceInfo(s)
  473. }
  474. func getNinjaWeightListSourceInMetric(s NinjaWeightListSource) *smpb.BuildConfig_NinjaWeightListSource {
  475. switch s {
  476. case NINJA_LOG:
  477. return smpb.BuildConfig_NINJA_LOG.Enum()
  478. case EVENLY_DISTRIBUTED:
  479. return smpb.BuildConfig_EVENLY_DISTRIBUTED.Enum()
  480. case EXTERNAL_FILE:
  481. return smpb.BuildConfig_EXTERNAL_FILE.Enum()
  482. case HINT_FROM_SOONG:
  483. return smpb.BuildConfig_HINT_FROM_SOONG.Enum()
  484. default:
  485. return smpb.BuildConfig_NOT_USED.Enum()
  486. }
  487. }
  488. func buildConfig(config Config) *smpb.BuildConfig {
  489. c := &smpb.BuildConfig{
  490. ForceUseGoma: proto.Bool(config.ForceUseGoma()),
  491. UseGoma: proto.Bool(config.UseGoma()),
  492. UseRbe: proto.Bool(config.UseRBE()),
  493. BazelMixedBuild: proto.Bool(config.BazelBuildEnabled()),
  494. ForceDisableBazelMixedBuild: proto.Bool(config.IsBazelMixedBuildForceDisabled()),
  495. NinjaWeightListSource: getNinjaWeightListSourceInMetric(config.NinjaWeightListSource()),
  496. }
  497. c.Targets = append(c.Targets, config.arguments...)
  498. return c
  499. }
  500. // getConfigArgs processes the command arguments based on the build action and creates a set of new
  501. // arguments to be accepted by Config.
  502. func getConfigArgs(action BuildAction, dir string, ctx Context, args []string) []string {
  503. // The next block of code verifies that the current directory is the root directory of the source
  504. // tree. It then finds the relative path of dir based on the root directory of the source tree
  505. // and verify that dir is inside of the source tree.
  506. checkTopDir(ctx)
  507. topDir, err := os.Getwd()
  508. if err != nil {
  509. ctx.Fatalf("Error retrieving top directory: %v", err)
  510. }
  511. dir, err = filepath.EvalSymlinks(dir)
  512. if err != nil {
  513. ctx.Fatalf("Unable to evaluate symlink of %s: %v", dir, err)
  514. }
  515. dir, err = filepath.Abs(dir)
  516. if err != nil {
  517. ctx.Fatalf("Unable to find absolute path %s: %v", dir, err)
  518. }
  519. relDir, err := filepath.Rel(topDir, dir)
  520. if err != nil {
  521. ctx.Fatalf("Unable to find relative path %s of %s: %v", relDir, topDir, err)
  522. }
  523. // If there are ".." in the path, it's not in the source tree.
  524. if strings.Contains(relDir, "..") {
  525. ctx.Fatalf("Directory %s is not under the source tree %s", dir, topDir)
  526. }
  527. configArgs := args[:]
  528. // If the arguments contains GET-INSTALL-PATH, change the target name prefix from MODULES-IN- to
  529. // GET-INSTALL-PATH-IN- to extract the installation path instead of building the modules.
  530. targetNamePrefix := "MODULES-IN-"
  531. if inList("GET-INSTALL-PATH", configArgs) {
  532. targetNamePrefix = "GET-INSTALL-PATH-IN-"
  533. configArgs = removeFromList("GET-INSTALL-PATH", configArgs)
  534. }
  535. var targets []string
  536. switch action {
  537. case BUILD_MODULES:
  538. // No additional processing is required when building a list of specific modules or all modules.
  539. case BUILD_MODULES_IN_A_DIRECTORY:
  540. // If dir is the root source tree, all the modules are built of the source tree are built so
  541. // no need to find the build file.
  542. if topDir == dir {
  543. break
  544. }
  545. buildFile := findBuildFile(ctx, relDir)
  546. if buildFile == "" {
  547. ctx.Fatalf("Build file not found for %s directory", relDir)
  548. }
  549. targets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)}
  550. case BUILD_MODULES_IN_DIRECTORIES:
  551. newConfigArgs, dirs := splitArgs(configArgs)
  552. configArgs = newConfigArgs
  553. targets = getTargetsFromDirs(ctx, relDir, dirs, targetNamePrefix)
  554. }
  555. // Tidy only override all other specified targets.
  556. tidyOnly := os.Getenv("WITH_TIDY_ONLY")
  557. if tidyOnly == "true" || tidyOnly == "1" {
  558. configArgs = append(configArgs, "tidy_only")
  559. } else {
  560. configArgs = append(configArgs, targets...)
  561. }
  562. return configArgs
  563. }
  564. // convertToTarget replaces "/" to "-" in dir and pre-append the targetNamePrefix to the target name.
  565. func convertToTarget(dir string, targetNamePrefix string) string {
  566. return targetNamePrefix + strings.ReplaceAll(dir, "/", "-")
  567. }
  568. // hasBuildFile returns true if dir contains an Android build file.
  569. func hasBuildFile(ctx Context, dir string) bool {
  570. for _, buildFile := range buildFiles {
  571. _, err := os.Stat(filepath.Join(dir, buildFile))
  572. if err == nil {
  573. return true
  574. }
  575. if !os.IsNotExist(err) {
  576. ctx.Fatalf("Error retrieving the build file stats: %v", err)
  577. }
  578. }
  579. return false
  580. }
  581. // findBuildFile finds a build file (makefile or blueprint file) by looking if there is a build file
  582. // in the current and any sub directory of dir. If a build file is not found, traverse the path
  583. // up by one directory and repeat again until either a build file is found or reached to the root
  584. // source tree. The returned filename of build file is "Android.mk". If one was not found, a blank
  585. // string is returned.
  586. func findBuildFile(ctx Context, dir string) string {
  587. // If the string is empty or ".", assume it is top directory of the source tree.
  588. if dir == "" || dir == "." {
  589. return ""
  590. }
  591. found := false
  592. for buildDir := dir; buildDir != "."; buildDir = filepath.Dir(buildDir) {
  593. err := filepath.Walk(buildDir, func(path string, info os.FileInfo, err error) error {
  594. if err != nil {
  595. return err
  596. }
  597. if found {
  598. return filepath.SkipDir
  599. }
  600. if info.IsDir() {
  601. return nil
  602. }
  603. for _, buildFile := range buildFiles {
  604. if info.Name() == buildFile {
  605. found = true
  606. return filepath.SkipDir
  607. }
  608. }
  609. return nil
  610. })
  611. if err != nil {
  612. ctx.Fatalf("Error finding Android build file: %v", err)
  613. }
  614. if found {
  615. return filepath.Join(buildDir, "Android.mk")
  616. }
  617. }
  618. return ""
  619. }
  620. // splitArgs iterates over the arguments list and splits into two lists: arguments and directories.
  621. func splitArgs(args []string) (newArgs []string, dirs []string) {
  622. specialArgs := map[string]bool{
  623. "showcommands": true,
  624. "snod": true,
  625. "dist": true,
  626. "checkbuild": true,
  627. }
  628. newArgs = []string{}
  629. dirs = []string{}
  630. for _, arg := range args {
  631. // It's a dash argument if it starts with "-" or it's a key=value pair, it's not a directory.
  632. if strings.IndexRune(arg, '-') == 0 || strings.IndexRune(arg, '=') != -1 {
  633. newArgs = append(newArgs, arg)
  634. continue
  635. }
  636. if _, ok := specialArgs[arg]; ok {
  637. newArgs = append(newArgs, arg)
  638. continue
  639. }
  640. dirs = append(dirs, arg)
  641. }
  642. return newArgs, dirs
  643. }
  644. // getTargetsFromDirs iterates over the dirs list and creates a list of targets to build. If a
  645. // directory from the dirs list does not exist, a fatal error is raised. relDir is related to the
  646. // source root tree where the build action command was invoked. Each directory is validated if the
  647. // build file can be found and follows the format "dir1:target1,target2,...". Target is optional.
  648. func getTargetsFromDirs(ctx Context, relDir string, dirs []string, targetNamePrefix string) (targets []string) {
  649. for _, dir := range dirs {
  650. // The directory may have specified specific modules to build. ":" is the separator to separate
  651. // the directory and the list of modules.
  652. s := strings.Split(dir, ":")
  653. l := len(s)
  654. if l > 2 { // more than one ":" was specified.
  655. ctx.Fatalf("%s not in proper directory:target1,target2,... format (\":\" was specified more than once)", dir)
  656. }
  657. dir = filepath.Join(relDir, s[0])
  658. if _, err := os.Stat(dir); err != nil {
  659. ctx.Fatalf("couldn't find directory %s", dir)
  660. }
  661. // Verify that if there are any targets specified after ":". Each target is separated by ",".
  662. var newTargets []string
  663. if l == 2 && s[1] != "" {
  664. newTargets = strings.Split(s[1], ",")
  665. if inList("", newTargets) {
  666. ctx.Fatalf("%s not in proper directory:target1,target2,... format", dir)
  667. }
  668. }
  669. // If there are specified targets to build in dir, an android build file must exist for the one
  670. // shot build. For the non-targets case, find the appropriate build file and build all the
  671. // modules in dir (or the closest one in the dir path).
  672. if len(newTargets) > 0 {
  673. if !hasBuildFile(ctx, dir) {
  674. ctx.Fatalf("Couldn't locate a build file from %s directory", dir)
  675. }
  676. } else {
  677. buildFile := findBuildFile(ctx, dir)
  678. if buildFile == "" {
  679. ctx.Fatalf("Build file not found for %s directory", dir)
  680. }
  681. newTargets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)}
  682. }
  683. targets = append(targets, newTargets...)
  684. }
  685. return targets
  686. }
  687. func (c *configImpl) parseArgs(ctx Context, args []string) {
  688. for i := 0; i < len(args); i++ {
  689. arg := strings.TrimSpace(args[i])
  690. if arg == "showcommands" {
  691. c.verbose = true
  692. } else if arg == "--empty-ninja-file" {
  693. c.emptyNinjaFile = true
  694. } else if arg == "--skip-ninja" {
  695. c.skipNinja = true
  696. } else if arg == "--skip-make" {
  697. // TODO(ccross): deprecate this, it has confusing behaviors. It doesn't run kati,
  698. // but it does run a Kati ninja file if the .kati_enabled marker file was created
  699. // by a previous build.
  700. c.skipConfig = true
  701. c.skipKati = true
  702. } else if arg == "--skip-kati" {
  703. // TODO: remove --skip-kati once module builds have been migrated to --song-only
  704. c.skipKati = true
  705. } else if arg == "--soong-only" {
  706. c.skipKati = true
  707. c.skipKatiNinja = true
  708. } else if arg == "--config-only" {
  709. c.skipKati = true
  710. c.skipKatiNinja = true
  711. c.skipSoong = true
  712. } else if arg == "--skip-config" {
  713. c.skipConfig = true
  714. } else if arg == "--skip-soong-tests" {
  715. c.skipSoongTests = true
  716. } else if arg == "--skip-metrics-upload" {
  717. c.skipMetricsUpload = true
  718. } else if arg == "--mk-metrics" {
  719. c.reportMkMetrics = true
  720. } else if arg == "--multitree-build" {
  721. c.multitreeBuild = true
  722. } else if arg == "--bazel-mode" {
  723. c.bazelProdMode = true
  724. } else if arg == "--bazel-mode-dev" {
  725. c.bazelDevMode = true
  726. } else if arg == "--bazel-mode-staging" {
  727. c.bazelStagingMode = true
  728. } else if arg == "--search-api-dir" {
  729. c.searchApiDir = true
  730. } else if strings.HasPrefix(arg, "--ninja_weight_source=") {
  731. source := strings.TrimPrefix(arg, "--ninja_weight_source=")
  732. if source == "ninja_log" {
  733. c.ninjaWeightListSource = NINJA_LOG
  734. } else if source == "evenly_distributed" {
  735. c.ninjaWeightListSource = EVENLY_DISTRIBUTED
  736. } else if source == "not_used" {
  737. c.ninjaWeightListSource = NOT_USED
  738. } else if source == "soong" {
  739. c.ninjaWeightListSource = HINT_FROM_SOONG
  740. } else if strings.HasPrefix(source, "file,") {
  741. c.ninjaWeightListSource = EXTERNAL_FILE
  742. filePath := strings.TrimPrefix(source, "file,")
  743. err := validateNinjaWeightList(filePath)
  744. if err != nil {
  745. ctx.Fatalf("Malformed weight list from %s: %s", filePath, err)
  746. }
  747. _, err = copyFile(filePath, filepath.Join(c.OutDir(), ".ninja_weight_list"))
  748. if err != nil {
  749. ctx.Fatalf("Error to copy ninja weight list from %s: %s", filePath, err)
  750. }
  751. } else {
  752. ctx.Fatalf("unknown option for ninja_weight_source: %s", source)
  753. }
  754. } else if arg == "--build-from-text-stub" {
  755. c.buildFromTextStub = true
  756. } else if strings.HasPrefix(arg, "--build-command=") {
  757. buildCmd := strings.TrimPrefix(arg, "--build-command=")
  758. // remove quotations
  759. buildCmd = strings.TrimPrefix(buildCmd, "\"")
  760. buildCmd = strings.TrimSuffix(buildCmd, "\"")
  761. ctx.Metrics.SetBuildCommand([]string{buildCmd})
  762. } else if strings.HasPrefix(arg, "--bazel-force-enabled-modules=") {
  763. c.bazelForceEnabledModules = strings.TrimPrefix(arg, "--bazel-force-enabled-modules=")
  764. } else if strings.HasPrefix(arg, "--build-started-time-unix-millis=") {
  765. buildTimeStr := strings.TrimPrefix(arg, "--build-started-time-unix-millis=")
  766. val, err := strconv.ParseInt(buildTimeStr, 10, 64)
  767. if err == nil {
  768. c.buildStartedTime = val
  769. } else {
  770. ctx.Fatalf("Error parsing build-time-started-unix-millis", err)
  771. }
  772. } else if arg == "--ensure-allowlist-integrity" {
  773. c.ensureAllowlistIntegrity = true
  774. } else if len(arg) > 0 && arg[0] == '-' {
  775. parseArgNum := func(def int) int {
  776. if len(arg) > 2 {
  777. p, err := strconv.ParseUint(arg[2:], 10, 31)
  778. if err != nil {
  779. ctx.Fatalf("Failed to parse %q: %v", arg, err)
  780. }
  781. return int(p)
  782. } else if i+1 < len(args) {
  783. p, err := strconv.ParseUint(args[i+1], 10, 31)
  784. if err == nil {
  785. i++
  786. return int(p)
  787. }
  788. }
  789. return def
  790. }
  791. if len(arg) > 1 && arg[1] == 'j' {
  792. c.parallel = parseArgNum(c.parallel)
  793. } else if len(arg) > 1 && arg[1] == 'k' {
  794. c.keepGoing = parseArgNum(0)
  795. } else {
  796. ctx.Fatalln("Unknown option:", arg)
  797. }
  798. } else if k, v, ok := decodeKeyValue(arg); ok && len(k) > 0 {
  799. if k == "OUT_DIR" {
  800. ctx.Fatalln("OUT_DIR may only be set in the environment, not as a command line option.")
  801. }
  802. c.environ.Set(k, v)
  803. } else if arg == "dist" {
  804. c.dist = true
  805. } else if arg == "json-module-graph" {
  806. c.jsonModuleGraph = true
  807. } else if arg == "bp2build" {
  808. c.bp2build = true
  809. } else if arg == "api_bp2build" {
  810. c.apiBp2build = true
  811. } else if arg == "queryview" {
  812. c.queryview = true
  813. } else if arg == "soong_docs" {
  814. c.soongDocs = true
  815. } else {
  816. if arg == "checkbuild" {
  817. c.checkbuild = true
  818. }
  819. c.arguments = append(c.arguments, arg)
  820. }
  821. }
  822. if (!c.bazelProdMode) && (!c.bazelDevMode) && (!c.bazelStagingMode) {
  823. c.bazelProdMode = defaultBazelProdMode(c)
  824. }
  825. }
  826. func validateNinjaWeightList(weightListFilePath string) (err error) {
  827. data, err := os.ReadFile(weightListFilePath)
  828. if err != nil {
  829. return
  830. }
  831. lines := strings.Split(strings.TrimSpace(string(data)), "\n")
  832. for _, line := range lines {
  833. fields := strings.Split(line, ",")
  834. if len(fields) != 2 {
  835. return fmt.Errorf("wrong format, each line should have two fields, but '%s'", line)
  836. }
  837. _, err = strconv.Atoi(fields[1])
  838. if err != nil {
  839. return
  840. }
  841. }
  842. return
  843. }
  844. func (c *configImpl) configureLocale(ctx Context) {
  845. cmd := Command(ctx, Config{c}, "locale", "locale", "-a")
  846. output, err := cmd.Output()
  847. var locales []string
  848. if err == nil {
  849. locales = strings.Split(string(output), "\n")
  850. } else {
  851. // If we're unable to list the locales, let's assume en_US.UTF-8
  852. locales = []string{"en_US.UTF-8"}
  853. ctx.Verbosef("Failed to list locales (%q), falling back to %q", err, locales)
  854. }
  855. // gettext uses LANGUAGE, which is passed directly through
  856. // For LANG and LC_*, only preserve the evaluated version of
  857. // LC_MESSAGES
  858. userLang := ""
  859. if lc_all, ok := c.environ.Get("LC_ALL"); ok {
  860. userLang = lc_all
  861. } else if lc_messages, ok := c.environ.Get("LC_MESSAGES"); ok {
  862. userLang = lc_messages
  863. } else if lang, ok := c.environ.Get("LANG"); ok {
  864. userLang = lang
  865. }
  866. c.environ.UnsetWithPrefix("LC_")
  867. if userLang != "" {
  868. c.environ.Set("LC_MESSAGES", userLang)
  869. }
  870. // The for LANG, use C.UTF-8 if it exists (Debian currently, proposed
  871. // for others)
  872. if inList("C.UTF-8", locales) {
  873. c.environ.Set("LANG", "C.UTF-8")
  874. } else if inList("C.utf8", locales) {
  875. // These normalize to the same thing
  876. c.environ.Set("LANG", "C.UTF-8")
  877. } else if inList("en_US.UTF-8", locales) {
  878. c.environ.Set("LANG", "en_US.UTF-8")
  879. } else if inList("en_US.utf8", locales) {
  880. // These normalize to the same thing
  881. c.environ.Set("LANG", "en_US.UTF-8")
  882. } else {
  883. ctx.Fatalln("System doesn't support either C.UTF-8 or en_US.UTF-8")
  884. }
  885. }
  886. func (c *configImpl) Environment() *Environment {
  887. return c.environ
  888. }
  889. func (c *configImpl) Arguments() []string {
  890. return c.arguments
  891. }
  892. func (c *configImpl) SoongBuildInvocationNeeded() bool {
  893. if len(c.Arguments()) > 0 {
  894. // Explicit targets requested that are not special targets like b2pbuild
  895. // or the JSON module graph
  896. return true
  897. }
  898. if !c.JsonModuleGraph() && !c.Bp2Build() && !c.Queryview() && !c.SoongDocs() && !c.ApiBp2build() {
  899. // Command line was empty, the default Ninja target is built
  900. return true
  901. }
  902. // bp2build + dist may be used to dist bp2build logs but does not require SoongBuildInvocation
  903. if c.Dist() && !c.Bp2Build() {
  904. return true
  905. }
  906. // build.ninja doesn't need to be generated
  907. return false
  908. }
  909. func (c *configImpl) OutDir() string {
  910. if outDir, ok := c.environ.Get("OUT_DIR"); ok {
  911. return outDir
  912. }
  913. return "out"
  914. }
  915. func (c *configImpl) DistDir() string {
  916. return c.distDir
  917. }
  918. func (c *configImpl) RealDistDir() string {
  919. return c.distDir
  920. }
  921. func (c *configImpl) NinjaArgs() []string {
  922. if c.skipKati {
  923. return c.arguments
  924. }
  925. return c.ninjaArgs
  926. }
  927. func (c *configImpl) BazelOutDir() string {
  928. return filepath.Join(c.OutDir(), "bazel")
  929. }
  930. func (c *configImpl) bazelOutputBase() string {
  931. return filepath.Join(c.BazelOutDir(), "output")
  932. }
  933. func (c *configImpl) SoongOutDir() string {
  934. return filepath.Join(c.OutDir(), "soong")
  935. }
  936. func (c *configImpl) ApiSurfacesOutDir() string {
  937. return filepath.Join(c.OutDir(), "api_surfaces")
  938. }
  939. func (c *configImpl) PrebuiltOS() string {
  940. switch runtime.GOOS {
  941. case "linux":
  942. return "linux-x86"
  943. case "darwin":
  944. return "darwin-x86"
  945. default:
  946. panic("Unknown GOOS")
  947. }
  948. }
  949. func (c *configImpl) HostToolDir() string {
  950. if c.SkipKatiNinja() {
  951. return filepath.Join(c.SoongOutDir(), "host", c.PrebuiltOS(), "bin")
  952. } else {
  953. return filepath.Join(c.OutDir(), "host", c.PrebuiltOS(), "bin")
  954. }
  955. }
  956. func (c *configImpl) NamedGlobFile(name string) string {
  957. return shared.JoinPath(c.SoongOutDir(), "globs-"+name+".ninja")
  958. }
  959. func (c *configImpl) UsedEnvFile(tag string) string {
  960. return shared.JoinPath(c.SoongOutDir(), usedEnvFile+"."+tag)
  961. }
  962. func (c *configImpl) Bp2BuildFilesMarkerFile() string {
  963. return shared.JoinPath(c.SoongOutDir(), "bp2build_files_marker")
  964. }
  965. func (c *configImpl) Bp2BuildWorkspaceMarkerFile() string {
  966. return shared.JoinPath(c.SoongOutDir(), "bp2build_workspace_marker")
  967. }
  968. func (c *configImpl) SoongDocsHtml() string {
  969. return shared.JoinPath(c.SoongOutDir(), "docs/soong_build.html")
  970. }
  971. func (c *configImpl) QueryviewMarkerFile() string {
  972. return shared.JoinPath(c.SoongOutDir(), "queryview.marker")
  973. }
  974. func (c *configImpl) ApiBp2buildMarkerFile() string {
  975. return shared.JoinPath(c.SoongOutDir(), "api_bp2build.marker")
  976. }
  977. func (c *configImpl) ModuleGraphFile() string {
  978. return shared.JoinPath(c.SoongOutDir(), "module-graph.json")
  979. }
  980. func (c *configImpl) ModuleActionsFile() string {
  981. return shared.JoinPath(c.SoongOutDir(), "module-actions.json")
  982. }
  983. func (c *configImpl) TempDir() string {
  984. return shared.TempDirForOutDir(c.SoongOutDir())
  985. }
  986. func (c *configImpl) FileListDir() string {
  987. return filepath.Join(c.OutDir(), ".module_paths")
  988. }
  989. func (c *configImpl) KatiSuffix() string {
  990. if c.katiSuffix != "" {
  991. return c.katiSuffix
  992. }
  993. panic("SetKatiSuffix has not been called")
  994. }
  995. // Checkbuild returns true if "checkbuild" was one of the build goals, which means that the
  996. // user is interested in additional checks at the expense of build time.
  997. func (c *configImpl) Checkbuild() bool {
  998. return c.checkbuild
  999. }
  1000. func (c *configImpl) Dist() bool {
  1001. return c.dist
  1002. }
  1003. func (c *configImpl) JsonModuleGraph() bool {
  1004. return c.jsonModuleGraph
  1005. }
  1006. func (c *configImpl) Bp2Build() bool {
  1007. return c.bp2build
  1008. }
  1009. func (c *configImpl) ApiBp2build() bool {
  1010. return c.apiBp2build
  1011. }
  1012. func (c *configImpl) Queryview() bool {
  1013. return c.queryview
  1014. }
  1015. func (c *configImpl) SoongDocs() bool {
  1016. return c.soongDocs
  1017. }
  1018. func (c *configImpl) IsVerbose() bool {
  1019. return c.verbose
  1020. }
  1021. func (c *configImpl) MultitreeBuild() bool {
  1022. return c.multitreeBuild
  1023. }
  1024. func (c *configImpl) NinjaWeightListSource() NinjaWeightListSource {
  1025. return c.ninjaWeightListSource
  1026. }
  1027. func (c *configImpl) SkipKati() bool {
  1028. return c.skipKati
  1029. }
  1030. func (c *configImpl) SkipKatiNinja() bool {
  1031. return c.skipKatiNinja
  1032. }
  1033. func (c *configImpl) SkipSoong() bool {
  1034. return c.skipSoong
  1035. }
  1036. func (c *configImpl) SkipNinja() bool {
  1037. return c.skipNinja
  1038. }
  1039. func (c *configImpl) SetSkipNinja(v bool) {
  1040. c.skipNinja = v
  1041. }
  1042. func (c *configImpl) SkipConfig() bool {
  1043. return c.skipConfig
  1044. }
  1045. func (c *configImpl) BuildFromTextStub() bool {
  1046. return c.buildFromTextStub
  1047. }
  1048. func (c *configImpl) TargetProduct() string {
  1049. if v, ok := c.environ.Get("TARGET_PRODUCT"); ok {
  1050. return v
  1051. }
  1052. panic("TARGET_PRODUCT is not defined")
  1053. }
  1054. func (c *configImpl) TargetDevice() string {
  1055. return c.targetDevice
  1056. }
  1057. func (c *configImpl) SetTargetDevice(device string) {
  1058. c.targetDevice = device
  1059. }
  1060. func (c *configImpl) TargetBuildVariant() string {
  1061. if v, ok := c.environ.Get("TARGET_BUILD_VARIANT"); ok {
  1062. return v
  1063. }
  1064. panic("TARGET_BUILD_VARIANT is not defined")
  1065. }
  1066. func (c *configImpl) KatiArgs() []string {
  1067. return c.katiArgs
  1068. }
  1069. func (c *configImpl) Parallel() int {
  1070. return c.parallel
  1071. }
  1072. func (c *configImpl) GetSourceRootDirs() []string {
  1073. return c.sourceRootDirs
  1074. }
  1075. func (c *configImpl) SetSourceRootDirs(i []string) {
  1076. c.sourceRootDirs = i
  1077. }
  1078. func (c *configImpl) GetIncludeTags() []string {
  1079. return c.includeTags
  1080. }
  1081. func (c *configImpl) SetIncludeTags(i []string) {
  1082. c.includeTags = i
  1083. }
  1084. func (c *configImpl) GetLogsPrefix() string {
  1085. return c.logsPrefix
  1086. }
  1087. func (c *configImpl) SetLogsPrefix(prefix string) {
  1088. c.logsPrefix = prefix
  1089. }
  1090. func (c *configImpl) HighmemParallel() int {
  1091. if i, ok := c.environ.GetInt("NINJA_HIGHMEM_NUM_JOBS"); ok {
  1092. return i
  1093. }
  1094. const minMemPerHighmemProcess = 8 * 1024 * 1024 * 1024
  1095. parallel := c.Parallel()
  1096. if c.UseRemoteBuild() {
  1097. // Ninja doesn't support nested pools, and when remote builds are enabled the total ninja parallelism
  1098. // is set very high (i.e. 500). Using a large value here would cause the total number of running jobs
  1099. // to be the sum of the sizes of the local and highmem pools, which will cause extra CPU contention.
  1100. // Return 1/16th of the size of the local pool, rounding up.
  1101. return (parallel + 15) / 16
  1102. } else if c.totalRAM == 0 {
  1103. // Couldn't detect the total RAM, don't restrict highmem processes.
  1104. return parallel
  1105. } else if c.totalRAM <= 16*1024*1024*1024 {
  1106. // Less than 16GB of ram, restrict to 1 highmem processes
  1107. return 1
  1108. } else if c.totalRAM <= 32*1024*1024*1024 {
  1109. // Less than 32GB of ram, restrict to 2 highmem processes
  1110. return 2
  1111. } else if p := int(c.totalRAM / minMemPerHighmemProcess); p < parallel {
  1112. // If less than 8GB total RAM per process, reduce the number of highmem processes
  1113. return p
  1114. }
  1115. // No restriction on highmem processes
  1116. return parallel
  1117. }
  1118. func (c *configImpl) TotalRAM() uint64 {
  1119. return c.totalRAM
  1120. }
  1121. // ForceUseGoma determines whether we should override Goma deprecation
  1122. // and use Goma for the current build or not.
  1123. func (c *configImpl) ForceUseGoma() bool {
  1124. if v, ok := c.environ.Get("FORCE_USE_GOMA"); ok {
  1125. v = strings.TrimSpace(v)
  1126. if v != "" && v != "false" {
  1127. return true
  1128. }
  1129. }
  1130. return false
  1131. }
  1132. func (c *configImpl) UseGoma() bool {
  1133. if v, ok := c.environ.Get("USE_GOMA"); ok {
  1134. v = strings.TrimSpace(v)
  1135. if v != "" && v != "false" {
  1136. return true
  1137. }
  1138. }
  1139. return false
  1140. }
  1141. func (c *configImpl) StartGoma() bool {
  1142. if !c.UseGoma() {
  1143. return false
  1144. }
  1145. if v, ok := c.environ.Get("NOSTART_GOMA"); ok {
  1146. v = strings.TrimSpace(v)
  1147. if v != "" && v != "false" {
  1148. return false
  1149. }
  1150. }
  1151. return true
  1152. }
  1153. func (c *configImpl) UseRBE() bool {
  1154. if v, ok := c.Environment().Get("USE_RBE"); ok {
  1155. v = strings.TrimSpace(v)
  1156. if v != "" && v != "false" {
  1157. return true
  1158. }
  1159. }
  1160. return false
  1161. }
  1162. func (c *configImpl) BazelBuildEnabled() bool {
  1163. return c.bazelProdMode || c.bazelDevMode || c.bazelStagingMode
  1164. }
  1165. func (c *configImpl) StartRBE() bool {
  1166. if !c.UseRBE() {
  1167. return false
  1168. }
  1169. if v, ok := c.environ.Get("NOSTART_RBE"); ok {
  1170. v = strings.TrimSpace(v)
  1171. if v != "" && v != "false" {
  1172. return false
  1173. }
  1174. }
  1175. return true
  1176. }
  1177. func (c *configImpl) rbeProxyLogsDir() string {
  1178. for _, f := range []string{"RBE_proxy_log_dir", "FLAG_output_dir"} {
  1179. if v, ok := c.environ.Get(f); ok {
  1180. return v
  1181. }
  1182. }
  1183. buildTmpDir := shared.TempDirForOutDir(c.SoongOutDir())
  1184. return filepath.Join(buildTmpDir, "rbe")
  1185. }
  1186. func (c *configImpl) rbeCacheDir() string {
  1187. for _, f := range []string{"RBE_cache_dir", "FLAG_cache_dir"} {
  1188. if v, ok := c.environ.Get(f); ok {
  1189. return v
  1190. }
  1191. }
  1192. return shared.JoinPath(c.SoongOutDir(), "rbe")
  1193. }
  1194. func (c *configImpl) shouldCleanupRBELogsDir() bool {
  1195. // Perform a log directory cleanup only when the log directory
  1196. // is auto created by the build rather than user-specified.
  1197. for _, f := range []string{"RBE_proxy_log_dir", "FLAG_output_dir"} {
  1198. if _, ok := c.environ.Get(f); ok {
  1199. return false
  1200. }
  1201. }
  1202. return true
  1203. }
  1204. func (c *configImpl) rbeExecRoot() string {
  1205. for _, f := range []string{"RBE_exec_root", "FLAG_exec_root"} {
  1206. if v, ok := c.environ.Get(f); ok {
  1207. return v
  1208. }
  1209. }
  1210. wd, err := os.Getwd()
  1211. if err != nil {
  1212. return ""
  1213. }
  1214. return wd
  1215. }
  1216. func (c *configImpl) rbeDir() string {
  1217. if v, ok := c.environ.Get("RBE_DIR"); ok {
  1218. return v
  1219. }
  1220. return "prebuilts/remoteexecution-client/live/"
  1221. }
  1222. func (c *configImpl) rbeReproxy() string {
  1223. for _, f := range []string{"RBE_re_proxy", "FLAG_re_proxy"} {
  1224. if v, ok := c.environ.Get(f); ok {
  1225. return v
  1226. }
  1227. }
  1228. return filepath.Join(c.rbeDir(), "reproxy")
  1229. }
  1230. func (c *configImpl) rbeAuth() (string, string) {
  1231. credFlags := []string{
  1232. "use_application_default_credentials",
  1233. "use_gce_credentials",
  1234. "credential_file",
  1235. "use_google_prod_creds",
  1236. }
  1237. for _, cf := range credFlags {
  1238. for _, f := range []string{"RBE_" + cf, "FLAG_" + cf} {
  1239. if v, ok := c.environ.Get(f); ok {
  1240. v = strings.TrimSpace(v)
  1241. if v != "" && v != "false" && v != "0" {
  1242. return "RBE_" + cf, v
  1243. }
  1244. }
  1245. }
  1246. }
  1247. return "RBE_use_application_default_credentials", "true"
  1248. }
  1249. func (c *configImpl) rbeSockAddr(dir string) (string, error) {
  1250. maxNameLen := len(syscall.RawSockaddrUnix{}.Path)
  1251. base := fmt.Sprintf("reproxy_%v.sock", rbeRandPrefix)
  1252. name := filepath.Join(dir, base)
  1253. if len(name) < maxNameLen {
  1254. return name, nil
  1255. }
  1256. name = filepath.Join("/tmp", base)
  1257. if len(name) < maxNameLen {
  1258. return name, nil
  1259. }
  1260. return "", fmt.Errorf("cannot generate a proxy socket address shorter than the limit of %v", maxNameLen)
  1261. }
  1262. // IsGooglerEnvironment returns true if the current build is running
  1263. // on a Google developer machine and false otherwise.
  1264. func (c *configImpl) IsGooglerEnvironment() bool {
  1265. cf := "ANDROID_BUILD_ENVIRONMENT_CONFIG"
  1266. if v, ok := c.environ.Get(cf); ok {
  1267. return v == "googler"
  1268. }
  1269. return false
  1270. }
  1271. // GoogleProdCredsExist determine whether credentials exist on the
  1272. // Googler machine to use remote execution.
  1273. func (c *configImpl) GoogleProdCredsExist() bool {
  1274. if googleProdCredsExistCache {
  1275. return googleProdCredsExistCache
  1276. }
  1277. if _, err := exec.Command("/usr/bin/prodcertstatus", "--simple_output", "--nocheck_loas").Output(); err != nil {
  1278. return false
  1279. }
  1280. googleProdCredsExistCache = true
  1281. return true
  1282. }
  1283. // UseRemoteBuild indicates whether to use a remote build acceleration system
  1284. // to speed up the build.
  1285. func (c *configImpl) UseRemoteBuild() bool {
  1286. return c.UseGoma() || c.UseRBE()
  1287. }
  1288. // StubbyExists checks whether the stubby binary exists on the machine running
  1289. // the build.
  1290. func (c *configImpl) StubbyExists() bool {
  1291. if _, err := exec.LookPath("stubby"); err != nil {
  1292. return false
  1293. }
  1294. return true
  1295. }
  1296. // RemoteParallel controls how many remote jobs (i.e., commands which contain
  1297. // gomacc) are run in parallel. Note the parallelism of all other jobs is
  1298. // still limited by Parallel()
  1299. func (c *configImpl) RemoteParallel() int {
  1300. if !c.UseRemoteBuild() {
  1301. return 0
  1302. }
  1303. if i, ok := c.environ.GetInt("NINJA_REMOTE_NUM_JOBS"); ok {
  1304. return i
  1305. }
  1306. return 500
  1307. }
  1308. func (c *configImpl) SetKatiArgs(args []string) {
  1309. c.katiArgs = args
  1310. }
  1311. func (c *configImpl) SetNinjaArgs(args []string) {
  1312. c.ninjaArgs = args
  1313. }
  1314. func (c *configImpl) SetKatiSuffix(suffix string) {
  1315. c.katiSuffix = suffix
  1316. }
  1317. func (c *configImpl) LastKatiSuffixFile() string {
  1318. return filepath.Join(c.OutDir(), "last_kati_suffix")
  1319. }
  1320. func (c *configImpl) HasKatiSuffix() bool {
  1321. return c.katiSuffix != ""
  1322. }
  1323. func (c *configImpl) KatiEnvFile() string {
  1324. return filepath.Join(c.OutDir(), "env"+c.KatiSuffix()+".sh")
  1325. }
  1326. func (c *configImpl) KatiBuildNinjaFile() string {
  1327. return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiBuildSuffix+".ninja")
  1328. }
  1329. func (c *configImpl) KatiPackageNinjaFile() string {
  1330. return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiPackageSuffix+".ninja")
  1331. }
  1332. func (c *configImpl) SoongVarsFile() string {
  1333. return filepath.Join(c.SoongOutDir(), "soong.variables")
  1334. }
  1335. func (c *configImpl) SoongNinjaFile() string {
  1336. return filepath.Join(c.SoongOutDir(), "build.ninja")
  1337. }
  1338. func (c *configImpl) CombinedNinjaFile() string {
  1339. if c.katiSuffix == "" {
  1340. return filepath.Join(c.OutDir(), "combined.ninja")
  1341. }
  1342. return filepath.Join(c.OutDir(), "combined"+c.KatiSuffix()+".ninja")
  1343. }
  1344. func (c *configImpl) SoongAndroidMk() string {
  1345. return filepath.Join(c.SoongOutDir(), "Android-"+c.TargetProduct()+".mk")
  1346. }
  1347. func (c *configImpl) SoongMakeVarsMk() string {
  1348. return filepath.Join(c.SoongOutDir(), "make_vars-"+c.TargetProduct()+".mk")
  1349. }
  1350. func (c *configImpl) ProductOut() string {
  1351. return filepath.Join(c.OutDir(), "target", "product", c.TargetDevice())
  1352. }
  1353. func (c *configImpl) DevicePreviousProductConfig() string {
  1354. return filepath.Join(c.ProductOut(), "previous_build_config.mk")
  1355. }
  1356. func (c *configImpl) KatiPackageMkDir() string {
  1357. return filepath.Join(c.ProductOut(), "obj", "CONFIG", "kati_packaging")
  1358. }
  1359. func (c *configImpl) hostOutRoot() string {
  1360. return filepath.Join(c.OutDir(), "host")
  1361. }
  1362. func (c *configImpl) HostOut() string {
  1363. return filepath.Join(c.hostOutRoot(), c.HostPrebuiltTag())
  1364. }
  1365. // This probably needs to be multi-valued, so not exporting it for now
  1366. func (c *configImpl) hostCrossOut() string {
  1367. if runtime.GOOS == "linux" {
  1368. return filepath.Join(c.hostOutRoot(), "windows-x86")
  1369. } else {
  1370. return ""
  1371. }
  1372. }
  1373. func (c *configImpl) HostPrebuiltTag() string {
  1374. if runtime.GOOS == "linux" {
  1375. return "linux-x86"
  1376. } else if runtime.GOOS == "darwin" {
  1377. return "darwin-x86"
  1378. } else {
  1379. panic("Unsupported OS")
  1380. }
  1381. }
  1382. func (c *configImpl) PrebuiltBuildTool(name string) string {
  1383. if v, ok := c.environ.Get("SANITIZE_HOST"); ok {
  1384. if sanitize := strings.Fields(v); inList("address", sanitize) {
  1385. asan := filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "asan/bin", name)
  1386. if _, err := os.Stat(asan); err == nil {
  1387. return asan
  1388. }
  1389. }
  1390. }
  1391. return filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "bin", name)
  1392. }
  1393. func (c *configImpl) SetBuildBrokenDupRules(val bool) {
  1394. c.brokenDupRules = val
  1395. }
  1396. func (c *configImpl) BuildBrokenDupRules() bool {
  1397. return c.brokenDupRules
  1398. }
  1399. func (c *configImpl) SetBuildBrokenUsesNetwork(val bool) {
  1400. c.brokenUsesNetwork = val
  1401. }
  1402. func (c *configImpl) BuildBrokenUsesNetwork() bool {
  1403. return c.brokenUsesNetwork
  1404. }
  1405. func (c *configImpl) SetBuildBrokenNinjaUsesEnvVars(val []string) {
  1406. c.brokenNinjaEnvVars = val
  1407. }
  1408. func (c *configImpl) BuildBrokenNinjaUsesEnvVars() []string {
  1409. return c.brokenNinjaEnvVars
  1410. }
  1411. func (c *configImpl) SetTargetDeviceDir(dir string) {
  1412. c.targetDeviceDir = dir
  1413. }
  1414. func (c *configImpl) TargetDeviceDir() string {
  1415. return c.targetDeviceDir
  1416. }
  1417. func (c *configImpl) BuildDateTime() string {
  1418. return c.buildDateTime
  1419. }
  1420. func (c *configImpl) MetricsUploaderApp() string {
  1421. return c.metricsUploader
  1422. }
  1423. // LogsDir returns the absolute path to the logs directory where build log and
  1424. // metrics files are located. By default, the logs directory is the out
  1425. // directory. If the argument dist is specified, the logs directory
  1426. // is <dist_dir>/logs.
  1427. func (c *configImpl) LogsDir() string {
  1428. dir := c.OutDir()
  1429. if c.Dist() {
  1430. // Always write logs to the real dist dir, even if Bazel is using a rigged dist dir for other files
  1431. dir = filepath.Join(c.RealDistDir(), "logs")
  1432. }
  1433. absDir, err := filepath.Abs(dir)
  1434. if err != nil {
  1435. fmt.Fprintf(os.Stderr, "\nError making log dir '%s' absolute: %s\n", dir, err.Error())
  1436. os.Exit(1)
  1437. }
  1438. return absDir
  1439. }
  1440. // BazelMetricsDir returns the <logs dir>/bazel_metrics directory
  1441. // where the bazel profiles are located.
  1442. func (c *configImpl) BazelMetricsDir() string {
  1443. return filepath.Join(c.LogsDir(), "bazel_metrics")
  1444. }
  1445. // MkFileMetrics returns the file path for make-related metrics.
  1446. func (c *configImpl) MkMetrics() string {
  1447. return filepath.Join(c.LogsDir(), "mk_metrics.pb")
  1448. }
  1449. func (c *configImpl) SetEmptyNinjaFile(v bool) {
  1450. c.emptyNinjaFile = v
  1451. }
  1452. func (c *configImpl) EmptyNinjaFile() bool {
  1453. return c.emptyNinjaFile
  1454. }
  1455. func (c *configImpl) IsBazelMixedBuildForceDisabled() bool {
  1456. return c.Environment().IsEnvTrue("BUILD_BROKEN_DISABLE_BAZEL")
  1457. }
  1458. func (c *configImpl) IsPersistentBazelEnabled() bool {
  1459. return c.Environment().IsEnvTrue("USE_PERSISTENT_BAZEL")
  1460. }
  1461. func (c *configImpl) BazelModulesForceEnabledByFlag() string {
  1462. return c.bazelForceEnabledModules
  1463. }
  1464. func (c *configImpl) SkipMetricsUpload() bool {
  1465. return c.skipMetricsUpload
  1466. }
  1467. func (c *configImpl) EnsureAllowlistIntegrity() bool {
  1468. return c.ensureAllowlistIntegrity
  1469. }
  1470. // Returns a Time object if one was passed via a command-line flag.
  1471. // Otherwise returns the passed default.
  1472. func (c *configImpl) BuildStartedTimeOrDefault(defaultTime time.Time) time.Time {
  1473. if c.buildStartedTime == 0 {
  1474. return defaultTime
  1475. }
  1476. return time.UnixMilli(c.buildStartedTime)
  1477. }
  1478. func GetMetricsUploader(topDir string, env *Environment) string {
  1479. if p, ok := env.Get("METRICS_UPLOADER"); ok {
  1480. metricsUploader := filepath.Join(topDir, p)
  1481. if _, err := os.Stat(metricsUploader); err == nil {
  1482. return metricsUploader
  1483. }
  1484. }
  1485. return ""
  1486. }