config.go 44 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601
  1. // Copyright 2017 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package build
  15. import (
  16. "context"
  17. "encoding/json"
  18. "fmt"
  19. "io/ioutil"
  20. "math/rand"
  21. "os"
  22. "os/exec"
  23. "path/filepath"
  24. "runtime"
  25. "strconv"
  26. "strings"
  27. "syscall"
  28. "time"
  29. "android/soong/shared"
  30. "google.golang.org/protobuf/proto"
  31. smpb "android/soong/ui/metrics/metrics_proto"
  32. )
  33. const (
  34. envConfigDir = "vendor/google/tools/soong_config"
  35. jsonSuffix = "json"
  36. configFetcher = "vendor/google/tools/soong/expconfigfetcher"
  37. envConfigFetchTimeout = 10 * time.Second
  38. )
  39. var (
  40. rbeRandPrefix int
  41. )
  42. func init() {
  43. rand.Seed(time.Now().UnixNano())
  44. rbeRandPrefix = rand.Intn(1000)
  45. }
  46. type Config struct{ *configImpl }
  47. type configImpl struct {
  48. // Some targets that are implemented in soong_build
  49. // (bp2build, json-module-graph) are not here and have their own bits below.
  50. arguments []string
  51. goma bool
  52. environ *Environment
  53. distDir string
  54. buildDateTime string
  55. logsPrefix string
  56. // From the arguments
  57. parallel int
  58. keepGoing int
  59. verbose bool
  60. checkbuild bool
  61. dist bool
  62. jsonModuleGraph bool
  63. apiBp2build bool // Generate BUILD files for Soong modules that contribute APIs
  64. bp2build bool
  65. queryview bool
  66. reportMkMetrics bool // Collect and report mk2bp migration progress metrics.
  67. soongDocs bool
  68. skipConfig bool
  69. skipKati bool
  70. skipKatiNinja bool
  71. skipSoong bool
  72. skipNinja bool
  73. skipSoongTests bool
  74. searchApiDir bool // Scan the Android.bp files generated in out/api_surfaces
  75. skipMetricsUpload bool
  76. buildStartedTime int64 // For metrics-upload-only - manually specify a build-started time
  77. // From the product config
  78. katiArgs []string
  79. ninjaArgs []string
  80. katiSuffix string
  81. targetDevice string
  82. targetDeviceDir string
  83. sandboxConfig *SandboxConfig
  84. // Autodetected
  85. totalRAM uint64
  86. brokenDupRules bool
  87. brokenUsesNetwork bool
  88. brokenNinjaEnvVars []string
  89. pathReplaced bool
  90. bazelProdMode bool
  91. bazelDevMode bool
  92. bazelStagingMode bool
  93. // Set by multiproduct_kati
  94. emptyNinjaFile bool
  95. metricsUploader string
  96. bazelForceEnabledModules string
  97. includeTags []string
  98. }
  99. const srcDirFileCheck = "build/soong/root.bp"
  100. var buildFiles = []string{"Android.mk", "Android.bp"}
  101. type BuildAction uint
  102. const (
  103. // Builds all of the modules and their dependencies of a specified directory, relative to the root
  104. // directory of the source tree.
  105. BUILD_MODULES_IN_A_DIRECTORY BuildAction = iota
  106. // Builds all of the modules and their dependencies of a list of specified directories. All specified
  107. // directories are relative to the root directory of the source tree.
  108. BUILD_MODULES_IN_DIRECTORIES
  109. // Build a list of specified modules. If none was specified, simply build the whole source tree.
  110. BUILD_MODULES
  111. )
  112. // checkTopDir validates that the current directory is at the root directory of the source tree.
  113. func checkTopDir(ctx Context) {
  114. if _, err := os.Stat(srcDirFileCheck); err != nil {
  115. if os.IsNotExist(err) {
  116. ctx.Fatalf("Current working directory must be the source tree. %q not found.", srcDirFileCheck)
  117. }
  118. ctx.Fatalln("Error verifying tree state:", err)
  119. }
  120. }
  121. // fetchEnvConfig optionally fetches a configuration file that can then subsequently be
  122. // loaded into Soong environment to control certain aspects of build behavior (e.g., enabling RBE).
  123. // If a configuration file already exists on disk, the fetch is run in the background
  124. // so as to NOT block the rest of the build execution.
  125. func fetchEnvConfig(ctx Context, config *configImpl, envConfigName string) error {
  126. configName := envConfigName + "." + jsonSuffix
  127. expConfigFetcher := &smpb.ExpConfigFetcher{Filename: &configName}
  128. defer func() {
  129. ctx.Metrics.ExpConfigFetcher(expConfigFetcher)
  130. }()
  131. if !config.GoogleProdCredsExist() {
  132. status := smpb.ExpConfigFetcher_MISSING_GCERT
  133. expConfigFetcher.Status = &status
  134. return nil
  135. }
  136. s, err := os.Stat(configFetcher)
  137. if err != nil {
  138. if os.IsNotExist(err) {
  139. return nil
  140. }
  141. return err
  142. }
  143. if s.Mode()&0111 == 0 {
  144. status := smpb.ExpConfigFetcher_ERROR
  145. expConfigFetcher.Status = &status
  146. return fmt.Errorf("configuration fetcher binary %v is not executable: %v", configFetcher, s.Mode())
  147. }
  148. configExists := false
  149. outConfigFilePath := filepath.Join(config.OutDir(), configName)
  150. if _, err := os.Stat(outConfigFilePath); err == nil {
  151. configExists = true
  152. }
  153. tCtx, cancel := context.WithTimeout(ctx, envConfigFetchTimeout)
  154. fetchStart := time.Now()
  155. cmd := exec.CommandContext(tCtx, configFetcher, "-output_config_dir", config.OutDir(),
  156. "-output_config_name", configName)
  157. if err := cmd.Start(); err != nil {
  158. status := smpb.ExpConfigFetcher_ERROR
  159. expConfigFetcher.Status = &status
  160. return err
  161. }
  162. fetchCfg := func() error {
  163. if err := cmd.Wait(); err != nil {
  164. status := smpb.ExpConfigFetcher_ERROR
  165. expConfigFetcher.Status = &status
  166. return err
  167. }
  168. fetchEnd := time.Now()
  169. expConfigFetcher.Micros = proto.Uint64(uint64(fetchEnd.Sub(fetchStart).Microseconds()))
  170. expConfigFetcher.Filename = proto.String(outConfigFilePath)
  171. if _, err := os.Stat(outConfigFilePath); err != nil {
  172. status := smpb.ExpConfigFetcher_NO_CONFIG
  173. expConfigFetcher.Status = &status
  174. return err
  175. }
  176. status := smpb.ExpConfigFetcher_CONFIG
  177. expConfigFetcher.Status = &status
  178. return nil
  179. }
  180. // If a config file does not exist, wait for the config file to be fetched. Otherwise
  181. // fetch the config file in the background and return immediately.
  182. if !configExists {
  183. defer cancel()
  184. return fetchCfg()
  185. }
  186. go func() {
  187. defer cancel()
  188. if err := fetchCfg(); err != nil {
  189. ctx.Verbosef("Failed to fetch config file %v: %v\n", configName, err)
  190. }
  191. }()
  192. return nil
  193. }
  194. func loadEnvConfig(ctx Context, config *configImpl, bc string) error {
  195. if bc == "" {
  196. return nil
  197. }
  198. configDirs := []string{
  199. config.OutDir(),
  200. os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG_DIR"),
  201. envConfigDir,
  202. }
  203. for _, dir := range configDirs {
  204. cfgFile := filepath.Join(os.Getenv("TOP"), dir, fmt.Sprintf("%s.%s", bc, jsonSuffix))
  205. envVarsJSON, err := ioutil.ReadFile(cfgFile)
  206. if err != nil {
  207. continue
  208. }
  209. ctx.Verbosef("Loading config file %v\n", cfgFile)
  210. var envVars map[string]map[string]string
  211. if err := json.Unmarshal(envVarsJSON, &envVars); err != nil {
  212. fmt.Fprintf(os.Stderr, "Env vars config file %s did not parse correctly: %s", cfgFile, err.Error())
  213. continue
  214. }
  215. for k, v := range envVars["env"] {
  216. if os.Getenv(k) != "" {
  217. continue
  218. }
  219. config.environ.Set(k, v)
  220. }
  221. ctx.Verbosef("Finished loading config file %v\n", cfgFile)
  222. break
  223. }
  224. return nil
  225. }
  226. func defaultBazelProdMode(cfg *configImpl) bool {
  227. // Environment flag to disable Bazel for users which experience
  228. // broken bazel-handled builds, or significant performance regressions.
  229. if cfg.IsBazelMixedBuildForceDisabled() {
  230. return false
  231. }
  232. // Darwin-host builds are currently untested with Bazel.
  233. if runtime.GOOS == "darwin" {
  234. return false
  235. }
  236. return true
  237. }
  238. func UploadOnlyConfig(ctx Context, _ ...string) Config {
  239. ret := &configImpl{
  240. environ: OsEnvironment(),
  241. sandboxConfig: &SandboxConfig{},
  242. }
  243. srcDir := absPath(ctx, ".")
  244. bc := os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG")
  245. if err := loadEnvConfig(ctx, ret, bc); err != nil {
  246. ctx.Fatalln("Failed to parse env config files: %v", err)
  247. }
  248. ret.metricsUploader = GetMetricsUploader(srcDir, ret.environ)
  249. return Config{ret}
  250. }
  251. func NewConfig(ctx Context, args ...string) Config {
  252. ret := &configImpl{
  253. environ: OsEnvironment(),
  254. sandboxConfig: &SandboxConfig{},
  255. }
  256. // Default matching ninja
  257. ret.parallel = runtime.NumCPU() + 2
  258. ret.keepGoing = 1
  259. ret.totalRAM = detectTotalRAM(ctx)
  260. ret.parseArgs(ctx, args)
  261. // Make sure OUT_DIR is set appropriately
  262. if outDir, ok := ret.environ.Get("OUT_DIR"); ok {
  263. ret.environ.Set("OUT_DIR", filepath.Clean(outDir))
  264. } else {
  265. outDir := "out"
  266. if baseDir, ok := ret.environ.Get("OUT_DIR_COMMON_BASE"); ok {
  267. if wd, err := os.Getwd(); err != nil {
  268. ctx.Fatalln("Failed to get working directory:", err)
  269. } else {
  270. outDir = filepath.Join(baseDir, filepath.Base(wd))
  271. }
  272. }
  273. ret.environ.Set("OUT_DIR", outDir)
  274. }
  275. // loadEnvConfig needs to know what the OUT_DIR is, so it should
  276. // be called after we determine the appropriate out directory.
  277. bc := os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG")
  278. if bc != "" {
  279. if err := fetchEnvConfig(ctx, ret, bc); err != nil {
  280. ctx.Verbosef("Failed to fetch config file: %v\n", err)
  281. }
  282. if err := loadEnvConfig(ctx, ret, bc); err != nil {
  283. ctx.Fatalln("Failed to parse env config files: %v", err)
  284. }
  285. }
  286. if distDir, ok := ret.environ.Get("DIST_DIR"); ok {
  287. ret.distDir = filepath.Clean(distDir)
  288. } else {
  289. ret.distDir = filepath.Join(ret.OutDir(), "dist")
  290. }
  291. if srcDirIsWritable, ok := ret.environ.Get("BUILD_BROKEN_SRC_DIR_IS_WRITABLE"); ok {
  292. ret.sandboxConfig.SetSrcDirIsRO(srcDirIsWritable == "false")
  293. }
  294. ret.environ.Unset(
  295. // We're already using it
  296. "USE_SOONG_UI",
  297. // We should never use GOROOT/GOPATH from the shell environment
  298. "GOROOT",
  299. "GOPATH",
  300. // These should only come from Soong, not the environment.
  301. "CLANG",
  302. "CLANG_CXX",
  303. "CCC_CC",
  304. "CCC_CXX",
  305. // Used by the goma compiler wrapper, but should only be set by
  306. // gomacc
  307. "GOMACC_PATH",
  308. // We handle this above
  309. "OUT_DIR_COMMON_BASE",
  310. // This is handled above too, and set for individual commands later
  311. "DIST_DIR",
  312. // Variables that have caused problems in the past
  313. "BASH_ENV",
  314. "CDPATH",
  315. "DISPLAY",
  316. "GREP_OPTIONS",
  317. "JAVAC",
  318. "NDK_ROOT",
  319. "POSIXLY_CORRECT",
  320. // Drop make flags
  321. "MAKEFLAGS",
  322. "MAKELEVEL",
  323. "MFLAGS",
  324. // Set in envsetup.sh, reset in makefiles
  325. "ANDROID_JAVA_TOOLCHAIN",
  326. // Set by envsetup.sh, but shouldn't be used inside the build because envsetup.sh is optional
  327. "ANDROID_BUILD_TOP",
  328. "ANDROID_HOST_OUT",
  329. "ANDROID_PRODUCT_OUT",
  330. "ANDROID_HOST_OUT_TESTCASES",
  331. "ANDROID_TARGET_OUT_TESTCASES",
  332. "ANDROID_TOOLCHAIN",
  333. "ANDROID_TOOLCHAIN_2ND_ARCH",
  334. "ANDROID_DEV_SCRIPTS",
  335. "ANDROID_EMULATOR_PREBUILTS",
  336. "ANDROID_PRE_BUILD_PATHS",
  337. )
  338. if ret.UseGoma() || ret.ForceUseGoma() {
  339. ctx.Println("Goma for Android has been deprecated and replaced with RBE. See go/rbe_for_android for instructions on how to use RBE.")
  340. ctx.Fatalln("USE_GOMA / FORCE_USE_GOMA flag is no longer supported.")
  341. }
  342. // Tell python not to spam the source tree with .pyc files.
  343. ret.environ.Set("PYTHONDONTWRITEBYTECODE", "1")
  344. tmpDir := absPath(ctx, ret.TempDir())
  345. ret.environ.Set("TMPDIR", tmpDir)
  346. // Always set ASAN_SYMBOLIZER_PATH so that ASAN-based tools can symbolize any crashes
  347. symbolizerPath := filepath.Join("prebuilts/clang/host", ret.HostPrebuiltTag(),
  348. "llvm-binutils-stable/llvm-symbolizer")
  349. ret.environ.Set("ASAN_SYMBOLIZER_PATH", absPath(ctx, symbolizerPath))
  350. // Precondition: the current directory is the top of the source tree
  351. checkTopDir(ctx)
  352. srcDir := absPath(ctx, ".")
  353. if strings.ContainsRune(srcDir, ' ') {
  354. ctx.Println("You are building in a directory whose absolute path contains a space character:")
  355. ctx.Println()
  356. ctx.Printf("%q\n", srcDir)
  357. ctx.Println()
  358. ctx.Fatalln("Directory names containing spaces are not supported")
  359. }
  360. ret.metricsUploader = GetMetricsUploader(srcDir, ret.environ)
  361. if outDir := ret.OutDir(); strings.ContainsRune(outDir, ' ') {
  362. ctx.Println("The absolute path of your output directory ($OUT_DIR) contains a space character:")
  363. ctx.Println()
  364. ctx.Printf("%q\n", outDir)
  365. ctx.Println()
  366. ctx.Fatalln("Directory names containing spaces are not supported")
  367. }
  368. if distDir := ret.RealDistDir(); strings.ContainsRune(distDir, ' ') {
  369. ctx.Println("The absolute path of your dist directory ($DIST_DIR) contains a space character:")
  370. ctx.Println()
  371. ctx.Printf("%q\n", distDir)
  372. ctx.Println()
  373. ctx.Fatalln("Directory names containing spaces are not supported")
  374. }
  375. // Configure Java-related variables, including adding it to $PATH
  376. java8Home := filepath.Join("prebuilts/jdk/jdk8", ret.HostPrebuiltTag())
  377. java9Home := filepath.Join("prebuilts/jdk/jdk9", ret.HostPrebuiltTag())
  378. java11Home := filepath.Join("prebuilts/jdk/jdk11", ret.HostPrebuiltTag())
  379. java17Home := filepath.Join("prebuilts/jdk/jdk17", ret.HostPrebuiltTag())
  380. javaHome := func() string {
  381. if override, ok := ret.environ.Get("OVERRIDE_ANDROID_JAVA_HOME"); ok {
  382. return override
  383. }
  384. if toolchain11, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN"); ok && toolchain11 != "true" {
  385. ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN is no longer supported. An OpenJDK 11 toolchain is now the global default.")
  386. }
  387. if toolchain17, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK17_TOOLCHAIN"); ok && toolchain17 != "true" {
  388. ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK17_TOOLCHAIN is no longer supported. An OpenJDK 17 toolchain is now the global default.")
  389. }
  390. return java17Home
  391. }()
  392. absJavaHome := absPath(ctx, javaHome)
  393. ret.configureLocale(ctx)
  394. newPath := []string{filepath.Join(absJavaHome, "bin")}
  395. if path, ok := ret.environ.Get("PATH"); ok && path != "" {
  396. newPath = append(newPath, path)
  397. }
  398. ret.environ.Unset("OVERRIDE_ANDROID_JAVA_HOME")
  399. ret.environ.Set("JAVA_HOME", absJavaHome)
  400. ret.environ.Set("ANDROID_JAVA_HOME", javaHome)
  401. ret.environ.Set("ANDROID_JAVA8_HOME", java8Home)
  402. ret.environ.Set("ANDROID_JAVA9_HOME", java9Home)
  403. ret.environ.Set("ANDROID_JAVA11_HOME", java11Home)
  404. ret.environ.Set("PATH", strings.Join(newPath, string(filepath.ListSeparator)))
  405. outDir := ret.OutDir()
  406. buildDateTimeFile := filepath.Join(outDir, "build_date.txt")
  407. if buildDateTime, ok := ret.environ.Get("BUILD_DATETIME"); ok && buildDateTime != "" {
  408. ret.buildDateTime = buildDateTime
  409. } else {
  410. ret.buildDateTime = strconv.FormatInt(time.Now().Unix(), 10)
  411. }
  412. ret.environ.Set("BUILD_DATETIME_FILE", buildDateTimeFile)
  413. if ret.UseRBE() {
  414. for k, v := range getRBEVars(ctx, Config{ret}) {
  415. ret.environ.Set(k, v)
  416. }
  417. }
  418. bpd := ret.BazelMetricsDir()
  419. if err := os.RemoveAll(bpd); err != nil {
  420. ctx.Fatalf("Unable to remove bazel profile directory %q: %v", bpd, err)
  421. }
  422. c := Config{ret}
  423. storeConfigMetrics(ctx, c)
  424. return c
  425. }
  426. // NewBuildActionConfig returns a build configuration based on the build action. The arguments are
  427. // processed based on the build action and extracts any arguments that belongs to the build action.
  428. func NewBuildActionConfig(action BuildAction, dir string, ctx Context, args ...string) Config {
  429. return NewConfig(ctx, getConfigArgs(action, dir, ctx, args)...)
  430. }
  431. // storeConfigMetrics selects a set of configuration information and store in
  432. // the metrics system for further analysis.
  433. func storeConfigMetrics(ctx Context, config Config) {
  434. if ctx.Metrics == nil {
  435. return
  436. }
  437. ctx.Metrics.BuildConfig(buildConfig(config))
  438. s := &smpb.SystemResourceInfo{
  439. TotalPhysicalMemory: proto.Uint64(config.TotalRAM()),
  440. AvailableCpus: proto.Int32(int32(runtime.NumCPU())),
  441. }
  442. ctx.Metrics.SystemResourceInfo(s)
  443. }
  444. func buildConfig(config Config) *smpb.BuildConfig {
  445. c := &smpb.BuildConfig{
  446. ForceUseGoma: proto.Bool(config.ForceUseGoma()),
  447. UseGoma: proto.Bool(config.UseGoma()),
  448. UseRbe: proto.Bool(config.UseRBE()),
  449. BazelMixedBuild: proto.Bool(config.BazelBuildEnabled()),
  450. ForceDisableBazelMixedBuild: proto.Bool(config.IsBazelMixedBuildForceDisabled()),
  451. }
  452. c.Targets = append(c.Targets, config.arguments...)
  453. return c
  454. }
  455. // getConfigArgs processes the command arguments based on the build action and creates a set of new
  456. // arguments to be accepted by Config.
  457. func getConfigArgs(action BuildAction, dir string, ctx Context, args []string) []string {
  458. // The next block of code verifies that the current directory is the root directory of the source
  459. // tree. It then finds the relative path of dir based on the root directory of the source tree
  460. // and verify that dir is inside of the source tree.
  461. checkTopDir(ctx)
  462. topDir, err := os.Getwd()
  463. if err != nil {
  464. ctx.Fatalf("Error retrieving top directory: %v", err)
  465. }
  466. dir, err = filepath.EvalSymlinks(dir)
  467. if err != nil {
  468. ctx.Fatalf("Unable to evaluate symlink of %s: %v", dir, err)
  469. }
  470. dir, err = filepath.Abs(dir)
  471. if err != nil {
  472. ctx.Fatalf("Unable to find absolute path %s: %v", dir, err)
  473. }
  474. relDir, err := filepath.Rel(topDir, dir)
  475. if err != nil {
  476. ctx.Fatalf("Unable to find relative path %s of %s: %v", relDir, topDir, err)
  477. }
  478. // If there are ".." in the path, it's not in the source tree.
  479. if strings.Contains(relDir, "..") {
  480. ctx.Fatalf("Directory %s is not under the source tree %s", dir, topDir)
  481. }
  482. configArgs := args[:]
  483. // If the arguments contains GET-INSTALL-PATH, change the target name prefix from MODULES-IN- to
  484. // GET-INSTALL-PATH-IN- to extract the installation path instead of building the modules.
  485. targetNamePrefix := "MODULES-IN-"
  486. if inList("GET-INSTALL-PATH", configArgs) {
  487. targetNamePrefix = "GET-INSTALL-PATH-IN-"
  488. configArgs = removeFromList("GET-INSTALL-PATH", configArgs)
  489. }
  490. var targets []string
  491. switch action {
  492. case BUILD_MODULES:
  493. // No additional processing is required when building a list of specific modules or all modules.
  494. case BUILD_MODULES_IN_A_DIRECTORY:
  495. // If dir is the root source tree, all the modules are built of the source tree are built so
  496. // no need to find the build file.
  497. if topDir == dir {
  498. break
  499. }
  500. buildFile := findBuildFile(ctx, relDir)
  501. if buildFile == "" {
  502. ctx.Fatalf("Build file not found for %s directory", relDir)
  503. }
  504. targets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)}
  505. case BUILD_MODULES_IN_DIRECTORIES:
  506. newConfigArgs, dirs := splitArgs(configArgs)
  507. configArgs = newConfigArgs
  508. targets = getTargetsFromDirs(ctx, relDir, dirs, targetNamePrefix)
  509. }
  510. // Tidy only override all other specified targets.
  511. tidyOnly := os.Getenv("WITH_TIDY_ONLY")
  512. if tidyOnly == "true" || tidyOnly == "1" {
  513. configArgs = append(configArgs, "tidy_only")
  514. } else {
  515. configArgs = append(configArgs, targets...)
  516. }
  517. return configArgs
  518. }
  519. // convertToTarget replaces "/" to "-" in dir and pre-append the targetNamePrefix to the target name.
  520. func convertToTarget(dir string, targetNamePrefix string) string {
  521. return targetNamePrefix + strings.ReplaceAll(dir, "/", "-")
  522. }
  523. // hasBuildFile returns true if dir contains an Android build file.
  524. func hasBuildFile(ctx Context, dir string) bool {
  525. for _, buildFile := range buildFiles {
  526. _, err := os.Stat(filepath.Join(dir, buildFile))
  527. if err == nil {
  528. return true
  529. }
  530. if !os.IsNotExist(err) {
  531. ctx.Fatalf("Error retrieving the build file stats: %v", err)
  532. }
  533. }
  534. return false
  535. }
  536. // findBuildFile finds a build file (makefile or blueprint file) by looking if there is a build file
  537. // in the current and any sub directory of dir. If a build file is not found, traverse the path
  538. // up by one directory and repeat again until either a build file is found or reached to the root
  539. // source tree. The returned filename of build file is "Android.mk". If one was not found, a blank
  540. // string is returned.
  541. func findBuildFile(ctx Context, dir string) string {
  542. // If the string is empty or ".", assume it is top directory of the source tree.
  543. if dir == "" || dir == "." {
  544. return ""
  545. }
  546. found := false
  547. for buildDir := dir; buildDir != "."; buildDir = filepath.Dir(buildDir) {
  548. err := filepath.Walk(buildDir, func(path string, info os.FileInfo, err error) error {
  549. if err != nil {
  550. return err
  551. }
  552. if found {
  553. return filepath.SkipDir
  554. }
  555. if info.IsDir() {
  556. return nil
  557. }
  558. for _, buildFile := range buildFiles {
  559. if info.Name() == buildFile {
  560. found = true
  561. return filepath.SkipDir
  562. }
  563. }
  564. return nil
  565. })
  566. if err != nil {
  567. ctx.Fatalf("Error finding Android build file: %v", err)
  568. }
  569. if found {
  570. return filepath.Join(buildDir, "Android.mk")
  571. }
  572. }
  573. return ""
  574. }
  575. // splitArgs iterates over the arguments list and splits into two lists: arguments and directories.
  576. func splitArgs(args []string) (newArgs []string, dirs []string) {
  577. specialArgs := map[string]bool{
  578. "showcommands": true,
  579. "snod": true,
  580. "dist": true,
  581. "checkbuild": true,
  582. }
  583. newArgs = []string{}
  584. dirs = []string{}
  585. for _, arg := range args {
  586. // It's a dash argument if it starts with "-" or it's a key=value pair, it's not a directory.
  587. if strings.IndexRune(arg, '-') == 0 || strings.IndexRune(arg, '=') != -1 {
  588. newArgs = append(newArgs, arg)
  589. continue
  590. }
  591. if _, ok := specialArgs[arg]; ok {
  592. newArgs = append(newArgs, arg)
  593. continue
  594. }
  595. dirs = append(dirs, arg)
  596. }
  597. return newArgs, dirs
  598. }
  599. // getTargetsFromDirs iterates over the dirs list and creates a list of targets to build. If a
  600. // directory from the dirs list does not exist, a fatal error is raised. relDir is related to the
  601. // source root tree where the build action command was invoked. Each directory is validated if the
  602. // build file can be found and follows the format "dir1:target1,target2,...". Target is optional.
  603. func getTargetsFromDirs(ctx Context, relDir string, dirs []string, targetNamePrefix string) (targets []string) {
  604. for _, dir := range dirs {
  605. // The directory may have specified specific modules to build. ":" is the separator to separate
  606. // the directory and the list of modules.
  607. s := strings.Split(dir, ":")
  608. l := len(s)
  609. if l > 2 { // more than one ":" was specified.
  610. ctx.Fatalf("%s not in proper directory:target1,target2,... format (\":\" was specified more than once)", dir)
  611. }
  612. dir = filepath.Join(relDir, s[0])
  613. if _, err := os.Stat(dir); err != nil {
  614. ctx.Fatalf("couldn't find directory %s", dir)
  615. }
  616. // Verify that if there are any targets specified after ":". Each target is separated by ",".
  617. var newTargets []string
  618. if l == 2 && s[1] != "" {
  619. newTargets = strings.Split(s[1], ",")
  620. if inList("", newTargets) {
  621. ctx.Fatalf("%s not in proper directory:target1,target2,... format", dir)
  622. }
  623. }
  624. // If there are specified targets to build in dir, an android build file must exist for the one
  625. // shot build. For the non-targets case, find the appropriate build file and build all the
  626. // modules in dir (or the closest one in the dir path).
  627. if len(newTargets) > 0 {
  628. if !hasBuildFile(ctx, dir) {
  629. ctx.Fatalf("Couldn't locate a build file from %s directory", dir)
  630. }
  631. } else {
  632. buildFile := findBuildFile(ctx, dir)
  633. if buildFile == "" {
  634. ctx.Fatalf("Build file not found for %s directory", dir)
  635. }
  636. newTargets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)}
  637. }
  638. targets = append(targets, newTargets...)
  639. }
  640. return targets
  641. }
  642. func (c *configImpl) parseArgs(ctx Context, args []string) {
  643. for i := 0; i < len(args); i++ {
  644. arg := strings.TrimSpace(args[i])
  645. if arg == "showcommands" {
  646. c.verbose = true
  647. } else if arg == "--empty-ninja-file" {
  648. c.emptyNinjaFile = true
  649. } else if arg == "--skip-ninja" {
  650. c.skipNinja = true
  651. } else if arg == "--skip-make" {
  652. // TODO(ccross): deprecate this, it has confusing behaviors. It doesn't run kati,
  653. // but it does run a Kati ninja file if the .kati_enabled marker file was created
  654. // by a previous build.
  655. c.skipConfig = true
  656. c.skipKati = true
  657. } else if arg == "--skip-kati" {
  658. // TODO: remove --skip-kati once module builds have been migrated to --song-only
  659. c.skipKati = true
  660. } else if arg == "--soong-only" {
  661. c.skipKati = true
  662. c.skipKatiNinja = true
  663. } else if arg == "--config-only" {
  664. c.skipKati = true
  665. c.skipKatiNinja = true
  666. c.skipSoong = true
  667. } else if arg == "--skip-config" {
  668. c.skipConfig = true
  669. } else if arg == "--skip-soong-tests" {
  670. c.skipSoongTests = true
  671. } else if arg == "--skip-metrics-upload" {
  672. c.skipMetricsUpload = true
  673. } else if arg == "--mk-metrics" {
  674. c.reportMkMetrics = true
  675. } else if arg == "--bazel-mode" {
  676. c.bazelProdMode = true
  677. } else if arg == "--bazel-mode-dev" {
  678. c.bazelDevMode = true
  679. } else if arg == "--bazel-mode-staging" {
  680. c.bazelStagingMode = true
  681. } else if arg == "--search-api-dir" {
  682. c.searchApiDir = true
  683. } else if strings.HasPrefix(arg, "--build-command=") {
  684. buildCmd := strings.TrimPrefix(arg, "--build-command=")
  685. // remove quotations
  686. buildCmd = strings.TrimPrefix(buildCmd, "\"")
  687. buildCmd = strings.TrimSuffix(buildCmd, "\"")
  688. ctx.Metrics.SetBuildCommand([]string{buildCmd})
  689. } else if strings.HasPrefix(arg, "--bazel-force-enabled-modules=") {
  690. c.bazelForceEnabledModules = strings.TrimPrefix(arg, "--bazel-force-enabled-modules=")
  691. } else if strings.HasPrefix(arg, "--build-started-time-unix-millis=") {
  692. buildTimeStr := strings.TrimPrefix(arg, "--build-started-time-unix-millis=")
  693. val, err := strconv.ParseInt(buildTimeStr, 10, 64)
  694. if err == nil {
  695. c.buildStartedTime = val
  696. } else {
  697. ctx.Fatalf("Error parsing build-time-started-unix-millis", err)
  698. }
  699. } else if len(arg) > 0 && arg[0] == '-' {
  700. parseArgNum := func(def int) int {
  701. if len(arg) > 2 {
  702. p, err := strconv.ParseUint(arg[2:], 10, 31)
  703. if err != nil {
  704. ctx.Fatalf("Failed to parse %q: %v", arg, err)
  705. }
  706. return int(p)
  707. } else if i+1 < len(args) {
  708. p, err := strconv.ParseUint(args[i+1], 10, 31)
  709. if err == nil {
  710. i++
  711. return int(p)
  712. }
  713. }
  714. return def
  715. }
  716. if len(arg) > 1 && arg[1] == 'j' {
  717. c.parallel = parseArgNum(c.parallel)
  718. } else if len(arg) > 1 && arg[1] == 'k' {
  719. c.keepGoing = parseArgNum(0)
  720. } else {
  721. ctx.Fatalln("Unknown option:", arg)
  722. }
  723. } else if k, v, ok := decodeKeyValue(arg); ok && len(k) > 0 {
  724. if k == "OUT_DIR" {
  725. ctx.Fatalln("OUT_DIR may only be set in the environment, not as a command line option.")
  726. }
  727. c.environ.Set(k, v)
  728. } else if arg == "dist" {
  729. c.dist = true
  730. } else if arg == "json-module-graph" {
  731. c.jsonModuleGraph = true
  732. } else if arg == "bp2build" {
  733. c.bp2build = true
  734. } else if arg == "api_bp2build" {
  735. c.apiBp2build = true
  736. } else if arg == "queryview" {
  737. c.queryview = true
  738. } else if arg == "soong_docs" {
  739. c.soongDocs = true
  740. } else {
  741. if arg == "checkbuild" {
  742. c.checkbuild = true
  743. }
  744. c.arguments = append(c.arguments, arg)
  745. }
  746. }
  747. if (!c.bazelProdMode) && (!c.bazelDevMode) && (!c.bazelStagingMode) {
  748. c.bazelProdMode = defaultBazelProdMode(c)
  749. }
  750. }
  751. func (c *configImpl) configureLocale(ctx Context) {
  752. cmd := Command(ctx, Config{c}, "locale", "locale", "-a")
  753. output, err := cmd.Output()
  754. var locales []string
  755. if err == nil {
  756. locales = strings.Split(string(output), "\n")
  757. } else {
  758. // If we're unable to list the locales, let's assume en_US.UTF-8
  759. locales = []string{"en_US.UTF-8"}
  760. ctx.Verbosef("Failed to list locales (%q), falling back to %q", err, locales)
  761. }
  762. // gettext uses LANGUAGE, which is passed directly through
  763. // For LANG and LC_*, only preserve the evaluated version of
  764. // LC_MESSAGES
  765. userLang := ""
  766. if lc_all, ok := c.environ.Get("LC_ALL"); ok {
  767. userLang = lc_all
  768. } else if lc_messages, ok := c.environ.Get("LC_MESSAGES"); ok {
  769. userLang = lc_messages
  770. } else if lang, ok := c.environ.Get("LANG"); ok {
  771. userLang = lang
  772. }
  773. c.environ.UnsetWithPrefix("LC_")
  774. if userLang != "" {
  775. c.environ.Set("LC_MESSAGES", userLang)
  776. }
  777. // The for LANG, use C.UTF-8 if it exists (Debian currently, proposed
  778. // for others)
  779. if inList("C.UTF-8", locales) {
  780. c.environ.Set("LANG", "C.UTF-8")
  781. } else if inList("C.utf8", locales) {
  782. // These normalize to the same thing
  783. c.environ.Set("LANG", "C.UTF-8")
  784. } else if inList("en_US.UTF-8", locales) {
  785. c.environ.Set("LANG", "en_US.UTF-8")
  786. } else if inList("en_US.utf8", locales) {
  787. // These normalize to the same thing
  788. c.environ.Set("LANG", "en_US.UTF-8")
  789. } else {
  790. ctx.Fatalln("System doesn't support either C.UTF-8 or en_US.UTF-8")
  791. }
  792. }
  793. func (c *configImpl) Environment() *Environment {
  794. return c.environ
  795. }
  796. func (c *configImpl) Arguments() []string {
  797. return c.arguments
  798. }
  799. func (c *configImpl) SoongBuildInvocationNeeded() bool {
  800. if len(c.Arguments()) > 0 {
  801. // Explicit targets requested that are not special targets like b2pbuild
  802. // or the JSON module graph
  803. return true
  804. }
  805. if !c.JsonModuleGraph() && !c.Bp2Build() && !c.Queryview() && !c.SoongDocs() && !c.ApiBp2build() {
  806. // Command line was empty, the default Ninja target is built
  807. return true
  808. }
  809. // bp2build + dist may be used to dist bp2build logs but does not require SoongBuildInvocation
  810. if c.Dist() && !c.Bp2Build() {
  811. return true
  812. }
  813. // build.ninja doesn't need to be generated
  814. return false
  815. }
  816. func (c *configImpl) OutDir() string {
  817. if outDir, ok := c.environ.Get("OUT_DIR"); ok {
  818. return outDir
  819. }
  820. return "out"
  821. }
  822. func (c *configImpl) DistDir() string {
  823. return c.distDir
  824. }
  825. func (c *configImpl) RealDistDir() string {
  826. return c.distDir
  827. }
  828. func (c *configImpl) NinjaArgs() []string {
  829. if c.skipKati {
  830. return c.arguments
  831. }
  832. return c.ninjaArgs
  833. }
  834. func (c *configImpl) BazelOutDir() string {
  835. return filepath.Join(c.OutDir(), "bazel")
  836. }
  837. func (c *configImpl) bazelOutputBase() string {
  838. return filepath.Join(c.BazelOutDir(), "output")
  839. }
  840. func (c *configImpl) SoongOutDir() string {
  841. return filepath.Join(c.OutDir(), "soong")
  842. }
  843. func (c *configImpl) ApiSurfacesOutDir() string {
  844. return filepath.Join(c.OutDir(), "api_surfaces")
  845. }
  846. func (c *configImpl) PrebuiltOS() string {
  847. switch runtime.GOOS {
  848. case "linux":
  849. return "linux-x86"
  850. case "darwin":
  851. return "darwin-x86"
  852. default:
  853. panic("Unknown GOOS")
  854. }
  855. }
  856. func (c *configImpl) HostToolDir() string {
  857. if c.SkipKatiNinja() {
  858. return filepath.Join(c.SoongOutDir(), "host", c.PrebuiltOS(), "bin")
  859. } else {
  860. return filepath.Join(c.OutDir(), "host", c.PrebuiltOS(), "bin")
  861. }
  862. }
  863. func (c *configImpl) NamedGlobFile(name string) string {
  864. return shared.JoinPath(c.SoongOutDir(), "globs-"+name+".ninja")
  865. }
  866. func (c *configImpl) UsedEnvFile(tag string) string {
  867. return shared.JoinPath(c.SoongOutDir(), usedEnvFile+"."+tag)
  868. }
  869. func (c *configImpl) Bp2BuildFilesMarkerFile() string {
  870. return shared.JoinPath(c.SoongOutDir(), "bp2build_files_marker")
  871. }
  872. func (c *configImpl) Bp2BuildWorkspaceMarkerFile() string {
  873. return shared.JoinPath(c.SoongOutDir(), "bp2build_workspace_marker")
  874. }
  875. func (c *configImpl) SoongDocsHtml() string {
  876. return shared.JoinPath(c.SoongOutDir(), "docs/soong_build.html")
  877. }
  878. func (c *configImpl) QueryviewMarkerFile() string {
  879. return shared.JoinPath(c.SoongOutDir(), "queryview.marker")
  880. }
  881. func (c *configImpl) ApiBp2buildMarkerFile() string {
  882. return shared.JoinPath(c.SoongOutDir(), "api_bp2build.marker")
  883. }
  884. func (c *configImpl) ModuleGraphFile() string {
  885. return shared.JoinPath(c.SoongOutDir(), "module-graph.json")
  886. }
  887. func (c *configImpl) ModuleActionsFile() string {
  888. return shared.JoinPath(c.SoongOutDir(), "module-actions.json")
  889. }
  890. func (c *configImpl) TempDir() string {
  891. return shared.TempDirForOutDir(c.SoongOutDir())
  892. }
  893. func (c *configImpl) FileListDir() string {
  894. return filepath.Join(c.OutDir(), ".module_paths")
  895. }
  896. func (c *configImpl) KatiSuffix() string {
  897. if c.katiSuffix != "" {
  898. return c.katiSuffix
  899. }
  900. panic("SetKatiSuffix has not been called")
  901. }
  902. // Checkbuild returns true if "checkbuild" was one of the build goals, which means that the
  903. // user is interested in additional checks at the expense of build time.
  904. func (c *configImpl) Checkbuild() bool {
  905. return c.checkbuild
  906. }
  907. func (c *configImpl) Dist() bool {
  908. return c.dist
  909. }
  910. func (c *configImpl) JsonModuleGraph() bool {
  911. return c.jsonModuleGraph
  912. }
  913. func (c *configImpl) Bp2Build() bool {
  914. return c.bp2build
  915. }
  916. func (c *configImpl) ApiBp2build() bool {
  917. return c.apiBp2build
  918. }
  919. func (c *configImpl) Queryview() bool {
  920. return c.queryview
  921. }
  922. func (c *configImpl) SoongDocs() bool {
  923. return c.soongDocs
  924. }
  925. func (c *configImpl) IsVerbose() bool {
  926. return c.verbose
  927. }
  928. func (c *configImpl) SkipKati() bool {
  929. return c.skipKati
  930. }
  931. func (c *configImpl) SkipKatiNinja() bool {
  932. return c.skipKatiNinja
  933. }
  934. func (c *configImpl) SkipSoong() bool {
  935. return c.skipSoong
  936. }
  937. func (c *configImpl) SkipNinja() bool {
  938. return c.skipNinja
  939. }
  940. func (c *configImpl) SetSkipNinja(v bool) {
  941. c.skipNinja = v
  942. }
  943. func (c *configImpl) SkipConfig() bool {
  944. return c.skipConfig
  945. }
  946. func (c *configImpl) TargetProduct() string {
  947. if v, ok := c.environ.Get("TARGET_PRODUCT"); ok {
  948. return v
  949. }
  950. panic("TARGET_PRODUCT is not defined")
  951. }
  952. func (c *configImpl) TargetDevice() string {
  953. return c.targetDevice
  954. }
  955. func (c *configImpl) SetTargetDevice(device string) {
  956. c.targetDevice = device
  957. }
  958. func (c *configImpl) TargetBuildVariant() string {
  959. if v, ok := c.environ.Get("TARGET_BUILD_VARIANT"); ok {
  960. return v
  961. }
  962. panic("TARGET_BUILD_VARIANT is not defined")
  963. }
  964. func (c *configImpl) KatiArgs() []string {
  965. return c.katiArgs
  966. }
  967. func (c *configImpl) Parallel() int {
  968. return c.parallel
  969. }
  970. func (c *configImpl) GetIncludeTags() []string {
  971. return c.includeTags
  972. }
  973. func (c *configImpl) SetIncludeTags(i []string) {
  974. c.includeTags = i
  975. }
  976. func (c *configImpl) GetLogsPrefix() string {
  977. return c.logsPrefix
  978. }
  979. func (c *configImpl) SetLogsPrefix(prefix string) {
  980. c.logsPrefix = prefix
  981. }
  982. func (c *configImpl) HighmemParallel() int {
  983. if i, ok := c.environ.GetInt("NINJA_HIGHMEM_NUM_JOBS"); ok {
  984. return i
  985. }
  986. const minMemPerHighmemProcess = 8 * 1024 * 1024 * 1024
  987. parallel := c.Parallel()
  988. if c.UseRemoteBuild() {
  989. // Ninja doesn't support nested pools, and when remote builds are enabled the total ninja parallelism
  990. // is set very high (i.e. 500). Using a large value here would cause the total number of running jobs
  991. // to be the sum of the sizes of the local and highmem pools, which will cause extra CPU contention.
  992. // Return 1/16th of the size of the local pool, rounding up.
  993. return (parallel + 15) / 16
  994. } else if c.totalRAM == 0 {
  995. // Couldn't detect the total RAM, don't restrict highmem processes.
  996. return parallel
  997. } else if c.totalRAM <= 16*1024*1024*1024 {
  998. // Less than 16GB of ram, restrict to 1 highmem processes
  999. return 1
  1000. } else if c.totalRAM <= 32*1024*1024*1024 {
  1001. // Less than 32GB of ram, restrict to 2 highmem processes
  1002. return 2
  1003. } else if p := int(c.totalRAM / minMemPerHighmemProcess); p < parallel {
  1004. // If less than 8GB total RAM per process, reduce the number of highmem processes
  1005. return p
  1006. }
  1007. // No restriction on highmem processes
  1008. return parallel
  1009. }
  1010. func (c *configImpl) TotalRAM() uint64 {
  1011. return c.totalRAM
  1012. }
  1013. // ForceUseGoma determines whether we should override Goma deprecation
  1014. // and use Goma for the current build or not.
  1015. func (c *configImpl) ForceUseGoma() bool {
  1016. if v, ok := c.environ.Get("FORCE_USE_GOMA"); ok {
  1017. v = strings.TrimSpace(v)
  1018. if v != "" && v != "false" {
  1019. return true
  1020. }
  1021. }
  1022. return false
  1023. }
  1024. func (c *configImpl) UseGoma() bool {
  1025. if v, ok := c.environ.Get("USE_GOMA"); ok {
  1026. v = strings.TrimSpace(v)
  1027. if v != "" && v != "false" {
  1028. return true
  1029. }
  1030. }
  1031. return false
  1032. }
  1033. func (c *configImpl) StartGoma() bool {
  1034. if !c.UseGoma() {
  1035. return false
  1036. }
  1037. if v, ok := c.environ.Get("NOSTART_GOMA"); ok {
  1038. v = strings.TrimSpace(v)
  1039. if v != "" && v != "false" {
  1040. return false
  1041. }
  1042. }
  1043. return true
  1044. }
  1045. func (c *configImpl) UseRBE() bool {
  1046. if v, ok := c.Environment().Get("USE_RBE"); ok {
  1047. v = strings.TrimSpace(v)
  1048. if v != "" && v != "false" {
  1049. return true
  1050. }
  1051. }
  1052. return false
  1053. }
  1054. func (c *configImpl) BazelBuildEnabled() bool {
  1055. return c.bazelProdMode || c.bazelDevMode || c.bazelStagingMode
  1056. }
  1057. func (c *configImpl) StartRBE() bool {
  1058. if !c.UseRBE() {
  1059. return false
  1060. }
  1061. if v, ok := c.environ.Get("NOSTART_RBE"); ok {
  1062. v = strings.TrimSpace(v)
  1063. if v != "" && v != "false" {
  1064. return false
  1065. }
  1066. }
  1067. return true
  1068. }
  1069. func (c *configImpl) rbeProxyLogsDir() string {
  1070. for _, f := range []string{"RBE_proxy_log_dir", "FLAG_output_dir"} {
  1071. if v, ok := c.environ.Get(f); ok {
  1072. return v
  1073. }
  1074. }
  1075. buildTmpDir := shared.TempDirForOutDir(c.SoongOutDir())
  1076. return filepath.Join(buildTmpDir, "rbe")
  1077. }
  1078. func (c *configImpl) shouldCleanupRBELogsDir() bool {
  1079. // Perform a log directory cleanup only when the log directory
  1080. // is auto created by the build rather than user-specified.
  1081. for _, f := range []string{"RBE_proxy_log_dir", "FLAG_output_dir"} {
  1082. if _, ok := c.environ.Get(f); ok {
  1083. return false
  1084. }
  1085. }
  1086. return true
  1087. }
  1088. func (c *configImpl) rbeExecRoot() string {
  1089. for _, f := range []string{"RBE_exec_root", "FLAG_exec_root"} {
  1090. if v, ok := c.environ.Get(f); ok {
  1091. return v
  1092. }
  1093. }
  1094. wd, err := os.Getwd()
  1095. if err != nil {
  1096. return ""
  1097. }
  1098. return wd
  1099. }
  1100. func (c *configImpl) rbeDir() string {
  1101. if v, ok := c.environ.Get("RBE_DIR"); ok {
  1102. return v
  1103. }
  1104. return "prebuilts/remoteexecution-client/live/"
  1105. }
  1106. func (c *configImpl) rbeReproxy() string {
  1107. for _, f := range []string{"RBE_re_proxy", "FLAG_re_proxy"} {
  1108. if v, ok := c.environ.Get(f); ok {
  1109. return v
  1110. }
  1111. }
  1112. return filepath.Join(c.rbeDir(), "reproxy")
  1113. }
  1114. func (c *configImpl) rbeAuth() (string, string) {
  1115. credFlags := []string{
  1116. "use_application_default_credentials",
  1117. "use_gce_credentials",
  1118. "credential_file",
  1119. "use_google_prod_creds",
  1120. }
  1121. for _, cf := range credFlags {
  1122. for _, f := range []string{"RBE_" + cf, "FLAG_" + cf} {
  1123. if v, ok := c.environ.Get(f); ok {
  1124. v = strings.TrimSpace(v)
  1125. if v != "" && v != "false" && v != "0" {
  1126. return "RBE_" + cf, v
  1127. }
  1128. }
  1129. }
  1130. }
  1131. return "RBE_use_application_default_credentials", "true"
  1132. }
  1133. func (c *configImpl) rbeSockAddr(dir string) (string, error) {
  1134. maxNameLen := len(syscall.RawSockaddrUnix{}.Path)
  1135. base := fmt.Sprintf("reproxy_%v.sock", rbeRandPrefix)
  1136. name := filepath.Join(dir, base)
  1137. if len(name) < maxNameLen {
  1138. return name, nil
  1139. }
  1140. name = filepath.Join("/tmp", base)
  1141. if len(name) < maxNameLen {
  1142. return name, nil
  1143. }
  1144. return "", fmt.Errorf("cannot generate a proxy socket address shorter than the limit of %v", maxNameLen)
  1145. }
  1146. // IsGooglerEnvironment returns true if the current build is running
  1147. // on a Google developer machine and false otherwise.
  1148. func (c *configImpl) IsGooglerEnvironment() bool {
  1149. cf := "ANDROID_BUILD_ENVIRONMENT_CONFIG"
  1150. if v, ok := c.environ.Get(cf); ok {
  1151. return v == "googler"
  1152. }
  1153. return false
  1154. }
  1155. // GoogleProdCredsExist determine whether credentials exist on the
  1156. // Googler machine to use remote execution.
  1157. func (c *configImpl) GoogleProdCredsExist() bool {
  1158. if _, err := exec.Command("/usr/bin/prodcertstatus", "--simple_output", "--nocheck_loas").Output(); err != nil {
  1159. return false
  1160. }
  1161. return true
  1162. }
  1163. // UseRemoteBuild indicates whether to use a remote build acceleration system
  1164. // to speed up the build.
  1165. func (c *configImpl) UseRemoteBuild() bool {
  1166. return c.UseGoma() || c.UseRBE()
  1167. }
  1168. // StubbyExists checks whether the stubby binary exists on the machine running
  1169. // the build.
  1170. func (c *configImpl) StubbyExists() bool {
  1171. if _, err := exec.LookPath("stubby"); err != nil {
  1172. return false
  1173. }
  1174. return true
  1175. }
  1176. // RemoteParallel controls how many remote jobs (i.e., commands which contain
  1177. // gomacc) are run in parallel. Note the parallelism of all other jobs is
  1178. // still limited by Parallel()
  1179. func (c *configImpl) RemoteParallel() int {
  1180. if !c.UseRemoteBuild() {
  1181. return 0
  1182. }
  1183. if i, ok := c.environ.GetInt("NINJA_REMOTE_NUM_JOBS"); ok {
  1184. return i
  1185. }
  1186. return 500
  1187. }
  1188. func (c *configImpl) SetKatiArgs(args []string) {
  1189. c.katiArgs = args
  1190. }
  1191. func (c *configImpl) SetNinjaArgs(args []string) {
  1192. c.ninjaArgs = args
  1193. }
  1194. func (c *configImpl) SetKatiSuffix(suffix string) {
  1195. c.katiSuffix = suffix
  1196. }
  1197. func (c *configImpl) LastKatiSuffixFile() string {
  1198. return filepath.Join(c.OutDir(), "last_kati_suffix")
  1199. }
  1200. func (c *configImpl) HasKatiSuffix() bool {
  1201. return c.katiSuffix != ""
  1202. }
  1203. func (c *configImpl) KatiEnvFile() string {
  1204. return filepath.Join(c.OutDir(), "env"+c.KatiSuffix()+".sh")
  1205. }
  1206. func (c *configImpl) KatiBuildNinjaFile() string {
  1207. return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiBuildSuffix+".ninja")
  1208. }
  1209. func (c *configImpl) KatiPackageNinjaFile() string {
  1210. return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiPackageSuffix+".ninja")
  1211. }
  1212. func (c *configImpl) SoongVarsFile() string {
  1213. return filepath.Join(c.SoongOutDir(), "soong.variables")
  1214. }
  1215. func (c *configImpl) SoongNinjaFile() string {
  1216. return filepath.Join(c.SoongOutDir(), "build.ninja")
  1217. }
  1218. func (c *configImpl) CombinedNinjaFile() string {
  1219. if c.katiSuffix == "" {
  1220. return filepath.Join(c.OutDir(), "combined.ninja")
  1221. }
  1222. return filepath.Join(c.OutDir(), "combined"+c.KatiSuffix()+".ninja")
  1223. }
  1224. func (c *configImpl) SoongAndroidMk() string {
  1225. return filepath.Join(c.SoongOutDir(), "Android-"+c.TargetProduct()+".mk")
  1226. }
  1227. func (c *configImpl) SoongMakeVarsMk() string {
  1228. return filepath.Join(c.SoongOutDir(), "make_vars-"+c.TargetProduct()+".mk")
  1229. }
  1230. func (c *configImpl) ProductOut() string {
  1231. return filepath.Join(c.OutDir(), "target", "product", c.TargetDevice())
  1232. }
  1233. func (c *configImpl) DevicePreviousProductConfig() string {
  1234. return filepath.Join(c.ProductOut(), "previous_build_config.mk")
  1235. }
  1236. func (c *configImpl) KatiPackageMkDir() string {
  1237. return filepath.Join(c.ProductOut(), "obj", "CONFIG", "kati_packaging")
  1238. }
  1239. func (c *configImpl) hostOutRoot() string {
  1240. return filepath.Join(c.OutDir(), "host")
  1241. }
  1242. func (c *configImpl) HostOut() string {
  1243. return filepath.Join(c.hostOutRoot(), c.HostPrebuiltTag())
  1244. }
  1245. // This probably needs to be multi-valued, so not exporting it for now
  1246. func (c *configImpl) hostCrossOut() string {
  1247. if runtime.GOOS == "linux" {
  1248. return filepath.Join(c.hostOutRoot(), "windows-x86")
  1249. } else {
  1250. return ""
  1251. }
  1252. }
  1253. func (c *configImpl) HostPrebuiltTag() string {
  1254. if runtime.GOOS == "linux" {
  1255. return "linux-x86"
  1256. } else if runtime.GOOS == "darwin" {
  1257. return "darwin-x86"
  1258. } else {
  1259. panic("Unsupported OS")
  1260. }
  1261. }
  1262. func (c *configImpl) PrebuiltBuildTool(name string) string {
  1263. if v, ok := c.environ.Get("SANITIZE_HOST"); ok {
  1264. if sanitize := strings.Fields(v); inList("address", sanitize) {
  1265. asan := filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "asan/bin", name)
  1266. if _, err := os.Stat(asan); err == nil {
  1267. return asan
  1268. }
  1269. }
  1270. }
  1271. return filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "bin", name)
  1272. }
  1273. func (c *configImpl) SetBuildBrokenDupRules(val bool) {
  1274. c.brokenDupRules = val
  1275. }
  1276. func (c *configImpl) BuildBrokenDupRules() bool {
  1277. return c.brokenDupRules
  1278. }
  1279. func (c *configImpl) SetBuildBrokenUsesNetwork(val bool) {
  1280. c.brokenUsesNetwork = val
  1281. }
  1282. func (c *configImpl) BuildBrokenUsesNetwork() bool {
  1283. return c.brokenUsesNetwork
  1284. }
  1285. func (c *configImpl) SetBuildBrokenNinjaUsesEnvVars(val []string) {
  1286. c.brokenNinjaEnvVars = val
  1287. }
  1288. func (c *configImpl) BuildBrokenNinjaUsesEnvVars() []string {
  1289. return c.brokenNinjaEnvVars
  1290. }
  1291. func (c *configImpl) SetTargetDeviceDir(dir string) {
  1292. c.targetDeviceDir = dir
  1293. }
  1294. func (c *configImpl) TargetDeviceDir() string {
  1295. return c.targetDeviceDir
  1296. }
  1297. func (c *configImpl) BuildDateTime() string {
  1298. return c.buildDateTime
  1299. }
  1300. func (c *configImpl) MetricsUploaderApp() string {
  1301. return c.metricsUploader
  1302. }
  1303. // LogsDir returns the absolute path to the logs directory where build log and
  1304. // metrics files are located. By default, the logs directory is the out
  1305. // directory. If the argument dist is specified, the logs directory
  1306. // is <dist_dir>/logs.
  1307. func (c *configImpl) LogsDir() string {
  1308. dir := c.OutDir()
  1309. if c.Dist() {
  1310. // Always write logs to the real dist dir, even if Bazel is using a rigged dist dir for other files
  1311. dir = filepath.Join(c.RealDistDir(), "logs")
  1312. }
  1313. absDir, err := filepath.Abs(dir)
  1314. if err != nil {
  1315. fmt.Fprintf(os.Stderr, "\nError making log dir '%s' absolute: %s\n", dir, err.Error())
  1316. os.Exit(1)
  1317. }
  1318. return absDir
  1319. }
  1320. // BazelMetricsDir returns the <logs dir>/bazel_metrics directory
  1321. // where the bazel profiles are located.
  1322. func (c *configImpl) BazelMetricsDir() string {
  1323. return filepath.Join(c.LogsDir(), "bazel_metrics")
  1324. }
  1325. // MkFileMetrics returns the file path for make-related metrics.
  1326. func (c *configImpl) MkMetrics() string {
  1327. return filepath.Join(c.LogsDir(), "mk_metrics.pb")
  1328. }
  1329. func (c *configImpl) SetEmptyNinjaFile(v bool) {
  1330. c.emptyNinjaFile = v
  1331. }
  1332. func (c *configImpl) EmptyNinjaFile() bool {
  1333. return c.emptyNinjaFile
  1334. }
  1335. func (c *configImpl) IsBazelMixedBuildForceDisabled() bool {
  1336. return c.Environment().IsEnvTrue("BUILD_BROKEN_DISABLE_BAZEL")
  1337. }
  1338. func (c *configImpl) IsPersistentBazelEnabled() bool {
  1339. return c.Environment().IsEnvTrue("USE_PERSISTENT_BAZEL")
  1340. }
  1341. func (c *configImpl) BazelModulesForceEnabledByFlag() string {
  1342. return c.bazelForceEnabledModules
  1343. }
  1344. func (c *configImpl) SkipMetricsUpload() bool {
  1345. return c.skipMetricsUpload
  1346. }
  1347. // Returns a Time object if one was passed via a command-line flag.
  1348. // Otherwise returns the passed default.
  1349. func (c *configImpl) BuildStartedTimeOrDefault(defaultTime time.Time) time.Time {
  1350. if c.buildStartedTime == 0 {
  1351. return defaultTime
  1352. }
  1353. return time.UnixMilli(c.buildStartedTime)
  1354. }
  1355. func GetMetricsUploader(topDir string, env *Environment) string {
  1356. if p, ok := env.Get("METRICS_UPLOADER"); ok {
  1357. metricsUploader := filepath.Join(topDir, p)
  1358. if _, err := os.Stat(metricsUploader); err == nil {
  1359. return metricsUploader
  1360. }
  1361. }
  1362. return ""
  1363. }