12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808 |
- // Copyright 2017 Google Inc. All rights reserved.
- //
- // Licensed under the Apache License, Version 2.0 (the "License");
- // you may not use this file except in compliance with the License.
- // You may obtain a copy of the License at
- //
- // http://www.apache.org/licenses/LICENSE-2.0
- //
- // Unless required by applicable law or agreed to in writing, software
- // distributed under the License is distributed on an "AS IS" BASIS,
- // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- // See the License for the specific language governing permissions and
- // limitations under the License.
- package build
- import (
- "context"
- "encoding/json"
- "errors"
- "fmt"
- "io/ioutil"
- "math/rand"
- "os"
- "os/exec"
- "path/filepath"
- "runtime"
- "strconv"
- "strings"
- "syscall"
- "time"
- "android/soong/shared"
- "google.golang.org/protobuf/proto"
- smpb "android/soong/ui/metrics/metrics_proto"
- )
- const (
- envConfigDir = "vendor/google/tools/soong_config"
- jsonSuffix = "json"
- configFetcher = "vendor/google/tools/soong/expconfigfetcher"
- envConfigFetchTimeout = 20 * time.Second
- )
- var (
- rbeRandPrefix int
- googleProdCredsExistCache bool
- )
- func init() {
- rand.Seed(time.Now().UnixNano())
- rbeRandPrefix = rand.Intn(1000)
- }
- type Config struct{ *configImpl }
- type configImpl struct {
- // Some targets that are implemented in soong_build
- // (bp2build, json-module-graph) are not here and have their own bits below.
- arguments []string
- goma bool
- environ *Environment
- distDir string
- buildDateTime string
- logsPrefix string
- // From the arguments
- parallel int
- keepGoing int
- verbose bool
- checkbuild bool
- dist bool
- jsonModuleGraph bool
- apiBp2build bool // Generate BUILD files for Soong modules that contribute APIs
- bp2build bool
- queryview bool
- reportMkMetrics bool // Collect and report mk2bp migration progress metrics.
- soongDocs bool
- multitreeBuild bool // This is a multitree build.
- skipConfig bool
- skipKati bool
- skipKatiNinja bool
- skipSoong bool
- skipNinja bool
- skipSoongTests bool
- searchApiDir bool // Scan the Android.bp files generated in out/api_surfaces
- skipMetricsUpload bool
- buildStartedTime int64 // For metrics-upload-only - manually specify a build-started time
- buildFromTextStub bool
- ensureAllowlistIntegrity bool // For CI builds - make sure modules are mixed-built
- bazelExitCode int32 // For b runs - necessary for updating NonZeroExit
- besId string // For b runs, to identify the BuildEventService logs
- // From the product config
- katiArgs []string
- ninjaArgs []string
- katiSuffix string
- targetDevice string
- targetDeviceDir string
- sandboxConfig *SandboxConfig
- // Autodetected
- totalRAM uint64
- brokenDupRules bool
- brokenUsesNetwork bool
- brokenNinjaEnvVars []string
- pathReplaced bool
- bazelProdMode bool
- bazelStagingMode bool
- // Set by multiproduct_kati
- emptyNinjaFile bool
- metricsUploader string
- bazelForceEnabledModules string
- includeTags []string
- sourceRootDirs []string
- // Data source to write ninja weight list
- ninjaWeightListSource NinjaWeightListSource
- }
- type NinjaWeightListSource uint
- const (
- // ninja doesn't use weight list.
- NOT_USED NinjaWeightListSource = iota
- // ninja uses weight list based on previous builds by ninja log
- NINJA_LOG
- // ninja thinks every task has the same weight.
- EVENLY_DISTRIBUTED
- // ninja uses an external custom weight list
- EXTERNAL_FILE
- // ninja uses a prioritized module list from Soong
- HINT_FROM_SOONG
- // If ninja log exists, use NINJA_LOG, if not, use HINT_FROM_SOONG instead.
- // We can assume it is an incremental build if ninja log exists.
- DEFAULT
- )
- const srcDirFileCheck = "build/soong/root.bp"
- var buildFiles = []string{"Android.mk", "Android.bp"}
- type BuildAction uint
- const (
- // Builds all of the modules and their dependencies of a specified directory, relative to the root
- // directory of the source tree.
- BUILD_MODULES_IN_A_DIRECTORY BuildAction = iota
- // Builds all of the modules and their dependencies of a list of specified directories. All specified
- // directories are relative to the root directory of the source tree.
- BUILD_MODULES_IN_DIRECTORIES
- // Build a list of specified modules. If none was specified, simply build the whole source tree.
- BUILD_MODULES
- )
- // checkTopDir validates that the current directory is at the root directory of the source tree.
- func checkTopDir(ctx Context) {
- if _, err := os.Stat(srcDirFileCheck); err != nil {
- if os.IsNotExist(err) {
- ctx.Fatalf("Current working directory must be the source tree. %q not found.", srcDirFileCheck)
- }
- ctx.Fatalln("Error verifying tree state:", err)
- }
- }
- // fetchEnvConfig optionally fetches a configuration file that can then subsequently be
- // loaded into Soong environment to control certain aspects of build behavior (e.g., enabling RBE).
- // If a configuration file already exists on disk, the fetch is run in the background
- // so as to NOT block the rest of the build execution.
- func fetchEnvConfig(ctx Context, config *configImpl, envConfigName string) error {
- configName := envConfigName + "." + jsonSuffix
- expConfigFetcher := &smpb.ExpConfigFetcher{Filename: &configName}
- defer func() {
- ctx.Metrics.ExpConfigFetcher(expConfigFetcher)
- }()
- if !config.GoogleProdCredsExist() {
- status := smpb.ExpConfigFetcher_MISSING_GCERT
- expConfigFetcher.Status = &status
- return nil
- }
- s, err := os.Stat(configFetcher)
- if err != nil {
- if os.IsNotExist(err) {
- return nil
- }
- return err
- }
- if s.Mode()&0111 == 0 {
- status := smpb.ExpConfigFetcher_ERROR
- expConfigFetcher.Status = &status
- return fmt.Errorf("configuration fetcher binary %v is not executable: %v", configFetcher, s.Mode())
- }
- configExists := false
- outConfigFilePath := filepath.Join(config.OutDir(), configName)
- if _, err := os.Stat(outConfigFilePath); err == nil {
- configExists = true
- }
- tCtx, cancel := context.WithTimeout(ctx, envConfigFetchTimeout)
- fetchStart := time.Now()
- cmd := exec.CommandContext(tCtx, configFetcher, "-output_config_dir", config.OutDir(),
- "-output_config_name", configName)
- if err := cmd.Start(); err != nil {
- status := smpb.ExpConfigFetcher_ERROR
- expConfigFetcher.Status = &status
- return err
- }
- fetchCfg := func() error {
- if err := cmd.Wait(); err != nil {
- status := smpb.ExpConfigFetcher_ERROR
- expConfigFetcher.Status = &status
- return err
- }
- fetchEnd := time.Now()
- expConfigFetcher.Micros = proto.Uint64(uint64(fetchEnd.Sub(fetchStart).Microseconds()))
- expConfigFetcher.Filename = proto.String(outConfigFilePath)
- if _, err := os.Stat(outConfigFilePath); err != nil {
- status := smpb.ExpConfigFetcher_NO_CONFIG
- expConfigFetcher.Status = &status
- return err
- }
- status := smpb.ExpConfigFetcher_CONFIG
- expConfigFetcher.Status = &status
- return nil
- }
- // If a config file does not exist, wait for the config file to be fetched. Otherwise
- // fetch the config file in the background and return immediately.
- if !configExists {
- defer cancel()
- return fetchCfg()
- }
- go func() {
- defer cancel()
- if err := fetchCfg(); err != nil {
- ctx.Verbosef("Failed to fetch config file %v: %v\n", configName, err)
- }
- }()
- return nil
- }
- func loadEnvConfig(ctx Context, config *configImpl, bc string) error {
- if bc == "" {
- return nil
- }
- configDirs := []string{
- config.OutDir(),
- os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG_DIR"),
- envConfigDir,
- }
- for _, dir := range configDirs {
- cfgFile := filepath.Join(os.Getenv("TOP"), dir, fmt.Sprintf("%s.%s", bc, jsonSuffix))
- envVarsJSON, err := ioutil.ReadFile(cfgFile)
- if err != nil {
- continue
- }
- ctx.Verbosef("Loading config file %v\n", cfgFile)
- var envVars map[string]map[string]string
- if err := json.Unmarshal(envVarsJSON, &envVars); err != nil {
- fmt.Fprintf(os.Stderr, "Env vars config file %s did not parse correctly: %s", cfgFile, err.Error())
- continue
- }
- for k, v := range envVars["env"] {
- if os.Getenv(k) != "" {
- continue
- }
- config.environ.Set(k, v)
- }
- ctx.Verbosef("Finished loading config file %v\n", cfgFile)
- break
- }
- return nil
- }
- func defaultBazelProdMode(cfg *configImpl) bool {
- // Environment flag to disable Bazel for users which experience
- // broken bazel-handled builds, or significant performance regressions.
- if cfg.IsBazelMixedBuildForceDisabled() {
- return false
- }
- // Darwin-host builds are currently untested with Bazel.
- if runtime.GOOS == "darwin" {
- return false
- }
- return true
- }
- func UploadOnlyConfig(ctx Context, args ...string) Config {
- ret := &configImpl{
- environ: OsEnvironment(),
- sandboxConfig: &SandboxConfig{},
- }
- ret.parseArgs(ctx, args)
- srcDir := absPath(ctx, ".")
- bc := os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG")
- if err := loadEnvConfig(ctx, ret, bc); err != nil {
- ctx.Fatalln("Failed to parse env config files: %v", err)
- }
- ret.metricsUploader = GetMetricsUploader(srcDir, ret.environ)
- return Config{ret}
- }
- func NewConfig(ctx Context, args ...string) Config {
- ret := &configImpl{
- environ: OsEnvironment(),
- sandboxConfig: &SandboxConfig{},
- ninjaWeightListSource: DEFAULT,
- }
- // Default matching ninja
- ret.parallel = runtime.NumCPU() + 2
- ret.keepGoing = 1
- ret.totalRAM = detectTotalRAM(ctx)
- ret.parseArgs(ctx, args)
- if ret.ninjaWeightListSource == HINT_FROM_SOONG {
- ret.environ.Set("SOONG_GENERATES_NINJA_HINT", "always")
- } else if ret.ninjaWeightListSource == DEFAULT {
- defaultNinjaWeightListSource := NINJA_LOG
- if _, err := os.Stat(filepath.Join(ret.OutDir(), ninjaLogFileName)); errors.Is(err, os.ErrNotExist) {
- ctx.Verboseln("$OUT/.ninja_log doesn't exist, use HINT_FROM_SOONG instead")
- defaultNinjaWeightListSource = HINT_FROM_SOONG
- } else {
- ctx.Verboseln("$OUT/.ninja_log exist, use NINJA_LOG")
- }
- ret.ninjaWeightListSource = defaultNinjaWeightListSource
- // soong_build generates ninja hint depending on ninja log existence.
- // Set it "depend" to avoid soong re-run due to env variable change.
- ret.environ.Set("SOONG_GENERATES_NINJA_HINT", "depend")
- }
- // Make sure OUT_DIR is set appropriately
- if outDir, ok := ret.environ.Get("OUT_DIR"); ok {
- ret.environ.Set("OUT_DIR", filepath.Clean(outDir))
- } else {
- outDir := "out"
- if baseDir, ok := ret.environ.Get("OUT_DIR_COMMON_BASE"); ok {
- if wd, err := os.Getwd(); err != nil {
- ctx.Fatalln("Failed to get working directory:", err)
- } else {
- outDir = filepath.Join(baseDir, filepath.Base(wd))
- }
- }
- ret.environ.Set("OUT_DIR", outDir)
- }
- // loadEnvConfig needs to know what the OUT_DIR is, so it should
- // be called after we determine the appropriate out directory.
- bc := os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG")
- if bc != "" {
- if err := fetchEnvConfig(ctx, ret, bc); err != nil {
- ctx.Verbosef("Failed to fetch config file: %v\n", err)
- }
- if err := loadEnvConfig(ctx, ret, bc); err != nil {
- ctx.Fatalln("Failed to parse env config files: %v", err)
- }
- }
- if distDir, ok := ret.environ.Get("DIST_DIR"); ok {
- ret.distDir = filepath.Clean(distDir)
- } else {
- ret.distDir = filepath.Join(ret.OutDir(), "dist")
- }
- if srcDirIsWritable, ok := ret.environ.Get("BUILD_BROKEN_SRC_DIR_IS_WRITABLE"); ok {
- ret.sandboxConfig.SetSrcDirIsRO(srcDirIsWritable == "false")
- }
- ret.environ.Unset(
- // We're already using it
- "USE_SOONG_UI",
- // We should never use GOROOT/GOPATH from the shell environment
- "GOROOT",
- "GOPATH",
- // These should only come from Soong, not the environment.
- "CLANG",
- "CLANG_CXX",
- "CCC_CC",
- "CCC_CXX",
- // Used by the goma compiler wrapper, but should only be set by
- // gomacc
- "GOMACC_PATH",
- // We handle this above
- "OUT_DIR_COMMON_BASE",
- // This is handled above too, and set for individual commands later
- "DIST_DIR",
- // Variables that have caused problems in the past
- "BASH_ENV",
- "CDPATH",
- "DISPLAY",
- "GREP_OPTIONS",
- "JAVAC",
- "NDK_ROOT",
- "POSIXLY_CORRECT",
- // Drop make flags
- "MAKEFLAGS",
- "MAKELEVEL",
- "MFLAGS",
- // Set in envsetup.sh, reset in makefiles
- "ANDROID_JAVA_TOOLCHAIN",
- // Set by envsetup.sh, but shouldn't be used inside the build because envsetup.sh is optional
- "ANDROID_BUILD_TOP",
- "ANDROID_HOST_OUT",
- "ANDROID_PRODUCT_OUT",
- "ANDROID_HOST_OUT_TESTCASES",
- "ANDROID_TARGET_OUT_TESTCASES",
- "ANDROID_TOOLCHAIN",
- "ANDROID_TOOLCHAIN_2ND_ARCH",
- "ANDROID_DEV_SCRIPTS",
- "ANDROID_EMULATOR_PREBUILTS",
- "ANDROID_PRE_BUILD_PATHS",
- )
- if ret.UseGoma() || ret.ForceUseGoma() {
- ctx.Println("Goma for Android has been deprecated and replaced with RBE. See go/rbe_for_android for instructions on how to use RBE.")
- ctx.Fatalln("USE_GOMA / FORCE_USE_GOMA flag is no longer supported.")
- }
- // Tell python not to spam the source tree with .pyc files.
- ret.environ.Set("PYTHONDONTWRITEBYTECODE", "1")
- tmpDir := absPath(ctx, ret.TempDir())
- ret.environ.Set("TMPDIR", tmpDir)
- // Always set ASAN_SYMBOLIZER_PATH so that ASAN-based tools can symbolize any crashes
- symbolizerPath := filepath.Join("prebuilts/clang/host", ret.HostPrebuiltTag(),
- "llvm-binutils-stable/llvm-symbolizer")
- ret.environ.Set("ASAN_SYMBOLIZER_PATH", absPath(ctx, symbolizerPath))
- // Precondition: the current directory is the top of the source tree
- checkTopDir(ctx)
- srcDir := absPath(ctx, ".")
- if strings.ContainsRune(srcDir, ' ') {
- ctx.Println("You are building in a directory whose absolute path contains a space character:")
- ctx.Println()
- ctx.Printf("%q\n", srcDir)
- ctx.Println()
- ctx.Fatalln("Directory names containing spaces are not supported")
- }
- ret.metricsUploader = GetMetricsUploader(srcDir, ret.environ)
- if outDir := ret.OutDir(); strings.ContainsRune(outDir, ' ') {
- ctx.Println("The absolute path of your output directory ($OUT_DIR) contains a space character:")
- ctx.Println()
- ctx.Printf("%q\n", outDir)
- ctx.Println()
- ctx.Fatalln("Directory names containing spaces are not supported")
- }
- if distDir := ret.RealDistDir(); strings.ContainsRune(distDir, ' ') {
- ctx.Println("The absolute path of your dist directory ($DIST_DIR) contains a space character:")
- ctx.Println()
- ctx.Printf("%q\n", distDir)
- ctx.Println()
- ctx.Fatalln("Directory names containing spaces are not supported")
- }
- // Configure Java-related variables, including adding it to $PATH
- java8Home := filepath.Join("prebuilts/jdk/jdk8", ret.HostPrebuiltTag())
- java9Home := filepath.Join("prebuilts/jdk/jdk9", ret.HostPrebuiltTag())
- java11Home := filepath.Join("prebuilts/jdk/jdk11", ret.HostPrebuiltTag())
- java17Home := filepath.Join("prebuilts/jdk/jdk17", ret.HostPrebuiltTag())
- javaHome := func() string {
- if override, ok := ret.environ.Get("OVERRIDE_ANDROID_JAVA_HOME"); ok {
- return override
- }
- if toolchain11, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN"); ok && toolchain11 != "true" {
- ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN is no longer supported. An OpenJDK 11 toolchain is now the global default.")
- }
- if toolchain17, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK17_TOOLCHAIN"); ok && toolchain17 != "true" {
- ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK17_TOOLCHAIN is no longer supported. An OpenJDK 17 toolchain is now the global default.")
- }
- return java17Home
- }()
- absJavaHome := absPath(ctx, javaHome)
- ret.configureLocale(ctx)
- newPath := []string{filepath.Join(absJavaHome, "bin")}
- if path, ok := ret.environ.Get("PATH"); ok && path != "" {
- newPath = append(newPath, path)
- }
- ret.environ.Unset("OVERRIDE_ANDROID_JAVA_HOME")
- ret.environ.Set("JAVA_HOME", absJavaHome)
- ret.environ.Set("ANDROID_JAVA_HOME", javaHome)
- ret.environ.Set("ANDROID_JAVA8_HOME", java8Home)
- ret.environ.Set("ANDROID_JAVA9_HOME", java9Home)
- ret.environ.Set("ANDROID_JAVA11_HOME", java11Home)
- ret.environ.Set("PATH", strings.Join(newPath, string(filepath.ListSeparator)))
- if ret.MultitreeBuild() {
- ret.environ.Set("MULTITREE_BUILD", "true")
- }
- outDir := ret.OutDir()
- buildDateTimeFile := filepath.Join(outDir, "build_date.txt")
- if buildDateTime, ok := ret.environ.Get("BUILD_DATETIME"); ok && buildDateTime != "" {
- ret.buildDateTime = buildDateTime
- } else {
- ret.buildDateTime = strconv.FormatInt(time.Now().Unix(), 10)
- }
- ret.environ.Set("BUILD_DATETIME_FILE", buildDateTimeFile)
- if ret.UseRBE() {
- for k, v := range getRBEVars(ctx, Config{ret}) {
- ret.environ.Set(k, v)
- }
- }
- if ret.BuildFromTextStub() {
- // TODO(b/271443071): support hidden api check for from-text stub build
- ret.environ.Set("UNSAFE_DISABLE_HIDDENAPI_FLAGS", "true")
- }
- bpd := ret.BazelMetricsDir()
- if err := os.RemoveAll(bpd); err != nil {
- ctx.Fatalf("Unable to remove bazel profile directory %q: %v", bpd, err)
- }
- c := Config{ret}
- storeConfigMetrics(ctx, c)
- return c
- }
- // NewBuildActionConfig returns a build configuration based on the build action. The arguments are
- // processed based on the build action and extracts any arguments that belongs to the build action.
- func NewBuildActionConfig(action BuildAction, dir string, ctx Context, args ...string) Config {
- return NewConfig(ctx, getConfigArgs(action, dir, ctx, args)...)
- }
- // storeConfigMetrics selects a set of configuration information and store in
- // the metrics system for further analysis.
- func storeConfigMetrics(ctx Context, config Config) {
- if ctx.Metrics == nil {
- return
- }
- ctx.Metrics.BuildConfig(buildConfig(config))
- s := &smpb.SystemResourceInfo{
- TotalPhysicalMemory: proto.Uint64(config.TotalRAM()),
- AvailableCpus: proto.Int32(int32(runtime.NumCPU())),
- }
- ctx.Metrics.SystemResourceInfo(s)
- }
- func getNinjaWeightListSourceInMetric(s NinjaWeightListSource) *smpb.BuildConfig_NinjaWeightListSource {
- switch s {
- case NINJA_LOG:
- return smpb.BuildConfig_NINJA_LOG.Enum()
- case EVENLY_DISTRIBUTED:
- return smpb.BuildConfig_EVENLY_DISTRIBUTED.Enum()
- case EXTERNAL_FILE:
- return smpb.BuildConfig_EXTERNAL_FILE.Enum()
- case HINT_FROM_SOONG:
- return smpb.BuildConfig_HINT_FROM_SOONG.Enum()
- default:
- return smpb.BuildConfig_NOT_USED.Enum()
- }
- }
- func buildConfig(config Config) *smpb.BuildConfig {
- c := &smpb.BuildConfig{
- ForceUseGoma: proto.Bool(config.ForceUseGoma()),
- UseGoma: proto.Bool(config.UseGoma()),
- UseRbe: proto.Bool(config.UseRBE()),
- BazelMixedBuild: proto.Bool(config.BazelBuildEnabled()),
- ForceDisableBazelMixedBuild: proto.Bool(config.IsBazelMixedBuildForceDisabled()),
- NinjaWeightListSource: getNinjaWeightListSourceInMetric(config.NinjaWeightListSource()),
- }
- c.Targets = append(c.Targets, config.arguments...)
- return c
- }
- // getConfigArgs processes the command arguments based on the build action and creates a set of new
- // arguments to be accepted by Config.
- func getConfigArgs(action BuildAction, dir string, ctx Context, args []string) []string {
- // The next block of code verifies that the current directory is the root directory of the source
- // tree. It then finds the relative path of dir based on the root directory of the source tree
- // and verify that dir is inside of the source tree.
- checkTopDir(ctx)
- topDir, err := os.Getwd()
- if err != nil {
- ctx.Fatalf("Error retrieving top directory: %v", err)
- }
- dir, err = filepath.EvalSymlinks(dir)
- if err != nil {
- ctx.Fatalf("Unable to evaluate symlink of %s: %v", dir, err)
- }
- dir, err = filepath.Abs(dir)
- if err != nil {
- ctx.Fatalf("Unable to find absolute path %s: %v", dir, err)
- }
- relDir, err := filepath.Rel(topDir, dir)
- if err != nil {
- ctx.Fatalf("Unable to find relative path %s of %s: %v", relDir, topDir, err)
- }
- // If there are ".." in the path, it's not in the source tree.
- if strings.Contains(relDir, "..") {
- ctx.Fatalf("Directory %s is not under the source tree %s", dir, topDir)
- }
- configArgs := args[:]
- // If the arguments contains GET-INSTALL-PATH, change the target name prefix from MODULES-IN- to
- // GET-INSTALL-PATH-IN- to extract the installation path instead of building the modules.
- targetNamePrefix := "MODULES-IN-"
- if inList("GET-INSTALL-PATH", configArgs) {
- targetNamePrefix = "GET-INSTALL-PATH-IN-"
- configArgs = removeFromList("GET-INSTALL-PATH", configArgs)
- }
- var targets []string
- switch action {
- case BUILD_MODULES:
- // No additional processing is required when building a list of specific modules or all modules.
- case BUILD_MODULES_IN_A_DIRECTORY:
- // If dir is the root source tree, all the modules are built of the source tree are built so
- // no need to find the build file.
- if topDir == dir {
- break
- }
- buildFile := findBuildFile(ctx, relDir)
- if buildFile == "" {
- ctx.Fatalf("Build file not found for %s directory", relDir)
- }
- targets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)}
- case BUILD_MODULES_IN_DIRECTORIES:
- newConfigArgs, dirs := splitArgs(configArgs)
- configArgs = newConfigArgs
- targets = getTargetsFromDirs(ctx, relDir, dirs, targetNamePrefix)
- }
- // Tidy only override all other specified targets.
- tidyOnly := os.Getenv("WITH_TIDY_ONLY")
- if tidyOnly == "true" || tidyOnly == "1" {
- configArgs = append(configArgs, "tidy_only")
- } else {
- configArgs = append(configArgs, targets...)
- }
- return configArgs
- }
- // convertToTarget replaces "/" to "-" in dir and pre-append the targetNamePrefix to the target name.
- func convertToTarget(dir string, targetNamePrefix string) string {
- return targetNamePrefix + strings.ReplaceAll(dir, "/", "-")
- }
- // hasBuildFile returns true if dir contains an Android build file.
- func hasBuildFile(ctx Context, dir string) bool {
- for _, buildFile := range buildFiles {
- _, err := os.Stat(filepath.Join(dir, buildFile))
- if err == nil {
- return true
- }
- if !os.IsNotExist(err) {
- ctx.Fatalf("Error retrieving the build file stats: %v", err)
- }
- }
- return false
- }
- // findBuildFile finds a build file (makefile or blueprint file) by looking if there is a build file
- // in the current and any sub directory of dir. If a build file is not found, traverse the path
- // up by one directory and repeat again until either a build file is found or reached to the root
- // source tree. The returned filename of build file is "Android.mk". If one was not found, a blank
- // string is returned.
- func findBuildFile(ctx Context, dir string) string {
- // If the string is empty or ".", assume it is top directory of the source tree.
- if dir == "" || dir == "." {
- return ""
- }
- found := false
- for buildDir := dir; buildDir != "."; buildDir = filepath.Dir(buildDir) {
- err := filepath.Walk(buildDir, func(path string, info os.FileInfo, err error) error {
- if err != nil {
- return err
- }
- if found {
- return filepath.SkipDir
- }
- if info.IsDir() {
- return nil
- }
- for _, buildFile := range buildFiles {
- if info.Name() == buildFile {
- found = true
- return filepath.SkipDir
- }
- }
- return nil
- })
- if err != nil {
- ctx.Fatalf("Error finding Android build file: %v", err)
- }
- if found {
- return filepath.Join(buildDir, "Android.mk")
- }
- }
- return ""
- }
- // splitArgs iterates over the arguments list and splits into two lists: arguments and directories.
- func splitArgs(args []string) (newArgs []string, dirs []string) {
- specialArgs := map[string]bool{
- "showcommands": true,
- "snod": true,
- "dist": true,
- "checkbuild": true,
- }
- newArgs = []string{}
- dirs = []string{}
- for _, arg := range args {
- // It's a dash argument if it starts with "-" or it's a key=value pair, it's not a directory.
- if strings.IndexRune(arg, '-') == 0 || strings.IndexRune(arg, '=') != -1 {
- newArgs = append(newArgs, arg)
- continue
- }
- if _, ok := specialArgs[arg]; ok {
- newArgs = append(newArgs, arg)
- continue
- }
- dirs = append(dirs, arg)
- }
- return newArgs, dirs
- }
- // getTargetsFromDirs iterates over the dirs list and creates a list of targets to build. If a
- // directory from the dirs list does not exist, a fatal error is raised. relDir is related to the
- // source root tree where the build action command was invoked. Each directory is validated if the
- // build file can be found and follows the format "dir1:target1,target2,...". Target is optional.
- func getTargetsFromDirs(ctx Context, relDir string, dirs []string, targetNamePrefix string) (targets []string) {
- for _, dir := range dirs {
- // The directory may have specified specific modules to build. ":" is the separator to separate
- // the directory and the list of modules.
- s := strings.Split(dir, ":")
- l := len(s)
- if l > 2 { // more than one ":" was specified.
- ctx.Fatalf("%s not in proper directory:target1,target2,... format (\":\" was specified more than once)", dir)
- }
- dir = filepath.Join(relDir, s[0])
- if _, err := os.Stat(dir); err != nil {
- ctx.Fatalf("couldn't find directory %s", dir)
- }
- // Verify that if there are any targets specified after ":". Each target is separated by ",".
- var newTargets []string
- if l == 2 && s[1] != "" {
- newTargets = strings.Split(s[1], ",")
- if inList("", newTargets) {
- ctx.Fatalf("%s not in proper directory:target1,target2,... format", dir)
- }
- }
- // If there are specified targets to build in dir, an android build file must exist for the one
- // shot build. For the non-targets case, find the appropriate build file and build all the
- // modules in dir (or the closest one in the dir path).
- if len(newTargets) > 0 {
- if !hasBuildFile(ctx, dir) {
- ctx.Fatalf("Couldn't locate a build file from %s directory", dir)
- }
- } else {
- buildFile := findBuildFile(ctx, dir)
- if buildFile == "" {
- ctx.Fatalf("Build file not found for %s directory", dir)
- }
- newTargets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)}
- }
- targets = append(targets, newTargets...)
- }
- return targets
- }
- func (c *configImpl) parseArgs(ctx Context, args []string) {
- for i := 0; i < len(args); i++ {
- arg := strings.TrimSpace(args[i])
- if arg == "showcommands" {
- c.verbose = true
- } else if arg == "--empty-ninja-file" {
- c.emptyNinjaFile = true
- } else if arg == "--skip-ninja" {
- c.skipNinja = true
- } else if arg == "--skip-make" {
- // TODO(ccross): deprecate this, it has confusing behaviors. It doesn't run kati,
- // but it does run a Kati ninja file if the .kati_enabled marker file was created
- // by a previous build.
- c.skipConfig = true
- c.skipKati = true
- } else if arg == "--soong-only" {
- c.skipKati = true
- c.skipKatiNinja = true
- } else if arg == "--config-only" {
- c.skipKati = true
- c.skipKatiNinja = true
- c.skipSoong = true
- } else if arg == "--skip-config" {
- c.skipConfig = true
- } else if arg == "--skip-soong-tests" {
- c.skipSoongTests = true
- } else if arg == "--skip-metrics-upload" {
- c.skipMetricsUpload = true
- } else if arg == "--mk-metrics" {
- c.reportMkMetrics = true
- } else if arg == "--multitree-build" {
- c.multitreeBuild = true
- } else if arg == "--bazel-mode" {
- c.bazelProdMode = true
- } else if arg == "--bazel-mode-staging" {
- c.bazelStagingMode = true
- } else if arg == "--search-api-dir" {
- c.searchApiDir = true
- } else if strings.HasPrefix(arg, "--ninja_weight_source=") {
- source := strings.TrimPrefix(arg, "--ninja_weight_source=")
- if source == "ninja_log" {
- c.ninjaWeightListSource = NINJA_LOG
- } else if source == "evenly_distributed" {
- c.ninjaWeightListSource = EVENLY_DISTRIBUTED
- } else if source == "not_used" {
- c.ninjaWeightListSource = NOT_USED
- } else if source == "soong" {
- c.ninjaWeightListSource = HINT_FROM_SOONG
- } else if strings.HasPrefix(source, "file,") {
- c.ninjaWeightListSource = EXTERNAL_FILE
- filePath := strings.TrimPrefix(source, "file,")
- err := validateNinjaWeightList(filePath)
- if err != nil {
- ctx.Fatalf("Malformed weight list from %s: %s", filePath, err)
- }
- _, err = copyFile(filePath, filepath.Join(c.OutDir(), ".ninja_weight_list"))
- if err != nil {
- ctx.Fatalf("Error to copy ninja weight list from %s: %s", filePath, err)
- }
- } else {
- ctx.Fatalf("unknown option for ninja_weight_source: %s", source)
- }
- } else if arg == "--build-from-text-stub" {
- c.buildFromTextStub = true
- } else if strings.HasPrefix(arg, "--build-command=") {
- buildCmd := strings.TrimPrefix(arg, "--build-command=")
- // remove quotations
- buildCmd = strings.TrimPrefix(buildCmd, "\"")
- buildCmd = strings.TrimSuffix(buildCmd, "\"")
- ctx.Metrics.SetBuildCommand([]string{buildCmd})
- } else if strings.HasPrefix(arg, "--bazel-force-enabled-modules=") {
- c.bazelForceEnabledModules = strings.TrimPrefix(arg, "--bazel-force-enabled-modules=")
- } else if strings.HasPrefix(arg, "--build-started-time-unix-millis=") {
- buildTimeStr := strings.TrimPrefix(arg, "--build-started-time-unix-millis=")
- val, err := strconv.ParseInt(buildTimeStr, 10, 64)
- if err == nil {
- c.buildStartedTime = val
- } else {
- ctx.Fatalf("Error parsing build-time-started-unix-millis", err)
- }
- } else if arg == "--ensure-allowlist-integrity" {
- c.ensureAllowlistIntegrity = true
- } else if strings.HasPrefix(arg, "--bazel-exit-code=") {
- bazelExitCodeStr := strings.TrimPrefix(arg, "--bazel-exit-code=")
- val, err := strconv.Atoi(bazelExitCodeStr)
- if err == nil {
- c.bazelExitCode = int32(val)
- } else {
- ctx.Fatalf("Error parsing bazel-exit-code", err)
- }
- } else if strings.HasPrefix(arg, "--bes-id=") {
- c.besId = strings.TrimPrefix(arg, "--bes-id=")
- } else if len(arg) > 0 && arg[0] == '-' {
- parseArgNum := func(def int) int {
- if len(arg) > 2 {
- p, err := strconv.ParseUint(arg[2:], 10, 31)
- if err != nil {
- ctx.Fatalf("Failed to parse %q: %v", arg, err)
- }
- return int(p)
- } else if i+1 < len(args) {
- p, err := strconv.ParseUint(args[i+1], 10, 31)
- if err == nil {
- i++
- return int(p)
- }
- }
- return def
- }
- if len(arg) > 1 && arg[1] == 'j' {
- c.parallel = parseArgNum(c.parallel)
- } else if len(arg) > 1 && arg[1] == 'k' {
- c.keepGoing = parseArgNum(0)
- } else {
- ctx.Fatalln("Unknown option:", arg)
- }
- } else if k, v, ok := decodeKeyValue(arg); ok && len(k) > 0 {
- if k == "OUT_DIR" {
- ctx.Fatalln("OUT_DIR may only be set in the environment, not as a command line option.")
- }
- c.environ.Set(k, v)
- } else if arg == "dist" {
- c.dist = true
- } else if arg == "json-module-graph" {
- c.jsonModuleGraph = true
- } else if arg == "bp2build" {
- c.bp2build = true
- } else if arg == "api_bp2build" {
- c.apiBp2build = true
- } else if arg == "queryview" {
- c.queryview = true
- } else if arg == "soong_docs" {
- c.soongDocs = true
- } else {
- if arg == "checkbuild" {
- c.checkbuild = true
- }
- c.arguments = append(c.arguments, arg)
- }
- }
- if (!c.bazelProdMode) && (!c.bazelStagingMode) {
- c.bazelProdMode = defaultBazelProdMode(c)
- }
- }
- func validateNinjaWeightList(weightListFilePath string) (err error) {
- data, err := os.ReadFile(weightListFilePath)
- if err != nil {
- return
- }
- lines := strings.Split(strings.TrimSpace(string(data)), "\n")
- for _, line := range lines {
- fields := strings.Split(line, ",")
- if len(fields) != 2 {
- return fmt.Errorf("wrong format, each line should have two fields, but '%s'", line)
- }
- _, err = strconv.Atoi(fields[1])
- if err != nil {
- return
- }
- }
- return
- }
- func (c *configImpl) configureLocale(ctx Context) {
- cmd := Command(ctx, Config{c}, "locale", "locale", "-a")
- output, err := cmd.Output()
- var locales []string
- if err == nil {
- locales = strings.Split(string(output), "\n")
- } else {
- // If we're unable to list the locales, let's assume en_US.UTF-8
- locales = []string{"en_US.UTF-8"}
- ctx.Verbosef("Failed to list locales (%q), falling back to %q", err, locales)
- }
- // gettext uses LANGUAGE, which is passed directly through
- // For LANG and LC_*, only preserve the evaluated version of
- // LC_MESSAGES
- userLang := ""
- if lc_all, ok := c.environ.Get("LC_ALL"); ok {
- userLang = lc_all
- } else if lc_messages, ok := c.environ.Get("LC_MESSAGES"); ok {
- userLang = lc_messages
- } else if lang, ok := c.environ.Get("LANG"); ok {
- userLang = lang
- }
- c.environ.UnsetWithPrefix("LC_")
- if userLang != "" {
- c.environ.Set("LC_MESSAGES", userLang)
- }
- // The for LANG, use C.UTF-8 if it exists (Debian currently, proposed
- // for others)
- if inList("C.UTF-8", locales) {
- c.environ.Set("LANG", "C.UTF-8")
- } else if inList("C.utf8", locales) {
- // These normalize to the same thing
- c.environ.Set("LANG", "C.UTF-8")
- } else if inList("en_US.UTF-8", locales) {
- c.environ.Set("LANG", "en_US.UTF-8")
- } else if inList("en_US.utf8", locales) {
- // These normalize to the same thing
- c.environ.Set("LANG", "en_US.UTF-8")
- } else {
- ctx.Fatalln("System doesn't support either C.UTF-8 or en_US.UTF-8")
- }
- }
- func (c *configImpl) Environment() *Environment {
- return c.environ
- }
- func (c *configImpl) Arguments() []string {
- return c.arguments
- }
- func (c *configImpl) SoongBuildInvocationNeeded() bool {
- if len(c.Arguments()) > 0 {
- // Explicit targets requested that are not special targets like b2pbuild
- // or the JSON module graph
- return true
- }
- if !c.JsonModuleGraph() && !c.Bp2Build() && !c.Queryview() && !c.SoongDocs() && !c.ApiBp2build() {
- // Command line was empty, the default Ninja target is built
- return true
- }
- // bp2build + dist may be used to dist bp2build logs but does not require SoongBuildInvocation
- if c.Dist() && !c.Bp2Build() {
- return true
- }
- // build.ninja doesn't need to be generated
- return false
- }
- func (c *configImpl) OutDir() string {
- if outDir, ok := c.environ.Get("OUT_DIR"); ok {
- return outDir
- }
- return "out"
- }
- func (c *configImpl) DistDir() string {
- return c.distDir
- }
- func (c *configImpl) RealDistDir() string {
- return c.distDir
- }
- func (c *configImpl) NinjaArgs() []string {
- if c.skipKati {
- return c.arguments
- }
- return c.ninjaArgs
- }
- func (c *configImpl) BazelOutDir() string {
- return filepath.Join(c.OutDir(), "bazel")
- }
- func (c *configImpl) bazelOutputBase() string {
- return filepath.Join(c.BazelOutDir(), "output")
- }
- func (c *configImpl) SoongOutDir() string {
- return filepath.Join(c.OutDir(), "soong")
- }
- func (c *configImpl) ApiSurfacesOutDir() string {
- return filepath.Join(c.OutDir(), "api_surfaces")
- }
- func (c *configImpl) PrebuiltOS() string {
- switch runtime.GOOS {
- case "linux":
- return "linux-x86"
- case "darwin":
- return "darwin-x86"
- default:
- panic("Unknown GOOS")
- }
- }
- func (c *configImpl) HostToolDir() string {
- if c.SkipKatiNinja() {
- return filepath.Join(c.SoongOutDir(), "host", c.PrebuiltOS(), "bin")
- } else {
- return filepath.Join(c.OutDir(), "host", c.PrebuiltOS(), "bin")
- }
- }
- func (c *configImpl) NamedGlobFile(name string) string {
- return shared.JoinPath(c.SoongOutDir(), "globs-"+name+".ninja")
- }
- func (c *configImpl) UsedEnvFile(tag string) string {
- if v, ok := c.environ.Get("TARGET_PRODUCT"); ok {
- return shared.JoinPath(c.SoongOutDir(), usedEnvFile+"."+v+"."+tag)
- }
- return shared.JoinPath(c.SoongOutDir(), usedEnvFile+"."+tag)
- }
- func (c *configImpl) Bp2BuildFilesMarkerFile() string {
- return shared.JoinPath(c.SoongOutDir(), "bp2build_files_marker")
- }
- func (c *configImpl) Bp2BuildWorkspaceMarkerFile() string {
- return shared.JoinPath(c.SoongOutDir(), "bp2build_workspace_marker")
- }
- func (c *configImpl) SoongDocsHtml() string {
- return shared.JoinPath(c.SoongOutDir(), "docs/soong_build.html")
- }
- func (c *configImpl) QueryviewMarkerFile() string {
- return shared.JoinPath(c.SoongOutDir(), "queryview.marker")
- }
- func (c *configImpl) ApiBp2buildMarkerFile() string {
- return shared.JoinPath(c.SoongOutDir(), "api_bp2build.marker")
- }
- func (c *configImpl) ModuleGraphFile() string {
- return shared.JoinPath(c.SoongOutDir(), "module-graph.json")
- }
- func (c *configImpl) ModuleActionsFile() string {
- return shared.JoinPath(c.SoongOutDir(), "module-actions.json")
- }
- func (c *configImpl) TempDir() string {
- return shared.TempDirForOutDir(c.SoongOutDir())
- }
- func (c *configImpl) FileListDir() string {
- return filepath.Join(c.OutDir(), ".module_paths")
- }
- func (c *configImpl) KatiSuffix() string {
- if c.katiSuffix != "" {
- return c.katiSuffix
- }
- panic("SetKatiSuffix has not been called")
- }
- // Checkbuild returns true if "checkbuild" was one of the build goals, which means that the
- // user is interested in additional checks at the expense of build time.
- func (c *configImpl) Checkbuild() bool {
- return c.checkbuild
- }
- func (c *configImpl) Dist() bool {
- return c.dist
- }
- func (c *configImpl) JsonModuleGraph() bool {
- return c.jsonModuleGraph
- }
- func (c *configImpl) Bp2Build() bool {
- return c.bp2build
- }
- func (c *configImpl) ApiBp2build() bool {
- return c.apiBp2build
- }
- func (c *configImpl) Queryview() bool {
- return c.queryview
- }
- func (c *configImpl) SoongDocs() bool {
- return c.soongDocs
- }
- func (c *configImpl) IsVerbose() bool {
- return c.verbose
- }
- func (c *configImpl) MultitreeBuild() bool {
- return c.multitreeBuild
- }
- func (c *configImpl) NinjaWeightListSource() NinjaWeightListSource {
- return c.ninjaWeightListSource
- }
- func (c *configImpl) SkipKati() bool {
- return c.skipKati
- }
- func (c *configImpl) SkipKatiNinja() bool {
- return c.skipKatiNinja
- }
- func (c *configImpl) SkipSoong() bool {
- return c.skipSoong
- }
- func (c *configImpl) SkipNinja() bool {
- return c.skipNinja
- }
- func (c *configImpl) SetSkipNinja(v bool) {
- c.skipNinja = v
- }
- func (c *configImpl) SkipConfig() bool {
- return c.skipConfig
- }
- func (c *configImpl) BuildFromTextStub() bool {
- return c.buildFromTextStub
- }
- func (c *configImpl) TargetProduct() string {
- if v, ok := c.environ.Get("TARGET_PRODUCT"); ok {
- return v
- }
- panic("TARGET_PRODUCT is not defined")
- }
- func (c *configImpl) TargetProductOrErr() (string, error) {
- if v, ok := c.environ.Get("TARGET_PRODUCT"); ok {
- return v, nil
- }
- return "", fmt.Errorf("TARGET_PRODUCT is not defined")
- }
- func (c *configImpl) TargetDevice() string {
- return c.targetDevice
- }
- func (c *configImpl) SetTargetDevice(device string) {
- c.targetDevice = device
- }
- func (c *configImpl) TargetBuildVariant() string {
- if v, ok := c.environ.Get("TARGET_BUILD_VARIANT"); ok {
- return v
- }
- panic("TARGET_BUILD_VARIANT is not defined")
- }
- func (c *configImpl) KatiArgs() []string {
- return c.katiArgs
- }
- func (c *configImpl) Parallel() int {
- return c.parallel
- }
- func (c *configImpl) GetSourceRootDirs() []string {
- return c.sourceRootDirs
- }
- func (c *configImpl) SetSourceRootDirs(i []string) {
- c.sourceRootDirs = i
- }
- func (c *configImpl) GetIncludeTags() []string {
- return c.includeTags
- }
- func (c *configImpl) SetIncludeTags(i []string) {
- c.includeTags = i
- }
- func (c *configImpl) GetLogsPrefix() string {
- return c.logsPrefix
- }
- func (c *configImpl) SetLogsPrefix(prefix string) {
- c.logsPrefix = prefix
- }
- func (c *configImpl) HighmemParallel() int {
- if i, ok := c.environ.GetInt("NINJA_HIGHMEM_NUM_JOBS"); ok {
- return i
- }
- const minMemPerHighmemProcess = 8 * 1024 * 1024 * 1024
- parallel := c.Parallel()
- if c.UseRemoteBuild() {
- // Ninja doesn't support nested pools, and when remote builds are enabled the total ninja parallelism
- // is set very high (i.e. 500). Using a large value here would cause the total number of running jobs
- // to be the sum of the sizes of the local and highmem pools, which will cause extra CPU contention.
- // Return 1/16th of the size of the local pool, rounding up.
- return (parallel + 15) / 16
- } else if c.totalRAM == 0 {
- // Couldn't detect the total RAM, don't restrict highmem processes.
- return parallel
- } else if c.totalRAM <= 16*1024*1024*1024 {
- // Less than 16GB of ram, restrict to 1 highmem processes
- return 1
- } else if c.totalRAM <= 32*1024*1024*1024 {
- // Less than 32GB of ram, restrict to 2 highmem processes
- return 2
- } else if p := int(c.totalRAM / minMemPerHighmemProcess); p < parallel {
- // If less than 8GB total RAM per process, reduce the number of highmem processes
- return p
- }
- // No restriction on highmem processes
- return parallel
- }
- func (c *configImpl) TotalRAM() uint64 {
- return c.totalRAM
- }
- // ForceUseGoma determines whether we should override Goma deprecation
- // and use Goma for the current build or not.
- func (c *configImpl) ForceUseGoma() bool {
- if v, ok := c.environ.Get("FORCE_USE_GOMA"); ok {
- v = strings.TrimSpace(v)
- if v != "" && v != "false" {
- return true
- }
- }
- return false
- }
- func (c *configImpl) UseGoma() bool {
- if v, ok := c.environ.Get("USE_GOMA"); ok {
- v = strings.TrimSpace(v)
- if v != "" && v != "false" {
- return true
- }
- }
- return false
- }
- func (c *configImpl) StartGoma() bool {
- if !c.UseGoma() {
- return false
- }
- if v, ok := c.environ.Get("NOSTART_GOMA"); ok {
- v = strings.TrimSpace(v)
- if v != "" && v != "false" {
- return false
- }
- }
- return true
- }
- func (c *configImpl) UseRBE() bool {
- // These alternate modes of running Soong do not use RBE / reclient.
- if c.Bp2Build() || c.Queryview() || c.ApiBp2build() || c.JsonModuleGraph() {
- return false
- }
- authType, _ := c.rbeAuth()
- // Do not use RBE with prod credentials in scenarios when stubby doesn't exist, since
- // its unlikely that we will be able to obtain necessary creds without stubby.
- if !c.StubbyExists() && strings.Contains(authType, "use_google_prod_creds") {
- return false
- }
- if v, ok := c.Environment().Get("USE_RBE"); ok {
- v = strings.TrimSpace(v)
- if v != "" && v != "false" {
- return true
- }
- }
- return false
- }
- func (c *configImpl) BazelBuildEnabled() bool {
- return c.bazelProdMode || c.bazelStagingMode
- }
- func (c *configImpl) StartRBE() bool {
- if !c.UseRBE() {
- return false
- }
- if v, ok := c.environ.Get("NOSTART_RBE"); ok {
- v = strings.TrimSpace(v)
- if v != "" && v != "false" {
- return false
- }
- }
- return true
- }
- func (c *configImpl) rbeProxyLogsDir() string {
- for _, f := range []string{"RBE_proxy_log_dir", "FLAG_output_dir"} {
- if v, ok := c.environ.Get(f); ok {
- return v
- }
- }
- buildTmpDir := shared.TempDirForOutDir(c.SoongOutDir())
- return filepath.Join(buildTmpDir, "rbe")
- }
- func (c *configImpl) rbeCacheDir() string {
- for _, f := range []string{"RBE_cache_dir", "FLAG_cache_dir"} {
- if v, ok := c.environ.Get(f); ok {
- return v
- }
- }
- return shared.JoinPath(c.SoongOutDir(), "rbe")
- }
- func (c *configImpl) shouldCleanupRBELogsDir() bool {
- // Perform a log directory cleanup only when the log directory
- // is auto created by the build rather than user-specified.
- for _, f := range []string{"RBE_proxy_log_dir", "FLAG_output_dir"} {
- if _, ok := c.environ.Get(f); ok {
- return false
- }
- }
- return true
- }
- func (c *configImpl) rbeExecRoot() string {
- for _, f := range []string{"RBE_exec_root", "FLAG_exec_root"} {
- if v, ok := c.environ.Get(f); ok {
- return v
- }
- }
- wd, err := os.Getwd()
- if err != nil {
- return ""
- }
- return wd
- }
- func (c *configImpl) rbeDir() string {
- if v, ok := c.environ.Get("RBE_DIR"); ok {
- return v
- }
- return "prebuilts/remoteexecution-client/live/"
- }
- func (c *configImpl) rbeReproxy() string {
- for _, f := range []string{"RBE_re_proxy", "FLAG_re_proxy"} {
- if v, ok := c.environ.Get(f); ok {
- return v
- }
- }
- return filepath.Join(c.rbeDir(), "reproxy")
- }
- func (c *configImpl) rbeAuth() (string, string) {
- credFlags := []string{
- "use_application_default_credentials",
- "use_gce_credentials",
- "credential_file",
- "use_google_prod_creds",
- }
- for _, cf := range credFlags {
- for _, f := range []string{"RBE_" + cf, "FLAG_" + cf} {
- if v, ok := c.environ.Get(f); ok {
- v = strings.TrimSpace(v)
- if v != "" && v != "false" && v != "0" {
- return "RBE_" + cf, v
- }
- }
- }
- }
- return "RBE_use_application_default_credentials", "true"
- }
- func (c *configImpl) rbeSockAddr(dir string) (string, error) {
- maxNameLen := len(syscall.RawSockaddrUnix{}.Path)
- base := fmt.Sprintf("reproxy_%v.sock", rbeRandPrefix)
- name := filepath.Join(dir, base)
- if len(name) < maxNameLen {
- return name, nil
- }
- name = filepath.Join("/tmp", base)
- if len(name) < maxNameLen {
- return name, nil
- }
- return "", fmt.Errorf("cannot generate a proxy socket address shorter than the limit of %v", maxNameLen)
- }
- // IsGooglerEnvironment returns true if the current build is running
- // on a Google developer machine and false otherwise.
- func (c *configImpl) IsGooglerEnvironment() bool {
- cf := "ANDROID_BUILD_ENVIRONMENT_CONFIG"
- if v, ok := c.environ.Get(cf); ok {
- return v == "googler"
- }
- return false
- }
- // GoogleProdCredsExist determine whether credentials exist on the
- // Googler machine to use remote execution.
- func (c *configImpl) GoogleProdCredsExist() bool {
- if googleProdCredsExistCache {
- return googleProdCredsExistCache
- }
- if _, err := exec.Command("/usr/bin/gcertstatus", "-nocheck_ssh").Output(); err != nil {
- return false
- }
- googleProdCredsExistCache = true
- return true
- }
- // UseRemoteBuild indicates whether to use a remote build acceleration system
- // to speed up the build.
- func (c *configImpl) UseRemoteBuild() bool {
- return c.UseGoma() || c.UseRBE()
- }
- // StubbyExists checks whether the stubby binary exists on the machine running
- // the build.
- func (c *configImpl) StubbyExists() bool {
- if _, err := exec.LookPath("stubby"); err != nil {
- return false
- }
- return true
- }
- // RemoteParallel controls how many remote jobs (i.e., commands which contain
- // gomacc) are run in parallel. Note the parallelism of all other jobs is
- // still limited by Parallel()
- func (c *configImpl) RemoteParallel() int {
- if !c.UseRemoteBuild() {
- return 0
- }
- if i, ok := c.environ.GetInt("NINJA_REMOTE_NUM_JOBS"); ok {
- return i
- }
- return 500
- }
- func (c *configImpl) SetKatiArgs(args []string) {
- c.katiArgs = args
- }
- func (c *configImpl) SetNinjaArgs(args []string) {
- c.ninjaArgs = args
- }
- func (c *configImpl) SetKatiSuffix(suffix string) {
- c.katiSuffix = suffix
- }
- func (c *configImpl) LastKatiSuffixFile() string {
- return filepath.Join(c.OutDir(), "last_kati_suffix")
- }
- func (c *configImpl) HasKatiSuffix() bool {
- return c.katiSuffix != ""
- }
- func (c *configImpl) KatiEnvFile() string {
- return filepath.Join(c.OutDir(), "env"+c.KatiSuffix()+".sh")
- }
- func (c *configImpl) KatiBuildNinjaFile() string {
- return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiBuildSuffix+".ninja")
- }
- func (c *configImpl) KatiPackageNinjaFile() string {
- return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiPackageSuffix+".ninja")
- }
- func (c *configImpl) SoongVarsFile() string {
- targetProduct, err := c.TargetProductOrErr()
- if err != nil {
- return filepath.Join(c.SoongOutDir(), "soong.variables")
- } else {
- return filepath.Join(c.SoongOutDir(), "soong."+targetProduct+".variables")
- }
- }
- func (c *configImpl) SoongNinjaFile() string {
- targetProduct, err := c.TargetProductOrErr()
- if err != nil {
- return filepath.Join(c.SoongOutDir(), "build.ninja")
- } else {
- return filepath.Join(c.SoongOutDir(), "build."+targetProduct+".ninja")
- }
- }
- func (c *configImpl) CombinedNinjaFile() string {
- if c.katiSuffix == "" {
- return filepath.Join(c.OutDir(), "combined.ninja")
- }
- return filepath.Join(c.OutDir(), "combined"+c.KatiSuffix()+".ninja")
- }
- func (c *configImpl) SoongAndroidMk() string {
- return filepath.Join(c.SoongOutDir(), "Android-"+c.TargetProduct()+".mk")
- }
- func (c *configImpl) SoongMakeVarsMk() string {
- return filepath.Join(c.SoongOutDir(), "make_vars-"+c.TargetProduct()+".mk")
- }
- func (c *configImpl) ProductOut() string {
- return filepath.Join(c.OutDir(), "target", "product", c.TargetDevice())
- }
- func (c *configImpl) DevicePreviousProductConfig() string {
- return filepath.Join(c.ProductOut(), "previous_build_config.mk")
- }
- func (c *configImpl) KatiPackageMkDir() string {
- return filepath.Join(c.ProductOut(), "obj", "CONFIG", "kati_packaging")
- }
- func (c *configImpl) hostOutRoot() string {
- return filepath.Join(c.OutDir(), "host")
- }
- func (c *configImpl) HostOut() string {
- return filepath.Join(c.hostOutRoot(), c.HostPrebuiltTag())
- }
- // This probably needs to be multi-valued, so not exporting it for now
- func (c *configImpl) hostCrossOut() string {
- if runtime.GOOS == "linux" {
- return filepath.Join(c.hostOutRoot(), "windows-x86")
- } else {
- return ""
- }
- }
- func (c *configImpl) HostPrebuiltTag() string {
- if runtime.GOOS == "linux" {
- return "linux-x86"
- } else if runtime.GOOS == "darwin" {
- return "darwin-x86"
- } else {
- panic("Unsupported OS")
- }
- }
- func (c *configImpl) PrebuiltBuildTool(name string) string {
- if v, ok := c.environ.Get("SANITIZE_HOST"); ok {
- if sanitize := strings.Fields(v); inList("address", sanitize) {
- asan := filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "asan/bin", name)
- if _, err := os.Stat(asan); err == nil {
- return asan
- }
- }
- }
- return filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "bin", name)
- }
- func (c *configImpl) SetBuildBrokenDupRules(val bool) {
- c.brokenDupRules = val
- }
- func (c *configImpl) BuildBrokenDupRules() bool {
- return c.brokenDupRules
- }
- func (c *configImpl) SetBuildBrokenUsesNetwork(val bool) {
- c.brokenUsesNetwork = val
- }
- func (c *configImpl) BuildBrokenUsesNetwork() bool {
- return c.brokenUsesNetwork
- }
- func (c *configImpl) SetBuildBrokenNinjaUsesEnvVars(val []string) {
- c.brokenNinjaEnvVars = val
- }
- func (c *configImpl) BuildBrokenNinjaUsesEnvVars() []string {
- return c.brokenNinjaEnvVars
- }
- func (c *configImpl) SetTargetDeviceDir(dir string) {
- c.targetDeviceDir = dir
- }
- func (c *configImpl) TargetDeviceDir() string {
- return c.targetDeviceDir
- }
- func (c *configImpl) BuildDateTime() string {
- return c.buildDateTime
- }
- func (c *configImpl) MetricsUploaderApp() string {
- return c.metricsUploader
- }
- // LogsDir returns the absolute path to the logs directory where build log and
- // metrics files are located. By default, the logs directory is the out
- // directory. If the argument dist is specified, the logs directory
- // is <dist_dir>/logs.
- func (c *configImpl) LogsDir() string {
- dir := c.OutDir()
- if c.Dist() {
- // Always write logs to the real dist dir, even if Bazel is using a rigged dist dir for other files
- dir = filepath.Join(c.RealDistDir(), "logs")
- }
- absDir, err := filepath.Abs(dir)
- if err != nil {
- fmt.Fprintf(os.Stderr, "\nError making log dir '%s' absolute: %s\n", dir, err.Error())
- os.Exit(1)
- }
- return absDir
- }
- // BazelMetricsDir returns the <logs dir>/bazel_metrics directory
- // where the bazel profiles are located.
- func (c *configImpl) BazelMetricsDir() string {
- return filepath.Join(c.LogsDir(), "bazel_metrics")
- }
- // MkFileMetrics returns the file path for make-related metrics.
- func (c *configImpl) MkMetrics() string {
- return filepath.Join(c.LogsDir(), "mk_metrics.pb")
- }
- func (c *configImpl) SetEmptyNinjaFile(v bool) {
- c.emptyNinjaFile = v
- }
- func (c *configImpl) EmptyNinjaFile() bool {
- return c.emptyNinjaFile
- }
- func (c *configImpl) IsBazelMixedBuildForceDisabled() bool {
- return c.Environment().IsEnvTrue("BUILD_BROKEN_DISABLE_BAZEL")
- }
- func (c *configImpl) IsPersistentBazelEnabled() bool {
- return c.Environment().IsEnvTrue("USE_PERSISTENT_BAZEL")
- }
- // GetBazeliskBazelVersion returns the Bazel version to use for this build,
- // or the empty string if the current canonical prod Bazel should be used.
- // This environment variable should only be set to debug the build system.
- // The Bazel version, if set, will be passed to Bazelisk, and Bazelisk will
- // handle downloading and invoking the correct Bazel binary.
- func (c *configImpl) GetBazeliskBazelVersion() string {
- value, _ := c.Environment().Get("USE_BAZEL_VERSION")
- return value
- }
- func (c *configImpl) BazelModulesForceEnabledByFlag() string {
- return c.bazelForceEnabledModules
- }
- func (c *configImpl) SkipMetricsUpload() bool {
- return c.skipMetricsUpload
- }
- func (c *configImpl) EnsureAllowlistIntegrity() bool {
- return c.ensureAllowlistIntegrity
- }
- // Returns a Time object if one was passed via a command-line flag.
- // Otherwise returns the passed default.
- func (c *configImpl) BuildStartedTimeOrDefault(defaultTime time.Time) time.Time {
- if c.buildStartedTime == 0 {
- return defaultTime
- }
- return time.UnixMilli(c.buildStartedTime)
- }
- func (c *configImpl) BazelExitCode() int32 {
- return c.bazelExitCode
- }
- func GetMetricsUploader(topDir string, env *Environment) string {
- if p, ok := env.Get("METRICS_UPLOADER"); ok {
- metricsUploader := filepath.Join(topDir, p)
- if _, err := os.Stat(metricsUploader); err == nil {
- return metricsUploader
- }
- }
- return ""
- }
|