// Copyright 2021 Google Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package android import ( "bufio" "errors" "fmt" "strings" "android/soong/ui/metrics/bp2build_metrics_proto" "github.com/google/blueprint" "github.com/google/blueprint/proptools" "android/soong/android/allowlists" ) const ( // A sentinel value to be used as a key in Bp2BuildConfig for modules with // no package path. This is also the module dir for top level Android.bp // modules. Bp2BuildTopLevel = "." ) type MixedBuildEnabledStatus int const ( // This module can be mixed_built. MixedBuildEnabled = iota // There is a technical incompatibility preventing this module from being // bazel-analyzed. Note: the module might also be incompatible. TechnicalIncompatibility // This module cannot be mixed_built due to some incompatibility with it // that is not a platform incompatibility. Example: the module-type is not // enabled, or is not bp2build-converted. ModuleIncompatibility // Missing dependencies. We can't query Bazel for modules if it has missing dependencies, there // will be failures. ModuleMissingDeps ) // FileGroupAsLibrary describes a filegroup module that is converted to some library // such as aidl_library or proto_library. type FileGroupAsLibrary interface { ShouldConvertToAidlLibrary(ctx BazelConversionPathContext) bool ShouldConvertToProtoLibrary(ctx BazelConversionPathContext) bool GetAidlLibraryLabel(ctx BazelConversionPathContext) string GetProtoLibraryLabel(ctx BazelConversionPathContext) string } type BazelConversionStatus struct { // Information about _all_ bp2build targets generated by this module. Multiple targets are // supported as Soong handles some things within a single target that we may choose to split into // multiple targets, e.g. renderscript, protos, yacc within a cc module. Bp2buildInfo []bp2buildInfo `blueprint:"mutated"` // UnconvertedBp2buildDep stores the module names of direct dependency that were not converted to // Bazel UnconvertedDeps []string `blueprint:"mutated"` // MissingBp2buildDep stores the module names of direct dependency that were not found MissingDeps []string `blueprint:"mutated"` // If non-nil, indicates that the module could not be converted successfully // with bp2build. This will describe the reason the module could not be converted. UnconvertedReason *UnconvertedReason } // The reason a module could not be converted to a BUILD target via bp2build. // This should match bp2build_metrics_proto.UnconvertedReason, but omits private // proto-related fields that prevent copying this struct. type UnconvertedReason struct { // Should correspond to a valid value in bp2build_metrics_proto.UnconvertedReasonType. // A raw int is used here instead, because blueprint logic requires that all transitive // fields of module definitions be primitives. ReasonType int Detail string } type BazelModuleProperties struct { // The label of the Bazel target replacing this Soong module. When run in conversion mode, this // will import the handcrafted build target into the autogenerated file. Note: this may result in // a conflict due to duplicate targets if bp2build_available is also set. Label *string // If true, bp2build will generate the converted Bazel target for this module. Note: this may // cause a conflict due to the duplicate targets if label is also set. // // This is a bool pointer to support tristates: true, false, not set. // // To opt in a module, set bazel_module: { bp2build_available: true } // To opt out a module, set bazel_module: { bp2build_available: false } // To defer the default setting for the directory, do not set the value. Bp2build_available *bool // CanConvertToBazel is set via InitBazelModule to indicate that a module type can be converted to // Bazel with Bp2build. CanConvertToBazel bool `blueprint:"mutated"` } // Properties contains common module properties for Bazel migration purposes. type properties struct { // In "Bazel mixed build" mode, this represents the Bazel target replacing // this Soong module. Bazel_module BazelModuleProperties } // namespacedVariableProperties is a map from a string representing a Soong // config variable namespace, like "android" or "vendor_name" to a slice of // pointer to a struct containing a single field called Soong_config_variables // whose value mirrors the structure in the Blueprint file. type namespacedVariableProperties map[string][]interface{} // BazelModuleBase contains the property structs with metadata for modules which can be converted to // Bazel. type BazelModuleBase struct { bazelProperties properties // namespacedVariableProperties is used for soong_config_module_type support // in bp2build. Soong config modules allow users to set module properties // based on custom product variables defined in Android.bp files. These // variables are namespaced to prevent clobbering, especially when set from // Makefiles. namespacedVariableProperties namespacedVariableProperties // baseModuleType is set when this module was created from a module type // defined by a soong_config_module_type. Every soong_config_module_type // "wraps" another module type, e.g. a soong_config_module_type can wrap a // cc_defaults to a custom_cc_defaults, or cc_binary to a custom_cc_binary. // This baseModuleType is set to the wrapped module type. baseModuleType string } // Bazelable is specifies the interface for modules that can be converted to Bazel. type Bazelable interface { bazelProps() *properties HasHandcraftedLabel() bool HandcraftedLabel() string GetBazelLabel(ctx BazelConversionPathContext, module blueprint.Module) string ShouldConvertWithBp2build(ctx BazelConversionContext) bool shouldConvertWithBp2build(ctx bazelOtherModuleContext, module blueprint.Module) bool // ConvertWithBp2build either converts the module to a Bazel build target or // declares the module as unconvertible (for logging and metrics). // Modules must implement this function to be bp2build convertible. The function // must either create at least one Bazel target module (using ctx.CreateBazelTargetModule or // its related functions), or declare itself unconvertible using ctx.MarkBp2buildUnconvertible. ConvertWithBp2build(ctx TopDownMutatorContext) // namespacedVariableProps is a map from a soong config variable namespace // (e.g. acme, android) to a map of interfaces{}, which are really // reflect.Struct pointers, representing the value of the // soong_config_variables property of a module. The struct pointer is the // one with the single member called Soong_config_variables, which itself is // a struct containing fields for each supported feature in that namespace. // // The reason for using a slice of interface{} is to support defaults // propagation of the struct pointers. namespacedVariableProps() namespacedVariableProperties setNamespacedVariableProps(props namespacedVariableProperties) BaseModuleType() string SetBaseModuleType(baseModuleType string) } // ApiProvider is implemented by modules that contribute to an API surface type ApiProvider interface { ConvertWithApiBp2build(ctx TopDownMutatorContext) } // MixedBuildBuildable is an interface that module types should implement in order // to be "handled by Bazel" in a mixed build. type MixedBuildBuildable interface { // IsMixedBuildSupported returns true if and only if this module should be // "handled by Bazel" in a mixed build. // This "escape hatch" allows modules with corner-case scenarios to opt out // of being built with Bazel. IsMixedBuildSupported(ctx BaseModuleContext) bool // QueueBazelCall invokes request-queueing functions on the BazelContext // so that these requests are handled when Bazel's cquery is invoked. QueueBazelCall(ctx BaseModuleContext) // ProcessBazelQueryResponse uses Bazel information (obtained from the BazelContext) // to set module fields and providers to propagate this module's metadata upstream. // This effectively "bridges the gap" between Bazel and Soong in a mixed build. // Soong modules depending on this module should be oblivious to the fact that // this module was handled by Bazel. ProcessBazelQueryResponse(ctx ModuleContext) } // BazelModule is a lightweight wrapper interface around Module for Bazel-convertible modules. type BazelModule interface { Module Bazelable } // InitBazelModule is a wrapper function that decorates a BazelModule with Bazel-conversion // properties. func InitBazelModule(module BazelModule) { module.AddProperties(module.bazelProps()) module.bazelProps().Bazel_module.CanConvertToBazel = true } // bazelProps returns the Bazel properties for the given BazelModuleBase. func (b *BazelModuleBase) bazelProps() *properties { return &b.bazelProperties } func (b *BazelModuleBase) namespacedVariableProps() namespacedVariableProperties { return b.namespacedVariableProperties } func (b *BazelModuleBase) setNamespacedVariableProps(props namespacedVariableProperties) { b.namespacedVariableProperties = props } func (b *BazelModuleBase) BaseModuleType() string { return b.baseModuleType } func (b *BazelModuleBase) SetBaseModuleType(baseModuleType string) { b.baseModuleType = baseModuleType } // HasHandcraftedLabel returns whether this module has a handcrafted Bazel label. func (b *BazelModuleBase) HasHandcraftedLabel() bool { return b.bazelProperties.Bazel_module.Label != nil } // HandcraftedLabel returns the handcrafted label for this module, or empty string if there is none func (b *BazelModuleBase) HandcraftedLabel() string { return proptools.String(b.bazelProperties.Bazel_module.Label) } // GetBazelLabel returns the Bazel label for the given BazelModuleBase. func (b *BazelModuleBase) GetBazelLabel(ctx BazelConversionPathContext, module blueprint.Module) string { if b.HasHandcraftedLabel() { return b.HandcraftedLabel() } if b.ShouldConvertWithBp2build(ctx) { return bp2buildModuleLabel(ctx, module) } panic(fmt.Errorf("requested non-existent label for module ", module.Name())) } type Bp2BuildConversionAllowlist struct { // Configure modules in these directories to enable bp2build_available: true or false by default. defaultConfig allowlists.Bp2BuildConfig // Keep any existing BUILD files (and do not generate new BUILD files) for these directories // in the synthetic Bazel workspace. keepExistingBuildFile map[string]bool // Per-module allowlist to always opt modules into both bp2build and Bazel Dev Mode mixed // builds. These modules are usually in directories with many other modules that are not ready // for conversion. // // A module can either be in this list or its directory allowlisted entirely // in bp2buildDefaultConfig, but not both at the same time. moduleAlwaysConvert map[string]bool // Per-module-type allowlist to always opt modules in to both bp2build and // Bazel Dev Mode mixed builds when they have the same type as one listed. moduleTypeAlwaysConvert map[string]bool // Per-module denylist to always opt modules out of bp2build conversion. moduleDoNotConvert map[string]bool } // NewBp2BuildAllowlist creates a new, empty Bp2BuildConversionAllowlist // which can be populated using builder pattern Set* methods func NewBp2BuildAllowlist() Bp2BuildConversionAllowlist { return Bp2BuildConversionAllowlist{ allowlists.Bp2BuildConfig{}, map[string]bool{}, map[string]bool{}, map[string]bool{}, map[string]bool{}, } } // SetDefaultConfig copies the entries from defaultConfig into the allowlist func (a Bp2BuildConversionAllowlist) SetDefaultConfig(defaultConfig allowlists.Bp2BuildConfig) Bp2BuildConversionAllowlist { if a.defaultConfig == nil { a.defaultConfig = allowlists.Bp2BuildConfig{} } for k, v := range defaultConfig { a.defaultConfig[k] = v } return a } // SetKeepExistingBuildFile copies the entries from keepExistingBuildFile into the allowlist func (a Bp2BuildConversionAllowlist) SetKeepExistingBuildFile(keepExistingBuildFile map[string]bool) Bp2BuildConversionAllowlist { if a.keepExistingBuildFile == nil { a.keepExistingBuildFile = map[string]bool{} } for k, v := range keepExistingBuildFile { a.keepExistingBuildFile[k] = v } return a } // SetModuleAlwaysConvertList copies the entries from moduleAlwaysConvert into the allowlist func (a Bp2BuildConversionAllowlist) SetModuleAlwaysConvertList(moduleAlwaysConvert []string) Bp2BuildConversionAllowlist { if a.moduleAlwaysConvert == nil { a.moduleAlwaysConvert = map[string]bool{} } for _, m := range moduleAlwaysConvert { a.moduleAlwaysConvert[m] = true } return a } // SetModuleTypeAlwaysConvertList copies the entries from moduleTypeAlwaysConvert into the allowlist func (a Bp2BuildConversionAllowlist) SetModuleTypeAlwaysConvertList(moduleTypeAlwaysConvert []string) Bp2BuildConversionAllowlist { if a.moduleTypeAlwaysConvert == nil { a.moduleTypeAlwaysConvert = map[string]bool{} } for _, m := range moduleTypeAlwaysConvert { a.moduleTypeAlwaysConvert[m] = true } return a } // SetModuleDoNotConvertList copies the entries from moduleDoNotConvert into the allowlist func (a Bp2BuildConversionAllowlist) SetModuleDoNotConvertList(moduleDoNotConvert []string) Bp2BuildConversionAllowlist { if a.moduleDoNotConvert == nil { a.moduleDoNotConvert = map[string]bool{} } for _, m := range moduleDoNotConvert { a.moduleDoNotConvert[m] = true } return a } // ShouldKeepExistingBuildFileForDir returns whether an existing BUILD file should be // added to the build symlink forest based on the current global configuration. func (a Bp2BuildConversionAllowlist) ShouldKeepExistingBuildFileForDir(dir string) bool { if _, ok := a.keepExistingBuildFile[dir]; ok { // Exact dir match return true } var i int // Check if subtree match for { j := strings.Index(dir[i:], "/") if j == -1 { return false //default } prefix := dir[0 : i+j] i = i + j + 1 // skip the "/" if recursive, ok := a.keepExistingBuildFile[prefix]; ok && recursive { return true } } } var bp2BuildAllowListKey = NewOnceKey("Bp2BuildAllowlist") var bp2buildAllowlist OncePer func GetBp2BuildAllowList() Bp2BuildConversionAllowlist { return bp2buildAllowlist.Once(bp2BuildAllowListKey, func() interface{} { return NewBp2BuildAllowlist().SetDefaultConfig(allowlists.Bp2buildDefaultConfig). SetKeepExistingBuildFile(allowlists.Bp2buildKeepExistingBuildFile). SetModuleAlwaysConvertList(allowlists.Bp2buildModuleAlwaysConvertList). SetModuleTypeAlwaysConvertList(allowlists.Bp2buildModuleTypeAlwaysConvertList). SetModuleDoNotConvertList(allowlists.Bp2buildModuleDoNotConvertList) }).(Bp2BuildConversionAllowlist) } // MixedBuildsEnabled returns a MixedBuildEnabledStatus regarding whether // a module is ready to be replaced by a converted or handcrafted Bazel target. // As a side effect, calling this method will also log whether this module is // mixed build enabled for metrics reporting. func MixedBuildsEnabled(ctx BaseModuleContext) MixedBuildEnabledStatus { platformIncompatible := isPlatformIncompatible(ctx.Os(), ctx.Arch().ArchType) if platformIncompatible { ctx.Config().LogMixedBuild(ctx, false) return TechnicalIncompatibility } if ctx.Config().AllowMissingDependencies() { missingDeps := ctx.getMissingDependencies() // If there are missing dependencies, querying Bazel will fail. Soong instead fails at execution // time, not loading/analysis. disable mixed builds and fall back to Soong to maintain that // behavior. if len(missingDeps) > 0 { ctx.Config().LogMixedBuild(ctx, false) return ModuleMissingDeps } } module := ctx.Module() apexInfo := ctx.Provider(ApexInfoProvider).(ApexInfo) withinApex := !apexInfo.IsForPlatform() mixedBuildEnabled := ctx.Config().IsMixedBuildsEnabled() && module.Enabled() && convertedToBazel(ctx, module) && ctx.Config().BazelContext.IsModuleNameAllowed(module.Name(), withinApex) ctx.Config().LogMixedBuild(ctx, mixedBuildEnabled) if mixedBuildEnabled { return MixedBuildEnabled } return ModuleIncompatibility } // ConvertedToBazel returns whether this module has been converted (with bp2build or manually) to Bazel. func convertedToBazel(ctx BazelConversionContext, module blueprint.Module) bool { b, ok := module.(Bazelable) if !ok { return false } return b.shouldConvertWithBp2build(ctx, module) || b.HasHandcraftedLabel() } // ShouldConvertWithBp2build returns whether the given BazelModuleBase should be converted with bp2build func (b *BazelModuleBase) ShouldConvertWithBp2build(ctx BazelConversionContext) bool { return b.shouldConvertWithBp2build(ctx, ctx.Module()) } type bazelOtherModuleContext interface { ModuleErrorf(format string, args ...interface{}) Config() Config OtherModuleType(m blueprint.Module) string OtherModuleName(m blueprint.Module) string OtherModuleDir(m blueprint.Module) string } func isPlatformIncompatible(osType OsType, arch ArchType) bool { return osType == Windows || // Windows toolchains are not currently supported. osType == LinuxBionic || // Linux Bionic toolchains are not currently supported. osType == LinuxMusl || // Linux musl toolchains are not currently supported (b/259266326). arch == Riscv64 // TODO(b/262192655) Riscv64 toolchains are not currently supported. } func (b *BazelModuleBase) shouldConvertWithBp2build(ctx bazelOtherModuleContext, module blueprint.Module) bool { if !b.bazelProps().Bazel_module.CanConvertToBazel { return false } // In api_bp2build mode, all soong modules that can provide API contributions should be converted // This is irrespective of its presence/absence in bp2build allowlists if ctx.Config().BuildMode == ApiBp2build { _, providesApis := module.(ApiProvider) return providesApis } propValue := b.bazelProperties.Bazel_module.Bp2build_available packagePath := moduleDirWithPossibleOverride(ctx, module) // Modules in unit tests which are enabled in the allowlist by type or name // trigger this conditional because unit tests run under the "." package path isTestModule := packagePath == Bp2BuildTopLevel && proptools.BoolDefault(propValue, false) if isTestModule { return true } moduleName := moduleNameWithPossibleOverride(ctx, module) allowlist := ctx.Config().Bp2buildPackageConfig moduleNameAllowed := allowlist.moduleAlwaysConvert[moduleName] moduleTypeAllowed := allowlist.moduleTypeAlwaysConvert[ctx.OtherModuleType(module)] allowlistConvert := moduleNameAllowed || moduleTypeAllowed if moduleNameAllowed && moduleTypeAllowed { ctx.ModuleErrorf("A module cannot be in moduleAlwaysConvert and also be in moduleTypeAlwaysConvert") return false } if allowlist.moduleDoNotConvert[moduleName] { if moduleNameAllowed { ctx.ModuleErrorf("a module cannot be in moduleDoNotConvert and also be in moduleAlwaysConvert") } return false } // This is a tristate value: true, false, or unset. if ok, directoryPath := bp2buildDefaultTrueRecursively(packagePath, allowlist.defaultConfig); ok { if moduleNameAllowed { ctx.ModuleErrorf("A module cannot be in a directory marked Bp2BuildDefaultTrue"+ " or Bp2BuildDefaultTrueRecursively and also be in moduleAlwaysConvert. Directory: '%s'"+ " Module: '%s'", directoryPath, moduleName) return false } // Allow modules to explicitly opt-out. return proptools.BoolDefault(propValue, true) } // Allow modules to explicitly opt-in. return proptools.BoolDefault(propValue, allowlistConvert) } // bp2buildDefaultTrueRecursively checks that the package contains a prefix from the // set of package prefixes where all modules must be converted. That is, if the // package is x/y/z, and the list contains either x, x/y, or x/y/z, this function will // return true. // // However, if the package is x/y, and it matches a Bp2BuildDefaultFalse "x/y" entry // exactly, this module will return false early. // // This function will also return false if the package doesn't match anything in // the config. // // This function will also return the allowlist entry which caused a particular // package to be enabled. Since packages can be enabled via a recursive declaration, // the path returned will not always be the same as the one provided. func bp2buildDefaultTrueRecursively(packagePath string, config allowlists.Bp2BuildConfig) (bool, string) { // Check if the package path has an exact match in the config. if config[packagePath] == allowlists.Bp2BuildDefaultTrue || config[packagePath] == allowlists.Bp2BuildDefaultTrueRecursively { return true, packagePath } else if config[packagePath] == allowlists.Bp2BuildDefaultFalse || config[packagePath] == allowlists.Bp2BuildDefaultFalseRecursively { return false, packagePath } // If not, check for the config recursively. packagePrefix := packagePath // e.g. for x/y/z, iterate over x/y, then x, taking the most-specific value from the allowlist. for strings.Contains(packagePrefix, "/") { dirIndex := strings.LastIndex(packagePrefix, "/") packagePrefix = packagePrefix[:dirIndex] switch value := config[packagePrefix]; value { case allowlists.Bp2BuildDefaultTrueRecursively: // package contains this prefix and this prefix should convert all modules return true, packagePrefix case allowlists.Bp2BuildDefaultFalseRecursively: //package contains this prefix and this prefix should NOT convert any modules return false, packagePrefix } // Continue to the next part of the package dir. } return false, packagePath } func registerBp2buildConversionMutator(ctx RegisterMutatorsContext) { ctx.TopDown("bp2build_conversion", bp2buildConversionMutator).Parallel() } func bp2buildConversionMutator(ctx TopDownMutatorContext) { if ctx.Config().HasBazelBuildTargetInSource(ctx) { // Defer to the BUILD target. Generating an additional target would // cause a BUILD file conflict. ctx.MarkBp2buildUnconvertible(bp2build_metrics_proto.UnconvertedReasonType_DEFINED_IN_BUILD_FILE, "") return } bModule, ok := ctx.Module().(Bazelable) if !ok { ctx.MarkBp2buildUnconvertible(bp2build_metrics_proto.UnconvertedReasonType_TYPE_UNSUPPORTED, "") return } // TODO: b/285631638 - Differentiate between denylisted modules and missing bp2build capabilities. if !bModule.shouldConvertWithBp2build(ctx, ctx.Module()) { ctx.MarkBp2buildUnconvertible(bp2build_metrics_proto.UnconvertedReasonType_UNSUPPORTED, "") return } bModule.ConvertWithBp2build(ctx) if !ctx.Module().base().IsConvertedByBp2build() && ctx.Module().base().GetUnconvertedReason() == nil { panic(fmt.Errorf("illegal bp2build invariant: module '%s' was neither converted nor marked unconvertible", ctx.ModuleName())) } } func registerApiBp2buildConversionMutator(ctx RegisterMutatorsContext) { ctx.TopDown("apiBp2build_conversion", convertWithApiBp2build).Parallel() } // Generate API contribution targets if the Soong module provides APIs func convertWithApiBp2build(ctx TopDownMutatorContext) { if m, ok := ctx.Module().(ApiProvider); ok { m.ConvertWithApiBp2build(ctx) } } // GetMainClassInManifest scans the manifest file specified in filepath and returns // the value of attribute Main-Class in the manifest file if it exists, or returns error. // WARNING: this is for bp2build converters of java_* modules only. func GetMainClassInManifest(c Config, filepath string) (string, error) { file, err := c.fs.Open(filepath) if err != nil { return "", err } defer file.Close() scanner := bufio.NewScanner(file) for scanner.Scan() { line := scanner.Text() if strings.HasPrefix(line, "Main-Class:") { return strings.TrimSpace(line[len("Main-Class:"):]), nil } } return "", errors.New("Main-Class is not found.") } func AttachValidationActions(ctx ModuleContext, outputFilePath Path, validations Paths) ModuleOutPath { validatedOutputFilePath := PathForModuleOut(ctx, "validated", outputFilePath.Base()) ctx.Build(pctx, BuildParams{ Rule: CpNoPreserveSymlink, Description: "run validations " + outputFilePath.Base(), Output: validatedOutputFilePath, Input: outputFilePath, Validations: validations, }) return validatedOutputFilePath }