123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832 |
- // Copyright 2017 Google Inc. All rights reserved.
- //
- // Licensed under the Apache License, Version 2.0 (the "License");
- // you may not use this file except in compliance with the License.
- // You may obtain a copy of the License at
- //
- // http://www.apache.org/licenses/LICENSE-2.0
- //
- // Unless required by applicable law or agreed to in writing, software
- // distributed under the License is distributed on an "AS IS" BASIS,
- // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- // See the License for the specific language governing permissions and
- // limitations under the License.
- package python
- // This file contains the "Base" module type for building Python program.
- import (
- "fmt"
- "path/filepath"
- "regexp"
- "strings"
- "android/soong/bazel"
- "github.com/google/blueprint"
- "github.com/google/blueprint/proptools"
- "android/soong/android"
- )
- func init() {
- registerPythonMutators(android.InitRegistrationContext)
- }
- func registerPythonMutators(ctx android.RegistrationContext) {
- ctx.PreDepsMutators(RegisterPythonPreDepsMutators)
- }
- // Exported to support other packages using Python modules in tests.
- func RegisterPythonPreDepsMutators(ctx android.RegisterMutatorsContext) {
- ctx.BottomUp("python_version", versionSplitMutator()).Parallel()
- }
- // the version-specific properties that apply to python modules.
- type VersionProperties struct {
- // whether the module is required to be built with this version.
- // Defaults to true for Python 3, and false otherwise.
- Enabled *bool
- // list of source files specific to this Python version.
- // Using the syntax ":module", srcs may reference the outputs of other modules that produce source files,
- // e.g. genrule or filegroup.
- Srcs []string `android:"path,arch_variant"`
- // list of source files that should not be used to build the Python module for this version.
- // This is most useful to remove files that are not common to all Python versions.
- Exclude_srcs []string `android:"path,arch_variant"`
- // list of the Python libraries used only for this Python version.
- Libs []string `android:"arch_variant"`
- // whether the binary is required to be built with embedded launcher for this version, defaults to false.
- Embedded_launcher *bool // TODO(b/174041232): Remove this property
- }
- // properties that apply to all python modules
- type BaseProperties struct {
- // the package path prefix within the output artifact at which to place the source/data
- // files of the current module.
- // eg. Pkg_path = "a/b/c"; Other packages can reference this module by using
- // (from a.b.c import ...) statement.
- // if left unspecified, all the source/data files path is unchanged within zip file.
- Pkg_path *string
- // true, if the Python module is used internally, eg, Python std libs.
- Is_internal *bool
- // list of source (.py) files compatible both with Python2 and Python3 used to compile the
- // Python module.
- // srcs may reference the outputs of other modules that produce source files like genrule
- // or filegroup using the syntax ":module".
- // Srcs has to be non-empty.
- Srcs []string `android:"path,arch_variant"`
- // list of source files that should not be used to build the C/C++ module.
- // This is most useful in the arch/multilib variants to remove non-common files
- Exclude_srcs []string `android:"path,arch_variant"`
- // list of files or filegroup modules that provide data that should be installed alongside
- // the test. the file extension can be arbitrary except for (.py).
- Data []string `android:"path,arch_variant"`
- // list of java modules that provide data that should be installed alongside the test.
- Java_data []string
- // list of the Python libraries compatible both with Python2 and Python3.
- Libs []string `android:"arch_variant"`
- Version struct {
- // Python2-specific properties, including whether Python2 is supported for this module
- // and version-specific sources, exclusions and dependencies.
- Py2 VersionProperties `android:"arch_variant"`
- // Python3-specific properties, including whether Python3 is supported for this module
- // and version-specific sources, exclusions and dependencies.
- Py3 VersionProperties `android:"arch_variant"`
- } `android:"arch_variant"`
- // the actual version each module uses after variations created.
- // this property name is hidden from users' perspectives, and soong will populate it during
- // runtime.
- Actual_version string `blueprint:"mutated"`
- // whether the module is required to be built with actual_version.
- // this is set by the python version mutator based on version-specific properties
- Enabled *bool `blueprint:"mutated"`
- // whether the binary is required to be built with embedded launcher for this actual_version.
- // this is set by the python version mutator based on version-specific properties
- Embedded_launcher *bool `blueprint:"mutated"`
- }
- type baseAttributes struct {
- // TODO(b/200311466): Probably not translate b/c Bazel has no good equiv
- //Pkg_path bazel.StringAttribute
- // TODO: Related to Pkg_bath and similarLy gated
- //Is_internal bazel.BoolAttribute
- // Combines Srcs and Exclude_srcs
- Srcs bazel.LabelListAttribute
- Deps bazel.LabelListAttribute
- // Combines Data and Java_data (invariant)
- Data bazel.LabelListAttribute
- Imports bazel.StringListAttribute
- }
- // Used to store files of current module after expanding dependencies
- type pathMapping struct {
- dest string
- src android.Path
- }
- type Module struct {
- android.ModuleBase
- android.DefaultableModuleBase
- android.BazelModuleBase
- properties BaseProperties
- protoProperties android.ProtoProperties
- // initialize before calling Init
- hod android.HostOrDeviceSupported
- multilib android.Multilib
- // interface used to bootstrap .par executable when embedded_launcher is true
- // this should be set by Python modules which are runnable, e.g. binaries and tests
- // bootstrapper might be nil (e.g. Python library module).
- bootstrapper bootstrapper
- // interface that implements functions required for installation
- // this should be set by Python modules which are runnable, e.g. binaries and tests
- // installer might be nil (e.g. Python library module).
- installer installer
- // the Python files of current module after expanding source dependencies.
- // pathMapping: <dest: runfile_path, src: source_path>
- srcsPathMappings []pathMapping
- // the data files of current module after expanding source dependencies.
- // pathMapping: <dest: runfile_path, src: source_path>
- dataPathMappings []pathMapping
- // the zip filepath for zipping current module source/data files.
- srcsZip android.Path
- // dependency modules' zip filepath for zipping current module source/data files.
- depsSrcsZips android.Paths
- // (.intermediate) module output path as installation source.
- installSource android.OptionalPath
- // Map to ensure sub-part of the AndroidMk for this module is only added once
- subAndroidMkOnce map[subAndroidMkProvider]bool
- }
- // newModule generates new Python base module
- func newModule(hod android.HostOrDeviceSupported, multilib android.Multilib) *Module {
- return &Module{
- hod: hod,
- multilib: multilib,
- }
- }
- func (m *Module) makeArchVariantBaseAttributes(ctx android.TopDownMutatorContext) baseAttributes {
- var attrs baseAttributes
- archVariantBaseProps := m.GetArchVariantProperties(ctx, &BaseProperties{})
- for axis, configToProps := range archVariantBaseProps {
- for config, props := range configToProps {
- if baseProps, ok := props.(*BaseProperties); ok {
- attrs.Srcs.SetSelectValue(axis, config,
- android.BazelLabelForModuleSrcExcludes(ctx, baseProps.Srcs, baseProps.Exclude_srcs))
- attrs.Deps.SetSelectValue(axis, config,
- android.BazelLabelForModuleDeps(ctx, baseProps.Libs))
- data := android.BazelLabelForModuleSrc(ctx, baseProps.Data)
- data.Append(android.BazelLabelForModuleSrc(ctx, baseProps.Java_data))
- attrs.Data.SetSelectValue(axis, config, data)
- }
- }
- }
- partitionedSrcs := bazel.PartitionLabelListAttribute(ctx, &attrs.Srcs, bazel.LabelPartitions{
- "proto": android.ProtoSrcLabelPartition,
- "py": bazel.LabelPartition{Keep_remainder: true},
- })
- attrs.Srcs = partitionedSrcs["py"]
- if !partitionedSrcs["proto"].IsEmpty() {
- protoInfo, _ := android.Bp2buildProtoProperties(ctx, &m.ModuleBase, partitionedSrcs["proto"])
- protoLabel := bazel.Label{Label: ":" + protoInfo.Name}
- pyProtoLibraryName := m.Name() + "_py_proto"
- ctx.CreateBazelTargetModule(bazel.BazelTargetModuleProperties{
- Rule_class: "py_proto_library",
- Bzl_load_location: "//build/bazel/rules/python:py_proto.bzl",
- }, android.CommonAttributes{
- Name: pyProtoLibraryName,
- }, &bazelPythonProtoLibraryAttributes{
- Deps: bazel.MakeSingleLabelListAttribute(protoLabel),
- })
- attrs.Deps.Add(bazel.MakeLabelAttribute(":" + pyProtoLibraryName))
- }
- // Bazel normally requires `import path.from.top.of.tree` statements in
- // python code, but with soong you can directly import modules from libraries.
- // Add "imports" attributes to the bazel library so it matches soong's behavior.
- imports := "."
- if m.properties.Pkg_path != nil {
- // TODO(b/215119317) This is a hack to handle the fact that we don't convert
- // pkg_path properly right now. If the folder structure that contains this
- // Android.bp file matches pkg_path, we can set imports to an appropriate
- // number of ../..s to emulate moving the files under a pkg_path folder.
- pkg_path := filepath.Clean(*m.properties.Pkg_path)
- if strings.HasPrefix(pkg_path, "/") {
- ctx.ModuleErrorf("pkg_path cannot start with a /: %s", pkg_path)
- }
- if !strings.HasSuffix(ctx.ModuleDir(), "/"+pkg_path) && ctx.ModuleDir() != pkg_path {
- ctx.ModuleErrorf("Currently, bp2build only supports pkg_paths that are the same as the folders the Android.bp file is in. pkg_path: %s, module directory: %s", pkg_path, ctx.ModuleDir())
- }
- numFolders := strings.Count(pkg_path, "/") + 1
- dots := make([]string, numFolders)
- for i := 0; i < numFolders; i++ {
- dots[i] = ".."
- }
- imports = strings.Join(dots, "/")
- }
- attrs.Imports = bazel.MakeStringListAttribute([]string{imports})
- return attrs
- }
- // bootstrapper interface should be implemented for runnable modules, e.g. binary and test
- type bootstrapper interface {
- bootstrapperProps() []interface{}
- bootstrap(ctx android.ModuleContext, ActualVersion string, embeddedLauncher bool,
- srcsPathMappings []pathMapping, srcsZip android.Path,
- depsSrcsZips android.Paths) android.OptionalPath
- autorun() bool
- }
- // installer interface should be implemented for installable modules, e.g. binary and test
- type installer interface {
- install(ctx android.ModuleContext, path android.Path)
- setAndroidMkSharedLibs(sharedLibs []string)
- }
- // interface implemented by Python modules to provide source and data mappings and zip to python
- // modules that depend on it
- type pythonDependency interface {
- getSrcsPathMappings() []pathMapping
- getDataPathMappings() []pathMapping
- getSrcsZip() android.Path
- }
- // getSrcsPathMappings gets this module's path mapping of src source path : runfiles destination
- func (p *Module) getSrcsPathMappings() []pathMapping {
- return p.srcsPathMappings
- }
- // getSrcsPathMappings gets this module's path mapping of data source path : runfiles destination
- func (p *Module) getDataPathMappings() []pathMapping {
- return p.dataPathMappings
- }
- // getSrcsZip returns the filepath where the current module's source/data files are zipped.
- func (p *Module) getSrcsZip() android.Path {
- return p.srcsZip
- }
- var _ pythonDependency = (*Module)(nil)
- var _ android.AndroidMkEntriesProvider = (*Module)(nil)
- func (p *Module) init(additionalProps ...interface{}) android.Module {
- p.AddProperties(&p.properties, &p.protoProperties)
- // Add additional properties for bootstrapping/installation
- // This is currently tied to the bootstrapper interface;
- // however, these are a combination of properties for the installation and bootstrapping of a module
- if p.bootstrapper != nil {
- p.AddProperties(p.bootstrapper.bootstrapperProps()...)
- }
- android.InitAndroidArchModule(p, p.hod, p.multilib)
- android.InitDefaultableModule(p)
- return p
- }
- // Python-specific tag to transfer information on the purpose of a dependency.
- // This is used when adding a dependency on a module, which can later be accessed when visiting
- // dependencies.
- type dependencyTag struct {
- blueprint.BaseDependencyTag
- name string
- }
- // Python-specific tag that indicates that installed files of this module should depend on installed
- // files of the dependency
- type installDependencyTag struct {
- blueprint.BaseDependencyTag
- // embedding this struct provides the installation dependency requirement
- android.InstallAlwaysNeededDependencyTag
- name string
- }
- var (
- pythonLibTag = dependencyTag{name: "pythonLib"}
- javaDataTag = dependencyTag{name: "javaData"}
- launcherTag = dependencyTag{name: "launcher"}
- launcherSharedLibTag = installDependencyTag{name: "launcherSharedLib"}
- pathComponentRegexp = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_-]*$`)
- pyExt = ".py"
- protoExt = ".proto"
- pyVersion2 = "PY2"
- pyVersion3 = "PY3"
- initFileName = "__init__.py"
- mainFileName = "__main__.py"
- entryPointFile = "entry_point.txt"
- parFileExt = ".zip"
- internalPath = "internal"
- )
- // versionSplitMutator creates version variants for modules and appends the version-specific
- // properties for a given variant to the properties in the variant module
- func versionSplitMutator() func(android.BottomUpMutatorContext) {
- return func(mctx android.BottomUpMutatorContext) {
- if base, ok := mctx.Module().(*Module); ok {
- versionNames := []string{}
- // collect version specific properties, so that we can merge version-specific properties
- // into the module's overall properties
- versionProps := []VersionProperties{}
- // PY3 is first so that we alias the PY3 variant rather than PY2 if both
- // are available
- if proptools.BoolDefault(base.properties.Version.Py3.Enabled, true) {
- versionNames = append(versionNames, pyVersion3)
- versionProps = append(versionProps, base.properties.Version.Py3)
- }
- if proptools.BoolDefault(base.properties.Version.Py2.Enabled, false) {
- versionNames = append(versionNames, pyVersion2)
- versionProps = append(versionProps, base.properties.Version.Py2)
- }
- modules := mctx.CreateLocalVariations(versionNames...)
- // Alias module to the first variant
- if len(versionNames) > 0 {
- mctx.AliasVariation(versionNames[0])
- }
- for i, v := range versionNames {
- // set the actual version for Python module.
- modules[i].(*Module).properties.Actual_version = v
- // append versioned properties for the Python module to the overall properties
- err := proptools.AppendMatchingProperties([]interface{}{&modules[i].(*Module).properties}, &versionProps[i], nil)
- if err != nil {
- panic(err)
- }
- }
- }
- }
- }
- // HostToolPath returns a path if appropriate such that this module can be used as a host tool,
- // fulfilling HostToolProvider interface.
- func (p *Module) HostToolPath() android.OptionalPath {
- if p.installer != nil {
- if bin, ok := p.installer.(*binaryDecorator); ok {
- // TODO: This should only be set when building host binaries -- tests built for device would be
- // setting this incorrectly.
- return android.OptionalPathForPath(bin.path)
- }
- }
- return android.OptionalPath{}
- }
- // OutputFiles returns output files based on given tag, returns an error if tag is unsupported.
- func (p *Module) OutputFiles(tag string) (android.Paths, error) {
- switch tag {
- case "":
- if outputFile := p.installSource; outputFile.Valid() {
- return android.Paths{outputFile.Path()}, nil
- }
- return android.Paths{}, nil
- default:
- return nil, fmt.Errorf("unsupported module reference tag %q", tag)
- }
- }
- func (p *Module) isEmbeddedLauncherEnabled() bool {
- return p.installer != nil && Bool(p.properties.Embedded_launcher)
- }
- func anyHasExt(paths []string, ext string) bool {
- for _, p := range paths {
- if filepath.Ext(p) == ext {
- return true
- }
- }
- return false
- }
- func (p *Module) anySrcHasExt(ctx android.BottomUpMutatorContext, ext string) bool {
- return anyHasExt(p.properties.Srcs, ext)
- }
- // DepsMutator mutates dependencies for this module:
- // - handles proto dependencies,
- // - if required, specifies launcher and adds launcher dependencies,
- // - applies python version mutations to Python dependencies
- func (p *Module) DepsMutator(ctx android.BottomUpMutatorContext) {
- android.ProtoDeps(ctx, &p.protoProperties)
- versionVariation := []blueprint.Variation{
- {"python_version", p.properties.Actual_version},
- }
- // If sources contain a proto file, add dependency on libprotobuf-python
- if p.anySrcHasExt(ctx, protoExt) && p.Name() != "libprotobuf-python" {
- ctx.AddVariationDependencies(versionVariation, pythonLibTag, "libprotobuf-python")
- }
- // Add python library dependencies for this python version variation
- ctx.AddVariationDependencies(versionVariation, pythonLibTag, android.LastUniqueStrings(p.properties.Libs)...)
- // If this module will be installed and has an embedded launcher, we need to add dependencies for:
- // * standard library
- // * launcher
- // * shared dependencies of the launcher
- if p.installer != nil && p.isEmbeddedLauncherEnabled() {
- var stdLib string
- var launcherModule string
- // Add launcher shared lib dependencies. Ideally, these should be
- // derived from the `shared_libs` property of the launcher. However, we
- // cannot read the property at this stage and it will be too late to add
- // dependencies later.
- launcherSharedLibDeps := []string{
- "libsqlite",
- }
- // Add launcher-specific dependencies for bionic
- if ctx.Target().Os.Bionic() {
- launcherSharedLibDeps = append(launcherSharedLibDeps, "libc", "libdl", "libm")
- }
- if ctx.Target().Os == android.LinuxMusl && !ctx.Config().HostStaticBinaries() {
- launcherSharedLibDeps = append(launcherSharedLibDeps, "libc_musl")
- }
- switch p.properties.Actual_version {
- case pyVersion2:
- stdLib = "py2-stdlib"
- launcherModule = "py2-launcher"
- if p.bootstrapper.autorun() {
- launcherModule = "py2-launcher-autorun"
- }
- launcherSharedLibDeps = append(launcherSharedLibDeps, "libc++")
- case pyVersion3:
- stdLib = "py3-stdlib"
- launcherModule = "py3-launcher"
- if p.bootstrapper.autorun() {
- launcherModule = "py3-launcher-autorun"
- }
- if ctx.Config().HostStaticBinaries() && ctx.Target().Os == android.LinuxMusl {
- launcherModule += "-static"
- }
- if ctx.Device() {
- launcherSharedLibDeps = append(launcherSharedLibDeps, "liblog")
- }
- default:
- panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.",
- p.properties.Actual_version, ctx.ModuleName()))
- }
- ctx.AddVariationDependencies(versionVariation, pythonLibTag, stdLib)
- ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherTag, launcherModule)
- ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherSharedLibTag, launcherSharedLibDeps...)
- }
- // Emulate the data property for java_data but with the arch variation overridden to "common"
- // so that it can point to java modules.
- javaDataVariation := []blueprint.Variation{{"arch", android.Common.String()}}
- ctx.AddVariationDependencies(javaDataVariation, javaDataTag, p.properties.Java_data...)
- }
- func (p *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) {
- p.generatePythonBuildActions(ctx)
- // Only Python binary and test modules have non-empty bootstrapper.
- if p.bootstrapper != nil {
- // if the module is being installed, we need to collect all transitive dependencies to embed in
- // the final par
- p.collectPathsFromTransitiveDeps(ctx)
- // bootstrap the module, including resolving main file, getting launcher path, and
- // registering actions to build the par file
- // bootstrap returns the binary output path
- p.installSource = p.bootstrapper.bootstrap(ctx, p.properties.Actual_version,
- p.isEmbeddedLauncherEnabled(), p.srcsPathMappings, p.srcsZip, p.depsSrcsZips)
- }
- // Only Python binary and test modules have non-empty installer.
- if p.installer != nil {
- var sharedLibs []string
- // if embedded launcher is enabled, we need to collect the shared library depenendencies of the
- // launcher
- for _, dep := range ctx.GetDirectDepsWithTag(launcherSharedLibTag) {
- sharedLibs = append(sharedLibs, ctx.OtherModuleName(dep))
- }
- p.installer.setAndroidMkSharedLibs(sharedLibs)
- // Install the par file from installSource
- if p.installSource.Valid() {
- p.installer.install(ctx, p.installSource.Path())
- }
- }
- }
- // generatePythonBuildActions performs build actions common to all Python modules
- func (p *Module) generatePythonBuildActions(ctx android.ModuleContext) {
- expandedSrcs := android.PathsForModuleSrcExcludes(ctx, p.properties.Srcs, p.properties.Exclude_srcs)
- requiresSrcs := true
- if p.bootstrapper != nil && !p.bootstrapper.autorun() {
- requiresSrcs = false
- }
- if len(expandedSrcs) == 0 && requiresSrcs {
- ctx.ModuleErrorf("doesn't have any source files!")
- }
- // expand data files from "data" property.
- expandedData := android.PathsForModuleSrc(ctx, p.properties.Data)
- // Emulate the data property for java_data dependencies.
- for _, javaData := range ctx.GetDirectDepsWithTag(javaDataTag) {
- expandedData = append(expandedData, android.OutputFilesForModule(ctx, javaData, "")...)
- }
- // Validate pkg_path property
- pkgPath := String(p.properties.Pkg_path)
- if pkgPath != "" {
- // TODO: export validation from android/paths.go handling to replace this duplicated functionality
- pkgPath = filepath.Clean(String(p.properties.Pkg_path))
- if pkgPath == ".." || strings.HasPrefix(pkgPath, "../") ||
- strings.HasPrefix(pkgPath, "/") {
- ctx.PropertyErrorf("pkg_path",
- "%q must be a relative path contained in par file.",
- String(p.properties.Pkg_path))
- return
- }
- }
- // If property Is_internal is set, prepend pkgPath with internalPath
- if proptools.BoolDefault(p.properties.Is_internal, false) {
- pkgPath = filepath.Join(internalPath, pkgPath)
- }
- // generate src:destination path mappings for this module
- p.genModulePathMappings(ctx, pkgPath, expandedSrcs, expandedData)
- // generate the zipfile of all source and data files
- p.srcsZip = p.createSrcsZip(ctx, pkgPath)
- }
- func isValidPythonPath(path string) error {
- identifiers := strings.Split(strings.TrimSuffix(path, filepath.Ext(path)), "/")
- for _, token := range identifiers {
- if !pathComponentRegexp.MatchString(token) {
- return fmt.Errorf("the path %q contains invalid subpath %q. "+
- "Subpaths must be at least one character long. "+
- "The first character must an underscore or letter. "+
- "Following characters may be any of: letter, digit, underscore, hyphen.",
- path, token)
- }
- }
- return nil
- }
- // For this module, generate unique pathMappings: <dest: runfiles_path, src: source_path>
- // for python/data files expanded from properties.
- func (p *Module) genModulePathMappings(ctx android.ModuleContext, pkgPath string,
- expandedSrcs, expandedData android.Paths) {
- // fetch <runfiles_path, source_path> pairs from "src" and "data" properties to
- // check current module duplicates.
- destToPySrcs := make(map[string]string)
- destToPyData := make(map[string]string)
- for _, s := range expandedSrcs {
- if s.Ext() != pyExt && s.Ext() != protoExt {
- ctx.PropertyErrorf("srcs", "found non (.py|.proto) file: %q!", s.String())
- continue
- }
- runfilesPath := filepath.Join(pkgPath, s.Rel())
- if err := isValidPythonPath(runfilesPath); err != nil {
- ctx.PropertyErrorf("srcs", err.Error())
- }
- if !checkForDuplicateOutputPath(ctx, destToPySrcs, runfilesPath, s.String(), p.Name(), p.Name()) {
- p.srcsPathMappings = append(p.srcsPathMappings, pathMapping{dest: runfilesPath, src: s})
- }
- }
- for _, d := range expandedData {
- if d.Ext() == pyExt || d.Ext() == protoExt {
- ctx.PropertyErrorf("data", "found (.py|.proto) file: %q!", d.String())
- continue
- }
- runfilesPath := filepath.Join(pkgPath, d.Rel())
- if !checkForDuplicateOutputPath(ctx, destToPyData, runfilesPath, d.String(), p.Name(), p.Name()) {
- p.dataPathMappings = append(p.dataPathMappings,
- pathMapping{dest: runfilesPath, src: d})
- }
- }
- }
- // createSrcsZip registers build actions to zip current module's sources and data.
- func (p *Module) createSrcsZip(ctx android.ModuleContext, pkgPath string) android.Path {
- relativeRootMap := make(map[string]android.Paths)
- pathMappings := append(p.srcsPathMappings, p.dataPathMappings...)
- var protoSrcs android.Paths
- // "srcs" or "data" properties may contain filegroup so it might happen that
- // the root directory for each source path is different.
- for _, path := range pathMappings {
- // handle proto sources separately
- if path.src.Ext() == protoExt {
- protoSrcs = append(protoSrcs, path.src)
- } else {
- var relativeRoot string
- relativeRoot = strings.TrimSuffix(path.src.String(), path.src.Rel())
- if v, found := relativeRootMap[relativeRoot]; found {
- relativeRootMap[relativeRoot] = append(v, path.src)
- } else {
- relativeRootMap[relativeRoot] = android.Paths{path.src}
- }
- }
- }
- var zips android.Paths
- if len(protoSrcs) > 0 {
- protoFlags := android.GetProtoFlags(ctx, &p.protoProperties)
- protoFlags.OutTypeFlag = "--python_out"
- for _, srcFile := range protoSrcs {
- zip := genProto(ctx, srcFile, protoFlags, pkgPath)
- zips = append(zips, zip)
- }
- }
- if len(relativeRootMap) > 0 {
- // in order to keep stable order of soong_zip params, we sort the keys here.
- roots := android.SortedStringKeys(relativeRootMap)
- // Use -symlinks=false so that the symlinks in the bazel output directory are followed
- parArgs := []string{"-symlinks=false"}
- if pkgPath != "" {
- // use package path as path prefix
- parArgs = append(parArgs, `-P `+pkgPath)
- }
- paths := android.Paths{}
- for _, root := range roots {
- // specify relative root of file in following -f arguments
- parArgs = append(parArgs, `-C `+root)
- for _, path := range relativeRootMap[root] {
- parArgs = append(parArgs, `-f `+path.String())
- paths = append(paths, path)
- }
- }
- origSrcsZip := android.PathForModuleOut(ctx, ctx.ModuleName()+".py.srcszip")
- ctx.Build(pctx, android.BuildParams{
- Rule: zip,
- Description: "python library archive",
- Output: origSrcsZip,
- // as zip rule does not use $in, there is no real need to distinguish between Inputs and Implicits
- Implicits: paths,
- Args: map[string]string{
- "args": strings.Join(parArgs, " "),
- },
- })
- zips = append(zips, origSrcsZip)
- }
- // we may have multiple zips due to separate handling of proto source files
- if len(zips) == 1 {
- return zips[0]
- } else {
- combinedSrcsZip := android.PathForModuleOut(ctx, ctx.ModuleName()+".srcszip")
- ctx.Build(pctx, android.BuildParams{
- Rule: combineZip,
- Description: "combine python library archive",
- Output: combinedSrcsZip,
- Inputs: zips,
- })
- return combinedSrcsZip
- }
- }
- // isPythonLibModule returns whether the given module is a Python library Module or not
- func isPythonLibModule(module blueprint.Module) bool {
- if m, ok := module.(*Module); ok {
- return m.isLibrary()
- }
- return false
- }
- // This is distinguished by the fact that Python libraries are not installable, while other Python
- // modules are.
- func (p *Module) isLibrary() bool {
- // Python library has no bootstrapper or installer
- return p.bootstrapper == nil && p.installer == nil
- }
- func (p *Module) isBinary() bool {
- _, ok := p.bootstrapper.(*binaryDecorator)
- return ok
- }
- // collectPathsFromTransitiveDeps checks for source/data files for duplicate paths
- // for module and its transitive dependencies and collects list of data/source file
- // zips for transitive dependencies.
- func (p *Module) collectPathsFromTransitiveDeps(ctx android.ModuleContext) {
- // fetch <runfiles_path, source_path> pairs from "src" and "data" properties to
- // check duplicates.
- destToPySrcs := make(map[string]string)
- destToPyData := make(map[string]string)
- for _, path := range p.srcsPathMappings {
- destToPySrcs[path.dest] = path.src.String()
- }
- for _, path := range p.dataPathMappings {
- destToPyData[path.dest] = path.src.String()
- }
- seen := make(map[android.Module]bool)
- // visit all its dependencies in depth first.
- ctx.WalkDeps(func(child, parent android.Module) bool {
- // we only collect dependencies tagged as python library deps
- if ctx.OtherModuleDependencyTag(child) != pythonLibTag {
- return false
- }
- if seen[child] {
- return false
- }
- seen[child] = true
- // Python modules only can depend on Python libraries.
- if !isPythonLibModule(child) {
- ctx.PropertyErrorf("libs",
- "the dependency %q of module %q is not Python library!",
- ctx.OtherModuleName(child), ctx.ModuleName())
- }
- // collect source and data paths, checking that there are no duplicate output file conflicts
- if dep, ok := child.(pythonDependency); ok {
- srcs := dep.getSrcsPathMappings()
- for _, path := range srcs {
- checkForDuplicateOutputPath(ctx, destToPySrcs,
- path.dest, path.src.String(), ctx.ModuleName(), ctx.OtherModuleName(child))
- }
- data := dep.getDataPathMappings()
- for _, path := range data {
- checkForDuplicateOutputPath(ctx, destToPyData,
- path.dest, path.src.String(), ctx.ModuleName(), ctx.OtherModuleName(child))
- }
- p.depsSrcsZips = append(p.depsSrcsZips, dep.getSrcsZip())
- }
- return true
- })
- }
- // chckForDuplicateOutputPath checks whether outputPath has already been included in map m, which
- // would result in two files being placed in the same location.
- // If there is a duplicate path, an error is thrown and true is returned
- // Otherwise, outputPath: srcPath is added to m and returns false
- func checkForDuplicateOutputPath(ctx android.ModuleContext, m map[string]string, outputPath, srcPath, curModule, otherModule string) bool {
- if oldSrcPath, found := m[outputPath]; found {
- ctx.ModuleErrorf("found two files to be placed at the same location within zip %q."+
- " First file: in module %s at path %q."+
- " Second file: in module %s at path %q.",
- outputPath, curModule, oldSrcPath, otherModule, srcPath)
- return true
- }
- m[outputPath] = srcPath
- return false
- }
- // InstallInData returns true as Python is not supported in the system partition
- func (p *Module) InstallInData() bool {
- return true
- }
- func (p *Module) ConvertWithBp2build(ctx android.TopDownMutatorContext) {
- if p.isLibrary() {
- pythonLibBp2Build(ctx, p)
- } else if p.isBinary() {
- pythonBinaryBp2Build(ctx, p)
- }
- }
- var Bool = proptools.Bool
- var BoolDefault = proptools.BoolDefault
- var String = proptools.String
|