python.go 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738
  1. // Copyright 2017 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package python
  15. // This file contains the "Base" module type for building Python program.
  16. import (
  17. "fmt"
  18. "path/filepath"
  19. "regexp"
  20. "strings"
  21. "github.com/google/blueprint"
  22. "github.com/google/blueprint/proptools"
  23. "android/soong/android"
  24. )
  25. func init() {
  26. registerPythonMutators(android.InitRegistrationContext)
  27. }
  28. func registerPythonMutators(ctx android.RegistrationContext) {
  29. ctx.PreDepsMutators(RegisterPythonPreDepsMutators)
  30. }
  31. // Exported to support other packages using Python modules in tests.
  32. func RegisterPythonPreDepsMutators(ctx android.RegisterMutatorsContext) {
  33. ctx.BottomUp("python_version", versionSplitMutator()).Parallel()
  34. }
  35. // the version-specific properties that apply to python modules.
  36. type VersionProperties struct {
  37. // whether the module is required to be built with this version.
  38. // Defaults to true for Python 3, and false otherwise.
  39. Enabled *bool
  40. // list of source files specific to this Python version.
  41. // Using the syntax ":module", srcs may reference the outputs of other modules that produce source files,
  42. // e.g. genrule or filegroup.
  43. Srcs []string `android:"path,arch_variant"`
  44. // list of source files that should not be used to build the Python module for this version.
  45. // This is most useful to remove files that are not common to all Python versions.
  46. Exclude_srcs []string `android:"path,arch_variant"`
  47. // list of the Python libraries used only for this Python version.
  48. Libs []string `android:"arch_variant"`
  49. // whether the binary is required to be built with embedded launcher for this version, defaults to false.
  50. Embedded_launcher *bool // TODO(b/174041232): Remove this property
  51. }
  52. // properties that apply to all python modules
  53. type BaseProperties struct {
  54. // the package path prefix within the output artifact at which to place the source/data
  55. // files of the current module.
  56. // eg. Pkg_path = "a/b/c"; Other packages can reference this module by using
  57. // (from a.b.c import ...) statement.
  58. // if left unspecified, all the source/data files path is unchanged within zip file.
  59. Pkg_path *string
  60. // true, if the Python module is used internally, eg, Python std libs.
  61. Is_internal *bool
  62. // list of source (.py) files compatible both with Python2 and Python3 used to compile the
  63. // Python module.
  64. // srcs may reference the outputs of other modules that produce source files like genrule
  65. // or filegroup using the syntax ":module".
  66. // Srcs has to be non-empty.
  67. Srcs []string `android:"path,arch_variant"`
  68. // list of source files that should not be used to build the C/C++ module.
  69. // This is most useful in the arch/multilib variants to remove non-common files
  70. Exclude_srcs []string `android:"path,arch_variant"`
  71. // list of files or filegroup modules that provide data that should be installed alongside
  72. // the test. the file extension can be arbitrary except for (.py).
  73. Data []string `android:"path,arch_variant"`
  74. // list of java modules that provide data that should be installed alongside the test.
  75. Java_data []string
  76. // list of the Python libraries compatible both with Python2 and Python3.
  77. Libs []string `android:"arch_variant"`
  78. Version struct {
  79. // Python2-specific properties, including whether Python2 is supported for this module
  80. // and version-specific sources, exclusions and dependencies.
  81. Py2 VersionProperties `android:"arch_variant"`
  82. // Python3-specific properties, including whether Python3 is supported for this module
  83. // and version-specific sources, exclusions and dependencies.
  84. Py3 VersionProperties `android:"arch_variant"`
  85. } `android:"arch_variant"`
  86. // the actual version each module uses after variations created.
  87. // this property name is hidden from users' perspectives, and soong will populate it during
  88. // runtime.
  89. Actual_version string `blueprint:"mutated"`
  90. // whether the module is required to be built with actual_version.
  91. // this is set by the python version mutator based on version-specific properties
  92. Enabled *bool `blueprint:"mutated"`
  93. // whether the binary is required to be built with embedded launcher for this actual_version.
  94. // this is set by the python version mutator based on version-specific properties
  95. Embedded_launcher *bool `blueprint:"mutated"`
  96. }
  97. // Used to store files of current module after expanding dependencies
  98. type pathMapping struct {
  99. dest string
  100. src android.Path
  101. }
  102. type PythonLibraryModule struct {
  103. android.ModuleBase
  104. android.DefaultableModuleBase
  105. android.BazelModuleBase
  106. properties BaseProperties
  107. protoProperties android.ProtoProperties
  108. // initialize before calling Init
  109. hod android.HostOrDeviceSupported
  110. multilib android.Multilib
  111. // the Python files of current module after expanding source dependencies.
  112. // pathMapping: <dest: runfile_path, src: source_path>
  113. srcsPathMappings []pathMapping
  114. // the data files of current module after expanding source dependencies.
  115. // pathMapping: <dest: runfile_path, src: source_path>
  116. dataPathMappings []pathMapping
  117. // The zip file containing the current module's source/data files.
  118. srcsZip android.Path
  119. // The zip file containing the current module's source/data files, with the
  120. // source files precompiled.
  121. precompiledSrcsZip android.Path
  122. }
  123. // newModule generates new Python base module
  124. func newModule(hod android.HostOrDeviceSupported, multilib android.Multilib) *PythonLibraryModule {
  125. return &PythonLibraryModule{
  126. hod: hod,
  127. multilib: multilib,
  128. }
  129. }
  130. // interface implemented by Python modules to provide source and data mappings and zip to python
  131. // modules that depend on it
  132. type pythonDependency interface {
  133. getSrcsPathMappings() []pathMapping
  134. getDataPathMappings() []pathMapping
  135. getSrcsZip() android.Path
  136. getPrecompiledSrcsZip() android.Path
  137. }
  138. // getSrcsPathMappings gets this module's path mapping of src source path : runfiles destination
  139. func (p *PythonLibraryModule) getSrcsPathMappings() []pathMapping {
  140. return p.srcsPathMappings
  141. }
  142. // getSrcsPathMappings gets this module's path mapping of data source path : runfiles destination
  143. func (p *PythonLibraryModule) getDataPathMappings() []pathMapping {
  144. return p.dataPathMappings
  145. }
  146. // getSrcsZip returns the filepath where the current module's source/data files are zipped.
  147. func (p *PythonLibraryModule) getSrcsZip() android.Path {
  148. return p.srcsZip
  149. }
  150. // getSrcsZip returns the filepath where the current module's source/data files are zipped.
  151. func (p *PythonLibraryModule) getPrecompiledSrcsZip() android.Path {
  152. return p.precompiledSrcsZip
  153. }
  154. func (p *PythonLibraryModule) getBaseProperties() *BaseProperties {
  155. return &p.properties
  156. }
  157. var _ pythonDependency = (*PythonLibraryModule)(nil)
  158. func (p *PythonLibraryModule) init() android.Module {
  159. p.AddProperties(&p.properties, &p.protoProperties)
  160. android.InitAndroidArchModule(p, p.hod, p.multilib)
  161. android.InitDefaultableModule(p)
  162. android.InitBazelModule(p)
  163. return p
  164. }
  165. // Python-specific tag to transfer information on the purpose of a dependency.
  166. // This is used when adding a dependency on a module, which can later be accessed when visiting
  167. // dependencies.
  168. type dependencyTag struct {
  169. blueprint.BaseDependencyTag
  170. name string
  171. }
  172. // Python-specific tag that indicates that installed files of this module should depend on installed
  173. // files of the dependency
  174. type installDependencyTag struct {
  175. blueprint.BaseDependencyTag
  176. // embedding this struct provides the installation dependency requirement
  177. android.InstallAlwaysNeededDependencyTag
  178. name string
  179. }
  180. var (
  181. pythonLibTag = dependencyTag{name: "pythonLib"}
  182. javaDataTag = dependencyTag{name: "javaData"}
  183. // The python interpreter, with soong module name "py3-launcher" or "py3-launcher-autorun".
  184. launcherTag = dependencyTag{name: "launcher"}
  185. launcherSharedLibTag = installDependencyTag{name: "launcherSharedLib"}
  186. // The python interpreter built for host so that we can precompile python sources.
  187. // This only works because the precompiled sources don't vary by architecture.
  188. // The soong module name is "py3-launcher".
  189. hostLauncherTag = dependencyTag{name: "hostLauncher"}
  190. hostlauncherSharedLibTag = dependencyTag{name: "hostlauncherSharedLib"}
  191. hostStdLibTag = dependencyTag{name: "hostStdLib"}
  192. pathComponentRegexp = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_-]*$`)
  193. pyExt = ".py"
  194. protoExt = ".proto"
  195. pyVersion2 = "PY2"
  196. pyVersion3 = "PY3"
  197. pyVersion2And3 = "PY2ANDPY3"
  198. internalPath = "internal"
  199. )
  200. type basePropertiesProvider interface {
  201. getBaseProperties() *BaseProperties
  202. }
  203. // versionSplitMutator creates version variants for modules and appends the version-specific
  204. // properties for a given variant to the properties in the variant module
  205. func versionSplitMutator() func(android.BottomUpMutatorContext) {
  206. return func(mctx android.BottomUpMutatorContext) {
  207. if base, ok := mctx.Module().(basePropertiesProvider); ok {
  208. props := base.getBaseProperties()
  209. var versionNames []string
  210. // collect version specific properties, so that we can merge version-specific properties
  211. // into the module's overall properties
  212. var versionProps []VersionProperties
  213. // PY3 is first so that we alias the PY3 variant rather than PY2 if both
  214. // are available
  215. if proptools.BoolDefault(props.Version.Py3.Enabled, true) {
  216. versionNames = append(versionNames, pyVersion3)
  217. versionProps = append(versionProps, props.Version.Py3)
  218. }
  219. if proptools.BoolDefault(props.Version.Py2.Enabled, false) {
  220. if !mctx.DeviceConfig().BuildBrokenUsesSoongPython2Modules() &&
  221. mctx.ModuleName() != "py2-cmd" &&
  222. mctx.ModuleName() != "py2-stdlib" {
  223. mctx.PropertyErrorf("version.py2.enabled", "Python 2 is no longer supported, please convert to python 3. This error can be temporarily overridden by setting BUILD_BROKEN_USES_SOONG_PYTHON2_MODULES := true in the product configuration")
  224. }
  225. versionNames = append(versionNames, pyVersion2)
  226. versionProps = append(versionProps, props.Version.Py2)
  227. }
  228. modules := mctx.CreateLocalVariations(versionNames...)
  229. // Alias module to the first variant
  230. if len(versionNames) > 0 {
  231. mctx.AliasVariation(versionNames[0])
  232. }
  233. for i, v := range versionNames {
  234. // set the actual version for Python module.
  235. newProps := modules[i].(basePropertiesProvider).getBaseProperties()
  236. newProps.Actual_version = v
  237. // append versioned properties for the Python module to the overall properties
  238. err := proptools.AppendMatchingProperties([]interface{}{newProps}, &versionProps[i], nil)
  239. if err != nil {
  240. panic(err)
  241. }
  242. }
  243. }
  244. }
  245. }
  246. func anyHasExt(paths []string, ext string) bool {
  247. for _, p := range paths {
  248. if filepath.Ext(p) == ext {
  249. return true
  250. }
  251. }
  252. return false
  253. }
  254. func (p *PythonLibraryModule) anySrcHasExt(ctx android.BottomUpMutatorContext, ext string) bool {
  255. return anyHasExt(p.properties.Srcs, ext)
  256. }
  257. // DepsMutator mutates dependencies for this module:
  258. // - handles proto dependencies,
  259. // - if required, specifies launcher and adds launcher dependencies,
  260. // - applies python version mutations to Python dependencies
  261. func (p *PythonLibraryModule) DepsMutator(ctx android.BottomUpMutatorContext) {
  262. android.ProtoDeps(ctx, &p.protoProperties)
  263. versionVariation := []blueprint.Variation{
  264. {"python_version", p.properties.Actual_version},
  265. }
  266. // If sources contain a proto file, add dependency on libprotobuf-python
  267. if p.anySrcHasExt(ctx, protoExt) && p.Name() != "libprotobuf-python" {
  268. ctx.AddVariationDependencies(versionVariation, pythonLibTag, "libprotobuf-python")
  269. }
  270. // Add python library dependencies for this python version variation
  271. ctx.AddVariationDependencies(versionVariation, pythonLibTag, android.LastUniqueStrings(p.properties.Libs)...)
  272. // Emulate the data property for java_data but with the arch variation overridden to "common"
  273. // so that it can point to java modules.
  274. javaDataVariation := []blueprint.Variation{{"arch", android.Common.String()}}
  275. ctx.AddVariationDependencies(javaDataVariation, javaDataTag, p.properties.Java_data...)
  276. p.AddDepsOnPythonLauncherAndStdlib(ctx, hostStdLibTag, hostLauncherTag, hostlauncherSharedLibTag, false, ctx.Config().BuildOSTarget)
  277. }
  278. // AddDepsOnPythonLauncherAndStdlib will make the current module depend on the python stdlib,
  279. // launcher (interpreter), and the launcher's shared libraries. If autorun is true, it will use
  280. // the autorun launcher instead of the regular one. This function acceps a targetForDeps argument
  281. // as the target to use for these dependencies. For embedded launcher python binaries, the launcher
  282. // that will be embedded will be under the same target as the python module itself. But when
  283. // precompiling python code, we need to get the python launcher built for host, even if we're
  284. // compiling the python module for device, so we pass a different target to this function.
  285. func (p *PythonLibraryModule) AddDepsOnPythonLauncherAndStdlib(ctx android.BottomUpMutatorContext,
  286. stdLibTag, launcherTag, launcherSharedLibTag blueprint.DependencyTag,
  287. autorun bool, targetForDeps android.Target) {
  288. var stdLib string
  289. var launcherModule string
  290. // Add launcher shared lib dependencies. Ideally, these should be
  291. // derived from the `shared_libs` property of the launcher. TODO: read these from
  292. // the python launcher itself using ctx.OtherModuleProvider() or similar on the result
  293. // of ctx.AddFarVariationDependencies()
  294. launcherSharedLibDeps := []string{
  295. "libsqlite",
  296. }
  297. // Add launcher-specific dependencies for bionic
  298. if targetForDeps.Os.Bionic() {
  299. launcherSharedLibDeps = append(launcherSharedLibDeps, "libc", "libdl", "libm")
  300. }
  301. if targetForDeps.Os == android.LinuxMusl && !ctx.Config().HostStaticBinaries() {
  302. launcherSharedLibDeps = append(launcherSharedLibDeps, "libc_musl")
  303. }
  304. switch p.properties.Actual_version {
  305. case pyVersion2:
  306. stdLib = "py2-stdlib"
  307. launcherModule = "py2-launcher"
  308. if autorun {
  309. launcherModule = "py2-launcher-autorun"
  310. }
  311. launcherSharedLibDeps = append(launcherSharedLibDeps, "libc++")
  312. case pyVersion3:
  313. stdLib = "py3-stdlib"
  314. launcherModule = "py3-launcher"
  315. if autorun {
  316. launcherModule = "py3-launcher-autorun"
  317. }
  318. if ctx.Config().HostStaticBinaries() && targetForDeps.Os == android.LinuxMusl {
  319. launcherModule += "-static"
  320. }
  321. if ctx.Device() {
  322. launcherSharedLibDeps = append(launcherSharedLibDeps, "liblog")
  323. }
  324. default:
  325. panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.",
  326. p.properties.Actual_version, ctx.ModuleName()))
  327. }
  328. targetVariations := targetForDeps.Variations()
  329. if ctx.ModuleName() != stdLib {
  330. stdLibVariations := make([]blueprint.Variation, 0, len(targetVariations)+1)
  331. stdLibVariations = append(stdLibVariations, blueprint.Variation{Mutator: "python_version", Variation: p.properties.Actual_version})
  332. stdLibVariations = append(stdLibVariations, targetVariations...)
  333. // Using AddFarVariationDependencies for all of these because they can be for a different
  334. // platform, like if the python module itself was being compiled for device, we may want
  335. // the python interpreter built for host so that we can precompile python sources.
  336. ctx.AddFarVariationDependencies(stdLibVariations, stdLibTag, stdLib)
  337. }
  338. ctx.AddFarVariationDependencies(targetVariations, launcherTag, launcherModule)
  339. ctx.AddFarVariationDependencies(targetVariations, launcherSharedLibTag, launcherSharedLibDeps...)
  340. }
  341. // GenerateAndroidBuildActions performs build actions common to all Python modules
  342. func (p *PythonLibraryModule) GenerateAndroidBuildActions(ctx android.ModuleContext) {
  343. expandedSrcs := android.PathsForModuleSrcExcludes(ctx, p.properties.Srcs, p.properties.Exclude_srcs)
  344. // expand data files from "data" property.
  345. expandedData := android.PathsForModuleSrc(ctx, p.properties.Data)
  346. // Emulate the data property for java_data dependencies.
  347. for _, javaData := range ctx.GetDirectDepsWithTag(javaDataTag) {
  348. expandedData = append(expandedData, android.OutputFilesForModule(ctx, javaData, "")...)
  349. }
  350. // Validate pkg_path property
  351. pkgPath := String(p.properties.Pkg_path)
  352. if pkgPath != "" {
  353. // TODO: export validation from android/paths.go handling to replace this duplicated functionality
  354. pkgPath = filepath.Clean(String(p.properties.Pkg_path))
  355. if pkgPath == ".." || strings.HasPrefix(pkgPath, "../") ||
  356. strings.HasPrefix(pkgPath, "/") {
  357. ctx.PropertyErrorf("pkg_path",
  358. "%q must be a relative path contained in par file.",
  359. String(p.properties.Pkg_path))
  360. return
  361. }
  362. }
  363. // If property Is_internal is set, prepend pkgPath with internalPath
  364. if proptools.BoolDefault(p.properties.Is_internal, false) {
  365. pkgPath = filepath.Join(internalPath, pkgPath)
  366. }
  367. // generate src:destination path mappings for this module
  368. p.genModulePathMappings(ctx, pkgPath, expandedSrcs, expandedData)
  369. // generate the zipfile of all source and data files
  370. p.srcsZip = p.createSrcsZip(ctx, pkgPath)
  371. p.precompiledSrcsZip = p.precompileSrcs(ctx)
  372. }
  373. func isValidPythonPath(path string) error {
  374. identifiers := strings.Split(strings.TrimSuffix(path, filepath.Ext(path)), "/")
  375. for _, token := range identifiers {
  376. if !pathComponentRegexp.MatchString(token) {
  377. return fmt.Errorf("the path %q contains invalid subpath %q. "+
  378. "Subpaths must be at least one character long. "+
  379. "The first character must an underscore or letter. "+
  380. "Following characters may be any of: letter, digit, underscore, hyphen.",
  381. path, token)
  382. }
  383. }
  384. return nil
  385. }
  386. // For this module, generate unique pathMappings: <dest: runfiles_path, src: source_path>
  387. // for python/data files expanded from properties.
  388. func (p *PythonLibraryModule) genModulePathMappings(ctx android.ModuleContext, pkgPath string,
  389. expandedSrcs, expandedData android.Paths) {
  390. // fetch <runfiles_path, source_path> pairs from "src" and "data" properties to
  391. // check current module duplicates.
  392. destToPySrcs := make(map[string]string)
  393. destToPyData := make(map[string]string)
  394. for _, s := range expandedSrcs {
  395. if s.Ext() != pyExt && s.Ext() != protoExt {
  396. ctx.PropertyErrorf("srcs", "found non (.py|.proto) file: %q!", s.String())
  397. continue
  398. }
  399. runfilesPath := filepath.Join(pkgPath, s.Rel())
  400. if err := isValidPythonPath(runfilesPath); err != nil {
  401. ctx.PropertyErrorf("srcs", err.Error())
  402. }
  403. if !checkForDuplicateOutputPath(ctx, destToPySrcs, runfilesPath, s.String(), p.Name(), p.Name()) {
  404. p.srcsPathMappings = append(p.srcsPathMappings, pathMapping{dest: runfilesPath, src: s})
  405. }
  406. }
  407. for _, d := range expandedData {
  408. if d.Ext() == pyExt || d.Ext() == protoExt {
  409. ctx.PropertyErrorf("data", "found (.py|.proto) file: %q!", d.String())
  410. continue
  411. }
  412. runfilesPath := filepath.Join(pkgPath, d.Rel())
  413. if !checkForDuplicateOutputPath(ctx, destToPyData, runfilesPath, d.String(), p.Name(), p.Name()) {
  414. p.dataPathMappings = append(p.dataPathMappings,
  415. pathMapping{dest: runfilesPath, src: d})
  416. }
  417. }
  418. }
  419. // createSrcsZip registers build actions to zip current module's sources and data.
  420. func (p *PythonLibraryModule) createSrcsZip(ctx android.ModuleContext, pkgPath string) android.Path {
  421. relativeRootMap := make(map[string]android.Paths)
  422. var protoSrcs android.Paths
  423. addPathMapping := func(path pathMapping) {
  424. // handle proto sources separately
  425. if path.src.Ext() == protoExt {
  426. protoSrcs = append(protoSrcs, path.src)
  427. } else {
  428. relativeRoot := strings.TrimSuffix(path.src.String(), path.src.Rel())
  429. relativeRootMap[relativeRoot] = append(relativeRootMap[relativeRoot], path.src)
  430. }
  431. }
  432. // "srcs" or "data" properties may contain filegroups so it might happen that
  433. // the root directory for each source path is different.
  434. for _, path := range p.srcsPathMappings {
  435. addPathMapping(path)
  436. }
  437. for _, path := range p.dataPathMappings {
  438. addPathMapping(path)
  439. }
  440. var zips android.Paths
  441. if len(protoSrcs) > 0 {
  442. protoFlags := android.GetProtoFlags(ctx, &p.protoProperties)
  443. protoFlags.OutTypeFlag = "--python_out"
  444. if pkgPath != "" {
  445. pkgPathStagingDir := android.PathForModuleGen(ctx, "protos_staged_for_pkg_path")
  446. rule := android.NewRuleBuilder(pctx, ctx)
  447. var stagedProtoSrcs android.Paths
  448. for _, srcFile := range protoSrcs {
  449. stagedProtoSrc := pkgPathStagingDir.Join(ctx, pkgPath, srcFile.Rel())
  450. rule.Command().Text("mkdir -p").Flag(filepath.Base(stagedProtoSrc.String()))
  451. rule.Command().Text("cp -f").Input(srcFile).Output(stagedProtoSrc)
  452. stagedProtoSrcs = append(stagedProtoSrcs, stagedProtoSrc)
  453. }
  454. rule.Build("stage_protos_for_pkg_path", "Stage protos for pkg_path")
  455. protoSrcs = stagedProtoSrcs
  456. }
  457. for _, srcFile := range protoSrcs {
  458. zip := genProto(ctx, srcFile, protoFlags)
  459. zips = append(zips, zip)
  460. }
  461. }
  462. if len(relativeRootMap) > 0 {
  463. // in order to keep stable order of soong_zip params, we sort the keys here.
  464. roots := android.SortedKeys(relativeRootMap)
  465. // Use -symlinks=false so that the symlinks in the bazel output directory are followed
  466. parArgs := []string{"-symlinks=false"}
  467. if pkgPath != "" {
  468. // use package path as path prefix
  469. parArgs = append(parArgs, `-P `+pkgPath)
  470. }
  471. paths := android.Paths{}
  472. for _, root := range roots {
  473. // specify relative root of file in following -f arguments
  474. parArgs = append(parArgs, `-C `+root)
  475. for _, path := range relativeRootMap[root] {
  476. parArgs = append(parArgs, `-f `+path.String())
  477. paths = append(paths, path)
  478. }
  479. }
  480. origSrcsZip := android.PathForModuleOut(ctx, ctx.ModuleName()+".py.srcszip")
  481. ctx.Build(pctx, android.BuildParams{
  482. Rule: zip,
  483. Description: "python library archive",
  484. Output: origSrcsZip,
  485. // as zip rule does not use $in, there is no real need to distinguish between Inputs and Implicits
  486. Implicits: paths,
  487. Args: map[string]string{
  488. "args": strings.Join(parArgs, " "),
  489. },
  490. })
  491. zips = append(zips, origSrcsZip)
  492. }
  493. // we may have multiple zips due to separate handling of proto source files
  494. if len(zips) == 1 {
  495. return zips[0]
  496. } else {
  497. combinedSrcsZip := android.PathForModuleOut(ctx, ctx.ModuleName()+".srcszip")
  498. ctx.Build(pctx, android.BuildParams{
  499. Rule: combineZip,
  500. Description: "combine python library archive",
  501. Output: combinedSrcsZip,
  502. Inputs: zips,
  503. })
  504. return combinedSrcsZip
  505. }
  506. }
  507. func (p *PythonLibraryModule) precompileSrcs(ctx android.ModuleContext) android.Path {
  508. // To precompile the python sources, we need a python interpreter and stdlib built
  509. // for host. We then use those to compile the python sources, which may be used on either
  510. // host of device. Python bytecode is architecture agnostic, so we're essentially
  511. // "cross compiling" for device here purely by virtue of host and device python bytecode
  512. // being the same.
  513. var stdLib android.Path
  514. var launcher android.Path
  515. if ctx.ModuleName() == "py3-stdlib" || ctx.ModuleName() == "py2-stdlib" {
  516. stdLib = p.srcsZip
  517. } else {
  518. ctx.VisitDirectDepsWithTag(hostStdLibTag, func(module android.Module) {
  519. if dep, ok := module.(pythonDependency); ok {
  520. stdLib = dep.getPrecompiledSrcsZip()
  521. }
  522. })
  523. }
  524. ctx.VisitDirectDepsWithTag(hostLauncherTag, func(module android.Module) {
  525. if dep, ok := module.(IntermPathProvider); ok {
  526. optionalLauncher := dep.IntermPathForModuleOut()
  527. if optionalLauncher.Valid() {
  528. launcher = optionalLauncher.Path()
  529. }
  530. }
  531. })
  532. var launcherSharedLibs android.Paths
  533. var ldLibraryPath []string
  534. ctx.VisitDirectDepsWithTag(hostlauncherSharedLibTag, func(module android.Module) {
  535. if dep, ok := module.(IntermPathProvider); ok {
  536. optionalPath := dep.IntermPathForModuleOut()
  537. if optionalPath.Valid() {
  538. launcherSharedLibs = append(launcherSharedLibs, optionalPath.Path())
  539. ldLibraryPath = append(ldLibraryPath, filepath.Dir(optionalPath.Path().String()))
  540. }
  541. }
  542. })
  543. out := android.PathForModuleOut(ctx, ctx.ModuleName()+".srcszipprecompiled")
  544. if stdLib == nil || launcher == nil {
  545. // This shouldn't happen in a real build because we'll error out when adding dependencies
  546. // on the stdlib and launcher if they don't exist. But some tests set
  547. // AllowMissingDependencies.
  548. return out
  549. }
  550. ctx.Build(pctx, android.BuildParams{
  551. Rule: precompile,
  552. Input: p.srcsZip,
  553. Output: out,
  554. Implicits: launcherSharedLibs,
  555. Description: "Precompile the python sources of " + ctx.ModuleName(),
  556. Args: map[string]string{
  557. "stdlibZip": stdLib.String(),
  558. "launcher": launcher.String(),
  559. "ldLibraryPath": strings.Join(ldLibraryPath, ":"),
  560. },
  561. })
  562. return out
  563. }
  564. // isPythonLibModule returns whether the given module is a Python library PythonLibraryModule or not
  565. func isPythonLibModule(module blueprint.Module) bool {
  566. if _, ok := module.(*PythonLibraryModule); ok {
  567. if _, ok := module.(*PythonBinaryModule); !ok {
  568. return true
  569. }
  570. }
  571. return false
  572. }
  573. // collectPathsFromTransitiveDeps checks for source/data files for duplicate paths
  574. // for module and its transitive dependencies and collects list of data/source file
  575. // zips for transitive dependencies.
  576. func (p *PythonLibraryModule) collectPathsFromTransitiveDeps(ctx android.ModuleContext, precompiled bool) android.Paths {
  577. // fetch <runfiles_path, source_path> pairs from "src" and "data" properties to
  578. // check duplicates.
  579. destToPySrcs := make(map[string]string)
  580. destToPyData := make(map[string]string)
  581. for _, path := range p.srcsPathMappings {
  582. destToPySrcs[path.dest] = path.src.String()
  583. }
  584. for _, path := range p.dataPathMappings {
  585. destToPyData[path.dest] = path.src.String()
  586. }
  587. seen := make(map[android.Module]bool)
  588. var result android.Paths
  589. // visit all its dependencies in depth first.
  590. ctx.WalkDeps(func(child, parent android.Module) bool {
  591. // we only collect dependencies tagged as python library deps
  592. if ctx.OtherModuleDependencyTag(child) != pythonLibTag {
  593. return false
  594. }
  595. if seen[child] {
  596. return false
  597. }
  598. seen[child] = true
  599. // Python modules only can depend on Python libraries.
  600. if !isPythonLibModule(child) {
  601. ctx.PropertyErrorf("libs",
  602. "the dependency %q of module %q is not Python library!",
  603. ctx.OtherModuleName(child), ctx.ModuleName())
  604. }
  605. // collect source and data paths, checking that there are no duplicate output file conflicts
  606. if dep, ok := child.(pythonDependency); ok {
  607. srcs := dep.getSrcsPathMappings()
  608. for _, path := range srcs {
  609. checkForDuplicateOutputPath(ctx, destToPySrcs,
  610. path.dest, path.src.String(), ctx.ModuleName(), ctx.OtherModuleName(child))
  611. }
  612. data := dep.getDataPathMappings()
  613. for _, path := range data {
  614. checkForDuplicateOutputPath(ctx, destToPyData,
  615. path.dest, path.src.String(), ctx.ModuleName(), ctx.OtherModuleName(child))
  616. }
  617. if precompiled {
  618. result = append(result, dep.getPrecompiledSrcsZip())
  619. } else {
  620. result = append(result, dep.getSrcsZip())
  621. }
  622. }
  623. return true
  624. })
  625. return result
  626. }
  627. // chckForDuplicateOutputPath checks whether outputPath has already been included in map m, which
  628. // would result in two files being placed in the same location.
  629. // If there is a duplicate path, an error is thrown and true is returned
  630. // Otherwise, outputPath: srcPath is added to m and returns false
  631. func checkForDuplicateOutputPath(ctx android.ModuleContext, m map[string]string, outputPath, srcPath, curModule, otherModule string) bool {
  632. if oldSrcPath, found := m[outputPath]; found {
  633. ctx.ModuleErrorf("found two files to be placed at the same location within zip %q."+
  634. " First file: in module %s at path %q."+
  635. " Second file: in module %s at path %q.",
  636. outputPath, curModule, oldSrcPath, otherModule, srcPath)
  637. return true
  638. }
  639. m[outputPath] = srcPath
  640. return false
  641. }
  642. // InstallInData returns true as Python is not supported in the system partition
  643. func (p *PythonLibraryModule) InstallInData() bool {
  644. return true
  645. }
  646. var Bool = proptools.Bool
  647. var BoolDefault = proptools.BoolDefault
  648. var String = proptools.String