python.go 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723
  1. // Copyright 2017 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package python
  15. // This file contains the "Base" module type for building Python program.
  16. import (
  17. "fmt"
  18. "path/filepath"
  19. "regexp"
  20. "strings"
  21. "github.com/google/blueprint"
  22. "github.com/google/blueprint/proptools"
  23. "android/soong/android"
  24. )
  25. func init() {
  26. registerPythonMutators(android.InitRegistrationContext)
  27. }
  28. func registerPythonMutators(ctx android.RegistrationContext) {
  29. ctx.PreDepsMutators(RegisterPythonPreDepsMutators)
  30. }
  31. // Exported to support other packages using Python modules in tests.
  32. func RegisterPythonPreDepsMutators(ctx android.RegisterMutatorsContext) {
  33. ctx.BottomUp("python_version", versionSplitMutator()).Parallel()
  34. }
  35. // the version-specific properties that apply to python modules.
  36. type VersionProperties struct {
  37. // whether the module is required to be built with this version.
  38. // Defaults to true for Python 3, and false otherwise.
  39. Enabled *bool `android:"arch_variant"`
  40. // list of source files specific to this Python version.
  41. // Using the syntax ":module", srcs may reference the outputs of other modules that produce source files,
  42. // e.g. genrule or filegroup.
  43. Srcs []string `android:"path,arch_variant"`
  44. // list of source files that should not be used to build the Python module for this version.
  45. // This is most useful to remove files that are not common to all Python versions.
  46. Exclude_srcs []string `android:"path,arch_variant"`
  47. // list of the Python libraries used only for this Python version.
  48. Libs []string `android:"arch_variant"`
  49. // whether the binary is required to be built with embedded launcher for this version, defaults to false.
  50. Embedded_launcher *bool `android:"arch_variant"` // TODO(b/174041232): Remove this property
  51. }
  52. // properties that apply to all python modules
  53. type BaseProperties struct {
  54. // the package path prefix within the output artifact at which to place the source/data
  55. // files of the current module.
  56. // eg. Pkg_path = "a/b/c"; Other packages can reference this module by using
  57. // (from a.b.c import ...) statement.
  58. // if left unspecified, all the source/data files path is unchanged within zip file.
  59. Pkg_path *string `android:"arch_variant"`
  60. // true, if the Python module is used internally, eg, Python std libs.
  61. Is_internal *bool `android:"arch_variant"`
  62. // list of source (.py) files compatible both with Python2 and Python3 used to compile the
  63. // Python module.
  64. // srcs may reference the outputs of other modules that produce source files like genrule
  65. // or filegroup using the syntax ":module".
  66. // Srcs has to be non-empty.
  67. Srcs []string `android:"path,arch_variant"`
  68. // list of source files that should not be used to build the C/C++ module.
  69. // This is most useful in the arch/multilib variants to remove non-common files
  70. Exclude_srcs []string `android:"path,arch_variant"`
  71. // list of files or filegroup modules that provide data that should be installed alongside
  72. // the test. the file extension can be arbitrary except for (.py).
  73. Data []string `android:"path,arch_variant"`
  74. // list of java modules that provide data that should be installed alongside the test.
  75. Java_data []string
  76. // list of the Python libraries compatible both with Python2 and Python3.
  77. Libs []string `android:"arch_variant"`
  78. Version struct {
  79. // Python2-specific properties, including whether Python2 is supported for this module
  80. // and version-specific sources, exclusions and dependencies.
  81. Py2 VersionProperties `android:"arch_variant"`
  82. // Python3-specific properties, including whether Python3 is supported for this module
  83. // and version-specific sources, exclusions and dependencies.
  84. Py3 VersionProperties `android:"arch_variant"`
  85. } `android:"arch_variant"`
  86. // the actual version each module uses after variations created.
  87. // this property name is hidden from users' perspectives, and soong will populate it during
  88. // runtime.
  89. Actual_version string `blueprint:"mutated"`
  90. // whether the module is required to be built with actual_version.
  91. // this is set by the python version mutator based on version-specific properties
  92. Enabled *bool `blueprint:"mutated"`
  93. // whether the binary is required to be built with embedded launcher for this actual_version.
  94. // this is set by the python version mutator based on version-specific properties
  95. Embedded_launcher *bool `blueprint:"mutated"`
  96. }
  97. // Used to store files of current module after expanding dependencies
  98. type pathMapping struct {
  99. dest string
  100. src android.Path
  101. }
  102. type Module struct {
  103. android.ModuleBase
  104. android.DefaultableModuleBase
  105. android.BazelModuleBase
  106. properties BaseProperties
  107. protoProperties android.ProtoProperties
  108. // initialize before calling Init
  109. hod android.HostOrDeviceSupported
  110. multilib android.Multilib
  111. // interface used to bootstrap .par executable when embedded_launcher is true
  112. // this should be set by Python modules which are runnable, e.g. binaries and tests
  113. // bootstrapper might be nil (e.g. Python library module).
  114. bootstrapper bootstrapper
  115. // interface that implements functions required for installation
  116. // this should be set by Python modules which are runnable, e.g. binaries and tests
  117. // installer might be nil (e.g. Python library module).
  118. installer installer
  119. // the Python files of current module after expanding source dependencies.
  120. // pathMapping: <dest: runfile_path, src: source_path>
  121. srcsPathMappings []pathMapping
  122. // the data files of current module after expanding source dependencies.
  123. // pathMapping: <dest: runfile_path, src: source_path>
  124. dataPathMappings []pathMapping
  125. // the zip filepath for zipping current module source/data files.
  126. srcsZip android.Path
  127. // dependency modules' zip filepath for zipping current module source/data files.
  128. depsSrcsZips android.Paths
  129. // (.intermediate) module output path as installation source.
  130. installSource android.OptionalPath
  131. // Map to ensure sub-part of the AndroidMk for this module is only added once
  132. subAndroidMkOnce map[subAndroidMkProvider]bool
  133. }
  134. // newModule generates new Python base module
  135. func newModule(hod android.HostOrDeviceSupported, multilib android.Multilib) *Module {
  136. return &Module{
  137. hod: hod,
  138. multilib: multilib,
  139. }
  140. }
  141. // bootstrapper interface should be implemented for runnable modules, e.g. binary and test
  142. type bootstrapper interface {
  143. bootstrapperProps() []interface{}
  144. bootstrap(ctx android.ModuleContext, ActualVersion string, embeddedLauncher bool,
  145. srcsPathMappings []pathMapping, srcsZip android.Path,
  146. depsSrcsZips android.Paths) android.OptionalPath
  147. autorun() bool
  148. }
  149. // installer interface should be implemented for installable modules, e.g. binary and test
  150. type installer interface {
  151. install(ctx android.ModuleContext, path android.Path)
  152. setAndroidMkSharedLibs(sharedLibs []string)
  153. }
  154. // interface implemented by Python modules to provide source and data mappings and zip to python
  155. // modules that depend on it
  156. type pythonDependency interface {
  157. getSrcsPathMappings() []pathMapping
  158. getDataPathMappings() []pathMapping
  159. getSrcsZip() android.Path
  160. }
  161. // getSrcsPathMappings gets this module's path mapping of src source path : runfiles destination
  162. func (p *Module) getSrcsPathMappings() []pathMapping {
  163. return p.srcsPathMappings
  164. }
  165. // getSrcsPathMappings gets this module's path mapping of data source path : runfiles destination
  166. func (p *Module) getDataPathMappings() []pathMapping {
  167. return p.dataPathMappings
  168. }
  169. // getSrcsZip returns the filepath where the current module's source/data files are zipped.
  170. func (p *Module) getSrcsZip() android.Path {
  171. return p.srcsZip
  172. }
  173. var _ pythonDependency = (*Module)(nil)
  174. var _ android.AndroidMkEntriesProvider = (*Module)(nil)
  175. func (p *Module) init(additionalProps ...interface{}) android.Module {
  176. p.AddProperties(&p.properties, &p.protoProperties)
  177. // Add additional properties for bootstrapping/installation
  178. // This is currently tied to the bootstrapper interface;
  179. // however, these are a combination of properties for the installation and bootstrapping of a module
  180. if p.bootstrapper != nil {
  181. p.AddProperties(p.bootstrapper.bootstrapperProps()...)
  182. }
  183. android.InitAndroidArchModule(p, p.hod, p.multilib)
  184. android.InitDefaultableModule(p)
  185. return p
  186. }
  187. // Python-specific tag to transfer information on the purpose of a dependency.
  188. // This is used when adding a dependency on a module, which can later be accessed when visiting
  189. // dependencies.
  190. type dependencyTag struct {
  191. blueprint.BaseDependencyTag
  192. name string
  193. }
  194. // Python-specific tag that indicates that installed files of this module should depend on installed
  195. // files of the dependency
  196. type installDependencyTag struct {
  197. blueprint.BaseDependencyTag
  198. // embedding this struct provides the installation dependency requirement
  199. android.InstallAlwaysNeededDependencyTag
  200. name string
  201. }
  202. var (
  203. pythonLibTag = dependencyTag{name: "pythonLib"}
  204. javaDataTag = dependencyTag{name: "javaData"}
  205. launcherTag = dependencyTag{name: "launcher"}
  206. launcherSharedLibTag = installDependencyTag{name: "launcherSharedLib"}
  207. pathComponentRegexp = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_-]*$`)
  208. pyExt = ".py"
  209. protoExt = ".proto"
  210. pyVersion2 = "PY2"
  211. pyVersion3 = "PY3"
  212. initFileName = "__init__.py"
  213. mainFileName = "__main__.py"
  214. entryPointFile = "entry_point.txt"
  215. parFileExt = ".zip"
  216. internalPath = "internal"
  217. )
  218. // versionSplitMutator creates version variants for modules and appends the version-specific
  219. // properties for a given variant to the properties in the variant module
  220. func versionSplitMutator() func(android.BottomUpMutatorContext) {
  221. return func(mctx android.BottomUpMutatorContext) {
  222. if base, ok := mctx.Module().(*Module); ok {
  223. versionNames := []string{}
  224. // collect version specific properties, so that we can merge version-specific properties
  225. // into the module's overall properties
  226. versionProps := []VersionProperties{}
  227. // PY3 is first so that we alias the PY3 variant rather than PY2 if both
  228. // are available
  229. if proptools.BoolDefault(base.properties.Version.Py3.Enabled, true) {
  230. versionNames = append(versionNames, pyVersion3)
  231. versionProps = append(versionProps, base.properties.Version.Py3)
  232. }
  233. if proptools.BoolDefault(base.properties.Version.Py2.Enabled, false) {
  234. versionNames = append(versionNames, pyVersion2)
  235. versionProps = append(versionProps, base.properties.Version.Py2)
  236. }
  237. modules := mctx.CreateLocalVariations(versionNames...)
  238. // Alias module to the first variant
  239. if len(versionNames) > 0 {
  240. mctx.AliasVariation(versionNames[0])
  241. }
  242. for i, v := range versionNames {
  243. // set the actual version for Python module.
  244. modules[i].(*Module).properties.Actual_version = v
  245. // append versioned properties for the Python module to the overall properties
  246. err := proptools.AppendMatchingProperties([]interface{}{&modules[i].(*Module).properties}, &versionProps[i], nil)
  247. if err != nil {
  248. panic(err)
  249. }
  250. }
  251. }
  252. }
  253. }
  254. // HostToolPath returns a path if appropriate such that this module can be used as a host tool,
  255. // fulfilling HostToolProvider interface.
  256. func (p *Module) HostToolPath() android.OptionalPath {
  257. if p.installer == nil {
  258. // python_library is just meta module, and doesn't have any installer.
  259. return android.OptionalPath{}
  260. }
  261. // TODO: This should only be set when building host binaries -- tests built for device would be
  262. // setting this incorrectly.
  263. return android.OptionalPathForPath(p.installer.(*binaryDecorator).path)
  264. }
  265. // OutputFiles returns output files based on given tag, returns an error if tag is unsupported.
  266. func (p *Module) OutputFiles(tag string) (android.Paths, error) {
  267. switch tag {
  268. case "":
  269. if outputFile := p.installSource; outputFile.Valid() {
  270. return android.Paths{outputFile.Path()}, nil
  271. }
  272. return android.Paths{}, nil
  273. default:
  274. return nil, fmt.Errorf("unsupported module reference tag %q", tag)
  275. }
  276. }
  277. func (p *Module) isEmbeddedLauncherEnabled() bool {
  278. return p.installer != nil && Bool(p.properties.Embedded_launcher)
  279. }
  280. func anyHasExt(paths []string, ext string) bool {
  281. for _, p := range paths {
  282. if filepath.Ext(p) == ext {
  283. return true
  284. }
  285. }
  286. return false
  287. }
  288. func (p *Module) anySrcHasExt(ctx android.BottomUpMutatorContext, ext string) bool {
  289. return anyHasExt(p.properties.Srcs, ext)
  290. }
  291. // DepsMutator mutates dependencies for this module:
  292. // * handles proto dependencies,
  293. // * if required, specifies launcher and adds launcher dependencies,
  294. // * applies python version mutations to Python dependencies
  295. func (p *Module) DepsMutator(ctx android.BottomUpMutatorContext) {
  296. android.ProtoDeps(ctx, &p.protoProperties)
  297. versionVariation := []blueprint.Variation{
  298. {"python_version", p.properties.Actual_version},
  299. }
  300. // If sources contain a proto file, add dependency on libprotobuf-python
  301. if p.anySrcHasExt(ctx, protoExt) && p.Name() != "libprotobuf-python" {
  302. ctx.AddVariationDependencies(versionVariation, pythonLibTag, "libprotobuf-python")
  303. }
  304. // Add python library dependencies for this python version variation
  305. ctx.AddVariationDependencies(versionVariation, pythonLibTag, android.LastUniqueStrings(p.properties.Libs)...)
  306. // If this module will be installed and has an embedded launcher, we need to add dependencies for:
  307. // * standard library
  308. // * launcher
  309. // * shared dependencies of the launcher
  310. if p.installer != nil && p.isEmbeddedLauncherEnabled() {
  311. var stdLib string
  312. var launcherModule string
  313. // Add launcher shared lib dependencies. Ideally, these should be
  314. // derived from the `shared_libs` property of the launcher. However, we
  315. // cannot read the property at this stage and it will be too late to add
  316. // dependencies later.
  317. launcherSharedLibDeps := []string{
  318. "libsqlite",
  319. }
  320. // Add launcher-specific dependencies for bionic
  321. if ctx.Target().Os.Bionic() {
  322. launcherSharedLibDeps = append(launcherSharedLibDeps, "libc", "libdl", "libm")
  323. }
  324. switch p.properties.Actual_version {
  325. case pyVersion2:
  326. stdLib = "py2-stdlib"
  327. launcherModule = "py2-launcher"
  328. if p.bootstrapper.autorun() {
  329. launcherModule = "py2-launcher-autorun"
  330. }
  331. launcherSharedLibDeps = append(launcherSharedLibDeps, "libc++")
  332. case pyVersion3:
  333. stdLib = "py3-stdlib"
  334. launcherModule = "py3-launcher"
  335. if p.bootstrapper.autorun() {
  336. launcherModule = "py3-launcher-autorun"
  337. }
  338. if ctx.Device() {
  339. launcherSharedLibDeps = append(launcherSharedLibDeps, "liblog")
  340. }
  341. default:
  342. panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.",
  343. p.properties.Actual_version, ctx.ModuleName()))
  344. }
  345. ctx.AddVariationDependencies(versionVariation, pythonLibTag, stdLib)
  346. ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherTag, launcherModule)
  347. ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherSharedLibTag, launcherSharedLibDeps...)
  348. }
  349. // Emulate the data property for java_data but with the arch variation overridden to "common"
  350. // so that it can point to java modules.
  351. javaDataVariation := []blueprint.Variation{{"arch", android.Common.String()}}
  352. ctx.AddVariationDependencies(javaDataVariation, javaDataTag, p.properties.Java_data...)
  353. }
  354. func (p *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) {
  355. p.generatePythonBuildActions(ctx)
  356. // Only Python binary and test modules have non-empty bootstrapper.
  357. if p.bootstrapper != nil {
  358. // if the module is being installed, we need to collect all transitive dependencies to embed in
  359. // the final par
  360. p.collectPathsFromTransitiveDeps(ctx)
  361. // bootstrap the module, including resolving main file, getting launcher path, and
  362. // registering actions to build the par file
  363. // bootstrap returns the binary output path
  364. p.installSource = p.bootstrapper.bootstrap(ctx, p.properties.Actual_version,
  365. p.isEmbeddedLauncherEnabled(), p.srcsPathMappings, p.srcsZip, p.depsSrcsZips)
  366. }
  367. // Only Python binary and test modules have non-empty installer.
  368. if p.installer != nil {
  369. var sharedLibs []string
  370. // if embedded launcher is enabled, we need to collect the shared library depenendencies of the
  371. // launcher
  372. ctx.VisitDirectDeps(func(dep android.Module) {
  373. if ctx.OtherModuleDependencyTag(dep) == launcherSharedLibTag {
  374. sharedLibs = append(sharedLibs, ctx.OtherModuleName(dep))
  375. }
  376. })
  377. p.installer.setAndroidMkSharedLibs(sharedLibs)
  378. // Install the par file from installSource
  379. if p.installSource.Valid() {
  380. p.installer.install(ctx, p.installSource.Path())
  381. }
  382. }
  383. }
  384. // generatePythonBuildActions performs build actions common to all Python modules
  385. func (p *Module) generatePythonBuildActions(ctx android.ModuleContext) {
  386. expandedSrcs := android.PathsForModuleSrcExcludes(ctx, p.properties.Srcs, p.properties.Exclude_srcs)
  387. requiresSrcs := true
  388. if p.bootstrapper != nil && !p.bootstrapper.autorun() {
  389. requiresSrcs = false
  390. }
  391. if len(expandedSrcs) == 0 && requiresSrcs {
  392. ctx.ModuleErrorf("doesn't have any source files!")
  393. }
  394. // expand data files from "data" property.
  395. expandedData := android.PathsForModuleSrc(ctx, p.properties.Data)
  396. // Emulate the data property for java_data dependencies.
  397. for _, javaData := range ctx.GetDirectDepsWithTag(javaDataTag) {
  398. expandedData = append(expandedData, android.OutputFilesForModule(ctx, javaData, "")...)
  399. }
  400. // Validate pkg_path property
  401. pkgPath := String(p.properties.Pkg_path)
  402. if pkgPath != "" {
  403. // TODO: export validation from android/paths.go handling to replace this duplicated functionality
  404. pkgPath = filepath.Clean(String(p.properties.Pkg_path))
  405. if pkgPath == ".." || strings.HasPrefix(pkgPath, "../") ||
  406. strings.HasPrefix(pkgPath, "/") {
  407. ctx.PropertyErrorf("pkg_path",
  408. "%q must be a relative path contained in par file.",
  409. String(p.properties.Pkg_path))
  410. return
  411. }
  412. }
  413. // If property Is_internal is set, prepend pkgPath with internalPath
  414. if proptools.BoolDefault(p.properties.Is_internal, false) {
  415. pkgPath = filepath.Join(internalPath, pkgPath)
  416. }
  417. // generate src:destination path mappings for this module
  418. p.genModulePathMappings(ctx, pkgPath, expandedSrcs, expandedData)
  419. // generate the zipfile of all source and data files
  420. p.srcsZip = p.createSrcsZip(ctx, pkgPath)
  421. }
  422. func isValidPythonPath(path string) error {
  423. identifiers := strings.Split(strings.TrimSuffix(path, filepath.Ext(path)), "/")
  424. for _, token := range identifiers {
  425. if !pathComponentRegexp.MatchString(token) {
  426. return fmt.Errorf("the path %q contains invalid subpath %q. "+
  427. "Subpaths must be at least one character long. "+
  428. "The first character must an underscore or letter. "+
  429. "Following characters may be any of: letter, digit, underscore, hyphen.",
  430. path, token)
  431. }
  432. }
  433. return nil
  434. }
  435. // For this module, generate unique pathMappings: <dest: runfiles_path, src: source_path>
  436. // for python/data files expanded from properties.
  437. func (p *Module) genModulePathMappings(ctx android.ModuleContext, pkgPath string,
  438. expandedSrcs, expandedData android.Paths) {
  439. // fetch <runfiles_path, source_path> pairs from "src" and "data" properties to
  440. // check current module duplicates.
  441. destToPySrcs := make(map[string]string)
  442. destToPyData := make(map[string]string)
  443. for _, s := range expandedSrcs {
  444. if s.Ext() != pyExt && s.Ext() != protoExt {
  445. ctx.PropertyErrorf("srcs", "found non (.py|.proto) file: %q!", s.String())
  446. continue
  447. }
  448. runfilesPath := filepath.Join(pkgPath, s.Rel())
  449. if err := isValidPythonPath(runfilesPath); err != nil {
  450. ctx.PropertyErrorf("srcs", err.Error())
  451. }
  452. if !checkForDuplicateOutputPath(ctx, destToPySrcs, runfilesPath, s.String(), p.Name(), p.Name()) {
  453. p.srcsPathMappings = append(p.srcsPathMappings, pathMapping{dest: runfilesPath, src: s})
  454. }
  455. }
  456. for _, d := range expandedData {
  457. if d.Ext() == pyExt || d.Ext() == protoExt {
  458. ctx.PropertyErrorf("data", "found (.py|.proto) file: %q!", d.String())
  459. continue
  460. }
  461. runfilesPath := filepath.Join(pkgPath, d.Rel())
  462. if !checkForDuplicateOutputPath(ctx, destToPyData, runfilesPath, d.String(), p.Name(), p.Name()) {
  463. p.dataPathMappings = append(p.dataPathMappings,
  464. pathMapping{dest: runfilesPath, src: d})
  465. }
  466. }
  467. }
  468. // createSrcsZip registers build actions to zip current module's sources and data.
  469. func (p *Module) createSrcsZip(ctx android.ModuleContext, pkgPath string) android.Path {
  470. relativeRootMap := make(map[string]android.Paths)
  471. pathMappings := append(p.srcsPathMappings, p.dataPathMappings...)
  472. var protoSrcs android.Paths
  473. // "srcs" or "data" properties may contain filegroup so it might happen that
  474. // the root directory for each source path is different.
  475. for _, path := range pathMappings {
  476. // handle proto sources separately
  477. if path.src.Ext() == protoExt {
  478. protoSrcs = append(protoSrcs, path.src)
  479. } else {
  480. var relativeRoot string
  481. relativeRoot = strings.TrimSuffix(path.src.String(), path.src.Rel())
  482. if v, found := relativeRootMap[relativeRoot]; found {
  483. relativeRootMap[relativeRoot] = append(v, path.src)
  484. } else {
  485. relativeRootMap[relativeRoot] = android.Paths{path.src}
  486. }
  487. }
  488. }
  489. var zips android.Paths
  490. if len(protoSrcs) > 0 {
  491. protoFlags := android.GetProtoFlags(ctx, &p.protoProperties)
  492. protoFlags.OutTypeFlag = "--python_out"
  493. for _, srcFile := range protoSrcs {
  494. zip := genProto(ctx, srcFile, protoFlags, pkgPath)
  495. zips = append(zips, zip)
  496. }
  497. }
  498. if len(relativeRootMap) > 0 {
  499. // in order to keep stable order of soong_zip params, we sort the keys here.
  500. roots := android.SortedStringKeys(relativeRootMap)
  501. parArgs := []string{}
  502. if pkgPath != "" {
  503. // use package path as path prefix
  504. parArgs = append(parArgs, `-P `+pkgPath)
  505. }
  506. paths := android.Paths{}
  507. for _, root := range roots {
  508. // specify relative root of file in following -f arguments
  509. parArgs = append(parArgs, `-C `+root)
  510. for _, path := range relativeRootMap[root] {
  511. parArgs = append(parArgs, `-f `+path.String())
  512. paths = append(paths, path)
  513. }
  514. }
  515. origSrcsZip := android.PathForModuleOut(ctx, ctx.ModuleName()+".py.srcszip")
  516. ctx.Build(pctx, android.BuildParams{
  517. Rule: zip,
  518. Description: "python library archive",
  519. Output: origSrcsZip,
  520. // as zip rule does not use $in, there is no real need to distinguish between Inputs and Implicits
  521. Implicits: paths,
  522. Args: map[string]string{
  523. "args": strings.Join(parArgs, " "),
  524. },
  525. })
  526. zips = append(zips, origSrcsZip)
  527. }
  528. // we may have multiple zips due to separate handling of proto source files
  529. if len(zips) == 1 {
  530. return zips[0]
  531. } else {
  532. combinedSrcsZip := android.PathForModuleOut(ctx, ctx.ModuleName()+".srcszip")
  533. ctx.Build(pctx, android.BuildParams{
  534. Rule: combineZip,
  535. Description: "combine python library archive",
  536. Output: combinedSrcsZip,
  537. Inputs: zips,
  538. })
  539. return combinedSrcsZip
  540. }
  541. }
  542. // isPythonLibModule returns whether the given module is a Python library Module or not
  543. // This is distinguished by the fact that Python libraries are not installable, while other Python
  544. // modules are.
  545. func isPythonLibModule(module blueprint.Module) bool {
  546. if m, ok := module.(*Module); ok {
  547. // Python library has no bootstrapper or installer
  548. if m.bootstrapper == nil && m.installer == nil {
  549. return true
  550. }
  551. }
  552. return false
  553. }
  554. // collectPathsFromTransitiveDeps checks for source/data files for duplicate paths
  555. // for module and its transitive dependencies and collects list of data/source file
  556. // zips for transitive dependencies.
  557. func (p *Module) collectPathsFromTransitiveDeps(ctx android.ModuleContext) {
  558. // fetch <runfiles_path, source_path> pairs from "src" and "data" properties to
  559. // check duplicates.
  560. destToPySrcs := make(map[string]string)
  561. destToPyData := make(map[string]string)
  562. for _, path := range p.srcsPathMappings {
  563. destToPySrcs[path.dest] = path.src.String()
  564. }
  565. for _, path := range p.dataPathMappings {
  566. destToPyData[path.dest] = path.src.String()
  567. }
  568. seen := make(map[android.Module]bool)
  569. // visit all its dependencies in depth first.
  570. ctx.WalkDeps(func(child, parent android.Module) bool {
  571. // we only collect dependencies tagged as python library deps
  572. if ctx.OtherModuleDependencyTag(child) != pythonLibTag {
  573. return false
  574. }
  575. if seen[child] {
  576. return false
  577. }
  578. seen[child] = true
  579. // Python modules only can depend on Python libraries.
  580. if !isPythonLibModule(child) {
  581. ctx.PropertyErrorf("libs",
  582. "the dependency %q of module %q is not Python library!",
  583. ctx.ModuleName(), ctx.OtherModuleName(child))
  584. }
  585. // collect source and data paths, checking that there are no duplicate output file conflicts
  586. if dep, ok := child.(pythonDependency); ok {
  587. srcs := dep.getSrcsPathMappings()
  588. for _, path := range srcs {
  589. checkForDuplicateOutputPath(ctx, destToPySrcs,
  590. path.dest, path.src.String(), ctx.ModuleName(), ctx.OtherModuleName(child))
  591. }
  592. data := dep.getDataPathMappings()
  593. for _, path := range data {
  594. checkForDuplicateOutputPath(ctx, destToPyData,
  595. path.dest, path.src.String(), ctx.ModuleName(), ctx.OtherModuleName(child))
  596. }
  597. p.depsSrcsZips = append(p.depsSrcsZips, dep.getSrcsZip())
  598. }
  599. return true
  600. })
  601. }
  602. // chckForDuplicateOutputPath checks whether outputPath has already been included in map m, which
  603. // would result in two files being placed in the same location.
  604. // If there is a duplicate path, an error is thrown and true is returned
  605. // Otherwise, outputPath: srcPath is added to m and returns false
  606. func checkForDuplicateOutputPath(ctx android.ModuleContext, m map[string]string, outputPath, srcPath, curModule, otherModule string) bool {
  607. if oldSrcPath, found := m[outputPath]; found {
  608. ctx.ModuleErrorf("found two files to be placed at the same location within zip %q."+
  609. " First file: in module %s at path %q."+
  610. " Second file: in module %s at path %q.",
  611. outputPath, curModule, oldSrcPath, otherModule, srcPath)
  612. return true
  613. }
  614. m[outputPath] = srcPath
  615. return false
  616. }
  617. // InstallInData returns true as Python is not supported in the system partition
  618. func (p *Module) InstallInData() bool {
  619. return true
  620. }
  621. var Bool = proptools.Bool
  622. var BoolDefault = proptools.BoolDefault
  623. var String = proptools.String