python.go 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832
  1. // Copyright 2017 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package python
  15. // This file contains the "Base" module type for building Python program.
  16. import (
  17. "fmt"
  18. "path/filepath"
  19. "regexp"
  20. "strings"
  21. "android/soong/bazel"
  22. "github.com/google/blueprint"
  23. "github.com/google/blueprint/proptools"
  24. "android/soong/android"
  25. )
  26. func init() {
  27. registerPythonMutators(android.InitRegistrationContext)
  28. }
  29. func registerPythonMutators(ctx android.RegistrationContext) {
  30. ctx.PreDepsMutators(RegisterPythonPreDepsMutators)
  31. }
  32. // Exported to support other packages using Python modules in tests.
  33. func RegisterPythonPreDepsMutators(ctx android.RegisterMutatorsContext) {
  34. ctx.BottomUp("python_version", versionSplitMutator()).Parallel()
  35. }
  36. // the version-specific properties that apply to python modules.
  37. type VersionProperties struct {
  38. // whether the module is required to be built with this version.
  39. // Defaults to true for Python 3, and false otherwise.
  40. Enabled *bool
  41. // list of source files specific to this Python version.
  42. // Using the syntax ":module", srcs may reference the outputs of other modules that produce source files,
  43. // e.g. genrule or filegroup.
  44. Srcs []string `android:"path,arch_variant"`
  45. // list of source files that should not be used to build the Python module for this version.
  46. // This is most useful to remove files that are not common to all Python versions.
  47. Exclude_srcs []string `android:"path,arch_variant"`
  48. // list of the Python libraries used only for this Python version.
  49. Libs []string `android:"arch_variant"`
  50. // whether the binary is required to be built with embedded launcher for this version, defaults to false.
  51. Embedded_launcher *bool // TODO(b/174041232): Remove this property
  52. }
  53. // properties that apply to all python modules
  54. type BaseProperties struct {
  55. // the package path prefix within the output artifact at which to place the source/data
  56. // files of the current module.
  57. // eg. Pkg_path = "a/b/c"; Other packages can reference this module by using
  58. // (from a.b.c import ...) statement.
  59. // if left unspecified, all the source/data files path is unchanged within zip file.
  60. Pkg_path *string
  61. // true, if the Python module is used internally, eg, Python std libs.
  62. Is_internal *bool
  63. // list of source (.py) files compatible both with Python2 and Python3 used to compile the
  64. // Python module.
  65. // srcs may reference the outputs of other modules that produce source files like genrule
  66. // or filegroup using the syntax ":module".
  67. // Srcs has to be non-empty.
  68. Srcs []string `android:"path,arch_variant"`
  69. // list of source files that should not be used to build the C/C++ module.
  70. // This is most useful in the arch/multilib variants to remove non-common files
  71. Exclude_srcs []string `android:"path,arch_variant"`
  72. // list of files or filegroup modules that provide data that should be installed alongside
  73. // the test. the file extension can be arbitrary except for (.py).
  74. Data []string `android:"path,arch_variant"`
  75. // list of java modules that provide data that should be installed alongside the test.
  76. Java_data []string
  77. // list of the Python libraries compatible both with Python2 and Python3.
  78. Libs []string `android:"arch_variant"`
  79. Version struct {
  80. // Python2-specific properties, including whether Python2 is supported for this module
  81. // and version-specific sources, exclusions and dependencies.
  82. Py2 VersionProperties `android:"arch_variant"`
  83. // Python3-specific properties, including whether Python3 is supported for this module
  84. // and version-specific sources, exclusions and dependencies.
  85. Py3 VersionProperties `android:"arch_variant"`
  86. } `android:"arch_variant"`
  87. // the actual version each module uses after variations created.
  88. // this property name is hidden from users' perspectives, and soong will populate it during
  89. // runtime.
  90. Actual_version string `blueprint:"mutated"`
  91. // whether the module is required to be built with actual_version.
  92. // this is set by the python version mutator based on version-specific properties
  93. Enabled *bool `blueprint:"mutated"`
  94. // whether the binary is required to be built with embedded launcher for this actual_version.
  95. // this is set by the python version mutator based on version-specific properties
  96. Embedded_launcher *bool `blueprint:"mutated"`
  97. }
  98. type baseAttributes struct {
  99. // TODO(b/200311466): Probably not translate b/c Bazel has no good equiv
  100. //Pkg_path bazel.StringAttribute
  101. // TODO: Related to Pkg_bath and similarLy gated
  102. //Is_internal bazel.BoolAttribute
  103. // Combines Srcs and Exclude_srcs
  104. Srcs bazel.LabelListAttribute
  105. Deps bazel.LabelListAttribute
  106. // Combines Data and Java_data (invariant)
  107. Data bazel.LabelListAttribute
  108. Imports bazel.StringListAttribute
  109. }
  110. // Used to store files of current module after expanding dependencies
  111. type pathMapping struct {
  112. dest string
  113. src android.Path
  114. }
  115. type Module struct {
  116. android.ModuleBase
  117. android.DefaultableModuleBase
  118. android.BazelModuleBase
  119. properties BaseProperties
  120. protoProperties android.ProtoProperties
  121. // initialize before calling Init
  122. hod android.HostOrDeviceSupported
  123. multilib android.Multilib
  124. // interface used to bootstrap .par executable when embedded_launcher is true
  125. // this should be set by Python modules which are runnable, e.g. binaries and tests
  126. // bootstrapper might be nil (e.g. Python library module).
  127. bootstrapper bootstrapper
  128. // interface that implements functions required for installation
  129. // this should be set by Python modules which are runnable, e.g. binaries and tests
  130. // installer might be nil (e.g. Python library module).
  131. installer installer
  132. // the Python files of current module after expanding source dependencies.
  133. // pathMapping: <dest: runfile_path, src: source_path>
  134. srcsPathMappings []pathMapping
  135. // the data files of current module after expanding source dependencies.
  136. // pathMapping: <dest: runfile_path, src: source_path>
  137. dataPathMappings []pathMapping
  138. // the zip filepath for zipping current module source/data files.
  139. srcsZip android.Path
  140. // dependency modules' zip filepath for zipping current module source/data files.
  141. depsSrcsZips android.Paths
  142. // (.intermediate) module output path as installation source.
  143. installSource android.OptionalPath
  144. // Map to ensure sub-part of the AndroidMk for this module is only added once
  145. subAndroidMkOnce map[subAndroidMkProvider]bool
  146. }
  147. // newModule generates new Python base module
  148. func newModule(hod android.HostOrDeviceSupported, multilib android.Multilib) *Module {
  149. return &Module{
  150. hod: hod,
  151. multilib: multilib,
  152. }
  153. }
  154. func (m *Module) makeArchVariantBaseAttributes(ctx android.TopDownMutatorContext) baseAttributes {
  155. var attrs baseAttributes
  156. archVariantBaseProps := m.GetArchVariantProperties(ctx, &BaseProperties{})
  157. for axis, configToProps := range archVariantBaseProps {
  158. for config, props := range configToProps {
  159. if baseProps, ok := props.(*BaseProperties); ok {
  160. attrs.Srcs.SetSelectValue(axis, config,
  161. android.BazelLabelForModuleSrcExcludes(ctx, baseProps.Srcs, baseProps.Exclude_srcs))
  162. attrs.Deps.SetSelectValue(axis, config,
  163. android.BazelLabelForModuleDeps(ctx, baseProps.Libs))
  164. data := android.BazelLabelForModuleSrc(ctx, baseProps.Data)
  165. data.Append(android.BazelLabelForModuleSrc(ctx, baseProps.Java_data))
  166. attrs.Data.SetSelectValue(axis, config, data)
  167. }
  168. }
  169. }
  170. partitionedSrcs := bazel.PartitionLabelListAttribute(ctx, &attrs.Srcs, bazel.LabelPartitions{
  171. "proto": android.ProtoSrcLabelPartition,
  172. "py": bazel.LabelPartition{Keep_remainder: true},
  173. })
  174. attrs.Srcs = partitionedSrcs["py"]
  175. if !partitionedSrcs["proto"].IsEmpty() {
  176. protoInfo, _ := android.Bp2buildProtoProperties(ctx, &m.ModuleBase, partitionedSrcs["proto"])
  177. protoLabel := bazel.Label{Label: ":" + protoInfo.Name}
  178. pyProtoLibraryName := m.Name() + "_py_proto"
  179. ctx.CreateBazelTargetModule(bazel.BazelTargetModuleProperties{
  180. Rule_class: "py_proto_library",
  181. Bzl_load_location: "//build/bazel/rules/python:py_proto.bzl",
  182. }, android.CommonAttributes{
  183. Name: pyProtoLibraryName,
  184. }, &bazelPythonProtoLibraryAttributes{
  185. Deps: bazel.MakeSingleLabelListAttribute(protoLabel),
  186. })
  187. attrs.Deps.Add(bazel.MakeLabelAttribute(":" + pyProtoLibraryName))
  188. }
  189. // Bazel normally requires `import path.from.top.of.tree` statements in
  190. // python code, but with soong you can directly import modules from libraries.
  191. // Add "imports" attributes to the bazel library so it matches soong's behavior.
  192. imports := "."
  193. if m.properties.Pkg_path != nil {
  194. // TODO(b/215119317) This is a hack to handle the fact that we don't convert
  195. // pkg_path properly right now. If the folder structure that contains this
  196. // Android.bp file matches pkg_path, we can set imports to an appropriate
  197. // number of ../..s to emulate moving the files under a pkg_path folder.
  198. pkg_path := filepath.Clean(*m.properties.Pkg_path)
  199. if strings.HasPrefix(pkg_path, "/") {
  200. ctx.ModuleErrorf("pkg_path cannot start with a /: %s", pkg_path)
  201. }
  202. if !strings.HasSuffix(ctx.ModuleDir(), "/"+pkg_path) && ctx.ModuleDir() != pkg_path {
  203. ctx.ModuleErrorf("Currently, bp2build only supports pkg_paths that are the same as the folders the Android.bp file is in. pkg_path: %s, module directory: %s", pkg_path, ctx.ModuleDir())
  204. }
  205. numFolders := strings.Count(pkg_path, "/") + 1
  206. dots := make([]string, numFolders)
  207. for i := 0; i < numFolders; i++ {
  208. dots[i] = ".."
  209. }
  210. imports = strings.Join(dots, "/")
  211. }
  212. attrs.Imports = bazel.MakeStringListAttribute([]string{imports})
  213. return attrs
  214. }
  215. // bootstrapper interface should be implemented for runnable modules, e.g. binary and test
  216. type bootstrapper interface {
  217. bootstrapperProps() []interface{}
  218. bootstrap(ctx android.ModuleContext, ActualVersion string, embeddedLauncher bool,
  219. srcsPathMappings []pathMapping, srcsZip android.Path,
  220. depsSrcsZips android.Paths) android.OptionalPath
  221. autorun() bool
  222. }
  223. // installer interface should be implemented for installable modules, e.g. binary and test
  224. type installer interface {
  225. install(ctx android.ModuleContext, path android.Path)
  226. setAndroidMkSharedLibs(sharedLibs []string)
  227. }
  228. // interface implemented by Python modules to provide source and data mappings and zip to python
  229. // modules that depend on it
  230. type pythonDependency interface {
  231. getSrcsPathMappings() []pathMapping
  232. getDataPathMappings() []pathMapping
  233. getSrcsZip() android.Path
  234. }
  235. // getSrcsPathMappings gets this module's path mapping of src source path : runfiles destination
  236. func (p *Module) getSrcsPathMappings() []pathMapping {
  237. return p.srcsPathMappings
  238. }
  239. // getSrcsPathMappings gets this module's path mapping of data source path : runfiles destination
  240. func (p *Module) getDataPathMappings() []pathMapping {
  241. return p.dataPathMappings
  242. }
  243. // getSrcsZip returns the filepath where the current module's source/data files are zipped.
  244. func (p *Module) getSrcsZip() android.Path {
  245. return p.srcsZip
  246. }
  247. var _ pythonDependency = (*Module)(nil)
  248. var _ android.AndroidMkEntriesProvider = (*Module)(nil)
  249. func (p *Module) init(additionalProps ...interface{}) android.Module {
  250. p.AddProperties(&p.properties, &p.protoProperties)
  251. // Add additional properties for bootstrapping/installation
  252. // This is currently tied to the bootstrapper interface;
  253. // however, these are a combination of properties for the installation and bootstrapping of a module
  254. if p.bootstrapper != nil {
  255. p.AddProperties(p.bootstrapper.bootstrapperProps()...)
  256. }
  257. android.InitAndroidArchModule(p, p.hod, p.multilib)
  258. android.InitDefaultableModule(p)
  259. return p
  260. }
  261. // Python-specific tag to transfer information on the purpose of a dependency.
  262. // This is used when adding a dependency on a module, which can later be accessed when visiting
  263. // dependencies.
  264. type dependencyTag struct {
  265. blueprint.BaseDependencyTag
  266. name string
  267. }
  268. // Python-specific tag that indicates that installed files of this module should depend on installed
  269. // files of the dependency
  270. type installDependencyTag struct {
  271. blueprint.BaseDependencyTag
  272. // embedding this struct provides the installation dependency requirement
  273. android.InstallAlwaysNeededDependencyTag
  274. name string
  275. }
  276. var (
  277. pythonLibTag = dependencyTag{name: "pythonLib"}
  278. javaDataTag = dependencyTag{name: "javaData"}
  279. launcherTag = dependencyTag{name: "launcher"}
  280. launcherSharedLibTag = installDependencyTag{name: "launcherSharedLib"}
  281. pathComponentRegexp = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_-]*$`)
  282. pyExt = ".py"
  283. protoExt = ".proto"
  284. pyVersion2 = "PY2"
  285. pyVersion3 = "PY3"
  286. initFileName = "__init__.py"
  287. mainFileName = "__main__.py"
  288. entryPointFile = "entry_point.txt"
  289. parFileExt = ".zip"
  290. internalPath = "internal"
  291. )
  292. // versionSplitMutator creates version variants for modules and appends the version-specific
  293. // properties for a given variant to the properties in the variant module
  294. func versionSplitMutator() func(android.BottomUpMutatorContext) {
  295. return func(mctx android.BottomUpMutatorContext) {
  296. if base, ok := mctx.Module().(*Module); ok {
  297. versionNames := []string{}
  298. // collect version specific properties, so that we can merge version-specific properties
  299. // into the module's overall properties
  300. versionProps := []VersionProperties{}
  301. // PY3 is first so that we alias the PY3 variant rather than PY2 if both
  302. // are available
  303. if proptools.BoolDefault(base.properties.Version.Py3.Enabled, true) {
  304. versionNames = append(versionNames, pyVersion3)
  305. versionProps = append(versionProps, base.properties.Version.Py3)
  306. }
  307. if proptools.BoolDefault(base.properties.Version.Py2.Enabled, false) {
  308. versionNames = append(versionNames, pyVersion2)
  309. versionProps = append(versionProps, base.properties.Version.Py2)
  310. }
  311. modules := mctx.CreateLocalVariations(versionNames...)
  312. // Alias module to the first variant
  313. if len(versionNames) > 0 {
  314. mctx.AliasVariation(versionNames[0])
  315. }
  316. for i, v := range versionNames {
  317. // set the actual version for Python module.
  318. modules[i].(*Module).properties.Actual_version = v
  319. // append versioned properties for the Python module to the overall properties
  320. err := proptools.AppendMatchingProperties([]interface{}{&modules[i].(*Module).properties}, &versionProps[i], nil)
  321. if err != nil {
  322. panic(err)
  323. }
  324. }
  325. }
  326. }
  327. }
  328. // HostToolPath returns a path if appropriate such that this module can be used as a host tool,
  329. // fulfilling HostToolProvider interface.
  330. func (p *Module) HostToolPath() android.OptionalPath {
  331. if p.installer != nil {
  332. if bin, ok := p.installer.(*binaryDecorator); ok {
  333. // TODO: This should only be set when building host binaries -- tests built for device would be
  334. // setting this incorrectly.
  335. return android.OptionalPathForPath(bin.path)
  336. }
  337. }
  338. return android.OptionalPath{}
  339. }
  340. // OutputFiles returns output files based on given tag, returns an error if tag is unsupported.
  341. func (p *Module) OutputFiles(tag string) (android.Paths, error) {
  342. switch tag {
  343. case "":
  344. if outputFile := p.installSource; outputFile.Valid() {
  345. return android.Paths{outputFile.Path()}, nil
  346. }
  347. return android.Paths{}, nil
  348. default:
  349. return nil, fmt.Errorf("unsupported module reference tag %q", tag)
  350. }
  351. }
  352. func (p *Module) isEmbeddedLauncherEnabled() bool {
  353. return p.installer != nil && Bool(p.properties.Embedded_launcher)
  354. }
  355. func anyHasExt(paths []string, ext string) bool {
  356. for _, p := range paths {
  357. if filepath.Ext(p) == ext {
  358. return true
  359. }
  360. }
  361. return false
  362. }
  363. func (p *Module) anySrcHasExt(ctx android.BottomUpMutatorContext, ext string) bool {
  364. return anyHasExt(p.properties.Srcs, ext)
  365. }
  366. // DepsMutator mutates dependencies for this module:
  367. // - handles proto dependencies,
  368. // - if required, specifies launcher and adds launcher dependencies,
  369. // - applies python version mutations to Python dependencies
  370. func (p *Module) DepsMutator(ctx android.BottomUpMutatorContext) {
  371. android.ProtoDeps(ctx, &p.protoProperties)
  372. versionVariation := []blueprint.Variation{
  373. {"python_version", p.properties.Actual_version},
  374. }
  375. // If sources contain a proto file, add dependency on libprotobuf-python
  376. if p.anySrcHasExt(ctx, protoExt) && p.Name() != "libprotobuf-python" {
  377. ctx.AddVariationDependencies(versionVariation, pythonLibTag, "libprotobuf-python")
  378. }
  379. // Add python library dependencies for this python version variation
  380. ctx.AddVariationDependencies(versionVariation, pythonLibTag, android.LastUniqueStrings(p.properties.Libs)...)
  381. // If this module will be installed and has an embedded launcher, we need to add dependencies for:
  382. // * standard library
  383. // * launcher
  384. // * shared dependencies of the launcher
  385. if p.installer != nil && p.isEmbeddedLauncherEnabled() {
  386. var stdLib string
  387. var launcherModule string
  388. // Add launcher shared lib dependencies. Ideally, these should be
  389. // derived from the `shared_libs` property of the launcher. However, we
  390. // cannot read the property at this stage and it will be too late to add
  391. // dependencies later.
  392. launcherSharedLibDeps := []string{
  393. "libsqlite",
  394. }
  395. // Add launcher-specific dependencies for bionic
  396. if ctx.Target().Os.Bionic() {
  397. launcherSharedLibDeps = append(launcherSharedLibDeps, "libc", "libdl", "libm")
  398. }
  399. if ctx.Target().Os == android.LinuxMusl && !ctx.Config().HostStaticBinaries() {
  400. launcherSharedLibDeps = append(launcherSharedLibDeps, "libc_musl")
  401. }
  402. switch p.properties.Actual_version {
  403. case pyVersion2:
  404. stdLib = "py2-stdlib"
  405. launcherModule = "py2-launcher"
  406. if p.bootstrapper.autorun() {
  407. launcherModule = "py2-launcher-autorun"
  408. }
  409. launcherSharedLibDeps = append(launcherSharedLibDeps, "libc++")
  410. case pyVersion3:
  411. stdLib = "py3-stdlib"
  412. launcherModule = "py3-launcher"
  413. if p.bootstrapper.autorun() {
  414. launcherModule = "py3-launcher-autorun"
  415. }
  416. if ctx.Config().HostStaticBinaries() && ctx.Target().Os == android.LinuxMusl {
  417. launcherModule += "-static"
  418. }
  419. if ctx.Device() {
  420. launcherSharedLibDeps = append(launcherSharedLibDeps, "liblog")
  421. }
  422. default:
  423. panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.",
  424. p.properties.Actual_version, ctx.ModuleName()))
  425. }
  426. ctx.AddVariationDependencies(versionVariation, pythonLibTag, stdLib)
  427. ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherTag, launcherModule)
  428. ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherSharedLibTag, launcherSharedLibDeps...)
  429. }
  430. // Emulate the data property for java_data but with the arch variation overridden to "common"
  431. // so that it can point to java modules.
  432. javaDataVariation := []blueprint.Variation{{"arch", android.Common.String()}}
  433. ctx.AddVariationDependencies(javaDataVariation, javaDataTag, p.properties.Java_data...)
  434. }
  435. func (p *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) {
  436. p.generatePythonBuildActions(ctx)
  437. // Only Python binary and test modules have non-empty bootstrapper.
  438. if p.bootstrapper != nil {
  439. // if the module is being installed, we need to collect all transitive dependencies to embed in
  440. // the final par
  441. p.collectPathsFromTransitiveDeps(ctx)
  442. // bootstrap the module, including resolving main file, getting launcher path, and
  443. // registering actions to build the par file
  444. // bootstrap returns the binary output path
  445. p.installSource = p.bootstrapper.bootstrap(ctx, p.properties.Actual_version,
  446. p.isEmbeddedLauncherEnabled(), p.srcsPathMappings, p.srcsZip, p.depsSrcsZips)
  447. }
  448. // Only Python binary and test modules have non-empty installer.
  449. if p.installer != nil {
  450. var sharedLibs []string
  451. // if embedded launcher is enabled, we need to collect the shared library depenendencies of the
  452. // launcher
  453. for _, dep := range ctx.GetDirectDepsWithTag(launcherSharedLibTag) {
  454. sharedLibs = append(sharedLibs, ctx.OtherModuleName(dep))
  455. }
  456. p.installer.setAndroidMkSharedLibs(sharedLibs)
  457. // Install the par file from installSource
  458. if p.installSource.Valid() {
  459. p.installer.install(ctx, p.installSource.Path())
  460. }
  461. }
  462. }
  463. // generatePythonBuildActions performs build actions common to all Python modules
  464. func (p *Module) generatePythonBuildActions(ctx android.ModuleContext) {
  465. expandedSrcs := android.PathsForModuleSrcExcludes(ctx, p.properties.Srcs, p.properties.Exclude_srcs)
  466. requiresSrcs := true
  467. if p.bootstrapper != nil && !p.bootstrapper.autorun() {
  468. requiresSrcs = false
  469. }
  470. if len(expandedSrcs) == 0 && requiresSrcs {
  471. ctx.ModuleErrorf("doesn't have any source files!")
  472. }
  473. // expand data files from "data" property.
  474. expandedData := android.PathsForModuleSrc(ctx, p.properties.Data)
  475. // Emulate the data property for java_data dependencies.
  476. for _, javaData := range ctx.GetDirectDepsWithTag(javaDataTag) {
  477. expandedData = append(expandedData, android.OutputFilesForModule(ctx, javaData, "")...)
  478. }
  479. // Validate pkg_path property
  480. pkgPath := String(p.properties.Pkg_path)
  481. if pkgPath != "" {
  482. // TODO: export validation from android/paths.go handling to replace this duplicated functionality
  483. pkgPath = filepath.Clean(String(p.properties.Pkg_path))
  484. if pkgPath == ".." || strings.HasPrefix(pkgPath, "../") ||
  485. strings.HasPrefix(pkgPath, "/") {
  486. ctx.PropertyErrorf("pkg_path",
  487. "%q must be a relative path contained in par file.",
  488. String(p.properties.Pkg_path))
  489. return
  490. }
  491. }
  492. // If property Is_internal is set, prepend pkgPath with internalPath
  493. if proptools.BoolDefault(p.properties.Is_internal, false) {
  494. pkgPath = filepath.Join(internalPath, pkgPath)
  495. }
  496. // generate src:destination path mappings for this module
  497. p.genModulePathMappings(ctx, pkgPath, expandedSrcs, expandedData)
  498. // generate the zipfile of all source and data files
  499. p.srcsZip = p.createSrcsZip(ctx, pkgPath)
  500. }
  501. func isValidPythonPath(path string) error {
  502. identifiers := strings.Split(strings.TrimSuffix(path, filepath.Ext(path)), "/")
  503. for _, token := range identifiers {
  504. if !pathComponentRegexp.MatchString(token) {
  505. return fmt.Errorf("the path %q contains invalid subpath %q. "+
  506. "Subpaths must be at least one character long. "+
  507. "The first character must an underscore or letter. "+
  508. "Following characters may be any of: letter, digit, underscore, hyphen.",
  509. path, token)
  510. }
  511. }
  512. return nil
  513. }
  514. // For this module, generate unique pathMappings: <dest: runfiles_path, src: source_path>
  515. // for python/data files expanded from properties.
  516. func (p *Module) genModulePathMappings(ctx android.ModuleContext, pkgPath string,
  517. expandedSrcs, expandedData android.Paths) {
  518. // fetch <runfiles_path, source_path> pairs from "src" and "data" properties to
  519. // check current module duplicates.
  520. destToPySrcs := make(map[string]string)
  521. destToPyData := make(map[string]string)
  522. for _, s := range expandedSrcs {
  523. if s.Ext() != pyExt && s.Ext() != protoExt {
  524. ctx.PropertyErrorf("srcs", "found non (.py|.proto) file: %q!", s.String())
  525. continue
  526. }
  527. runfilesPath := filepath.Join(pkgPath, s.Rel())
  528. if err := isValidPythonPath(runfilesPath); err != nil {
  529. ctx.PropertyErrorf("srcs", err.Error())
  530. }
  531. if !checkForDuplicateOutputPath(ctx, destToPySrcs, runfilesPath, s.String(), p.Name(), p.Name()) {
  532. p.srcsPathMappings = append(p.srcsPathMappings, pathMapping{dest: runfilesPath, src: s})
  533. }
  534. }
  535. for _, d := range expandedData {
  536. if d.Ext() == pyExt || d.Ext() == protoExt {
  537. ctx.PropertyErrorf("data", "found (.py|.proto) file: %q!", d.String())
  538. continue
  539. }
  540. runfilesPath := filepath.Join(pkgPath, d.Rel())
  541. if !checkForDuplicateOutputPath(ctx, destToPyData, runfilesPath, d.String(), p.Name(), p.Name()) {
  542. p.dataPathMappings = append(p.dataPathMappings,
  543. pathMapping{dest: runfilesPath, src: d})
  544. }
  545. }
  546. }
  547. // createSrcsZip registers build actions to zip current module's sources and data.
  548. func (p *Module) createSrcsZip(ctx android.ModuleContext, pkgPath string) android.Path {
  549. relativeRootMap := make(map[string]android.Paths)
  550. pathMappings := append(p.srcsPathMappings, p.dataPathMappings...)
  551. var protoSrcs android.Paths
  552. // "srcs" or "data" properties may contain filegroup so it might happen that
  553. // the root directory for each source path is different.
  554. for _, path := range pathMappings {
  555. // handle proto sources separately
  556. if path.src.Ext() == protoExt {
  557. protoSrcs = append(protoSrcs, path.src)
  558. } else {
  559. var relativeRoot string
  560. relativeRoot = strings.TrimSuffix(path.src.String(), path.src.Rel())
  561. if v, found := relativeRootMap[relativeRoot]; found {
  562. relativeRootMap[relativeRoot] = append(v, path.src)
  563. } else {
  564. relativeRootMap[relativeRoot] = android.Paths{path.src}
  565. }
  566. }
  567. }
  568. var zips android.Paths
  569. if len(protoSrcs) > 0 {
  570. protoFlags := android.GetProtoFlags(ctx, &p.protoProperties)
  571. protoFlags.OutTypeFlag = "--python_out"
  572. for _, srcFile := range protoSrcs {
  573. zip := genProto(ctx, srcFile, protoFlags, pkgPath)
  574. zips = append(zips, zip)
  575. }
  576. }
  577. if len(relativeRootMap) > 0 {
  578. // in order to keep stable order of soong_zip params, we sort the keys here.
  579. roots := android.SortedStringKeys(relativeRootMap)
  580. // Use -symlinks=false so that the symlinks in the bazel output directory are followed
  581. parArgs := []string{"-symlinks=false"}
  582. if pkgPath != "" {
  583. // use package path as path prefix
  584. parArgs = append(parArgs, `-P `+pkgPath)
  585. }
  586. paths := android.Paths{}
  587. for _, root := range roots {
  588. // specify relative root of file in following -f arguments
  589. parArgs = append(parArgs, `-C `+root)
  590. for _, path := range relativeRootMap[root] {
  591. parArgs = append(parArgs, `-f `+path.String())
  592. paths = append(paths, path)
  593. }
  594. }
  595. origSrcsZip := android.PathForModuleOut(ctx, ctx.ModuleName()+".py.srcszip")
  596. ctx.Build(pctx, android.BuildParams{
  597. Rule: zip,
  598. Description: "python library archive",
  599. Output: origSrcsZip,
  600. // as zip rule does not use $in, there is no real need to distinguish between Inputs and Implicits
  601. Implicits: paths,
  602. Args: map[string]string{
  603. "args": strings.Join(parArgs, " "),
  604. },
  605. })
  606. zips = append(zips, origSrcsZip)
  607. }
  608. // we may have multiple zips due to separate handling of proto source files
  609. if len(zips) == 1 {
  610. return zips[0]
  611. } else {
  612. combinedSrcsZip := android.PathForModuleOut(ctx, ctx.ModuleName()+".srcszip")
  613. ctx.Build(pctx, android.BuildParams{
  614. Rule: combineZip,
  615. Description: "combine python library archive",
  616. Output: combinedSrcsZip,
  617. Inputs: zips,
  618. })
  619. return combinedSrcsZip
  620. }
  621. }
  622. // isPythonLibModule returns whether the given module is a Python library Module or not
  623. func isPythonLibModule(module blueprint.Module) bool {
  624. if m, ok := module.(*Module); ok {
  625. return m.isLibrary()
  626. }
  627. return false
  628. }
  629. // This is distinguished by the fact that Python libraries are not installable, while other Python
  630. // modules are.
  631. func (p *Module) isLibrary() bool {
  632. // Python library has no bootstrapper or installer
  633. return p.bootstrapper == nil && p.installer == nil
  634. }
  635. func (p *Module) isBinary() bool {
  636. _, ok := p.bootstrapper.(*binaryDecorator)
  637. return ok
  638. }
  639. // collectPathsFromTransitiveDeps checks for source/data files for duplicate paths
  640. // for module and its transitive dependencies and collects list of data/source file
  641. // zips for transitive dependencies.
  642. func (p *Module) collectPathsFromTransitiveDeps(ctx android.ModuleContext) {
  643. // fetch <runfiles_path, source_path> pairs from "src" and "data" properties to
  644. // check duplicates.
  645. destToPySrcs := make(map[string]string)
  646. destToPyData := make(map[string]string)
  647. for _, path := range p.srcsPathMappings {
  648. destToPySrcs[path.dest] = path.src.String()
  649. }
  650. for _, path := range p.dataPathMappings {
  651. destToPyData[path.dest] = path.src.String()
  652. }
  653. seen := make(map[android.Module]bool)
  654. // visit all its dependencies in depth first.
  655. ctx.WalkDeps(func(child, parent android.Module) bool {
  656. // we only collect dependencies tagged as python library deps
  657. if ctx.OtherModuleDependencyTag(child) != pythonLibTag {
  658. return false
  659. }
  660. if seen[child] {
  661. return false
  662. }
  663. seen[child] = true
  664. // Python modules only can depend on Python libraries.
  665. if !isPythonLibModule(child) {
  666. ctx.PropertyErrorf("libs",
  667. "the dependency %q of module %q is not Python library!",
  668. ctx.OtherModuleName(child), ctx.ModuleName())
  669. }
  670. // collect source and data paths, checking that there are no duplicate output file conflicts
  671. if dep, ok := child.(pythonDependency); ok {
  672. srcs := dep.getSrcsPathMappings()
  673. for _, path := range srcs {
  674. checkForDuplicateOutputPath(ctx, destToPySrcs,
  675. path.dest, path.src.String(), ctx.ModuleName(), ctx.OtherModuleName(child))
  676. }
  677. data := dep.getDataPathMappings()
  678. for _, path := range data {
  679. checkForDuplicateOutputPath(ctx, destToPyData,
  680. path.dest, path.src.String(), ctx.ModuleName(), ctx.OtherModuleName(child))
  681. }
  682. p.depsSrcsZips = append(p.depsSrcsZips, dep.getSrcsZip())
  683. }
  684. return true
  685. })
  686. }
  687. // chckForDuplicateOutputPath checks whether outputPath has already been included in map m, which
  688. // would result in two files being placed in the same location.
  689. // If there is a duplicate path, an error is thrown and true is returned
  690. // Otherwise, outputPath: srcPath is added to m and returns false
  691. func checkForDuplicateOutputPath(ctx android.ModuleContext, m map[string]string, outputPath, srcPath, curModule, otherModule string) bool {
  692. if oldSrcPath, found := m[outputPath]; found {
  693. ctx.ModuleErrorf("found two files to be placed at the same location within zip %q."+
  694. " First file: in module %s at path %q."+
  695. " Second file: in module %s at path %q.",
  696. outputPath, curModule, oldSrcPath, otherModule, srcPath)
  697. return true
  698. }
  699. m[outputPath] = srcPath
  700. return false
  701. }
  702. // InstallInData returns true as Python is not supported in the system partition
  703. func (p *Module) InstallInData() bool {
  704. return true
  705. }
  706. func (p *Module) ConvertWithBp2build(ctx android.TopDownMutatorContext) {
  707. if p.isLibrary() {
  708. pythonLibBp2Build(ctx, p)
  709. } else if p.isBinary() {
  710. pythonBinaryBp2Build(ctx, p)
  711. }
  712. }
  713. var Bool = proptools.Bool
  714. var BoolDefault = proptools.BoolDefault
  715. var String = proptools.String