genrule.go 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936
  1. // Copyright 2015 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. // A genrule module takes a list of source files ("srcs" property), an optional
  15. // list of tools ("tools" property), and a command line ("cmd" property), to
  16. // generate output files ("out" property).
  17. package genrule
  18. import (
  19. "fmt"
  20. "io"
  21. "path/filepath"
  22. "strconv"
  23. "strings"
  24. "github.com/google/blueprint"
  25. "github.com/google/blueprint/bootstrap"
  26. "github.com/google/blueprint/proptools"
  27. "android/soong/android"
  28. "android/soong/bazel"
  29. )
  30. func init() {
  31. RegisterGenruleBuildComponents(android.InitRegistrationContext)
  32. }
  33. // Test fixture preparer that will register most genrule build components.
  34. //
  35. // Singletons and mutators should only be added here if they are needed for a majority of genrule
  36. // module types, otherwise they should be added under a separate preparer to allow them to be
  37. // selected only when needed to reduce test execution time.
  38. //
  39. // Module types do not have much of an overhead unless they are used so this should include as many
  40. // module types as possible. The exceptions are those module types that require mutators and/or
  41. // singletons in order to function in which case they should be kept together in a separate
  42. // preparer.
  43. var PrepareForTestWithGenRuleBuildComponents = android.GroupFixturePreparers(
  44. android.FixtureRegisterWithContext(RegisterGenruleBuildComponents),
  45. )
  46. // Prepare a fixture to use all genrule module types, mutators and singletons fully.
  47. //
  48. // This should only be used by tests that want to run with as much of the build enabled as possible.
  49. var PrepareForIntegrationTestWithGenrule = android.GroupFixturePreparers(
  50. PrepareForTestWithGenRuleBuildComponents,
  51. )
  52. func RegisterGenruleBuildComponents(ctx android.RegistrationContext) {
  53. ctx.RegisterModuleType("genrule_defaults", defaultsFactory)
  54. ctx.RegisterModuleType("gensrcs", GenSrcsFactory)
  55. ctx.RegisterModuleType("genrule", GenRuleFactory)
  56. ctx.FinalDepsMutators(func(ctx android.RegisterMutatorsContext) {
  57. ctx.BottomUp("genrule_tool_deps", toolDepsMutator).Parallel()
  58. })
  59. }
  60. var (
  61. pctx = android.NewPackageContext("android/soong/genrule")
  62. // Used by gensrcs when there is more than 1 shard to merge the outputs
  63. // of each shard into a zip file.
  64. gensrcsMerge = pctx.AndroidStaticRule("gensrcsMerge", blueprint.RuleParams{
  65. Command: "${soongZip} -o ${tmpZip} @${tmpZip}.rsp && ${zipSync} -d ${genDir} ${tmpZip}",
  66. CommandDeps: []string{"${soongZip}", "${zipSync}"},
  67. Rspfile: "${tmpZip}.rsp",
  68. RspfileContent: "${zipArgs}",
  69. }, "tmpZip", "genDir", "zipArgs")
  70. )
  71. func init() {
  72. pctx.Import("android/soong/android")
  73. pctx.HostBinToolVariable("soongZip", "soong_zip")
  74. pctx.HostBinToolVariable("zipSync", "zipsync")
  75. }
  76. type SourceFileGenerator interface {
  77. GeneratedSourceFiles() android.Paths
  78. GeneratedHeaderDirs() android.Paths
  79. GeneratedDeps() android.Paths
  80. }
  81. // Alias for android.HostToolProvider
  82. // Deprecated: use android.HostToolProvider instead.
  83. type HostToolProvider interface {
  84. android.HostToolProvider
  85. }
  86. type hostToolDependencyTag struct {
  87. blueprint.BaseDependencyTag
  88. android.LicenseAnnotationToolchainDependencyTag
  89. label string
  90. }
  91. func (t hostToolDependencyTag) AllowDisabledModuleDependency(target android.Module) bool {
  92. // Allow depending on a disabled module if it's replaced by a prebuilt
  93. // counterpart. We get the prebuilt through android.PrebuiltGetPreferred in
  94. // GenerateAndroidBuildActions.
  95. return target.IsReplacedByPrebuilt()
  96. }
  97. var _ android.AllowDisabledModuleDependency = (*hostToolDependencyTag)(nil)
  98. type generatorProperties struct {
  99. // The command to run on one or more input files. Cmd supports substitution of a few variables.
  100. //
  101. // Available variables for substitution:
  102. //
  103. // $(location): the path to the first entry in tools or tool_files.
  104. // $(location <label>): the path to the tool, tool_file, input or output with name <label>. Use $(location) if <label> refers to a rule that outputs exactly one file.
  105. // $(locations <label>): the paths to the tools, tool_files, inputs or outputs with name <label>. Use $(locations) if <label> refers to a rule that outputs two or more files.
  106. // $(in): one or more input files.
  107. // $(out): a single output file.
  108. // $(depfile): a file to which dependencies will be written, if the depfile property is set to true.
  109. // $(genDir): the sandbox directory for this tool; contains $(out).
  110. // $$: a literal $
  111. Cmd *string
  112. // Enable reading a file containing dependencies in gcc format after the command completes
  113. Depfile *bool
  114. // name of the modules (if any) that produces the host executable. Leave empty for
  115. // prebuilts or scripts that do not need a module to build them.
  116. Tools []string
  117. // Local file that is used as the tool
  118. Tool_files []string `android:"path"`
  119. // List of directories to export generated headers from
  120. Export_include_dirs []string
  121. // list of input files
  122. Srcs []string `android:"path,arch_variant"`
  123. // input files to exclude
  124. Exclude_srcs []string `android:"path,arch_variant"`
  125. }
  126. type Module struct {
  127. android.ModuleBase
  128. android.DefaultableModuleBase
  129. android.BazelModuleBase
  130. android.ApexModuleBase
  131. // For other packages to make their own genrules with extra
  132. // properties
  133. Extra interface{}
  134. // CmdModifier can be set by wrappers around genrule to modify the command, for example to
  135. // prefix environment variables to it.
  136. CmdModifier func(ctx android.ModuleContext, cmd string) string
  137. android.ImageInterface
  138. properties generatorProperties
  139. // For the different tasks that genrule and gensrc generate. genrule will
  140. // generate 1 task, and gensrc will generate 1 or more tasks based on the
  141. // number of shards the input files are sharded into.
  142. taskGenerator taskFunc
  143. rule blueprint.Rule
  144. rawCommands []string
  145. exportedIncludeDirs android.Paths
  146. outputFiles android.Paths
  147. outputDeps android.Paths
  148. subName string
  149. subDir string
  150. // Collect the module directory for IDE info in java/jdeps.go.
  151. modulePaths []string
  152. }
  153. type taskFunc func(ctx android.ModuleContext, rawCommand string, srcFiles android.Paths) []generateTask
  154. type generateTask struct {
  155. in android.Paths
  156. out android.WritablePaths
  157. depFile android.WritablePath
  158. copyTo android.WritablePaths // For gensrcs to set on gensrcsMerge rule.
  159. genDir android.WritablePath
  160. extraTools android.Paths // dependencies on tools used by the generator
  161. cmd string
  162. // For gensrsc sharding.
  163. shard int
  164. shards int
  165. }
  166. func (g *Module) GeneratedSourceFiles() android.Paths {
  167. return g.outputFiles
  168. }
  169. func (g *Module) Srcs() android.Paths {
  170. return append(android.Paths{}, g.outputFiles...)
  171. }
  172. func (g *Module) GeneratedHeaderDirs() android.Paths {
  173. return g.exportedIncludeDirs
  174. }
  175. func (g *Module) GeneratedDeps() android.Paths {
  176. return g.outputDeps
  177. }
  178. func (g *Module) OutputFiles(tag string) (android.Paths, error) {
  179. if tag == "" {
  180. return append(android.Paths{}, g.outputFiles...), nil
  181. }
  182. // otherwise, tag should match one of outputs
  183. for _, outputFile := range g.outputFiles {
  184. if outputFile.Rel() == tag {
  185. return android.Paths{outputFile}, nil
  186. }
  187. }
  188. return nil, fmt.Errorf("unsupported module reference tag %q", tag)
  189. }
  190. var _ android.SourceFileProducer = (*Module)(nil)
  191. var _ android.OutputFileProducer = (*Module)(nil)
  192. func toolDepsMutator(ctx android.BottomUpMutatorContext) {
  193. if g, ok := ctx.Module().(*Module); ok {
  194. for _, tool := range g.properties.Tools {
  195. tag := hostToolDependencyTag{label: tool}
  196. if m := android.SrcIsModule(tool); m != "" {
  197. tool = m
  198. }
  199. ctx.AddFarVariationDependencies(ctx.Config().BuildOSTarget.Variations(), tag, tool)
  200. }
  201. }
  202. }
  203. // Returns true if information was available from Bazel, false if bazel invocation still needs to occur.
  204. func (c *Module) GenerateBazelBuildActions(ctx android.ModuleContext, label string) bool {
  205. bazelCtx := ctx.Config().BazelContext
  206. filePaths, ok := bazelCtx.GetOutputFiles(label, android.GetConfigKey(ctx))
  207. if ok {
  208. var bazelOutputFiles android.Paths
  209. exportIncludeDirs := map[string]bool{}
  210. for _, bazelOutputFile := range filePaths {
  211. bazelOutputFiles = append(bazelOutputFiles, android.PathForBazelOut(ctx, bazelOutputFile))
  212. exportIncludeDirs[filepath.Dir(bazelOutputFile)] = true
  213. }
  214. c.outputFiles = bazelOutputFiles
  215. c.outputDeps = bazelOutputFiles
  216. for includePath, _ := range exportIncludeDirs {
  217. c.exportedIncludeDirs = append(c.exportedIncludeDirs, android.PathForBazelOut(ctx, includePath))
  218. }
  219. }
  220. return ok
  221. }
  222. func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) {
  223. g.subName = ctx.ModuleSubDir()
  224. // Collect the module directory for IDE info in java/jdeps.go.
  225. g.modulePaths = append(g.modulePaths, ctx.ModuleDir())
  226. if len(g.properties.Export_include_dirs) > 0 {
  227. for _, dir := range g.properties.Export_include_dirs {
  228. g.exportedIncludeDirs = append(g.exportedIncludeDirs,
  229. android.PathForModuleGen(ctx, g.subDir, ctx.ModuleDir(), dir))
  230. }
  231. } else {
  232. g.exportedIncludeDirs = append(g.exportedIncludeDirs, android.PathForModuleGen(ctx, g.subDir))
  233. }
  234. locationLabels := map[string]location{}
  235. firstLabel := ""
  236. addLocationLabel := func(label string, loc location) {
  237. if firstLabel == "" {
  238. firstLabel = label
  239. }
  240. if _, exists := locationLabels[label]; !exists {
  241. locationLabels[label] = loc
  242. } else {
  243. ctx.ModuleErrorf("multiple labels for %q, %q and %q",
  244. label, locationLabels[label], loc)
  245. }
  246. }
  247. var tools android.Paths
  248. var packagedTools []android.PackagingSpec
  249. if len(g.properties.Tools) > 0 {
  250. seenTools := make(map[string]bool)
  251. ctx.VisitDirectDepsBlueprint(func(module blueprint.Module) {
  252. switch tag := ctx.OtherModuleDependencyTag(module).(type) {
  253. case hostToolDependencyTag:
  254. tool := ctx.OtherModuleName(module)
  255. if m, ok := module.(android.Module); ok {
  256. // Necessary to retrieve any prebuilt replacement for the tool, since
  257. // toolDepsMutator runs too late for the prebuilt mutators to have
  258. // replaced the dependency.
  259. module = android.PrebuiltGetPreferred(ctx, m)
  260. }
  261. switch t := module.(type) {
  262. case android.HostToolProvider:
  263. // A HostToolProvider provides the path to a tool, which will be copied
  264. // into the sandbox.
  265. if !t.(android.Module).Enabled() {
  266. if ctx.Config().AllowMissingDependencies() {
  267. ctx.AddMissingDependencies([]string{tool})
  268. } else {
  269. ctx.ModuleErrorf("depends on disabled module %q", tool)
  270. }
  271. return
  272. }
  273. path := t.HostToolPath()
  274. if !path.Valid() {
  275. ctx.ModuleErrorf("host tool %q missing output file", tool)
  276. return
  277. }
  278. if specs := t.TransitivePackagingSpecs(); specs != nil {
  279. // If the HostToolProvider has PackgingSpecs, which are definitions of the
  280. // required relative locations of the tool and its dependencies, use those
  281. // instead. They will be copied to those relative locations in the sbox
  282. // sandbox.
  283. packagedTools = append(packagedTools, specs...)
  284. // Assume that the first PackagingSpec of the module is the tool.
  285. addLocationLabel(tag.label, packagedToolLocation{specs[0]})
  286. } else {
  287. tools = append(tools, path.Path())
  288. addLocationLabel(tag.label, toolLocation{android.Paths{path.Path()}})
  289. }
  290. case bootstrap.GoBinaryTool:
  291. // A GoBinaryTool provides the install path to a tool, which will be copied.
  292. p := android.PathForGoBinary(ctx, t)
  293. tools = append(tools, p)
  294. addLocationLabel(tag.label, toolLocation{android.Paths{p}})
  295. default:
  296. ctx.ModuleErrorf("%q is not a host tool provider", tool)
  297. return
  298. }
  299. seenTools[tag.label] = true
  300. }
  301. })
  302. // If AllowMissingDependencies is enabled, the build will not have stopped when
  303. // AddFarVariationDependencies was called on a missing tool, which will result in nonsensical
  304. // "cmd: unknown location label ..." errors later. Add a placeholder file to the local label.
  305. // The command that uses this placeholder file will never be executed because the rule will be
  306. // replaced with an android.Error rule reporting the missing dependencies.
  307. if ctx.Config().AllowMissingDependencies() {
  308. for _, tool := range g.properties.Tools {
  309. if !seenTools[tool] {
  310. addLocationLabel(tool, errorLocation{"***missing tool " + tool + "***"})
  311. }
  312. }
  313. }
  314. }
  315. if ctx.Failed() {
  316. return
  317. }
  318. for _, toolFile := range g.properties.Tool_files {
  319. paths := android.PathsForModuleSrc(ctx, []string{toolFile})
  320. tools = append(tools, paths...)
  321. addLocationLabel(toolFile, toolLocation{paths})
  322. }
  323. var srcFiles android.Paths
  324. for _, in := range g.properties.Srcs {
  325. paths, missingDeps := android.PathsAndMissingDepsForModuleSrcExcludes(ctx, []string{in}, g.properties.Exclude_srcs)
  326. if len(missingDeps) > 0 {
  327. if !ctx.Config().AllowMissingDependencies() {
  328. panic(fmt.Errorf("should never get here, the missing dependencies %q should have been reported in DepsMutator",
  329. missingDeps))
  330. }
  331. // If AllowMissingDependencies is enabled, the build will not have stopped when
  332. // the dependency was added on a missing SourceFileProducer module, which will result in nonsensical
  333. // "cmd: label ":..." has no files" errors later. Add a placeholder file to the local label.
  334. // The command that uses this placeholder file will never be executed because the rule will be
  335. // replaced with an android.Error rule reporting the missing dependencies.
  336. ctx.AddMissingDependencies(missingDeps)
  337. addLocationLabel(in, errorLocation{"***missing srcs " + in + "***"})
  338. } else {
  339. srcFiles = append(srcFiles, paths...)
  340. addLocationLabel(in, inputLocation{paths})
  341. }
  342. }
  343. var copyFrom android.Paths
  344. var outputFiles android.WritablePaths
  345. var zipArgs strings.Builder
  346. cmd := String(g.properties.Cmd)
  347. if g.CmdModifier != nil {
  348. cmd = g.CmdModifier(ctx, cmd)
  349. }
  350. // Generate tasks, either from genrule or gensrcs.
  351. for _, task := range g.taskGenerator(ctx, cmd, srcFiles) {
  352. if len(task.out) == 0 {
  353. ctx.ModuleErrorf("must have at least one output file")
  354. return
  355. }
  356. // Pick a unique path outside the task.genDir for the sbox manifest textproto,
  357. // a unique rule name, and the user-visible description.
  358. manifestName := "genrule.sbox.textproto"
  359. desc := "generate"
  360. name := "generator"
  361. if task.shards > 0 {
  362. manifestName = "genrule_" + strconv.Itoa(task.shard) + ".sbox.textproto"
  363. desc += " " + strconv.Itoa(task.shard)
  364. name += strconv.Itoa(task.shard)
  365. } else if len(task.out) == 1 {
  366. desc += " " + task.out[0].Base()
  367. }
  368. manifestPath := android.PathForModuleOut(ctx, manifestName)
  369. // Use a RuleBuilder to create a rule that runs the command inside an sbox sandbox.
  370. rule := android.NewRuleBuilder(pctx, ctx).Sbox(task.genDir, manifestPath).SandboxTools()
  371. cmd := rule.Command()
  372. for _, out := range task.out {
  373. addLocationLabel(out.Rel(), outputLocation{out})
  374. }
  375. referencedDepfile := false
  376. rawCommand, err := android.Expand(task.cmd, func(name string) (string, error) {
  377. // report the error directly without returning an error to android.Expand to catch multiple errors in a
  378. // single run
  379. reportError := func(fmt string, args ...interface{}) (string, error) {
  380. ctx.PropertyErrorf("cmd", fmt, args...)
  381. return "SOONG_ERROR", nil
  382. }
  383. switch name {
  384. case "location":
  385. if len(g.properties.Tools) == 0 && len(g.properties.Tool_files) == 0 {
  386. return reportError("at least one `tools` or `tool_files` is required if $(location) is used")
  387. }
  388. loc := locationLabels[firstLabel]
  389. paths := loc.Paths(cmd)
  390. if len(paths) == 0 {
  391. return reportError("default label %q has no files", firstLabel)
  392. } else if len(paths) > 1 {
  393. return reportError("default label %q has multiple files, use $(locations %s) to reference it",
  394. firstLabel, firstLabel)
  395. }
  396. return paths[0], nil
  397. case "in":
  398. return strings.Join(cmd.PathsForInputs(srcFiles), " "), nil
  399. case "out":
  400. var sandboxOuts []string
  401. for _, out := range task.out {
  402. sandboxOuts = append(sandboxOuts, cmd.PathForOutput(out))
  403. }
  404. return strings.Join(sandboxOuts, " "), nil
  405. case "depfile":
  406. referencedDepfile = true
  407. if !Bool(g.properties.Depfile) {
  408. return reportError("$(depfile) used without depfile property")
  409. }
  410. return "__SBOX_DEPFILE__", nil
  411. case "genDir":
  412. return cmd.PathForOutput(task.genDir), nil
  413. default:
  414. if strings.HasPrefix(name, "location ") {
  415. label := strings.TrimSpace(strings.TrimPrefix(name, "location "))
  416. if loc, ok := locationLabels[label]; ok {
  417. paths := loc.Paths(cmd)
  418. if len(paths) == 0 {
  419. return reportError("label %q has no files", label)
  420. } else if len(paths) > 1 {
  421. return reportError("label %q has multiple files, use $(locations %s) to reference it",
  422. label, label)
  423. }
  424. return paths[0], nil
  425. } else {
  426. return reportError("unknown location label %q", label)
  427. }
  428. } else if strings.HasPrefix(name, "locations ") {
  429. label := strings.TrimSpace(strings.TrimPrefix(name, "locations "))
  430. if loc, ok := locationLabels[label]; ok {
  431. paths := loc.Paths(cmd)
  432. if len(paths) == 0 {
  433. return reportError("label %q has no files", label)
  434. }
  435. return strings.Join(paths, " "), nil
  436. } else {
  437. return reportError("unknown locations label %q", label)
  438. }
  439. } else {
  440. return reportError("unknown variable '$(%s)'", name)
  441. }
  442. }
  443. })
  444. if err != nil {
  445. ctx.PropertyErrorf("cmd", "%s", err.Error())
  446. return
  447. }
  448. if Bool(g.properties.Depfile) && !referencedDepfile {
  449. ctx.PropertyErrorf("cmd", "specified depfile=true but did not include a reference to '${depfile}' in cmd")
  450. return
  451. }
  452. g.rawCommands = append(g.rawCommands, rawCommand)
  453. cmd.Text(rawCommand)
  454. cmd.ImplicitOutputs(task.out)
  455. cmd.Implicits(task.in)
  456. cmd.ImplicitTools(tools)
  457. cmd.ImplicitTools(task.extraTools)
  458. cmd.ImplicitPackagedTools(packagedTools)
  459. if Bool(g.properties.Depfile) {
  460. cmd.ImplicitDepFile(task.depFile)
  461. }
  462. // Create the rule to run the genrule command inside sbox.
  463. rule.Build(name, desc)
  464. if len(task.copyTo) > 0 {
  465. // If copyTo is set, multiple shards need to be copied into a single directory.
  466. // task.out contains the per-shard paths, and copyTo contains the corresponding
  467. // final path. The files need to be copied into the final directory by a
  468. // single rule so it can remove the directory before it starts to ensure no
  469. // old files remain. zipsync already does this, so build up zipArgs that
  470. // zip all the per-shard directories into a single zip.
  471. outputFiles = append(outputFiles, task.copyTo...)
  472. copyFrom = append(copyFrom, task.out.Paths()...)
  473. zipArgs.WriteString(" -C " + task.genDir.String())
  474. zipArgs.WriteString(android.JoinWithPrefix(task.out.Strings(), " -f "))
  475. } else {
  476. outputFiles = append(outputFiles, task.out...)
  477. }
  478. }
  479. if len(copyFrom) > 0 {
  480. // Create a rule that zips all the per-shard directories into a single zip and then
  481. // uses zipsync to unzip it into the final directory.
  482. ctx.Build(pctx, android.BuildParams{
  483. Rule: gensrcsMerge,
  484. Implicits: copyFrom,
  485. Outputs: outputFiles,
  486. Description: "merge shards",
  487. Args: map[string]string{
  488. "zipArgs": zipArgs.String(),
  489. "tmpZip": android.PathForModuleGen(ctx, g.subDir+".zip").String(),
  490. "genDir": android.PathForModuleGen(ctx, g.subDir).String(),
  491. },
  492. })
  493. }
  494. g.outputFiles = outputFiles.Paths()
  495. bazelModuleLabel := g.GetBazelLabel(ctx, g)
  496. bazelActionsUsed := false
  497. if g.MixedBuildsEnabled(ctx) {
  498. bazelActionsUsed = g.GenerateBazelBuildActions(ctx, bazelModuleLabel)
  499. }
  500. if !bazelActionsUsed {
  501. // For <= 6 outputs, just embed those directly in the users. Right now, that covers >90% of
  502. // the genrules on AOSP. That will make things simpler to look at the graph in the common
  503. // case. For larger sets of outputs, inject a phony target in between to limit ninja file
  504. // growth.
  505. if len(g.outputFiles) <= 6 {
  506. g.outputDeps = g.outputFiles
  507. } else {
  508. phonyFile := android.PathForModuleGen(ctx, "genrule-phony")
  509. ctx.Build(pctx, android.BuildParams{
  510. Rule: blueprint.Phony,
  511. Output: phonyFile,
  512. Inputs: g.outputFiles,
  513. })
  514. g.outputDeps = android.Paths{phonyFile}
  515. }
  516. }
  517. }
  518. // Collect information for opening IDE project files in java/jdeps.go.
  519. func (g *Module) IDEInfo(dpInfo *android.IdeInfo) {
  520. dpInfo.Srcs = append(dpInfo.Srcs, g.Srcs().Strings()...)
  521. for _, src := range g.properties.Srcs {
  522. if strings.HasPrefix(src, ":") {
  523. src = strings.Trim(src, ":")
  524. dpInfo.Deps = append(dpInfo.Deps, src)
  525. }
  526. }
  527. dpInfo.Paths = append(dpInfo.Paths, g.modulePaths...)
  528. }
  529. func (g *Module) AndroidMk() android.AndroidMkData {
  530. return android.AndroidMkData{
  531. Class: "ETC",
  532. OutputFile: android.OptionalPathForPath(g.outputFiles[0]),
  533. SubName: g.subName,
  534. Extra: []android.AndroidMkExtraFunc{
  535. func(w io.Writer, outputFile android.Path) {
  536. fmt.Fprintln(w, "LOCAL_UNINSTALLABLE_MODULE := true")
  537. },
  538. },
  539. Custom: func(w io.Writer, name, prefix, moduleDir string, data android.AndroidMkData) {
  540. android.WriteAndroidMkData(w, data)
  541. if data.SubName != "" {
  542. fmt.Fprintln(w, ".PHONY:", name)
  543. fmt.Fprintln(w, name, ":", name+g.subName)
  544. }
  545. },
  546. }
  547. }
  548. var _ android.ApexModule = (*Module)(nil)
  549. // Implements android.ApexModule
  550. func (g *Module) ShouldSupportSdkVersion(ctx android.BaseModuleContext,
  551. sdkVersion android.ApiLevel) error {
  552. // Because generated outputs are checked by client modules(e.g. cc_library, ...)
  553. // we can safely ignore the check here.
  554. return nil
  555. }
  556. func generatorFactory(taskGenerator taskFunc, props ...interface{}) *Module {
  557. module := &Module{
  558. taskGenerator: taskGenerator,
  559. }
  560. module.AddProperties(props...)
  561. module.AddProperties(&module.properties)
  562. module.ImageInterface = noopImageInterface{}
  563. return module
  564. }
  565. type noopImageInterface struct{}
  566. func (x noopImageInterface) ImageMutatorBegin(android.BaseModuleContext) {}
  567. func (x noopImageInterface) CoreVariantNeeded(android.BaseModuleContext) bool { return false }
  568. func (x noopImageInterface) RamdiskVariantNeeded(android.BaseModuleContext) bool { return false }
  569. func (x noopImageInterface) VendorRamdiskVariantNeeded(android.BaseModuleContext) bool { return false }
  570. func (x noopImageInterface) DebugRamdiskVariantNeeded(android.BaseModuleContext) bool { return false }
  571. func (x noopImageInterface) RecoveryVariantNeeded(android.BaseModuleContext) bool { return false }
  572. func (x noopImageInterface) ExtraImageVariations(ctx android.BaseModuleContext) []string { return nil }
  573. func (x noopImageInterface) SetImageVariation(ctx android.BaseModuleContext, variation string, module android.Module) {
  574. }
  575. func NewGenSrcs() *Module {
  576. properties := &genSrcsProperties{}
  577. // finalSubDir is the name of the subdirectory that output files will be generated into.
  578. // It is used so that per-shard directories can be placed alongside it an then finally
  579. // merged into it.
  580. const finalSubDir = "gensrcs"
  581. taskGenerator := func(ctx android.ModuleContext, rawCommand string, srcFiles android.Paths) []generateTask {
  582. shardSize := defaultShardSize
  583. if s := properties.Shard_size; s != nil {
  584. shardSize = int(*s)
  585. }
  586. // gensrcs rules can easily hit command line limits by repeating the command for
  587. // every input file. Shard the input files into groups.
  588. shards := android.ShardPaths(srcFiles, shardSize)
  589. var generateTasks []generateTask
  590. for i, shard := range shards {
  591. var commands []string
  592. var outFiles android.WritablePaths
  593. var commandDepFiles []string
  594. var copyTo android.WritablePaths
  595. // When sharding is enabled (i.e. len(shards) > 1), the sbox rules for each
  596. // shard will be write to their own directories and then be merged together
  597. // into finalSubDir. If sharding is not enabled (i.e. len(shards) == 1),
  598. // the sbox rule will write directly to finalSubDir.
  599. genSubDir := finalSubDir
  600. if len(shards) > 1 {
  601. genSubDir = strconv.Itoa(i)
  602. }
  603. genDir := android.PathForModuleGen(ctx, genSubDir)
  604. // TODO(ccross): this RuleBuilder is a hack to be able to call
  605. // rule.Command().PathForOutput. Replace this with passing the rule into the
  606. // generator.
  607. rule := android.NewRuleBuilder(pctx, ctx).Sbox(genDir, nil).SandboxTools()
  608. for _, in := range shard {
  609. outFile := android.GenPathWithExt(ctx, finalSubDir, in, String(properties.Output_extension))
  610. // If sharding is enabled, then outFile is the path to the output file in
  611. // the shard directory, and copyTo is the path to the output file in the
  612. // final directory.
  613. if len(shards) > 1 {
  614. shardFile := android.GenPathWithExt(ctx, genSubDir, in, String(properties.Output_extension))
  615. copyTo = append(copyTo, outFile)
  616. outFile = shardFile
  617. }
  618. outFiles = append(outFiles, outFile)
  619. // pre-expand the command line to replace $in and $out with references to
  620. // a single input and output file.
  621. command, err := android.Expand(rawCommand, func(name string) (string, error) {
  622. switch name {
  623. case "in":
  624. return in.String(), nil
  625. case "out":
  626. return rule.Command().PathForOutput(outFile), nil
  627. case "depfile":
  628. // Generate a depfile for each output file. Store the list for
  629. // later in order to combine them all into a single depfile.
  630. depFile := rule.Command().PathForOutput(outFile.ReplaceExtension(ctx, "d"))
  631. commandDepFiles = append(commandDepFiles, depFile)
  632. return depFile, nil
  633. default:
  634. return "$(" + name + ")", nil
  635. }
  636. })
  637. if err != nil {
  638. ctx.PropertyErrorf("cmd", err.Error())
  639. }
  640. // escape the command in case for example it contains '#', an odd number of '"', etc
  641. command = fmt.Sprintf("bash -c %v", proptools.ShellEscape(command))
  642. commands = append(commands, command)
  643. }
  644. fullCommand := strings.Join(commands, " && ")
  645. var outputDepfile android.WritablePath
  646. var extraTools android.Paths
  647. if len(commandDepFiles) > 0 {
  648. // Each command wrote to a depfile, but ninja can only handle one
  649. // depfile per rule. Use the dep_fixer tool at the end of the
  650. // command to combine all the depfiles into a single output depfile.
  651. outputDepfile = android.PathForModuleGen(ctx, genSubDir, "gensrcs.d")
  652. depFixerTool := ctx.Config().HostToolPath(ctx, "dep_fixer")
  653. fullCommand += fmt.Sprintf(" && %s -o $(depfile) %s",
  654. rule.Command().PathForTool(depFixerTool),
  655. strings.Join(commandDepFiles, " "))
  656. extraTools = append(extraTools, depFixerTool)
  657. }
  658. generateTasks = append(generateTasks, generateTask{
  659. in: shard,
  660. out: outFiles,
  661. depFile: outputDepfile,
  662. copyTo: copyTo,
  663. genDir: genDir,
  664. cmd: fullCommand,
  665. shard: i,
  666. shards: len(shards),
  667. extraTools: extraTools,
  668. })
  669. }
  670. return generateTasks
  671. }
  672. g := generatorFactory(taskGenerator, properties)
  673. g.subDir = finalSubDir
  674. return g
  675. }
  676. func GenSrcsFactory() android.Module {
  677. m := NewGenSrcs()
  678. android.InitAndroidModule(m)
  679. return m
  680. }
  681. type genSrcsProperties struct {
  682. // extension that will be substituted for each output file
  683. Output_extension *string
  684. // maximum number of files that will be passed on a single command line.
  685. Shard_size *int64
  686. }
  687. const defaultShardSize = 50
  688. func NewGenRule() *Module {
  689. properties := &genRuleProperties{}
  690. taskGenerator := func(ctx android.ModuleContext, rawCommand string, srcFiles android.Paths) []generateTask {
  691. outs := make(android.WritablePaths, len(properties.Out))
  692. var depFile android.WritablePath
  693. for i, out := range properties.Out {
  694. outPath := android.PathForModuleGen(ctx, out)
  695. if i == 0 {
  696. depFile = outPath.ReplaceExtension(ctx, "d")
  697. }
  698. outs[i] = outPath
  699. }
  700. return []generateTask{{
  701. in: srcFiles,
  702. out: outs,
  703. depFile: depFile,
  704. genDir: android.PathForModuleGen(ctx),
  705. cmd: rawCommand,
  706. }}
  707. }
  708. return generatorFactory(taskGenerator, properties)
  709. }
  710. func GenRuleFactory() android.Module {
  711. m := NewGenRule()
  712. android.InitAndroidModule(m)
  713. android.InitDefaultableModule(m)
  714. android.InitBazelModule(m)
  715. return m
  716. }
  717. type genRuleProperties struct {
  718. // names of the output files that will be generated
  719. Out []string `android:"arch_variant"`
  720. }
  721. type bazelGenruleAttributes struct {
  722. Srcs bazel.LabelListAttribute
  723. Outs []string
  724. Tools bazel.LabelListAttribute
  725. Cmd string
  726. }
  727. // ConvertWithBp2build converts a Soong module -> Bazel target.
  728. func (m *Module) ConvertWithBp2build(ctx android.TopDownMutatorContext) {
  729. // Bazel only has the "tools" attribute.
  730. tools_prop := android.BazelLabelForModuleDeps(ctx, m.properties.Tools)
  731. tool_files_prop := android.BazelLabelForModuleSrc(ctx, m.properties.Tool_files)
  732. tools_prop.Append(tool_files_prop)
  733. tools := bazel.MakeLabelListAttribute(tools_prop)
  734. srcs := bazel.MakeLabelListAttribute(android.BazelLabelForModuleSrc(ctx, m.properties.Srcs))
  735. var allReplacements bazel.LabelList
  736. allReplacements.Append(tools.Value)
  737. allReplacements.Append(srcs.Value)
  738. // Replace in and out variables with $< and $@
  739. var cmd string
  740. if m.properties.Cmd != nil {
  741. cmd = strings.Replace(*m.properties.Cmd, "$(in)", "$(SRCS)", -1)
  742. cmd = strings.Replace(cmd, "$(out)", "$(OUTS)", -1)
  743. genDir := "$(GENDIR)"
  744. if t := ctx.ModuleType(); t == "cc_genrule" || t == "java_genrule" || t == "java_genrule_host" {
  745. genDir = "$(RULEDIR)"
  746. }
  747. cmd = strings.Replace(cmd, "$(genDir)", genDir, -1)
  748. if len(tools.Value.Includes) > 0 {
  749. cmd = strings.Replace(cmd, "$(location)", fmt.Sprintf("$(location %s)", tools.Value.Includes[0].Label), -1)
  750. cmd = strings.Replace(cmd, "$(locations)", fmt.Sprintf("$(locations %s)", tools.Value.Includes[0].Label), -1)
  751. }
  752. for _, l := range allReplacements.Includes {
  753. bpLoc := fmt.Sprintf("$(location %s)", l.OriginalModuleName)
  754. bpLocs := fmt.Sprintf("$(locations %s)", l.OriginalModuleName)
  755. bazelLoc := fmt.Sprintf("$(location %s)", l.Label)
  756. bazelLocs := fmt.Sprintf("$(locations %s)", l.Label)
  757. cmd = strings.Replace(cmd, bpLoc, bazelLoc, -1)
  758. cmd = strings.Replace(cmd, bpLocs, bazelLocs, -1)
  759. }
  760. }
  761. // The Out prop is not in an immediately accessible field
  762. // in the Module struct, so use GetProperties and cast it
  763. // to the known struct prop.
  764. var outs []string
  765. for _, propIntf := range m.GetProperties() {
  766. if props, ok := propIntf.(*genRuleProperties); ok {
  767. outs = props.Out
  768. break
  769. }
  770. }
  771. attrs := &bazelGenruleAttributes{
  772. Srcs: srcs,
  773. Outs: outs,
  774. Cmd: cmd,
  775. Tools: tools,
  776. }
  777. props := bazel.BazelTargetModuleProperties{
  778. Rule_class: "genrule",
  779. }
  780. // Create the BazelTargetModule.
  781. ctx.CreateBazelTargetModule(props, android.CommonAttributes{Name: m.Name()}, attrs)
  782. }
  783. var Bool = proptools.Bool
  784. var String = proptools.String
  785. //
  786. // Defaults
  787. //
  788. type Defaults struct {
  789. android.ModuleBase
  790. android.DefaultsModuleBase
  791. }
  792. func defaultsFactory() android.Module {
  793. return DefaultsFactory()
  794. }
  795. func DefaultsFactory(props ...interface{}) android.Module {
  796. module := &Defaults{}
  797. module.AddProperties(props...)
  798. module.AddProperties(
  799. &generatorProperties{},
  800. &genRuleProperties{},
  801. )
  802. android.InitDefaultsModule(module)
  803. return module
  804. }