genrule.go 35 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032
  1. // Copyright 2015 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. // A genrule module takes a list of source files ("srcs" property), an optional
  15. // list of tools ("tools" property), and a command line ("cmd" property), to
  16. // generate output files ("out" property).
  17. package genrule
  18. import (
  19. "fmt"
  20. "io"
  21. "path/filepath"
  22. "strconv"
  23. "strings"
  24. "android/soong/bazel/cquery"
  25. "github.com/google/blueprint"
  26. "github.com/google/blueprint/bootstrap"
  27. "github.com/google/blueprint/proptools"
  28. "android/soong/android"
  29. "android/soong/bazel"
  30. )
  31. func init() {
  32. RegisterGenruleBuildComponents(android.InitRegistrationContext)
  33. }
  34. // Test fixture preparer that will register most genrule build components.
  35. //
  36. // Singletons and mutators should only be added here if they are needed for a majority of genrule
  37. // module types, otherwise they should be added under a separate preparer to allow them to be
  38. // selected only when needed to reduce test execution time.
  39. //
  40. // Module types do not have much of an overhead unless they are used so this should include as many
  41. // module types as possible. The exceptions are those module types that require mutators and/or
  42. // singletons in order to function in which case they should be kept together in a separate
  43. // preparer.
  44. var PrepareForTestWithGenRuleBuildComponents = android.GroupFixturePreparers(
  45. android.FixtureRegisterWithContext(RegisterGenruleBuildComponents),
  46. )
  47. // Prepare a fixture to use all genrule module types, mutators and singletons fully.
  48. //
  49. // This should only be used by tests that want to run with as much of the build enabled as possible.
  50. var PrepareForIntegrationTestWithGenrule = android.GroupFixturePreparers(
  51. PrepareForTestWithGenRuleBuildComponents,
  52. )
  53. func RegisterGenruleBuildComponents(ctx android.RegistrationContext) {
  54. ctx.RegisterModuleType("genrule_defaults", defaultsFactory)
  55. ctx.RegisterModuleType("gensrcs", GenSrcsFactory)
  56. ctx.RegisterModuleType("genrule", GenRuleFactory)
  57. ctx.FinalDepsMutators(func(ctx android.RegisterMutatorsContext) {
  58. ctx.BottomUp("genrule_tool_deps", toolDepsMutator).Parallel()
  59. })
  60. }
  61. var (
  62. pctx = android.NewPackageContext("android/soong/genrule")
  63. // Used by gensrcs when there is more than 1 shard to merge the outputs
  64. // of each shard into a zip file.
  65. gensrcsMerge = pctx.AndroidStaticRule("gensrcsMerge", blueprint.RuleParams{
  66. Command: "${soongZip} -o ${tmpZip} @${tmpZip}.rsp && ${zipSync} -d ${genDir} ${tmpZip}",
  67. CommandDeps: []string{"${soongZip}", "${zipSync}"},
  68. Rspfile: "${tmpZip}.rsp",
  69. RspfileContent: "${zipArgs}",
  70. }, "tmpZip", "genDir", "zipArgs")
  71. )
  72. func init() {
  73. pctx.Import("android/soong/android")
  74. pctx.HostBinToolVariable("soongZip", "soong_zip")
  75. pctx.HostBinToolVariable("zipSync", "zipsync")
  76. }
  77. type SourceFileGenerator interface {
  78. GeneratedSourceFiles() android.Paths
  79. GeneratedHeaderDirs() android.Paths
  80. GeneratedDeps() android.Paths
  81. }
  82. // Alias for android.HostToolProvider
  83. // Deprecated: use android.HostToolProvider instead.
  84. type HostToolProvider interface {
  85. android.HostToolProvider
  86. }
  87. type hostToolDependencyTag struct {
  88. blueprint.BaseDependencyTag
  89. android.LicenseAnnotationToolchainDependencyTag
  90. label string
  91. }
  92. func (t hostToolDependencyTag) AllowDisabledModuleDependency(target android.Module) bool {
  93. // Allow depending on a disabled module if it's replaced by a prebuilt
  94. // counterpart. We get the prebuilt through android.PrebuiltGetPreferred in
  95. // GenerateAndroidBuildActions.
  96. return target.IsReplacedByPrebuilt()
  97. }
  98. var _ android.AllowDisabledModuleDependency = (*hostToolDependencyTag)(nil)
  99. type generatorProperties struct {
  100. // The command to run on one or more input files. Cmd supports substitution of a few variables.
  101. //
  102. // Available variables for substitution:
  103. //
  104. // $(location): the path to the first entry in tools or tool_files.
  105. // $(location <label>): the path to the tool, tool_file, input or output with name <label>. Use $(location) if <label> refers to a rule that outputs exactly one file.
  106. // $(locations <label>): the paths to the tools, tool_files, inputs or outputs with name <label>. Use $(locations) if <label> refers to a rule that outputs two or more files.
  107. // $(in): one or more input files.
  108. // $(out): a single output file.
  109. // $(depfile): a file to which dependencies will be written, if the depfile property is set to true.
  110. // $(genDir): the sandbox directory for this tool; contains $(out).
  111. // $$: a literal $
  112. Cmd *string
  113. // Enable reading a file containing dependencies in gcc format after the command completes
  114. Depfile *bool
  115. // name of the modules (if any) that produces the host executable. Leave empty for
  116. // prebuilts or scripts that do not need a module to build them.
  117. Tools []string
  118. // Local file that is used as the tool
  119. Tool_files []string `android:"path"`
  120. // List of directories to export generated headers from
  121. Export_include_dirs []string
  122. // list of input files
  123. Srcs []string `android:"path,arch_variant"`
  124. // input files to exclude
  125. Exclude_srcs []string `android:"path,arch_variant"`
  126. }
  127. type Module struct {
  128. android.ModuleBase
  129. android.DefaultableModuleBase
  130. android.BazelModuleBase
  131. android.ApexModuleBase
  132. // For other packages to make their own genrules with extra
  133. // properties
  134. Extra interface{}
  135. // CmdModifier can be set by wrappers around genrule to modify the command, for example to
  136. // prefix environment variables to it.
  137. CmdModifier func(ctx android.ModuleContext, cmd string) string
  138. android.ImageInterface
  139. properties generatorProperties
  140. // For the different tasks that genrule and gensrc generate. genrule will
  141. // generate 1 task, and gensrc will generate 1 or more tasks based on the
  142. // number of shards the input files are sharded into.
  143. taskGenerator taskFunc
  144. rule blueprint.Rule
  145. rawCommands []string
  146. exportedIncludeDirs android.Paths
  147. outputFiles android.Paths
  148. outputDeps android.Paths
  149. subName string
  150. subDir string
  151. // Collect the module directory for IDE info in java/jdeps.go.
  152. modulePaths []string
  153. }
  154. var _ android.MixedBuildBuildable = (*Module)(nil)
  155. type taskFunc func(ctx android.ModuleContext, rawCommand string, srcFiles android.Paths) []generateTask
  156. type generateTask struct {
  157. in android.Paths
  158. out android.WritablePaths
  159. depFile android.WritablePath
  160. copyTo android.WritablePaths // For gensrcs to set on gensrcsMerge rule.
  161. genDir android.WritablePath
  162. extraTools android.Paths // dependencies on tools used by the generator
  163. cmd string
  164. // For gensrsc sharding.
  165. shard int
  166. shards int
  167. }
  168. func (g *Module) GeneratedSourceFiles() android.Paths {
  169. return g.outputFiles
  170. }
  171. func (g *Module) Srcs() android.Paths {
  172. return append(android.Paths{}, g.outputFiles...)
  173. }
  174. func (g *Module) GeneratedHeaderDirs() android.Paths {
  175. return g.exportedIncludeDirs
  176. }
  177. func (g *Module) GeneratedDeps() android.Paths {
  178. return g.outputDeps
  179. }
  180. func (g *Module) OutputFiles(tag string) (android.Paths, error) {
  181. if tag == "" {
  182. return append(android.Paths{}, g.outputFiles...), nil
  183. }
  184. // otherwise, tag should match one of outputs
  185. for _, outputFile := range g.outputFiles {
  186. if outputFile.Rel() == tag {
  187. return android.Paths{outputFile}, nil
  188. }
  189. }
  190. return nil, fmt.Errorf("unsupported module reference tag %q", tag)
  191. }
  192. var _ android.SourceFileProducer = (*Module)(nil)
  193. var _ android.OutputFileProducer = (*Module)(nil)
  194. func toolDepsMutator(ctx android.BottomUpMutatorContext) {
  195. if g, ok := ctx.Module().(*Module); ok {
  196. for _, tool := range g.properties.Tools {
  197. tag := hostToolDependencyTag{label: tool}
  198. if m := android.SrcIsModule(tool); m != "" {
  199. tool = m
  200. }
  201. ctx.AddFarVariationDependencies(ctx.Config().BuildOSTarget.Variations(), tag, tool)
  202. }
  203. }
  204. }
  205. func (g *Module) ProcessBazelQueryResponse(ctx android.ModuleContext) {
  206. g.generateCommonBuildActions(ctx)
  207. label := g.GetBazelLabel(ctx, g)
  208. bazelCtx := ctx.Config().BazelContext
  209. filePaths, err := bazelCtx.GetOutputFiles(label, android.GetConfigKey(ctx))
  210. if err != nil {
  211. ctx.ModuleErrorf(err.Error())
  212. return
  213. }
  214. var bazelOutputFiles android.Paths
  215. exportIncludeDirs := map[string]bool{}
  216. for _, bazelOutputFile := range filePaths {
  217. bazelOutputFiles = append(bazelOutputFiles, android.PathForBazelOutRelative(ctx, ctx.ModuleDir(), bazelOutputFile))
  218. exportIncludeDirs[filepath.Dir(bazelOutputFile)] = true
  219. }
  220. g.outputFiles = bazelOutputFiles
  221. g.outputDeps = bazelOutputFiles
  222. for includePath, _ := range exportIncludeDirs {
  223. g.exportedIncludeDirs = append(g.exportedIncludeDirs, android.PathForBazelOut(ctx, includePath))
  224. }
  225. }
  226. // generateCommonBuildActions contains build action generation logic
  227. // common to both the mixed build case and the legacy case of genrule processing.
  228. // To fully support genrule in mixed builds, the contents of this function should
  229. // approach zero; there should be no genrule action registration done directly
  230. // by Soong logic in the mixed-build case.
  231. func (g *Module) generateCommonBuildActions(ctx android.ModuleContext) {
  232. g.subName = ctx.ModuleSubDir()
  233. // Collect the module directory for IDE info in java/jdeps.go.
  234. g.modulePaths = append(g.modulePaths, ctx.ModuleDir())
  235. if len(g.properties.Export_include_dirs) > 0 {
  236. for _, dir := range g.properties.Export_include_dirs {
  237. g.exportedIncludeDirs = append(g.exportedIncludeDirs,
  238. android.PathForModuleGen(ctx, g.subDir, ctx.ModuleDir(), dir))
  239. }
  240. } else {
  241. g.exportedIncludeDirs = append(g.exportedIncludeDirs, android.PathForModuleGen(ctx, g.subDir))
  242. }
  243. locationLabels := map[string]location{}
  244. firstLabel := ""
  245. addLocationLabel := func(label string, loc location) {
  246. if firstLabel == "" {
  247. firstLabel = label
  248. }
  249. if _, exists := locationLabels[label]; !exists {
  250. locationLabels[label] = loc
  251. } else {
  252. ctx.ModuleErrorf("multiple locations for label %q: %q and %q (do you have duplicate srcs entries?)",
  253. label, locationLabels[label], loc)
  254. }
  255. }
  256. var tools android.Paths
  257. var packagedTools []android.PackagingSpec
  258. if len(g.properties.Tools) > 0 {
  259. seenTools := make(map[string]bool)
  260. ctx.VisitDirectDepsBlueprint(func(module blueprint.Module) {
  261. switch tag := ctx.OtherModuleDependencyTag(module).(type) {
  262. case hostToolDependencyTag:
  263. tool := ctx.OtherModuleName(module)
  264. if m, ok := module.(android.Module); ok {
  265. // Necessary to retrieve any prebuilt replacement for the tool, since
  266. // toolDepsMutator runs too late for the prebuilt mutators to have
  267. // replaced the dependency.
  268. module = android.PrebuiltGetPreferred(ctx, m)
  269. }
  270. switch t := module.(type) {
  271. case android.HostToolProvider:
  272. // A HostToolProvider provides the path to a tool, which will be copied
  273. // into the sandbox.
  274. if !t.(android.Module).Enabled() {
  275. if ctx.Config().AllowMissingDependencies() {
  276. ctx.AddMissingDependencies([]string{tool})
  277. } else {
  278. ctx.ModuleErrorf("depends on disabled module %q", tool)
  279. }
  280. return
  281. }
  282. path := t.HostToolPath()
  283. if !path.Valid() {
  284. ctx.ModuleErrorf("host tool %q missing output file", tool)
  285. return
  286. }
  287. if specs := t.TransitivePackagingSpecs(); specs != nil {
  288. // If the HostToolProvider has PackgingSpecs, which are definitions of the
  289. // required relative locations of the tool and its dependencies, use those
  290. // instead. They will be copied to those relative locations in the sbox
  291. // sandbox.
  292. packagedTools = append(packagedTools, specs...)
  293. // Assume that the first PackagingSpec of the module is the tool.
  294. addLocationLabel(tag.label, packagedToolLocation{specs[0]})
  295. } else {
  296. tools = append(tools, path.Path())
  297. addLocationLabel(tag.label, toolLocation{android.Paths{path.Path()}})
  298. }
  299. case bootstrap.GoBinaryTool:
  300. // A GoBinaryTool provides the install path to a tool, which will be copied.
  301. p := android.PathForGoBinary(ctx, t)
  302. tools = append(tools, p)
  303. addLocationLabel(tag.label, toolLocation{android.Paths{p}})
  304. default:
  305. ctx.ModuleErrorf("%q is not a host tool provider", tool)
  306. return
  307. }
  308. seenTools[tag.label] = true
  309. }
  310. })
  311. // If AllowMissingDependencies is enabled, the build will not have stopped when
  312. // AddFarVariationDependencies was called on a missing tool, which will result in nonsensical
  313. // "cmd: unknown location label ..." errors later. Add a placeholder file to the local label.
  314. // The command that uses this placeholder file will never be executed because the rule will be
  315. // replaced with an android.Error rule reporting the missing dependencies.
  316. if ctx.Config().AllowMissingDependencies() {
  317. for _, tool := range g.properties.Tools {
  318. if !seenTools[tool] {
  319. addLocationLabel(tool, errorLocation{"***missing tool " + tool + "***"})
  320. }
  321. }
  322. }
  323. }
  324. if ctx.Failed() {
  325. return
  326. }
  327. for _, toolFile := range g.properties.Tool_files {
  328. paths := android.PathsForModuleSrc(ctx, []string{toolFile})
  329. tools = append(tools, paths...)
  330. addLocationLabel(toolFile, toolLocation{paths})
  331. }
  332. includeDirInPaths := ctx.DeviceConfig().BuildBrokenInputDir(g.Name())
  333. var srcFiles android.Paths
  334. for _, in := range g.properties.Srcs {
  335. paths, missingDeps := android.PathsAndMissingDepsRelativeToModuleSourceDir(android.SourceInput{
  336. Context: ctx, Paths: []string{in}, ExcludePaths: g.properties.Exclude_srcs, IncludeDirs: includeDirInPaths,
  337. })
  338. if len(missingDeps) > 0 {
  339. if !ctx.Config().AllowMissingDependencies() {
  340. panic(fmt.Errorf("should never get here, the missing dependencies %q should have been reported in DepsMutator",
  341. missingDeps))
  342. }
  343. // If AllowMissingDependencies is enabled, the build will not have stopped when
  344. // the dependency was added on a missing SourceFileProducer module, which will result in nonsensical
  345. // "cmd: label ":..." has no files" errors later. Add a placeholder file to the local label.
  346. // The command that uses this placeholder file will never be executed because the rule will be
  347. // replaced with an android.Error rule reporting the missing dependencies.
  348. ctx.AddMissingDependencies(missingDeps)
  349. addLocationLabel(in, errorLocation{"***missing srcs " + in + "***"})
  350. } else {
  351. srcFiles = append(srcFiles, paths...)
  352. addLocationLabel(in, inputLocation{paths})
  353. }
  354. }
  355. var copyFrom android.Paths
  356. var outputFiles android.WritablePaths
  357. var zipArgs strings.Builder
  358. cmd := String(g.properties.Cmd)
  359. if g.CmdModifier != nil {
  360. cmd = g.CmdModifier(ctx, cmd)
  361. }
  362. // Generate tasks, either from genrule or gensrcs.
  363. for _, task := range g.taskGenerator(ctx, cmd, srcFiles) {
  364. if len(task.out) == 0 {
  365. ctx.ModuleErrorf("must have at least one output file")
  366. return
  367. }
  368. // Pick a unique path outside the task.genDir for the sbox manifest textproto,
  369. // a unique rule name, and the user-visible description.
  370. manifestName := "genrule.sbox.textproto"
  371. desc := "generate"
  372. name := "generator"
  373. if task.shards > 0 {
  374. manifestName = "genrule_" + strconv.Itoa(task.shard) + ".sbox.textproto"
  375. desc += " " + strconv.Itoa(task.shard)
  376. name += strconv.Itoa(task.shard)
  377. } else if len(task.out) == 1 {
  378. desc += " " + task.out[0].Base()
  379. }
  380. manifestPath := android.PathForModuleOut(ctx, manifestName)
  381. // Use a RuleBuilder to create a rule that runs the command inside an sbox sandbox.
  382. rule := getSandboxedRuleBuilder(ctx, android.NewRuleBuilder(pctx, ctx).Sbox(task.genDir, manifestPath))
  383. cmd := rule.Command()
  384. for _, out := range task.out {
  385. addLocationLabel(out.Rel(), outputLocation{out})
  386. }
  387. referencedDepfile := false
  388. rawCommand, err := android.Expand(task.cmd, func(name string) (string, error) {
  389. // report the error directly without returning an error to android.Expand to catch multiple errors in a
  390. // single run
  391. reportError := func(fmt string, args ...interface{}) (string, error) {
  392. ctx.PropertyErrorf("cmd", fmt, args...)
  393. return "SOONG_ERROR", nil
  394. }
  395. // Apply shell escape to each cases to prevent source file paths containing $ from being evaluated in shell
  396. switch name {
  397. case "location":
  398. if len(g.properties.Tools) == 0 && len(g.properties.Tool_files) == 0 {
  399. return reportError("at least one `tools` or `tool_files` is required if $(location) is used")
  400. }
  401. loc := locationLabels[firstLabel]
  402. paths := loc.Paths(cmd)
  403. if len(paths) == 0 {
  404. return reportError("default label %q has no files", firstLabel)
  405. } else if len(paths) > 1 {
  406. return reportError("default label %q has multiple files, use $(locations %s) to reference it",
  407. firstLabel, firstLabel)
  408. }
  409. return proptools.ShellEscape(paths[0]), nil
  410. case "in":
  411. return strings.Join(proptools.ShellEscapeList(cmd.PathsForInputs(srcFiles)), " "), nil
  412. case "out":
  413. var sandboxOuts []string
  414. for _, out := range task.out {
  415. sandboxOuts = append(sandboxOuts, cmd.PathForOutput(out))
  416. }
  417. return strings.Join(proptools.ShellEscapeList(sandboxOuts), " "), nil
  418. case "depfile":
  419. referencedDepfile = true
  420. if !Bool(g.properties.Depfile) {
  421. return reportError("$(depfile) used without depfile property")
  422. }
  423. return "__SBOX_DEPFILE__", nil
  424. case "genDir":
  425. return proptools.ShellEscape(cmd.PathForOutput(task.genDir)), nil
  426. default:
  427. if strings.HasPrefix(name, "location ") {
  428. label := strings.TrimSpace(strings.TrimPrefix(name, "location "))
  429. if loc, ok := locationLabels[label]; ok {
  430. paths := loc.Paths(cmd)
  431. if len(paths) == 0 {
  432. return reportError("label %q has no files", label)
  433. } else if len(paths) > 1 {
  434. return reportError("label %q has multiple files, use $(locations %s) to reference it",
  435. label, label)
  436. }
  437. return proptools.ShellEscape(paths[0]), nil
  438. } else {
  439. return reportError("unknown location label %q is not in srcs, out, tools or tool_files.", label)
  440. }
  441. } else if strings.HasPrefix(name, "locations ") {
  442. label := strings.TrimSpace(strings.TrimPrefix(name, "locations "))
  443. if loc, ok := locationLabels[label]; ok {
  444. paths := loc.Paths(cmd)
  445. if len(paths) == 0 {
  446. return reportError("label %q has no files", label)
  447. }
  448. return proptools.ShellEscape(strings.Join(paths, " ")), nil
  449. } else {
  450. return reportError("unknown locations label %q is not in srcs, out, tools or tool_files.", label)
  451. }
  452. } else {
  453. return reportError("unknown variable '$(%s)'", name)
  454. }
  455. }
  456. })
  457. if err != nil {
  458. ctx.PropertyErrorf("cmd", "%s", err.Error())
  459. return
  460. }
  461. if Bool(g.properties.Depfile) && !referencedDepfile {
  462. ctx.PropertyErrorf("cmd", "specified depfile=true but did not include a reference to '${depfile}' in cmd")
  463. return
  464. }
  465. g.rawCommands = append(g.rawCommands, rawCommand)
  466. cmd.Text(rawCommand)
  467. cmd.ImplicitOutputs(task.out)
  468. cmd.Implicits(task.in)
  469. cmd.ImplicitTools(tools)
  470. cmd.ImplicitTools(task.extraTools)
  471. cmd.ImplicitPackagedTools(packagedTools)
  472. if Bool(g.properties.Depfile) {
  473. cmd.ImplicitDepFile(task.depFile)
  474. }
  475. // Create the rule to run the genrule command inside sbox.
  476. rule.Build(name, desc)
  477. if len(task.copyTo) > 0 {
  478. // If copyTo is set, multiple shards need to be copied into a single directory.
  479. // task.out contains the per-shard paths, and copyTo contains the corresponding
  480. // final path. The files need to be copied into the final directory by a
  481. // single rule so it can remove the directory before it starts to ensure no
  482. // old files remain. zipsync already does this, so build up zipArgs that
  483. // zip all the per-shard directories into a single zip.
  484. outputFiles = append(outputFiles, task.copyTo...)
  485. copyFrom = append(copyFrom, task.out.Paths()...)
  486. zipArgs.WriteString(" -C " + task.genDir.String())
  487. zipArgs.WriteString(android.JoinWithPrefix(task.out.Strings(), " -f "))
  488. } else {
  489. outputFiles = append(outputFiles, task.out...)
  490. }
  491. }
  492. if len(copyFrom) > 0 {
  493. // Create a rule that zips all the per-shard directories into a single zip and then
  494. // uses zipsync to unzip it into the final directory.
  495. ctx.Build(pctx, android.BuildParams{
  496. Rule: gensrcsMerge,
  497. Implicits: copyFrom,
  498. Outputs: outputFiles,
  499. Description: "merge shards",
  500. Args: map[string]string{
  501. "zipArgs": zipArgs.String(),
  502. "tmpZip": android.PathForModuleGen(ctx, g.subDir+".zip").String(),
  503. "genDir": android.PathForModuleGen(ctx, g.subDir).String(),
  504. },
  505. })
  506. }
  507. g.outputFiles = outputFiles.Paths()
  508. }
  509. func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) {
  510. // Allowlist genrule to use depfile until we have a solution to remove it.
  511. // TODO(b/235582219): Remove allowlist for genrule
  512. if Bool(g.properties.Depfile) {
  513. // TODO(b/283852474): Checking the GenruleSandboxing flag is temporary in
  514. // order to pass the presubmit before internal master is updated.
  515. if ctx.DeviceConfig().GenruleSandboxing() && !DepfileAllowSet[g.Name()] {
  516. ctx.PropertyErrorf(
  517. "depfile",
  518. "Deprecated to ensure the module type is convertible to Bazel. "+
  519. "Try specifying the dependencies explicitly so that there is no need to use depfile. "+
  520. "If not possible, the escape hatch is to add the module to allowlists.go to bypass the error.")
  521. }
  522. }
  523. g.generateCommonBuildActions(ctx)
  524. // For <= 6 outputs, just embed those directly in the users. Right now, that covers >90% of
  525. // the genrules on AOSP. That will make things simpler to look at the graph in the common
  526. // case. For larger sets of outputs, inject a phony target in between to limit ninja file
  527. // growth.
  528. if len(g.outputFiles) <= 6 {
  529. g.outputDeps = g.outputFiles
  530. } else {
  531. phonyFile := android.PathForModuleGen(ctx, "genrule-phony")
  532. ctx.Build(pctx, android.BuildParams{
  533. Rule: blueprint.Phony,
  534. Output: phonyFile,
  535. Inputs: g.outputFiles,
  536. })
  537. g.outputDeps = android.Paths{phonyFile}
  538. }
  539. }
  540. func (g *Module) QueueBazelCall(ctx android.BaseModuleContext) {
  541. bazelCtx := ctx.Config().BazelContext
  542. bazelCtx.QueueBazelRequest(g.GetBazelLabel(ctx, g), cquery.GetOutputFiles, android.GetConfigKey(ctx))
  543. }
  544. func (g *Module) IsMixedBuildSupported(ctx android.BaseModuleContext) bool {
  545. return true
  546. }
  547. // Collect information for opening IDE project files in java/jdeps.go.
  548. func (g *Module) IDEInfo(dpInfo *android.IdeInfo) {
  549. dpInfo.Srcs = append(dpInfo.Srcs, g.Srcs().Strings()...)
  550. for _, src := range g.properties.Srcs {
  551. if strings.HasPrefix(src, ":") {
  552. src = strings.Trim(src, ":")
  553. dpInfo.Deps = append(dpInfo.Deps, src)
  554. }
  555. }
  556. dpInfo.Paths = append(dpInfo.Paths, g.modulePaths...)
  557. }
  558. func (g *Module) AndroidMk() android.AndroidMkData {
  559. return android.AndroidMkData{
  560. Class: "ETC",
  561. OutputFile: android.OptionalPathForPath(g.outputFiles[0]),
  562. SubName: g.subName,
  563. Extra: []android.AndroidMkExtraFunc{
  564. func(w io.Writer, outputFile android.Path) {
  565. fmt.Fprintln(w, "LOCAL_UNINSTALLABLE_MODULE := true")
  566. },
  567. },
  568. Custom: func(w io.Writer, name, prefix, moduleDir string, data android.AndroidMkData) {
  569. android.WriteAndroidMkData(w, data)
  570. if data.SubName != "" {
  571. fmt.Fprintln(w, ".PHONY:", name)
  572. fmt.Fprintln(w, name, ":", name+g.subName)
  573. }
  574. },
  575. }
  576. }
  577. var _ android.ApexModule = (*Module)(nil)
  578. // Implements android.ApexModule
  579. func (g *Module) ShouldSupportSdkVersion(ctx android.BaseModuleContext,
  580. sdkVersion android.ApiLevel) error {
  581. // Because generated outputs are checked by client modules(e.g. cc_library, ...)
  582. // we can safely ignore the check here.
  583. return nil
  584. }
  585. func generatorFactory(taskGenerator taskFunc, props ...interface{}) *Module {
  586. module := &Module{
  587. taskGenerator: taskGenerator,
  588. }
  589. module.AddProperties(props...)
  590. module.AddProperties(&module.properties)
  591. module.ImageInterface = noopImageInterface{}
  592. return module
  593. }
  594. type noopImageInterface struct{}
  595. func (x noopImageInterface) ImageMutatorBegin(android.BaseModuleContext) {}
  596. func (x noopImageInterface) CoreVariantNeeded(android.BaseModuleContext) bool { return false }
  597. func (x noopImageInterface) RamdiskVariantNeeded(android.BaseModuleContext) bool { return false }
  598. func (x noopImageInterface) VendorRamdiskVariantNeeded(android.BaseModuleContext) bool { return false }
  599. func (x noopImageInterface) DebugRamdiskVariantNeeded(android.BaseModuleContext) bool { return false }
  600. func (x noopImageInterface) RecoveryVariantNeeded(android.BaseModuleContext) bool { return false }
  601. func (x noopImageInterface) ExtraImageVariations(ctx android.BaseModuleContext) []string { return nil }
  602. func (x noopImageInterface) SetImageVariation(ctx android.BaseModuleContext, variation string, module android.Module) {
  603. }
  604. func NewGenSrcs() *Module {
  605. properties := &genSrcsProperties{}
  606. // finalSubDir is the name of the subdirectory that output files will be generated into.
  607. // It is used so that per-shard directories can be placed alongside it an then finally
  608. // merged into it.
  609. const finalSubDir = "gensrcs"
  610. taskGenerator := func(ctx android.ModuleContext, rawCommand string, srcFiles android.Paths) []generateTask {
  611. shardSize := defaultShardSize
  612. if s := properties.Shard_size; s != nil {
  613. shardSize = int(*s)
  614. }
  615. // gensrcs rules can easily hit command line limits by repeating the command for
  616. // every input file. Shard the input files into groups.
  617. shards := android.ShardPaths(srcFiles, shardSize)
  618. var generateTasks []generateTask
  619. for i, shard := range shards {
  620. var commands []string
  621. var outFiles android.WritablePaths
  622. var commandDepFiles []string
  623. var copyTo android.WritablePaths
  624. // When sharding is enabled (i.e. len(shards) > 1), the sbox rules for each
  625. // shard will be write to their own directories and then be merged together
  626. // into finalSubDir. If sharding is not enabled (i.e. len(shards) == 1),
  627. // the sbox rule will write directly to finalSubDir.
  628. genSubDir := finalSubDir
  629. if len(shards) > 1 {
  630. genSubDir = strconv.Itoa(i)
  631. }
  632. genDir := android.PathForModuleGen(ctx, genSubDir)
  633. // TODO(ccross): this RuleBuilder is a hack to be able to call
  634. // rule.Command().PathForOutput. Replace this with passing the rule into the
  635. // generator.
  636. rule := getSandboxedRuleBuilder(ctx, android.NewRuleBuilder(pctx, ctx).Sbox(genDir, nil))
  637. for _, in := range shard {
  638. outFile := android.GenPathWithExt(ctx, finalSubDir, in, String(properties.Output_extension))
  639. // If sharding is enabled, then outFile is the path to the output file in
  640. // the shard directory, and copyTo is the path to the output file in the
  641. // final directory.
  642. if len(shards) > 1 {
  643. shardFile := android.GenPathWithExt(ctx, genSubDir, in, String(properties.Output_extension))
  644. copyTo = append(copyTo, outFile)
  645. outFile = shardFile
  646. }
  647. outFiles = append(outFiles, outFile)
  648. // pre-expand the command line to replace $in and $out with references to
  649. // a single input and output file.
  650. command, err := android.Expand(rawCommand, func(name string) (string, error) {
  651. switch name {
  652. case "in":
  653. return in.String(), nil
  654. case "out":
  655. return rule.Command().PathForOutput(outFile), nil
  656. case "depfile":
  657. // Generate a depfile for each output file. Store the list for
  658. // later in order to combine them all into a single depfile.
  659. depFile := rule.Command().PathForOutput(outFile.ReplaceExtension(ctx, "d"))
  660. commandDepFiles = append(commandDepFiles, depFile)
  661. return depFile, nil
  662. default:
  663. return "$(" + name + ")", nil
  664. }
  665. })
  666. if err != nil {
  667. ctx.PropertyErrorf("cmd", err.Error())
  668. }
  669. // escape the command in case for example it contains '#', an odd number of '"', etc
  670. command = fmt.Sprintf("bash -c %v", proptools.ShellEscape(command))
  671. commands = append(commands, command)
  672. }
  673. fullCommand := strings.Join(commands, " && ")
  674. var outputDepfile android.WritablePath
  675. var extraTools android.Paths
  676. if len(commandDepFiles) > 0 {
  677. // Each command wrote to a depfile, but ninja can only handle one
  678. // depfile per rule. Use the dep_fixer tool at the end of the
  679. // command to combine all the depfiles into a single output depfile.
  680. outputDepfile = android.PathForModuleGen(ctx, genSubDir, "gensrcs.d")
  681. depFixerTool := ctx.Config().HostToolPath(ctx, "dep_fixer")
  682. fullCommand += fmt.Sprintf(" && %s -o $(depfile) %s",
  683. rule.Command().PathForTool(depFixerTool),
  684. strings.Join(commandDepFiles, " "))
  685. extraTools = append(extraTools, depFixerTool)
  686. }
  687. generateTasks = append(generateTasks, generateTask{
  688. in: shard,
  689. out: outFiles,
  690. depFile: outputDepfile,
  691. copyTo: copyTo,
  692. genDir: genDir,
  693. cmd: fullCommand,
  694. shard: i,
  695. shards: len(shards),
  696. extraTools: extraTools,
  697. })
  698. }
  699. return generateTasks
  700. }
  701. g := generatorFactory(taskGenerator, properties)
  702. g.subDir = finalSubDir
  703. return g
  704. }
  705. func GenSrcsFactory() android.Module {
  706. m := NewGenSrcs()
  707. android.InitAndroidModule(m)
  708. android.InitBazelModule(m)
  709. return m
  710. }
  711. type genSrcsProperties struct {
  712. // extension that will be substituted for each output file
  713. Output_extension *string
  714. // maximum number of files that will be passed on a single command line.
  715. Shard_size *int64
  716. }
  717. type bazelGensrcsAttributes struct {
  718. Srcs bazel.LabelListAttribute
  719. Output_extension *string
  720. Tools bazel.LabelListAttribute
  721. Cmd string
  722. }
  723. const defaultShardSize = 50
  724. func NewGenRule() *Module {
  725. properties := &genRuleProperties{}
  726. taskGenerator := func(ctx android.ModuleContext, rawCommand string, srcFiles android.Paths) []generateTask {
  727. outs := make(android.WritablePaths, len(properties.Out))
  728. var depFile android.WritablePath
  729. for i, out := range properties.Out {
  730. outPath := android.PathForModuleGen(ctx, out)
  731. if i == 0 {
  732. depFile = outPath.ReplaceExtension(ctx, "d")
  733. }
  734. outs[i] = outPath
  735. }
  736. return []generateTask{{
  737. in: srcFiles,
  738. out: outs,
  739. depFile: depFile,
  740. genDir: android.PathForModuleGen(ctx),
  741. cmd: rawCommand,
  742. }}
  743. }
  744. return generatorFactory(taskGenerator, properties)
  745. }
  746. func GenRuleFactory() android.Module {
  747. m := NewGenRule()
  748. android.InitAndroidModule(m)
  749. android.InitDefaultableModule(m)
  750. android.InitBazelModule(m)
  751. return m
  752. }
  753. type genRuleProperties struct {
  754. // names of the output files that will be generated
  755. Out []string
  756. }
  757. type bazelGenruleAttributes struct {
  758. Srcs bazel.LabelListAttribute
  759. Outs []string
  760. Tools bazel.LabelListAttribute
  761. Cmd string
  762. }
  763. // ConvertWithBp2build converts a Soong module -> Bazel target.
  764. func (m *Module) ConvertWithBp2build(ctx android.TopDownMutatorContext) {
  765. // Bazel only has the "tools" attribute.
  766. tools_prop := android.BazelLabelForModuleDeps(ctx, m.properties.Tools)
  767. tool_files_prop := android.BazelLabelForModuleSrc(ctx, m.properties.Tool_files)
  768. tools_prop.Append(tool_files_prop)
  769. tools := bazel.MakeLabelListAttribute(tools_prop)
  770. srcs := bazel.LabelListAttribute{}
  771. srcs_labels := bazel.LabelList{}
  772. // Only cc_genrule is arch specific
  773. if ctx.ModuleType() == "cc_genrule" {
  774. for axis, configToProps := range m.GetArchVariantProperties(ctx, &generatorProperties{}) {
  775. for config, props := range configToProps {
  776. if props, ok := props.(*generatorProperties); ok {
  777. labels := android.BazelLabelForModuleSrcExcludes(ctx, props.Srcs, props.Exclude_srcs)
  778. srcs_labels.Append(labels)
  779. srcs.SetSelectValue(axis, config, labels)
  780. }
  781. }
  782. }
  783. } else {
  784. srcs_labels = android.BazelLabelForModuleSrcExcludes(ctx, m.properties.Srcs, m.properties.Exclude_srcs)
  785. srcs = bazel.MakeLabelListAttribute(srcs_labels)
  786. }
  787. var allReplacements bazel.LabelList
  788. allReplacements.Append(tools.Value)
  789. allReplacements.Append(bazel.FirstUniqueBazelLabelList(srcs_labels))
  790. // Replace in and out variables with $< and $@
  791. var cmd string
  792. if m.properties.Cmd != nil {
  793. if ctx.ModuleType() == "gensrcs" {
  794. cmd = strings.ReplaceAll(*m.properties.Cmd, "$(in)", "$(SRC)")
  795. cmd = strings.ReplaceAll(cmd, "$(out)", "$(OUT)")
  796. } else {
  797. cmd = strings.Replace(*m.properties.Cmd, "$(in)", "$(SRCS)", -1)
  798. cmd = strings.Replace(cmd, "$(out)", "$(OUTS)", -1)
  799. }
  800. cmd = strings.Replace(cmd, "$(genDir)", "$(RULEDIR)", -1)
  801. if len(tools.Value.Includes) > 0 {
  802. cmd = strings.Replace(cmd, "$(location)", fmt.Sprintf("$(location %s)", tools.Value.Includes[0].Label), -1)
  803. cmd = strings.Replace(cmd, "$(locations)", fmt.Sprintf("$(locations %s)", tools.Value.Includes[0].Label), -1)
  804. }
  805. for _, l := range allReplacements.Includes {
  806. bpLoc := fmt.Sprintf("$(location %s)", l.OriginalModuleName)
  807. bpLocs := fmt.Sprintf("$(locations %s)", l.OriginalModuleName)
  808. bazelLoc := fmt.Sprintf("$(location %s)", l.Label)
  809. bazelLocs := fmt.Sprintf("$(locations %s)", l.Label)
  810. cmd = strings.Replace(cmd, bpLoc, bazelLoc, -1)
  811. cmd = strings.Replace(cmd, bpLocs, bazelLocs, -1)
  812. }
  813. }
  814. tags := android.ApexAvailableTagsWithoutTestApexes(ctx, m)
  815. if ctx.ModuleType() == "gensrcs" {
  816. // The Output_extension prop is not in an immediately accessible field
  817. // in the Module struct, so use GetProperties and cast it
  818. // to the known struct prop.
  819. var outputExtension *string
  820. for _, propIntf := range m.GetProperties() {
  821. if props, ok := propIntf.(*genSrcsProperties); ok {
  822. outputExtension = props.Output_extension
  823. break
  824. }
  825. }
  826. props := bazel.BazelTargetModuleProperties{
  827. Rule_class: "gensrcs",
  828. Bzl_load_location: "//build/bazel/rules:gensrcs.bzl",
  829. }
  830. attrs := &bazelGensrcsAttributes{
  831. Srcs: srcs,
  832. Output_extension: outputExtension,
  833. Cmd: cmd,
  834. Tools: tools,
  835. }
  836. ctx.CreateBazelTargetModule(props, android.CommonAttributes{
  837. Name: m.Name(),
  838. Tags: tags,
  839. }, attrs)
  840. } else {
  841. // The Out prop is not in an immediately accessible field
  842. // in the Module struct, so use GetProperties and cast it
  843. // to the known struct prop.
  844. var outs []string
  845. for _, propIntf := range m.GetProperties() {
  846. if props, ok := propIntf.(*genRuleProperties); ok {
  847. outs = props.Out
  848. break
  849. }
  850. }
  851. attrs := &bazelGenruleAttributes{
  852. Srcs: srcs,
  853. Outs: outs,
  854. Cmd: cmd,
  855. Tools: tools,
  856. }
  857. props := bazel.BazelTargetModuleProperties{
  858. Rule_class: "genrule",
  859. }
  860. ctx.CreateBazelTargetModule(props, android.CommonAttributes{
  861. Name: m.Name(),
  862. Tags: tags,
  863. }, attrs)
  864. }
  865. }
  866. var Bool = proptools.Bool
  867. var String = proptools.String
  868. // Defaults
  869. type Defaults struct {
  870. android.ModuleBase
  871. android.DefaultsModuleBase
  872. }
  873. func defaultsFactory() android.Module {
  874. return DefaultsFactory()
  875. }
  876. func DefaultsFactory(props ...interface{}) android.Module {
  877. module := &Defaults{}
  878. module.AddProperties(props...)
  879. module.AddProperties(
  880. &generatorProperties{},
  881. &genRuleProperties{},
  882. )
  883. android.InitDefaultsModule(module)
  884. return module
  885. }
  886. func getSandboxedRuleBuilder(ctx android.ModuleContext, r *android.RuleBuilder) *android.RuleBuilder {
  887. if !ctx.DeviceConfig().GenruleSandboxing() || SandboxingDenyPathSet[ctx.ModuleDir()] ||
  888. SandboxingDenyModuleSet[ctx.ModuleName()] {
  889. return r.SandboxTools()
  890. }
  891. return r.SandboxInputs()
  892. }