genrule.go 37 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087
  1. // Copyright 2015 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. // A genrule module takes a list of source files ("srcs" property), an optional
  15. // list of tools ("tools" property), and a command line ("cmd" property), to
  16. // generate output files ("out" property).
  17. package genrule
  18. import (
  19. "fmt"
  20. "io"
  21. "path/filepath"
  22. "strconv"
  23. "strings"
  24. "sync"
  25. "android/soong/bazel/cquery"
  26. "github.com/google/blueprint"
  27. "github.com/google/blueprint/bootstrap"
  28. "github.com/google/blueprint/proptools"
  29. "android/soong/android"
  30. "android/soong/bazel"
  31. )
  32. func init() {
  33. RegisterGenruleBuildComponents(android.InitRegistrationContext)
  34. }
  35. // Test fixture preparer that will register most genrule build components.
  36. //
  37. // Singletons and mutators should only be added here if they are needed for a majority of genrule
  38. // module types, otherwise they should be added under a separate preparer to allow them to be
  39. // selected only when needed to reduce test execution time.
  40. //
  41. // Module types do not have much of an overhead unless they are used so this should include as many
  42. // module types as possible. The exceptions are those module types that require mutators and/or
  43. // singletons in order to function in which case they should be kept together in a separate
  44. // preparer.
  45. var PrepareForTestWithGenRuleBuildComponents = android.GroupFixturePreparers(
  46. android.FixtureRegisterWithContext(RegisterGenruleBuildComponents),
  47. )
  48. // Prepare a fixture to use all genrule module types, mutators and singletons fully.
  49. //
  50. // This should only be used by tests that want to run with as much of the build enabled as possible.
  51. var PrepareForIntegrationTestWithGenrule = android.GroupFixturePreparers(
  52. PrepareForTestWithGenRuleBuildComponents,
  53. )
  54. var DepfileAllowSet map[string]bool
  55. var SandboxingDenyModuleSet map[string]bool
  56. var SandboxingDenyPathSet map[string]bool
  57. var SandboxingDenyModuleSetLock sync.Mutex
  58. var DepfileAllowSetLock sync.Mutex
  59. func RegisterGenruleBuildComponents(ctx android.RegistrationContext) {
  60. ctx.RegisterModuleType("genrule_defaults", defaultsFactory)
  61. ctx.RegisterModuleType("gensrcs", GenSrcsFactory)
  62. ctx.RegisterModuleType("genrule", GenRuleFactory)
  63. ctx.FinalDepsMutators(func(ctx android.RegisterMutatorsContext) {
  64. ctx.BottomUp("genrule_tool_deps", toolDepsMutator).Parallel()
  65. })
  66. }
  67. var (
  68. pctx = android.NewPackageContext("android/soong/genrule")
  69. // Used by gensrcs when there is more than 1 shard to merge the outputs
  70. // of each shard into a zip file.
  71. gensrcsMerge = pctx.AndroidStaticRule("gensrcsMerge", blueprint.RuleParams{
  72. Command: "${soongZip} -o ${tmpZip} @${tmpZip}.rsp && ${zipSync} -d ${genDir} ${tmpZip}",
  73. CommandDeps: []string{"${soongZip}", "${zipSync}"},
  74. Rspfile: "${tmpZip}.rsp",
  75. RspfileContent: "${zipArgs}",
  76. }, "tmpZip", "genDir", "zipArgs")
  77. )
  78. func init() {
  79. pctx.Import("android/soong/android")
  80. pctx.HostBinToolVariable("soongZip", "soong_zip")
  81. pctx.HostBinToolVariable("zipSync", "zipsync")
  82. }
  83. type SourceFileGenerator interface {
  84. GeneratedSourceFiles() android.Paths
  85. GeneratedHeaderDirs() android.Paths
  86. GeneratedDeps() android.Paths
  87. }
  88. // Alias for android.HostToolProvider
  89. // Deprecated: use android.HostToolProvider instead.
  90. type HostToolProvider interface {
  91. android.HostToolProvider
  92. }
  93. type hostToolDependencyTag struct {
  94. blueprint.BaseDependencyTag
  95. android.LicenseAnnotationToolchainDependencyTag
  96. label string
  97. }
  98. func (t hostToolDependencyTag) AllowDisabledModuleDependency(target android.Module) bool {
  99. // Allow depending on a disabled module if it's replaced by a prebuilt
  100. // counterpart. We get the prebuilt through android.PrebuiltGetPreferred in
  101. // GenerateAndroidBuildActions.
  102. return target.IsReplacedByPrebuilt()
  103. }
  104. var _ android.AllowDisabledModuleDependency = (*hostToolDependencyTag)(nil)
  105. type generatorProperties struct {
  106. // The command to run on one or more input files. Cmd supports substitution of a few variables.
  107. //
  108. // Available variables for substitution:
  109. //
  110. // $(location): the path to the first entry in tools or tool_files.
  111. // $(location <label>): the path to the tool, tool_file, input or output with name <label>. Use $(location) if <label> refers to a rule that outputs exactly one file.
  112. // $(locations <label>): the paths to the tools, tool_files, inputs or outputs with name <label>. Use $(locations) if <label> refers to a rule that outputs two or more files.
  113. // $(in): one or more input files.
  114. // $(out): a single output file.
  115. // $(depfile): a file to which dependencies will be written, if the depfile property is set to true.
  116. // $(genDir): the sandbox directory for this tool; contains $(out).
  117. // $$: a literal $
  118. Cmd *string
  119. // Enable reading a file containing dependencies in gcc format after the command completes
  120. Depfile *bool
  121. // name of the modules (if any) that produces the host executable. Leave empty for
  122. // prebuilts or scripts that do not need a module to build them.
  123. Tools []string
  124. // Local file that is used as the tool
  125. Tool_files []string `android:"path"`
  126. // List of directories to export generated headers from
  127. Export_include_dirs []string
  128. // list of input files
  129. Srcs []string `android:"path,arch_variant"`
  130. // input files to exclude
  131. Exclude_srcs []string `android:"path,arch_variant"`
  132. }
  133. type Module struct {
  134. android.ModuleBase
  135. android.DefaultableModuleBase
  136. android.BazelModuleBase
  137. android.ApexModuleBase
  138. // For other packages to make their own genrules with extra
  139. // properties
  140. Extra interface{}
  141. // CmdModifier can be set by wrappers around genrule to modify the command, for example to
  142. // prefix environment variables to it.
  143. CmdModifier func(ctx android.ModuleContext, cmd string) string
  144. android.ImageInterface
  145. properties generatorProperties
  146. // For the different tasks that genrule and gensrc generate. genrule will
  147. // generate 1 task, and gensrc will generate 1 or more tasks based on the
  148. // number of shards the input files are sharded into.
  149. taskGenerator taskFunc
  150. rule blueprint.Rule
  151. rawCommands []string
  152. exportedIncludeDirs android.Paths
  153. outputFiles android.Paths
  154. outputDeps android.Paths
  155. subName string
  156. subDir string
  157. // Collect the module directory for IDE info in java/jdeps.go.
  158. modulePaths []string
  159. }
  160. var _ android.MixedBuildBuildable = (*Module)(nil)
  161. type taskFunc func(ctx android.ModuleContext, rawCommand string, srcFiles android.Paths) []generateTask
  162. type generateTask struct {
  163. in android.Paths
  164. out android.WritablePaths
  165. depFile android.WritablePath
  166. copyTo android.WritablePaths // For gensrcs to set on gensrcsMerge rule.
  167. genDir android.WritablePath
  168. extraTools android.Paths // dependencies on tools used by the generator
  169. extraInputs map[string][]string
  170. cmd string
  171. // For gensrsc sharding.
  172. shard int
  173. shards int
  174. }
  175. func (g *Module) GeneratedSourceFiles() android.Paths {
  176. return g.outputFiles
  177. }
  178. func (g *Module) Srcs() android.Paths {
  179. return append(android.Paths{}, g.outputFiles...)
  180. }
  181. func (g *Module) GeneratedHeaderDirs() android.Paths {
  182. return g.exportedIncludeDirs
  183. }
  184. func (g *Module) GeneratedDeps() android.Paths {
  185. return g.outputDeps
  186. }
  187. func (g *Module) OutputFiles(tag string) (android.Paths, error) {
  188. if tag == "" {
  189. return append(android.Paths{}, g.outputFiles...), nil
  190. }
  191. // otherwise, tag should match one of outputs
  192. for _, outputFile := range g.outputFiles {
  193. if outputFile.Rel() == tag {
  194. return android.Paths{outputFile}, nil
  195. }
  196. }
  197. return nil, fmt.Errorf("unsupported module reference tag %q", tag)
  198. }
  199. var _ android.SourceFileProducer = (*Module)(nil)
  200. var _ android.OutputFileProducer = (*Module)(nil)
  201. func toolDepsMutator(ctx android.BottomUpMutatorContext) {
  202. if g, ok := ctx.Module().(*Module); ok {
  203. for _, tool := range g.properties.Tools {
  204. tag := hostToolDependencyTag{label: tool}
  205. if m := android.SrcIsModule(tool); m != "" {
  206. tool = m
  207. }
  208. ctx.AddFarVariationDependencies(ctx.Config().BuildOSTarget.Variations(), tag, tool)
  209. }
  210. }
  211. }
  212. func (g *Module) ProcessBazelQueryResponse(ctx android.ModuleContext) {
  213. g.generateCommonBuildActions(ctx)
  214. label := g.GetBazelLabel(ctx, g)
  215. bazelCtx := ctx.Config().BazelContext
  216. filePaths, err := bazelCtx.GetOutputFiles(label, android.GetConfigKey(ctx))
  217. if err != nil {
  218. ctx.ModuleErrorf(err.Error())
  219. return
  220. }
  221. var bazelOutputFiles android.Paths
  222. exportIncludeDirs := map[string]bool{}
  223. for _, bazelOutputFile := range filePaths {
  224. bazelOutputFiles = append(bazelOutputFiles, android.PathForBazelOutRelative(ctx, ctx.ModuleDir(), bazelOutputFile))
  225. exportIncludeDirs[filepath.Dir(bazelOutputFile)] = true
  226. }
  227. g.outputFiles = bazelOutputFiles
  228. g.outputDeps = bazelOutputFiles
  229. for includePath, _ := range exportIncludeDirs {
  230. g.exportedIncludeDirs = append(g.exportedIncludeDirs, android.PathForBazelOut(ctx, includePath))
  231. }
  232. }
  233. // generateCommonBuildActions contains build action generation logic
  234. // common to both the mixed build case and the legacy case of genrule processing.
  235. // To fully support genrule in mixed builds, the contents of this function should
  236. // approach zero; there should be no genrule action registration done directly
  237. // by Soong logic in the mixed-build case.
  238. func (g *Module) generateCommonBuildActions(ctx android.ModuleContext) {
  239. g.subName = ctx.ModuleSubDir()
  240. // Collect the module directory for IDE info in java/jdeps.go.
  241. g.modulePaths = append(g.modulePaths, ctx.ModuleDir())
  242. if len(g.properties.Export_include_dirs) > 0 {
  243. for _, dir := range g.properties.Export_include_dirs {
  244. g.exportedIncludeDirs = append(g.exportedIncludeDirs,
  245. android.PathForModuleGen(ctx, g.subDir, ctx.ModuleDir(), dir))
  246. }
  247. } else {
  248. g.exportedIncludeDirs = append(g.exportedIncludeDirs, android.PathForModuleGen(ctx, g.subDir))
  249. }
  250. locationLabels := map[string]location{}
  251. firstLabel := ""
  252. addLocationLabel := func(label string, loc location) {
  253. if firstLabel == "" {
  254. firstLabel = label
  255. }
  256. if _, exists := locationLabels[label]; !exists {
  257. locationLabels[label] = loc
  258. } else {
  259. ctx.ModuleErrorf("multiple locations for label %q: %q and %q (do you have duplicate srcs entries?)",
  260. label, locationLabels[label], loc)
  261. }
  262. }
  263. var tools android.Paths
  264. var packagedTools []android.PackagingSpec
  265. if len(g.properties.Tools) > 0 {
  266. seenTools := make(map[string]bool)
  267. ctx.VisitDirectDepsBlueprint(func(module blueprint.Module) {
  268. switch tag := ctx.OtherModuleDependencyTag(module).(type) {
  269. case hostToolDependencyTag:
  270. tool := ctx.OtherModuleName(module)
  271. if m, ok := module.(android.Module); ok {
  272. // Necessary to retrieve any prebuilt replacement for the tool, since
  273. // toolDepsMutator runs too late for the prebuilt mutators to have
  274. // replaced the dependency.
  275. module = android.PrebuiltGetPreferred(ctx, m)
  276. }
  277. switch t := module.(type) {
  278. case android.HostToolProvider:
  279. // A HostToolProvider provides the path to a tool, which will be copied
  280. // into the sandbox.
  281. if !t.(android.Module).Enabled() {
  282. if ctx.Config().AllowMissingDependencies() {
  283. ctx.AddMissingDependencies([]string{tool})
  284. } else {
  285. ctx.ModuleErrorf("depends on disabled module %q", tool)
  286. }
  287. return
  288. }
  289. path := t.HostToolPath()
  290. if !path.Valid() {
  291. ctx.ModuleErrorf("host tool %q missing output file", tool)
  292. return
  293. }
  294. if specs := t.TransitivePackagingSpecs(); specs != nil {
  295. // If the HostToolProvider has PackgingSpecs, which are definitions of the
  296. // required relative locations of the tool and its dependencies, use those
  297. // instead. They will be copied to those relative locations in the sbox
  298. // sandbox.
  299. packagedTools = append(packagedTools, specs...)
  300. // Assume that the first PackagingSpec of the module is the tool.
  301. addLocationLabel(tag.label, packagedToolLocation{specs[0]})
  302. } else {
  303. tools = append(tools, path.Path())
  304. addLocationLabel(tag.label, toolLocation{android.Paths{path.Path()}})
  305. }
  306. case bootstrap.GoBinaryTool:
  307. // A GoBinaryTool provides the install path to a tool, which will be copied.
  308. p := android.PathForGoBinary(ctx, t)
  309. tools = append(tools, p)
  310. addLocationLabel(tag.label, toolLocation{android.Paths{p}})
  311. default:
  312. ctx.ModuleErrorf("%q is not a host tool provider", tool)
  313. return
  314. }
  315. seenTools[tag.label] = true
  316. }
  317. })
  318. // If AllowMissingDependencies is enabled, the build will not have stopped when
  319. // AddFarVariationDependencies was called on a missing tool, which will result in nonsensical
  320. // "cmd: unknown location label ..." errors later. Add a placeholder file to the local label.
  321. // The command that uses this placeholder file will never be executed because the rule will be
  322. // replaced with an android.Error rule reporting the missing dependencies.
  323. if ctx.Config().AllowMissingDependencies() {
  324. for _, tool := range g.properties.Tools {
  325. if !seenTools[tool] {
  326. addLocationLabel(tool, errorLocation{"***missing tool " + tool + "***"})
  327. }
  328. }
  329. }
  330. }
  331. if ctx.Failed() {
  332. return
  333. }
  334. for _, toolFile := range g.properties.Tool_files {
  335. paths := android.PathsForModuleSrc(ctx, []string{toolFile})
  336. tools = append(tools, paths...)
  337. addLocationLabel(toolFile, toolLocation{paths})
  338. }
  339. addLabelsForInputs := func(propName string, include, exclude []string) android.Paths {
  340. includeDirInPaths := ctx.DeviceConfig().BuildBrokenInputDir(g.Name())
  341. var srcFiles android.Paths
  342. for _, in := range include {
  343. paths, missingDeps := android.PathsAndMissingDepsRelativeToModuleSourceDir(android.SourceInput{
  344. Context: ctx, Paths: []string{in}, ExcludePaths: exclude, IncludeDirs: includeDirInPaths,
  345. })
  346. if len(missingDeps) > 0 {
  347. if !ctx.Config().AllowMissingDependencies() {
  348. panic(fmt.Errorf("should never get here, the missing dependencies %q should have been reported in DepsMutator",
  349. missingDeps))
  350. }
  351. // If AllowMissingDependencies is enabled, the build will not have stopped when
  352. // the dependency was added on a missing SourceFileProducer module, which will result in nonsensical
  353. // "cmd: label ":..." has no files" errors later. Add a placeholder file to the local label.
  354. // The command that uses this placeholder file will never be executed because the rule will be
  355. // replaced with an android.Error rule reporting the missing dependencies.
  356. ctx.AddMissingDependencies(missingDeps)
  357. addLocationLabel(in, errorLocation{"***missing " + propName + " " + in + "***"})
  358. } else {
  359. srcFiles = append(srcFiles, paths...)
  360. addLocationLabel(in, inputLocation{paths})
  361. }
  362. }
  363. return srcFiles
  364. }
  365. srcFiles := addLabelsForInputs("srcs", g.properties.Srcs, g.properties.Exclude_srcs)
  366. var copyFrom android.Paths
  367. var outputFiles android.WritablePaths
  368. var zipArgs strings.Builder
  369. cmd := String(g.properties.Cmd)
  370. if g.CmdModifier != nil {
  371. cmd = g.CmdModifier(ctx, cmd)
  372. }
  373. // Generate tasks, either from genrule or gensrcs.
  374. for i, task := range g.taskGenerator(ctx, cmd, srcFiles) {
  375. if len(task.out) == 0 {
  376. ctx.ModuleErrorf("must have at least one output file")
  377. return
  378. }
  379. var extraInputs android.Paths
  380. // Only handle extra inputs once as these currently are the same across all tasks
  381. if i == 0 {
  382. for name, values := range task.extraInputs {
  383. extraInputs = append(extraInputs, addLabelsForInputs(name, values, []string{})...)
  384. }
  385. }
  386. // Pick a unique path outside the task.genDir for the sbox manifest textproto,
  387. // a unique rule name, and the user-visible description.
  388. manifestName := "genrule.sbox.textproto"
  389. desc := "generate"
  390. name := "generator"
  391. if task.shards > 0 {
  392. manifestName = "genrule_" + strconv.Itoa(task.shard) + ".sbox.textproto"
  393. desc += " " + strconv.Itoa(task.shard)
  394. name += strconv.Itoa(task.shard)
  395. } else if len(task.out) == 1 {
  396. desc += " " + task.out[0].Base()
  397. }
  398. manifestPath := android.PathForModuleOut(ctx, manifestName)
  399. // Use a RuleBuilder to create a rule that runs the command inside an sbox sandbox.
  400. rule := getSandboxedRuleBuilder(ctx, android.NewRuleBuilder(pctx, ctx).Sbox(task.genDir, manifestPath))
  401. cmd := rule.Command()
  402. for _, out := range task.out {
  403. addLocationLabel(out.Rel(), outputLocation{out})
  404. }
  405. referencedDepfile := false
  406. rawCommand, err := android.Expand(task.cmd, func(name string) (string, error) {
  407. // report the error directly without returning an error to android.Expand to catch multiple errors in a
  408. // single run
  409. reportError := func(fmt string, args ...interface{}) (string, error) {
  410. ctx.PropertyErrorf("cmd", fmt, args...)
  411. return "SOONG_ERROR", nil
  412. }
  413. // Apply shell escape to each cases to prevent source file paths containing $ from being evaluated in shell
  414. switch name {
  415. case "location":
  416. if len(g.properties.Tools) == 0 && len(g.properties.Tool_files) == 0 {
  417. return reportError("at least one `tools` or `tool_files` is required if $(location) is used")
  418. }
  419. loc := locationLabels[firstLabel]
  420. paths := loc.Paths(cmd)
  421. if len(paths) == 0 {
  422. return reportError("default label %q has no files", firstLabel)
  423. } else if len(paths) > 1 {
  424. return reportError("default label %q has multiple files, use $(locations %s) to reference it",
  425. firstLabel, firstLabel)
  426. }
  427. return proptools.ShellEscape(paths[0]), nil
  428. case "in":
  429. return strings.Join(proptools.ShellEscapeList(cmd.PathsForInputs(srcFiles)), " "), nil
  430. case "out":
  431. var sandboxOuts []string
  432. for _, out := range task.out {
  433. sandboxOuts = append(sandboxOuts, cmd.PathForOutput(out))
  434. }
  435. return strings.Join(proptools.ShellEscapeList(sandboxOuts), " "), nil
  436. case "depfile":
  437. referencedDepfile = true
  438. if !Bool(g.properties.Depfile) {
  439. return reportError("$(depfile) used without depfile property")
  440. }
  441. return "__SBOX_DEPFILE__", nil
  442. case "genDir":
  443. return proptools.ShellEscape(cmd.PathForOutput(task.genDir)), nil
  444. default:
  445. if strings.HasPrefix(name, "location ") {
  446. label := strings.TrimSpace(strings.TrimPrefix(name, "location "))
  447. if loc, ok := locationLabels[label]; ok {
  448. paths := loc.Paths(cmd)
  449. if len(paths) == 0 {
  450. return reportError("label %q has no files", label)
  451. } else if len(paths) > 1 {
  452. return reportError("label %q has multiple files, use $(locations %s) to reference it",
  453. label, label)
  454. }
  455. return proptools.ShellEscape(paths[0]), nil
  456. } else {
  457. return reportError("unknown location label %q is not in srcs, out, tools or tool_files.", label)
  458. }
  459. } else if strings.HasPrefix(name, "locations ") {
  460. label := strings.TrimSpace(strings.TrimPrefix(name, "locations "))
  461. if loc, ok := locationLabels[label]; ok {
  462. paths := loc.Paths(cmd)
  463. if len(paths) == 0 {
  464. return reportError("label %q has no files", label)
  465. }
  466. return proptools.ShellEscape(strings.Join(paths, " ")), nil
  467. } else {
  468. return reportError("unknown locations label %q is not in srcs, out, tools or tool_files.", label)
  469. }
  470. } else {
  471. return reportError("unknown variable '$(%s)'", name)
  472. }
  473. }
  474. })
  475. if err != nil {
  476. ctx.PropertyErrorf("cmd", "%s", err.Error())
  477. return
  478. }
  479. if Bool(g.properties.Depfile) && !referencedDepfile {
  480. ctx.PropertyErrorf("cmd", "specified depfile=true but did not include a reference to '${depfile}' in cmd")
  481. return
  482. }
  483. g.rawCommands = append(g.rawCommands, rawCommand)
  484. cmd.Text(rawCommand)
  485. cmd.Implicits(srcFiles) // need to be able to reference other srcs
  486. cmd.Implicits(extraInputs)
  487. cmd.ImplicitOutputs(task.out)
  488. cmd.Implicits(task.in)
  489. cmd.ImplicitTools(tools)
  490. cmd.ImplicitTools(task.extraTools)
  491. cmd.ImplicitPackagedTools(packagedTools)
  492. if Bool(g.properties.Depfile) {
  493. cmd.ImplicitDepFile(task.depFile)
  494. }
  495. // Create the rule to run the genrule command inside sbox.
  496. rule.Build(name, desc)
  497. if len(task.copyTo) > 0 {
  498. // If copyTo is set, multiple shards need to be copied into a single directory.
  499. // task.out contains the per-shard paths, and copyTo contains the corresponding
  500. // final path. The files need to be copied into the final directory by a
  501. // single rule so it can remove the directory before it starts to ensure no
  502. // old files remain. zipsync already does this, so build up zipArgs that
  503. // zip all the per-shard directories into a single zip.
  504. outputFiles = append(outputFiles, task.copyTo...)
  505. copyFrom = append(copyFrom, task.out.Paths()...)
  506. zipArgs.WriteString(" -C " + task.genDir.String())
  507. zipArgs.WriteString(android.JoinWithPrefix(task.out.Strings(), " -f "))
  508. } else {
  509. outputFiles = append(outputFiles, task.out...)
  510. }
  511. }
  512. if len(copyFrom) > 0 {
  513. // Create a rule that zips all the per-shard directories into a single zip and then
  514. // uses zipsync to unzip it into the final directory.
  515. ctx.Build(pctx, android.BuildParams{
  516. Rule: gensrcsMerge,
  517. Implicits: copyFrom,
  518. Outputs: outputFiles,
  519. Description: "merge shards",
  520. Args: map[string]string{
  521. "zipArgs": zipArgs.String(),
  522. "tmpZip": android.PathForModuleGen(ctx, g.subDir+".zip").String(),
  523. "genDir": android.PathForModuleGen(ctx, g.subDir).String(),
  524. },
  525. })
  526. }
  527. g.outputFiles = outputFiles.Paths()
  528. }
  529. func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) {
  530. // Allowlist genrule to use depfile until we have a solution to remove it.
  531. // TODO(b/235582219): Remove allowlist for genrule
  532. if Bool(g.properties.Depfile) {
  533. if DepfileAllowSet == nil {
  534. DepfileAllowSetLock.Lock()
  535. defer DepfileAllowSetLock.Unlock()
  536. DepfileAllowSet = map[string]bool{}
  537. android.AddToStringSet(DepfileAllowSet, DepfileAllowList)
  538. }
  539. // TODO(b/283852474): Checking the GenruleSandboxing flag is temporary in
  540. // order to pass the presubmit before internal master is updated.
  541. if ctx.DeviceConfig().GenruleSandboxing() && !DepfileAllowSet[g.Name()] {
  542. ctx.PropertyErrorf(
  543. "depfile",
  544. "Deprecated to ensure the module type is convertible to Bazel. "+
  545. "Try specifying the dependencies explicitly so that there is no need to use depfile. "+
  546. "If not possible, the escape hatch is to add the module to allowlists.go to bypass the error.")
  547. }
  548. }
  549. g.generateCommonBuildActions(ctx)
  550. // For <= 6 outputs, just embed those directly in the users. Right now, that covers >90% of
  551. // the genrules on AOSP. That will make things simpler to look at the graph in the common
  552. // case. For larger sets of outputs, inject a phony target in between to limit ninja file
  553. // growth.
  554. if len(g.outputFiles) <= 6 {
  555. g.outputDeps = g.outputFiles
  556. } else {
  557. phonyFile := android.PathForModuleGen(ctx, "genrule-phony")
  558. ctx.Build(pctx, android.BuildParams{
  559. Rule: blueprint.Phony,
  560. Output: phonyFile,
  561. Inputs: g.outputFiles,
  562. })
  563. g.outputDeps = android.Paths{phonyFile}
  564. }
  565. }
  566. func (g *Module) QueueBazelCall(ctx android.BaseModuleContext) {
  567. bazelCtx := ctx.Config().BazelContext
  568. bazelCtx.QueueBazelRequest(g.GetBazelLabel(ctx, g), cquery.GetOutputFiles, android.GetConfigKey(ctx))
  569. }
  570. func (g *Module) IsMixedBuildSupported(ctx android.BaseModuleContext) bool {
  571. return true
  572. }
  573. // Collect information for opening IDE project files in java/jdeps.go.
  574. func (g *Module) IDEInfo(dpInfo *android.IdeInfo) {
  575. dpInfo.Srcs = append(dpInfo.Srcs, g.Srcs().Strings()...)
  576. for _, src := range g.properties.Srcs {
  577. if strings.HasPrefix(src, ":") {
  578. src = strings.Trim(src, ":")
  579. dpInfo.Deps = append(dpInfo.Deps, src)
  580. }
  581. }
  582. dpInfo.Paths = append(dpInfo.Paths, g.modulePaths...)
  583. }
  584. func (g *Module) AndroidMk() android.AndroidMkData {
  585. return android.AndroidMkData{
  586. Class: "ETC",
  587. OutputFile: android.OptionalPathForPath(g.outputFiles[0]),
  588. SubName: g.subName,
  589. Extra: []android.AndroidMkExtraFunc{
  590. func(w io.Writer, outputFile android.Path) {
  591. fmt.Fprintln(w, "LOCAL_UNINSTALLABLE_MODULE := true")
  592. },
  593. },
  594. Custom: func(w io.Writer, name, prefix, moduleDir string, data android.AndroidMkData) {
  595. android.WriteAndroidMkData(w, data)
  596. if data.SubName != "" {
  597. fmt.Fprintln(w, ".PHONY:", name)
  598. fmt.Fprintln(w, name, ":", name+g.subName)
  599. }
  600. },
  601. }
  602. }
  603. var _ android.ApexModule = (*Module)(nil)
  604. // Implements android.ApexModule
  605. func (g *Module) ShouldSupportSdkVersion(ctx android.BaseModuleContext,
  606. sdkVersion android.ApiLevel) error {
  607. // Because generated outputs are checked by client modules(e.g. cc_library, ...)
  608. // we can safely ignore the check here.
  609. return nil
  610. }
  611. func generatorFactory(taskGenerator taskFunc, props ...interface{}) *Module {
  612. module := &Module{
  613. taskGenerator: taskGenerator,
  614. }
  615. module.AddProperties(props...)
  616. module.AddProperties(&module.properties)
  617. module.ImageInterface = noopImageInterface{}
  618. return module
  619. }
  620. type noopImageInterface struct{}
  621. func (x noopImageInterface) ImageMutatorBegin(android.BaseModuleContext) {}
  622. func (x noopImageInterface) CoreVariantNeeded(android.BaseModuleContext) bool { return false }
  623. func (x noopImageInterface) RamdiskVariantNeeded(android.BaseModuleContext) bool { return false }
  624. func (x noopImageInterface) VendorRamdiskVariantNeeded(android.BaseModuleContext) bool { return false }
  625. func (x noopImageInterface) DebugRamdiskVariantNeeded(android.BaseModuleContext) bool { return false }
  626. func (x noopImageInterface) RecoveryVariantNeeded(android.BaseModuleContext) bool { return false }
  627. func (x noopImageInterface) ExtraImageVariations(ctx android.BaseModuleContext) []string { return nil }
  628. func (x noopImageInterface) SetImageVariation(ctx android.BaseModuleContext, variation string, module android.Module) {
  629. }
  630. func NewGenSrcs() *Module {
  631. properties := &genSrcsProperties{}
  632. // finalSubDir is the name of the subdirectory that output files will be generated into.
  633. // It is used so that per-shard directories can be placed alongside it an then finally
  634. // merged into it.
  635. const finalSubDir = "gensrcs"
  636. taskGenerator := func(ctx android.ModuleContext, rawCommand string, srcFiles android.Paths) []generateTask {
  637. shardSize := defaultShardSize
  638. if s := properties.Shard_size; s != nil {
  639. shardSize = int(*s)
  640. }
  641. // gensrcs rules can easily hit command line limits by repeating the command for
  642. // every input file. Shard the input files into groups.
  643. shards := android.ShardPaths(srcFiles, shardSize)
  644. var generateTasks []generateTask
  645. for i, shard := range shards {
  646. var commands []string
  647. var outFiles android.WritablePaths
  648. var commandDepFiles []string
  649. var copyTo android.WritablePaths
  650. // When sharding is enabled (i.e. len(shards) > 1), the sbox rules for each
  651. // shard will be write to their own directories and then be merged together
  652. // into finalSubDir. If sharding is not enabled (i.e. len(shards) == 1),
  653. // the sbox rule will write directly to finalSubDir.
  654. genSubDir := finalSubDir
  655. if len(shards) > 1 {
  656. genSubDir = strconv.Itoa(i)
  657. }
  658. genDir := android.PathForModuleGen(ctx, genSubDir)
  659. // TODO(ccross): this RuleBuilder is a hack to be able to call
  660. // rule.Command().PathForOutput. Replace this with passing the rule into the
  661. // generator.
  662. rule := getSandboxedRuleBuilder(ctx, android.NewRuleBuilder(pctx, ctx).Sbox(genDir, nil))
  663. for _, in := range shard {
  664. outFile := android.GenPathWithExt(ctx, finalSubDir, in, String(properties.Output_extension))
  665. // If sharding is enabled, then outFile is the path to the output file in
  666. // the shard directory, and copyTo is the path to the output file in the
  667. // final directory.
  668. if len(shards) > 1 {
  669. shardFile := android.GenPathWithExt(ctx, genSubDir, in, String(properties.Output_extension))
  670. copyTo = append(copyTo, outFile)
  671. outFile = shardFile
  672. }
  673. outFiles = append(outFiles, outFile)
  674. // pre-expand the command line to replace $in and $out with references to
  675. // a single input and output file.
  676. command, err := android.Expand(rawCommand, func(name string) (string, error) {
  677. switch name {
  678. case "in":
  679. return in.String(), nil
  680. case "out":
  681. return rule.Command().PathForOutput(outFile), nil
  682. case "depfile":
  683. // Generate a depfile for each output file. Store the list for
  684. // later in order to combine them all into a single depfile.
  685. depFile := rule.Command().PathForOutput(outFile.ReplaceExtension(ctx, "d"))
  686. commandDepFiles = append(commandDepFiles, depFile)
  687. return depFile, nil
  688. default:
  689. return "$(" + name + ")", nil
  690. }
  691. })
  692. if err != nil {
  693. ctx.PropertyErrorf("cmd", err.Error())
  694. }
  695. // escape the command in case for example it contains '#', an odd number of '"', etc
  696. command = fmt.Sprintf("bash -c %v", proptools.ShellEscape(command))
  697. commands = append(commands, command)
  698. }
  699. fullCommand := strings.Join(commands, " && ")
  700. var outputDepfile android.WritablePath
  701. var extraTools android.Paths
  702. if len(commandDepFiles) > 0 {
  703. // Each command wrote to a depfile, but ninja can only handle one
  704. // depfile per rule. Use the dep_fixer tool at the end of the
  705. // command to combine all the depfiles into a single output depfile.
  706. outputDepfile = android.PathForModuleGen(ctx, genSubDir, "gensrcs.d")
  707. depFixerTool := ctx.Config().HostToolPath(ctx, "dep_fixer")
  708. fullCommand += fmt.Sprintf(" && %s -o $(depfile) %s",
  709. rule.Command().PathForTool(depFixerTool),
  710. strings.Join(commandDepFiles, " "))
  711. extraTools = append(extraTools, depFixerTool)
  712. }
  713. generateTasks = append(generateTasks, generateTask{
  714. in: shard,
  715. out: outFiles,
  716. depFile: outputDepfile,
  717. copyTo: copyTo,
  718. genDir: genDir,
  719. cmd: fullCommand,
  720. shard: i,
  721. shards: len(shards),
  722. extraTools: extraTools,
  723. extraInputs: map[string][]string{
  724. "data": properties.Data,
  725. },
  726. })
  727. }
  728. return generateTasks
  729. }
  730. g := generatorFactory(taskGenerator, properties)
  731. g.subDir = finalSubDir
  732. return g
  733. }
  734. func GenSrcsFactory() android.Module {
  735. m := NewGenSrcs()
  736. android.InitAndroidModule(m)
  737. android.InitBazelModule(m)
  738. return m
  739. }
  740. type genSrcsProperties struct {
  741. // extension that will be substituted for each output file
  742. Output_extension *string
  743. // maximum number of files that will be passed on a single command line.
  744. Shard_size *int64
  745. // Additional files needed for build that are not tooling related.
  746. Data []string `android:"path"`
  747. }
  748. type bazelGensrcsAttributes struct {
  749. Srcs bazel.LabelListAttribute
  750. Output_extension *string
  751. Tools bazel.LabelListAttribute
  752. Cmd string
  753. Data bazel.LabelListAttribute
  754. }
  755. const defaultShardSize = 50
  756. func NewGenRule() *Module {
  757. properties := &genRuleProperties{}
  758. taskGenerator := func(ctx android.ModuleContext, rawCommand string, srcFiles android.Paths) []generateTask {
  759. outs := make(android.WritablePaths, len(properties.Out))
  760. var depFile android.WritablePath
  761. for i, out := range properties.Out {
  762. outPath := android.PathForModuleGen(ctx, out)
  763. if i == 0 {
  764. depFile = outPath.ReplaceExtension(ctx, "d")
  765. }
  766. outs[i] = outPath
  767. }
  768. return []generateTask{{
  769. in: srcFiles,
  770. out: outs,
  771. depFile: depFile,
  772. genDir: android.PathForModuleGen(ctx),
  773. cmd: rawCommand,
  774. }}
  775. }
  776. return generatorFactory(taskGenerator, properties)
  777. }
  778. func GenRuleFactory() android.Module {
  779. m := NewGenRule()
  780. android.InitAndroidModule(m)
  781. android.InitDefaultableModule(m)
  782. android.InitBazelModule(m)
  783. return m
  784. }
  785. type genRuleProperties struct {
  786. // names of the output files that will be generated
  787. Out []string
  788. }
  789. type bazelGenruleAttributes struct {
  790. Srcs bazel.LabelListAttribute
  791. Outs []string
  792. Tools bazel.LabelListAttribute
  793. Cmd string
  794. }
  795. // ConvertWithBp2build converts a Soong module -> Bazel target.
  796. func (m *Module) ConvertWithBp2build(ctx android.TopDownMutatorContext) {
  797. // Bazel only has the "tools" attribute.
  798. tools_prop := android.BazelLabelForModuleDeps(ctx, m.properties.Tools)
  799. tool_files_prop := android.BazelLabelForModuleSrc(ctx, m.properties.Tool_files)
  800. tools_prop.Append(tool_files_prop)
  801. tools := bazel.MakeLabelListAttribute(tools_prop)
  802. srcs := bazel.LabelListAttribute{}
  803. srcs_labels := bazel.LabelList{}
  804. // Only cc_genrule is arch specific
  805. if ctx.ModuleType() == "cc_genrule" {
  806. for axis, configToProps := range m.GetArchVariantProperties(ctx, &generatorProperties{}) {
  807. for config, props := range configToProps {
  808. if props, ok := props.(*generatorProperties); ok {
  809. labels := android.BazelLabelForModuleSrcExcludes(ctx, props.Srcs, props.Exclude_srcs)
  810. srcs_labels.Append(labels)
  811. srcs.SetSelectValue(axis, config, labels)
  812. }
  813. }
  814. }
  815. } else {
  816. srcs_labels = android.BazelLabelForModuleSrcExcludes(ctx, m.properties.Srcs, m.properties.Exclude_srcs)
  817. srcs = bazel.MakeLabelListAttribute(srcs_labels)
  818. }
  819. var allReplacements bazel.LabelList
  820. allReplacements.Append(tools.Value)
  821. allReplacements.Append(bazel.FirstUniqueBazelLabelList(srcs_labels))
  822. // The Output_extension prop is not in an immediately accessible field
  823. // in the Module struct, so use GetProperties and cast it
  824. // to the known struct prop.
  825. var outputExtension *string
  826. var data bazel.LabelListAttribute
  827. if ctx.ModuleType() == "gensrcs" {
  828. for _, propIntf := range m.GetProperties() {
  829. if props, ok := propIntf.(*genSrcsProperties); ok {
  830. outputExtension = props.Output_extension
  831. dataFiles := android.BazelLabelForModuleSrc(ctx, props.Data)
  832. allReplacements.Append(bazel.FirstUniqueBazelLabelList(dataFiles))
  833. data = bazel.MakeLabelListAttribute(dataFiles)
  834. break
  835. }
  836. }
  837. }
  838. // Replace in and out variables with $< and $@
  839. var cmd string
  840. if m.properties.Cmd != nil {
  841. if ctx.ModuleType() == "gensrcs" {
  842. cmd = strings.ReplaceAll(*m.properties.Cmd, "$(in)", "$(SRC)")
  843. cmd = strings.ReplaceAll(cmd, "$(out)", "$(OUT)")
  844. } else {
  845. cmd = strings.Replace(*m.properties.Cmd, "$(in)", "$(SRCS)", -1)
  846. cmd = strings.Replace(cmd, "$(out)", "$(OUTS)", -1)
  847. }
  848. cmd = strings.Replace(cmd, "$(genDir)", "$(RULEDIR)", -1)
  849. if len(tools.Value.Includes) > 0 {
  850. cmd = strings.Replace(cmd, "$(location)", fmt.Sprintf("$(location %s)", tools.Value.Includes[0].Label), -1)
  851. cmd = strings.Replace(cmd, "$(locations)", fmt.Sprintf("$(locations %s)", tools.Value.Includes[0].Label), -1)
  852. }
  853. for _, l := range allReplacements.Includes {
  854. bpLoc := fmt.Sprintf("$(location %s)", l.OriginalModuleName)
  855. bpLocs := fmt.Sprintf("$(locations %s)", l.OriginalModuleName)
  856. bazelLoc := fmt.Sprintf("$(location %s)", l.Label)
  857. bazelLocs := fmt.Sprintf("$(locations %s)", l.Label)
  858. cmd = strings.Replace(cmd, bpLoc, bazelLoc, -1)
  859. cmd = strings.Replace(cmd, bpLocs, bazelLocs, -1)
  860. }
  861. }
  862. tags := android.ApexAvailableTagsWithoutTestApexes(ctx, m)
  863. if ctx.ModuleType() == "gensrcs" {
  864. props := bazel.BazelTargetModuleProperties{
  865. Rule_class: "gensrcs",
  866. Bzl_load_location: "//build/bazel/rules:gensrcs.bzl",
  867. }
  868. attrs := &bazelGensrcsAttributes{
  869. Srcs: srcs,
  870. Output_extension: outputExtension,
  871. Cmd: cmd,
  872. Tools: tools,
  873. Data: data,
  874. }
  875. ctx.CreateBazelTargetModule(props, android.CommonAttributes{
  876. Name: m.Name(),
  877. Tags: tags,
  878. }, attrs)
  879. } else {
  880. // The Out prop is not in an immediately accessible field
  881. // in the Module struct, so use GetProperties and cast it
  882. // to the known struct prop.
  883. var outs []string
  884. for _, propIntf := range m.GetProperties() {
  885. if props, ok := propIntf.(*genRuleProperties); ok {
  886. outs = props.Out
  887. break
  888. }
  889. }
  890. attrs := &bazelGenruleAttributes{
  891. Srcs: srcs,
  892. Outs: outs,
  893. Cmd: cmd,
  894. Tools: tools,
  895. }
  896. props := bazel.BazelTargetModuleProperties{
  897. Rule_class: "genrule",
  898. }
  899. ctx.CreateBazelTargetModule(props, android.CommonAttributes{
  900. Name: m.Name(),
  901. Tags: tags,
  902. }, attrs)
  903. }
  904. }
  905. var Bool = proptools.Bool
  906. var String = proptools.String
  907. // Defaults
  908. type Defaults struct {
  909. android.ModuleBase
  910. android.DefaultsModuleBase
  911. }
  912. func defaultsFactory() android.Module {
  913. return DefaultsFactory()
  914. }
  915. func DefaultsFactory(props ...interface{}) android.Module {
  916. module := &Defaults{}
  917. module.AddProperties(props...)
  918. module.AddProperties(
  919. &generatorProperties{},
  920. &genRuleProperties{},
  921. )
  922. android.InitDefaultsModule(module)
  923. return module
  924. }
  925. func getSandboxedRuleBuilder(ctx android.ModuleContext, r *android.RuleBuilder) *android.RuleBuilder {
  926. if !ctx.DeviceConfig().GenruleSandboxing() {
  927. return r.SandboxTools()
  928. }
  929. if SandboxingDenyModuleSet == nil {
  930. SandboxingDenyModuleSetLock.Lock()
  931. defer SandboxingDenyModuleSetLock.Unlock()
  932. SandboxingDenyModuleSet = map[string]bool{}
  933. SandboxingDenyPathSet = map[string]bool{}
  934. android.AddToStringSet(SandboxingDenyModuleSet, append(DepfileAllowList, SandboxingDenyModuleList...))
  935. android.AddToStringSet(SandboxingDenyPathSet, SandboxingDenyPathList)
  936. }
  937. if SandboxingDenyPathSet[ctx.ModuleDir()] || SandboxingDenyModuleSet[ctx.ModuleName()] {
  938. return r.SandboxTools()
  939. }
  940. return r.SandboxInputs()
  941. }