genrule.go 37 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103
  1. // Copyright 2015 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. // A genrule module takes a list of source files ("srcs" property), an optional
  15. // list of tools ("tools" property), and a command line ("cmd" property), to
  16. // generate output files ("out" property).
  17. package genrule
  18. import (
  19. "fmt"
  20. "io"
  21. "path/filepath"
  22. "strconv"
  23. "strings"
  24. "android/soong/bazel/cquery"
  25. "github.com/google/blueprint"
  26. "github.com/google/blueprint/bootstrap"
  27. "github.com/google/blueprint/proptools"
  28. "android/soong/android"
  29. "android/soong/bazel"
  30. )
  31. func init() {
  32. RegisterGenruleBuildComponents(android.InitRegistrationContext)
  33. }
  34. // Test fixture preparer that will register most genrule build components.
  35. //
  36. // Singletons and mutators should only be added here if they are needed for a majority of genrule
  37. // module types, otherwise they should be added under a separate preparer to allow them to be
  38. // selected only when needed to reduce test execution time.
  39. //
  40. // Module types do not have much of an overhead unless they are used so this should include as many
  41. // module types as possible. The exceptions are those module types that require mutators and/or
  42. // singletons in order to function in which case they should be kept together in a separate
  43. // preparer.
  44. var PrepareForTestWithGenRuleBuildComponents = android.GroupFixturePreparers(
  45. android.FixtureRegisterWithContext(RegisterGenruleBuildComponents),
  46. )
  47. // Prepare a fixture to use all genrule module types, mutators and singletons fully.
  48. //
  49. // This should only be used by tests that want to run with as much of the build enabled as possible.
  50. var PrepareForIntegrationTestWithGenrule = android.GroupFixturePreparers(
  51. PrepareForTestWithGenRuleBuildComponents,
  52. )
  53. func RegisterGenruleBuildComponents(ctx android.RegistrationContext) {
  54. ctx.RegisterModuleType("genrule_defaults", defaultsFactory)
  55. ctx.RegisterModuleType("gensrcs", GenSrcsFactory)
  56. ctx.RegisterModuleType("genrule", GenRuleFactory)
  57. ctx.FinalDepsMutators(func(ctx android.RegisterMutatorsContext) {
  58. ctx.BottomUp("genrule_tool_deps", toolDepsMutator).Parallel()
  59. })
  60. }
  61. var (
  62. pctx = android.NewPackageContext("android/soong/genrule")
  63. // Used by gensrcs when there is more than 1 shard to merge the outputs
  64. // of each shard into a zip file.
  65. gensrcsMerge = pctx.AndroidStaticRule("gensrcsMerge", blueprint.RuleParams{
  66. Command: "${soongZip} -o ${tmpZip} @${tmpZip}.rsp && ${zipSync} -d ${genDir} ${tmpZip}",
  67. CommandDeps: []string{"${soongZip}", "${zipSync}"},
  68. Rspfile: "${tmpZip}.rsp",
  69. RspfileContent: "${zipArgs}",
  70. }, "tmpZip", "genDir", "zipArgs")
  71. )
  72. func init() {
  73. pctx.Import("android/soong/android")
  74. pctx.HostBinToolVariable("soongZip", "soong_zip")
  75. pctx.HostBinToolVariable("zipSync", "zipsync")
  76. }
  77. type SourceFileGenerator interface {
  78. GeneratedSourceFiles() android.Paths
  79. GeneratedHeaderDirs() android.Paths
  80. GeneratedDeps() android.Paths
  81. }
  82. // Alias for android.HostToolProvider
  83. // Deprecated: use android.HostToolProvider instead.
  84. type HostToolProvider interface {
  85. android.HostToolProvider
  86. }
  87. type hostToolDependencyTag struct {
  88. blueprint.BaseDependencyTag
  89. android.LicenseAnnotationToolchainDependencyTag
  90. label string
  91. }
  92. func (t hostToolDependencyTag) AllowDisabledModuleDependency(target android.Module) bool {
  93. // Allow depending on a disabled module if it's replaced by a prebuilt
  94. // counterpart. We get the prebuilt through android.PrebuiltGetPreferred in
  95. // GenerateAndroidBuildActions.
  96. return target.IsReplacedByPrebuilt()
  97. }
  98. var _ android.AllowDisabledModuleDependency = (*hostToolDependencyTag)(nil)
  99. type generatorProperties struct {
  100. // The command to run on one or more input files. Cmd supports substitution of a few variables.
  101. //
  102. // Available variables for substitution:
  103. //
  104. // $(location): the path to the first entry in tools or tool_files.
  105. // $(location <label>): the path to the tool, tool_file, input or output with name <label>. Use $(location) if <label> refers to a rule that outputs exactly one file.
  106. // $(locations <label>): the paths to the tools, tool_files, inputs or outputs with name <label>. Use $(locations) if <label> refers to a rule that outputs two or more files.
  107. // $(in): one or more input files.
  108. // $(out): a single output file.
  109. // $(depfile): a file to which dependencies will be written, if the depfile property is set to true.
  110. // $(genDir): the sandbox directory for this tool; contains $(out).
  111. // $$: a literal $
  112. Cmd *string
  113. // Enable reading a file containing dependencies in gcc format after the command completes
  114. Depfile *bool
  115. // name of the modules (if any) that produces the host executable. Leave empty for
  116. // prebuilts or scripts that do not need a module to build them.
  117. Tools []string
  118. // Local file that is used as the tool
  119. Tool_files []string `android:"path"`
  120. // List of directories to export generated headers from
  121. Export_include_dirs []string
  122. // list of input files
  123. Srcs []string `android:"path,arch_variant"`
  124. // input files to exclude
  125. Exclude_srcs []string `android:"path,arch_variant"`
  126. }
  127. type Module struct {
  128. android.ModuleBase
  129. android.DefaultableModuleBase
  130. android.BazelModuleBase
  131. android.ApexModuleBase
  132. // For other packages to make their own genrules with extra
  133. // properties
  134. Extra interface{}
  135. // CmdModifier can be set by wrappers around genrule to modify the command, for example to
  136. // prefix environment variables to it.
  137. CmdModifier func(ctx android.ModuleContext, cmd string) string
  138. android.ImageInterface
  139. properties generatorProperties
  140. // For the different tasks that genrule and gensrc generate. genrule will
  141. // generate 1 task, and gensrc will generate 1 or more tasks based on the
  142. // number of shards the input files are sharded into.
  143. taskGenerator taskFunc
  144. rule blueprint.Rule
  145. rawCommands []string
  146. exportedIncludeDirs android.Paths
  147. outputFiles android.Paths
  148. outputDeps android.Paths
  149. subName string
  150. subDir string
  151. // Collect the module directory for IDE info in java/jdeps.go.
  152. modulePaths []string
  153. }
  154. var _ android.MixedBuildBuildable = (*Module)(nil)
  155. type taskFunc func(ctx android.ModuleContext, rawCommand string, srcFiles android.Paths) []generateTask
  156. type generateTask struct {
  157. in android.Paths
  158. out android.WritablePaths
  159. depFile android.WritablePath
  160. copyTo android.WritablePaths // For gensrcs to set on gensrcsMerge rule.
  161. genDir android.WritablePath
  162. extraTools android.Paths // dependencies on tools used by the generator
  163. extraInputs map[string][]string
  164. cmd string
  165. // For gensrsc sharding.
  166. shard int
  167. shards int
  168. }
  169. func (g *Module) GeneratedSourceFiles() android.Paths {
  170. return g.outputFiles
  171. }
  172. func (g *Module) Srcs() android.Paths {
  173. return append(android.Paths{}, g.outputFiles...)
  174. }
  175. func (g *Module) GeneratedHeaderDirs() android.Paths {
  176. return g.exportedIncludeDirs
  177. }
  178. func (g *Module) GeneratedDeps() android.Paths {
  179. return g.outputDeps
  180. }
  181. func (g *Module) OutputFiles(tag string) (android.Paths, error) {
  182. if tag == "" {
  183. return append(android.Paths{}, g.outputFiles...), nil
  184. }
  185. // otherwise, tag should match one of outputs
  186. for _, outputFile := range g.outputFiles {
  187. if outputFile.Rel() == tag {
  188. return android.Paths{outputFile}, nil
  189. }
  190. }
  191. return nil, fmt.Errorf("unsupported module reference tag %q", tag)
  192. }
  193. var _ android.SourceFileProducer = (*Module)(nil)
  194. var _ android.OutputFileProducer = (*Module)(nil)
  195. func toolDepsMutator(ctx android.BottomUpMutatorContext) {
  196. if g, ok := ctx.Module().(*Module); ok {
  197. for _, tool := range g.properties.Tools {
  198. tag := hostToolDependencyTag{label: tool}
  199. if m := android.SrcIsModule(tool); m != "" {
  200. tool = m
  201. }
  202. ctx.AddFarVariationDependencies(ctx.Config().BuildOSTarget.Variations(), tag, tool)
  203. }
  204. }
  205. }
  206. func (g *Module) ProcessBazelQueryResponse(ctx android.ModuleContext) {
  207. g.generateCommonBuildActions(ctx)
  208. label := g.GetBazelLabel(ctx, g)
  209. bazelCtx := ctx.Config().BazelContext
  210. filePaths, err := bazelCtx.GetOutputFiles(label, android.GetConfigKey(ctx))
  211. if err != nil {
  212. ctx.ModuleErrorf(err.Error())
  213. return
  214. }
  215. var bazelOutputFiles android.Paths
  216. exportIncludeDirs := map[string]bool{}
  217. for _, bazelOutputFile := range filePaths {
  218. bazelOutputFiles = append(bazelOutputFiles, android.PathForBazelOutRelative(ctx, ctx.ModuleDir(), bazelOutputFile))
  219. exportIncludeDirs[filepath.Dir(bazelOutputFile)] = true
  220. }
  221. g.outputFiles = bazelOutputFiles
  222. g.outputDeps = bazelOutputFiles
  223. for includePath, _ := range exportIncludeDirs {
  224. g.exportedIncludeDirs = append(g.exportedIncludeDirs, android.PathForBazelOut(ctx, includePath))
  225. }
  226. }
  227. // generateCommonBuildActions contains build action generation logic
  228. // common to both the mixed build case and the legacy case of genrule processing.
  229. // To fully support genrule in mixed builds, the contents of this function should
  230. // approach zero; there should be no genrule action registration done directly
  231. // by Soong logic in the mixed-build case.
  232. func (g *Module) generateCommonBuildActions(ctx android.ModuleContext) {
  233. g.subName = ctx.ModuleSubDir()
  234. // Collect the module directory for IDE info in java/jdeps.go.
  235. g.modulePaths = append(g.modulePaths, ctx.ModuleDir())
  236. if len(g.properties.Export_include_dirs) > 0 {
  237. for _, dir := range g.properties.Export_include_dirs {
  238. g.exportedIncludeDirs = append(g.exportedIncludeDirs,
  239. android.PathForModuleGen(ctx, g.subDir, ctx.ModuleDir(), dir))
  240. }
  241. } else {
  242. g.exportedIncludeDirs = append(g.exportedIncludeDirs, android.PathForModuleGen(ctx, g.subDir))
  243. }
  244. locationLabels := map[string]location{}
  245. firstLabel := ""
  246. addLocationLabel := func(label string, loc location) {
  247. if firstLabel == "" {
  248. firstLabel = label
  249. }
  250. if _, exists := locationLabels[label]; !exists {
  251. locationLabels[label] = loc
  252. } else {
  253. ctx.ModuleErrorf("multiple locations for label %q: %q and %q (do you have duplicate srcs entries?)",
  254. label, locationLabels[label], loc)
  255. }
  256. }
  257. var tools android.Paths
  258. var packagedTools []android.PackagingSpec
  259. if len(g.properties.Tools) > 0 {
  260. seenTools := make(map[string]bool)
  261. ctx.VisitDirectDepsBlueprint(func(module blueprint.Module) {
  262. switch tag := ctx.OtherModuleDependencyTag(module).(type) {
  263. case hostToolDependencyTag:
  264. tool := ctx.OtherModuleName(module)
  265. if m, ok := module.(android.Module); ok {
  266. // Necessary to retrieve any prebuilt replacement for the tool, since
  267. // toolDepsMutator runs too late for the prebuilt mutators to have
  268. // replaced the dependency.
  269. module = android.PrebuiltGetPreferred(ctx, m)
  270. }
  271. switch t := module.(type) {
  272. case android.HostToolProvider:
  273. // A HostToolProvider provides the path to a tool, which will be copied
  274. // into the sandbox.
  275. if !t.(android.Module).Enabled() {
  276. if ctx.Config().AllowMissingDependencies() {
  277. ctx.AddMissingDependencies([]string{tool})
  278. } else {
  279. ctx.ModuleErrorf("depends on disabled module %q", tool)
  280. }
  281. return
  282. }
  283. path := t.HostToolPath()
  284. if !path.Valid() {
  285. ctx.ModuleErrorf("host tool %q missing output file", tool)
  286. return
  287. }
  288. if specs := t.TransitivePackagingSpecs(); specs != nil {
  289. // If the HostToolProvider has PackgingSpecs, which are definitions of the
  290. // required relative locations of the tool and its dependencies, use those
  291. // instead. They will be copied to those relative locations in the sbox
  292. // sandbox.
  293. packagedTools = append(packagedTools, specs...)
  294. // Assume that the first PackagingSpec of the module is the tool.
  295. addLocationLabel(tag.label, packagedToolLocation{specs[0]})
  296. } else {
  297. tools = append(tools, path.Path())
  298. addLocationLabel(tag.label, toolLocation{android.Paths{path.Path()}})
  299. }
  300. case bootstrap.GoBinaryTool:
  301. // A GoBinaryTool provides the install path to a tool, which will be copied.
  302. p := android.PathForGoBinary(ctx, t)
  303. tools = append(tools, p)
  304. addLocationLabel(tag.label, toolLocation{android.Paths{p}})
  305. default:
  306. ctx.ModuleErrorf("%q is not a host tool provider", tool)
  307. return
  308. }
  309. seenTools[tag.label] = true
  310. }
  311. })
  312. // If AllowMissingDependencies is enabled, the build will not have stopped when
  313. // AddFarVariationDependencies was called on a missing tool, which will result in nonsensical
  314. // "cmd: unknown location label ..." errors later. Add a placeholder file to the local label.
  315. // The command that uses this placeholder file will never be executed because the rule will be
  316. // replaced with an android.Error rule reporting the missing dependencies.
  317. if ctx.Config().AllowMissingDependencies() {
  318. for _, tool := range g.properties.Tools {
  319. if !seenTools[tool] {
  320. addLocationLabel(tool, errorLocation{"***missing tool " + tool + "***"})
  321. }
  322. }
  323. }
  324. }
  325. if ctx.Failed() {
  326. return
  327. }
  328. for _, toolFile := range g.properties.Tool_files {
  329. paths := android.PathsForModuleSrc(ctx, []string{toolFile})
  330. tools = append(tools, paths...)
  331. addLocationLabel(toolFile, toolLocation{paths})
  332. }
  333. addLabelsForInputs := func(propName string, include, exclude []string) android.Paths {
  334. includeDirInPaths := ctx.DeviceConfig().BuildBrokenInputDir(g.Name())
  335. var srcFiles android.Paths
  336. for _, in := range include {
  337. paths, missingDeps := android.PathsAndMissingDepsRelativeToModuleSourceDir(android.SourceInput{
  338. Context: ctx, Paths: []string{in}, ExcludePaths: exclude, IncludeDirs: includeDirInPaths,
  339. })
  340. if len(missingDeps) > 0 {
  341. if !ctx.Config().AllowMissingDependencies() {
  342. panic(fmt.Errorf("should never get here, the missing dependencies %q should have been reported in DepsMutator",
  343. missingDeps))
  344. }
  345. // If AllowMissingDependencies is enabled, the build will not have stopped when
  346. // the dependency was added on a missing SourceFileProducer module, which will result in nonsensical
  347. // "cmd: label ":..." has no files" errors later. Add a placeholder file to the local label.
  348. // The command that uses this placeholder file will never be executed because the rule will be
  349. // replaced with an android.Error rule reporting the missing dependencies.
  350. ctx.AddMissingDependencies(missingDeps)
  351. addLocationLabel(in, errorLocation{"***missing " + propName + " " + in + "***"})
  352. } else {
  353. srcFiles = append(srcFiles, paths...)
  354. addLocationLabel(in, inputLocation{paths})
  355. }
  356. }
  357. return srcFiles
  358. }
  359. srcFiles := addLabelsForInputs("srcs", g.properties.Srcs, g.properties.Exclude_srcs)
  360. var copyFrom android.Paths
  361. var outputFiles android.WritablePaths
  362. var zipArgs strings.Builder
  363. cmd := String(g.properties.Cmd)
  364. if g.CmdModifier != nil {
  365. cmd = g.CmdModifier(ctx, cmd)
  366. }
  367. // Generate tasks, either from genrule or gensrcs.
  368. for i, task := range g.taskGenerator(ctx, cmd, srcFiles) {
  369. if len(task.out) == 0 {
  370. ctx.ModuleErrorf("must have at least one output file")
  371. return
  372. }
  373. var extraInputs android.Paths
  374. // Only handle extra inputs once as these currently are the same across all tasks
  375. if i == 0 {
  376. for name, values := range task.extraInputs {
  377. extraInputs = append(extraInputs, addLabelsForInputs(name, values, []string{})...)
  378. }
  379. }
  380. // Pick a unique path outside the task.genDir for the sbox manifest textproto,
  381. // a unique rule name, and the user-visible description.
  382. manifestName := "genrule.sbox.textproto"
  383. desc := "generate"
  384. name := "generator"
  385. if task.shards > 0 {
  386. manifestName = "genrule_" + strconv.Itoa(task.shard) + ".sbox.textproto"
  387. desc += " " + strconv.Itoa(task.shard)
  388. name += strconv.Itoa(task.shard)
  389. } else if len(task.out) == 1 {
  390. desc += " " + task.out[0].Base()
  391. }
  392. manifestPath := android.PathForModuleOut(ctx, manifestName)
  393. // Use a RuleBuilder to create a rule that runs the command inside an sbox sandbox.
  394. rule := getSandboxedRuleBuilder(ctx, android.NewRuleBuilder(pctx, ctx).Sbox(task.genDir, manifestPath))
  395. cmd := rule.Command()
  396. for _, out := range task.out {
  397. addLocationLabel(out.Rel(), outputLocation{out})
  398. }
  399. referencedDepfile := false
  400. rawCommand, err := android.Expand(task.cmd, func(name string) (string, error) {
  401. // report the error directly without returning an error to android.Expand to catch multiple errors in a
  402. // single run
  403. reportError := func(fmt string, args ...interface{}) (string, error) {
  404. ctx.PropertyErrorf("cmd", fmt, args...)
  405. return "SOONG_ERROR", nil
  406. }
  407. // Apply shell escape to each cases to prevent source file paths containing $ from being evaluated in shell
  408. switch name {
  409. case "location":
  410. if len(g.properties.Tools) == 0 && len(g.properties.Tool_files) == 0 {
  411. return reportError("at least one `tools` or `tool_files` is required if $(location) is used")
  412. }
  413. loc := locationLabels[firstLabel]
  414. paths := loc.Paths(cmd)
  415. if len(paths) == 0 {
  416. return reportError("default label %q has no files", firstLabel)
  417. } else if len(paths) > 1 {
  418. return reportError("default label %q has multiple files, use $(locations %s) to reference it",
  419. firstLabel, firstLabel)
  420. }
  421. return proptools.ShellEscape(paths[0]), nil
  422. case "in":
  423. return strings.Join(proptools.ShellEscapeList(cmd.PathsForInputs(srcFiles)), " "), nil
  424. case "out":
  425. var sandboxOuts []string
  426. for _, out := range task.out {
  427. sandboxOuts = append(sandboxOuts, cmd.PathForOutput(out))
  428. }
  429. return strings.Join(proptools.ShellEscapeList(sandboxOuts), " "), nil
  430. case "depfile":
  431. referencedDepfile = true
  432. if !Bool(g.properties.Depfile) {
  433. return reportError("$(depfile) used without depfile property")
  434. }
  435. return "__SBOX_DEPFILE__", nil
  436. case "genDir":
  437. return proptools.ShellEscape(cmd.PathForOutput(task.genDir)), nil
  438. default:
  439. if strings.HasPrefix(name, "location ") {
  440. label := strings.TrimSpace(strings.TrimPrefix(name, "location "))
  441. if loc, ok := locationLabels[label]; ok {
  442. paths := loc.Paths(cmd)
  443. if len(paths) == 0 {
  444. return reportError("label %q has no files", label)
  445. } else if len(paths) > 1 {
  446. return reportError("label %q has multiple files, use $(locations %s) to reference it",
  447. label, label)
  448. }
  449. return proptools.ShellEscape(paths[0]), nil
  450. } else {
  451. return reportError("unknown location label %q is not in srcs, out, tools or tool_files.", label)
  452. }
  453. } else if strings.HasPrefix(name, "locations ") {
  454. label := strings.TrimSpace(strings.TrimPrefix(name, "locations "))
  455. if loc, ok := locationLabels[label]; ok {
  456. paths := loc.Paths(cmd)
  457. if len(paths) == 0 {
  458. return reportError("label %q has no files", label)
  459. }
  460. return proptools.ShellEscape(strings.Join(paths, " ")), nil
  461. } else {
  462. return reportError("unknown locations label %q is not in srcs, out, tools or tool_files.", label)
  463. }
  464. } else {
  465. return reportError("unknown variable '$(%s)'", name)
  466. }
  467. }
  468. })
  469. if err != nil {
  470. ctx.PropertyErrorf("cmd", "%s", err.Error())
  471. return
  472. }
  473. if Bool(g.properties.Depfile) && !referencedDepfile {
  474. ctx.PropertyErrorf("cmd", "specified depfile=true but did not include a reference to '${depfile}' in cmd")
  475. return
  476. }
  477. g.rawCommands = append(g.rawCommands, rawCommand)
  478. cmd.Text(rawCommand)
  479. cmd.Implicits(srcFiles) // need to be able to reference other srcs
  480. cmd.Implicits(extraInputs)
  481. cmd.ImplicitOutputs(task.out)
  482. cmd.Implicits(task.in)
  483. cmd.ImplicitTools(tools)
  484. cmd.ImplicitTools(task.extraTools)
  485. cmd.ImplicitPackagedTools(packagedTools)
  486. if Bool(g.properties.Depfile) {
  487. cmd.ImplicitDepFile(task.depFile)
  488. }
  489. // Create the rule to run the genrule command inside sbox.
  490. rule.Build(name, desc)
  491. if len(task.copyTo) > 0 {
  492. // If copyTo is set, multiple shards need to be copied into a single directory.
  493. // task.out contains the per-shard paths, and copyTo contains the corresponding
  494. // final path. The files need to be copied into the final directory by a
  495. // single rule so it can remove the directory before it starts to ensure no
  496. // old files remain. zipsync already does this, so build up zipArgs that
  497. // zip all the per-shard directories into a single zip.
  498. outputFiles = append(outputFiles, task.copyTo...)
  499. copyFrom = append(copyFrom, task.out.Paths()...)
  500. zipArgs.WriteString(" -C " + task.genDir.String())
  501. zipArgs.WriteString(android.JoinWithPrefix(task.out.Strings(), " -f "))
  502. } else {
  503. outputFiles = append(outputFiles, task.out...)
  504. }
  505. }
  506. if len(copyFrom) > 0 {
  507. // Create a rule that zips all the per-shard directories into a single zip and then
  508. // uses zipsync to unzip it into the final directory.
  509. ctx.Build(pctx, android.BuildParams{
  510. Rule: gensrcsMerge,
  511. Implicits: copyFrom,
  512. Outputs: outputFiles,
  513. Description: "merge shards",
  514. Args: map[string]string{
  515. "zipArgs": zipArgs.String(),
  516. "tmpZip": android.PathForModuleGen(ctx, g.subDir+".zip").String(),
  517. "genDir": android.PathForModuleGen(ctx, g.subDir).String(),
  518. },
  519. })
  520. }
  521. g.outputFiles = outputFiles.Paths()
  522. }
  523. func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) {
  524. // Allowlist genrule to use depfile until we have a solution to remove it.
  525. // TODO(b/235582219): Remove allowlist for genrule
  526. if Bool(g.properties.Depfile) {
  527. sandboxingAllowlistSets := getSandboxingAllowlistSets(ctx)
  528. // TODO(b/283852474): Checking the GenruleSandboxing flag is temporary in
  529. // order to pass the presubmit before internal master is updated.
  530. if ctx.DeviceConfig().GenruleSandboxing() && !sandboxingAllowlistSets.depfileAllowSet[g.Name()] {
  531. ctx.PropertyErrorf(
  532. "depfile",
  533. "Deprecated to ensure the module type is convertible to Bazel. "+
  534. "Try specifying the dependencies explicitly so that there is no need to use depfile. "+
  535. "If not possible, the escape hatch is to add the module to allowlists.go to bypass the error.")
  536. }
  537. }
  538. g.generateCommonBuildActions(ctx)
  539. // For <= 6 outputs, just embed those directly in the users. Right now, that covers >90% of
  540. // the genrules on AOSP. That will make things simpler to look at the graph in the common
  541. // case. For larger sets of outputs, inject a phony target in between to limit ninja file
  542. // growth.
  543. if len(g.outputFiles) <= 6 {
  544. g.outputDeps = g.outputFiles
  545. } else {
  546. phonyFile := android.PathForModuleGen(ctx, "genrule-phony")
  547. ctx.Build(pctx, android.BuildParams{
  548. Rule: blueprint.Phony,
  549. Output: phonyFile,
  550. Inputs: g.outputFiles,
  551. })
  552. g.outputDeps = android.Paths{phonyFile}
  553. }
  554. }
  555. func (g *Module) QueueBazelCall(ctx android.BaseModuleContext) {
  556. bazelCtx := ctx.Config().BazelContext
  557. bazelCtx.QueueBazelRequest(g.GetBazelLabel(ctx, g), cquery.GetOutputFiles, android.GetConfigKey(ctx))
  558. }
  559. func (g *Module) IsMixedBuildSupported(ctx android.BaseModuleContext) bool {
  560. return true
  561. }
  562. // Collect information for opening IDE project files in java/jdeps.go.
  563. func (g *Module) IDEInfo(dpInfo *android.IdeInfo) {
  564. dpInfo.Srcs = append(dpInfo.Srcs, g.Srcs().Strings()...)
  565. for _, src := range g.properties.Srcs {
  566. if strings.HasPrefix(src, ":") {
  567. src = strings.Trim(src, ":")
  568. dpInfo.Deps = append(dpInfo.Deps, src)
  569. }
  570. }
  571. dpInfo.Paths = append(dpInfo.Paths, g.modulePaths...)
  572. }
  573. func (g *Module) AndroidMk() android.AndroidMkData {
  574. return android.AndroidMkData{
  575. Class: "ETC",
  576. OutputFile: android.OptionalPathForPath(g.outputFiles[0]),
  577. SubName: g.subName,
  578. Extra: []android.AndroidMkExtraFunc{
  579. func(w io.Writer, outputFile android.Path) {
  580. fmt.Fprintln(w, "LOCAL_UNINSTALLABLE_MODULE := true")
  581. },
  582. },
  583. Custom: func(w io.Writer, name, prefix, moduleDir string, data android.AndroidMkData) {
  584. android.WriteAndroidMkData(w, data)
  585. if data.SubName != "" {
  586. fmt.Fprintln(w, ".PHONY:", name)
  587. fmt.Fprintln(w, name, ":", name+g.subName)
  588. }
  589. },
  590. }
  591. }
  592. var _ android.ApexModule = (*Module)(nil)
  593. // Implements android.ApexModule
  594. func (g *Module) ShouldSupportSdkVersion(ctx android.BaseModuleContext,
  595. sdkVersion android.ApiLevel) error {
  596. // Because generated outputs are checked by client modules(e.g. cc_library, ...)
  597. // we can safely ignore the check here.
  598. return nil
  599. }
  600. func generatorFactory(taskGenerator taskFunc, props ...interface{}) *Module {
  601. module := &Module{
  602. taskGenerator: taskGenerator,
  603. }
  604. module.AddProperties(props...)
  605. module.AddProperties(&module.properties)
  606. module.ImageInterface = noopImageInterface{}
  607. return module
  608. }
  609. type noopImageInterface struct{}
  610. func (x noopImageInterface) ImageMutatorBegin(android.BaseModuleContext) {}
  611. func (x noopImageInterface) CoreVariantNeeded(android.BaseModuleContext) bool { return false }
  612. func (x noopImageInterface) RamdiskVariantNeeded(android.BaseModuleContext) bool { return false }
  613. func (x noopImageInterface) VendorRamdiskVariantNeeded(android.BaseModuleContext) bool { return false }
  614. func (x noopImageInterface) DebugRamdiskVariantNeeded(android.BaseModuleContext) bool { return false }
  615. func (x noopImageInterface) RecoveryVariantNeeded(android.BaseModuleContext) bool { return false }
  616. func (x noopImageInterface) ExtraImageVariations(ctx android.BaseModuleContext) []string { return nil }
  617. func (x noopImageInterface) SetImageVariation(ctx android.BaseModuleContext, variation string, module android.Module) {
  618. }
  619. func NewGenSrcs() *Module {
  620. properties := &genSrcsProperties{}
  621. // finalSubDir is the name of the subdirectory that output files will be generated into.
  622. // It is used so that per-shard directories can be placed alongside it an then finally
  623. // merged into it.
  624. const finalSubDir = "gensrcs"
  625. taskGenerator := func(ctx android.ModuleContext, rawCommand string, srcFiles android.Paths) []generateTask {
  626. shardSize := defaultShardSize
  627. if s := properties.Shard_size; s != nil {
  628. shardSize = int(*s)
  629. }
  630. // gensrcs rules can easily hit command line limits by repeating the command for
  631. // every input file. Shard the input files into groups.
  632. shards := android.ShardPaths(srcFiles, shardSize)
  633. var generateTasks []generateTask
  634. for i, shard := range shards {
  635. var commands []string
  636. var outFiles android.WritablePaths
  637. var commandDepFiles []string
  638. var copyTo android.WritablePaths
  639. // When sharding is enabled (i.e. len(shards) > 1), the sbox rules for each
  640. // shard will be write to their own directories and then be merged together
  641. // into finalSubDir. If sharding is not enabled (i.e. len(shards) == 1),
  642. // the sbox rule will write directly to finalSubDir.
  643. genSubDir := finalSubDir
  644. if len(shards) > 1 {
  645. genSubDir = strconv.Itoa(i)
  646. }
  647. genDir := android.PathForModuleGen(ctx, genSubDir)
  648. // TODO(ccross): this RuleBuilder is a hack to be able to call
  649. // rule.Command().PathForOutput. Replace this with passing the rule into the
  650. // generator.
  651. rule := getSandboxedRuleBuilder(ctx, android.NewRuleBuilder(pctx, ctx).Sbox(genDir, nil))
  652. for _, in := range shard {
  653. outFile := android.GenPathWithExt(ctx, finalSubDir, in, String(properties.Output_extension))
  654. // If sharding is enabled, then outFile is the path to the output file in
  655. // the shard directory, and copyTo is the path to the output file in the
  656. // final directory.
  657. if len(shards) > 1 {
  658. shardFile := android.GenPathWithExt(ctx, genSubDir, in, String(properties.Output_extension))
  659. copyTo = append(copyTo, outFile)
  660. outFile = shardFile
  661. }
  662. outFiles = append(outFiles, outFile)
  663. // pre-expand the command line to replace $in and $out with references to
  664. // a single input and output file.
  665. command, err := android.Expand(rawCommand, func(name string) (string, error) {
  666. switch name {
  667. case "in":
  668. return in.String(), nil
  669. case "out":
  670. return rule.Command().PathForOutput(outFile), nil
  671. case "depfile":
  672. // Generate a depfile for each output file. Store the list for
  673. // later in order to combine them all into a single depfile.
  674. depFile := rule.Command().PathForOutput(outFile.ReplaceExtension(ctx, "d"))
  675. commandDepFiles = append(commandDepFiles, depFile)
  676. return depFile, nil
  677. default:
  678. return "$(" + name + ")", nil
  679. }
  680. })
  681. if err != nil {
  682. ctx.PropertyErrorf("cmd", err.Error())
  683. }
  684. // escape the command in case for example it contains '#', an odd number of '"', etc
  685. command = fmt.Sprintf("bash -c %v", proptools.ShellEscape(command))
  686. commands = append(commands, command)
  687. }
  688. fullCommand := strings.Join(commands, " && ")
  689. var outputDepfile android.WritablePath
  690. var extraTools android.Paths
  691. if len(commandDepFiles) > 0 {
  692. // Each command wrote to a depfile, but ninja can only handle one
  693. // depfile per rule. Use the dep_fixer tool at the end of the
  694. // command to combine all the depfiles into a single output depfile.
  695. outputDepfile = android.PathForModuleGen(ctx, genSubDir, "gensrcs.d")
  696. depFixerTool := ctx.Config().HostToolPath(ctx, "dep_fixer")
  697. fullCommand += fmt.Sprintf(" && %s -o $(depfile) %s",
  698. rule.Command().PathForTool(depFixerTool),
  699. strings.Join(commandDepFiles, " "))
  700. extraTools = append(extraTools, depFixerTool)
  701. }
  702. generateTasks = append(generateTasks, generateTask{
  703. in: shard,
  704. out: outFiles,
  705. depFile: outputDepfile,
  706. copyTo: copyTo,
  707. genDir: genDir,
  708. cmd: fullCommand,
  709. shard: i,
  710. shards: len(shards),
  711. extraTools: extraTools,
  712. extraInputs: map[string][]string{
  713. "data": properties.Data,
  714. },
  715. })
  716. }
  717. return generateTasks
  718. }
  719. g := generatorFactory(taskGenerator, properties)
  720. g.subDir = finalSubDir
  721. return g
  722. }
  723. func GenSrcsFactory() android.Module {
  724. m := NewGenSrcs()
  725. android.InitAndroidModule(m)
  726. android.InitBazelModule(m)
  727. return m
  728. }
  729. type genSrcsProperties struct {
  730. // extension that will be substituted for each output file
  731. Output_extension *string
  732. // maximum number of files that will be passed on a single command line.
  733. Shard_size *int64
  734. // Additional files needed for build that are not tooling related.
  735. Data []string `android:"path"`
  736. }
  737. type bazelGensrcsAttributes struct {
  738. Srcs bazel.LabelListAttribute
  739. Output_extension *string
  740. Tools bazel.LabelListAttribute
  741. Cmd string
  742. Data bazel.LabelListAttribute
  743. }
  744. const defaultShardSize = 50
  745. func NewGenRule() *Module {
  746. properties := &genRuleProperties{}
  747. taskGenerator := func(ctx android.ModuleContext, rawCommand string, srcFiles android.Paths) []generateTask {
  748. outs := make(android.WritablePaths, len(properties.Out))
  749. var depFile android.WritablePath
  750. for i, out := range properties.Out {
  751. outPath := android.PathForModuleGen(ctx, out)
  752. if i == 0 {
  753. depFile = outPath.ReplaceExtension(ctx, "d")
  754. }
  755. outs[i] = outPath
  756. }
  757. return []generateTask{{
  758. in: srcFiles,
  759. out: outs,
  760. depFile: depFile,
  761. genDir: android.PathForModuleGen(ctx),
  762. cmd: rawCommand,
  763. }}
  764. }
  765. return generatorFactory(taskGenerator, properties)
  766. }
  767. func GenRuleFactory() android.Module {
  768. m := NewGenRule()
  769. android.InitAndroidModule(m)
  770. android.InitDefaultableModule(m)
  771. android.InitBazelModule(m)
  772. return m
  773. }
  774. type genRuleProperties struct {
  775. // names of the output files that will be generated
  776. Out []string
  777. }
  778. type bazelGenruleAttributes struct {
  779. Srcs bazel.LabelListAttribute
  780. Outs []string
  781. Tools bazel.LabelListAttribute
  782. Cmd string
  783. }
  784. // ConvertWithBp2build converts a Soong module -> Bazel target.
  785. func (m *Module) ConvertWithBp2build(ctx android.TopDownMutatorContext) {
  786. // Bazel only has the "tools" attribute.
  787. tools_prop := android.BazelLabelForModuleDeps(ctx, m.properties.Tools)
  788. tool_files_prop := android.BazelLabelForModuleSrc(ctx, m.properties.Tool_files)
  789. tools_prop.Append(tool_files_prop)
  790. tools := bazel.MakeLabelListAttribute(tools_prop)
  791. srcs := bazel.LabelListAttribute{}
  792. srcs_labels := bazel.LabelList{}
  793. // Only cc_genrule is arch specific
  794. if ctx.ModuleType() == "cc_genrule" {
  795. for axis, configToProps := range m.GetArchVariantProperties(ctx, &generatorProperties{}) {
  796. for config, props := range configToProps {
  797. if props, ok := props.(*generatorProperties); ok {
  798. labels := android.BazelLabelForModuleSrcExcludes(ctx, props.Srcs, props.Exclude_srcs)
  799. srcs_labels.Append(labels)
  800. srcs.SetSelectValue(axis, config, labels)
  801. }
  802. }
  803. }
  804. } else {
  805. srcs_labels = android.BazelLabelForModuleSrcExcludes(ctx, m.properties.Srcs, m.properties.Exclude_srcs)
  806. srcs = bazel.MakeLabelListAttribute(srcs_labels)
  807. }
  808. var allReplacements bazel.LabelList
  809. allReplacements.Append(tools.Value)
  810. allReplacements.Append(bazel.FirstUniqueBazelLabelList(srcs_labels))
  811. // The Output_extension prop is not in an immediately accessible field
  812. // in the Module struct, so use GetProperties and cast it
  813. // to the known struct prop.
  814. var outputExtension *string
  815. var data bazel.LabelListAttribute
  816. if ctx.ModuleType() == "gensrcs" {
  817. for _, propIntf := range m.GetProperties() {
  818. if props, ok := propIntf.(*genSrcsProperties); ok {
  819. outputExtension = props.Output_extension
  820. dataFiles := android.BazelLabelForModuleSrc(ctx, props.Data)
  821. allReplacements.Append(bazel.FirstUniqueBazelLabelList(dataFiles))
  822. data = bazel.MakeLabelListAttribute(dataFiles)
  823. break
  824. }
  825. }
  826. }
  827. // Replace in and out variables with $< and $@
  828. var cmd string
  829. if m.properties.Cmd != nil {
  830. if ctx.ModuleType() == "gensrcs" {
  831. cmd = strings.ReplaceAll(*m.properties.Cmd, "$(in)", "$(SRC)")
  832. cmd = strings.ReplaceAll(cmd, "$(out)", "$(OUT)")
  833. } else {
  834. cmd = strings.Replace(*m.properties.Cmd, "$(in)", "$(SRCS)", -1)
  835. cmd = strings.Replace(cmd, "$(out)", "$(OUTS)", -1)
  836. }
  837. cmd = strings.Replace(cmd, "$(genDir)", "$(RULEDIR)", -1)
  838. if len(tools.Value.Includes) > 0 {
  839. cmd = strings.Replace(cmd, "$(location)", fmt.Sprintf("$(location %s)", tools.Value.Includes[0].Label), -1)
  840. cmd = strings.Replace(cmd, "$(locations)", fmt.Sprintf("$(locations %s)", tools.Value.Includes[0].Label), -1)
  841. }
  842. for _, l := range allReplacements.Includes {
  843. bpLoc := fmt.Sprintf("$(location %s)", l.OriginalModuleName)
  844. bpLocs := fmt.Sprintf("$(locations %s)", l.OriginalModuleName)
  845. bazelLoc := fmt.Sprintf("$(location %s)", l.Label)
  846. bazelLocs := fmt.Sprintf("$(locations %s)", l.Label)
  847. cmd = strings.Replace(cmd, bpLoc, bazelLoc, -1)
  848. cmd = strings.Replace(cmd, bpLocs, bazelLocs, -1)
  849. }
  850. }
  851. tags := android.ApexAvailableTagsWithoutTestApexes(ctx, m)
  852. if ctx.ModuleType() == "gensrcs" {
  853. props := bazel.BazelTargetModuleProperties{
  854. Rule_class: "gensrcs",
  855. Bzl_load_location: "//build/bazel/rules:gensrcs.bzl",
  856. }
  857. attrs := &bazelGensrcsAttributes{
  858. Srcs: srcs,
  859. Output_extension: outputExtension,
  860. Cmd: cmd,
  861. Tools: tools,
  862. Data: data,
  863. }
  864. ctx.CreateBazelTargetModule(props, android.CommonAttributes{
  865. Name: m.Name(),
  866. Tags: tags,
  867. }, attrs)
  868. } else {
  869. // The Out prop is not in an immediately accessible field
  870. // in the Module struct, so use GetProperties and cast it
  871. // to the known struct prop.
  872. var outs []string
  873. for _, propIntf := range m.GetProperties() {
  874. if props, ok := propIntf.(*genRuleProperties); ok {
  875. outs = props.Out
  876. break
  877. }
  878. }
  879. bazelName := m.Name()
  880. for _, out := range outs {
  881. if out == bazelName {
  882. // This is a workaround to circumvent a Bazel warning where a genrule's
  883. // out may not have the same name as the target itself. This makes no
  884. // difference for reverse dependencies, because they may depend on the
  885. // out file by name.
  886. bazelName = bazelName + "-gen"
  887. break
  888. }
  889. }
  890. attrs := &bazelGenruleAttributes{
  891. Srcs: srcs,
  892. Outs: outs,
  893. Cmd: cmd,
  894. Tools: tools,
  895. }
  896. props := bazel.BazelTargetModuleProperties{
  897. Rule_class: "genrule",
  898. }
  899. ctx.CreateBazelTargetModule(props, android.CommonAttributes{
  900. Name: bazelName,
  901. Tags: tags,
  902. }, attrs)
  903. }
  904. }
  905. var Bool = proptools.Bool
  906. var String = proptools.String
  907. // Defaults
  908. type Defaults struct {
  909. android.ModuleBase
  910. android.DefaultsModuleBase
  911. }
  912. func defaultsFactory() android.Module {
  913. return DefaultsFactory()
  914. }
  915. func DefaultsFactory(props ...interface{}) android.Module {
  916. module := &Defaults{}
  917. module.AddProperties(props...)
  918. module.AddProperties(
  919. &generatorProperties{},
  920. &genRuleProperties{},
  921. )
  922. android.InitDefaultsModule(module)
  923. return module
  924. }
  925. var sandboxingAllowlistKey = android.NewOnceKey("genruleSandboxingAllowlistKey")
  926. type sandboxingAllowlistSets struct {
  927. sandboxingDenyModuleSet map[string]bool
  928. sandboxingDenyPathSet map[string]bool
  929. depfileAllowSet map[string]bool
  930. }
  931. func getSandboxingAllowlistSets(ctx android.PathContext) *sandboxingAllowlistSets {
  932. return ctx.Config().Once(sandboxingAllowlistKey, func() interface{} {
  933. sandboxingDenyModuleSet := map[string]bool{}
  934. sandboxingDenyPathSet := map[string]bool{}
  935. depfileAllowSet := map[string]bool{}
  936. android.AddToStringSet(sandboxingDenyModuleSet, append(DepfileAllowList, SandboxingDenyModuleList...))
  937. android.AddToStringSet(sandboxingDenyPathSet, SandboxingDenyPathList)
  938. android.AddToStringSet(depfileAllowSet, DepfileAllowList)
  939. return &sandboxingAllowlistSets{
  940. sandboxingDenyModuleSet: sandboxingDenyModuleSet,
  941. sandboxingDenyPathSet: sandboxingDenyPathSet,
  942. depfileAllowSet: depfileAllowSet,
  943. }
  944. }).(*sandboxingAllowlistSets)
  945. }
  946. func getSandboxedRuleBuilder(ctx android.ModuleContext, r *android.RuleBuilder) *android.RuleBuilder {
  947. if !ctx.DeviceConfig().GenruleSandboxing() {
  948. return r.SandboxTools()
  949. }
  950. sandboxingAllowlistSets := getSandboxingAllowlistSets(ctx)
  951. if sandboxingAllowlistSets.sandboxingDenyPathSet[ctx.ModuleDir()] ||
  952. sandboxingAllowlistSets.sandboxingDenyModuleSet[ctx.ModuleName()] {
  953. return r.SandboxTools()
  954. }
  955. return r.SandboxInputs()
  956. }