rule_builder.go 48 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366
  1. // Copyright 2018 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package android
  15. import (
  16. "crypto/sha256"
  17. "fmt"
  18. "path/filepath"
  19. "sort"
  20. "strings"
  21. "testing"
  22. "github.com/google/blueprint"
  23. "github.com/google/blueprint/proptools"
  24. "google.golang.org/protobuf/encoding/prototext"
  25. "google.golang.org/protobuf/proto"
  26. "android/soong/cmd/sbox/sbox_proto"
  27. "android/soong/remoteexec"
  28. "android/soong/response"
  29. "android/soong/shared"
  30. )
  31. const sboxSandboxBaseDir = "__SBOX_SANDBOX_DIR__"
  32. const sboxOutSubDir = "out"
  33. const sboxToolsSubDir = "tools"
  34. const sboxOutDir = sboxSandboxBaseDir + "/" + sboxOutSubDir
  35. // RuleBuilder provides an alternative to ModuleContext.Rule and ModuleContext.Build to add a command line to the build
  36. // graph.
  37. type RuleBuilder struct {
  38. pctx PackageContext
  39. ctx BuilderContext
  40. commands []*RuleBuilderCommand
  41. installs RuleBuilderInstalls
  42. temporariesSet map[WritablePath]bool
  43. restat bool
  44. sbox bool
  45. highmem bool
  46. remoteable RemoteRuleSupports
  47. rbeParams *remoteexec.REParams
  48. outDir WritablePath
  49. sboxTools bool
  50. sboxInputs bool
  51. sboxManifestPath WritablePath
  52. missingDeps []string
  53. }
  54. // NewRuleBuilder returns a newly created RuleBuilder.
  55. func NewRuleBuilder(pctx PackageContext, ctx BuilderContext) *RuleBuilder {
  56. return &RuleBuilder{
  57. pctx: pctx,
  58. ctx: ctx,
  59. temporariesSet: make(map[WritablePath]bool),
  60. }
  61. }
  62. // RuleBuilderInstall is a tuple of install from and to locations.
  63. type RuleBuilderInstall struct {
  64. From Path
  65. To string
  66. }
  67. type RuleBuilderInstalls []RuleBuilderInstall
  68. // String returns the RuleBuilderInstalls in the form used by $(call copy-many-files) in Make, a space separated
  69. // list of from:to tuples.
  70. func (installs RuleBuilderInstalls) String() string {
  71. sb := strings.Builder{}
  72. for i, install := range installs {
  73. if i != 0 {
  74. sb.WriteRune(' ')
  75. }
  76. sb.WriteString(install.From.String())
  77. sb.WriteRune(':')
  78. sb.WriteString(install.To)
  79. }
  80. return sb.String()
  81. }
  82. // MissingDeps adds modules to the list of missing dependencies. If MissingDeps
  83. // is called with a non-empty input, any call to Build will result in a rule
  84. // that will print an error listing the missing dependencies and fail.
  85. // MissingDeps should only be called if Config.AllowMissingDependencies() is
  86. // true.
  87. func (r *RuleBuilder) MissingDeps(missingDeps []string) {
  88. r.missingDeps = append(r.missingDeps, missingDeps...)
  89. }
  90. // Restat marks the rule as a restat rule, which will be passed to ModuleContext.Rule in BuildParams.Restat.
  91. func (r *RuleBuilder) Restat() *RuleBuilder {
  92. r.restat = true
  93. return r
  94. }
  95. // HighMem marks the rule as a high memory rule, which will limit how many run in parallel with other high memory
  96. // rules.
  97. func (r *RuleBuilder) HighMem() *RuleBuilder {
  98. r.highmem = true
  99. return r
  100. }
  101. // Remoteable marks the rule as supporting remote execution.
  102. func (r *RuleBuilder) Remoteable(supports RemoteRuleSupports) *RuleBuilder {
  103. r.remoteable = supports
  104. return r
  105. }
  106. // Rewrapper marks the rule as running inside rewrapper using the given params in order to support
  107. // running on RBE. During RuleBuilder.Build the params will be combined with the inputs, outputs
  108. // and tools known to RuleBuilder to prepend an appropriate rewrapper command line to the rule's
  109. // command line.
  110. func (r *RuleBuilder) Rewrapper(params *remoteexec.REParams) *RuleBuilder {
  111. if !r.sboxInputs {
  112. panic(fmt.Errorf("RuleBuilder.Rewrapper must be called after RuleBuilder.SandboxInputs"))
  113. }
  114. r.rbeParams = params
  115. return r
  116. }
  117. // Sbox marks the rule as needing to be wrapped by sbox. The outputDir should point to the output
  118. // directory that sbox will wipe. It should not be written to by any other rule. manifestPath should
  119. // point to a location where sbox's manifest will be written and must be outside outputDir. sbox
  120. // will ensure that all outputs have been written, and will discard any output files that were not
  121. // specified.
  122. func (r *RuleBuilder) Sbox(outputDir WritablePath, manifestPath WritablePath) *RuleBuilder {
  123. if r.sbox {
  124. panic("Sbox() may not be called more than once")
  125. }
  126. if len(r.commands) > 0 {
  127. panic("Sbox() may not be called after Command()")
  128. }
  129. r.sbox = true
  130. r.outDir = outputDir
  131. r.sboxManifestPath = manifestPath
  132. return r
  133. }
  134. // SandboxTools enables tool sandboxing for the rule by copying any referenced tools into the
  135. // sandbox.
  136. func (r *RuleBuilder) SandboxTools() *RuleBuilder {
  137. if !r.sbox {
  138. panic("SandboxTools() must be called after Sbox()")
  139. }
  140. if len(r.commands) > 0 {
  141. panic("SandboxTools() may not be called after Command()")
  142. }
  143. r.sboxTools = true
  144. return r
  145. }
  146. // SandboxInputs enables input sandboxing for the rule by copying any referenced inputs into the
  147. // sandbox. It also implies SandboxTools().
  148. //
  149. // Sandboxing inputs requires RuleBuilder to be aware of all references to input paths. Paths
  150. // that are passed to RuleBuilder outside of the methods that expect inputs, for example
  151. // FlagWithArg, must use RuleBuilderCommand.PathForInput to translate the path to one that matches
  152. // the sandbox layout.
  153. func (r *RuleBuilder) SandboxInputs() *RuleBuilder {
  154. if !r.sbox {
  155. panic("SandboxInputs() must be called after Sbox()")
  156. }
  157. if len(r.commands) > 0 {
  158. panic("SandboxInputs() may not be called after Command()")
  159. }
  160. r.sboxTools = true
  161. r.sboxInputs = true
  162. return r
  163. }
  164. // Install associates an output of the rule with an install location, which can be retrieved later using
  165. // RuleBuilder.Installs.
  166. func (r *RuleBuilder) Install(from Path, to string) {
  167. r.installs = append(r.installs, RuleBuilderInstall{from, to})
  168. }
  169. // Command returns a new RuleBuilderCommand for the rule. The commands will be ordered in the rule by when they were
  170. // created by this method. That can be mutated through their methods in any order, as long as the mutations do not
  171. // race with any call to Build.
  172. func (r *RuleBuilder) Command() *RuleBuilderCommand {
  173. command := &RuleBuilderCommand{
  174. rule: r,
  175. }
  176. r.commands = append(r.commands, command)
  177. return command
  178. }
  179. // Temporary marks an output of a command as an intermediate file that will be used as an input to another command
  180. // in the same rule, and should not be listed in Outputs.
  181. func (r *RuleBuilder) Temporary(path WritablePath) {
  182. r.temporariesSet[path] = true
  183. }
  184. // DeleteTemporaryFiles adds a command to the rule that deletes any outputs that have been marked using Temporary
  185. // when the rule runs. DeleteTemporaryFiles should be called after all calls to Temporary.
  186. func (r *RuleBuilder) DeleteTemporaryFiles() {
  187. var temporariesList WritablePaths
  188. for intermediate := range r.temporariesSet {
  189. temporariesList = append(temporariesList, intermediate)
  190. }
  191. sort.Slice(temporariesList, func(i, j int) bool {
  192. return temporariesList[i].String() < temporariesList[j].String()
  193. })
  194. r.Command().Text("rm").Flag("-f").Outputs(temporariesList)
  195. }
  196. // Inputs returns the list of paths that were passed to the RuleBuilderCommand methods that take
  197. // input paths, such as RuleBuilderCommand.Input, RuleBuilderCommand.Implicit, or
  198. // RuleBuilderCommand.FlagWithInput. Inputs to a command that are also outputs of another command
  199. // in the same RuleBuilder are filtered out. The list is sorted and duplicates removed.
  200. func (r *RuleBuilder) Inputs() Paths {
  201. outputs := r.outputSet()
  202. depFiles := r.depFileSet()
  203. inputs := make(map[string]Path)
  204. for _, c := range r.commands {
  205. for _, input := range append(c.inputs, c.implicits...) {
  206. inputStr := input.String()
  207. if _, isOutput := outputs[inputStr]; !isOutput {
  208. if _, isDepFile := depFiles[inputStr]; !isDepFile {
  209. inputs[input.String()] = input
  210. }
  211. }
  212. }
  213. }
  214. var inputList Paths
  215. for _, input := range inputs {
  216. inputList = append(inputList, input)
  217. }
  218. sort.Slice(inputList, func(i, j int) bool {
  219. return inputList[i].String() < inputList[j].String()
  220. })
  221. return inputList
  222. }
  223. // OrderOnlys returns the list of paths that were passed to the RuleBuilderCommand.OrderOnly or
  224. // RuleBuilderCommand.OrderOnlys. The list is sorted and duplicates removed.
  225. func (r *RuleBuilder) OrderOnlys() Paths {
  226. orderOnlys := make(map[string]Path)
  227. for _, c := range r.commands {
  228. for _, orderOnly := range c.orderOnlys {
  229. orderOnlys[orderOnly.String()] = orderOnly
  230. }
  231. }
  232. var orderOnlyList Paths
  233. for _, orderOnly := range orderOnlys {
  234. orderOnlyList = append(orderOnlyList, orderOnly)
  235. }
  236. sort.Slice(orderOnlyList, func(i, j int) bool {
  237. return orderOnlyList[i].String() < orderOnlyList[j].String()
  238. })
  239. return orderOnlyList
  240. }
  241. // Validations returns the list of paths that were passed to RuleBuilderCommand.Validation or
  242. // RuleBuilderCommand.Validations. The list is sorted and duplicates removed.
  243. func (r *RuleBuilder) Validations() Paths {
  244. validations := make(map[string]Path)
  245. for _, c := range r.commands {
  246. for _, validation := range c.validations {
  247. validations[validation.String()] = validation
  248. }
  249. }
  250. var validationList Paths
  251. for _, validation := range validations {
  252. validationList = append(validationList, validation)
  253. }
  254. sort.Slice(validationList, func(i, j int) bool {
  255. return validationList[i].String() < validationList[j].String()
  256. })
  257. return validationList
  258. }
  259. func (r *RuleBuilder) outputSet() map[string]WritablePath {
  260. outputs := make(map[string]WritablePath)
  261. for _, c := range r.commands {
  262. for _, output := range c.outputs {
  263. outputs[output.String()] = output
  264. }
  265. }
  266. return outputs
  267. }
  268. // Outputs returns the list of paths that were passed to the RuleBuilderCommand methods that take
  269. // output paths, such as RuleBuilderCommand.Output, RuleBuilderCommand.ImplicitOutput, or
  270. // RuleBuilderCommand.FlagWithInput. The list is sorted and duplicates removed.
  271. func (r *RuleBuilder) Outputs() WritablePaths {
  272. outputs := r.outputSet()
  273. var outputList WritablePaths
  274. for _, output := range outputs {
  275. if !r.temporariesSet[output] {
  276. outputList = append(outputList, output)
  277. }
  278. }
  279. sort.Slice(outputList, func(i, j int) bool {
  280. return outputList[i].String() < outputList[j].String()
  281. })
  282. return outputList
  283. }
  284. func (r *RuleBuilder) symlinkOutputSet() map[string]WritablePath {
  285. symlinkOutputs := make(map[string]WritablePath)
  286. for _, c := range r.commands {
  287. for _, symlinkOutput := range c.symlinkOutputs {
  288. symlinkOutputs[symlinkOutput.String()] = symlinkOutput
  289. }
  290. }
  291. return symlinkOutputs
  292. }
  293. // SymlinkOutputs returns the list of paths that the executor (Ninja) would
  294. // verify, after build edge completion, that:
  295. //
  296. // 1) Created output symlinks match the list of paths in this list exactly (no more, no fewer)
  297. // 2) Created output files are *not* declared in this list.
  298. //
  299. // These symlink outputs are expected to be a subset of outputs or implicit
  300. // outputs, or they would fail validation at build param construction time
  301. // later, to support other non-rule-builder approaches for constructing
  302. // statements.
  303. func (r *RuleBuilder) SymlinkOutputs() WritablePaths {
  304. symlinkOutputs := r.symlinkOutputSet()
  305. var symlinkOutputList WritablePaths
  306. for _, symlinkOutput := range symlinkOutputs {
  307. symlinkOutputList = append(symlinkOutputList, symlinkOutput)
  308. }
  309. sort.Slice(symlinkOutputList, func(i, j int) bool {
  310. return symlinkOutputList[i].String() < symlinkOutputList[j].String()
  311. })
  312. return symlinkOutputList
  313. }
  314. func (r *RuleBuilder) depFileSet() map[string]WritablePath {
  315. depFiles := make(map[string]WritablePath)
  316. for _, c := range r.commands {
  317. for _, depFile := range c.depFiles {
  318. depFiles[depFile.String()] = depFile
  319. }
  320. }
  321. return depFiles
  322. }
  323. // DepFiles returns the list of paths that were passed to the RuleBuilderCommand methods that take depfile paths, such
  324. // as RuleBuilderCommand.DepFile or RuleBuilderCommand.FlagWithDepFile.
  325. func (r *RuleBuilder) DepFiles() WritablePaths {
  326. var depFiles WritablePaths
  327. for _, c := range r.commands {
  328. for _, depFile := range c.depFiles {
  329. depFiles = append(depFiles, depFile)
  330. }
  331. }
  332. return depFiles
  333. }
  334. // Installs returns the list of tuples passed to Install.
  335. func (r *RuleBuilder) Installs() RuleBuilderInstalls {
  336. return append(RuleBuilderInstalls(nil), r.installs...)
  337. }
  338. func (r *RuleBuilder) toolsSet() map[string]Path {
  339. tools := make(map[string]Path)
  340. for _, c := range r.commands {
  341. for _, tool := range c.tools {
  342. tools[tool.String()] = tool
  343. }
  344. }
  345. return tools
  346. }
  347. // Tools returns the list of paths that were passed to the RuleBuilderCommand.Tool method. The
  348. // list is sorted and duplicates removed.
  349. func (r *RuleBuilder) Tools() Paths {
  350. toolsSet := r.toolsSet()
  351. var toolsList Paths
  352. for _, tool := range toolsSet {
  353. toolsList = append(toolsList, tool)
  354. }
  355. sort.Slice(toolsList, func(i, j int) bool {
  356. return toolsList[i].String() < toolsList[j].String()
  357. })
  358. return toolsList
  359. }
  360. // RspFileInputs returns the list of paths that were passed to the RuleBuilderCommand.FlagWithRspFileInputList method.
  361. func (r *RuleBuilder) RspFileInputs() Paths {
  362. var rspFileInputs Paths
  363. for _, c := range r.commands {
  364. for _, rspFile := range c.rspFiles {
  365. rspFileInputs = append(rspFileInputs, rspFile.paths...)
  366. }
  367. }
  368. return rspFileInputs
  369. }
  370. func (r *RuleBuilder) rspFiles() []rspFileAndPaths {
  371. var rspFiles []rspFileAndPaths
  372. for _, c := range r.commands {
  373. rspFiles = append(rspFiles, c.rspFiles...)
  374. }
  375. return rspFiles
  376. }
  377. // Commands returns a slice containing the built command line for each call to RuleBuilder.Command.
  378. func (r *RuleBuilder) Commands() []string {
  379. var commands []string
  380. for _, c := range r.commands {
  381. commands = append(commands, c.String())
  382. }
  383. return commands
  384. }
  385. // BuilderContext is a subset of ModuleContext and SingletonContext.
  386. type BuilderContext interface {
  387. PathContext
  388. Rule(PackageContext, string, blueprint.RuleParams, ...string) blueprint.Rule
  389. Build(PackageContext, BuildParams)
  390. }
  391. var _ BuilderContext = ModuleContext(nil)
  392. var _ BuilderContext = SingletonContext(nil)
  393. func (r *RuleBuilder) depFileMergerCmd(depFiles WritablePaths) *RuleBuilderCommand {
  394. return r.Command().
  395. builtToolWithoutDeps("dep_fixer").
  396. Inputs(depFiles.Paths())
  397. }
  398. // Build adds the built command line to the build graph, with dependencies on Inputs and Tools, and output files for
  399. // Outputs.
  400. func (r *RuleBuilder) Build(name string, desc string) {
  401. name = ninjaNameEscape(name)
  402. if len(r.missingDeps) > 0 {
  403. r.ctx.Build(pctx, BuildParams{
  404. Rule: ErrorRule,
  405. Outputs: r.Outputs(),
  406. Description: desc,
  407. Args: map[string]string{
  408. "error": "missing dependencies: " + strings.Join(r.missingDeps, ", "),
  409. },
  410. })
  411. return
  412. }
  413. var depFile WritablePath
  414. var depFormat blueprint.Deps
  415. if depFiles := r.DepFiles(); len(depFiles) > 0 {
  416. depFile = depFiles[0]
  417. depFormat = blueprint.DepsGCC
  418. if len(depFiles) > 1 {
  419. // Add a command locally that merges all depfiles together into the first depfile.
  420. r.depFileMergerCmd(depFiles)
  421. if r.sbox {
  422. // Check for Rel() errors, as all depfiles should be in the output dir. Errors
  423. // will be reported to the ctx.
  424. for _, path := range depFiles[1:] {
  425. Rel(r.ctx, r.outDir.String(), path.String())
  426. }
  427. }
  428. }
  429. }
  430. tools := r.Tools()
  431. commands := r.Commands()
  432. outputs := r.Outputs()
  433. inputs := r.Inputs()
  434. rspFiles := r.rspFiles()
  435. if len(commands) == 0 {
  436. return
  437. }
  438. if len(outputs) == 0 {
  439. panic("No outputs specified from any Commands")
  440. }
  441. commandString := strings.Join(commands, " && ")
  442. if r.sbox {
  443. // If running the command inside sbox, write the rule data out to an sbox
  444. // manifest.textproto.
  445. manifest := sbox_proto.Manifest{}
  446. command := sbox_proto.Command{}
  447. manifest.Commands = append(manifest.Commands, &command)
  448. command.Command = proto.String(commandString)
  449. if depFile != nil {
  450. manifest.OutputDepfile = proto.String(depFile.String())
  451. }
  452. // If sandboxing tools is enabled, add copy rules to the manifest to copy each tool
  453. // into the sbox directory.
  454. if r.sboxTools {
  455. for _, tool := range tools {
  456. command.CopyBefore = append(command.CopyBefore, &sbox_proto.Copy{
  457. From: proto.String(tool.String()),
  458. To: proto.String(sboxPathForToolRel(r.ctx, tool)),
  459. })
  460. }
  461. for _, c := range r.commands {
  462. for _, tool := range c.packagedTools {
  463. command.CopyBefore = append(command.CopyBefore, &sbox_proto.Copy{
  464. From: proto.String(tool.srcPath.String()),
  465. To: proto.String(sboxPathForPackagedToolRel(tool)),
  466. Executable: proto.Bool(tool.executable),
  467. })
  468. tools = append(tools, tool.srcPath)
  469. }
  470. }
  471. }
  472. // If sandboxing inputs is enabled, add copy rules to the manifest to copy each input
  473. // into the sbox directory.
  474. if r.sboxInputs {
  475. for _, input := range inputs {
  476. command.CopyBefore = append(command.CopyBefore, &sbox_proto.Copy{
  477. From: proto.String(input.String()),
  478. To: proto.String(r.sboxPathForInputRel(input)),
  479. })
  480. }
  481. // If using rsp files copy them and their contents into the sbox directory with
  482. // the appropriate path mappings.
  483. for _, rspFile := range rspFiles {
  484. command.RspFiles = append(command.RspFiles, &sbox_proto.RspFile{
  485. File: proto.String(rspFile.file.String()),
  486. // These have to match the logic in sboxPathForInputRel
  487. PathMappings: []*sbox_proto.PathMapping{
  488. {
  489. From: proto.String(r.outDir.String()),
  490. To: proto.String(sboxOutSubDir),
  491. },
  492. {
  493. From: proto.String(PathForOutput(r.ctx).String()),
  494. To: proto.String(sboxOutSubDir),
  495. },
  496. },
  497. })
  498. }
  499. command.Chdir = proto.Bool(true)
  500. }
  501. // Add copy rules to the manifest to copy each output file from the sbox directory.
  502. // to the output directory after running the commands.
  503. sboxOutputs := make([]string, len(outputs))
  504. for i, output := range outputs {
  505. rel := Rel(r.ctx, r.outDir.String(), output.String())
  506. sboxOutputs[i] = filepath.Join(sboxOutDir, rel)
  507. command.CopyAfter = append(command.CopyAfter, &sbox_proto.Copy{
  508. From: proto.String(filepath.Join(sboxOutSubDir, rel)),
  509. To: proto.String(output.String()),
  510. })
  511. }
  512. // Outputs that were marked Temporary will not be checked that they are in the output
  513. // directory by the loop above, check them here.
  514. for path := range r.temporariesSet {
  515. Rel(r.ctx, r.outDir.String(), path.String())
  516. }
  517. // Add a hash of the list of input files to the manifest so that the textproto file
  518. // changes when the list of input files changes and causes the sbox rule that
  519. // depends on it to rerun.
  520. command.InputHash = proto.String(hashSrcFiles(inputs))
  521. // Verify that the manifest textproto is not inside the sbox output directory, otherwise
  522. // it will get deleted when the sbox rule clears its output directory.
  523. _, manifestInOutDir := MaybeRel(r.ctx, r.outDir.String(), r.sboxManifestPath.String())
  524. if manifestInOutDir {
  525. ReportPathErrorf(r.ctx, "sbox rule %q manifestPath %q must not be in outputDir %q",
  526. name, r.sboxManifestPath.String(), r.outDir.String())
  527. }
  528. // Create a rule to write the manifest as a the textproto.
  529. pbText, err := prototext.Marshal(&manifest)
  530. if err != nil {
  531. ReportPathErrorf(r.ctx, "sbox manifest failed to marshal: %q", err)
  532. }
  533. WriteFileRule(r.ctx, r.sboxManifestPath, string(pbText))
  534. // Generate a new string to use as the command line of the sbox rule. This uses
  535. // a RuleBuilderCommand as a convenience method of building the command line, then
  536. // converts it to a string to replace commandString.
  537. sboxCmd := &RuleBuilderCommand{
  538. rule: &RuleBuilder{
  539. ctx: r.ctx,
  540. },
  541. }
  542. sboxCmd.builtToolWithoutDeps("sbox").
  543. FlagWithArg("--sandbox-path ", shared.TempDirForOutDir(PathForOutput(r.ctx).String())).
  544. FlagWithArg("--output-dir ", r.outDir.String()).
  545. FlagWithInput("--manifest ", r.sboxManifestPath)
  546. if r.restat {
  547. sboxCmd.Flag("--write-if-changed")
  548. }
  549. // Replace the command string, and add the sbox tool and manifest textproto to the
  550. // dependencies of the final sbox rule.
  551. commandString = sboxCmd.buf.String()
  552. tools = append(tools, sboxCmd.tools...)
  553. inputs = append(inputs, sboxCmd.inputs...)
  554. if r.rbeParams != nil {
  555. // RBE needs a list of input files to copy to the remote builder. For inputs already
  556. // listed in an rsp file, pass the rsp file directly to rewrapper. For the rest,
  557. // create a new rsp file to pass to rewrapper.
  558. var remoteRspFiles Paths
  559. var remoteInputs Paths
  560. remoteInputs = append(remoteInputs, inputs...)
  561. remoteInputs = append(remoteInputs, tools...)
  562. for _, rspFile := range rspFiles {
  563. remoteInputs = append(remoteInputs, rspFile.file)
  564. remoteRspFiles = append(remoteRspFiles, rspFile.file)
  565. }
  566. if len(remoteInputs) > 0 {
  567. inputsListFile := r.sboxManifestPath.ReplaceExtension(r.ctx, "rbe_inputs.list")
  568. writeRspFileRule(r.ctx, inputsListFile, remoteInputs)
  569. remoteRspFiles = append(remoteRspFiles, inputsListFile)
  570. // Add the new rsp file as an extra input to the rule.
  571. inputs = append(inputs, inputsListFile)
  572. }
  573. r.rbeParams.OutputFiles = outputs.Strings()
  574. r.rbeParams.RSPFiles = remoteRspFiles.Strings()
  575. rewrapperCommand := r.rbeParams.NoVarTemplate(r.ctx.Config().RBEWrapper())
  576. commandString = rewrapperCommand + " bash -c '" + strings.ReplaceAll(commandString, `'`, `'\''`) + "'"
  577. }
  578. } else {
  579. // If not using sbox the rule will run the command directly, put the hash of the
  580. // list of input files in a comment at the end of the command line to ensure ninja
  581. // reruns the rule when the list of input files changes.
  582. commandString += " # hash of input list: " + hashSrcFiles(inputs)
  583. }
  584. // Ninja doesn't like multiple outputs when depfiles are enabled, move all but the first output to
  585. // ImplicitOutputs. RuleBuilder doesn't use "$out", so the distinction between Outputs and
  586. // ImplicitOutputs doesn't matter.
  587. output := outputs[0]
  588. implicitOutputs := outputs[1:]
  589. var rspFile, rspFileContent string
  590. var rspFileInputs Paths
  591. if len(rspFiles) > 0 {
  592. // The first rsp files uses Ninja's rsp file support for the rule
  593. rspFile = rspFiles[0].file.String()
  594. // Use "$in" for rspFileContent to avoid duplicating the list of files in the dependency
  595. // list and in the contents of the rsp file. Inputs to the rule that are not in the
  596. // rsp file will be listed in Implicits instead of Inputs so they don't show up in "$in".
  597. rspFileContent = "$in"
  598. rspFileInputs = append(rspFileInputs, rspFiles[0].paths...)
  599. for _, rspFile := range rspFiles[1:] {
  600. // Any additional rsp files need an extra rule to write the file.
  601. writeRspFileRule(r.ctx, rspFile.file, rspFile.paths)
  602. // The main rule needs to depend on the inputs listed in the extra rsp file.
  603. inputs = append(inputs, rspFile.paths...)
  604. // The main rule needs to depend on the extra rsp file.
  605. inputs = append(inputs, rspFile.file)
  606. }
  607. }
  608. var pool blueprint.Pool
  609. if r.ctx.Config().UseGoma() && r.remoteable.Goma {
  610. // When USE_GOMA=true is set and the rule is supported by goma, allow jobs to run outside the local pool.
  611. } else if r.ctx.Config().UseRBE() && r.remoteable.RBE {
  612. // When USE_RBE=true is set and the rule is supported by RBE, use the remotePool.
  613. pool = remotePool
  614. } else if r.highmem {
  615. pool = highmemPool
  616. } else if r.ctx.Config().UseRemoteBuild() {
  617. pool = localPool
  618. }
  619. r.ctx.Build(r.pctx, BuildParams{
  620. Rule: r.ctx.Rule(pctx, name, blueprint.RuleParams{
  621. Command: proptools.NinjaEscape(commandString),
  622. CommandDeps: proptools.NinjaEscapeList(tools.Strings()),
  623. Restat: r.restat,
  624. Rspfile: proptools.NinjaEscape(rspFile),
  625. RspfileContent: rspFileContent,
  626. Pool: pool,
  627. }),
  628. Inputs: rspFileInputs,
  629. Implicits: inputs,
  630. OrderOnly: r.OrderOnlys(),
  631. Validations: r.Validations(),
  632. Output: output,
  633. ImplicitOutputs: implicitOutputs,
  634. SymlinkOutputs: r.SymlinkOutputs(),
  635. Depfile: depFile,
  636. Deps: depFormat,
  637. Description: desc,
  638. })
  639. }
  640. // RuleBuilderCommand is a builder for a command in a command line. It can be mutated by its methods to add to the
  641. // command and track dependencies. The methods mutate the RuleBuilderCommand in place, as well as return the
  642. // RuleBuilderCommand, so they can be used chained or unchained. All methods that add text implicitly add a single
  643. // space as a separator from the previous method.
  644. type RuleBuilderCommand struct {
  645. rule *RuleBuilder
  646. buf strings.Builder
  647. inputs Paths
  648. implicits Paths
  649. orderOnlys Paths
  650. validations Paths
  651. outputs WritablePaths
  652. symlinkOutputs WritablePaths
  653. depFiles WritablePaths
  654. tools Paths
  655. packagedTools []PackagingSpec
  656. rspFiles []rspFileAndPaths
  657. }
  658. type rspFileAndPaths struct {
  659. file WritablePath
  660. paths Paths
  661. }
  662. func checkPathNotNil(path Path) {
  663. if path == nil {
  664. panic("rule_builder paths cannot be nil")
  665. }
  666. }
  667. func (c *RuleBuilderCommand) addInput(path Path) string {
  668. checkPathNotNil(path)
  669. c.inputs = append(c.inputs, path)
  670. return c.PathForInput(path)
  671. }
  672. func (c *RuleBuilderCommand) addImplicit(path Path) {
  673. checkPathNotNil(path)
  674. c.implicits = append(c.implicits, path)
  675. }
  676. func (c *RuleBuilderCommand) addOrderOnly(path Path) {
  677. checkPathNotNil(path)
  678. c.orderOnlys = append(c.orderOnlys, path)
  679. }
  680. // PathForInput takes an input path and returns the appropriate path to use on the command line. If
  681. // sbox was enabled via a call to RuleBuilder.Sbox() and the path was an output path it returns a
  682. // path with the placeholder prefix used for outputs in sbox. If sbox is not enabled it returns the
  683. // original path.
  684. func (c *RuleBuilderCommand) PathForInput(path Path) string {
  685. if c.rule.sbox {
  686. rel, inSandbox := c.rule._sboxPathForInputRel(path)
  687. if inSandbox {
  688. rel = filepath.Join(sboxSandboxBaseDir, rel)
  689. }
  690. return rel
  691. }
  692. return path.String()
  693. }
  694. // PathsForInputs takes a list of input paths and returns the appropriate paths to use on the
  695. // command line. If sbox was enabled via a call to RuleBuilder.Sbox() a path was an output path, it
  696. // returns the path with the placeholder prefix used for outputs in sbox. If sbox is not enabled it
  697. // returns the original paths.
  698. func (c *RuleBuilderCommand) PathsForInputs(paths Paths) []string {
  699. ret := make([]string, len(paths))
  700. for i, path := range paths {
  701. ret[i] = c.PathForInput(path)
  702. }
  703. return ret
  704. }
  705. // PathForOutput takes an output path and returns the appropriate path to use on the command
  706. // line. If sbox was enabled via a call to RuleBuilder.Sbox(), it returns a path with the
  707. // placeholder prefix used for outputs in sbox. If sbox is not enabled it returns the
  708. // original path.
  709. func (c *RuleBuilderCommand) PathForOutput(path WritablePath) string {
  710. if c.rule.sbox {
  711. // Errors will be handled in RuleBuilder.Build where we have a context to report them
  712. rel, _, _ := maybeRelErr(c.rule.outDir.String(), path.String())
  713. return filepath.Join(sboxOutDir, rel)
  714. }
  715. return path.String()
  716. }
  717. func sboxPathForToolRel(ctx BuilderContext, path Path) string {
  718. // Errors will be handled in RuleBuilder.Build where we have a context to report them
  719. toolDir := pathForInstall(ctx, ctx.Config().BuildOS, ctx.Config().BuildArch, "", false)
  720. relOutSoong, isRelOutSoong, _ := maybeRelErr(toolDir.String(), path.String())
  721. if isRelOutSoong {
  722. // The tool is in the Soong output directory, it will be copied to __SBOX_OUT_DIR__/tools/out
  723. return filepath.Join(sboxToolsSubDir, "out", relOutSoong)
  724. }
  725. // The tool is in the source directory, it will be copied to __SBOX_OUT_DIR__/tools/src
  726. return filepath.Join(sboxToolsSubDir, "src", path.String())
  727. }
  728. func (r *RuleBuilder) _sboxPathForInputRel(path Path) (rel string, inSandbox bool) {
  729. // Errors will be handled in RuleBuilder.Build where we have a context to report them
  730. rel, isRelSboxOut, _ := maybeRelErr(r.outDir.String(), path.String())
  731. if isRelSboxOut {
  732. return filepath.Join(sboxOutSubDir, rel), true
  733. }
  734. if r.sboxInputs {
  735. // When sandboxing inputs all inputs have to be copied into the sandbox. Input files that
  736. // are outputs of other rules could be an arbitrary absolute path if OUT_DIR is set, so they
  737. // will be copied to relative paths under __SBOX_OUT_DIR__/out.
  738. rel, isRelOut, _ := maybeRelErr(PathForOutput(r.ctx).String(), path.String())
  739. if isRelOut {
  740. return filepath.Join(sboxOutSubDir, rel), true
  741. }
  742. }
  743. return path.String(), false
  744. }
  745. func (r *RuleBuilder) sboxPathForInputRel(path Path) string {
  746. rel, _ := r._sboxPathForInputRel(path)
  747. return rel
  748. }
  749. func (r *RuleBuilder) sboxPathsForInputsRel(paths Paths) []string {
  750. ret := make([]string, len(paths))
  751. for i, path := range paths {
  752. ret[i] = r.sboxPathForInputRel(path)
  753. }
  754. return ret
  755. }
  756. func sboxPathForPackagedToolRel(spec PackagingSpec) string {
  757. return filepath.Join(sboxToolsSubDir, "out", spec.relPathInPackage)
  758. }
  759. // PathForPackagedTool takes a PackageSpec for a tool and returns the corresponding path for the
  760. // tool after copying it into the sandbox. This can be used on the RuleBuilder command line to
  761. // reference the tool.
  762. func (c *RuleBuilderCommand) PathForPackagedTool(spec PackagingSpec) string {
  763. if !c.rule.sboxTools {
  764. panic("PathForPackagedTool() requires SandboxTools()")
  765. }
  766. return filepath.Join(sboxSandboxBaseDir, sboxPathForPackagedToolRel(spec))
  767. }
  768. // PathForTool takes a path to a tool, which may be an output file or a source file, and returns
  769. // the corresponding path for the tool in the sbox sandbox if sbox is enabled, or the original path
  770. // if it is not. This can be used on the RuleBuilder command line to reference the tool.
  771. func (c *RuleBuilderCommand) PathForTool(path Path) string {
  772. if c.rule.sbox && c.rule.sboxTools {
  773. return filepath.Join(sboxSandboxBaseDir, sboxPathForToolRel(c.rule.ctx, path))
  774. }
  775. return path.String()
  776. }
  777. // PathsForTools takes a list of paths to tools, which may be output files or source files, and
  778. // returns the corresponding paths for the tools in the sbox sandbox if sbox is enabled, or the
  779. // original paths if it is not. This can be used on the RuleBuilder command line to reference the tool.
  780. func (c *RuleBuilderCommand) PathsForTools(paths Paths) []string {
  781. if c.rule.sbox && c.rule.sboxTools {
  782. var ret []string
  783. for _, path := range paths {
  784. ret = append(ret, filepath.Join(sboxSandboxBaseDir, sboxPathForToolRel(c.rule.ctx, path)))
  785. }
  786. return ret
  787. }
  788. return paths.Strings()
  789. }
  790. // PackagedTool adds the specified tool path to the command line. It can only be used with tool
  791. // sandboxing enabled by SandboxTools(), and will copy the tool into the sandbox.
  792. func (c *RuleBuilderCommand) PackagedTool(spec PackagingSpec) *RuleBuilderCommand {
  793. if !c.rule.sboxTools {
  794. panic("PackagedTool() requires SandboxTools()")
  795. }
  796. c.packagedTools = append(c.packagedTools, spec)
  797. c.Text(sboxPathForPackagedToolRel(spec))
  798. return c
  799. }
  800. // ImplicitPackagedTool copies the specified tool into the sandbox without modifying the command
  801. // line. It can only be used with tool sandboxing enabled by SandboxTools().
  802. func (c *RuleBuilderCommand) ImplicitPackagedTool(spec PackagingSpec) *RuleBuilderCommand {
  803. if !c.rule.sboxTools {
  804. panic("ImplicitPackagedTool() requires SandboxTools()")
  805. }
  806. c.packagedTools = append(c.packagedTools, spec)
  807. return c
  808. }
  809. // ImplicitPackagedTools copies the specified tools into the sandbox without modifying the command
  810. // line. It can only be used with tool sandboxing enabled by SandboxTools().
  811. func (c *RuleBuilderCommand) ImplicitPackagedTools(specs []PackagingSpec) *RuleBuilderCommand {
  812. if !c.rule.sboxTools {
  813. panic("ImplicitPackagedTools() requires SandboxTools()")
  814. }
  815. c.packagedTools = append(c.packagedTools, specs...)
  816. return c
  817. }
  818. // Text adds the specified raw text to the command line. The text should not contain input or output paths or the
  819. // rule will not have them listed in its dependencies or outputs.
  820. func (c *RuleBuilderCommand) Text(text string) *RuleBuilderCommand {
  821. if c.buf.Len() > 0 {
  822. c.buf.WriteByte(' ')
  823. }
  824. c.buf.WriteString(text)
  825. return c
  826. }
  827. // Textf adds the specified formatted text to the command line. The text should not contain input or output paths or
  828. // the rule will not have them listed in its dependencies or outputs.
  829. func (c *RuleBuilderCommand) Textf(format string, a ...interface{}) *RuleBuilderCommand {
  830. return c.Text(fmt.Sprintf(format, a...))
  831. }
  832. // Flag adds the specified raw text to the command line. The text should not contain input or output paths or the
  833. // rule will not have them listed in its dependencies or outputs.
  834. func (c *RuleBuilderCommand) Flag(flag string) *RuleBuilderCommand {
  835. return c.Text(flag)
  836. }
  837. // OptionalFlag adds the specified raw text to the command line if it is not nil. The text should not contain input or
  838. // output paths or the rule will not have them listed in its dependencies or outputs.
  839. func (c *RuleBuilderCommand) OptionalFlag(flag *string) *RuleBuilderCommand {
  840. if flag != nil {
  841. c.Text(*flag)
  842. }
  843. return c
  844. }
  845. // Flags adds the specified raw text to the command line. The text should not contain input or output paths or the
  846. // rule will not have them listed in its dependencies or outputs.
  847. func (c *RuleBuilderCommand) Flags(flags []string) *RuleBuilderCommand {
  848. for _, flag := range flags {
  849. c.Text(flag)
  850. }
  851. return c
  852. }
  853. // FlagWithArg adds the specified flag and argument text to the command line, with no separator between them. The flag
  854. // and argument should not contain input or output paths or the rule will not have them listed in its dependencies or
  855. // outputs.
  856. func (c *RuleBuilderCommand) FlagWithArg(flag, arg string) *RuleBuilderCommand {
  857. return c.Text(flag + arg)
  858. }
  859. // FlagForEachArg adds the specified flag joined with each argument to the command line. The result is identical to
  860. // calling FlagWithArg for argument.
  861. func (c *RuleBuilderCommand) FlagForEachArg(flag string, args []string) *RuleBuilderCommand {
  862. for _, arg := range args {
  863. c.FlagWithArg(flag, arg)
  864. }
  865. return c
  866. }
  867. // FlagWithList adds the specified flag and list of arguments to the command line, with the arguments joined by sep
  868. // and no separator between the flag and arguments. The flag and arguments should not contain input or output paths or
  869. // the rule will not have them listed in its dependencies or outputs.
  870. func (c *RuleBuilderCommand) FlagWithList(flag string, list []string, sep string) *RuleBuilderCommand {
  871. return c.Text(flag + strings.Join(list, sep))
  872. }
  873. // Tool adds the specified tool path to the command line. The path will be also added to the dependencies returned by
  874. // RuleBuilder.Tools.
  875. func (c *RuleBuilderCommand) Tool(path Path) *RuleBuilderCommand {
  876. checkPathNotNil(path)
  877. c.tools = append(c.tools, path)
  878. return c.Text(c.PathForTool(path))
  879. }
  880. // Tool adds the specified tool path to the dependencies returned by RuleBuilder.Tools.
  881. func (c *RuleBuilderCommand) ImplicitTool(path Path) *RuleBuilderCommand {
  882. checkPathNotNil(path)
  883. c.tools = append(c.tools, path)
  884. return c
  885. }
  886. // Tool adds the specified tool path to the dependencies returned by RuleBuilder.Tools.
  887. func (c *RuleBuilderCommand) ImplicitTools(paths Paths) *RuleBuilderCommand {
  888. for _, path := range paths {
  889. c.ImplicitTool(path)
  890. }
  891. return c
  892. }
  893. // BuiltTool adds the specified tool path that was built using a host Soong module to the command line. The path will
  894. // be also added to the dependencies returned by RuleBuilder.Tools.
  895. //
  896. // It is equivalent to:
  897. //
  898. // cmd.Tool(ctx.Config().HostToolPath(ctx, tool))
  899. func (c *RuleBuilderCommand) BuiltTool(tool string) *RuleBuilderCommand {
  900. if c.rule.ctx.Config().UseHostMusl() {
  901. // If the host is using musl, assume that the tool was built against musl libc and include
  902. // libc_musl.so in the sandbox.
  903. // TODO(ccross): if we supported adding new dependencies during GenerateAndroidBuildActions
  904. // this could be a dependency + TransitivePackagingSpecs.
  905. c.ImplicitTool(c.rule.ctx.Config().HostJNIToolPath(c.rule.ctx, "libc_musl"))
  906. }
  907. return c.builtToolWithoutDeps(tool)
  908. }
  909. // builtToolWithoutDeps is similar to BuiltTool, but doesn't add any dependencies. It is used
  910. // internally by RuleBuilder for helper tools that are known to be compiled statically.
  911. func (c *RuleBuilderCommand) builtToolWithoutDeps(tool string) *RuleBuilderCommand {
  912. return c.Tool(c.rule.ctx.Config().HostToolPath(c.rule.ctx, tool))
  913. }
  914. // PrebuiltBuildTool adds the specified tool path from prebuils/build-tools. The path will be also added to the
  915. // dependencies returned by RuleBuilder.Tools.
  916. //
  917. // It is equivalent to:
  918. //
  919. // cmd.Tool(ctx.Config().PrebuiltBuildTool(ctx, tool))
  920. func (c *RuleBuilderCommand) PrebuiltBuildTool(ctx PathContext, tool string) *RuleBuilderCommand {
  921. return c.Tool(ctx.Config().PrebuiltBuildTool(ctx, tool))
  922. }
  923. // Input adds the specified input path to the command line. The path will also be added to the dependencies returned by
  924. // RuleBuilder.Inputs.
  925. func (c *RuleBuilderCommand) Input(path Path) *RuleBuilderCommand {
  926. return c.Text(c.addInput(path))
  927. }
  928. // Inputs adds the specified input paths to the command line, separated by spaces. The paths will also be added to the
  929. // dependencies returned by RuleBuilder.Inputs.
  930. func (c *RuleBuilderCommand) Inputs(paths Paths) *RuleBuilderCommand {
  931. for _, path := range paths {
  932. c.Input(path)
  933. }
  934. return c
  935. }
  936. // Implicit adds the specified input path to the dependencies returned by RuleBuilder.Inputs without modifying the
  937. // command line.
  938. func (c *RuleBuilderCommand) Implicit(path Path) *RuleBuilderCommand {
  939. c.addImplicit(path)
  940. return c
  941. }
  942. // Implicits adds the specified input paths to the dependencies returned by RuleBuilder.Inputs without modifying the
  943. // command line.
  944. func (c *RuleBuilderCommand) Implicits(paths Paths) *RuleBuilderCommand {
  945. for _, path := range paths {
  946. c.addImplicit(path)
  947. }
  948. return c
  949. }
  950. // GetImplicits returns the command's implicit inputs.
  951. func (c *RuleBuilderCommand) GetImplicits() Paths {
  952. return c.implicits
  953. }
  954. // OrderOnly adds the specified input path to the dependencies returned by RuleBuilder.OrderOnlys
  955. // without modifying the command line.
  956. func (c *RuleBuilderCommand) OrderOnly(path Path) *RuleBuilderCommand {
  957. c.addOrderOnly(path)
  958. return c
  959. }
  960. // OrderOnlys adds the specified input paths to the dependencies returned by RuleBuilder.OrderOnlys
  961. // without modifying the command line.
  962. func (c *RuleBuilderCommand) OrderOnlys(paths Paths) *RuleBuilderCommand {
  963. for _, path := range paths {
  964. c.addOrderOnly(path)
  965. }
  966. return c
  967. }
  968. // Validation adds the specified input path to the validation dependencies by
  969. // RuleBuilder.Validations without modifying the command line.
  970. func (c *RuleBuilderCommand) Validation(path Path) *RuleBuilderCommand {
  971. checkPathNotNil(path)
  972. c.validations = append(c.validations, path)
  973. return c
  974. }
  975. // Validations adds the specified input paths to the validation dependencies by
  976. // RuleBuilder.Validations without modifying the command line.
  977. func (c *RuleBuilderCommand) Validations(paths Paths) *RuleBuilderCommand {
  978. for _, path := range paths {
  979. c.Validation(path)
  980. }
  981. return c
  982. }
  983. // Output adds the specified output path to the command line. The path will also be added to the outputs returned by
  984. // RuleBuilder.Outputs.
  985. func (c *RuleBuilderCommand) Output(path WritablePath) *RuleBuilderCommand {
  986. checkPathNotNil(path)
  987. c.outputs = append(c.outputs, path)
  988. return c.Text(c.PathForOutput(path))
  989. }
  990. // Outputs adds the specified output paths to the command line, separated by spaces. The paths will also be added to
  991. // the outputs returned by RuleBuilder.Outputs.
  992. func (c *RuleBuilderCommand) Outputs(paths WritablePaths) *RuleBuilderCommand {
  993. for _, path := range paths {
  994. c.Output(path)
  995. }
  996. return c
  997. }
  998. // OutputDir adds the output directory to the command line. This is only available when used with RuleBuilder.Sbox,
  999. // and will be the temporary output directory managed by sbox, not the final one.
  1000. func (c *RuleBuilderCommand) OutputDir() *RuleBuilderCommand {
  1001. if !c.rule.sbox {
  1002. panic("OutputDir only valid with Sbox")
  1003. }
  1004. return c.Text(sboxOutDir)
  1005. }
  1006. // DepFile adds the specified depfile path to the paths returned by RuleBuilder.DepFiles and adds it to the command
  1007. // line, and causes RuleBuilder.Build file to set the depfile flag for ninja. If multiple depfiles are added to
  1008. // commands in a single RuleBuilder then RuleBuilder.Build will add an extra command to merge the depfiles together.
  1009. func (c *RuleBuilderCommand) DepFile(path WritablePath) *RuleBuilderCommand {
  1010. checkPathNotNil(path)
  1011. c.depFiles = append(c.depFiles, path)
  1012. return c.Text(c.PathForOutput(path))
  1013. }
  1014. // ImplicitOutput adds the specified output path to the dependencies returned by RuleBuilder.Outputs without modifying
  1015. // the command line.
  1016. func (c *RuleBuilderCommand) ImplicitOutput(path WritablePath) *RuleBuilderCommand {
  1017. c.outputs = append(c.outputs, path)
  1018. return c
  1019. }
  1020. // ImplicitOutputs adds the specified output paths to the dependencies returned by RuleBuilder.Outputs without modifying
  1021. // the command line.
  1022. func (c *RuleBuilderCommand) ImplicitOutputs(paths WritablePaths) *RuleBuilderCommand {
  1023. c.outputs = append(c.outputs, paths...)
  1024. return c
  1025. }
  1026. // ImplicitSymlinkOutput declares the specified path as an implicit output that
  1027. // will be a symlink instead of a regular file. Does not modify the command
  1028. // line.
  1029. func (c *RuleBuilderCommand) ImplicitSymlinkOutput(path WritablePath) *RuleBuilderCommand {
  1030. checkPathNotNil(path)
  1031. c.symlinkOutputs = append(c.symlinkOutputs, path)
  1032. return c.ImplicitOutput(path)
  1033. }
  1034. // ImplicitSymlinkOutputs declares the specified paths as implicit outputs that
  1035. // will be a symlinks instead of regular files. Does not modify the command
  1036. // line.
  1037. func (c *RuleBuilderCommand) ImplicitSymlinkOutputs(paths WritablePaths) *RuleBuilderCommand {
  1038. for _, path := range paths {
  1039. c.ImplicitSymlinkOutput(path)
  1040. }
  1041. return c
  1042. }
  1043. // SymlinkOutput declares the specified path as an output that will be a symlink
  1044. // instead of a regular file. Modifies the command line.
  1045. func (c *RuleBuilderCommand) SymlinkOutput(path WritablePath) *RuleBuilderCommand {
  1046. checkPathNotNil(path)
  1047. c.symlinkOutputs = append(c.symlinkOutputs, path)
  1048. return c.Output(path)
  1049. }
  1050. // SymlinkOutputsl declares the specified paths as outputs that will be symlinks
  1051. // instead of regular files. Modifies the command line.
  1052. func (c *RuleBuilderCommand) SymlinkOutputs(paths WritablePaths) *RuleBuilderCommand {
  1053. for _, path := range paths {
  1054. c.SymlinkOutput(path)
  1055. }
  1056. return c
  1057. }
  1058. // ImplicitDepFile adds the specified depfile path to the paths returned by RuleBuilder.DepFiles without modifying
  1059. // the command line, and causes RuleBuilder.Build file to set the depfile flag for ninja. If multiple depfiles
  1060. // are added to commands in a single RuleBuilder then RuleBuilder.Build will add an extra command to merge the
  1061. // depfiles together.
  1062. func (c *RuleBuilderCommand) ImplicitDepFile(path WritablePath) *RuleBuilderCommand {
  1063. c.depFiles = append(c.depFiles, path)
  1064. return c
  1065. }
  1066. // FlagWithInput adds the specified flag and input path to the command line, with no separator between them. The path
  1067. // will also be added to the dependencies returned by RuleBuilder.Inputs.
  1068. func (c *RuleBuilderCommand) FlagWithInput(flag string, path Path) *RuleBuilderCommand {
  1069. return c.Text(flag + c.addInput(path))
  1070. }
  1071. // FlagWithInputList adds the specified flag and input paths to the command line, with the inputs joined by sep
  1072. // and no separator between the flag and inputs. The input paths will also be added to the dependencies returned by
  1073. // RuleBuilder.Inputs.
  1074. func (c *RuleBuilderCommand) FlagWithInputList(flag string, paths Paths, sep string) *RuleBuilderCommand {
  1075. strs := make([]string, len(paths))
  1076. for i, path := range paths {
  1077. strs[i] = c.addInput(path)
  1078. }
  1079. return c.FlagWithList(flag, strs, sep)
  1080. }
  1081. // FlagForEachInput adds the specified flag joined with each input path to the command line. The input paths will also
  1082. // be added to the dependencies returned by RuleBuilder.Inputs. The result is identical to calling FlagWithInput for
  1083. // each input path.
  1084. func (c *RuleBuilderCommand) FlagForEachInput(flag string, paths Paths) *RuleBuilderCommand {
  1085. for _, path := range paths {
  1086. c.FlagWithInput(flag, path)
  1087. }
  1088. return c
  1089. }
  1090. // FlagWithOutput adds the specified flag and output path to the command line, with no separator between them. The path
  1091. // will also be added to the outputs returned by RuleBuilder.Outputs.
  1092. func (c *RuleBuilderCommand) FlagWithOutput(flag string, path WritablePath) *RuleBuilderCommand {
  1093. c.outputs = append(c.outputs, path)
  1094. return c.Text(flag + c.PathForOutput(path))
  1095. }
  1096. // FlagWithDepFile adds the specified flag and depfile path to the command line, with no separator between them. The path
  1097. // will also be added to the outputs returned by RuleBuilder.Outputs.
  1098. func (c *RuleBuilderCommand) FlagWithDepFile(flag string, path WritablePath) *RuleBuilderCommand {
  1099. c.depFiles = append(c.depFiles, path)
  1100. return c.Text(flag + c.PathForOutput(path))
  1101. }
  1102. // FlagWithRspFileInputList adds the specified flag and path to an rspfile to the command line, with
  1103. // no separator between them. The paths will be written to the rspfile. If sbox is enabled, the
  1104. // rspfile must be outside the sbox directory. The first use of FlagWithRspFileInputList in any
  1105. // RuleBuilderCommand of a RuleBuilder will use Ninja's rsp file support for the rule, additional
  1106. // uses will result in an auxiliary rules to write the rspFile contents.
  1107. func (c *RuleBuilderCommand) FlagWithRspFileInputList(flag string, rspFile WritablePath, paths Paths) *RuleBuilderCommand {
  1108. // Use an empty slice if paths is nil, the non-nil slice is used as an indicator that the rsp file must be
  1109. // generated.
  1110. if paths == nil {
  1111. paths = Paths{}
  1112. }
  1113. c.rspFiles = append(c.rspFiles, rspFileAndPaths{rspFile, paths})
  1114. if c.rule.sbox {
  1115. if _, isRel, _ := maybeRelErr(c.rule.outDir.String(), rspFile.String()); isRel {
  1116. panic(fmt.Errorf("FlagWithRspFileInputList rspfile %q must not be inside out dir %q",
  1117. rspFile.String(), c.rule.outDir.String()))
  1118. }
  1119. }
  1120. c.FlagWithArg(flag, c.PathForInput(rspFile))
  1121. return c
  1122. }
  1123. // String returns the command line.
  1124. func (c *RuleBuilderCommand) String() string {
  1125. return c.buf.String()
  1126. }
  1127. // RuleBuilderSboxProtoForTests takes the BuildParams for the manifest passed to RuleBuilder.Sbox()
  1128. // and returns sbox testproto generated by the RuleBuilder.
  1129. func RuleBuilderSboxProtoForTests(t *testing.T, params TestingBuildParams) *sbox_proto.Manifest {
  1130. t.Helper()
  1131. content := ContentFromFileRuleForTests(t, params)
  1132. manifest := sbox_proto.Manifest{}
  1133. err := prototext.Unmarshal([]byte(content), &manifest)
  1134. if err != nil {
  1135. t.Fatalf("failed to unmarshal manifest: %s", err.Error())
  1136. }
  1137. return &manifest
  1138. }
  1139. func ninjaNameEscape(s string) string {
  1140. b := []byte(s)
  1141. escaped := false
  1142. for i, c := range b {
  1143. valid := (c >= 'a' && c <= 'z') ||
  1144. (c >= 'A' && c <= 'Z') ||
  1145. (c >= '0' && c <= '9') ||
  1146. (c == '_') ||
  1147. (c == '-') ||
  1148. (c == '.')
  1149. if !valid {
  1150. b[i] = '_'
  1151. escaped = true
  1152. }
  1153. }
  1154. if escaped {
  1155. s = string(b)
  1156. }
  1157. return s
  1158. }
  1159. // hashSrcFiles returns a hash of the list of source files. It is used to ensure the command line
  1160. // or the sbox textproto manifest change even if the input files are not listed on the command line.
  1161. func hashSrcFiles(srcFiles Paths) string {
  1162. h := sha256.New()
  1163. srcFileList := strings.Join(srcFiles.Strings(), "\n")
  1164. h.Write([]byte(srcFileList))
  1165. return fmt.Sprintf("%x", h.Sum(nil))
  1166. }
  1167. // BuilderContextForTesting returns a BuilderContext for the given config that can be used for tests
  1168. // that need to call methods that take a BuilderContext.
  1169. func BuilderContextForTesting(config Config) BuilderContext {
  1170. pathCtx := PathContextForTesting(config)
  1171. return builderContextForTests{
  1172. PathContext: pathCtx,
  1173. }
  1174. }
  1175. type builderContextForTests struct {
  1176. PathContext
  1177. }
  1178. func (builderContextForTests) Rule(PackageContext, string, blueprint.RuleParams, ...string) blueprint.Rule {
  1179. return nil
  1180. }
  1181. func (builderContextForTests) Build(PackageContext, BuildParams) {}
  1182. func writeRspFileRule(ctx BuilderContext, rspFile WritablePath, paths Paths) {
  1183. buf := &strings.Builder{}
  1184. err := response.WriteRspFile(buf, paths.Strings())
  1185. if err != nil {
  1186. // There should never be I/O errors writing to a bytes.Buffer.
  1187. panic(err)
  1188. }
  1189. WriteFileRule(ctx, rspFile, buf.String())
  1190. }