rule_builder.go 45 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294
  1. // Copyright 2018 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package android
  15. import (
  16. "crypto/sha256"
  17. "fmt"
  18. "path/filepath"
  19. "sort"
  20. "strings"
  21. "testing"
  22. "github.com/golang/protobuf/proto"
  23. "github.com/google/blueprint"
  24. "github.com/google/blueprint/proptools"
  25. "android/soong/cmd/sbox/sbox_proto"
  26. "android/soong/remoteexec"
  27. "android/soong/response"
  28. "android/soong/shared"
  29. )
  30. const sboxSandboxBaseDir = "__SBOX_SANDBOX_DIR__"
  31. const sboxOutSubDir = "out"
  32. const sboxToolsSubDir = "tools"
  33. const sboxOutDir = sboxSandboxBaseDir + "/" + sboxOutSubDir
  34. // RuleBuilder provides an alternative to ModuleContext.Rule and ModuleContext.Build to add a command line to the build
  35. // graph.
  36. type RuleBuilder struct {
  37. pctx PackageContext
  38. ctx BuilderContext
  39. commands []*RuleBuilderCommand
  40. installs RuleBuilderInstalls
  41. temporariesSet map[WritablePath]bool
  42. restat bool
  43. sbox bool
  44. highmem bool
  45. remoteable RemoteRuleSupports
  46. rbeParams *remoteexec.REParams
  47. outDir WritablePath
  48. sboxTools bool
  49. sboxInputs bool
  50. sboxManifestPath WritablePath
  51. missingDeps []string
  52. }
  53. // NewRuleBuilder returns a newly created RuleBuilder.
  54. func NewRuleBuilder(pctx PackageContext, ctx BuilderContext) *RuleBuilder {
  55. return &RuleBuilder{
  56. pctx: pctx,
  57. ctx: ctx,
  58. temporariesSet: make(map[WritablePath]bool),
  59. }
  60. }
  61. // RuleBuilderInstall is a tuple of install from and to locations.
  62. type RuleBuilderInstall struct {
  63. From Path
  64. To string
  65. }
  66. type RuleBuilderInstalls []RuleBuilderInstall
  67. // String returns the RuleBuilderInstalls in the form used by $(call copy-many-files) in Make, a space separated
  68. // list of from:to tuples.
  69. func (installs RuleBuilderInstalls) String() string {
  70. sb := strings.Builder{}
  71. for i, install := range installs {
  72. if i != 0 {
  73. sb.WriteRune(' ')
  74. }
  75. sb.WriteString(install.From.String())
  76. sb.WriteRune(':')
  77. sb.WriteString(install.To)
  78. }
  79. return sb.String()
  80. }
  81. // MissingDeps adds modules to the list of missing dependencies. If MissingDeps
  82. // is called with a non-empty input, any call to Build will result in a rule
  83. // that will print an error listing the missing dependencies and fail.
  84. // MissingDeps should only be called if Config.AllowMissingDependencies() is
  85. // true.
  86. func (r *RuleBuilder) MissingDeps(missingDeps []string) {
  87. r.missingDeps = append(r.missingDeps, missingDeps...)
  88. }
  89. // Restat marks the rule as a restat rule, which will be passed to ModuleContext.Rule in BuildParams.Restat.
  90. //
  91. // Restat is not compatible with Sbox()
  92. func (r *RuleBuilder) Restat() *RuleBuilder {
  93. if r.sbox {
  94. panic("Restat() is not compatible with Sbox()")
  95. }
  96. r.restat = true
  97. return r
  98. }
  99. // HighMem marks the rule as a high memory rule, which will limit how many run in parallel with other high memory
  100. // rules.
  101. func (r *RuleBuilder) HighMem() *RuleBuilder {
  102. r.highmem = true
  103. return r
  104. }
  105. // Remoteable marks the rule as supporting remote execution.
  106. func (r *RuleBuilder) Remoteable(supports RemoteRuleSupports) *RuleBuilder {
  107. r.remoteable = supports
  108. return r
  109. }
  110. // Rewrapper marks the rule as running inside rewrapper using the given params in order to support
  111. // running on RBE. During RuleBuilder.Build the params will be combined with the inputs, outputs
  112. // and tools known to RuleBuilder to prepend an appropriate rewrapper command line to the rule's
  113. // command line.
  114. func (r *RuleBuilder) Rewrapper(params *remoteexec.REParams) *RuleBuilder {
  115. if !r.sboxInputs {
  116. panic(fmt.Errorf("RuleBuilder.Rewrapper must be called after RuleBuilder.SandboxInputs"))
  117. }
  118. r.rbeParams = params
  119. return r
  120. }
  121. // Sbox marks the rule as needing to be wrapped by sbox. The outputDir should point to the output
  122. // directory that sbox will wipe. It should not be written to by any other rule. manifestPath should
  123. // point to a location where sbox's manifest will be written and must be outside outputDir. sbox
  124. // will ensure that all outputs have been written, and will discard any output files that were not
  125. // specified.
  126. //
  127. // Sbox is not compatible with Restat()
  128. func (r *RuleBuilder) Sbox(outputDir WritablePath, manifestPath WritablePath) *RuleBuilder {
  129. if r.sbox {
  130. panic("Sbox() may not be called more than once")
  131. }
  132. if len(r.commands) > 0 {
  133. panic("Sbox() may not be called after Command()")
  134. }
  135. if r.restat {
  136. panic("Sbox() is not compatible with Restat()")
  137. }
  138. r.sbox = true
  139. r.outDir = outputDir
  140. r.sboxManifestPath = manifestPath
  141. return r
  142. }
  143. // SandboxTools enables tool sandboxing for the rule by copying any referenced tools into the
  144. // sandbox.
  145. func (r *RuleBuilder) SandboxTools() *RuleBuilder {
  146. if !r.sbox {
  147. panic("SandboxTools() must be called after Sbox()")
  148. }
  149. if len(r.commands) > 0 {
  150. panic("SandboxTools() may not be called after Command()")
  151. }
  152. r.sboxTools = true
  153. return r
  154. }
  155. // SandboxInputs enables input sandboxing for the rule by copying any referenced inputs into the
  156. // sandbox. It also implies SandboxTools().
  157. //
  158. // Sandboxing inputs requires RuleBuilder to be aware of all references to input paths. Paths
  159. // that are passed to RuleBuilder outside of the methods that expect inputs, for example
  160. // FlagWithArg, must use RuleBuilderCommand.PathForInput to translate the path to one that matches
  161. // the sandbox layout.
  162. func (r *RuleBuilder) SandboxInputs() *RuleBuilder {
  163. if !r.sbox {
  164. panic("SandboxInputs() must be called after Sbox()")
  165. }
  166. if len(r.commands) > 0 {
  167. panic("SandboxInputs() may not be called after Command()")
  168. }
  169. r.sboxTools = true
  170. r.sboxInputs = true
  171. return r
  172. }
  173. // Install associates an output of the rule with an install location, which can be retrieved later using
  174. // RuleBuilder.Installs.
  175. func (r *RuleBuilder) Install(from Path, to string) {
  176. r.installs = append(r.installs, RuleBuilderInstall{from, to})
  177. }
  178. // Command returns a new RuleBuilderCommand for the rule. The commands will be ordered in the rule by when they were
  179. // created by this method. That can be mutated through their methods in any order, as long as the mutations do not
  180. // race with any call to Build.
  181. func (r *RuleBuilder) Command() *RuleBuilderCommand {
  182. command := &RuleBuilderCommand{
  183. rule: r,
  184. }
  185. r.commands = append(r.commands, command)
  186. return command
  187. }
  188. // Temporary marks an output of a command as an intermediate file that will be used as an input to another command
  189. // in the same rule, and should not be listed in Outputs.
  190. func (r *RuleBuilder) Temporary(path WritablePath) {
  191. r.temporariesSet[path] = true
  192. }
  193. // DeleteTemporaryFiles adds a command to the rule that deletes any outputs that have been marked using Temporary
  194. // when the rule runs. DeleteTemporaryFiles should be called after all calls to Temporary.
  195. func (r *RuleBuilder) DeleteTemporaryFiles() {
  196. var temporariesList WritablePaths
  197. for intermediate := range r.temporariesSet {
  198. temporariesList = append(temporariesList, intermediate)
  199. }
  200. sort.Slice(temporariesList, func(i, j int) bool {
  201. return temporariesList[i].String() < temporariesList[j].String()
  202. })
  203. r.Command().Text("rm").Flag("-f").Outputs(temporariesList)
  204. }
  205. // Inputs returns the list of paths that were passed to the RuleBuilderCommand methods that take
  206. // input paths, such as RuleBuilderCommand.Input, RuleBuilderCommand.Implicit, or
  207. // RuleBuilderCommand.FlagWithInput. Inputs to a command that are also outputs of another command
  208. // in the same RuleBuilder are filtered out. The list is sorted and duplicates removed.
  209. func (r *RuleBuilder) Inputs() Paths {
  210. outputs := r.outputSet()
  211. depFiles := r.depFileSet()
  212. inputs := make(map[string]Path)
  213. for _, c := range r.commands {
  214. for _, input := range append(c.inputs, c.implicits...) {
  215. inputStr := input.String()
  216. if _, isOutput := outputs[inputStr]; !isOutput {
  217. if _, isDepFile := depFiles[inputStr]; !isDepFile {
  218. inputs[input.String()] = input
  219. }
  220. }
  221. }
  222. }
  223. var inputList Paths
  224. for _, input := range inputs {
  225. inputList = append(inputList, input)
  226. }
  227. sort.Slice(inputList, func(i, j int) bool {
  228. return inputList[i].String() < inputList[j].String()
  229. })
  230. return inputList
  231. }
  232. // OrderOnlys returns the list of paths that were passed to the RuleBuilderCommand.OrderOnly or
  233. // RuleBuilderCommand.OrderOnlys. The list is sorted and duplicates removed.
  234. func (r *RuleBuilder) OrderOnlys() Paths {
  235. orderOnlys := make(map[string]Path)
  236. for _, c := range r.commands {
  237. for _, orderOnly := range c.orderOnlys {
  238. orderOnlys[orderOnly.String()] = orderOnly
  239. }
  240. }
  241. var orderOnlyList Paths
  242. for _, orderOnly := range orderOnlys {
  243. orderOnlyList = append(orderOnlyList, orderOnly)
  244. }
  245. sort.Slice(orderOnlyList, func(i, j int) bool {
  246. return orderOnlyList[i].String() < orderOnlyList[j].String()
  247. })
  248. return orderOnlyList
  249. }
  250. func (r *RuleBuilder) outputSet() map[string]WritablePath {
  251. outputs := make(map[string]WritablePath)
  252. for _, c := range r.commands {
  253. for _, output := range c.outputs {
  254. outputs[output.String()] = output
  255. }
  256. }
  257. return outputs
  258. }
  259. // Outputs returns the list of paths that were passed to the RuleBuilderCommand methods that take
  260. // output paths, such as RuleBuilderCommand.Output, RuleBuilderCommand.ImplicitOutput, or
  261. // RuleBuilderCommand.FlagWithInput. The list is sorted and duplicates removed.
  262. func (r *RuleBuilder) Outputs() WritablePaths {
  263. outputs := r.outputSet()
  264. var outputList WritablePaths
  265. for _, output := range outputs {
  266. if !r.temporariesSet[output] {
  267. outputList = append(outputList, output)
  268. }
  269. }
  270. sort.Slice(outputList, func(i, j int) bool {
  271. return outputList[i].String() < outputList[j].String()
  272. })
  273. return outputList
  274. }
  275. func (r *RuleBuilder) symlinkOutputSet() map[string]WritablePath {
  276. symlinkOutputs := make(map[string]WritablePath)
  277. for _, c := range r.commands {
  278. for _, symlinkOutput := range c.symlinkOutputs {
  279. symlinkOutputs[symlinkOutput.String()] = symlinkOutput
  280. }
  281. }
  282. return symlinkOutputs
  283. }
  284. // SymlinkOutputs returns the list of paths that the executor (Ninja) would
  285. // verify, after build edge completion, that:
  286. //
  287. // 1) Created output symlinks match the list of paths in this list exactly (no more, no fewer)
  288. // 2) Created output files are *not* declared in this list.
  289. //
  290. // These symlink outputs are expected to be a subset of outputs or implicit
  291. // outputs, or they would fail validation at build param construction time
  292. // later, to support other non-rule-builder approaches for constructing
  293. // statements.
  294. func (r *RuleBuilder) SymlinkOutputs() WritablePaths {
  295. symlinkOutputs := r.symlinkOutputSet()
  296. var symlinkOutputList WritablePaths
  297. for _, symlinkOutput := range symlinkOutputs {
  298. symlinkOutputList = append(symlinkOutputList, symlinkOutput)
  299. }
  300. sort.Slice(symlinkOutputList, func(i, j int) bool {
  301. return symlinkOutputList[i].String() < symlinkOutputList[j].String()
  302. })
  303. return symlinkOutputList
  304. }
  305. func (r *RuleBuilder) depFileSet() map[string]WritablePath {
  306. depFiles := make(map[string]WritablePath)
  307. for _, c := range r.commands {
  308. for _, depFile := range c.depFiles {
  309. depFiles[depFile.String()] = depFile
  310. }
  311. }
  312. return depFiles
  313. }
  314. // DepFiles returns the list of paths that were passed to the RuleBuilderCommand methods that take depfile paths, such
  315. // as RuleBuilderCommand.DepFile or RuleBuilderCommand.FlagWithDepFile.
  316. func (r *RuleBuilder) DepFiles() WritablePaths {
  317. var depFiles WritablePaths
  318. for _, c := range r.commands {
  319. for _, depFile := range c.depFiles {
  320. depFiles = append(depFiles, depFile)
  321. }
  322. }
  323. return depFiles
  324. }
  325. // Installs returns the list of tuples passed to Install.
  326. func (r *RuleBuilder) Installs() RuleBuilderInstalls {
  327. return append(RuleBuilderInstalls(nil), r.installs...)
  328. }
  329. func (r *RuleBuilder) toolsSet() map[string]Path {
  330. tools := make(map[string]Path)
  331. for _, c := range r.commands {
  332. for _, tool := range c.tools {
  333. tools[tool.String()] = tool
  334. }
  335. }
  336. return tools
  337. }
  338. // Tools returns the list of paths that were passed to the RuleBuilderCommand.Tool method. The
  339. // list is sorted and duplicates removed.
  340. func (r *RuleBuilder) Tools() Paths {
  341. toolsSet := r.toolsSet()
  342. var toolsList Paths
  343. for _, tool := range toolsSet {
  344. toolsList = append(toolsList, tool)
  345. }
  346. sort.Slice(toolsList, func(i, j int) bool {
  347. return toolsList[i].String() < toolsList[j].String()
  348. })
  349. return toolsList
  350. }
  351. // RspFileInputs returns the list of paths that were passed to the RuleBuilderCommand.FlagWithRspFileInputList method.
  352. func (r *RuleBuilder) RspFileInputs() Paths {
  353. var rspFileInputs Paths
  354. for _, c := range r.commands {
  355. for _, rspFile := range c.rspFiles {
  356. rspFileInputs = append(rspFileInputs, rspFile.paths...)
  357. }
  358. }
  359. return rspFileInputs
  360. }
  361. func (r *RuleBuilder) rspFiles() []rspFileAndPaths {
  362. var rspFiles []rspFileAndPaths
  363. for _, c := range r.commands {
  364. rspFiles = append(rspFiles, c.rspFiles...)
  365. }
  366. return rspFiles
  367. }
  368. // Commands returns a slice containing the built command line for each call to RuleBuilder.Command.
  369. func (r *RuleBuilder) Commands() []string {
  370. var commands []string
  371. for _, c := range r.commands {
  372. commands = append(commands, c.String())
  373. }
  374. return commands
  375. }
  376. // BuilderContext is a subset of ModuleContext and SingletonContext.
  377. type BuilderContext interface {
  378. PathContext
  379. Rule(PackageContext, string, blueprint.RuleParams, ...string) blueprint.Rule
  380. Build(PackageContext, BuildParams)
  381. }
  382. var _ BuilderContext = ModuleContext(nil)
  383. var _ BuilderContext = SingletonContext(nil)
  384. func (r *RuleBuilder) depFileMergerCmd(depFiles WritablePaths) *RuleBuilderCommand {
  385. return r.Command().
  386. BuiltTool("dep_fixer").
  387. Inputs(depFiles.Paths())
  388. }
  389. // Build adds the built command line to the build graph, with dependencies on Inputs and Tools, and output files for
  390. // Outputs.
  391. func (r *RuleBuilder) Build(name string, desc string) {
  392. name = ninjaNameEscape(name)
  393. if len(r.missingDeps) > 0 {
  394. r.ctx.Build(pctx, BuildParams{
  395. Rule: ErrorRule,
  396. Outputs: r.Outputs(),
  397. OrderOnly: r.OrderOnlys(),
  398. Description: desc,
  399. Args: map[string]string{
  400. "error": "missing dependencies: " + strings.Join(r.missingDeps, ", "),
  401. },
  402. })
  403. return
  404. }
  405. var depFile WritablePath
  406. var depFormat blueprint.Deps
  407. if depFiles := r.DepFiles(); len(depFiles) > 0 {
  408. depFile = depFiles[0]
  409. depFormat = blueprint.DepsGCC
  410. if len(depFiles) > 1 {
  411. // Add a command locally that merges all depfiles together into the first depfile.
  412. r.depFileMergerCmd(depFiles)
  413. if r.sbox {
  414. // Check for Rel() errors, as all depfiles should be in the output dir. Errors
  415. // will be reported to the ctx.
  416. for _, path := range depFiles[1:] {
  417. Rel(r.ctx, r.outDir.String(), path.String())
  418. }
  419. }
  420. }
  421. }
  422. tools := r.Tools()
  423. commands := r.Commands()
  424. outputs := r.Outputs()
  425. inputs := r.Inputs()
  426. rspFiles := r.rspFiles()
  427. if len(commands) == 0 {
  428. return
  429. }
  430. if len(outputs) == 0 {
  431. panic("No outputs specified from any Commands")
  432. }
  433. commandString := strings.Join(commands, " && ")
  434. if r.sbox {
  435. // If running the command inside sbox, write the rule data out to an sbox
  436. // manifest.textproto.
  437. manifest := sbox_proto.Manifest{}
  438. command := sbox_proto.Command{}
  439. manifest.Commands = append(manifest.Commands, &command)
  440. command.Command = proto.String(commandString)
  441. if depFile != nil {
  442. manifest.OutputDepfile = proto.String(depFile.String())
  443. }
  444. // If sandboxing tools is enabled, add copy rules to the manifest to copy each tool
  445. // into the sbox directory.
  446. if r.sboxTools {
  447. for _, tool := range tools {
  448. command.CopyBefore = append(command.CopyBefore, &sbox_proto.Copy{
  449. From: proto.String(tool.String()),
  450. To: proto.String(sboxPathForToolRel(r.ctx, tool)),
  451. })
  452. }
  453. for _, c := range r.commands {
  454. for _, tool := range c.packagedTools {
  455. command.CopyBefore = append(command.CopyBefore, &sbox_proto.Copy{
  456. From: proto.String(tool.srcPath.String()),
  457. To: proto.String(sboxPathForPackagedToolRel(tool)),
  458. Executable: proto.Bool(tool.executable),
  459. })
  460. tools = append(tools, tool.srcPath)
  461. }
  462. }
  463. }
  464. // If sandboxing inputs is enabled, add copy rules to the manifest to copy each input
  465. // into the sbox directory.
  466. if r.sboxInputs {
  467. for _, input := range inputs {
  468. command.CopyBefore = append(command.CopyBefore, &sbox_proto.Copy{
  469. From: proto.String(input.String()),
  470. To: proto.String(r.sboxPathForInputRel(input)),
  471. })
  472. }
  473. // If using rsp files copy them and their contents into the sbox directory with
  474. // the appropriate path mappings.
  475. for _, rspFile := range rspFiles {
  476. command.RspFiles = append(command.RspFiles, &sbox_proto.RspFile{
  477. File: proto.String(rspFile.file.String()),
  478. // These have to match the logic in sboxPathForInputRel
  479. PathMappings: []*sbox_proto.PathMapping{
  480. {
  481. From: proto.String(r.outDir.String()),
  482. To: proto.String(sboxOutSubDir),
  483. },
  484. {
  485. From: proto.String(PathForOutput(r.ctx).String()),
  486. To: proto.String(sboxOutSubDir),
  487. },
  488. },
  489. })
  490. }
  491. command.Chdir = proto.Bool(true)
  492. }
  493. // Add copy rules to the manifest to copy each output file from the sbox directory.
  494. // to the output directory after running the commands.
  495. sboxOutputs := make([]string, len(outputs))
  496. for i, output := range outputs {
  497. rel := Rel(r.ctx, r.outDir.String(), output.String())
  498. sboxOutputs[i] = filepath.Join(sboxOutDir, rel)
  499. command.CopyAfter = append(command.CopyAfter, &sbox_proto.Copy{
  500. From: proto.String(filepath.Join(sboxOutSubDir, rel)),
  501. To: proto.String(output.String()),
  502. })
  503. }
  504. // Outputs that were marked Temporary will not be checked that they are in the output
  505. // directory by the loop above, check them here.
  506. for path := range r.temporariesSet {
  507. Rel(r.ctx, r.outDir.String(), path.String())
  508. }
  509. // Add a hash of the list of input files to the manifest so that the textproto file
  510. // changes when the list of input files changes and causes the sbox rule that
  511. // depends on it to rerun.
  512. command.InputHash = proto.String(hashSrcFiles(inputs))
  513. // Verify that the manifest textproto is not inside the sbox output directory, otherwise
  514. // it will get deleted when the sbox rule clears its output directory.
  515. _, manifestInOutDir := MaybeRel(r.ctx, r.outDir.String(), r.sboxManifestPath.String())
  516. if manifestInOutDir {
  517. ReportPathErrorf(r.ctx, "sbox rule %q manifestPath %q must not be in outputDir %q",
  518. name, r.sboxManifestPath.String(), r.outDir.String())
  519. }
  520. // Create a rule to write the manifest as a the textproto.
  521. WriteFileRule(r.ctx, r.sboxManifestPath, proto.MarshalTextString(&manifest))
  522. // Generate a new string to use as the command line of the sbox rule. This uses
  523. // a RuleBuilderCommand as a convenience method of building the command line, then
  524. // converts it to a string to replace commandString.
  525. sboxCmd := &RuleBuilderCommand{
  526. rule: &RuleBuilder{
  527. ctx: r.ctx,
  528. },
  529. }
  530. sboxCmd.Text("rm -rf").Output(r.outDir)
  531. sboxCmd.Text("&&")
  532. sboxCmd.BuiltTool("sbox").
  533. Flag("--sandbox-path").Text(shared.TempDirForOutDir(PathForOutput(r.ctx).String())).
  534. Flag("--manifest").Input(r.sboxManifestPath)
  535. // Replace the command string, and add the sbox tool and manifest textproto to the
  536. // dependencies of the final sbox rule.
  537. commandString = sboxCmd.buf.String()
  538. tools = append(tools, sboxCmd.tools...)
  539. inputs = append(inputs, sboxCmd.inputs...)
  540. if r.rbeParams != nil {
  541. // RBE needs a list of input files to copy to the remote builder. For inputs already
  542. // listed in an rsp file, pass the rsp file directly to rewrapper. For the rest,
  543. // create a new rsp file to pass to rewrapper.
  544. var remoteRspFiles Paths
  545. var remoteInputs Paths
  546. remoteInputs = append(remoteInputs, inputs...)
  547. remoteInputs = append(remoteInputs, tools...)
  548. for _, rspFile := range rspFiles {
  549. remoteInputs = append(remoteInputs, rspFile.file)
  550. remoteRspFiles = append(remoteRspFiles, rspFile.file)
  551. }
  552. if len(remoteInputs) > 0 {
  553. inputsListFile := r.sboxManifestPath.ReplaceExtension(r.ctx, "rbe_inputs.list")
  554. writeRspFileRule(r.ctx, inputsListFile, remoteInputs)
  555. remoteRspFiles = append(remoteRspFiles, inputsListFile)
  556. // Add the new rsp file as an extra input to the rule.
  557. inputs = append(inputs, inputsListFile)
  558. }
  559. r.rbeParams.OutputFiles = outputs.Strings()
  560. r.rbeParams.RSPFiles = remoteRspFiles.Strings()
  561. rewrapperCommand := r.rbeParams.NoVarTemplate(r.ctx.Config().RBEWrapper())
  562. commandString = rewrapperCommand + " bash -c '" + strings.ReplaceAll(commandString, `'`, `'\''`) + "'"
  563. }
  564. } else {
  565. // If not using sbox the rule will run the command directly, put the hash of the
  566. // list of input files in a comment at the end of the command line to ensure ninja
  567. // reruns the rule when the list of input files changes.
  568. commandString += " # hash of input list: " + hashSrcFiles(inputs)
  569. }
  570. // Ninja doesn't like multiple outputs when depfiles are enabled, move all but the first output to
  571. // ImplicitOutputs. RuleBuilder doesn't use "$out", so the distinction between Outputs and
  572. // ImplicitOutputs doesn't matter.
  573. output := outputs[0]
  574. implicitOutputs := outputs[1:]
  575. var rspFile, rspFileContent string
  576. var rspFileInputs Paths
  577. if len(rspFiles) > 0 {
  578. // The first rsp files uses Ninja's rsp file support for the rule
  579. rspFile = rspFiles[0].file.String()
  580. // Use "$in" for rspFileContent to avoid duplicating the list of files in the dependency
  581. // list and in the contents of the rsp file. Inputs to the rule that are not in the
  582. // rsp file will be listed in Implicits instead of Inputs so they don't show up in "$in".
  583. rspFileContent = "$in"
  584. rspFileInputs = append(rspFileInputs, rspFiles[0].paths...)
  585. for _, rspFile := range rspFiles[1:] {
  586. // Any additional rsp files need an extra rule to write the file.
  587. writeRspFileRule(r.ctx, rspFile.file, rspFile.paths)
  588. // The main rule needs to depend on the inputs listed in the extra rsp file.
  589. inputs = append(inputs, rspFile.paths...)
  590. // The main rule needs to depend on the extra rsp file.
  591. inputs = append(inputs, rspFile.file)
  592. }
  593. }
  594. var pool blueprint.Pool
  595. if r.ctx.Config().UseGoma() && r.remoteable.Goma {
  596. // When USE_GOMA=true is set and the rule is supported by goma, allow jobs to run outside the local pool.
  597. } else if r.ctx.Config().UseRBE() && r.remoteable.RBE {
  598. // When USE_RBE=true is set and the rule is supported by RBE, use the remotePool.
  599. pool = remotePool
  600. } else if r.highmem {
  601. pool = highmemPool
  602. } else if r.ctx.Config().UseRemoteBuild() {
  603. pool = localPool
  604. }
  605. r.ctx.Build(r.pctx, BuildParams{
  606. Rule: r.ctx.Rule(pctx, name, blueprint.RuleParams{
  607. Command: proptools.NinjaEscape(commandString),
  608. CommandDeps: proptools.NinjaEscapeList(tools.Strings()),
  609. Restat: r.restat,
  610. Rspfile: proptools.NinjaEscape(rspFile),
  611. RspfileContent: rspFileContent,
  612. Pool: pool,
  613. }),
  614. Inputs: rspFileInputs,
  615. Implicits: inputs,
  616. Output: output,
  617. ImplicitOutputs: implicitOutputs,
  618. SymlinkOutputs: r.SymlinkOutputs(),
  619. Depfile: depFile,
  620. Deps: depFormat,
  621. Description: desc,
  622. })
  623. }
  624. // RuleBuilderCommand is a builder for a command in a command line. It can be mutated by its methods to add to the
  625. // command and track dependencies. The methods mutate the RuleBuilderCommand in place, as well as return the
  626. // RuleBuilderCommand, so they can be used chained or unchained. All methods that add text implicitly add a single
  627. // space as a separator from the previous method.
  628. type RuleBuilderCommand struct {
  629. rule *RuleBuilder
  630. buf strings.Builder
  631. inputs Paths
  632. implicits Paths
  633. orderOnlys Paths
  634. outputs WritablePaths
  635. symlinkOutputs WritablePaths
  636. depFiles WritablePaths
  637. tools Paths
  638. packagedTools []PackagingSpec
  639. rspFiles []rspFileAndPaths
  640. }
  641. type rspFileAndPaths struct {
  642. file WritablePath
  643. paths Paths
  644. }
  645. func (c *RuleBuilderCommand) addInput(path Path) string {
  646. c.inputs = append(c.inputs, path)
  647. return c.PathForInput(path)
  648. }
  649. func (c *RuleBuilderCommand) addImplicit(path Path) {
  650. c.implicits = append(c.implicits, path)
  651. }
  652. func (c *RuleBuilderCommand) addOrderOnly(path Path) {
  653. c.orderOnlys = append(c.orderOnlys, path)
  654. }
  655. // PathForInput takes an input path and returns the appropriate path to use on the command line. If
  656. // sbox was enabled via a call to RuleBuilder.Sbox() and the path was an output path it returns a
  657. // path with the placeholder prefix used for outputs in sbox. If sbox is not enabled it returns the
  658. // original path.
  659. func (c *RuleBuilderCommand) PathForInput(path Path) string {
  660. if c.rule.sbox {
  661. rel, inSandbox := c.rule._sboxPathForInputRel(path)
  662. if inSandbox {
  663. rel = filepath.Join(sboxSandboxBaseDir, rel)
  664. }
  665. return rel
  666. }
  667. return path.String()
  668. }
  669. // PathsForInputs takes a list of input paths and returns the appropriate paths to use on the
  670. // command line. If sbox was enabled via a call to RuleBuilder.Sbox() a path was an output path, it
  671. // returns the path with the placeholder prefix used for outputs in sbox. If sbox is not enabled it
  672. // returns the original paths.
  673. func (c *RuleBuilderCommand) PathsForInputs(paths Paths) []string {
  674. ret := make([]string, len(paths))
  675. for i, path := range paths {
  676. ret[i] = c.PathForInput(path)
  677. }
  678. return ret
  679. }
  680. // PathForOutput takes an output path and returns the appropriate path to use on the command
  681. // line. If sbox was enabled via a call to RuleBuilder.Sbox(), it returns a path with the
  682. // placeholder prefix used for outputs in sbox. If sbox is not enabled it returns the
  683. // original path.
  684. func (c *RuleBuilderCommand) PathForOutput(path WritablePath) string {
  685. if c.rule.sbox {
  686. // Errors will be handled in RuleBuilder.Build where we have a context to report them
  687. rel, _, _ := maybeRelErr(c.rule.outDir.String(), path.String())
  688. return filepath.Join(sboxOutDir, rel)
  689. }
  690. return path.String()
  691. }
  692. func sboxPathForToolRel(ctx BuilderContext, path Path) string {
  693. // Errors will be handled in RuleBuilder.Build where we have a context to report them
  694. relOut, isRelOut, _ := maybeRelErr(PathForOutput(ctx, "host", ctx.Config().PrebuiltOS()).String(), path.String())
  695. if isRelOut {
  696. // The tool is in the output directory, it will be copied to __SBOX_OUT_DIR__/tools/out
  697. return filepath.Join(sboxToolsSubDir, "out", relOut)
  698. }
  699. // The tool is in the source directory, it will be copied to __SBOX_OUT_DIR__/tools/src
  700. return filepath.Join(sboxToolsSubDir, "src", path.String())
  701. }
  702. func (r *RuleBuilder) _sboxPathForInputRel(path Path) (rel string, inSandbox bool) {
  703. // Errors will be handled in RuleBuilder.Build where we have a context to report them
  704. rel, isRelSboxOut, _ := maybeRelErr(r.outDir.String(), path.String())
  705. if isRelSboxOut {
  706. return filepath.Join(sboxOutSubDir, rel), true
  707. }
  708. if r.sboxInputs {
  709. // When sandboxing inputs all inputs have to be copied into the sandbox. Input files that
  710. // are outputs of other rules could be an arbitrary absolute path if OUT_DIR is set, so they
  711. // will be copied to relative paths under __SBOX_OUT_DIR__/out.
  712. rel, isRelOut, _ := maybeRelErr(PathForOutput(r.ctx).String(), path.String())
  713. if isRelOut {
  714. return filepath.Join(sboxOutSubDir, rel), true
  715. }
  716. }
  717. return path.String(), false
  718. }
  719. func (r *RuleBuilder) sboxPathForInputRel(path Path) string {
  720. rel, _ := r._sboxPathForInputRel(path)
  721. return rel
  722. }
  723. func (r *RuleBuilder) sboxPathsForInputsRel(paths Paths) []string {
  724. ret := make([]string, len(paths))
  725. for i, path := range paths {
  726. ret[i] = r.sboxPathForInputRel(path)
  727. }
  728. return ret
  729. }
  730. func sboxPathForPackagedToolRel(spec PackagingSpec) string {
  731. return filepath.Join(sboxToolsSubDir, "out", spec.relPathInPackage)
  732. }
  733. // PathForPackagedTool takes a PackageSpec for a tool and returns the corresponding path for the
  734. // tool after copying it into the sandbox. This can be used on the RuleBuilder command line to
  735. // reference the tool.
  736. func (c *RuleBuilderCommand) PathForPackagedTool(spec PackagingSpec) string {
  737. if !c.rule.sboxTools {
  738. panic("PathForPackagedTool() requires SandboxTools()")
  739. }
  740. return filepath.Join(sboxSandboxBaseDir, sboxPathForPackagedToolRel(spec))
  741. }
  742. // PathForTool takes a path to a tool, which may be an output file or a source file, and returns
  743. // the corresponding path for the tool in the sbox sandbox if sbox is enabled, or the original path
  744. // if it is not. This can be used on the RuleBuilder command line to reference the tool.
  745. func (c *RuleBuilderCommand) PathForTool(path Path) string {
  746. if c.rule.sbox && c.rule.sboxTools {
  747. return filepath.Join(sboxSandboxBaseDir, sboxPathForToolRel(c.rule.ctx, path))
  748. }
  749. return path.String()
  750. }
  751. // PathsForTools takes a list of paths to tools, which may be output files or source files, and
  752. // returns the corresponding paths for the tools in the sbox sandbox if sbox is enabled, or the
  753. // original paths if it is not. This can be used on the RuleBuilder command line to reference the tool.
  754. func (c *RuleBuilderCommand) PathsForTools(paths Paths) []string {
  755. if c.rule.sbox && c.rule.sboxTools {
  756. var ret []string
  757. for _, path := range paths {
  758. ret = append(ret, filepath.Join(sboxSandboxBaseDir, sboxPathForToolRel(c.rule.ctx, path)))
  759. }
  760. return ret
  761. }
  762. return paths.Strings()
  763. }
  764. // PackagedTool adds the specified tool path to the command line. It can only be used with tool
  765. // sandboxing enabled by SandboxTools(), and will copy the tool into the sandbox.
  766. func (c *RuleBuilderCommand) PackagedTool(spec PackagingSpec) *RuleBuilderCommand {
  767. if !c.rule.sboxTools {
  768. panic("PackagedTool() requires SandboxTools()")
  769. }
  770. c.packagedTools = append(c.packagedTools, spec)
  771. c.Text(sboxPathForPackagedToolRel(spec))
  772. return c
  773. }
  774. // ImplicitPackagedTool copies the specified tool into the sandbox without modifying the command
  775. // line. It can only be used with tool sandboxing enabled by SandboxTools().
  776. func (c *RuleBuilderCommand) ImplicitPackagedTool(spec PackagingSpec) *RuleBuilderCommand {
  777. if !c.rule.sboxTools {
  778. panic("ImplicitPackagedTool() requires SandboxTools()")
  779. }
  780. c.packagedTools = append(c.packagedTools, spec)
  781. return c
  782. }
  783. // ImplicitPackagedTools copies the specified tools into the sandbox without modifying the command
  784. // line. It can only be used with tool sandboxing enabled by SandboxTools().
  785. func (c *RuleBuilderCommand) ImplicitPackagedTools(specs []PackagingSpec) *RuleBuilderCommand {
  786. if !c.rule.sboxTools {
  787. panic("ImplicitPackagedTools() requires SandboxTools()")
  788. }
  789. c.packagedTools = append(c.packagedTools, specs...)
  790. return c
  791. }
  792. // Text adds the specified raw text to the command line. The text should not contain input or output paths or the
  793. // rule will not have them listed in its dependencies or outputs.
  794. func (c *RuleBuilderCommand) Text(text string) *RuleBuilderCommand {
  795. if c.buf.Len() > 0 {
  796. c.buf.WriteByte(' ')
  797. }
  798. c.buf.WriteString(text)
  799. return c
  800. }
  801. // Textf adds the specified formatted text to the command line. The text should not contain input or output paths or
  802. // the rule will not have them listed in its dependencies or outputs.
  803. func (c *RuleBuilderCommand) Textf(format string, a ...interface{}) *RuleBuilderCommand {
  804. return c.Text(fmt.Sprintf(format, a...))
  805. }
  806. // Flag adds the specified raw text to the command line. The text should not contain input or output paths or the
  807. // rule will not have them listed in its dependencies or outputs.
  808. func (c *RuleBuilderCommand) Flag(flag string) *RuleBuilderCommand {
  809. return c.Text(flag)
  810. }
  811. // OptionalFlag adds the specified raw text to the command line if it is not nil. The text should not contain input or
  812. // output paths or the rule will not have them listed in its dependencies or outputs.
  813. func (c *RuleBuilderCommand) OptionalFlag(flag *string) *RuleBuilderCommand {
  814. if flag != nil {
  815. c.Text(*flag)
  816. }
  817. return c
  818. }
  819. // Flags adds the specified raw text to the command line. The text should not contain input or output paths or the
  820. // rule will not have them listed in its dependencies or outputs.
  821. func (c *RuleBuilderCommand) Flags(flags []string) *RuleBuilderCommand {
  822. for _, flag := range flags {
  823. c.Text(flag)
  824. }
  825. return c
  826. }
  827. // FlagWithArg adds the specified flag and argument text to the command line, with no separator between them. The flag
  828. // and argument should not contain input or output paths or the rule will not have them listed in its dependencies or
  829. // outputs.
  830. func (c *RuleBuilderCommand) FlagWithArg(flag, arg string) *RuleBuilderCommand {
  831. return c.Text(flag + arg)
  832. }
  833. // FlagForEachArg adds the specified flag joined with each argument to the command line. The result is identical to
  834. // calling FlagWithArg for argument.
  835. func (c *RuleBuilderCommand) FlagForEachArg(flag string, args []string) *RuleBuilderCommand {
  836. for _, arg := range args {
  837. c.FlagWithArg(flag, arg)
  838. }
  839. return c
  840. }
  841. // FlagWithList adds the specified flag and list of arguments to the command line, with the arguments joined by sep
  842. // and no separator between the flag and arguments. The flag and arguments should not contain input or output paths or
  843. // the rule will not have them listed in its dependencies or outputs.
  844. func (c *RuleBuilderCommand) FlagWithList(flag string, list []string, sep string) *RuleBuilderCommand {
  845. return c.Text(flag + strings.Join(list, sep))
  846. }
  847. // Tool adds the specified tool path to the command line. The path will be also added to the dependencies returned by
  848. // RuleBuilder.Tools.
  849. func (c *RuleBuilderCommand) Tool(path Path) *RuleBuilderCommand {
  850. c.tools = append(c.tools, path)
  851. return c.Text(c.PathForTool(path))
  852. }
  853. // Tool adds the specified tool path to the dependencies returned by RuleBuilder.Tools.
  854. func (c *RuleBuilderCommand) ImplicitTool(path Path) *RuleBuilderCommand {
  855. c.tools = append(c.tools, path)
  856. return c
  857. }
  858. // Tool adds the specified tool path to the dependencies returned by RuleBuilder.Tools.
  859. func (c *RuleBuilderCommand) ImplicitTools(paths Paths) *RuleBuilderCommand {
  860. c.tools = append(c.tools, paths...)
  861. return c
  862. }
  863. // BuiltTool adds the specified tool path that was built using a host Soong module to the command line. The path will
  864. // be also added to the dependencies returned by RuleBuilder.Tools.
  865. //
  866. // It is equivalent to:
  867. // cmd.Tool(ctx.Config().HostToolPath(ctx, tool))
  868. func (c *RuleBuilderCommand) BuiltTool(tool string) *RuleBuilderCommand {
  869. return c.Tool(c.rule.ctx.Config().HostToolPath(c.rule.ctx, tool))
  870. }
  871. // PrebuiltBuildTool adds the specified tool path from prebuils/build-tools. The path will be also added to the
  872. // dependencies returned by RuleBuilder.Tools.
  873. //
  874. // It is equivalent to:
  875. // cmd.Tool(ctx.Config().PrebuiltBuildTool(ctx, tool))
  876. func (c *RuleBuilderCommand) PrebuiltBuildTool(ctx PathContext, tool string) *RuleBuilderCommand {
  877. return c.Tool(ctx.Config().PrebuiltBuildTool(ctx, tool))
  878. }
  879. // Input adds the specified input path to the command line. The path will also be added to the dependencies returned by
  880. // RuleBuilder.Inputs.
  881. func (c *RuleBuilderCommand) Input(path Path) *RuleBuilderCommand {
  882. return c.Text(c.addInput(path))
  883. }
  884. // Inputs adds the specified input paths to the command line, separated by spaces. The paths will also be added to the
  885. // dependencies returned by RuleBuilder.Inputs.
  886. func (c *RuleBuilderCommand) Inputs(paths Paths) *RuleBuilderCommand {
  887. for _, path := range paths {
  888. c.Input(path)
  889. }
  890. return c
  891. }
  892. // Implicit adds the specified input path to the dependencies returned by RuleBuilder.Inputs without modifying the
  893. // command line.
  894. func (c *RuleBuilderCommand) Implicit(path Path) *RuleBuilderCommand {
  895. c.addImplicit(path)
  896. return c
  897. }
  898. // Implicits adds the specified input paths to the dependencies returned by RuleBuilder.Inputs without modifying the
  899. // command line.
  900. func (c *RuleBuilderCommand) Implicits(paths Paths) *RuleBuilderCommand {
  901. for _, path := range paths {
  902. c.addImplicit(path)
  903. }
  904. return c
  905. }
  906. // GetImplicits returns the command's implicit inputs.
  907. func (c *RuleBuilderCommand) GetImplicits() Paths {
  908. return c.implicits
  909. }
  910. // OrderOnly adds the specified input path to the dependencies returned by RuleBuilder.OrderOnlys
  911. // without modifying the command line.
  912. func (c *RuleBuilderCommand) OrderOnly(path Path) *RuleBuilderCommand {
  913. c.addOrderOnly(path)
  914. return c
  915. }
  916. // OrderOnlys adds the specified input paths to the dependencies returned by RuleBuilder.OrderOnlys
  917. // without modifying the command line.
  918. func (c *RuleBuilderCommand) OrderOnlys(paths Paths) *RuleBuilderCommand {
  919. for _, path := range paths {
  920. c.addOrderOnly(path)
  921. }
  922. return c
  923. }
  924. // Output adds the specified output path to the command line. The path will also be added to the outputs returned by
  925. // RuleBuilder.Outputs.
  926. func (c *RuleBuilderCommand) Output(path WritablePath) *RuleBuilderCommand {
  927. c.outputs = append(c.outputs, path)
  928. return c.Text(c.PathForOutput(path))
  929. }
  930. // Outputs adds the specified output paths to the command line, separated by spaces. The paths will also be added to
  931. // the outputs returned by RuleBuilder.Outputs.
  932. func (c *RuleBuilderCommand) Outputs(paths WritablePaths) *RuleBuilderCommand {
  933. for _, path := range paths {
  934. c.Output(path)
  935. }
  936. return c
  937. }
  938. // OutputDir adds the output directory to the command line. This is only available when used with RuleBuilder.Sbox,
  939. // and will be the temporary output directory managed by sbox, not the final one.
  940. func (c *RuleBuilderCommand) OutputDir() *RuleBuilderCommand {
  941. if !c.rule.sbox {
  942. panic("OutputDir only valid with Sbox")
  943. }
  944. return c.Text(sboxOutDir)
  945. }
  946. // DepFile adds the specified depfile path to the paths returned by RuleBuilder.DepFiles and adds it to the command
  947. // line, and causes RuleBuilder.Build file to set the depfile flag for ninja. If multiple depfiles are added to
  948. // commands in a single RuleBuilder then RuleBuilder.Build will add an extra command to merge the depfiles together.
  949. func (c *RuleBuilderCommand) DepFile(path WritablePath) *RuleBuilderCommand {
  950. c.depFiles = append(c.depFiles, path)
  951. return c.Text(c.PathForOutput(path))
  952. }
  953. // ImplicitOutput adds the specified output path to the dependencies returned by RuleBuilder.Outputs without modifying
  954. // the command line.
  955. func (c *RuleBuilderCommand) ImplicitOutput(path WritablePath) *RuleBuilderCommand {
  956. c.outputs = append(c.outputs, path)
  957. return c
  958. }
  959. // ImplicitOutputs adds the specified output paths to the dependencies returned by RuleBuilder.Outputs without modifying
  960. // the command line.
  961. func (c *RuleBuilderCommand) ImplicitOutputs(paths WritablePaths) *RuleBuilderCommand {
  962. c.outputs = append(c.outputs, paths...)
  963. return c
  964. }
  965. // ImplicitSymlinkOutput declares the specified path as an implicit output that
  966. // will be a symlink instead of a regular file. Does not modify the command
  967. // line.
  968. func (c *RuleBuilderCommand) ImplicitSymlinkOutput(path WritablePath) *RuleBuilderCommand {
  969. c.symlinkOutputs = append(c.symlinkOutputs, path)
  970. return c.ImplicitOutput(path)
  971. }
  972. // ImplicitSymlinkOutputs declares the specified paths as implicit outputs that
  973. // will be a symlinks instead of regular files. Does not modify the command
  974. // line.
  975. func (c *RuleBuilderCommand) ImplicitSymlinkOutputs(paths WritablePaths) *RuleBuilderCommand {
  976. for _, path := range paths {
  977. c.ImplicitSymlinkOutput(path)
  978. }
  979. return c
  980. }
  981. // SymlinkOutput declares the specified path as an output that will be a symlink
  982. // instead of a regular file. Modifies the command line.
  983. func (c *RuleBuilderCommand) SymlinkOutput(path WritablePath) *RuleBuilderCommand {
  984. c.symlinkOutputs = append(c.symlinkOutputs, path)
  985. return c.Output(path)
  986. }
  987. // SymlinkOutputsl declares the specified paths as outputs that will be symlinks
  988. // instead of regular files. Modifies the command line.
  989. func (c *RuleBuilderCommand) SymlinkOutputs(paths WritablePaths) *RuleBuilderCommand {
  990. for _, path := range paths {
  991. c.SymlinkOutput(path)
  992. }
  993. return c
  994. }
  995. // ImplicitDepFile adds the specified depfile path to the paths returned by RuleBuilder.DepFiles without modifying
  996. // the command line, and causes RuleBuilder.Build file to set the depfile flag for ninja. If multiple depfiles
  997. // are added to commands in a single RuleBuilder then RuleBuilder.Build will add an extra command to merge the
  998. // depfiles together.
  999. func (c *RuleBuilderCommand) ImplicitDepFile(path WritablePath) *RuleBuilderCommand {
  1000. c.depFiles = append(c.depFiles, path)
  1001. return c
  1002. }
  1003. // FlagWithInput adds the specified flag and input path to the command line, with no separator between them. The path
  1004. // will also be added to the dependencies returned by RuleBuilder.Inputs.
  1005. func (c *RuleBuilderCommand) FlagWithInput(flag string, path Path) *RuleBuilderCommand {
  1006. return c.Text(flag + c.addInput(path))
  1007. }
  1008. // FlagWithInputList adds the specified flag and input paths to the command line, with the inputs joined by sep
  1009. // and no separator between the flag and inputs. The input paths will also be added to the dependencies returned by
  1010. // RuleBuilder.Inputs.
  1011. func (c *RuleBuilderCommand) FlagWithInputList(flag string, paths Paths, sep string) *RuleBuilderCommand {
  1012. strs := make([]string, len(paths))
  1013. for i, path := range paths {
  1014. strs[i] = c.addInput(path)
  1015. }
  1016. return c.FlagWithList(flag, strs, sep)
  1017. }
  1018. // FlagForEachInput adds the specified flag joined with each input path to the command line. The input paths will also
  1019. // be added to the dependencies returned by RuleBuilder.Inputs. The result is identical to calling FlagWithInput for
  1020. // each input path.
  1021. func (c *RuleBuilderCommand) FlagForEachInput(flag string, paths Paths) *RuleBuilderCommand {
  1022. for _, path := range paths {
  1023. c.FlagWithInput(flag, path)
  1024. }
  1025. return c
  1026. }
  1027. // FlagWithOutput adds the specified flag and output path to the command line, with no separator between them. The path
  1028. // will also be added to the outputs returned by RuleBuilder.Outputs.
  1029. func (c *RuleBuilderCommand) FlagWithOutput(flag string, path WritablePath) *RuleBuilderCommand {
  1030. c.outputs = append(c.outputs, path)
  1031. return c.Text(flag + c.PathForOutput(path))
  1032. }
  1033. // FlagWithDepFile adds the specified flag and depfile path to the command line, with no separator between them. The path
  1034. // will also be added to the outputs returned by RuleBuilder.Outputs.
  1035. func (c *RuleBuilderCommand) FlagWithDepFile(flag string, path WritablePath) *RuleBuilderCommand {
  1036. c.depFiles = append(c.depFiles, path)
  1037. return c.Text(flag + c.PathForOutput(path))
  1038. }
  1039. // FlagWithRspFileInputList adds the specified flag and path to an rspfile to the command line, with
  1040. // no separator between them. The paths will be written to the rspfile. If sbox is enabled, the
  1041. // rspfile must be outside the sbox directory. The first use of FlagWithRspFileInputList in any
  1042. // RuleBuilderCommand of a RuleBuilder will use Ninja's rsp file support for the rule, additional
  1043. // uses will result in an auxiliary rules to write the rspFile contents.
  1044. func (c *RuleBuilderCommand) FlagWithRspFileInputList(flag string, rspFile WritablePath, paths Paths) *RuleBuilderCommand {
  1045. // Use an empty slice if paths is nil, the non-nil slice is used as an indicator that the rsp file must be
  1046. // generated.
  1047. if paths == nil {
  1048. paths = Paths{}
  1049. }
  1050. c.rspFiles = append(c.rspFiles, rspFileAndPaths{rspFile, paths})
  1051. if c.rule.sbox {
  1052. if _, isRel, _ := maybeRelErr(c.rule.outDir.String(), rspFile.String()); isRel {
  1053. panic(fmt.Errorf("FlagWithRspFileInputList rspfile %q must not be inside out dir %q",
  1054. rspFile.String(), c.rule.outDir.String()))
  1055. }
  1056. }
  1057. c.FlagWithArg(flag, c.PathForInput(rspFile))
  1058. return c
  1059. }
  1060. // String returns the command line.
  1061. func (c *RuleBuilderCommand) String() string {
  1062. return c.buf.String()
  1063. }
  1064. // RuleBuilderSboxProtoForTests takes the BuildParams for the manifest passed to RuleBuilder.Sbox()
  1065. // and returns sbox testproto generated by the RuleBuilder.
  1066. func RuleBuilderSboxProtoForTests(t *testing.T, params TestingBuildParams) *sbox_proto.Manifest {
  1067. t.Helper()
  1068. content := ContentFromFileRuleForTests(t, params)
  1069. manifest := sbox_proto.Manifest{}
  1070. err := proto.UnmarshalText(content, &manifest)
  1071. if err != nil {
  1072. t.Fatalf("failed to unmarshal manifest: %s", err.Error())
  1073. }
  1074. return &manifest
  1075. }
  1076. func ninjaNameEscape(s string) string {
  1077. b := []byte(s)
  1078. escaped := false
  1079. for i, c := range b {
  1080. valid := (c >= 'a' && c <= 'z') ||
  1081. (c >= 'A' && c <= 'Z') ||
  1082. (c >= '0' && c <= '9') ||
  1083. (c == '_') ||
  1084. (c == '-') ||
  1085. (c == '.')
  1086. if !valid {
  1087. b[i] = '_'
  1088. escaped = true
  1089. }
  1090. }
  1091. if escaped {
  1092. s = string(b)
  1093. }
  1094. return s
  1095. }
  1096. // hashSrcFiles returns a hash of the list of source files. It is used to ensure the command line
  1097. // or the sbox textproto manifest change even if the input files are not listed on the command line.
  1098. func hashSrcFiles(srcFiles Paths) string {
  1099. h := sha256.New()
  1100. srcFileList := strings.Join(srcFiles.Strings(), "\n")
  1101. h.Write([]byte(srcFileList))
  1102. return fmt.Sprintf("%x", h.Sum(nil))
  1103. }
  1104. // BuilderContextForTesting returns a BuilderContext for the given config that can be used for tests
  1105. // that need to call methods that take a BuilderContext.
  1106. func BuilderContextForTesting(config Config) BuilderContext {
  1107. pathCtx := PathContextForTesting(config)
  1108. return builderContextForTests{
  1109. PathContext: pathCtx,
  1110. }
  1111. }
  1112. type builderContextForTests struct {
  1113. PathContext
  1114. }
  1115. func (builderContextForTests) Rule(PackageContext, string, blueprint.RuleParams, ...string) blueprint.Rule {
  1116. return nil
  1117. }
  1118. func (builderContextForTests) Build(PackageContext, BuildParams) {}
  1119. func writeRspFileRule(ctx BuilderContext, rspFile WritablePath, paths Paths) {
  1120. buf := &strings.Builder{}
  1121. err := response.WriteRspFile(buf, paths.Strings())
  1122. if err != nil {
  1123. // There should never be I/O errors writing to a bytes.Buffer.
  1124. panic(err)
  1125. }
  1126. WriteFileRule(ctx, rspFile, buf.String())
  1127. }