rule_builder.go 47 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358
  1. // Copyright 2018 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package android
  15. import (
  16. "crypto/sha256"
  17. "fmt"
  18. "path/filepath"
  19. "sort"
  20. "strings"
  21. "testing"
  22. "github.com/google/blueprint"
  23. "github.com/google/blueprint/proptools"
  24. "google.golang.org/protobuf/encoding/prototext"
  25. "google.golang.org/protobuf/proto"
  26. "android/soong/cmd/sbox/sbox_proto"
  27. "android/soong/remoteexec"
  28. "android/soong/response"
  29. "android/soong/shared"
  30. )
  31. const sboxSandboxBaseDir = "__SBOX_SANDBOX_DIR__"
  32. const sboxOutSubDir = "out"
  33. const sboxToolsSubDir = "tools"
  34. const sboxOutDir = sboxSandboxBaseDir + "/" + sboxOutSubDir
  35. // RuleBuilder provides an alternative to ModuleContext.Rule and ModuleContext.Build to add a command line to the build
  36. // graph.
  37. type RuleBuilder struct {
  38. pctx PackageContext
  39. ctx BuilderContext
  40. commands []*RuleBuilderCommand
  41. installs RuleBuilderInstalls
  42. temporariesSet map[WritablePath]bool
  43. restat bool
  44. sbox bool
  45. highmem bool
  46. remoteable RemoteRuleSupports
  47. rbeParams *remoteexec.REParams
  48. outDir WritablePath
  49. sboxTools bool
  50. sboxInputs bool
  51. sboxManifestPath WritablePath
  52. missingDeps []string
  53. }
  54. // NewRuleBuilder returns a newly created RuleBuilder.
  55. func NewRuleBuilder(pctx PackageContext, ctx BuilderContext) *RuleBuilder {
  56. return &RuleBuilder{
  57. pctx: pctx,
  58. ctx: ctx,
  59. temporariesSet: make(map[WritablePath]bool),
  60. }
  61. }
  62. // RuleBuilderInstall is a tuple of install from and to locations.
  63. type RuleBuilderInstall struct {
  64. From Path
  65. To string
  66. }
  67. type RuleBuilderInstalls []RuleBuilderInstall
  68. // String returns the RuleBuilderInstalls in the form used by $(call copy-many-files) in Make, a space separated
  69. // list of from:to tuples.
  70. func (installs RuleBuilderInstalls) String() string {
  71. sb := strings.Builder{}
  72. for i, install := range installs {
  73. if i != 0 {
  74. sb.WriteRune(' ')
  75. }
  76. sb.WriteString(install.From.String())
  77. sb.WriteRune(':')
  78. sb.WriteString(install.To)
  79. }
  80. return sb.String()
  81. }
  82. // MissingDeps adds modules to the list of missing dependencies. If MissingDeps
  83. // is called with a non-empty input, any call to Build will result in a rule
  84. // that will print an error listing the missing dependencies and fail.
  85. // MissingDeps should only be called if Config.AllowMissingDependencies() is
  86. // true.
  87. func (r *RuleBuilder) MissingDeps(missingDeps []string) {
  88. r.missingDeps = append(r.missingDeps, missingDeps...)
  89. }
  90. // Restat marks the rule as a restat rule, which will be passed to ModuleContext.Rule in BuildParams.Restat.
  91. //
  92. // Restat is not compatible with Sbox()
  93. func (r *RuleBuilder) Restat() *RuleBuilder {
  94. if r.sbox {
  95. panic("Restat() is not compatible with Sbox()")
  96. }
  97. r.restat = true
  98. return r
  99. }
  100. // HighMem marks the rule as a high memory rule, which will limit how many run in parallel with other high memory
  101. // rules.
  102. func (r *RuleBuilder) HighMem() *RuleBuilder {
  103. r.highmem = true
  104. return r
  105. }
  106. // Remoteable marks the rule as supporting remote execution.
  107. func (r *RuleBuilder) Remoteable(supports RemoteRuleSupports) *RuleBuilder {
  108. r.remoteable = supports
  109. return r
  110. }
  111. // Rewrapper marks the rule as running inside rewrapper using the given params in order to support
  112. // running on RBE. During RuleBuilder.Build the params will be combined with the inputs, outputs
  113. // and tools known to RuleBuilder to prepend an appropriate rewrapper command line to the rule's
  114. // command line.
  115. func (r *RuleBuilder) Rewrapper(params *remoteexec.REParams) *RuleBuilder {
  116. if !r.sboxInputs {
  117. panic(fmt.Errorf("RuleBuilder.Rewrapper must be called after RuleBuilder.SandboxInputs"))
  118. }
  119. r.rbeParams = params
  120. return r
  121. }
  122. // Sbox marks the rule as needing to be wrapped by sbox. The outputDir should point to the output
  123. // directory that sbox will wipe. It should not be written to by any other rule. manifestPath should
  124. // point to a location where sbox's manifest will be written and must be outside outputDir. sbox
  125. // will ensure that all outputs have been written, and will discard any output files that were not
  126. // specified.
  127. //
  128. // Sbox is not compatible with Restat()
  129. func (r *RuleBuilder) Sbox(outputDir WritablePath, manifestPath WritablePath) *RuleBuilder {
  130. if r.sbox {
  131. panic("Sbox() may not be called more than once")
  132. }
  133. if len(r.commands) > 0 {
  134. panic("Sbox() may not be called after Command()")
  135. }
  136. if r.restat {
  137. panic("Sbox() is not compatible with Restat()")
  138. }
  139. r.sbox = true
  140. r.outDir = outputDir
  141. r.sboxManifestPath = manifestPath
  142. return r
  143. }
  144. // SandboxTools enables tool sandboxing for the rule by copying any referenced tools into the
  145. // sandbox.
  146. func (r *RuleBuilder) SandboxTools() *RuleBuilder {
  147. if !r.sbox {
  148. panic("SandboxTools() must be called after Sbox()")
  149. }
  150. if len(r.commands) > 0 {
  151. panic("SandboxTools() may not be called after Command()")
  152. }
  153. r.sboxTools = true
  154. return r
  155. }
  156. // SandboxInputs enables input sandboxing for the rule by copying any referenced inputs into the
  157. // sandbox. It also implies SandboxTools().
  158. //
  159. // Sandboxing inputs requires RuleBuilder to be aware of all references to input paths. Paths
  160. // that are passed to RuleBuilder outside of the methods that expect inputs, for example
  161. // FlagWithArg, must use RuleBuilderCommand.PathForInput to translate the path to one that matches
  162. // the sandbox layout.
  163. func (r *RuleBuilder) SandboxInputs() *RuleBuilder {
  164. if !r.sbox {
  165. panic("SandboxInputs() must be called after Sbox()")
  166. }
  167. if len(r.commands) > 0 {
  168. panic("SandboxInputs() may not be called after Command()")
  169. }
  170. r.sboxTools = true
  171. r.sboxInputs = true
  172. return r
  173. }
  174. // Install associates an output of the rule with an install location, which can be retrieved later using
  175. // RuleBuilder.Installs.
  176. func (r *RuleBuilder) Install(from Path, to string) {
  177. r.installs = append(r.installs, RuleBuilderInstall{from, to})
  178. }
  179. // Command returns a new RuleBuilderCommand for the rule. The commands will be ordered in the rule by when they were
  180. // created by this method. That can be mutated through their methods in any order, as long as the mutations do not
  181. // race with any call to Build.
  182. func (r *RuleBuilder) Command() *RuleBuilderCommand {
  183. command := &RuleBuilderCommand{
  184. rule: r,
  185. }
  186. r.commands = append(r.commands, command)
  187. return command
  188. }
  189. // Temporary marks an output of a command as an intermediate file that will be used as an input to another command
  190. // in the same rule, and should not be listed in Outputs.
  191. func (r *RuleBuilder) Temporary(path WritablePath) {
  192. r.temporariesSet[path] = true
  193. }
  194. // DeleteTemporaryFiles adds a command to the rule that deletes any outputs that have been marked using Temporary
  195. // when the rule runs. DeleteTemporaryFiles should be called after all calls to Temporary.
  196. func (r *RuleBuilder) DeleteTemporaryFiles() {
  197. var temporariesList WritablePaths
  198. for intermediate := range r.temporariesSet {
  199. temporariesList = append(temporariesList, intermediate)
  200. }
  201. sort.Slice(temporariesList, func(i, j int) bool {
  202. return temporariesList[i].String() < temporariesList[j].String()
  203. })
  204. r.Command().Text("rm").Flag("-f").Outputs(temporariesList)
  205. }
  206. // Inputs returns the list of paths that were passed to the RuleBuilderCommand methods that take
  207. // input paths, such as RuleBuilderCommand.Input, RuleBuilderCommand.Implicit, or
  208. // RuleBuilderCommand.FlagWithInput. Inputs to a command that are also outputs of another command
  209. // in the same RuleBuilder are filtered out. The list is sorted and duplicates removed.
  210. func (r *RuleBuilder) Inputs() Paths {
  211. outputs := r.outputSet()
  212. depFiles := r.depFileSet()
  213. inputs := make(map[string]Path)
  214. for _, c := range r.commands {
  215. for _, input := range append(c.inputs, c.implicits...) {
  216. inputStr := input.String()
  217. if _, isOutput := outputs[inputStr]; !isOutput {
  218. if _, isDepFile := depFiles[inputStr]; !isDepFile {
  219. inputs[input.String()] = input
  220. }
  221. }
  222. }
  223. }
  224. var inputList Paths
  225. for _, input := range inputs {
  226. inputList = append(inputList, input)
  227. }
  228. sort.Slice(inputList, func(i, j int) bool {
  229. return inputList[i].String() < inputList[j].String()
  230. })
  231. return inputList
  232. }
  233. // OrderOnlys returns the list of paths that were passed to the RuleBuilderCommand.OrderOnly or
  234. // RuleBuilderCommand.OrderOnlys. The list is sorted and duplicates removed.
  235. func (r *RuleBuilder) OrderOnlys() Paths {
  236. orderOnlys := make(map[string]Path)
  237. for _, c := range r.commands {
  238. for _, orderOnly := range c.orderOnlys {
  239. orderOnlys[orderOnly.String()] = orderOnly
  240. }
  241. }
  242. var orderOnlyList Paths
  243. for _, orderOnly := range orderOnlys {
  244. orderOnlyList = append(orderOnlyList, orderOnly)
  245. }
  246. sort.Slice(orderOnlyList, func(i, j int) bool {
  247. return orderOnlyList[i].String() < orderOnlyList[j].String()
  248. })
  249. return orderOnlyList
  250. }
  251. // Validations returns the list of paths that were passed to RuleBuilderCommand.Validation or
  252. // RuleBuilderCommand.Validations. The list is sorted and duplicates removed.
  253. func (r *RuleBuilder) Validations() Paths {
  254. validations := make(map[string]Path)
  255. for _, c := range r.commands {
  256. for _, validation := range c.validations {
  257. validations[validation.String()] = validation
  258. }
  259. }
  260. var validationList Paths
  261. for _, validation := range validations {
  262. validationList = append(validationList, validation)
  263. }
  264. sort.Slice(validationList, func(i, j int) bool {
  265. return validationList[i].String() < validationList[j].String()
  266. })
  267. return validationList
  268. }
  269. func (r *RuleBuilder) outputSet() map[string]WritablePath {
  270. outputs := make(map[string]WritablePath)
  271. for _, c := range r.commands {
  272. for _, output := range c.outputs {
  273. outputs[output.String()] = output
  274. }
  275. }
  276. return outputs
  277. }
  278. // Outputs returns the list of paths that were passed to the RuleBuilderCommand methods that take
  279. // output paths, such as RuleBuilderCommand.Output, RuleBuilderCommand.ImplicitOutput, or
  280. // RuleBuilderCommand.FlagWithInput. The list is sorted and duplicates removed.
  281. func (r *RuleBuilder) Outputs() WritablePaths {
  282. outputs := r.outputSet()
  283. var outputList WritablePaths
  284. for _, output := range outputs {
  285. if !r.temporariesSet[output] {
  286. outputList = append(outputList, output)
  287. }
  288. }
  289. sort.Slice(outputList, func(i, j int) bool {
  290. return outputList[i].String() < outputList[j].String()
  291. })
  292. return outputList
  293. }
  294. func (r *RuleBuilder) symlinkOutputSet() map[string]WritablePath {
  295. symlinkOutputs := make(map[string]WritablePath)
  296. for _, c := range r.commands {
  297. for _, symlinkOutput := range c.symlinkOutputs {
  298. symlinkOutputs[symlinkOutput.String()] = symlinkOutput
  299. }
  300. }
  301. return symlinkOutputs
  302. }
  303. // SymlinkOutputs returns the list of paths that the executor (Ninja) would
  304. // verify, after build edge completion, that:
  305. //
  306. // 1) Created output symlinks match the list of paths in this list exactly (no more, no fewer)
  307. // 2) Created output files are *not* declared in this list.
  308. //
  309. // These symlink outputs are expected to be a subset of outputs or implicit
  310. // outputs, or they would fail validation at build param construction time
  311. // later, to support other non-rule-builder approaches for constructing
  312. // statements.
  313. func (r *RuleBuilder) SymlinkOutputs() WritablePaths {
  314. symlinkOutputs := r.symlinkOutputSet()
  315. var symlinkOutputList WritablePaths
  316. for _, symlinkOutput := range symlinkOutputs {
  317. symlinkOutputList = append(symlinkOutputList, symlinkOutput)
  318. }
  319. sort.Slice(symlinkOutputList, func(i, j int) bool {
  320. return symlinkOutputList[i].String() < symlinkOutputList[j].String()
  321. })
  322. return symlinkOutputList
  323. }
  324. func (r *RuleBuilder) depFileSet() map[string]WritablePath {
  325. depFiles := make(map[string]WritablePath)
  326. for _, c := range r.commands {
  327. for _, depFile := range c.depFiles {
  328. depFiles[depFile.String()] = depFile
  329. }
  330. }
  331. return depFiles
  332. }
  333. // DepFiles returns the list of paths that were passed to the RuleBuilderCommand methods that take depfile paths, such
  334. // as RuleBuilderCommand.DepFile or RuleBuilderCommand.FlagWithDepFile.
  335. func (r *RuleBuilder) DepFiles() WritablePaths {
  336. var depFiles WritablePaths
  337. for _, c := range r.commands {
  338. for _, depFile := range c.depFiles {
  339. depFiles = append(depFiles, depFile)
  340. }
  341. }
  342. return depFiles
  343. }
  344. // Installs returns the list of tuples passed to Install.
  345. func (r *RuleBuilder) Installs() RuleBuilderInstalls {
  346. return append(RuleBuilderInstalls(nil), r.installs...)
  347. }
  348. func (r *RuleBuilder) toolsSet() map[string]Path {
  349. tools := make(map[string]Path)
  350. for _, c := range r.commands {
  351. for _, tool := range c.tools {
  352. tools[tool.String()] = tool
  353. }
  354. }
  355. return tools
  356. }
  357. // Tools returns the list of paths that were passed to the RuleBuilderCommand.Tool method. The
  358. // list is sorted and duplicates removed.
  359. func (r *RuleBuilder) Tools() Paths {
  360. toolsSet := r.toolsSet()
  361. var toolsList Paths
  362. for _, tool := range toolsSet {
  363. toolsList = append(toolsList, tool)
  364. }
  365. sort.Slice(toolsList, func(i, j int) bool {
  366. return toolsList[i].String() < toolsList[j].String()
  367. })
  368. return toolsList
  369. }
  370. // RspFileInputs returns the list of paths that were passed to the RuleBuilderCommand.FlagWithRspFileInputList method.
  371. func (r *RuleBuilder) RspFileInputs() Paths {
  372. var rspFileInputs Paths
  373. for _, c := range r.commands {
  374. for _, rspFile := range c.rspFiles {
  375. rspFileInputs = append(rspFileInputs, rspFile.paths...)
  376. }
  377. }
  378. return rspFileInputs
  379. }
  380. func (r *RuleBuilder) rspFiles() []rspFileAndPaths {
  381. var rspFiles []rspFileAndPaths
  382. for _, c := range r.commands {
  383. rspFiles = append(rspFiles, c.rspFiles...)
  384. }
  385. return rspFiles
  386. }
  387. // Commands returns a slice containing the built command line for each call to RuleBuilder.Command.
  388. func (r *RuleBuilder) Commands() []string {
  389. var commands []string
  390. for _, c := range r.commands {
  391. commands = append(commands, c.String())
  392. }
  393. return commands
  394. }
  395. // BuilderContext is a subset of ModuleContext and SingletonContext.
  396. type BuilderContext interface {
  397. PathContext
  398. Rule(PackageContext, string, blueprint.RuleParams, ...string) blueprint.Rule
  399. Build(PackageContext, BuildParams)
  400. }
  401. var _ BuilderContext = ModuleContext(nil)
  402. var _ BuilderContext = SingletonContext(nil)
  403. func (r *RuleBuilder) depFileMergerCmd(depFiles WritablePaths) *RuleBuilderCommand {
  404. return r.Command().
  405. BuiltTool("dep_fixer").
  406. Inputs(depFiles.Paths())
  407. }
  408. // Build adds the built command line to the build graph, with dependencies on Inputs and Tools, and output files for
  409. // Outputs.
  410. func (r *RuleBuilder) Build(name string, desc string) {
  411. name = ninjaNameEscape(name)
  412. if len(r.missingDeps) > 0 {
  413. r.ctx.Build(pctx, BuildParams{
  414. Rule: ErrorRule,
  415. Outputs: r.Outputs(),
  416. Description: desc,
  417. Args: map[string]string{
  418. "error": "missing dependencies: " + strings.Join(r.missingDeps, ", "),
  419. },
  420. })
  421. return
  422. }
  423. var depFile WritablePath
  424. var depFormat blueprint.Deps
  425. if depFiles := r.DepFiles(); len(depFiles) > 0 {
  426. depFile = depFiles[0]
  427. depFormat = blueprint.DepsGCC
  428. if len(depFiles) > 1 {
  429. // Add a command locally that merges all depfiles together into the first depfile.
  430. r.depFileMergerCmd(depFiles)
  431. if r.sbox {
  432. // Check for Rel() errors, as all depfiles should be in the output dir. Errors
  433. // will be reported to the ctx.
  434. for _, path := range depFiles[1:] {
  435. Rel(r.ctx, r.outDir.String(), path.String())
  436. }
  437. }
  438. }
  439. }
  440. tools := r.Tools()
  441. commands := r.Commands()
  442. outputs := r.Outputs()
  443. inputs := r.Inputs()
  444. rspFiles := r.rspFiles()
  445. if len(commands) == 0 {
  446. return
  447. }
  448. if len(outputs) == 0 {
  449. panic("No outputs specified from any Commands")
  450. }
  451. commandString := strings.Join(commands, " && ")
  452. if r.sbox {
  453. // If running the command inside sbox, write the rule data out to an sbox
  454. // manifest.textproto.
  455. manifest := sbox_proto.Manifest{}
  456. command := sbox_proto.Command{}
  457. manifest.Commands = append(manifest.Commands, &command)
  458. command.Command = proto.String(commandString)
  459. if depFile != nil {
  460. manifest.OutputDepfile = proto.String(depFile.String())
  461. }
  462. // If sandboxing tools is enabled, add copy rules to the manifest to copy each tool
  463. // into the sbox directory.
  464. if r.sboxTools {
  465. for _, tool := range tools {
  466. command.CopyBefore = append(command.CopyBefore, &sbox_proto.Copy{
  467. From: proto.String(tool.String()),
  468. To: proto.String(sboxPathForToolRel(r.ctx, tool)),
  469. })
  470. }
  471. for _, c := range r.commands {
  472. for _, tool := range c.packagedTools {
  473. command.CopyBefore = append(command.CopyBefore, &sbox_proto.Copy{
  474. From: proto.String(tool.srcPath.String()),
  475. To: proto.String(sboxPathForPackagedToolRel(tool)),
  476. Executable: proto.Bool(tool.executable),
  477. })
  478. tools = append(tools, tool.srcPath)
  479. }
  480. }
  481. }
  482. // If sandboxing inputs is enabled, add copy rules to the manifest to copy each input
  483. // into the sbox directory.
  484. if r.sboxInputs {
  485. for _, input := range inputs {
  486. command.CopyBefore = append(command.CopyBefore, &sbox_proto.Copy{
  487. From: proto.String(input.String()),
  488. To: proto.String(r.sboxPathForInputRel(input)),
  489. })
  490. }
  491. // If using rsp files copy them and their contents into the sbox directory with
  492. // the appropriate path mappings.
  493. for _, rspFile := range rspFiles {
  494. command.RspFiles = append(command.RspFiles, &sbox_proto.RspFile{
  495. File: proto.String(rspFile.file.String()),
  496. // These have to match the logic in sboxPathForInputRel
  497. PathMappings: []*sbox_proto.PathMapping{
  498. {
  499. From: proto.String(r.outDir.String()),
  500. To: proto.String(sboxOutSubDir),
  501. },
  502. {
  503. From: proto.String(PathForOutput(r.ctx).String()),
  504. To: proto.String(sboxOutSubDir),
  505. },
  506. },
  507. })
  508. }
  509. command.Chdir = proto.Bool(true)
  510. }
  511. // Add copy rules to the manifest to copy each output file from the sbox directory.
  512. // to the output directory after running the commands.
  513. sboxOutputs := make([]string, len(outputs))
  514. for i, output := range outputs {
  515. rel := Rel(r.ctx, r.outDir.String(), output.String())
  516. sboxOutputs[i] = filepath.Join(sboxOutDir, rel)
  517. command.CopyAfter = append(command.CopyAfter, &sbox_proto.Copy{
  518. From: proto.String(filepath.Join(sboxOutSubDir, rel)),
  519. To: proto.String(output.String()),
  520. })
  521. }
  522. // Outputs that were marked Temporary will not be checked that they are in the output
  523. // directory by the loop above, check them here.
  524. for path := range r.temporariesSet {
  525. Rel(r.ctx, r.outDir.String(), path.String())
  526. }
  527. // Add a hash of the list of input files to the manifest so that the textproto file
  528. // changes when the list of input files changes and causes the sbox rule that
  529. // depends on it to rerun.
  530. command.InputHash = proto.String(hashSrcFiles(inputs))
  531. // Verify that the manifest textproto is not inside the sbox output directory, otherwise
  532. // it will get deleted when the sbox rule clears its output directory.
  533. _, manifestInOutDir := MaybeRel(r.ctx, r.outDir.String(), r.sboxManifestPath.String())
  534. if manifestInOutDir {
  535. ReportPathErrorf(r.ctx, "sbox rule %q manifestPath %q must not be in outputDir %q",
  536. name, r.sboxManifestPath.String(), r.outDir.String())
  537. }
  538. // Create a rule to write the manifest as a the textproto.
  539. pbText, err := prototext.Marshal(&manifest)
  540. if err != nil {
  541. ReportPathErrorf(r.ctx, "sbox manifest failed to marshal: %q", err)
  542. }
  543. WriteFileRule(r.ctx, r.sboxManifestPath, string(pbText))
  544. // Generate a new string to use as the command line of the sbox rule. This uses
  545. // a RuleBuilderCommand as a convenience method of building the command line, then
  546. // converts it to a string to replace commandString.
  547. sboxCmd := &RuleBuilderCommand{
  548. rule: &RuleBuilder{
  549. ctx: r.ctx,
  550. },
  551. }
  552. sboxCmd.Text("rm -rf").Output(r.outDir)
  553. sboxCmd.Text("&&")
  554. sboxCmd.BuiltTool("sbox").
  555. Flag("--sandbox-path").Text(shared.TempDirForOutDir(PathForOutput(r.ctx).String())).
  556. Flag("--manifest").Input(r.sboxManifestPath)
  557. // Replace the command string, and add the sbox tool and manifest textproto to the
  558. // dependencies of the final sbox rule.
  559. commandString = sboxCmd.buf.String()
  560. tools = append(tools, sboxCmd.tools...)
  561. inputs = append(inputs, sboxCmd.inputs...)
  562. if r.rbeParams != nil {
  563. // RBE needs a list of input files to copy to the remote builder. For inputs already
  564. // listed in an rsp file, pass the rsp file directly to rewrapper. For the rest,
  565. // create a new rsp file to pass to rewrapper.
  566. var remoteRspFiles Paths
  567. var remoteInputs Paths
  568. remoteInputs = append(remoteInputs, inputs...)
  569. remoteInputs = append(remoteInputs, tools...)
  570. for _, rspFile := range rspFiles {
  571. remoteInputs = append(remoteInputs, rspFile.file)
  572. remoteRspFiles = append(remoteRspFiles, rspFile.file)
  573. }
  574. if len(remoteInputs) > 0 {
  575. inputsListFile := r.sboxManifestPath.ReplaceExtension(r.ctx, "rbe_inputs.list")
  576. writeRspFileRule(r.ctx, inputsListFile, remoteInputs)
  577. remoteRspFiles = append(remoteRspFiles, inputsListFile)
  578. // Add the new rsp file as an extra input to the rule.
  579. inputs = append(inputs, inputsListFile)
  580. }
  581. r.rbeParams.OutputFiles = outputs.Strings()
  582. r.rbeParams.RSPFiles = remoteRspFiles.Strings()
  583. rewrapperCommand := r.rbeParams.NoVarTemplate(r.ctx.Config().RBEWrapper())
  584. commandString = rewrapperCommand + " bash -c '" + strings.ReplaceAll(commandString, `'`, `'\''`) + "'"
  585. }
  586. } else {
  587. // If not using sbox the rule will run the command directly, put the hash of the
  588. // list of input files in a comment at the end of the command line to ensure ninja
  589. // reruns the rule when the list of input files changes.
  590. commandString += " # hash of input list: " + hashSrcFiles(inputs)
  591. }
  592. // Ninja doesn't like multiple outputs when depfiles are enabled, move all but the first output to
  593. // ImplicitOutputs. RuleBuilder doesn't use "$out", so the distinction between Outputs and
  594. // ImplicitOutputs doesn't matter.
  595. output := outputs[0]
  596. implicitOutputs := outputs[1:]
  597. var rspFile, rspFileContent string
  598. var rspFileInputs Paths
  599. if len(rspFiles) > 0 {
  600. // The first rsp files uses Ninja's rsp file support for the rule
  601. rspFile = rspFiles[0].file.String()
  602. // Use "$in" for rspFileContent to avoid duplicating the list of files in the dependency
  603. // list and in the contents of the rsp file. Inputs to the rule that are not in the
  604. // rsp file will be listed in Implicits instead of Inputs so they don't show up in "$in".
  605. rspFileContent = "$in"
  606. rspFileInputs = append(rspFileInputs, rspFiles[0].paths...)
  607. for _, rspFile := range rspFiles[1:] {
  608. // Any additional rsp files need an extra rule to write the file.
  609. writeRspFileRule(r.ctx, rspFile.file, rspFile.paths)
  610. // The main rule needs to depend on the inputs listed in the extra rsp file.
  611. inputs = append(inputs, rspFile.paths...)
  612. // The main rule needs to depend on the extra rsp file.
  613. inputs = append(inputs, rspFile.file)
  614. }
  615. }
  616. var pool blueprint.Pool
  617. if r.ctx.Config().UseGoma() && r.remoteable.Goma {
  618. // When USE_GOMA=true is set and the rule is supported by goma, allow jobs to run outside the local pool.
  619. } else if r.ctx.Config().UseRBE() && r.remoteable.RBE {
  620. // When USE_RBE=true is set and the rule is supported by RBE, use the remotePool.
  621. pool = remotePool
  622. } else if r.highmem {
  623. pool = highmemPool
  624. } else if r.ctx.Config().UseRemoteBuild() {
  625. pool = localPool
  626. }
  627. r.ctx.Build(r.pctx, BuildParams{
  628. Rule: r.ctx.Rule(pctx, name, blueprint.RuleParams{
  629. Command: proptools.NinjaEscape(commandString),
  630. CommandDeps: proptools.NinjaEscapeList(tools.Strings()),
  631. Restat: r.restat,
  632. Rspfile: proptools.NinjaEscape(rspFile),
  633. RspfileContent: rspFileContent,
  634. Pool: pool,
  635. }),
  636. Inputs: rspFileInputs,
  637. Implicits: inputs,
  638. OrderOnly: r.OrderOnlys(),
  639. Validations: r.Validations(),
  640. Output: output,
  641. ImplicitOutputs: implicitOutputs,
  642. SymlinkOutputs: r.SymlinkOutputs(),
  643. Depfile: depFile,
  644. Deps: depFormat,
  645. Description: desc,
  646. })
  647. }
  648. // RuleBuilderCommand is a builder for a command in a command line. It can be mutated by its methods to add to the
  649. // command and track dependencies. The methods mutate the RuleBuilderCommand in place, as well as return the
  650. // RuleBuilderCommand, so they can be used chained or unchained. All methods that add text implicitly add a single
  651. // space as a separator from the previous method.
  652. type RuleBuilderCommand struct {
  653. rule *RuleBuilder
  654. buf strings.Builder
  655. inputs Paths
  656. implicits Paths
  657. orderOnlys Paths
  658. validations Paths
  659. outputs WritablePaths
  660. symlinkOutputs WritablePaths
  661. depFiles WritablePaths
  662. tools Paths
  663. packagedTools []PackagingSpec
  664. rspFiles []rspFileAndPaths
  665. }
  666. type rspFileAndPaths struct {
  667. file WritablePath
  668. paths Paths
  669. }
  670. func checkPathNotNil(path Path) {
  671. if path == nil {
  672. panic("rule_builder paths cannot be nil")
  673. }
  674. }
  675. func (c *RuleBuilderCommand) addInput(path Path) string {
  676. checkPathNotNil(path)
  677. c.inputs = append(c.inputs, path)
  678. return c.PathForInput(path)
  679. }
  680. func (c *RuleBuilderCommand) addImplicit(path Path) {
  681. checkPathNotNil(path)
  682. c.implicits = append(c.implicits, path)
  683. }
  684. func (c *RuleBuilderCommand) addOrderOnly(path Path) {
  685. checkPathNotNil(path)
  686. c.orderOnlys = append(c.orderOnlys, path)
  687. }
  688. // PathForInput takes an input path and returns the appropriate path to use on the command line. If
  689. // sbox was enabled via a call to RuleBuilder.Sbox() and the path was an output path it returns a
  690. // path with the placeholder prefix used for outputs in sbox. If sbox is not enabled it returns the
  691. // original path.
  692. func (c *RuleBuilderCommand) PathForInput(path Path) string {
  693. if c.rule.sbox {
  694. rel, inSandbox := c.rule._sboxPathForInputRel(path)
  695. if inSandbox {
  696. rel = filepath.Join(sboxSandboxBaseDir, rel)
  697. }
  698. return rel
  699. }
  700. return path.String()
  701. }
  702. // PathsForInputs takes a list of input paths and returns the appropriate paths to use on the
  703. // command line. If sbox was enabled via a call to RuleBuilder.Sbox() a path was an output path, it
  704. // returns the path with the placeholder prefix used for outputs in sbox. If sbox is not enabled it
  705. // returns the original paths.
  706. func (c *RuleBuilderCommand) PathsForInputs(paths Paths) []string {
  707. ret := make([]string, len(paths))
  708. for i, path := range paths {
  709. ret[i] = c.PathForInput(path)
  710. }
  711. return ret
  712. }
  713. // PathForOutput takes an output path and returns the appropriate path to use on the command
  714. // line. If sbox was enabled via a call to RuleBuilder.Sbox(), it returns a path with the
  715. // placeholder prefix used for outputs in sbox. If sbox is not enabled it returns the
  716. // original path.
  717. func (c *RuleBuilderCommand) PathForOutput(path WritablePath) string {
  718. if c.rule.sbox {
  719. // Errors will be handled in RuleBuilder.Build where we have a context to report them
  720. rel, _, _ := maybeRelErr(c.rule.outDir.String(), path.String())
  721. return filepath.Join(sboxOutDir, rel)
  722. }
  723. return path.String()
  724. }
  725. func sboxPathForToolRel(ctx BuilderContext, path Path) string {
  726. // Errors will be handled in RuleBuilder.Build where we have a context to report them
  727. toolDir := pathForInstall(ctx, ctx.Config().BuildOS, ctx.Config().BuildArch, "", false)
  728. relOutSoong, isRelOutSoong, _ := maybeRelErr(toolDir.String(), path.String())
  729. if isRelOutSoong {
  730. // The tool is in the Soong output directory, it will be copied to __SBOX_OUT_DIR__/tools/out
  731. return filepath.Join(sboxToolsSubDir, "out", relOutSoong)
  732. }
  733. // The tool is in the source directory, it will be copied to __SBOX_OUT_DIR__/tools/src
  734. return filepath.Join(sboxToolsSubDir, "src", path.String())
  735. }
  736. func (r *RuleBuilder) _sboxPathForInputRel(path Path) (rel string, inSandbox bool) {
  737. // Errors will be handled in RuleBuilder.Build where we have a context to report them
  738. rel, isRelSboxOut, _ := maybeRelErr(r.outDir.String(), path.String())
  739. if isRelSboxOut {
  740. return filepath.Join(sboxOutSubDir, rel), true
  741. }
  742. if r.sboxInputs {
  743. // When sandboxing inputs all inputs have to be copied into the sandbox. Input files that
  744. // are outputs of other rules could be an arbitrary absolute path if OUT_DIR is set, so they
  745. // will be copied to relative paths under __SBOX_OUT_DIR__/out.
  746. rel, isRelOut, _ := maybeRelErr(PathForOutput(r.ctx).String(), path.String())
  747. if isRelOut {
  748. return filepath.Join(sboxOutSubDir, rel), true
  749. }
  750. }
  751. return path.String(), false
  752. }
  753. func (r *RuleBuilder) sboxPathForInputRel(path Path) string {
  754. rel, _ := r._sboxPathForInputRel(path)
  755. return rel
  756. }
  757. func (r *RuleBuilder) sboxPathsForInputsRel(paths Paths) []string {
  758. ret := make([]string, len(paths))
  759. for i, path := range paths {
  760. ret[i] = r.sboxPathForInputRel(path)
  761. }
  762. return ret
  763. }
  764. func sboxPathForPackagedToolRel(spec PackagingSpec) string {
  765. return filepath.Join(sboxToolsSubDir, "out", spec.relPathInPackage)
  766. }
  767. // PathForPackagedTool takes a PackageSpec for a tool and returns the corresponding path for the
  768. // tool after copying it into the sandbox. This can be used on the RuleBuilder command line to
  769. // reference the tool.
  770. func (c *RuleBuilderCommand) PathForPackagedTool(spec PackagingSpec) string {
  771. if !c.rule.sboxTools {
  772. panic("PathForPackagedTool() requires SandboxTools()")
  773. }
  774. return filepath.Join(sboxSandboxBaseDir, sboxPathForPackagedToolRel(spec))
  775. }
  776. // PathForTool takes a path to a tool, which may be an output file or a source file, and returns
  777. // the corresponding path for the tool in the sbox sandbox if sbox is enabled, or the original path
  778. // if it is not. This can be used on the RuleBuilder command line to reference the tool.
  779. func (c *RuleBuilderCommand) PathForTool(path Path) string {
  780. if c.rule.sbox && c.rule.sboxTools {
  781. return filepath.Join(sboxSandboxBaseDir, sboxPathForToolRel(c.rule.ctx, path))
  782. }
  783. return path.String()
  784. }
  785. // PathsForTools takes a list of paths to tools, which may be output files or source files, and
  786. // returns the corresponding paths for the tools in the sbox sandbox if sbox is enabled, or the
  787. // original paths if it is not. This can be used on the RuleBuilder command line to reference the tool.
  788. func (c *RuleBuilderCommand) PathsForTools(paths Paths) []string {
  789. if c.rule.sbox && c.rule.sboxTools {
  790. var ret []string
  791. for _, path := range paths {
  792. ret = append(ret, filepath.Join(sboxSandboxBaseDir, sboxPathForToolRel(c.rule.ctx, path)))
  793. }
  794. return ret
  795. }
  796. return paths.Strings()
  797. }
  798. // PackagedTool adds the specified tool path to the command line. It can only be used with tool
  799. // sandboxing enabled by SandboxTools(), and will copy the tool into the sandbox.
  800. func (c *RuleBuilderCommand) PackagedTool(spec PackagingSpec) *RuleBuilderCommand {
  801. if !c.rule.sboxTools {
  802. panic("PackagedTool() requires SandboxTools()")
  803. }
  804. c.packagedTools = append(c.packagedTools, spec)
  805. c.Text(sboxPathForPackagedToolRel(spec))
  806. return c
  807. }
  808. // ImplicitPackagedTool copies the specified tool into the sandbox without modifying the command
  809. // line. It can only be used with tool sandboxing enabled by SandboxTools().
  810. func (c *RuleBuilderCommand) ImplicitPackagedTool(spec PackagingSpec) *RuleBuilderCommand {
  811. if !c.rule.sboxTools {
  812. panic("ImplicitPackagedTool() requires SandboxTools()")
  813. }
  814. c.packagedTools = append(c.packagedTools, spec)
  815. return c
  816. }
  817. // ImplicitPackagedTools copies the specified tools into the sandbox without modifying the command
  818. // line. It can only be used with tool sandboxing enabled by SandboxTools().
  819. func (c *RuleBuilderCommand) ImplicitPackagedTools(specs []PackagingSpec) *RuleBuilderCommand {
  820. if !c.rule.sboxTools {
  821. panic("ImplicitPackagedTools() requires SandboxTools()")
  822. }
  823. c.packagedTools = append(c.packagedTools, specs...)
  824. return c
  825. }
  826. // Text adds the specified raw text to the command line. The text should not contain input or output paths or the
  827. // rule will not have them listed in its dependencies or outputs.
  828. func (c *RuleBuilderCommand) Text(text string) *RuleBuilderCommand {
  829. if c.buf.Len() > 0 {
  830. c.buf.WriteByte(' ')
  831. }
  832. c.buf.WriteString(text)
  833. return c
  834. }
  835. // Textf adds the specified formatted text to the command line. The text should not contain input or output paths or
  836. // the rule will not have them listed in its dependencies or outputs.
  837. func (c *RuleBuilderCommand) Textf(format string, a ...interface{}) *RuleBuilderCommand {
  838. return c.Text(fmt.Sprintf(format, a...))
  839. }
  840. // Flag adds the specified raw text to the command line. The text should not contain input or output paths or the
  841. // rule will not have them listed in its dependencies or outputs.
  842. func (c *RuleBuilderCommand) Flag(flag string) *RuleBuilderCommand {
  843. return c.Text(flag)
  844. }
  845. // OptionalFlag adds the specified raw text to the command line if it is not nil. The text should not contain input or
  846. // output paths or the rule will not have them listed in its dependencies or outputs.
  847. func (c *RuleBuilderCommand) OptionalFlag(flag *string) *RuleBuilderCommand {
  848. if flag != nil {
  849. c.Text(*flag)
  850. }
  851. return c
  852. }
  853. // Flags adds the specified raw text to the command line. The text should not contain input or output paths or the
  854. // rule will not have them listed in its dependencies or outputs.
  855. func (c *RuleBuilderCommand) Flags(flags []string) *RuleBuilderCommand {
  856. for _, flag := range flags {
  857. c.Text(flag)
  858. }
  859. return c
  860. }
  861. // FlagWithArg adds the specified flag and argument text to the command line, with no separator between them. The flag
  862. // and argument should not contain input or output paths or the rule will not have them listed in its dependencies or
  863. // outputs.
  864. func (c *RuleBuilderCommand) FlagWithArg(flag, arg string) *RuleBuilderCommand {
  865. return c.Text(flag + arg)
  866. }
  867. // FlagForEachArg adds the specified flag joined with each argument to the command line. The result is identical to
  868. // calling FlagWithArg for argument.
  869. func (c *RuleBuilderCommand) FlagForEachArg(flag string, args []string) *RuleBuilderCommand {
  870. for _, arg := range args {
  871. c.FlagWithArg(flag, arg)
  872. }
  873. return c
  874. }
  875. // FlagWithList adds the specified flag and list of arguments to the command line, with the arguments joined by sep
  876. // and no separator between the flag and arguments. The flag and arguments should not contain input or output paths or
  877. // the rule will not have them listed in its dependencies or outputs.
  878. func (c *RuleBuilderCommand) FlagWithList(flag string, list []string, sep string) *RuleBuilderCommand {
  879. return c.Text(flag + strings.Join(list, sep))
  880. }
  881. // Tool adds the specified tool path to the command line. The path will be also added to the dependencies returned by
  882. // RuleBuilder.Tools.
  883. func (c *RuleBuilderCommand) Tool(path Path) *RuleBuilderCommand {
  884. checkPathNotNil(path)
  885. c.tools = append(c.tools, path)
  886. return c.Text(c.PathForTool(path))
  887. }
  888. // Tool adds the specified tool path to the dependencies returned by RuleBuilder.Tools.
  889. func (c *RuleBuilderCommand) ImplicitTool(path Path) *RuleBuilderCommand {
  890. checkPathNotNil(path)
  891. c.tools = append(c.tools, path)
  892. return c
  893. }
  894. // Tool adds the specified tool path to the dependencies returned by RuleBuilder.Tools.
  895. func (c *RuleBuilderCommand) ImplicitTools(paths Paths) *RuleBuilderCommand {
  896. for _, path := range paths {
  897. c.ImplicitTool(path)
  898. }
  899. return c
  900. }
  901. // BuiltTool adds the specified tool path that was built using a host Soong module to the command line. The path will
  902. // be also added to the dependencies returned by RuleBuilder.Tools.
  903. //
  904. // It is equivalent to:
  905. // cmd.Tool(ctx.Config().HostToolPath(ctx, tool))
  906. func (c *RuleBuilderCommand) BuiltTool(tool string) *RuleBuilderCommand {
  907. return c.Tool(c.rule.ctx.Config().HostToolPath(c.rule.ctx, tool))
  908. }
  909. // PrebuiltBuildTool adds the specified tool path from prebuils/build-tools. The path will be also added to the
  910. // dependencies returned by RuleBuilder.Tools.
  911. //
  912. // It is equivalent to:
  913. // cmd.Tool(ctx.Config().PrebuiltBuildTool(ctx, tool))
  914. func (c *RuleBuilderCommand) PrebuiltBuildTool(ctx PathContext, tool string) *RuleBuilderCommand {
  915. return c.Tool(ctx.Config().PrebuiltBuildTool(ctx, tool))
  916. }
  917. // Input adds the specified input path to the command line. The path will also be added to the dependencies returned by
  918. // RuleBuilder.Inputs.
  919. func (c *RuleBuilderCommand) Input(path Path) *RuleBuilderCommand {
  920. return c.Text(c.addInput(path))
  921. }
  922. // Inputs adds the specified input paths to the command line, separated by spaces. The paths will also be added to the
  923. // dependencies returned by RuleBuilder.Inputs.
  924. func (c *RuleBuilderCommand) Inputs(paths Paths) *RuleBuilderCommand {
  925. for _, path := range paths {
  926. c.Input(path)
  927. }
  928. return c
  929. }
  930. // Implicit adds the specified input path to the dependencies returned by RuleBuilder.Inputs without modifying the
  931. // command line.
  932. func (c *RuleBuilderCommand) Implicit(path Path) *RuleBuilderCommand {
  933. c.addImplicit(path)
  934. return c
  935. }
  936. // Implicits adds the specified input paths to the dependencies returned by RuleBuilder.Inputs without modifying the
  937. // command line.
  938. func (c *RuleBuilderCommand) Implicits(paths Paths) *RuleBuilderCommand {
  939. for _, path := range paths {
  940. c.addImplicit(path)
  941. }
  942. return c
  943. }
  944. // GetImplicits returns the command's implicit inputs.
  945. func (c *RuleBuilderCommand) GetImplicits() Paths {
  946. return c.implicits
  947. }
  948. // OrderOnly adds the specified input path to the dependencies returned by RuleBuilder.OrderOnlys
  949. // without modifying the command line.
  950. func (c *RuleBuilderCommand) OrderOnly(path Path) *RuleBuilderCommand {
  951. c.addOrderOnly(path)
  952. return c
  953. }
  954. // OrderOnlys adds the specified input paths to the dependencies returned by RuleBuilder.OrderOnlys
  955. // without modifying the command line.
  956. func (c *RuleBuilderCommand) OrderOnlys(paths Paths) *RuleBuilderCommand {
  957. for _, path := range paths {
  958. c.addOrderOnly(path)
  959. }
  960. return c
  961. }
  962. // Validation adds the specified input path to the validation dependencies by
  963. // RuleBuilder.Validations without modifying the command line.
  964. func (c *RuleBuilderCommand) Validation(path Path) *RuleBuilderCommand {
  965. checkPathNotNil(path)
  966. c.validations = append(c.validations, path)
  967. return c
  968. }
  969. // Validations adds the specified input paths to the validation dependencies by
  970. // RuleBuilder.Validations without modifying the command line.
  971. func (c *RuleBuilderCommand) Validations(paths Paths) *RuleBuilderCommand {
  972. for _, path := range paths {
  973. c.Validation(path)
  974. }
  975. return c
  976. }
  977. // Output adds the specified output path to the command line. The path will also be added to the outputs returned by
  978. // RuleBuilder.Outputs.
  979. func (c *RuleBuilderCommand) Output(path WritablePath) *RuleBuilderCommand {
  980. checkPathNotNil(path)
  981. c.outputs = append(c.outputs, path)
  982. return c.Text(c.PathForOutput(path))
  983. }
  984. // Outputs adds the specified output paths to the command line, separated by spaces. The paths will also be added to
  985. // the outputs returned by RuleBuilder.Outputs.
  986. func (c *RuleBuilderCommand) Outputs(paths WritablePaths) *RuleBuilderCommand {
  987. for _, path := range paths {
  988. c.Output(path)
  989. }
  990. return c
  991. }
  992. // OutputDir adds the output directory to the command line. This is only available when used with RuleBuilder.Sbox,
  993. // and will be the temporary output directory managed by sbox, not the final one.
  994. func (c *RuleBuilderCommand) OutputDir() *RuleBuilderCommand {
  995. if !c.rule.sbox {
  996. panic("OutputDir only valid with Sbox")
  997. }
  998. return c.Text(sboxOutDir)
  999. }
  1000. // DepFile adds the specified depfile path to the paths returned by RuleBuilder.DepFiles and adds it to the command
  1001. // line, and causes RuleBuilder.Build file to set the depfile flag for ninja. If multiple depfiles are added to
  1002. // commands in a single RuleBuilder then RuleBuilder.Build will add an extra command to merge the depfiles together.
  1003. func (c *RuleBuilderCommand) DepFile(path WritablePath) *RuleBuilderCommand {
  1004. checkPathNotNil(path)
  1005. c.depFiles = append(c.depFiles, path)
  1006. return c.Text(c.PathForOutput(path))
  1007. }
  1008. // ImplicitOutput adds the specified output path to the dependencies returned by RuleBuilder.Outputs without modifying
  1009. // the command line.
  1010. func (c *RuleBuilderCommand) ImplicitOutput(path WritablePath) *RuleBuilderCommand {
  1011. c.outputs = append(c.outputs, path)
  1012. return c
  1013. }
  1014. // ImplicitOutputs adds the specified output paths to the dependencies returned by RuleBuilder.Outputs without modifying
  1015. // the command line.
  1016. func (c *RuleBuilderCommand) ImplicitOutputs(paths WritablePaths) *RuleBuilderCommand {
  1017. c.outputs = append(c.outputs, paths...)
  1018. return c
  1019. }
  1020. // ImplicitSymlinkOutput declares the specified path as an implicit output that
  1021. // will be a symlink instead of a regular file. Does not modify the command
  1022. // line.
  1023. func (c *RuleBuilderCommand) ImplicitSymlinkOutput(path WritablePath) *RuleBuilderCommand {
  1024. checkPathNotNil(path)
  1025. c.symlinkOutputs = append(c.symlinkOutputs, path)
  1026. return c.ImplicitOutput(path)
  1027. }
  1028. // ImplicitSymlinkOutputs declares the specified paths as implicit outputs that
  1029. // will be a symlinks instead of regular files. Does not modify the command
  1030. // line.
  1031. func (c *RuleBuilderCommand) ImplicitSymlinkOutputs(paths WritablePaths) *RuleBuilderCommand {
  1032. for _, path := range paths {
  1033. c.ImplicitSymlinkOutput(path)
  1034. }
  1035. return c
  1036. }
  1037. // SymlinkOutput declares the specified path as an output that will be a symlink
  1038. // instead of a regular file. Modifies the command line.
  1039. func (c *RuleBuilderCommand) SymlinkOutput(path WritablePath) *RuleBuilderCommand {
  1040. checkPathNotNil(path)
  1041. c.symlinkOutputs = append(c.symlinkOutputs, path)
  1042. return c.Output(path)
  1043. }
  1044. // SymlinkOutputsl declares the specified paths as outputs that will be symlinks
  1045. // instead of regular files. Modifies the command line.
  1046. func (c *RuleBuilderCommand) SymlinkOutputs(paths WritablePaths) *RuleBuilderCommand {
  1047. for _, path := range paths {
  1048. c.SymlinkOutput(path)
  1049. }
  1050. return c
  1051. }
  1052. // ImplicitDepFile adds the specified depfile path to the paths returned by RuleBuilder.DepFiles without modifying
  1053. // the command line, and causes RuleBuilder.Build file to set the depfile flag for ninja. If multiple depfiles
  1054. // are added to commands in a single RuleBuilder then RuleBuilder.Build will add an extra command to merge the
  1055. // depfiles together.
  1056. func (c *RuleBuilderCommand) ImplicitDepFile(path WritablePath) *RuleBuilderCommand {
  1057. c.depFiles = append(c.depFiles, path)
  1058. return c
  1059. }
  1060. // FlagWithInput adds the specified flag and input path to the command line, with no separator between them. The path
  1061. // will also be added to the dependencies returned by RuleBuilder.Inputs.
  1062. func (c *RuleBuilderCommand) FlagWithInput(flag string, path Path) *RuleBuilderCommand {
  1063. return c.Text(flag + c.addInput(path))
  1064. }
  1065. // FlagWithInputList adds the specified flag and input paths to the command line, with the inputs joined by sep
  1066. // and no separator between the flag and inputs. The input paths will also be added to the dependencies returned by
  1067. // RuleBuilder.Inputs.
  1068. func (c *RuleBuilderCommand) FlagWithInputList(flag string, paths Paths, sep string) *RuleBuilderCommand {
  1069. strs := make([]string, len(paths))
  1070. for i, path := range paths {
  1071. strs[i] = c.addInput(path)
  1072. }
  1073. return c.FlagWithList(flag, strs, sep)
  1074. }
  1075. // FlagForEachInput adds the specified flag joined with each input path to the command line. The input paths will also
  1076. // be added to the dependencies returned by RuleBuilder.Inputs. The result is identical to calling FlagWithInput for
  1077. // each input path.
  1078. func (c *RuleBuilderCommand) FlagForEachInput(flag string, paths Paths) *RuleBuilderCommand {
  1079. for _, path := range paths {
  1080. c.FlagWithInput(flag, path)
  1081. }
  1082. return c
  1083. }
  1084. // FlagWithOutput adds the specified flag and output path to the command line, with no separator between them. The path
  1085. // will also be added to the outputs returned by RuleBuilder.Outputs.
  1086. func (c *RuleBuilderCommand) FlagWithOutput(flag string, path WritablePath) *RuleBuilderCommand {
  1087. c.outputs = append(c.outputs, path)
  1088. return c.Text(flag + c.PathForOutput(path))
  1089. }
  1090. // FlagWithDepFile adds the specified flag and depfile path to the command line, with no separator between them. The path
  1091. // will also be added to the outputs returned by RuleBuilder.Outputs.
  1092. func (c *RuleBuilderCommand) FlagWithDepFile(flag string, path WritablePath) *RuleBuilderCommand {
  1093. c.depFiles = append(c.depFiles, path)
  1094. return c.Text(flag + c.PathForOutput(path))
  1095. }
  1096. // FlagWithRspFileInputList adds the specified flag and path to an rspfile to the command line, with
  1097. // no separator between them. The paths will be written to the rspfile. If sbox is enabled, the
  1098. // rspfile must be outside the sbox directory. The first use of FlagWithRspFileInputList in any
  1099. // RuleBuilderCommand of a RuleBuilder will use Ninja's rsp file support for the rule, additional
  1100. // uses will result in an auxiliary rules to write the rspFile contents.
  1101. func (c *RuleBuilderCommand) FlagWithRspFileInputList(flag string, rspFile WritablePath, paths Paths) *RuleBuilderCommand {
  1102. // Use an empty slice if paths is nil, the non-nil slice is used as an indicator that the rsp file must be
  1103. // generated.
  1104. if paths == nil {
  1105. paths = Paths{}
  1106. }
  1107. c.rspFiles = append(c.rspFiles, rspFileAndPaths{rspFile, paths})
  1108. if c.rule.sbox {
  1109. if _, isRel, _ := maybeRelErr(c.rule.outDir.String(), rspFile.String()); isRel {
  1110. panic(fmt.Errorf("FlagWithRspFileInputList rspfile %q must not be inside out dir %q",
  1111. rspFile.String(), c.rule.outDir.String()))
  1112. }
  1113. }
  1114. c.FlagWithArg(flag, c.PathForInput(rspFile))
  1115. return c
  1116. }
  1117. // String returns the command line.
  1118. func (c *RuleBuilderCommand) String() string {
  1119. return c.buf.String()
  1120. }
  1121. // RuleBuilderSboxProtoForTests takes the BuildParams for the manifest passed to RuleBuilder.Sbox()
  1122. // and returns sbox testproto generated by the RuleBuilder.
  1123. func RuleBuilderSboxProtoForTests(t *testing.T, params TestingBuildParams) *sbox_proto.Manifest {
  1124. t.Helper()
  1125. content := ContentFromFileRuleForTests(t, params)
  1126. manifest := sbox_proto.Manifest{}
  1127. err := prototext.Unmarshal([]byte(content), &manifest)
  1128. if err != nil {
  1129. t.Fatalf("failed to unmarshal manifest: %s", err.Error())
  1130. }
  1131. return &manifest
  1132. }
  1133. func ninjaNameEscape(s string) string {
  1134. b := []byte(s)
  1135. escaped := false
  1136. for i, c := range b {
  1137. valid := (c >= 'a' && c <= 'z') ||
  1138. (c >= 'A' && c <= 'Z') ||
  1139. (c >= '0' && c <= '9') ||
  1140. (c == '_') ||
  1141. (c == '-') ||
  1142. (c == '.')
  1143. if !valid {
  1144. b[i] = '_'
  1145. escaped = true
  1146. }
  1147. }
  1148. if escaped {
  1149. s = string(b)
  1150. }
  1151. return s
  1152. }
  1153. // hashSrcFiles returns a hash of the list of source files. It is used to ensure the command line
  1154. // or the sbox textproto manifest change even if the input files are not listed on the command line.
  1155. func hashSrcFiles(srcFiles Paths) string {
  1156. h := sha256.New()
  1157. srcFileList := strings.Join(srcFiles.Strings(), "\n")
  1158. h.Write([]byte(srcFileList))
  1159. return fmt.Sprintf("%x", h.Sum(nil))
  1160. }
  1161. // BuilderContextForTesting returns a BuilderContext for the given config that can be used for tests
  1162. // that need to call methods that take a BuilderContext.
  1163. func BuilderContextForTesting(config Config) BuilderContext {
  1164. pathCtx := PathContextForTesting(config)
  1165. return builderContextForTests{
  1166. PathContext: pathCtx,
  1167. }
  1168. }
  1169. type builderContextForTests struct {
  1170. PathContext
  1171. }
  1172. func (builderContextForTests) Rule(PackageContext, string, blueprint.RuleParams, ...string) blueprint.Rule {
  1173. return nil
  1174. }
  1175. func (builderContextForTests) Build(PackageContext, BuildParams) {}
  1176. func writeRspFileRule(ctx BuilderContext, rspFile WritablePath, paths Paths) {
  1177. buf := &strings.Builder{}
  1178. err := response.WriteRspFile(buf, paths.Strings())
  1179. if err != nil {
  1180. // There should never be I/O errors writing to a bytes.Buffer.
  1181. panic(err)
  1182. }
  1183. WriteFileRule(ctx, rspFile, buf.String())
  1184. }