rule_builder.go 48 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364
  1. // Copyright 2018 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package android
  15. import (
  16. "crypto/sha256"
  17. "fmt"
  18. "path/filepath"
  19. "sort"
  20. "strings"
  21. "testing"
  22. "github.com/google/blueprint"
  23. "github.com/google/blueprint/proptools"
  24. "google.golang.org/protobuf/encoding/prototext"
  25. "google.golang.org/protobuf/proto"
  26. "android/soong/cmd/sbox/sbox_proto"
  27. "android/soong/remoteexec"
  28. "android/soong/response"
  29. "android/soong/shared"
  30. )
  31. const sboxSandboxBaseDir = "__SBOX_SANDBOX_DIR__"
  32. const sboxOutSubDir = "out"
  33. const sboxToolsSubDir = "tools"
  34. const sboxOutDir = sboxSandboxBaseDir + "/" + sboxOutSubDir
  35. // RuleBuilder provides an alternative to ModuleContext.Rule and ModuleContext.Build to add a command line to the build
  36. // graph.
  37. type RuleBuilder struct {
  38. pctx PackageContext
  39. ctx BuilderContext
  40. commands []*RuleBuilderCommand
  41. installs RuleBuilderInstalls
  42. temporariesSet map[WritablePath]bool
  43. restat bool
  44. sbox bool
  45. highmem bool
  46. remoteable RemoteRuleSupports
  47. rbeParams *remoteexec.REParams
  48. outDir WritablePath
  49. sboxTools bool
  50. sboxInputs bool
  51. sboxManifestPath WritablePath
  52. missingDeps []string
  53. }
  54. // NewRuleBuilder returns a newly created RuleBuilder.
  55. func NewRuleBuilder(pctx PackageContext, ctx BuilderContext) *RuleBuilder {
  56. return &RuleBuilder{
  57. pctx: pctx,
  58. ctx: ctx,
  59. temporariesSet: make(map[WritablePath]bool),
  60. }
  61. }
  62. // RuleBuilderInstall is a tuple of install from and to locations.
  63. type RuleBuilderInstall struct {
  64. From Path
  65. To string
  66. }
  67. type RuleBuilderInstalls []RuleBuilderInstall
  68. // String returns the RuleBuilderInstalls in the form used by $(call copy-many-files) in Make, a space separated
  69. // list of from:to tuples.
  70. func (installs RuleBuilderInstalls) String() string {
  71. sb := strings.Builder{}
  72. for i, install := range installs {
  73. if i != 0 {
  74. sb.WriteRune(' ')
  75. }
  76. sb.WriteString(install.From.String())
  77. sb.WriteRune(':')
  78. sb.WriteString(install.To)
  79. }
  80. return sb.String()
  81. }
  82. // MissingDeps adds modules to the list of missing dependencies. If MissingDeps
  83. // is called with a non-empty input, any call to Build will result in a rule
  84. // that will print an error listing the missing dependencies and fail.
  85. // MissingDeps should only be called if Config.AllowMissingDependencies() is
  86. // true.
  87. func (r *RuleBuilder) MissingDeps(missingDeps []string) {
  88. r.missingDeps = append(r.missingDeps, missingDeps...)
  89. }
  90. // Restat marks the rule as a restat rule, which will be passed to ModuleContext.Rule in BuildParams.Restat.
  91. func (r *RuleBuilder) Restat() *RuleBuilder {
  92. r.restat = true
  93. return r
  94. }
  95. // HighMem marks the rule as a high memory rule, which will limit how many run in parallel with other high memory
  96. // rules.
  97. func (r *RuleBuilder) HighMem() *RuleBuilder {
  98. r.highmem = true
  99. return r
  100. }
  101. // Remoteable marks the rule as supporting remote execution.
  102. func (r *RuleBuilder) Remoteable(supports RemoteRuleSupports) *RuleBuilder {
  103. r.remoteable = supports
  104. return r
  105. }
  106. // Rewrapper marks the rule as running inside rewrapper using the given params in order to support
  107. // running on RBE. During RuleBuilder.Build the params will be combined with the inputs, outputs
  108. // and tools known to RuleBuilder to prepend an appropriate rewrapper command line to the rule's
  109. // command line.
  110. func (r *RuleBuilder) Rewrapper(params *remoteexec.REParams) *RuleBuilder {
  111. if !r.sboxInputs {
  112. panic(fmt.Errorf("RuleBuilder.Rewrapper must be called after RuleBuilder.SandboxInputs"))
  113. }
  114. r.rbeParams = params
  115. return r
  116. }
  117. // Sbox marks the rule as needing to be wrapped by sbox. The outputDir should point to the output
  118. // directory that sbox will wipe. It should not be written to by any other rule. manifestPath should
  119. // point to a location where sbox's manifest will be written and must be outside outputDir. sbox
  120. // will ensure that all outputs have been written, and will discard any output files that were not
  121. // specified.
  122. func (r *RuleBuilder) Sbox(outputDir WritablePath, manifestPath WritablePath) *RuleBuilder {
  123. if r.sbox {
  124. panic("Sbox() may not be called more than once")
  125. }
  126. if len(r.commands) > 0 {
  127. panic("Sbox() may not be called after Command()")
  128. }
  129. r.sbox = true
  130. r.outDir = outputDir
  131. r.sboxManifestPath = manifestPath
  132. return r
  133. }
  134. // SandboxTools enables tool sandboxing for the rule by copying any referenced tools into the
  135. // sandbox.
  136. func (r *RuleBuilder) SandboxTools() *RuleBuilder {
  137. if !r.sbox {
  138. panic("SandboxTools() must be called after Sbox()")
  139. }
  140. if len(r.commands) > 0 {
  141. panic("SandboxTools() may not be called after Command()")
  142. }
  143. r.sboxTools = true
  144. return r
  145. }
  146. // SandboxInputs enables input sandboxing for the rule by copying any referenced inputs into the
  147. // sandbox. It also implies SandboxTools().
  148. //
  149. // Sandboxing inputs requires RuleBuilder to be aware of all references to input paths. Paths
  150. // that are passed to RuleBuilder outside of the methods that expect inputs, for example
  151. // FlagWithArg, must use RuleBuilderCommand.PathForInput to translate the path to one that matches
  152. // the sandbox layout.
  153. func (r *RuleBuilder) SandboxInputs() *RuleBuilder {
  154. if !r.sbox {
  155. panic("SandboxInputs() must be called after Sbox()")
  156. }
  157. if len(r.commands) > 0 {
  158. panic("SandboxInputs() may not be called after Command()")
  159. }
  160. r.sboxTools = true
  161. r.sboxInputs = true
  162. return r
  163. }
  164. // Install associates an output of the rule with an install location, which can be retrieved later using
  165. // RuleBuilder.Installs.
  166. func (r *RuleBuilder) Install(from Path, to string) {
  167. r.installs = append(r.installs, RuleBuilderInstall{from, to})
  168. }
  169. // Command returns a new RuleBuilderCommand for the rule. The commands will be ordered in the rule by when they were
  170. // created by this method. That can be mutated through their methods in any order, as long as the mutations do not
  171. // race with any call to Build.
  172. func (r *RuleBuilder) Command() *RuleBuilderCommand {
  173. command := &RuleBuilderCommand{
  174. rule: r,
  175. }
  176. r.commands = append(r.commands, command)
  177. return command
  178. }
  179. // Temporary marks an output of a command as an intermediate file that will be used as an input to another command
  180. // in the same rule, and should not be listed in Outputs.
  181. func (r *RuleBuilder) Temporary(path WritablePath) {
  182. r.temporariesSet[path] = true
  183. }
  184. // DeleteTemporaryFiles adds a command to the rule that deletes any outputs that have been marked using Temporary
  185. // when the rule runs. DeleteTemporaryFiles should be called after all calls to Temporary.
  186. func (r *RuleBuilder) DeleteTemporaryFiles() {
  187. var temporariesList WritablePaths
  188. for intermediate := range r.temporariesSet {
  189. temporariesList = append(temporariesList, intermediate)
  190. }
  191. sort.Slice(temporariesList, func(i, j int) bool {
  192. return temporariesList[i].String() < temporariesList[j].String()
  193. })
  194. r.Command().Text("rm").Flag("-f").Outputs(temporariesList)
  195. }
  196. // Inputs returns the list of paths that were passed to the RuleBuilderCommand methods that take
  197. // input paths, such as RuleBuilderCommand.Input, RuleBuilderCommand.Implicit, or
  198. // RuleBuilderCommand.FlagWithInput. Inputs to a command that are also outputs of another command
  199. // in the same RuleBuilder are filtered out. The list is sorted and duplicates removed.
  200. func (r *RuleBuilder) Inputs() Paths {
  201. outputs := r.outputSet()
  202. depFiles := r.depFileSet()
  203. inputs := make(map[string]Path)
  204. for _, c := range r.commands {
  205. for _, input := range append(c.inputs, c.implicits...) {
  206. inputStr := input.String()
  207. if _, isOutput := outputs[inputStr]; !isOutput {
  208. if _, isDepFile := depFiles[inputStr]; !isDepFile {
  209. inputs[input.String()] = input
  210. }
  211. }
  212. }
  213. }
  214. var inputList Paths
  215. for _, input := range inputs {
  216. inputList = append(inputList, input)
  217. }
  218. sort.Slice(inputList, func(i, j int) bool {
  219. return inputList[i].String() < inputList[j].String()
  220. })
  221. return inputList
  222. }
  223. // OrderOnlys returns the list of paths that were passed to the RuleBuilderCommand.OrderOnly or
  224. // RuleBuilderCommand.OrderOnlys. The list is sorted and duplicates removed.
  225. func (r *RuleBuilder) OrderOnlys() Paths {
  226. orderOnlys := make(map[string]Path)
  227. for _, c := range r.commands {
  228. for _, orderOnly := range c.orderOnlys {
  229. orderOnlys[orderOnly.String()] = orderOnly
  230. }
  231. }
  232. var orderOnlyList Paths
  233. for _, orderOnly := range orderOnlys {
  234. orderOnlyList = append(orderOnlyList, orderOnly)
  235. }
  236. sort.Slice(orderOnlyList, func(i, j int) bool {
  237. return orderOnlyList[i].String() < orderOnlyList[j].String()
  238. })
  239. return orderOnlyList
  240. }
  241. // Validations returns the list of paths that were passed to RuleBuilderCommand.Validation or
  242. // RuleBuilderCommand.Validations. The list is sorted and duplicates removed.
  243. func (r *RuleBuilder) Validations() Paths {
  244. validations := make(map[string]Path)
  245. for _, c := range r.commands {
  246. for _, validation := range c.validations {
  247. validations[validation.String()] = validation
  248. }
  249. }
  250. var validationList Paths
  251. for _, validation := range validations {
  252. validationList = append(validationList, validation)
  253. }
  254. sort.Slice(validationList, func(i, j int) bool {
  255. return validationList[i].String() < validationList[j].String()
  256. })
  257. return validationList
  258. }
  259. func (r *RuleBuilder) outputSet() map[string]WritablePath {
  260. outputs := make(map[string]WritablePath)
  261. for _, c := range r.commands {
  262. for _, output := range c.outputs {
  263. outputs[output.String()] = output
  264. }
  265. }
  266. return outputs
  267. }
  268. // Outputs returns the list of paths that were passed to the RuleBuilderCommand methods that take
  269. // output paths, such as RuleBuilderCommand.Output, RuleBuilderCommand.ImplicitOutput, or
  270. // RuleBuilderCommand.FlagWithInput. The list is sorted and duplicates removed.
  271. func (r *RuleBuilder) Outputs() WritablePaths {
  272. outputs := r.outputSet()
  273. var outputList WritablePaths
  274. for _, output := range outputs {
  275. if !r.temporariesSet[output] {
  276. outputList = append(outputList, output)
  277. }
  278. }
  279. sort.Slice(outputList, func(i, j int) bool {
  280. return outputList[i].String() < outputList[j].String()
  281. })
  282. return outputList
  283. }
  284. func (r *RuleBuilder) symlinkOutputSet() map[string]WritablePath {
  285. symlinkOutputs := make(map[string]WritablePath)
  286. for _, c := range r.commands {
  287. for _, symlinkOutput := range c.symlinkOutputs {
  288. symlinkOutputs[symlinkOutput.String()] = symlinkOutput
  289. }
  290. }
  291. return symlinkOutputs
  292. }
  293. // SymlinkOutputs returns the list of paths that the executor (Ninja) would
  294. // verify, after build edge completion, that:
  295. //
  296. // 1) Created output symlinks match the list of paths in this list exactly (no more, no fewer)
  297. // 2) Created output files are *not* declared in this list.
  298. //
  299. // These symlink outputs are expected to be a subset of outputs or implicit
  300. // outputs, or they would fail validation at build param construction time
  301. // later, to support other non-rule-builder approaches for constructing
  302. // statements.
  303. func (r *RuleBuilder) SymlinkOutputs() WritablePaths {
  304. symlinkOutputs := r.symlinkOutputSet()
  305. var symlinkOutputList WritablePaths
  306. for _, symlinkOutput := range symlinkOutputs {
  307. symlinkOutputList = append(symlinkOutputList, symlinkOutput)
  308. }
  309. sort.Slice(symlinkOutputList, func(i, j int) bool {
  310. return symlinkOutputList[i].String() < symlinkOutputList[j].String()
  311. })
  312. return symlinkOutputList
  313. }
  314. func (r *RuleBuilder) depFileSet() map[string]WritablePath {
  315. depFiles := make(map[string]WritablePath)
  316. for _, c := range r.commands {
  317. for _, depFile := range c.depFiles {
  318. depFiles[depFile.String()] = depFile
  319. }
  320. }
  321. return depFiles
  322. }
  323. // DepFiles returns the list of paths that were passed to the RuleBuilderCommand methods that take depfile paths, such
  324. // as RuleBuilderCommand.DepFile or RuleBuilderCommand.FlagWithDepFile.
  325. func (r *RuleBuilder) DepFiles() WritablePaths {
  326. var depFiles WritablePaths
  327. for _, c := range r.commands {
  328. for _, depFile := range c.depFiles {
  329. depFiles = append(depFiles, depFile)
  330. }
  331. }
  332. return depFiles
  333. }
  334. // Installs returns the list of tuples passed to Install.
  335. func (r *RuleBuilder) Installs() RuleBuilderInstalls {
  336. return append(RuleBuilderInstalls(nil), r.installs...)
  337. }
  338. func (r *RuleBuilder) toolsSet() map[string]Path {
  339. tools := make(map[string]Path)
  340. for _, c := range r.commands {
  341. for _, tool := range c.tools {
  342. tools[tool.String()] = tool
  343. }
  344. }
  345. return tools
  346. }
  347. // Tools returns the list of paths that were passed to the RuleBuilderCommand.Tool method. The
  348. // list is sorted and duplicates removed.
  349. func (r *RuleBuilder) Tools() Paths {
  350. toolsSet := r.toolsSet()
  351. var toolsList Paths
  352. for _, tool := range toolsSet {
  353. toolsList = append(toolsList, tool)
  354. }
  355. sort.Slice(toolsList, func(i, j int) bool {
  356. return toolsList[i].String() < toolsList[j].String()
  357. })
  358. return toolsList
  359. }
  360. // RspFileInputs returns the list of paths that were passed to the RuleBuilderCommand.FlagWithRspFileInputList method.
  361. func (r *RuleBuilder) RspFileInputs() Paths {
  362. var rspFileInputs Paths
  363. for _, c := range r.commands {
  364. for _, rspFile := range c.rspFiles {
  365. rspFileInputs = append(rspFileInputs, rspFile.paths...)
  366. }
  367. }
  368. return rspFileInputs
  369. }
  370. func (r *RuleBuilder) rspFiles() []rspFileAndPaths {
  371. var rspFiles []rspFileAndPaths
  372. for _, c := range r.commands {
  373. rspFiles = append(rspFiles, c.rspFiles...)
  374. }
  375. return rspFiles
  376. }
  377. // Commands returns a slice containing the built command line for each call to RuleBuilder.Command.
  378. func (r *RuleBuilder) Commands() []string {
  379. var commands []string
  380. for _, c := range r.commands {
  381. commands = append(commands, c.String())
  382. }
  383. return commands
  384. }
  385. // BuilderContext is a subset of ModuleContext and SingletonContext.
  386. type BuilderContext interface {
  387. PathContext
  388. Rule(PackageContext, string, blueprint.RuleParams, ...string) blueprint.Rule
  389. Build(PackageContext, BuildParams)
  390. }
  391. var _ BuilderContext = ModuleContext(nil)
  392. var _ BuilderContext = SingletonContext(nil)
  393. func (r *RuleBuilder) depFileMergerCmd(depFiles WritablePaths) *RuleBuilderCommand {
  394. return r.Command().
  395. builtToolWithoutDeps("dep_fixer").
  396. Inputs(depFiles.Paths())
  397. }
  398. // Build adds the built command line to the build graph, with dependencies on Inputs and Tools, and output files for
  399. // Outputs.
  400. func (r *RuleBuilder) Build(name string, desc string) {
  401. name = ninjaNameEscape(name)
  402. if len(r.missingDeps) > 0 {
  403. r.ctx.Build(pctx, BuildParams{
  404. Rule: ErrorRule,
  405. Outputs: r.Outputs(),
  406. Description: desc,
  407. Args: map[string]string{
  408. "error": "missing dependencies: " + strings.Join(r.missingDeps, ", "),
  409. },
  410. })
  411. return
  412. }
  413. var depFile WritablePath
  414. var depFormat blueprint.Deps
  415. if depFiles := r.DepFiles(); len(depFiles) > 0 {
  416. depFile = depFiles[0]
  417. depFormat = blueprint.DepsGCC
  418. if len(depFiles) > 1 {
  419. // Add a command locally that merges all depfiles together into the first depfile.
  420. r.depFileMergerCmd(depFiles)
  421. if r.sbox {
  422. // Check for Rel() errors, as all depfiles should be in the output dir. Errors
  423. // will be reported to the ctx.
  424. for _, path := range depFiles[1:] {
  425. Rel(r.ctx, r.outDir.String(), path.String())
  426. }
  427. }
  428. }
  429. }
  430. tools := r.Tools()
  431. commands := r.Commands()
  432. outputs := r.Outputs()
  433. inputs := r.Inputs()
  434. rspFiles := r.rspFiles()
  435. if len(commands) == 0 {
  436. return
  437. }
  438. if len(outputs) == 0 {
  439. panic("No outputs specified from any Commands")
  440. }
  441. commandString := strings.Join(commands, " && ")
  442. if r.sbox {
  443. // If running the command inside sbox, write the rule data out to an sbox
  444. // manifest.textproto.
  445. manifest := sbox_proto.Manifest{}
  446. command := sbox_proto.Command{}
  447. manifest.Commands = append(manifest.Commands, &command)
  448. command.Command = proto.String(commandString)
  449. if depFile != nil {
  450. manifest.OutputDepfile = proto.String(depFile.String())
  451. }
  452. // If sandboxing tools is enabled, add copy rules to the manifest to copy each tool
  453. // into the sbox directory.
  454. if r.sboxTools {
  455. for _, tool := range tools {
  456. command.CopyBefore = append(command.CopyBefore, &sbox_proto.Copy{
  457. From: proto.String(tool.String()),
  458. To: proto.String(sboxPathForToolRel(r.ctx, tool)),
  459. })
  460. }
  461. for _, c := range r.commands {
  462. for _, tool := range c.packagedTools {
  463. command.CopyBefore = append(command.CopyBefore, &sbox_proto.Copy{
  464. From: proto.String(tool.srcPath.String()),
  465. To: proto.String(sboxPathForPackagedToolRel(tool)),
  466. Executable: proto.Bool(tool.executable),
  467. })
  468. tools = append(tools, tool.srcPath)
  469. }
  470. }
  471. }
  472. // If sandboxing inputs is enabled, add copy rules to the manifest to copy each input
  473. // into the sbox directory.
  474. if r.sboxInputs {
  475. for _, input := range inputs {
  476. command.CopyBefore = append(command.CopyBefore, &sbox_proto.Copy{
  477. From: proto.String(input.String()),
  478. To: proto.String(r.sboxPathForInputRel(input)),
  479. })
  480. }
  481. // If using rsp files copy them and their contents into the sbox directory with
  482. // the appropriate path mappings.
  483. for _, rspFile := range rspFiles {
  484. command.RspFiles = append(command.RspFiles, &sbox_proto.RspFile{
  485. File: proto.String(rspFile.file.String()),
  486. // These have to match the logic in sboxPathForInputRel
  487. PathMappings: []*sbox_proto.PathMapping{
  488. {
  489. From: proto.String(r.outDir.String()),
  490. To: proto.String(sboxOutSubDir),
  491. },
  492. {
  493. From: proto.String(PathForOutput(r.ctx).String()),
  494. To: proto.String(sboxOutSubDir),
  495. },
  496. },
  497. })
  498. }
  499. command.Chdir = proto.Bool(true)
  500. }
  501. // Add copy rules to the manifest to copy each output file from the sbox directory.
  502. // to the output directory after running the commands.
  503. sboxOutputs := make([]string, len(outputs))
  504. for i, output := range outputs {
  505. rel := Rel(r.ctx, r.outDir.String(), output.String())
  506. sboxOutputs[i] = filepath.Join(sboxOutDir, rel)
  507. command.CopyAfter = append(command.CopyAfter, &sbox_proto.Copy{
  508. From: proto.String(filepath.Join(sboxOutSubDir, rel)),
  509. To: proto.String(output.String()),
  510. })
  511. }
  512. // Outputs that were marked Temporary will not be checked that they are in the output
  513. // directory by the loop above, check them here.
  514. for path := range r.temporariesSet {
  515. Rel(r.ctx, r.outDir.String(), path.String())
  516. }
  517. // Add a hash of the list of input files to the manifest so that the textproto file
  518. // changes when the list of input files changes and causes the sbox rule that
  519. // depends on it to rerun.
  520. command.InputHash = proto.String(hashSrcFiles(inputs))
  521. // Verify that the manifest textproto is not inside the sbox output directory, otherwise
  522. // it will get deleted when the sbox rule clears its output directory.
  523. _, manifestInOutDir := MaybeRel(r.ctx, r.outDir.String(), r.sboxManifestPath.String())
  524. if manifestInOutDir {
  525. ReportPathErrorf(r.ctx, "sbox rule %q manifestPath %q must not be in outputDir %q",
  526. name, r.sboxManifestPath.String(), r.outDir.String())
  527. }
  528. // Create a rule to write the manifest as a the textproto.
  529. pbText, err := prototext.Marshal(&manifest)
  530. if err != nil {
  531. ReportPathErrorf(r.ctx, "sbox manifest failed to marshal: %q", err)
  532. }
  533. WriteFileRule(r.ctx, r.sboxManifestPath, string(pbText))
  534. // Generate a new string to use as the command line of the sbox rule. This uses
  535. // a RuleBuilderCommand as a convenience method of building the command line, then
  536. // converts it to a string to replace commandString.
  537. sboxCmd := &RuleBuilderCommand{
  538. rule: &RuleBuilder{
  539. ctx: r.ctx,
  540. },
  541. }
  542. sboxCmd.builtToolWithoutDeps("sbox").
  543. FlagWithArg("--sandbox-path ", shared.TempDirForOutDir(PathForOutput(r.ctx).String())).
  544. FlagWithArg("--output-dir ", r.outDir.String()).
  545. FlagWithInput("--manifest ", r.sboxManifestPath)
  546. if r.restat {
  547. sboxCmd.Flag("--write-if-changed")
  548. }
  549. // Replace the command string, and add the sbox tool and manifest textproto to the
  550. // dependencies of the final sbox rule.
  551. commandString = sboxCmd.buf.String()
  552. tools = append(tools, sboxCmd.tools...)
  553. inputs = append(inputs, sboxCmd.inputs...)
  554. if r.rbeParams != nil {
  555. // RBE needs a list of input files to copy to the remote builder. For inputs already
  556. // listed in an rsp file, pass the rsp file directly to rewrapper. For the rest,
  557. // create a new rsp file to pass to rewrapper.
  558. var remoteRspFiles Paths
  559. var remoteInputs Paths
  560. remoteInputs = append(remoteInputs, inputs...)
  561. remoteInputs = append(remoteInputs, tools...)
  562. for _, rspFile := range rspFiles {
  563. remoteInputs = append(remoteInputs, rspFile.file)
  564. remoteRspFiles = append(remoteRspFiles, rspFile.file)
  565. }
  566. if len(remoteInputs) > 0 {
  567. inputsListFile := r.sboxManifestPath.ReplaceExtension(r.ctx, "rbe_inputs.list")
  568. writeRspFileRule(r.ctx, inputsListFile, remoteInputs)
  569. remoteRspFiles = append(remoteRspFiles, inputsListFile)
  570. // Add the new rsp file as an extra input to the rule.
  571. inputs = append(inputs, inputsListFile)
  572. }
  573. r.rbeParams.OutputFiles = outputs.Strings()
  574. r.rbeParams.RSPFiles = remoteRspFiles.Strings()
  575. rewrapperCommand := r.rbeParams.NoVarTemplate(r.ctx.Config().RBEWrapper())
  576. commandString = rewrapperCommand + " bash -c '" + strings.ReplaceAll(commandString, `'`, `'\''`) + "'"
  577. }
  578. } else {
  579. // If not using sbox the rule will run the command directly, put the hash of the
  580. // list of input files in a comment at the end of the command line to ensure ninja
  581. // reruns the rule when the list of input files changes.
  582. commandString += " # hash of input list: " + hashSrcFiles(inputs)
  583. }
  584. // Ninja doesn't like multiple outputs when depfiles are enabled, move all but the first output to
  585. // ImplicitOutputs. RuleBuilder doesn't use "$out", so the distinction between Outputs and
  586. // ImplicitOutputs doesn't matter.
  587. output := outputs[0]
  588. implicitOutputs := outputs[1:]
  589. var rspFile, rspFileContent string
  590. var rspFileInputs Paths
  591. if len(rspFiles) > 0 {
  592. // The first rsp files uses Ninja's rsp file support for the rule
  593. rspFile = rspFiles[0].file.String()
  594. // Use "$in" for rspFileContent to avoid duplicating the list of files in the dependency
  595. // list and in the contents of the rsp file. Inputs to the rule that are not in the
  596. // rsp file will be listed in Implicits instead of Inputs so they don't show up in "$in".
  597. rspFileContent = "$in"
  598. rspFileInputs = append(rspFileInputs, rspFiles[0].paths...)
  599. for _, rspFile := range rspFiles[1:] {
  600. // Any additional rsp files need an extra rule to write the file.
  601. writeRspFileRule(r.ctx, rspFile.file, rspFile.paths)
  602. // The main rule needs to depend on the inputs listed in the extra rsp file.
  603. inputs = append(inputs, rspFile.paths...)
  604. // The main rule needs to depend on the extra rsp file.
  605. inputs = append(inputs, rspFile.file)
  606. }
  607. }
  608. var pool blueprint.Pool
  609. if r.ctx.Config().UseGoma() && r.remoteable.Goma {
  610. // When USE_GOMA=true is set and the rule is supported by goma, allow jobs to run outside the local pool.
  611. } else if r.ctx.Config().UseRBE() && r.remoteable.RBE {
  612. // When USE_RBE=true is set and the rule is supported by RBE, use the remotePool.
  613. pool = remotePool
  614. } else if r.highmem {
  615. pool = highmemPool
  616. } else if r.ctx.Config().UseRemoteBuild() {
  617. pool = localPool
  618. }
  619. r.ctx.Build(r.pctx, BuildParams{
  620. Rule: r.ctx.Rule(pctx, name, blueprint.RuleParams{
  621. Command: proptools.NinjaEscape(commandString),
  622. CommandDeps: proptools.NinjaEscapeList(tools.Strings()),
  623. Restat: r.restat,
  624. Rspfile: proptools.NinjaEscape(rspFile),
  625. RspfileContent: rspFileContent,
  626. Pool: pool,
  627. }),
  628. Inputs: rspFileInputs,
  629. Implicits: inputs,
  630. OrderOnly: r.OrderOnlys(),
  631. Validations: r.Validations(),
  632. Output: output,
  633. ImplicitOutputs: implicitOutputs,
  634. SymlinkOutputs: r.SymlinkOutputs(),
  635. Depfile: depFile,
  636. Deps: depFormat,
  637. Description: desc,
  638. })
  639. }
  640. // RuleBuilderCommand is a builder for a command in a command line. It can be mutated by its methods to add to the
  641. // command and track dependencies. The methods mutate the RuleBuilderCommand in place, as well as return the
  642. // RuleBuilderCommand, so they can be used chained or unchained. All methods that add text implicitly add a single
  643. // space as a separator from the previous method.
  644. type RuleBuilderCommand struct {
  645. rule *RuleBuilder
  646. buf strings.Builder
  647. inputs Paths
  648. implicits Paths
  649. orderOnlys Paths
  650. validations Paths
  651. outputs WritablePaths
  652. symlinkOutputs WritablePaths
  653. depFiles WritablePaths
  654. tools Paths
  655. packagedTools []PackagingSpec
  656. rspFiles []rspFileAndPaths
  657. }
  658. type rspFileAndPaths struct {
  659. file WritablePath
  660. paths Paths
  661. }
  662. func checkPathNotNil(path Path) {
  663. if path == nil {
  664. panic("rule_builder paths cannot be nil")
  665. }
  666. }
  667. func (c *RuleBuilderCommand) addInput(path Path) string {
  668. checkPathNotNil(path)
  669. c.inputs = append(c.inputs, path)
  670. return c.PathForInput(path)
  671. }
  672. func (c *RuleBuilderCommand) addImplicit(path Path) {
  673. checkPathNotNil(path)
  674. c.implicits = append(c.implicits, path)
  675. }
  676. func (c *RuleBuilderCommand) addOrderOnly(path Path) {
  677. checkPathNotNil(path)
  678. c.orderOnlys = append(c.orderOnlys, path)
  679. }
  680. // PathForInput takes an input path and returns the appropriate path to use on the command line. If
  681. // sbox was enabled via a call to RuleBuilder.Sbox() and the path was an output path it returns a
  682. // path with the placeholder prefix used for outputs in sbox. If sbox is not enabled it returns the
  683. // original path.
  684. func (c *RuleBuilderCommand) PathForInput(path Path) string {
  685. if c.rule.sbox {
  686. rel, inSandbox := c.rule._sboxPathForInputRel(path)
  687. if inSandbox {
  688. rel = filepath.Join(sboxSandboxBaseDir, rel)
  689. }
  690. return rel
  691. }
  692. return path.String()
  693. }
  694. // PathsForInputs takes a list of input paths and returns the appropriate paths to use on the
  695. // command line. If sbox was enabled via a call to RuleBuilder.Sbox() a path was an output path, it
  696. // returns the path with the placeholder prefix used for outputs in sbox. If sbox is not enabled it
  697. // returns the original paths.
  698. func (c *RuleBuilderCommand) PathsForInputs(paths Paths) []string {
  699. ret := make([]string, len(paths))
  700. for i, path := range paths {
  701. ret[i] = c.PathForInput(path)
  702. }
  703. return ret
  704. }
  705. // PathForOutput takes an output path and returns the appropriate path to use on the command
  706. // line. If sbox was enabled via a call to RuleBuilder.Sbox(), it returns a path with the
  707. // placeholder prefix used for outputs in sbox. If sbox is not enabled it returns the
  708. // original path.
  709. func (c *RuleBuilderCommand) PathForOutput(path WritablePath) string {
  710. if c.rule.sbox {
  711. // Errors will be handled in RuleBuilder.Build where we have a context to report them
  712. rel, _, _ := maybeRelErr(c.rule.outDir.String(), path.String())
  713. return filepath.Join(sboxOutDir, rel)
  714. }
  715. return path.String()
  716. }
  717. func sboxPathForToolRel(ctx BuilderContext, path Path) string {
  718. // Errors will be handled in RuleBuilder.Build where we have a context to report them
  719. toolDir := pathForInstall(ctx, ctx.Config().BuildOS, ctx.Config().BuildArch, "", false)
  720. relOutSoong, isRelOutSoong, _ := maybeRelErr(toolDir.String(), path.String())
  721. if isRelOutSoong {
  722. // The tool is in the Soong output directory, it will be copied to __SBOX_OUT_DIR__/tools/out
  723. return filepath.Join(sboxToolsSubDir, "out", relOutSoong)
  724. }
  725. // The tool is in the source directory, it will be copied to __SBOX_OUT_DIR__/tools/src
  726. return filepath.Join(sboxToolsSubDir, "src", path.String())
  727. }
  728. func (r *RuleBuilder) _sboxPathForInputRel(path Path) (rel string, inSandbox bool) {
  729. // Errors will be handled in RuleBuilder.Build where we have a context to report them
  730. rel, isRelSboxOut, _ := maybeRelErr(r.outDir.String(), path.String())
  731. if isRelSboxOut {
  732. return filepath.Join(sboxOutSubDir, rel), true
  733. }
  734. if r.sboxInputs {
  735. // When sandboxing inputs all inputs have to be copied into the sandbox. Input files that
  736. // are outputs of other rules could be an arbitrary absolute path if OUT_DIR is set, so they
  737. // will be copied to relative paths under __SBOX_OUT_DIR__/out.
  738. rel, isRelOut, _ := maybeRelErr(PathForOutput(r.ctx).String(), path.String())
  739. if isRelOut {
  740. return filepath.Join(sboxOutSubDir, rel), true
  741. }
  742. }
  743. return path.String(), false
  744. }
  745. func (r *RuleBuilder) sboxPathForInputRel(path Path) string {
  746. rel, _ := r._sboxPathForInputRel(path)
  747. return rel
  748. }
  749. func (r *RuleBuilder) sboxPathsForInputsRel(paths Paths) []string {
  750. ret := make([]string, len(paths))
  751. for i, path := range paths {
  752. ret[i] = r.sboxPathForInputRel(path)
  753. }
  754. return ret
  755. }
  756. func sboxPathForPackagedToolRel(spec PackagingSpec) string {
  757. return filepath.Join(sboxToolsSubDir, "out", spec.relPathInPackage)
  758. }
  759. // PathForPackagedTool takes a PackageSpec for a tool and returns the corresponding path for the
  760. // tool after copying it into the sandbox. This can be used on the RuleBuilder command line to
  761. // reference the tool.
  762. func (c *RuleBuilderCommand) PathForPackagedTool(spec PackagingSpec) string {
  763. if !c.rule.sboxTools {
  764. panic("PathForPackagedTool() requires SandboxTools()")
  765. }
  766. return filepath.Join(sboxSandboxBaseDir, sboxPathForPackagedToolRel(spec))
  767. }
  768. // PathForTool takes a path to a tool, which may be an output file or a source file, and returns
  769. // the corresponding path for the tool in the sbox sandbox if sbox is enabled, or the original path
  770. // if it is not. This can be used on the RuleBuilder command line to reference the tool.
  771. func (c *RuleBuilderCommand) PathForTool(path Path) string {
  772. if c.rule.sbox && c.rule.sboxTools {
  773. return filepath.Join(sboxSandboxBaseDir, sboxPathForToolRel(c.rule.ctx, path))
  774. }
  775. return path.String()
  776. }
  777. // PathsForTools takes a list of paths to tools, which may be output files or source files, and
  778. // returns the corresponding paths for the tools in the sbox sandbox if sbox is enabled, or the
  779. // original paths if it is not. This can be used on the RuleBuilder command line to reference the tool.
  780. func (c *RuleBuilderCommand) PathsForTools(paths Paths) []string {
  781. if c.rule.sbox && c.rule.sboxTools {
  782. var ret []string
  783. for _, path := range paths {
  784. ret = append(ret, filepath.Join(sboxSandboxBaseDir, sboxPathForToolRel(c.rule.ctx, path)))
  785. }
  786. return ret
  787. }
  788. return paths.Strings()
  789. }
  790. // PackagedTool adds the specified tool path to the command line. It can only be used with tool
  791. // sandboxing enabled by SandboxTools(), and will copy the tool into the sandbox.
  792. func (c *RuleBuilderCommand) PackagedTool(spec PackagingSpec) *RuleBuilderCommand {
  793. if !c.rule.sboxTools {
  794. panic("PackagedTool() requires SandboxTools()")
  795. }
  796. c.packagedTools = append(c.packagedTools, spec)
  797. c.Text(sboxPathForPackagedToolRel(spec))
  798. return c
  799. }
  800. // ImplicitPackagedTool copies the specified tool into the sandbox without modifying the command
  801. // line. It can only be used with tool sandboxing enabled by SandboxTools().
  802. func (c *RuleBuilderCommand) ImplicitPackagedTool(spec PackagingSpec) *RuleBuilderCommand {
  803. if !c.rule.sboxTools {
  804. panic("ImplicitPackagedTool() requires SandboxTools()")
  805. }
  806. c.packagedTools = append(c.packagedTools, spec)
  807. return c
  808. }
  809. // ImplicitPackagedTools copies the specified tools into the sandbox without modifying the command
  810. // line. It can only be used with tool sandboxing enabled by SandboxTools().
  811. func (c *RuleBuilderCommand) ImplicitPackagedTools(specs []PackagingSpec) *RuleBuilderCommand {
  812. if !c.rule.sboxTools {
  813. panic("ImplicitPackagedTools() requires SandboxTools()")
  814. }
  815. c.packagedTools = append(c.packagedTools, specs...)
  816. return c
  817. }
  818. // Text adds the specified raw text to the command line. The text should not contain input or output paths or the
  819. // rule will not have them listed in its dependencies or outputs.
  820. func (c *RuleBuilderCommand) Text(text string) *RuleBuilderCommand {
  821. if c.buf.Len() > 0 {
  822. c.buf.WriteByte(' ')
  823. }
  824. c.buf.WriteString(text)
  825. return c
  826. }
  827. // Textf adds the specified formatted text to the command line. The text should not contain input or output paths or
  828. // the rule will not have them listed in its dependencies or outputs.
  829. func (c *RuleBuilderCommand) Textf(format string, a ...interface{}) *RuleBuilderCommand {
  830. return c.Text(fmt.Sprintf(format, a...))
  831. }
  832. // Flag adds the specified raw text to the command line. The text should not contain input or output paths or the
  833. // rule will not have them listed in its dependencies or outputs.
  834. func (c *RuleBuilderCommand) Flag(flag string) *RuleBuilderCommand {
  835. return c.Text(flag)
  836. }
  837. // OptionalFlag adds the specified raw text to the command line if it is not nil. The text should not contain input or
  838. // output paths or the rule will not have them listed in its dependencies or outputs.
  839. func (c *RuleBuilderCommand) OptionalFlag(flag *string) *RuleBuilderCommand {
  840. if flag != nil {
  841. c.Text(*flag)
  842. }
  843. return c
  844. }
  845. // Flags adds the specified raw text to the command line. The text should not contain input or output paths or the
  846. // rule will not have them listed in its dependencies or outputs.
  847. func (c *RuleBuilderCommand) Flags(flags []string) *RuleBuilderCommand {
  848. for _, flag := range flags {
  849. c.Text(flag)
  850. }
  851. return c
  852. }
  853. // FlagWithArg adds the specified flag and argument text to the command line, with no separator between them. The flag
  854. // and argument should not contain input or output paths or the rule will not have them listed in its dependencies or
  855. // outputs.
  856. func (c *RuleBuilderCommand) FlagWithArg(flag, arg string) *RuleBuilderCommand {
  857. return c.Text(flag + arg)
  858. }
  859. // FlagForEachArg adds the specified flag joined with each argument to the command line. The result is identical to
  860. // calling FlagWithArg for argument.
  861. func (c *RuleBuilderCommand) FlagForEachArg(flag string, args []string) *RuleBuilderCommand {
  862. for _, arg := range args {
  863. c.FlagWithArg(flag, arg)
  864. }
  865. return c
  866. }
  867. // FlagWithList adds the specified flag and list of arguments to the command line, with the arguments joined by sep
  868. // and no separator between the flag and arguments. The flag and arguments should not contain input or output paths or
  869. // the rule will not have them listed in its dependencies or outputs.
  870. func (c *RuleBuilderCommand) FlagWithList(flag string, list []string, sep string) *RuleBuilderCommand {
  871. return c.Text(flag + strings.Join(list, sep))
  872. }
  873. // Tool adds the specified tool path to the command line. The path will be also added to the dependencies returned by
  874. // RuleBuilder.Tools.
  875. func (c *RuleBuilderCommand) Tool(path Path) *RuleBuilderCommand {
  876. checkPathNotNil(path)
  877. c.tools = append(c.tools, path)
  878. return c.Text(c.PathForTool(path))
  879. }
  880. // Tool adds the specified tool path to the dependencies returned by RuleBuilder.Tools.
  881. func (c *RuleBuilderCommand) ImplicitTool(path Path) *RuleBuilderCommand {
  882. checkPathNotNil(path)
  883. c.tools = append(c.tools, path)
  884. return c
  885. }
  886. // Tool adds the specified tool path to the dependencies returned by RuleBuilder.Tools.
  887. func (c *RuleBuilderCommand) ImplicitTools(paths Paths) *RuleBuilderCommand {
  888. for _, path := range paths {
  889. c.ImplicitTool(path)
  890. }
  891. return c
  892. }
  893. // BuiltTool adds the specified tool path that was built using a host Soong module to the command line. The path will
  894. // be also added to the dependencies returned by RuleBuilder.Tools.
  895. //
  896. // It is equivalent to:
  897. // cmd.Tool(ctx.Config().HostToolPath(ctx, tool))
  898. func (c *RuleBuilderCommand) BuiltTool(tool string) *RuleBuilderCommand {
  899. if c.rule.ctx.Config().UseHostMusl() {
  900. // If the host is using musl, assume that the tool was built against musl libc and include
  901. // libc_musl.so in the sandbox.
  902. // TODO(ccross): if we supported adding new dependencies during GenerateAndroidBuildActions
  903. // this could be a dependency + TransitivePackagingSpecs.
  904. c.ImplicitTool(c.rule.ctx.Config().HostJNIToolPath(c.rule.ctx, "libc_musl"))
  905. }
  906. return c.builtToolWithoutDeps(tool)
  907. }
  908. // builtToolWithoutDeps is similar to BuiltTool, but doesn't add any dependencies. It is used
  909. // internally by RuleBuilder for helper tools that are known to be compiled statically.
  910. func (c *RuleBuilderCommand) builtToolWithoutDeps(tool string) *RuleBuilderCommand {
  911. return c.Tool(c.rule.ctx.Config().HostToolPath(c.rule.ctx, tool))
  912. }
  913. // PrebuiltBuildTool adds the specified tool path from prebuils/build-tools. The path will be also added to the
  914. // dependencies returned by RuleBuilder.Tools.
  915. //
  916. // It is equivalent to:
  917. // cmd.Tool(ctx.Config().PrebuiltBuildTool(ctx, tool))
  918. func (c *RuleBuilderCommand) PrebuiltBuildTool(ctx PathContext, tool string) *RuleBuilderCommand {
  919. return c.Tool(ctx.Config().PrebuiltBuildTool(ctx, tool))
  920. }
  921. // Input adds the specified input path to the command line. The path will also be added to the dependencies returned by
  922. // RuleBuilder.Inputs.
  923. func (c *RuleBuilderCommand) Input(path Path) *RuleBuilderCommand {
  924. return c.Text(c.addInput(path))
  925. }
  926. // Inputs adds the specified input paths to the command line, separated by spaces. The paths will also be added to the
  927. // dependencies returned by RuleBuilder.Inputs.
  928. func (c *RuleBuilderCommand) Inputs(paths Paths) *RuleBuilderCommand {
  929. for _, path := range paths {
  930. c.Input(path)
  931. }
  932. return c
  933. }
  934. // Implicit adds the specified input path to the dependencies returned by RuleBuilder.Inputs without modifying the
  935. // command line.
  936. func (c *RuleBuilderCommand) Implicit(path Path) *RuleBuilderCommand {
  937. c.addImplicit(path)
  938. return c
  939. }
  940. // Implicits adds the specified input paths to the dependencies returned by RuleBuilder.Inputs without modifying the
  941. // command line.
  942. func (c *RuleBuilderCommand) Implicits(paths Paths) *RuleBuilderCommand {
  943. for _, path := range paths {
  944. c.addImplicit(path)
  945. }
  946. return c
  947. }
  948. // GetImplicits returns the command's implicit inputs.
  949. func (c *RuleBuilderCommand) GetImplicits() Paths {
  950. return c.implicits
  951. }
  952. // OrderOnly adds the specified input path to the dependencies returned by RuleBuilder.OrderOnlys
  953. // without modifying the command line.
  954. func (c *RuleBuilderCommand) OrderOnly(path Path) *RuleBuilderCommand {
  955. c.addOrderOnly(path)
  956. return c
  957. }
  958. // OrderOnlys adds the specified input paths to the dependencies returned by RuleBuilder.OrderOnlys
  959. // without modifying the command line.
  960. func (c *RuleBuilderCommand) OrderOnlys(paths Paths) *RuleBuilderCommand {
  961. for _, path := range paths {
  962. c.addOrderOnly(path)
  963. }
  964. return c
  965. }
  966. // Validation adds the specified input path to the validation dependencies by
  967. // RuleBuilder.Validations without modifying the command line.
  968. func (c *RuleBuilderCommand) Validation(path Path) *RuleBuilderCommand {
  969. checkPathNotNil(path)
  970. c.validations = append(c.validations, path)
  971. return c
  972. }
  973. // Validations adds the specified input paths to the validation dependencies by
  974. // RuleBuilder.Validations without modifying the command line.
  975. func (c *RuleBuilderCommand) Validations(paths Paths) *RuleBuilderCommand {
  976. for _, path := range paths {
  977. c.Validation(path)
  978. }
  979. return c
  980. }
  981. // Output adds the specified output path to the command line. The path will also be added to the outputs returned by
  982. // RuleBuilder.Outputs.
  983. func (c *RuleBuilderCommand) Output(path WritablePath) *RuleBuilderCommand {
  984. checkPathNotNil(path)
  985. c.outputs = append(c.outputs, path)
  986. return c.Text(c.PathForOutput(path))
  987. }
  988. // Outputs adds the specified output paths to the command line, separated by spaces. The paths will also be added to
  989. // the outputs returned by RuleBuilder.Outputs.
  990. func (c *RuleBuilderCommand) Outputs(paths WritablePaths) *RuleBuilderCommand {
  991. for _, path := range paths {
  992. c.Output(path)
  993. }
  994. return c
  995. }
  996. // OutputDir adds the output directory to the command line. This is only available when used with RuleBuilder.Sbox,
  997. // and will be the temporary output directory managed by sbox, not the final one.
  998. func (c *RuleBuilderCommand) OutputDir() *RuleBuilderCommand {
  999. if !c.rule.sbox {
  1000. panic("OutputDir only valid with Sbox")
  1001. }
  1002. return c.Text(sboxOutDir)
  1003. }
  1004. // DepFile adds the specified depfile path to the paths returned by RuleBuilder.DepFiles and adds it to the command
  1005. // line, and causes RuleBuilder.Build file to set the depfile flag for ninja. If multiple depfiles are added to
  1006. // commands in a single RuleBuilder then RuleBuilder.Build will add an extra command to merge the depfiles together.
  1007. func (c *RuleBuilderCommand) DepFile(path WritablePath) *RuleBuilderCommand {
  1008. checkPathNotNil(path)
  1009. c.depFiles = append(c.depFiles, path)
  1010. return c.Text(c.PathForOutput(path))
  1011. }
  1012. // ImplicitOutput adds the specified output path to the dependencies returned by RuleBuilder.Outputs without modifying
  1013. // the command line.
  1014. func (c *RuleBuilderCommand) ImplicitOutput(path WritablePath) *RuleBuilderCommand {
  1015. c.outputs = append(c.outputs, path)
  1016. return c
  1017. }
  1018. // ImplicitOutputs adds the specified output paths to the dependencies returned by RuleBuilder.Outputs without modifying
  1019. // the command line.
  1020. func (c *RuleBuilderCommand) ImplicitOutputs(paths WritablePaths) *RuleBuilderCommand {
  1021. c.outputs = append(c.outputs, paths...)
  1022. return c
  1023. }
  1024. // ImplicitSymlinkOutput declares the specified path as an implicit output that
  1025. // will be a symlink instead of a regular file. Does not modify the command
  1026. // line.
  1027. func (c *RuleBuilderCommand) ImplicitSymlinkOutput(path WritablePath) *RuleBuilderCommand {
  1028. checkPathNotNil(path)
  1029. c.symlinkOutputs = append(c.symlinkOutputs, path)
  1030. return c.ImplicitOutput(path)
  1031. }
  1032. // ImplicitSymlinkOutputs declares the specified paths as implicit outputs that
  1033. // will be a symlinks instead of regular files. Does not modify the command
  1034. // line.
  1035. func (c *RuleBuilderCommand) ImplicitSymlinkOutputs(paths WritablePaths) *RuleBuilderCommand {
  1036. for _, path := range paths {
  1037. c.ImplicitSymlinkOutput(path)
  1038. }
  1039. return c
  1040. }
  1041. // SymlinkOutput declares the specified path as an output that will be a symlink
  1042. // instead of a regular file. Modifies the command line.
  1043. func (c *RuleBuilderCommand) SymlinkOutput(path WritablePath) *RuleBuilderCommand {
  1044. checkPathNotNil(path)
  1045. c.symlinkOutputs = append(c.symlinkOutputs, path)
  1046. return c.Output(path)
  1047. }
  1048. // SymlinkOutputsl declares the specified paths as outputs that will be symlinks
  1049. // instead of regular files. Modifies the command line.
  1050. func (c *RuleBuilderCommand) SymlinkOutputs(paths WritablePaths) *RuleBuilderCommand {
  1051. for _, path := range paths {
  1052. c.SymlinkOutput(path)
  1053. }
  1054. return c
  1055. }
  1056. // ImplicitDepFile adds the specified depfile path to the paths returned by RuleBuilder.DepFiles without modifying
  1057. // the command line, and causes RuleBuilder.Build file to set the depfile flag for ninja. If multiple depfiles
  1058. // are added to commands in a single RuleBuilder then RuleBuilder.Build will add an extra command to merge the
  1059. // depfiles together.
  1060. func (c *RuleBuilderCommand) ImplicitDepFile(path WritablePath) *RuleBuilderCommand {
  1061. c.depFiles = append(c.depFiles, path)
  1062. return c
  1063. }
  1064. // FlagWithInput adds the specified flag and input path to the command line, with no separator between them. The path
  1065. // will also be added to the dependencies returned by RuleBuilder.Inputs.
  1066. func (c *RuleBuilderCommand) FlagWithInput(flag string, path Path) *RuleBuilderCommand {
  1067. return c.Text(flag + c.addInput(path))
  1068. }
  1069. // FlagWithInputList adds the specified flag and input paths to the command line, with the inputs joined by sep
  1070. // and no separator between the flag and inputs. The input paths will also be added to the dependencies returned by
  1071. // RuleBuilder.Inputs.
  1072. func (c *RuleBuilderCommand) FlagWithInputList(flag string, paths Paths, sep string) *RuleBuilderCommand {
  1073. strs := make([]string, len(paths))
  1074. for i, path := range paths {
  1075. strs[i] = c.addInput(path)
  1076. }
  1077. return c.FlagWithList(flag, strs, sep)
  1078. }
  1079. // FlagForEachInput adds the specified flag joined with each input path to the command line. The input paths will also
  1080. // be added to the dependencies returned by RuleBuilder.Inputs. The result is identical to calling FlagWithInput for
  1081. // each input path.
  1082. func (c *RuleBuilderCommand) FlagForEachInput(flag string, paths Paths) *RuleBuilderCommand {
  1083. for _, path := range paths {
  1084. c.FlagWithInput(flag, path)
  1085. }
  1086. return c
  1087. }
  1088. // FlagWithOutput adds the specified flag and output path to the command line, with no separator between them. The path
  1089. // will also be added to the outputs returned by RuleBuilder.Outputs.
  1090. func (c *RuleBuilderCommand) FlagWithOutput(flag string, path WritablePath) *RuleBuilderCommand {
  1091. c.outputs = append(c.outputs, path)
  1092. return c.Text(flag + c.PathForOutput(path))
  1093. }
  1094. // FlagWithDepFile adds the specified flag and depfile path to the command line, with no separator between them. The path
  1095. // will also be added to the outputs returned by RuleBuilder.Outputs.
  1096. func (c *RuleBuilderCommand) FlagWithDepFile(flag string, path WritablePath) *RuleBuilderCommand {
  1097. c.depFiles = append(c.depFiles, path)
  1098. return c.Text(flag + c.PathForOutput(path))
  1099. }
  1100. // FlagWithRspFileInputList adds the specified flag and path to an rspfile to the command line, with
  1101. // no separator between them. The paths will be written to the rspfile. If sbox is enabled, the
  1102. // rspfile must be outside the sbox directory. The first use of FlagWithRspFileInputList in any
  1103. // RuleBuilderCommand of a RuleBuilder will use Ninja's rsp file support for the rule, additional
  1104. // uses will result in an auxiliary rules to write the rspFile contents.
  1105. func (c *RuleBuilderCommand) FlagWithRspFileInputList(flag string, rspFile WritablePath, paths Paths) *RuleBuilderCommand {
  1106. // Use an empty slice if paths is nil, the non-nil slice is used as an indicator that the rsp file must be
  1107. // generated.
  1108. if paths == nil {
  1109. paths = Paths{}
  1110. }
  1111. c.rspFiles = append(c.rspFiles, rspFileAndPaths{rspFile, paths})
  1112. if c.rule.sbox {
  1113. if _, isRel, _ := maybeRelErr(c.rule.outDir.String(), rspFile.String()); isRel {
  1114. panic(fmt.Errorf("FlagWithRspFileInputList rspfile %q must not be inside out dir %q",
  1115. rspFile.String(), c.rule.outDir.String()))
  1116. }
  1117. }
  1118. c.FlagWithArg(flag, c.PathForInput(rspFile))
  1119. return c
  1120. }
  1121. // String returns the command line.
  1122. func (c *RuleBuilderCommand) String() string {
  1123. return c.buf.String()
  1124. }
  1125. // RuleBuilderSboxProtoForTests takes the BuildParams for the manifest passed to RuleBuilder.Sbox()
  1126. // and returns sbox testproto generated by the RuleBuilder.
  1127. func RuleBuilderSboxProtoForTests(t *testing.T, params TestingBuildParams) *sbox_proto.Manifest {
  1128. t.Helper()
  1129. content := ContentFromFileRuleForTests(t, params)
  1130. manifest := sbox_proto.Manifest{}
  1131. err := prototext.Unmarshal([]byte(content), &manifest)
  1132. if err != nil {
  1133. t.Fatalf("failed to unmarshal manifest: %s", err.Error())
  1134. }
  1135. return &manifest
  1136. }
  1137. func ninjaNameEscape(s string) string {
  1138. b := []byte(s)
  1139. escaped := false
  1140. for i, c := range b {
  1141. valid := (c >= 'a' && c <= 'z') ||
  1142. (c >= 'A' && c <= 'Z') ||
  1143. (c >= '0' && c <= '9') ||
  1144. (c == '_') ||
  1145. (c == '-') ||
  1146. (c == '.')
  1147. if !valid {
  1148. b[i] = '_'
  1149. escaped = true
  1150. }
  1151. }
  1152. if escaped {
  1153. s = string(b)
  1154. }
  1155. return s
  1156. }
  1157. // hashSrcFiles returns a hash of the list of source files. It is used to ensure the command line
  1158. // or the sbox textproto manifest change even if the input files are not listed on the command line.
  1159. func hashSrcFiles(srcFiles Paths) string {
  1160. h := sha256.New()
  1161. srcFileList := strings.Join(srcFiles.Strings(), "\n")
  1162. h.Write([]byte(srcFileList))
  1163. return fmt.Sprintf("%x", h.Sum(nil))
  1164. }
  1165. // BuilderContextForTesting returns a BuilderContext for the given config that can be used for tests
  1166. // that need to call methods that take a BuilderContext.
  1167. func BuilderContextForTesting(config Config) BuilderContext {
  1168. pathCtx := PathContextForTesting(config)
  1169. return builderContextForTests{
  1170. PathContext: pathCtx,
  1171. }
  1172. }
  1173. type builderContextForTests struct {
  1174. PathContext
  1175. }
  1176. func (builderContextForTests) Rule(PackageContext, string, blueprint.RuleParams, ...string) blueprint.Rule {
  1177. return nil
  1178. }
  1179. func (builderContextForTests) Build(PackageContext, BuildParams) {}
  1180. func writeRspFileRule(ctx BuilderContext, rspFile WritablePath, paths Paths) {
  1181. buf := &strings.Builder{}
  1182. err := response.WriteRspFile(buf, paths.Strings())
  1183. if err != nil {
  1184. // There should never be I/O errors writing to a bytes.Buffer.
  1185. panic(err)
  1186. }
  1187. WriteFileRule(ctx, rspFile, buf.String())
  1188. }