sbox.go 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803
  1. // Copyright 2017 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package main
  15. import (
  16. "bytes"
  17. "crypto/sha1"
  18. "encoding/hex"
  19. "errors"
  20. "flag"
  21. "fmt"
  22. "io"
  23. "io/ioutil"
  24. "os"
  25. "os/exec"
  26. "path/filepath"
  27. "strconv"
  28. "strings"
  29. "time"
  30. "android/soong/cmd/sbox/sbox_proto"
  31. "android/soong/makedeps"
  32. "android/soong/response"
  33. "google.golang.org/protobuf/encoding/prototext"
  34. )
  35. var (
  36. sandboxesRoot string
  37. outputDir string
  38. manifestFile string
  39. keepOutDir bool
  40. writeIfChanged bool
  41. )
  42. const (
  43. depFilePlaceholder = "__SBOX_DEPFILE__"
  44. sandboxDirPlaceholder = "__SBOX_SANDBOX_DIR__"
  45. )
  46. func init() {
  47. flag.StringVar(&sandboxesRoot, "sandbox-path", "",
  48. "root of temp directory to put the sandbox into")
  49. flag.StringVar(&outputDir, "output-dir", "",
  50. "directory which will contain all output files and only output files")
  51. flag.StringVar(&manifestFile, "manifest", "",
  52. "textproto manifest describing the sandboxed command(s)")
  53. flag.BoolVar(&keepOutDir, "keep-out-dir", false,
  54. "whether to keep the sandbox directory when done")
  55. flag.BoolVar(&writeIfChanged, "write-if-changed", false,
  56. "only write the output files if they have changed")
  57. }
  58. func usageViolation(violation string) {
  59. if violation != "" {
  60. fmt.Fprintf(os.Stderr, "Usage error: %s.\n\n", violation)
  61. }
  62. fmt.Fprintf(os.Stderr,
  63. "Usage: sbox --manifest <manifest> --sandbox-path <sandboxPath>\n")
  64. flag.PrintDefaults()
  65. os.Exit(1)
  66. }
  67. func main() {
  68. flag.Usage = func() {
  69. usageViolation("")
  70. }
  71. flag.Parse()
  72. error := run()
  73. if error != nil {
  74. fmt.Fprintln(os.Stderr, error)
  75. os.Exit(1)
  76. }
  77. }
  78. func findAllFilesUnder(root string) (paths []string) {
  79. paths = []string{}
  80. filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
  81. if !info.IsDir() {
  82. relPath, err := filepath.Rel(root, path)
  83. if err != nil {
  84. // couldn't find relative path from ancestor?
  85. panic(err)
  86. }
  87. paths = append(paths, relPath)
  88. }
  89. return nil
  90. })
  91. return paths
  92. }
  93. func run() error {
  94. if manifestFile == "" {
  95. usageViolation("--manifest <manifest> is required and must be non-empty")
  96. }
  97. if sandboxesRoot == "" {
  98. // In practice, the value of sandboxesRoot will mostly likely be at a fixed location relative to OUT_DIR,
  99. // and the sbox executable will most likely be at a fixed location relative to OUT_DIR too, so
  100. // the value of sandboxesRoot will most likely be at a fixed location relative to the sbox executable
  101. // However, Soong also needs to be able to separately remove the sandbox directory on startup (if it has anything left in it)
  102. // and by passing it as a parameter we don't need to duplicate its value
  103. usageViolation("--sandbox-path <sandboxPath> is required and must be non-empty")
  104. }
  105. manifest, err := readManifest(manifestFile)
  106. if len(manifest.Commands) == 0 {
  107. return fmt.Errorf("at least one commands entry is required in %q", manifestFile)
  108. }
  109. // setup sandbox directory
  110. err = os.MkdirAll(sandboxesRoot, 0777)
  111. if err != nil {
  112. return fmt.Errorf("failed to create %q: %w", sandboxesRoot, err)
  113. }
  114. // This tool assumes that there are no two concurrent runs with the same
  115. // manifestFile. It should therefore be safe to use the hash of the
  116. // manifestFile as the temporary directory name. We do this because it
  117. // makes the temporary directory name deterministic. There are some
  118. // tools that embed the name of the temporary output in the output, and
  119. // they otherwise cause non-determinism, which then poisons actions
  120. // depending on this one.
  121. hash := sha1.New()
  122. hash.Write([]byte(manifestFile))
  123. tempDir := filepath.Join(sandboxesRoot, "sbox", hex.EncodeToString(hash.Sum(nil)))
  124. err = os.RemoveAll(tempDir)
  125. if err != nil {
  126. return err
  127. }
  128. err = os.MkdirAll(tempDir, 0777)
  129. if err != nil {
  130. return fmt.Errorf("failed to create temporary dir in %q: %w", sandboxesRoot, err)
  131. }
  132. // In the common case, the following line of code is what removes the sandbox
  133. // If a fatal error occurs (such as if our Go process is killed unexpectedly),
  134. // then at the beginning of the next build, Soong will wipe the temporary
  135. // directory.
  136. defer func() {
  137. // in some cases we decline to remove the temp dir, to facilitate debugging
  138. if !keepOutDir {
  139. os.RemoveAll(tempDir)
  140. }
  141. }()
  142. // If there is more than one command in the manifest use a separate directory for each one.
  143. useSubDir := len(manifest.Commands) > 1
  144. var commandDepFiles []string
  145. for i, command := range manifest.Commands {
  146. localTempDir := tempDir
  147. if useSubDir {
  148. localTempDir = filepath.Join(localTempDir, strconv.Itoa(i))
  149. }
  150. depFile, err := runCommand(command, localTempDir, i)
  151. if err != nil {
  152. // Running the command failed, keep the temporary output directory around in
  153. // case a user wants to inspect it for debugging purposes. Soong will delete
  154. // it at the beginning of the next build anyway.
  155. keepOutDir = true
  156. return err
  157. }
  158. if depFile != "" {
  159. commandDepFiles = append(commandDepFiles, depFile)
  160. }
  161. }
  162. outputDepFile := manifest.GetOutputDepfile()
  163. if len(commandDepFiles) > 0 && outputDepFile == "" {
  164. return fmt.Errorf("Sandboxed commands used %s but output depfile is not set in manifest file",
  165. depFilePlaceholder)
  166. }
  167. if outputDepFile != "" {
  168. // Merge the depfiles from each command in the manifest to a single output depfile.
  169. err = rewriteDepFiles(commandDepFiles, outputDepFile)
  170. if err != nil {
  171. return fmt.Errorf("failed merging depfiles: %w", err)
  172. }
  173. }
  174. return nil
  175. }
  176. // createCommandScript will create and return an exec.Cmd that runs rawCommand.
  177. //
  178. // rawCommand is executed via a script in the sandbox.
  179. // tempDir is the temporary where the script is created.
  180. // toDirInSandBox is the path containing the script in the sbox environment.
  181. // toDirInSandBox is the path containing the script in the sbox environment.
  182. // seed is a unique integer used to distinguish different scripts that might be at location.
  183. //
  184. // returns an exec.Cmd that can be ran from within sbox context if no error, or nil if error.
  185. // caller must ensure script is cleaned up if function succeeds.
  186. //
  187. func createCommandScript(rawCommand string, tempDir, toDirInSandbox string, seed int) (*exec.Cmd, error) {
  188. scriptName := fmt.Sprintf("sbox_command.%d.bash", seed)
  189. scriptPathAndName := joinPath(tempDir, scriptName)
  190. err := os.WriteFile(scriptPathAndName, []byte(rawCommand), 0644)
  191. if err != nil {
  192. return nil, fmt.Errorf("failed to write command %s... to %s",
  193. rawCommand[0:40], scriptPathAndName)
  194. }
  195. return exec.Command("bash", joinPath(toDirInSandbox, filepath.Base(scriptName))), nil
  196. }
  197. // readManifest reads an sbox manifest from a textproto file.
  198. func readManifest(file string) (*sbox_proto.Manifest, error) {
  199. manifestData, err := ioutil.ReadFile(file)
  200. if err != nil {
  201. return nil, fmt.Errorf("error reading manifest %q: %w", file, err)
  202. }
  203. manifest := sbox_proto.Manifest{}
  204. err = prototext.Unmarshal(manifestData, &manifest)
  205. if err != nil {
  206. return nil, fmt.Errorf("error parsing manifest %q: %w", file, err)
  207. }
  208. return &manifest, nil
  209. }
  210. // runCommand runs a single command from a manifest. If the command references the
  211. // __SBOX_DEPFILE__ placeholder it returns the name of the depfile that was used.
  212. func runCommand(command *sbox_proto.Command, tempDir string, commandIndex int) (depFile string, err error) {
  213. rawCommand := command.GetCommand()
  214. if rawCommand == "" {
  215. return "", fmt.Errorf("command is required")
  216. }
  217. // Remove files from the output directory
  218. err = clearOutputDirectory(command.CopyAfter, outputDir, writeType(writeIfChanged))
  219. if err != nil {
  220. return "", err
  221. }
  222. pathToTempDirInSbox := tempDir
  223. if command.GetChdir() {
  224. pathToTempDirInSbox = "."
  225. }
  226. err = os.MkdirAll(tempDir, 0777)
  227. if err != nil {
  228. return "", fmt.Errorf("failed to create %q: %w", tempDir, err)
  229. }
  230. // Copy in any files specified by the manifest.
  231. err = copyFiles(command.CopyBefore, "", tempDir, requireFromExists, alwaysWrite)
  232. if err != nil {
  233. return "", err
  234. }
  235. err = copyRspFiles(command.RspFiles, tempDir, pathToTempDirInSbox)
  236. if err != nil {
  237. return "", err
  238. }
  239. if strings.Contains(rawCommand, depFilePlaceholder) {
  240. depFile = filepath.Join(pathToTempDirInSbox, "deps.d")
  241. rawCommand = strings.Replace(rawCommand, depFilePlaceholder, depFile, -1)
  242. }
  243. if strings.Contains(rawCommand, sandboxDirPlaceholder) {
  244. rawCommand = strings.Replace(rawCommand, sandboxDirPlaceholder, pathToTempDirInSbox, -1)
  245. }
  246. // Emulate ninja's behavior of creating the directories for any output files before
  247. // running the command.
  248. err = makeOutputDirs(command.CopyAfter, tempDir)
  249. if err != nil {
  250. return "", err
  251. }
  252. cmd, err := createCommandScript(rawCommand, tempDir, pathToTempDirInSbox, commandIndex)
  253. if err != nil {
  254. return "", err
  255. }
  256. buf := &bytes.Buffer{}
  257. cmd.Stdin = os.Stdin
  258. cmd.Stdout = buf
  259. cmd.Stderr = buf
  260. if command.GetChdir() {
  261. cmd.Dir = tempDir
  262. path := os.Getenv("PATH")
  263. absPath, err := makeAbsPathEnv(path)
  264. if err != nil {
  265. return "", err
  266. }
  267. err = os.Setenv("PATH", absPath)
  268. if err != nil {
  269. return "", fmt.Errorf("Failed to update PATH: %w", err)
  270. }
  271. }
  272. err = cmd.Run()
  273. if err != nil {
  274. // The command failed, do a best effort copy of output files out of the sandbox. This is
  275. // especially useful for linters with baselines that print an error message on failure
  276. // with a command to copy the output lint errors to the new baseline. Use a copy instead of
  277. // a move to leave the sandbox intact for manual inspection
  278. copyFiles(command.CopyAfter, tempDir, "", allowFromNotExists, writeType(writeIfChanged))
  279. }
  280. // If the command was executed but failed with an error, print a debugging message before
  281. // the command's output so it doesn't scroll the real error message off the screen.
  282. if exit, ok := err.(*exec.ExitError); ok && !exit.Success() {
  283. fmt.Fprintf(os.Stderr,
  284. "The failing command was run inside an sbox sandbox in temporary directory\n"+
  285. "%s\n"+
  286. "The failing command line was:\n"+
  287. "%s\n",
  288. tempDir, rawCommand)
  289. }
  290. // Write the command's combined stdout/stderr.
  291. os.Stdout.Write(buf.Bytes())
  292. if err != nil {
  293. return "", err
  294. }
  295. err = validateOutputFiles(command.CopyAfter, tempDir, outputDir, rawCommand)
  296. if err != nil {
  297. return "", err
  298. }
  299. // the created files match the declared files; now move them
  300. err = moveFiles(command.CopyAfter, tempDir, "", writeType(writeIfChanged))
  301. if err != nil {
  302. return "", err
  303. }
  304. return depFile, nil
  305. }
  306. // makeOutputDirs creates directories in the sandbox dir for every file that has a rule to be copied
  307. // out of the sandbox. This emulate's Ninja's behavior of creating directories for output files
  308. // so that the tools don't have to.
  309. func makeOutputDirs(copies []*sbox_proto.Copy, sandboxDir string) error {
  310. for _, copyPair := range copies {
  311. dir := joinPath(sandboxDir, filepath.Dir(copyPair.GetFrom()))
  312. err := os.MkdirAll(dir, 0777)
  313. if err != nil {
  314. return err
  315. }
  316. }
  317. return nil
  318. }
  319. // validateOutputFiles verifies that all files that have a rule to be copied out of the sandbox
  320. // were created by the command.
  321. func validateOutputFiles(copies []*sbox_proto.Copy, sandboxDir, outputDir, rawCommand string) error {
  322. var missingOutputErrors []error
  323. var incorrectOutputDirectoryErrors []error
  324. for _, copyPair := range copies {
  325. fromPath := joinPath(sandboxDir, copyPair.GetFrom())
  326. fileInfo, err := os.Stat(fromPath)
  327. if err != nil {
  328. missingOutputErrors = append(missingOutputErrors, fmt.Errorf("%s: does not exist", fromPath))
  329. continue
  330. }
  331. if fileInfo.IsDir() {
  332. missingOutputErrors = append(missingOutputErrors, fmt.Errorf("%s: not a file", fromPath))
  333. }
  334. toPath := copyPair.GetTo()
  335. if rel, err := filepath.Rel(outputDir, toPath); err != nil {
  336. return err
  337. } else if strings.HasPrefix(rel, "../") {
  338. incorrectOutputDirectoryErrors = append(incorrectOutputDirectoryErrors,
  339. fmt.Errorf("%s is not under %s", toPath, outputDir))
  340. }
  341. }
  342. const maxErrors = 10
  343. if len(incorrectOutputDirectoryErrors) > 0 {
  344. errorMessage := ""
  345. more := 0
  346. if len(incorrectOutputDirectoryErrors) > maxErrors {
  347. more = len(incorrectOutputDirectoryErrors) - maxErrors
  348. incorrectOutputDirectoryErrors = incorrectOutputDirectoryErrors[:maxErrors]
  349. }
  350. for _, err := range incorrectOutputDirectoryErrors {
  351. errorMessage += err.Error() + "\n"
  352. }
  353. if more > 0 {
  354. errorMessage += fmt.Sprintf("...%v more", more)
  355. }
  356. return errors.New(errorMessage)
  357. }
  358. if len(missingOutputErrors) > 0 {
  359. // find all created files for making a more informative error message
  360. createdFiles := findAllFilesUnder(sandboxDir)
  361. // build error message
  362. errorMessage := "mismatch between declared and actual outputs\n"
  363. errorMessage += "in sbox command(" + rawCommand + ")\n\n"
  364. errorMessage += "in sandbox " + sandboxDir + ",\n"
  365. errorMessage += fmt.Sprintf("failed to create %v files:\n", len(missingOutputErrors))
  366. for _, missingOutputError := range missingOutputErrors {
  367. errorMessage += " " + missingOutputError.Error() + "\n"
  368. }
  369. if len(createdFiles) < 1 {
  370. errorMessage += "created 0 files."
  371. } else {
  372. errorMessage += fmt.Sprintf("did create %v files:\n", len(createdFiles))
  373. creationMessages := createdFiles
  374. if len(creationMessages) > maxErrors {
  375. creationMessages = creationMessages[:maxErrors]
  376. creationMessages = append(creationMessages, fmt.Sprintf("...%v more", len(createdFiles)-maxErrors))
  377. }
  378. for _, creationMessage := range creationMessages {
  379. errorMessage += " " + creationMessage + "\n"
  380. }
  381. }
  382. return errors.New(errorMessage)
  383. }
  384. return nil
  385. }
  386. type existsType bool
  387. const (
  388. requireFromExists existsType = false
  389. allowFromNotExists = true
  390. )
  391. type writeType bool
  392. const (
  393. alwaysWrite writeType = false
  394. onlyWriteIfChanged = true
  395. )
  396. // copyFiles copies files in or out of the sandbox. If exists is allowFromNotExists then errors
  397. // caused by a from path not existing are ignored. If write is onlyWriteIfChanged then the output
  398. // file is compared to the input file and not written to if it is the same, avoiding updating
  399. // the timestamp.
  400. func copyFiles(copies []*sbox_proto.Copy, fromDir, toDir string, exists existsType, write writeType) error {
  401. for _, copyPair := range copies {
  402. fromPath := joinPath(fromDir, copyPair.GetFrom())
  403. toPath := joinPath(toDir, copyPair.GetTo())
  404. err := copyOneFile(fromPath, toPath, copyPair.GetExecutable(), exists, write)
  405. if err != nil {
  406. return fmt.Errorf("error copying %q to %q: %w", fromPath, toPath, err)
  407. }
  408. }
  409. return nil
  410. }
  411. // copyOneFile copies a file and its permissions. If forceExecutable is true it adds u+x to the
  412. // permissions. If exists is allowFromNotExists it returns nil if the from path doesn't exist.
  413. // If write is onlyWriteIfChanged then the output file is compared to the input file and not written to
  414. // if it is the same, avoiding updating the timestamp.
  415. func copyOneFile(from string, to string, forceExecutable bool, exists existsType,
  416. write writeType) error {
  417. err := os.MkdirAll(filepath.Dir(to), 0777)
  418. if err != nil {
  419. return err
  420. }
  421. stat, err := os.Stat(from)
  422. if err != nil {
  423. if os.IsNotExist(err) && exists == allowFromNotExists {
  424. return nil
  425. }
  426. return err
  427. }
  428. perm := stat.Mode()
  429. if forceExecutable {
  430. perm = perm | 0100 // u+x
  431. }
  432. if write == onlyWriteIfChanged && filesHaveSameContents(from, to) {
  433. return nil
  434. }
  435. in, err := os.Open(from)
  436. if err != nil {
  437. return err
  438. }
  439. defer in.Close()
  440. // Remove the target before copying. In most cases the file won't exist, but if there are
  441. // duplicate copy rules for a file and the source file was read-only the second copy could
  442. // fail.
  443. err = os.Remove(to)
  444. if err != nil && !os.IsNotExist(err) {
  445. return err
  446. }
  447. out, err := os.Create(to)
  448. if err != nil {
  449. return err
  450. }
  451. defer func() {
  452. out.Close()
  453. if err != nil {
  454. os.Remove(to)
  455. }
  456. }()
  457. _, err = io.Copy(out, in)
  458. if err != nil {
  459. return err
  460. }
  461. if err = out.Close(); err != nil {
  462. return err
  463. }
  464. if err = os.Chmod(to, perm); err != nil {
  465. return err
  466. }
  467. return nil
  468. }
  469. // copyRspFiles copies rsp files into the sandbox with path mappings, and also copies the files
  470. // listed into the sandbox.
  471. func copyRspFiles(rspFiles []*sbox_proto.RspFile, toDir, toDirInSandbox string) error {
  472. for _, rspFile := range rspFiles {
  473. err := copyOneRspFile(rspFile, toDir, toDirInSandbox)
  474. if err != nil {
  475. return err
  476. }
  477. }
  478. return nil
  479. }
  480. // copyOneRspFiles copies an rsp file into the sandbox with path mappings, and also copies the files
  481. // listed into the sandbox.
  482. func copyOneRspFile(rspFile *sbox_proto.RspFile, toDir, toDirInSandbox string) error {
  483. in, err := os.Open(rspFile.GetFile())
  484. if err != nil {
  485. return err
  486. }
  487. defer in.Close()
  488. files, err := response.ReadRspFile(in)
  489. if err != nil {
  490. return err
  491. }
  492. for i, from := range files {
  493. // Convert the real path of the input file into the path inside the sandbox using the
  494. // path mappings.
  495. to := applyPathMappings(rspFile.PathMappings, from)
  496. // Copy the file into the sandbox.
  497. err := copyOneFile(from, joinPath(toDir, to), false, requireFromExists, alwaysWrite)
  498. if err != nil {
  499. return err
  500. }
  501. // Rewrite the name in the list of files to be relative to the sandbox directory.
  502. files[i] = joinPath(toDirInSandbox, to)
  503. }
  504. // Convert the real path of the rsp file into the path inside the sandbox using the path
  505. // mappings.
  506. outRspFile := joinPath(toDir, applyPathMappings(rspFile.PathMappings, rspFile.GetFile()))
  507. err = os.MkdirAll(filepath.Dir(outRspFile), 0777)
  508. if err != nil {
  509. return err
  510. }
  511. out, err := os.Create(outRspFile)
  512. if err != nil {
  513. return err
  514. }
  515. defer out.Close()
  516. // Write the rsp file with converted paths into the sandbox.
  517. err = response.WriteRspFile(out, files)
  518. if err != nil {
  519. return err
  520. }
  521. return nil
  522. }
  523. // applyPathMappings takes a list of path mappings and a path, and returns the path with the first
  524. // matching path mapping applied. If the path does not match any of the path mappings then it is
  525. // returned unmodified.
  526. func applyPathMappings(pathMappings []*sbox_proto.PathMapping, path string) string {
  527. for _, mapping := range pathMappings {
  528. if strings.HasPrefix(path, mapping.GetFrom()+"/") {
  529. return joinPath(mapping.GetTo()+"/", strings.TrimPrefix(path, mapping.GetFrom()+"/"))
  530. }
  531. }
  532. return path
  533. }
  534. // moveFiles moves files specified by a set of copy rules. It uses os.Rename, so it is restricted
  535. // to moving files where the source and destination are in the same filesystem. This is OK for
  536. // sbox because the temporary directory is inside the out directory. If write is onlyWriteIfChanged
  537. // then the output file is compared to the input file and not written to if it is the same, avoiding
  538. // updating the timestamp. Otherwise it always updates the timestamp of the new file.
  539. func moveFiles(copies []*sbox_proto.Copy, fromDir, toDir string, write writeType) error {
  540. for _, copyPair := range copies {
  541. fromPath := joinPath(fromDir, copyPair.GetFrom())
  542. toPath := joinPath(toDir, copyPair.GetTo())
  543. err := os.MkdirAll(filepath.Dir(toPath), 0777)
  544. if err != nil {
  545. return err
  546. }
  547. if write == onlyWriteIfChanged && filesHaveSameContents(fromPath, toPath) {
  548. continue
  549. }
  550. err = os.Rename(fromPath, toPath)
  551. if err != nil {
  552. return err
  553. }
  554. // Update the timestamp of the output file in case the tool wrote an old timestamp (for example, tar can extract
  555. // files with old timestamps).
  556. now := time.Now()
  557. err = os.Chtimes(toPath, now, now)
  558. if err != nil {
  559. return err
  560. }
  561. }
  562. return nil
  563. }
  564. // clearOutputDirectory removes all files in the output directory if write is alwaysWrite, or
  565. // any files not listed in copies if write is onlyWriteIfChanged
  566. func clearOutputDirectory(copies []*sbox_proto.Copy, outputDir string, write writeType) error {
  567. if outputDir == "" {
  568. return fmt.Errorf("output directory must be set")
  569. }
  570. if write == alwaysWrite {
  571. // When writing all the output files remove the whole output directory
  572. return os.RemoveAll(outputDir)
  573. }
  574. outputFiles := make(map[string]bool, len(copies))
  575. for _, copyPair := range copies {
  576. outputFiles[copyPair.GetTo()] = true
  577. }
  578. existingFiles := findAllFilesUnder(outputDir)
  579. for _, existingFile := range existingFiles {
  580. fullExistingFile := filepath.Join(outputDir, existingFile)
  581. if !outputFiles[fullExistingFile] {
  582. err := os.Remove(fullExistingFile)
  583. if err != nil {
  584. return fmt.Errorf("failed to remove obsolete output file %s: %w", fullExistingFile, err)
  585. }
  586. }
  587. }
  588. return nil
  589. }
  590. // Rewrite one or more depfiles so that it doesn't include the (randomized) sandbox directory
  591. // to an output file.
  592. func rewriteDepFiles(ins []string, out string) error {
  593. var mergedDeps []string
  594. for _, in := range ins {
  595. data, err := ioutil.ReadFile(in)
  596. if err != nil {
  597. return err
  598. }
  599. deps, err := makedeps.Parse(in, bytes.NewBuffer(data))
  600. if err != nil {
  601. return err
  602. }
  603. mergedDeps = append(mergedDeps, deps.Inputs...)
  604. }
  605. deps := makedeps.Deps{
  606. // Ninja doesn't care what the output file is, so we can use any string here.
  607. Output: "outputfile",
  608. Inputs: mergedDeps,
  609. }
  610. // Make the directory for the output depfile in case it is in a different directory
  611. // than any of the output files.
  612. outDir := filepath.Dir(out)
  613. err := os.MkdirAll(outDir, 0777)
  614. if err != nil {
  615. return fmt.Errorf("failed to create %q: %w", outDir, err)
  616. }
  617. return ioutil.WriteFile(out, deps.Print(), 0666)
  618. }
  619. // joinPath wraps filepath.Join but returns file without appending to dir if file is
  620. // absolute.
  621. func joinPath(dir, file string) string {
  622. if filepath.IsAbs(file) {
  623. return file
  624. }
  625. return filepath.Join(dir, file)
  626. }
  627. // filesHaveSameContents compares the contents if two files, returning true if they are the same
  628. // and returning false if they are different or any errors occur.
  629. func filesHaveSameContents(a, b string) bool {
  630. // Compare the sizes of the two files
  631. statA, err := os.Stat(a)
  632. if err != nil {
  633. return false
  634. }
  635. statB, err := os.Stat(b)
  636. if err != nil {
  637. return false
  638. }
  639. if statA.Size() != statB.Size() {
  640. return false
  641. }
  642. // Open the two files
  643. fileA, err := os.Open(a)
  644. if err != nil {
  645. return false
  646. }
  647. defer fileA.Close()
  648. fileB, err := os.Open(a)
  649. if err != nil {
  650. return false
  651. }
  652. defer fileB.Close()
  653. // Compare the files 1MB at a time
  654. const bufSize = 1 * 1024 * 1024
  655. bufA := make([]byte, bufSize)
  656. bufB := make([]byte, bufSize)
  657. remain := statA.Size()
  658. for remain > 0 {
  659. toRead := int64(bufSize)
  660. if toRead > remain {
  661. toRead = remain
  662. }
  663. _, err = io.ReadFull(fileA, bufA[:toRead])
  664. if err != nil {
  665. return false
  666. }
  667. _, err = io.ReadFull(fileB, bufB[:toRead])
  668. if err != nil {
  669. return false
  670. }
  671. if bytes.Compare(bufA[:toRead], bufB[:toRead]) != 0 {
  672. return false
  673. }
  674. remain -= toRead
  675. }
  676. return true
  677. }
  678. func makeAbsPathEnv(pathEnv string) (string, error) {
  679. pathEnvElements := filepath.SplitList(pathEnv)
  680. for i, p := range pathEnvElements {
  681. if !filepath.IsAbs(p) {
  682. absPath, err := filepath.Abs(p)
  683. if err != nil {
  684. return "", fmt.Errorf("failed to make PATH entry %q absolute: %w", p, err)
  685. }
  686. pathEnvElements[i] = absPath
  687. }
  688. }
  689. return strings.Join(pathEnvElements, string(filepath.ListSeparator)), nil
  690. }