sbox.go 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801
  1. // Copyright 2017 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package main
  15. import (
  16. "bytes"
  17. "crypto/sha1"
  18. "encoding/hex"
  19. "errors"
  20. "flag"
  21. "fmt"
  22. "io"
  23. "io/ioutil"
  24. "os"
  25. "os/exec"
  26. "path/filepath"
  27. "strconv"
  28. "strings"
  29. "time"
  30. "android/soong/cmd/sbox/sbox_proto"
  31. "android/soong/makedeps"
  32. "android/soong/response"
  33. "google.golang.org/protobuf/encoding/prototext"
  34. )
  35. var (
  36. sandboxesRoot string
  37. outputDir string
  38. manifestFile string
  39. keepOutDir bool
  40. writeIfChanged bool
  41. )
  42. const (
  43. depFilePlaceholder = "__SBOX_DEPFILE__"
  44. sandboxDirPlaceholder = "__SBOX_SANDBOX_DIR__"
  45. )
  46. func init() {
  47. flag.StringVar(&sandboxesRoot, "sandbox-path", "",
  48. "root of temp directory to put the sandbox into")
  49. flag.StringVar(&outputDir, "output-dir", "",
  50. "directory which will contain all output files and only output files")
  51. flag.StringVar(&manifestFile, "manifest", "",
  52. "textproto manifest describing the sandboxed command(s)")
  53. flag.BoolVar(&keepOutDir, "keep-out-dir", false,
  54. "whether to keep the sandbox directory when done")
  55. flag.BoolVar(&writeIfChanged, "write-if-changed", false,
  56. "only write the output files if they have changed")
  57. }
  58. func usageViolation(violation string) {
  59. if violation != "" {
  60. fmt.Fprintf(os.Stderr, "Usage error: %s.\n\n", violation)
  61. }
  62. fmt.Fprintf(os.Stderr,
  63. "Usage: sbox --manifest <manifest> --sandbox-path <sandboxPath>\n")
  64. flag.PrintDefaults()
  65. os.Exit(1)
  66. }
  67. func main() {
  68. flag.Usage = func() {
  69. usageViolation("")
  70. }
  71. flag.Parse()
  72. error := run()
  73. if error != nil {
  74. fmt.Fprintln(os.Stderr, error)
  75. os.Exit(1)
  76. }
  77. }
  78. func findAllFilesUnder(root string) (paths []string) {
  79. paths = []string{}
  80. filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
  81. if !info.IsDir() {
  82. relPath, err := filepath.Rel(root, path)
  83. if err != nil {
  84. // couldn't find relative path from ancestor?
  85. panic(err)
  86. }
  87. paths = append(paths, relPath)
  88. }
  89. return nil
  90. })
  91. return paths
  92. }
  93. func run() error {
  94. if manifestFile == "" {
  95. usageViolation("--manifest <manifest> is required and must be non-empty")
  96. }
  97. if sandboxesRoot == "" {
  98. // In practice, the value of sandboxesRoot will mostly likely be at a fixed location relative to OUT_DIR,
  99. // and the sbox executable will most likely be at a fixed location relative to OUT_DIR too, so
  100. // the value of sandboxesRoot will most likely be at a fixed location relative to the sbox executable
  101. // However, Soong also needs to be able to separately remove the sandbox directory on startup (if it has anything left in it)
  102. // and by passing it as a parameter we don't need to duplicate its value
  103. usageViolation("--sandbox-path <sandboxPath> is required and must be non-empty")
  104. }
  105. manifest, err := readManifest(manifestFile)
  106. if len(manifest.Commands) == 0 {
  107. return fmt.Errorf("at least one commands entry is required in %q", manifestFile)
  108. }
  109. // setup sandbox directory
  110. err = os.MkdirAll(sandboxesRoot, 0777)
  111. if err != nil {
  112. return fmt.Errorf("failed to create %q: %w", sandboxesRoot, err)
  113. }
  114. // This tool assumes that there are no two concurrent runs with the same
  115. // manifestFile. It should therefore be safe to use the hash of the
  116. // manifestFile as the temporary directory name. We do this because it
  117. // makes the temporary directory name deterministic. There are some
  118. // tools that embed the name of the temporary output in the output, and
  119. // they otherwise cause non-determinism, which then poisons actions
  120. // depending on this one.
  121. hash := sha1.New()
  122. hash.Write([]byte(manifestFile))
  123. tempDir := filepath.Join(sandboxesRoot, "sbox", hex.EncodeToString(hash.Sum(nil)))
  124. err = os.RemoveAll(tempDir)
  125. if err != nil {
  126. return err
  127. }
  128. err = os.MkdirAll(tempDir, 0777)
  129. if err != nil {
  130. return fmt.Errorf("failed to create temporary dir in %q: %w", sandboxesRoot, err)
  131. }
  132. // In the common case, the following line of code is what removes the sandbox
  133. // If a fatal error occurs (such as if our Go process is killed unexpectedly),
  134. // then at the beginning of the next build, Soong will wipe the temporary
  135. // directory.
  136. defer func() {
  137. // in some cases we decline to remove the temp dir, to facilitate debugging
  138. if !keepOutDir {
  139. os.RemoveAll(tempDir)
  140. }
  141. }()
  142. // If there is more than one command in the manifest use a separate directory for each one.
  143. useSubDir := len(manifest.Commands) > 1
  144. var commandDepFiles []string
  145. for i, command := range manifest.Commands {
  146. localTempDir := tempDir
  147. if useSubDir {
  148. localTempDir = filepath.Join(localTempDir, strconv.Itoa(i))
  149. }
  150. depFile, err := runCommand(command, localTempDir, i)
  151. if err != nil {
  152. // Running the command failed, keep the temporary output directory around in
  153. // case a user wants to inspect it for debugging purposes. Soong will delete
  154. // it at the beginning of the next build anyway.
  155. keepOutDir = true
  156. return err
  157. }
  158. if depFile != "" {
  159. commandDepFiles = append(commandDepFiles, depFile)
  160. }
  161. }
  162. outputDepFile := manifest.GetOutputDepfile()
  163. if len(commandDepFiles) > 0 && outputDepFile == "" {
  164. return fmt.Errorf("Sandboxed commands used %s but output depfile is not set in manifest file",
  165. depFilePlaceholder)
  166. }
  167. if outputDepFile != "" {
  168. // Merge the depfiles from each command in the manifest to a single output depfile.
  169. err = rewriteDepFiles(commandDepFiles, outputDepFile)
  170. if err != nil {
  171. return fmt.Errorf("failed merging depfiles: %w", err)
  172. }
  173. }
  174. return nil
  175. }
  176. // createCommandScript will create and return an exec.Cmd that runs rawCommand.
  177. //
  178. // rawCommand is executed via a script in the sandbox.
  179. // scriptPath is the temporary where the script is created.
  180. // scriptPathInSandbox is the path to the script in the sbox environment.
  181. //
  182. // returns an exec.Cmd that can be ran from within sbox context if no error, or nil if error.
  183. // caller must ensure script is cleaned up if function succeeds.
  184. func createCommandScript(rawCommand, scriptPath, scriptPathInSandbox string) (*exec.Cmd, error) {
  185. err := os.WriteFile(scriptPath, []byte(rawCommand), 0644)
  186. if err != nil {
  187. return nil, fmt.Errorf("failed to write command %s... to %s",
  188. rawCommand[0:40], scriptPath)
  189. }
  190. return exec.Command("bash", scriptPathInSandbox), nil
  191. }
  192. // readManifest reads an sbox manifest from a textproto file.
  193. func readManifest(file string) (*sbox_proto.Manifest, error) {
  194. manifestData, err := ioutil.ReadFile(file)
  195. if err != nil {
  196. return nil, fmt.Errorf("error reading manifest %q: %w", file, err)
  197. }
  198. manifest := sbox_proto.Manifest{}
  199. err = prototext.Unmarshal(manifestData, &manifest)
  200. if err != nil {
  201. return nil, fmt.Errorf("error parsing manifest %q: %w", file, err)
  202. }
  203. return &manifest, nil
  204. }
  205. // runCommand runs a single command from a manifest. If the command references the
  206. // __SBOX_DEPFILE__ placeholder it returns the name of the depfile that was used.
  207. func runCommand(command *sbox_proto.Command, tempDir string, commandIndex int) (depFile string, err error) {
  208. rawCommand := command.GetCommand()
  209. if rawCommand == "" {
  210. return "", fmt.Errorf("command is required")
  211. }
  212. // Remove files from the output directory
  213. err = clearOutputDirectory(command.CopyAfter, outputDir, writeType(writeIfChanged))
  214. if err != nil {
  215. return "", err
  216. }
  217. pathToTempDirInSbox := tempDir
  218. if command.GetChdir() {
  219. pathToTempDirInSbox = "."
  220. }
  221. err = os.MkdirAll(tempDir, 0777)
  222. if err != nil {
  223. return "", fmt.Errorf("failed to create %q: %w", tempDir, err)
  224. }
  225. // Copy in any files specified by the manifest.
  226. err = copyFiles(command.CopyBefore, "", tempDir, requireFromExists, alwaysWrite)
  227. if err != nil {
  228. return "", err
  229. }
  230. err = copyRspFiles(command.RspFiles, tempDir, pathToTempDirInSbox)
  231. if err != nil {
  232. return "", err
  233. }
  234. if strings.Contains(rawCommand, depFilePlaceholder) {
  235. depFile = filepath.Join(pathToTempDirInSbox, "deps.d")
  236. rawCommand = strings.Replace(rawCommand, depFilePlaceholder, depFile, -1)
  237. }
  238. if strings.Contains(rawCommand, sandboxDirPlaceholder) {
  239. rawCommand = strings.Replace(rawCommand, sandboxDirPlaceholder, pathToTempDirInSbox, -1)
  240. }
  241. // Emulate ninja's behavior of creating the directories for any output files before
  242. // running the command.
  243. err = makeOutputDirs(command.CopyAfter, tempDir)
  244. if err != nil {
  245. return "", err
  246. }
  247. scriptName := fmt.Sprintf("sbox_command.%d.bash", commandIndex)
  248. scriptPath := joinPath(tempDir, scriptName)
  249. scriptPathInSandbox := joinPath(pathToTempDirInSbox, scriptName)
  250. cmd, err := createCommandScript(rawCommand, scriptPath, scriptPathInSandbox)
  251. if err != nil {
  252. return "", err
  253. }
  254. buf := &bytes.Buffer{}
  255. cmd.Stdin = os.Stdin
  256. cmd.Stdout = buf
  257. cmd.Stderr = buf
  258. if command.GetChdir() {
  259. cmd.Dir = tempDir
  260. path := os.Getenv("PATH")
  261. absPath, err := makeAbsPathEnv(path)
  262. if err != nil {
  263. return "", err
  264. }
  265. err = os.Setenv("PATH", absPath)
  266. if err != nil {
  267. return "", fmt.Errorf("Failed to update PATH: %w", err)
  268. }
  269. }
  270. err = cmd.Run()
  271. if err != nil {
  272. // The command failed, do a best effort copy of output files out of the sandbox. This is
  273. // especially useful for linters with baselines that print an error message on failure
  274. // with a command to copy the output lint errors to the new baseline. Use a copy instead of
  275. // a move to leave the sandbox intact for manual inspection
  276. copyFiles(command.CopyAfter, tempDir, "", allowFromNotExists, writeType(writeIfChanged))
  277. }
  278. // If the command was executed but failed with an error, print a debugging message before
  279. // the command's output so it doesn't scroll the real error message off the screen.
  280. if exit, ok := err.(*exec.ExitError); ok && !exit.Success() {
  281. fmt.Fprintf(os.Stderr,
  282. "The failing command was run inside an sbox sandbox in temporary directory\n"+
  283. "%s\n"+
  284. "The failing command line can be found in\n"+
  285. "%s\n",
  286. tempDir, scriptPath)
  287. }
  288. // Write the command's combined stdout/stderr.
  289. os.Stdout.Write(buf.Bytes())
  290. if err != nil {
  291. return "", err
  292. }
  293. err = validateOutputFiles(command.CopyAfter, tempDir, outputDir, rawCommand)
  294. if err != nil {
  295. return "", err
  296. }
  297. // the created files match the declared files; now move them
  298. err = moveFiles(command.CopyAfter, tempDir, "", writeType(writeIfChanged))
  299. if err != nil {
  300. return "", err
  301. }
  302. return depFile, nil
  303. }
  304. // makeOutputDirs creates directories in the sandbox dir for every file that has a rule to be copied
  305. // out of the sandbox. This emulate's Ninja's behavior of creating directories for output files
  306. // so that the tools don't have to.
  307. func makeOutputDirs(copies []*sbox_proto.Copy, sandboxDir string) error {
  308. for _, copyPair := range copies {
  309. dir := joinPath(sandboxDir, filepath.Dir(copyPair.GetFrom()))
  310. err := os.MkdirAll(dir, 0777)
  311. if err != nil {
  312. return err
  313. }
  314. }
  315. return nil
  316. }
  317. // validateOutputFiles verifies that all files that have a rule to be copied out of the sandbox
  318. // were created by the command.
  319. func validateOutputFiles(copies []*sbox_proto.Copy, sandboxDir, outputDir, rawCommand string) error {
  320. var missingOutputErrors []error
  321. var incorrectOutputDirectoryErrors []error
  322. for _, copyPair := range copies {
  323. fromPath := joinPath(sandboxDir, copyPair.GetFrom())
  324. fileInfo, err := os.Stat(fromPath)
  325. if err != nil {
  326. missingOutputErrors = append(missingOutputErrors, fmt.Errorf("%s: does not exist", fromPath))
  327. continue
  328. }
  329. if fileInfo.IsDir() {
  330. missingOutputErrors = append(missingOutputErrors, fmt.Errorf("%s: not a file", fromPath))
  331. }
  332. toPath := copyPair.GetTo()
  333. if rel, err := filepath.Rel(outputDir, toPath); err != nil {
  334. return err
  335. } else if strings.HasPrefix(rel, "../") {
  336. incorrectOutputDirectoryErrors = append(incorrectOutputDirectoryErrors,
  337. fmt.Errorf("%s is not under %s", toPath, outputDir))
  338. }
  339. }
  340. const maxErrors = 10
  341. if len(incorrectOutputDirectoryErrors) > 0 {
  342. errorMessage := ""
  343. more := 0
  344. if len(incorrectOutputDirectoryErrors) > maxErrors {
  345. more = len(incorrectOutputDirectoryErrors) - maxErrors
  346. incorrectOutputDirectoryErrors = incorrectOutputDirectoryErrors[:maxErrors]
  347. }
  348. for _, err := range incorrectOutputDirectoryErrors {
  349. errorMessage += err.Error() + "\n"
  350. }
  351. if more > 0 {
  352. errorMessage += fmt.Sprintf("...%v more", more)
  353. }
  354. return errors.New(errorMessage)
  355. }
  356. if len(missingOutputErrors) > 0 {
  357. // find all created files for making a more informative error message
  358. createdFiles := findAllFilesUnder(sandboxDir)
  359. // build error message
  360. errorMessage := "mismatch between declared and actual outputs\n"
  361. errorMessage += "in sbox command(" + rawCommand + ")\n\n"
  362. errorMessage += "in sandbox " + sandboxDir + ",\n"
  363. errorMessage += fmt.Sprintf("failed to create %v files:\n", len(missingOutputErrors))
  364. for _, missingOutputError := range missingOutputErrors {
  365. errorMessage += " " + missingOutputError.Error() + "\n"
  366. }
  367. if len(createdFiles) < 1 {
  368. errorMessage += "created 0 files."
  369. } else {
  370. errorMessage += fmt.Sprintf("did create %v files:\n", len(createdFiles))
  371. creationMessages := createdFiles
  372. if len(creationMessages) > maxErrors {
  373. creationMessages = creationMessages[:maxErrors]
  374. creationMessages = append(creationMessages, fmt.Sprintf("...%v more", len(createdFiles)-maxErrors))
  375. }
  376. for _, creationMessage := range creationMessages {
  377. errorMessage += " " + creationMessage + "\n"
  378. }
  379. }
  380. return errors.New(errorMessage)
  381. }
  382. return nil
  383. }
  384. type existsType bool
  385. const (
  386. requireFromExists existsType = false
  387. allowFromNotExists = true
  388. )
  389. type writeType bool
  390. const (
  391. alwaysWrite writeType = false
  392. onlyWriteIfChanged = true
  393. )
  394. // copyFiles copies files in or out of the sandbox. If exists is allowFromNotExists then errors
  395. // caused by a from path not existing are ignored. If write is onlyWriteIfChanged then the output
  396. // file is compared to the input file and not written to if it is the same, avoiding updating
  397. // the timestamp.
  398. func copyFiles(copies []*sbox_proto.Copy, fromDir, toDir string, exists existsType, write writeType) error {
  399. for _, copyPair := range copies {
  400. fromPath := joinPath(fromDir, copyPair.GetFrom())
  401. toPath := joinPath(toDir, copyPair.GetTo())
  402. err := copyOneFile(fromPath, toPath, copyPair.GetExecutable(), exists, write)
  403. if err != nil {
  404. return fmt.Errorf("error copying %q to %q: %w", fromPath, toPath, err)
  405. }
  406. }
  407. return nil
  408. }
  409. // copyOneFile copies a file and its permissions. If forceExecutable is true it adds u+x to the
  410. // permissions. If exists is allowFromNotExists it returns nil if the from path doesn't exist.
  411. // If write is onlyWriteIfChanged then the output file is compared to the input file and not written to
  412. // if it is the same, avoiding updating the timestamp.
  413. func copyOneFile(from string, to string, forceExecutable bool, exists existsType,
  414. write writeType) error {
  415. err := os.MkdirAll(filepath.Dir(to), 0777)
  416. if err != nil {
  417. return err
  418. }
  419. stat, err := os.Stat(from)
  420. if err != nil {
  421. if os.IsNotExist(err) && exists == allowFromNotExists {
  422. return nil
  423. }
  424. return err
  425. }
  426. perm := stat.Mode()
  427. if forceExecutable {
  428. perm = perm | 0100 // u+x
  429. }
  430. if write == onlyWriteIfChanged && filesHaveSameContents(from, to) {
  431. return nil
  432. }
  433. in, err := os.Open(from)
  434. if err != nil {
  435. return err
  436. }
  437. defer in.Close()
  438. // Remove the target before copying. In most cases the file won't exist, but if there are
  439. // duplicate copy rules for a file and the source file was read-only the second copy could
  440. // fail.
  441. err = os.Remove(to)
  442. if err != nil && !os.IsNotExist(err) {
  443. return err
  444. }
  445. out, err := os.Create(to)
  446. if err != nil {
  447. return err
  448. }
  449. defer func() {
  450. out.Close()
  451. if err != nil {
  452. os.Remove(to)
  453. }
  454. }()
  455. _, err = io.Copy(out, in)
  456. if err != nil {
  457. return err
  458. }
  459. if err = out.Close(); err != nil {
  460. return err
  461. }
  462. if err = os.Chmod(to, perm); err != nil {
  463. return err
  464. }
  465. return nil
  466. }
  467. // copyRspFiles copies rsp files into the sandbox with path mappings, and also copies the files
  468. // listed into the sandbox.
  469. func copyRspFiles(rspFiles []*sbox_proto.RspFile, toDir, toDirInSandbox string) error {
  470. for _, rspFile := range rspFiles {
  471. err := copyOneRspFile(rspFile, toDir, toDirInSandbox)
  472. if err != nil {
  473. return err
  474. }
  475. }
  476. return nil
  477. }
  478. // copyOneRspFiles copies an rsp file into the sandbox with path mappings, and also copies the files
  479. // listed into the sandbox.
  480. func copyOneRspFile(rspFile *sbox_proto.RspFile, toDir, toDirInSandbox string) error {
  481. in, err := os.Open(rspFile.GetFile())
  482. if err != nil {
  483. return err
  484. }
  485. defer in.Close()
  486. files, err := response.ReadRspFile(in)
  487. if err != nil {
  488. return err
  489. }
  490. for i, from := range files {
  491. // Convert the real path of the input file into the path inside the sandbox using the
  492. // path mappings.
  493. to := applyPathMappings(rspFile.PathMappings, from)
  494. // Copy the file into the sandbox.
  495. err := copyOneFile(from, joinPath(toDir, to), false, requireFromExists, alwaysWrite)
  496. if err != nil {
  497. return err
  498. }
  499. // Rewrite the name in the list of files to be relative to the sandbox directory.
  500. files[i] = joinPath(toDirInSandbox, to)
  501. }
  502. // Convert the real path of the rsp file into the path inside the sandbox using the path
  503. // mappings.
  504. outRspFile := joinPath(toDir, applyPathMappings(rspFile.PathMappings, rspFile.GetFile()))
  505. err = os.MkdirAll(filepath.Dir(outRspFile), 0777)
  506. if err != nil {
  507. return err
  508. }
  509. out, err := os.Create(outRspFile)
  510. if err != nil {
  511. return err
  512. }
  513. defer out.Close()
  514. // Write the rsp file with converted paths into the sandbox.
  515. err = response.WriteRspFile(out, files)
  516. if err != nil {
  517. return err
  518. }
  519. return nil
  520. }
  521. // applyPathMappings takes a list of path mappings and a path, and returns the path with the first
  522. // matching path mapping applied. If the path does not match any of the path mappings then it is
  523. // returned unmodified.
  524. func applyPathMappings(pathMappings []*sbox_proto.PathMapping, path string) string {
  525. for _, mapping := range pathMappings {
  526. if strings.HasPrefix(path, mapping.GetFrom()+"/") {
  527. return joinPath(mapping.GetTo()+"/", strings.TrimPrefix(path, mapping.GetFrom()+"/"))
  528. }
  529. }
  530. return path
  531. }
  532. // moveFiles moves files specified by a set of copy rules. It uses os.Rename, so it is restricted
  533. // to moving files where the source and destination are in the same filesystem. This is OK for
  534. // sbox because the temporary directory is inside the out directory. If write is onlyWriteIfChanged
  535. // then the output file is compared to the input file and not written to if it is the same, avoiding
  536. // updating the timestamp. Otherwise it always updates the timestamp of the new file.
  537. func moveFiles(copies []*sbox_proto.Copy, fromDir, toDir string, write writeType) error {
  538. for _, copyPair := range copies {
  539. fromPath := joinPath(fromDir, copyPair.GetFrom())
  540. toPath := joinPath(toDir, copyPair.GetTo())
  541. err := os.MkdirAll(filepath.Dir(toPath), 0777)
  542. if err != nil {
  543. return err
  544. }
  545. if write == onlyWriteIfChanged && filesHaveSameContents(fromPath, toPath) {
  546. continue
  547. }
  548. err = os.Rename(fromPath, toPath)
  549. if err != nil {
  550. return err
  551. }
  552. // Update the timestamp of the output file in case the tool wrote an old timestamp (for example, tar can extract
  553. // files with old timestamps).
  554. now := time.Now()
  555. err = os.Chtimes(toPath, now, now)
  556. if err != nil {
  557. return err
  558. }
  559. }
  560. return nil
  561. }
  562. // clearOutputDirectory removes all files in the output directory if write is alwaysWrite, or
  563. // any files not listed in copies if write is onlyWriteIfChanged
  564. func clearOutputDirectory(copies []*sbox_proto.Copy, outputDir string, write writeType) error {
  565. if outputDir == "" {
  566. return fmt.Errorf("output directory must be set")
  567. }
  568. if write == alwaysWrite {
  569. // When writing all the output files remove the whole output directory
  570. return os.RemoveAll(outputDir)
  571. }
  572. outputFiles := make(map[string]bool, len(copies))
  573. for _, copyPair := range copies {
  574. outputFiles[copyPair.GetTo()] = true
  575. }
  576. existingFiles := findAllFilesUnder(outputDir)
  577. for _, existingFile := range existingFiles {
  578. fullExistingFile := filepath.Join(outputDir, existingFile)
  579. if !outputFiles[fullExistingFile] {
  580. err := os.Remove(fullExistingFile)
  581. if err != nil {
  582. return fmt.Errorf("failed to remove obsolete output file %s: %w", fullExistingFile, err)
  583. }
  584. }
  585. }
  586. return nil
  587. }
  588. // Rewrite one or more depfiles so that it doesn't include the (randomized) sandbox directory
  589. // to an output file.
  590. func rewriteDepFiles(ins []string, out string) error {
  591. var mergedDeps []string
  592. for _, in := range ins {
  593. data, err := ioutil.ReadFile(in)
  594. if err != nil {
  595. return err
  596. }
  597. deps, err := makedeps.Parse(in, bytes.NewBuffer(data))
  598. if err != nil {
  599. return err
  600. }
  601. mergedDeps = append(mergedDeps, deps.Inputs...)
  602. }
  603. deps := makedeps.Deps{
  604. // Ninja doesn't care what the output file is, so we can use any string here.
  605. Output: "outputfile",
  606. Inputs: mergedDeps,
  607. }
  608. // Make the directory for the output depfile in case it is in a different directory
  609. // than any of the output files.
  610. outDir := filepath.Dir(out)
  611. err := os.MkdirAll(outDir, 0777)
  612. if err != nil {
  613. return fmt.Errorf("failed to create %q: %w", outDir, err)
  614. }
  615. return ioutil.WriteFile(out, deps.Print(), 0666)
  616. }
  617. // joinPath wraps filepath.Join but returns file without appending to dir if file is
  618. // absolute.
  619. func joinPath(dir, file string) string {
  620. if filepath.IsAbs(file) {
  621. return file
  622. }
  623. return filepath.Join(dir, file)
  624. }
  625. // filesHaveSameContents compares the contents if two files, returning true if they are the same
  626. // and returning false if they are different or any errors occur.
  627. func filesHaveSameContents(a, b string) bool {
  628. // Compare the sizes of the two files
  629. statA, err := os.Stat(a)
  630. if err != nil {
  631. return false
  632. }
  633. statB, err := os.Stat(b)
  634. if err != nil {
  635. return false
  636. }
  637. if statA.Size() != statB.Size() {
  638. return false
  639. }
  640. // Open the two files
  641. fileA, err := os.Open(a)
  642. if err != nil {
  643. return false
  644. }
  645. defer fileA.Close()
  646. fileB, err := os.Open(b)
  647. if err != nil {
  648. return false
  649. }
  650. defer fileB.Close()
  651. // Compare the files 1MB at a time
  652. const bufSize = 1 * 1024 * 1024
  653. bufA := make([]byte, bufSize)
  654. bufB := make([]byte, bufSize)
  655. remain := statA.Size()
  656. for remain > 0 {
  657. toRead := int64(bufSize)
  658. if toRead > remain {
  659. toRead = remain
  660. }
  661. _, err = io.ReadFull(fileA, bufA[:toRead])
  662. if err != nil {
  663. return false
  664. }
  665. _, err = io.ReadFull(fileB, bufB[:toRead])
  666. if err != nil {
  667. return false
  668. }
  669. if bytes.Compare(bufA[:toRead], bufB[:toRead]) != 0 {
  670. return false
  671. }
  672. remain -= toRead
  673. }
  674. return true
  675. }
  676. func makeAbsPathEnv(pathEnv string) (string, error) {
  677. pathEnvElements := filepath.SplitList(pathEnv)
  678. for i, p := range pathEnvElements {
  679. if !filepath.IsAbs(p) {
  680. absPath, err := filepath.Abs(p)
  681. if err != nil {
  682. return "", fmt.Errorf("failed to make PATH entry %q absolute: %w", p, err)
  683. }
  684. pathEnvElements[i] = absPath
  685. }
  686. }
  687. return strings.Join(pathEnvElements, string(filepath.ListSeparator)), nil
  688. }