sbox.go 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578
  1. // Copyright 2017 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package main
  15. import (
  16. "bytes"
  17. "crypto/sha1"
  18. "encoding/hex"
  19. "errors"
  20. "flag"
  21. "fmt"
  22. "io"
  23. "io/ioutil"
  24. "os"
  25. "os/exec"
  26. "path/filepath"
  27. "strconv"
  28. "strings"
  29. "time"
  30. "android/soong/cmd/sbox/sbox_proto"
  31. "android/soong/makedeps"
  32. "android/soong/response"
  33. "github.com/golang/protobuf/proto"
  34. )
  35. var (
  36. sandboxesRoot string
  37. manifestFile string
  38. keepOutDir bool
  39. )
  40. const (
  41. depFilePlaceholder = "__SBOX_DEPFILE__"
  42. sandboxDirPlaceholder = "__SBOX_SANDBOX_DIR__"
  43. )
  44. func init() {
  45. flag.StringVar(&sandboxesRoot, "sandbox-path", "",
  46. "root of temp directory to put the sandbox into")
  47. flag.StringVar(&manifestFile, "manifest", "",
  48. "textproto manifest describing the sandboxed command(s)")
  49. flag.BoolVar(&keepOutDir, "keep-out-dir", false,
  50. "whether to keep the sandbox directory when done")
  51. }
  52. func usageViolation(violation string) {
  53. if violation != "" {
  54. fmt.Fprintf(os.Stderr, "Usage error: %s.\n\n", violation)
  55. }
  56. fmt.Fprintf(os.Stderr,
  57. "Usage: sbox --manifest <manifest> --sandbox-path <sandboxPath>\n")
  58. flag.PrintDefaults()
  59. os.Exit(1)
  60. }
  61. func main() {
  62. flag.Usage = func() {
  63. usageViolation("")
  64. }
  65. flag.Parse()
  66. error := run()
  67. if error != nil {
  68. fmt.Fprintln(os.Stderr, error)
  69. os.Exit(1)
  70. }
  71. }
  72. func findAllFilesUnder(root string) (paths []string) {
  73. paths = []string{}
  74. filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
  75. if !info.IsDir() {
  76. relPath, err := filepath.Rel(root, path)
  77. if err != nil {
  78. // couldn't find relative path from ancestor?
  79. panic(err)
  80. }
  81. paths = append(paths, relPath)
  82. }
  83. return nil
  84. })
  85. return paths
  86. }
  87. func run() error {
  88. if manifestFile == "" {
  89. usageViolation("--manifest <manifest> is required and must be non-empty")
  90. }
  91. if sandboxesRoot == "" {
  92. // In practice, the value of sandboxesRoot will mostly likely be at a fixed location relative to OUT_DIR,
  93. // and the sbox executable will most likely be at a fixed location relative to OUT_DIR too, so
  94. // the value of sandboxesRoot will most likely be at a fixed location relative to the sbox executable
  95. // However, Soong also needs to be able to separately remove the sandbox directory on startup (if it has anything left in it)
  96. // and by passing it as a parameter we don't need to duplicate its value
  97. usageViolation("--sandbox-path <sandboxPath> is required and must be non-empty")
  98. }
  99. manifest, err := readManifest(manifestFile)
  100. if len(manifest.Commands) == 0 {
  101. return fmt.Errorf("at least one commands entry is required in %q", manifestFile)
  102. }
  103. // setup sandbox directory
  104. err = os.MkdirAll(sandboxesRoot, 0777)
  105. if err != nil {
  106. return fmt.Errorf("failed to create %q: %w", sandboxesRoot, err)
  107. }
  108. // This tool assumes that there are no two concurrent runs with the same
  109. // manifestFile. It should therefore be safe to use the hash of the
  110. // manifestFile as the temporary directory name. We do this because it
  111. // makes the temporary directory name deterministic. There are some
  112. // tools that embed the name of the temporary output in the output, and
  113. // they otherwise cause non-determinism, which then poisons actions
  114. // depending on this one.
  115. hash := sha1.New()
  116. hash.Write([]byte(manifestFile))
  117. tempDir := filepath.Join(sandboxesRoot, "sbox", hex.EncodeToString(hash.Sum(nil)))
  118. err = os.RemoveAll(tempDir)
  119. if err != nil {
  120. return err
  121. }
  122. err = os.MkdirAll(tempDir, 0777)
  123. if err != nil {
  124. return fmt.Errorf("failed to create temporary dir in %q: %w", sandboxesRoot, err)
  125. }
  126. // In the common case, the following line of code is what removes the sandbox
  127. // If a fatal error occurs (such as if our Go process is killed unexpectedly),
  128. // then at the beginning of the next build, Soong will wipe the temporary
  129. // directory.
  130. defer func() {
  131. // in some cases we decline to remove the temp dir, to facilitate debugging
  132. if !keepOutDir {
  133. os.RemoveAll(tempDir)
  134. }
  135. }()
  136. // If there is more than one command in the manifest use a separate directory for each one.
  137. useSubDir := len(manifest.Commands) > 1
  138. var commandDepFiles []string
  139. for i, command := range manifest.Commands {
  140. localTempDir := tempDir
  141. if useSubDir {
  142. localTempDir = filepath.Join(localTempDir, strconv.Itoa(i))
  143. }
  144. depFile, err := runCommand(command, localTempDir)
  145. if err != nil {
  146. // Running the command failed, keep the temporary output directory around in
  147. // case a user wants to inspect it for debugging purposes. Soong will delete
  148. // it at the beginning of the next build anyway.
  149. keepOutDir = true
  150. return err
  151. }
  152. if depFile != "" {
  153. commandDepFiles = append(commandDepFiles, depFile)
  154. }
  155. }
  156. outputDepFile := manifest.GetOutputDepfile()
  157. if len(commandDepFiles) > 0 && outputDepFile == "" {
  158. return fmt.Errorf("Sandboxed commands used %s but output depfile is not set in manifest file",
  159. depFilePlaceholder)
  160. }
  161. if outputDepFile != "" {
  162. // Merge the depfiles from each command in the manifest to a single output depfile.
  163. err = rewriteDepFiles(commandDepFiles, outputDepFile)
  164. if err != nil {
  165. return fmt.Errorf("failed merging depfiles: %w", err)
  166. }
  167. }
  168. return nil
  169. }
  170. // readManifest reads an sbox manifest from a textproto file.
  171. func readManifest(file string) (*sbox_proto.Manifest, error) {
  172. manifestData, err := ioutil.ReadFile(file)
  173. if err != nil {
  174. return nil, fmt.Errorf("error reading manifest %q: %w", file, err)
  175. }
  176. manifest := sbox_proto.Manifest{}
  177. err = proto.UnmarshalText(string(manifestData), &manifest)
  178. if err != nil {
  179. return nil, fmt.Errorf("error parsing manifest %q: %w", file, err)
  180. }
  181. return &manifest, nil
  182. }
  183. // runCommand runs a single command from a manifest. If the command references the
  184. // __SBOX_DEPFILE__ placeholder it returns the name of the depfile that was used.
  185. func runCommand(command *sbox_proto.Command, tempDir string) (depFile string, err error) {
  186. rawCommand := command.GetCommand()
  187. if rawCommand == "" {
  188. return "", fmt.Errorf("command is required")
  189. }
  190. pathToTempDirInSbox := tempDir
  191. if command.GetChdir() {
  192. pathToTempDirInSbox = "."
  193. }
  194. err = os.MkdirAll(tempDir, 0777)
  195. if err != nil {
  196. return "", fmt.Errorf("failed to create %q: %w", tempDir, err)
  197. }
  198. // Copy in any files specified by the manifest.
  199. err = copyFiles(command.CopyBefore, "", tempDir)
  200. if err != nil {
  201. return "", err
  202. }
  203. err = copyRspFiles(command.RspFiles, tempDir, pathToTempDirInSbox)
  204. if err != nil {
  205. return "", err
  206. }
  207. if strings.Contains(rawCommand, depFilePlaceholder) {
  208. depFile = filepath.Join(pathToTempDirInSbox, "deps.d")
  209. rawCommand = strings.Replace(rawCommand, depFilePlaceholder, depFile, -1)
  210. }
  211. if strings.Contains(rawCommand, sandboxDirPlaceholder) {
  212. rawCommand = strings.Replace(rawCommand, sandboxDirPlaceholder, pathToTempDirInSbox, -1)
  213. }
  214. // Emulate ninja's behavior of creating the directories for any output files before
  215. // running the command.
  216. err = makeOutputDirs(command.CopyAfter, tempDir)
  217. if err != nil {
  218. return "", err
  219. }
  220. commandDescription := rawCommand
  221. cmd := exec.Command("bash", "-c", rawCommand)
  222. cmd.Stdin = os.Stdin
  223. cmd.Stdout = os.Stdout
  224. cmd.Stderr = os.Stderr
  225. if command.GetChdir() {
  226. cmd.Dir = tempDir
  227. path := os.Getenv("PATH")
  228. absPath, err := makeAbsPathEnv(path)
  229. if err != nil {
  230. return "", err
  231. }
  232. err = os.Setenv("PATH", absPath)
  233. if err != nil {
  234. return "", fmt.Errorf("Failed to update PATH: %w", err)
  235. }
  236. }
  237. err = cmd.Run()
  238. if exit, ok := err.(*exec.ExitError); ok && !exit.Success() {
  239. return "", fmt.Errorf("sbox command failed with err:\n%s\n%w\n", commandDescription, err)
  240. } else if err != nil {
  241. return "", err
  242. }
  243. missingOutputErrors := validateOutputFiles(command.CopyAfter, tempDir)
  244. if len(missingOutputErrors) > 0 {
  245. // find all created files for making a more informative error message
  246. createdFiles := findAllFilesUnder(tempDir)
  247. // build error message
  248. errorMessage := "mismatch between declared and actual outputs\n"
  249. errorMessage += "in sbox command(" + commandDescription + ")\n\n"
  250. errorMessage += "in sandbox " + tempDir + ",\n"
  251. errorMessage += fmt.Sprintf("failed to create %v files:\n", len(missingOutputErrors))
  252. for _, missingOutputError := range missingOutputErrors {
  253. errorMessage += " " + missingOutputError.Error() + "\n"
  254. }
  255. if len(createdFiles) < 1 {
  256. errorMessage += "created 0 files."
  257. } else {
  258. errorMessage += fmt.Sprintf("did create %v files:\n", len(createdFiles))
  259. creationMessages := createdFiles
  260. maxNumCreationLines := 10
  261. if len(creationMessages) > maxNumCreationLines {
  262. creationMessages = creationMessages[:maxNumCreationLines]
  263. creationMessages = append(creationMessages, fmt.Sprintf("...%v more", len(createdFiles)-maxNumCreationLines))
  264. }
  265. for _, creationMessage := range creationMessages {
  266. errorMessage += " " + creationMessage + "\n"
  267. }
  268. }
  269. return "", errors.New(errorMessage)
  270. }
  271. // the created files match the declared files; now move them
  272. err = moveFiles(command.CopyAfter, tempDir, "")
  273. return depFile, nil
  274. }
  275. // makeOutputDirs creates directories in the sandbox dir for every file that has a rule to be copied
  276. // out of the sandbox. This emulate's Ninja's behavior of creating directories for output files
  277. // so that the tools don't have to.
  278. func makeOutputDirs(copies []*sbox_proto.Copy, sandboxDir string) error {
  279. for _, copyPair := range copies {
  280. dir := joinPath(sandboxDir, filepath.Dir(copyPair.GetFrom()))
  281. err := os.MkdirAll(dir, 0777)
  282. if err != nil {
  283. return err
  284. }
  285. }
  286. return nil
  287. }
  288. // validateOutputFiles verifies that all files that have a rule to be copied out of the sandbox
  289. // were created by the command.
  290. func validateOutputFiles(copies []*sbox_proto.Copy, sandboxDir string) []error {
  291. var missingOutputErrors []error
  292. for _, copyPair := range copies {
  293. fromPath := joinPath(sandboxDir, copyPair.GetFrom())
  294. fileInfo, err := os.Stat(fromPath)
  295. if err != nil {
  296. missingOutputErrors = append(missingOutputErrors, fmt.Errorf("%s: does not exist", fromPath))
  297. continue
  298. }
  299. if fileInfo.IsDir() {
  300. missingOutputErrors = append(missingOutputErrors, fmt.Errorf("%s: not a file", fromPath))
  301. }
  302. }
  303. return missingOutputErrors
  304. }
  305. // copyFiles copies files in or out of the sandbox.
  306. func copyFiles(copies []*sbox_proto.Copy, fromDir, toDir string) error {
  307. for _, copyPair := range copies {
  308. fromPath := joinPath(fromDir, copyPair.GetFrom())
  309. toPath := joinPath(toDir, copyPair.GetTo())
  310. err := copyOneFile(fromPath, toPath, copyPair.GetExecutable())
  311. if err != nil {
  312. return fmt.Errorf("error copying %q to %q: %w", fromPath, toPath, err)
  313. }
  314. }
  315. return nil
  316. }
  317. // copyOneFile copies a file.
  318. func copyOneFile(from string, to string, executable bool) error {
  319. err := os.MkdirAll(filepath.Dir(to), 0777)
  320. if err != nil {
  321. return err
  322. }
  323. stat, err := os.Stat(from)
  324. if err != nil {
  325. return err
  326. }
  327. perm := stat.Mode()
  328. if executable {
  329. perm = perm | 0100 // u+x
  330. }
  331. in, err := os.Open(from)
  332. if err != nil {
  333. return err
  334. }
  335. defer in.Close()
  336. out, err := os.Create(to)
  337. if err != nil {
  338. return err
  339. }
  340. defer func() {
  341. out.Close()
  342. if err != nil {
  343. os.Remove(to)
  344. }
  345. }()
  346. _, err = io.Copy(out, in)
  347. if err != nil {
  348. return err
  349. }
  350. if err = out.Close(); err != nil {
  351. return err
  352. }
  353. if err = os.Chmod(to, perm); err != nil {
  354. return err
  355. }
  356. return nil
  357. }
  358. // copyRspFiles copies rsp files into the sandbox with path mappings, and also copies the files
  359. // listed into the sandbox.
  360. func copyRspFiles(rspFiles []*sbox_proto.RspFile, toDir, toDirInSandbox string) error {
  361. for _, rspFile := range rspFiles {
  362. err := copyOneRspFile(rspFile, toDir, toDirInSandbox)
  363. if err != nil {
  364. return err
  365. }
  366. }
  367. return nil
  368. }
  369. // copyOneRspFiles copies an rsp file into the sandbox with path mappings, and also copies the files
  370. // listed into the sandbox.
  371. func copyOneRspFile(rspFile *sbox_proto.RspFile, toDir, toDirInSandbox string) error {
  372. in, err := os.Open(rspFile.GetFile())
  373. if err != nil {
  374. return err
  375. }
  376. defer in.Close()
  377. files, err := response.ReadRspFile(in)
  378. if err != nil {
  379. return err
  380. }
  381. for i, from := range files {
  382. // Convert the real path of the input file into the path inside the sandbox using the
  383. // path mappings.
  384. to := applyPathMappings(rspFile.PathMappings, from)
  385. // Copy the file into the sandbox.
  386. err := copyOneFile(from, joinPath(toDir, to), false)
  387. if err != nil {
  388. return err
  389. }
  390. // Rewrite the name in the list of files to be relative to the sandbox directory.
  391. files[i] = joinPath(toDirInSandbox, to)
  392. }
  393. // Convert the real path of the rsp file into the path inside the sandbox using the path
  394. // mappings.
  395. outRspFile := joinPath(toDir, applyPathMappings(rspFile.PathMappings, rspFile.GetFile()))
  396. err = os.MkdirAll(filepath.Dir(outRspFile), 0777)
  397. if err != nil {
  398. return err
  399. }
  400. out, err := os.Create(outRspFile)
  401. if err != nil {
  402. return err
  403. }
  404. defer out.Close()
  405. // Write the rsp file with converted paths into the sandbox.
  406. err = response.WriteRspFile(out, files)
  407. if err != nil {
  408. return err
  409. }
  410. return nil
  411. }
  412. // applyPathMappings takes a list of path mappings and a path, and returns the path with the first
  413. // matching path mapping applied. If the path does not match any of the path mappings then it is
  414. // returned unmodified.
  415. func applyPathMappings(pathMappings []*sbox_proto.PathMapping, path string) string {
  416. for _, mapping := range pathMappings {
  417. if strings.HasPrefix(path, mapping.GetFrom()+"/") {
  418. return joinPath(mapping.GetTo()+"/", strings.TrimPrefix(path, mapping.GetFrom()+"/"))
  419. }
  420. }
  421. return path
  422. }
  423. // moveFiles moves files specified by a set of copy rules. It uses os.Rename, so it is restricted
  424. // to moving files where the source and destination are in the same filesystem. This is OK for
  425. // sbox because the temporary directory is inside the out directory. It updates the timestamp
  426. // of the new file.
  427. func moveFiles(copies []*sbox_proto.Copy, fromDir, toDir string) error {
  428. for _, copyPair := range copies {
  429. fromPath := joinPath(fromDir, copyPair.GetFrom())
  430. toPath := joinPath(toDir, copyPair.GetTo())
  431. err := os.MkdirAll(filepath.Dir(toPath), 0777)
  432. if err != nil {
  433. return err
  434. }
  435. err = os.Rename(fromPath, toPath)
  436. if err != nil {
  437. return err
  438. }
  439. // Update the timestamp of the output file in case the tool wrote an old timestamp (for example, tar can extract
  440. // files with old timestamps).
  441. now := time.Now()
  442. err = os.Chtimes(toPath, now, now)
  443. if err != nil {
  444. return err
  445. }
  446. }
  447. return nil
  448. }
  449. // Rewrite one or more depfiles so that it doesn't include the (randomized) sandbox directory
  450. // to an output file.
  451. func rewriteDepFiles(ins []string, out string) error {
  452. var mergedDeps []string
  453. for _, in := range ins {
  454. data, err := ioutil.ReadFile(in)
  455. if err != nil {
  456. return err
  457. }
  458. deps, err := makedeps.Parse(in, bytes.NewBuffer(data))
  459. if err != nil {
  460. return err
  461. }
  462. mergedDeps = append(mergedDeps, deps.Inputs...)
  463. }
  464. deps := makedeps.Deps{
  465. // Ninja doesn't care what the output file is, so we can use any string here.
  466. Output: "outputfile",
  467. Inputs: mergedDeps,
  468. }
  469. // Make the directory for the output depfile in case it is in a different directory
  470. // than any of the output files.
  471. outDir := filepath.Dir(out)
  472. err := os.MkdirAll(outDir, 0777)
  473. if err != nil {
  474. return fmt.Errorf("failed to create %q: %w", outDir, err)
  475. }
  476. return ioutil.WriteFile(out, deps.Print(), 0666)
  477. }
  478. // joinPath wraps filepath.Join but returns file without appending to dir if file is
  479. // absolute.
  480. func joinPath(dir, file string) string {
  481. if filepath.IsAbs(file) {
  482. return file
  483. }
  484. return filepath.Join(dir, file)
  485. }
  486. func makeAbsPathEnv(pathEnv string) (string, error) {
  487. pathEnvElements := filepath.SplitList(pathEnv)
  488. for i, p := range pathEnvElements {
  489. if !filepath.IsAbs(p) {
  490. absPath, err := filepath.Abs(p)
  491. if err != nil {
  492. return "", fmt.Errorf("failed to make PATH entry %q absolute: %w", p, err)
  493. }
  494. pathEnvElements[i] = absPath
  495. }
  496. }
  497. return strings.Join(pathEnvElements, string(filepath.ListSeparator)), nil
  498. }