123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578 |
- // Copyright 2017 Google Inc. All rights reserved.
- //
- // Licensed under the Apache License, Version 2.0 (the "License");
- // you may not use this file except in compliance with the License.
- // You may obtain a copy of the License at
- //
- // http://www.apache.org/licenses/LICENSE-2.0
- //
- // Unless required by applicable law or agreed to in writing, software
- // distributed under the License is distributed on an "AS IS" BASIS,
- // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- // See the License for the specific language governing permissions and
- // limitations under the License.
- package main
- import (
- "bytes"
- "crypto/sha1"
- "encoding/hex"
- "errors"
- "flag"
- "fmt"
- "io"
- "io/ioutil"
- "os"
- "os/exec"
- "path/filepath"
- "strconv"
- "strings"
- "time"
- "android/soong/cmd/sbox/sbox_proto"
- "android/soong/makedeps"
- "android/soong/response"
- "github.com/golang/protobuf/proto"
- )
- var (
- sandboxesRoot string
- manifestFile string
- keepOutDir bool
- )
- const (
- depFilePlaceholder = "__SBOX_DEPFILE__"
- sandboxDirPlaceholder = "__SBOX_SANDBOX_DIR__"
- )
- func init() {
- flag.StringVar(&sandboxesRoot, "sandbox-path", "",
- "root of temp directory to put the sandbox into")
- flag.StringVar(&manifestFile, "manifest", "",
- "textproto manifest describing the sandboxed command(s)")
- flag.BoolVar(&keepOutDir, "keep-out-dir", false,
- "whether to keep the sandbox directory when done")
- }
- func usageViolation(violation string) {
- if violation != "" {
- fmt.Fprintf(os.Stderr, "Usage error: %s.\n\n", violation)
- }
- fmt.Fprintf(os.Stderr,
- "Usage: sbox --manifest <manifest> --sandbox-path <sandboxPath>\n")
- flag.PrintDefaults()
- os.Exit(1)
- }
- func main() {
- flag.Usage = func() {
- usageViolation("")
- }
- flag.Parse()
- error := run()
- if error != nil {
- fmt.Fprintln(os.Stderr, error)
- os.Exit(1)
- }
- }
- func findAllFilesUnder(root string) (paths []string) {
- paths = []string{}
- filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
- if !info.IsDir() {
- relPath, err := filepath.Rel(root, path)
- if err != nil {
- // couldn't find relative path from ancestor?
- panic(err)
- }
- paths = append(paths, relPath)
- }
- return nil
- })
- return paths
- }
- func run() error {
- if manifestFile == "" {
- usageViolation("--manifest <manifest> is required and must be non-empty")
- }
- if sandboxesRoot == "" {
- // In practice, the value of sandboxesRoot will mostly likely be at a fixed location relative to OUT_DIR,
- // and the sbox executable will most likely be at a fixed location relative to OUT_DIR too, so
- // the value of sandboxesRoot will most likely be at a fixed location relative to the sbox executable
- // However, Soong also needs to be able to separately remove the sandbox directory on startup (if it has anything left in it)
- // and by passing it as a parameter we don't need to duplicate its value
- usageViolation("--sandbox-path <sandboxPath> is required and must be non-empty")
- }
- manifest, err := readManifest(manifestFile)
- if len(manifest.Commands) == 0 {
- return fmt.Errorf("at least one commands entry is required in %q", manifestFile)
- }
- // setup sandbox directory
- err = os.MkdirAll(sandboxesRoot, 0777)
- if err != nil {
- return fmt.Errorf("failed to create %q: %w", sandboxesRoot, err)
- }
- // This tool assumes that there are no two concurrent runs with the same
- // manifestFile. It should therefore be safe to use the hash of the
- // manifestFile as the temporary directory name. We do this because it
- // makes the temporary directory name deterministic. There are some
- // tools that embed the name of the temporary output in the output, and
- // they otherwise cause non-determinism, which then poisons actions
- // depending on this one.
- hash := sha1.New()
- hash.Write([]byte(manifestFile))
- tempDir := filepath.Join(sandboxesRoot, "sbox", hex.EncodeToString(hash.Sum(nil)))
- err = os.RemoveAll(tempDir)
- if err != nil {
- return err
- }
- err = os.MkdirAll(tempDir, 0777)
- if err != nil {
- return fmt.Errorf("failed to create temporary dir in %q: %w", sandboxesRoot, err)
- }
- // In the common case, the following line of code is what removes the sandbox
- // If a fatal error occurs (such as if our Go process is killed unexpectedly),
- // then at the beginning of the next build, Soong will wipe the temporary
- // directory.
- defer func() {
- // in some cases we decline to remove the temp dir, to facilitate debugging
- if !keepOutDir {
- os.RemoveAll(tempDir)
- }
- }()
- // If there is more than one command in the manifest use a separate directory for each one.
- useSubDir := len(manifest.Commands) > 1
- var commandDepFiles []string
- for i, command := range manifest.Commands {
- localTempDir := tempDir
- if useSubDir {
- localTempDir = filepath.Join(localTempDir, strconv.Itoa(i))
- }
- depFile, err := runCommand(command, localTempDir)
- if err != nil {
- // Running the command failed, keep the temporary output directory around in
- // case a user wants to inspect it for debugging purposes. Soong will delete
- // it at the beginning of the next build anyway.
- keepOutDir = true
- return err
- }
- if depFile != "" {
- commandDepFiles = append(commandDepFiles, depFile)
- }
- }
- outputDepFile := manifest.GetOutputDepfile()
- if len(commandDepFiles) > 0 && outputDepFile == "" {
- return fmt.Errorf("Sandboxed commands used %s but output depfile is not set in manifest file",
- depFilePlaceholder)
- }
- if outputDepFile != "" {
- // Merge the depfiles from each command in the manifest to a single output depfile.
- err = rewriteDepFiles(commandDepFiles, outputDepFile)
- if err != nil {
- return fmt.Errorf("failed merging depfiles: %w", err)
- }
- }
- return nil
- }
- // readManifest reads an sbox manifest from a textproto file.
- func readManifest(file string) (*sbox_proto.Manifest, error) {
- manifestData, err := ioutil.ReadFile(file)
- if err != nil {
- return nil, fmt.Errorf("error reading manifest %q: %w", file, err)
- }
- manifest := sbox_proto.Manifest{}
- err = proto.UnmarshalText(string(manifestData), &manifest)
- if err != nil {
- return nil, fmt.Errorf("error parsing manifest %q: %w", file, err)
- }
- return &manifest, nil
- }
- // runCommand runs a single command from a manifest. If the command references the
- // __SBOX_DEPFILE__ placeholder it returns the name of the depfile that was used.
- func runCommand(command *sbox_proto.Command, tempDir string) (depFile string, err error) {
- rawCommand := command.GetCommand()
- if rawCommand == "" {
- return "", fmt.Errorf("command is required")
- }
- pathToTempDirInSbox := tempDir
- if command.GetChdir() {
- pathToTempDirInSbox = "."
- }
- err = os.MkdirAll(tempDir, 0777)
- if err != nil {
- return "", fmt.Errorf("failed to create %q: %w", tempDir, err)
- }
- // Copy in any files specified by the manifest.
- err = copyFiles(command.CopyBefore, "", tempDir)
- if err != nil {
- return "", err
- }
- err = copyRspFiles(command.RspFiles, tempDir, pathToTempDirInSbox)
- if err != nil {
- return "", err
- }
- if strings.Contains(rawCommand, depFilePlaceholder) {
- depFile = filepath.Join(pathToTempDirInSbox, "deps.d")
- rawCommand = strings.Replace(rawCommand, depFilePlaceholder, depFile, -1)
- }
- if strings.Contains(rawCommand, sandboxDirPlaceholder) {
- rawCommand = strings.Replace(rawCommand, sandboxDirPlaceholder, pathToTempDirInSbox, -1)
- }
- // Emulate ninja's behavior of creating the directories for any output files before
- // running the command.
- err = makeOutputDirs(command.CopyAfter, tempDir)
- if err != nil {
- return "", err
- }
- commandDescription := rawCommand
- cmd := exec.Command("bash", "-c", rawCommand)
- cmd.Stdin = os.Stdin
- cmd.Stdout = os.Stdout
- cmd.Stderr = os.Stderr
- if command.GetChdir() {
- cmd.Dir = tempDir
- path := os.Getenv("PATH")
- absPath, err := makeAbsPathEnv(path)
- if err != nil {
- return "", err
- }
- err = os.Setenv("PATH", absPath)
- if err != nil {
- return "", fmt.Errorf("Failed to update PATH: %w", err)
- }
- }
- err = cmd.Run()
- if exit, ok := err.(*exec.ExitError); ok && !exit.Success() {
- return "", fmt.Errorf("sbox command failed with err:\n%s\n%w\n", commandDescription, err)
- } else if err != nil {
- return "", err
- }
- missingOutputErrors := validateOutputFiles(command.CopyAfter, tempDir)
- if len(missingOutputErrors) > 0 {
- // find all created files for making a more informative error message
- createdFiles := findAllFilesUnder(tempDir)
- // build error message
- errorMessage := "mismatch between declared and actual outputs\n"
- errorMessage += "in sbox command(" + commandDescription + ")\n\n"
- errorMessage += "in sandbox " + tempDir + ",\n"
- errorMessage += fmt.Sprintf("failed to create %v files:\n", len(missingOutputErrors))
- for _, missingOutputError := range missingOutputErrors {
- errorMessage += " " + missingOutputError.Error() + "\n"
- }
- if len(createdFiles) < 1 {
- errorMessage += "created 0 files."
- } else {
- errorMessage += fmt.Sprintf("did create %v files:\n", len(createdFiles))
- creationMessages := createdFiles
- maxNumCreationLines := 10
- if len(creationMessages) > maxNumCreationLines {
- creationMessages = creationMessages[:maxNumCreationLines]
- creationMessages = append(creationMessages, fmt.Sprintf("...%v more", len(createdFiles)-maxNumCreationLines))
- }
- for _, creationMessage := range creationMessages {
- errorMessage += " " + creationMessage + "\n"
- }
- }
- return "", errors.New(errorMessage)
- }
- // the created files match the declared files; now move them
- err = moveFiles(command.CopyAfter, tempDir, "")
- return depFile, nil
- }
- // makeOutputDirs creates directories in the sandbox dir for every file that has a rule to be copied
- // out of the sandbox. This emulate's Ninja's behavior of creating directories for output files
- // so that the tools don't have to.
- func makeOutputDirs(copies []*sbox_proto.Copy, sandboxDir string) error {
- for _, copyPair := range copies {
- dir := joinPath(sandboxDir, filepath.Dir(copyPair.GetFrom()))
- err := os.MkdirAll(dir, 0777)
- if err != nil {
- return err
- }
- }
- return nil
- }
- // validateOutputFiles verifies that all files that have a rule to be copied out of the sandbox
- // were created by the command.
- func validateOutputFiles(copies []*sbox_proto.Copy, sandboxDir string) []error {
- var missingOutputErrors []error
- for _, copyPair := range copies {
- fromPath := joinPath(sandboxDir, copyPair.GetFrom())
- fileInfo, err := os.Stat(fromPath)
- if err != nil {
- missingOutputErrors = append(missingOutputErrors, fmt.Errorf("%s: does not exist", fromPath))
- continue
- }
- if fileInfo.IsDir() {
- missingOutputErrors = append(missingOutputErrors, fmt.Errorf("%s: not a file", fromPath))
- }
- }
- return missingOutputErrors
- }
- // copyFiles copies files in or out of the sandbox.
- func copyFiles(copies []*sbox_proto.Copy, fromDir, toDir string) error {
- for _, copyPair := range copies {
- fromPath := joinPath(fromDir, copyPair.GetFrom())
- toPath := joinPath(toDir, copyPair.GetTo())
- err := copyOneFile(fromPath, toPath, copyPair.GetExecutable())
- if err != nil {
- return fmt.Errorf("error copying %q to %q: %w", fromPath, toPath, err)
- }
- }
- return nil
- }
- // copyOneFile copies a file.
- func copyOneFile(from string, to string, executable bool) error {
- err := os.MkdirAll(filepath.Dir(to), 0777)
- if err != nil {
- return err
- }
- stat, err := os.Stat(from)
- if err != nil {
- return err
- }
- perm := stat.Mode()
- if executable {
- perm = perm | 0100 // u+x
- }
- in, err := os.Open(from)
- if err != nil {
- return err
- }
- defer in.Close()
- out, err := os.Create(to)
- if err != nil {
- return err
- }
- defer func() {
- out.Close()
- if err != nil {
- os.Remove(to)
- }
- }()
- _, err = io.Copy(out, in)
- if err != nil {
- return err
- }
- if err = out.Close(); err != nil {
- return err
- }
- if err = os.Chmod(to, perm); err != nil {
- return err
- }
- return nil
- }
- // copyRspFiles copies rsp files into the sandbox with path mappings, and also copies the files
- // listed into the sandbox.
- func copyRspFiles(rspFiles []*sbox_proto.RspFile, toDir, toDirInSandbox string) error {
- for _, rspFile := range rspFiles {
- err := copyOneRspFile(rspFile, toDir, toDirInSandbox)
- if err != nil {
- return err
- }
- }
- return nil
- }
- // copyOneRspFiles copies an rsp file into the sandbox with path mappings, and also copies the files
- // listed into the sandbox.
- func copyOneRspFile(rspFile *sbox_proto.RspFile, toDir, toDirInSandbox string) error {
- in, err := os.Open(rspFile.GetFile())
- if err != nil {
- return err
- }
- defer in.Close()
- files, err := response.ReadRspFile(in)
- if err != nil {
- return err
- }
- for i, from := range files {
- // Convert the real path of the input file into the path inside the sandbox using the
- // path mappings.
- to := applyPathMappings(rspFile.PathMappings, from)
- // Copy the file into the sandbox.
- err := copyOneFile(from, joinPath(toDir, to), false)
- if err != nil {
- return err
- }
- // Rewrite the name in the list of files to be relative to the sandbox directory.
- files[i] = joinPath(toDirInSandbox, to)
- }
- // Convert the real path of the rsp file into the path inside the sandbox using the path
- // mappings.
- outRspFile := joinPath(toDir, applyPathMappings(rspFile.PathMappings, rspFile.GetFile()))
- err = os.MkdirAll(filepath.Dir(outRspFile), 0777)
- if err != nil {
- return err
- }
- out, err := os.Create(outRspFile)
- if err != nil {
- return err
- }
- defer out.Close()
- // Write the rsp file with converted paths into the sandbox.
- err = response.WriteRspFile(out, files)
- if err != nil {
- return err
- }
- return nil
- }
- // applyPathMappings takes a list of path mappings and a path, and returns the path with the first
- // matching path mapping applied. If the path does not match any of the path mappings then it is
- // returned unmodified.
- func applyPathMappings(pathMappings []*sbox_proto.PathMapping, path string) string {
- for _, mapping := range pathMappings {
- if strings.HasPrefix(path, mapping.GetFrom()+"/") {
- return joinPath(mapping.GetTo()+"/", strings.TrimPrefix(path, mapping.GetFrom()+"/"))
- }
- }
- return path
- }
- // moveFiles moves files specified by a set of copy rules. It uses os.Rename, so it is restricted
- // to moving files where the source and destination are in the same filesystem. This is OK for
- // sbox because the temporary directory is inside the out directory. It updates the timestamp
- // of the new file.
- func moveFiles(copies []*sbox_proto.Copy, fromDir, toDir string) error {
- for _, copyPair := range copies {
- fromPath := joinPath(fromDir, copyPair.GetFrom())
- toPath := joinPath(toDir, copyPair.GetTo())
- err := os.MkdirAll(filepath.Dir(toPath), 0777)
- if err != nil {
- return err
- }
- err = os.Rename(fromPath, toPath)
- if err != nil {
- return err
- }
- // Update the timestamp of the output file in case the tool wrote an old timestamp (for example, tar can extract
- // files with old timestamps).
- now := time.Now()
- err = os.Chtimes(toPath, now, now)
- if err != nil {
- return err
- }
- }
- return nil
- }
- // Rewrite one or more depfiles so that it doesn't include the (randomized) sandbox directory
- // to an output file.
- func rewriteDepFiles(ins []string, out string) error {
- var mergedDeps []string
- for _, in := range ins {
- data, err := ioutil.ReadFile(in)
- if err != nil {
- return err
- }
- deps, err := makedeps.Parse(in, bytes.NewBuffer(data))
- if err != nil {
- return err
- }
- mergedDeps = append(mergedDeps, deps.Inputs...)
- }
- deps := makedeps.Deps{
- // Ninja doesn't care what the output file is, so we can use any string here.
- Output: "outputfile",
- Inputs: mergedDeps,
- }
- // Make the directory for the output depfile in case it is in a different directory
- // than any of the output files.
- outDir := filepath.Dir(out)
- err := os.MkdirAll(outDir, 0777)
- if err != nil {
- return fmt.Errorf("failed to create %q: %w", outDir, err)
- }
- return ioutil.WriteFile(out, deps.Print(), 0666)
- }
- // joinPath wraps filepath.Join but returns file without appending to dir if file is
- // absolute.
- func joinPath(dir, file string) string {
- if filepath.IsAbs(file) {
- return file
- }
- return filepath.Join(dir, file)
- }
- func makeAbsPathEnv(pathEnv string) (string, error) {
- pathEnvElements := filepath.SplitList(pathEnv)
- for i, p := range pathEnvElements {
- if !filepath.IsAbs(p) {
- absPath, err := filepath.Abs(p)
- if err != nil {
- return "", fmt.Errorf("failed to make PATH entry %q absolute: %w", p, err)
- }
- pathEnvElements[i] = absPath
- }
- }
- return strings.Join(pathEnvElements, string(filepath.ListSeparator)), nil
- }
|