123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353 |
- # python3
- # Copyright 2021 The Chromium Authors. All rights reserved.
- # Use of this source code is governed by a BSD-style license that can be
- # found in the LICENSE file.
- """Provide abstractions and helpers for the Rust Cargo build tool.
- In addition to data types representing Cargo concepts, this module has helpers
- for parsing and generating Cargo.toml files, for running the Cargo tool, and for
- parsing its output."""
- from __future__ import annotations
- from enum import Enum
- import os
- import subprocess
- import sys
- import toml
- import typing
- from typing import Any, Optional
- from lib import common
- from lib import consts
- class CrateKey:
- """A unique identifier for any given third-party crate.
- This is a combination of the crate's name and its epoch, since we have at
- most one crate for a given epoch.
- The name and version/epoch are directly from the crate and are not
- normalized.
- """
- def __init__(self, name: str, version: str):
- self.name = name
- self.epoch = common.version_epoch_dots(version)
- def __repr__(self) -> str:
- return "CrateKey({} v{})".format(self.name, self.epoch)
- def __eq__(self, other: object) -> bool:
- if not isinstance(other, CrateKey):
- return NotImplemented
- return self.name == other.name and self.epoch == other.epoch
- def __hash__(self) -> int:
- return hash("{}: {}".format(self.name, self.epoch))
- class CrateUsage(Enum):
- """The ways that a crate's library can be used from other crates."""
- FOR_NORMAL = 1, # Used from another crate's lib/binary outputs.
- FOR_BUILDRS = 2, # Used from another crates's build.rs.
- FOR_TESTS = 3, # Used from another crate's tests.
- def gn_target_name(self) -> str:
- """The name to use for a gn target.
- This is the name of the target used for generating the target in the
- BUILD.gn file. The name is based on how the target will be used, since
- crates have different features enabled when being built for use in
- tests, or for use from a build.rs build script."""
- if self == CrateUsage.FOR_NORMAL:
- return CrateBuildOutput.NORMAL.gn_target_name_for_dep()
- elif self == CrateUsage.FOR_BUILDRS:
- return CrateBuildOutput.BUILDRS.gn_target_name_for_dep()
- elif self == CrateUsage.FOR_TESTS:
- return CrateBuildOutput.TESTS.gn_target_name_for_dep()
- else:
- return NotImplemented
- class CrateBuildOutput(Enum):
- """The various build outputs when building a crate."""
- NORMAL = 1 # Building the crate's normal output.
- BUILDRS = 2 # Building the crate's build.rs.
- TESTS = 3 # Building the crate's tests.
- def as_dep_usage(self) -> CrateUsage:
- if self == CrateBuildOutput.NORMAL:
- return CrateUsage.FOR_NORMAL
- elif self == CrateBuildOutput.BUILDRS:
- return CrateUsage.FOR_BUILDRS
- elif self == CrateBuildOutput.TESTS:
- return CrateUsage.FOR_TESTS
- else:
- assert False # Unhandled CrateBuildOutput?
- def gn_target_name_for_dep(self):
- """The name to use for gn dependency targets.
- This is the name of the target to use for a dependency in the `deps`,
- `build_deps`, or `dev_deps` section of a BUILD.gn target. The name
- depends on what kind of dependency it is, since crates have different
- features enabled when being built for use in tests, or for use from a
- build.rs build script."""
- if self == CrateBuildOutput.NORMAL:
- return "lib"
- if self == CrateBuildOutput.BUILDRS:
- return "buildrs_support"
- if self == CrateBuildOutput.TESTS:
- return "test_support"
- def _cargo_tree_edges(self) -> str:
- """Get the argument for `cargo tree --edges`
- Returns what to pass to the --edges argument when running `cargo tree`
- to see the dependencies of a given build output."""
- if self == CrateBuildOutput.NORMAL:
- return "normal"
- elif self == CrateBuildOutput.BUILDRS:
- return "build"
- elif self == CrateBuildOutput.TESTS:
- return "dev"
- else:
- return NotImplemented
- def run_cargo_tree(path: str, build: CrateBuildOutput,
- target_arch: Optional[str], depth: Optional[int],
- features: list) -> list[str]:
- """Runs `cargo tree` on the Cargo.toml file at `path`.
- Note that `cargo tree` actually invokes `rustc` a bunch to collect its
- output, but it does not appear to actually compile anything. Additionally,
- we are running `cargo tree` in a temp directory with placeholder rust files
- present to satisfy `cargo tree`, so no source code from crates.io should
- be compiled, or run, by this tool.
- Args:
- target_arch: one of the ALL_RUSTC_ARCH which are targets understood by
- rustc, and shown by `rustc --print target-list`. Or none, in which
- case the current machine's architecture is used.
- Returns:
- The output of cargo tree, with split by lines into a list.
- """
- tree_cmd = [
- "cargo",
- "tree",
- "--manifest-path",
- path,
- "--edges",
- build._cargo_tree_edges(),
- "--format={p} {f}",
- "-v",
- ]
- if target_arch:
- tree_cmd += ["--target", target_arch]
- if depth is not None:
- tree_cmd += ["--depth", str(depth)]
- if "default" not in features:
- tree_cmd += ["--no-default-features"]
- features = [f for f in features if not f == "default"]
- if features:
- tree_cmd += ["--features", ",".join(features)]
- try:
- r = subprocess.check_output(tree_cmd, text=True, stderr=subprocess.PIPE)
- except subprocess.CalledProcessError as e:
- print()
- print(' '.join(tree_cmd))
- print(e.stderr)
- raise e
- return r.splitlines()
- def add_required_cargo_fields(toml_3p):
- """Add required fields for a Cargo.toml to be parsed by `cargo tree`."""
- toml_3p["package"] = {
- "name": "chromium",
- "version": "1.0.0",
- }
- return toml_3p
- class ListOf3pCargoToml:
- """A typesafe cache of info about local third-party Cargo.toml files."""
- class CargoToml:
- def __init__(self, name: str, epoch: str, path: str):
- self.name = name
- self.epoch = epoch
- self.path = path
- def __init__(self, list_of: list[CargoToml]):
- self._list_of = list_of
- def write_cargo_toml_in_tempdir(
- dir: str,
- all_3p_tomls: ListOf3pCargoToml,
- orig_toml_parsed: Optional[dict[str, Any]] = None,
- orig_toml_path: Optional[str] = None,
- verbose: bool = False) -> str:
- """Write a temporary Cargo.toml file that will work with `cargo tree`.
- Creates a copy of a Cargo.toml, specified in `orig_toml_path`, in to the
- temp directory specified by `dir` and sets up the temp dir so that running
- `cargo` will succeed. Also points all crates named in `all_3p_tomls` to
- the downloaded versions.
- Exactly one of `orig_toml_parsed` or `orig_toml_path` must be specified.
- Args:
- dir: An OS path to a temp directory where the Cargo.toml file is to be
- written.
- all_3p_tomls: A cache of local third-party Cargo.toml files, crated by
- gen_list_of_3p_cargo_toml(). The generated Cargo.toml will be patched
- to point `cargo tree` to local Cargo.tomls for dependencies in order
- to see local changes.
- orig_toml_parsed: The Cargo.toml file contents to write, as a
- dictionary.
- orig_toml_path: An OS path to the Cargo.toml file which should be copied
- into the output Cargo.toml.
- verbose: Whether to print verbose output, including the full TOML
- content.
- Returns:
- The OS path to the output Cargo.toml file in `dir`, for convenience.
- """
- assert bool(orig_toml_parsed) ^ bool(orig_toml_path)
- orig_toml_text: Optional[str] = None
- if orig_toml_path:
- with open(orig_toml_path, "r") as f:
- orig_toml_text = f.read()
- orig_toml_parsed = dict(toml.loads(orig_toml_text))
- # This assertion is necessary for type checking. Now mypy deduces
- # orig_toml_parsed's type as dict[str, Any] instead of Optional[...]
- assert orig_toml_parsed is not None
- orig_name = orig_toml_parsed["package"]["name"]
- orig_epoch = common.version_epoch_dots(
- orig_toml_parsed["package"]["version"])
- if all_3p_tomls is None:
- all_3p_tomls = ListOf3pCargoToml([])
- # Since we're putting a Cargo.toml in a temp dir, cargo won't be
- # able to find the src/lib.rs and will bail out, so we make it.
- os.mkdir(os.path.join(dir, "src"))
- with open(os.path.join(dir, "src", "lib.rs"), mode="w") as f:
- f.write("lib.rs")
- # Same thing for build.rs, as some Cargo.toml flags make it go looking
- # for a build script to verify it exists.
- if not "build" in orig_toml_parsed["package"]:
- with open(os.path.join(dir, "build.rs"), mode="w") as f:
- f.write("build.rs")
- # And [[bin]] targets, if they have a name but no path, expect to
- # find a file at src/bin/%name%.rs or at src/main.rs, though when
- # one is preferred is unclear. It seems to always work with the
- # former one though, but not always with the latter.
- if "bin" in orig_toml_parsed:
- os.mkdir(os.path.join(dir, "src", "bin"))
- for bin in orig_toml_parsed["bin"]:
- if "path" not in bin and "name" in bin:
- with open(os.path.join(dir, "src", "bin",
- "{}.rs".format(bin["name"])),
- mode="w") as f:
- f.write("bin main.rs")
- # Workspaces in a crate's Cargo.toml need to point to other Cargo.toml files
- # on disk, and those Cargo.toml files require a lib or binary source as
- # well. We don't support building workspaces, but cargo will die if it can't
- # find them.
- if "workspace" in orig_toml_parsed:
- for m in orig_toml_parsed["workspace"].get("members", []):
- workspace_dir = os.path.join(dir, *(m.split("/")))
- os.makedirs(workspace_dir)
- with open(os.path.join(workspace_dir, "Cargo.toml"), mode="w") as f:
- f.write(consts.FAKE_EMPTY_CARGO_TOML)
- bin_dir = os.path.join(workspace_dir, "src", "bin")
- os.makedirs(bin_dir)
- with open(os.path.join(bin_dir, "main.rs"), mode="w") as f:
- f.write("workspace {} bin main.rs".format(m))
- # Generate a patch that points the current crate, to the temp dir, and all
- # others to `consts.THIRD_PARTY`. This is to deal with build/dev deps that
- # transitively depend back on the current crate. Otherwise it gets seen in
- # 2 paths.
- patch: dict[str, Any] = {"patch": {"crates-io": {}}}
- cwd = os.getcwd()
- for in_3p in all_3p_tomls._list_of:
- if in_3p.name == orig_name and in_3p.epoch == orig_epoch:
- # If this is the crate we're creating a temp Cargo.toml for, point
- # the patch to the temp dir.
- abspath = dir
- else:
- # Otherwise, point the patch to the downloaded third-party crate's
- # dir.
- abspath = os.path.join(cwd, in_3p.path)
- patch_name = ("{}_v{}".format(
- in_3p.name, common.version_epoch_normalized(in_3p.epoch)))
- patch["patch"]["crates-io"][patch_name] = {
- "version": in_3p.epoch,
- "path": abspath,
- "package": in_3p.name,
- }
- tmp_cargo_toml_path = os.path.join(dir, "Cargo.toml")
- # This is the third-party Cargo.toml file. Note that we do not write
- # the `orig_toml_parsed` as the python parser does not like the contents
- # of some Cargo.toml files that cargo is just fine with. So we write the
- # contents without a round trip through the parser.
- if orig_toml_text:
- cargo_toml_text = orig_toml_text
- else:
- cargo_toml_text = toml.dumps(orig_toml_parsed)
- # We attach our "patch" keys onto it to redirect all crates.io
- # dependencies into `consts.THIRD_PARTY`.
- cargo_toml_text = cargo_toml_text + toml.dumps(patch)
- # Generate our own (temp) copy of a Cargo.toml for the dependency
- # that we will run `cargo tree` against.
- with open(tmp_cargo_toml_path, mode="w") as tmp_cargo_toml:
- tmp_cargo_toml.write(cargo_toml_text)
- if verbose:
- print("Writing to %s:" % tmp_cargo_toml_path)
- print("=======")
- print(cargo_toml_text)
- print("=======")
- return tmp_cargo_toml_path
- def gen_list_of_3p_cargo_toml() -> ListOf3pCargoToml:
- """Create a cached view of existing third-party crates.
- Find all the third-party crates present and cache them for generating
- Cargo.toml files in temp dirs that will point to them."""
- list_of: list[ListOf3pCargoToml.CargoToml] = []
- for normalized_crate_name in os.listdir(common.os_third_party_dir()):
- crate_dir = common.os_crate_name_dir(normalized_crate_name)
- if not os.path.isdir(crate_dir):
- continue
- for v_epoch in os.listdir(crate_dir):
- epoch = v_epoch.replace("v", "").replace("_", ".")
- filepath = common.os_crate_cargo_dir(normalized_crate_name,
- epoch,
- rel_path=["Cargo.toml"])
- if os.path.exists(filepath):
- cargo_toml = toml.load(filepath)
- # Note this can't use the directory name because it was
- # normalized, so we read the real name from the Cargo.toml.
- name = cargo_toml["package"]["name"]
- assert common.crate_name_normalized(
- name) == normalized_crate_name
- # The version epoch comes from the directory name.
- list_of += [
- ListOf3pCargoToml.CargoToml(
- name, epoch,
- common.os_crate_cargo_dir(normalized_crate_name, epoch))
- ]
- return ListOf3pCargoToml(list_of)
|