cargo.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353
  1. # python3
  2. # Copyright 2021 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """Provide abstractions and helpers for the Rust Cargo build tool.
  6. In addition to data types representing Cargo concepts, this module has helpers
  7. for parsing and generating Cargo.toml files, for running the Cargo tool, and for
  8. parsing its output."""
  9. from __future__ import annotations
  10. from enum import Enum
  11. import os
  12. import subprocess
  13. import sys
  14. import toml
  15. import typing
  16. from typing import Any, Optional
  17. from lib import common
  18. from lib import consts
  19. class CrateKey:
  20. """A unique identifier for any given third-party crate.
  21. This is a combination of the crate's name and its epoch, since we have at
  22. most one crate for a given epoch.
  23. The name and version/epoch are directly from the crate and are not
  24. normalized.
  25. """
  26. def __init__(self, name: str, version: str):
  27. self.name = name
  28. self.epoch = common.version_epoch_dots(version)
  29. def __repr__(self) -> str:
  30. return "CrateKey({} v{})".format(self.name, self.epoch)
  31. def __eq__(self, other: object) -> bool:
  32. if not isinstance(other, CrateKey):
  33. return NotImplemented
  34. return self.name == other.name and self.epoch == other.epoch
  35. def __hash__(self) -> int:
  36. return hash("{}: {}".format(self.name, self.epoch))
  37. class CrateUsage(Enum):
  38. """The ways that a crate's library can be used from other crates."""
  39. FOR_NORMAL = 1, # Used from another crate's lib/binary outputs.
  40. FOR_BUILDRS = 2, # Used from another crates's build.rs.
  41. FOR_TESTS = 3, # Used from another crate's tests.
  42. def gn_target_name(self) -> str:
  43. """The name to use for a gn target.
  44. This is the name of the target used for generating the target in the
  45. BUILD.gn file. The name is based on how the target will be used, since
  46. crates have different features enabled when being built for use in
  47. tests, or for use from a build.rs build script."""
  48. if self == CrateUsage.FOR_NORMAL:
  49. return CrateBuildOutput.NORMAL.gn_target_name_for_dep()
  50. elif self == CrateUsage.FOR_BUILDRS:
  51. return CrateBuildOutput.BUILDRS.gn_target_name_for_dep()
  52. elif self == CrateUsage.FOR_TESTS:
  53. return CrateBuildOutput.TESTS.gn_target_name_for_dep()
  54. else:
  55. return NotImplemented
  56. class CrateBuildOutput(Enum):
  57. """The various build outputs when building a crate."""
  58. NORMAL = 1 # Building the crate's normal output.
  59. BUILDRS = 2 # Building the crate's build.rs.
  60. TESTS = 3 # Building the crate's tests.
  61. def as_dep_usage(self) -> CrateUsage:
  62. if self == CrateBuildOutput.NORMAL:
  63. return CrateUsage.FOR_NORMAL
  64. elif self == CrateBuildOutput.BUILDRS:
  65. return CrateUsage.FOR_BUILDRS
  66. elif self == CrateBuildOutput.TESTS:
  67. return CrateUsage.FOR_TESTS
  68. else:
  69. assert False # Unhandled CrateBuildOutput?
  70. def gn_target_name_for_dep(self):
  71. """The name to use for gn dependency targets.
  72. This is the name of the target to use for a dependency in the `deps`,
  73. `build_deps`, or `dev_deps` section of a BUILD.gn target. The name
  74. depends on what kind of dependency it is, since crates have different
  75. features enabled when being built for use in tests, or for use from a
  76. build.rs build script."""
  77. if self == CrateBuildOutput.NORMAL:
  78. return "lib"
  79. if self == CrateBuildOutput.BUILDRS:
  80. return "buildrs_support"
  81. if self == CrateBuildOutput.TESTS:
  82. return "test_support"
  83. def _cargo_tree_edges(self) -> str:
  84. """Get the argument for `cargo tree --edges`
  85. Returns what to pass to the --edges argument when running `cargo tree`
  86. to see the dependencies of a given build output."""
  87. if self == CrateBuildOutput.NORMAL:
  88. return "normal"
  89. elif self == CrateBuildOutput.BUILDRS:
  90. return "build"
  91. elif self == CrateBuildOutput.TESTS:
  92. return "dev"
  93. else:
  94. return NotImplemented
  95. def run_cargo_tree(path: str, build: CrateBuildOutput,
  96. target_arch: Optional[str], depth: Optional[int],
  97. features: list) -> list[str]:
  98. """Runs `cargo tree` on the Cargo.toml file at `path`.
  99. Note that `cargo tree` actually invokes `rustc` a bunch to collect its
  100. output, but it does not appear to actually compile anything. Additionally,
  101. we are running `cargo tree` in a temp directory with placeholder rust files
  102. present to satisfy `cargo tree`, so no source code from crates.io should
  103. be compiled, or run, by this tool.
  104. Args:
  105. target_arch: one of the ALL_RUSTC_ARCH which are targets understood by
  106. rustc, and shown by `rustc --print target-list`. Or none, in which
  107. case the current machine's architecture is used.
  108. Returns:
  109. The output of cargo tree, with split by lines into a list.
  110. """
  111. tree_cmd = [
  112. "cargo",
  113. "tree",
  114. "--manifest-path",
  115. path,
  116. "--edges",
  117. build._cargo_tree_edges(),
  118. "--format={p} {f}",
  119. "-v",
  120. ]
  121. if target_arch:
  122. tree_cmd += ["--target", target_arch]
  123. if depth is not None:
  124. tree_cmd += ["--depth", str(depth)]
  125. if "default" not in features:
  126. tree_cmd += ["--no-default-features"]
  127. features = [f for f in features if not f == "default"]
  128. if features:
  129. tree_cmd += ["--features", ",".join(features)]
  130. try:
  131. r = subprocess.check_output(tree_cmd, text=True, stderr=subprocess.PIPE)
  132. except subprocess.CalledProcessError as e:
  133. print()
  134. print(' '.join(tree_cmd))
  135. print(e.stderr)
  136. raise e
  137. return r.splitlines()
  138. def add_required_cargo_fields(toml_3p):
  139. """Add required fields for a Cargo.toml to be parsed by `cargo tree`."""
  140. toml_3p["package"] = {
  141. "name": "chromium",
  142. "version": "1.0.0",
  143. }
  144. return toml_3p
  145. class ListOf3pCargoToml:
  146. """A typesafe cache of info about local third-party Cargo.toml files."""
  147. class CargoToml:
  148. def __init__(self, name: str, epoch: str, path: str):
  149. self.name = name
  150. self.epoch = epoch
  151. self.path = path
  152. def __init__(self, list_of: list[CargoToml]):
  153. self._list_of = list_of
  154. def write_cargo_toml_in_tempdir(
  155. dir: str,
  156. all_3p_tomls: ListOf3pCargoToml,
  157. orig_toml_parsed: Optional[dict[str, Any]] = None,
  158. orig_toml_path: Optional[str] = None,
  159. verbose: bool = False) -> str:
  160. """Write a temporary Cargo.toml file that will work with `cargo tree`.
  161. Creates a copy of a Cargo.toml, specified in `orig_toml_path`, in to the
  162. temp directory specified by `dir` and sets up the temp dir so that running
  163. `cargo` will succeed. Also points all crates named in `all_3p_tomls` to
  164. the downloaded versions.
  165. Exactly one of `orig_toml_parsed` or `orig_toml_path` must be specified.
  166. Args:
  167. dir: An OS path to a temp directory where the Cargo.toml file is to be
  168. written.
  169. all_3p_tomls: A cache of local third-party Cargo.toml files, crated by
  170. gen_list_of_3p_cargo_toml(). The generated Cargo.toml will be patched
  171. to point `cargo tree` to local Cargo.tomls for dependencies in order
  172. to see local changes.
  173. orig_toml_parsed: The Cargo.toml file contents to write, as a
  174. dictionary.
  175. orig_toml_path: An OS path to the Cargo.toml file which should be copied
  176. into the output Cargo.toml.
  177. verbose: Whether to print verbose output, including the full TOML
  178. content.
  179. Returns:
  180. The OS path to the output Cargo.toml file in `dir`, for convenience.
  181. """
  182. assert bool(orig_toml_parsed) ^ bool(orig_toml_path)
  183. orig_toml_text: Optional[str] = None
  184. if orig_toml_path:
  185. with open(orig_toml_path, "r") as f:
  186. orig_toml_text = f.read()
  187. orig_toml_parsed = dict(toml.loads(orig_toml_text))
  188. # This assertion is necessary for type checking. Now mypy deduces
  189. # orig_toml_parsed's type as dict[str, Any] instead of Optional[...]
  190. assert orig_toml_parsed is not None
  191. orig_name = orig_toml_parsed["package"]["name"]
  192. orig_epoch = common.version_epoch_dots(
  193. orig_toml_parsed["package"]["version"])
  194. if all_3p_tomls is None:
  195. all_3p_tomls = ListOf3pCargoToml([])
  196. # Since we're putting a Cargo.toml in a temp dir, cargo won't be
  197. # able to find the src/lib.rs and will bail out, so we make it.
  198. os.mkdir(os.path.join(dir, "src"))
  199. with open(os.path.join(dir, "src", "lib.rs"), mode="w") as f:
  200. f.write("lib.rs")
  201. # Same thing for build.rs, as some Cargo.toml flags make it go looking
  202. # for a build script to verify it exists.
  203. if not "build" in orig_toml_parsed["package"]:
  204. with open(os.path.join(dir, "build.rs"), mode="w") as f:
  205. f.write("build.rs")
  206. # And [[bin]] targets, if they have a name but no path, expect to
  207. # find a file at src/bin/%name%.rs or at src/main.rs, though when
  208. # one is preferred is unclear. It seems to always work with the
  209. # former one though, but not always with the latter.
  210. if "bin" in orig_toml_parsed:
  211. os.mkdir(os.path.join(dir, "src", "bin"))
  212. for bin in orig_toml_parsed["bin"]:
  213. if "path" not in bin and "name" in bin:
  214. with open(os.path.join(dir, "src", "bin",
  215. "{}.rs".format(bin["name"])),
  216. mode="w") as f:
  217. f.write("bin main.rs")
  218. # Workspaces in a crate's Cargo.toml need to point to other Cargo.toml files
  219. # on disk, and those Cargo.toml files require a lib or binary source as
  220. # well. We don't support building workspaces, but cargo will die if it can't
  221. # find them.
  222. if "workspace" in orig_toml_parsed:
  223. for m in orig_toml_parsed["workspace"].get("members", []):
  224. workspace_dir = os.path.join(dir, *(m.split("/")))
  225. os.makedirs(workspace_dir)
  226. with open(os.path.join(workspace_dir, "Cargo.toml"), mode="w") as f:
  227. f.write(consts.FAKE_EMPTY_CARGO_TOML)
  228. bin_dir = os.path.join(workspace_dir, "src", "bin")
  229. os.makedirs(bin_dir)
  230. with open(os.path.join(bin_dir, "main.rs"), mode="w") as f:
  231. f.write("workspace {} bin main.rs".format(m))
  232. # Generate a patch that points the current crate, to the temp dir, and all
  233. # others to `consts.THIRD_PARTY`. This is to deal with build/dev deps that
  234. # transitively depend back on the current crate. Otherwise it gets seen in
  235. # 2 paths.
  236. patch: dict[str, Any] = {"patch": {"crates-io": {}}}
  237. cwd = os.getcwd()
  238. for in_3p in all_3p_tomls._list_of:
  239. if in_3p.name == orig_name and in_3p.epoch == orig_epoch:
  240. # If this is the crate we're creating a temp Cargo.toml for, point
  241. # the patch to the temp dir.
  242. abspath = dir
  243. else:
  244. # Otherwise, point the patch to the downloaded third-party crate's
  245. # dir.
  246. abspath = os.path.join(cwd, in_3p.path)
  247. patch_name = ("{}_v{}".format(
  248. in_3p.name, common.version_epoch_normalized(in_3p.epoch)))
  249. patch["patch"]["crates-io"][patch_name] = {
  250. "version": in_3p.epoch,
  251. "path": abspath,
  252. "package": in_3p.name,
  253. }
  254. tmp_cargo_toml_path = os.path.join(dir, "Cargo.toml")
  255. # This is the third-party Cargo.toml file. Note that we do not write
  256. # the `orig_toml_parsed` as the python parser does not like the contents
  257. # of some Cargo.toml files that cargo is just fine with. So we write the
  258. # contents without a round trip through the parser.
  259. if orig_toml_text:
  260. cargo_toml_text = orig_toml_text
  261. else:
  262. cargo_toml_text = toml.dumps(orig_toml_parsed)
  263. # We attach our "patch" keys onto it to redirect all crates.io
  264. # dependencies into `consts.THIRD_PARTY`.
  265. cargo_toml_text = cargo_toml_text + toml.dumps(patch)
  266. # Generate our own (temp) copy of a Cargo.toml for the dependency
  267. # that we will run `cargo tree` against.
  268. with open(tmp_cargo_toml_path, mode="w") as tmp_cargo_toml:
  269. tmp_cargo_toml.write(cargo_toml_text)
  270. if verbose:
  271. print("Writing to %s:" % tmp_cargo_toml_path)
  272. print("=======")
  273. print(cargo_toml_text)
  274. print("=======")
  275. return tmp_cargo_toml_path
  276. def gen_list_of_3p_cargo_toml() -> ListOf3pCargoToml:
  277. """Create a cached view of existing third-party crates.
  278. Find all the third-party crates present and cache them for generating
  279. Cargo.toml files in temp dirs that will point to them."""
  280. list_of: list[ListOf3pCargoToml.CargoToml] = []
  281. for normalized_crate_name in os.listdir(common.os_third_party_dir()):
  282. crate_dir = common.os_crate_name_dir(normalized_crate_name)
  283. if not os.path.isdir(crate_dir):
  284. continue
  285. for v_epoch in os.listdir(crate_dir):
  286. epoch = v_epoch.replace("v", "").replace("_", ".")
  287. filepath = common.os_crate_cargo_dir(normalized_crate_name,
  288. epoch,
  289. rel_path=["Cargo.toml"])
  290. if os.path.exists(filepath):
  291. cargo_toml = toml.load(filepath)
  292. # Note this can't use the directory name because it was
  293. # normalized, so we read the real name from the Cargo.toml.
  294. name = cargo_toml["package"]["name"]
  295. assert common.crate_name_normalized(
  296. name) == normalized_crate_name
  297. # The version epoch comes from the directory name.
  298. list_of += [
  299. ListOf3pCargoToml.CargoToml(
  300. name, epoch,
  301. common.os_crate_cargo_dir(normalized_crate_name, epoch))
  302. ]
  303. return ListOf3pCargoToml(list_of)