build_rust.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373
  1. #!/usr/bin/env python3
  2. # Copyright 2022 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. '''Assembles a Rust toolchain in-tree linked against in-tree LLVM.
  6. !!! DO NOT USE IN PRODUCTION
  7. Builds a Rust toolchain bootstrapped from an untrusted rustc build.
  8. Rust has an official boostrapping build. At a high level:
  9. 1. A "stage 0" Rust is downloaded from Rust's official server. This
  10. is one major release before the version being built. E.g. if building trunk
  11. the latest beta is downloaded. If building stable 1.57.0, stage0 is stable
  12. 1.56.1.
  13. 2. Stage 0 libstd is built. This is different than the libstd downloaded above.
  14. 3. Stage 1 rustc is built with rustc from (1) and libstd from (2)
  15. 2. Stage 1 libstd is built with stage 1 rustc. Later artifacts built with
  16. stage 1 rustc are built with stage 1 libstd.
  17. Further stages are possible and continue as expected. Additionally, the build
  18. can be extensively customized. See for details:
  19. https://rustc-dev-guide.rust-lang.org/building/bootstrapping.html
  20. This script clones the Rust repository, checks it out to a defined revision,
  21. then builds stage 1 rustc and libstd using the LLVM build from
  22. //tools/clang/scripts/build.py or clang-libs fetched from
  23. //tools/clang/scripts/update.py.
  24. Ideally our build would begin with our own trusted stage0 rustc. As it is
  25. simpler, for now we use an official build.
  26. TODO(https://crbug.com/1245714): Do a proper 3-stage build
  27. '''
  28. import argparse
  29. import collections
  30. import hashlib
  31. import os
  32. import pipes
  33. import shutil
  34. import string
  35. import subprocess
  36. import sys
  37. from pathlib import Path
  38. # Get variables and helpers from Clang update script.
  39. sys.path.append(
  40. os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'clang',
  41. 'scripts'))
  42. from update import (CLANG_REVISION, CLANG_SUB_REVISION, LLVM_BUILD_DIR,
  43. GetDefaultHostOs, RmTree, UpdatePackage)
  44. import build
  45. from update_rust import (CHROMIUM_DIR, RUST_REVISION, RUST_SUB_REVISION,
  46. RUST_TOOLCHAIN_OUT_DIR, STAGE0_JSON_SHA256,
  47. THIRD_PARTY_DIR, VERSION_STAMP_PATH, GetPackageVersion)
  48. RUST_GIT_URL = 'https://github.com/rust-lang/rust/'
  49. RUST_SRC_DIR = os.path.join(THIRD_PARTY_DIR, 'rust-src')
  50. STAGE0_JSON_PATH = os.path.join(RUST_SRC_DIR, 'src', 'stage0.json')
  51. # Download crates.io dependencies to rust-src subdir (rather than $HOME/.cargo)
  52. CARGO_HOME_DIR = os.path.join(RUST_SRC_DIR, 'cargo-home')
  53. RUST_SRC_VERSION_FILE_PATH = os.path.join(RUST_SRC_DIR, 'src', 'version')
  54. RUST_TOOLCHAIN_LIB_DIR = os.path.join(RUST_TOOLCHAIN_OUT_DIR, 'lib')
  55. RUST_TOOLCHAIN_SRC_DIST_DIR = os.path.join(RUST_TOOLCHAIN_LIB_DIR, 'rustlib',
  56. 'src', 'rust')
  57. RUST_TOOLCHAIN_SRC_DIST_VENDOR_DIR = os.path.join(RUST_TOOLCHAIN_SRC_DIST_DIR,
  58. 'vendor')
  59. RUST_CONFIG_TEMPLATE_PATH = os.path.join(
  60. os.path.dirname(os.path.abspath(__file__)), 'config.toml.template')
  61. RUST_SRC_VENDOR_DIR = os.path.join(RUST_SRC_DIR, 'vendor')
  62. # Desired tools and libraries in our Rust toolchain.
  63. DISTRIBUTION_ARTIFACTS = [
  64. 'cargo', 'clippy', 'compiler/rustc', 'library/std', 'rust-analyzer',
  65. 'rustfmt', 'src'
  66. ]
  67. # Which test suites to run. Any failure will fail the build.
  68. TEST_SUITES = [
  69. 'library/std',
  70. 'src/test/codegen',
  71. 'src/test/ui',
  72. ]
  73. EXCLUDED_TESTS = [
  74. # Temporarily disabled due to https://github.com/rust-lang/rust/issues/94322
  75. 'src/test/ui/numeric/numeric-cast.rs',
  76. # Temporarily disabled due to https://github.com/rust-lang/rust/issues/96497
  77. 'src/test/codegen/issue-96497-slice-size-nowrap.rs',
  78. # TODO(crbug.com/1347563): Re-enable when fixed.
  79. 'src/test/codegen/sanitizer-cfi-emit-type-checks.rs',
  80. 'src/test/codegen/sanitizer-cfi-emit-type-metadata-itanium-cxx-abi.rs',
  81. ]
  82. def RunCommand(command, env=None, fail_hard=True):
  83. print('Running', command)
  84. if subprocess.run(command, env=env,
  85. shell=sys.platform == 'win32').returncode == 0:
  86. return True
  87. print('Failed.')
  88. if fail_hard:
  89. sys.exit(1)
  90. return False
  91. def CheckoutRust(commit, dir):
  92. # Submodules we must update early since bootstrap wants them before it
  93. # starts managing them.
  94. force_update_submodules = [
  95. 'src/tools/rust-analyzer', 'compiler/rustc_codegen_cranelift'
  96. ]
  97. # Shared between first checkout and subsequent updates.
  98. def UpdateSubmodules():
  99. return RunCommand(
  100. ['git', 'submodule', 'update', '--init', '--recursive'] +
  101. force_update_submodules,
  102. fail_hard=False)
  103. # Try updating the current repo if it exists and has no local diff.
  104. if os.path.isdir(dir):
  105. os.chdir(dir)
  106. # git diff-index --quiet returns success when there is no diff.
  107. # Also check that the first commit is reachable.
  108. if (RunCommand(['git', 'diff-index', '--quiet', 'HEAD'],
  109. fail_hard=False)
  110. and RunCommand(['git', 'fetch'], fail_hard=False)
  111. and RunCommand(['git', 'checkout', commit], fail_hard=False)
  112. and UpdateSubmodules()):
  113. return
  114. # If we can't use the current repo, delete it.
  115. os.chdir(CHROMIUM_DIR) # Can't remove dir if we're in it.
  116. print('Removing %s.' % dir)
  117. RmTree(dir)
  118. clone_cmd = ['git', 'clone', RUST_GIT_URL, dir]
  119. if RunCommand(clone_cmd, fail_hard=False):
  120. os.chdir(dir)
  121. if (RunCommand(['git', 'checkout', commit], fail_hard=False)
  122. and UpdateSubmodules()):
  123. return
  124. print('CheckoutRust failed.')
  125. sys.exit(1)
  126. def VerifyStage0JsonHash():
  127. hasher = hashlib.sha256()
  128. with open(STAGE0_JSON_PATH, 'rb') as input:
  129. hasher.update(input.read())
  130. actual_hash = hasher.hexdigest()
  131. if actual_hash == STAGE0_JSON_SHA256:
  132. return
  133. print('src/stage0.json hash is different than expected!')
  134. print('Expected hash: ' + STAGE0_JSON_SHA256)
  135. print('Actual hash: ' + actual_hash)
  136. sys.exit(1)
  137. def Configure(llvm_libs_root):
  138. # Read the config.toml template file...
  139. with open(RUST_CONFIG_TEMPLATE_PATH, 'r') as input:
  140. template = string.Template(input.read())
  141. subs = {}
  142. subs['INSTALL_DIR'] = RUST_TOOLCHAIN_OUT_DIR
  143. subs['LLVM_ROOT'] = llvm_libs_root
  144. subs['PACKAGE_VERSION'] = GetPackageVersion()
  145. # ...and apply substitutions, writing to config.toml in Rust tree.
  146. with open(os.path.join(RUST_SRC_DIR, 'config.toml'), 'w') as output:
  147. output.write(template.substitute(subs))
  148. def RunXPy(sub, args, gcc_toolchain_path, verbose):
  149. ''' Run x.py, Rust's build script'''
  150. clang_path = os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')
  151. RUSTENV = collections.defaultdict(str, os.environ)
  152. # Cargo normally stores files in $HOME. Override this.
  153. RUSTENV['CARGO_HOME'] = CARGO_HOME_DIR
  154. RUSTENV['AR'] = os.path.join(LLVM_BUILD_DIR, 'bin', 'llvm-ar')
  155. RUSTENV['CC'] = clang_path
  156. RUSTENV['CXX'] = os.path.join(LLVM_BUILD_DIR, 'bin', 'clang++')
  157. RUSTENV['LD'] = clang_path
  158. # We use these flags to avoid linking with the system libstdc++.
  159. gcc_toolchain_flag = (f'--gcc-toolchain={gcc_toolchain_path}'
  160. if gcc_toolchain_path else '')
  161. # These affect how C/C++ files are compiled, but not Rust libs/exes.
  162. RUSTENV['CFLAGS'] += f' {gcc_toolchain_flag}'
  163. RUSTENV['CXXFLAGS'] += f' {gcc_toolchain_flag}'
  164. RUSTENV['LDFLAGS'] += f' {gcc_toolchain_flag}'
  165. # These affect how Rust crates are built. A `-Clink-arg=<foo>` arg passes
  166. # foo to the clang invocation used to link.
  167. #
  168. # TODO(https://crbug.com/1281664): remove --no-gc-sections argument.
  169. # Workaround for a bug causing std::env::args() to return an empty list,
  170. # making Rust binaries unable to take command line arguments. Fix is landed
  171. # upstream in LLVM but hasn't rolled into Chromium. Also see:
  172. # * https://github.com/rust-lang/rust/issues/92181
  173. # * https://reviews.llvm.org/D116528
  174. RUSTENV['RUSTFLAGS_BOOTSTRAP'] = (
  175. f'-Clinker={clang_path} -Clink-arg=-fuse-ld=lld '
  176. f'-Clink-arg=-Wl,--no-gc-sections -Clink-arg={gcc_toolchain_flag} '
  177. f'-L native={gcc_toolchain_path}/lib64')
  178. RUSTENV['RUSTFLAGS_NOT_BOOTSTRAP'] = RUSTENV['RUSTFLAGS_BOOTSTRAP']
  179. os.chdir(RUST_SRC_DIR)
  180. cmd = [sys.executable, 'x.py', sub]
  181. if verbose and verbose > 0:
  182. cmd.append('-' + verbose * 'v')
  183. RunCommand(cmd + args, env=RUSTENV)
  184. # Get arguments to run desired test suites, minus disabled tests.
  185. def GetTestArgs():
  186. args = TEST_SUITES
  187. for excluded in EXCLUDED_TESTS:
  188. args.append('--skip')
  189. args.append(excluded)
  190. return args
  191. def main():
  192. parser = argparse.ArgumentParser(
  193. description='Build and package Rust toolchain')
  194. parser.add_argument('-v',
  195. '--verbose',
  196. action='count',
  197. help='run subcommands with verbosity')
  198. parser.add_argument(
  199. '--verify-stage0-hash',
  200. action='store_true',
  201. help=
  202. 'checkout Rust, verify the stage0 hash, then quit without building. '
  203. 'Will print the actual hash if different than expected.')
  204. parser.add_argument(
  205. '--skip-checkout',
  206. action='store_true',
  207. help='skip Rust git checkout. Useful for trying local changes')
  208. parser.add_argument('--skip-clean',
  209. action='store_true',
  210. help='skip x.py clean step')
  211. parser.add_argument('--skip-test',
  212. action='store_true',
  213. help='skip running rustc and libstd tests')
  214. parser.add_argument('--skip-install',
  215. action='store_true',
  216. help='do not install to RUST_TOOLCHAIN_OUT_DIR')
  217. parser.add_argument(
  218. '--fetch-llvm-libs',
  219. action='store_true',
  220. help='fetch Clang/LLVM libs and extract into LLVM_BUILD_DIR. Useless '
  221. 'without --use-final-llvm-build-dir.')
  222. parser.add_argument(
  223. '--use-final-llvm-build-dir',
  224. action='store_true',
  225. help='use libs in LLVM_BUILD_DIR instead of LLVM_BOOTSTRAP_DIR. Useful '
  226. 'with --fetch-llvm-libs for local builds.')
  227. parser.add_argument(
  228. '--run-xpy',
  229. action='store_true',
  230. help='run x.py command in configured Rust checkout. Quits after '
  231. 'running specified command, skipping all normal build steps. For '
  232. 'debugging. Running x.py directly will not set the appropriate env '
  233. 'variables nor update config.toml')
  234. args, rest = parser.parse_known_args()
  235. # Get the LLVM root for libs. We use LLVM_BUILD_DIR tools either way.
  236. #
  237. # TODO(https://crbug.com/1245714): use LTO libs from LLVM_BUILD_DIR for
  238. # stage 2+.
  239. if args.use_final_llvm_build_dir:
  240. llvm_libs_root = LLVM_BUILD_DIR
  241. else:
  242. llvm_libs_root = build.LLVM_BOOTSTRAP_DIR
  243. if not args.skip_checkout:
  244. CheckoutRust(RUST_REVISION, RUST_SRC_DIR)
  245. VerifyStage0JsonHash()
  246. if args.verify_stage0_hash:
  247. # The above function exits and prints the actual hash if verification
  248. # failed so we just quit here; if we reach this point, the hash is
  249. # valid.
  250. return 0
  251. if args.fetch_llvm_libs:
  252. UpdatePackage('clang-libs', GetDefaultHostOs())
  253. # Fetch GCC package to build against same libstdc++ as Clang. This function
  254. # will only download it if necessary.
  255. args.gcc_toolchain = None
  256. build.MaybeDownloadHostGcc(args)
  257. # Set up config.toml in Rust source tree to configure build.
  258. Configure(llvm_libs_root)
  259. if args.run_xpy:
  260. if rest[0] == '--':
  261. rest = rest[1:]
  262. RunXPy(rest[0], rest[1:], args.gcc_toolchain, args.verbose)
  263. return 0
  264. else:
  265. assert not rest
  266. # Delete vendored sources and .cargo subdir. Otherwise when updating an
  267. # existing checkout, vendored sources will not be re-fetched leaving deps
  268. # out of date.
  269. if not args.skip_checkout:
  270. for dir in [
  271. os.path.join(RUST_SRC_DIR, d) for d in ['vendor', '.cargo']
  272. ]:
  273. if os.path.exists(dir):
  274. shutil.rmtree(dir)
  275. if not args.skip_clean:
  276. print('Cleaning build artifacts...')
  277. RunXPy('clean', [], args.gcc_toolchain, args.verbose)
  278. if not args.skip_test:
  279. print('Running stage 2 tests...')
  280. # Run a subset of tests. Tell x.py to keep the rustc we already built.
  281. RunXPy('test', GetTestArgs(), args.gcc_toolchain, args.verbose)
  282. targets = [
  283. 'library/proc_macro', 'library/std', 'src/tools/cargo',
  284. 'src/tools/clippy', 'src/tools/rustfmt'
  285. ]
  286. # Build stage 2 compiler, tools, and libraries. This should reuse earlier
  287. # stages from the test command (if run).
  288. print('Building stage 2 artifacts...')
  289. RunXPy('build', ['--stage', '2'] + targets, args.gcc_toolchain,
  290. args.verbose)
  291. if args.skip_install:
  292. # Rust is fully built. We can quit.
  293. return 0
  294. print(f'Installing to {RUST_TOOLCHAIN_OUT_DIR} ...')
  295. # Clean output directory.
  296. if os.path.exists(RUST_TOOLCHAIN_OUT_DIR):
  297. shutil.rmtree(RUST_TOOLCHAIN_OUT_DIR)
  298. RunXPy('install', DISTRIBUTION_ARTIFACTS, args.gcc_toolchain, args.verbose)
  299. # Write expected `rustc --version` string to our toolchain directory.
  300. with open(RUST_SRC_VERSION_FILE_PATH) as version_file:
  301. rust_version = version_file.readline().rstrip()
  302. with open(VERSION_STAMP_PATH, 'w') as stamp:
  303. stamp.write('rustc %s-dev (%s chromium)\n' %
  304. (rust_version, GetPackageVersion()))
  305. return 0
  306. # TODO(crbug.com/1342708): fix vendoring and re-enable.
  307. # x.py installed library sources to our toolchain directory. We also need to
  308. # copy the vendor directory so Chromium checkouts have all the deps needed
  309. # to build std.
  310. # shutil.copytree(RUST_SRC_VENDOR_DIR, RUST_TOOLCHAIN_SRC_DIST_VENDOR_DIR)
  311. if __name__ == '__main__':
  312. sys.exit(main())