123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373 |
- #!/usr/bin/env python3
- # Copyright 2022 The Chromium Authors. All rights reserved.
- # Use of this source code is governed by a BSD-style license that can be
- # found in the LICENSE file.
- '''Assembles a Rust toolchain in-tree linked against in-tree LLVM.
- !!! DO NOT USE IN PRODUCTION
- Builds a Rust toolchain bootstrapped from an untrusted rustc build.
- Rust has an official boostrapping build. At a high level:
- 1. A "stage 0" Rust is downloaded from Rust's official server. This
- is one major release before the version being built. E.g. if building trunk
- the latest beta is downloaded. If building stable 1.57.0, stage0 is stable
- 1.56.1.
- 2. Stage 0 libstd is built. This is different than the libstd downloaded above.
- 3. Stage 1 rustc is built with rustc from (1) and libstd from (2)
- 2. Stage 1 libstd is built with stage 1 rustc. Later artifacts built with
- stage 1 rustc are built with stage 1 libstd.
- Further stages are possible and continue as expected. Additionally, the build
- can be extensively customized. See for details:
- https://rustc-dev-guide.rust-lang.org/building/bootstrapping.html
- This script clones the Rust repository, checks it out to a defined revision,
- then builds stage 1 rustc and libstd using the LLVM build from
- //tools/clang/scripts/build.py or clang-libs fetched from
- //tools/clang/scripts/update.py.
- Ideally our build would begin with our own trusted stage0 rustc. As it is
- simpler, for now we use an official build.
- TODO(https://crbug.com/1245714): Do a proper 3-stage build
- '''
- import argparse
- import collections
- import hashlib
- import os
- import pipes
- import shutil
- import string
- import subprocess
- import sys
- from pathlib import Path
- # Get variables and helpers from Clang update script.
- sys.path.append(
- os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'clang',
- 'scripts'))
- from update import (CLANG_REVISION, CLANG_SUB_REVISION, LLVM_BUILD_DIR,
- GetDefaultHostOs, RmTree, UpdatePackage)
- import build
- from update_rust import (CHROMIUM_DIR, RUST_REVISION, RUST_SUB_REVISION,
- RUST_TOOLCHAIN_OUT_DIR, STAGE0_JSON_SHA256,
- THIRD_PARTY_DIR, VERSION_STAMP_PATH, GetPackageVersion)
- RUST_GIT_URL = 'https://github.com/rust-lang/rust/'
- RUST_SRC_DIR = os.path.join(THIRD_PARTY_DIR, 'rust-src')
- STAGE0_JSON_PATH = os.path.join(RUST_SRC_DIR, 'src', 'stage0.json')
- # Download crates.io dependencies to rust-src subdir (rather than $HOME/.cargo)
- CARGO_HOME_DIR = os.path.join(RUST_SRC_DIR, 'cargo-home')
- RUST_SRC_VERSION_FILE_PATH = os.path.join(RUST_SRC_DIR, 'src', 'version')
- RUST_TOOLCHAIN_LIB_DIR = os.path.join(RUST_TOOLCHAIN_OUT_DIR, 'lib')
- RUST_TOOLCHAIN_SRC_DIST_DIR = os.path.join(RUST_TOOLCHAIN_LIB_DIR, 'rustlib',
- 'src', 'rust')
- RUST_TOOLCHAIN_SRC_DIST_VENDOR_DIR = os.path.join(RUST_TOOLCHAIN_SRC_DIST_DIR,
- 'vendor')
- RUST_CONFIG_TEMPLATE_PATH = os.path.join(
- os.path.dirname(os.path.abspath(__file__)), 'config.toml.template')
- RUST_SRC_VENDOR_DIR = os.path.join(RUST_SRC_DIR, 'vendor')
- # Desired tools and libraries in our Rust toolchain.
- DISTRIBUTION_ARTIFACTS = [
- 'cargo', 'clippy', 'compiler/rustc', 'library/std', 'rust-analyzer',
- 'rustfmt', 'src'
- ]
- # Which test suites to run. Any failure will fail the build.
- TEST_SUITES = [
- 'library/std',
- 'src/test/codegen',
- 'src/test/ui',
- ]
- EXCLUDED_TESTS = [
- # Temporarily disabled due to https://github.com/rust-lang/rust/issues/94322
- 'src/test/ui/numeric/numeric-cast.rs',
- # Temporarily disabled due to https://github.com/rust-lang/rust/issues/96497
- 'src/test/codegen/issue-96497-slice-size-nowrap.rs',
- # TODO(crbug.com/1347563): Re-enable when fixed.
- 'src/test/codegen/sanitizer-cfi-emit-type-checks.rs',
- 'src/test/codegen/sanitizer-cfi-emit-type-metadata-itanium-cxx-abi.rs',
- ]
- def RunCommand(command, env=None, fail_hard=True):
- print('Running', command)
- if subprocess.run(command, env=env,
- shell=sys.platform == 'win32').returncode == 0:
- return True
- print('Failed.')
- if fail_hard:
- sys.exit(1)
- return False
- def CheckoutRust(commit, dir):
- # Submodules we must update early since bootstrap wants them before it
- # starts managing them.
- force_update_submodules = [
- 'src/tools/rust-analyzer', 'compiler/rustc_codegen_cranelift'
- ]
- # Shared between first checkout and subsequent updates.
- def UpdateSubmodules():
- return RunCommand(
- ['git', 'submodule', 'update', '--init', '--recursive'] +
- force_update_submodules,
- fail_hard=False)
- # Try updating the current repo if it exists and has no local diff.
- if os.path.isdir(dir):
- os.chdir(dir)
- # git diff-index --quiet returns success when there is no diff.
- # Also check that the first commit is reachable.
- if (RunCommand(['git', 'diff-index', '--quiet', 'HEAD'],
- fail_hard=False)
- and RunCommand(['git', 'fetch'], fail_hard=False)
- and RunCommand(['git', 'checkout', commit], fail_hard=False)
- and UpdateSubmodules()):
- return
- # If we can't use the current repo, delete it.
- os.chdir(CHROMIUM_DIR) # Can't remove dir if we're in it.
- print('Removing %s.' % dir)
- RmTree(dir)
- clone_cmd = ['git', 'clone', RUST_GIT_URL, dir]
- if RunCommand(clone_cmd, fail_hard=False):
- os.chdir(dir)
- if (RunCommand(['git', 'checkout', commit], fail_hard=False)
- and UpdateSubmodules()):
- return
- print('CheckoutRust failed.')
- sys.exit(1)
- def VerifyStage0JsonHash():
- hasher = hashlib.sha256()
- with open(STAGE0_JSON_PATH, 'rb') as input:
- hasher.update(input.read())
- actual_hash = hasher.hexdigest()
- if actual_hash == STAGE0_JSON_SHA256:
- return
- print('src/stage0.json hash is different than expected!')
- print('Expected hash: ' + STAGE0_JSON_SHA256)
- print('Actual hash: ' + actual_hash)
- sys.exit(1)
- def Configure(llvm_libs_root):
- # Read the config.toml template file...
- with open(RUST_CONFIG_TEMPLATE_PATH, 'r') as input:
- template = string.Template(input.read())
- subs = {}
- subs['INSTALL_DIR'] = RUST_TOOLCHAIN_OUT_DIR
- subs['LLVM_ROOT'] = llvm_libs_root
- subs['PACKAGE_VERSION'] = GetPackageVersion()
- # ...and apply substitutions, writing to config.toml in Rust tree.
- with open(os.path.join(RUST_SRC_DIR, 'config.toml'), 'w') as output:
- output.write(template.substitute(subs))
- def RunXPy(sub, args, gcc_toolchain_path, verbose):
- ''' Run x.py, Rust's build script'''
- clang_path = os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')
- RUSTENV = collections.defaultdict(str, os.environ)
- # Cargo normally stores files in $HOME. Override this.
- RUSTENV['CARGO_HOME'] = CARGO_HOME_DIR
- RUSTENV['AR'] = os.path.join(LLVM_BUILD_DIR, 'bin', 'llvm-ar')
- RUSTENV['CC'] = clang_path
- RUSTENV['CXX'] = os.path.join(LLVM_BUILD_DIR, 'bin', 'clang++')
- RUSTENV['LD'] = clang_path
- # We use these flags to avoid linking with the system libstdc++.
- gcc_toolchain_flag = (f'--gcc-toolchain={gcc_toolchain_path}'
- if gcc_toolchain_path else '')
- # These affect how C/C++ files are compiled, but not Rust libs/exes.
- RUSTENV['CFLAGS'] += f' {gcc_toolchain_flag}'
- RUSTENV['CXXFLAGS'] += f' {gcc_toolchain_flag}'
- RUSTENV['LDFLAGS'] += f' {gcc_toolchain_flag}'
- # These affect how Rust crates are built. A `-Clink-arg=<foo>` arg passes
- # foo to the clang invocation used to link.
- #
- # TODO(https://crbug.com/1281664): remove --no-gc-sections argument.
- # Workaround for a bug causing std::env::args() to return an empty list,
- # making Rust binaries unable to take command line arguments. Fix is landed
- # upstream in LLVM but hasn't rolled into Chromium. Also see:
- # * https://github.com/rust-lang/rust/issues/92181
- # * https://reviews.llvm.org/D116528
- RUSTENV['RUSTFLAGS_BOOTSTRAP'] = (
- f'-Clinker={clang_path} -Clink-arg=-fuse-ld=lld '
- f'-Clink-arg=-Wl,--no-gc-sections -Clink-arg={gcc_toolchain_flag} '
- f'-L native={gcc_toolchain_path}/lib64')
- RUSTENV['RUSTFLAGS_NOT_BOOTSTRAP'] = RUSTENV['RUSTFLAGS_BOOTSTRAP']
- os.chdir(RUST_SRC_DIR)
- cmd = [sys.executable, 'x.py', sub]
- if verbose and verbose > 0:
- cmd.append('-' + verbose * 'v')
- RunCommand(cmd + args, env=RUSTENV)
- # Get arguments to run desired test suites, minus disabled tests.
- def GetTestArgs():
- args = TEST_SUITES
- for excluded in EXCLUDED_TESTS:
- args.append('--skip')
- args.append(excluded)
- return args
- def main():
- parser = argparse.ArgumentParser(
- description='Build and package Rust toolchain')
- parser.add_argument('-v',
- '--verbose',
- action='count',
- help='run subcommands with verbosity')
- parser.add_argument(
- '--verify-stage0-hash',
- action='store_true',
- help=
- 'checkout Rust, verify the stage0 hash, then quit without building. '
- 'Will print the actual hash if different than expected.')
- parser.add_argument(
- '--skip-checkout',
- action='store_true',
- help='skip Rust git checkout. Useful for trying local changes')
- parser.add_argument('--skip-clean',
- action='store_true',
- help='skip x.py clean step')
- parser.add_argument('--skip-test',
- action='store_true',
- help='skip running rustc and libstd tests')
- parser.add_argument('--skip-install',
- action='store_true',
- help='do not install to RUST_TOOLCHAIN_OUT_DIR')
- parser.add_argument(
- '--fetch-llvm-libs',
- action='store_true',
- help='fetch Clang/LLVM libs and extract into LLVM_BUILD_DIR. Useless '
- 'without --use-final-llvm-build-dir.')
- parser.add_argument(
- '--use-final-llvm-build-dir',
- action='store_true',
- help='use libs in LLVM_BUILD_DIR instead of LLVM_BOOTSTRAP_DIR. Useful '
- 'with --fetch-llvm-libs for local builds.')
- parser.add_argument(
- '--run-xpy',
- action='store_true',
- help='run x.py command in configured Rust checkout. Quits after '
- 'running specified command, skipping all normal build steps. For '
- 'debugging. Running x.py directly will not set the appropriate env '
- 'variables nor update config.toml')
- args, rest = parser.parse_known_args()
- # Get the LLVM root for libs. We use LLVM_BUILD_DIR tools either way.
- #
- # TODO(https://crbug.com/1245714): use LTO libs from LLVM_BUILD_DIR for
- # stage 2+.
- if args.use_final_llvm_build_dir:
- llvm_libs_root = LLVM_BUILD_DIR
- else:
- llvm_libs_root = build.LLVM_BOOTSTRAP_DIR
- if not args.skip_checkout:
- CheckoutRust(RUST_REVISION, RUST_SRC_DIR)
- VerifyStage0JsonHash()
- if args.verify_stage0_hash:
- # The above function exits and prints the actual hash if verification
- # failed so we just quit here; if we reach this point, the hash is
- # valid.
- return 0
- if args.fetch_llvm_libs:
- UpdatePackage('clang-libs', GetDefaultHostOs())
- # Fetch GCC package to build against same libstdc++ as Clang. This function
- # will only download it if necessary.
- args.gcc_toolchain = None
- build.MaybeDownloadHostGcc(args)
- # Set up config.toml in Rust source tree to configure build.
- Configure(llvm_libs_root)
- if args.run_xpy:
- if rest[0] == '--':
- rest = rest[1:]
- RunXPy(rest[0], rest[1:], args.gcc_toolchain, args.verbose)
- return 0
- else:
- assert not rest
- # Delete vendored sources and .cargo subdir. Otherwise when updating an
- # existing checkout, vendored sources will not be re-fetched leaving deps
- # out of date.
- if not args.skip_checkout:
- for dir in [
- os.path.join(RUST_SRC_DIR, d) for d in ['vendor', '.cargo']
- ]:
- if os.path.exists(dir):
- shutil.rmtree(dir)
- if not args.skip_clean:
- print('Cleaning build artifacts...')
- RunXPy('clean', [], args.gcc_toolchain, args.verbose)
- if not args.skip_test:
- print('Running stage 2 tests...')
- # Run a subset of tests. Tell x.py to keep the rustc we already built.
- RunXPy('test', GetTestArgs(), args.gcc_toolchain, args.verbose)
- targets = [
- 'library/proc_macro', 'library/std', 'src/tools/cargo',
- 'src/tools/clippy', 'src/tools/rustfmt'
- ]
- # Build stage 2 compiler, tools, and libraries. This should reuse earlier
- # stages from the test command (if run).
- print('Building stage 2 artifacts...')
- RunXPy('build', ['--stage', '2'] + targets, args.gcc_toolchain,
- args.verbose)
- if args.skip_install:
- # Rust is fully built. We can quit.
- return 0
- print(f'Installing to {RUST_TOOLCHAIN_OUT_DIR} ...')
- # Clean output directory.
- if os.path.exists(RUST_TOOLCHAIN_OUT_DIR):
- shutil.rmtree(RUST_TOOLCHAIN_OUT_DIR)
- RunXPy('install', DISTRIBUTION_ARTIFACTS, args.gcc_toolchain, args.verbose)
- # Write expected `rustc --version` string to our toolchain directory.
- with open(RUST_SRC_VERSION_FILE_PATH) as version_file:
- rust_version = version_file.readline().rstrip()
- with open(VERSION_STAMP_PATH, 'w') as stamp:
- stamp.write('rustc %s-dev (%s chromium)\n' %
- (rust_version, GetPackageVersion()))
- return 0
- # TODO(crbug.com/1342708): fix vendoring and re-enable.
- # x.py installed library sources to our toolchain directory. We also need to
- # copy the vendor directory so Chromium checkouts have all the deps needed
- # to build std.
- # shutil.copytree(RUST_SRC_VENDOR_DIR, RUST_TOOLCHAIN_SRC_DIST_VENDOR_DIR)
- if __name__ == '__main__':
- sys.exit(main())
|