12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307 |
- #!/usr/bin/env python3
- # Copyright 2019 The Chromium Authors. All rights reserved.
- # Use of this source code is governed by a BSD-style license that can be
- # found in the LICENSE file.
- """This script is used to build clang binaries. It is used by package.py to
- create the prebuilt binaries downloaded by update.py and used by developers.
- The expectation is that update.py downloads prebuilt binaries for everyone, and
- nobody should run this script as part of normal development.
- """
- from __future__ import print_function
- import argparse
- import glob
- import io
- import json
- import os
- import pipes
- import platform
- import re
- import shutil
- import subprocess
- import sys
- import tempfile
- from update import (CDS_URL, CHROMIUM_DIR, CLANG_REVISION, LLVM_BUILD_DIR,
- FORCE_HEAD_REVISION_FILE, PACKAGE_VERSION, RELEASE_VERSION,
- STAMP_FILE, DownloadUrl, DownloadAndUnpack, EnsureDirExists,
- ReadStampFile, RmTree, WriteStampFile)
- # Path constants. (All of these should be absolute paths.)
- THIRD_PARTY_DIR = os.path.join(CHROMIUM_DIR, 'third_party')
- LLVM_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm')
- COMPILER_RT_DIR = os.path.join(LLVM_DIR, 'compiler-rt')
- LLVM_BOOTSTRAP_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-bootstrap')
- LLVM_BOOTSTRAP_INSTALL_DIR = os.path.join(THIRD_PARTY_DIR,
- 'llvm-bootstrap-install')
- LLVM_INSTRUMENTED_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-instrumented')
- LLVM_PROFDATA_FILE = os.path.join(LLVM_INSTRUMENTED_DIR, 'profdata.prof')
- LLVM_BUILD_TOOLS_DIR = os.path.abspath(
- os.path.join(LLVM_DIR, '..', 'llvm-build-tools'))
- ANDROID_NDK_DIR = os.path.join(
- CHROMIUM_DIR, 'third_party', 'android_ndk')
- FUCHSIA_SDK_DIR = os.path.join(CHROMIUM_DIR, 'third_party', 'fuchsia-sdk',
- 'sdk')
- PINNED_CLANG_DIR = os.path.join(LLVM_BUILD_TOOLS_DIR, 'pinned-clang')
- BUG_REPORT_URL = ('https://crbug.com and run'
- ' tools/clang/scripts/process_crashreports.py'
- ' (only works inside Google) which will upload a report')
- win_sdk_dir = None
- def GetWinSDKDir():
- """Get the location of the current SDK."""
- global win_sdk_dir
- if win_sdk_dir:
- return win_sdk_dir
- # Don't let vs_toolchain overwrite our environment.
- environ_bak = os.environ
- sys.path.append(os.path.join(CHROMIUM_DIR, 'build'))
- import vs_toolchain
- win_sdk_dir = vs_toolchain.SetEnvironmentAndGetSDKDir()
- msvs_version = vs_toolchain.GetVisualStudioVersion()
- if bool(int(os.environ.get('DEPOT_TOOLS_WIN_TOOLCHAIN', '1'))):
- dia_path = os.path.join(win_sdk_dir, '..', 'DIA SDK', 'bin', 'amd64')
- else:
- if 'GYP_MSVS_OVERRIDE_PATH' not in os.environ:
- vs_path = vs_toolchain.DetectVisualStudioPath()
- else:
- vs_path = os.environ['GYP_MSVS_OVERRIDE_PATH']
- dia_path = os.path.join(vs_path, 'DIA SDK', 'bin', 'amd64')
- os.environ = environ_bak
- return win_sdk_dir
- def RunCommand(command, msvc_arch=None, env=None, fail_hard=True):
- """Run command and return success (True) or failure; or if fail_hard is
- True, exit on failure. If msvc_arch is set, runs the command in a
- shell with the msvc tools for that architecture."""
- if msvc_arch and sys.platform == 'win32':
- command = [os.path.join(GetWinSDKDir(), 'bin', 'SetEnv.cmd'),
- "/" + msvc_arch, '&&'] + command
- # https://docs.python.org/2/library/subprocess.html:
- # "On Unix with shell=True [...] if args is a sequence, the first item
- # specifies the command string, and any additional items will be treated as
- # additional arguments to the shell itself. That is to say, Popen does the
- # equivalent of:
- # Popen(['/bin/sh', '-c', args[0], args[1], ...])"
- #
- # We want to pass additional arguments to command[0], not to the shell,
- # so manually join everything into a single string.
- # Annoyingly, for "svn co url c:\path", pipes.quote() thinks that it should
- # quote c:\path but svn can't handle quoted paths on Windows. Since on
- # Windows follow-on args are passed to args[0] instead of the shell, don't
- # do the single-string transformation there.
- if sys.platform != 'win32':
- command = ' '.join([pipes.quote(c) for c in command])
- print('Running', command)
- if subprocess.call(command, env=env, shell=True) == 0:
- return True
- print('Failed.')
- if fail_hard:
- sys.exit(1)
- return False
- def CopyFile(src, dst):
- """Copy a file from src to dst."""
- print("Copying %s to %s" % (src, dst))
- shutil.copy(src, dst)
- def CopyDirectoryContents(src, dst):
- """Copy the files from directory src to dst."""
- dst = os.path.realpath(dst) # realpath() in case dst ends in /..
- EnsureDirExists(dst)
- for f in os.listdir(src):
- CopyFile(os.path.join(src, f), dst)
- def CheckoutLLVM(commit, dir):
- """Checkout the LLVM monorepo at a certain git commit in dir. Any local
- modifications in dir will be lost."""
- print('Checking out LLVM monorepo %s into %s' % (commit, dir))
- # Try updating the current repo if it exists and has no local diff.
- if os.path.isdir(dir):
- os.chdir(dir)
- # git diff-index --quiet returns success when there is no diff.
- # Also check that the first commit is reachable.
- if (RunCommand(['git', 'diff-index', '--quiet', 'HEAD'], fail_hard=False)
- and RunCommand(['git', 'fetch'], fail_hard=False)
- and RunCommand(['git', 'checkout', commit], fail_hard=False)):
- return
- # If we can't use the current repo, delete it.
- os.chdir(CHROMIUM_DIR) # Can't remove dir if we're in it.
- print('Removing %s.' % dir)
- RmTree(dir)
- clone_cmd = ['git', 'clone', 'https://github.com/llvm/llvm-project/', dir]
- if RunCommand(clone_cmd, fail_hard=False):
- os.chdir(dir)
- if RunCommand(['git', 'checkout', commit], fail_hard=False):
- return
- print('CheckoutLLVM failed.')
- sys.exit(1)
- def UrlOpen(url):
- # TODO(crbug.com/1067752): Use urllib once certificates are fixed.
- return subprocess.check_output(['curl', '--silent', url],
- universal_newlines=True)
- def GetLatestLLVMCommit():
- """Get the latest commit hash in the LLVM monorepo."""
- ref = json.loads(
- UrlOpen(('https://api.github.com/repos/'
- 'llvm/llvm-project/git/refs/heads/main')))
- assert ref['object']['type'] == 'commit'
- return ref['object']['sha']
- def GetCommitDescription(commit):
- """Get the output of `git describe`.
- Needs to be called from inside the git repository dir."""
- git_exe = 'git.bat' if sys.platform.startswith('win') else 'git'
- return subprocess.check_output(
- [git_exe, 'describe', '--long', '--abbrev=8', commit],
- universal_newlines=True).rstrip()
- def DeleteChromeToolsShim():
- # TODO: These dirs are no longer used. Remove this code after a while.
- OLD_SHIM_DIR = os.path.join(LLVM_DIR, 'tools', 'zzz-chrometools')
- shutil.rmtree(OLD_SHIM_DIR, ignore_errors=True)
- CHROME_TOOLS_SHIM_DIR = os.path.join(LLVM_DIR, 'llvm', 'tools', 'chrometools')
- shutil.rmtree(CHROME_TOOLS_SHIM_DIR, ignore_errors=True)
- def AddCMakeToPath(args):
- """Download CMake and add it to PATH."""
- if args.use_system_cmake:
- return
- if sys.platform == 'win32':
- zip_name = 'cmake-3.23.0-windows-x86_64.zip'
- dir_name = ['cmake-3.23.0-windows-x86_64', 'bin']
- elif sys.platform == 'darwin':
- zip_name = 'cmake-3.23.0-macos-universal.tar.gz'
- dir_name = ['cmake-3.23.0-macos-universal', 'CMake.app', 'Contents', 'bin']
- else:
- zip_name = 'cmake-3.23.0-linux-x86_64.tar.gz'
- dir_name = ['cmake-3.23.0-linux-x86_64', 'bin']
- cmake_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, *dir_name)
- if not os.path.exists(cmake_dir):
- DownloadAndUnpack(CDS_URL + '/tools/' + zip_name, LLVM_BUILD_TOOLS_DIR)
- os.environ['PATH'] = cmake_dir + os.pathsep + os.environ.get('PATH', '')
- def AddGnuWinToPath():
- """Download some GNU win tools and add them to PATH."""
- if sys.platform != 'win32':
- return
- gnuwin_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'gnuwin')
- GNUWIN_VERSION = '14'
- GNUWIN_STAMP = os.path.join(gnuwin_dir, 'stamp')
- if ReadStampFile(GNUWIN_STAMP) == GNUWIN_VERSION:
- print('GNU Win tools already up to date.')
- else:
- zip_name = 'gnuwin-%s.zip' % GNUWIN_VERSION
- DownloadAndUnpack(CDS_URL + '/tools/' + zip_name, LLVM_BUILD_TOOLS_DIR)
- WriteStampFile(GNUWIN_VERSION, GNUWIN_STAMP)
- os.environ['PATH'] = gnuwin_dir + os.pathsep + os.environ.get('PATH', '')
- # find.exe, mv.exe and rm.exe are from MSYS (see crrev.com/389632). MSYS uses
- # Cygwin under the hood, and initializing Cygwin has a race-condition when
- # getting group and user data from the Active Directory is slow. To work
- # around this, use a horrible hack telling it not to do that.
- # See https://crbug.com/905289
- etc = os.path.join(gnuwin_dir, '..', '..', 'etc')
- EnsureDirExists(etc)
- with open(os.path.join(etc, 'nsswitch.conf'), 'w') as f:
- f.write('passwd: files\n')
- f.write('group: files\n')
- def AddZlibToPath():
- """Download and build zlib, and add to PATH."""
- zlib_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'zlib-1.2.11')
- if os.path.exists(zlib_dir):
- RmTree(zlib_dir)
- zip_name = 'zlib-1.2.11.tar.gz'
- DownloadAndUnpack(CDS_URL + '/tools/' + zip_name, LLVM_BUILD_TOOLS_DIR)
- os.chdir(zlib_dir)
- zlib_files = [
- 'adler32', 'compress', 'crc32', 'deflate', 'gzclose', 'gzlib', 'gzread',
- 'gzwrite', 'inflate', 'infback', 'inftrees', 'inffast', 'trees',
- 'uncompr', 'zutil'
- ]
- cl_flags = [
- '/nologo', '/O2', '/DZLIB_DLL', '/c', '/D_CRT_SECURE_NO_DEPRECATE',
- '/D_CRT_NONSTDC_NO_DEPRECATE'
- ]
- RunCommand(
- ['cl.exe'] + [f + '.c' for f in zlib_files] + cl_flags, msvc_arch='x64')
- RunCommand(
- ['lib.exe'] + [f + '.obj'
- for f in zlib_files] + ['/nologo', '/out:zlib.lib'],
- msvc_arch='x64')
- # Remove the test directory so it isn't found when trying to find
- # test.exe.
- shutil.rmtree('test')
- os.environ['PATH'] = zlib_dir + os.pathsep + os.environ.get('PATH', '')
- return zlib_dir
- def BuildLibXml2():
- """Download and build libxml2"""
- # The .tar.gz on GCS was uploaded as follows.
- # The gitlab page has more up-to-date packages than http://xmlsoft.org/,
- # and the official releases on xmlsoft.org are only available over ftp too.
- # $ VER=v2.9.12
- # $ curl -O \
- # https://gitlab.gnome.org/GNOME/libxml2/-/archive/$VER/libxml2-$VER.tar.gz
- # $ gsutil cp -n -a public-read libxml2-$VER.tar.gz \
- # gs://chromium-browser-clang/tools
- libxml2_version = 'libxml2-v2.9.12'
- libxml2_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, libxml2_version)
- if os.path.exists(libxml2_dir):
- RmTree(libxml2_dir)
- zip_name = libxml2_version + '.tar.gz'
- DownloadAndUnpack(CDS_URL + '/tools/' + zip_name, LLVM_BUILD_TOOLS_DIR)
- os.chdir(libxml2_dir)
- os.mkdir('build')
- os.chdir('build')
- libxml2_install_dir = os.path.join(libxml2_dir, 'build', 'install')
- # Disable everything except WITH_TREE and WITH_OUTPUT, both needed by LLVM's
- # WindowsManifestMerger.
- # Also enable WITH_THREADS, else libxml doesn't compile on Linux.
- RunCommand(
- [
- 'cmake',
- '-GNinja',
- '-DCMAKE_BUILD_TYPE=Release',
- '-DCMAKE_INSTALL_PREFIX=install',
- # The mac_arm bot builds a clang arm binary, but currently on an intel
- # host. If we ever move it to run on an arm mac, this can go. We
- # could pass this only if args.build_mac_arm, but libxml is small, so
- # might as well build it universal always for a few years.
- '-DCMAKE_OSX_ARCHITECTURES=arm64;x86_64',
- '-DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded', # /MT to match LLVM.
- '-DBUILD_SHARED_LIBS=OFF',
- '-DLIBXML2_WITH_C14N=OFF',
- '-DLIBXML2_WITH_CATALOG=OFF',
- '-DLIBXML2_WITH_DEBUG=OFF',
- '-DLIBXML2_WITH_DOCB=OFF',
- '-DLIBXML2_WITH_FTP=OFF',
- '-DLIBXML2_WITH_HTML=OFF',
- '-DLIBXML2_WITH_HTTP=OFF',
- '-DLIBXML2_WITH_ICONV=OFF',
- '-DLIBXML2_WITH_ICU=OFF',
- '-DLIBXML2_WITH_ISO8859X=OFF',
- '-DLIBXML2_WITH_LEGACY=OFF',
- '-DLIBXML2_WITH_LZMA=OFF',
- '-DLIBXML2_WITH_MEM_DEBUG=OFF',
- '-DLIBXML2_WITH_MODULES=OFF',
- '-DLIBXML2_WITH_OUTPUT=ON',
- '-DLIBXML2_WITH_PATTERN=OFF',
- '-DLIBXML2_WITH_PROGRAMS=OFF',
- '-DLIBXML2_WITH_PUSH=OFF',
- '-DLIBXML2_WITH_PYTHON=OFF',
- '-DLIBXML2_WITH_READER=OFF',
- '-DLIBXML2_WITH_REGEXPS=OFF',
- '-DLIBXML2_WITH_RUN_DEBUG=OFF',
- '-DLIBXML2_WITH_SAX1=OFF',
- '-DLIBXML2_WITH_SCHEMAS=OFF',
- '-DLIBXML2_WITH_SCHEMATRON=OFF',
- '-DLIBXML2_WITH_TESTS=OFF',
- '-DLIBXML2_WITH_THREADS=ON',
- '-DLIBXML2_WITH_THREAD_ALLOC=OFF',
- '-DLIBXML2_WITH_TREE=ON',
- '-DLIBXML2_WITH_VALID=OFF',
- '-DLIBXML2_WITH_WRITER=OFF',
- '-DLIBXML2_WITH_XINCLUDE=OFF',
- '-DLIBXML2_WITH_XPATH=OFF',
- '-DLIBXML2_WITH_XPTR=OFF',
- '-DLIBXML2_WITH_ZLIB=OFF',
- '..',
- ],
- msvc_arch='x64')
- RunCommand(['ninja', 'install'], msvc_arch='x64')
- libxml2_include_dir = os.path.join(libxml2_install_dir, 'include', 'libxml2')
- if sys.platform == 'win32':
- libxml2_lib = os.path.join(libxml2_install_dir, 'lib', 'libxml2s.lib')
- else:
- libxml2_lib = os.path.join(libxml2_install_dir, 'lib', 'libxml2.a')
- extra_cmake_flags = [
- '-DLLVM_ENABLE_LIBXML2=FORCE_ON',
- '-DLIBXML2_INCLUDE_DIR=' + libxml2_include_dir.replace('\\', '/'),
- '-DLIBXML2_LIBRARIES=' + libxml2_lib.replace('\\', '/'),
- ]
- extra_cflags = ['-DLIBXML_STATIC']
- return extra_cmake_flags, extra_cflags
- def DownloadRPMalloc():
- """Download rpmalloc."""
- rpmalloc_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'rpmalloc')
- if os.path.exists(rpmalloc_dir):
- RmTree(rpmalloc_dir)
- # Using rpmalloc bc1923f rather than the latest release (1.4.1) because
- # it contains the fix for https://github.com/mjansson/rpmalloc/pull/186
- # which would cause lld to deadlock.
- # The zip file was created and uploaded as follows:
- # $ mkdir rpmalloc
- # $ curl -L https://github.com/mjansson/rpmalloc/archive/bc1923f436539327707b08ef9751a7a87bdd9d2f.tar.gz \
- # | tar -C rpmalloc --strip-components=1 -xzf -
- # $ GZIP=-9 tar vzcf rpmalloc-bc1923f.tgz rpmalloc
- # $ gsutil.py cp -n -a public-read rpmalloc-bc1923f.tgz \
- # gs://chromium-browser-clang/tools/
- zip_name = 'rpmalloc-bc1923f.tgz'
- DownloadAndUnpack(CDS_URL + '/tools/' + zip_name, LLVM_BUILD_TOOLS_DIR)
- rpmalloc_dir = rpmalloc_dir.replace('\\', '/')
- return rpmalloc_dir
- def DownloadPinnedClang():
- # The update.py in this current revision may have a patched revision while
- # building new clang packages. Get update.py off HEAD~ to pull the current
- # pinned clang.
- with tempfile.NamedTemporaryFile() as f:
- subprocess.check_call(
- ['git', 'show', 'HEAD~:tools/clang/scripts/update.py'],
- stdout=f,
- cwd=CHROMIUM_DIR)
- print("Running update.py")
- # Without the flush, the subprocess call below doesn't work.
- f.flush()
- subprocess.check_call(
- [sys.executable, f.name, '--output-dir=' + PINNED_CLANG_DIR])
- # TODO(crbug.com/929645): Remove once we don't need gcc's libstdc++.
- def MaybeDownloadHostGcc(args):
- """Download a modern GCC host compiler on Linux."""
- assert sys.platform.startswith('linux')
- if args.gcc_toolchain:
- return
- gcc_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'gcc-10.2.0-trusty')
- if not os.path.exists(gcc_dir):
- DownloadAndUnpack(CDS_URL + '/tools/gcc-10.2.0-trusty.tgz', gcc_dir)
- args.gcc_toolchain = gcc_dir
- def VerifyVersionOfBuiltClangMatchesVERSION():
- """Checks that `clang --version` outputs RELEASE_VERSION. If this
- fails, update.RELEASE_VERSION is out-of-date and needs to be updated (possibly
- in an `if args.llvm_force_head_revision:` block inupdate. main() first)."""
- clang = os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')
- if sys.platform == 'win32':
- clang += '-cl.exe'
- version_out = subprocess.check_output([clang, '--version'],
- universal_newlines=True)
- version_out = re.match(r'clang version ([0-9.]+)', version_out).group(1)
- if version_out != RELEASE_VERSION:
- print(('unexpected clang version %s (not %s), '
- 'update RELEASE_VERSION in update.py')
- % (version_out, RELEASE_VERSION))
- sys.exit(1)
- def VerifyZlibSupport():
- """Check that clang was built with zlib support enabled."""
- clang = os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')
- test_file = '/dev/null'
- if sys.platform == 'win32':
- clang += '.exe'
- test_file = 'nul'
- print('Checking for zlib support')
- clang_out = subprocess.check_output([
- clang, '-target', 'x86_64-unknown-linux-gnu', '-gz', '-c', '-###', '-x',
- 'c', test_file
- ],
- stderr=subprocess.STDOUT,
- universal_newlines=True)
- if (re.search(r'--compress-debug-sections', clang_out)):
- print('OK')
- else:
- print(('Failed to detect zlib support!\n\n(driver output: %s)') % clang_out)
- sys.exit(1)
- # TODO(https://crbug.com/1286289): remove once Chrome targets don't rely on
- # libstdc++.so existing in the clang package.
- def CopyLibstdcpp(args, build_dir):
- if not args.gcc_toolchain:
- return
- # Find libstdc++.so.6
- libstdcpp = subprocess.check_output([
- os.path.join(args.gcc_toolchain, 'bin', 'g++'),
- '-print-file-name=libstdc++.so.6'
- ],
- universal_newlines=True).rstrip()
- EnsureDirExists(os.path.join(build_dir, 'lib'))
- CopyFile(libstdcpp, os.path.join(build_dir, 'lib'))
- def gn_arg(v):
- if v == 'True':
- return True
- if v == 'False':
- return False
- raise argparse.ArgumentTypeError('Expected one of %r or %r' % (
- 'True', 'False'))
- def main():
- parser = argparse.ArgumentParser(description='Build Clang.')
- parser.add_argument('--bootstrap', action='store_true',
- help='first build clang with CC, then with itself.')
- parser.add_argument('--build-mac-arm', action='store_true',
- help='Build arm binaries. Only valid on macOS.')
- parser.add_argument('--disable-asserts', action='store_true',
- help='build with asserts disabled')
- parser.add_argument('--gcc-toolchain', help='what gcc toolchain to use for '
- 'building; --gcc-toolchain=/opt/foo picks '
- '/opt/foo/bin/gcc')
- parser.add_argument('--pgo', action='store_true', help='build with PGO')
- parser.add_argument('--thinlto',
- action='store_true',
- help='build with ThinLTO')
- parser.add_argument('--llvm-force-head-revision', action='store_true',
- help='build the latest revision')
- parser.add_argument('--run-tests', action='store_true',
- help='run tests after building')
- parser.add_argument('--skip-build', action='store_true',
- help='do not build anything')
- parser.add_argument('--skip-checkout', action='store_true',
- help='do not create or update any checkouts')
- parser.add_argument('--build-dir',
- help='Override build directory')
- parser.add_argument('--extra-tools', nargs='*', default=[],
- help='select additional chrome tools to build')
- parser.add_argument('--use-system-cmake', action='store_true',
- help='use the cmake from PATH instead of downloading '
- 'and using prebuilt cmake binaries')
- parser.add_argument('--with-android', type=gn_arg, nargs='?', const=True,
- help='build the Android ASan runtime (linux only)',
- default=sys.platform.startswith('linux'))
- parser.add_argument('--with-fuchsia',
- type=gn_arg,
- nargs='?',
- const=True,
- help='build the Fuchsia runtimes (linux and mac only)',
- default=sys.platform.startswith('linux')
- or sys.platform.startswith('darwin'))
- parser.add_argument('--without-android', action='store_false',
- help='don\'t build Android ASan runtime (linux only)',
- dest='with_android')
- parser.add_argument('--without-fuchsia', action='store_false',
- help='don\'t build Fuchsia clang_rt runtime (linux/mac)',
- dest='with_fuchsia',
- default=sys.platform in ('linux2', 'darwin'))
- args = parser.parse_args()
- global CLANG_REVISION, PACKAGE_VERSION, LLVM_BUILD_DIR
- if (args.pgo or args.thinlto) and not args.bootstrap:
- print('--pgo/--thinlto requires --bootstrap')
- return 1
- if args.with_android and not os.path.exists(ANDROID_NDK_DIR):
- print('Android NDK not found at ' + ANDROID_NDK_DIR)
- print('The Android NDK is needed to build a Clang whose -fsanitize=address')
- print('works on Android. See ')
- print('https://www.chromium.org/developers/how-tos/android-build-instructions')
- print('for how to install the NDK, or pass --without-android.')
- return 1
- if args.with_fuchsia and not os.path.exists(FUCHSIA_SDK_DIR):
- print('Fuchsia SDK not found at ' + FUCHSIA_SDK_DIR)
- print('The Fuchsia SDK is needed to build libclang_rt for Fuchsia.')
- print('Install the Fuchsia SDK by adding fuchsia to the ')
- print('target_os section in your .gclient and running hooks, ')
- print('or pass --without-fuchsia.')
- print(
- 'https://chromium.googlesource.com/chromium/src/+/main/docs/fuchsia/build_instructions.md'
- )
- print('for general Fuchsia build instructions.')
- return 1
- if args.build_mac_arm and sys.platform != 'darwin':
- print('--build-mac-arm only valid on macOS')
- return 1
- if args.build_mac_arm and platform.machine() == 'arm64':
- print('--build-mac-arm only valid on intel to cross-build arm')
- return 1
- # Don't buffer stdout, so that print statements are immediately flushed.
- # LLVM tests print output without newlines, so with buffering they won't be
- # immediately printed.
- major, _, _, _, _ = sys.version_info
- if major == 3:
- # Python3 only allows unbuffered output for binary streams. This
- # workaround comes from https://stackoverflow.com/a/181654/4052492.
- sys.stdout = io.TextIOWrapper(open(sys.stdout.fileno(), 'wb', 0),
- write_through=True)
- else:
- sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
- # The gnuwin package also includes curl, which is needed to interact with the
- # github API below.
- # TODO(crbug.com/1067752): Use urllib once certificates are fixed, and
- # move this down to where we fetch other build tools.
- AddGnuWinToPath()
- if sys.platform == 'darwin':
- isysroot = subprocess.check_output(['xcrun', '--show-sdk-path'],
- universal_newlines=True).rstrip()
- if args.build_dir:
- LLVM_BUILD_DIR = args.build_dir
- if args.llvm_force_head_revision:
- checkout_revision = GetLatestLLVMCommit()
- else:
- checkout_revision = CLANG_REVISION
- if not args.skip_checkout:
- CheckoutLLVM(checkout_revision, LLVM_DIR)
- if args.llvm_force_head_revision:
- CLANG_REVISION = GetCommitDescription(checkout_revision)
- PACKAGE_VERSION = '%s-0' % CLANG_REVISION
- print('Locally building clang %s...' % PACKAGE_VERSION)
- WriteStampFile('', STAMP_FILE)
- WriteStampFile('', FORCE_HEAD_REVISION_FILE)
- AddCMakeToPath(args)
- DeleteChromeToolsShim()
- if args.skip_build:
- return 0
- # The variable "lld" is only used on Windows because only there does setting
- # CMAKE_LINKER have an effect: On Windows, the linker is called directly,
- # while elsewhere it's called through the compiler driver, and we pass
- # -fuse-ld=lld there to make the compiler driver call the linker (by setting
- # LLVM_ENABLE_LLD).
- cc, cxx, lld = None, None, None
- cflags = []
- cxxflags = []
- ldflags = []
- targets = 'AArch64;ARM;Mips;PowerPC;RISCV;SystemZ;WebAssembly;X86'
- projects = 'clang;compiler-rt;lld;clang-tools-extra'
- if sys.platform == 'darwin':
- # clang needs libc++, else -stdlib=libc++ won't find includes
- # (this is needed for bootstrap builds and for building the fuchsia runtime)
- projects += ';libcxx'
- base_cmake_args = [
- '-GNinja',
- '-DCMAKE_BUILD_TYPE=Release',
- '-DLLVM_ENABLE_ASSERTIONS=%s' % ('OFF' if args.disable_asserts else 'ON'),
- '-DLLVM_ENABLE_PROJECTS=' + projects,
- '-DLLVM_TARGETS_TO_BUILD=' + targets,
- # PIC needed for Rust build (links LLVM into shared object)
- '-DLLVM_ENABLE_PIC=ON',
- '-DLLVM_ENABLE_UNWIND_TABLES=OFF',
- '-DLLVM_ENABLE_TERMINFO=OFF',
- '-DLLVM_ENABLE_Z3_SOLVER=OFF',
- '-DCLANG_PLUGIN_SUPPORT=OFF',
- '-DCLANG_ENABLE_STATIC_ANALYZER=OFF',
- '-DCLANG_ENABLE_ARCMT=OFF',
- '-DBUG_REPORT_URL=' + BUG_REPORT_URL,
- # Don't run Go bindings tests; PGO makes them confused.
- '-DLLVM_INCLUDE_GO_TESTS=OFF',
- # TODO(crbug.com/1113475): Update binutils.
- '-DENABLE_X86_RELAX_RELOCATIONS=NO',
- # See crbug.com/1126219: Use native symbolizer instead of DIA
- '-DLLVM_ENABLE_DIA_SDK=OFF',
- # StarFive: Required for native RISCV build
- '-DSANITIZER_COMMON_LINK_LIBS=-latomic',
- # See crbug.com/1205046: don't build scudo (and others we don't need).
- '-DCOMPILER_RT_SANITIZERS_TO_BUILD=asan;dfsan;msan;hwasan;tsan;cfi',
- # The default value differs per platform, force it off everywhere.
- '-DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF',
- # Don't use curl.
- '-DLLVM_ENABLE_CURL=OFF',
- # Build libclang.a as well as libclang.so
- '-DLIBCLANG_BUILD_STATIC=ON',
- ]
- if sys.platform.startswith('linux'):
- MaybeDownloadHostGcc(args)
- DownloadPinnedClang()
- cc = os.path.join(PINNED_CLANG_DIR, 'bin', 'clang')
- cxx = os.path.join(PINNED_CLANG_DIR, 'bin', 'clang++')
- # Use the libraries in the specified gcc installation for building.
- cflags.append('--gcc-toolchain=' + args.gcc_toolchain)
- cxxflags.append('--gcc-toolchain=' + args.gcc_toolchain)
- base_cmake_args += [
- # The host clang has lld.
- '-DLLVM_ENABLE_LLD=ON',
- '-DLLVM_STATIC_LINK_CXX_STDLIB=ON',
- # Force compiler-rt tests to use our gcc toolchain
- # because the one on the host may be too old.
- # Even with -static-libstdc++ the compiler-rt tests add -lstdc++
- # which adds a DT_NEEDED to libstdc++.so so we need to add RPATHs
- # to the gcc toolchain.
- '-DCOMPILER_RT_TEST_COMPILER_CFLAGS=--gcc-toolchain=' +
- args.gcc_toolchain + ' -Wl,-rpath,' +
- os.path.join(args.gcc_toolchain, 'lib64') + ' -Wl,-rpath,' +
- os.path.join(args.gcc_toolchain, 'lib32')
- ]
- if sys.platform == 'darwin':
- # For libc++, we only want the headers.
- base_cmake_args.extend([
- '-DLIBCXX_ENABLE_SHARED=OFF',
- '-DLIBCXX_ENABLE_STATIC=OFF',
- '-DLIBCXX_INCLUDE_TESTS=OFF',
- '-DLIBCXX_ENABLE_EXPERIMENTAL_LIBRARY=OFF',
- ])
- if sys.platform == 'win32':
- base_cmake_args.append('-DLLVM_USE_CRT_RELEASE=MT')
- # Require zlib compression.
- zlib_dir = AddZlibToPath()
- cflags.append('-I' + zlib_dir)
- cxxflags.append('-I' + zlib_dir)
- ldflags.append('-LIBPATH:' + zlib_dir)
- # Use rpmalloc. For faster ThinLTO linking.
- rpmalloc_dir = DownloadRPMalloc()
- base_cmake_args.append('-DLLVM_INTEGRATED_CRT_ALLOC=' + rpmalloc_dir)
- # Statically link libxml2 to make lld-link not require mt.exe on Windows,
- # and to make sure lld-link output on other platforms is identical to
- # lld-link on Windows (for cross-builds).
- libxml_cmake_args, libxml_cflags = BuildLibXml2()
- base_cmake_args += libxml_cmake_args
- cflags += libxml_cflags
- cxxflags += libxml_cflags
- if args.bootstrap:
- print('Building bootstrap compiler')
- if os.path.exists(LLVM_BOOTSTRAP_DIR):
- RmTree(LLVM_BOOTSTRAP_DIR)
- EnsureDirExists(LLVM_BOOTSTRAP_DIR)
- os.chdir(LLVM_BOOTSTRAP_DIR)
- projects = 'clang'
- if args.pgo:
- # Need libclang_rt.profile
- projects += ';compiler-rt'
- if sys.platform != 'darwin':
- projects += ';lld'
- if sys.platform == 'darwin':
- # Need libc++ and compiler-rt for the bootstrap compiler on mac.
- projects += ';libcxx;compiler-rt'
- bootstrap_targets = 'X86'
- if sys.platform == 'darwin':
- # Need ARM and AArch64 for building the ios clang_rt.
- bootstrap_targets += ';ARM;AArch64'
- bootstrap_args = base_cmake_args + [
- '-DLLVM_TARGETS_TO_BUILD=' + bootstrap_targets,
- '-DLLVM_ENABLE_PROJECTS=' + projects,
- '-DCMAKE_INSTALL_PREFIX=' + LLVM_BOOTSTRAP_INSTALL_DIR,
- '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
- '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
- '-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(ldflags),
- '-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(ldflags),
- '-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(ldflags),
- # Ignore args.disable_asserts for the bootstrap compiler.
- '-DLLVM_ENABLE_ASSERTIONS=ON',
- ]
- if sys.platform == 'darwin':
- # On macOS, the bootstrap toolchain needs to have compiler-rt because
- # dsymutil's link needs libclang_rt.osx.a. Only the x86_64 osx
- # libraries are needed though, and only libclang_rt (i.e.
- # COMPILER_RT_BUILD_BUILTINS).
- bootstrap_args.extend([
- '-DCOMPILER_RT_BUILD_BUILTINS=ON',
- '-DCOMPILER_RT_BUILD_CRT=OFF',
- '-DCOMPILER_RT_BUILD_LIBFUZZER=OFF',
- '-DCOMPILER_RT_BUILD_MEMPROF=OFF',
- '-DCOMPILER_RT_BUILD_ORC=OFF',
- '-DCOMPILER_RT_BUILD_SANITIZERS=OFF',
- '-DCOMPILER_RT_BUILD_XRAY=OFF',
- '-DCOMPILER_RT_ENABLE_IOS=OFF',
- '-DCOMPILER_RT_ENABLE_WATCHOS=OFF',
- '-DCOMPILER_RT_ENABLE_TVOS=OFF',
- ])
- if platform.machine() == 'arm64':
- bootstrap_args.extend(['-DDARWIN_osx_ARCHS=arm64'])
- else:
- bootstrap_args.extend(['-DDARWIN_osx_ARCHS=x86_64'])
- elif args.pgo:
- # PGO needs libclang_rt.profile but none of the other compiler-rt stuff.
- bootstrap_args.extend([
- '-DCOMPILER_RT_BUILD_BUILTINS=OFF',
- '-DCOMPILER_RT_BUILD_CRT=OFF',
- '-DCOMPILER_RT_BUILD_LIBFUZZER=OFF',
- '-DCOMPILER_RT_BUILD_MEMPROF=OFF',
- '-DCOMPILER_RT_BUILD_ORC=OFF',
- '-DCOMPILER_RT_BUILD_PROFILE=ON',
- '-DCOMPILER_RT_BUILD_SANITIZERS=OFF',
- '-DCOMPILER_RT_BUILD_XRAY=OFF',
- ])
- if cc is not None: bootstrap_args.append('-DCMAKE_C_COMPILER=' + cc)
- if cxx is not None: bootstrap_args.append('-DCMAKE_CXX_COMPILER=' + cxx)
- if lld is not None: bootstrap_args.append('-DCMAKE_LINKER=' + lld)
- RunCommand(['cmake'] + bootstrap_args + [os.path.join(LLVM_DIR, 'llvm')],
- msvc_arch='x64')
- RunCommand(['ninja'], msvc_arch='x64')
- if args.run_tests:
- test_targets = ['check-all']
- if sys.platform == 'darwin' and platform.machine() == 'arm64':
- # TODO(llvm.org/PR49918): Run check-all on mac/arm too.
- test_targets = ['check-llvm', 'check-clang']
- RunCommand(['ninja'] + test_targets, msvc_arch='x64')
- RunCommand(['ninja', 'install'], msvc_arch='x64')
- if sys.platform == 'win32':
- cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe')
- cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe')
- lld = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'lld-link.exe')
- # CMake has a hard time with backslashes in compiler paths:
- # https://stackoverflow.com/questions/13050827
- cc = cc.replace('\\', '/')
- cxx = cxx.replace('\\', '/')
- lld = lld.replace('\\', '/')
- else:
- cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang')
- cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang++')
- print('Bootstrap compiler installed.')
- if args.pgo:
- print('Building instrumented compiler')
- if os.path.exists(LLVM_INSTRUMENTED_DIR):
- RmTree(LLVM_INSTRUMENTED_DIR)
- EnsureDirExists(LLVM_INSTRUMENTED_DIR)
- os.chdir(LLVM_INSTRUMENTED_DIR)
- projects = 'clang'
- if sys.platform == 'darwin':
- projects += ';libcxx;compiler-rt'
- instrument_args = base_cmake_args + [
- '-DLLVM_ENABLE_PROJECTS=' + projects,
- '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
- '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
- '-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(ldflags),
- '-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(ldflags),
- '-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(ldflags),
- # Build with instrumentation.
- '-DLLVM_BUILD_INSTRUMENTED=IR',
- ]
- # Build with the bootstrap compiler.
- if cc is not None: instrument_args.append('-DCMAKE_C_COMPILER=' + cc)
- if cxx is not None: instrument_args.append('-DCMAKE_CXX_COMPILER=' + cxx)
- if lld is not None: instrument_args.append('-DCMAKE_LINKER=' + lld)
- RunCommand(['cmake'] + instrument_args + [os.path.join(LLVM_DIR, 'llvm')],
- msvc_arch='x64')
- RunCommand(['ninja'], msvc_arch='x64')
- print('Instrumented compiler built.')
- # Train by building some C++ code.
- #
- # pgo_training-1.ii is a preprocessed (on Linux) version of
- # src/third_party/blink/renderer/core/layout/layout_object.cc, selected
- # because it's a large translation unit in Blink, which is normally the
- # slowest part of Chromium to compile. Using this, we get ~20% shorter
- # build times for Linux, Android, and Mac, which is also what we got when
- # training by actually building a target in Chromium. (For comparison, a
- # C++-y "Hello World" program only resulted in 14% faster builds.)
- # See https://crbug.com/966403#c16 for all numbers.
- #
- # Although the training currently only exercises Clang, it does involve LLVM
- # internals, and so LLD also benefits when used for ThinLTO links.
- #
- # NOTE: Tidy uses binaries built with this profile, but doesn't seem to
- # gain much from it. If tidy's execution time becomes a concern, it might
- # be good to investigate that.
- #
- # TODO(hans): Enhance the training, perhaps by including preprocessed code
- # from more platforms, and by doing some linking so that lld can benefit
- # from PGO as well. Perhaps the training could be done asynchronously by
- # dedicated buildbots that upload profiles to the cloud.
- training_source = 'pgo_training-1.ii'
- with open(training_source, 'wb') as f:
- DownloadUrl(CDS_URL + '/' + training_source, f)
- train_cmd = [os.path.join(LLVM_INSTRUMENTED_DIR, 'bin', 'clang++'),
- '-target', 'x86_64-unknown-unknown', '-O2', '-g', '-std=c++14',
- '-fno-exceptions', '-fno-rtti', '-w', '-c', training_source]
- if sys.platform == 'darwin':
- train_cmd.extend(['-stdlib=libc++', '-isysroot', isysroot])
- RunCommand(train_cmd, msvc_arch='x64')
- # Merge profiles.
- profdata = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'llvm-profdata')
- RunCommand([profdata, 'merge', '-output=' + LLVM_PROFDATA_FILE] +
- glob.glob(os.path.join(LLVM_INSTRUMENTED_DIR, 'profiles',
- '*.profraw')), msvc_arch='x64')
- print('Profile generated.')
- compiler_rt_args = [
- # Build crtbegin/crtend. It's just two tiny TUs, so just enable this
- # everywhere, even though we only need it on Linux.
- '-DCOMPILER_RT_BUILD_CRT=ON',
- '-DCOMPILER_RT_BUILD_LIBFUZZER=OFF',
- '-DCOMPILER_RT_BUILD_MEMPROF=OFF',
- '-DCOMPILER_RT_BUILD_ORC=OFF',
- '-DCOMPILER_RT_BUILD_PROFILE=ON',
- '-DCOMPILER_RT_BUILD_SANITIZERS=ON',
- '-DCOMPILER_RT_BUILD_XRAY=OFF',
- ]
- if sys.platform == 'darwin':
- compiler_rt_args.extend([
- '-DCOMPILER_RT_ENABLE_IOS=ON',
- '-DCOMPILER_RT_ENABLE_WATCHOS=OFF',
- '-DCOMPILER_RT_ENABLE_TVOS=OFF',
- # armv7 is A5 and earlier, armv7s is A6+ (2012 and later, before 64-bit
- # iPhones). armv7k is Apple Watch, which we don't need.
- '-DDARWIN_ios_ARCHS=armv7;armv7s;arm64',
- '-DDARWIN_iossim_ARCHS=i386;x86_64;arm64',
- # We don't need 32-bit intel support for macOS, we only ship 64-bit.
- '-DDARWIN_osx_ARCHS=arm64;x86_64',
- ])
- if sys.platform == 'win32':
- # https://crbug.com/1293778
- compiler_rt_args.append('-DCOMPILER_RT_BUILD_BUILTINS=OFF')
- else:
- compiler_rt_args.append('-DCOMPILER_RT_BUILD_BUILTINS=ON')
- # LLVM uses C++11 starting in llvm 3.5. On Linux, this means libstdc++4.7+ is
- # needed, on OS X it requires libc++. clang only automatically links to libc++
- # when targeting OS X 10.9+, so add stdlib=libc++ explicitly so clang can run
- # on OS X versions as old as 10.7.
- deployment_target = ''
- if sys.platform == 'darwin' and args.bootstrap:
- # When building on 10.9, /usr/include usually doesn't exist, and while
- # Xcode's clang automatically sets a sysroot, self-built clangs don't.
- cflags = ['-isysroot', isysroot]
- cxxflags = ['-stdlib=libc++'] + cflags
- ldflags += ['-stdlib=libc++']
- deployment_target = '10.7'
- # If building at head, define a macro that plugins can use for #ifdefing
- # out code that builds at head, but not at CLANG_REVISION or vice versa.
- if args.llvm_force_head_revision:
- cflags += ['-DLLVM_FORCE_HEAD_REVISION']
- cxxflags += ['-DLLVM_FORCE_HEAD_REVISION']
- # Build PDBs for archival on Windows. Don't use RelWithDebInfo since it
- # has different optimization defaults than Release.
- # Also disable stack cookies (/GS-) for performance.
- if sys.platform == 'win32':
- cflags += ['/Zi', '/GS-']
- cxxflags += ['/Zi', '/GS-']
- ldflags += ['/DEBUG', '/OPT:REF', '/OPT:ICF']
- deployment_env = None
- if deployment_target:
- deployment_env = os.environ.copy()
- deployment_env['MACOSX_DEPLOYMENT_TARGET'] = deployment_target
- print('Building final compiler.')
- default_tools = ['plugins', 'blink_gc_plugin', 'translation_unit']
- chrome_tools = list(set(default_tools + args.extra_tools))
- if cc is not None: base_cmake_args.append('-DCMAKE_C_COMPILER=' + cc)
- if cxx is not None: base_cmake_args.append('-DCMAKE_CXX_COMPILER=' + cxx)
- if lld is not None: base_cmake_args.append('-DCMAKE_LINKER=' + lld)
- cmake_args = base_cmake_args + compiler_rt_args + [
- '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
- '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
- '-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(ldflags),
- '-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(ldflags),
- '-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(ldflags),
- '-DCMAKE_INSTALL_PREFIX=' + LLVM_BUILD_DIR,
- '-DLLVM_EXTERNAL_PROJECTS=chrometools',
- '-DLLVM_EXTERNAL_CHROMETOOLS_SOURCE_DIR=' +
- os.path.join(CHROMIUM_DIR, 'tools', 'clang'),
- '-DCHROMIUM_TOOLS=%s' % ';'.join(chrome_tools)]
- if args.pgo:
- cmake_args.append('-DLLVM_PROFDATA_FILE=' + LLVM_PROFDATA_FILE)
- if args.thinlto:
- cmake_args.append('-DLLVM_ENABLE_LTO=Thin')
- if sys.platform == 'win32':
- cmake_args.append('-DLLVM_ENABLE_ZLIB=FORCE_ON')
- if sys.platform == 'darwin':
- cmake_args += ['-DCOMPILER_RT_ENABLE_IOS=ON',
- '-DSANITIZER_MIN_OSX_VERSION=10.7']
- if args.build_mac_arm:
- assert platform.machine() != 'arm64', 'build_mac_arm for cross build only'
- cmake_args += ['-DCMAKE_OSX_ARCHITECTURES=arm64',
- '-DLLVM_USE_HOST_TOOLS=ON']
- # The default LLVM_DEFAULT_TARGET_TRIPLE depends on the host machine.
- # Set it explicitly to make the build of clang more hermetic, and also to
- # set it to arm64 when cross-building clang for mac/arm.
- if sys.platform == 'darwin':
- if args.build_mac_arm or platform.machine() == 'arm64':
- cmake_args.append('-DLLVM_DEFAULT_TARGET_TRIPLE=arm64-apple-darwin')
- else:
- cmake_args.append('-DLLVM_DEFAULT_TARGET_TRIPLE=x86_64-apple-darwin')
- elif sys.platform.startswith('linux'):
- if platform.machine() == 'aarch64':
- cmake_args.append(
- '-DLLVM_DEFAULT_TARGET_TRIPLE=aarch64-unknown-linux-gnu')
- elif platform.machine() == 'riscv64':
- cmake_args.append(
- '-DLLVM_DEFAULT_TARGET_TRIPLE=riscv64-linux-gnu')
- else:
- cmake_args.append('-DLLVM_DEFAULT_TARGET_TRIPLE=x86_64-unknown-linux-gnu')
- cmake_args.append('-DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=ON')
- elif sys.platform == 'win32':
- cmake_args.append('-DLLVM_DEFAULT_TARGET_TRIPLE=x86_64-pc-windows-msvc')
- if os.path.exists(LLVM_BUILD_DIR):
- RmTree(LLVM_BUILD_DIR)
- EnsureDirExists(LLVM_BUILD_DIR)
- os.chdir(LLVM_BUILD_DIR)
- RunCommand(['cmake'] + cmake_args + [os.path.join(LLVM_DIR, 'llvm')],
- msvc_arch='x64', env=deployment_env)
- CopyLibstdcpp(args, LLVM_BUILD_DIR)
- RunCommand(['ninja'], msvc_arch='x64')
- if chrome_tools:
- # If any Chromium tools were built, install those now.
- RunCommand(['ninja', 'cr-install'], msvc_arch='x64')
- if not args.build_mac_arm:
- VerifyVersionOfBuiltClangMatchesVERSION()
- VerifyZlibSupport()
- if sys.platform == 'win32':
- rt_platform = 'windows'
- elif sys.platform == 'darwin':
- rt_platform = 'darwin'
- else:
- assert sys.platform.startswith('linux')
- rt_platform = 'linux'
- rt_lib_dst_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang', RELEASE_VERSION,
- 'lib', rt_platform)
- # Make sure the directory exists; this will not be implicilty created if
- # built with per-target runtime directories.
- if not os.path.exists(rt_lib_dst_dir):
- os.makedirs(rt_lib_dst_dir)
- # Do an out-of-tree build of compiler-rt for 32-bit Win clang_rt.profile.lib.
- if sys.platform == 'win32':
- compiler_rt_build_dir = os.path.join(LLVM_BUILD_DIR, 'compiler-rt')
- if os.path.isdir(compiler_rt_build_dir):
- RmTree(compiler_rt_build_dir)
- os.makedirs(compiler_rt_build_dir)
- os.chdir(compiler_rt_build_dir)
- if args.bootstrap:
- # The bootstrap compiler produces 64-bit binaries by default.
- cflags += ['-m32']
- cxxflags += ['-m32']
- compiler_rt_args = base_cmake_args + [
- '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
- '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
- '-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(ldflags),
- '-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(ldflags),
- '-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(ldflags),
- '-DCOMPILER_RT_BUILD_BUILTINS=OFF',
- '-DCOMPILER_RT_BUILD_CRT=OFF',
- '-DCOMPILER_RT_BUILD_LIBFUZZER=OFF',
- '-DCOMPILER_RT_BUILD_MEMPROF=OFF',
- '-DCOMPILER_RT_BUILD_ORC=OFF',
- '-DCOMPILER_RT_BUILD_PROFILE=ON',
- '-DCOMPILER_RT_BUILD_SANITIZERS=OFF',
- '-DCOMPILER_RT_BUILD_XRAY=OFF',
- # The libxml2 we built above is 64-bit. Since it's only needed by
- # lld-link and not compiler-rt, just turn it off down here.
- '-DLLVM_ENABLE_LIBXML2=OFF',
- ]
- RunCommand(['cmake'] + compiler_rt_args +
- [os.path.join(LLVM_DIR, 'llvm')],
- msvc_arch='x86', env=deployment_env)
- RunCommand(['ninja', 'compiler-rt'], msvc_arch='x86')
- # Copy select output to the main tree.
- rt_lib_src_dir = os.path.join(compiler_rt_build_dir, 'lib', 'clang',
- RELEASE_VERSION, 'lib', rt_platform)
- # Static and dynamic libraries:
- CopyDirectoryContents(rt_lib_src_dir, rt_lib_dst_dir)
- if args.with_android:
- # TODO(thakis): Now that the NDK uses clang, try to build all archs in
- # one LLVM build instead of building 3 times.
- toolchain_dir = ANDROID_NDK_DIR + '/toolchains/llvm/prebuilt/linux-x86_64'
- for target_arch in ['aarch64', 'arm', 'i686', 'x86_64']:
- # Build compiler-rt runtimes needed for Android in a separate build tree.
- build_dir = os.path.join(LLVM_BUILD_DIR, 'android-' + target_arch)
- if not os.path.exists(build_dir):
- os.mkdir(os.path.join(build_dir))
- os.chdir(build_dir)
- target_triple = target_arch
- if target_arch == 'arm':
- target_triple = 'armv7'
- api_level = '19'
- if target_arch == 'aarch64' or target_arch == 'x86_64':
- api_level = '21'
- target_triple += '-linux-android' + api_level
- cflags = [
- '--target=' + target_triple,
- '--sysroot=%s/sysroot' % toolchain_dir,
- # pylint: disable=line-too-long
- # android_ndk/toolchains/llvm/prebuilt/linux-x86_64/aarch64-linux-android/bin/ld
- # depends on a newer version of libxml2.so than what's available on
- # the bots. To make things work, use our just-built lld as linker.
- # pylint: enable=line-too-long
- '-fuse-ld=lld',
- # We don't have an unwinder ready, and don't need it either.
- '--unwindlib=none',
- ]
- if target_arch == 'aarch64':
- # Use PAC/BTI instructions for AArch64
- cflags += [ '-mbranch-protection=standard' ]
- android_args = base_cmake_args + [
- '-DCMAKE_C_COMPILER=' + os.path.join(LLVM_BUILD_DIR, 'bin/clang'),
- '-DCMAKE_CXX_COMPILER=' + os.path.join(LLVM_BUILD_DIR, 'bin/clang++'),
- '-DLLVM_CONFIG_PATH=' + os.path.join(LLVM_BUILD_DIR, 'bin/llvm-config'),
- '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
- '-DCMAKE_CXX_FLAGS=' + ' '.join(cflags),
- '-DCMAKE_ASM_FLAGS=' + ' '.join(cflags),
- '-DCOMPILER_RT_BUILD_BUILTINS=ON',
- '-DCOMPILER_RT_BUILD_CRT=OFF',
- '-DCOMPILER_RT_BUILD_LIBFUZZER=OFF',
- '-DCOMPILER_RT_BUILD_MEMPROF=OFF',
- '-DCOMPILER_RT_BUILD_ORC=OFF',
- '-DCOMPILER_RT_BUILD_PROFILE=OFF',
- '-DCOMPILER_RT_BUILD_SANITIZERS=OFF',
- '-DCOMPILER_RT_BUILD_XRAY=OFF',
- '-DSANITIZER_CXX_ABI=libcxxabi',
- '-DCMAKE_SHARED_LINKER_FLAGS=-Wl,-u__cxa_demangle',
- '-DANDROID=1'
- # These are necessary because otherwise CMake tries to build an
- # executable to test to see if the compiler is working, but in doing so,
- # it links against the builtins.a that we're about to build.
- '-DCMAKE_CXX_COMPILER_WORKS=ON',
- '-DCMAKE_C_COMPILER_WORKS=ON',
- '-DCMAKE_ASM_COMPILER_WORKS=ON',
- ]
- # First build the builtins and copy to the main build tree.
- RunCommand(['cmake'] +
- android_args +
- [os.path.join(COMPILER_RT_DIR, 'lib', 'builtins')])
- builtins_a = 'lib/linux/libclang_rt.builtins-%s-android.a' % target_arch
- RunCommand(['ninja', builtins_a])
- shutil.copy(builtins_a, rt_lib_dst_dir)
- # With the builtins in place, we can build the other runtimes.
- build_dir_2 = build_dir + '-phase2'
- if not os.path.exists(build_dir_2):
- os.mkdir(os.path.join(build_dir_2))
- os.chdir(build_dir_2)
- android_args.extend([
- '-DCOMPILER_RT_BUILD_BUILTINS=OFF',
- '-DCOMPILER_RT_USE_BUILTINS_LIBRARY=ON',
- '-DCOMPILER_RT_BUILD_PROFILE=ON',
- '-DCOMPILER_RT_BUILD_SANITIZERS=ON',
- ])
- RunCommand(['cmake'] + android_args + [COMPILER_RT_DIR])
- libs_want = [
- 'lib/linux/libclang_rt.asan-{0}-android.so',
- 'lib/linux/libclang_rt.asan_static-{0}-android.a',
- 'lib/linux/libclang_rt.ubsan_standalone-{0}-android.so',
- 'lib/linux/libclang_rt.profile-{0}-android.a',
- ]
- # Only build HWASan for AArch64.
- if target_arch == 'aarch64':
- libs_want += ['lib/linux/libclang_rt.hwasan-{0}-android.so']
- libs_want = [lib.format(target_arch) for lib in libs_want]
- RunCommand(['ninja'] + libs_want)
- # And copy them into the main build tree.
- for p in libs_want:
- shutil.copy(p, rt_lib_dst_dir)
- if args.with_fuchsia:
- # Fuchsia links against libclang_rt.builtins-<arch>.a instead of libgcc.a.
- for target_arch in ['aarch64', 'x86_64']:
- fuchsia_arch_name = {'aarch64': 'arm64', 'x86_64': 'x64'}[target_arch]
- toolchain_dir = os.path.join(
- FUCHSIA_SDK_DIR, 'arch', fuchsia_arch_name, 'sysroot')
- # Build clang_rt runtime for Fuchsia in a separate build tree.
- build_dir = os.path.join(LLVM_BUILD_DIR, 'fuchsia-' + target_arch)
- if not os.path.exists(build_dir):
- os.mkdir(os.path.join(build_dir))
- os.chdir(build_dir)
- target_spec = target_arch + '-unknown-fuchsia'
- if args.build_mac_arm:
- # Just-built clang can't run (it's an arm binary on an intel host), so
- # use the bootstrap compiler instead.
- host_path = LLVM_BOOTSTRAP_INSTALL_DIR
- else:
- host_path = LLVM_BUILD_DIR
- # TODO(thakis): Might have to pass -B here once sysroot contains
- # binaries (e.g. gas for arm64?)
- fuchsia_args = base_cmake_args + [
- '-DCMAKE_C_COMPILER=' + os.path.join(host_path, 'bin/clang'),
- '-DCMAKE_CXX_COMPILER=' + os.path.join(host_path, 'bin/clang++'),
- '-DCMAKE_LINKER=' + os.path.join(host_path, 'bin/clang'),
- '-DCMAKE_AR=' + os.path.join(host_path, 'bin/llvm-ar'),
- '-DLLVM_CONFIG_PATH=' + os.path.join(host_path, 'bin/llvm-config'),
- '-DCMAKE_SYSTEM_NAME=Fuchsia',
- '-DCMAKE_CXX_COMPILER_TARGET=' + target_spec,
- '-DCMAKE_C_COMPILER_TARGET=' + target_spec,
- '-DCMAKE_ASM_COMPILER_TARGET=' + target_spec,
- '-DCOMPILER_RT_BUILD_BUILTINS=ON',
- '-DCOMPILER_RT_BUILD_CRT=OFF',
- '-DCOMPILER_RT_BUILD_LIBFUZZER=OFF',
- '-DCOMPILER_RT_BUILD_MEMPROF=OFF',
- '-DCOMPILER_RT_BUILD_ORC=OFF',
- '-DCOMPILER_RT_BUILD_PROFILE=OFF',
- '-DCOMPILER_RT_BUILD_SANITIZERS=OFF',
- '-DCOMPILER_RT_BUILD_XRAY=OFF',
- '-DCOMPILER_RT_DEFAULT_TARGET_ONLY=ON',
- '-DCMAKE_SYSROOT=%s' % toolchain_dir,
- # TODO(thakis|scottmg): Use PER_TARGET_RUNTIME_DIR for all platforms.
- # https://crbug.com/882485.
- '-DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=ON',
- # These are necessary because otherwise CMake tries to build an
- # executable to test to see if the compiler is working, but in doing so,
- # it links against the builtins.a that we're about to build.
- '-DCMAKE_CXX_COMPILER_WORKS=ON',
- '-DCMAKE_C_COMPILER_WORKS=ON',
- '-DCMAKE_ASM_COMPILER_WORKS=ON',
- ]
- RunCommand(['cmake'] +
- fuchsia_args +
- [os.path.join(COMPILER_RT_DIR, 'lib', 'builtins')])
- builtins_a = 'libclang_rt.builtins.a'
- RunCommand(['ninja', builtins_a])
- # And copy it into the main build tree.
- fuchsia_lib_dst_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
- RELEASE_VERSION, 'lib', target_spec)
- if not os.path.exists(fuchsia_lib_dst_dir):
- os.makedirs(fuchsia_lib_dst_dir)
- CopyFile(os.path.join(build_dir, 'lib', target_spec, builtins_a),
- fuchsia_lib_dst_dir)
- # Build the Fuchsia profile and asan runtimes. This is done after the rt
- # builtins have been created because the CMake build runs link checks that
- # require that the builtins already exist to succeed.
- # TODO(thakis): Figure out why this doesn't build with the stage0
- # compiler in arm cross builds.
- if target_arch == 'x86_64' and not args.build_mac_arm:
- fuchsia_args.extend([
- '-DCOMPILER_RT_BUILD_BUILTINS=OFF',
- '-DCOMPILER_RT_BUILD_PROFILE=ON',
- ])
- # Build the asan runtime only on non-Mac platforms. Macs are excluded
- # because the asan install changes library RPATHs which CMake only
- # supports on ELF platforms and MacOS uses Mach-O instead of ELF.
- if sys.platform != 'darwin':
- fuchsia_args.extend([
- '-DCOMPILER_RT_BUILD_SANITIZERS=ON',
- '-DSANITIZER_NO_UNDEFINED_SYMBOLS=OFF',
- ])
- build_phase2_dir = os.path.join(LLVM_BUILD_DIR,
- 'fuchsia-phase2-' + target_arch)
- if not os.path.exists(build_phase2_dir):
- os.mkdir(os.path.join(build_phase2_dir))
- os.chdir(build_phase2_dir)
- RunCommand(['cmake'] +
- fuchsia_args +
- [COMPILER_RT_DIR])
- profile_a = 'libclang_rt.profile.a'
- asan_preinit_a = 'libclang_rt.asan-preinit.a'
- asan_static_a = 'libclang_rt.asan_static.a'
- asan_so = 'libclang_rt.asan.so'
- ninja_command = ['ninja', profile_a]
- if sys.platform != 'darwin':
- ninja_command.append(asan_so)
- ninja_command.append(asan_preinit_a)
- ninja_command.append(asan_static_a)
- RunCommand(ninja_command)
- CopyFile(os.path.join(build_phase2_dir, 'lib', target_spec, profile_a),
- fuchsia_lib_dst_dir)
- if sys.platform != 'darwin':
- CopyFile(os.path.join(build_phase2_dir, 'lib', target_spec, asan_so),
- fuchsia_lib_dst_dir)
- CopyFile(
- os.path.join(build_phase2_dir, 'lib', target_spec,
- asan_preinit_a), fuchsia_lib_dst_dir)
- CopyFile(
- os.path.join(build_phase2_dir, 'lib', target_spec, asan_static_a),
- fuchsia_lib_dst_dir)
- # Run tests.
- if (not args.build_mac_arm and
- (args.run_tests or args.llvm_force_head_revision)):
- RunCommand(['ninja', '-C', LLVM_BUILD_DIR, 'cr-check-all'], msvc_arch='x64')
- if not args.build_mac_arm and args.run_tests:
- test_targets = [ 'check-all' ]
- if sys.platform == 'darwin':
- # TODO(thakis): Run check-all on Darwin too, https://crbug.com/959361
- test_targets = [ 'check-llvm', 'check-clang', 'check-lld' ]
- RunCommand(['ninja', '-C', LLVM_BUILD_DIR] + test_targets, msvc_arch='x64')
- WriteStampFile(PACKAGE_VERSION, STAMP_FILE)
- WriteStampFile(PACKAGE_VERSION, FORCE_HEAD_REVISION_FILE)
- print('Clang build was successful.')
- return 0
- if __name__ == '__main__':
- sys.exit(main())
|