123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472 |
- #!/usr/bin/env python3
- import os.path
- import re
- import requests
- import textwrap
- BASE_URL = "https://toolchains.bootlin.com/downloads/releases/toolchains"
- AUTOGENERATED_COMMENT = """# This file was auto-generated by support/scripts/gen-bootlin-toolchains
- # Do not edit
- """
- # In the below dict:
- # - 'conditions' indicate the cumulative conditions under which the
- # toolchain will be made available. In several situations, a given
- # toolchain is usable on several architectures variants (for
- # example, an ARMv6 toolchain can be used on ARMv7)
- # - 'test_options' indicate one specific configuration where the
- # toolchain can be used. It is used to create the runtime test
- # cases. If 'test_options' does not exist, the code assumes it can
- # be made equal to 'conditions'
- # - 'prefix' is the prefix of the cross-compilation toolchain tools
- arches = {
- 'aarch64': {
- 'conditions': ['BR2_aarch64'],
- 'prefix': 'aarch64',
- },
- 'aarch64be': {
- 'conditions': ['BR2_aarch64_be'],
- 'prefix': 'aarch64_be',
- },
- 'arcle-750d': {
- 'conditions': ['BR2_arcle', 'BR2_arc750d'],
- 'prefix': 'arc',
- },
- 'arcle-hs38': {
- 'conditions': ['BR2_arcle', 'BR2_archs38'],
- 'prefix': 'arc',
- },
- 'armv5-eabi': {
- 'conditions': ['BR2_ARM_CPU_ARMV5', 'BR2_ARM_EABI'],
- 'test_options': ['BR2_arm', 'BR2_arm926t', 'BR2_ARM_EABI'],
- 'prefix': 'arm',
- },
- 'armv6-eabihf': {
- 'conditions': ['BR2_ARM_CPU_ARMV6', 'BR2_ARM_EABIHF'],
- 'test_options': ['BR2_arm', 'BR2_arm1176jzf_s', 'BR2_ARM_EABIHF'],
- 'prefix': 'arm',
- },
- 'armv7-eabihf': {
- 'conditions': ['BR2_ARM_CPU_ARMV7A', 'BR2_ARM_EABIHF'],
- 'test_options': ['BR2_arm', 'BR2_cortex_a8', 'BR2_ARM_EABIHF'],
- 'prefix': 'arm',
- },
- 'armv7m': {
- 'conditions': ['BR2_ARM_CPU_ARMV7M'],
- 'test_options': ['BR2_arm', 'BR2_cortex_m4'],
- 'prefix': 'arm',
- },
- 'm68k-68xxx': {
- 'conditions': ['BR2_m68k_m68k'],
- 'test_options': ['BR2_m68k', 'BR2_m68k_68040'],
- 'prefix': 'm68k',
- },
- 'm68k-coldfire': {
- 'conditions': ['BR2_m68k_cf'],
- 'test_options': ['BR2_m68k', 'BR2_m68k_cf5208'],
- 'prefix': 'm68k',
- },
- 'microblazebe': {
- 'conditions': ['BR2_microblazebe'],
- 'prefix': 'microblaze',
- },
- 'microblazeel': {
- 'conditions': ['BR2_microblazeel'],
- 'prefix': 'microblazeel',
- },
- 'mips32': {
- # Not sure it could be used by other mips32 variants?
- 'conditions': ['BR2_mips', 'BR2_mips_32', '!BR2_MIPS_SOFT_FLOAT'],
- 'prefix': 'mips',
- },
- 'mips32el': {
- # Not sure it could be used by other mips32el variants?
- 'conditions': ['BR2_mipsel', 'BR2_mips_32', '!BR2_MIPS_SOFT_FLOAT'],
- 'prefix': 'mipsel',
- },
- 'mips32r5el': {
- 'conditions': ['BR2_mipsel', 'BR2_mips_32r5', '!BR2_MIPS_SOFT_FLOAT'],
- 'prefix': 'mipsel',
- },
- 'mips32r6el': {
- 'conditions': ['BR2_mipsel', 'BR2_mips_32r6', '!BR2_MIPS_SOFT_FLOAT'],
- 'prefix': 'mipsel',
- },
- 'mips64': {
- # Not sure it could be used by other mips64 variants?
- 'conditions': ['BR2_mips64', 'BR2_mips_64', '!BR2_MIPS_SOFT_FLOAT'],
- 'prefix': 'mips64',
- },
- 'mips64-n32': {
- # Not sure it could be used by other mips64 variants?
- 'conditions': ['BR2_mips64', 'BR2_mips_64', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
- 'prefix': 'mips64',
- },
- 'mips64el-n32': {
- # Not sure it could be used by other mips64el variants?
- 'conditions': ['BR2_mips64el', 'BR2_mips_64', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
- 'prefix': 'mips64el',
- },
- 'mips64r6el-n32': {
- 'conditions': ['BR2_mips64el', 'BR2_mips_64r6', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
- 'prefix': 'mips64el',
- },
- 'nios2': {
- 'conditions': ['BR2_nios2'],
- 'prefix': 'nios2',
- },
- 'openrisc': {
- 'conditions': ['BR2_or1k'],
- 'prefix': 'or1k',
- },
- 'powerpc-e500mc': {
- # Not sure it could be used by other powerpc variants?
- 'conditions': ['BR2_powerpc', 'BR2_powerpc_e500mc'],
- 'prefix': 'powerpc',
- },
- 'powerpc64-e5500': {
- 'conditions': ['BR2_powerpc64', 'BR2_powerpc_e5500'],
- 'prefix': 'powerpc64',
- },
- 'powerpc64-power8': {
- 'conditions': ['BR2_powerpc64', 'BR2_powerpc_power8'],
- 'prefix': 'powerpc64',
- },
- 'powerpc64le-power8': {
- 'conditions': ['BR2_powerpc64le', 'BR2_powerpc_power8'],
- 'prefix': 'powerpc64le',
- },
- 'riscv32-ilp32d': {
- 'conditions': ['BR2_riscv', 'BR2_riscv_g', 'BR2_RISCV_32', 'BR2_RISCV_ABI_ILP32D'],
- 'prefix': 'riscv32',
- },
- 'riscv64': {
- 'conditions': ['BR2_riscv', 'BR2_riscv_g', 'BR2_RISCV_64', 'BR2_RISCV_ABI_LP64'],
- 'prefix': 'riscv64',
- },
- 'sh-sh4': {
- 'conditions': ['BR2_sh', 'BR2_sh4'],
- 'prefix': 'sh4',
- },
- 'sh-sh4aeb': {
- 'conditions': ['BR2_sh', 'BR2_sh4aeb'],
- 'prefix': 'sh4aeb',
- },
- 'sparc64': {
- 'conditions': ['BR2_sparc64', 'BR2_sparc_v9'],
- 'prefix': 'sparc64',
- },
- 'sparcv8': {
- 'conditions': ['BR2_sparc', 'BR2_sparc_v8'],
- 'prefix': 'sparc',
- },
- 'x86-64-core-i7': {
- 'conditions': ['BR2_x86_64',
- 'BR2_X86_CPU_HAS_MMX',
- 'BR2_X86_CPU_HAS_SSE',
- 'BR2_X86_CPU_HAS_SSE2',
- 'BR2_X86_CPU_HAS_SSE3',
- 'BR2_X86_CPU_HAS_SSSE3',
- 'BR2_X86_CPU_HAS_SSE4',
- 'BR2_X86_CPU_HAS_SSE42'],
- 'test_options': ['BR2_x86_64', 'BR2_x86_corei7'],
- 'prefix': 'x86_64',
- },
- 'x86-core2': {
- 'conditions': ['BR2_i386',
- 'BR2_X86_CPU_HAS_MMX',
- 'BR2_X86_CPU_HAS_SSE',
- 'BR2_X86_CPU_HAS_SSE2',
- 'BR2_X86_CPU_HAS_SSE3',
- 'BR2_X86_CPU_HAS_SSSE3'],
- 'test_options': ['BR2_i386', 'BR2_x86_core2'],
- 'prefix': 'i686',
- },
- 'x86-i686': {
- 'conditions': ['BR2_i386',
- '!BR2_x86_i486',
- '!BR2_x86_i586',
- '!BR2_x86_x1000'],
- 'test_options': ['BR2_i386',
- 'BR2_x86_i686'],
- 'prefix': 'i686',
- },
- 'xtensa-lx60': {
- 'conditions': ['BR2_xtensa', 'BR2_xtensa_fsf'],
- 'prefix': 'xtensa',
- },
- }
- class Toolchain:
- def __init__(self, arch, libc, variant, version):
- self.arch = arch
- self.libc = libc
- self.variant = variant
- self.version = version
- self.fname_prefix = "%s--%s--%s-%s" % (self.arch, self.libc, self.variant, self.version)
- self.option_name = "BR2_TOOLCHAIN_EXTERNAL_BOOTLIN_%s_%s_%s" % \
- (self.arch.replace("-", "_").upper(), self.libc.upper(), self.variant.replace("-", "_").upper())
- self.fragment = requests.get(self.fragment_url).text.split("\n")
- self.sha256 = requests.get(self.hash_url).text.split(" ")[0]
- @property
- def tarball_url(self):
- return os.path.join(BASE_URL, self.arch, "tarballs",
- self.fname_prefix + ".tar.bz2")
- @property
- def hash_url(self):
- return os.path.join(BASE_URL, self.arch, "tarballs",
- self.fname_prefix + ".sha256")
- @property
- def fragment_url(self):
- return os.path.join(BASE_URL, self.arch, "fragments",
- self.fname_prefix + ".frag")
- def gen_config_in_options(self, f):
- f.write("config %s\n" % self.option_name)
- f.write("\tbool \"%s %s %s %s\"\n" %
- (self.arch, self.libc, self.variant, self.version))
- for c in arches[self.arch]['conditions']:
- f.write("\tdepends on %s\n" % c)
- selects = []
- for frag in self.fragment:
- # libc type
- if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_UCLIBC"):
- selects.append("BR2_TOOLCHAIN_EXTERNAL_UCLIBC")
- elif frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_GLIBC"):
- selects.append("BR2_TOOLCHAIN_EXTERNAL_GLIBC")
- # all glibc toolchains have RPC support
- selects.append("BR2_TOOLCHAIN_HAS_NATIVE_RPC")
- elif frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_MUSL"):
- selects.append("BR2_TOOLCHAIN_EXTERNAL_MUSL")
- # gcc version
- if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_GCC_"):
- m = re.match("^BR2_TOOLCHAIN_EXTERNAL_GCC_([0-9_]*)=y$", frag)
- assert m, "Cannot get gcc version for toolchain %s" % self.fname_prefix
- selects.append("BR2_TOOLCHAIN_GCC_AT_LEAST_%s" % m[1])
- # kernel headers version
- if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HEADERS_"):
- m = re.match("^BR2_TOOLCHAIN_EXTERNAL_HEADERS_([0-9_]*)=y$", frag)
- assert m, "Cannot get kernel headers version for toolchain %s" % self.fname_prefix
- selects.append("BR2_TOOLCHAIN_HEADERS_AT_LEAST_%s" % m[1])
- # C++
- if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CXX"):
- selects.append("BR2_INSTALL_LIBSTDCPP")
- # SSP
- if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_SSP"):
- selects.append("BR2_TOOLCHAIN_HAS_SSP")
- # wchar
- if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_WCHAR"):
- selects.append("BR2_USE_WCHAR")
- # locale
- if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_LOCALE"):
- # locale implies the availability of wchar
- selects.append("BR2_USE_WCHAR")
- selects.append("BR2_ENABLE_LOCALE")
- # thread support
- if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS"):
- selects.append("BR2_TOOLCHAIN_HAS_THREADS")
- if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS_DEBUG"):
- selects.append("BR2_TOOLCHAIN_HAS_THREADS_DEBUG")
- if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS_NPTL"):
- selects.append("BR2_TOOLCHAIN_HAS_THREADS_NPTL")
- # RPC
- if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_INET_RPC"):
- selects.append("BR2_TOOLCHAIN_HAS_NATIVE_RPC")
- # D language
- if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_DLANG"):
- selects.append("BR2_TOOLCHAIN_HAS_DLANG")
- # fortran
- if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_FORTRAN"):
- selects.append("BR2_TOOLCHAIN_HAS_FORTRAN")
- # OpenMP
- if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_OPENMP"):
- selects.append("BR2_TOOLCHAIN_HAS_OPENMP")
- for select in selects:
- f.write("\tselect %s\n" % select)
- f.write("\thelp\n")
- desc = "Bootlin toolchain for the %s architecture, using the %s C library. " % \
- (self.arch, self.libc)
- if self.variant == "stable":
- desc += "This is a stable version, which means it is using stable and proven versions of gcc, gdb and binutils."
- else:
- desc += "This is a bleeding-edge version, which means it is using the latest versions of gcc, gdb and binutils."
- f.write(textwrap.fill(desc, width=62, initial_indent="\t ", subsequent_indent="\t ") + "\n")
- f.write("\n")
- f.write("\t https://toolchains.bootlin.com/\n")
- f.write("\n")
- def gen_mk(self, f):
- f.write("ifeq ($(%s),y)\n" % self.option_name)
- f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_VERSION = %s\n" % self.version)
- f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_SOURCE = %s--%s--%s-$(TOOLCHAIN_EXTERNAL_BOOTLIN_VERSION).tar.bz2\n" %
- (self.arch, self.libc, self.variant))
- f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_SITE = %s\n" %
- os.path.join(BASE_URL, self.arch, "tarballs"))
- f.write("endif\n\n")
- pass
- def gen_hash(self, f):
- f.write("# From %s\n" % self.hash_url)
- f.write("sha256 %s %s\n" % (self.sha256, os.path.basename(self.tarball_url)))
- def gen_test(self, f):
- if self.variant == "stable":
- variant = "Stable"
- else:
- variant = "BleedingEdge"
- testname = "TestExternalToolchainBootlin" + \
- self.arch.replace("-", "").capitalize() + \
- self.libc.capitalize() + variant
- f.write("\n\n")
- f.write("class %s(TestExternalToolchain):\n" % testname)
- f.write(" config = \"\"\"\n")
- if 'test_options' in arches[self.arch]:
- test_options = arches[self.arch]['test_options']
- else:
- test_options = arches[self.arch]['conditions']
- for opt in test_options:
- if opt.startswith("!"):
- f.write(" # %s is not set\n" % opt[1:])
- else:
- f.write(" %s=y\n" % opt)
- f.write(" BR2_TOOLCHAIN_EXTERNAL=y\n")
- f.write(" BR2_TOOLCHAIN_EXTERNAL_BOOTLIN=y\n")
- f.write(" %s=y\n" % self.option_name)
- f.write(" # BR2_TARGET_ROOTFS_TAR is not set\n")
- f.write(" \"\"\"\n")
- f.write(" toolchain_prefix = \"%s-linux\"\n" % arches[self.arch]['prefix'])
- f.write("\n")
- f.write(" def test_run(self):\n")
- f.write(" TestExternalToolchain.common_check(self)\n")
- def __repr__(self):
- return "Toolchain(arch=%s libc=%s variant=%s version=%s, option=%s)" % \
- (self.arch, self.libc, self.variant, self.version, self.option_name)
- def get_toolchains():
- toolchains = list()
- for arch, details in arches.items():
- print(arch)
- url = os.path.join(BASE_URL, arch, "available_toolchains")
- page = requests.get(url).text
- fnames = sorted(re.findall(r'<td><a href="(\w[^"]+)"', page))
- # This dict will allow us to keep only the latest version for
- # each toolchain.
- tmp = dict()
- for fname in fnames:
- parts = fname.split('--')
- assert parts[0] == arch, "Arch does not match: %s vs. %s" % (parts[0], arch)
- libc = parts[1]
- if parts[2].startswith("stable-"):
- variant = "stable"
- version = parts[2][len("stable-"):]
- elif parts[2].startswith("bleeding-edge-"):
- variant = "bleeding-edge"
- version = parts[2][len("bleeding-edge-"):]
- tmp[(arch, libc, variant)] = version
- toolchains += [Toolchain(k[0], k[1], k[2], v) for k, v in tmp.items()]
- return toolchains
- def gen_config_in_options(toolchains, fpath):
- with open(fpath, "w") as f:
- f.write(AUTOGENERATED_COMMENT)
- f.write("config BR2_TOOLCHAIN_EXTERNAL_BOOTLIN_ARCH_SUPPORTS\n")
- f.write("\tbool\n")
- for arch, details in arches.items():
- f.write("\tdefault y if %s\n" % " && ".join(details['conditions']))
- f.write("\n")
- f.write("if BR2_TOOLCHAIN_EXTERNAL_BOOTLIN\n\n")
- f.write("config BR2_TOOLCHAIN_EXTERNAL_PREFIX\n")
- f.write("\tdefault \"$(ARCH)-linux\"\n")
- f.write("\n")
- f.write("config BR2_PACKAGE_PROVIDES_TOOLCHAIN_EXTERNAL\n")
- f.write("\tdefault \"toolchain-external-bootlin\"\n")
- f.write("\n")
- f.write("choice\n")
- f.write("\tprompt \"Bootlin toolchain variant\"\n")
- for toolchain in toolchains:
- toolchain.gen_config_in_options(f)
- f.write("endchoice\n")
- f.write("endif\n")
- def gen_mk(toolchains, fpath):
- with open(fpath, "w") as f:
- f.write("#" * 80 + "\n")
- f.write("#\n")
- f.write("# toolchain-external-bootlin\n")
- f.write("#\n")
- f.write("#" * 80 + "\n")
- f.write("\n")
- f.write(AUTOGENERATED_COMMENT)
- for toolchain in toolchains:
- toolchain.gen_mk(f)
- f.write("$(eval $(toolchain-external-package))\n")
- def gen_hash(toolchains, fpath):
- with open(fpath, "w") as f:
- f.write(AUTOGENERATED_COMMENT)
- for toolchain in toolchains:
- toolchain.gen_hash(f)
- def gen_runtime_test(toolchains, fpath):
- with open(fpath, "w") as f:
- f.write(AUTOGENERATED_COMMENT)
- f.write("from tests.toolchain.test_external import TestExternalToolchain\n")
- for toolchain in toolchains:
- toolchain.gen_test(f)
- def gen_toolchains(toolchains):
- maindir = "toolchain/toolchain-external/toolchain-external-bootlin"
- gen_config_in_options(toolchains, os.path.join(maindir, "Config.in.options"))
- gen_mk(toolchains, os.path.join(maindir, "toolchain-external-bootlin.mk"))
- gen_hash(toolchains, os.path.join(maindir, "toolchain-external-bootlin.hash"))
- gen_runtime_test(toolchains,
- os.path.join("support", "testing", "tests", "toolchain", "test_external_bootlin.py"))
- toolchains = get_toolchains()
- gen_toolchains(toolchains)
|