gen-bootlin-toolchains 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496
  1. #!/usr/bin/env python3
  2. import os.path
  3. import re
  4. import requests
  5. import textwrap
  6. BASE_URL = "https://toolchains.bootlin.com/downloads/releases/toolchains"
  7. AUTOGENERATED_COMMENT = """# This file was auto-generated by support/scripts/gen-bootlin-toolchains
  8. # Do not edit
  9. """
  10. # In the below dict:
  11. # - 'conditions' indicate the cumulative conditions under which the
  12. # toolchain will be made available. In several situations, a given
  13. # toolchain is usable on several architectures variants (for
  14. # example, an ARMv6 toolchain can be used on ARMv7)
  15. # - 'test_options' indicate one specific configuration where the
  16. # toolchain can be used. It is used to create the runtime test
  17. # cases. If 'test_options' does not exist, the code assumes it can
  18. # be made equal to 'conditions'
  19. # - 'prefix' is the prefix of the cross-compilation toolchain tools
  20. arches = {
  21. 'aarch64': {
  22. 'conditions': ['BR2_aarch64'],
  23. 'prefix': 'aarch64',
  24. },
  25. 'aarch64be': {
  26. 'conditions': ['BR2_aarch64_be'],
  27. 'prefix': 'aarch64_be',
  28. },
  29. 'arcle-750d': {
  30. 'conditions': ['BR2_arcle', 'BR2_arc750d'],
  31. 'prefix': 'arc',
  32. },
  33. 'arcle-hs38': {
  34. 'conditions': ['BR2_arcle', 'BR2_archs38'],
  35. 'prefix': 'arc',
  36. },
  37. 'armv5-eabi': {
  38. 'conditions': ['BR2_ARM_CPU_ARMV5', 'BR2_ARM_EABI'],
  39. 'test_options': ['BR2_arm', 'BR2_arm926t', 'BR2_ARM_EABI'],
  40. 'prefix': 'arm',
  41. },
  42. 'armv6-eabihf': {
  43. 'conditions': ['BR2_ARM_CPU_ARMV6', 'BR2_ARM_EABIHF'],
  44. 'test_options': ['BR2_arm', 'BR2_arm1176jzf_s', 'BR2_ARM_EABIHF'],
  45. 'prefix': 'arm',
  46. },
  47. 'armv7-eabihf': {
  48. 'conditions': ['BR2_ARM_CPU_ARMV7A', 'BR2_ARM_EABIHF'],
  49. 'test_options': ['BR2_arm', 'BR2_cortex_a8', 'BR2_ARM_EABIHF'],
  50. 'prefix': 'arm',
  51. },
  52. 'armv7m': {
  53. 'conditions': ['BR2_ARM_CPU_ARMV7M'],
  54. 'test_options': ['BR2_arm', 'BR2_cortex_m4'],
  55. 'prefix': 'arm',
  56. },
  57. 'm68k-68xxx': {
  58. 'conditions': ['BR2_m68k_m68k'],
  59. 'test_options': ['BR2_m68k', 'BR2_m68k_68040'],
  60. 'prefix': 'm68k',
  61. },
  62. 'm68k-coldfire': {
  63. 'conditions': ['BR2_m68k_cf'],
  64. 'test_options': ['BR2_m68k', 'BR2_m68k_cf5208'],
  65. 'prefix': 'm68k',
  66. },
  67. 'microblazebe': {
  68. 'conditions': ['BR2_microblazebe'],
  69. 'prefix': 'microblaze',
  70. },
  71. 'microblazeel': {
  72. 'conditions': ['BR2_microblazeel'],
  73. 'prefix': 'microblazeel',
  74. },
  75. 'mips32': {
  76. # Not sure it could be used by other mips32 variants?
  77. 'conditions': ['BR2_mips', 'BR2_mips_32', '!BR2_MIPS_SOFT_FLOAT'],
  78. 'prefix': 'mips',
  79. },
  80. 'mips32el': {
  81. # Not sure it could be used by other mips32el variants?
  82. 'conditions': ['BR2_mipsel', 'BR2_mips_32', '!BR2_MIPS_SOFT_FLOAT'],
  83. 'prefix': 'mipsel',
  84. },
  85. 'mips32r5el': {
  86. 'conditions': ['BR2_mipsel', 'BR2_mips_32r5', '!BR2_MIPS_SOFT_FLOAT'],
  87. 'prefix': 'mipsel',
  88. },
  89. 'mips32r6el': {
  90. 'conditions': ['BR2_mipsel', 'BR2_mips_32r6', '!BR2_MIPS_SOFT_FLOAT'],
  91. 'prefix': 'mipsel',
  92. },
  93. 'mips64': {
  94. # Not sure it could be used by other mips64 variants?
  95. 'conditions': ['BR2_mips64', 'BR2_mips_64', '!BR2_MIPS_SOFT_FLOAT'],
  96. 'prefix': 'mips64',
  97. },
  98. 'mips64-n32': {
  99. # Not sure it could be used by other mips64 variants?
  100. 'conditions': ['BR2_mips64', 'BR2_mips_64', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
  101. 'prefix': 'mips64',
  102. },
  103. 'mips64el-n32': {
  104. # Not sure it could be used by other mips64el variants?
  105. 'conditions': ['BR2_mips64el', 'BR2_mips_64', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
  106. 'prefix': 'mips64el',
  107. },
  108. 'mips64r6el-n32': {
  109. 'conditions': ['BR2_mips64el', 'BR2_mips_64r6', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
  110. 'prefix': 'mips64el',
  111. },
  112. 'nios2': {
  113. 'conditions': ['BR2_nios2'],
  114. 'prefix': 'nios2',
  115. },
  116. 'openrisc': {
  117. 'conditions': ['BR2_or1k'],
  118. 'prefix': 'or1k',
  119. },
  120. 'powerpc-440fp': {
  121. # Not sure it could be used by other powerpc variants?
  122. 'conditions': ['BR2_powerpc', 'BR2_powerpc_440fp'],
  123. 'prefix': 'powerpc',
  124. },
  125. 'powerpc-e300c3': {
  126. # Not sure it could be used by other powerpc variants?
  127. 'conditions': ['BR2_powerpc', 'BR2_powerpc_e300c3'],
  128. 'prefix': 'powerpc',
  129. },
  130. 'powerpc-e500mc': {
  131. # Not sure it could be used by other powerpc variants?
  132. 'conditions': ['BR2_powerpc', 'BR2_powerpc_e500mc'],
  133. 'prefix': 'powerpc',
  134. },
  135. 'powerpc64-e5500': {
  136. 'conditions': ['BR2_powerpc64', 'BR2_powerpc_e5500'],
  137. 'prefix': 'powerpc64',
  138. },
  139. 'powerpc64-e6500': {
  140. 'conditions': ['BR2_powerpc64', 'BR2_powerpc_e6500'],
  141. 'prefix': 'powerpc64',
  142. },
  143. 'powerpc64-power8': {
  144. 'conditions': ['BR2_powerpc64', 'BR2_powerpc_power8'],
  145. 'prefix': 'powerpc64',
  146. },
  147. 'powerpc64le-power8': {
  148. 'conditions': ['BR2_powerpc64le', 'BR2_powerpc_power8'],
  149. 'prefix': 'powerpc64le',
  150. },
  151. 'riscv32-ilp32d': {
  152. 'conditions': ['BR2_riscv', 'BR2_riscv_g', 'BR2_RISCV_32', 'BR2_RISCV_ABI_ILP32D'],
  153. 'prefix': 'riscv32',
  154. },
  155. 'riscv64': {
  156. 'conditions': ['BR2_riscv', 'BR2_riscv_g', 'BR2_RISCV_64', 'BR2_RISCV_ABI_LP64'],
  157. 'prefix': 'riscv64',
  158. },
  159. 'sh-sh4': {
  160. 'conditions': ['BR2_sh', 'BR2_sh4'],
  161. 'prefix': 'sh4',
  162. },
  163. 'sh-sh4aeb': {
  164. 'conditions': ['BR2_sh', 'BR2_sh4aeb'],
  165. 'prefix': 'sh4aeb',
  166. },
  167. 'sparc64': {
  168. 'conditions': ['BR2_sparc64', 'BR2_sparc_v9'],
  169. 'prefix': 'sparc64',
  170. },
  171. 'sparcv8': {
  172. 'conditions': ['BR2_sparc', 'BR2_sparc_v8'],
  173. 'prefix': 'sparc',
  174. },
  175. 'x86-64-core-i7': {
  176. 'conditions': ['BR2_x86_64',
  177. 'BR2_X86_CPU_HAS_MMX',
  178. 'BR2_X86_CPU_HAS_SSE',
  179. 'BR2_X86_CPU_HAS_SSE2',
  180. 'BR2_X86_CPU_HAS_SSE3',
  181. 'BR2_X86_CPU_HAS_SSSE3',
  182. 'BR2_X86_CPU_HAS_SSE4',
  183. 'BR2_X86_CPU_HAS_SSE42'],
  184. 'test_options': ['BR2_x86_64', 'BR2_x86_corei7'],
  185. 'prefix': 'x86_64',
  186. },
  187. 'x86-core2': {
  188. 'conditions': ['BR2_i386',
  189. 'BR2_X86_CPU_HAS_MMX',
  190. 'BR2_X86_CPU_HAS_SSE',
  191. 'BR2_X86_CPU_HAS_SSE2',
  192. 'BR2_X86_CPU_HAS_SSE3',
  193. 'BR2_X86_CPU_HAS_SSSE3'],
  194. 'test_options': ['BR2_i386', 'BR2_x86_core2'],
  195. 'prefix': 'i686',
  196. },
  197. 'x86-i686': {
  198. 'conditions': ['BR2_i386',
  199. '!BR2_x86_i486',
  200. '!BR2_x86_i586',
  201. '!BR2_x86_x1000'],
  202. 'test_options': ['BR2_i386',
  203. 'BR2_x86_i686'],
  204. 'prefix': 'i686',
  205. },
  206. 'xtensa-lx60': {
  207. 'conditions': ['BR2_xtensa', 'BR2_XTENSA_CUSTOM', 'BR2_XTENSA_LITTLE_ENDIAN'],
  208. 'prefix': 'xtensa',
  209. },
  210. }
  211. class Toolchain:
  212. def __init__(self, arch, libc, variant, version):
  213. self.arch = arch
  214. self.libc = libc
  215. self.variant = variant
  216. self.version = version
  217. self.fname_prefix = "%s--%s--%s-%s" % (self.arch, self.libc, self.variant, self.version)
  218. self.option_name = "BR2_TOOLCHAIN_EXTERNAL_BOOTLIN_%s_%s_%s" % \
  219. (self.arch.replace("-", "_").upper(), self.libc.upper(), self.variant.replace("-", "_").upper())
  220. self.fragment = requests.get(self.fragment_url).text.split("\n")
  221. self.sha256 = requests.get(self.hash_url).text.split(" ")[0]
  222. @property
  223. def tarball_url(self):
  224. return os.path.join(BASE_URL, self.arch, "tarballs",
  225. self.fname_prefix + ".tar.bz2")
  226. @property
  227. def hash_url(self):
  228. return os.path.join(BASE_URL, self.arch, "tarballs",
  229. self.fname_prefix + ".sha256")
  230. @property
  231. def fragment_url(self):
  232. return os.path.join(BASE_URL, self.arch, "fragments",
  233. self.fname_prefix + ".frag")
  234. def gen_config_in_options(self, f):
  235. f.write("config %s\n" % self.option_name)
  236. f.write("\tbool \"%s %s %s %s\"\n" %
  237. (self.arch, self.libc, self.variant, self.version))
  238. depends = []
  239. selects = []
  240. for c in arches[self.arch]['conditions']:
  241. depends.append(c)
  242. for frag in self.fragment:
  243. # libc type
  244. if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_UCLIBC"):
  245. selects.append("BR2_TOOLCHAIN_EXTERNAL_UCLIBC")
  246. elif frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_GLIBC"):
  247. # glibc needs mmu support
  248. depends.append("BR2_USE_MMU")
  249. # glibc doesn't support static only configuration
  250. depends.append("!BR2_STATIC_LIBS")
  251. selects.append("BR2_TOOLCHAIN_EXTERNAL_GLIBC")
  252. elif frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_MUSL"):
  253. # musl needs mmu support
  254. depends.append("BR2_USE_MMU")
  255. selects.append("BR2_TOOLCHAIN_EXTERNAL_MUSL")
  256. # gcc version
  257. if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_GCC_"):
  258. m = re.match("^BR2_TOOLCHAIN_EXTERNAL_GCC_([0-9_]*)=y$", frag)
  259. assert m, "Cannot get gcc version for toolchain %s" % self.fname_prefix
  260. selects.append("BR2_TOOLCHAIN_GCC_AT_LEAST_%s" % m[1])
  261. # kernel headers version
  262. if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HEADERS_"):
  263. m = re.match("^BR2_TOOLCHAIN_EXTERNAL_HEADERS_([0-9_]*)=y$", frag)
  264. assert m, "Cannot get kernel headers version for toolchain %s" % self.fname_prefix
  265. selects.append("BR2_TOOLCHAIN_HEADERS_AT_LEAST_%s" % m[1])
  266. # C++
  267. if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CXX"):
  268. selects.append("BR2_INSTALL_LIBSTDCPP")
  269. # SSP
  270. if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_SSP"):
  271. selects.append("BR2_TOOLCHAIN_HAS_SSP")
  272. # wchar
  273. if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_WCHAR"):
  274. selects.append("BR2_USE_WCHAR")
  275. # locale
  276. if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_LOCALE"):
  277. # locale implies the availability of wchar
  278. selects.append("BR2_USE_WCHAR")
  279. selects.append("BR2_ENABLE_LOCALE")
  280. # thread support
  281. if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS"):
  282. selects.append("BR2_TOOLCHAIN_HAS_THREADS")
  283. if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS_DEBUG"):
  284. selects.append("BR2_TOOLCHAIN_HAS_THREADS_DEBUG")
  285. if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS_NPTL"):
  286. selects.append("BR2_TOOLCHAIN_HAS_THREADS_NPTL")
  287. # RPC
  288. if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_INET_RPC"):
  289. selects.append("BR2_TOOLCHAIN_HAS_NATIVE_RPC")
  290. # D language
  291. if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_DLANG"):
  292. selects.append("BR2_TOOLCHAIN_HAS_DLANG")
  293. # fortran
  294. if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_FORTRAN"):
  295. selects.append("BR2_TOOLCHAIN_HAS_FORTRAN")
  296. # OpenMP
  297. if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_OPENMP"):
  298. selects.append("BR2_TOOLCHAIN_HAS_OPENMP")
  299. for depend in depends:
  300. f.write("\tdepends on %s\n" % depend)
  301. for select in selects:
  302. f.write("\tselect %s\n" % select)
  303. f.write("\thelp\n")
  304. desc = "Bootlin toolchain for the %s architecture, using the %s C library. " % \
  305. (self.arch, self.libc)
  306. if self.variant == "stable":
  307. desc += "This is a stable version, which means it is using stable and proven versions of gcc, gdb and binutils."
  308. else:
  309. desc += "This is a bleeding-edge version, which means it is using the latest versions of gcc, gdb and binutils."
  310. f.write(textwrap.fill(desc, width=62, initial_indent="\t ", subsequent_indent="\t ") + "\n")
  311. f.write("\n")
  312. f.write("\t https://toolchains.bootlin.com/\n")
  313. f.write("\n")
  314. def gen_mk(self, f):
  315. f.write("ifeq ($(%s),y)\n" % self.option_name)
  316. f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_VERSION = %s\n" % self.version)
  317. f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_SOURCE = %s--%s--%s-$(TOOLCHAIN_EXTERNAL_BOOTLIN_VERSION).tar.bz2\n" %
  318. (self.arch, self.libc, self.variant))
  319. f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_SITE = %s\n" %
  320. os.path.join(BASE_URL, self.arch, "tarballs"))
  321. f.write("endif\n\n")
  322. pass
  323. def gen_hash(self, f):
  324. f.write("# From %s\n" % self.hash_url)
  325. f.write("sha256 %s %s\n" % (self.sha256, os.path.basename(self.tarball_url)))
  326. def gen_test(self, f):
  327. if self.variant == "stable":
  328. variant = "Stable"
  329. else:
  330. variant = "BleedingEdge"
  331. testname = "TestExternalToolchainBootlin" + \
  332. self.arch.replace("-", "").capitalize() + \
  333. self.libc.capitalize() + variant
  334. f.write("\n\n")
  335. f.write("class %s(TestExternalToolchain):\n" % testname)
  336. f.write(" config = \"\"\"\n")
  337. if 'test_options' in arches[self.arch]:
  338. test_options = arches[self.arch]['test_options']
  339. else:
  340. test_options = arches[self.arch]['conditions']
  341. for opt in test_options:
  342. if opt.startswith("!"):
  343. f.write(" # %s is not set\n" % opt[1:])
  344. else:
  345. f.write(" %s=y\n" % opt)
  346. f.write(" BR2_TOOLCHAIN_EXTERNAL=y\n")
  347. f.write(" BR2_TOOLCHAIN_EXTERNAL_BOOTLIN=y\n")
  348. f.write(" %s=y\n" % self.option_name)
  349. f.write(" # BR2_TARGET_ROOTFS_TAR is not set\n")
  350. f.write(" \"\"\"\n")
  351. f.write(" toolchain_prefix = \"%s-linux\"\n" % arches[self.arch]['prefix'])
  352. f.write("\n")
  353. f.write(" def test_run(self):\n")
  354. f.write(" TestExternalToolchain.common_check(self)\n")
  355. def __repr__(self):
  356. return "Toolchain(arch=%s libc=%s variant=%s version=%s, option=%s)" % \
  357. (self.arch, self.libc, self.variant, self.version, self.option_name)
  358. def get_toolchains():
  359. toolchains = list()
  360. for arch, details in arches.items():
  361. print(arch)
  362. url = os.path.join(BASE_URL, arch, "available_toolchains")
  363. page = requests.get(url).text
  364. fnames = sorted(re.findall(r'<td><a href="(\w[^"]+)"', page))
  365. # This dict will allow us to keep only the latest version for
  366. # each toolchain.
  367. tmp = dict()
  368. for fname in fnames:
  369. parts = fname.split('--')
  370. assert parts[0] == arch, "Arch does not match: %s vs. %s" % (parts[0], arch)
  371. libc = parts[1]
  372. if parts[2].startswith("stable-"):
  373. variant = "stable"
  374. version = parts[2][len("stable-"):]
  375. elif parts[2].startswith("bleeding-edge-"):
  376. variant = "bleeding-edge"
  377. version = parts[2][len("bleeding-edge-"):]
  378. tmp[(arch, libc, variant)] = version
  379. toolchains += [Toolchain(k[0], k[1], k[2], v) for k, v in tmp.items()]
  380. return toolchains
  381. def gen_config_in_options(toolchains, fpath):
  382. with open(fpath, "w") as f:
  383. f.write(AUTOGENERATED_COMMENT)
  384. f.write("config BR2_TOOLCHAIN_EXTERNAL_BOOTLIN_ARCH_SUPPORTS\n")
  385. f.write("\tbool\n")
  386. for arch, details in arches.items():
  387. f.write("\tdefault y if %s\n" % " && ".join(details['conditions']))
  388. f.write("\n")
  389. f.write("if BR2_TOOLCHAIN_EXTERNAL_BOOTLIN\n\n")
  390. f.write("config BR2_TOOLCHAIN_EXTERNAL_PREFIX\n")
  391. f.write("\tdefault \"$(ARCH)-linux\"\n")
  392. f.write("\n")
  393. f.write("config BR2_PACKAGE_PROVIDES_TOOLCHAIN_EXTERNAL\n")
  394. f.write("\tdefault \"toolchain-external-bootlin\"\n")
  395. f.write("\n")
  396. f.write("choice\n")
  397. f.write("\tprompt \"Bootlin toolchain variant\"\n")
  398. for toolchain in toolchains:
  399. toolchain.gen_config_in_options(f)
  400. f.write("endchoice\n")
  401. f.write("endif\n")
  402. def gen_mk(toolchains, fpath):
  403. with open(fpath, "w") as f:
  404. f.write("#" * 80 + "\n")
  405. f.write("#\n")
  406. f.write("# toolchain-external-bootlin\n")
  407. f.write("#\n")
  408. f.write("#" * 80 + "\n")
  409. f.write("\n")
  410. f.write(AUTOGENERATED_COMMENT)
  411. for toolchain in toolchains:
  412. toolchain.gen_mk(f)
  413. f.write("$(eval $(toolchain-external-package))\n")
  414. def gen_hash(toolchains, fpath):
  415. with open(fpath, "w") as f:
  416. f.write(AUTOGENERATED_COMMENT)
  417. for toolchain in toolchains:
  418. toolchain.gen_hash(f)
  419. def gen_runtime_test(toolchains, fpath):
  420. with open(fpath, "w") as f:
  421. f.write(AUTOGENERATED_COMMENT)
  422. f.write("from tests.toolchain.test_external import TestExternalToolchain\n")
  423. for toolchain in toolchains:
  424. toolchain.gen_test(f)
  425. def gen_toolchains(toolchains):
  426. maindir = "toolchain/toolchain-external/toolchain-external-bootlin"
  427. gen_config_in_options(toolchains, os.path.join(maindir, "Config.in.options"))
  428. gen_mk(toolchains, os.path.join(maindir, "toolchain-external-bootlin.mk"))
  429. gen_hash(toolchains, os.path.join(maindir, "toolchain-external-bootlin.hash"))
  430. gen_runtime_test(toolchains,
  431. os.path.join("support", "testing", "tests", "toolchain", "test_external_bootlin.py"))
  432. toolchains = get_toolchains()
  433. gen_toolchains(toolchains)