gen-bootlin-toolchains 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472
  1. #!/usr/bin/env python3
  2. import os.path
  3. import re
  4. import requests
  5. import textwrap
  6. BASE_URL = "https://toolchains.bootlin.com/downloads/releases/toolchains"
  7. AUTOGENERATED_COMMENT = """# This file was auto-generated by support/scripts/gen-bootlin-toolchains
  8. # Do not edit
  9. """
  10. # In the below dict:
  11. # - 'conditions' indicate the cumulative conditions under which the
  12. # toolchain will be made available. In several situations, a given
  13. # toolchain is usable on several architectures variants (for
  14. # example, an ARMv6 toolchain can be used on ARMv7)
  15. # - 'test_options' indicate one specific configuration where the
  16. # toolchain can be used. It is used to create the runtime test
  17. # cases. If 'test_options' does not exist, the code assumes it can
  18. # be made equal to 'conditions'
  19. # - 'prefix' is the prefix of the cross-compilation toolchain tools
  20. arches = {
  21. 'aarch64': {
  22. 'conditions': ['BR2_aarch64'],
  23. 'prefix': 'aarch64',
  24. },
  25. 'aarch64be': {
  26. 'conditions': ['BR2_aarch64_be'],
  27. 'prefix': 'aarch64_be',
  28. },
  29. 'arcle-750d': {
  30. 'conditions': ['BR2_arcle', 'BR2_arc750d'],
  31. 'prefix': 'arc',
  32. },
  33. 'arcle-hs38': {
  34. 'conditions': ['BR2_arcle', 'BR2_archs38'],
  35. 'prefix': 'arc',
  36. },
  37. 'armv5-eabi': {
  38. 'conditions': ['BR2_ARM_CPU_ARMV5', 'BR2_ARM_EABI'],
  39. 'test_options': ['BR2_arm', 'BR2_arm926t', 'BR2_ARM_EABI'],
  40. 'prefix': 'arm',
  41. },
  42. 'armv6-eabihf': {
  43. 'conditions': ['BR2_ARM_CPU_ARMV6', 'BR2_ARM_EABIHF'],
  44. 'test_options': ['BR2_arm', 'BR2_arm1176jzf_s', 'BR2_ARM_EABIHF'],
  45. 'prefix': 'arm',
  46. },
  47. 'armv7-eabihf': {
  48. 'conditions': ['BR2_ARM_CPU_ARMV7A', 'BR2_ARM_EABIHF'],
  49. 'test_options': ['BR2_arm', 'BR2_cortex_a8', 'BR2_ARM_EABIHF'],
  50. 'prefix': 'arm',
  51. },
  52. 'armv7m': {
  53. 'conditions': ['BR2_ARM_CPU_ARMV7M'],
  54. 'test_options': ['BR2_arm', 'BR2_cortex_m4'],
  55. 'prefix': 'arm',
  56. },
  57. 'm68k-68xxx': {
  58. 'conditions': ['BR2_m68k_m68k'],
  59. 'test_options': ['BR2_m68k', 'BR2_m68k_68040'],
  60. 'prefix': 'm68k',
  61. },
  62. 'm68k-coldfire': {
  63. 'conditions': ['BR2_m68k_cf'],
  64. 'test_options': ['BR2_m68k', 'BR2_m68k_cf5208'],
  65. 'prefix': 'm68k',
  66. },
  67. 'microblazebe': {
  68. 'conditions': ['BR2_microblazebe'],
  69. 'prefix': 'microblaze',
  70. },
  71. 'microblazeel': {
  72. 'conditions': ['BR2_microblazeel'],
  73. 'prefix': 'microblazeel',
  74. },
  75. 'mips32': {
  76. # Not sure it could be used by other mips32 variants?
  77. 'conditions': ['BR2_mips', 'BR2_mips_32', '!BR2_MIPS_SOFT_FLOAT'],
  78. 'prefix': 'mips',
  79. },
  80. 'mips32el': {
  81. # Not sure it could be used by other mips32el variants?
  82. 'conditions': ['BR2_mipsel', 'BR2_mips_32', '!BR2_MIPS_SOFT_FLOAT'],
  83. 'prefix': 'mipsel',
  84. },
  85. 'mips32r5el': {
  86. 'conditions': ['BR2_mipsel', 'BR2_mips_32r5', '!BR2_MIPS_SOFT_FLOAT'],
  87. 'prefix': 'mipsel',
  88. },
  89. 'mips32r6el': {
  90. 'conditions': ['BR2_mipsel', 'BR2_mips_32r6', '!BR2_MIPS_SOFT_FLOAT'],
  91. 'prefix': 'mipsel',
  92. },
  93. 'mips64': {
  94. # Not sure it could be used by other mips64 variants?
  95. 'conditions': ['BR2_mips64', 'BR2_mips_64', '!BR2_MIPS_SOFT_FLOAT'],
  96. 'prefix': 'mips64',
  97. },
  98. 'mips64-n32': {
  99. # Not sure it could be used by other mips64 variants?
  100. 'conditions': ['BR2_mips64', 'BR2_mips_64', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
  101. 'prefix': 'mips64',
  102. },
  103. 'mips64el-n32': {
  104. # Not sure it could be used by other mips64el variants?
  105. 'conditions': ['BR2_mips64el', 'BR2_mips_64', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
  106. 'prefix': 'mips64el',
  107. },
  108. 'mips64r6el-n32': {
  109. 'conditions': ['BR2_mips64el', 'BR2_mips_64r6', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
  110. 'prefix': 'mips64el',
  111. },
  112. 'nios2': {
  113. 'conditions': ['BR2_nios2'],
  114. 'prefix': 'nios2',
  115. },
  116. 'openrisc': {
  117. 'conditions': ['BR2_or1k'],
  118. 'prefix': 'or1k',
  119. },
  120. 'powerpc-e500mc': {
  121. # Not sure it could be used by other powerpc variants?
  122. 'conditions': ['BR2_powerpc', 'BR2_powerpc_e500mc'],
  123. 'prefix': 'powerpc',
  124. },
  125. 'powerpc64-e5500': {
  126. 'conditions': ['BR2_powerpc64', 'BR2_powerpc_e5500'],
  127. 'prefix': 'powerpc64',
  128. },
  129. 'powerpc64-power8': {
  130. 'conditions': ['BR2_powerpc64', 'BR2_powerpc_power8'],
  131. 'prefix': 'powerpc64',
  132. },
  133. 'powerpc64le-power8': {
  134. 'conditions': ['BR2_powerpc64le', 'BR2_powerpc_power8'],
  135. 'prefix': 'powerpc64le',
  136. },
  137. 'riscv32-ilp32d': {
  138. 'conditions': ['BR2_riscv', 'BR2_riscv_g', 'BR2_RISCV_32', 'BR2_RISCV_ABI_ILP32D'],
  139. 'prefix': 'riscv32',
  140. },
  141. 'riscv64': {
  142. 'conditions': ['BR2_riscv', 'BR2_riscv_g', 'BR2_RISCV_64', 'BR2_RISCV_ABI_LP64'],
  143. 'prefix': 'riscv64',
  144. },
  145. 'sh-sh4': {
  146. 'conditions': ['BR2_sh', 'BR2_sh4'],
  147. 'prefix': 'sh4',
  148. },
  149. 'sh-sh4aeb': {
  150. 'conditions': ['BR2_sh', 'BR2_sh4aeb'],
  151. 'prefix': 'sh4aeb',
  152. },
  153. 'sparc64': {
  154. 'conditions': ['BR2_sparc64', 'BR2_sparc_v9'],
  155. 'prefix': 'sparc64',
  156. },
  157. 'sparcv8': {
  158. 'conditions': ['BR2_sparc', 'BR2_sparc_v8'],
  159. 'prefix': 'sparc',
  160. },
  161. 'x86-64-core-i7': {
  162. 'conditions': ['BR2_x86_64',
  163. 'BR2_X86_CPU_HAS_MMX',
  164. 'BR2_X86_CPU_HAS_SSE',
  165. 'BR2_X86_CPU_HAS_SSE2',
  166. 'BR2_X86_CPU_HAS_SSE3',
  167. 'BR2_X86_CPU_HAS_SSSE3',
  168. 'BR2_X86_CPU_HAS_SSE4',
  169. 'BR2_X86_CPU_HAS_SSE42'],
  170. 'test_options': ['BR2_x86_64', 'BR2_x86_corei7'],
  171. 'prefix': 'x86_64',
  172. },
  173. 'x86-core2': {
  174. 'conditions': ['BR2_i386',
  175. 'BR2_X86_CPU_HAS_MMX',
  176. 'BR2_X86_CPU_HAS_SSE',
  177. 'BR2_X86_CPU_HAS_SSE2',
  178. 'BR2_X86_CPU_HAS_SSE3',
  179. 'BR2_X86_CPU_HAS_SSSE3'],
  180. 'test_options': ['BR2_i386', 'BR2_x86_core2'],
  181. 'prefix': 'i686',
  182. },
  183. 'x86-i686': {
  184. 'conditions': ['BR2_i386',
  185. '!BR2_x86_i486',
  186. '!BR2_x86_i586',
  187. '!BR2_x86_x1000'],
  188. 'test_options': ['BR2_i386',
  189. 'BR2_x86_i686'],
  190. 'prefix': 'i686',
  191. },
  192. 'xtensa-lx60': {
  193. 'conditions': ['BR2_xtensa', 'BR2_xtensa_fsf'],
  194. 'prefix': 'xtensa',
  195. },
  196. }
  197. class Toolchain:
  198. def __init__(self, arch, libc, variant, version):
  199. self.arch = arch
  200. self.libc = libc
  201. self.variant = variant
  202. self.version = version
  203. self.fname_prefix = "%s--%s--%s-%s" % (self.arch, self.libc, self.variant, self.version)
  204. self.option_name = "BR2_TOOLCHAIN_EXTERNAL_BOOTLIN_%s_%s_%s" % \
  205. (self.arch.replace("-", "_").upper(), self.libc.upper(), self.variant.replace("-", "_").upper())
  206. self.fragment = requests.get(self.fragment_url).text.split("\n")
  207. self.sha256 = requests.get(self.hash_url).text.split(" ")[0]
  208. @property
  209. def tarball_url(self):
  210. return os.path.join(BASE_URL, self.arch, "tarballs",
  211. self.fname_prefix + ".tar.bz2")
  212. @property
  213. def hash_url(self):
  214. return os.path.join(BASE_URL, self.arch, "tarballs",
  215. self.fname_prefix + ".sha256")
  216. @property
  217. def fragment_url(self):
  218. return os.path.join(BASE_URL, self.arch, "fragments",
  219. self.fname_prefix + ".frag")
  220. def gen_config_in_options(self, f):
  221. f.write("config %s\n" % self.option_name)
  222. f.write("\tbool \"%s %s %s %s\"\n" %
  223. (self.arch, self.libc, self.variant, self.version))
  224. for c in arches[self.arch]['conditions']:
  225. f.write("\tdepends on %s\n" % c)
  226. selects = []
  227. for frag in self.fragment:
  228. # libc type
  229. if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_UCLIBC"):
  230. selects.append("BR2_TOOLCHAIN_EXTERNAL_UCLIBC")
  231. elif frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_GLIBC"):
  232. selects.append("BR2_TOOLCHAIN_EXTERNAL_GLIBC")
  233. # all glibc toolchains have RPC support
  234. selects.append("BR2_TOOLCHAIN_HAS_NATIVE_RPC")
  235. elif frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_MUSL"):
  236. selects.append("BR2_TOOLCHAIN_EXTERNAL_MUSL")
  237. # gcc version
  238. if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_GCC_"):
  239. m = re.match("^BR2_TOOLCHAIN_EXTERNAL_GCC_([0-9_]*)=y$", frag)
  240. assert m, "Cannot get gcc version for toolchain %s" % self.fname_prefix
  241. selects.append("BR2_TOOLCHAIN_GCC_AT_LEAST_%s" % m[1])
  242. # kernel headers version
  243. if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HEADERS_"):
  244. m = re.match("^BR2_TOOLCHAIN_EXTERNAL_HEADERS_([0-9_]*)=y$", frag)
  245. assert m, "Cannot get kernel headers version for toolchain %s" % self.fname_prefix
  246. selects.append("BR2_TOOLCHAIN_HEADERS_AT_LEAST_%s" % m[1])
  247. # C++
  248. if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CXX"):
  249. selects.append("BR2_INSTALL_LIBSTDCPP")
  250. # SSP
  251. if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_SSP"):
  252. selects.append("BR2_TOOLCHAIN_HAS_SSP")
  253. # wchar
  254. if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_WCHAR"):
  255. selects.append("BR2_USE_WCHAR")
  256. # locale
  257. if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_LOCALE"):
  258. # locale implies the availability of wchar
  259. selects.append("BR2_USE_WCHAR")
  260. selects.append("BR2_ENABLE_LOCALE")
  261. # thread support
  262. if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS"):
  263. selects.append("BR2_TOOLCHAIN_HAS_THREADS")
  264. if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS_DEBUG"):
  265. selects.append("BR2_TOOLCHAIN_HAS_THREADS_DEBUG")
  266. if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS_NPTL"):
  267. selects.append("BR2_TOOLCHAIN_HAS_THREADS_NPTL")
  268. # RPC
  269. if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_INET_RPC"):
  270. selects.append("BR2_TOOLCHAIN_HAS_NATIVE_RPC")
  271. # D language
  272. if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_DLANG"):
  273. selects.append("BR2_TOOLCHAIN_HAS_DLANG")
  274. # fortran
  275. if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_FORTRAN"):
  276. selects.append("BR2_TOOLCHAIN_HAS_FORTRAN")
  277. # OpenMP
  278. if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_OPENMP"):
  279. selects.append("BR2_TOOLCHAIN_HAS_OPENMP")
  280. for select in selects:
  281. f.write("\tselect %s\n" % select)
  282. f.write("\thelp\n")
  283. desc = "Bootlin toolchain for the %s architecture, using the %s C library. " % \
  284. (self.arch, self.libc)
  285. if self.variant == "stable":
  286. desc += "This is a stable version, which means it is using stable and proven versions of gcc, gdb and binutils."
  287. else:
  288. desc += "This is a bleeding-edge version, which means it is using the latest versions of gcc, gdb and binutils."
  289. f.write(textwrap.fill(desc, width=62, initial_indent="\t ", subsequent_indent="\t ") + "\n")
  290. f.write("\n")
  291. f.write("\t https://toolchains.bootlin.com/\n")
  292. f.write("\n")
  293. def gen_mk(self, f):
  294. f.write("ifeq ($(%s),y)\n" % self.option_name)
  295. f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_VERSION = %s\n" % self.version)
  296. f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_SOURCE = %s--%s--%s-$(TOOLCHAIN_EXTERNAL_BOOTLIN_VERSION).tar.bz2\n" %
  297. (self.arch, self.libc, self.variant))
  298. f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_SITE = %s\n" %
  299. os.path.join(BASE_URL, self.arch, "tarballs"))
  300. f.write("endif\n\n")
  301. pass
  302. def gen_hash(self, f):
  303. f.write("# From %s\n" % self.hash_url)
  304. f.write("sha256 %s %s\n" % (self.sha256, os.path.basename(self.tarball_url)))
  305. def gen_test(self, f):
  306. if self.variant == "stable":
  307. variant = "Stable"
  308. else:
  309. variant = "BleedingEdge"
  310. testname = "TestExternalToolchainBootlin" + \
  311. self.arch.replace("-", "").capitalize() + \
  312. self.libc.capitalize() + variant
  313. f.write("\n\n")
  314. f.write("class %s(TestExternalToolchain):\n" % testname)
  315. f.write(" config = \"\"\"\n")
  316. if 'test_options' in arches[self.arch]:
  317. test_options = arches[self.arch]['test_options']
  318. else:
  319. test_options = arches[self.arch]['conditions']
  320. for opt in test_options:
  321. if opt.startswith("!"):
  322. f.write(" # %s is not set\n" % opt[1:])
  323. else:
  324. f.write(" %s=y\n" % opt)
  325. f.write(" BR2_TOOLCHAIN_EXTERNAL=y\n")
  326. f.write(" BR2_TOOLCHAIN_EXTERNAL_BOOTLIN=y\n")
  327. f.write(" %s=y\n" % self.option_name)
  328. f.write(" # BR2_TARGET_ROOTFS_TAR is not set\n")
  329. f.write(" \"\"\"\n")
  330. f.write(" toolchain_prefix = \"%s-linux\"\n" % arches[self.arch]['prefix'])
  331. f.write("\n")
  332. f.write(" def test_run(self):\n")
  333. f.write(" TestExternalToolchain.common_check(self)\n")
  334. def __repr__(self):
  335. return "Toolchain(arch=%s libc=%s variant=%s version=%s, option=%s)" % \
  336. (self.arch, self.libc, self.variant, self.version, self.option_name)
  337. def get_toolchains():
  338. toolchains = list()
  339. for arch, details in arches.items():
  340. print(arch)
  341. url = os.path.join(BASE_URL, arch, "available_toolchains")
  342. page = requests.get(url).text
  343. fnames = sorted(re.findall(r'<td><a href="(\w[^"]+)"', page))
  344. # This dict will allow us to keep only the latest version for
  345. # each toolchain.
  346. tmp = dict()
  347. for fname in fnames:
  348. parts = fname.split('--')
  349. assert parts[0] == arch, "Arch does not match: %s vs. %s" % (parts[0], arch)
  350. libc = parts[1]
  351. if parts[2].startswith("stable-"):
  352. variant = "stable"
  353. version = parts[2][len("stable-"):]
  354. elif parts[2].startswith("bleeding-edge-"):
  355. variant = "bleeding-edge"
  356. version = parts[2][len("bleeding-edge-"):]
  357. tmp[(arch, libc, variant)] = version
  358. toolchains += [Toolchain(k[0], k[1], k[2], v) for k, v in tmp.items()]
  359. return toolchains
  360. def gen_config_in_options(toolchains, fpath):
  361. with open(fpath, "w") as f:
  362. f.write(AUTOGENERATED_COMMENT)
  363. f.write("config BR2_TOOLCHAIN_EXTERNAL_BOOTLIN_ARCH_SUPPORTS\n")
  364. f.write("\tbool\n")
  365. for arch, details in arches.items():
  366. f.write("\tdefault y if %s\n" % " && ".join(details['conditions']))
  367. f.write("\n")
  368. f.write("if BR2_TOOLCHAIN_EXTERNAL_BOOTLIN\n\n")
  369. f.write("config BR2_TOOLCHAIN_EXTERNAL_PREFIX\n")
  370. f.write("\tdefault \"$(ARCH)-linux\"\n")
  371. f.write("\n")
  372. f.write("config BR2_PACKAGE_PROVIDES_TOOLCHAIN_EXTERNAL\n")
  373. f.write("\tdefault \"toolchain-external-bootlin\"\n")
  374. f.write("\n")
  375. f.write("choice\n")
  376. f.write("\tprompt \"Bootlin toolchain variant\"\n")
  377. for toolchain in toolchains:
  378. toolchain.gen_config_in_options(f)
  379. f.write("endchoice\n")
  380. f.write("endif\n")
  381. def gen_mk(toolchains, fpath):
  382. with open(fpath, "w") as f:
  383. f.write("#" * 80 + "\n")
  384. f.write("#\n")
  385. f.write("# toolchain-external-bootlin\n")
  386. f.write("#\n")
  387. f.write("#" * 80 + "\n")
  388. f.write("\n")
  389. f.write(AUTOGENERATED_COMMENT)
  390. for toolchain in toolchains:
  391. toolchain.gen_mk(f)
  392. f.write("$(eval $(toolchain-external-package))\n")
  393. def gen_hash(toolchains, fpath):
  394. with open(fpath, "w") as f:
  395. f.write(AUTOGENERATED_COMMENT)
  396. for toolchain in toolchains:
  397. toolchain.gen_hash(f)
  398. def gen_runtime_test(toolchains, fpath):
  399. with open(fpath, "w") as f:
  400. f.write(AUTOGENERATED_COMMENT)
  401. f.write("from tests.toolchain.test_external import TestExternalToolchain\n")
  402. for toolchain in toolchains:
  403. toolchain.gen_test(f)
  404. def gen_toolchains(toolchains):
  405. maindir = "toolchain/toolchain-external/toolchain-external-bootlin"
  406. gen_config_in_options(toolchains, os.path.join(maindir, "Config.in.options"))
  407. gen_mk(toolchains, os.path.join(maindir, "toolchain-external-bootlin.mk"))
  408. gen_hash(toolchains, os.path.join(maindir, "toolchain-external-bootlin.hash"))
  409. gen_runtime_test(toolchains,
  410. os.path.join("support", "testing", "tests", "toolchain", "test_external_bootlin.py"))
  411. toolchains = get_toolchains()
  412. gen_toolchains(toolchains)