toolchain.py 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645
  1. # SPDX-License-Identifier: GPL-2.0+
  2. # Copyright (c) 2012 The Chromium OS Authors.
  3. #
  4. import re
  5. import glob
  6. from html.parser import HTMLParser
  7. import os
  8. import sys
  9. import tempfile
  10. import urllib.request, urllib.error, urllib.parse
  11. from buildman import bsettings
  12. from patman import command
  13. from patman import terminal
  14. from patman import tools
  15. (PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH,
  16. PRIORITY_CALC) = list(range(4))
  17. (VAR_CROSS_COMPILE, VAR_PATH, VAR_ARCH, VAR_MAKE_ARGS) = range(4)
  18. # Simple class to collect links from a page
  19. class MyHTMLParser(HTMLParser):
  20. def __init__(self, arch):
  21. """Create a new parser
  22. After the parser runs, self.links will be set to a list of the links
  23. to .xz archives found in the page, and self.arch_link will be set to
  24. the one for the given architecture (or None if not found).
  25. Args:
  26. arch: Architecture to search for
  27. """
  28. HTMLParser.__init__(self)
  29. self.arch_link = None
  30. self.links = []
  31. self.re_arch = re.compile('[-_]%s-' % arch)
  32. def handle_starttag(self, tag, attrs):
  33. if tag == 'a':
  34. for tag, value in attrs:
  35. if tag == 'href':
  36. if value and value.endswith('.xz'):
  37. self.links.append(value)
  38. if self.re_arch.search(value):
  39. self.arch_link = value
  40. class Toolchain:
  41. """A single toolchain
  42. Public members:
  43. gcc: Full path to C compiler
  44. path: Directory path containing C compiler
  45. cross: Cross compile string, e.g. 'arm-linux-'
  46. arch: Architecture of toolchain as determined from the first
  47. component of the filename. E.g. arm-linux-gcc becomes arm
  48. priority: Toolchain priority (0=highest, 20=lowest)
  49. override_toolchain: Toolchain to use for sandbox, overriding the normal
  50. one
  51. """
  52. def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC,
  53. arch=None, override_toolchain=None):
  54. """Create a new toolchain object.
  55. Args:
  56. fname: Filename of the gcc component
  57. test: True to run the toolchain to test it
  58. verbose: True to print out the information
  59. priority: Priority to use for this toolchain, or PRIORITY_CALC to
  60. calculate it
  61. """
  62. self.gcc = fname
  63. self.path = os.path.dirname(fname)
  64. self.override_toolchain = override_toolchain
  65. # Find the CROSS_COMPILE prefix to use for U-Boot. For example,
  66. # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'.
  67. basename = os.path.basename(fname)
  68. pos = basename.rfind('-')
  69. self.cross = basename[:pos + 1] if pos != -1 else ''
  70. # The architecture is the first part of the name
  71. pos = self.cross.find('-')
  72. if arch:
  73. self.arch = arch
  74. else:
  75. self.arch = self.cross[:pos] if pos != -1 else 'sandbox'
  76. if self.arch == 'sandbox' and override_toolchain:
  77. self.gcc = override_toolchain
  78. env = self.MakeEnvironment(False)
  79. # As a basic sanity check, run the C compiler with --version
  80. cmd = [fname, '--version']
  81. if priority == PRIORITY_CALC:
  82. self.priority = self.GetPriority(fname)
  83. else:
  84. self.priority = priority
  85. if test:
  86. result = command.RunPipe([cmd], capture=True, env=env,
  87. raise_on_error=False)
  88. self.ok = result.return_code == 0
  89. if verbose:
  90. print('Tool chain test: ', end=' ')
  91. if self.ok:
  92. print("OK, arch='%s', priority %d" % (self.arch,
  93. self.priority))
  94. else:
  95. print('BAD')
  96. print('Command: ', cmd)
  97. print(result.stdout)
  98. print(result.stderr)
  99. else:
  100. self.ok = True
  101. def GetPriority(self, fname):
  102. """Return the priority of the toolchain.
  103. Toolchains are ranked according to their suitability by their
  104. filename prefix.
  105. Args:
  106. fname: Filename of toolchain
  107. Returns:
  108. Priority of toolchain, PRIORITY_CALC=highest, 20=lowest.
  109. """
  110. priority_list = ['-elf', '-unknown-linux-gnu', '-linux',
  111. '-none-linux-gnueabi', '-none-linux-gnueabihf', '-uclinux',
  112. '-none-eabi', '-gentoo-linux-gnu', '-linux-gnueabi',
  113. '-linux-gnueabihf', '-le-linux', '-uclinux']
  114. for prio in range(len(priority_list)):
  115. if priority_list[prio] in fname:
  116. return PRIORITY_CALC + prio
  117. return PRIORITY_CALC + prio
  118. def GetWrapper(self, show_warning=True):
  119. """Get toolchain wrapper from the setting file.
  120. """
  121. value = ''
  122. for name, value in bsettings.GetItems('toolchain-wrapper'):
  123. if not value:
  124. print("Warning: Wrapper not found")
  125. if value:
  126. value = value + ' '
  127. return value
  128. def GetEnvArgs(self, which):
  129. """Get an environment variable/args value based on the the toolchain
  130. Args:
  131. which: VAR_... value to get
  132. Returns:
  133. Value of that environment variable or arguments
  134. """
  135. wrapper = self.GetWrapper()
  136. if which == VAR_CROSS_COMPILE:
  137. return wrapper + os.path.join(self.path, self.cross)
  138. elif which == VAR_PATH:
  139. return self.path
  140. elif which == VAR_ARCH:
  141. return self.arch
  142. elif which == VAR_MAKE_ARGS:
  143. args = self.MakeArgs()
  144. if args:
  145. return ' '.join(args)
  146. return ''
  147. else:
  148. raise ValueError('Unknown arg to GetEnvArgs (%d)' % which)
  149. def MakeEnvironment(self, full_path):
  150. """Returns an environment for using the toolchain.
  151. Thie takes the current environment and adds CROSS_COMPILE so that
  152. the tool chain will operate correctly. This also disables localized
  153. output and possibly unicode encoded output of all build tools by
  154. adding LC_ALL=C.
  155. Note that os.environb is used to obtain the environment, since in some
  156. cases the environment many contain non-ASCII characters and we see
  157. errors like:
  158. UnicodeEncodeError: 'utf-8' codec can't encode characters in position
  159. 569-570: surrogates not allowed
  160. Args:
  161. full_path: Return the full path in CROSS_COMPILE and don't set
  162. PATH
  163. Returns:
  164. Dict containing the (bytes) environment to use. This is based on the
  165. current environment, with changes as needed to CROSS_COMPILE, PATH
  166. and LC_ALL.
  167. """
  168. env = dict(os.environb)
  169. wrapper = self.GetWrapper()
  170. if self.override_toolchain:
  171. # We'll use MakeArgs() to provide this
  172. pass
  173. elif full_path:
  174. env[b'CROSS_COMPILE'] = tools.ToBytes(
  175. wrapper + os.path.join(self.path, self.cross))
  176. else:
  177. env[b'CROSS_COMPILE'] = tools.ToBytes(wrapper + self.cross)
  178. env[b'PATH'] = tools.ToBytes(self.path) + b':' + env[b'PATH']
  179. env[b'LC_ALL'] = b'C'
  180. return env
  181. def MakeArgs(self):
  182. """Create the 'make' arguments for a toolchain
  183. This is only used when the toolchain is being overridden. Since the
  184. U-Boot Makefile sets CC and HOSTCC explicitly we cannot rely on the
  185. environment (and MakeEnvironment()) to override these values. This
  186. function returns the arguments to accomplish this.
  187. Returns:
  188. List of arguments to pass to 'make'
  189. """
  190. if self.override_toolchain:
  191. return ['HOSTCC=%s' % self.override_toolchain,
  192. 'CC=%s' % self.override_toolchain]
  193. return []
  194. class Toolchains:
  195. """Manage a list of toolchains for building U-Boot
  196. We select one toolchain for each architecture type
  197. Public members:
  198. toolchains: Dict of Toolchain objects, keyed by architecture name
  199. prefixes: Dict of prefixes to check, keyed by architecture. This can
  200. be a full path and toolchain prefix, for example
  201. {'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of
  202. something on the search path, for example
  203. {'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported.
  204. paths: List of paths to check for toolchains (may contain wildcards)
  205. """
  206. def __init__(self, override_toolchain=None):
  207. self.toolchains = {}
  208. self.prefixes = {}
  209. self.paths = []
  210. self.override_toolchain = override_toolchain
  211. self._make_flags = dict(bsettings.GetItems('make-flags'))
  212. def GetPathList(self, show_warning=True):
  213. """Get a list of available toolchain paths
  214. Args:
  215. show_warning: True to show a warning if there are no tool chains.
  216. Returns:
  217. List of strings, each a path to a toolchain mentioned in the
  218. [toolchain] section of the settings file.
  219. """
  220. toolchains = bsettings.GetItems('toolchain')
  221. if show_warning and not toolchains:
  222. print(("Warning: No tool chains. Please run 'buildman "
  223. "--fetch-arch all' to download all available toolchains, or "
  224. "add a [toolchain] section to your buildman config file "
  225. "%s. See README for details" %
  226. bsettings.config_fname))
  227. paths = []
  228. for name, value in toolchains:
  229. if '*' in value:
  230. paths += glob.glob(value)
  231. else:
  232. paths.append(value)
  233. return paths
  234. def GetSettings(self, show_warning=True):
  235. """Get toolchain settings from the settings file.
  236. Args:
  237. show_warning: True to show a warning if there are no tool chains.
  238. """
  239. self.prefixes = bsettings.GetItems('toolchain-prefix')
  240. self.paths += self.GetPathList(show_warning)
  241. def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC,
  242. arch=None):
  243. """Add a toolchain to our list
  244. We select the given toolchain as our preferred one for its
  245. architecture if it is a higher priority than the others.
  246. Args:
  247. fname: Filename of toolchain's gcc driver
  248. test: True to run the toolchain to test it
  249. priority: Priority to use for this toolchain
  250. arch: Toolchain architecture, or None if not known
  251. """
  252. toolchain = Toolchain(fname, test, verbose, priority, arch,
  253. self.override_toolchain)
  254. add_it = toolchain.ok
  255. if toolchain.arch in self.toolchains:
  256. add_it = (toolchain.priority <
  257. self.toolchains[toolchain.arch].priority)
  258. if add_it:
  259. self.toolchains[toolchain.arch] = toolchain
  260. elif verbose:
  261. print(("Toolchain '%s' at priority %d will be ignored because "
  262. "another toolchain for arch '%s' has priority %d" %
  263. (toolchain.gcc, toolchain.priority, toolchain.arch,
  264. self.toolchains[toolchain.arch].priority)))
  265. def ScanPath(self, path, verbose):
  266. """Scan a path for a valid toolchain
  267. Args:
  268. path: Path to scan
  269. verbose: True to print out progress information
  270. Returns:
  271. Filename of C compiler if found, else None
  272. """
  273. fnames = []
  274. for subdir in ['.', 'bin', 'usr/bin']:
  275. dirname = os.path.join(path, subdir)
  276. if verbose: print(" - looking in '%s'" % dirname)
  277. for fname in glob.glob(dirname + '/*gcc'):
  278. if verbose: print(" - found '%s'" % fname)
  279. fnames.append(fname)
  280. return fnames
  281. def ScanPathEnv(self, fname):
  282. """Scan the PATH environment variable for a given filename.
  283. Args:
  284. fname: Filename to scan for
  285. Returns:
  286. List of matching pathanames, or [] if none
  287. """
  288. pathname_list = []
  289. for path in os.environ["PATH"].split(os.pathsep):
  290. path = path.strip('"')
  291. pathname = os.path.join(path, fname)
  292. if os.path.exists(pathname):
  293. pathname_list.append(pathname)
  294. return pathname_list
  295. def Scan(self, verbose):
  296. """Scan for available toolchains and select the best for each arch.
  297. We look for all the toolchains we can file, figure out the
  298. architecture for each, and whether it works. Then we select the
  299. highest priority toolchain for each arch.
  300. Args:
  301. verbose: True to print out progress information
  302. """
  303. if verbose: print('Scanning for tool chains')
  304. for name, value in self.prefixes:
  305. if verbose: print(" - scanning prefix '%s'" % value)
  306. if os.path.exists(value):
  307. self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name)
  308. continue
  309. fname = value + 'gcc'
  310. if os.path.exists(fname):
  311. self.Add(fname, True, verbose, PRIORITY_PREFIX_GCC, name)
  312. continue
  313. fname_list = self.ScanPathEnv(fname)
  314. for f in fname_list:
  315. self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name)
  316. if not fname_list:
  317. raise ValueError("No tool chain found for prefix '%s'" %
  318. value)
  319. for path in self.paths:
  320. if verbose: print(" - scanning path '%s'" % path)
  321. fnames = self.ScanPath(path, verbose)
  322. for fname in fnames:
  323. self.Add(fname, True, verbose)
  324. def List(self):
  325. """List out the selected toolchains for each architecture"""
  326. col = terminal.Color()
  327. print(col.Color(col.BLUE, 'List of available toolchains (%d):' %
  328. len(self.toolchains)))
  329. if len(self.toolchains):
  330. for key, value in sorted(self.toolchains.items()):
  331. print('%-10s: %s' % (key, value.gcc))
  332. else:
  333. print('None')
  334. def Select(self, arch):
  335. """Returns the toolchain for a given architecture
  336. Args:
  337. args: Name of architecture (e.g. 'arm', 'ppc_8xx')
  338. returns:
  339. toolchain object, or None if none found
  340. """
  341. for tag, value in bsettings.GetItems('toolchain-alias'):
  342. if arch == tag:
  343. for alias in value.split():
  344. if alias in self.toolchains:
  345. return self.toolchains[alias]
  346. if not arch in self.toolchains:
  347. raise ValueError("No tool chain found for arch '%s'" % arch)
  348. return self.toolchains[arch]
  349. def ResolveReferences(self, var_dict, args):
  350. """Resolve variable references in a string
  351. This converts ${blah} within the string to the value of blah.
  352. This function works recursively.
  353. Args:
  354. var_dict: Dictionary containing variables and their values
  355. args: String containing make arguments
  356. Returns:
  357. Resolved string
  358. >>> bsettings.Setup()
  359. >>> tcs = Toolchains()
  360. >>> tcs.Add('fred', False)
  361. >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \
  362. 'second' : '2nd'}
  363. >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set')
  364. 'this=OBLIQUE_set'
  365. >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd')
  366. 'this=OBLIQUE_setfi2ndrstnd'
  367. """
  368. re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})')
  369. while True:
  370. m = re_var.search(args)
  371. if not m:
  372. break
  373. lookup = m.group(0)[2:-1]
  374. value = var_dict.get(lookup, '')
  375. args = args[:m.start(0)] + value + args[m.end(0):]
  376. return args
  377. def GetMakeArguments(self, board):
  378. """Returns 'make' arguments for a given board
  379. The flags are in a section called 'make-flags'. Flags are named
  380. after the target they represent, for example snapper9260=TESTING=1
  381. will pass TESTING=1 to make when building the snapper9260 board.
  382. References to other boards can be added in the string also. For
  383. example:
  384. [make-flags]
  385. at91-boards=ENABLE_AT91_TEST=1
  386. snapper9260=${at91-boards} BUILD_TAG=442
  387. snapper9g45=${at91-boards} BUILD_TAG=443
  388. This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260
  389. and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45.
  390. A special 'target' variable is set to the board target.
  391. Args:
  392. board: Board object for the board to check.
  393. Returns:
  394. 'make' flags for that board, or '' if none
  395. """
  396. self._make_flags['target'] = board.target
  397. arg_str = self.ResolveReferences(self._make_flags,
  398. self._make_flags.get(board.target, ''))
  399. args = re.findall("(?:\".*?\"|\S)+", arg_str)
  400. i = 0
  401. while i < len(args):
  402. args[i] = args[i].replace('"', '')
  403. if not args[i]:
  404. del args[i]
  405. else:
  406. i += 1
  407. return args
  408. def LocateArchUrl(self, fetch_arch):
  409. """Find a toolchain available online
  410. Look in standard places for available toolchains. At present the
  411. only standard place is at kernel.org.
  412. Args:
  413. arch: Architecture to look for, or 'list' for all
  414. Returns:
  415. If fetch_arch is 'list', a tuple:
  416. Machine architecture (e.g. x86_64)
  417. List of toolchains
  418. else
  419. URL containing this toolchain, if avaialble, else None
  420. """
  421. arch = command.OutputOneLine('uname', '-m')
  422. if arch == 'aarch64':
  423. arch = 'arm64'
  424. base = 'https://www.kernel.org/pub/tools/crosstool/files/bin'
  425. versions = ['9.2.0', '7.3.0', '6.4.0', '4.9.4']
  426. links = []
  427. for version in versions:
  428. url = '%s/%s/%s/' % (base, arch, version)
  429. print('Checking: %s' % url)
  430. response = urllib.request.urlopen(url)
  431. html = tools.ToString(response.read())
  432. parser = MyHTMLParser(fetch_arch)
  433. parser.feed(html)
  434. if fetch_arch == 'list':
  435. links += parser.links
  436. elif parser.arch_link:
  437. return url + parser.arch_link
  438. if fetch_arch == 'list':
  439. return arch, links
  440. return None
  441. def Download(self, url):
  442. """Download a file to a temporary directory
  443. Args:
  444. url: URL to download
  445. Returns:
  446. Tuple:
  447. Temporary directory name
  448. Full path to the downloaded archive file in that directory,
  449. or None if there was an error while downloading
  450. """
  451. print('Downloading: %s' % url)
  452. leaf = url.split('/')[-1]
  453. tmpdir = tempfile.mkdtemp('.buildman')
  454. response = urllib.request.urlopen(url)
  455. fname = os.path.join(tmpdir, leaf)
  456. fd = open(fname, 'wb')
  457. meta = response.info()
  458. size = int(meta.get('Content-Length'))
  459. done = 0
  460. block_size = 1 << 16
  461. status = ''
  462. # Read the file in chunks and show progress as we go
  463. while True:
  464. buffer = response.read(block_size)
  465. if not buffer:
  466. print(chr(8) * (len(status) + 1), '\r', end=' ')
  467. break
  468. done += len(buffer)
  469. fd.write(buffer)
  470. status = r'%10d MiB [%3d%%]' % (done // 1024 // 1024,
  471. done * 100 // size)
  472. status = status + chr(8) * (len(status) + 1)
  473. print(status, end=' ')
  474. sys.stdout.flush()
  475. fd.close()
  476. if done != size:
  477. print('Error, failed to download')
  478. os.remove(fname)
  479. fname = None
  480. return tmpdir, fname
  481. def Unpack(self, fname, dest):
  482. """Unpack a tar file
  483. Args:
  484. fname: Filename to unpack
  485. dest: Destination directory
  486. Returns:
  487. Directory name of the first entry in the archive, without the
  488. trailing /
  489. """
  490. stdout = command.Output('tar', 'xvfJ', fname, '-C', dest)
  491. dirs = stdout.splitlines()[1].split('/')[:2]
  492. return '/'.join(dirs)
  493. def TestSettingsHasPath(self, path):
  494. """Check if buildman will find this toolchain
  495. Returns:
  496. True if the path is in settings, False if not
  497. """
  498. paths = self.GetPathList(False)
  499. return path in paths
  500. def ListArchs(self):
  501. """List architectures with available toolchains to download"""
  502. host_arch, archives = self.LocateArchUrl('list')
  503. re_arch = re.compile('[-a-z0-9.]*[-_]([^-]*)-.*')
  504. arch_set = set()
  505. for archive in archives:
  506. # Remove the host architecture from the start
  507. arch = re_arch.match(archive[len(host_arch):])
  508. if arch:
  509. if arch.group(1) != '2.0' and arch.group(1) != '64':
  510. arch_set.add(arch.group(1))
  511. return sorted(arch_set)
  512. def FetchAndInstall(self, arch):
  513. """Fetch and install a new toolchain
  514. arch:
  515. Architecture to fetch, or 'list' to list
  516. """
  517. # Fist get the URL for this architecture
  518. col = terminal.Color()
  519. print(col.Color(col.BLUE, "Downloading toolchain for arch '%s'" % arch))
  520. url = self.LocateArchUrl(arch)
  521. if not url:
  522. print(("Cannot find toolchain for arch '%s' - use 'list' to list" %
  523. arch))
  524. return 2
  525. home = os.environ['HOME']
  526. dest = os.path.join(home, '.buildman-toolchains')
  527. if not os.path.exists(dest):
  528. os.mkdir(dest)
  529. # Download the tar file for this toolchain and unpack it
  530. tmpdir, tarfile = self.Download(url)
  531. if not tarfile:
  532. return 1
  533. print(col.Color(col.GREEN, 'Unpacking to: %s' % dest), end=' ')
  534. sys.stdout.flush()
  535. path = self.Unpack(tarfile, dest)
  536. os.remove(tarfile)
  537. os.rmdir(tmpdir)
  538. print()
  539. # Check that the toolchain works
  540. print(col.Color(col.GREEN, 'Testing'))
  541. dirpath = os.path.join(dest, path)
  542. compiler_fname_list = self.ScanPath(dirpath, True)
  543. if not compiler_fname_list:
  544. print('Could not locate C compiler - fetch failed.')
  545. return 1
  546. if len(compiler_fname_list) != 1:
  547. print(col.Color(col.RED, 'Warning, ambiguous toolchains: %s' %
  548. ', '.join(compiler_fname_list)))
  549. toolchain = Toolchain(compiler_fname_list[0], True, True)
  550. # Make sure that it will be found by buildman
  551. if not self.TestSettingsHasPath(dirpath):
  552. print(("Adding 'download' to config file '%s'" %
  553. bsettings.config_fname))
  554. bsettings.SetItem('toolchain', 'download', '%s/*/*' % dest)
  555. return 0