|
@@ -5,11 +5,42 @@
|
|
|
|
|
|
import re
|
|
|
import glob
|
|
|
+from HTMLParser import HTMLParser
|
|
|
import os
|
|
|
+import sys
|
|
|
+import tempfile
|
|
|
+import urllib2
|
|
|
|
|
|
import bsettings
|
|
|
import command
|
|
|
|
|
|
+# Simple class to collect links from a page
|
|
|
+class MyHTMLParser(HTMLParser):
|
|
|
+ def __init__(self, arch):
|
|
|
+ """Create a new parser
|
|
|
+
|
|
|
+ After the parser runs, self.links will be set to a list of the links
|
|
|
+ to .xz archives found in the page, and self.arch_link will be set to
|
|
|
+ the one for the given architecture (or None if not found).
|
|
|
+
|
|
|
+ Args:
|
|
|
+ arch: Architecture to search for
|
|
|
+ """
|
|
|
+ HTMLParser.__init__(self)
|
|
|
+ self.arch_link = None
|
|
|
+ self.links = []
|
|
|
+ self._match = '_%s-' % arch
|
|
|
+
|
|
|
+ def handle_starttag(self, tag, attrs):
|
|
|
+ if tag == 'a':
|
|
|
+ for tag, value in attrs:
|
|
|
+ if tag == 'href':
|
|
|
+ if value and value.endswith('.xz'):
|
|
|
+ self.links.append(value)
|
|
|
+ if self._match in value:
|
|
|
+ self.arch_link = value
|
|
|
+
|
|
|
+
|
|
|
class Toolchain:
|
|
|
"""A single toolchain
|
|
|
|
|
@@ -20,7 +51,6 @@ class Toolchain:
|
|
|
arch: Architecture of toolchain as determined from the first
|
|
|
component of the filename. E.g. arm-linux-gcc becomes arm
|
|
|
"""
|
|
|
-
|
|
|
def __init__(self, fname, test, verbose=False):
|
|
|
"""Create a new toolchain object.
|
|
|
|
|
@@ -116,18 +146,29 @@ class Toolchains:
|
|
|
self.paths = []
|
|
|
self._make_flags = dict(bsettings.GetItems('make-flags'))
|
|
|
|
|
|
- def GetSettings(self):
|
|
|
+ def GetPathList(self):
|
|
|
+ """Get a list of available toolchain paths
|
|
|
+
|
|
|
+ Returns:
|
|
|
+ List of strings, each a path to a toolchain mentioned in the
|
|
|
+ [toolchain] section of the settings file.
|
|
|
+ """
|
|
|
toolchains = bsettings.GetItems('toolchain')
|
|
|
if not toolchains:
|
|
|
print ("Warning: No tool chains - please add a [toolchain] section"
|
|
|
" to your buildman config file %s. See README for details" %
|
|
|
bsettings.config_fname)
|
|
|
|
|
|
+ paths = []
|
|
|
for name, value in toolchains:
|
|
|
if '*' in value:
|
|
|
- self.paths += glob.glob(value)
|
|
|
+ paths += glob.glob(value)
|
|
|
else:
|
|
|
- self.paths.append(value)
|
|
|
+ paths.append(value)
|
|
|
+ return paths
|
|
|
+
|
|
|
+ def GetSettings(self):
|
|
|
+ self.paths += self.GetPathList()
|
|
|
|
|
|
def Add(self, fname, test=True, verbose=False):
|
|
|
"""Add a toolchain to our list
|
|
@@ -147,6 +188,24 @@ class Toolchains:
|
|
|
if add_it:
|
|
|
self.toolchains[toolchain.arch] = toolchain
|
|
|
|
|
|
+ def ScanPath(self, path, verbose):
|
|
|
+ """Scan a path for a valid toolchain
|
|
|
+
|
|
|
+ Args:
|
|
|
+ path: Path to scan
|
|
|
+ verbose: True to print out progress information
|
|
|
+ Returns:
|
|
|
+ Filename of C compiler if found, else None
|
|
|
+ """
|
|
|
+ for subdir in ['.', 'bin', 'usr/bin']:
|
|
|
+ dirname = os.path.join(path, subdir)
|
|
|
+ if verbose: print " - looking in '%s'" % dirname
|
|
|
+ for fname in glob.glob(dirname + '/*gcc'):
|
|
|
+ if verbose: print " - found '%s'" % fname
|
|
|
+ return fname
|
|
|
+ return None
|
|
|
+
|
|
|
+
|
|
|
def Scan(self, verbose):
|
|
|
"""Scan for available toolchains and select the best for each arch.
|
|
|
|
|
@@ -160,12 +219,9 @@ class Toolchains:
|
|
|
if verbose: print 'Scanning for tool chains'
|
|
|
for path in self.paths:
|
|
|
if verbose: print " - scanning path '%s'" % path
|
|
|
- for subdir in ['.', 'bin', 'usr/bin']:
|
|
|
- dirname = os.path.join(path, subdir)
|
|
|
- if verbose: print " - looking in '%s'" % dirname
|
|
|
- for fname in glob.glob(dirname + '/*gcc'):
|
|
|
- if verbose: print " - found '%s'" % fname
|
|
|
- self.Add(fname, True, verbose)
|
|
|
+ fname = self.ScanPath(path, verbose)
|
|
|
+ if fname:
|
|
|
+ self.Add(fname, True, verbose)
|
|
|
|
|
|
def List(self):
|
|
|
"""List out the selected toolchains for each architecture"""
|
|
@@ -264,3 +320,160 @@ class Toolchains:
|
|
|
else:
|
|
|
i += 1
|
|
|
return args
|
|
|
+
|
|
|
+ def LocateArchUrl(self, fetch_arch):
|
|
|
+ """Find a toolchain available online
|
|
|
+
|
|
|
+ Look in standard places for available toolchains. At present the
|
|
|
+ only standard place is at kernel.org.
|
|
|
+
|
|
|
+ Args:
|
|
|
+ arch: Architecture to look for, or 'list' for all
|
|
|
+ Returns:
|
|
|
+ If fetch_arch is 'list', a tuple:
|
|
|
+ Machine architecture (e.g. x86_64)
|
|
|
+ List of toolchains
|
|
|
+ else
|
|
|
+ URL containing this toolchain, if avaialble, else None
|
|
|
+ """
|
|
|
+ arch = command.OutputOneLine('uname', '-m')
|
|
|
+ base = 'https://www.kernel.org/pub/tools/crosstool/files/bin'
|
|
|
+ versions = ['4.6.3', '4.6.2', '4.5.1', '4.2.4']
|
|
|
+ links = []
|
|
|
+ for version in versions:
|
|
|
+ url = '%s/%s/%s/' % (base, arch, version)
|
|
|
+ print 'Checking: %s' % url
|
|
|
+ response = urllib2.urlopen(url)
|
|
|
+ html = response.read()
|
|
|
+ parser = MyHTMLParser(fetch_arch)
|
|
|
+ parser.feed(html)
|
|
|
+ if fetch_arch == 'list':
|
|
|
+ links += parser.links
|
|
|
+ elif parser.arch_link:
|
|
|
+ return url + parser.arch_link
|
|
|
+ if fetch_arch == 'list':
|
|
|
+ return arch, links
|
|
|
+ return None
|
|
|
+
|
|
|
+ def Download(self, url):
|
|
|
+ """Download a file to a temporary directory
|
|
|
+
|
|
|
+ Args:
|
|
|
+ url: URL to download
|
|
|
+ Returns:
|
|
|
+ Tuple:
|
|
|
+ Temporary directory name
|
|
|
+ Full path to the downloaded archive file in that directory,
|
|
|
+ or None if there was an error while downloading
|
|
|
+ """
|
|
|
+ print "Downloading: %s" % url
|
|
|
+ leaf = url.split('/')[-1]
|
|
|
+ tmpdir = tempfile.mkdtemp('.buildman')
|
|
|
+ response = urllib2.urlopen(url)
|
|
|
+ fname = os.path.join(tmpdir, leaf)
|
|
|
+ fd = open(fname, 'wb')
|
|
|
+ meta = response.info()
|
|
|
+ size = int(meta.getheaders("Content-Length")[0])
|
|
|
+ done = 0
|
|
|
+ block_size = 1 << 16
|
|
|
+ status = ''
|
|
|
+
|
|
|
+ # Read the file in chunks and show progress as we go
|
|
|
+ while True:
|
|
|
+ buffer = response.read(block_size)
|
|
|
+ if not buffer:
|
|
|
+ print chr(8) * (len(status) + 1), '\r',
|
|
|
+ break
|
|
|
+
|
|
|
+ done += len(buffer)
|
|
|
+ fd.write(buffer)
|
|
|
+ status = r"%10d MiB [%3d%%]" % (done / 1024 / 1024,
|
|
|
+ done * 100 / size)
|
|
|
+ status = status + chr(8) * (len(status) + 1)
|
|
|
+ print status,
|
|
|
+ sys.stdout.flush()
|
|
|
+ fd.close()
|
|
|
+ if done != size:
|
|
|
+ print 'Error, failed to download'
|
|
|
+ os.remove(fname)
|
|
|
+ fname = None
|
|
|
+ return tmpdir, fname
|
|
|
+
|
|
|
+ def Unpack(self, fname, dest):
|
|
|
+ """Unpack a tar file
|
|
|
+
|
|
|
+ Args:
|
|
|
+ fname: Filename to unpack
|
|
|
+ dest: Destination directory
|
|
|
+ Returns:
|
|
|
+ Directory name of the first entry in the archive, without the
|
|
|
+ trailing /
|
|
|
+ """
|
|
|
+ stdout = command.Output('tar', 'xvfJ', fname, '-C', dest)
|
|
|
+ return stdout.splitlines()[0][:-1]
|
|
|
+
|
|
|
+ def TestSettingsHasPath(self, path):
|
|
|
+ """Check if builmand will find this toolchain
|
|
|
+
|
|
|
+ Returns:
|
|
|
+ True if the path is in settings, False if not
|
|
|
+ """
|
|
|
+ paths = self.GetPathList()
|
|
|
+ return path in paths
|
|
|
+
|
|
|
+ def ListArchs(self):
|
|
|
+ """List architectures with available toolchains to download"""
|
|
|
+ host_arch, archives = self.LocateArchUrl('list')
|
|
|
+ re_arch = re.compile('[-a-z0-9.]*_([^-]*)-.*')
|
|
|
+ arch_set = set()
|
|
|
+ for archive in archives:
|
|
|
+ # Remove the host architecture from the start
|
|
|
+ arch = re_arch.match(archive[len(host_arch):])
|
|
|
+ if arch:
|
|
|
+ arch_set.add(arch.group(1))
|
|
|
+ return sorted(arch_set)
|
|
|
+
|
|
|
+ def FetchAndInstall(self, arch):
|
|
|
+ """Fetch and install a new toolchain
|
|
|
+
|
|
|
+ arch:
|
|
|
+ Architecture to fetch, or 'list' to list
|
|
|
+ """
|
|
|
+ # Fist get the URL for this architecture
|
|
|
+ url = self.LocateArchUrl(arch)
|
|
|
+ if not url:
|
|
|
+ print ("Cannot find toolchain for arch '%s' - use 'list' to list" %
|
|
|
+ arch)
|
|
|
+ return 2
|
|
|
+ home = os.environ['HOME']
|
|
|
+ dest = os.path.join(home, '.buildman-toolchains')
|
|
|
+ if not os.path.exists(dest):
|
|
|
+ os.mkdir(dest)
|
|
|
+
|
|
|
+ # Download the tar file for this toolchain and unpack it
|
|
|
+ tmpdir, tarfile = self.Download(url)
|
|
|
+ if not tarfile:
|
|
|
+ return 1
|
|
|
+ print 'Unpacking to: %s' % dest,
|
|
|
+ sys.stdout.flush()
|
|
|
+ path = self.Unpack(tarfile, dest)
|
|
|
+ os.remove(tarfile)
|
|
|
+ os.rmdir(tmpdir)
|
|
|
+ print
|
|
|
+
|
|
|
+ # Check that the toolchain works
|
|
|
+ print 'Testing'
|
|
|
+ dirpath = os.path.join(dest, path)
|
|
|
+ compiler_fname = self.ScanPath(dirpath, True)
|
|
|
+ if not compiler_fname:
|
|
|
+ print 'Could not locate C compiler - fetch failed.'
|
|
|
+ return 1
|
|
|
+ toolchain = Toolchain(compiler_fname, True, True)
|
|
|
+
|
|
|
+ # Make sure that it will be found by buildman
|
|
|
+ if not self.TestSettingsHasPath(dirpath):
|
|
|
+ print ("Adding 'download' to config file '%s'" %
|
|
|
+ bsettings.config_fname)
|
|
|
+ tools_dir = os.path.dirname(dirpath)
|
|
|
+ bsettings.SetItem('toolchain', 'download', '%s/*' % tools_dir)
|
|
|
+ return 0
|