Browse Source

oe.external: move common functions here

Also provide external-common.bbclass to wrap its inclusion and setup
associated metadata. This will make easier to search sysroots for files
without pulling in the rest of the external toolchain class.

Signed-off-by: Christopher Larson <chris_larson@mentor.com>
Christopher Larson 8 years ago
parent
commit
bfc86fd9c4

+ 30 - 0
classes/external-common.bbclass

@@ -0,0 +1,30 @@
+OE_IMPORTS += "oe.external"
+
+EXTERNAL_TOOLCHAIN_SYSROOT ?= "${@oe.external.run(d, 'gcc', *(TARGET_CC_ARCH.split() + ['-print-sysroot'])).rstrip()}"
+EXTERNAL_TOOLCHAIN_LIBROOT ?= "${@oe.external.run(d, 'gcc', *(TARGET_CC_ARCH.split() + ['-print-file-name=crtbegin.o'])).rstrip().replace('/crtbegin.o', '')}"
+
+EXTERNAL_INSTALL_SOURCE_PATHS = "\
+    ${EXTERNAL_TOOLCHAIN_SYSROOT} \
+    ${EXTERNAL_TOOLCHAIN}/${EXTERNAL_TARGET_SYS} \
+    ${EXTERNAL_TOOLCHAIN_SYSROOT}/.. \
+    ${EXTERNAL_TOOLCHAIN} \
+    ${D} \
+"
+
+# Potential locations within the external toolchain sysroot
+FILES_MIRRORS = "\
+    ${bindir}/|/usr/${baselib}/bin/\n \
+    ${base_libdir}/|/usr/${baselib}/\n \
+    ${libexecdir}/|/usr/libexec/\n \
+    ${libexecdir}/|/usr/${baselib}/${PN}\n \
+    ${mandir}/|/usr/share/man/\n \
+    ${mandir}/|/usr/man/\n \
+    ${mandir}/|/man/\n \
+    ${mandir}/|/share/doc/*-${EXTERNAL_TARGET_SYS}/man/\n \
+    ${prefix}/|${base_prefix}/\n \
+"
+
+def external_run(d, *args):
+    """Convenience wrapper"""
+    oe_import(d)
+    return oe.external.run(d, *args)

+ 9 - 119
classes/external-toolchain.bbclass

@@ -29,30 +29,6 @@ PROVIDES += "${EXTERNAL_PN}"
 LICENSE = "CLOSED"
 LIC_FILES_CHKSUM = "${COMMON_LIC_CHKSUM}"
 
-EXTERNAL_TOOLCHAIN_SYSROOT ?= "${@external_run(d, 'gcc', *(TARGET_CC_ARCH.split() + ['-print-sysroot'])).rstrip()}"
-EXTERNAL_TOOLCHAIN_LIBROOT ?= "${@external_run(d, 'gcc', *(TARGET_CC_ARCH.split() + ['-print-file-name=crtbegin.o'])).rstrip().replace('/crtbegin.o', '')}"
-
-EXTERNAL_INSTALL_SOURCE_PATHS = "\
-    ${EXTERNAL_TOOLCHAIN_SYSROOT} \
-    ${EXTERNAL_TOOLCHAIN}/${EXTERNAL_TARGET_SYS} \
-    ${EXTERNAL_TOOLCHAIN_SYSROOT}/.. \
-    ${EXTERNAL_TOOLCHAIN} \
-    ${D} \
-"
-
-# Potential locations within the external toolchain sysroot
-FILES_MIRRORS = "\
-    ${bindir}/|/usr/${baselib}/bin/\n \
-    ${base_libdir}/|/usr/${baselib}/\n \
-    ${libexecdir}/|/usr/libexec/\n \
-    ${libexecdir}/|/usr/${baselib}/${PN}\n \
-    ${mandir}/|/usr/share/man/\n \
-    ${mandir}/|/usr/man/\n \
-    ${mandir}/|/man/\n \
-    ${mandir}/|/share/doc/*-${EXTERNAL_TARGET_SYS}/man/\n \
-    ${prefix}/|${base_prefix}/\n \
-"
-
 do_configure[noexec] = "1"
 do_compile[noexec] = "1"
 
@@ -88,20 +64,20 @@ python () {
     if not oe.data.typed_value('EXTERNAL_AUTO_PROVIDE', d):
         return
 
-    sysroots, mirrors = get_file_search_metadata(d)
+    sysroots, mirrors = oe.external.get_file_search_metadata(d)
     pattern = d.getVar('EXTERNAL_PROVIDE_PATTERN', True)
     if pattern is None:
-        files = list(gather_pkg_files(d))
+        files = list(oe.external.gather_pkg_files(d))
         files = filter(lambda f: '.debug' not in f, files)
-        expanded = expand_paths(files, mirrors)
-        paths = search_sysroots(expanded, sysroots)
+        expanded = oe.external.expand_paths(files, mirrors)
+        paths = oe.external.search_sysroots(expanded, sysroots)
         if not any(f for p, f in paths):
             raise bb.parse.SkipPackage('No files found in external toolchain sysroot for `{}`'.format(', '.join(files)))
     elif not pattern:
         return
     else:
-        expanded = oe.external_toolchain.expand_paths([pattern], mirrors)
-        paths = oe.external_toolchain.search_sysroots(expanded, sysroots)
+        expanded = oe.external.expand_paths([pattern], mirrors)
+        paths = oe.external.search_sysroots(expanded, sysroots)
         if not any(f for p, f in paths):
             raise bb.parse.SkipPackage('No files found in external toolchain sysroot for `{}`'.format(pattern))
 }
@@ -115,99 +91,13 @@ python do_install () {
 python external_toolchain_do_install () {
     import subprocess
     installdest = d.getVar('D', True)
-    sysroots, mirrors = get_file_search_metadata(d)
-    files = gather_pkg_files(d)
-    copy_from_sysroots(files, sysroots, mirrors, installdest)
+    sysroots, mirrors = oe.external.get_file_search_metadata(d)
+    files = oe.external.gather_pkg_files(d)
+    oe.external.copy_from_sysroots(files, sysroots, mirrors, installdest)
     subprocess.check_call(['chown', '-R', 'root:root', installdest])
 }
 external_toolchain_do_install[vardeps] += "${@' '.join('FILES_%s' % pkg for pkg in '${PACKAGES}'.split())}"
 
-def get_file_search_metadata(d):
-    '''Given the metadata, return the mirrors and sysroots to operate against.'''
-    from collections import defaultdict
-
-    mirrors = []
-    for entry in d.getVar('FILES_MIRRORS', True).replace('\\n', '\n').split('\n'):
-        entry = entry.strip()
-        if not entry:
-            continue
-        pattern, subst = entry.strip().split('|', 1)
-        mirrors.append(('^' + pattern, subst))
-
-    source_paths = [os.path.realpath(p)
-                    for p in d.getVar('EXTERNAL_INSTALL_SOURCE_PATHS', True).split()]
-
-    return source_paths, mirrors
-
-def gather_pkg_files(d):
-    '''Given the metadata, return all the files we want to copy to ${D} for
-    this recipe.'''
-    import itertools
-    files = []
-    for pkg in d.getVar('PACKAGES', True).split():
-        files = itertools.chain(files, (d.getVar('FILES_{}'.format(pkg), True) or '').split())
-    files = itertools.chain(files, d.getVar('EXTERNAL_EXTRA_FILES', True).split())
-    return files
-
-def copy_from_sysroots(pathnames, sysroots, mirrors, installdest):
-    '''Copy the specified files from the specified sysroots, also checking the
-    specified mirror patterns as alternate paths, to the specified destination.'''
-    import subprocess
-
-    expanded_pathnames = expand_paths(pathnames, mirrors)
-    searched_paths = search_sysroots(expanded_pathnames, sysroots)
-    for path, files in searched_paths:
-        if not files:
-            bb.debug(1, 'Failed to find `{}`'.format(path))
-        else:
-            destdir = oe.path.join(installdest, os.path.dirname(path))
-            bb.utils.mkdirhier(destdir)
-            subprocess.check_call(['cp', '-pPR'] + list(files) + [destdir + '/'])
-            bb.note('Copied `{}`  to `{}/`'.format(', '.join(files), destdir))
-
-def expand_paths(pathnames, mirrors):
-    '''Apply search/replace to paths to get alternate search paths.
-
-    Returns a generator with tuples of (pathname, expanded_paths).'''
-    import re
-    for pathname in pathnames:
-        expanded_paths = [pathname]
-
-        for search, replace in mirrors:
-            try:
-                new_pathname = re.sub(search, replace, pathname, count=1)
-            except re.error as exc:
-                bb.warn("Invalid pattern for `%s`" % search)
-                continue
-            if new_pathname != pathname:
-                expanded_paths.append(new_pathname)
-
-        yield pathname, expanded_paths
-
-def search_sysroots(path_entries, sysroots):
-    '''Search the supplied sysroots for the supplied paths, checking supplied
-    alternate paths. Expects entries in the format (pathname, all_paths).
-
-    Returns a generator with tuples of (pathname, found_paths).'''
-    import glob
-    import itertools
-    for path, pathnames in path_entries:
-        for sysroot, pathname in ((s, p) for s in sysroots
-                                         for p in itertools.chain([path], pathnames)):
-            check_path = sysroot + os.sep + pathname
-            found_paths = glob.glob(check_path)
-            if found_paths:
-                yield path, found_paths
-                break
-        else:
-            yield path, None
-
-def find_sysroot_files(paths, d):
-    sysroots, mirrors = get_file_search_metadata(d)
-    expanded = expand_paths(paths, mirrors)
-    search_results = search_sysroots(expanded, sysroots)
-    return [v for k, v in search_results]
-
 # Change do_install's CWD to EXTERNAL_TOOLCHAIN for convenience
 do_install[dirs] = "${D} ${EXTERNAL_TOOLCHAIN}"
 

+ 0 - 24
conf/distro/include/external-run.inc

@@ -1,24 +0,0 @@
-def external_run(d, cmd, *args):
-    import subprocess
-
-    topdir = d.getVar('TMPDIR', True)
-    toolchain_path = d.getVar('EXTERNAL_TOOLCHAIN', True)
-    if toolchain_path:
-        target_prefix = d.getVar('EXTERNAL_TARGET_SYS', True) + '-'
-        path = os.path.join(toolchain_path, 'bin', target_prefix + cmd)
-        args = [path] + list(args)
-
-        try:
-            output = oe.path.check_output(args, cwd=topdir, stderr=subprocess.STDOUT)
-        except oe.path.CalledProcessError as exc:
-            import pipes
-            bb.debug(1, "{0} failed: {1}".format(' '.join(pipes.quote(a) for a in args), exc.output))
-        except OSError as exc:
-            import pipes
-            bb.debug(1, "{0} failed: {1}".format(' '.join(pipes.quote(a) for a in args), str(exc)))
-        else:
-            return output
-
-    return 'UNKNOWN'
-
-external_run[vardepsexclude] += "EXTERNAL_TOOLCHAIN TMPDIR"

+ 4 - 3
conf/distro/include/tcmode-external-sourcery.inc

@@ -44,6 +44,9 @@ PREFERRED_PROVIDER_oprofile ??= "oprofile"
 PREFERRED_PROVIDER_libgcc = "libgcc-external"
 PREFERRED_PROVIDER_linux-libc-headers = "linux-libc-headers-external"
 
+# Pull in our utility functions for use elsewhere
+INHERIT += "external-common"
+
 # Ensure that we don't pull in any internal toolchain recipes
 INHERIT += "blacklist blacklist_dynamic"
 PNBLACKLIST[uclibc] = "not building with an external toolchain"
@@ -189,15 +192,13 @@ python toolchain_sanity_check () {
 toolchain_sanity_check[eventmask] = "bb.event.TreeDataPreparationStarted"
 addhandler toolchain_sanity_check
 
-require conf/distro/include/external-run.inc
-
 GCCVERSION ?= "${@'.'.join('${GCC_VERSION}'.split('.')[:2])}%"
 GCC_VERSION = "${@external_run(d, 'gcc', '-dumpversion').rstrip()}"
 GCC_VERSION_allarch = ""
 GCC_VERSION[vardepvalue] = "${GCC_VERSION}"
 
 def sourcery_version(d):
-    version = external_run(d, 'gcc', '-v').splitlines()[-1]
+    version = oe.external.run(d, 'gcc', '-v').splitlines()[-1]
     if version != 'UNKNOWN':
         main_version = version.split()[-1].rstrip(')')
         if main_version == 'Preview':

+ 2 - 0
lib/oe/__init__.py

@@ -0,0 +1,2 @@
+from pkgutil import extend_path
+__path__ = extend_path(__path__, __name__)

+ 116 - 0
lib/oe/external.py

@@ -0,0 +1,116 @@
+import os.path
+import oe.path
+import bb
+
+
+def run(d, cmd, *args):
+    import subprocess
+
+    topdir = d.getVar('TMPDIR', True)
+    toolchain_path = d.getVar('EXTERNAL_TOOLCHAIN', True)
+    if toolchain_path:
+        target_prefix = d.getVar('EXTERNAL_TARGET_SYS', True) + '-'
+        path = os.path.join(toolchain_path, 'bin', target_prefix + cmd)
+        args = [path] + list(args)
+
+        try:
+            output = oe.path.check_output(args, cwd=topdir, stderr=subprocess.STDOUT)
+        except oe.path.CalledProcessError as exc:
+            import pipes
+            bb.debug(1, "{0} failed: {1}".format(' '.join(pipes.quote(a) for a in args), exc.output))
+        except OSError as exc:
+            import pipes
+            bb.debug(1, "{0} failed: {1}".format(' '.join(pipes.quote(a) for a in args), str(exc)))
+        else:
+            return output
+
+    return 'UNKNOWN'
+
+
+def get_file_search_metadata(d):
+    '''Given the metadata, return the mirrors and sysroots to operate against.'''
+
+    mirrors = []
+    for entry in d.getVar('FILES_MIRRORS', True).replace('\\n', '\n').split('\n'):
+        entry = entry.strip()
+        if not entry:
+            continue
+        pattern, subst = entry.strip().split('|', 1)
+        mirrors.append(('^' + pattern, subst))
+
+    source_paths = [os.path.realpath(p)
+                    for p in d.getVar('EXTERNAL_INSTALL_SOURCE_PATHS', True).split()]
+
+    return source_paths, mirrors
+
+
+def gather_pkg_files(d):
+    '''Given the metadata, return all the files we want to copy to ${D} for
+    this recipe.'''
+    import itertools
+    files = []
+    for pkg in d.getVar('PACKAGES', True).split():
+        files = itertools.chain(files, (d.getVar('FILES_{}'.format(pkg), True) or '').split())
+    files = itertools.chain(files, d.getVar('EXTERNAL_EXTRA_FILES', True).split())
+    return files
+
+
+def copy_from_sysroots(pathnames, sysroots, mirrors, installdest):
+    '''Copy the specified files from the specified sysroots, also checking the
+    specified mirror patterns as alternate paths, to the specified destination.'''
+    import subprocess
+
+    expanded_pathnames = expand_paths(pathnames, mirrors)
+    searched_paths = search_sysroots(expanded_pathnames, sysroots)
+    for path, files in searched_paths:
+        if not files:
+            bb.debug(1, 'Failed to find `{}`'.format(path))
+        else:
+            destdir = oe.path.join(installdest, os.path.dirname(path))
+            bb.utils.mkdirhier(destdir)
+            subprocess.check_call(['cp', '-pPR'] + list(files) + [destdir + '/'])
+            bb.note('Copied `{}`  to `{}/`'.format(', '.join(files), destdir))
+
+def expand_paths(pathnames, mirrors):
+    '''Apply search/replace to paths to get alternate search paths.
+
+    Returns a generator with tuples of (pathname, expanded_paths).'''
+    import re
+    for pathname in pathnames:
+        expanded_paths = [pathname]
+
+        for search, replace in mirrors:
+            try:
+                new_pathname = re.sub(search, replace, pathname, count=1)
+            except re.error as exc:
+                bb.warn("Invalid pattern for `%s`" % search)
+                continue
+            if new_pathname != pathname:
+                expanded_paths.append(new_pathname)
+
+        yield pathname, expanded_paths
+
+def search_sysroots(path_entries, sysroots):
+    '''Search the supplied sysroots for the supplied paths, checking supplied
+    alternate paths. Expects entries in the format (pathname, all_paths).
+
+    Returns a generator with tuples of (pathname, found_paths).'''
+    import glob
+    import itertools
+    for path, pathnames in path_entries:
+        for sysroot, pathname in ((s, p) for s in sysroots
+                                         for p in itertools.chain([path], pathnames)):
+            check_path = sysroot + os.sep + pathname
+            found_paths = glob.glob(check_path)
+            if found_paths:
+                yield path, found_paths
+                break
+        else:
+            yield path, None
+
+
+def find_sysroot_files(paths, d):
+    sysroots, mirrors = get_file_search_metadata(d)
+    expanded = expand_paths(paths, mirrors)
+    search_results = list(search_sysroots(expanded, sysroots))
+    return [v for k, v in search_results]

+ 1 - 1
recipes-external/binutils/binutils-external-cross.bb

@@ -5,7 +5,7 @@ HOMEPAGE = "http://www.gnu.org/software/binutils/"
 BUGTRACKER = "http://sourceware.org/bugzilla/"
 SECTION = "devel"
 PN .= "-${TARGET_ARCH}"
-PV := "${@external_run(d, 'ld', '-v').splitlines()[0].split()[-1].rstrip()}"
+PV := "${@oe.external.run(d, 'ld', '-v').splitlines()[0].split()[-1].rstrip()}"
 LICENSE = "${@'GPLv3' if '${PV}'.split('.') > '2.17.50.0.12'.split('.') else 'GPLv2'}"
 
 PROVIDES += "\

+ 1 - 1
recipes-external/gcc/gcc-external.inc

@@ -1,7 +1,7 @@
 SUMMARY = "The GNU Compiler Collection"
 HOMEPAGE = "http://www.gnu.org/software/gcc/"
 SECTION = "devel"
-GCC_VERSION := "${@external_run(d, 'gcc', '-dumpversion').rstrip()}"
+GCC_VERSION := "${@oe.external.run(d, 'gcc', '-dumpversion').rstrip()}"
 PV = "${GCC_VERSION}"
 
 inherit external-toolchain

+ 2 - 2
recipes-external/gcc/gcc-runtime-external.bb

@@ -1,4 +1,4 @@
-GCC_VERSION := "${@external_run(d, 'gcc', '-dumpversion').rstrip()}"
+GCC_VERSION := "${@oe.external.run(d, 'gcc', '-dumpversion').rstrip()}"
 PV = "${GCC_VERSION}"
 BINV = "${GCC_VERSION}"
 
@@ -20,7 +20,7 @@ python () {
 }
 
 target_libdir = "${libdir}"
-HEADERS_MULTILIB_SUFFIX ?= "${@external_run(d, 'gcc', *(TARGET_CC_ARCH.split() + ['-print-sysroot-headers-suffix'])).rstrip()}"
+HEADERS_MULTILIB_SUFFIX ?= "${@oe.external.run(d, 'gcc', *(TARGET_CC_ARCH.split() + ['-print-sysroot-headers-suffix'])).rstrip()}"
 external_libroot = "${@os.path.realpath('${EXTERNAL_TOOLCHAIN_LIBROOT}').replace(os.path.realpath('${EXTERNAL_TOOLCHAIN}') + '/', '/')}"
 FILES_MIRRORS =. "\
     ${libdir}/gcc/${TARGET_SYS}/${GCC_VERSION}/|${external_libroot}/\n \

+ 1 - 1
recipes-external/gcc/gcc-sanitizers-external.bb

@@ -1,4 +1,4 @@
-GCC_VERSION := "${@external_run(d, 'gcc', '-dumpversion').rstrip()}"
+GCC_VERSION := "${@oe.external.run(d, 'gcc', '-dumpversion').rstrip()}"
 PV = "${GCC_VERSION}"
 BINV = "${GCC_VERSION}"
 

+ 1 - 1
recipes-external/gcc/libgcc-external.bb

@@ -1,7 +1,7 @@
 SUMMARY = "The GNU Compiler Collection - libgcc"
 HOMEPAGE = "http://www.gnu.org/software/gcc/"
 SECTION = "devel"
-GCC_VERSION := "${@external_run(d, 'gcc', '-dumpversion').rstrip()}"
+GCC_VERSION := "${@oe.external.run(d, 'gcc', '-dumpversion').rstrip()}"
 PV = "${GCC_VERSION}"
 
 inherit external-toolchain

+ 2 - 2
recipes-external/gdb/gdbserver-external.bb

@@ -1,12 +1,12 @@
 SUMMARY = "gdb - GNU debugger"
 HOMEPAGE = "http://www.gnu.org/software/gdb/"
 SECTION = "devel"
-PV := "${@external_run(d, 'gdb', '-v').splitlines()[0].split()[-1]}"
+PV := "${@oe.external.run(d, 'gdb', '-v').splitlines()[0].split()[-1]}"
 
 inherit external-toolchain
 
 def get_gdb_license(d):
-    output = external_run(d, 'gdb', '-v')
+    output = oe.external.run(d, 'gdb', '-v')
     if output != 'UNKNOWN':
         for line in output.splitlines():
             if line.startswith('License '):

+ 1 - 1
recipes-external/glibc/glibc-external.bb

@@ -27,7 +27,7 @@ PROVIDES += "glibc \
 
 def get_external_libc_license(d):
     errnosearch = os.path.join(d.getVar('includedir', True), 'errno.h')
-    found = find_sysroot_files([errnosearch], d)
+    found = oe.external.find_sysroot_files([errnosearch], d)
     if found:
         errno_paths = found[0]
         if errno_paths:

+ 1 - 1
recipes-external/glibc/glibc-sourcery.bb

@@ -1,7 +1,7 @@
 require recipes-core/glibc/glibc.inc
 
 
-EXTERNAL_TOOLCHAIN_SYSROOT ?= "${@external_run(d, 'gcc', *(TARGET_CC_ARCH.split() + ['-print-sysroot'])).rstrip()}"
+EXTERNAL_TOOLCHAIN_SYSROOT ?= "${@oe.external.run(d, 'gcc', *(TARGET_CC_ARCH.split() + ['-print-sysroot'])).rstrip()}"
 
 LICENSE = "CLOSED"
 LIC_FILES_CHKSUM = ""

+ 1 - 1
recipes-external/glibc/glibc-sysroot-setup.inc

@@ -2,7 +2,7 @@ def sysroot_multilib_suffix(d):
     PATH = d.getVar('PATH', True)
     target_cc_arch = d.getVar('TARGET_CC_ARCH', True)
     options = d.getVar('TOOLCHAIN_OPTIONS', True) or d.expand('--sysroot=${STAGING_DIR_TARGET}')
-    sysroot = external_run(d, 'gcc', *(target_cc_arch.split() + options.split() + ['-print-sysroot'])).rstrip()
+    sysroot = oe.external.run(d, 'gcc', *(target_cc_arch.split() + options.split() + ['-print-sysroot'])).rstrip()
 
     staging = d.getVar('STAGING_DIR_HOST', True)
     if sysroot == staging: