123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345 |
- #!/usr/bin/env python3
- # Buildtools and buildtools extended installer helper script
- #
- # Copyright (C) 2017-2020 Intel Corporation
- #
- # SPDX-License-Identifier: GPL-2.0-only
- #
- # NOTE: --with-extended-buildtools is on by default
- #
- # Example usage (extended buildtools from milestone):
- # (1) using --url and --filename
- # $ install-buildtools \
- # --url http://downloads.yoctoproject.org/releases/yocto/milestones/yocto-3.1_M3/buildtools \
- # --filename x86_64-buildtools-extended-nativesdk-standalone-3.0+snapshot-20200315.sh
- # (2) using --base-url, --release, --installer-version and --build-date
- # $ install-buildtools \
- # --base-url http://downloads.yoctoproject.org/releases/yocto \
- # --release yocto-3.1_M3 \
- # --installer-version 3.0+snapshot
- # --build-date 202000315
- #
- # Example usage (standard buildtools from release):
- # (3) using --url and --filename
- # $ install-buildtools --without-extended-buildtools \
- # --url http://downloads.yoctoproject.org/releases/yocto/yocto-3.0.2/buildtools \
- # --filename x86_64-buildtools-nativesdk-standalone-3.0.2.sh
- # (4) using --base-url, --release and --installer-version
- # $ install-buildtools --without-extended-buildtools \
- # --base-url http://downloads.yoctoproject.org/releases/yocto \
- # --release yocto-3.0.2 \
- # --installer-version 3.0.2
- #
- import argparse
- import logging
- import os
- import platform
- import re
- import shutil
- import shlex
- import stat
- import subprocess
- import sys
- import tempfile
- from urllib.parse import quote
- scripts_path = os.path.dirname(os.path.realpath(__file__))
- lib_path = scripts_path + '/lib'
- sys.path = sys.path + [lib_path]
- import scriptutils
- import scriptpath
- PROGNAME = 'install-buildtools'
- logger = scriptutils.logger_create(PROGNAME, stream=sys.stdout)
- DEFAULT_INSTALL_DIR = os.path.join(os.path.split(scripts_path)[0],'buildtools')
- DEFAULT_BASE_URL = 'http://downloads.yoctoproject.org/releases/yocto'
- DEFAULT_RELEASE = 'yocto-3.2_M3'
- DEFAULT_INSTALLER_VERSION = '3.1+snapshot'
- DEFAULT_BUILDDATE = '20200923'
- # Python version sanity check
- if not (sys.version_info.major == 3 and sys.version_info.minor >= 4):
- logger.error("This script requires Python 3.4 or greater")
- logger.error("You have Python %s.%s" %
- (sys.version_info.major, sys.version_info.minor))
- sys.exit(1)
- # The following three functions are copied directly from
- # bitbake/lib/bb/utils.py, in order to allow this script
- # to run on versions of python earlier than what bitbake
- # supports (e.g. less than Python 3.5 for YP 3.1 release)
- def _hasher(method, filename):
- import mmap
- with open(filename, "rb") as f:
- try:
- with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm:
- for chunk in iter(lambda: mm.read(8192), b''):
- method.update(chunk)
- except ValueError:
- # You can't mmap() an empty file so silence this exception
- pass
- return method.hexdigest()
- def md5_file(filename):
- """
- Return the hex string representation of the MD5 checksum of filename.
- """
- import hashlib
- return _hasher(hashlib.md5(), filename)
- def sha256_file(filename):
- """
- Return the hex string representation of the 256-bit SHA checksum of
- filename.
- """
- import hashlib
- return _hasher(hashlib.sha256(), filename)
- def main():
- global DEFAULT_INSTALL_DIR
- global DEFAULT_BASE_URL
- global DEFAULT_RELEASE
- global DEFAULT_INSTALLER_VERSION
- global DEFAULT_BUILDDATE
- filename = ""
- release = ""
- buildtools_url = ""
- install_dir = ""
- arch = platform.machine()
- parser = argparse.ArgumentParser(
- description="Buildtools installation helper",
- add_help=False)
- parser.add_argument('-u', '--url',
- help='URL from where to fetch buildtools SDK installer, not '
- 'including filename (optional)\n'
- 'Requires --filename.',
- action='store')
- parser.add_argument('-f', '--filename',
- help='filename for the buildtools SDK installer to be installed '
- '(optional)\nRequires --url',
- action='store')
- parser.add_argument('-d', '--directory',
- default=DEFAULT_INSTALL_DIR,
- help='directory where buildtools SDK will be installed (optional)',
- action='store')
- parser.add_argument('-r', '--release',
- default=DEFAULT_RELEASE,
- help='Yocto Project release string for SDK which will be '
- 'installed (optional)',
- action='store')
- parser.add_argument('-V', '--installer-version',
- default=DEFAULT_INSTALLER_VERSION,
- help='version string for the SDK to be installed (optional)',
- action='store')
- parser.add_argument('-b', '--base-url',
- default=DEFAULT_BASE_URL,
- help='base URL from which to fetch SDK (optional)', action='store')
- parser.add_argument('-t', '--build-date',
- default=DEFAULT_BUILDDATE,
- help='Build date of pre-release SDK (optional)', action='store')
- group = parser.add_mutually_exclusive_group()
- group.add_argument('--with-extended-buildtools', action='store_true',
- dest='with_extended_buildtools',
- default=True,
- help='enable extended buildtools tarball (on by default)')
- group.add_argument('--without-extended-buildtools', action='store_false',
- dest='with_extended_buildtools',
- help='disable extended buildtools (traditional buildtools tarball)')
- group = parser.add_mutually_exclusive_group()
- group.add_argument('-c', '--check', help='enable checksum validation',
- default=True, action='store_true')
- group.add_argument('-n', '--no-check', help='disable checksum validation',
- dest="check", action='store_false')
- parser.add_argument('-D', '--debug', help='enable debug output',
- action='store_true')
- parser.add_argument('-q', '--quiet', help='print only errors',
- action='store_true')
- parser.add_argument('-h', '--help', action='help',
- default=argparse.SUPPRESS,
- help='show this help message and exit')
- args = parser.parse_args()
- if args.debug:
- logger.setLevel(logging.DEBUG)
- elif args.quiet:
- logger.setLevel(logging.ERROR)
- if args.url and args.filename:
- logger.debug("--url and --filename detected. Ignoring --base-url "
- "--release --installer-version arguments.")
- filename = args.filename
- buildtools_url = "%s/%s" % (args.url, filename)
- else:
- if args.base_url:
- base_url = args.base_url
- else:
- base_url = DEFAULT_BASE_URL
- if args.release:
- # check if this is a pre-release "milestone" SDK
- m = re.search(r"^(?P<distro>[a-zA-Z\-]+)(?P<version>[0-9.]+)(?P<milestone>_M[1-9])$",
- args.release)
- logger.debug("milestone regex: %s" % m)
- if m and m.group('milestone'):
- logger.debug("release[distro]: %s" % m.group('distro'))
- logger.debug("release[version]: %s" % m.group('version'))
- logger.debug("release[milestone]: %s" % m.group('milestone'))
- if not args.build_date:
- logger.error("Milestone installers require --build-date")
- else:
- if args.with_extended_buildtools:
- filename = "%s-buildtools-extended-nativesdk-standalone-%s-%s.sh" % (
- arch, args.installer_version, args.build_date)
- else:
- filename = "%s-buildtools-nativesdk-standalone-%s-%s.sh" % (
- arch, args.installer_version, args.build_date)
- safe_filename = quote(filename)
- buildtools_url = "%s/milestones/%s/buildtools/%s" % (base_url, args.release, safe_filename)
- # regular release SDK
- else:
- if args.with_extended_buildtools:
- filename = "%s-buildtools-extended-nativesdk-standalone-%s.sh" % (arch, args.installer_version)
- else:
- filename = "%s-buildtools-nativesdk-standalone-%s.sh" % (arch, args.installer_version)
- safe_filename = quote(filename)
- buildtools_url = "%s/%s/buildtools/%s" % (base_url, args.release, safe_filename)
- tmpsdk_dir = tempfile.mkdtemp()
- try:
- # Fetch installer
- logger.info("Fetching buildtools installer")
- tmpbuildtools = os.path.join(tmpsdk_dir, filename)
- ret = subprocess.call("wget -q -O %s %s" %
- (tmpbuildtools, buildtools_url), shell=True)
- if ret != 0:
- logger.error("Could not download file from %s" % buildtools_url)
- return ret
- # Verify checksum
- if args.check:
- logger.info("Fetching buildtools installer checksum")
- checksum_type = ""
- for checksum_type in ["md5sum", "sha256sum"]:
- check_url = "{}.{}".format(buildtools_url, checksum_type)
- checksum_filename = "{}.{}".format(filename, checksum_type)
- tmpbuildtools_checksum = os.path.join(tmpsdk_dir, checksum_filename)
- ret = subprocess.call("wget -q -O %s %s" %
- (tmpbuildtools_checksum, check_url), shell=True)
- if ret == 0:
- break
- else:
- if ret != 0:
- logger.error("Could not download file from %s" % check_url)
- return ret
- regex = re.compile(r"^(?P<checksum>[0-9a-f]+)\s+(?P<path>.*/)?(?P<filename>.*)$")
- with open(tmpbuildtools_checksum, 'rb') as f:
- original = f.read()
- m = re.search(regex, original.decode("utf-8"))
- logger.debug("checksum regex match: %s" % m)
- logger.debug("checksum: %s" % m.group('checksum'))
- logger.debug("path: %s" % m.group('path'))
- logger.debug("filename: %s" % m.group('filename'))
- if filename != m.group('filename'):
- logger.error("Filename does not match name in checksum")
- return 1
- checksum = m.group('checksum')
- if checksum_type == "md5sum":
- checksum_value = md5_file(tmpbuildtools)
- else:
- checksum_value = sha256_file(tmpbuildtools)
- if checksum == checksum_value:
- logger.info("Checksum success")
- else:
- logger.error("Checksum %s expected. Actual checksum is %s." %
- (checksum, checksum_value))
- return 1
- # Make installer executable
- logger.info("Making installer executable")
- st = os.stat(tmpbuildtools)
- os.chmod(tmpbuildtools, st.st_mode | stat.S_IEXEC)
- logger.debug(os.stat(tmpbuildtools))
- if args.directory:
- install_dir = args.directory
- ret = subprocess.call("%s -d %s -y" %
- (tmpbuildtools, install_dir), shell=True)
- else:
- install_dir = "/opt/poky/%s" % args.installer_version
- ret = subprocess.call("%s -y" % tmpbuildtools, shell=True)
- if ret != 0:
- logger.error("Could not run buildtools installer")
- return ret
- # Setup the environment
- logger.info("Setting up the environment")
- regex = re.compile(r'^(?P<export>export )?(?P<env_var>[A-Z_]+)=(?P<env_val>.+)$')
- with open("%s/environment-setup-%s-pokysdk-linux" %
- (install_dir, arch), 'rb') as f:
- for line in f:
- match = regex.search(line.decode('utf-8'))
- logger.debug("export regex: %s" % match)
- if match:
- env_var = match.group('env_var')
- logger.debug("env_var: %s" % env_var)
- env_val = match.group('env_val')
- logger.debug("env_val: %s" % env_val)
- os.environ[env_var] = env_val
- # Test installation
- logger.info("Testing installation")
- tool = ""
- m = re.search("extended", tmpbuildtools)
- logger.debug("extended regex: %s" % m)
- if args.with_extended_buildtools and not m:
- logger.info("Ignoring --with-extended-buildtools as filename "
- "does not contain 'extended'")
- if args.with_extended_buildtools and m:
- tool = 'gcc'
- else:
- tool = 'tar'
- logger.debug("install_dir: %s" % install_dir)
- cmd = shlex.split("/usr/bin/which %s" % tool)
- logger.debug("cmd: %s" % cmd)
- logger.debug("tool: %s" % tool)
- proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
- output, errors = proc.communicate()
- logger.debug("proc.args: %s" % proc.args)
- logger.debug("proc.communicate(): output %s" % output)
- logger.debug("proc.communicate(): errors %s" % errors)
- which_tool = output.decode('utf-8')
- logger.debug("which %s: %s" % (tool, which_tool))
- ret = proc.returncode
- if not which_tool.startswith(install_dir):
- logger.error("Something went wrong: %s not found in %s" %
- (tool, install_dir))
- if ret != 0:
- logger.error("Something went wrong: installation failed")
- else:
- logger.info("Installation successful. Remember to source the "
- "environment setup script now and in any new session.")
- return ret
- finally:
- # cleanup tmp directory
- shutil.rmtree(tmpsdk_dir)
- if __name__ == '__main__':
- try:
- ret = main()
- except Exception:
- ret = 1
- import traceback
- traceback.print_exc()
- sys.exit(ret)
|