install-buildtools 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345
  1. #!/usr/bin/env python3
  2. # Buildtools and buildtools extended installer helper script
  3. #
  4. # Copyright (C) 2017-2020 Intel Corporation
  5. #
  6. # SPDX-License-Identifier: GPL-2.0-only
  7. #
  8. # NOTE: --with-extended-buildtools is on by default
  9. #
  10. # Example usage (extended buildtools from milestone):
  11. # (1) using --url and --filename
  12. # $ install-buildtools \
  13. # --url http://downloads.yoctoproject.org/releases/yocto/milestones/yocto-3.1_M3/buildtools \
  14. # --filename x86_64-buildtools-extended-nativesdk-standalone-3.0+snapshot-20200315.sh
  15. # (2) using --base-url, --release, --installer-version and --build-date
  16. # $ install-buildtools \
  17. # --base-url http://downloads.yoctoproject.org/releases/yocto \
  18. # --release yocto-3.1_M3 \
  19. # --installer-version 3.0+snapshot
  20. # --build-date 202000315
  21. #
  22. # Example usage (standard buildtools from release):
  23. # (3) using --url and --filename
  24. # $ install-buildtools --without-extended-buildtools \
  25. # --url http://downloads.yoctoproject.org/releases/yocto/yocto-3.0.2/buildtools \
  26. # --filename x86_64-buildtools-nativesdk-standalone-3.0.2.sh
  27. # (4) using --base-url, --release and --installer-version
  28. # $ install-buildtools --without-extended-buildtools \
  29. # --base-url http://downloads.yoctoproject.org/releases/yocto \
  30. # --release yocto-3.0.2 \
  31. # --installer-version 3.0.2
  32. #
  33. import argparse
  34. import logging
  35. import os
  36. import platform
  37. import re
  38. import shutil
  39. import shlex
  40. import stat
  41. import subprocess
  42. import sys
  43. import tempfile
  44. from urllib.parse import quote
  45. scripts_path = os.path.dirname(os.path.realpath(__file__))
  46. lib_path = scripts_path + '/lib'
  47. sys.path = sys.path + [lib_path]
  48. import scriptutils
  49. import scriptpath
  50. PROGNAME = 'install-buildtools'
  51. logger = scriptutils.logger_create(PROGNAME, stream=sys.stdout)
  52. DEFAULT_INSTALL_DIR = os.path.join(os.path.split(scripts_path)[0],'buildtools')
  53. DEFAULT_BASE_URL = 'http://downloads.yoctoproject.org/releases/yocto'
  54. DEFAULT_RELEASE = 'yocto-3.2_M3'
  55. DEFAULT_INSTALLER_VERSION = '3.1+snapshot'
  56. DEFAULT_BUILDDATE = '20200923'
  57. # Python version sanity check
  58. if not (sys.version_info.major == 3 and sys.version_info.minor >= 4):
  59. logger.error("This script requires Python 3.4 or greater")
  60. logger.error("You have Python %s.%s" %
  61. (sys.version_info.major, sys.version_info.minor))
  62. sys.exit(1)
  63. # The following three functions are copied directly from
  64. # bitbake/lib/bb/utils.py, in order to allow this script
  65. # to run on versions of python earlier than what bitbake
  66. # supports (e.g. less than Python 3.5 for YP 3.1 release)
  67. def _hasher(method, filename):
  68. import mmap
  69. with open(filename, "rb") as f:
  70. try:
  71. with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm:
  72. for chunk in iter(lambda: mm.read(8192), b''):
  73. method.update(chunk)
  74. except ValueError:
  75. # You can't mmap() an empty file so silence this exception
  76. pass
  77. return method.hexdigest()
  78. def md5_file(filename):
  79. """
  80. Return the hex string representation of the MD5 checksum of filename.
  81. """
  82. import hashlib
  83. return _hasher(hashlib.md5(), filename)
  84. def sha256_file(filename):
  85. """
  86. Return the hex string representation of the 256-bit SHA checksum of
  87. filename.
  88. """
  89. import hashlib
  90. return _hasher(hashlib.sha256(), filename)
  91. def main():
  92. global DEFAULT_INSTALL_DIR
  93. global DEFAULT_BASE_URL
  94. global DEFAULT_RELEASE
  95. global DEFAULT_INSTALLER_VERSION
  96. global DEFAULT_BUILDDATE
  97. filename = ""
  98. release = ""
  99. buildtools_url = ""
  100. install_dir = ""
  101. arch = platform.machine()
  102. parser = argparse.ArgumentParser(
  103. description="Buildtools installation helper",
  104. add_help=False)
  105. parser.add_argument('-u', '--url',
  106. help='URL from where to fetch buildtools SDK installer, not '
  107. 'including filename (optional)\n'
  108. 'Requires --filename.',
  109. action='store')
  110. parser.add_argument('-f', '--filename',
  111. help='filename for the buildtools SDK installer to be installed '
  112. '(optional)\nRequires --url',
  113. action='store')
  114. parser.add_argument('-d', '--directory',
  115. default=DEFAULT_INSTALL_DIR,
  116. help='directory where buildtools SDK will be installed (optional)',
  117. action='store')
  118. parser.add_argument('-r', '--release',
  119. default=DEFAULT_RELEASE,
  120. help='Yocto Project release string for SDK which will be '
  121. 'installed (optional)',
  122. action='store')
  123. parser.add_argument('-V', '--installer-version',
  124. default=DEFAULT_INSTALLER_VERSION,
  125. help='version string for the SDK to be installed (optional)',
  126. action='store')
  127. parser.add_argument('-b', '--base-url',
  128. default=DEFAULT_BASE_URL,
  129. help='base URL from which to fetch SDK (optional)', action='store')
  130. parser.add_argument('-t', '--build-date',
  131. default=DEFAULT_BUILDDATE,
  132. help='Build date of pre-release SDK (optional)', action='store')
  133. group = parser.add_mutually_exclusive_group()
  134. group.add_argument('--with-extended-buildtools', action='store_true',
  135. dest='with_extended_buildtools',
  136. default=True,
  137. help='enable extended buildtools tarball (on by default)')
  138. group.add_argument('--without-extended-buildtools', action='store_false',
  139. dest='with_extended_buildtools',
  140. help='disable extended buildtools (traditional buildtools tarball)')
  141. group = parser.add_mutually_exclusive_group()
  142. group.add_argument('-c', '--check', help='enable checksum validation',
  143. default=True, action='store_true')
  144. group.add_argument('-n', '--no-check', help='disable checksum validation',
  145. dest="check", action='store_false')
  146. parser.add_argument('-D', '--debug', help='enable debug output',
  147. action='store_true')
  148. parser.add_argument('-q', '--quiet', help='print only errors',
  149. action='store_true')
  150. parser.add_argument('-h', '--help', action='help',
  151. default=argparse.SUPPRESS,
  152. help='show this help message and exit')
  153. args = parser.parse_args()
  154. if args.debug:
  155. logger.setLevel(logging.DEBUG)
  156. elif args.quiet:
  157. logger.setLevel(logging.ERROR)
  158. if args.url and args.filename:
  159. logger.debug("--url and --filename detected. Ignoring --base-url "
  160. "--release --installer-version arguments.")
  161. filename = args.filename
  162. buildtools_url = "%s/%s" % (args.url, filename)
  163. else:
  164. if args.base_url:
  165. base_url = args.base_url
  166. else:
  167. base_url = DEFAULT_BASE_URL
  168. if args.release:
  169. # check if this is a pre-release "milestone" SDK
  170. m = re.search(r"^(?P<distro>[a-zA-Z\-]+)(?P<version>[0-9.]+)(?P<milestone>_M[1-9])$",
  171. args.release)
  172. logger.debug("milestone regex: %s" % m)
  173. if m and m.group('milestone'):
  174. logger.debug("release[distro]: %s" % m.group('distro'))
  175. logger.debug("release[version]: %s" % m.group('version'))
  176. logger.debug("release[milestone]: %s" % m.group('milestone'))
  177. if not args.build_date:
  178. logger.error("Milestone installers require --build-date")
  179. else:
  180. if args.with_extended_buildtools:
  181. filename = "%s-buildtools-extended-nativesdk-standalone-%s-%s.sh" % (
  182. arch, args.installer_version, args.build_date)
  183. else:
  184. filename = "%s-buildtools-nativesdk-standalone-%s-%s.sh" % (
  185. arch, args.installer_version, args.build_date)
  186. safe_filename = quote(filename)
  187. buildtools_url = "%s/milestones/%s/buildtools/%s" % (base_url, args.release, safe_filename)
  188. # regular release SDK
  189. else:
  190. if args.with_extended_buildtools:
  191. filename = "%s-buildtools-extended-nativesdk-standalone-%s.sh" % (arch, args.installer_version)
  192. else:
  193. filename = "%s-buildtools-nativesdk-standalone-%s.sh" % (arch, args.installer_version)
  194. safe_filename = quote(filename)
  195. buildtools_url = "%s/%s/buildtools/%s" % (base_url, args.release, safe_filename)
  196. tmpsdk_dir = tempfile.mkdtemp()
  197. try:
  198. # Fetch installer
  199. logger.info("Fetching buildtools installer")
  200. tmpbuildtools = os.path.join(tmpsdk_dir, filename)
  201. ret = subprocess.call("wget -q -O %s %s" %
  202. (tmpbuildtools, buildtools_url), shell=True)
  203. if ret != 0:
  204. logger.error("Could not download file from %s" % buildtools_url)
  205. return ret
  206. # Verify checksum
  207. if args.check:
  208. logger.info("Fetching buildtools installer checksum")
  209. checksum_type = ""
  210. for checksum_type in ["md5sum", "sha256sum"]:
  211. check_url = "{}.{}".format(buildtools_url, checksum_type)
  212. checksum_filename = "{}.{}".format(filename, checksum_type)
  213. tmpbuildtools_checksum = os.path.join(tmpsdk_dir, checksum_filename)
  214. ret = subprocess.call("wget -q -O %s %s" %
  215. (tmpbuildtools_checksum, check_url), shell=True)
  216. if ret == 0:
  217. break
  218. else:
  219. if ret != 0:
  220. logger.error("Could not download file from %s" % check_url)
  221. return ret
  222. regex = re.compile(r"^(?P<checksum>[0-9a-f]+)\s+(?P<path>.*/)?(?P<filename>.*)$")
  223. with open(tmpbuildtools_checksum, 'rb') as f:
  224. original = f.read()
  225. m = re.search(regex, original.decode("utf-8"))
  226. logger.debug("checksum regex match: %s" % m)
  227. logger.debug("checksum: %s" % m.group('checksum'))
  228. logger.debug("path: %s" % m.group('path'))
  229. logger.debug("filename: %s" % m.group('filename'))
  230. if filename != m.group('filename'):
  231. logger.error("Filename does not match name in checksum")
  232. return 1
  233. checksum = m.group('checksum')
  234. if checksum_type == "md5sum":
  235. checksum_value = md5_file(tmpbuildtools)
  236. else:
  237. checksum_value = sha256_file(tmpbuildtools)
  238. if checksum == checksum_value:
  239. logger.info("Checksum success")
  240. else:
  241. logger.error("Checksum %s expected. Actual checksum is %s." %
  242. (checksum, checksum_value))
  243. return 1
  244. # Make installer executable
  245. logger.info("Making installer executable")
  246. st = os.stat(tmpbuildtools)
  247. os.chmod(tmpbuildtools, st.st_mode | stat.S_IEXEC)
  248. logger.debug(os.stat(tmpbuildtools))
  249. if args.directory:
  250. install_dir = args.directory
  251. ret = subprocess.call("%s -d %s -y" %
  252. (tmpbuildtools, install_dir), shell=True)
  253. else:
  254. install_dir = "/opt/poky/%s" % args.installer_version
  255. ret = subprocess.call("%s -y" % tmpbuildtools, shell=True)
  256. if ret != 0:
  257. logger.error("Could not run buildtools installer")
  258. return ret
  259. # Setup the environment
  260. logger.info("Setting up the environment")
  261. regex = re.compile(r'^(?P<export>export )?(?P<env_var>[A-Z_]+)=(?P<env_val>.+)$')
  262. with open("%s/environment-setup-%s-pokysdk-linux" %
  263. (install_dir, arch), 'rb') as f:
  264. for line in f:
  265. match = regex.search(line.decode('utf-8'))
  266. logger.debug("export regex: %s" % match)
  267. if match:
  268. env_var = match.group('env_var')
  269. logger.debug("env_var: %s" % env_var)
  270. env_val = match.group('env_val')
  271. logger.debug("env_val: %s" % env_val)
  272. os.environ[env_var] = env_val
  273. # Test installation
  274. logger.info("Testing installation")
  275. tool = ""
  276. m = re.search("extended", tmpbuildtools)
  277. logger.debug("extended regex: %s" % m)
  278. if args.with_extended_buildtools and not m:
  279. logger.info("Ignoring --with-extended-buildtools as filename "
  280. "does not contain 'extended'")
  281. if args.with_extended_buildtools and m:
  282. tool = 'gcc'
  283. else:
  284. tool = 'tar'
  285. logger.debug("install_dir: %s" % install_dir)
  286. cmd = shlex.split("/usr/bin/which %s" % tool)
  287. logger.debug("cmd: %s" % cmd)
  288. logger.debug("tool: %s" % tool)
  289. proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
  290. output, errors = proc.communicate()
  291. logger.debug("proc.args: %s" % proc.args)
  292. logger.debug("proc.communicate(): output %s" % output)
  293. logger.debug("proc.communicate(): errors %s" % errors)
  294. which_tool = output.decode('utf-8')
  295. logger.debug("which %s: %s" % (tool, which_tool))
  296. ret = proc.returncode
  297. if not which_tool.startswith(install_dir):
  298. logger.error("Something went wrong: %s not found in %s" %
  299. (tool, install_dir))
  300. if ret != 0:
  301. logger.error("Something went wrong: installation failed")
  302. else:
  303. logger.info("Installation successful. Remember to source the "
  304. "environment setup script now and in any new session.")
  305. return ret
  306. finally:
  307. # cleanup tmp directory
  308. shutil.rmtree(tmpsdk_dir)
  309. if __name__ == '__main__':
  310. try:
  311. ret = main()
  312. except Exception:
  313. ret = 1
  314. import traceback
  315. traceback.print_exc()
  316. sys.exit(ret)