install-buildtools 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339
  1. #!/usr/bin/env python3
  2. # Buildtools and buildtools extended installer helper script
  3. #
  4. # Copyright (C) 2017-2020 Intel Corporation
  5. #
  6. # SPDX-License-Identifier: GPL-2.0-only
  7. #
  8. # NOTE: --with-extended-buildtools is on by default
  9. #
  10. # Example usage (extended buildtools from milestone):
  11. # (1) using --url and --filename
  12. # $ install-buildtools \
  13. # --url http://downloads.yoctoproject.org/releases/yocto/milestones/yocto-3.1_M3/buildtools \
  14. # --filename x86_64-buildtools-extended-nativesdk-standalone-3.0+snapshot-20200315.sh
  15. # (2) using --base-url, --release, --installer-version and --build-date
  16. # $ install-buildtools \
  17. # --base-url http://downloads.yoctoproject.org/releases/yocto \
  18. # --release yocto-3.1_M3 \
  19. # --installer-version 3.0+snapshot
  20. # --build-date 202000315
  21. #
  22. # Example usage (standard buildtools from release):
  23. # (3) using --url and --filename
  24. # $ install-buildtools --without-extended-buildtools \
  25. # --url http://downloads.yoctoproject.org/releases/yocto/yocto-3.0.2/buildtools \
  26. # --filename x86_64-buildtools-nativesdk-standalone-3.0.2.sh
  27. # (4) using --base-url, --release and --installer-version
  28. # $ install-buildtools --without-extended-buildtools \
  29. # --base-url http://downloads.yoctoproject.org/releases/yocto \
  30. # --release yocto-3.0.2 \
  31. # --installer-version 3.0.2
  32. #
  33. import argparse
  34. import logging
  35. import os
  36. import re
  37. import shutil
  38. import shlex
  39. import stat
  40. import subprocess
  41. import sys
  42. import tempfile
  43. from urllib.parse import quote
  44. scripts_path = os.path.dirname(os.path.realpath(__file__))
  45. lib_path = scripts_path + '/lib'
  46. sys.path = sys.path + [lib_path]
  47. import scriptutils
  48. import scriptpath
  49. PROGNAME = 'install-buildtools'
  50. logger = scriptutils.logger_create(PROGNAME, stream=sys.stdout)
  51. DEFAULT_INSTALL_DIR = os.path.join(os.path.split(scripts_path)[0],'buildtools')
  52. DEFAULT_BASE_URL = 'http://downloads.yoctoproject.org/releases/yocto'
  53. DEFAULT_RELEASE = 'yocto-3.1_M3'
  54. DEFAULT_INSTALLER_VERSION = '3.0+snapshot'
  55. DEFAULT_BUILDDATE = "20200315"
  56. # Python version sanity check
  57. if not (sys.version_info.major == 3 and sys.version_info.minor >= 4):
  58. logger.error("This script requires Python 3.4 or greater")
  59. logger.error("You have Python %s.%s" %
  60. (sys.version_info.major, sys.version_info.minor))
  61. sys.exit(1)
  62. # The following three functions are copied directly from
  63. # bitbake/lib/bb/utils.py, in order to allow this script
  64. # to run on versions of python earlier than what bitbake
  65. # supports (e.g. less than Python 3.5 for YP 3.1 release)
  66. def _hasher(method, filename):
  67. import mmap
  68. with open(filename, "rb") as f:
  69. try:
  70. with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm:
  71. for chunk in iter(lambda: mm.read(8192), b''):
  72. method.update(chunk)
  73. except ValueError:
  74. # You can't mmap() an empty file so silence this exception
  75. pass
  76. return method.hexdigest()
  77. def md5_file(filename):
  78. """
  79. Return the hex string representation of the MD5 checksum of filename.
  80. """
  81. import hashlib
  82. return _hasher(hashlib.md5(), filename)
  83. def sha256_file(filename):
  84. """
  85. Return the hex string representation of the 256-bit SHA checksum of
  86. filename.
  87. """
  88. import hashlib
  89. return _hasher(hashlib.sha256(), filename)
  90. def main():
  91. global DEFAULT_INSTALL_DIR
  92. global DEFAULT_BASE_URL
  93. global DEFAULT_RELEASE
  94. global DEFAULT_INSTALLER_VERSION
  95. global DEFAULT_BUILDDATE
  96. filename = ""
  97. release = ""
  98. buildtools_url = ""
  99. install_dir = ""
  100. parser = argparse.ArgumentParser(
  101. description="Buildtools installation helper",
  102. add_help=False)
  103. parser.add_argument('-u', '--url',
  104. help='URL from where to fetch buildtools SDK installer, not '
  105. 'including filename (optional)\n'
  106. 'Requires --filename.',
  107. action='store')
  108. parser.add_argument('-f', '--filename',
  109. help='filename for the buildtools SDK installer to be installed '
  110. '(optional)\nRequires --url',
  111. action='store')
  112. parser.add_argument('-d', '--directory',
  113. default=DEFAULT_INSTALL_DIR,
  114. help='directory where buildtools SDK will be installed (optional)',
  115. action='store')
  116. parser.add_argument('-r', '--release',
  117. default=DEFAULT_RELEASE,
  118. help='Yocto Project release string for SDK which will be '
  119. 'installed (optional)',
  120. action='store')
  121. parser.add_argument('-V', '--installer-version',
  122. default=DEFAULT_INSTALLER_VERSION,
  123. help='version string for the SDK to be installed (optional)',
  124. action='store')
  125. parser.add_argument('-b', '--base-url',
  126. default=DEFAULT_BASE_URL,
  127. help='base URL from which to fetch SDK (optional)', action='store')
  128. parser.add_argument('-t', '--build-date',
  129. default=DEFAULT_BUILDDATE,
  130. help='Build date of pre-release SDK (optional)', action='store')
  131. group = parser.add_mutually_exclusive_group()
  132. group.add_argument('--with-extended-buildtools', action='store_true',
  133. dest='with_extended_buildtools',
  134. default=True,
  135. help='enable extended buildtools tarball (on by default)')
  136. group.add_argument('--without-extended-buildtools', action='store_false',
  137. dest='with_extended_buildtools',
  138. help='disable extended buildtools (traditional buildtools tarball)')
  139. parser.add_argument('-c', '--check', help='enable md5 checksum checking',
  140. default=True,
  141. action='store_true')
  142. parser.add_argument('-D', '--debug', help='enable debug output',
  143. action='store_true')
  144. parser.add_argument('-q', '--quiet', help='print only errors',
  145. action='store_true')
  146. parser.add_argument('-h', '--help', action='help',
  147. default=argparse.SUPPRESS,
  148. help='show this help message and exit')
  149. args = parser.parse_args()
  150. if args.debug:
  151. logger.setLevel(logging.DEBUG)
  152. elif args.quiet:
  153. logger.setLevel(logging.ERROR)
  154. if args.url and args.filename:
  155. logger.debug("--url and --filename detected. Ignoring --base-url "
  156. "--release --installer-version arguments.")
  157. filename = args.filename
  158. buildtools_url = "%s/%s" % (args.url, filename)
  159. else:
  160. if args.base_url:
  161. base_url = args.base_url
  162. else:
  163. base_url = DEFAULT_BASE_URL
  164. if args.release:
  165. # check if this is a pre-release "milestone" SDK
  166. m = re.search(r"^(?P<distro>[a-zA-Z\-]+)(?P<version>[0-9.]+)(?P<milestone>_M[1-9])$",
  167. args.release)
  168. logger.debug("milestone regex: %s" % m)
  169. if m and m.group('milestone'):
  170. logger.debug("release[distro]: %s" % m.group('distro'))
  171. logger.debug("release[version]: %s" % m.group('version'))
  172. logger.debug("release[milestone]: %s" % m.group('milestone'))
  173. if not args.build_date:
  174. logger.error("Milestone installers require --build-date")
  175. else:
  176. if args.with_extended_buildtools:
  177. filename = "x86_64-buildtools-extended-nativesdk-standalone-%s-%s.sh" % (
  178. args.installer_version, args.build_date)
  179. else:
  180. filename = "x86_64-buildtools-nativesdk-standalone-%s-%s.sh" % (
  181. args.installer_version, args.build_date)
  182. safe_filename = quote(filename)
  183. buildtools_url = "%s/milestones/%s/buildtools/%s" % (base_url, args.release, safe_filename)
  184. # regular release SDK
  185. else:
  186. if args.with_extended_buildtools:
  187. filename = "x86_64-buildtools-extended-nativesdk-standalone-%s.sh" % args.installer_version
  188. else:
  189. filename = "x86_64-buildtools-nativesdk-standalone-%s.sh" % args.installer_version
  190. safe_filename = quote(filename)
  191. buildtools_url = "%s/%s/buildtools/%s" % (base_url, args.release, safe_filename)
  192. tmpsdk_dir = tempfile.mkdtemp()
  193. try:
  194. # Fetch installer
  195. logger.info("Fetching buildtools installer")
  196. tmpbuildtools = os.path.join(tmpsdk_dir, filename)
  197. ret = subprocess.call("wget -q -O %s %s" %
  198. (tmpbuildtools, buildtools_url), shell=True)
  199. if ret != 0:
  200. logger.error("Could not download file from %s" % buildtools_url)
  201. return ret
  202. # Verify checksum
  203. if args.check:
  204. logger.info("Fetching buildtools installer checksum")
  205. checksum_type = ""
  206. for checksum_type in ["md5sum", "sha256"]:
  207. check_url = "{}.{}".format(buildtools_url, checksum_type)
  208. checksum_filename = "{}.{}".format(filename, checksum_type)
  209. tmpbuildtools_checksum = os.path.join(tmpsdk_dir, checksum_filename)
  210. ret = subprocess.call("wget -q -O %s %s" %
  211. (tmpbuildtools_checksum, check_url), shell=True)
  212. if ret == 0:
  213. break
  214. else:
  215. if ret != 0:
  216. logger.error("Could not download file from %s" % check_url)
  217. return ret
  218. regex = re.compile(r"^(?P<checksum>[0-9a-f]+)\s\s(?P<path>.*/)?(?P<filename>.*)$")
  219. with open(tmpbuildtools_checksum, 'rb') as f:
  220. original = f.read()
  221. m = re.search(regex, original.decode("utf-8"))
  222. logger.debug("checksum regex match: %s" % m)
  223. logger.debug("checksum: %s" % m.group('checksum'))
  224. logger.debug("path: %s" % m.group('path'))
  225. logger.debug("filename: %s" % m.group('filename'))
  226. if filename != m.group('filename'):
  227. logger.error("Filename does not match name in checksum")
  228. return 1
  229. checksum = m.group('checksum')
  230. if checksum_type == "md5sum":
  231. checksum_value = md5_file(tmpbuildtools)
  232. else:
  233. checksum_value = sha256_file(tmpbuildtools)
  234. if checksum == checksum_value:
  235. logger.info("Checksum success")
  236. else:
  237. logger.error("Checksum %s expected. Actual checksum is %s." %
  238. (checksum, checksum_value))
  239. # Make installer executable
  240. logger.info("Making installer executable")
  241. st = os.stat(tmpbuildtools)
  242. os.chmod(tmpbuildtools, st.st_mode | stat.S_IEXEC)
  243. logger.debug(os.stat(tmpbuildtools))
  244. if args.directory:
  245. install_dir = args.directory
  246. ret = subprocess.call("%s -d %s -y" %
  247. (tmpbuildtools, install_dir), shell=True)
  248. else:
  249. install_dir = "/opt/poky/%s" % args.installer_version
  250. ret = subprocess.call("%s -y" % tmpbuildtools, shell=True)
  251. if ret != 0:
  252. logger.error("Could not run buildtools installer")
  253. # Setup the environment
  254. logger.info("Setting up the environment")
  255. regex = re.compile(r'^(?P<export>export )?(?P<env_var>[A-Z_]+)=(?P<env_val>.+)$')
  256. with open("%s/environment-setup-x86_64-pokysdk-linux" %
  257. install_dir, 'rb') as f:
  258. for line in f:
  259. match = regex.search(line.decode('utf-8'))
  260. logger.debug("export regex: %s" % match)
  261. if match:
  262. env_var = match.group('env_var')
  263. logger.debug("env_var: %s" % env_var)
  264. env_val = match.group('env_val')
  265. logger.debug("env_val: %s" % env_val)
  266. os.environ[env_var] = env_val
  267. # Test installation
  268. logger.info("Testing installation")
  269. tool = ""
  270. m = re.search("extended", tmpbuildtools)
  271. logger.debug("extended regex: %s" % m)
  272. if args.with_extended_buildtools and not m:
  273. logger.info("Ignoring --with-extended-buildtools as filename "
  274. "does not contain 'extended'")
  275. if args.with_extended_buildtools and m:
  276. tool = 'gcc'
  277. else:
  278. tool = 'tar'
  279. logger.debug("install_dir: %s" % install_dir)
  280. cmd = shlex.split("/usr/bin/which %s" % tool)
  281. logger.debug("cmd: %s" % cmd)
  282. logger.debug("tool: %s" % tool)
  283. proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
  284. output, errors = proc.communicate()
  285. logger.debug("proc.args: %s" % proc.args)
  286. logger.debug("proc.communicate(): output %s" % output)
  287. logger.debug("proc.communicate(): errors %s" % errors)
  288. which_tool = output.decode('utf-8')
  289. logger.debug("which %s: %s" % (tool, which_tool))
  290. ret = proc.returncode
  291. if not which_tool.startswith(install_dir):
  292. logger.error("Something went wrong: %s not found in %s" %
  293. (tool, install_dir))
  294. if ret != 0:
  295. logger.error("Something went wrong: installation failed")
  296. else:
  297. logger.info("Installation successful. Remember to source the "
  298. "environment setup script now and in any new session.")
  299. return ret
  300. finally:
  301. # cleanup tmp directory
  302. shutil.rmtree(tmpsdk_dir)
  303. if __name__ == '__main__':
  304. try:
  305. ret = main()
  306. except Exception:
  307. ret = 1
  308. import traceback
  309. traceback.print_exc()
  310. sys.exit(ret)