scriptutils.py 9.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270
  1. # Script utility functions
  2. #
  3. # Copyright (C) 2014 Intel Corporation
  4. #
  5. # SPDX-License-Identifier: GPL-2.0-only
  6. #
  7. import argparse
  8. import glob
  9. import logging
  10. import os
  11. import random
  12. import shlex
  13. import shutil
  14. import string
  15. import subprocess
  16. import sys
  17. import tempfile
  18. import threading
  19. import importlib
  20. from importlib import machinery
  21. class KeepAliveStreamHandler(logging.StreamHandler):
  22. def __init__(self, keepalive=True, **kwargs):
  23. super().__init__(**kwargs)
  24. if keepalive is True:
  25. keepalive = 5000 # default timeout
  26. self._timeout = threading.Condition()
  27. self._stop = False
  28. # background thread waits on condition, if the condition does not
  29. # happen emit a keep alive message
  30. def thread():
  31. while not self._stop:
  32. with self._timeout:
  33. if not self._timeout.wait(keepalive):
  34. self.emit(logging.LogRecord("keepalive", logging.INFO,
  35. None, None, "Keepalive message", None, None))
  36. self._thread = threading.Thread(target = thread, daemon = True)
  37. self._thread.start()
  38. def close(self):
  39. # mark the thread to stop and notify it
  40. self._stop = True
  41. with self._timeout:
  42. self._timeout.notify()
  43. # wait for it to join
  44. self._thread.join()
  45. super().close()
  46. def emit(self, record):
  47. super().emit(record)
  48. # trigger timer reset
  49. with self._timeout:
  50. self._timeout.notify()
  51. def logger_create(name, stream=None, keepalive=None):
  52. logger = logging.getLogger(name)
  53. if keepalive is not None:
  54. loggerhandler = KeepAliveStreamHandler(stream=stream, keepalive=keepalive)
  55. else:
  56. loggerhandler = logging.StreamHandler(stream=stream)
  57. loggerhandler.setFormatter(logging.Formatter("%(levelname)s: %(message)s"))
  58. logger.addHandler(loggerhandler)
  59. logger.setLevel(logging.INFO)
  60. return logger
  61. def logger_setup_color(logger, color='auto'):
  62. from bb.msg import BBLogFormatter
  63. for handler in logger.handlers:
  64. if (isinstance(handler, logging.StreamHandler) and
  65. isinstance(handler.formatter, BBLogFormatter)):
  66. if color == 'always' or (color == 'auto' and handler.stream.isatty()):
  67. handler.formatter.enable_color()
  68. def load_plugins(logger, plugins, pluginpath):
  69. import imp
  70. def load_plugin(name):
  71. logger.debug('Loading plugin %s' % name)
  72. spec = importlib.machinery.PathFinder.find_spec(name, path=[pluginpath] )
  73. if spec:
  74. return spec.loader.load_module()
  75. def plugin_name(filename):
  76. return os.path.splitext(os.path.basename(filename))[0]
  77. known_plugins = [plugin_name(p.__name__) for p in plugins]
  78. logger.debug('Loading plugins from %s...' % pluginpath)
  79. for fn in glob.glob(os.path.join(pluginpath, '*.py')):
  80. name = plugin_name(fn)
  81. if name != '__init__' and name not in known_plugins:
  82. plugin = load_plugin(name)
  83. if hasattr(plugin, 'plugin_init'):
  84. plugin.plugin_init(plugins)
  85. plugins.append(plugin)
  86. def git_convert_standalone_clone(repodir):
  87. """If specified directory is a git repository, ensure it's a standalone clone"""
  88. import bb.process
  89. if os.path.exists(os.path.join(repodir, '.git')):
  90. alternatesfile = os.path.join(repodir, '.git', 'objects', 'info', 'alternates')
  91. if os.path.exists(alternatesfile):
  92. # This will have been cloned with -s, so we need to convert it so none
  93. # of the contents is shared
  94. bb.process.run('git repack -a', cwd=repodir)
  95. os.remove(alternatesfile)
  96. def _get_temp_recipe_dir(d):
  97. # This is a little bit hacky but we need to find a place where we can put
  98. # the recipe so that bitbake can find it. We're going to delete it at the
  99. # end so it doesn't really matter where we put it.
  100. bbfiles = d.getVar('BBFILES').split()
  101. fetchrecipedir = None
  102. for pth in bbfiles:
  103. if pth.endswith('.bb'):
  104. pthdir = os.path.dirname(pth)
  105. if os.access(os.path.dirname(os.path.dirname(pthdir)), os.W_OK):
  106. fetchrecipedir = pthdir.replace('*', 'recipetool')
  107. if pthdir.endswith('workspace/recipes/*'):
  108. # Prefer the workspace
  109. break
  110. return fetchrecipedir
  111. class FetchUrlFailure(Exception):
  112. def __init__(self, url):
  113. self.url = url
  114. def __str__(self):
  115. return "Failed to fetch URL %s" % self.url
  116. def fetch_url(tinfoil, srcuri, srcrev, destdir, logger, preserve_tmp=False, mirrors=False):
  117. """
  118. Fetch the specified URL using normal do_fetch and do_unpack tasks, i.e.
  119. any dependencies that need to be satisfied in order to support the fetch
  120. operation will be taken care of
  121. """
  122. import bb
  123. checksums = {}
  124. fetchrecipepn = None
  125. # We need to put our temp directory under ${BASE_WORKDIR} otherwise
  126. # we may have problems with the recipe-specific sysroot population
  127. tmpparent = tinfoil.config_data.getVar('BASE_WORKDIR')
  128. bb.utils.mkdirhier(tmpparent)
  129. tmpdir = tempfile.mkdtemp(prefix='recipetool-', dir=tmpparent)
  130. try:
  131. tmpworkdir = os.path.join(tmpdir, 'work')
  132. logger.debug('fetch_url: temp dir is %s' % tmpdir)
  133. fetchrecipedir = _get_temp_recipe_dir(tinfoil.config_data)
  134. if not fetchrecipedir:
  135. logger.error('Searched BBFILES but unable to find a writeable place to put temporary recipe')
  136. sys.exit(1)
  137. fetchrecipe = None
  138. bb.utils.mkdirhier(fetchrecipedir)
  139. try:
  140. # Generate a dummy recipe so we can follow more or less normal paths
  141. # for do_fetch and do_unpack
  142. # I'd use tempfile functions here but underscores can be produced by that and those
  143. # aren't allowed in recipe file names except to separate the version
  144. rndstring = ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(8))
  145. fetchrecipe = os.path.join(fetchrecipedir, 'tmp-recipetool-%s.bb' % rndstring)
  146. fetchrecipepn = os.path.splitext(os.path.basename(fetchrecipe))[0]
  147. logger.debug('Generating initial recipe %s for fetching' % fetchrecipe)
  148. with open(fetchrecipe, 'w') as f:
  149. # We don't want to have to specify LIC_FILES_CHKSUM
  150. f.write('LICENSE = "CLOSED"\n')
  151. # We don't need the cross-compiler
  152. f.write('INHIBIT_DEFAULT_DEPS = "1"\n')
  153. # We don't have the checksums yet so we can't require them
  154. f.write('BB_STRICT_CHECKSUM = "ignore"\n')
  155. f.write('SRC_URI = "%s"\n' % srcuri)
  156. f.write('SRCREV = "%s"\n' % srcrev)
  157. f.write('WORKDIR = "%s"\n' % tmpworkdir)
  158. # Set S out of the way so it doesn't get created under the workdir
  159. f.write('S = "%s"\n' % os.path.join(tmpdir, 'emptysrc'))
  160. if not mirrors:
  161. # We do not need PREMIRRORS since we are almost certainly
  162. # fetching new source rather than something that has already
  163. # been fetched. Hence, we disable them by default.
  164. # However, we provide an option for users to enable it.
  165. f.write('PREMIRRORS = ""\n')
  166. f.write('MIRRORS = ""\n')
  167. logger.info('Fetching %s...' % srcuri)
  168. # FIXME this is too noisy at the moment
  169. # Parse recipes so our new recipe gets picked up
  170. tinfoil.parse_recipes()
  171. def eventhandler(event):
  172. if isinstance(event, bb.fetch2.MissingChecksumEvent):
  173. checksums.update(event.checksums)
  174. return True
  175. return False
  176. # Run the fetch + unpack tasks
  177. res = tinfoil.build_targets(fetchrecipepn,
  178. 'do_unpack',
  179. handle_events=True,
  180. extra_events=['bb.fetch2.MissingChecksumEvent'],
  181. event_callback=eventhandler)
  182. if not res:
  183. raise FetchUrlFailure(srcuri)
  184. # Remove unneeded directories
  185. rd = tinfoil.parse_recipe(fetchrecipepn)
  186. if rd:
  187. pathvars = ['T', 'RECIPE_SYSROOT', 'RECIPE_SYSROOT_NATIVE']
  188. for pathvar in pathvars:
  189. path = rd.getVar(pathvar)
  190. shutil.rmtree(path)
  191. finally:
  192. if fetchrecipe:
  193. try:
  194. os.remove(fetchrecipe)
  195. except FileNotFoundError:
  196. pass
  197. try:
  198. os.rmdir(fetchrecipedir)
  199. except OSError as e:
  200. import errno
  201. if e.errno != errno.ENOTEMPTY:
  202. raise
  203. bb.utils.mkdirhier(destdir)
  204. for fn in os.listdir(tmpworkdir):
  205. shutil.move(os.path.join(tmpworkdir, fn), destdir)
  206. finally:
  207. if not preserve_tmp:
  208. shutil.rmtree(tmpdir)
  209. tmpdir = None
  210. return checksums, tmpdir
  211. def run_editor(fn, logger=None):
  212. if isinstance(fn, str):
  213. files = [fn]
  214. else:
  215. files = fn
  216. editor = os.getenv('VISUAL', os.getenv('EDITOR', 'vi'))
  217. try:
  218. #print(shlex.split(editor) + files)
  219. return subprocess.check_call(shlex.split(editor) + files)
  220. except subprocess.CalledProcessError as exc:
  221. logger.error("Execution of '%s' failed: %s" % (editor, exc))
  222. return 1
  223. def is_src_url(param):
  224. """
  225. Check if a parameter is a URL and return True if so
  226. NOTE: be careful about changing this as it will influence how devtool/recipetool command line handling works
  227. """
  228. if not param:
  229. return False
  230. elif '://' in param:
  231. return True
  232. elif param.startswith('git@') or ('@' in param and param.endswith('.git')):
  233. return True
  234. return False