scriptutils.py 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279
  1. # Script utility functions
  2. #
  3. # Copyright (C) 2014 Intel Corporation
  4. #
  5. # SPDX-License-Identifier: GPL-2.0-only
  6. #
  7. import argparse
  8. import glob
  9. import logging
  10. import os
  11. import random
  12. import shlex
  13. import shutil
  14. import string
  15. import subprocess
  16. import sys
  17. import tempfile
  18. import threading
  19. import importlib
  20. from importlib import machinery
  21. class KeepAliveStreamHandler(logging.StreamHandler):
  22. def __init__(self, keepalive=True, **kwargs):
  23. super().__init__(**kwargs)
  24. if keepalive is True:
  25. keepalive = 5000 # default timeout
  26. self._timeout = threading.Condition()
  27. self._stop = False
  28. # background thread waits on condition, if the condition does not
  29. # happen emit a keep alive message
  30. def thread():
  31. while not self._stop:
  32. with self._timeout:
  33. if not self._timeout.wait(keepalive):
  34. self.emit(logging.LogRecord("keepalive", logging.INFO,
  35. None, None, "Keepalive message", None, None))
  36. self._thread = threading.Thread(target = thread, daemon = True)
  37. self._thread.start()
  38. def close(self):
  39. # mark the thread to stop and notify it
  40. self._stop = True
  41. with self._timeout:
  42. self._timeout.notify()
  43. # wait for it to join
  44. self._thread.join()
  45. super().close()
  46. def emit(self, record):
  47. super().emit(record)
  48. # trigger timer reset
  49. with self._timeout:
  50. self._timeout.notify()
  51. def logger_create(name, stream=None, keepalive=None):
  52. logger = logging.getLogger(name)
  53. if keepalive is not None:
  54. loggerhandler = KeepAliveStreamHandler(stream=stream, keepalive=keepalive)
  55. else:
  56. loggerhandler = logging.StreamHandler(stream=stream)
  57. loggerhandler.setFormatter(logging.Formatter("%(levelname)s: %(message)s"))
  58. logger.addHandler(loggerhandler)
  59. logger.setLevel(logging.INFO)
  60. return logger
  61. def logger_setup_color(logger, color='auto'):
  62. from bb.msg import BBLogFormatter
  63. for handler in logger.handlers:
  64. if (isinstance(handler, logging.StreamHandler) and
  65. isinstance(handler.formatter, BBLogFormatter)):
  66. if color == 'always' or (color == 'auto' and handler.stream.isatty()):
  67. handler.formatter.enable_color()
  68. def load_plugins(logger, plugins, pluginpath):
  69. def load_plugin(name):
  70. logger.debug('Loading plugin %s' % name)
  71. spec = importlib.machinery.PathFinder.find_spec(name, path=[pluginpath] )
  72. if spec:
  73. return spec.loader.load_module()
  74. def plugin_name(filename):
  75. return os.path.splitext(os.path.basename(filename))[0]
  76. known_plugins = [plugin_name(p.__name__) for p in plugins]
  77. logger.debug('Loading plugins from %s...' % pluginpath)
  78. for fn in glob.glob(os.path.join(pluginpath, '*.py')):
  79. name = plugin_name(fn)
  80. if name != '__init__' and name not in known_plugins:
  81. plugin = load_plugin(name)
  82. if hasattr(plugin, 'plugin_init'):
  83. plugin.plugin_init(plugins)
  84. plugins.append(plugin)
  85. def git_convert_standalone_clone(repodir):
  86. """If specified directory is a git repository, ensure it's a standalone clone"""
  87. import bb.process
  88. if os.path.exists(os.path.join(repodir, '.git')):
  89. alternatesfile = os.path.join(repodir, '.git', 'objects', 'info', 'alternates')
  90. if os.path.exists(alternatesfile):
  91. # This will have been cloned with -s, so we need to convert it so none
  92. # of the contents is shared
  93. bb.process.run('git repack -a', cwd=repodir)
  94. os.remove(alternatesfile)
  95. def _get_temp_recipe_dir(d):
  96. # This is a little bit hacky but we need to find a place where we can put
  97. # the recipe so that bitbake can find it. We're going to delete it at the
  98. # end so it doesn't really matter where we put it.
  99. bbfiles = d.getVar('BBFILES').split()
  100. fetchrecipedir = None
  101. for pth in bbfiles:
  102. if pth.endswith('.bb'):
  103. pthdir = os.path.dirname(pth)
  104. if os.access(os.path.dirname(os.path.dirname(pthdir)), os.W_OK):
  105. fetchrecipedir = pthdir.replace('*', 'recipetool')
  106. if pthdir.endswith('workspace/recipes/*'):
  107. # Prefer the workspace
  108. break
  109. return fetchrecipedir
  110. class FetchUrlFailure(Exception):
  111. def __init__(self, url):
  112. self.url = url
  113. def __str__(self):
  114. return "Failed to fetch URL %s" % self.url
  115. def fetch_url(tinfoil, srcuri, srcrev, destdir, logger, preserve_tmp=False, mirrors=False):
  116. """
  117. Fetch the specified URL using normal do_fetch and do_unpack tasks, i.e.
  118. any dependencies that need to be satisfied in order to support the fetch
  119. operation will be taken care of
  120. """
  121. import bb
  122. checksums = {}
  123. fetchrecipepn = None
  124. # We need to put our temp directory under ${BASE_WORKDIR} otherwise
  125. # we may have problems with the recipe-specific sysroot population
  126. tmpparent = tinfoil.config_data.getVar('BASE_WORKDIR')
  127. bb.utils.mkdirhier(tmpparent)
  128. tmpdir = tempfile.mkdtemp(prefix='recipetool-', dir=tmpparent)
  129. try:
  130. tmpworkdir = os.path.join(tmpdir, 'work')
  131. logger.debug('fetch_url: temp dir is %s' % tmpdir)
  132. fetchrecipedir = _get_temp_recipe_dir(tinfoil.config_data)
  133. if not fetchrecipedir:
  134. logger.error('Searched BBFILES but unable to find a writeable place to put temporary recipe')
  135. sys.exit(1)
  136. fetchrecipe = None
  137. bb.utils.mkdirhier(fetchrecipedir)
  138. try:
  139. # Generate a dummy recipe so we can follow more or less normal paths
  140. # for do_fetch and do_unpack
  141. # I'd use tempfile functions here but underscores can be produced by that and those
  142. # aren't allowed in recipe file names except to separate the version
  143. rndstring = ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(8))
  144. fetchrecipe = os.path.join(fetchrecipedir, 'tmp-recipetool-%s.bb' % rndstring)
  145. fetchrecipepn = os.path.splitext(os.path.basename(fetchrecipe))[0]
  146. logger.debug('Generating initial recipe %s for fetching' % fetchrecipe)
  147. with open(fetchrecipe, 'w') as f:
  148. # We don't want to have to specify LIC_FILES_CHKSUM
  149. f.write('LICENSE = "CLOSED"\n')
  150. # We don't need the cross-compiler
  151. f.write('INHIBIT_DEFAULT_DEPS = "1"\n')
  152. # We don't have the checksums yet so we can't require them
  153. f.write('BB_STRICT_CHECKSUM = "ignore"\n')
  154. f.write('SRC_URI = "%s"\n' % srcuri)
  155. f.write('SRCREV = "%s"\n' % srcrev)
  156. f.write('WORKDIR = "%s"\n' % tmpworkdir)
  157. # Set S out of the way so it doesn't get created under the workdir
  158. f.write('S = "%s"\n' % os.path.join(tmpdir, 'emptysrc'))
  159. if not mirrors:
  160. # We do not need PREMIRRORS since we are almost certainly
  161. # fetching new source rather than something that has already
  162. # been fetched. Hence, we disable them by default.
  163. # However, we provide an option for users to enable it.
  164. f.write('PREMIRRORS = ""\n')
  165. f.write('MIRRORS = ""\n')
  166. logger.info('Fetching %s...' % srcuri)
  167. # FIXME this is too noisy at the moment
  168. # Parse recipes so our new recipe gets picked up
  169. tinfoil.parse_recipes()
  170. def eventhandler(event):
  171. if isinstance(event, bb.fetch2.MissingChecksumEvent):
  172. checksums.update(event.checksums)
  173. return True
  174. return False
  175. # Run the fetch + unpack tasks
  176. res = tinfoil.build_targets(fetchrecipepn,
  177. 'do_unpack',
  178. handle_events=True,
  179. extra_events=['bb.fetch2.MissingChecksumEvent'],
  180. event_callback=eventhandler)
  181. if not res:
  182. raise FetchUrlFailure(srcuri)
  183. # Remove unneeded directories
  184. rd = tinfoil.parse_recipe(fetchrecipepn)
  185. if rd:
  186. pathvars = ['T', 'RECIPE_SYSROOT', 'RECIPE_SYSROOT_NATIVE']
  187. for pathvar in pathvars:
  188. path = rd.getVar(pathvar)
  189. shutil.rmtree(path)
  190. finally:
  191. if fetchrecipe:
  192. try:
  193. os.remove(fetchrecipe)
  194. except FileNotFoundError:
  195. pass
  196. try:
  197. os.rmdir(fetchrecipedir)
  198. except OSError as e:
  199. import errno
  200. if e.errno != errno.ENOTEMPTY:
  201. raise
  202. bb.utils.mkdirhier(destdir)
  203. for fn in os.listdir(tmpworkdir):
  204. shutil.move(os.path.join(tmpworkdir, fn), destdir)
  205. finally:
  206. if not preserve_tmp:
  207. shutil.rmtree(tmpdir)
  208. tmpdir = None
  209. return checksums, tmpdir
  210. def run_editor(fn, logger=None):
  211. if isinstance(fn, str):
  212. files = [fn]
  213. else:
  214. files = fn
  215. editor = os.getenv('VISUAL', os.getenv('EDITOR', 'vi'))
  216. try:
  217. #print(shlex.split(editor) + files)
  218. return subprocess.check_call(shlex.split(editor) + files)
  219. except subprocess.CalledProcessError as exc:
  220. logger.error("Execution of '%s' failed: %s" % (editor, exc))
  221. return 1
  222. def is_src_url(param):
  223. """
  224. Check if a parameter is a URL and return True if so
  225. NOTE: be careful about changing this as it will influence how devtool/recipetool command line handling works
  226. """
  227. if not param:
  228. return False
  229. elif '://' in param:
  230. return True
  231. elif param.startswith('git@') or ('@' in param and param.endswith('.git')):
  232. return True
  233. return False
  234. def filter_src_subdirs(pth):
  235. """
  236. Filter out subdirectories of initial unpacked source trees that we do not care about.
  237. Used by devtool and recipetool.
  238. """
  239. dirlist = os.listdir(pth)
  240. filterout = ['git.indirectionsymlink', 'source-date-epoch']
  241. dirlist = [x for x in dirlist if x not in filterout]
  242. return dirlist