__init__.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420
  1. # Yocto Project layer check tool
  2. #
  3. # Copyright (C) 2017 Intel Corporation
  4. #
  5. # SPDX-License-Identifier: MIT
  6. #
  7. import os
  8. import re
  9. import subprocess
  10. from enum import Enum
  11. import bb.tinfoil
  12. class LayerType(Enum):
  13. BSP = 0
  14. DISTRO = 1
  15. SOFTWARE = 2
  16. ERROR_NO_LAYER_CONF = 98
  17. ERROR_BSP_DISTRO = 99
  18. def _get_configurations(path):
  19. configs = []
  20. for f in os.listdir(path):
  21. file_path = os.path.join(path, f)
  22. if os.path.isfile(file_path) and f.endswith('.conf'):
  23. configs.append(f[:-5]) # strip .conf
  24. return configs
  25. def _get_layer_collections(layer_path, lconf=None, data=None):
  26. import bb.parse
  27. import bb.data
  28. if lconf is None:
  29. lconf = os.path.join(layer_path, 'conf', 'layer.conf')
  30. if data is None:
  31. ldata = bb.data.init()
  32. bb.parse.init_parser(ldata)
  33. else:
  34. ldata = data.createCopy()
  35. ldata.setVar('LAYERDIR', layer_path)
  36. try:
  37. ldata = bb.parse.handle(lconf, ldata, include=True)
  38. except:
  39. raise RuntimeError("Parsing of layer.conf from layer: %s failed" % layer_path)
  40. ldata.expandVarref('LAYERDIR')
  41. collections = (ldata.getVar('BBFILE_COLLECTIONS') or '').split()
  42. if not collections:
  43. name = os.path.basename(layer_path)
  44. collections = [name]
  45. collections = {c: {} for c in collections}
  46. for name in collections:
  47. priority = ldata.getVar('BBFILE_PRIORITY_%s' % name)
  48. pattern = ldata.getVar('BBFILE_PATTERN_%s' % name)
  49. depends = ldata.getVar('LAYERDEPENDS_%s' % name)
  50. compat = ldata.getVar('LAYERSERIES_COMPAT_%s' % name)
  51. try:
  52. depDict = bb.utils.explode_dep_versions2(depends or "")
  53. except bb.utils.VersionStringException as vse:
  54. bb.fatal('Error parsing LAYERDEPENDS_%s: %s' % (name, str(vse)))
  55. collections[name]['priority'] = priority
  56. collections[name]['pattern'] = pattern
  57. collections[name]['depends'] = ' '.join(depDict.keys())
  58. collections[name]['compat'] = compat
  59. return collections
  60. def _detect_layer(layer_path):
  61. """
  62. Scans layer directory to detect what type of layer
  63. is BSP, Distro or Software.
  64. Returns a dictionary with layer name, type and path.
  65. """
  66. layer = {}
  67. layer_name = os.path.basename(layer_path)
  68. layer['name'] = layer_name
  69. layer['path'] = layer_path
  70. layer['conf'] = {}
  71. if not os.path.isfile(os.path.join(layer_path, 'conf', 'layer.conf')):
  72. layer['type'] = LayerType.ERROR_NO_LAYER_CONF
  73. return layer
  74. machine_conf = os.path.join(layer_path, 'conf', 'machine')
  75. distro_conf = os.path.join(layer_path, 'conf', 'distro')
  76. is_bsp = False
  77. is_distro = False
  78. if os.path.isdir(machine_conf):
  79. machines = _get_configurations(machine_conf)
  80. if machines:
  81. is_bsp = True
  82. if os.path.isdir(distro_conf):
  83. distros = _get_configurations(distro_conf)
  84. if distros:
  85. is_distro = True
  86. if is_bsp and is_distro:
  87. layer['type'] = LayerType.ERROR_BSP_DISTRO
  88. elif is_bsp:
  89. layer['type'] = LayerType.BSP
  90. layer['conf']['machines'] = machines
  91. elif is_distro:
  92. layer['type'] = LayerType.DISTRO
  93. layer['conf']['distros'] = distros
  94. else:
  95. layer['type'] = LayerType.SOFTWARE
  96. layer['collections'] = _get_layer_collections(layer['path'])
  97. return layer
  98. def detect_layers(layer_directories, no_auto):
  99. layers = []
  100. for directory in layer_directories:
  101. directory = os.path.realpath(directory)
  102. if directory[-1] == '/':
  103. directory = directory[0:-1]
  104. if no_auto:
  105. conf_dir = os.path.join(directory, 'conf')
  106. if os.path.isdir(conf_dir):
  107. layer = _detect_layer(directory)
  108. if layer:
  109. layers.append(layer)
  110. else:
  111. for root, dirs, files in os.walk(directory):
  112. dir_name = os.path.basename(root)
  113. conf_dir = os.path.join(root, 'conf')
  114. if os.path.isdir(conf_dir):
  115. layer = _detect_layer(root)
  116. if layer:
  117. layers.append(layer)
  118. return layers
  119. def _find_layer_depends(depend, layers):
  120. for layer in layers:
  121. if 'collections' not in layer:
  122. continue
  123. for collection in layer['collections']:
  124. if depend == collection:
  125. return layer
  126. return None
  127. def add_layer_dependencies(bblayersconf, layer, layers, logger):
  128. def recurse_dependencies(depends, layer, layers, logger, ret = []):
  129. logger.debug('Processing dependencies %s for layer %s.' % \
  130. (depends, layer['name']))
  131. for depend in depends.split():
  132. # core (oe-core) is suppose to be provided
  133. if depend == 'core':
  134. continue
  135. layer_depend = _find_layer_depends(depend, layers)
  136. if not layer_depend:
  137. logger.error('Layer %s depends on %s and isn\'t found.' % \
  138. (layer['name'], depend))
  139. ret = None
  140. continue
  141. # We keep processing, even if ret is None, this allows us to report
  142. # multiple errors at once
  143. if ret is not None and layer_depend not in ret:
  144. ret.append(layer_depend)
  145. else:
  146. # we might have processed this dependency already, in which case
  147. # we should not do it again (avoid recursive loop)
  148. continue
  149. # Recursively process...
  150. if 'collections' not in layer_depend:
  151. continue
  152. for collection in layer_depend['collections']:
  153. collect_deps = layer_depend['collections'][collection]['depends']
  154. if not collect_deps:
  155. continue
  156. ret = recurse_dependencies(collect_deps, layer_depend, layers, logger, ret)
  157. return ret
  158. layer_depends = []
  159. for collection in layer['collections']:
  160. depends = layer['collections'][collection]['depends']
  161. if not depends:
  162. continue
  163. layer_depends = recurse_dependencies(depends, layer, layers, logger, layer_depends)
  164. # Note: [] (empty) is allowed, None is not!
  165. if layer_depends is None:
  166. return False
  167. else:
  168. add_layers(bblayersconf, layer_depends, logger)
  169. return True
  170. def add_layers(bblayersconf, layers, logger):
  171. # Don't add a layer that is already present.
  172. added = set()
  173. output = check_command('Getting existing layers failed.', 'bitbake-layers show-layers').decode('utf-8')
  174. for layer, path, pri in re.findall(r'^(\S+) +([^\n]*?) +(\d+)$', output, re.MULTILINE):
  175. added.add(path)
  176. with open(bblayersconf, 'a+') as f:
  177. for layer in layers:
  178. logger.info('Adding layer %s' % layer['name'])
  179. name = layer['name']
  180. path = layer['path']
  181. if path in added:
  182. logger.info('%s is already in %s' % (name, bblayersconf))
  183. else:
  184. added.add(path)
  185. f.write("\nBBLAYERS += \"%s\"\n" % path)
  186. return True
  187. def check_bblayers(bblayersconf, layer_path, logger):
  188. '''
  189. If layer_path found in BBLAYERS return True
  190. '''
  191. import bb.parse
  192. import bb.data
  193. ldata = bb.parse.handle(bblayersconf, bb.data.init(), include=True)
  194. for bblayer in (ldata.getVar('BBLAYERS') or '').split():
  195. if os.path.normpath(bblayer) == os.path.normpath(layer_path):
  196. return True
  197. return False
  198. def check_command(error_msg, cmd, cwd=None):
  199. '''
  200. Run a command under a shell, capture stdout and stderr in a single stream,
  201. throw an error when command returns non-zero exit code. Returns the output.
  202. '''
  203. p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd)
  204. output, _ = p.communicate()
  205. if p.returncode:
  206. msg = "%s\nCommand: %s\nOutput:\n%s" % (error_msg, cmd, output.decode('utf-8'))
  207. raise RuntimeError(msg)
  208. return output
  209. def get_signatures(builddir, failsafe=False, machine=None):
  210. import re
  211. # some recipes needs to be excluded like meta-world-pkgdata
  212. # because a layer can add recipes to a world build so signature
  213. # will be change
  214. exclude_recipes = ('meta-world-pkgdata',)
  215. sigs = {}
  216. tune2tasks = {}
  217. cmd = 'BB_ENV_EXTRAWHITE="$BB_ENV_EXTRAWHITE BB_SIGNATURE_HANDLER" BB_SIGNATURE_HANDLER="OEBasicHash" '
  218. if machine:
  219. cmd += 'MACHINE=%s ' % machine
  220. cmd += 'bitbake '
  221. if failsafe:
  222. cmd += '-k '
  223. cmd += '-S none world'
  224. sigs_file = os.path.join(builddir, 'locked-sigs.inc')
  225. if os.path.exists(sigs_file):
  226. os.unlink(sigs_file)
  227. try:
  228. check_command('Generating signatures failed. This might be due to some parse error and/or general layer incompatibilities.',
  229. cmd, builddir)
  230. except RuntimeError as ex:
  231. if failsafe and os.path.exists(sigs_file):
  232. # Ignore the error here. Most likely some recipes active
  233. # in a world build lack some dependencies. There is a
  234. # separate test_machine_world_build which exposes the
  235. # failure.
  236. pass
  237. else:
  238. raise
  239. sig_regex = re.compile("^(?P<task>.*:.*):(?P<hash>.*) .$")
  240. tune_regex = re.compile("(^|\s)SIGGEN_LOCKEDSIGS_t-(?P<tune>\S*)\s*=\s*")
  241. current_tune = None
  242. with open(sigs_file, 'r') as f:
  243. for line in f.readlines():
  244. line = line.strip()
  245. t = tune_regex.search(line)
  246. if t:
  247. current_tune = t.group('tune')
  248. s = sig_regex.match(line)
  249. if s:
  250. exclude = False
  251. for er in exclude_recipes:
  252. (recipe, task) = s.group('task').split(':')
  253. if er == recipe:
  254. exclude = True
  255. break
  256. if exclude:
  257. continue
  258. sigs[s.group('task')] = s.group('hash')
  259. tune2tasks.setdefault(current_tune, []).append(s.group('task'))
  260. if not sigs:
  261. raise RuntimeError('Can\'t load signatures from %s' % sigs_file)
  262. return (sigs, tune2tasks)
  263. def get_depgraph(targets=['world'], failsafe=False):
  264. '''
  265. Returns the dependency graph for the given target(s).
  266. The dependency graph is taken directly from DepTreeEvent.
  267. '''
  268. depgraph = None
  269. with bb.tinfoil.Tinfoil() as tinfoil:
  270. tinfoil.prepare(config_only=False)
  271. tinfoil.set_event_mask(['bb.event.NoProvider', 'bb.event.DepTreeGenerated', 'bb.command.CommandCompleted'])
  272. if not tinfoil.run_command('generateDepTreeEvent', targets, 'do_build'):
  273. raise RuntimeError('starting generateDepTreeEvent failed')
  274. while True:
  275. event = tinfoil.wait_event(timeout=1000)
  276. if event:
  277. if isinstance(event, bb.command.CommandFailed):
  278. raise RuntimeError('Generating dependency information failed: %s' % event.error)
  279. elif isinstance(event, bb.command.CommandCompleted):
  280. break
  281. elif isinstance(event, bb.event.NoProvider):
  282. if failsafe:
  283. # The event is informational, we will get information about the
  284. # remaining dependencies eventually and thus can ignore this
  285. # here like we do in get_signatures(), if desired.
  286. continue
  287. if event._reasons:
  288. raise RuntimeError('Nothing provides %s: %s' % (event._item, event._reasons))
  289. else:
  290. raise RuntimeError('Nothing provides %s.' % (event._item))
  291. elif isinstance(event, bb.event.DepTreeGenerated):
  292. depgraph = event._depgraph
  293. if depgraph is None:
  294. raise RuntimeError('Could not retrieve the depgraph.')
  295. return depgraph
  296. def compare_signatures(old_sigs, curr_sigs):
  297. '''
  298. Compares the result of two get_signatures() calls. Returns None if no
  299. problems found, otherwise a string that can be used as additional
  300. explanation in self.fail().
  301. '''
  302. # task -> (old signature, new signature)
  303. sig_diff = {}
  304. for task in old_sigs:
  305. if task in curr_sigs and \
  306. old_sigs[task] != curr_sigs[task]:
  307. sig_diff[task] = (old_sigs[task], curr_sigs[task])
  308. if not sig_diff:
  309. return None
  310. # Beware, depgraph uses task=<pn>.<taskname> whereas get_signatures()
  311. # uses <pn>:<taskname>. Need to convert sometimes. The output follows
  312. # the convention from get_signatures() because that seems closer to
  313. # normal bitbake output.
  314. def sig2graph(task):
  315. pn, taskname = task.rsplit(':', 1)
  316. return pn + '.' + taskname
  317. def graph2sig(task):
  318. pn, taskname = task.rsplit('.', 1)
  319. return pn + ':' + taskname
  320. depgraph = get_depgraph(failsafe=True)
  321. depends = depgraph['tdepends']
  322. # If a task A has a changed signature, but none of its
  323. # dependencies, then we need to report it because it is
  324. # the one which introduces a change. Any task depending on
  325. # A (directly or indirectly) will also have a changed
  326. # signature, but we don't need to report it. It might have
  327. # its own changes, which will become apparent once the
  328. # issues that we do report are fixed and the test gets run
  329. # again.
  330. sig_diff_filtered = []
  331. for task, (old_sig, new_sig) in sig_diff.items():
  332. deps_tainted = False
  333. for dep in depends.get(sig2graph(task), ()):
  334. if graph2sig(dep) in sig_diff:
  335. deps_tainted = True
  336. break
  337. if not deps_tainted:
  338. sig_diff_filtered.append((task, old_sig, new_sig))
  339. msg = []
  340. msg.append('%d signatures changed, initial differences (first hash before, second after):' %
  341. len(sig_diff))
  342. for diff in sorted(sig_diff_filtered):
  343. recipe, taskname = diff[0].rsplit(':', 1)
  344. cmd = 'bitbake-diffsigs --task %s %s --signature %s %s' % \
  345. (recipe, taskname, diff[1], diff[2])
  346. msg.append(' %s: %s -> %s' % diff)
  347. msg.append(' %s' % cmd)
  348. try:
  349. output = check_command('Determining signature difference failed.',
  350. cmd).decode('utf-8')
  351. except RuntimeError as error:
  352. output = str(error)
  353. if output:
  354. msg.extend([' ' + line for line in output.splitlines()])
  355. msg.append('')
  356. return '\n'.join(msg)