yocto-check-layer 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257
  1. #!/usr/bin/env python3
  2. # Yocto Project layer checking tool
  3. #
  4. # Copyright (C) 2017 Intel Corporation
  5. #
  6. # SPDX-License-Identifier: MIT
  7. #
  8. import os
  9. import sys
  10. import argparse
  11. import logging
  12. import time
  13. import signal
  14. import shutil
  15. import collections
  16. scripts_path = os.path.dirname(os.path.realpath(__file__))
  17. lib_path = scripts_path + '/lib'
  18. sys.path = sys.path + [lib_path]
  19. import scriptutils
  20. import scriptpath
  21. scriptpath.add_oe_lib_path()
  22. scriptpath.add_bitbake_lib_path()
  23. from checklayer import LayerType, detect_layers, add_layers, add_layer_dependencies, get_layer_dependencies, get_signatures, check_bblayers, sanity_check_layers
  24. from oeqa.utils.commands import get_bb_vars
  25. PROGNAME = 'yocto-check-layer'
  26. CASES_PATHS = [os.path.join(os.path.abspath(os.path.dirname(__file__)),
  27. 'lib', 'checklayer', 'cases')]
  28. logger = scriptutils.logger_create(PROGNAME, stream=sys.stdout)
  29. def test_layer(td, layer, test_software_layer_signatures):
  30. from checklayer.context import CheckLayerTestContext
  31. logger.info("Starting to analyze: %s" % layer['name'])
  32. logger.info("----------------------------------------------------------------------")
  33. tc = CheckLayerTestContext(td=td, logger=logger, layer=layer, test_software_layer_signatures=test_software_layer_signatures)
  34. tc.loadTests(CASES_PATHS)
  35. return tc.runTests()
  36. def dump_layer_debug(layer):
  37. logger.debug("Found layer %s (%s)" % (layer["name"], layer["path"]))
  38. collections = layer.get("collections", {})
  39. if collections:
  40. logger.debug("%s collections: %s" % (layer["name"], ", ".join(collections)))
  41. def main():
  42. parser = argparse.ArgumentParser(
  43. description="Yocto Project layer checking tool",
  44. add_help=False)
  45. parser.add_argument('layers', metavar='LAYER_DIR', nargs='+',
  46. help='Layer to check')
  47. parser.add_argument('-o', '--output-log',
  48. help='File to output log (optional)', action='store')
  49. parser.add_argument('--dependency', nargs="+",
  50. help='Layers to process for dependencies', action='store')
  51. parser.add_argument('--no-auto-dependency', help='Disable automatic testing of dependencies',
  52. action='store_true')
  53. parser.add_argument('--machines', nargs="+",
  54. help='List of MACHINEs to be used during testing', action='store')
  55. parser.add_argument('--additional-layers', nargs="+",
  56. help='List of additional layers to add during testing', action='store')
  57. group = parser.add_mutually_exclusive_group()
  58. group.add_argument('--with-software-layer-signature-check', action='store_true', dest='test_software_layer_signatures',
  59. default=True,
  60. help='check that software layers do not change signatures (on by default)')
  61. group.add_argument('--without-software-layer-signature-check', action='store_false', dest='test_software_layer_signatures',
  62. help='disable signature checking for software layers')
  63. parser.add_argument('-n', '--no-auto', help='Disable auto layer discovery',
  64. action='store_true')
  65. parser.add_argument('-d', '--debug', help='Enable debug output',
  66. action='store_true')
  67. parser.add_argument('-q', '--quiet', help='Print only errors',
  68. action='store_true')
  69. parser.add_argument('-h', '--help', action='help',
  70. default=argparse.SUPPRESS,
  71. help='show this help message and exit')
  72. args = parser.parse_args()
  73. if args.output_log:
  74. fh = logging.FileHandler(args.output_log)
  75. fh.setFormatter(logging.Formatter("%(levelname)s: %(message)s"))
  76. logger.addHandler(fh)
  77. if args.debug:
  78. logger.setLevel(logging.DEBUG)
  79. elif args.quiet:
  80. logger.setLevel(logging.ERROR)
  81. if not 'BUILDDIR' in os.environ:
  82. logger.error("You must source the environment before running this script.")
  83. logger.error("$ source oe-init-build-env")
  84. return 1
  85. builddir = os.environ['BUILDDIR']
  86. bblayersconf = os.path.join(builddir, 'conf', 'bblayers.conf')
  87. layers = detect_layers(args.layers, args.no_auto)
  88. if not layers:
  89. logger.error("Failed to detect layers")
  90. return 1
  91. if args.additional_layers:
  92. additional_layers = detect_layers(args.additional_layers, args.no_auto)
  93. else:
  94. additional_layers = []
  95. if args.dependency:
  96. dep_layers = detect_layers(args.dependency, args.no_auto)
  97. dep_layers = dep_layers + layers
  98. else:
  99. dep_layers = layers
  100. logger.debug("Found additional layers:")
  101. for l in additional_layers:
  102. dump_layer_debug(l)
  103. logger.debug("Found dependency layers:")
  104. for l in dep_layers:
  105. dump_layer_debug(l)
  106. if not sanity_check_layers(additional_layers + dep_layers, logger):
  107. logger.error("Failed layer validation")
  108. return 1
  109. logger.info("Detected layers:")
  110. for layer in layers:
  111. if layer['type'] == LayerType.ERROR_BSP_DISTRO:
  112. logger.error("%s: Can't be DISTRO and BSP type at the same time."\
  113. " Both conf/distro and conf/machine folders were found."\
  114. % layer['name'])
  115. layers.remove(layer)
  116. elif layer['type'] == LayerType.ERROR_NO_LAYER_CONF:
  117. logger.info("%s: Doesn't have conf/layer.conf file, so ignoring"\
  118. % layer['name'])
  119. layers.remove(layer)
  120. else:
  121. logger.info("%s: %s, %s" % (layer['name'], layer['type'],
  122. layer['path']))
  123. if not layers:
  124. return 1
  125. # Find all dependencies, and get them checked too
  126. if not args.no_auto_dependency:
  127. depends = []
  128. for layer in layers:
  129. layer_depends = get_layer_dependencies(layer, dep_layers, logger)
  130. if layer_depends:
  131. for d in layer_depends:
  132. if d not in depends:
  133. depends.append(d)
  134. for d in depends:
  135. if d not in layers:
  136. logger.info("Adding %s to the list of layers to test, as a dependency", d['name'])
  137. layers.append(d)
  138. shutil.copyfile(bblayersconf, bblayersconf + '.backup')
  139. def cleanup_bblayers(signum, frame):
  140. shutil.copyfile(bblayersconf + '.backup', bblayersconf)
  141. os.unlink(bblayersconf + '.backup')
  142. signal.signal(signal.SIGTERM, cleanup_bblayers)
  143. signal.signal(signal.SIGINT, cleanup_bblayers)
  144. td = {}
  145. results = collections.OrderedDict()
  146. results_status = collections.OrderedDict()
  147. layers_tested = 0
  148. for layer in layers:
  149. if layer['type'] == LayerType.ERROR_NO_LAYER_CONF or \
  150. layer['type'] == LayerType.ERROR_BSP_DISTRO:
  151. continue
  152. # Reset to a clean backup copy for each run
  153. shutil.copyfile(bblayersconf + '.backup', bblayersconf)
  154. if check_bblayers(bblayersconf, layer['path'], logger):
  155. logger.info("%s already in %s. To capture initial signatures, layer under test should not present "
  156. "in BBLAYERS. Please remove %s from BBLAYERS." % (layer['name'], bblayersconf, layer['name']))
  157. results[layer['name']] = None
  158. results_status[layer['name']] = 'SKIPPED (Layer under test should not present in BBLAYERS)'
  159. continue
  160. logger.info('')
  161. logger.info("Setting up for %s(%s), %s" % (layer['name'], layer['type'],
  162. layer['path']))
  163. missing_dependencies = not add_layer_dependencies(bblayersconf, layer, dep_layers, logger)
  164. if not missing_dependencies:
  165. for additional_layer in additional_layers:
  166. if not add_layer_dependencies(bblayersconf, additional_layer, dep_layers, logger):
  167. missing_dependencies = True
  168. break
  169. if missing_dependencies:
  170. logger.info('Skipping %s due to missing dependencies.' % layer['name'])
  171. results[layer['name']] = None
  172. results_status[layer['name']] = 'SKIPPED (Missing dependencies)'
  173. layers_tested = layers_tested + 1
  174. continue
  175. if any(map(lambda additional_layer: not add_layers(bblayersconf, [additional_layer], logger),
  176. additional_layers)):
  177. logger.info('Skipping %s due to missing additional layers.' % layer['name'])
  178. results[layer['name']] = None
  179. results_status[layer['name']] = 'SKIPPED (Missing additional layers)'
  180. layers_tested = layers_tested + 1
  181. continue
  182. logger.info('Getting initial bitbake variables ...')
  183. td['bbvars'] = get_bb_vars()
  184. logger.info('Getting initial signatures ...')
  185. td['builddir'] = builddir
  186. try:
  187. td['sigs'], td['tunetasks'] = get_signatures(td['builddir'])
  188. except RuntimeError as e:
  189. logger.info(str(e))
  190. results[layer['name']] = None
  191. results_status[layer['name']] = 'FAIL (Generating world signatures)'
  192. layers_tested = layers_tested + 1
  193. continue
  194. td['machines'] = args.machines
  195. if not add_layers(bblayersconf, [layer], logger):
  196. logger.info('Skipping %s ???.' % layer['name'])
  197. results[layer['name']] = None
  198. results_status[layer['name']] = 'SKIPPED (Unknown)'
  199. layers_tested = layers_tested + 1
  200. continue
  201. result = test_layer(td, layer, args.test_software_layer_signatures)
  202. results[layer['name']] = result
  203. results_status[layer['name']] = 'PASS' if results[layer['name']].wasSuccessful() else 'FAIL'
  204. layers_tested = layers_tested + 1
  205. ret = 0
  206. if layers_tested:
  207. logger.info('')
  208. logger.info('Summary of results:')
  209. logger.info('')
  210. for layer_name in results_status:
  211. logger.info('%s ... %s' % (layer_name, results_status[layer_name]))
  212. if not results[layer_name] or not results[layer_name].wasSuccessful():
  213. ret = 2 # ret = 1 used for initialization errors
  214. cleanup_bblayers(None, None)
  215. return ret
  216. if __name__ == '__main__':
  217. try:
  218. ret = main()
  219. except Exception:
  220. ret = 1
  221. import traceback
  222. traceback.print_exc()
  223. sys.exit(ret)