copy_buildsystem.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295
  1. #
  2. # SPDX-License-Identifier: GPL-2.0-only
  3. #
  4. # This class should provide easy access to the different aspects of the
  5. # buildsystem such as layers, bitbake location, etc.
  6. #
  7. # SDK_LAYERS_EXCLUDE: Layers which will be excluded from SDK layers.
  8. # SDK_LAYERS_EXCLUDE_PATTERN: The simiar to SDK_LAYERS_EXCLUDE, this supports
  9. # python regular expression, use space as separator,
  10. # e.g.: ".*-downloads closed-.*"
  11. #
  12. import stat
  13. import shutil
  14. def _smart_copy(src, dest):
  15. import subprocess
  16. # smart_copy will choose the correct function depending on whether the
  17. # source is a file or a directory.
  18. mode = os.stat(src).st_mode
  19. if stat.S_ISDIR(mode):
  20. bb.utils.mkdirhier(dest)
  21. cmd = "tar --exclude='.git' --xattrs --xattrs-include='*' -chf - -C %s -p . \
  22. | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dest)
  23. subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
  24. else:
  25. shutil.copyfile(src, dest)
  26. shutil.copymode(src, dest)
  27. class BuildSystem(object):
  28. def __init__(self, context, d):
  29. self.d = d
  30. self.context = context
  31. self.layerdirs = [os.path.abspath(pth) for pth in d.getVar('BBLAYERS').split()]
  32. self.layers_exclude = (d.getVar('SDK_LAYERS_EXCLUDE') or "").split()
  33. self.layers_exclude_pattern = d.getVar('SDK_LAYERS_EXCLUDE_PATTERN')
  34. def copy_bitbake_and_layers(self, destdir, workspace_name=None):
  35. import re
  36. # Copy in all metadata layers + bitbake (as repositories)
  37. copied_corebase = None
  38. layers_copied = []
  39. bb.utils.mkdirhier(destdir)
  40. layers = list(self.layerdirs)
  41. corebase = os.path.abspath(self.d.getVar('COREBASE'))
  42. layers.append(corebase)
  43. # Get relationship between TOPDIR and COREBASE
  44. # Layers should respect it
  45. corebase_relative = os.path.dirname(os.path.relpath(os.path.abspath(self.d.getVar('TOPDIR')), corebase))
  46. # The bitbake build system uses the meta-skeleton layer as a layout
  47. # for common recipies, e.g: the recipetool script to create kernel recipies
  48. # Add the meta-skeleton layer to be included as part of the eSDK installation
  49. layers.append(os.path.join(corebase, 'meta-skeleton'))
  50. # Exclude layers
  51. for layer_exclude in self.layers_exclude:
  52. if layer_exclude in layers:
  53. bb.note('Excluded %s from sdk layers since it is in SDK_LAYERS_EXCLUDE' % layer_exclude)
  54. layers.remove(layer_exclude)
  55. if self.layers_exclude_pattern:
  56. layers_cp = layers[:]
  57. for pattern in self.layers_exclude_pattern.split():
  58. for layer in layers_cp:
  59. if re.match(pattern, layer):
  60. bb.note('Excluded %s from sdk layers since matched SDK_LAYERS_EXCLUDE_PATTERN' % layer)
  61. layers.remove(layer)
  62. workspace_newname = workspace_name
  63. if workspace_newname:
  64. layernames = [os.path.basename(layer) for layer in layers]
  65. extranum = 0
  66. while workspace_newname in layernames:
  67. extranum += 1
  68. workspace_newname = '%s-%d' % (workspace_name, extranum)
  69. corebase_files = self.d.getVar('COREBASE_FILES').split()
  70. corebase_files = [corebase + '/' +x for x in corebase_files]
  71. # Make sure bitbake goes in
  72. bitbake_dir = bb.__file__.rsplit('/', 3)[0]
  73. corebase_files.append(bitbake_dir)
  74. for layer in layers:
  75. layerconf = os.path.join(layer, 'conf', 'layer.conf')
  76. layernewname = os.path.basename(layer)
  77. workspace = False
  78. if os.path.exists(layerconf):
  79. with open(layerconf, 'r') as f:
  80. if f.readline().startswith("# ### workspace layer auto-generated by devtool ###"):
  81. if workspace_newname:
  82. layernewname = workspace_newname
  83. workspace = True
  84. else:
  85. bb.plain("NOTE: Excluding local workspace layer %s from %s" % (layer, self.context))
  86. continue
  87. # If the layer was already under corebase, leave it there
  88. # since layers such as meta have issues when moved.
  89. layerdestpath = destdir
  90. if corebase == os.path.dirname(layer):
  91. layerdestpath += '/' + os.path.basename(corebase)
  92. else:
  93. layer_relative = os.path.relpath(layer, corebase)
  94. if os.path.dirname(layer_relative) == corebase_relative:
  95. layer_relative = os.path.dirname(corebase_relative) + '/' + layernewname
  96. layer_relative = os.path.basename(corebase) + '/' + layer_relative
  97. if os.path.dirname(layer_relative) != layernewname:
  98. layerdestpath += '/' + os.path.dirname(layer_relative)
  99. layerdestpath += '/' + layernewname
  100. layer_relative = os.path.relpath(layerdestpath,
  101. destdir)
  102. # Treat corebase as special since it typically will contain
  103. # build directories or other custom items.
  104. if corebase == layer:
  105. copied_corebase = layer_relative
  106. bb.utils.mkdirhier(layerdestpath)
  107. for f in corebase_files:
  108. f_basename = os.path.basename(f)
  109. destname = os.path.join(layerdestpath, f_basename)
  110. _smart_copy(f, destname)
  111. else:
  112. layers_copied.append(layer_relative)
  113. if os.path.exists(os.path.join(layerdestpath, 'conf/layer.conf')):
  114. bb.note("Skipping layer %s, already handled" % layer)
  115. else:
  116. _smart_copy(layer, layerdestpath)
  117. if workspace:
  118. # Make some adjustments original workspace layer
  119. # Drop sources (recipe tasks will be locked, so we don't need them)
  120. srcdir = os.path.join(layerdestpath, 'sources')
  121. if os.path.isdir(srcdir):
  122. shutil.rmtree(srcdir)
  123. # Drop all bbappends except the one for the image the SDK is being built for
  124. # (because of externalsrc, the workspace bbappends will interfere with the
  125. # locked signatures if present, and we don't need them anyway)
  126. image_bbappend = os.path.splitext(os.path.basename(self.d.getVar('FILE')))[0] + '.bbappend'
  127. appenddir = os.path.join(layerdestpath, 'appends')
  128. if os.path.isdir(appenddir):
  129. for fn in os.listdir(appenddir):
  130. if fn == image_bbappend:
  131. continue
  132. else:
  133. os.remove(os.path.join(appenddir, fn))
  134. # Drop README
  135. readme = os.path.join(layerdestpath, 'README')
  136. if os.path.exists(readme):
  137. os.remove(readme)
  138. # Filter out comments in layer.conf and change layer name
  139. layerconf = os.path.join(layerdestpath, 'conf', 'layer.conf')
  140. with open(layerconf, 'r') as f:
  141. origlines = f.readlines()
  142. with open(layerconf, 'w') as f:
  143. for line in origlines:
  144. if line.startswith('#'):
  145. continue
  146. line = line.replace('workspacelayer', workspace_newname)
  147. f.write(line)
  148. # meta-skeleton layer is added as part of the build system
  149. # but not as a layer included in the build, therefore it is
  150. # not reported to the function caller.
  151. for layer in layers_copied:
  152. if layer.endswith('/meta-skeleton'):
  153. layers_copied.remove(layer)
  154. break
  155. return copied_corebase, layers_copied
  156. def generate_locked_sigs(sigfile, d):
  157. bb.utils.mkdirhier(os.path.dirname(sigfile))
  158. depd = d.getVar('BB_TASKDEPDATA', False)
  159. tasks = ['%s:%s' % (v[2], v[1]) for v in depd.values()]
  160. bb.parse.siggen.dump_lockedsigs(sigfile, tasks)
  161. def prune_lockedsigs(excluded_tasks, excluded_targets, lockedsigs, onlynative, pruned_output):
  162. with open(lockedsigs, 'r') as infile:
  163. bb.utils.mkdirhier(os.path.dirname(pruned_output))
  164. with open(pruned_output, 'w') as f:
  165. invalue = False
  166. for line in infile:
  167. if invalue:
  168. if line.endswith('\\\n'):
  169. splitval = line.strip().split(':')
  170. if not splitval[1] in excluded_tasks and not splitval[0] in excluded_targets:
  171. if onlynative:
  172. if 'nativesdk' in splitval[0]:
  173. f.write(line)
  174. else:
  175. f.write(line)
  176. else:
  177. f.write(line)
  178. invalue = False
  179. elif line.startswith('SIGGEN_LOCKEDSIGS'):
  180. invalue = True
  181. f.write(line)
  182. def merge_lockedsigs(copy_tasks, lockedsigs_main, lockedsigs_extra, merged_output, copy_output=None):
  183. merged = {}
  184. arch_order = []
  185. with open(lockedsigs_main, 'r') as f:
  186. invalue = None
  187. for line in f:
  188. if invalue:
  189. if line.endswith('\\\n'):
  190. merged[invalue].append(line)
  191. else:
  192. invalue = None
  193. elif line.startswith('SIGGEN_LOCKEDSIGS_t-'):
  194. invalue = line[18:].split('=', 1)[0].rstrip()
  195. merged[invalue] = []
  196. arch_order.append(invalue)
  197. with open(lockedsigs_extra, 'r') as f:
  198. invalue = None
  199. tocopy = {}
  200. for line in f:
  201. if invalue:
  202. if line.endswith('\\\n'):
  203. if not line in merged[invalue]:
  204. target, task = line.strip().split(':')[:2]
  205. if not copy_tasks or task in copy_tasks:
  206. tocopy[invalue].append(line)
  207. merged[invalue].append(line)
  208. else:
  209. invalue = None
  210. elif line.startswith('SIGGEN_LOCKEDSIGS_t-'):
  211. invalue = line[18:].split('=', 1)[0].rstrip()
  212. if not invalue in merged:
  213. merged[invalue] = []
  214. arch_order.append(invalue)
  215. tocopy[invalue] = []
  216. def write_sigs_file(fn, types, sigs):
  217. fulltypes = []
  218. bb.utils.mkdirhier(os.path.dirname(fn))
  219. with open(fn, 'w') as f:
  220. for typename in types:
  221. lines = sigs[typename]
  222. if lines:
  223. f.write('SIGGEN_LOCKEDSIGS_%s = "\\\n' % typename)
  224. for line in lines:
  225. f.write(line)
  226. f.write(' "\n')
  227. fulltypes.append(typename)
  228. f.write('SIGGEN_LOCKEDSIGS_TYPES = "%s"\n' % ' '.join(fulltypes))
  229. if copy_output:
  230. write_sigs_file(copy_output, list(tocopy.keys()), tocopy)
  231. if merged_output:
  232. write_sigs_file(merged_output, arch_order, merged)
  233. def create_locked_sstate_cache(lockedsigs, input_sstate_cache, output_sstate_cache, d, fixedlsbstring="", filterfile=None):
  234. import shutil
  235. bb.note('Generating sstate-cache...')
  236. nativelsbstring = d.getVar('NATIVELSBSTRING')
  237. bb.process.run("gen-lockedsig-cache %s %s %s %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache, nativelsbstring, filterfile or ''))
  238. if fixedlsbstring and nativelsbstring != fixedlsbstring:
  239. nativedir = output_sstate_cache + '/' + nativelsbstring
  240. if os.path.isdir(nativedir):
  241. destdir = os.path.join(output_sstate_cache, fixedlsbstring)
  242. for root, _, files in os.walk(nativedir):
  243. for fn in files:
  244. src = os.path.join(root, fn)
  245. dest = os.path.join(destdir, os.path.relpath(src, nativedir))
  246. if os.path.exists(dest):
  247. # Already exists, and it'll be the same file, so just delete it
  248. os.unlink(src)
  249. else:
  250. bb.utils.mkdirhier(os.path.dirname(dest))
  251. shutil.move(src, dest)
  252. def check_sstate_task_list(d, targets, filteroutfile, cmdprefix='', cwd=None, logfile=None):
  253. import subprocess
  254. bb.note('Generating sstate task list...')
  255. if not cwd:
  256. cwd = os.getcwd()
  257. if logfile:
  258. logparam = '-l %s' % logfile
  259. else:
  260. logparam = ''
  261. cmd = "%sBB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 oe-check-sstate %s -s -o %s %s" % (cmdprefix, targets, filteroutfile, logparam)
  262. env = dict(d.getVar('BB_ORIGENV', False))
  263. env.pop('BUILDDIR', '')
  264. env.pop('BBPATH', '')
  265. pathitems = env['PATH'].split(':')
  266. env['PATH'] = ':'.join([item for item in pathitems if not item.endswith('/bitbake/bin')])
  267. bb.process.run(cmd, stderr=subprocess.STDOUT, env=env, cwd=cwd, executable='/bin/bash')