toaster.bbclass 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388
  1. #
  2. # Toaster helper class
  3. #
  4. # Copyright (C) 2013 Intel Corporation
  5. #
  6. # Released under the MIT license (see COPYING.MIT)
  7. #
  8. # This bbclass is designed to extract data used by OE-Core during the build process,
  9. # for recording in the Toaster system.
  10. # The data access is synchronous, preserving the build data integrity across
  11. # different builds.
  12. #
  13. # The data is transferred through the event system, using the MetadataEvent objects.
  14. #
  15. # The model is to enable the datadump functions as postfuncs, and have the dump
  16. # executed after the real taskfunc has been executed. This prevents task signature changing
  17. # is toaster is enabled or not. Build performance is not affected if Toaster is not enabled.
  18. #
  19. # To enable, use INHERIT in local.conf:
  20. #
  21. # INHERIT += "toaster"
  22. #
  23. #
  24. #
  25. #
  26. # Find and dump layer info when we got the layers parsed
  27. python toaster_layerinfo_dumpdata() {
  28. import subprocess
  29. def _get_git_branch(layer_path):
  30. branch = subprocess.Popen("git symbolic-ref HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0]
  31. branch = branch.decode('utf-8')
  32. branch = branch.replace('refs/heads/', '').rstrip()
  33. return branch
  34. def _get_git_revision(layer_path):
  35. revision = subprocess.Popen("git rev-parse HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0].rstrip()
  36. return revision
  37. def _get_url_map_name(layer_name):
  38. """ Some layers have a different name on openembedded.org site,
  39. this method returns the correct name to use in the URL
  40. """
  41. url_name = layer_name
  42. url_mapping = {'meta': 'openembedded-core'}
  43. for key in url_mapping.keys():
  44. if key == layer_name:
  45. url_name = url_mapping[key]
  46. return url_name
  47. def _get_layer_version_information(layer_path):
  48. layer_version_info = {}
  49. layer_version_info['branch'] = _get_git_branch(layer_path)
  50. layer_version_info['commit'] = _get_git_revision(layer_path)
  51. layer_version_info['priority'] = 0
  52. return layer_version_info
  53. def _get_layer_dict(layer_path):
  54. layer_info = {}
  55. layer_name = layer_path.split('/')[-1]
  56. layer_url = 'http://layers.openembedded.org/layerindex/layer/{layer}/'
  57. layer_url_name = _get_url_map_name(layer_name)
  58. layer_info['name'] = layer_url_name
  59. layer_info['local_path'] = layer_path
  60. layer_info['layer_index_url'] = layer_url.format(layer=layer_url_name)
  61. layer_info['version'] = _get_layer_version_information(layer_path)
  62. return layer_info
  63. bblayers = e.data.getVar("BBLAYERS")
  64. llayerinfo = {}
  65. for layer in { l for l in bblayers.strip().split(" ") if len(l) }:
  66. llayerinfo[layer] = _get_layer_dict(layer)
  67. bb.event.fire(bb.event.MetadataEvent("LayerInfo", llayerinfo), e.data)
  68. }
  69. # Dump package file info data
  70. def _toaster_load_pkgdatafile(dirpath, filepath):
  71. import json
  72. import re
  73. pkgdata = {}
  74. with open(os.path.join(dirpath, filepath), "r") as fin:
  75. for line in fin:
  76. try:
  77. kn, kv = line.strip().split(": ", 1)
  78. m = re.match(r"^PKG_([^A-Z:]*)", kn)
  79. if m:
  80. pkgdata['OPKGN'] = m.group(1)
  81. kn = "_".join([x for x in kn.split("_") if x.isupper()])
  82. pkgdata[kn] = kv.strip()
  83. if kn == 'FILES_INFO':
  84. pkgdata[kn] = json.loads(kv)
  85. except ValueError:
  86. pass # ignore lines without valid key: value pairs
  87. return pkgdata
  88. def _toaster_dumpdata(pkgdatadir, d):
  89. """
  90. Dumps the data about the packages created by a recipe
  91. """
  92. # No need to try and dumpdata if the recipe isn't generating packages
  93. if not d.getVar('PACKAGES'):
  94. return
  95. lpkgdata = {}
  96. datadir = os.path.join(pkgdatadir, 'runtime')
  97. # scan and send data for each generated package
  98. if os.path.exists(datadir):
  99. for datafile in os.listdir(datadir):
  100. if not datafile.endswith('.packaged'):
  101. lpkgdata = _toaster_load_pkgdatafile(datadir, datafile)
  102. # Fire an event containing the pkg data
  103. bb.event.fire(bb.event.MetadataEvent("SinglePackageInfo", lpkgdata), d)
  104. python toaster_package_dumpdata() {
  105. _toaster_dumpdata(d.getVar('PKGDESTWORK'), d)
  106. }
  107. python toaster_packagedata_dumpdata() {
  108. # This path needs to match do_packagedata[sstate-inputdirs]
  109. _toaster_dumpdata(os.path.join(d.getVar('WORKDIR'), 'pkgdata-pdata-input'), d)
  110. }
  111. # 2. Dump output image files information
  112. python toaster_artifact_dumpdata() {
  113. """
  114. Dump data about SDK variables
  115. """
  116. event_data = {
  117. "TOOLCHAIN_OUTPUTNAME": d.getVar("TOOLCHAIN_OUTPUTNAME")
  118. }
  119. bb.event.fire(bb.event.MetadataEvent("SDKArtifactInfo", event_data), d)
  120. }
  121. # collect list of buildstats files based on fired events; when the build completes, collect all stats and fire an event with collected data
  122. python toaster_collect_task_stats() {
  123. import bb.build
  124. import bb.event
  125. import bb.data
  126. import bb.utils
  127. import os
  128. if not e.data.getVar('BUILDSTATS_BASE'):
  129. return # if we don't have buildstats, we cannot collect stats
  130. toaster_statlist_file = os.path.join(e.data.getVar('BUILDSTATS_BASE'), "toasterstatlist")
  131. def stat_to_float(value):
  132. return float(value.strip('% \n\r'))
  133. def _append_read_list(v):
  134. lock = bb.utils.lockfile(e.data.expand("${TOPDIR}/toaster.lock"), False, True)
  135. with open(toaster_statlist_file, "a") as fout:
  136. taskdir = e.data.expand("${BUILDSTATS_BASE}/${BUILDNAME}/${PF}")
  137. fout.write("%s::%s::%s::%s\n" % (e.taskfile, e.taskname, os.path.join(taskdir, e.task), e.data.expand("${PN}")))
  138. bb.utils.unlockfile(lock)
  139. def _read_stats(filename):
  140. # seconds
  141. cpu_time_user = 0
  142. cpu_time_system = 0
  143. # bytes
  144. disk_io_read = 0
  145. disk_io_write = 0
  146. started = 0
  147. ended = 0
  148. taskname = ''
  149. statinfo = {}
  150. with open(filename, 'r') as task_bs:
  151. for line in task_bs.readlines():
  152. k,v = line.strip().split(": ", 1)
  153. statinfo[k] = v
  154. if "Started" in statinfo:
  155. started = stat_to_float(statinfo["Started"])
  156. if "Ended" in statinfo:
  157. ended = stat_to_float(statinfo["Ended"])
  158. if "Child rusage ru_utime" in statinfo:
  159. cpu_time_user = cpu_time_user + stat_to_float(statinfo["Child rusage ru_utime"])
  160. if "Child rusage ru_stime" in statinfo:
  161. cpu_time_system = cpu_time_system + stat_to_float(statinfo["Child rusage ru_stime"])
  162. if "IO write_bytes" in statinfo:
  163. write_bytes = int(statinfo["IO write_bytes"].strip('% \n\r'))
  164. disk_io_write = disk_io_write + write_bytes
  165. if "IO read_bytes" in statinfo:
  166. read_bytes = int(statinfo["IO read_bytes"].strip('% \n\r'))
  167. disk_io_read = disk_io_read + read_bytes
  168. return {
  169. 'stat_file': filename,
  170. 'cpu_time_user': cpu_time_user,
  171. 'cpu_time_system': cpu_time_system,
  172. 'disk_io_read': disk_io_read,
  173. 'disk_io_write': disk_io_write,
  174. 'started': started,
  175. 'ended': ended
  176. }
  177. if isinstance(e, (bb.build.TaskSucceeded, bb.build.TaskFailed)):
  178. _append_read_list(e)
  179. pass
  180. if isinstance(e, bb.event.BuildCompleted) and os.path.exists(toaster_statlist_file):
  181. events = []
  182. with open(toaster_statlist_file, "r") as fin:
  183. for line in fin:
  184. (taskfile, taskname, filename, recipename) = line.strip().split("::")
  185. stats = _read_stats(filename)
  186. events.append((taskfile, taskname, stats, recipename))
  187. bb.event.fire(bb.event.MetadataEvent("BuildStatsList", events), e.data)
  188. os.unlink(toaster_statlist_file)
  189. }
  190. # dump relevant build history data as an event when the build is completed
  191. python toaster_buildhistory_dump() {
  192. import re
  193. BUILDHISTORY_DIR = e.data.expand("${TOPDIR}/buildhistory")
  194. BUILDHISTORY_DIR_IMAGE_BASE = e.data.expand("%s/images/${MACHINE_ARCH}/${TCLIBC}/"% BUILDHISTORY_DIR)
  195. pkgdata_dir = e.data.getVar("PKGDATA_DIR")
  196. # scan the build targets for this build
  197. images = {}
  198. allpkgs = {}
  199. files = {}
  200. for target in e._pkgs:
  201. target = target.split(':')[0] # strip ':<task>' suffix from the target
  202. installed_img_path = e.data.expand(os.path.join(BUILDHISTORY_DIR_IMAGE_BASE, target))
  203. if os.path.exists(installed_img_path):
  204. images[target] = {}
  205. files[target] = {}
  206. files[target]['dirs'] = []
  207. files[target]['syms'] = []
  208. files[target]['files'] = []
  209. with open("%s/installed-package-sizes.txt" % installed_img_path, "r") as fin:
  210. for line in fin:
  211. line = line.rstrip(";")
  212. psize, punit, pname = line.split()
  213. # this size is "installed-size" as it measures how much space it takes on disk
  214. images[target][pname.strip()] = {'size':int(psize)*1024, 'depends' : []}
  215. with open("%s/depends.dot" % installed_img_path, "r") as fin:
  216. p = re.compile(r'\s*"(?P<name>[^"]+)"\s*->\s*"(?P<dep>[^"]+)"(?P<rec>.*?\[style=dotted\])?')
  217. for line in fin:
  218. m = p.match(line)
  219. if not m:
  220. continue
  221. pname = m.group('name')
  222. dependsname = m.group('dep')
  223. deptype = 'recommends' if m.group('rec') else 'depends'
  224. # If RPM is used for packaging, then there may be
  225. # dependencies such as "/bin/sh", which will confuse
  226. # _toaster_load_pkgdatafile() later on. While at it, ignore
  227. # any dependencies that contain parentheses, e.g.,
  228. # "libc.so.6(GLIBC_2.7)".
  229. if dependsname.startswith('/') or '(' in dependsname:
  230. continue
  231. if not pname in images[target]:
  232. images[target][pname] = {'size': 0, 'depends' : []}
  233. if not dependsname in images[target]:
  234. images[target][dependsname] = {'size': 0, 'depends' : []}
  235. images[target][pname]['depends'].append((dependsname, deptype))
  236. # files-in-image.txt is only generated if an image file is created,
  237. # so the file entries ('syms', 'dirs', 'files') for a target will be
  238. # empty for rootfs builds and other "image" tasks which don't
  239. # produce image files
  240. # (e.g. "bitbake core-image-minimal -c populate_sdk")
  241. files_in_image_path = "%s/files-in-image.txt" % installed_img_path
  242. if os.path.exists(files_in_image_path):
  243. with open(files_in_image_path, "r") as fin:
  244. for line in fin:
  245. lc = [ x for x in line.strip().split(" ") if len(x) > 0 ]
  246. if lc[0].startswith("l"):
  247. files[target]['syms'].append(lc)
  248. elif lc[0].startswith("d"):
  249. files[target]['dirs'].append(lc)
  250. else:
  251. files[target]['files'].append(lc)
  252. for pname in images[target]:
  253. if not pname in allpkgs:
  254. try:
  255. pkgdata = _toaster_load_pkgdatafile("%s/runtime-reverse/" % pkgdata_dir, pname)
  256. except IOError as err:
  257. if err.errno == 2:
  258. # We expect this e.g. for RRECOMMENDS that are unsatisfied at runtime
  259. continue
  260. else:
  261. raise
  262. allpkgs[pname] = pkgdata
  263. data = { 'pkgdata' : allpkgs, 'imgdata' : images, 'filedata' : files }
  264. bb.event.fire(bb.event.MetadataEvent("ImagePkgList", data), e.data)
  265. }
  266. # get list of artifacts from sstate manifest
  267. python toaster_artifacts() {
  268. if e.taskname in ["do_deploy", "do_image_complete", "do_populate_sdk", "do_populate_sdk_ext"]:
  269. d2 = d.createCopy()
  270. d2.setVar('FILE', e.taskfile)
  271. # Use 'stamp-extra-info' if present, else use workaround
  272. # to determine 'SSTATE_MANMACH'
  273. extrainf = d2.getVarFlag(e.taskname, 'stamp-extra-info')
  274. if extrainf:
  275. d2.setVar('SSTATE_MANMACH', extrainf)
  276. else:
  277. if "do_populate_sdk" == e.taskname:
  278. d2.setVar('SSTATE_MANMACH', d2.expand("${MACHINE}${SDKMACHINE}"))
  279. else:
  280. d2.setVar('SSTATE_MANMACH', d2.expand("${MACHINE}"))
  281. manifest = oe.sstatesig.sstate_get_manifest_filename(e.taskname[3:], d2)[0]
  282. if os.access(manifest, os.R_OK):
  283. with open(manifest) as fmanifest:
  284. artifacts = [fname.strip() for fname in fmanifest]
  285. data = {"task": e.taskid, "artifacts": artifacts}
  286. bb.event.fire(bb.event.MetadataEvent("TaskArtifacts", data), d2)
  287. }
  288. # set event handlers
  289. addhandler toaster_layerinfo_dumpdata
  290. toaster_layerinfo_dumpdata[eventmask] = "bb.event.TreeDataPreparationCompleted"
  291. addhandler toaster_collect_task_stats
  292. toaster_collect_task_stats[eventmask] = "bb.event.BuildCompleted bb.build.TaskSucceeded bb.build.TaskFailed"
  293. addhandler toaster_buildhistory_dump
  294. toaster_buildhistory_dump[eventmask] = "bb.event.BuildCompleted"
  295. addhandler toaster_artifacts
  296. toaster_artifacts[eventmask] = "bb.runqueue.runQueueTaskSkipped bb.runqueue.runQueueTaskCompleted"
  297. do_packagedata_setscene[postfuncs] += "toaster_packagedata_dumpdata "
  298. do_packagedata_setscene[vardepsexclude] += "toaster_packagedata_dumpdata "
  299. do_package[postfuncs] += "toaster_package_dumpdata "
  300. do_package[vardepsexclude] += "toaster_package_dumpdata "
  301. #do_populate_sdk[postfuncs] += "toaster_artifact_dumpdata "
  302. #do_populate_sdk[vardepsexclude] += "toaster_artifact_dumpdata "
  303. #do_populate_sdk_ext[postfuncs] += "toaster_artifact_dumpdata "
  304. #do_populate_sdk_ext[vardepsexclude] += "toaster_artifact_dumpdata "