bitbake-whatchanged 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320
  1. #!/usr/bin/env python3
  2. # ex:ts=4:sw=4:sts=4:et
  3. # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
  4. # Copyright (c) 2013 Wind River Systems, Inc.
  5. #
  6. # SPDX-License-Identifier: GPL-2.0-only
  7. #
  8. import os
  9. import sys
  10. import getopt
  11. import shutil
  12. import re
  13. import warnings
  14. import subprocess
  15. import argparse
  16. scripts_path = os.path.abspath(os.path.dirname(os.path.abspath(sys.argv[0])))
  17. lib_path = scripts_path + '/lib'
  18. sys.path = sys.path + [lib_path]
  19. import scriptpath
  20. # Figure out where is the bitbake/lib/bb since we need bb.siggen and bb.process
  21. bitbakepath = scriptpath.add_bitbake_lib_path()
  22. if not bitbakepath:
  23. sys.stderr.write("Unable to find bitbake by searching parent directory of this script or PATH\n")
  24. sys.exit(1)
  25. scriptpath.add_oe_lib_path()
  26. import argparse_oe
  27. import bb.siggen
  28. import bb.process
  29. # Match the stamp's filename
  30. # group(1): PE_PV (may no PE)
  31. # group(2): PR
  32. # group(3): TASK
  33. # group(4): HASH
  34. stamp_re = re.compile("(?P<pv>.*)-(?P<pr>r\d+)\.(?P<task>do_\w+)\.(?P<hash>[^\.]*)")
  35. sigdata_re = re.compile(".*\.sigdata\..*")
  36. def gen_dict(stamps):
  37. """
  38. Generate the dict from the stamps dir.
  39. The output dict format is:
  40. {fake_f: {pn: PN, pv: PV, pr: PR, task: TASK, path: PATH}}
  41. Where:
  42. fake_f: pv + task + hash
  43. path: the path to the stamp file
  44. """
  45. # The member of the sub dict (A "path" will be appended below)
  46. sub_mem = ("pv", "pr", "task")
  47. d = {}
  48. for dirpath, _, files in os.walk(stamps):
  49. for f in files:
  50. # The "bitbake -S" would generate ".sigdata", but no "_setscene".
  51. fake_f = re.sub('_setscene.', '.', f)
  52. fake_f = re.sub('.sigdata', '', fake_f)
  53. subdict = {}
  54. tmp = stamp_re.match(fake_f)
  55. if tmp:
  56. for i in sub_mem:
  57. subdict[i] = tmp.group(i)
  58. if len(subdict) != 0:
  59. pn = os.path.basename(dirpath)
  60. subdict['pn'] = pn
  61. # The path will be used by os.stat() and bb.siggen
  62. subdict['path'] = dirpath + "/" + f
  63. fake_f = tmp.group('pv') + tmp.group('task') + tmp.group('hash')
  64. d[fake_f] = subdict
  65. return d
  66. # Re-construct the dict
  67. def recon_dict(dict_in):
  68. """
  69. The output dict format is:
  70. {pn_task: {pv: PV, pr: PR, path: PATH}}
  71. """
  72. dict_out = {}
  73. for k in dict_in.keys():
  74. subdict = {}
  75. # The key
  76. pn_task = "%s_%s" % (dict_in.get(k).get('pn'), dict_in.get(k).get('task'))
  77. # If more than one stamps are found, use the latest one.
  78. if pn_task in dict_out:
  79. full_path_pre = dict_out.get(pn_task).get('path')
  80. full_path_cur = dict_in.get(k).get('path')
  81. if os.stat(full_path_pre).st_mtime > os.stat(full_path_cur).st_mtime:
  82. continue
  83. subdict['pv'] = dict_in.get(k).get('pv')
  84. subdict['pr'] = dict_in.get(k).get('pr')
  85. subdict['path'] = dict_in.get(k).get('path')
  86. dict_out[pn_task] = subdict
  87. return dict_out
  88. def split_pntask(s):
  89. """
  90. Split the pn_task in to (pn, task) and return it
  91. """
  92. tmp = re.match("(.*)_(do_.*)", s)
  93. return (tmp.group(1), tmp.group(2))
  94. def print_added(d_new = None, d_old = None):
  95. """
  96. Print the newly added tasks
  97. """
  98. added = {}
  99. for k in list(d_new.keys()):
  100. if k not in d_old:
  101. # Add the new one to added dict, and remove it from
  102. # d_new, so the remaining ones are the changed ones
  103. added[k] = d_new.get(k)
  104. del(d_new[k])
  105. if not added:
  106. return 0
  107. # Format the output, the dict format is:
  108. # {pn: task1, task2 ...}
  109. added_format = {}
  110. counter = 0
  111. for k in added.keys():
  112. pn, task = split_pntask(k)
  113. if pn in added_format:
  114. # Append the value
  115. added_format[pn] = "%s %s" % (added_format.get(pn), task)
  116. else:
  117. added_format[pn] = task
  118. counter += 1
  119. print("=== Newly added tasks: (%s tasks)" % counter)
  120. for k in added_format.keys():
  121. print(" %s: %s" % (k, added_format.get(k)))
  122. return counter
  123. def print_vrchanged(d_new = None, d_old = None, vr = None):
  124. """
  125. Print the pv or pr changed tasks.
  126. The arg "vr" is "pv" or "pr"
  127. """
  128. pvchanged = {}
  129. counter = 0
  130. for k in list(d_new.keys()):
  131. if d_new.get(k).get(vr) != d_old.get(k).get(vr):
  132. counter += 1
  133. pn, task = split_pntask(k)
  134. if pn not in pvchanged:
  135. # Format the output, we only print pn (no task) since
  136. # all the tasks would be changed when pn or pr changed,
  137. # the dict format is:
  138. # {pn: pv/pr_old -> pv/pr_new}
  139. pvchanged[pn] = "%s -> %s" % (d_old.get(k).get(vr), d_new.get(k).get(vr))
  140. del(d_new[k])
  141. if not pvchanged:
  142. return 0
  143. print("\n=== %s changed: (%s tasks)" % (vr.upper(), counter))
  144. for k in pvchanged.keys():
  145. print(" %s: %s" % (k, pvchanged.get(k)))
  146. return counter
  147. def print_depchanged(d_new = None, d_old = None, verbose = False):
  148. """
  149. Print the dependency changes
  150. """
  151. depchanged = {}
  152. counter = 0
  153. for k in d_new.keys():
  154. counter += 1
  155. pn, task = split_pntask(k)
  156. if (verbose):
  157. full_path_old = d_old.get(k).get("path")
  158. full_path_new = d_new.get(k).get("path")
  159. # No counter since it is not ready here
  160. if sigdata_re.match(full_path_old) and sigdata_re.match(full_path_new):
  161. output = bb.siggen.compare_sigfiles(full_path_old, full_path_new)
  162. if output:
  163. print("\n=== The verbose changes of %s.%s:" % (pn, task))
  164. print('\n'.join(output))
  165. else:
  166. # Format the output, the format is:
  167. # {pn: task1, task2, ...}
  168. if pn in depchanged:
  169. depchanged[pn] = "%s %s" % (depchanged.get(pn), task)
  170. else:
  171. depchanged[pn] = task
  172. if len(depchanged) > 0:
  173. print("\n=== Dependencies changed: (%s tasks)" % counter)
  174. for k in depchanged.keys():
  175. print(" %s: %s" % (k, depchanged[k]))
  176. return counter
  177. def main():
  178. """
  179. Print what will be done between the current and last builds:
  180. 1) Run "STAMPS_DIR=<path> bitbake -S recipe" to re-generate the stamps
  181. 2) Figure out what are newly added and changed, can't figure out
  182. what are removed since we can't know the previous stamps
  183. clearly, for example, if there are several builds, we can't know
  184. which stamps the last build has used exactly.
  185. 3) Use bb.siggen.compare_sigfiles to diff the old and new stamps
  186. """
  187. parser = argparse_oe.ArgumentParser(usage = """%(prog)s [options] [package ...]
  188. print what will be done between the current and last builds, for example:
  189. $ bitbake core-image-sato
  190. # Edit the recipes
  191. $ bitbake-whatchanged core-image-sato
  192. The changes will be printed"
  193. Note:
  194. The amount of tasks is not accurate when the task is "do_build" since
  195. it usually depends on other tasks.
  196. The "nostamp" task is not included.
  197. """
  198. )
  199. parser.add_argument("recipe", help="recipe to check")
  200. parser.add_argument("-v", "--verbose", help = "print the verbose changes", action = "store_true")
  201. args = parser.parse_args()
  202. # Get the STAMPS_DIR
  203. print("Figuring out the STAMPS_DIR ...")
  204. cmdline = "bitbake -e | sed -ne 's/^STAMPS_DIR=\"\(.*\)\"/\\1/p'"
  205. try:
  206. stampsdir, err = bb.process.run(cmdline)
  207. except:
  208. raise
  209. if not stampsdir:
  210. print("ERROR: No STAMPS_DIR found for '%s'" % args.recipe, file=sys.stderr)
  211. return 2
  212. stampsdir = stampsdir.rstrip("\n")
  213. if not os.path.isdir(stampsdir):
  214. print("ERROR: stamps directory \"%s\" not found!" % stampsdir, file=sys.stderr)
  215. return 2
  216. # The new stamps dir
  217. new_stampsdir = stampsdir + ".bbs"
  218. if os.path.exists(new_stampsdir):
  219. print("ERROR: %s already exists!" % new_stampsdir, file=sys.stderr)
  220. return 2
  221. try:
  222. # Generate the new stamps dir
  223. print("Generating the new stamps ... (need several minutes)")
  224. cmdline = "STAMPS_DIR=%s bitbake -S none %s" % (new_stampsdir, args.recipe)
  225. # FIXME
  226. # The "bitbake -S" may fail, not fatal error, the stamps will still
  227. # be generated, this might be a bug of "bitbake -S".
  228. try:
  229. bb.process.run(cmdline)
  230. except Exception as exc:
  231. print(exc)
  232. # The dict for the new and old stamps.
  233. old_dict = gen_dict(stampsdir)
  234. new_dict = gen_dict(new_stampsdir)
  235. # Remove the same one from both stamps.
  236. cnt_unchanged = 0
  237. for k in list(new_dict.keys()):
  238. if k in old_dict:
  239. cnt_unchanged += 1
  240. del(new_dict[k])
  241. del(old_dict[k])
  242. # Re-construct the dict to easily find out what is added or changed.
  243. # The dict format is:
  244. # {pn_task: {pv: PV, pr: PR, path: PATH}}
  245. new_recon = recon_dict(new_dict)
  246. old_recon = recon_dict(old_dict)
  247. del new_dict
  248. del old_dict
  249. # Figure out what are changed, the new_recon would be changed
  250. # by the print_xxx function.
  251. # Newly added
  252. cnt_added = print_added(new_recon, old_recon)
  253. # PV (including PE) and PR changed
  254. # Let the bb.siggen handle them if verbose
  255. cnt_rv = {}
  256. if not args.verbose:
  257. for i in ('pv', 'pr'):
  258. cnt_rv[i] = print_vrchanged(new_recon, old_recon, i)
  259. # Dependencies changed (use bitbake-diffsigs)
  260. cnt_dep = print_depchanged(new_recon, old_recon, args.verbose)
  261. total_changed = cnt_added + (cnt_rv.get('pv') or 0) + (cnt_rv.get('pr') or 0) + cnt_dep
  262. print("\n=== Summary: (%s changed, %s unchanged)" % (total_changed, cnt_unchanged))
  263. if args.verbose:
  264. print("Newly added: %s\nDependencies changed: %s\n" % \
  265. (cnt_added, cnt_dep))
  266. else:
  267. print("Newly added: %s\nPV changed: %s\nPR changed: %s\nDependencies changed: %s\n" % \
  268. (cnt_added, cnt_rv.get('pv') or 0, cnt_rv.get('pr') or 0, cnt_dep))
  269. except:
  270. print("ERROR occurred!")
  271. raise
  272. finally:
  273. # Remove the newly generated stamps dir
  274. if os.path.exists(new_stampsdir):
  275. print("Removing the newly generated stamps dir ...")
  276. shutil.rmtree(new_stampsdir)
  277. if __name__ == "__main__":
  278. sys.exit(main())