bitbake-whatchanged 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339
  1. #!/usr/bin/env python
  2. # ex:ts=4:sw=4:sts=4:et
  3. # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
  4. # Copyright (c) 2013 Wind River Systems, Inc.
  5. #
  6. # This program is free software; you can redistribute it and/or modify
  7. # it under the terms of the GNU General Public License version 2 as
  8. # published by the Free Software Foundation.
  9. #
  10. # This program is distributed in the hope that it will be useful,
  11. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  13. # See the GNU General Public License for more details.
  14. #
  15. # You should have received a copy of the GNU General Public License
  16. # along with this program; if not, write to the Free Software
  17. # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  18. from __future__ import print_function
  19. import os
  20. import sys
  21. import getopt
  22. import shutil
  23. import re
  24. import warnings
  25. import subprocess
  26. from optparse import OptionParser
  27. scripts_path = os.path.abspath(os.path.dirname(os.path.abspath(sys.argv[0])))
  28. lib_path = scripts_path + '/lib'
  29. sys.path = sys.path + [lib_path]
  30. import scriptpath
  31. # Figure out where is the bitbake/lib/bb since we need bb.siggen and bb.process
  32. bitbakepath = scriptpath.add_bitbake_lib_path()
  33. if not bitbakepath:
  34. sys.stderr.write("Unable to find bitbake by searching parent directory of this script or PATH\n")
  35. sys.exit(1)
  36. import bb.siggen
  37. import bb.process
  38. # Match the stamp's filename
  39. # group(1): PE_PV (may no PE)
  40. # group(2): PR
  41. # group(3): TASK
  42. # group(4): HASH
  43. stamp_re = re.compile("(?P<pv>.*)-(?P<pr>r\d+)\.(?P<task>do_\w+)\.(?P<hash>[^\.]*)")
  44. sigdata_re = re.compile(".*\.sigdata\..*")
  45. def gen_dict(stamps):
  46. """
  47. Generate the dict from the stamps dir.
  48. The output dict format is:
  49. {fake_f: {pn: PN, pv: PV, pr: PR, task: TASK, path: PATH}}
  50. Where:
  51. fake_f: pv + task + hash
  52. path: the path to the stamp file
  53. """
  54. # The member of the sub dict (A "path" will be appended below)
  55. sub_mem = ("pv", "pr", "task")
  56. d = {}
  57. for dirpath, _, files in os.walk(stamps):
  58. for f in files:
  59. # The "bitbake -S" would generate ".sigdata", but no "_setscene".
  60. fake_f = re.sub('_setscene.', '.', f)
  61. fake_f = re.sub('.sigdata', '', fake_f)
  62. subdict = {}
  63. tmp = stamp_re.match(fake_f)
  64. if tmp:
  65. for i in sub_mem:
  66. subdict[i] = tmp.group(i)
  67. if len(subdict) != 0:
  68. pn = os.path.basename(dirpath)
  69. subdict['pn'] = pn
  70. # The path will be used by os.stat() and bb.siggen
  71. subdict['path'] = dirpath + "/" + f
  72. fake_f = tmp.group('pv') + tmp.group('task') + tmp.group('hash')
  73. d[fake_f] = subdict
  74. return d
  75. # Re-construct the dict
  76. def recon_dict(dict_in):
  77. """
  78. The output dict format is:
  79. {pn_task: {pv: PV, pr: PR, path: PATH}}
  80. """
  81. dict_out = {}
  82. for k in dict_in.keys():
  83. subdict = {}
  84. # The key
  85. pn_task = "%s_%s" % (dict_in.get(k).get('pn'), dict_in.get(k).get('task'))
  86. # If more than one stamps are found, use the latest one.
  87. if pn_task in dict_out:
  88. full_path_pre = dict_out.get(pn_task).get('path')
  89. full_path_cur = dict_in.get(k).get('path')
  90. if os.stat(full_path_pre).st_mtime > os.stat(full_path_cur).st_mtime:
  91. continue
  92. subdict['pv'] = dict_in.get(k).get('pv')
  93. subdict['pr'] = dict_in.get(k).get('pr')
  94. subdict['path'] = dict_in.get(k).get('path')
  95. dict_out[pn_task] = subdict
  96. return dict_out
  97. def split_pntask(s):
  98. """
  99. Split the pn_task in to (pn, task) and return it
  100. """
  101. tmp = re.match("(.*)_(do_.*)", s)
  102. return (tmp.group(1), tmp.group(2))
  103. def print_added(d_new = None, d_old = None):
  104. """
  105. Print the newly added tasks
  106. """
  107. added = {}
  108. for k in d_new.keys():
  109. if k not in d_old:
  110. # Add the new one to added dict, and remove it from
  111. # d_new, so the remaining ones are the changed ones
  112. added[k] = d_new.get(k)
  113. del(d_new[k])
  114. if not added:
  115. return 0
  116. # Format the output, the dict format is:
  117. # {pn: task1, task2 ...}
  118. added_format = {}
  119. counter = 0
  120. for k in added.keys():
  121. pn, task = split_pntask(k)
  122. if pn in added_format:
  123. # Append the value
  124. added_format[pn] = "%s %s" % (added_format.get(pn), task)
  125. else:
  126. added_format[pn] = task
  127. counter += 1
  128. print("=== Newly added tasks: (%s tasks)" % counter)
  129. for k in added_format.keys():
  130. print(" %s: %s" % (k, added_format.get(k)))
  131. return counter
  132. def print_vrchanged(d_new = None, d_old = None, vr = None):
  133. """
  134. Print the pv or pr changed tasks.
  135. The arg "vr" is "pv" or "pr"
  136. """
  137. pvchanged = {}
  138. counter = 0
  139. for k in d_new.keys():
  140. if d_new.get(k).get(vr) != d_old.get(k).get(vr):
  141. counter += 1
  142. pn, task = split_pntask(k)
  143. if pn not in pvchanged:
  144. # Format the output, we only print pn (no task) since
  145. # all the tasks would be changed when pn or pr changed,
  146. # the dict format is:
  147. # {pn: pv/pr_old -> pv/pr_new}
  148. pvchanged[pn] = "%s -> %s" % (d_old.get(k).get(vr), d_new.get(k).get(vr))
  149. del(d_new[k])
  150. if not pvchanged:
  151. return 0
  152. print("\n=== %s changed: (%s tasks)" % (vr.upper(), counter))
  153. for k in pvchanged.keys():
  154. print(" %s: %s" % (k, pvchanged.get(k)))
  155. return counter
  156. def print_depchanged(d_new = None, d_old = None, verbose = False):
  157. """
  158. Print the dependency changes
  159. """
  160. depchanged = {}
  161. counter = 0
  162. for k in d_new.keys():
  163. counter += 1
  164. pn, task = split_pntask(k)
  165. if (verbose):
  166. full_path_old = d_old.get(k).get("path")
  167. full_path_new = d_new.get(k).get("path")
  168. # No counter since it is not ready here
  169. if sigdata_re.match(full_path_old) and sigdata_re.match(full_path_new):
  170. output = bb.siggen.compare_sigfiles(full_path_old, full_path_new)
  171. if output:
  172. print("\n=== The verbose changes of %s.%s:" % (pn, task))
  173. print('\n'.join(output))
  174. else:
  175. # Format the output, the format is:
  176. # {pn: task1, task2, ...}
  177. if pn in depchanged:
  178. depchanged[pn] = "%s %s" % (depchanged.get(pn), task)
  179. else:
  180. depchanged[pn] = task
  181. if len(depchanged) > 0:
  182. print("\n=== Dependencies changed: (%s tasks)" % counter)
  183. for k in depchanged.keys():
  184. print(" %s: %s" % (k, depchanged[k]))
  185. return counter
  186. def main():
  187. """
  188. Print what will be done between the current and last builds:
  189. 1) Run "STAMPS_DIR=<path> bitbake -S recipe" to re-generate the stamps
  190. 2) Figure out what are newly added and changed, can't figure out
  191. what are removed since we can't know the previous stamps
  192. clearly, for example, if there are several builds, we can't know
  193. which stamps the last build has used exactly.
  194. 3) Use bb.siggen.compare_sigfiles to diff the old and new stamps
  195. """
  196. parser = OptionParser(
  197. version = "1.0",
  198. usage = """%prog [options] [package ...]
  199. print what will be done between the current and last builds, for example:
  200. $ bitbake core-image-sato
  201. # Edit the recipes
  202. $ bitbake-whatchanged core-image-sato
  203. The changes will be printed"
  204. Note:
  205. The amount of tasks is not accurate when the task is "do_build" since
  206. it usually depends on other tasks.
  207. The "nostamp" task is not included.
  208. """
  209. )
  210. parser.add_option("-v", "--verbose", help = "print the verbose changes",
  211. action = "store_true", dest = "verbose")
  212. options, args = parser.parse_args(sys.argv)
  213. verbose = options.verbose
  214. if len(args) != 2:
  215. parser.error("Incorrect number of arguments")
  216. else:
  217. recipe = args[1]
  218. # Get the STAMPS_DIR
  219. print("Figuring out the STAMPS_DIR ...")
  220. cmdline = "bitbake -e | sed -ne 's/^STAMPS_DIR=\"\(.*\)\"/\\1/p'"
  221. try:
  222. stampsdir, err = bb.process.run(cmdline)
  223. except:
  224. raise
  225. if not stampsdir:
  226. print("ERROR: No STAMPS_DIR found for '%s'" % recipe, file=sys.stderr)
  227. return 2
  228. stampsdir = stampsdir.rstrip("\n")
  229. if not os.path.isdir(stampsdir):
  230. print("ERROR: stamps directory \"%s\" not found!" % stampsdir, file=sys.stderr)
  231. return 2
  232. # The new stamps dir
  233. new_stampsdir = stampsdir + ".bbs"
  234. if os.path.exists(new_stampsdir):
  235. print("ERROR: %s already exists!" % new_stampsdir, file=sys.stderr)
  236. return 2
  237. try:
  238. # Generate the new stamps dir
  239. print("Generating the new stamps ... (need several minutes)")
  240. cmdline = "STAMPS_DIR=%s bitbake -S none %s" % (new_stampsdir, recipe)
  241. # FIXME
  242. # The "bitbake -S" may fail, not fatal error, the stamps will still
  243. # be generated, this might be a bug of "bitbake -S".
  244. try:
  245. bb.process.run(cmdline)
  246. except Exception as exc:
  247. print(exc)
  248. # The dict for the new and old stamps.
  249. old_dict = gen_dict(stampsdir)
  250. new_dict = gen_dict(new_stampsdir)
  251. # Remove the same one from both stamps.
  252. cnt_unchanged = 0
  253. for k in new_dict.keys():
  254. if k in old_dict:
  255. cnt_unchanged += 1
  256. del(new_dict[k])
  257. del(old_dict[k])
  258. # Re-construct the dict to easily find out what is added or changed.
  259. # The dict format is:
  260. # {pn_task: {pv: PV, pr: PR, path: PATH}}
  261. new_recon = recon_dict(new_dict)
  262. old_recon = recon_dict(old_dict)
  263. del new_dict
  264. del old_dict
  265. # Figure out what are changed, the new_recon would be changed
  266. # by the print_xxx function.
  267. # Newly added
  268. cnt_added = print_added(new_recon, old_recon)
  269. # PV (including PE) and PR changed
  270. # Let the bb.siggen handle them if verbose
  271. cnt_rv = {}
  272. if not verbose:
  273. for i in ('pv', 'pr'):
  274. cnt_rv[i] = print_vrchanged(new_recon, old_recon, i)
  275. # Dependencies changed (use bitbake-diffsigs)
  276. cnt_dep = print_depchanged(new_recon, old_recon, verbose)
  277. total_changed = cnt_added + (cnt_rv.get('pv') or 0) + (cnt_rv.get('pr') or 0) + cnt_dep
  278. print("\n=== Summary: (%s changed, %s unchanged)" % (total_changed, cnt_unchanged))
  279. if verbose:
  280. print("Newly added: %s\nDependencies changed: %s\n" % \
  281. (cnt_added, cnt_dep))
  282. else:
  283. print("Newly added: %s\nPV changed: %s\nPR changed: %s\nDependencies changed: %s\n" % \
  284. (cnt_added, cnt_rv.get('pv') or 0, cnt_rv.get('pr') or 0, cnt_dep))
  285. except:
  286. print("ERROR occurred!")
  287. raise
  288. finally:
  289. # Remove the newly generated stamps dir
  290. if os.path.exists(new_stampsdir):
  291. print("Removing the newly generated stamps dir ...")
  292. shutil.rmtree(new_stampsdir)
  293. if __name__ == "__main__":
  294. sys.exit(main())