buildhistory_analysis.py 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723
  1. # Report significant differences in the buildhistory repository since a specific revision
  2. #
  3. # Copyright (C) 2012-2013, 2016-2017 Intel Corporation
  4. # Author: Paul Eggleton <paul.eggleton@linux.intel.com>
  5. #
  6. # SPDX-License-Identifier: GPL-2.0-only
  7. #
  8. # Note: requires GitPython 0.3.1+
  9. #
  10. # You can use this from the command line by running scripts/buildhistory-diff
  11. #
  12. import sys
  13. import os.path
  14. import difflib
  15. import git
  16. import re
  17. import shlex
  18. import hashlib
  19. import collections
  20. import bb.utils
  21. import bb.tinfoil
  22. # How to display fields
  23. list_fields = ['DEPENDS', 'RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RREPLACES', 'RCONFLICTS', 'FILES', 'FILELIST', 'USER_CLASSES', 'IMAGE_CLASSES', 'IMAGE_FEATURES', 'IMAGE_LINGUAS', 'IMAGE_INSTALL', 'BAD_RECOMMENDATIONS', 'PACKAGE_EXCLUDE']
  24. list_order_fields = ['PACKAGES']
  25. defaultval_map = {'PKG': 'PKG', 'PKGE': 'PE', 'PKGV': 'PV', 'PKGR': 'PR'}
  26. numeric_fields = ['PKGSIZE', 'IMAGESIZE']
  27. # Fields to monitor
  28. monitor_fields = ['RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RREPLACES', 'RCONFLICTS', 'PACKAGES', 'FILELIST', 'PKGSIZE', 'IMAGESIZE', 'PKG']
  29. ver_monitor_fields = ['PKGE', 'PKGV', 'PKGR']
  30. # Percentage change to alert for numeric fields
  31. monitor_numeric_threshold = 10
  32. # Image files to monitor (note that image-info.txt is handled separately)
  33. img_monitor_files = ['installed-package-names.txt', 'files-in-image.txt']
  34. colours = {
  35. 'colour_default': '',
  36. 'colour_add': '',
  37. 'colour_remove': '',
  38. }
  39. def init_colours(use_colours):
  40. global colours
  41. if use_colours:
  42. colours = {
  43. 'colour_default': '\033[0m',
  44. 'colour_add': '\033[1;32m',
  45. 'colour_remove': '\033[1;31m',
  46. }
  47. else:
  48. colours = {
  49. 'colour_default': '',
  50. 'colour_add': '',
  51. 'colour_remove': '',
  52. }
  53. class ChangeRecord:
  54. def __init__(self, path, fieldname, oldvalue, newvalue, monitored):
  55. self.path = path
  56. self.fieldname = fieldname
  57. self.oldvalue = oldvalue
  58. self.newvalue = newvalue
  59. self.monitored = monitored
  60. self.filechanges = None
  61. def __str__(self):
  62. return self._str_internal(True)
  63. def _str_internal(self, outer):
  64. if outer:
  65. if '/image-files/' in self.path:
  66. prefix = '%s: ' % self.path.split('/image-files/')[0]
  67. else:
  68. prefix = '%s: ' % self.path
  69. else:
  70. prefix = ''
  71. def pkglist_combine(depver):
  72. pkglist = []
  73. for k,v in depver.items():
  74. if v:
  75. pkglist.append("%s (%s)" % (k,v))
  76. else:
  77. pkglist.append(k)
  78. return pkglist
  79. def detect_renamed_dirs(aitems, bitems):
  80. adirs = set(map(os.path.dirname, aitems))
  81. bdirs = set(map(os.path.dirname, bitems))
  82. files_ab = [(name, sorted(os.path.basename(item) for item in aitems if os.path.dirname(item) == name)) \
  83. for name in adirs - bdirs]
  84. files_ba = [(name, sorted(os.path.basename(item) for item in bitems if os.path.dirname(item) == name)) \
  85. for name in bdirs - adirs]
  86. renamed_dirs = []
  87. for dir1, files1 in files_ab:
  88. rename = False
  89. for dir2, files2 in files_ba:
  90. if files1 == files2 and not rename:
  91. renamed_dirs.append((dir1,dir2))
  92. # Make sure that we don't use this (dir, files) pair again.
  93. files_ba.remove((dir2,files2))
  94. # If a dir has already been found to have a rename, stop and go no further.
  95. rename = True
  96. # remove files that belong to renamed dirs from aitems and bitems
  97. for dir1, dir2 in renamed_dirs:
  98. aitems = [item for item in aitems if os.path.dirname(item) not in (dir1, dir2)]
  99. bitems = [item for item in bitems if os.path.dirname(item) not in (dir1, dir2)]
  100. return renamed_dirs, aitems, bitems
  101. if self.fieldname in list_fields or self.fieldname in list_order_fields:
  102. renamed_dirs = []
  103. changed_order = False
  104. if self.fieldname in ['RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RREPLACES', 'RCONFLICTS']:
  105. (depvera, depverb) = compare_pkg_lists(self.oldvalue, self.newvalue)
  106. aitems = pkglist_combine(depvera)
  107. bitems = pkglist_combine(depverb)
  108. else:
  109. if self.fieldname == 'FILELIST':
  110. aitems = shlex.split(self.oldvalue)
  111. bitems = shlex.split(self.newvalue)
  112. renamed_dirs, aitems, bitems = detect_renamed_dirs(aitems, bitems)
  113. else:
  114. aitems = self.oldvalue.split()
  115. bitems = self.newvalue.split()
  116. removed = list(set(aitems) - set(bitems))
  117. added = list(set(bitems) - set(aitems))
  118. if not removed and not added and self.fieldname in ['RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RREPLACES', 'RCONFLICTS']:
  119. depvera = bb.utils.explode_dep_versions2(self.oldvalue, sort=False)
  120. depverb = bb.utils.explode_dep_versions2(self.newvalue, sort=False)
  121. for i, j in zip(depvera.items(), depverb.items()):
  122. if i[0] != j[0]:
  123. changed_order = True
  124. break
  125. lines = []
  126. if renamed_dirs:
  127. for dfrom, dto in renamed_dirs:
  128. lines.append('directory renamed {colour_remove}{}{colour_default} -> {colour_add}{}{colour_default}'.format(dfrom, dto, **colours))
  129. if removed or added:
  130. if removed and not bitems:
  131. lines.append('removed all items "{colour_remove}{}{colour_default}"'.format(' '.join(removed), **colours))
  132. else:
  133. if removed:
  134. lines.append('removed "{colour_remove}{value}{colour_default}"'.format(value=' '.join(removed), **colours))
  135. if added:
  136. lines.append('added "{colour_add}{value}{colour_default}"'.format(value=' '.join(added), **colours))
  137. else:
  138. lines.append('changed order')
  139. if not (removed or added or changed_order):
  140. out = ''
  141. else:
  142. out = '%s: %s' % (self.fieldname, ', '.join(lines))
  143. elif self.fieldname in numeric_fields:
  144. aval = int(self.oldvalue or 0)
  145. bval = int(self.newvalue or 0)
  146. if aval != 0:
  147. percentchg = ((bval - aval) / float(aval)) * 100
  148. else:
  149. percentchg = 100
  150. out = '{} changed from {colour_remove}{}{colour_default} to {colour_add}{}{colour_default} ({}{:.0f}%)'.format(self.fieldname, self.oldvalue or "''", self.newvalue or "''", '+' if percentchg > 0 else '', percentchg, **colours)
  151. elif self.fieldname in defaultval_map:
  152. out = '{} changed from {colour_remove}{}{colour_default} to {colour_add}{}{colour_default}'.format(self.fieldname, self.oldvalue, self.newvalue, **colours)
  153. if self.fieldname == 'PKG' and '[default]' in self.newvalue:
  154. out += ' - may indicate debian renaming failure'
  155. elif self.fieldname in ['pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm']:
  156. if self.oldvalue and self.newvalue:
  157. out = '%s changed:\n ' % self.fieldname
  158. elif self.newvalue:
  159. out = '%s added:\n ' % self.fieldname
  160. elif self.oldvalue:
  161. out = '%s cleared:\n ' % self.fieldname
  162. alines = self.oldvalue.splitlines()
  163. blines = self.newvalue.splitlines()
  164. diff = difflib.unified_diff(alines, blines, self.fieldname, self.fieldname, lineterm='')
  165. out += '\n '.join(list(diff)[2:])
  166. out += '\n --'
  167. elif self.fieldname in img_monitor_files or '/image-files/' in self.path or self.fieldname == "sysroot":
  168. if self.filechanges or (self.oldvalue and self.newvalue):
  169. fieldname = self.fieldname
  170. if '/image-files/' in self.path:
  171. fieldname = os.path.join('/' + self.path.split('/image-files/')[1], self.fieldname)
  172. out = 'Changes to %s:\n ' % fieldname
  173. else:
  174. if outer:
  175. prefix = 'Changes to %s ' % self.path
  176. out = '(%s):\n ' % self.fieldname
  177. if self.filechanges:
  178. out += '\n '.join(['%s' % i for i in self.filechanges])
  179. else:
  180. alines = self.oldvalue.splitlines()
  181. blines = self.newvalue.splitlines()
  182. diff = difflib.unified_diff(alines, blines, fieldname, fieldname, lineterm='')
  183. out += '\n '.join(list(diff))
  184. out += '\n --'
  185. else:
  186. out = ''
  187. else:
  188. out = '{} changed from "{colour_remove}{}{colour_default}" to "{colour_add}{}{colour_default}"'.format(self.fieldname, self.oldvalue, self.newvalue, **colours)
  189. return '%s%s' % (prefix, out) if out else ''
  190. class FileChange:
  191. changetype_add = 'A'
  192. changetype_remove = 'R'
  193. changetype_type = 'T'
  194. changetype_perms = 'P'
  195. changetype_ownergroup = 'O'
  196. changetype_link = 'L'
  197. changetype_move = 'M'
  198. def __init__(self, path, changetype, oldvalue = None, newvalue = None):
  199. self.path = path
  200. self.changetype = changetype
  201. self.oldvalue = oldvalue
  202. self.newvalue = newvalue
  203. def _ftype_str(self, ftype):
  204. if ftype == '-':
  205. return 'file'
  206. elif ftype == 'd':
  207. return 'directory'
  208. elif ftype == 'l':
  209. return 'symlink'
  210. elif ftype == 'c':
  211. return 'char device'
  212. elif ftype == 'b':
  213. return 'block device'
  214. elif ftype == 'p':
  215. return 'fifo'
  216. elif ftype == 's':
  217. return 'socket'
  218. else:
  219. return 'unknown (%s)' % ftype
  220. def __str__(self):
  221. if self.changetype == self.changetype_add:
  222. return '%s was added' % self.path
  223. elif self.changetype == self.changetype_remove:
  224. return '%s was removed' % self.path
  225. elif self.changetype == self.changetype_type:
  226. return '%s changed type from %s to %s' % (self.path, self._ftype_str(self.oldvalue), self._ftype_str(self.newvalue))
  227. elif self.changetype == self.changetype_perms:
  228. return '%s changed permissions from %s to %s' % (self.path, self.oldvalue, self.newvalue)
  229. elif self.changetype == self.changetype_ownergroup:
  230. return '%s changed owner/group from %s to %s' % (self.path, self.oldvalue, self.newvalue)
  231. elif self.changetype == self.changetype_link:
  232. return '%s changed symlink target from %s to %s' % (self.path, self.oldvalue, self.newvalue)
  233. elif self.changetype == self.changetype_move:
  234. return '%s moved to %s' % (self.path, self.oldvalue)
  235. else:
  236. return '%s changed (unknown)' % self.path
  237. def blob_to_dict(blob):
  238. alines = [line for line in blob.data_stream.read().decode('utf-8').splitlines()]
  239. adict = {}
  240. for line in alines:
  241. splitv = [i.strip() for i in line.split('=',1)]
  242. if len(splitv) > 1:
  243. adict[splitv[0]] = splitv[1]
  244. return adict
  245. def file_list_to_dict(lines):
  246. adict = {}
  247. for line in lines:
  248. # Leave the last few fields intact so we handle file names containing spaces
  249. splitv = line.split(None,4)
  250. # Grab the path and remove the leading .
  251. path = splitv[4][1:].strip()
  252. # Handle symlinks
  253. if(' -> ' in path):
  254. target = path.split(' -> ')[1]
  255. path = path.split(' -> ')[0]
  256. adict[path] = splitv[0:3] + [target]
  257. else:
  258. adict[path] = splitv[0:3]
  259. return adict
  260. numeric_removal = str.maketrans('0123456789', 'XXXXXXXXXX')
  261. def compare_file_lists(alines, blines, compare_ownership=True):
  262. adict = file_list_to_dict(alines)
  263. bdict = file_list_to_dict(blines)
  264. filechanges = []
  265. additions = []
  266. removals = []
  267. for path, splitv in adict.items():
  268. newsplitv = bdict.pop(path, None)
  269. if newsplitv:
  270. # Check type
  271. oldvalue = splitv[0][0]
  272. newvalue = newsplitv[0][0]
  273. if oldvalue != newvalue:
  274. filechanges.append(FileChange(path, FileChange.changetype_type, oldvalue, newvalue))
  275. # Check permissions
  276. oldvalue = splitv[0][1:]
  277. newvalue = newsplitv[0][1:]
  278. if oldvalue != newvalue:
  279. filechanges.append(FileChange(path, FileChange.changetype_perms, oldvalue, newvalue))
  280. if compare_ownership:
  281. # Check owner/group
  282. oldvalue = '%s/%s' % (splitv[1], splitv[2])
  283. newvalue = '%s/%s' % (newsplitv[1], newsplitv[2])
  284. if oldvalue != newvalue:
  285. filechanges.append(FileChange(path, FileChange.changetype_ownergroup, oldvalue, newvalue))
  286. # Check symlink target
  287. if newsplitv[0][0] == 'l':
  288. if len(splitv) > 3:
  289. oldvalue = splitv[3]
  290. else:
  291. oldvalue = None
  292. newvalue = newsplitv[3]
  293. if oldvalue != newvalue:
  294. filechanges.append(FileChange(path, FileChange.changetype_link, oldvalue, newvalue))
  295. else:
  296. removals.append(path)
  297. # Whatever is left over has been added
  298. for path in bdict:
  299. additions.append(path)
  300. # Rather than print additions and removals, its nicer to print file 'moves'
  301. # where names or paths are similar.
  302. revmap_remove = {}
  303. for removal in removals:
  304. translated = removal.translate(numeric_removal)
  305. if translated not in revmap_remove:
  306. revmap_remove[translated] = []
  307. revmap_remove[translated].append(removal)
  308. #
  309. # We want to detect renames of large trees of files like
  310. # /lib/modules/5.4.40-yocto-standard to /lib/modules/5.4.43-yocto-standard
  311. #
  312. renames = {}
  313. for addition in additions.copy():
  314. if addition not in additions:
  315. continue
  316. translated = addition.translate(numeric_removal)
  317. if translated in revmap_remove:
  318. if len(revmap_remove[translated]) != 1:
  319. continue
  320. removal = revmap_remove[translated][0]
  321. commondir = addition.split("/")
  322. commondir2 = removal.split("/")
  323. idx = None
  324. for i in range(len(commondir)):
  325. if commondir[i] != commondir2[i]:
  326. idx = i
  327. break
  328. commondir = "/".join(commondir[:i+1])
  329. commondir2 = "/".join(commondir2[:i+1])
  330. # If the common parent is in one dict and not the other its likely a rename
  331. # so iterate through those files and process as such
  332. if commondir2 not in bdict and commondir not in adict:
  333. if commondir not in renames:
  334. renames[commondir] = commondir2
  335. for addition2 in additions.copy():
  336. if addition2.startswith(commondir):
  337. removal2 = addition2.replace(commondir, commondir2)
  338. if removal2 in removals:
  339. additions.remove(addition2)
  340. removals.remove(removal2)
  341. continue
  342. filechanges.append(FileChange(removal, FileChange.changetype_move, addition))
  343. if addition in additions:
  344. additions.remove(addition)
  345. if removal in removals:
  346. removals.remove(removal)
  347. for rename in renames:
  348. filechanges.append(FileChange(renames[rename], FileChange.changetype_move, rename))
  349. for addition in additions:
  350. filechanges.append(FileChange(addition, FileChange.changetype_add))
  351. for removal in removals:
  352. filechanges.append(FileChange(removal, FileChange.changetype_remove))
  353. return filechanges
  354. def compare_lists(alines, blines):
  355. removed = list(set(alines) - set(blines))
  356. added = list(set(blines) - set(alines))
  357. filechanges = []
  358. for pkg in removed:
  359. filechanges.append(FileChange(pkg, FileChange.changetype_remove))
  360. for pkg in added:
  361. filechanges.append(FileChange(pkg, FileChange.changetype_add))
  362. return filechanges
  363. def compare_pkg_lists(astr, bstr):
  364. depvera = bb.utils.explode_dep_versions2(astr)
  365. depverb = bb.utils.explode_dep_versions2(bstr)
  366. # Strip out changes where the version has increased
  367. remove = []
  368. for k in depvera:
  369. if k in depverb:
  370. dva = depvera[k]
  371. dvb = depverb[k]
  372. if dva and dvb and len(dva) == len(dvb):
  373. # Since length is the same, sort so that prefixes (e.g. >=) will line up
  374. dva.sort()
  375. dvb.sort()
  376. removeit = True
  377. for dvai, dvbi in zip(dva, dvb):
  378. if dvai != dvbi:
  379. aiprefix = dvai.split(' ')[0]
  380. biprefix = dvbi.split(' ')[0]
  381. if aiprefix == biprefix and aiprefix in ['>=', '=']:
  382. if bb.utils.vercmp(bb.utils.split_version(dvai), bb.utils.split_version(dvbi)) > 0:
  383. removeit = False
  384. break
  385. else:
  386. removeit = False
  387. break
  388. if removeit:
  389. remove.append(k)
  390. for k in remove:
  391. depvera.pop(k)
  392. depverb.pop(k)
  393. return (depvera, depverb)
  394. def compare_dict_blobs(path, ablob, bblob, report_all, report_ver):
  395. adict = blob_to_dict(ablob)
  396. bdict = blob_to_dict(bblob)
  397. pkgname = os.path.basename(path)
  398. defaultvals = {}
  399. defaultvals['PKG'] = pkgname
  400. defaultvals['PKGE'] = '0'
  401. changes = []
  402. keys = list(set(adict.keys()) | set(bdict.keys()) | set(defaultval_map.keys()))
  403. for key in keys:
  404. astr = adict.get(key, '')
  405. bstr = bdict.get(key, '')
  406. if key in ver_monitor_fields:
  407. monitored = report_ver or astr or bstr
  408. else:
  409. monitored = key in monitor_fields
  410. mapped_key = defaultval_map.get(key, '')
  411. if mapped_key:
  412. if not astr:
  413. astr = '%s [default]' % adict.get(mapped_key, defaultvals.get(key, ''))
  414. if not bstr:
  415. bstr = '%s [default]' % bdict.get(mapped_key, defaultvals.get(key, ''))
  416. if astr != bstr:
  417. if (not report_all) and key in numeric_fields:
  418. aval = int(astr or 0)
  419. bval = int(bstr or 0)
  420. if aval != 0:
  421. percentchg = ((bval - aval) / float(aval)) * 100
  422. else:
  423. percentchg = 100
  424. if abs(percentchg) < monitor_numeric_threshold:
  425. continue
  426. elif (not report_all) and key in list_fields:
  427. if key == "FILELIST" and (path.endswith("-dbg") or path.endswith("-src")) and bstr.strip() != '':
  428. continue
  429. if key in ['RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RREPLACES', 'RCONFLICTS']:
  430. (depvera, depverb) = compare_pkg_lists(astr, bstr)
  431. if depvera == depverb:
  432. continue
  433. if key == 'FILELIST':
  434. alist = shlex.split(astr)
  435. blist = shlex.split(bstr)
  436. else:
  437. alist = astr.split()
  438. blist = bstr.split()
  439. alist.sort()
  440. blist.sort()
  441. # We don't care about the removal of self-dependencies
  442. if pkgname in alist and not pkgname in blist:
  443. alist.remove(pkgname)
  444. if ' '.join(alist) == ' '.join(blist):
  445. continue
  446. if key == 'PKGR' and not report_all:
  447. vers = []
  448. # strip leading 'r' and dots
  449. for ver in (astr.split()[0], bstr.split()[0]):
  450. if ver.startswith('r'):
  451. ver = ver[1:]
  452. vers.append(ver.replace('.', ''))
  453. maxlen = max(len(vers[0]), len(vers[1]))
  454. try:
  455. # pad with '0' and convert to int
  456. vers = [int(ver.ljust(maxlen, '0')) for ver in vers]
  457. except ValueError:
  458. pass
  459. else:
  460. # skip decrements and increments
  461. if abs(vers[0] - vers[1]) == 1:
  462. continue
  463. chg = ChangeRecord(path, key, astr, bstr, monitored)
  464. changes.append(chg)
  465. return changes
  466. def compare_siglists(a_blob, b_blob, taskdiff=False):
  467. # FIXME collapse down a recipe's tasks?
  468. alines = a_blob.data_stream.read().decode('utf-8').splitlines()
  469. blines = b_blob.data_stream.read().decode('utf-8').splitlines()
  470. keys = []
  471. pnmap = {}
  472. def readsigs(lines):
  473. sigs = {}
  474. for line in lines:
  475. linesplit = line.split()
  476. if len(linesplit) > 2:
  477. sigs[linesplit[0]] = linesplit[2]
  478. if not linesplit[0] in keys:
  479. keys.append(linesplit[0])
  480. pnmap[linesplit[1]] = linesplit[0].rsplit('.', 1)[0]
  481. return sigs
  482. adict = readsigs(alines)
  483. bdict = readsigs(blines)
  484. out = []
  485. changecount = 0
  486. addcount = 0
  487. removecount = 0
  488. if taskdiff:
  489. with bb.tinfoil.Tinfoil() as tinfoil:
  490. tinfoil.prepare(config_only=True)
  491. changes = collections.OrderedDict()
  492. def compare_hashfiles(pn, taskname, hash1, hash2):
  493. hashes = [hash1, hash2]
  494. hashfiles = bb.siggen.find_siginfo(pn, taskname, hashes, tinfoil.config_data)
  495. if not taskname:
  496. (pn, taskname) = pn.rsplit('.', 1)
  497. pn = pnmap.get(pn, pn)
  498. desc = '%s.%s' % (pn, taskname)
  499. if len(hashfiles) == 0:
  500. out.append("Unable to find matching sigdata for %s with hashes %s or %s" % (desc, hash1, hash2))
  501. elif not hash1 in hashfiles:
  502. out.append("Unable to find matching sigdata for %s with hash %s" % (desc, hash1))
  503. elif not hash2 in hashfiles:
  504. out.append("Unable to find matching sigdata for %s with hash %s" % (desc, hash2))
  505. else:
  506. out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb, collapsed=True)
  507. for line in out2:
  508. m = hashlib.sha256()
  509. m.update(line.encode('utf-8'))
  510. entry = changes.get(m.hexdigest(), (line, []))
  511. if desc not in entry[1]:
  512. changes[m.hexdigest()] = (line, entry[1] + [desc])
  513. # Define recursion callback
  514. def recursecb(key, hash1, hash2):
  515. compare_hashfiles(key, None, hash1, hash2)
  516. return []
  517. for key in keys:
  518. siga = adict.get(key, None)
  519. sigb = bdict.get(key, None)
  520. if siga is not None and sigb is not None and siga != sigb:
  521. changecount += 1
  522. (pn, taskname) = key.rsplit('.', 1)
  523. compare_hashfiles(pn, taskname, siga, sigb)
  524. elif siga is None:
  525. addcount += 1
  526. elif sigb is None:
  527. removecount += 1
  528. for key, item in changes.items():
  529. line, tasks = item
  530. if len(tasks) == 1:
  531. desc = tasks[0]
  532. elif len(tasks) == 2:
  533. desc = '%s and %s' % (tasks[0], tasks[1])
  534. else:
  535. desc = '%s and %d others' % (tasks[-1], len(tasks)-1)
  536. out.append('%s: %s' % (desc, line))
  537. else:
  538. for key in keys:
  539. siga = adict.get(key, None)
  540. sigb = bdict.get(key, None)
  541. if siga is not None and sigb is not None and siga != sigb:
  542. out.append('%s changed from %s to %s' % (key, siga, sigb))
  543. changecount += 1
  544. elif siga is None:
  545. out.append('%s was added' % key)
  546. addcount += 1
  547. elif sigb is None:
  548. out.append('%s was removed' % key)
  549. removecount += 1
  550. out.append('Summary: %d tasks added, %d tasks removed, %d tasks modified (%.1f%%)' % (addcount, removecount, changecount, (changecount / float(len(bdict)) * 100)))
  551. return '\n'.join(out)
  552. def process_changes(repopath, revision1, revision2='HEAD', report_all=False, report_ver=False,
  553. sigs=False, sigsdiff=False, exclude_path=None):
  554. repo = git.Repo(repopath)
  555. assert repo.bare == False
  556. commit = repo.commit(revision1)
  557. diff = commit.diff(revision2)
  558. changes = []
  559. if sigs or sigsdiff:
  560. for d in diff.iter_change_type('M'):
  561. if d.a_blob.path == 'siglist.txt':
  562. changes.append(compare_siglists(d.a_blob, d.b_blob, taskdiff=sigsdiff))
  563. return changes
  564. for d in diff.iter_change_type('M'):
  565. path = os.path.dirname(d.a_blob.path)
  566. if path.startswith('packages/'):
  567. filename = os.path.basename(d.a_blob.path)
  568. if filename == 'latest':
  569. changes.extend(compare_dict_blobs(path, d.a_blob, d.b_blob, report_all, report_ver))
  570. elif filename.startswith('latest.'):
  571. chg = ChangeRecord(path, filename, d.a_blob.data_stream.read().decode('utf-8'), d.b_blob.data_stream.read().decode('utf-8'), True)
  572. changes.append(chg)
  573. elif filename == 'sysroot':
  574. alines = d.a_blob.data_stream.read().decode('utf-8').splitlines()
  575. blines = d.b_blob.data_stream.read().decode('utf-8').splitlines()
  576. filechanges = compare_file_lists(alines,blines, compare_ownership=False)
  577. if filechanges:
  578. chg = ChangeRecord(path, filename, None, None, True)
  579. chg.filechanges = filechanges
  580. changes.append(chg)
  581. elif path.startswith('images/'):
  582. filename = os.path.basename(d.a_blob.path)
  583. if filename in img_monitor_files:
  584. if filename == 'files-in-image.txt':
  585. alines = d.a_blob.data_stream.read().decode('utf-8').splitlines()
  586. blines = d.b_blob.data_stream.read().decode('utf-8').splitlines()
  587. filechanges = compare_file_lists(alines,blines)
  588. if filechanges:
  589. chg = ChangeRecord(path, filename, None, None, True)
  590. chg.filechanges = filechanges
  591. changes.append(chg)
  592. elif filename == 'installed-package-names.txt':
  593. alines = d.a_blob.data_stream.read().decode('utf-8').splitlines()
  594. blines = d.b_blob.data_stream.read().decode('utf-8').splitlines()
  595. filechanges = compare_lists(alines,blines)
  596. if filechanges:
  597. chg = ChangeRecord(path, filename, None, None, True)
  598. chg.filechanges = filechanges
  599. changes.append(chg)
  600. else:
  601. chg = ChangeRecord(path, filename, d.a_blob.data_stream.read().decode('utf-8'), d.b_blob.data_stream.read().decode('utf-8'), True)
  602. changes.append(chg)
  603. elif filename == 'image-info.txt':
  604. changes.extend(compare_dict_blobs(path, d.a_blob, d.b_blob, report_all, report_ver))
  605. elif '/image-files/' in path:
  606. chg = ChangeRecord(path, filename, d.a_blob.data_stream.read().decode('utf-8'), d.b_blob.data_stream.read().decode('utf-8'), True)
  607. changes.append(chg)
  608. # Look for added preinst/postinst/prerm/postrm
  609. # (without reporting newly added recipes)
  610. addedpkgs = []
  611. addedchanges = []
  612. for d in diff.iter_change_type('A'):
  613. path = os.path.dirname(d.b_blob.path)
  614. if path.startswith('packages/'):
  615. filename = os.path.basename(d.b_blob.path)
  616. if filename == 'latest':
  617. addedpkgs.append(path)
  618. elif filename.startswith('latest.'):
  619. chg = ChangeRecord(path, filename[7:], '', d.b_blob.data_stream.read().decode('utf-8'), True)
  620. addedchanges.append(chg)
  621. for chg in addedchanges:
  622. found = False
  623. for pkg in addedpkgs:
  624. if chg.path.startswith(pkg):
  625. found = True
  626. break
  627. if not found:
  628. changes.append(chg)
  629. # Look for cleared preinst/postinst/prerm/postrm
  630. for d in diff.iter_change_type('D'):
  631. path = os.path.dirname(d.a_blob.path)
  632. if path.startswith('packages/'):
  633. filename = os.path.basename(d.a_blob.path)
  634. if filename != 'latest' and filename.startswith('latest.'):
  635. chg = ChangeRecord(path, filename[7:], d.a_blob.data_stream.read().decode('utf-8'), '', True)
  636. changes.append(chg)
  637. # filter out unwanted paths
  638. if exclude_path:
  639. for chg in changes:
  640. if chg.filechanges:
  641. fchgs = []
  642. for fchg in chg.filechanges:
  643. for epath in exclude_path:
  644. if fchg.path.startswith(epath):
  645. break
  646. else:
  647. fchgs.append(fchg)
  648. chg.filechanges = fchgs
  649. if report_all:
  650. return changes
  651. else:
  652. return [chg for chg in changes if chg.monitored]