buildhistory_analysis.py 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649
  1. # Report significant differences in the buildhistory repository since a specific revision
  2. #
  3. # Copyright (C) 2012-2013, 2016-2017 Intel Corporation
  4. # Author: Paul Eggleton <paul.eggleton@linux.intel.com>
  5. #
  6. # SPDX-License-Identifier: GPL-2.0-only
  7. #
  8. # Note: requires GitPython 0.3.1+
  9. #
  10. # You can use this from the command line by running scripts/buildhistory-diff
  11. #
  12. import sys
  13. import os.path
  14. import difflib
  15. import git
  16. import re
  17. import shlex
  18. import hashlib
  19. import collections
  20. import bb.utils
  21. import bb.tinfoil
  22. # How to display fields
  23. list_fields = ['DEPENDS', 'RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RREPLACES', 'RCONFLICTS', 'FILES', 'FILELIST', 'USER_CLASSES', 'IMAGE_CLASSES', 'IMAGE_FEATURES', 'IMAGE_LINGUAS', 'IMAGE_INSTALL', 'BAD_RECOMMENDATIONS', 'PACKAGE_EXCLUDE']
  24. list_order_fields = ['PACKAGES']
  25. defaultval_map = {'PKG': 'PKG', 'PKGE': 'PE', 'PKGV': 'PV', 'PKGR': 'PR'}
  26. numeric_fields = ['PKGSIZE', 'IMAGESIZE']
  27. # Fields to monitor
  28. monitor_fields = ['RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RREPLACES', 'RCONFLICTS', 'PACKAGES', 'FILELIST', 'PKGSIZE', 'IMAGESIZE', 'PKG']
  29. ver_monitor_fields = ['PKGE', 'PKGV', 'PKGR']
  30. # Percentage change to alert for numeric fields
  31. monitor_numeric_threshold = 10
  32. # Image files to monitor (note that image-info.txt is handled separately)
  33. img_monitor_files = ['installed-package-names.txt', 'files-in-image.txt']
  34. colours = {
  35. 'colour_default': '',
  36. 'colour_add': '',
  37. 'colour_remove': '',
  38. }
  39. def init_colours(use_colours):
  40. global colours
  41. if use_colours:
  42. colours = {
  43. 'colour_default': '\033[0m',
  44. 'colour_add': '\033[1;32m',
  45. 'colour_remove': '\033[1;31m',
  46. }
  47. else:
  48. colours = {
  49. 'colour_default': '',
  50. 'colour_add': '',
  51. 'colour_remove': '',
  52. }
  53. class ChangeRecord:
  54. def __init__(self, path, fieldname, oldvalue, newvalue, monitored):
  55. self.path = path
  56. self.fieldname = fieldname
  57. self.oldvalue = oldvalue
  58. self.newvalue = newvalue
  59. self.monitored = monitored
  60. self.filechanges = None
  61. def __str__(self):
  62. return self._str_internal(True)
  63. def _str_internal(self, outer):
  64. if outer:
  65. if '/image-files/' in self.path:
  66. prefix = '%s: ' % self.path.split('/image-files/')[0]
  67. else:
  68. prefix = '%s: ' % self.path
  69. else:
  70. prefix = ''
  71. def pkglist_combine(depver):
  72. pkglist = []
  73. for k,v in depver.items():
  74. if v:
  75. pkglist.append("%s (%s)" % (k,v))
  76. else:
  77. pkglist.append(k)
  78. return pkglist
  79. def detect_renamed_dirs(aitems, bitems):
  80. adirs = set(map(os.path.dirname, aitems))
  81. bdirs = set(map(os.path.dirname, bitems))
  82. files_ab = [(name, sorted(os.path.basename(item) for item in aitems if os.path.dirname(item) == name)) \
  83. for name in adirs - bdirs]
  84. files_ba = [(name, sorted(os.path.basename(item) for item in bitems if os.path.dirname(item) == name)) \
  85. for name in bdirs - adirs]
  86. renamed_dirs = []
  87. for dir1, files1 in files_ab:
  88. rename = False
  89. for dir2, files2 in files_ba:
  90. if files1 == files2 and not rename:
  91. renamed_dirs.append((dir1,dir2))
  92. # Make sure that we don't use this (dir, files) pair again.
  93. files_ba.remove((dir2,files2))
  94. # If a dir has already been found to have a rename, stop and go no further.
  95. rename = True
  96. # remove files that belong to renamed dirs from aitems and bitems
  97. for dir1, dir2 in renamed_dirs:
  98. aitems = [item for item in aitems if os.path.dirname(item) not in (dir1, dir2)]
  99. bitems = [item for item in bitems if os.path.dirname(item) not in (dir1, dir2)]
  100. return renamed_dirs, aitems, bitems
  101. if self.fieldname in list_fields or self.fieldname in list_order_fields:
  102. renamed_dirs = []
  103. changed_order = False
  104. if self.fieldname in ['RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RREPLACES', 'RCONFLICTS']:
  105. (depvera, depverb) = compare_pkg_lists(self.oldvalue, self.newvalue)
  106. aitems = pkglist_combine(depvera)
  107. bitems = pkglist_combine(depverb)
  108. else:
  109. if self.fieldname == 'FILELIST':
  110. aitems = shlex.split(self.oldvalue)
  111. bitems = shlex.split(self.newvalue)
  112. renamed_dirs, aitems, bitems = detect_renamed_dirs(aitems, bitems)
  113. else:
  114. aitems = self.oldvalue.split()
  115. bitems = self.newvalue.split()
  116. removed = list(set(aitems) - set(bitems))
  117. added = list(set(bitems) - set(aitems))
  118. if not removed and not added and self.fieldname in ['RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RREPLACES', 'RCONFLICTS']:
  119. depvera = bb.utils.explode_dep_versions2(self.oldvalue, sort=False)
  120. depverb = bb.utils.explode_dep_versions2(self.newvalue, sort=False)
  121. for i, j in zip(depvera.items(), depverb.items()):
  122. if i[0] != j[0]:
  123. changed_order = True
  124. break
  125. lines = []
  126. if renamed_dirs:
  127. for dfrom, dto in renamed_dirs:
  128. lines.append('directory renamed {colour_remove}{}{colour_default} -> {colour_add}{}{colour_default}'.format(dfrom, dto, **colours))
  129. if removed or added:
  130. if removed and not bitems:
  131. lines.append('removed all items "{colour_remove}{}{colour_default}"'.format(' '.join(removed), **colours))
  132. else:
  133. if removed:
  134. lines.append('removed "{colour_remove}{value}{colour_default}"'.format(value=' '.join(removed), **colours))
  135. if added:
  136. lines.append('added "{colour_add}{value}{colour_default}"'.format(value=' '.join(added), **colours))
  137. else:
  138. lines.append('changed order')
  139. if not (removed or added or changed_order):
  140. out = ''
  141. else:
  142. out = '%s: %s' % (self.fieldname, ', '.join(lines))
  143. elif self.fieldname in numeric_fields:
  144. aval = int(self.oldvalue or 0)
  145. bval = int(self.newvalue or 0)
  146. if aval != 0:
  147. percentchg = ((bval - aval) / float(aval)) * 100
  148. else:
  149. percentchg = 100
  150. out = '{} changed from {colour_remove}{}{colour_default} to {colour_add}{}{colour_default} ({}{:.0f}%)'.format(self.fieldname, self.oldvalue or "''", self.newvalue or "''", '+' if percentchg > 0 else '', percentchg, **colours)
  151. elif self.fieldname in defaultval_map:
  152. out = '{} changed from {colour_remove}{}{colour_default} to {colour_add}{}{colour_default}'.format(self.fieldname, self.oldvalue, self.newvalue, **colours)
  153. if self.fieldname == 'PKG' and '[default]' in self.newvalue:
  154. out += ' - may indicate debian renaming failure'
  155. elif self.fieldname in ['pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm']:
  156. if self.oldvalue and self.newvalue:
  157. out = '%s changed:\n ' % self.fieldname
  158. elif self.newvalue:
  159. out = '%s added:\n ' % self.fieldname
  160. elif self.oldvalue:
  161. out = '%s cleared:\n ' % self.fieldname
  162. alines = self.oldvalue.splitlines()
  163. blines = self.newvalue.splitlines()
  164. diff = difflib.unified_diff(alines, blines, self.fieldname, self.fieldname, lineterm='')
  165. out += '\n '.join(list(diff)[2:])
  166. out += '\n --'
  167. elif self.fieldname in img_monitor_files or '/image-files/' in self.path:
  168. if self.filechanges or (self.oldvalue and self.newvalue):
  169. fieldname = self.fieldname
  170. if '/image-files/' in self.path:
  171. fieldname = os.path.join('/' + self.path.split('/image-files/')[1], self.fieldname)
  172. out = 'Changes to %s:\n ' % fieldname
  173. else:
  174. if outer:
  175. prefix = 'Changes to %s ' % self.path
  176. out = '(%s):\n ' % self.fieldname
  177. if self.filechanges:
  178. out += '\n '.join(['%s' % i for i in self.filechanges])
  179. else:
  180. alines = self.oldvalue.splitlines()
  181. blines = self.newvalue.splitlines()
  182. diff = difflib.unified_diff(alines, blines, fieldname, fieldname, lineterm='')
  183. out += '\n '.join(list(diff))
  184. out += '\n --'
  185. else:
  186. out = ''
  187. else:
  188. out = '{} changed from "{colour_remove}{}{colour_default}" to "{colour_add}{}{colour_default}"'.format(self.fieldname, self.oldvalue, self.newvalue, **colours)
  189. return '%s%s' % (prefix, out) if out else ''
  190. class FileChange:
  191. changetype_add = 'A'
  192. changetype_remove = 'R'
  193. changetype_type = 'T'
  194. changetype_perms = 'P'
  195. changetype_ownergroup = 'O'
  196. changetype_link = 'L'
  197. def __init__(self, path, changetype, oldvalue = None, newvalue = None):
  198. self.path = path
  199. self.changetype = changetype
  200. self.oldvalue = oldvalue
  201. self.newvalue = newvalue
  202. def _ftype_str(self, ftype):
  203. if ftype == '-':
  204. return 'file'
  205. elif ftype == 'd':
  206. return 'directory'
  207. elif ftype == 'l':
  208. return 'symlink'
  209. elif ftype == 'c':
  210. return 'char device'
  211. elif ftype == 'b':
  212. return 'block device'
  213. elif ftype == 'p':
  214. return 'fifo'
  215. elif ftype == 's':
  216. return 'socket'
  217. else:
  218. return 'unknown (%s)' % ftype
  219. def __str__(self):
  220. if self.changetype == self.changetype_add:
  221. return '%s was added' % self.path
  222. elif self.changetype == self.changetype_remove:
  223. return '%s was removed' % self.path
  224. elif self.changetype == self.changetype_type:
  225. return '%s changed type from %s to %s' % (self.path, self._ftype_str(self.oldvalue), self._ftype_str(self.newvalue))
  226. elif self.changetype == self.changetype_perms:
  227. return '%s changed permissions from %s to %s' % (self.path, self.oldvalue, self.newvalue)
  228. elif self.changetype == self.changetype_ownergroup:
  229. return '%s changed owner/group from %s to %s' % (self.path, self.oldvalue, self.newvalue)
  230. elif self.changetype == self.changetype_link:
  231. return '%s changed symlink target from %s to %s' % (self.path, self.oldvalue, self.newvalue)
  232. else:
  233. return '%s changed (unknown)' % self.path
  234. def blob_to_dict(blob):
  235. alines = [line for line in blob.data_stream.read().decode('utf-8').splitlines()]
  236. adict = {}
  237. for line in alines:
  238. splitv = [i.strip() for i in line.split('=',1)]
  239. if len(splitv) > 1:
  240. adict[splitv[0]] = splitv[1]
  241. return adict
  242. def file_list_to_dict(lines):
  243. adict = {}
  244. for line in lines:
  245. # Leave the last few fields intact so we handle file names containing spaces
  246. splitv = line.split(None,4)
  247. # Grab the path and remove the leading .
  248. path = splitv[4][1:].strip()
  249. # Handle symlinks
  250. if(' -> ' in path):
  251. target = path.split(' -> ')[1]
  252. path = path.split(' -> ')[0]
  253. adict[path] = splitv[0:3] + [target]
  254. else:
  255. adict[path] = splitv[0:3]
  256. return adict
  257. def compare_file_lists(alines, blines):
  258. adict = file_list_to_dict(alines)
  259. bdict = file_list_to_dict(blines)
  260. filechanges = []
  261. for path, splitv in adict.items():
  262. newsplitv = bdict.pop(path, None)
  263. if newsplitv:
  264. # Check type
  265. oldvalue = splitv[0][0]
  266. newvalue = newsplitv[0][0]
  267. if oldvalue != newvalue:
  268. filechanges.append(FileChange(path, FileChange.changetype_type, oldvalue, newvalue))
  269. # Check permissions
  270. oldvalue = splitv[0][1:]
  271. newvalue = newsplitv[0][1:]
  272. if oldvalue != newvalue:
  273. filechanges.append(FileChange(path, FileChange.changetype_perms, oldvalue, newvalue))
  274. # Check owner/group
  275. oldvalue = '%s/%s' % (splitv[1], splitv[2])
  276. newvalue = '%s/%s' % (newsplitv[1], newsplitv[2])
  277. if oldvalue != newvalue:
  278. filechanges.append(FileChange(path, FileChange.changetype_ownergroup, oldvalue, newvalue))
  279. # Check symlink target
  280. if newsplitv[0][0] == 'l':
  281. if len(splitv) > 3:
  282. oldvalue = splitv[3]
  283. else:
  284. oldvalue = None
  285. newvalue = newsplitv[3]
  286. if oldvalue != newvalue:
  287. filechanges.append(FileChange(path, FileChange.changetype_link, oldvalue, newvalue))
  288. else:
  289. filechanges.append(FileChange(path, FileChange.changetype_remove))
  290. # Whatever is left over has been added
  291. for path in bdict:
  292. filechanges.append(FileChange(path, FileChange.changetype_add))
  293. return filechanges
  294. def compare_lists(alines, blines):
  295. removed = list(set(alines) - set(blines))
  296. added = list(set(blines) - set(alines))
  297. filechanges = []
  298. for pkg in removed:
  299. filechanges.append(FileChange(pkg, FileChange.changetype_remove))
  300. for pkg in added:
  301. filechanges.append(FileChange(pkg, FileChange.changetype_add))
  302. return filechanges
  303. def compare_pkg_lists(astr, bstr):
  304. depvera = bb.utils.explode_dep_versions2(astr)
  305. depverb = bb.utils.explode_dep_versions2(bstr)
  306. # Strip out changes where the version has increased
  307. remove = []
  308. for k in depvera:
  309. if k in depverb:
  310. dva = depvera[k]
  311. dvb = depverb[k]
  312. if dva and dvb and len(dva) == len(dvb):
  313. # Since length is the same, sort so that prefixes (e.g. >=) will line up
  314. dva.sort()
  315. dvb.sort()
  316. removeit = True
  317. for dvai, dvbi in zip(dva, dvb):
  318. if dvai != dvbi:
  319. aiprefix = dvai.split(' ')[0]
  320. biprefix = dvbi.split(' ')[0]
  321. if aiprefix == biprefix and aiprefix in ['>=', '=']:
  322. if bb.utils.vercmp(bb.utils.split_version(dvai), bb.utils.split_version(dvbi)) > 0:
  323. removeit = False
  324. break
  325. else:
  326. removeit = False
  327. break
  328. if removeit:
  329. remove.append(k)
  330. for k in remove:
  331. depvera.pop(k)
  332. depverb.pop(k)
  333. return (depvera, depverb)
  334. def compare_dict_blobs(path, ablob, bblob, report_all, report_ver):
  335. adict = blob_to_dict(ablob)
  336. bdict = blob_to_dict(bblob)
  337. pkgname = os.path.basename(path)
  338. defaultvals = {}
  339. defaultvals['PKG'] = pkgname
  340. defaultvals['PKGE'] = '0'
  341. changes = []
  342. keys = list(set(adict.keys()) | set(bdict.keys()) | set(defaultval_map.keys()))
  343. for key in keys:
  344. astr = adict.get(key, '')
  345. bstr = bdict.get(key, '')
  346. if key in ver_monitor_fields:
  347. monitored = report_ver or astr or bstr
  348. else:
  349. monitored = key in monitor_fields
  350. mapped_key = defaultval_map.get(key, '')
  351. if mapped_key:
  352. if not astr:
  353. astr = '%s [default]' % adict.get(mapped_key, defaultvals.get(key, ''))
  354. if not bstr:
  355. bstr = '%s [default]' % bdict.get(mapped_key, defaultvals.get(key, ''))
  356. if astr != bstr:
  357. if (not report_all) and key in numeric_fields:
  358. aval = int(astr or 0)
  359. bval = int(bstr or 0)
  360. if aval != 0:
  361. percentchg = ((bval - aval) / float(aval)) * 100
  362. else:
  363. percentchg = 100
  364. if abs(percentchg) < monitor_numeric_threshold:
  365. continue
  366. elif (not report_all) and key in list_fields:
  367. if key == "FILELIST" and path.endswith("-dbg") and bstr.strip() != '':
  368. continue
  369. if key in ['RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RREPLACES', 'RCONFLICTS']:
  370. (depvera, depverb) = compare_pkg_lists(astr, bstr)
  371. if depvera == depverb:
  372. continue
  373. if key == 'FILELIST':
  374. alist = shlex.split(astr)
  375. blist = shlex.split(bstr)
  376. else:
  377. alist = astr.split()
  378. blist = bstr.split()
  379. alist.sort()
  380. blist.sort()
  381. # We don't care about the removal of self-dependencies
  382. if pkgname in alist and not pkgname in blist:
  383. alist.remove(pkgname)
  384. if ' '.join(alist) == ' '.join(blist):
  385. continue
  386. if key == 'PKGR' and not report_all:
  387. vers = []
  388. # strip leading 'r' and dots
  389. for ver in (astr.split()[0], bstr.split()[0]):
  390. if ver.startswith('r'):
  391. ver = ver[1:]
  392. vers.append(ver.replace('.', ''))
  393. maxlen = max(len(vers[0]), len(vers[1]))
  394. try:
  395. # pad with '0' and convert to int
  396. vers = [int(ver.ljust(maxlen, '0')) for ver in vers]
  397. except ValueError:
  398. pass
  399. else:
  400. # skip decrements and increments
  401. if abs(vers[0] - vers[1]) == 1:
  402. continue
  403. chg = ChangeRecord(path, key, astr, bstr, monitored)
  404. changes.append(chg)
  405. return changes
  406. def compare_siglists(a_blob, b_blob, taskdiff=False):
  407. # FIXME collapse down a recipe's tasks?
  408. alines = a_blob.data_stream.read().decode('utf-8').splitlines()
  409. blines = b_blob.data_stream.read().decode('utf-8').splitlines()
  410. keys = []
  411. pnmap = {}
  412. def readsigs(lines):
  413. sigs = {}
  414. for line in lines:
  415. linesplit = line.split()
  416. if len(linesplit) > 2:
  417. sigs[linesplit[0]] = linesplit[2]
  418. if not linesplit[0] in keys:
  419. keys.append(linesplit[0])
  420. pnmap[linesplit[1]] = linesplit[0].rsplit('.', 1)[0]
  421. return sigs
  422. adict = readsigs(alines)
  423. bdict = readsigs(blines)
  424. out = []
  425. changecount = 0
  426. addcount = 0
  427. removecount = 0
  428. if taskdiff:
  429. with bb.tinfoil.Tinfoil() as tinfoil:
  430. tinfoil.prepare(config_only=True)
  431. changes = collections.OrderedDict()
  432. def compare_hashfiles(pn, taskname, hash1, hash2):
  433. hashes = [hash1, hash2]
  434. hashfiles = bb.siggen.find_siginfo(pn, taskname, hashes, tinfoil.config_data)
  435. if not taskname:
  436. (pn, taskname) = pn.rsplit('.', 1)
  437. pn = pnmap.get(pn, pn)
  438. desc = '%s.%s' % (pn, taskname)
  439. if len(hashfiles) == 0:
  440. out.append("Unable to find matching sigdata for %s with hashes %s or %s" % (desc, hash1, hash2))
  441. elif not hash1 in hashfiles:
  442. out.append("Unable to find matching sigdata for %s with hash %s" % (desc, hash1))
  443. elif not hash2 in hashfiles:
  444. out.append("Unable to find matching sigdata for %s with hash %s" % (desc, hash2))
  445. else:
  446. out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb, collapsed=True)
  447. for line in out2:
  448. m = hashlib.sha256()
  449. m.update(line.encode('utf-8'))
  450. entry = changes.get(m.hexdigest(), (line, []))
  451. if desc not in entry[1]:
  452. changes[m.hexdigest()] = (line, entry[1] + [desc])
  453. # Define recursion callback
  454. def recursecb(key, hash1, hash2):
  455. compare_hashfiles(key, None, hash1, hash2)
  456. return []
  457. for key in keys:
  458. siga = adict.get(key, None)
  459. sigb = bdict.get(key, None)
  460. if siga is not None and sigb is not None and siga != sigb:
  461. changecount += 1
  462. (pn, taskname) = key.rsplit('.', 1)
  463. compare_hashfiles(pn, taskname, siga, sigb)
  464. elif siga is None:
  465. addcount += 1
  466. elif sigb is None:
  467. removecount += 1
  468. for key, item in changes.items():
  469. line, tasks = item
  470. if len(tasks) == 1:
  471. desc = tasks[0]
  472. elif len(tasks) == 2:
  473. desc = '%s and %s' % (tasks[0], tasks[1])
  474. else:
  475. desc = '%s and %d others' % (tasks[-1], len(tasks)-1)
  476. out.append('%s: %s' % (desc, line))
  477. else:
  478. for key in keys:
  479. siga = adict.get(key, None)
  480. sigb = bdict.get(key, None)
  481. if siga is not None and sigb is not None and siga != sigb:
  482. out.append('%s changed from %s to %s' % (key, siga, sigb))
  483. changecount += 1
  484. elif siga is None:
  485. out.append('%s was added' % key)
  486. addcount += 1
  487. elif sigb is None:
  488. out.append('%s was removed' % key)
  489. removecount += 1
  490. out.append('Summary: %d tasks added, %d tasks removed, %d tasks modified (%.1f%%)' % (addcount, removecount, changecount, (changecount / float(len(bdict)) * 100)))
  491. return '\n'.join(out)
  492. def process_changes(repopath, revision1, revision2='HEAD', report_all=False, report_ver=False,
  493. sigs=False, sigsdiff=False, exclude_path=None):
  494. repo = git.Repo(repopath)
  495. assert repo.bare == False
  496. commit = repo.commit(revision1)
  497. diff = commit.diff(revision2)
  498. changes = []
  499. if sigs or sigsdiff:
  500. for d in diff.iter_change_type('M'):
  501. if d.a_blob.path == 'siglist.txt':
  502. changes.append(compare_siglists(d.a_blob, d.b_blob, taskdiff=sigsdiff))
  503. return changes
  504. for d in diff.iter_change_type('M'):
  505. path = os.path.dirname(d.a_blob.path)
  506. if path.startswith('packages/'):
  507. filename = os.path.basename(d.a_blob.path)
  508. if filename == 'latest':
  509. changes.extend(compare_dict_blobs(path, d.a_blob, d.b_blob, report_all, report_ver))
  510. elif filename.startswith('latest.'):
  511. chg = ChangeRecord(path, filename, d.a_blob.data_stream.read().decode('utf-8'), d.b_blob.data_stream.read().decode('utf-8'), True)
  512. changes.append(chg)
  513. elif path.startswith('images/'):
  514. filename = os.path.basename(d.a_blob.path)
  515. if filename in img_monitor_files:
  516. if filename == 'files-in-image.txt':
  517. alines = d.a_blob.data_stream.read().decode('utf-8').splitlines()
  518. blines = d.b_blob.data_stream.read().decode('utf-8').splitlines()
  519. filechanges = compare_file_lists(alines,blines)
  520. if filechanges:
  521. chg = ChangeRecord(path, filename, None, None, True)
  522. chg.filechanges = filechanges
  523. changes.append(chg)
  524. elif filename == 'installed-package-names.txt':
  525. alines = d.a_blob.data_stream.read().decode('utf-8').splitlines()
  526. blines = d.b_blob.data_stream.read().decode('utf-8').splitlines()
  527. filechanges = compare_lists(alines,blines)
  528. if filechanges:
  529. chg = ChangeRecord(path, filename, None, None, True)
  530. chg.filechanges = filechanges
  531. changes.append(chg)
  532. else:
  533. chg = ChangeRecord(path, filename, d.a_blob.data_stream.read().decode('utf-8'), d.b_blob.data_stream.read().decode('utf-8'), True)
  534. changes.append(chg)
  535. elif filename == 'image-info.txt':
  536. changes.extend(compare_dict_blobs(path, d.a_blob, d.b_blob, report_all, report_ver))
  537. elif '/image-files/' in path:
  538. chg = ChangeRecord(path, filename, d.a_blob.data_stream.read().decode('utf-8'), d.b_blob.data_stream.read().decode('utf-8'), True)
  539. changes.append(chg)
  540. # Look for added preinst/postinst/prerm/postrm
  541. # (without reporting newly added recipes)
  542. addedpkgs = []
  543. addedchanges = []
  544. for d in diff.iter_change_type('A'):
  545. path = os.path.dirname(d.b_blob.path)
  546. if path.startswith('packages/'):
  547. filename = os.path.basename(d.b_blob.path)
  548. if filename == 'latest':
  549. addedpkgs.append(path)
  550. elif filename.startswith('latest.'):
  551. chg = ChangeRecord(path, filename[7:], '', d.b_blob.data_stream.read().decode('utf-8'), True)
  552. addedchanges.append(chg)
  553. for chg in addedchanges:
  554. found = False
  555. for pkg in addedpkgs:
  556. if chg.path.startswith(pkg):
  557. found = True
  558. break
  559. if not found:
  560. changes.append(chg)
  561. # Look for cleared preinst/postinst/prerm/postrm
  562. for d in diff.iter_change_type('D'):
  563. path = os.path.dirname(d.a_blob.path)
  564. if path.startswith('packages/'):
  565. filename = os.path.basename(d.a_blob.path)
  566. if filename != 'latest' and filename.startswith('latest.'):
  567. chg = ChangeRecord(path, filename[7:], d.a_blob.data_stream.read().decode('utf-8'), '', True)
  568. changes.append(chg)
  569. # filter out unwanted paths
  570. if exclude_path:
  571. for chg in changes:
  572. if chg.filechanges:
  573. fchgs = []
  574. for fchg in chg.filechanges:
  575. for epath in exclude_path:
  576. if fchg.path.startswith(epath):
  577. break
  578. else:
  579. fchgs.append(fchg)
  580. chg.filechanges = fchgs
  581. if report_all:
  582. return changes
  583. else:
  584. return [chg for chg in changes if chg.monitored]