siggen.py 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737
  1. #
  2. # SPDX-License-Identifier: GPL-2.0-only
  3. #
  4. import hashlib
  5. import logging
  6. import os
  7. import re
  8. import tempfile
  9. import pickle
  10. import bb.data
  11. import difflib
  12. import simplediff
  13. from bb.checksum import FileChecksumCache
  14. logger = logging.getLogger('BitBake.SigGen')
  15. def init(d):
  16. siggens = [obj for obj in globals().values()
  17. if type(obj) is type and issubclass(obj, SignatureGenerator)]
  18. desired = d.getVar("BB_SIGNATURE_HANDLER") or "noop"
  19. for sg in siggens:
  20. if desired == sg.name:
  21. return sg(d)
  22. break
  23. else:
  24. logger.error("Invalid signature generator '%s', using default 'noop'\n"
  25. "Available generators: %s", desired,
  26. ', '.join(obj.name for obj in siggens))
  27. return SignatureGenerator(d)
  28. class SignatureGenerator(object):
  29. """
  30. """
  31. name = "noop"
  32. def __init__(self, data):
  33. self.basehash = {}
  34. self.taskhash = {}
  35. self.runtaskdeps = {}
  36. self.file_checksum_values = {}
  37. self.taints = {}
  38. def finalise(self, fn, d, varient):
  39. return
  40. def get_unihash(self, task):
  41. return self.taskhash[task]
  42. def get_taskhash(self, fn, task, deps, dataCache):
  43. return "0"
  44. def writeout_file_checksum_cache(self):
  45. """Write/update the file checksum cache onto disk"""
  46. return
  47. def stampfile(self, stampbase, file_name, taskname, extrainfo):
  48. return ("%s.%s.%s" % (stampbase, taskname, extrainfo)).rstrip('.')
  49. def stampcleanmask(self, stampbase, file_name, taskname, extrainfo):
  50. return ("%s.%s.%s" % (stampbase, taskname, extrainfo)).rstrip('.')
  51. def dump_sigtask(self, fn, task, stampbase, runtime):
  52. return
  53. def invalidate_task(self, task, d, fn):
  54. bb.build.del_stamp(task, d, fn)
  55. def dump_sigs(self, dataCache, options):
  56. return
  57. def get_taskdata(self):
  58. return (self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints, self.basehash)
  59. def set_taskdata(self, data):
  60. self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints, self.basehash = data
  61. def reset(self, data):
  62. self.__init__(data)
  63. class SignatureGeneratorBasic(SignatureGenerator):
  64. """
  65. """
  66. name = "basic"
  67. def __init__(self, data):
  68. self.basehash = {}
  69. self.taskhash = {}
  70. self.taskdeps = {}
  71. self.runtaskdeps = {}
  72. self.file_checksum_values = {}
  73. self.taints = {}
  74. self.gendeps = {}
  75. self.lookupcache = {}
  76. self.pkgnameextract = re.compile(r"(?P<fn>.*)\..*")
  77. self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST") or "").split())
  78. self.taskwhitelist = None
  79. self.init_rundepcheck(data)
  80. checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE")
  81. if checksum_cache_file:
  82. self.checksum_cache = FileChecksumCache()
  83. self.checksum_cache.init_cache(data, checksum_cache_file)
  84. else:
  85. self.checksum_cache = None
  86. def init_rundepcheck(self, data):
  87. self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST") or None
  88. if self.taskwhitelist:
  89. self.twl = re.compile(self.taskwhitelist)
  90. else:
  91. self.twl = None
  92. def _build_data(self, fn, d):
  93. ignore_mismatch = ((d.getVar("BB_HASH_IGNORE_MISMATCH") or '') == '1')
  94. tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d)
  95. taskdeps, basehash = bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, self.basewhitelist, fn)
  96. for task in tasklist:
  97. k = fn + "." + task
  98. if not ignore_mismatch and k in self.basehash and self.basehash[k] != basehash[k]:
  99. bb.error("When reparsing %s, the basehash value changed from %s to %s. The metadata is not deterministic and this needs to be fixed." % (k, self.basehash[k], basehash[k]))
  100. bb.error("The following commands may help:")
  101. cmd = "$ bitbake %s -c%s" % (d.getVar('PN'), task)
  102. # Make sure sigdata is dumped before run printdiff
  103. bb.error("%s -Snone" % cmd)
  104. bb.error("Then:")
  105. bb.error("%s -Sprintdiff\n" % cmd)
  106. self.basehash[k] = basehash[k]
  107. self.taskdeps[fn] = taskdeps
  108. self.gendeps[fn] = gendeps
  109. self.lookupcache[fn] = lookupcache
  110. return taskdeps
  111. def finalise(self, fn, d, variant):
  112. mc = d.getVar("__BBMULTICONFIG", False) or ""
  113. if variant or mc:
  114. fn = bb.cache.realfn2virtual(fn, variant, mc)
  115. try:
  116. taskdeps = self._build_data(fn, d)
  117. except bb.parse.SkipRecipe:
  118. raise
  119. except:
  120. bb.warn("Error during finalise of %s" % fn)
  121. raise
  122. #Slow but can be useful for debugging mismatched basehashes
  123. #for task in self.taskdeps[fn]:
  124. # self.dump_sigtask(fn, task, d.getVar("STAMP"), False)
  125. for task in taskdeps:
  126. d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + "." + task])
  127. def rundep_check(self, fn, recipename, task, dep, depname, dataCache):
  128. # Return True if we should keep the dependency, False to drop it
  129. # We only manipulate the dependencies for packages not in the whitelist
  130. if self.twl and not self.twl.search(recipename):
  131. # then process the actual dependencies
  132. if self.twl.search(depname):
  133. return False
  134. return True
  135. def read_taint(self, fn, task, stampbase):
  136. taint = None
  137. try:
  138. with open(stampbase + '.' + task + '.taint', 'r') as taintf:
  139. taint = taintf.read()
  140. except IOError:
  141. pass
  142. return taint
  143. def get_taskhash(self, fn, task, deps, dataCache):
  144. mc = ''
  145. if fn.startswith('multiconfig:'):
  146. mc = fn.split(':')[1]
  147. k = fn + "." + task
  148. data = dataCache.basetaskhash[k]
  149. self.basehash[k] = data
  150. self.runtaskdeps[k] = []
  151. self.file_checksum_values[k] = []
  152. recipename = dataCache.pkg_fn[fn]
  153. for dep in sorted(deps, key=clean_basepath):
  154. pkgname = self.pkgnameextract.search(dep).group('fn')
  155. if mc:
  156. depmc = pkgname.split(':')[1]
  157. if mc != depmc:
  158. continue
  159. if dep.startswith("multiconfig:") and not mc:
  160. continue
  161. depname = dataCache.pkg_fn[pkgname]
  162. if not self.rundep_check(fn, recipename, task, dep, depname, dataCache):
  163. continue
  164. if dep not in self.taskhash:
  165. bb.fatal("%s is not in taskhash, caller isn't calling in dependency order?" % dep)
  166. data = data + self.get_unihash(dep)
  167. self.runtaskdeps[k].append(dep)
  168. if task in dataCache.file_checksums[fn]:
  169. if self.checksum_cache:
  170. checksums = self.checksum_cache.get_checksums(dataCache.file_checksums[fn][task], recipename)
  171. else:
  172. checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename)
  173. for (f,cs) in checksums:
  174. self.file_checksum_values[k].append((f,cs))
  175. if cs:
  176. data = data + cs
  177. taskdep = dataCache.task_deps[fn]
  178. if 'nostamp' in taskdep and task in taskdep['nostamp']:
  179. # Nostamp tasks need an implicit taint so that they force any dependent tasks to run
  180. import uuid
  181. taint = str(uuid.uuid4())
  182. data = data + taint
  183. self.taints[k] = "nostamp:" + taint
  184. taint = self.read_taint(fn, task, dataCache.stamp[fn])
  185. if taint:
  186. data = data + taint
  187. self.taints[k] = taint
  188. logger.warning("%s is tainted from a forced run" % k)
  189. h = hashlib.sha256(data.encode("utf-8")).hexdigest()
  190. self.taskhash[k] = h
  191. #d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task])
  192. return h
  193. def writeout_file_checksum_cache(self):
  194. """Write/update the file checksum cache onto disk"""
  195. if self.checksum_cache:
  196. self.checksum_cache.save_extras()
  197. self.checksum_cache.save_merge()
  198. else:
  199. bb.fetch2.fetcher_parse_save()
  200. bb.fetch2.fetcher_parse_done()
  201. def dump_sigtask(self, fn, task, stampbase, runtime):
  202. k = fn + "." + task
  203. referencestamp = stampbase
  204. if isinstance(runtime, str) and runtime.startswith("customfile"):
  205. sigfile = stampbase
  206. referencestamp = runtime[11:]
  207. elif runtime and k in self.taskhash:
  208. sigfile = stampbase + "." + task + ".sigdata" + "." + self.taskhash[k]
  209. else:
  210. sigfile = stampbase + "." + task + ".sigbasedata" + "." + self.basehash[k]
  211. bb.utils.mkdirhier(os.path.dirname(sigfile))
  212. data = {}
  213. data['task'] = task
  214. data['basewhitelist'] = self.basewhitelist
  215. data['taskwhitelist'] = self.taskwhitelist
  216. data['taskdeps'] = self.taskdeps[fn][task]
  217. data['basehash'] = self.basehash[k]
  218. data['gendeps'] = {}
  219. data['varvals'] = {}
  220. data['varvals'][task] = self.lookupcache[fn][task]
  221. for dep in self.taskdeps[fn][task]:
  222. if dep in self.basewhitelist:
  223. continue
  224. data['gendeps'][dep] = self.gendeps[fn][dep]
  225. data['varvals'][dep] = self.lookupcache[fn][dep]
  226. if runtime and k in self.taskhash:
  227. data['runtaskdeps'] = self.runtaskdeps[k]
  228. data['file_checksum_values'] = [(os.path.basename(f), cs) for f,cs in self.file_checksum_values[k]]
  229. data['runtaskhashes'] = {}
  230. for dep in data['runtaskdeps']:
  231. data['runtaskhashes'][dep] = self.get_unihash(dep)
  232. data['taskhash'] = self.taskhash[k]
  233. taint = self.read_taint(fn, task, referencestamp)
  234. if taint:
  235. data['taint'] = taint
  236. if runtime and k in self.taints:
  237. if 'nostamp:' in self.taints[k]:
  238. data['taint'] = self.taints[k]
  239. computed_basehash = calc_basehash(data)
  240. if computed_basehash != self.basehash[k]:
  241. bb.error("Basehash mismatch %s versus %s for %s" % (computed_basehash, self.basehash[k], k))
  242. if runtime and k in self.taskhash:
  243. computed_taskhash = calc_taskhash(data)
  244. if computed_taskhash != self.taskhash[k]:
  245. bb.error("Taskhash mismatch %s versus %s for %s" % (computed_taskhash, self.taskhash[k], k))
  246. sigfile = sigfile.replace(self.taskhash[k], computed_taskhash)
  247. fd, tmpfile = tempfile.mkstemp(dir=os.path.dirname(sigfile), prefix="sigtask.")
  248. try:
  249. with os.fdopen(fd, "wb") as stream:
  250. p = pickle.dump(data, stream, -1)
  251. stream.flush()
  252. os.chmod(tmpfile, 0o664)
  253. os.rename(tmpfile, sigfile)
  254. except (OSError, IOError) as err:
  255. try:
  256. os.unlink(tmpfile)
  257. except OSError:
  258. pass
  259. raise err
  260. def dump_sigfn(self, fn, dataCaches, options):
  261. if fn in self.taskdeps:
  262. for task in self.taskdeps[fn]:
  263. tid = fn + ":" + task
  264. (mc, _, _) = bb.runqueue.split_tid(tid)
  265. k = fn + "." + task
  266. if k not in self.taskhash:
  267. continue
  268. if dataCaches[mc].basetaskhash[k] != self.basehash[k]:
  269. bb.error("Bitbake's cached basehash does not match the one we just generated (%s)!" % k)
  270. bb.error("The mismatched hashes were %s and %s" % (dataCaches[mc].basetaskhash[k], self.basehash[k]))
  271. self.dump_sigtask(fn, task, dataCaches[mc].stamp[fn], True)
  272. class SignatureGeneratorBasicHash(SignatureGeneratorBasic):
  273. name = "basichash"
  274. def get_stampfile_hash(self, task):
  275. if task in self.taskhash:
  276. return self.taskhash[task]
  277. # If task is not in basehash, then error
  278. return self.basehash[task]
  279. def stampfile(self, stampbase, fn, taskname, extrainfo, clean=False):
  280. if taskname != "do_setscene" and taskname.endswith("_setscene"):
  281. k = fn + "." + taskname[:-9]
  282. else:
  283. k = fn + "." + taskname
  284. if clean:
  285. h = "*"
  286. else:
  287. h = self.get_stampfile_hash(k)
  288. return ("%s.%s.%s.%s" % (stampbase, taskname, h, extrainfo)).rstrip('.')
  289. def stampcleanmask(self, stampbase, fn, taskname, extrainfo):
  290. return self.stampfile(stampbase, fn, taskname, extrainfo, clean=True)
  291. def invalidate_task(self, task, d, fn):
  292. bb.note("Tainting hash to force rebuild of task %s, %s" % (fn, task))
  293. bb.build.write_taint(task, d, fn)
  294. def dump_this_task(outfile, d):
  295. import bb.parse
  296. fn = d.getVar("BB_FILENAME")
  297. task = "do_" + d.getVar("BB_CURRENTTASK")
  298. referencestamp = bb.build.stamp_internal(task, d, None, True)
  299. bb.parse.siggen.dump_sigtask(fn, task, outfile, "customfile:" + referencestamp)
  300. def init_colors(enable_color):
  301. """Initialise colour dict for passing to compare_sigfiles()"""
  302. # First set up the colours
  303. colors = {'color_title': '\033[1m',
  304. 'color_default': '\033[0m',
  305. 'color_add': '\033[0;32m',
  306. 'color_remove': '\033[0;31m',
  307. }
  308. # Leave all keys present but clear the values
  309. if not enable_color:
  310. for k in colors.keys():
  311. colors[k] = ''
  312. return colors
  313. def worddiff_str(oldstr, newstr, colors=None):
  314. if not colors:
  315. colors = init_colors(False)
  316. diff = simplediff.diff(oldstr.split(' '), newstr.split(' '))
  317. ret = []
  318. for change, value in diff:
  319. value = ' '.join(value)
  320. if change == '=':
  321. ret.append(value)
  322. elif change == '+':
  323. item = '{color_add}{{+{value}+}}{color_default}'.format(value=value, **colors)
  324. ret.append(item)
  325. elif change == '-':
  326. item = '{color_remove}[-{value}-]{color_default}'.format(value=value, **colors)
  327. ret.append(item)
  328. whitespace_note = ''
  329. if oldstr != newstr and ' '.join(oldstr.split()) == ' '.join(newstr.split()):
  330. whitespace_note = ' (whitespace changed)'
  331. return '"%s"%s' % (' '.join(ret), whitespace_note)
  332. def list_inline_diff(oldlist, newlist, colors=None):
  333. if not colors:
  334. colors = init_colors(False)
  335. diff = simplediff.diff(oldlist, newlist)
  336. ret = []
  337. for change, value in diff:
  338. value = ' '.join(value)
  339. if change == '=':
  340. ret.append("'%s'" % value)
  341. elif change == '+':
  342. item = '{color_add}+{value}{color_default}'.format(value=value, **colors)
  343. ret.append(item)
  344. elif change == '-':
  345. item = '{color_remove}-{value}{color_default}'.format(value=value, **colors)
  346. ret.append(item)
  347. return '[%s]' % (', '.join(ret))
  348. def clean_basepath(a):
  349. mc = None
  350. if a.startswith("multiconfig:"):
  351. _, mc, a = a.split(":", 2)
  352. b = a.rsplit("/", 2)[1] + '/' + a.rsplit("/", 2)[2]
  353. if a.startswith("virtual:"):
  354. b = b + ":" + a.rsplit(":", 1)[0]
  355. if mc:
  356. b = b + ":multiconfig:" + mc
  357. return b
  358. def clean_basepaths(a):
  359. b = {}
  360. for x in a:
  361. b[clean_basepath(x)] = a[x]
  362. return b
  363. def clean_basepaths_list(a):
  364. b = []
  365. for x in a:
  366. b.append(clean_basepath(x))
  367. return b
  368. def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
  369. output = []
  370. colors = init_colors(color)
  371. def color_format(formatstr, **values):
  372. """
  373. Return colour formatted string.
  374. NOTE: call with the format string, not an already formatted string
  375. containing values (otherwise you could have trouble with { and }
  376. characters)
  377. """
  378. if not formatstr.endswith('{color_default}'):
  379. formatstr += '{color_default}'
  380. # In newer python 3 versions you can pass both of these directly,
  381. # but we only require 3.4 at the moment
  382. formatparams = {}
  383. formatparams.update(colors)
  384. formatparams.update(values)
  385. return formatstr.format(**formatparams)
  386. with open(a, 'rb') as f:
  387. p1 = pickle.Unpickler(f)
  388. a_data = p1.load()
  389. with open(b, 'rb') as f:
  390. p2 = pickle.Unpickler(f)
  391. b_data = p2.load()
  392. def dict_diff(a, b, whitelist=set()):
  393. sa = set(a.keys())
  394. sb = set(b.keys())
  395. common = sa & sb
  396. changed = set()
  397. for i in common:
  398. if a[i] != b[i] and i not in whitelist:
  399. changed.add(i)
  400. added = sb - sa
  401. removed = sa - sb
  402. return changed, added, removed
  403. def file_checksums_diff(a, b):
  404. from collections import Counter
  405. # Handle old siginfo format
  406. if isinstance(a, dict):
  407. a = [(os.path.basename(f), cs) for f, cs in a.items()]
  408. if isinstance(b, dict):
  409. b = [(os.path.basename(f), cs) for f, cs in b.items()]
  410. # Compare lists, ensuring we can handle duplicate filenames if they exist
  411. removedcount = Counter(a)
  412. removedcount.subtract(b)
  413. addedcount = Counter(b)
  414. addedcount.subtract(a)
  415. added = []
  416. for x in b:
  417. if addedcount[x] > 0:
  418. addedcount[x] -= 1
  419. added.append(x)
  420. removed = []
  421. changed = []
  422. for x in a:
  423. if removedcount[x] > 0:
  424. removedcount[x] -= 1
  425. for y in added:
  426. if y[0] == x[0]:
  427. changed.append((x[0], x[1], y[1]))
  428. added.remove(y)
  429. break
  430. else:
  431. removed.append(x)
  432. added = [x[0] for x in added]
  433. removed = [x[0] for x in removed]
  434. return changed, added, removed
  435. if 'basewhitelist' in a_data and a_data['basewhitelist'] != b_data['basewhitelist']:
  436. output.append(color_format("{color_title}basewhitelist changed{color_default} from '%s' to '%s'") % (a_data['basewhitelist'], b_data['basewhitelist']))
  437. if a_data['basewhitelist'] and b_data['basewhitelist']:
  438. output.append("changed items: %s" % a_data['basewhitelist'].symmetric_difference(b_data['basewhitelist']))
  439. if 'taskwhitelist' in a_data and a_data['taskwhitelist'] != b_data['taskwhitelist']:
  440. output.append(color_format("{color_title}taskwhitelist changed{color_default} from '%s' to '%s'") % (a_data['taskwhitelist'], b_data['taskwhitelist']))
  441. if a_data['taskwhitelist'] and b_data['taskwhitelist']:
  442. output.append("changed items: %s" % a_data['taskwhitelist'].symmetric_difference(b_data['taskwhitelist']))
  443. if a_data['taskdeps'] != b_data['taskdeps']:
  444. output.append(color_format("{color_title}Task dependencies changed{color_default} from:\n%s\nto:\n%s") % (sorted(a_data['taskdeps']), sorted(b_data['taskdeps'])))
  445. if a_data['basehash'] != b_data['basehash'] and not collapsed:
  446. output.append(color_format("{color_title}basehash changed{color_default} from %s to %s") % (a_data['basehash'], b_data['basehash']))
  447. changed, added, removed = dict_diff(a_data['gendeps'], b_data['gendeps'], a_data['basewhitelist'] & b_data['basewhitelist'])
  448. if changed:
  449. for dep in changed:
  450. output.append(color_format("{color_title}List of dependencies for variable %s changed from '{color_default}%s{color_title}' to '{color_default}%s{color_title}'") % (dep, a_data['gendeps'][dep], b_data['gendeps'][dep]))
  451. if a_data['gendeps'][dep] and b_data['gendeps'][dep]:
  452. output.append("changed items: %s" % a_data['gendeps'][dep].symmetric_difference(b_data['gendeps'][dep]))
  453. if added:
  454. for dep in added:
  455. output.append(color_format("{color_title}Dependency on variable %s was added") % (dep))
  456. if removed:
  457. for dep in removed:
  458. output.append(color_format("{color_title}Dependency on Variable %s was removed") % (dep))
  459. changed, added, removed = dict_diff(a_data['varvals'], b_data['varvals'])
  460. if changed:
  461. for dep in changed:
  462. oldval = a_data['varvals'][dep]
  463. newval = b_data['varvals'][dep]
  464. if newval and oldval and ('\n' in oldval or '\n' in newval):
  465. diff = difflib.unified_diff(oldval.splitlines(), newval.splitlines(), lineterm='')
  466. # Cut off the first two lines, since we aren't interested in
  467. # the old/new filename (they are blank anyway in this case)
  468. difflines = list(diff)[2:]
  469. if color:
  470. # Add colour to diff output
  471. for i, line in enumerate(difflines):
  472. if line.startswith('+'):
  473. line = color_format('{color_add}{line}', line=line)
  474. difflines[i] = line
  475. elif line.startswith('-'):
  476. line = color_format('{color_remove}{line}', line=line)
  477. difflines[i] = line
  478. output.append(color_format("{color_title}Variable {var} value changed:{color_default}\n{diff}", var=dep, diff='\n'.join(difflines)))
  479. elif newval and oldval and (' ' in oldval or ' ' in newval):
  480. output.append(color_format("{color_title}Variable {var} value changed:{color_default}\n{diff}", var=dep, diff=worddiff_str(oldval, newval, colors)))
  481. else:
  482. output.append(color_format("{color_title}Variable {var} value changed from '{color_default}{oldval}{color_title}' to '{color_default}{newval}{color_title}'{color_default}", var=dep, oldval=oldval, newval=newval))
  483. if not 'file_checksum_values' in a_data:
  484. a_data['file_checksum_values'] = {}
  485. if not 'file_checksum_values' in b_data:
  486. b_data['file_checksum_values'] = {}
  487. changed, added, removed = file_checksums_diff(a_data['file_checksum_values'], b_data['file_checksum_values'])
  488. if changed:
  489. for f, old, new in changed:
  490. output.append(color_format("{color_title}Checksum for file %s changed{color_default} from %s to %s") % (f, old, new))
  491. if added:
  492. for f in added:
  493. output.append(color_format("{color_title}Dependency on checksum of file %s was added") % (f))
  494. if removed:
  495. for f in removed:
  496. output.append(color_format("{color_title}Dependency on checksum of file %s was removed") % (f))
  497. if not 'runtaskdeps' in a_data:
  498. a_data['runtaskdeps'] = {}
  499. if not 'runtaskdeps' in b_data:
  500. b_data['runtaskdeps'] = {}
  501. if not collapsed:
  502. if len(a_data['runtaskdeps']) != len(b_data['runtaskdeps']):
  503. changed = ["Number of task dependencies changed"]
  504. else:
  505. changed = []
  506. for idx, task in enumerate(a_data['runtaskdeps']):
  507. a = a_data['runtaskdeps'][idx]
  508. b = b_data['runtaskdeps'][idx]
  509. if a_data['runtaskhashes'][a] != b_data['runtaskhashes'][b] and not collapsed:
  510. changed.append("%s with hash %s\n changed to\n%s with hash %s" % (clean_basepath(a), a_data['runtaskhashes'][a], clean_basepath(b), b_data['runtaskhashes'][b]))
  511. if changed:
  512. clean_a = clean_basepaths_list(a_data['runtaskdeps'])
  513. clean_b = clean_basepaths_list(b_data['runtaskdeps'])
  514. if clean_a != clean_b:
  515. output.append(color_format("{color_title}runtaskdeps changed:{color_default}\n%s") % list_inline_diff(clean_a, clean_b, colors))
  516. else:
  517. output.append(color_format("{color_title}runtaskdeps changed:"))
  518. output.append("\n".join(changed))
  519. if 'runtaskhashes' in a_data and 'runtaskhashes' in b_data:
  520. a = a_data['runtaskhashes']
  521. b = b_data['runtaskhashes']
  522. changed, added, removed = dict_diff(a, b)
  523. if added:
  524. for dep in added:
  525. bdep_found = False
  526. if removed:
  527. for bdep in removed:
  528. if b[dep] == a[bdep]:
  529. #output.append("Dependency on task %s was replaced by %s with same hash" % (dep, bdep))
  530. bdep_found = True
  531. if not bdep_found:
  532. output.append(color_format("{color_title}Dependency on task %s was added{color_default} with hash %s") % (clean_basepath(dep), b[dep]))
  533. if removed:
  534. for dep in removed:
  535. adep_found = False
  536. if added:
  537. for adep in added:
  538. if b[adep] == a[dep]:
  539. #output.append("Dependency on task %s was replaced by %s with same hash" % (adep, dep))
  540. adep_found = True
  541. if not adep_found:
  542. output.append(color_format("{color_title}Dependency on task %s was removed{color_default} with hash %s") % (clean_basepath(dep), a[dep]))
  543. if changed:
  544. for dep in changed:
  545. if not collapsed:
  546. output.append(color_format("{color_title}Hash for dependent task %s changed{color_default} from %s to %s") % (clean_basepath(dep), a[dep], b[dep]))
  547. if callable(recursecb):
  548. recout = recursecb(dep, a[dep], b[dep])
  549. if recout:
  550. if collapsed:
  551. output.extend(recout)
  552. else:
  553. # If a dependent hash changed, might as well print the line above and then defer to the changes in
  554. # that hash since in all likelyhood, they're the same changes this task also saw.
  555. output = [output[-1]] + recout
  556. a_taint = a_data.get('taint', None)
  557. b_taint = b_data.get('taint', None)
  558. if a_taint != b_taint:
  559. if a_taint.startswith('nostamp:'):
  560. a_taint = a_taint.replace('nostamp:', 'nostamp(uuid4):')
  561. if b_taint.startswith('nostamp:'):
  562. b_taint = b_taint.replace('nostamp:', 'nostamp(uuid4):')
  563. output.append(color_format("{color_title}Taint (by forced/invalidated task) changed{color_default} from %s to %s") % (a_taint, b_taint))
  564. return output
  565. def calc_basehash(sigdata):
  566. task = sigdata['task']
  567. basedata = sigdata['varvals'][task]
  568. if basedata is None:
  569. basedata = ''
  570. alldeps = sigdata['taskdeps']
  571. for dep in alldeps:
  572. basedata = basedata + dep
  573. val = sigdata['varvals'][dep]
  574. if val is not None:
  575. basedata = basedata + str(val)
  576. return hashlib.sha256(basedata.encode("utf-8")).hexdigest()
  577. def calc_taskhash(sigdata):
  578. data = sigdata['basehash']
  579. for dep in sigdata['runtaskdeps']:
  580. data = data + sigdata['runtaskhashes'][dep]
  581. for c in sigdata['file_checksum_values']:
  582. if c[1]:
  583. data = data + c[1]
  584. if 'taint' in sigdata:
  585. if 'nostamp:' in sigdata['taint']:
  586. data = data + sigdata['taint'][8:]
  587. else:
  588. data = data + sigdata['taint']
  589. return hashlib.sha256(data.encode("utf-8")).hexdigest()
  590. def dump_sigfile(a):
  591. output = []
  592. with open(a, 'rb') as f:
  593. p1 = pickle.Unpickler(f)
  594. a_data = p1.load()
  595. output.append("basewhitelist: %s" % (a_data['basewhitelist']))
  596. output.append("taskwhitelist: %s" % (a_data['taskwhitelist']))
  597. output.append("Task dependencies: %s" % (sorted(a_data['taskdeps'])))
  598. output.append("basehash: %s" % (a_data['basehash']))
  599. for dep in a_data['gendeps']:
  600. output.append("List of dependencies for variable %s is %s" % (dep, a_data['gendeps'][dep]))
  601. for dep in a_data['varvals']:
  602. output.append("Variable %s value is %s" % (dep, a_data['varvals'][dep]))
  603. if 'runtaskdeps' in a_data:
  604. output.append("Tasks this task depends on: %s" % (a_data['runtaskdeps']))
  605. if 'file_checksum_values' in a_data:
  606. output.append("This task depends on the checksums of files: %s" % (a_data['file_checksum_values']))
  607. if 'runtaskhashes' in a_data:
  608. for dep in a_data['runtaskhashes']:
  609. output.append("Hash for dependent task %s is %s" % (dep, a_data['runtaskhashes'][dep]))
  610. if 'taint' in a_data:
  611. if a_data['taint'].startswith('nostamp:'):
  612. msg = a_data['taint'].replace('nostamp:', 'nostamp(uuid4):')
  613. else:
  614. msg = a_data['taint']
  615. output.append("Tainted (by forced/invalidated task): %s" % msg)
  616. if 'task' in a_data:
  617. computed_basehash = calc_basehash(a_data)
  618. output.append("Computed base hash is %s and from file %s" % (computed_basehash, a_data['basehash']))
  619. else:
  620. output.append("Unable to compute base hash")
  621. computed_taskhash = calc_taskhash(a_data)
  622. output.append("Computed task hash is %s" % computed_taskhash)
  623. return output