siggen.py 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723
  1. #
  2. # SPDX-License-Identifier: GPL-2.0-only
  3. #
  4. import hashlib
  5. import logging
  6. import os
  7. import re
  8. import tempfile
  9. import pickle
  10. import bb.data
  11. import difflib
  12. import simplediff
  13. from bb.checksum import FileChecksumCache
  14. logger = logging.getLogger('BitBake.SigGen')
  15. def init(d):
  16. siggens = [obj for obj in globals().values()
  17. if type(obj) is type and issubclass(obj, SignatureGenerator)]
  18. desired = d.getVar("BB_SIGNATURE_HANDLER") or "noop"
  19. for sg in siggens:
  20. if desired == sg.name:
  21. return sg(d)
  22. break
  23. else:
  24. logger.error("Invalid signature generator '%s', using default 'noop'\n"
  25. "Available generators: %s", desired,
  26. ', '.join(obj.name for obj in siggens))
  27. return SignatureGenerator(d)
  28. class SignatureGenerator(object):
  29. """
  30. """
  31. name = "noop"
  32. def __init__(self, data):
  33. self.basehash = {}
  34. self.taskhash = {}
  35. self.runtaskdeps = {}
  36. self.file_checksum_values = {}
  37. self.taints = {}
  38. def finalise(self, fn, d, varient):
  39. return
  40. def get_unihash(self, task):
  41. return self.taskhash[task]
  42. def get_taskhash(self, fn, task, deps, dataCache):
  43. return "0"
  44. def writeout_file_checksum_cache(self):
  45. """Write/update the file checksum cache onto disk"""
  46. return
  47. def stampfile(self, stampbase, file_name, taskname, extrainfo):
  48. return ("%s.%s.%s" % (stampbase, taskname, extrainfo)).rstrip('.')
  49. def stampcleanmask(self, stampbase, file_name, taskname, extrainfo):
  50. return ("%s.%s.%s" % (stampbase, taskname, extrainfo)).rstrip('.')
  51. def dump_sigtask(self, fn, task, stampbase, runtime):
  52. return
  53. def invalidate_task(self, task, d, fn):
  54. bb.build.del_stamp(task, d, fn)
  55. def dump_sigs(self, dataCache, options):
  56. return
  57. def get_taskdata(self):
  58. return (self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints, self.basehash)
  59. def set_taskdata(self, data):
  60. self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints, self.basehash = data
  61. def reset(self, data):
  62. self.__init__(data)
  63. class SignatureGeneratorBasic(SignatureGenerator):
  64. """
  65. """
  66. name = "basic"
  67. def __init__(self, data):
  68. self.basehash = {}
  69. self.taskhash = {}
  70. self.taskdeps = {}
  71. self.runtaskdeps = {}
  72. self.file_checksum_values = {}
  73. self.taints = {}
  74. self.gendeps = {}
  75. self.lookupcache = {}
  76. self.pkgnameextract = re.compile(r"(?P<fn>.*)\..*")
  77. self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST") or "").split())
  78. self.taskwhitelist = None
  79. self.init_rundepcheck(data)
  80. checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE")
  81. if checksum_cache_file:
  82. self.checksum_cache = FileChecksumCache()
  83. self.checksum_cache.init_cache(data, checksum_cache_file)
  84. else:
  85. self.checksum_cache = None
  86. def init_rundepcheck(self, data):
  87. self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST") or None
  88. if self.taskwhitelist:
  89. self.twl = re.compile(self.taskwhitelist)
  90. else:
  91. self.twl = None
  92. def _build_data(self, fn, d):
  93. ignore_mismatch = ((d.getVar("BB_HASH_IGNORE_MISMATCH") or '') == '1')
  94. tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d)
  95. taskdeps, basehash = bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, self.basewhitelist, fn)
  96. for task in tasklist:
  97. k = fn + "." + task
  98. if not ignore_mismatch and k in self.basehash and self.basehash[k] != basehash[k]:
  99. bb.error("When reparsing %s, the basehash value changed from %s to %s. The metadata is not deterministic and this needs to be fixed." % (k, self.basehash[k], basehash[k]))
  100. self.basehash[k] = basehash[k]
  101. self.taskdeps[fn] = taskdeps
  102. self.gendeps[fn] = gendeps
  103. self.lookupcache[fn] = lookupcache
  104. return taskdeps
  105. def finalise(self, fn, d, variant):
  106. mc = d.getVar("__BBMULTICONFIG", False) or ""
  107. if variant or mc:
  108. fn = bb.cache.realfn2virtual(fn, variant, mc)
  109. try:
  110. taskdeps = self._build_data(fn, d)
  111. except bb.parse.SkipRecipe:
  112. raise
  113. except:
  114. bb.warn("Error during finalise of %s" % fn)
  115. raise
  116. #Slow but can be useful for debugging mismatched basehashes
  117. #for task in self.taskdeps[fn]:
  118. # self.dump_sigtask(fn, task, d.getVar("STAMP"), False)
  119. for task in taskdeps:
  120. d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + "." + task])
  121. def rundep_check(self, fn, recipename, task, dep, depname, dataCache):
  122. # Return True if we should keep the dependency, False to drop it
  123. # We only manipulate the dependencies for packages not in the whitelist
  124. if self.twl and not self.twl.search(recipename):
  125. # then process the actual dependencies
  126. if self.twl.search(depname):
  127. return False
  128. return True
  129. def read_taint(self, fn, task, stampbase):
  130. taint = None
  131. try:
  132. with open(stampbase + '.' + task + '.taint', 'r') as taintf:
  133. taint = taintf.read()
  134. except IOError:
  135. pass
  136. return taint
  137. def get_taskhash(self, fn, task, deps, dataCache):
  138. mc = ''
  139. if fn.startswith('multiconfig:'):
  140. mc = fn.split(':')[1]
  141. k = fn + "." + task
  142. data = dataCache.basetaskhash[k]
  143. self.basehash[k] = data
  144. self.runtaskdeps[k] = []
  145. self.file_checksum_values[k] = []
  146. recipename = dataCache.pkg_fn[fn]
  147. for dep in sorted(deps, key=clean_basepath):
  148. pkgname = self.pkgnameextract.search(dep).group('fn')
  149. if mc:
  150. depmc = pkgname.split(':')[1]
  151. if mc != depmc:
  152. continue
  153. if dep.startswith("multiconfig:") and not mc:
  154. continue
  155. depname = dataCache.pkg_fn[pkgname]
  156. if not self.rundep_check(fn, recipename, task, dep, depname, dataCache):
  157. continue
  158. if dep not in self.taskhash:
  159. bb.fatal("%s is not in taskhash, caller isn't calling in dependency order?" % dep)
  160. data = data + self.get_unihash(dep)
  161. self.runtaskdeps[k].append(dep)
  162. if task in dataCache.file_checksums[fn]:
  163. if self.checksum_cache:
  164. checksums = self.checksum_cache.get_checksums(dataCache.file_checksums[fn][task], recipename)
  165. else:
  166. checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename)
  167. for (f,cs) in checksums:
  168. self.file_checksum_values[k].append((f,cs))
  169. if cs:
  170. data = data + cs
  171. taskdep = dataCache.task_deps[fn]
  172. if 'nostamp' in taskdep and task in taskdep['nostamp']:
  173. # Nostamp tasks need an implicit taint so that they force any dependent tasks to run
  174. import uuid
  175. taint = str(uuid.uuid4())
  176. data = data + taint
  177. self.taints[k] = "nostamp:" + taint
  178. taint = self.read_taint(fn, task, dataCache.stamp[fn])
  179. if taint:
  180. data = data + taint
  181. self.taints[k] = taint
  182. logger.warning("%s is tainted from a forced run" % k)
  183. h = hashlib.sha256(data.encode("utf-8")).hexdigest()
  184. self.taskhash[k] = h
  185. #d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task])
  186. return h
  187. def writeout_file_checksum_cache(self):
  188. """Write/update the file checksum cache onto disk"""
  189. if self.checksum_cache:
  190. self.checksum_cache.save_extras()
  191. self.checksum_cache.save_merge()
  192. else:
  193. bb.fetch2.fetcher_parse_save()
  194. bb.fetch2.fetcher_parse_done()
  195. def dump_sigtask(self, fn, task, stampbase, runtime):
  196. k = fn + "." + task
  197. referencestamp = stampbase
  198. if isinstance(runtime, str) and runtime.startswith("customfile"):
  199. sigfile = stampbase
  200. referencestamp = runtime[11:]
  201. elif runtime and k in self.taskhash:
  202. sigfile = stampbase + "." + task + ".sigdata" + "." + self.taskhash[k]
  203. else:
  204. sigfile = stampbase + "." + task + ".sigbasedata" + "." + self.basehash[k]
  205. bb.utils.mkdirhier(os.path.dirname(sigfile))
  206. data = {}
  207. data['task'] = task
  208. data['basewhitelist'] = self.basewhitelist
  209. data['taskwhitelist'] = self.taskwhitelist
  210. data['taskdeps'] = self.taskdeps[fn][task]
  211. data['basehash'] = self.basehash[k]
  212. data['gendeps'] = {}
  213. data['varvals'] = {}
  214. data['varvals'][task] = self.lookupcache[fn][task]
  215. for dep in self.taskdeps[fn][task]:
  216. if dep in self.basewhitelist:
  217. continue
  218. data['gendeps'][dep] = self.gendeps[fn][dep]
  219. data['varvals'][dep] = self.lookupcache[fn][dep]
  220. if runtime and k in self.taskhash:
  221. data['runtaskdeps'] = self.runtaskdeps[k]
  222. data['file_checksum_values'] = [(os.path.basename(f), cs) for f,cs in self.file_checksum_values[k]]
  223. data['runtaskhashes'] = {}
  224. for dep in data['runtaskdeps']:
  225. data['runtaskhashes'][dep] = self.get_unihash(dep)
  226. data['taskhash'] = self.taskhash[k]
  227. taint = self.read_taint(fn, task, referencestamp)
  228. if taint:
  229. data['taint'] = taint
  230. if runtime and k in self.taints:
  231. if 'nostamp:' in self.taints[k]:
  232. data['taint'] = self.taints[k]
  233. computed_basehash = calc_basehash(data)
  234. if computed_basehash != self.basehash[k]:
  235. bb.error("Basehash mismatch %s versus %s for %s" % (computed_basehash, self.basehash[k], k))
  236. if runtime and k in self.taskhash:
  237. computed_taskhash = calc_taskhash(data)
  238. if computed_taskhash != self.taskhash[k]:
  239. bb.error("Taskhash mismatch %s versus %s for %s" % (computed_taskhash, self.taskhash[k], k))
  240. sigfile = sigfile.replace(self.taskhash[k], computed_taskhash)
  241. fd, tmpfile = tempfile.mkstemp(dir=os.path.dirname(sigfile), prefix="sigtask.")
  242. try:
  243. with os.fdopen(fd, "wb") as stream:
  244. p = pickle.dump(data, stream, -1)
  245. stream.flush()
  246. os.chmod(tmpfile, 0o664)
  247. os.rename(tmpfile, sigfile)
  248. except (OSError, IOError) as err:
  249. try:
  250. os.unlink(tmpfile)
  251. except OSError:
  252. pass
  253. raise err
  254. def dump_sigfn(self, fn, dataCaches, options):
  255. if fn in self.taskdeps:
  256. for task in self.taskdeps[fn]:
  257. tid = fn + ":" + task
  258. (mc, _, _) = bb.runqueue.split_tid(tid)
  259. k = fn + "." + task
  260. if k not in self.taskhash:
  261. continue
  262. if dataCaches[mc].basetaskhash[k] != self.basehash[k]:
  263. bb.error("Bitbake's cached basehash does not match the one we just generated (%s)!" % k)
  264. bb.error("The mismatched hashes were %s and %s" % (dataCaches[mc].basetaskhash[k], self.basehash[k]))
  265. self.dump_sigtask(fn, task, dataCaches[mc].stamp[fn], True)
  266. class SignatureGeneratorBasicHash(SignatureGeneratorBasic):
  267. name = "basichash"
  268. def get_stampfile_hash(self, task):
  269. if task in self.taskhash:
  270. return self.taskhash[task]
  271. # If task is not in basehash, then error
  272. return self.basehash[task]
  273. def stampfile(self, stampbase, fn, taskname, extrainfo, clean=False):
  274. if taskname != "do_setscene" and taskname.endswith("_setscene"):
  275. k = fn + "." + taskname[:-9]
  276. else:
  277. k = fn + "." + taskname
  278. if clean:
  279. h = "*"
  280. else:
  281. h = self.get_stampfile_hash(k)
  282. return ("%s.%s.%s.%s" % (stampbase, taskname, h, extrainfo)).rstrip('.')
  283. def stampcleanmask(self, stampbase, fn, taskname, extrainfo):
  284. return self.stampfile(stampbase, fn, taskname, extrainfo, clean=True)
  285. def invalidate_task(self, task, d, fn):
  286. bb.note("Tainting hash to force rebuild of task %s, %s" % (fn, task))
  287. bb.build.write_taint(task, d, fn)
  288. def dump_this_task(outfile, d):
  289. import bb.parse
  290. fn = d.getVar("BB_FILENAME")
  291. task = "do_" + d.getVar("BB_CURRENTTASK")
  292. referencestamp = bb.build.stamp_internal(task, d, None, True)
  293. bb.parse.siggen.dump_sigtask(fn, task, outfile, "customfile:" + referencestamp)
  294. def init_colors(enable_color):
  295. """Initialise colour dict for passing to compare_sigfiles()"""
  296. # First set up the colours
  297. colors = {'color_title': '\033[1m',
  298. 'color_default': '\033[0m',
  299. 'color_add': '\033[0;32m',
  300. 'color_remove': '\033[0;31m',
  301. }
  302. # Leave all keys present but clear the values
  303. if not enable_color:
  304. for k in colors.keys():
  305. colors[k] = ''
  306. return colors
  307. def worddiff_str(oldstr, newstr, colors=None):
  308. if not colors:
  309. colors = init_colors(False)
  310. diff = simplediff.diff(oldstr.split(' '), newstr.split(' '))
  311. ret = []
  312. for change, value in diff:
  313. value = ' '.join(value)
  314. if change == '=':
  315. ret.append(value)
  316. elif change == '+':
  317. item = '{color_add}{{+{value}+}}{color_default}'.format(value=value, **colors)
  318. ret.append(item)
  319. elif change == '-':
  320. item = '{color_remove}[-{value}-]{color_default}'.format(value=value, **colors)
  321. ret.append(item)
  322. whitespace_note = ''
  323. if oldstr != newstr and ' '.join(oldstr.split()) == ' '.join(newstr.split()):
  324. whitespace_note = ' (whitespace changed)'
  325. return '"%s"%s' % (' '.join(ret), whitespace_note)
  326. def list_inline_diff(oldlist, newlist, colors=None):
  327. if not colors:
  328. colors = init_colors(False)
  329. diff = simplediff.diff(oldlist, newlist)
  330. ret = []
  331. for change, value in diff:
  332. value = ' '.join(value)
  333. if change == '=':
  334. ret.append("'%s'" % value)
  335. elif change == '+':
  336. item = '{color_add}+{value}{color_default}'.format(value=value, **colors)
  337. ret.append(item)
  338. elif change == '-':
  339. item = '{color_remove}-{value}{color_default}'.format(value=value, **colors)
  340. ret.append(item)
  341. return '[%s]' % (', '.join(ret))
  342. def clean_basepath(a):
  343. mc = None
  344. if a.startswith("multiconfig:"):
  345. _, mc, a = a.split(":", 2)
  346. b = a.rsplit("/", 2)[1] + '/' + a.rsplit("/", 2)[2]
  347. if a.startswith("virtual:"):
  348. b = b + ":" + a.rsplit(":", 1)[0]
  349. if mc:
  350. b = b + ":multiconfig:" + mc
  351. return b
  352. def clean_basepaths(a):
  353. b = {}
  354. for x in a:
  355. b[clean_basepath(x)] = a[x]
  356. return b
  357. def clean_basepaths_list(a):
  358. b = []
  359. for x in a:
  360. b.append(clean_basepath(x))
  361. return b
  362. def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
  363. output = []
  364. colors = init_colors(color)
  365. def color_format(formatstr, **values):
  366. """
  367. Return colour formatted string.
  368. NOTE: call with the format string, not an already formatted string
  369. containing values (otherwise you could have trouble with { and }
  370. characters)
  371. """
  372. if not formatstr.endswith('{color_default}'):
  373. formatstr += '{color_default}'
  374. # In newer python 3 versions you can pass both of these directly,
  375. # but we only require 3.4 at the moment
  376. formatparams = {}
  377. formatparams.update(colors)
  378. formatparams.update(values)
  379. return formatstr.format(**formatparams)
  380. with open(a, 'rb') as f:
  381. p1 = pickle.Unpickler(f)
  382. a_data = p1.load()
  383. with open(b, 'rb') as f:
  384. p2 = pickle.Unpickler(f)
  385. b_data = p2.load()
  386. def dict_diff(a, b, whitelist=set()):
  387. sa = set(a.keys())
  388. sb = set(b.keys())
  389. common = sa & sb
  390. changed = set()
  391. for i in common:
  392. if a[i] != b[i] and i not in whitelist:
  393. changed.add(i)
  394. added = sb - sa
  395. removed = sa - sb
  396. return changed, added, removed
  397. def file_checksums_diff(a, b):
  398. from collections import Counter
  399. # Handle old siginfo format
  400. if isinstance(a, dict):
  401. a = [(os.path.basename(f), cs) for f, cs in a.items()]
  402. if isinstance(b, dict):
  403. b = [(os.path.basename(f), cs) for f, cs in b.items()]
  404. # Compare lists, ensuring we can handle duplicate filenames if they exist
  405. removedcount = Counter(a)
  406. removedcount.subtract(b)
  407. addedcount = Counter(b)
  408. addedcount.subtract(a)
  409. added = []
  410. for x in b:
  411. if addedcount[x] > 0:
  412. addedcount[x] -= 1
  413. added.append(x)
  414. removed = []
  415. changed = []
  416. for x in a:
  417. if removedcount[x] > 0:
  418. removedcount[x] -= 1
  419. for y in added:
  420. if y[0] == x[0]:
  421. changed.append((x[0], x[1], y[1]))
  422. added.remove(y)
  423. break
  424. else:
  425. removed.append(x)
  426. added = [x[0] for x in added]
  427. removed = [x[0] for x in removed]
  428. return changed, added, removed
  429. if 'basewhitelist' in a_data and a_data['basewhitelist'] != b_data['basewhitelist']:
  430. output.append(color_format("{color_title}basewhitelist changed{color_default} from '%s' to '%s'") % (a_data['basewhitelist'], b_data['basewhitelist']))
  431. if a_data['basewhitelist'] and b_data['basewhitelist']:
  432. output.append("changed items: %s" % a_data['basewhitelist'].symmetric_difference(b_data['basewhitelist']))
  433. if 'taskwhitelist' in a_data and a_data['taskwhitelist'] != b_data['taskwhitelist']:
  434. output.append(color_format("{color_title}taskwhitelist changed{color_default} from '%s' to '%s'") % (a_data['taskwhitelist'], b_data['taskwhitelist']))
  435. if a_data['taskwhitelist'] and b_data['taskwhitelist']:
  436. output.append("changed items: %s" % a_data['taskwhitelist'].symmetric_difference(b_data['taskwhitelist']))
  437. if a_data['taskdeps'] != b_data['taskdeps']:
  438. output.append(color_format("{color_title}Task dependencies changed{color_default} from:\n%s\nto:\n%s") % (sorted(a_data['taskdeps']), sorted(b_data['taskdeps'])))
  439. if a_data['basehash'] != b_data['basehash'] and not collapsed:
  440. output.append(color_format("{color_title}basehash changed{color_default} from %s to %s") % (a_data['basehash'], b_data['basehash']))
  441. changed, added, removed = dict_diff(a_data['gendeps'], b_data['gendeps'], a_data['basewhitelist'] & b_data['basewhitelist'])
  442. if changed:
  443. for dep in changed:
  444. output.append(color_format("{color_title}List of dependencies for variable %s changed from '{color_default}%s{color_title}' to '{color_default}%s{color_title}'") % (dep, a_data['gendeps'][dep], b_data['gendeps'][dep]))
  445. if a_data['gendeps'][dep] and b_data['gendeps'][dep]:
  446. output.append("changed items: %s" % a_data['gendeps'][dep].symmetric_difference(b_data['gendeps'][dep]))
  447. if added:
  448. for dep in added:
  449. output.append(color_format("{color_title}Dependency on variable %s was added") % (dep))
  450. if removed:
  451. for dep in removed:
  452. output.append(color_format("{color_title}Dependency on Variable %s was removed") % (dep))
  453. changed, added, removed = dict_diff(a_data['varvals'], b_data['varvals'])
  454. if changed:
  455. for dep in changed:
  456. oldval = a_data['varvals'][dep]
  457. newval = b_data['varvals'][dep]
  458. if newval and oldval and ('\n' in oldval or '\n' in newval):
  459. diff = difflib.unified_diff(oldval.splitlines(), newval.splitlines(), lineterm='')
  460. # Cut off the first two lines, since we aren't interested in
  461. # the old/new filename (they are blank anyway in this case)
  462. difflines = list(diff)[2:]
  463. if color:
  464. # Add colour to diff output
  465. for i, line in enumerate(difflines):
  466. if line.startswith('+'):
  467. line = color_format('{color_add}{line}', line=line)
  468. difflines[i] = line
  469. elif line.startswith('-'):
  470. line = color_format('{color_remove}{line}', line=line)
  471. difflines[i] = line
  472. output.append(color_format("{color_title}Variable {var} value changed:{color_default}\n{diff}", var=dep, diff='\n'.join(difflines)))
  473. elif newval and oldval and (' ' in oldval or ' ' in newval):
  474. output.append(color_format("{color_title}Variable {var} value changed:{color_default}\n{diff}", var=dep, diff=worddiff_str(oldval, newval, colors)))
  475. else:
  476. output.append(color_format("{color_title}Variable {var} value changed from '{color_default}{oldval}{color_title}' to '{color_default}{newval}{color_title}'{color_default}", var=dep, oldval=oldval, newval=newval))
  477. if not 'file_checksum_values' in a_data:
  478. a_data['file_checksum_values'] = {}
  479. if not 'file_checksum_values' in b_data:
  480. b_data['file_checksum_values'] = {}
  481. changed, added, removed = file_checksums_diff(a_data['file_checksum_values'], b_data['file_checksum_values'])
  482. if changed:
  483. for f, old, new in changed:
  484. output.append(color_format("{color_title}Checksum for file %s changed{color_default} from %s to %s") % (f, old, new))
  485. if added:
  486. for f in added:
  487. output.append(color_format("{color_title}Dependency on checksum of file %s was added") % (f))
  488. if removed:
  489. for f in removed:
  490. output.append(color_format("{color_title}Dependency on checksum of file %s was removed") % (f))
  491. if not 'runtaskdeps' in a_data:
  492. a_data['runtaskdeps'] = {}
  493. if not 'runtaskdeps' in b_data:
  494. b_data['runtaskdeps'] = {}
  495. if not collapsed:
  496. if len(a_data['runtaskdeps']) != len(b_data['runtaskdeps']):
  497. changed = ["Number of task dependencies changed"]
  498. else:
  499. changed = []
  500. for idx, task in enumerate(a_data['runtaskdeps']):
  501. a = a_data['runtaskdeps'][idx]
  502. b = b_data['runtaskdeps'][idx]
  503. if a_data['runtaskhashes'][a] != b_data['runtaskhashes'][b] and not collapsed:
  504. changed.append("%s with hash %s\n changed to\n%s with hash %s" % (clean_basepath(a), a_data['runtaskhashes'][a], clean_basepath(b), b_data['runtaskhashes'][b]))
  505. if changed:
  506. clean_a = clean_basepaths_list(a_data['runtaskdeps'])
  507. clean_b = clean_basepaths_list(b_data['runtaskdeps'])
  508. if clean_a != clean_b:
  509. output.append(color_format("{color_title}runtaskdeps changed:{color_default}\n%s") % list_inline_diff(clean_a, clean_b, colors))
  510. else:
  511. output.append(color_format("{color_title}runtaskdeps changed:"))
  512. output.append("\n".join(changed))
  513. if 'runtaskhashes' in a_data and 'runtaskhashes' in b_data:
  514. a = a_data['runtaskhashes']
  515. b = b_data['runtaskhashes']
  516. changed, added, removed = dict_diff(a, b)
  517. if added:
  518. for dep in added:
  519. bdep_found = False
  520. if removed:
  521. for bdep in removed:
  522. if b[dep] == a[bdep]:
  523. #output.append("Dependency on task %s was replaced by %s with same hash" % (dep, bdep))
  524. bdep_found = True
  525. if not bdep_found:
  526. output.append(color_format("{color_title}Dependency on task %s was added{color_default} with hash %s") % (clean_basepath(dep), b[dep]))
  527. if removed:
  528. for dep in removed:
  529. adep_found = False
  530. if added:
  531. for adep in added:
  532. if b[adep] == a[dep]:
  533. #output.append("Dependency on task %s was replaced by %s with same hash" % (adep, dep))
  534. adep_found = True
  535. if not adep_found:
  536. output.append(color_format("{color_title}Dependency on task %s was removed{color_default} with hash %s") % (clean_basepath(dep), a[dep]))
  537. if changed:
  538. for dep in changed:
  539. if not collapsed:
  540. output.append(color_format("{color_title}Hash for dependent task %s changed{color_default} from %s to %s") % (clean_basepath(dep), a[dep], b[dep]))
  541. if callable(recursecb):
  542. recout = recursecb(dep, a[dep], b[dep])
  543. if recout:
  544. if collapsed:
  545. output.extend(recout)
  546. else:
  547. # If a dependent hash changed, might as well print the line above and then defer to the changes in
  548. # that hash since in all likelyhood, they're the same changes this task also saw.
  549. output = [output[-1]] + recout
  550. a_taint = a_data.get('taint', None)
  551. b_taint = b_data.get('taint', None)
  552. if a_taint != b_taint:
  553. output.append(color_format("{color_title}Taint (by forced/invalidated task) changed{color_default} from %s to %s") % (a_taint, b_taint))
  554. return output
  555. def calc_basehash(sigdata):
  556. task = sigdata['task']
  557. basedata = sigdata['varvals'][task]
  558. if basedata is None:
  559. basedata = ''
  560. alldeps = sigdata['taskdeps']
  561. for dep in alldeps:
  562. basedata = basedata + dep
  563. val = sigdata['varvals'][dep]
  564. if val is not None:
  565. basedata = basedata + str(val)
  566. return hashlib.sha256(basedata.encode("utf-8")).hexdigest()
  567. def calc_taskhash(sigdata):
  568. data = sigdata['basehash']
  569. for dep in sigdata['runtaskdeps']:
  570. data = data + sigdata['runtaskhashes'][dep]
  571. for c in sigdata['file_checksum_values']:
  572. if c[1]:
  573. data = data + c[1]
  574. if 'taint' in sigdata:
  575. if 'nostamp:' in sigdata['taint']:
  576. data = data + sigdata['taint'][8:]
  577. else:
  578. data = data + sigdata['taint']
  579. return hashlib.sha256(data.encode("utf-8")).hexdigest()
  580. def dump_sigfile(a):
  581. output = []
  582. with open(a, 'rb') as f:
  583. p1 = pickle.Unpickler(f)
  584. a_data = p1.load()
  585. output.append("basewhitelist: %s" % (a_data['basewhitelist']))
  586. output.append("taskwhitelist: %s" % (a_data['taskwhitelist']))
  587. output.append("Task dependencies: %s" % (sorted(a_data['taskdeps'])))
  588. output.append("basehash: %s" % (a_data['basehash']))
  589. for dep in a_data['gendeps']:
  590. output.append("List of dependencies for variable %s is %s" % (dep, a_data['gendeps'][dep]))
  591. for dep in a_data['varvals']:
  592. output.append("Variable %s value is %s" % (dep, a_data['varvals'][dep]))
  593. if 'runtaskdeps' in a_data:
  594. output.append("Tasks this task depends on: %s" % (a_data['runtaskdeps']))
  595. if 'file_checksum_values' in a_data:
  596. output.append("This task depends on the checksums of files: %s" % (a_data['file_checksum_values']))
  597. if 'runtaskhashes' in a_data:
  598. for dep in a_data['runtaskhashes']:
  599. output.append("Hash for dependent task %s is %s" % (dep, a_data['runtaskhashes'][dep]))
  600. if 'taint' in a_data:
  601. output.append("Tainted (by forced/invalidated task): %s" % a_data['taint'])
  602. if 'task' in a_data:
  603. computed_basehash = calc_basehash(a_data)
  604. output.append("Computed base hash is %s and from file %s" % (computed_basehash, a_data['basehash']))
  605. else:
  606. output.append("Unable to compute base hash")
  607. computed_taskhash = calc_taskhash(a_data)
  608. output.append("Computed task hash is %s" % computed_taskhash)
  609. return output