siggen.py 42 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084
  1. #
  2. # SPDX-License-Identifier: GPL-2.0-only
  3. #
  4. import hashlib
  5. import logging
  6. import os
  7. import re
  8. import tempfile
  9. import pickle
  10. import bb.data
  11. import difflib
  12. import simplediff
  13. from bb.checksum import FileChecksumCache
  14. from bb import runqueue
  15. import hashserv
  16. import hashserv.client
  17. logger = logging.getLogger('BitBake.SigGen')
  18. hashequiv_logger = logging.getLogger('BitBake.SigGen.HashEquiv')
  19. def init(d):
  20. siggens = [obj for obj in globals().values()
  21. if type(obj) is type and issubclass(obj, SignatureGenerator)]
  22. desired = d.getVar("BB_SIGNATURE_HANDLER") or "noop"
  23. for sg in siggens:
  24. if desired == sg.name:
  25. return sg(d)
  26. break
  27. else:
  28. logger.error("Invalid signature generator '%s', using default 'noop'\n"
  29. "Available generators: %s", desired,
  30. ', '.join(obj.name for obj in siggens))
  31. return SignatureGenerator(d)
  32. class SignatureGenerator(object):
  33. """
  34. """
  35. name = "noop"
  36. # If the derived class supports multiconfig datacaches, set this to True
  37. # The default is False for backward compatibility with derived signature
  38. # generators that do not understand multiconfig caches
  39. supports_multiconfig_datacaches = False
  40. def __init__(self, data):
  41. self.basehash = {}
  42. self.taskhash = {}
  43. self.unihash = {}
  44. self.runtaskdeps = {}
  45. self.file_checksum_values = {}
  46. self.taints = {}
  47. self.unitaskhashes = {}
  48. self.tidtopn = {}
  49. self.setscenetasks = set()
  50. def finalise(self, fn, d, varient):
  51. return
  52. def postparsing_clean_cache(self):
  53. return
  54. def get_unihash(self, tid):
  55. return self.taskhash[tid]
  56. def prep_taskhash(self, tid, deps, dataCaches):
  57. return
  58. def get_taskhash(self, tid, deps, dataCaches):
  59. self.taskhash[tid] = hashlib.sha256(tid.encode("utf-8")).hexdigest()
  60. return self.taskhash[tid]
  61. def writeout_file_checksum_cache(self):
  62. """Write/update the file checksum cache onto disk"""
  63. return
  64. def stampfile(self, stampbase, file_name, taskname, extrainfo):
  65. return ("%s.%s.%s" % (stampbase, taskname, extrainfo)).rstrip('.')
  66. def stampcleanmask(self, stampbase, file_name, taskname, extrainfo):
  67. return ("%s.%s.%s" % (stampbase, taskname, extrainfo)).rstrip('.')
  68. def dump_sigtask(self, fn, task, stampbase, runtime):
  69. return
  70. def invalidate_task(self, task, d, fn):
  71. bb.build.del_stamp(task, d, fn)
  72. def dump_sigs(self, dataCache, options):
  73. return
  74. def get_taskdata(self):
  75. return (self.runtaskdeps, self.taskhash, self.unihash, self.file_checksum_values, self.taints, self.basehash, self.unitaskhashes, self.tidtopn, self.setscenetasks)
  76. def set_taskdata(self, data):
  77. self.runtaskdeps, self.taskhash, self.unihash, self.file_checksum_values, self.taints, self.basehash, self.unitaskhashes, self.tidtopn, self.setscenetasks = data
  78. def reset(self, data):
  79. self.__init__(data)
  80. def get_taskhashes(self):
  81. return self.taskhash, self.unihash, self.unitaskhashes, self.tidtopn
  82. def set_taskhashes(self, hashes):
  83. self.taskhash, self.unihash, self.unitaskhashes, self.tidtopn = hashes
  84. def save_unitaskhashes(self):
  85. return
  86. def set_setscene_tasks(self, setscene_tasks):
  87. return
  88. @classmethod
  89. def get_data_caches(cls, dataCaches, mc):
  90. """
  91. This function returns the datacaches that should be passed to signature
  92. generator functions. If the signature generator supports multiconfig
  93. caches, the entire dictionary of data caches is sent, otherwise a
  94. special proxy is sent that support both index access to all
  95. multiconfigs, and also direct access for the default multiconfig.
  96. The proxy class allows code in this class itself to always use
  97. multiconfig aware code (to ease maintenance), but derived classes that
  98. are unaware of multiconfig data caches can still access the default
  99. multiconfig as expected.
  100. Do not override this function in derived classes; it will be removed in
  101. the future when support for multiconfig data caches is mandatory
  102. """
  103. class DataCacheProxy(object):
  104. def __init__(self):
  105. pass
  106. def __getitem__(self, key):
  107. return dataCaches[key]
  108. def __getattr__(self, name):
  109. return getattr(dataCaches[mc], name)
  110. if cls.supports_multiconfig_datacaches:
  111. return dataCaches
  112. return DataCacheProxy()
  113. class SignatureGeneratorBasic(SignatureGenerator):
  114. """
  115. """
  116. name = "basic"
  117. def __init__(self, data):
  118. self.basehash = {}
  119. self.taskhash = {}
  120. self.unihash = {}
  121. self.taskdeps = {}
  122. self.runtaskdeps = {}
  123. self.file_checksum_values = {}
  124. self.taints = {}
  125. self.gendeps = {}
  126. self.lookupcache = {}
  127. self.setscenetasks = set()
  128. self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST") or "").split())
  129. self.taskwhitelist = None
  130. self.init_rundepcheck(data)
  131. checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE")
  132. if checksum_cache_file:
  133. self.checksum_cache = FileChecksumCache()
  134. self.checksum_cache.init_cache(data, checksum_cache_file)
  135. else:
  136. self.checksum_cache = None
  137. self.unihash_cache = bb.cache.SimpleCache("3")
  138. self.unitaskhashes = self.unihash_cache.init_cache(data, "bb_unihashes.dat", {})
  139. self.localdirsexclude = (data.getVar("BB_SIGNATURE_LOCAL_DIRS_EXCLUDE") or "CVS .bzr .git .hg .osc .p4 .repo .svn").split()
  140. self.tidtopn = {}
  141. def init_rundepcheck(self, data):
  142. self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST") or None
  143. if self.taskwhitelist:
  144. self.twl = re.compile(self.taskwhitelist)
  145. else:
  146. self.twl = None
  147. def _build_data(self, fn, d):
  148. ignore_mismatch = ((d.getVar("BB_HASH_IGNORE_MISMATCH") or '') == '1')
  149. tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d, self.basewhitelist)
  150. taskdeps, basehash = bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, self.basewhitelist, fn)
  151. for task in tasklist:
  152. tid = fn + ":" + task
  153. if not ignore_mismatch and tid in self.basehash and self.basehash[tid] != basehash[tid]:
  154. bb.error("When reparsing %s, the basehash value changed from %s to %s. The metadata is not deterministic and this needs to be fixed." % (tid, self.basehash[tid], basehash[tid]))
  155. bb.error("The following commands may help:")
  156. cmd = "$ bitbake %s -c%s" % (d.getVar('PN'), task)
  157. # Make sure sigdata is dumped before run printdiff
  158. bb.error("%s -Snone" % cmd)
  159. bb.error("Then:")
  160. bb.error("%s -Sprintdiff\n" % cmd)
  161. self.basehash[tid] = basehash[tid]
  162. self.taskdeps[fn] = taskdeps
  163. self.gendeps[fn] = gendeps
  164. self.lookupcache[fn] = lookupcache
  165. return taskdeps
  166. def set_setscene_tasks(self, setscene_tasks):
  167. self.setscenetasks = set(setscene_tasks)
  168. def finalise(self, fn, d, variant):
  169. mc = d.getVar("__BBMULTICONFIG", False) or ""
  170. if variant or mc:
  171. fn = bb.cache.realfn2virtual(fn, variant, mc)
  172. try:
  173. taskdeps = self._build_data(fn, d)
  174. except bb.parse.SkipRecipe:
  175. raise
  176. except:
  177. bb.warn("Error during finalise of %s" % fn)
  178. raise
  179. #Slow but can be useful for debugging mismatched basehashes
  180. #for task in self.taskdeps[fn]:
  181. # self.dump_sigtask(fn, task, d.getVar("STAMP"), False)
  182. for task in taskdeps:
  183. d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + ":" + task])
  184. def postparsing_clean_cache(self):
  185. #
  186. # After parsing we can remove some things from memory to reduce our memory footprint
  187. #
  188. self.gendeps = {}
  189. self.lookupcache = {}
  190. self.taskdeps = {}
  191. def rundep_check(self, fn, recipename, task, dep, depname, dataCaches):
  192. # Return True if we should keep the dependency, False to drop it
  193. # We only manipulate the dependencies for packages not in the whitelist
  194. if self.twl and not self.twl.search(recipename):
  195. # then process the actual dependencies
  196. if self.twl.search(depname):
  197. return False
  198. return True
  199. def read_taint(self, fn, task, stampbase):
  200. taint = None
  201. try:
  202. with open(stampbase + '.' + task + '.taint', 'r') as taintf:
  203. taint = taintf.read()
  204. except IOError:
  205. pass
  206. return taint
  207. def prep_taskhash(self, tid, deps, dataCaches):
  208. (mc, _, task, fn) = bb.runqueue.split_tid_mcfn(tid)
  209. self.basehash[tid] = dataCaches[mc].basetaskhash[tid]
  210. self.runtaskdeps[tid] = []
  211. self.file_checksum_values[tid] = []
  212. recipename = dataCaches[mc].pkg_fn[fn]
  213. self.tidtopn[tid] = recipename
  214. for dep in sorted(deps, key=clean_basepath):
  215. (depmc, _, _, depmcfn) = bb.runqueue.split_tid_mcfn(dep)
  216. depname = dataCaches[depmc].pkg_fn[depmcfn]
  217. if not self.supports_multiconfig_datacaches and mc != depmc:
  218. # If the signature generator doesn't understand multiconfig
  219. # data caches, any dependency not in the same multiconfig must
  220. # be skipped for backward compatibility
  221. continue
  222. if not self.rundep_check(fn, recipename, task, dep, depname, dataCaches):
  223. continue
  224. if dep not in self.taskhash:
  225. bb.fatal("%s is not in taskhash, caller isn't calling in dependency order?" % dep)
  226. self.runtaskdeps[tid].append(dep)
  227. if task in dataCaches[mc].file_checksums[fn]:
  228. if self.checksum_cache:
  229. checksums = self.checksum_cache.get_checksums(dataCaches[mc].file_checksums[fn][task], recipename, self.localdirsexclude)
  230. else:
  231. checksums = bb.fetch2.get_file_checksums(dataCaches[mc].file_checksums[fn][task], recipename, self.localdirsexclude)
  232. for (f,cs) in checksums:
  233. self.file_checksum_values[tid].append((f,cs))
  234. taskdep = dataCaches[mc].task_deps[fn]
  235. if 'nostamp' in taskdep and task in taskdep['nostamp']:
  236. # Nostamp tasks need an implicit taint so that they force any dependent tasks to run
  237. if tid in self.taints and self.taints[tid].startswith("nostamp:"):
  238. # Don't reset taint value upon every call
  239. pass
  240. else:
  241. import uuid
  242. taint = str(uuid.uuid4())
  243. self.taints[tid] = "nostamp:" + taint
  244. taint = self.read_taint(fn, task, dataCaches[mc].stamp[fn])
  245. if taint:
  246. self.taints[tid] = taint
  247. logger.warning("%s is tainted from a forced run" % tid)
  248. return
  249. def get_taskhash(self, tid, deps, dataCaches):
  250. data = self.basehash[tid]
  251. for dep in self.runtaskdeps[tid]:
  252. if dep in self.unihash:
  253. if self.unihash[dep] is None:
  254. data = data + self.taskhash[dep]
  255. else:
  256. data = data + self.unihash[dep]
  257. else:
  258. data = data + self.get_unihash(dep)
  259. for (f, cs) in self.file_checksum_values[tid]:
  260. if cs:
  261. data = data + cs
  262. if tid in self.taints:
  263. if self.taints[tid].startswith("nostamp:"):
  264. data = data + self.taints[tid][8:]
  265. else:
  266. data = data + self.taints[tid]
  267. h = hashlib.sha256(data.encode("utf-8")).hexdigest()
  268. self.taskhash[tid] = h
  269. #d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task])
  270. return h
  271. def writeout_file_checksum_cache(self):
  272. """Write/update the file checksum cache onto disk"""
  273. if self.checksum_cache:
  274. self.checksum_cache.save_extras()
  275. self.checksum_cache.save_merge()
  276. else:
  277. bb.fetch2.fetcher_parse_save()
  278. bb.fetch2.fetcher_parse_done()
  279. def save_unitaskhashes(self):
  280. self.unihash_cache.save(self.unitaskhashes)
  281. def dump_sigtask(self, fn, task, stampbase, runtime):
  282. tid = fn + ":" + task
  283. referencestamp = stampbase
  284. if isinstance(runtime, str) and runtime.startswith("customfile"):
  285. sigfile = stampbase
  286. referencestamp = runtime[11:]
  287. elif runtime and tid in self.taskhash:
  288. sigfile = stampbase + "." + task + ".sigdata" + "." + self.get_unihash(tid)
  289. else:
  290. sigfile = stampbase + "." + task + ".sigbasedata" + "." + self.basehash[tid]
  291. with bb.utils.umask(0o002):
  292. bb.utils.mkdirhier(os.path.dirname(sigfile))
  293. data = {}
  294. data['task'] = task
  295. data['basewhitelist'] = self.basewhitelist
  296. data['taskwhitelist'] = self.taskwhitelist
  297. data['taskdeps'] = self.taskdeps[fn][task]
  298. data['basehash'] = self.basehash[tid]
  299. data['gendeps'] = {}
  300. data['varvals'] = {}
  301. data['varvals'][task] = self.lookupcache[fn][task]
  302. for dep in self.taskdeps[fn][task]:
  303. if dep in self.basewhitelist:
  304. continue
  305. data['gendeps'][dep] = self.gendeps[fn][dep]
  306. data['varvals'][dep] = self.lookupcache[fn][dep]
  307. if runtime and tid in self.taskhash:
  308. data['runtaskdeps'] = self.runtaskdeps[tid]
  309. data['file_checksum_values'] = [(os.path.basename(f), cs) for f,cs in self.file_checksum_values[tid]]
  310. data['runtaskhashes'] = {}
  311. for dep in data['runtaskdeps']:
  312. data['runtaskhashes'][dep] = self.get_unihash(dep)
  313. data['taskhash'] = self.taskhash[tid]
  314. data['unihash'] = self.get_unihash(tid)
  315. taint = self.read_taint(fn, task, referencestamp)
  316. if taint:
  317. data['taint'] = taint
  318. if runtime and tid in self.taints:
  319. if 'nostamp:' in self.taints[tid]:
  320. data['taint'] = self.taints[tid]
  321. computed_basehash = calc_basehash(data)
  322. if computed_basehash != self.basehash[tid]:
  323. bb.error("Basehash mismatch %s versus %s for %s" % (computed_basehash, self.basehash[tid], tid))
  324. if runtime and tid in self.taskhash:
  325. computed_taskhash = calc_taskhash(data)
  326. if computed_taskhash != self.taskhash[tid]:
  327. bb.error("Taskhash mismatch %s versus %s for %s" % (computed_taskhash, self.taskhash[tid], tid))
  328. sigfile = sigfile.replace(self.taskhash[tid], computed_taskhash)
  329. fd, tmpfile = tempfile.mkstemp(dir=os.path.dirname(sigfile), prefix="sigtask.")
  330. try:
  331. with os.fdopen(fd, "wb") as stream:
  332. p = pickle.dump(data, stream, -1)
  333. stream.flush()
  334. os.chmod(tmpfile, 0o664)
  335. os.rename(tmpfile, sigfile)
  336. except (OSError, IOError) as err:
  337. try:
  338. os.unlink(tmpfile)
  339. except OSError:
  340. pass
  341. raise err
  342. def dump_sigfn(self, fn, dataCaches, options):
  343. if fn in self.taskdeps:
  344. for task in self.taskdeps[fn]:
  345. tid = fn + ":" + task
  346. mc = bb.runqueue.mc_from_tid(tid)
  347. if tid not in self.taskhash:
  348. continue
  349. if dataCaches[mc].basetaskhash[tid] != self.basehash[tid]:
  350. bb.error("Bitbake's cached basehash does not match the one we just generated (%s)!" % tid)
  351. bb.error("The mismatched hashes were %s and %s" % (dataCaches[mc].basetaskhash[tid], self.basehash[tid]))
  352. self.dump_sigtask(fn, task, dataCaches[mc].stamp[fn], True)
  353. class SignatureGeneratorBasicHash(SignatureGeneratorBasic):
  354. name = "basichash"
  355. def get_stampfile_hash(self, tid):
  356. if tid in self.taskhash:
  357. return self.taskhash[tid]
  358. # If task is not in basehash, then error
  359. return self.basehash[tid]
  360. def stampfile(self, stampbase, fn, taskname, extrainfo, clean=False):
  361. if taskname != "do_setscene" and taskname.endswith("_setscene"):
  362. tid = fn + ":" + taskname[:-9]
  363. else:
  364. tid = fn + ":" + taskname
  365. if clean:
  366. h = "*"
  367. else:
  368. h = self.get_stampfile_hash(tid)
  369. return ("%s.%s.%s.%s" % (stampbase, taskname, h, extrainfo)).rstrip('.')
  370. def stampcleanmask(self, stampbase, fn, taskname, extrainfo):
  371. return self.stampfile(stampbase, fn, taskname, extrainfo, clean=True)
  372. def invalidate_task(self, task, d, fn):
  373. bb.note("Tainting hash to force rebuild of task %s, %s" % (fn, task))
  374. bb.build.write_taint(task, d, fn)
  375. class SignatureGeneratorUniHashMixIn(object):
  376. def __init__(self, data):
  377. self.extramethod = {}
  378. super().__init__(data)
  379. def get_taskdata(self):
  380. return (self.server, self.method, self.extramethod) + super().get_taskdata()
  381. def set_taskdata(self, data):
  382. self.server, self.method, self.extramethod = data[:3]
  383. super().set_taskdata(data[3:])
  384. def client(self):
  385. if getattr(self, '_client', None) is None:
  386. self._client = hashserv.create_client(self.server)
  387. return self._client
  388. def get_stampfile_hash(self, tid):
  389. if tid in self.taskhash:
  390. # If a unique hash is reported, use it as the stampfile hash. This
  391. # ensures that if a task won't be re-run if the taskhash changes,
  392. # but it would result in the same output hash
  393. unihash = self._get_unihash(tid)
  394. if unihash is not None:
  395. return unihash
  396. return super().get_stampfile_hash(tid)
  397. def set_unihash(self, tid, unihash):
  398. (mc, fn, taskname, taskfn) = bb.runqueue.split_tid_mcfn(tid)
  399. key = mc + ":" + self.tidtopn[tid] + ":" + taskname
  400. self.unitaskhashes[key] = (self.taskhash[tid], unihash)
  401. self.unihash[tid] = unihash
  402. def _get_unihash(self, tid, checkkey=None):
  403. if tid not in self.tidtopn:
  404. return None
  405. (mc, fn, taskname, taskfn) = bb.runqueue.split_tid_mcfn(tid)
  406. key = mc + ":" + self.tidtopn[tid] + ":" + taskname
  407. if key not in self.unitaskhashes:
  408. return None
  409. if not checkkey:
  410. checkkey = self.taskhash[tid]
  411. (key, unihash) = self.unitaskhashes[key]
  412. if key != checkkey:
  413. return None
  414. return unihash
  415. def get_unihash(self, tid):
  416. taskhash = self.taskhash[tid]
  417. # If its not a setscene task we can return
  418. if self.setscenetasks and tid not in self.setscenetasks:
  419. self.unihash[tid] = None
  420. return taskhash
  421. # TODO: This cache can grow unbounded. It probably only needs to keep
  422. # for each task
  423. unihash = self._get_unihash(tid)
  424. if unihash is not None:
  425. self.unihash[tid] = unihash
  426. return unihash
  427. # In the absence of being able to discover a unique hash from the
  428. # server, make it be equivalent to the taskhash. The unique "hash" only
  429. # really needs to be a unique string (not even necessarily a hash), but
  430. # making it match the taskhash has a few advantages:
  431. #
  432. # 1) All of the sstate code that assumes hashes can be the same
  433. # 2) It provides maximal compatibility with builders that don't use
  434. # an equivalency server
  435. # 3) The value is easy for multiple independent builders to derive the
  436. # same unique hash from the same input. This means that if the
  437. # independent builders find the same taskhash, but it isn't reported
  438. # to the server, there is a better chance that they will agree on
  439. # the unique hash.
  440. unihash = taskhash
  441. try:
  442. method = self.method
  443. if tid in self.extramethod:
  444. method = method + self.extramethod[tid]
  445. data = self.client().get_unihash(method, self.taskhash[tid])
  446. if data:
  447. unihash = data
  448. # A unique hash equal to the taskhash is not very interesting,
  449. # so it is reported it at debug level 2. If they differ, that
  450. # is much more interesting, so it is reported at debug level 1
  451. hashequiv_logger.debug((1, 2)[unihash == taskhash], 'Found unihash %s in place of %s for %s from %s' % (unihash, taskhash, tid, self.server))
  452. else:
  453. hashequiv_logger.debug(2, 'No reported unihash for %s:%s from %s' % (tid, taskhash, self.server))
  454. except hashserv.client.HashConnectionError as e:
  455. bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
  456. self.set_unihash(tid, unihash)
  457. self.unihash[tid] = unihash
  458. return unihash
  459. def report_unihash(self, path, task, d):
  460. import importlib
  461. taskhash = d.getVar('BB_TASKHASH')
  462. unihash = d.getVar('BB_UNIHASH')
  463. report_taskdata = d.getVar('SSTATE_HASHEQUIV_REPORT_TASKDATA') == '1'
  464. tempdir = d.getVar('T')
  465. fn = d.getVar('BB_FILENAME')
  466. tid = fn + ':do_' + task
  467. key = tid + ':' + taskhash
  468. if self.setscenetasks and tid not in self.setscenetasks:
  469. return
  470. # This can happen if locked sigs are in action. Detect and just abort
  471. if taskhash != self.taskhash[tid]:
  472. return
  473. # Sanity checks
  474. cache_unihash = self._get_unihash(tid, checkkey=taskhash)
  475. if cache_unihash is None:
  476. bb.fatal('%s not in unihash cache. Please report this error' % key)
  477. if cache_unihash != unihash:
  478. bb.fatal("Cache unihash %s doesn't match BB_UNIHASH %s" % (cache_unihash, unihash))
  479. sigfile = None
  480. sigfile_name = "depsig.do_%s.%d" % (task, os.getpid())
  481. sigfile_link = "depsig.do_%s" % task
  482. try:
  483. sigfile = open(os.path.join(tempdir, sigfile_name), 'w+b')
  484. locs = {'path': path, 'sigfile': sigfile, 'task': task, 'd': d}
  485. if "." in self.method:
  486. (module, method) = self.method.rsplit('.', 1)
  487. locs['method'] = getattr(importlib.import_module(module), method)
  488. outhash = bb.utils.better_eval('method(path, sigfile, task, d)', locs)
  489. else:
  490. outhash = bb.utils.better_eval(self.method + '(path, sigfile, task, d)', locs)
  491. try:
  492. extra_data = {}
  493. owner = d.getVar('SSTATE_HASHEQUIV_OWNER')
  494. if owner:
  495. extra_data['owner'] = owner
  496. if report_taskdata:
  497. sigfile.seek(0)
  498. extra_data['PN'] = d.getVar('PN')
  499. extra_data['PV'] = d.getVar('PV')
  500. extra_data['PR'] = d.getVar('PR')
  501. extra_data['task'] = task
  502. extra_data['outhash_siginfo'] = sigfile.read().decode('utf-8')
  503. method = self.method
  504. if tid in self.extramethod:
  505. method = method + self.extramethod[tid]
  506. data = self.client().report_unihash(taskhash, method, outhash, unihash, extra_data)
  507. new_unihash = data['unihash']
  508. if new_unihash != unihash:
  509. hashequiv_logger.debug(1, 'Task %s unihash changed %s -> %s by server %s' % (taskhash, unihash, new_unihash, self.server))
  510. bb.event.fire(bb.runqueue.taskUniHashUpdate(fn + ':do_' + task, new_unihash), d)
  511. self.set_unihash(tid, new_unihash)
  512. d.setVar('BB_UNIHASH', new_unihash)
  513. else:
  514. hashequiv_logger.debug(1, 'Reported task %s as unihash %s to %s' % (taskhash, unihash, self.server))
  515. except hashserv.client.HashConnectionError as e:
  516. bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
  517. finally:
  518. if sigfile:
  519. sigfile.close()
  520. sigfile_link_path = os.path.join(tempdir, sigfile_link)
  521. bb.utils.remove(sigfile_link_path)
  522. try:
  523. os.symlink(sigfile_name, sigfile_link_path)
  524. except OSError:
  525. pass
  526. def report_unihash_equiv(self, tid, taskhash, wanted_unihash, current_unihash, datacaches):
  527. try:
  528. extra_data = {}
  529. method = self.method
  530. if tid in self.extramethod:
  531. method = method + self.extramethod[tid]
  532. data = self.client().report_unihash_equiv(taskhash, method, wanted_unihash, extra_data)
  533. hashequiv_logger.verbose('Reported task %s as unihash %s to %s (%s)' % (tid, wanted_unihash, self.server, str(data)))
  534. if data is None:
  535. bb.warn("Server unable to handle unihash report")
  536. return False
  537. finalunihash = data['unihash']
  538. if finalunihash == current_unihash:
  539. hashequiv_logger.verbose('Task %s unihash %s unchanged by server' % (tid, finalunihash))
  540. elif finalunihash == wanted_unihash:
  541. hashequiv_logger.verbose('Task %s unihash changed %s -> %s as wanted' % (tid, current_unihash, finalunihash))
  542. self.set_unihash(tid, finalunihash)
  543. return True
  544. else:
  545. # TODO: What to do here?
  546. hashequiv_logger.verbose('Task %s unihash reported as unwanted hash %s' % (tid, finalunihash))
  547. except hashserv.client.HashConnectionError as e:
  548. bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
  549. return False
  550. #
  551. # Dummy class used for bitbake-selftest
  552. #
  553. class SignatureGeneratorTestEquivHash(SignatureGeneratorUniHashMixIn, SignatureGeneratorBasicHash):
  554. name = "TestEquivHash"
  555. def init_rundepcheck(self, data):
  556. super().init_rundepcheck(data)
  557. self.server = data.getVar('BB_HASHSERVE')
  558. self.method = "sstate_output_hash"
  559. #
  560. # Dummy class used for bitbake-selftest
  561. #
  562. class SignatureGeneratorTestMulticonfigDepends(SignatureGeneratorBasicHash):
  563. name = "TestMulticonfigDepends"
  564. supports_multiconfig_datacaches = True
  565. def dump_this_task(outfile, d):
  566. import bb.parse
  567. fn = d.getVar("BB_FILENAME")
  568. task = "do_" + d.getVar("BB_CURRENTTASK")
  569. referencestamp = bb.build.stamp_internal(task, d, None, True)
  570. bb.parse.siggen.dump_sigtask(fn, task, outfile, "customfile:" + referencestamp)
  571. def init_colors(enable_color):
  572. """Initialise colour dict for passing to compare_sigfiles()"""
  573. # First set up the colours
  574. colors = {'color_title': '\033[1m',
  575. 'color_default': '\033[0m',
  576. 'color_add': '\033[0;32m',
  577. 'color_remove': '\033[0;31m',
  578. }
  579. # Leave all keys present but clear the values
  580. if not enable_color:
  581. for k in colors.keys():
  582. colors[k] = ''
  583. return colors
  584. def worddiff_str(oldstr, newstr, colors=None):
  585. if not colors:
  586. colors = init_colors(False)
  587. diff = simplediff.diff(oldstr.split(' '), newstr.split(' '))
  588. ret = []
  589. for change, value in diff:
  590. value = ' '.join(value)
  591. if change == '=':
  592. ret.append(value)
  593. elif change == '+':
  594. item = '{color_add}{{+{value}+}}{color_default}'.format(value=value, **colors)
  595. ret.append(item)
  596. elif change == '-':
  597. item = '{color_remove}[-{value}-]{color_default}'.format(value=value, **colors)
  598. ret.append(item)
  599. whitespace_note = ''
  600. if oldstr != newstr and ' '.join(oldstr.split()) == ' '.join(newstr.split()):
  601. whitespace_note = ' (whitespace changed)'
  602. return '"%s"%s' % (' '.join(ret), whitespace_note)
  603. def list_inline_diff(oldlist, newlist, colors=None):
  604. if not colors:
  605. colors = init_colors(False)
  606. diff = simplediff.diff(oldlist, newlist)
  607. ret = []
  608. for change, value in diff:
  609. value = ' '.join(value)
  610. if change == '=':
  611. ret.append("'%s'" % value)
  612. elif change == '+':
  613. item = '{color_add}+{value}{color_default}'.format(value=value, **colors)
  614. ret.append(item)
  615. elif change == '-':
  616. item = '{color_remove}-{value}{color_default}'.format(value=value, **colors)
  617. ret.append(item)
  618. return '[%s]' % (', '.join(ret))
  619. def clean_basepath(basepath):
  620. basepath, dir, recipe_task = basepath.rsplit("/", 2)
  621. cleaned = dir + '/' + recipe_task
  622. if basepath[0] == '/':
  623. return cleaned
  624. if basepath.startswith("mc:"):
  625. mc, mc_name, basepath = basepath.split(":", 2)
  626. mc_suffix = ':mc:' + mc_name
  627. else:
  628. mc_suffix = ''
  629. # mc stuff now removed from basepath. Whatever was next, if present will be the first
  630. # suffix. ':/', recipe path start, marks the end of this. Something like
  631. # 'virtual:a[:b[:c]]:/path...' (b and c being optional)
  632. if basepath[0] != '/':
  633. cleaned += ':' + basepath.split(':/', 1)[0]
  634. return cleaned + mc_suffix
  635. def clean_basepaths(a):
  636. b = {}
  637. for x in a:
  638. b[clean_basepath(x)] = a[x]
  639. return b
  640. def clean_basepaths_list(a):
  641. b = []
  642. for x in a:
  643. b.append(clean_basepath(x))
  644. return b
  645. def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
  646. output = []
  647. colors = init_colors(color)
  648. def color_format(formatstr, **values):
  649. """
  650. Return colour formatted string.
  651. NOTE: call with the format string, not an already formatted string
  652. containing values (otherwise you could have trouble with { and }
  653. characters)
  654. """
  655. if not formatstr.endswith('{color_default}'):
  656. formatstr += '{color_default}'
  657. # In newer python 3 versions you can pass both of these directly,
  658. # but we only require 3.4 at the moment
  659. formatparams = {}
  660. formatparams.update(colors)
  661. formatparams.update(values)
  662. return formatstr.format(**formatparams)
  663. with open(a, 'rb') as f:
  664. p1 = pickle.Unpickler(f)
  665. a_data = p1.load()
  666. with open(b, 'rb') as f:
  667. p2 = pickle.Unpickler(f)
  668. b_data = p2.load()
  669. def dict_diff(a, b, whitelist=set()):
  670. sa = set(a.keys())
  671. sb = set(b.keys())
  672. common = sa & sb
  673. changed = set()
  674. for i in common:
  675. if a[i] != b[i] and i not in whitelist:
  676. changed.add(i)
  677. added = sb - sa
  678. removed = sa - sb
  679. return changed, added, removed
  680. def file_checksums_diff(a, b):
  681. from collections import Counter
  682. # Handle old siginfo format
  683. if isinstance(a, dict):
  684. a = [(os.path.basename(f), cs) for f, cs in a.items()]
  685. if isinstance(b, dict):
  686. b = [(os.path.basename(f), cs) for f, cs in b.items()]
  687. # Compare lists, ensuring we can handle duplicate filenames if they exist
  688. removedcount = Counter(a)
  689. removedcount.subtract(b)
  690. addedcount = Counter(b)
  691. addedcount.subtract(a)
  692. added = []
  693. for x in b:
  694. if addedcount[x] > 0:
  695. addedcount[x] -= 1
  696. added.append(x)
  697. removed = []
  698. changed = []
  699. for x in a:
  700. if removedcount[x] > 0:
  701. removedcount[x] -= 1
  702. for y in added:
  703. if y[0] == x[0]:
  704. changed.append((x[0], x[1], y[1]))
  705. added.remove(y)
  706. break
  707. else:
  708. removed.append(x)
  709. added = [x[0] for x in added]
  710. removed = [x[0] for x in removed]
  711. return changed, added, removed
  712. if 'basewhitelist' in a_data and a_data['basewhitelist'] != b_data['basewhitelist']:
  713. output.append(color_format("{color_title}basewhitelist changed{color_default} from '%s' to '%s'") % (a_data['basewhitelist'], b_data['basewhitelist']))
  714. if a_data['basewhitelist'] and b_data['basewhitelist']:
  715. output.append("changed items: %s" % a_data['basewhitelist'].symmetric_difference(b_data['basewhitelist']))
  716. if 'taskwhitelist' in a_data and a_data['taskwhitelist'] != b_data['taskwhitelist']:
  717. output.append(color_format("{color_title}taskwhitelist changed{color_default} from '%s' to '%s'") % (a_data['taskwhitelist'], b_data['taskwhitelist']))
  718. if a_data['taskwhitelist'] and b_data['taskwhitelist']:
  719. output.append("changed items: %s" % a_data['taskwhitelist'].symmetric_difference(b_data['taskwhitelist']))
  720. if a_data['taskdeps'] != b_data['taskdeps']:
  721. output.append(color_format("{color_title}Task dependencies changed{color_default} from:\n%s\nto:\n%s") % (sorted(a_data['taskdeps']), sorted(b_data['taskdeps'])))
  722. if a_data['basehash'] != b_data['basehash'] and not collapsed:
  723. output.append(color_format("{color_title}basehash changed{color_default} from %s to %s") % (a_data['basehash'], b_data['basehash']))
  724. changed, added, removed = dict_diff(a_data['gendeps'], b_data['gendeps'], a_data['basewhitelist'] & b_data['basewhitelist'])
  725. if changed:
  726. for dep in changed:
  727. output.append(color_format("{color_title}List of dependencies for variable %s changed from '{color_default}%s{color_title}' to '{color_default}%s{color_title}'") % (dep, a_data['gendeps'][dep], b_data['gendeps'][dep]))
  728. if a_data['gendeps'][dep] and b_data['gendeps'][dep]:
  729. output.append("changed items: %s" % a_data['gendeps'][dep].symmetric_difference(b_data['gendeps'][dep]))
  730. if added:
  731. for dep in added:
  732. output.append(color_format("{color_title}Dependency on variable %s was added") % (dep))
  733. if removed:
  734. for dep in removed:
  735. output.append(color_format("{color_title}Dependency on Variable %s was removed") % (dep))
  736. changed, added, removed = dict_diff(a_data['varvals'], b_data['varvals'])
  737. if changed:
  738. for dep in changed:
  739. oldval = a_data['varvals'][dep]
  740. newval = b_data['varvals'][dep]
  741. if newval and oldval and ('\n' in oldval or '\n' in newval):
  742. diff = difflib.unified_diff(oldval.splitlines(), newval.splitlines(), lineterm='')
  743. # Cut off the first two lines, since we aren't interested in
  744. # the old/new filename (they are blank anyway in this case)
  745. difflines = list(diff)[2:]
  746. if color:
  747. # Add colour to diff output
  748. for i, line in enumerate(difflines):
  749. if line.startswith('+'):
  750. line = color_format('{color_add}{line}', line=line)
  751. difflines[i] = line
  752. elif line.startswith('-'):
  753. line = color_format('{color_remove}{line}', line=line)
  754. difflines[i] = line
  755. output.append(color_format("{color_title}Variable {var} value changed:{color_default}\n{diff}", var=dep, diff='\n'.join(difflines)))
  756. elif newval and oldval and (' ' in oldval or ' ' in newval):
  757. output.append(color_format("{color_title}Variable {var} value changed:{color_default}\n{diff}", var=dep, diff=worddiff_str(oldval, newval, colors)))
  758. else:
  759. output.append(color_format("{color_title}Variable {var} value changed from '{color_default}{oldval}{color_title}' to '{color_default}{newval}{color_title}'{color_default}", var=dep, oldval=oldval, newval=newval))
  760. if not 'file_checksum_values' in a_data:
  761. a_data['file_checksum_values'] = {}
  762. if not 'file_checksum_values' in b_data:
  763. b_data['file_checksum_values'] = {}
  764. changed, added, removed = file_checksums_diff(a_data['file_checksum_values'], b_data['file_checksum_values'])
  765. if changed:
  766. for f, old, new in changed:
  767. output.append(color_format("{color_title}Checksum for file %s changed{color_default} from %s to %s") % (f, old, new))
  768. if added:
  769. for f in added:
  770. output.append(color_format("{color_title}Dependency on checksum of file %s was added") % (f))
  771. if removed:
  772. for f in removed:
  773. output.append(color_format("{color_title}Dependency on checksum of file %s was removed") % (f))
  774. if not 'runtaskdeps' in a_data:
  775. a_data['runtaskdeps'] = {}
  776. if not 'runtaskdeps' in b_data:
  777. b_data['runtaskdeps'] = {}
  778. if not collapsed:
  779. if len(a_data['runtaskdeps']) != len(b_data['runtaskdeps']):
  780. changed = ["Number of task dependencies changed"]
  781. else:
  782. changed = []
  783. for idx, task in enumerate(a_data['runtaskdeps']):
  784. a = a_data['runtaskdeps'][idx]
  785. b = b_data['runtaskdeps'][idx]
  786. if a_data['runtaskhashes'][a] != b_data['runtaskhashes'][b] and not collapsed:
  787. changed.append("%s with hash %s\n changed to\n%s with hash %s" % (clean_basepath(a), a_data['runtaskhashes'][a], clean_basepath(b), b_data['runtaskhashes'][b]))
  788. if changed:
  789. clean_a = clean_basepaths_list(a_data['runtaskdeps'])
  790. clean_b = clean_basepaths_list(b_data['runtaskdeps'])
  791. if clean_a != clean_b:
  792. output.append(color_format("{color_title}runtaskdeps changed:{color_default}\n%s") % list_inline_diff(clean_a, clean_b, colors))
  793. else:
  794. output.append(color_format("{color_title}runtaskdeps changed:"))
  795. output.append("\n".join(changed))
  796. if 'runtaskhashes' in a_data and 'runtaskhashes' in b_data:
  797. a = a_data['runtaskhashes']
  798. b = b_data['runtaskhashes']
  799. changed, added, removed = dict_diff(a, b)
  800. if added:
  801. for dep in added:
  802. bdep_found = False
  803. if removed:
  804. for bdep in removed:
  805. if b[dep] == a[bdep]:
  806. #output.append("Dependency on task %s was replaced by %s with same hash" % (dep, bdep))
  807. bdep_found = True
  808. if not bdep_found:
  809. output.append(color_format("{color_title}Dependency on task %s was added{color_default} with hash %s") % (clean_basepath(dep), b[dep]))
  810. if removed:
  811. for dep in removed:
  812. adep_found = False
  813. if added:
  814. for adep in added:
  815. if b[adep] == a[dep]:
  816. #output.append("Dependency on task %s was replaced by %s with same hash" % (adep, dep))
  817. adep_found = True
  818. if not adep_found:
  819. output.append(color_format("{color_title}Dependency on task %s was removed{color_default} with hash %s") % (clean_basepath(dep), a[dep]))
  820. if changed:
  821. for dep in changed:
  822. if not collapsed:
  823. output.append(color_format("{color_title}Hash for dependent task %s changed{color_default} from %s to %s") % (clean_basepath(dep), a[dep], b[dep]))
  824. if callable(recursecb):
  825. recout = recursecb(dep, a[dep], b[dep])
  826. if recout:
  827. if collapsed:
  828. output.extend(recout)
  829. else:
  830. # If a dependent hash changed, might as well print the line above and then defer to the changes in
  831. # that hash since in all likelyhood, they're the same changes this task also saw.
  832. output = [output[-1]] + recout
  833. a_taint = a_data.get('taint', None)
  834. b_taint = b_data.get('taint', None)
  835. if a_taint != b_taint:
  836. if a_taint and a_taint.startswith('nostamp:'):
  837. a_taint = a_taint.replace('nostamp:', 'nostamp(uuid4):')
  838. if b_taint and b_taint.startswith('nostamp:'):
  839. b_taint = b_taint.replace('nostamp:', 'nostamp(uuid4):')
  840. output.append(color_format("{color_title}Taint (by forced/invalidated task) changed{color_default} from %s to %s") % (a_taint, b_taint))
  841. return output
  842. def calc_basehash(sigdata):
  843. task = sigdata['task']
  844. basedata = sigdata['varvals'][task]
  845. if basedata is None:
  846. basedata = ''
  847. alldeps = sigdata['taskdeps']
  848. for dep in alldeps:
  849. basedata = basedata + dep
  850. val = sigdata['varvals'][dep]
  851. if val is not None:
  852. basedata = basedata + str(val)
  853. return hashlib.sha256(basedata.encode("utf-8")).hexdigest()
  854. def calc_taskhash(sigdata):
  855. data = sigdata['basehash']
  856. for dep in sigdata['runtaskdeps']:
  857. data = data + sigdata['runtaskhashes'][dep]
  858. for c in sigdata['file_checksum_values']:
  859. if c[1]:
  860. data = data + c[1]
  861. if 'taint' in sigdata:
  862. if 'nostamp:' in sigdata['taint']:
  863. data = data + sigdata['taint'][8:]
  864. else:
  865. data = data + sigdata['taint']
  866. return hashlib.sha256(data.encode("utf-8")).hexdigest()
  867. def dump_sigfile(a):
  868. output = []
  869. with open(a, 'rb') as f:
  870. p1 = pickle.Unpickler(f)
  871. a_data = p1.load()
  872. output.append("basewhitelist: %s" % (a_data['basewhitelist']))
  873. output.append("taskwhitelist: %s" % (a_data['taskwhitelist']))
  874. output.append("Task dependencies: %s" % (sorted(a_data['taskdeps'])))
  875. output.append("basehash: %s" % (a_data['basehash']))
  876. for dep in a_data['gendeps']:
  877. output.append("List of dependencies for variable %s is %s" % (dep, a_data['gendeps'][dep]))
  878. for dep in a_data['varvals']:
  879. output.append("Variable %s value is %s" % (dep, a_data['varvals'][dep]))
  880. if 'runtaskdeps' in a_data:
  881. output.append("Tasks this task depends on: %s" % (a_data['runtaskdeps']))
  882. if 'file_checksum_values' in a_data:
  883. output.append("This task depends on the checksums of files: %s" % (a_data['file_checksum_values']))
  884. if 'runtaskhashes' in a_data:
  885. for dep in a_data['runtaskhashes']:
  886. output.append("Hash for dependent task %s is %s" % (dep, a_data['runtaskhashes'][dep]))
  887. if 'taint' in a_data:
  888. if a_data['taint'].startswith('nostamp:'):
  889. msg = a_data['taint'].replace('nostamp:', 'nostamp(uuid4):')
  890. else:
  891. msg = a_data['taint']
  892. output.append("Tainted (by forced/invalidated task): %s" % msg)
  893. if 'task' in a_data:
  894. computed_basehash = calc_basehash(a_data)
  895. output.append("Computed base hash is %s and from file %s" % (computed_basehash, a_data['basehash']))
  896. else:
  897. output.append("Unable to compute base hash")
  898. computed_taskhash = calc_taskhash(a_data)
  899. output.append("Computed task hash is %s" % computed_taskhash)
  900. return output