cache.py 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879
  1. #
  2. # BitBake Cache implementation
  3. #
  4. # Caching of bitbake variables before task execution
  5. # Copyright (C) 2006 Richard Purdie
  6. # Copyright (C) 2012 Intel Corporation
  7. # but small sections based on code from bin/bitbake:
  8. # Copyright (C) 2003, 2004 Chris Larson
  9. # Copyright (C) 2003, 2004 Phil Blundell
  10. # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
  11. # Copyright (C) 2005 Holger Hans Peter Freyther
  12. # Copyright (C) 2005 ROAD GmbH
  13. #
  14. # SPDX-License-Identifier: GPL-2.0-only
  15. #
  16. import os
  17. import sys
  18. import logging
  19. import pickle
  20. from collections import defaultdict
  21. import bb.utils
  22. logger = logging.getLogger("BitBake.Cache")
  23. __cache_version__ = "152"
  24. def getCacheFile(path, filename, data_hash):
  25. return os.path.join(path, filename + "." + data_hash)
  26. # RecipeInfoCommon defines common data retrieving methods
  27. # from meta data for caches. CoreRecipeInfo as well as other
  28. # Extra RecipeInfo needs to inherit this class
  29. class RecipeInfoCommon(object):
  30. @classmethod
  31. def listvar(cls, var, metadata):
  32. return cls.getvar(var, metadata).split()
  33. @classmethod
  34. def intvar(cls, var, metadata):
  35. return int(cls.getvar(var, metadata) or 0)
  36. @classmethod
  37. def depvar(cls, var, metadata):
  38. return bb.utils.explode_deps(cls.getvar(var, metadata))
  39. @classmethod
  40. def pkgvar(cls, var, packages, metadata):
  41. return dict((pkg, cls.depvar("%s_%s" % (var, pkg), metadata))
  42. for pkg in packages)
  43. @classmethod
  44. def taskvar(cls, var, tasks, metadata):
  45. return dict((task, cls.getvar("%s_task-%s" % (var, task), metadata))
  46. for task in tasks)
  47. @classmethod
  48. def flaglist(cls, flag, varlist, metadata, squash=False):
  49. out_dict = dict((var, metadata.getVarFlag(var, flag))
  50. for var in varlist)
  51. if squash:
  52. return dict((k,v) for (k,v) in out_dict.items() if v)
  53. else:
  54. return out_dict
  55. @classmethod
  56. def getvar(cls, var, metadata, expand = True):
  57. return metadata.getVar(var, expand) or ''
  58. class CoreRecipeInfo(RecipeInfoCommon):
  59. __slots__ = ()
  60. cachefile = "bb_cache.dat"
  61. def __init__(self, filename, metadata):
  62. self.file_depends = metadata.getVar('__depends', False)
  63. self.timestamp = bb.parse.cached_mtime(filename)
  64. self.variants = self.listvar('__VARIANTS', metadata) + ['']
  65. self.appends = self.listvar('__BBAPPEND', metadata)
  66. self.nocache = self.getvar('BB_DONT_CACHE', metadata)
  67. self.skipreason = self.getvar('__SKIPPED', metadata)
  68. if self.skipreason:
  69. self.pn = self.getvar('PN', metadata) or bb.parse.vars_from_file(filename,metadata)[0]
  70. self.skipped = True
  71. self.provides = self.depvar('PROVIDES', metadata)
  72. self.rprovides = self.depvar('RPROVIDES', metadata)
  73. return
  74. self.tasks = metadata.getVar('__BBTASKS', False)
  75. self.pn = self.getvar('PN', metadata)
  76. self.packages = self.listvar('PACKAGES', metadata)
  77. if not self.packages:
  78. self.packages.append(self.pn)
  79. self.basetaskhashes = self.taskvar('BB_BASEHASH', self.tasks, metadata)
  80. self.hashfilename = self.getvar('BB_HASHFILENAME', metadata)
  81. self.task_deps = metadata.getVar('_task_deps', False) or {'tasks': [], 'parents': {}}
  82. self.skipped = False
  83. self.pe = self.getvar('PE', metadata)
  84. self.pv = self.getvar('PV', metadata)
  85. self.pr = self.getvar('PR', metadata)
  86. self.defaultpref = self.intvar('DEFAULT_PREFERENCE', metadata)
  87. self.not_world = self.getvar('EXCLUDE_FROM_WORLD', metadata)
  88. self.stamp = self.getvar('STAMP', metadata)
  89. self.stampclean = self.getvar('STAMPCLEAN', metadata)
  90. self.stamp_extrainfo = self.flaglist('stamp-extra-info', self.tasks, metadata)
  91. self.file_checksums = self.flaglist('file-checksums', self.tasks, metadata, True)
  92. self.packages_dynamic = self.listvar('PACKAGES_DYNAMIC', metadata)
  93. self.depends = self.depvar('DEPENDS', metadata)
  94. self.provides = self.depvar('PROVIDES', metadata)
  95. self.rdepends = self.depvar('RDEPENDS', metadata)
  96. self.rprovides = self.depvar('RPROVIDES', metadata)
  97. self.rrecommends = self.depvar('RRECOMMENDS', metadata)
  98. self.rprovides_pkg = self.pkgvar('RPROVIDES', self.packages, metadata)
  99. self.rdepends_pkg = self.pkgvar('RDEPENDS', self.packages, metadata)
  100. self.rrecommends_pkg = self.pkgvar('RRECOMMENDS', self.packages, metadata)
  101. self.inherits = self.getvar('__inherit_cache', metadata, expand=False)
  102. self.fakerootenv = self.getvar('FAKEROOTENV', metadata)
  103. self.fakerootdirs = self.getvar('FAKEROOTDIRS', metadata)
  104. self.fakerootnoenv = self.getvar('FAKEROOTNOENV', metadata)
  105. self.extradepsfunc = self.getvar('calculate_extra_depends', metadata)
  106. @classmethod
  107. def init_cacheData(cls, cachedata):
  108. # CacheData in Core RecipeInfo Class
  109. cachedata.task_deps = {}
  110. cachedata.pkg_fn = {}
  111. cachedata.pkg_pn = defaultdict(list)
  112. cachedata.pkg_pepvpr = {}
  113. cachedata.pkg_dp = {}
  114. cachedata.stamp = {}
  115. cachedata.stampclean = {}
  116. cachedata.stamp_extrainfo = {}
  117. cachedata.file_checksums = {}
  118. cachedata.fn_provides = {}
  119. cachedata.pn_provides = defaultdict(list)
  120. cachedata.all_depends = []
  121. cachedata.deps = defaultdict(list)
  122. cachedata.packages = defaultdict(list)
  123. cachedata.providers = defaultdict(list)
  124. cachedata.rproviders = defaultdict(list)
  125. cachedata.packages_dynamic = defaultdict(list)
  126. cachedata.rundeps = defaultdict(lambda: defaultdict(list))
  127. cachedata.runrecs = defaultdict(lambda: defaultdict(list))
  128. cachedata.possible_world = []
  129. cachedata.universe_target = []
  130. cachedata.hashfn = {}
  131. cachedata.basetaskhash = {}
  132. cachedata.inherits = {}
  133. cachedata.fakerootenv = {}
  134. cachedata.fakerootnoenv = {}
  135. cachedata.fakerootdirs = {}
  136. cachedata.extradepsfunc = {}
  137. def add_cacheData(self, cachedata, fn):
  138. cachedata.task_deps[fn] = self.task_deps
  139. cachedata.pkg_fn[fn] = self.pn
  140. cachedata.pkg_pn[self.pn].append(fn)
  141. cachedata.pkg_pepvpr[fn] = (self.pe, self.pv, self.pr)
  142. cachedata.pkg_dp[fn] = self.defaultpref
  143. cachedata.stamp[fn] = self.stamp
  144. cachedata.stampclean[fn] = self.stampclean
  145. cachedata.stamp_extrainfo[fn] = self.stamp_extrainfo
  146. cachedata.file_checksums[fn] = self.file_checksums
  147. provides = [self.pn]
  148. for provide in self.provides:
  149. if provide not in provides:
  150. provides.append(provide)
  151. cachedata.fn_provides[fn] = provides
  152. for provide in provides:
  153. cachedata.providers[provide].append(fn)
  154. if provide not in cachedata.pn_provides[self.pn]:
  155. cachedata.pn_provides[self.pn].append(provide)
  156. for dep in self.depends:
  157. if dep not in cachedata.deps[fn]:
  158. cachedata.deps[fn].append(dep)
  159. if dep not in cachedata.all_depends:
  160. cachedata.all_depends.append(dep)
  161. rprovides = self.rprovides
  162. for package in self.packages:
  163. cachedata.packages[package].append(fn)
  164. rprovides += self.rprovides_pkg[package]
  165. for rprovide in rprovides:
  166. if fn not in cachedata.rproviders[rprovide]:
  167. cachedata.rproviders[rprovide].append(fn)
  168. for package in self.packages_dynamic:
  169. cachedata.packages_dynamic[package].append(fn)
  170. # Build hash of runtime depends and recommends
  171. for package in self.packages:
  172. cachedata.rundeps[fn][package] = list(self.rdepends) + self.rdepends_pkg[package]
  173. cachedata.runrecs[fn][package] = list(self.rrecommends) + self.rrecommends_pkg[package]
  174. # Collect files we may need for possible world-dep
  175. # calculations
  176. if self.not_world:
  177. logger.debug(1, "EXCLUDE FROM WORLD: %s", fn)
  178. else:
  179. cachedata.possible_world.append(fn)
  180. # create a collection of all targets for sanity checking
  181. # tasks, such as upstream versions, license, and tools for
  182. # task and image creation.
  183. cachedata.universe_target.append(self.pn)
  184. cachedata.hashfn[fn] = self.hashfilename
  185. for task, taskhash in self.basetaskhashes.items():
  186. identifier = '%s.%s' % (fn, task)
  187. cachedata.basetaskhash[identifier] = taskhash
  188. cachedata.inherits[fn] = self.inherits
  189. cachedata.fakerootenv[fn] = self.fakerootenv
  190. cachedata.fakerootnoenv[fn] = self.fakerootnoenv
  191. cachedata.fakerootdirs[fn] = self.fakerootdirs
  192. cachedata.extradepsfunc[fn] = self.extradepsfunc
  193. def virtualfn2realfn(virtualfn):
  194. """
  195. Convert a virtual file name to a real one + the associated subclass keyword
  196. """
  197. mc = ""
  198. if virtualfn.startswith('multiconfig:'):
  199. elems = virtualfn.split(':')
  200. mc = elems[1]
  201. virtualfn = ":".join(elems[2:])
  202. fn = virtualfn
  203. cls = ""
  204. if virtualfn.startswith('virtual:'):
  205. elems = virtualfn.split(':')
  206. cls = ":".join(elems[1:-1])
  207. fn = elems[-1]
  208. return (fn, cls, mc)
  209. def realfn2virtual(realfn, cls, mc):
  210. """
  211. Convert a real filename + the associated subclass keyword to a virtual filename
  212. """
  213. if cls:
  214. realfn = "virtual:" + cls + ":" + realfn
  215. if mc:
  216. realfn = "multiconfig:" + mc + ":" + realfn
  217. return realfn
  218. def variant2virtual(realfn, variant):
  219. """
  220. Convert a real filename + the associated subclass keyword to a virtual filename
  221. """
  222. if variant == "":
  223. return realfn
  224. if variant.startswith("multiconfig:"):
  225. elems = variant.split(":")
  226. if elems[2]:
  227. return "multiconfig:" + elems[1] + ":virtual:" + ":".join(elems[2:]) + ":" + realfn
  228. return "multiconfig:" + elems[1] + ":" + realfn
  229. return "virtual:" + variant + ":" + realfn
  230. def parse_recipe(bb_data, bbfile, appends, mc=''):
  231. """
  232. Parse a recipe
  233. """
  234. chdir_back = False
  235. bb_data.setVar("__BBMULTICONFIG", mc)
  236. # expand tmpdir to include this topdir
  237. bb_data.setVar('TMPDIR', bb_data.getVar('TMPDIR') or "")
  238. bbfile_loc = os.path.abspath(os.path.dirname(bbfile))
  239. oldpath = os.path.abspath(os.getcwd())
  240. bb.parse.cached_mtime_noerror(bbfile_loc)
  241. # The ConfHandler first looks if there is a TOPDIR and if not
  242. # then it would call getcwd().
  243. # Previously, we chdir()ed to bbfile_loc, called the handler
  244. # and finally chdir()ed back, a couple of thousand times. We now
  245. # just fill in TOPDIR to point to bbfile_loc if there is no TOPDIR yet.
  246. if not bb_data.getVar('TOPDIR', False):
  247. chdir_back = True
  248. bb_data.setVar('TOPDIR', bbfile_loc)
  249. try:
  250. if appends:
  251. bb_data.setVar('__BBAPPEND', " ".join(appends))
  252. bb_data = bb.parse.handle(bbfile, bb_data)
  253. if chdir_back:
  254. os.chdir(oldpath)
  255. return bb_data
  256. except:
  257. if chdir_back:
  258. os.chdir(oldpath)
  259. raise
  260. class NoCache(object):
  261. def __init__(self, databuilder):
  262. self.databuilder = databuilder
  263. self.data = databuilder.data
  264. def loadDataFull(self, virtualfn, appends):
  265. """
  266. Return a complete set of data for fn.
  267. To do this, we need to parse the file.
  268. """
  269. logger.debug(1, "Parsing %s (full)" % virtualfn)
  270. (fn, virtual, mc) = virtualfn2realfn(virtualfn)
  271. bb_data = self.load_bbfile(virtualfn, appends, virtonly=True)
  272. return bb_data[virtual]
  273. def load_bbfile(self, bbfile, appends, virtonly = False):
  274. """
  275. Load and parse one .bb build file
  276. Return the data and whether parsing resulted in the file being skipped
  277. """
  278. if virtonly:
  279. (bbfile, virtual, mc) = virtualfn2realfn(bbfile)
  280. bb_data = self.databuilder.mcdata[mc].createCopy()
  281. bb_data.setVar("__ONLYFINALISE", virtual or "default")
  282. datastores = parse_recipe(bb_data, bbfile, appends, mc)
  283. return datastores
  284. bb_data = self.data.createCopy()
  285. datastores = parse_recipe(bb_data, bbfile, appends)
  286. for mc in self.databuilder.mcdata:
  287. if not mc:
  288. continue
  289. bb_data = self.databuilder.mcdata[mc].createCopy()
  290. newstores = parse_recipe(bb_data, bbfile, appends, mc)
  291. for ns in newstores:
  292. datastores["multiconfig:%s:%s" % (mc, ns)] = newstores[ns]
  293. return datastores
  294. class Cache(NoCache):
  295. """
  296. BitBake Cache implementation
  297. """
  298. def __init__(self, databuilder, data_hash, caches_array):
  299. super().__init__(databuilder)
  300. data = databuilder.data
  301. # Pass caches_array information into Cache Constructor
  302. # It will be used later for deciding whether we
  303. # need extra cache file dump/load support
  304. self.caches_array = caches_array
  305. self.cachedir = data.getVar("CACHE")
  306. self.clean = set()
  307. self.checked = set()
  308. self.depends_cache = {}
  309. self.data_fn = None
  310. self.cacheclean = True
  311. self.data_hash = data_hash
  312. if self.cachedir in [None, '']:
  313. self.has_cache = False
  314. logger.info("Not using a cache. "
  315. "Set CACHE = <directory> to enable.")
  316. return
  317. self.has_cache = True
  318. self.cachefile = getCacheFile(self.cachedir, "bb_cache.dat", self.data_hash)
  319. logger.debug(1, "Cache dir: %s", self.cachedir)
  320. bb.utils.mkdirhier(self.cachedir)
  321. cache_ok = True
  322. if self.caches_array:
  323. for cache_class in self.caches_array:
  324. cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
  325. cache_ok = cache_ok and os.path.exists(cachefile)
  326. cache_class.init_cacheData(self)
  327. if cache_ok:
  328. self.load_cachefile()
  329. elif os.path.isfile(self.cachefile):
  330. logger.info("Out of date cache found, rebuilding...")
  331. else:
  332. logger.debug(1, "Cache file %s not found, building..." % self.cachefile)
  333. def load_cachefile(self):
  334. cachesize = 0
  335. previous_progress = 0
  336. previous_percent = 0
  337. # Calculate the correct cachesize of all those cache files
  338. for cache_class in self.caches_array:
  339. cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
  340. with open(cachefile, "rb") as cachefile:
  341. cachesize += os.fstat(cachefile.fileno()).st_size
  342. bb.event.fire(bb.event.CacheLoadStarted(cachesize), self.data)
  343. for cache_class in self.caches_array:
  344. cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
  345. logger.debug(1, 'Loading cache file: %s' % cachefile)
  346. with open(cachefile, "rb") as cachefile:
  347. pickled = pickle.Unpickler(cachefile)
  348. # Check cache version information
  349. try:
  350. cache_ver = pickled.load()
  351. bitbake_ver = pickled.load()
  352. except Exception:
  353. logger.info('Invalid cache, rebuilding...')
  354. return
  355. if cache_ver != __cache_version__:
  356. logger.info('Cache version mismatch, rebuilding...')
  357. return
  358. elif bitbake_ver != bb.__version__:
  359. logger.info('Bitbake version mismatch, rebuilding...')
  360. return
  361. # Load the rest of the cache file
  362. current_progress = 0
  363. while cachefile:
  364. try:
  365. key = pickled.load()
  366. value = pickled.load()
  367. except Exception:
  368. break
  369. if not isinstance(key, str):
  370. bb.warn("%s from extras cache is not a string?" % key)
  371. break
  372. if not isinstance(value, RecipeInfoCommon):
  373. bb.warn("%s from extras cache is not a RecipeInfoCommon class?" % value)
  374. break
  375. if key in self.depends_cache:
  376. self.depends_cache[key].append(value)
  377. else:
  378. self.depends_cache[key] = [value]
  379. # only fire events on even percentage boundaries
  380. current_progress = cachefile.tell() + previous_progress
  381. if current_progress > cachesize:
  382. # we might have calculated incorrect total size because a file
  383. # might've been written out just after we checked its size
  384. cachesize = current_progress
  385. current_percent = 100 * current_progress / cachesize
  386. if current_percent > previous_percent:
  387. previous_percent = current_percent
  388. bb.event.fire(bb.event.CacheLoadProgress(current_progress, cachesize),
  389. self.data)
  390. previous_progress += current_progress
  391. # Note: depends cache number is corresponding to the parsing file numbers.
  392. # The same file has several caches, still regarded as one item in the cache
  393. bb.event.fire(bb.event.CacheLoadCompleted(cachesize,
  394. len(self.depends_cache)),
  395. self.data)
  396. def parse(self, filename, appends):
  397. """Parse the specified filename, returning the recipe information"""
  398. logger.debug(1, "Parsing %s", filename)
  399. infos = []
  400. datastores = self.load_bbfile(filename, appends)
  401. depends = []
  402. variants = []
  403. # Process the "real" fn last so we can store variants list
  404. for variant, data in sorted(datastores.items(),
  405. key=lambda i: i[0],
  406. reverse=True):
  407. virtualfn = variant2virtual(filename, variant)
  408. variants.append(variant)
  409. depends = depends + (data.getVar("__depends", False) or [])
  410. if depends and not variant:
  411. data.setVar("__depends", depends)
  412. if virtualfn == filename:
  413. data.setVar("__VARIANTS", " ".join(variants))
  414. info_array = []
  415. for cache_class in self.caches_array:
  416. info = cache_class(filename, data)
  417. info_array.append(info)
  418. infos.append((virtualfn, info_array))
  419. return infos
  420. def load(self, filename, appends):
  421. """Obtain the recipe information for the specified filename,
  422. using cached values if available, otherwise parsing.
  423. Note that if it does parse to obtain the info, it will not
  424. automatically add the information to the cache or to your
  425. CacheData. Use the add or add_info method to do so after
  426. running this, or use loadData instead."""
  427. cached = self.cacheValid(filename, appends)
  428. if cached:
  429. infos = []
  430. # info_array item is a list of [CoreRecipeInfo, XXXRecipeInfo]
  431. info_array = self.depends_cache[filename]
  432. for variant in info_array[0].variants:
  433. virtualfn = variant2virtual(filename, variant)
  434. infos.append((virtualfn, self.depends_cache[virtualfn]))
  435. else:
  436. return self.parse(filename, appends, configdata, self.caches_array)
  437. return cached, infos
  438. def loadData(self, fn, appends, cacheData):
  439. """Load the recipe info for the specified filename,
  440. parsing and adding to the cache if necessary, and adding
  441. the recipe information to the supplied CacheData instance."""
  442. skipped, virtuals = 0, 0
  443. cached, infos = self.load(fn, appends)
  444. for virtualfn, info_array in infos:
  445. if info_array[0].skipped:
  446. logger.debug(1, "Skipping %s: %s", virtualfn, info_array[0].skipreason)
  447. skipped += 1
  448. else:
  449. self.add_info(virtualfn, info_array, cacheData, not cached)
  450. virtuals += 1
  451. return cached, skipped, virtuals
  452. def cacheValid(self, fn, appends):
  453. """
  454. Is the cache valid for fn?
  455. Fast version, no timestamps checked.
  456. """
  457. if fn not in self.checked:
  458. self.cacheValidUpdate(fn, appends)
  459. # Is cache enabled?
  460. if not self.has_cache:
  461. return False
  462. if fn in self.clean:
  463. return True
  464. return False
  465. def cacheValidUpdate(self, fn, appends):
  466. """
  467. Is the cache valid for fn?
  468. Make thorough (slower) checks including timestamps.
  469. """
  470. # Is cache enabled?
  471. if not self.has_cache:
  472. return False
  473. self.checked.add(fn)
  474. # File isn't in depends_cache
  475. if not fn in self.depends_cache:
  476. logger.debug(2, "Cache: %s is not cached", fn)
  477. return False
  478. mtime = bb.parse.cached_mtime_noerror(fn)
  479. # Check file still exists
  480. if mtime == 0:
  481. logger.debug(2, "Cache: %s no longer exists", fn)
  482. self.remove(fn)
  483. return False
  484. info_array = self.depends_cache[fn]
  485. # Check the file's timestamp
  486. if mtime != info_array[0].timestamp:
  487. logger.debug(2, "Cache: %s changed", fn)
  488. self.remove(fn)
  489. return False
  490. # Check dependencies are still valid
  491. depends = info_array[0].file_depends
  492. if depends:
  493. for f, old_mtime in depends:
  494. fmtime = bb.parse.cached_mtime_noerror(f)
  495. # Check if file still exists
  496. if old_mtime != 0 and fmtime == 0:
  497. logger.debug(2, "Cache: %s's dependency %s was removed",
  498. fn, f)
  499. self.remove(fn)
  500. return False
  501. if (fmtime != old_mtime):
  502. logger.debug(2, "Cache: %s's dependency %s changed",
  503. fn, f)
  504. self.remove(fn)
  505. return False
  506. if hasattr(info_array[0], 'file_checksums'):
  507. for _, fl in info_array[0].file_checksums.items():
  508. fl = fl.strip()
  509. while fl:
  510. # A .split() would be simpler but means spaces or colons in filenames would break
  511. a = fl.find(":True")
  512. b = fl.find(":False")
  513. if ((a < 0) and b) or ((b > 0) and (b < a)):
  514. f = fl[:b+6]
  515. fl = fl[b+7:]
  516. elif ((b < 0) and a) or ((a > 0) and (a < b)):
  517. f = fl[:a+5]
  518. fl = fl[a+6:]
  519. else:
  520. break
  521. fl = fl.strip()
  522. if "*" in f:
  523. continue
  524. f, exist = f.split(":")
  525. if (exist == "True" and not os.path.exists(f)) or (exist == "False" and os.path.exists(f)):
  526. logger.debug(2, "Cache: %s's file checksum list file %s changed",
  527. fn, f)
  528. self.remove(fn)
  529. return False
  530. if appends != info_array[0].appends:
  531. logger.debug(2, "Cache: appends for %s changed", fn)
  532. logger.debug(2, "%s to %s" % (str(appends), str(info_array[0].appends)))
  533. self.remove(fn)
  534. return False
  535. invalid = False
  536. for cls in info_array[0].variants:
  537. virtualfn = variant2virtual(fn, cls)
  538. self.clean.add(virtualfn)
  539. if virtualfn not in self.depends_cache:
  540. logger.debug(2, "Cache: %s is not cached", virtualfn)
  541. invalid = True
  542. elif len(self.depends_cache[virtualfn]) != len(self.caches_array):
  543. logger.debug(2, "Cache: Extra caches missing for %s?" % virtualfn)
  544. invalid = True
  545. # If any one of the variants is not present, mark as invalid for all
  546. if invalid:
  547. for cls in info_array[0].variants:
  548. virtualfn = variant2virtual(fn, cls)
  549. if virtualfn in self.clean:
  550. logger.debug(2, "Cache: Removing %s from cache", virtualfn)
  551. self.clean.remove(virtualfn)
  552. if fn in self.clean:
  553. logger.debug(2, "Cache: Marking %s as not clean", fn)
  554. self.clean.remove(fn)
  555. return False
  556. self.clean.add(fn)
  557. return True
  558. def remove(self, fn):
  559. """
  560. Remove a fn from the cache
  561. Called from the parser in error cases
  562. """
  563. if fn in self.depends_cache:
  564. logger.debug(1, "Removing %s from cache", fn)
  565. del self.depends_cache[fn]
  566. if fn in self.clean:
  567. logger.debug(1, "Marking %s as unclean", fn)
  568. self.clean.remove(fn)
  569. def sync(self):
  570. """
  571. Save the cache
  572. Called from the parser when complete (or exiting)
  573. """
  574. if not self.has_cache:
  575. return
  576. if self.cacheclean:
  577. logger.debug(2, "Cache is clean, not saving.")
  578. return
  579. for cache_class in self.caches_array:
  580. cache_class_name = cache_class.__name__
  581. cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
  582. with open(cachefile, "wb") as f:
  583. p = pickle.Pickler(f, pickle.HIGHEST_PROTOCOL)
  584. p.dump(__cache_version__)
  585. p.dump(bb.__version__)
  586. for key, info_array in self.depends_cache.items():
  587. for info in info_array:
  588. if isinstance(info, RecipeInfoCommon) and info.__class__.__name__ == cache_class_name:
  589. p.dump(key)
  590. p.dump(info)
  591. del self.depends_cache
  592. @staticmethod
  593. def mtime(cachefile):
  594. return bb.parse.cached_mtime_noerror(cachefile)
  595. def add_info(self, filename, info_array, cacheData, parsed=None, watcher=None):
  596. if isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped):
  597. cacheData.add_from_recipeinfo(filename, info_array)
  598. if watcher:
  599. watcher(info_array[0].file_depends)
  600. if not self.has_cache:
  601. return
  602. if (info_array[0].skipped or 'SRCREVINACTION' not in info_array[0].pv) and not info_array[0].nocache:
  603. if parsed:
  604. self.cacheclean = False
  605. self.depends_cache[filename] = info_array
  606. def add(self, file_name, data, cacheData, parsed=None):
  607. """
  608. Save data we need into the cache
  609. """
  610. realfn = virtualfn2realfn(file_name)[0]
  611. info_array = []
  612. for cache_class in self.caches_array:
  613. info_array.append(cache_class(realfn, data))
  614. self.add_info(file_name, info_array, cacheData, parsed)
  615. def init(cooker):
  616. """
  617. The Objective: Cache the minimum amount of data possible yet get to the
  618. stage of building packages (i.e. tryBuild) without reparsing any .bb files.
  619. To do this, we intercept getVar calls and only cache the variables we see
  620. being accessed. We rely on the cache getVar calls being made for all
  621. variables bitbake might need to use to reach this stage. For each cached
  622. file we need to track:
  623. * Its mtime
  624. * The mtimes of all its dependencies
  625. * Whether it caused a parse.SkipRecipe exception
  626. Files causing parsing errors are evicted from the cache.
  627. """
  628. return Cache(cooker.configuration.data, cooker.configuration.data_hash)
  629. class CacheData(object):
  630. """
  631. The data structures we compile from the cached data
  632. """
  633. def __init__(self, caches_array):
  634. self.caches_array = caches_array
  635. for cache_class in self.caches_array:
  636. if not issubclass(cache_class, RecipeInfoCommon):
  637. bb.error("Extra cache data class %s should subclass RecipeInfoCommon class" % cache_class)
  638. cache_class.init_cacheData(self)
  639. # Direct cache variables
  640. self.task_queues = {}
  641. self.preferred = {}
  642. self.tasks = {}
  643. # Indirect Cache variables (set elsewhere)
  644. self.ignored_dependencies = []
  645. self.world_target = set()
  646. self.bbfile_priority = {}
  647. def add_from_recipeinfo(self, fn, info_array):
  648. for info in info_array:
  649. info.add_cacheData(self, fn)
  650. class MultiProcessCache(object):
  651. """
  652. BitBake multi-process cache implementation
  653. Used by the codeparser & file checksum caches
  654. """
  655. def __init__(self):
  656. self.cachefile = None
  657. self.cachedata = self.create_cachedata()
  658. self.cachedata_extras = self.create_cachedata()
  659. def init_cache(self, d, cache_file_name=None):
  660. cachedir = (d.getVar("PERSISTENT_DIR") or
  661. d.getVar("CACHE"))
  662. if cachedir in [None, '']:
  663. return
  664. bb.utils.mkdirhier(cachedir)
  665. self.cachefile = os.path.join(cachedir,
  666. cache_file_name or self.__class__.cache_file_name)
  667. logger.debug(1, "Using cache in '%s'", self.cachefile)
  668. glf = bb.utils.lockfile(self.cachefile + ".lock")
  669. try:
  670. with open(self.cachefile, "rb") as f:
  671. p = pickle.Unpickler(f)
  672. data, version = p.load()
  673. except:
  674. bb.utils.unlockfile(glf)
  675. return
  676. bb.utils.unlockfile(glf)
  677. if version != self.__class__.CACHE_VERSION:
  678. return
  679. self.cachedata = data
  680. def create_cachedata(self):
  681. data = [{}]
  682. return data
  683. def save_extras(self):
  684. if not self.cachefile:
  685. return
  686. glf = bb.utils.lockfile(self.cachefile + ".lock", shared=True)
  687. i = os.getpid()
  688. lf = None
  689. while not lf:
  690. lf = bb.utils.lockfile(self.cachefile + ".lock." + str(i), retry=False)
  691. if not lf or os.path.exists(self.cachefile + "-" + str(i)):
  692. if lf:
  693. bb.utils.unlockfile(lf)
  694. lf = None
  695. i = i + 1
  696. continue
  697. with open(self.cachefile + "-" + str(i), "wb") as f:
  698. p = pickle.Pickler(f, -1)
  699. p.dump([self.cachedata_extras, self.__class__.CACHE_VERSION])
  700. bb.utils.unlockfile(lf)
  701. bb.utils.unlockfile(glf)
  702. def merge_data(self, source, dest):
  703. for j in range(0,len(dest)):
  704. for h in source[j]:
  705. if h not in dest[j]:
  706. dest[j][h] = source[j][h]
  707. def save_merge(self):
  708. if not self.cachefile:
  709. return
  710. glf = bb.utils.lockfile(self.cachefile + ".lock")
  711. data = self.cachedata
  712. for f in [y for y in os.listdir(os.path.dirname(self.cachefile)) if y.startswith(os.path.basename(self.cachefile) + '-')]:
  713. f = os.path.join(os.path.dirname(self.cachefile), f)
  714. try:
  715. with open(f, "rb") as fd:
  716. p = pickle.Unpickler(fd)
  717. extradata, version = p.load()
  718. except (IOError, EOFError):
  719. os.unlink(f)
  720. continue
  721. if version != self.__class__.CACHE_VERSION:
  722. os.unlink(f)
  723. continue
  724. self.merge_data(extradata, data)
  725. os.unlink(f)
  726. with open(self.cachefile, "wb") as f:
  727. p = pickle.Pickler(f, -1)
  728. p.dump([data, self.__class__.CACHE_VERSION])
  729. bb.utils.unlockfile(glf)