cache.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550
  1. # ex:ts=4:sw=4:sts=4:et
  2. # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
  3. #
  4. # BitBake 'Event' implementation
  5. #
  6. # Caching of bitbake variables before task execution
  7. # Copyright (C) 2006 Richard Purdie
  8. # but small sections based on code from bin/bitbake:
  9. # Copyright (C) 2003, 2004 Chris Larson
  10. # Copyright (C) 2003, 2004 Phil Blundell
  11. # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
  12. # Copyright (C) 2005 Holger Hans Peter Freyther
  13. # Copyright (C) 2005 ROAD GmbH
  14. #
  15. # This program is free software; you can redistribute it and/or modify
  16. # it under the terms of the GNU General Public License version 2 as
  17. # published by the Free Software Foundation.
  18. #
  19. # This program is distributed in the hope that it will be useful,
  20. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  21. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  22. # GNU General Public License for more details.
  23. #
  24. # You should have received a copy of the GNU General Public License along
  25. # with this program; if not, write to the Free Software Foundation, Inc.,
  26. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  27. import os
  28. import bb.data
  29. import bb.utils
  30. try:
  31. import cPickle as pickle
  32. except ImportError:
  33. import pickle
  34. bb.msg.note(1, bb.msg.domain.Cache, "Importing cPickle failed. Falling back to a very slow implementation.")
  35. __cache_version__ = "131"
  36. class Cache:
  37. """
  38. BitBake Cache implementation
  39. """
  40. def __init__(self, data):
  41. self.cachedir = bb.data.getVar("CACHE", data, True)
  42. self.clean = {}
  43. self.checked = {}
  44. self.depends_cache = {}
  45. self.data = None
  46. self.data_fn = None
  47. self.cacheclean = True
  48. if self.cachedir in [None, '']:
  49. self.has_cache = False
  50. bb.msg.note(1, bb.msg.domain.Cache, "Not using a cache. Set CACHE = <directory> to enable.")
  51. return
  52. self.has_cache = True
  53. self.cachefile = os.path.join(self.cachedir, "bb_cache.dat")
  54. bb.msg.debug(1, bb.msg.domain.Cache, "Using cache in '%s'" % self.cachedir)
  55. bb.utils.mkdirhier(self.cachedir)
  56. # If any of configuration.data's dependencies are newer than the
  57. # cache there isn't even any point in loading it...
  58. newest_mtime = 0
  59. deps = bb.data.getVar("__depends", data)
  60. old_mtimes = [old_mtime for f, old_mtime in deps]
  61. old_mtimes.append(newest_mtime)
  62. newest_mtime = max(old_mtimes)
  63. if bb.parse.cached_mtime_noerror(self.cachefile) >= newest_mtime:
  64. try:
  65. p = pickle.Unpickler(file(self.cachefile, "rb"))
  66. self.depends_cache, version_data = p.load()
  67. if version_data['CACHE_VER'] != __cache_version__:
  68. raise ValueError('Cache Version Mismatch')
  69. if version_data['BITBAKE_VER'] != bb.__version__:
  70. raise ValueError('Bitbake Version Mismatch')
  71. except EOFError:
  72. bb.msg.note(1, bb.msg.domain.Cache, "Truncated cache found, rebuilding...")
  73. self.depends_cache = {}
  74. except:
  75. bb.msg.note(1, bb.msg.domain.Cache, "Invalid cache found, rebuilding...")
  76. self.depends_cache = {}
  77. else:
  78. if os.path.isfile(self.cachefile):
  79. bb.msg.note(1, bb.msg.domain.Cache, "Out of date cache found, rebuilding...")
  80. def getVar(self, var, fn, exp = 0):
  81. """
  82. Gets the value of a variable
  83. (similar to getVar in the data class)
  84. There are two scenarios:
  85. 1. We have cached data - serve from depends_cache[fn]
  86. 2. We're learning what data to cache - serve from data
  87. backend but add a copy of the data to the cache.
  88. """
  89. if fn in self.clean:
  90. return self.depends_cache[fn][var]
  91. self.depends_cache.setdefault(fn, {})
  92. if fn != self.data_fn:
  93. # We're trying to access data in the cache which doesn't exist
  94. # yet setData hasn't been called to setup the right access. Very bad.
  95. bb.msg.error(bb.msg.domain.Cache, "Parsing error data_fn %s and fn %s don't match" % (self.data_fn, fn))
  96. self.cacheclean = False
  97. result = bb.data.getVar(var, self.data, exp)
  98. self.depends_cache[fn][var] = result
  99. return result
  100. def setData(self, virtualfn, fn, data):
  101. """
  102. Called to prime bb_cache ready to learn which variables to cache.
  103. Will be followed by calls to self.getVar which aren't cached
  104. but can be fulfilled from self.data.
  105. """
  106. self.data_fn = virtualfn
  107. self.data = data
  108. # Make sure __depends makes the depends_cache
  109. # If we're a virtual class we need to make sure all our depends are appended
  110. # to the depends of fn.
  111. depends = self.getVar("__depends", virtualfn) or set()
  112. self.depends_cache.setdefault(fn, {})
  113. if "__depends" not in self.depends_cache[fn] or not self.depends_cache[fn]["__depends"]:
  114. self.depends_cache[fn]["__depends"] = depends
  115. else:
  116. self.depends_cache[fn]["__depends"].update(depends)
  117. # Make sure the variants always make it into the cache too
  118. self.getVar('__VARIANTS', virtualfn, True)
  119. self.depends_cache[virtualfn]["CACHETIMESTAMP"] = bb.parse.cached_mtime(fn)
  120. def virtualfn2realfn(self, virtualfn):
  121. """
  122. Convert a virtual file name to a real one + the associated subclass keyword
  123. """
  124. fn = virtualfn
  125. cls = ""
  126. if virtualfn.startswith('virtual:'):
  127. cls = virtualfn.split(':', 2)[1]
  128. fn = virtualfn.replace('virtual:' + cls + ':', '')
  129. #bb.msg.debug(2, bb.msg.domain.Cache, "virtualfn2realfn %s to %s %s" % (virtualfn, fn, cls))
  130. return (fn, cls)
  131. def realfn2virtual(self, realfn, cls):
  132. """
  133. Convert a real filename + the associated subclass keyword to a virtual filename
  134. """
  135. if cls == "":
  136. #bb.msg.debug(2, bb.msg.domain.Cache, "realfn2virtual %s and '%s' to %s" % (realfn, cls, realfn))
  137. return realfn
  138. #bb.msg.debug(2, bb.msg.domain.Cache, "realfn2virtual %s and %s to %s" % (realfn, cls, "virtual:" + cls + ":" + realfn))
  139. return "virtual:" + cls + ":" + realfn
  140. def loadDataFull(self, virtualfn, appends, cfgData):
  141. """
  142. Return a complete set of data for fn.
  143. To do this, we need to parse the file.
  144. """
  145. (fn, cls) = self.virtualfn2realfn(virtualfn)
  146. bb.msg.debug(1, bb.msg.domain.Cache, "Parsing %s (full)" % fn)
  147. bb_data = self.load_bbfile(fn, appends, cfgData)
  148. return bb_data[cls]
  149. def loadData(self, fn, appends, cfgData, cacheData):
  150. """
  151. Load a subset of data for fn.
  152. If the cached data is valid we do nothing,
  153. To do this, we need to parse the file and set the system
  154. to record the variables accessed.
  155. Return the cache status and whether the file was skipped when parsed
  156. """
  157. skipped = 0
  158. virtuals = 0
  159. if fn not in self.checked:
  160. self.cacheValidUpdate(fn)
  161. if self.cacheValid(fn):
  162. multi = self.getVar('__VARIANTS', fn, True)
  163. for cls in (multi or "").split() + [""]:
  164. virtualfn = self.realfn2virtual(fn, cls)
  165. if self.depends_cache[virtualfn]["__SKIPPED"]:
  166. skipped += 1
  167. bb.msg.debug(1, bb.msg.domain.Cache, "Skipping %s" % virtualfn)
  168. continue
  169. self.handle_data(virtualfn, cacheData)
  170. virtuals += 1
  171. return True, skipped, virtuals
  172. bb.msg.debug(1, bb.msg.domain.Cache, "Parsing %s" % fn)
  173. bb_data = self.load_bbfile(fn, appends, cfgData)
  174. for data in bb_data:
  175. virtualfn = self.realfn2virtual(fn, data)
  176. self.setData(virtualfn, fn, bb_data[data])
  177. if self.getVar("__SKIPPED", virtualfn):
  178. skipped += 1
  179. bb.msg.debug(1, bb.msg.domain.Cache, "Skipping %s" % virtualfn)
  180. else:
  181. self.handle_data(virtualfn, cacheData)
  182. virtuals += 1
  183. return False, skipped, virtuals
  184. def cacheValid(self, fn):
  185. """
  186. Is the cache valid for fn?
  187. Fast version, no timestamps checked.
  188. """
  189. # Is cache enabled?
  190. if not self.has_cache:
  191. return False
  192. if fn in self.clean:
  193. return True
  194. return False
  195. def cacheValidUpdate(self, fn):
  196. """
  197. Is the cache valid for fn?
  198. Make thorough (slower) checks including timestamps.
  199. """
  200. # Is cache enabled?
  201. if not self.has_cache:
  202. return False
  203. self.checked[fn] = ""
  204. # Pretend we're clean so getVar works
  205. self.clean[fn] = ""
  206. # File isn't in depends_cache
  207. if not fn in self.depends_cache:
  208. bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s is not cached" % fn)
  209. self.remove(fn)
  210. return False
  211. mtime = bb.parse.cached_mtime_noerror(fn)
  212. # Check file still exists
  213. if mtime == 0:
  214. bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s no longer exists" % fn)
  215. self.remove(fn)
  216. return False
  217. # Check the file's timestamp
  218. if mtime != self.getVar("CACHETIMESTAMP", fn, True):
  219. bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s changed" % fn)
  220. self.remove(fn)
  221. return False
  222. # Check dependencies are still valid
  223. depends = self.getVar("__depends", fn, True)
  224. if depends:
  225. for f, old_mtime in depends:
  226. fmtime = bb.parse.cached_mtime_noerror(f)
  227. # Check if file still exists
  228. if old_mtime != 0 and fmtime == 0:
  229. self.remove(fn)
  230. return False
  231. if (fmtime != old_mtime):
  232. bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s's dependency %s changed" % (fn, f))
  233. self.remove(fn)
  234. return False
  235. #bb.msg.debug(2, bb.msg.domain.Cache, "Depends Cache: %s is clean" % fn)
  236. if not fn in self.clean:
  237. self.clean[fn] = ""
  238. invalid = False
  239. # Mark extended class data as clean too
  240. multi = self.getVar('__VARIANTS', fn, True)
  241. for cls in (multi or "").split():
  242. virtualfn = self.realfn2virtual(fn, cls)
  243. self.clean[virtualfn] = ""
  244. if not virtualfn in self.depends_cache:
  245. bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s is not cached" % virtualfn)
  246. invalid = True
  247. # If any one of the varients is not present, mark cache as invalid for all
  248. if invalid:
  249. for cls in (multi or "").split():
  250. virtualfn = self.realfn2virtual(fn, cls)
  251. bb.msg.debug(2, bb.msg.domain.Cache, "Cache: Removing %s from cache" % virtualfn)
  252. del self.clean[virtualfn]
  253. bb.msg.debug(2, bb.msg.domain.Cache, "Cache: Removing %s from cache" % fn)
  254. del self.clean[fn]
  255. return False
  256. return True
  257. def remove(self, fn):
  258. """
  259. Remove a fn from the cache
  260. Called from the parser in error cases
  261. """
  262. bb.msg.debug(1, bb.msg.domain.Cache, "Removing %s from cache" % fn)
  263. if fn in self.depends_cache:
  264. del self.depends_cache[fn]
  265. if fn in self.clean:
  266. del self.clean[fn]
  267. def sync(self):
  268. """
  269. Save the cache
  270. Called from the parser when complete (or exiting)
  271. """
  272. import copy
  273. if not self.has_cache:
  274. return
  275. if self.cacheclean:
  276. bb.msg.note(1, bb.msg.domain.Cache, "Cache is clean, not saving.")
  277. return
  278. version_data = {}
  279. version_data['CACHE_VER'] = __cache_version__
  280. version_data['BITBAKE_VER'] = bb.__version__
  281. cache_data = copy.copy(self.depends_cache)
  282. for fn in self.depends_cache:
  283. if '__BB_DONT_CACHE' in self.depends_cache[fn] and self.depends_cache[fn]['__BB_DONT_CACHE']:
  284. bb.msg.debug(2, bb.msg.domain.Cache, "Not caching %s, marked as not cacheable" % fn)
  285. del cache_data[fn]
  286. elif 'PV' in self.depends_cache[fn] and 'SRCREVINACTION' in self.depends_cache[fn]['PV']:
  287. bb.msg.error(bb.msg.domain.Cache, "Not caching %s as it had SRCREVINACTION in PV. Please report this bug" % fn)
  288. del cache_data[fn]
  289. p = pickle.Pickler(file(self.cachefile, "wb" ), -1 )
  290. p.dump([cache_data, version_data])
  291. def mtime(self, cachefile):
  292. return bb.parse.cached_mtime_noerror(cachefile)
  293. def handle_data(self, file_name, cacheData):
  294. """
  295. Save data we need into the cache
  296. """
  297. pn = self.getVar('PN', file_name, True)
  298. pe = self.getVar('PE', file_name, True) or "0"
  299. pv = self.getVar('PV', file_name, True)
  300. if 'SRCREVINACTION' in pv:
  301. bb.msg.note(1, bb.msg.domain.Cache, "Found SRCREVINACTION in PV (%s) or %s. Please report this bug." % (pv, file_name))
  302. pr = self.getVar('PR', file_name, True)
  303. dp = int(self.getVar('DEFAULT_PREFERENCE', file_name, True) or "0")
  304. depends = bb.utils.explode_deps(self.getVar("DEPENDS", file_name, True) or "")
  305. packages = (self.getVar('PACKAGES', file_name, True) or "").split()
  306. packages_dynamic = (self.getVar('PACKAGES_DYNAMIC', file_name, True) or "").split()
  307. rprovides = (self.getVar("RPROVIDES", file_name, True) or "").split()
  308. cacheData.task_deps[file_name] = self.getVar("_task_deps", file_name)
  309. # build PackageName to FileName lookup table
  310. if pn not in cacheData.pkg_pn:
  311. cacheData.pkg_pn[pn] = []
  312. cacheData.pkg_pn[pn].append(file_name)
  313. cacheData.stamp[file_name] = self.getVar('STAMP', file_name, True)
  314. # build FileName to PackageName lookup table
  315. cacheData.pkg_fn[file_name] = pn
  316. cacheData.pkg_pepvpr[file_name] = (pe, pv, pr)
  317. cacheData.pkg_dp[file_name] = dp
  318. provides = [pn]
  319. for provide in (self.getVar("PROVIDES", file_name, True) or "").split():
  320. if provide not in provides:
  321. provides.append(provide)
  322. # Build forward and reverse provider hashes
  323. # Forward: virtual -> [filenames]
  324. # Reverse: PN -> [virtuals]
  325. if pn not in cacheData.pn_provides:
  326. cacheData.pn_provides[pn] = []
  327. cacheData.fn_provides[file_name] = provides
  328. for provide in provides:
  329. if provide not in cacheData.providers:
  330. cacheData.providers[provide] = []
  331. cacheData.providers[provide].append(file_name)
  332. if not provide in cacheData.pn_provides[pn]:
  333. cacheData.pn_provides[pn].append(provide)
  334. cacheData.deps[file_name] = []
  335. for dep in depends:
  336. if not dep in cacheData.deps[file_name]:
  337. cacheData.deps[file_name].append(dep)
  338. if not dep in cacheData.all_depends:
  339. cacheData.all_depends.append(dep)
  340. # Build reverse hash for PACKAGES, so runtime dependencies
  341. # can be be resolved (RDEPENDS, RRECOMMENDS etc.)
  342. for package in packages:
  343. if not package in cacheData.packages:
  344. cacheData.packages[package] = []
  345. cacheData.packages[package].append(file_name)
  346. rprovides += (self.getVar("RPROVIDES_%s" % package, file_name, 1) or "").split()
  347. for package in packages_dynamic:
  348. if not package in cacheData.packages_dynamic:
  349. cacheData.packages_dynamic[package] = []
  350. cacheData.packages_dynamic[package].append(file_name)
  351. for rprovide in rprovides:
  352. if not rprovide in cacheData.rproviders:
  353. cacheData.rproviders[rprovide] = []
  354. cacheData.rproviders[rprovide].append(file_name)
  355. # Build hash of runtime depends and rececommends
  356. if not file_name in cacheData.rundeps:
  357. cacheData.rundeps[file_name] = {}
  358. if not file_name in cacheData.runrecs:
  359. cacheData.runrecs[file_name] = {}
  360. rdepends = self.getVar('RDEPENDS', file_name, True) or ""
  361. rrecommends = self.getVar('RRECOMMENDS', file_name, True) or ""
  362. for package in packages + [pn]:
  363. if not package in cacheData.rundeps[file_name]:
  364. cacheData.rundeps[file_name][package] = []
  365. if not package in cacheData.runrecs[file_name]:
  366. cacheData.runrecs[file_name][package] = []
  367. cacheData.rundeps[file_name][package] = rdepends + " " + (self.getVar("RDEPENDS_%s" % package, file_name, True) or "")
  368. cacheData.runrecs[file_name][package] = rrecommends + " " + (self.getVar("RRECOMMENDS_%s" % package, file_name, True) or "")
  369. # Collect files we may need for possible world-dep
  370. # calculations
  371. if not self.getVar('BROKEN', file_name, True) and not self.getVar('EXCLUDE_FROM_WORLD', file_name, True):
  372. cacheData.possible_world.append(file_name)
  373. # Touch this to make sure its in the cache
  374. self.getVar('__BB_DONT_CACHE', file_name, True)
  375. self.getVar('__VARIANTS', file_name, True)
  376. def load_bbfile(self, bbfile, appends, config):
  377. """
  378. Load and parse one .bb build file
  379. Return the data and whether parsing resulted in the file being skipped
  380. """
  381. chdir_back = False
  382. from bb import data, parse
  383. # expand tmpdir to include this topdir
  384. data.setVar('TMPDIR', data.getVar('TMPDIR', config, 1) or "", config)
  385. bbfile_loc = os.path.abspath(os.path.dirname(bbfile))
  386. oldpath = os.path.abspath(os.getcwd())
  387. parse.cached_mtime_noerror(bbfile_loc)
  388. bb_data = data.init_db(config)
  389. # The ConfHandler first looks if there is a TOPDIR and if not
  390. # then it would call getcwd().
  391. # Previously, we chdir()ed to bbfile_loc, called the handler
  392. # and finally chdir()ed back, a couple of thousand times. We now
  393. # just fill in TOPDIR to point to bbfile_loc if there is no TOPDIR yet.
  394. if not data.getVar('TOPDIR', bb_data):
  395. chdir_back = True
  396. data.setVar('TOPDIR', bbfile_loc, bb_data)
  397. try:
  398. if appends:
  399. data.setVar('__BBAPPEND', " ".join(appends), bb_data)
  400. bb_data = parse.handle(bbfile, bb_data) # read .bb data
  401. if chdir_back: os.chdir(oldpath)
  402. return bb_data
  403. except:
  404. if chdir_back: os.chdir(oldpath)
  405. raise
  406. def init(cooker):
  407. """
  408. The Objective: Cache the minimum amount of data possible yet get to the
  409. stage of building packages (i.e. tryBuild) without reparsing any .bb files.
  410. To do this, we intercept getVar calls and only cache the variables we see
  411. being accessed. We rely on the cache getVar calls being made for all
  412. variables bitbake might need to use to reach this stage. For each cached
  413. file we need to track:
  414. * Its mtime
  415. * The mtimes of all its dependencies
  416. * Whether it caused a parse.SkipPackage exception
  417. Files causing parsing errors are evicted from the cache.
  418. """
  419. return Cache(cooker.configuration.data)
  420. #============================================================================#
  421. # CacheData
  422. #============================================================================#
  423. class CacheData:
  424. """
  425. The data structures we compile from the cached data
  426. """
  427. def __init__(self):
  428. """
  429. Direct cache variables
  430. (from Cache.handle_data)
  431. """
  432. self.providers = {}
  433. self.rproviders = {}
  434. self.packages = {}
  435. self.packages_dynamic = {}
  436. self.possible_world = []
  437. self.pkg_pn = {}
  438. self.pkg_fn = {}
  439. self.pkg_pepvpr = {}
  440. self.pkg_dp = {}
  441. self.pn_provides = {}
  442. self.fn_provides = {}
  443. self.all_depends = []
  444. self.deps = {}
  445. self.rundeps = {}
  446. self.runrecs = {}
  447. self.task_queues = {}
  448. self.task_deps = {}
  449. self.stamp = {}
  450. self.preferred = {}
  451. """
  452. Indirect Cache variables
  453. (set elsewhere)
  454. """
  455. self.ignored_dependencies = []
  456. self.world_target = set()
  457. self.bbfile_priority = {}
  458. self.bbfile_config_priorities = []