cache.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430
  1. #!/usr/bin/env python
  2. # ex:ts=4:sw=4:sts=4:et
  3. # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
  4. #
  5. # BitBake 'Event' implementation
  6. #
  7. # Caching of bitbake variables before task execution
  8. # Copyright (C) 2006 Richard Purdie
  9. # but small sections based on code from bin/bitbake:
  10. # Copyright (C) 2003, 2004 Chris Larson
  11. # Copyright (C) 2003, 2004 Phil Blundell
  12. # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
  13. # Copyright (C) 2005 Holger Hans Peter Freyther
  14. # Copyright (C) 2005 ROAD GmbH
  15. #
  16. # This program is free software; you can redistribute it and/or modify
  17. # it under the terms of the GNU General Public License version 2 as
  18. # published by the Free Software Foundation.
  19. #
  20. # This program is distributed in the hope that it will be useful,
  21. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  22. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  23. # GNU General Public License for more details.
  24. #
  25. # You should have received a copy of the GNU General Public License along
  26. # with this program; if not, write to the Free Software Foundation, Inc.,
  27. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  28. import os, re
  29. import bb.data
  30. import bb.utils
  31. from sets import Set
  32. try:
  33. import cPickle as pickle
  34. except ImportError:
  35. import pickle
  36. bb.msg.note(1, bb.msg.domain.Cache, "Importing cPickle failed. Falling back to a very slow implementation.")
  37. __cache_version__ = "125"
  38. class Cache:
  39. """
  40. BitBake Cache implementation
  41. """
  42. def __init__(self, cooker):
  43. self.cachedir = bb.data.getVar("CACHE", cooker.configuration.data, True)
  44. self.clean = {}
  45. self.depends_cache = {}
  46. self.data = None
  47. self.data_fn = None
  48. if self.cachedir in [None, '']:
  49. self.has_cache = False
  50. bb.msg.note(1, bb.msg.domain.Cache, "Not using a cache. Set CACHE = <directory> to enable.")
  51. else:
  52. self.has_cache = True
  53. self.cachefile = os.path.join(self.cachedir,"bb_cache.dat")
  54. bb.msg.debug(1, bb.msg.domain.Cache, "Using cache in '%s'" % self.cachedir)
  55. try:
  56. os.stat( self.cachedir )
  57. except OSError:
  58. bb.mkdirhier( self.cachedir )
  59. if self.has_cache and (self.mtime(self.cachefile)):
  60. try:
  61. p = pickle.Unpickler( file(self.cachefile,"rb"))
  62. self.depends_cache, version_data = p.load()
  63. if version_data['CACHE_VER'] != __cache_version__:
  64. raise ValueError, 'Cache Version Mismatch'
  65. if version_data['BITBAKE_VER'] != bb.__version__:
  66. raise ValueError, 'Bitbake Version Mismatch'
  67. except (ValueError, KeyError):
  68. bb.msg.note(1, bb.msg.domain.Cache, "Invalid cache found, rebuilding...")
  69. self.depends_cache = {}
  70. if self.depends_cache:
  71. for fn in self.depends_cache.keys():
  72. self.clean[fn] = ""
  73. self.cacheValidUpdate(fn)
  74. def getVar(self, var, fn, exp = 0):
  75. """
  76. Gets the value of a variable
  77. (similar to getVar in the data class)
  78. There are two scenarios:
  79. 1. We have cached data - serve from depends_cache[fn]
  80. 2. We're learning what data to cache - serve from data
  81. backend but add a copy of the data to the cache.
  82. """
  83. if fn in self.clean:
  84. return self.depends_cache[fn][var]
  85. if not fn in self.depends_cache:
  86. self.depends_cache[fn] = {}
  87. if fn != self.data_fn:
  88. # We're trying to access data in the cache which doesn't exist
  89. # yet setData hasn't been called to setup the right access. Very bad.
  90. bb.msg.error(bb.msg.domain.Cache, "Parsing error data_fn %s and fn %s don't match" % (self.data_fn, fn))
  91. result = bb.data.getVar(var, self.data, exp)
  92. self.depends_cache[fn][var] = result
  93. return result
  94. def setData(self, fn, data):
  95. """
  96. Called to prime bb_cache ready to learn which variables to cache.
  97. Will be followed by calls to self.getVar which aren't cached
  98. but can be fulfilled from self.data.
  99. """
  100. self.data_fn = fn
  101. self.data = data
  102. # Make sure __depends makes the depends_cache
  103. self.getVar("__depends", fn, True)
  104. self.depends_cache[fn]["CACHETIMESTAMP"] = bb.parse.cached_mtime(fn)
  105. def loadDataFull(self, fn, cfgData):
  106. """
  107. Return a complete set of data for fn.
  108. To do this, we need to parse the file.
  109. """
  110. bb_data, skipped = self.load_bbfile(fn, cfgData)
  111. return bb_data
  112. def loadData(self, fn, cfgData):
  113. """
  114. Load a subset of data for fn.
  115. If the cached data is valid we do nothing,
  116. To do this, we need to parse the file and set the system
  117. to record the variables accessed.
  118. Return the cache status and whether the file was skipped when parsed
  119. """
  120. if self.cacheValid(fn):
  121. if "SKIPPED" in self.depends_cache[fn]:
  122. return True, True
  123. return True, False
  124. bb_data, skipped = self.load_bbfile(fn, cfgData)
  125. self.setData(fn, bb_data)
  126. return False, skipped
  127. def cacheValid(self, fn):
  128. """
  129. Is the cache valid for fn?
  130. Fast version, no timestamps checked.
  131. """
  132. # Is cache enabled?
  133. if not self.has_cache:
  134. return False
  135. if fn in self.clean:
  136. return True
  137. return False
  138. def cacheValidUpdate(self, fn):
  139. """
  140. Is the cache valid for fn?
  141. Make thorough (slower) checks including timestamps.
  142. """
  143. # Is cache enabled?
  144. if not self.has_cache:
  145. return False
  146. # Check file still exists
  147. if self.mtime(fn) == 0:
  148. bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s not longer exists" % fn)
  149. self.remove(fn)
  150. return False
  151. # File isn't in depends_cache
  152. if not fn in self.depends_cache:
  153. bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s is not cached" % fn)
  154. self.remove(fn)
  155. return False
  156. # Check the file's timestamp
  157. if bb.parse.cached_mtime(fn) > self.getVar("CACHETIMESTAMP", fn, True):
  158. bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s changed" % fn)
  159. self.remove(fn)
  160. return False
  161. # Check dependencies are still valid
  162. depends = self.getVar("__depends", fn, True)
  163. for f,old_mtime in depends:
  164. # Check if file still exists
  165. if self.mtime(f) == 0:
  166. return False
  167. new_mtime = bb.parse.cached_mtime(f)
  168. if (new_mtime > old_mtime):
  169. bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s's dependency %s changed" % (fn, f))
  170. self.remove(fn)
  171. return False
  172. bb.msg.debug(2, bb.msg.domain.Cache, "Depends Cache: %s is clean" % fn)
  173. if not fn in self.clean:
  174. self.clean[fn] = ""
  175. return True
  176. def skip(self, fn):
  177. """
  178. Mark a fn as skipped
  179. Called from the parser
  180. """
  181. if not fn in self.depends_cache:
  182. self.depends_cache[fn] = {}
  183. self.depends_cache[fn]["SKIPPED"] = "1"
  184. def remove(self, fn):
  185. """
  186. Remove a fn from the cache
  187. Called from the parser in error cases
  188. """
  189. bb.msg.debug(1, bb.msg.domain.Cache, "Removing %s from cache" % fn)
  190. if fn in self.depends_cache:
  191. del self.depends_cache[fn]
  192. if fn in self.clean:
  193. del self.clean[fn]
  194. def sync(self):
  195. """
  196. Save the cache
  197. Called from the parser when complete (or exiting)
  198. """
  199. if not self.has_cache:
  200. return
  201. version_data = {}
  202. version_data['CACHE_VER'] = __cache_version__
  203. version_data['BITBAKE_VER'] = bb.__version__
  204. p = pickle.Pickler(file(self.cachefile, "wb" ), -1 )
  205. p.dump([self.depends_cache, version_data])
  206. def mtime(self, cachefile):
  207. return bb.parse.cached_mtime_noerror(cachefile)
  208. def handle_data(self, file_name, cacheData):
  209. """
  210. Save data we need into the cache
  211. """
  212. pn = self.getVar('PN', file_name, True)
  213. pv = self.getVar('PV', file_name, True)
  214. pr = self.getVar('PR', file_name, True)
  215. dp = int(self.getVar('DEFAULT_PREFERENCE', file_name, True) or "0")
  216. provides = Set([pn] + (self.getVar("PROVIDES", file_name, True) or "").split())
  217. depends = bb.utils.explode_deps(self.getVar("DEPENDS", file_name, True) or "")
  218. packages = (self.getVar('PACKAGES', file_name, True) or "").split()
  219. packages_dynamic = (self.getVar('PACKAGES_DYNAMIC', file_name, True) or "").split()
  220. rprovides = (self.getVar("RPROVIDES", file_name, True) or "").split()
  221. cacheData.task_queues[file_name] = self.getVar("_task_graph", file_name, True)
  222. cacheData.task_deps[file_name] = self.getVar("_task_deps", file_name, True)
  223. # build PackageName to FileName lookup table
  224. if pn not in cacheData.pkg_pn:
  225. cacheData.pkg_pn[pn] = []
  226. cacheData.pkg_pn[pn].append(file_name)
  227. cacheData.stamp[file_name] = self.getVar('STAMP', file_name, True)
  228. # build FileName to PackageName lookup table
  229. cacheData.pkg_fn[file_name] = pn
  230. cacheData.pkg_pvpr[file_name] = (pv,pr)
  231. cacheData.pkg_dp[file_name] = dp
  232. # Build forward and reverse provider hashes
  233. # Forward: virtual -> [filenames]
  234. # Reverse: PN -> [virtuals]
  235. if pn not in cacheData.pn_provides:
  236. cacheData.pn_provides[pn] = Set()
  237. cacheData.pn_provides[pn] |= provides
  238. for provide in provides:
  239. if provide not in cacheData.providers:
  240. cacheData.providers[provide] = []
  241. cacheData.providers[provide].append(file_name)
  242. cacheData.deps[file_name] = Set()
  243. for dep in depends:
  244. cacheData.all_depends.add(dep)
  245. cacheData.deps[file_name].add(dep)
  246. # Build reverse hash for PACKAGES, so runtime dependencies
  247. # can be be resolved (RDEPENDS, RRECOMMENDS etc.)
  248. for package in packages:
  249. if not package in cacheData.packages:
  250. cacheData.packages[package] = []
  251. cacheData.packages[package].append(file_name)
  252. rprovides += (self.getVar("RPROVIDES_%s" % package, file_name, 1) or "").split()
  253. for package in packages_dynamic:
  254. if not package in cacheData.packages_dynamic:
  255. cacheData.packages_dynamic[package] = []
  256. cacheData.packages_dynamic[package].append(file_name)
  257. for rprovide in rprovides:
  258. if not rprovide in cacheData.rproviders:
  259. cacheData.rproviders[rprovide] = []
  260. cacheData.rproviders[rprovide].append(file_name)
  261. # Build hash of runtime depends and rececommends
  262. def add_dep(deplist, deps):
  263. for dep in deps:
  264. if not dep in deplist:
  265. deplist[dep] = ""
  266. if not file_name in cacheData.rundeps:
  267. cacheData.rundeps[file_name] = {}
  268. if not file_name in cacheData.runrecs:
  269. cacheData.runrecs[file_name] = {}
  270. for package in packages + [pn]:
  271. if not package in cacheData.rundeps[file_name]:
  272. cacheData.rundeps[file_name][package] = {}
  273. if not package in cacheData.runrecs[file_name]:
  274. cacheData.runrecs[file_name][package] = {}
  275. add_dep(cacheData.rundeps[file_name][package], bb.utils.explode_deps(self.getVar('RDEPENDS', file_name, True) or ""))
  276. add_dep(cacheData.runrecs[file_name][package], bb.utils.explode_deps(self.getVar('RRECOMMENDS', file_name, True) or ""))
  277. add_dep(cacheData.rundeps[file_name][package], bb.utils.explode_deps(self.getVar("RDEPENDS_%s" % package, file_name, True) or ""))
  278. add_dep(cacheData.runrecs[file_name][package], bb.utils.explode_deps(self.getVar("RRECOMMENDS_%s" % package, file_name, True) or ""))
  279. # Collect files we may need for possible world-dep
  280. # calculations
  281. if not self.getVar('BROKEN', file_name, True) and not self.getVar('EXCLUDE_FROM_WORLD', file_name, True):
  282. cacheData.possible_world.append(file_name)
  283. def load_bbfile( self, bbfile , config):
  284. """
  285. Load and parse one .bb build file
  286. Return the data and whether parsing resulted in the file being skipped
  287. """
  288. import bb
  289. from bb import utils, data, parse, debug, event, fatal
  290. # expand tmpdir to include this topdir
  291. data.setVar('TMPDIR', data.getVar('TMPDIR', config, 1) or "", config)
  292. bbfile_loc = os.path.abspath(os.path.dirname(bbfile))
  293. oldpath = os.path.abspath(os.getcwd())
  294. if self.mtime(bbfile_loc):
  295. os.chdir(bbfile_loc)
  296. bb_data = data.init_db(config)
  297. try:
  298. bb_data = parse.handle(bbfile, bb_data) # read .bb data
  299. os.chdir(oldpath)
  300. return bb_data, False
  301. except bb.parse.SkipPackage:
  302. os.chdir(oldpath)
  303. return bb_data, True
  304. except:
  305. os.chdir(oldpath)
  306. raise
  307. def init(cooker):
  308. """
  309. The Objective: Cache the minimum amount of data possible yet get to the
  310. stage of building packages (i.e. tryBuild) without reparsing any .bb files.
  311. To do this, we intercept getVar calls and only cache the variables we see
  312. being accessed. We rely on the cache getVar calls being made for all
  313. variables bitbake might need to use to reach this stage. For each cached
  314. file we need to track:
  315. * Its mtime
  316. * The mtimes of all its dependencies
  317. * Whether it caused a parse.SkipPackage exception
  318. Files causing parsing errors are evicted from the cache.
  319. """
  320. return Cache(cooker)
  321. #============================================================================#
  322. # CacheData
  323. #============================================================================#
  324. class CacheData:
  325. """
  326. The data structures we compile from the cached data
  327. """
  328. def __init__(self):
  329. """
  330. Direct cache variables
  331. (from Cache.handle_data)
  332. """
  333. self.providers = {}
  334. self.rproviders = {}
  335. self.packages = {}
  336. self.packages_dynamic = {}
  337. self.possible_world = []
  338. self.pkg_pn = {}
  339. self.pkg_fn = {}
  340. self.pkg_pvpr = {}
  341. self.pkg_dp = {}
  342. self.pn_provides = {}
  343. self.all_depends = Set()
  344. self.deps = {}
  345. self.rundeps = {}
  346. self.runrecs = {}
  347. self.task_queues = {}
  348. self.task_deps = {}
  349. self.stamp = {}
  350. self.preferred = {}
  351. """
  352. Indirect Cache variables
  353. (set elsewhere)
  354. """
  355. self.ignored_dependencies = []
  356. self.world_target = Set()
  357. self.bbfile_priority = {}
  358. self.bbfile_config_priorities = []