persist_data.py 6.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218
  1. """BitBake Persistent Data Store
  2. Used to store data in a central location such that other threads/tasks can
  3. access them at some future date. Acts as a convenience wrapper around sqlite,
  4. currently, providing a key/value store accessed by 'domain'.
  5. """
  6. # Copyright (C) 2007 Richard Purdie
  7. # Copyright (C) 2010 Chris Larson <chris_larson@mentor.com>
  8. #
  9. # This program is free software; you can redistribute it and/or modify
  10. # it under the terms of the GNU General Public License version 2 as
  11. # published by the Free Software Foundation.
  12. #
  13. # This program is distributed in the hope that it will be useful,
  14. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16. # GNU General Public License for more details.
  17. #
  18. # You should have received a copy of the GNU General Public License along
  19. # with this program; if not, write to the Free Software Foundation, Inc.,
  20. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  21. import collections
  22. import logging
  23. import os.path
  24. import sys
  25. import warnings
  26. from bb.compat import total_ordering
  27. from collections import Mapping
  28. try:
  29. import sqlite3
  30. except ImportError:
  31. from pysqlite2 import dbapi2 as sqlite3
  32. sqlversion = sqlite3.sqlite_version_info
  33. if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
  34. raise Exception("sqlite3 version 3.3.0 or later is required.")
  35. logger = logging.getLogger("BitBake.PersistData")
  36. if hasattr(sqlite3, 'enable_shared_cache'):
  37. try:
  38. sqlite3.enable_shared_cache(True)
  39. except sqlite3.OperationalError:
  40. pass
  41. @total_ordering
  42. class SQLTable(collections.MutableMapping):
  43. """Object representing a table/domain in the database"""
  44. def __init__(self, cachefile, table):
  45. self.cachefile = cachefile
  46. self.table = table
  47. self.cursor = connect(self.cachefile)
  48. self._execute("CREATE TABLE IF NOT EXISTS %s(key TEXT, value TEXT);"
  49. % table)
  50. def _execute(self, *query):
  51. """Execute a query, waiting to acquire a lock if necessary"""
  52. count = 0
  53. while True:
  54. try:
  55. return self.cursor.execute(*query)
  56. except sqlite3.OperationalError as exc:
  57. if 'database is locked' in str(exc) and count < 500:
  58. count = count + 1
  59. self.cursor.close()
  60. self.cursor = connect(self.cachefile)
  61. continue
  62. raise
  63. def __enter__(self):
  64. self.cursor.__enter__()
  65. return self
  66. def __exit__(self, *excinfo):
  67. self.cursor.__exit__(*excinfo)
  68. def __getitem__(self, key):
  69. data = self._execute("SELECT * from %s where key=?;" %
  70. self.table, [key])
  71. for row in data:
  72. return row[1]
  73. raise KeyError(key)
  74. def __delitem__(self, key):
  75. if key not in self:
  76. raise KeyError(key)
  77. self._execute("DELETE from %s where key=?;" % self.table, [key])
  78. def __setitem__(self, key, value):
  79. if not isinstance(key, str):
  80. raise TypeError('Only string keys are supported')
  81. elif not isinstance(value, str):
  82. raise TypeError('Only string values are supported')
  83. data = self._execute("SELECT * from %s where key=?;" %
  84. self.table, [key])
  85. exists = len(list(data))
  86. if exists:
  87. self._execute("UPDATE %s SET value=? WHERE key=?;" % self.table,
  88. [value, key])
  89. else:
  90. self._execute("INSERT into %s(key, value) values (?, ?);" %
  91. self.table, [key, value])
  92. def __contains__(self, key):
  93. return key in set(self)
  94. def __len__(self):
  95. data = self._execute("SELECT COUNT(key) FROM %s;" % self.table)
  96. for row in data:
  97. return row[0]
  98. def __iter__(self):
  99. data = self._execute("SELECT key FROM %s;" % self.table)
  100. return (row[0] for row in data)
  101. def __lt__(self, other):
  102. if not isinstance(other, Mapping):
  103. raise NotImplemented
  104. return len(self) < len(other)
  105. def get_by_pattern(self, pattern):
  106. data = self._execute("SELECT * FROM %s WHERE key LIKE ?;" %
  107. self.table, [pattern])
  108. return [row[1] for row in data]
  109. def values(self):
  110. return list(self.values())
  111. def itervalues(self):
  112. data = self._execute("SELECT value FROM %s;" % self.table)
  113. return (row[0] for row in data)
  114. def items(self):
  115. return list(self.items())
  116. def iteritems(self):
  117. return self._execute("SELECT * FROM %s;" % self.table)
  118. def clear(self):
  119. self._execute("DELETE FROM %s;" % self.table)
  120. def has_key(self, key):
  121. return key in self
  122. class PersistData(object):
  123. """Deprecated representation of the bitbake persistent data store"""
  124. def __init__(self, d):
  125. warnings.warn("Use of PersistData is deprecated. Please use "
  126. "persist(domain, d) instead.",
  127. category=DeprecationWarning,
  128. stacklevel=2)
  129. self.data = persist(d)
  130. logger.debug(1, "Using '%s' as the persistent data cache",
  131. self.data.filename)
  132. def addDomain(self, domain):
  133. """
  134. Add a domain (pending deprecation)
  135. """
  136. return self.data[domain]
  137. def delDomain(self, domain):
  138. """
  139. Removes a domain and all the data it contains
  140. """
  141. del self.data[domain]
  142. def getKeyValues(self, domain):
  143. """
  144. Return a list of key + value pairs for a domain
  145. """
  146. return list(self.data[domain].items())
  147. def getValue(self, domain, key):
  148. """
  149. Return the value of a key for a domain
  150. """
  151. return self.data[domain][key]
  152. def setValue(self, domain, key, value):
  153. """
  154. Sets the value of a key for a domain
  155. """
  156. self.data[domain][key] = value
  157. def delValue(self, domain, key):
  158. """
  159. Deletes a key/value pair
  160. """
  161. del self.data[domain][key]
  162. def connect(database):
  163. connection = sqlite3.connect(database, timeout=5, isolation_level=None)
  164. connection.execute("pragma synchronous = off;")
  165. connection.text_factory = str
  166. return connection
  167. def persist(domain, d):
  168. """Convenience factory for SQLTable objects based upon metadata"""
  169. import bb.utils
  170. cachedir = (d.getVar("PERSISTENT_DIR", True) or
  171. d.getVar("CACHE", True))
  172. if not cachedir:
  173. logger.critical("Please set the 'PERSISTENT_DIR' or 'CACHE' variable")
  174. sys.exit(1)
  175. bb.utils.mkdirhier(cachedir)
  176. cachefile = os.path.join(cachedir, "bb_persist_data.sqlite3")
  177. return SQLTable(cachefile, domain)