db.py 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280
  1. #
  2. # SPDX-License-Identifier: GPL-2.0-only
  3. #
  4. import logging
  5. import os.path
  6. import errno
  7. import prserv
  8. import time
  9. try:
  10. import sqlite3
  11. except ImportError:
  12. from pysqlite2 import dbapi2 as sqlite3
  13. logger = logging.getLogger("BitBake.PRserv")
  14. sqlversion = sqlite3.sqlite_version_info
  15. if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
  16. raise Exception("sqlite3 version 3.3.0 or later is required.")
  17. #
  18. # "No History" mode - for a given query tuple (version, pkgarch, checksum),
  19. # the returned value will be the largest among all the values of the same
  20. # (version, pkgarch). This means the PR value returned can NOT be decremented.
  21. #
  22. # "History" mode - Return a new higher value for previously unseen query
  23. # tuple (version, pkgarch, checksum), otherwise return historical value.
  24. # Value can decrement if returning to a previous build.
  25. #
  26. class PRTable(object):
  27. def __init__(self, conn, table, nohist):
  28. self.conn = conn
  29. self.nohist = nohist
  30. self.dirty = False
  31. if nohist:
  32. self.table = "%s_nohist" % table
  33. else:
  34. self.table = "%s_hist" % table
  35. self._execute("CREATE TABLE IF NOT EXISTS %s \
  36. (version TEXT NOT NULL, \
  37. pkgarch TEXT NOT NULL, \
  38. checksum TEXT NOT NULL, \
  39. value INTEGER, \
  40. PRIMARY KEY (version, pkgarch, checksum));" % self.table)
  41. def _execute(self, *query):
  42. """Execute a query, waiting to acquire a lock if necessary"""
  43. start = time.time()
  44. end = start + 20
  45. while True:
  46. try:
  47. return self.conn.execute(*query)
  48. except sqlite3.OperationalError as exc:
  49. if 'is locked' in str(exc) and end > time.time():
  50. continue
  51. raise exc
  52. def sync(self):
  53. self.conn.commit()
  54. self._execute("BEGIN EXCLUSIVE TRANSACTION")
  55. def sync_if_dirty(self):
  56. if self.dirty:
  57. self.sync()
  58. self.dirty = False
  59. def _getValueHist(self, version, pkgarch, checksum):
  60. data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
  61. (version, pkgarch, checksum))
  62. row=data.fetchone()
  63. if row is not None:
  64. return row[0]
  65. else:
  66. #no value found, try to insert
  67. try:
  68. self._execute("INSERT INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1,0) from %s where version=? AND pkgarch=?));"
  69. % (self.table,self.table),
  70. (version,pkgarch, checksum,version, pkgarch))
  71. except sqlite3.IntegrityError as exc:
  72. logger.error(str(exc))
  73. self.dirty = True
  74. data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
  75. (version, pkgarch, checksum))
  76. row=data.fetchone()
  77. if row is not None:
  78. return row[0]
  79. else:
  80. raise prserv.NotFoundError
  81. def _getValueNohist(self, version, pkgarch, checksum):
  82. data=self._execute("SELECT value FROM %s \
  83. WHERE version=? AND pkgarch=? AND checksum=? AND \
  84. value >= (select max(value) from %s where version=? AND pkgarch=?);"
  85. % (self.table, self.table),
  86. (version, pkgarch, checksum, version, pkgarch))
  87. row=data.fetchone()
  88. if row is not None:
  89. return row[0]
  90. else:
  91. #no value found, try to insert
  92. try:
  93. self._execute("INSERT OR REPLACE INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1,0) from %s where version=? AND pkgarch=?));"
  94. % (self.table,self.table),
  95. (version, pkgarch, checksum, version, pkgarch))
  96. except sqlite3.IntegrityError as exc:
  97. logger.error(str(exc))
  98. self.conn.rollback()
  99. self.dirty = True
  100. data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
  101. (version, pkgarch, checksum))
  102. row=data.fetchone()
  103. if row is not None:
  104. return row[0]
  105. else:
  106. raise prserv.NotFoundError
  107. def getValue(self, version, pkgarch, checksum):
  108. if self.nohist:
  109. return self._getValueNohist(version, pkgarch, checksum)
  110. else:
  111. return self._getValueHist(version, pkgarch, checksum)
  112. def _importHist(self, version, pkgarch, checksum, value):
  113. val = None
  114. data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
  115. (version, pkgarch, checksum))
  116. row = data.fetchone()
  117. if row is not None:
  118. val=row[0]
  119. else:
  120. #no value found, try to insert
  121. try:
  122. self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table),
  123. (version, pkgarch, checksum, value))
  124. except sqlite3.IntegrityError as exc:
  125. logger.error(str(exc))
  126. self.dirty = True
  127. data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
  128. (version, pkgarch, checksum))
  129. row = data.fetchone()
  130. if row is not None:
  131. val = row[0]
  132. return val
  133. def _importNohist(self, version, pkgarch, checksum, value):
  134. try:
  135. #try to insert
  136. self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table),
  137. (version, pkgarch, checksum,value))
  138. except sqlite3.IntegrityError as exc:
  139. #already have the record, try to update
  140. try:
  141. self._execute("UPDATE %s SET value=? WHERE version=? AND pkgarch=? AND checksum=? AND value<?"
  142. % (self.table),
  143. (value,version,pkgarch,checksum,value))
  144. except sqlite3.IntegrityError as exc:
  145. logger.error(str(exc))
  146. self.dirty = True
  147. data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=? AND value>=?;" % self.table,
  148. (version,pkgarch,checksum,value))
  149. row=data.fetchone()
  150. if row is not None:
  151. return row[0]
  152. else:
  153. return None
  154. def importone(self, version, pkgarch, checksum, value):
  155. if self.nohist:
  156. return self._importNohist(version, pkgarch, checksum, value)
  157. else:
  158. return self._importHist(version, pkgarch, checksum, value)
  159. def export(self, version, pkgarch, checksum, colinfo):
  160. metainfo = {}
  161. #column info
  162. if colinfo:
  163. metainfo['tbl_name'] = self.table
  164. metainfo['core_ver'] = prserv.__version__
  165. metainfo['col_info'] = []
  166. data = self._execute("PRAGMA table_info(%s);" % self.table)
  167. for row in data:
  168. col = {}
  169. col['name'] = row['name']
  170. col['type'] = row['type']
  171. col['notnull'] = row['notnull']
  172. col['dflt_value'] = row['dflt_value']
  173. col['pk'] = row['pk']
  174. metainfo['col_info'].append(col)
  175. #data info
  176. datainfo = []
  177. if self.nohist:
  178. sqlstmt = "SELECT T1.version, T1.pkgarch, T1.checksum, T1.value FROM %s as T1, \
  179. (SELECT version,pkgarch,max(value) as maxvalue FROM %s GROUP BY version,pkgarch) as T2 \
  180. WHERE T1.version=T2.version AND T1.pkgarch=T2.pkgarch AND T1.value=T2.maxvalue " % (self.table, self.table)
  181. else:
  182. sqlstmt = "SELECT * FROM %s as T1 WHERE 1=1 " % self.table
  183. sqlarg = []
  184. where = ""
  185. if version:
  186. where += "AND T1.version=? "
  187. sqlarg.append(str(version))
  188. if pkgarch:
  189. where += "AND T1.pkgarch=? "
  190. sqlarg.append(str(pkgarch))
  191. if checksum:
  192. where += "AND T1.checksum=? "
  193. sqlarg.append(str(checksum))
  194. sqlstmt += where + ";"
  195. if len(sqlarg):
  196. data = self._execute(sqlstmt, tuple(sqlarg))
  197. else:
  198. data = self._execute(sqlstmt)
  199. for row in data:
  200. if row['version']:
  201. col = {}
  202. col['version'] = row['version']
  203. col['pkgarch'] = row['pkgarch']
  204. col['checksum'] = row['checksum']
  205. col['value'] = row['value']
  206. datainfo.append(col)
  207. return (metainfo, datainfo)
  208. def dump_db(self, fd):
  209. writeCount = 0
  210. for line in self.conn.iterdump():
  211. writeCount = writeCount + len(line) + 1
  212. fd.write(line)
  213. fd.write('\n')
  214. return writeCount
  215. class PRData(object):
  216. """Object representing the PR database"""
  217. def __init__(self, filename, nohist=True):
  218. self.filename=os.path.abspath(filename)
  219. self.nohist=nohist
  220. #build directory hierarchy
  221. try:
  222. os.makedirs(os.path.dirname(self.filename))
  223. except OSError as e:
  224. if e.errno != errno.EEXIST:
  225. raise e
  226. self.connection=sqlite3.connect(self.filename, isolation_level="EXCLUSIVE", check_same_thread = False)
  227. self.connection.row_factory=sqlite3.Row
  228. self.connection.execute("pragma synchronous = off;")
  229. self.connection.execute("PRAGMA journal_mode = MEMORY;")
  230. self._tables={}
  231. def disconnect(self):
  232. self.connection.close()
  233. def __getitem__(self,tblname):
  234. if not isinstance(tblname, str):
  235. raise TypeError("tblname argument must be a string, not '%s'" %
  236. type(tblname))
  237. if tblname in self._tables:
  238. return self._tables[tblname]
  239. else:
  240. tableobj = self._tables[tblname] = PRTable(self.connection, tblname, self.nohist)
  241. return tableobj
  242. def __delitem__(self, tblname):
  243. if tblname in self._tables:
  244. del self._tables[tblname]
  245. logger.info("drop table %s" % (tblname))
  246. self.connection.execute("DROP TABLE IF EXISTS %s;" % tblname)