123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280 |
- #
- # SPDX-License-Identifier: GPL-2.0-only
- #
- import logging
- import os.path
- import errno
- import prserv
- import time
- try:
- import sqlite3
- except ImportError:
- from pysqlite2 import dbapi2 as sqlite3
- logger = logging.getLogger("BitBake.PRserv")
- sqlversion = sqlite3.sqlite_version_info
- if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
- raise Exception("sqlite3 version 3.3.0 or later is required.")
- #
- # "No History" mode - for a given query tuple (version, pkgarch, checksum),
- # the returned value will be the largest among all the values of the same
- # (version, pkgarch). This means the PR value returned can NOT be decremented.
- #
- # "History" mode - Return a new higher value for previously unseen query
- # tuple (version, pkgarch, checksum), otherwise return historical value.
- # Value can decrement if returning to a previous build.
- #
- class PRTable(object):
- def __init__(self, conn, table, nohist):
- self.conn = conn
- self.nohist = nohist
- self.dirty = False
- if nohist:
- self.table = "%s_nohist" % table
- else:
- self.table = "%s_hist" % table
- self._execute("CREATE TABLE IF NOT EXISTS %s \
- (version TEXT NOT NULL, \
- pkgarch TEXT NOT NULL, \
- checksum TEXT NOT NULL, \
- value INTEGER, \
- PRIMARY KEY (version, pkgarch, checksum));" % self.table)
- def _execute(self, *query):
- """Execute a query, waiting to acquire a lock if necessary"""
- start = time.time()
- end = start + 20
- while True:
- try:
- return self.conn.execute(*query)
- except sqlite3.OperationalError as exc:
- if 'is locked' in str(exc) and end > time.time():
- continue
- raise exc
- def sync(self):
- self.conn.commit()
- self._execute("BEGIN EXCLUSIVE TRANSACTION")
- def sync_if_dirty(self):
- if self.dirty:
- self.sync()
- self.dirty = False
- def _getValueHist(self, version, pkgarch, checksum):
- data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
- (version, pkgarch, checksum))
- row=data.fetchone()
- if row is not None:
- return row[0]
- else:
- #no value found, try to insert
- try:
- self._execute("INSERT INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1,0) from %s where version=? AND pkgarch=?));"
- % (self.table,self.table),
- (version,pkgarch, checksum,version, pkgarch))
- except sqlite3.IntegrityError as exc:
- logger.error(str(exc))
- self.dirty = True
- data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
- (version, pkgarch, checksum))
- row=data.fetchone()
- if row is not None:
- return row[0]
- else:
- raise prserv.NotFoundError
- def _getValueNohist(self, version, pkgarch, checksum):
- data=self._execute("SELECT value FROM %s \
- WHERE version=? AND pkgarch=? AND checksum=? AND \
- value >= (select max(value) from %s where version=? AND pkgarch=?);"
- % (self.table, self.table),
- (version, pkgarch, checksum, version, pkgarch))
- row=data.fetchone()
- if row is not None:
- return row[0]
- else:
- #no value found, try to insert
- try:
- self._execute("INSERT OR REPLACE INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1,0) from %s where version=? AND pkgarch=?));"
- % (self.table,self.table),
- (version, pkgarch, checksum, version, pkgarch))
- except sqlite3.IntegrityError as exc:
- logger.error(str(exc))
- self.conn.rollback()
- self.dirty = True
- data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
- (version, pkgarch, checksum))
- row=data.fetchone()
- if row is not None:
- return row[0]
- else:
- raise prserv.NotFoundError
- def getValue(self, version, pkgarch, checksum):
- if self.nohist:
- return self._getValueNohist(version, pkgarch, checksum)
- else:
- return self._getValueHist(version, pkgarch, checksum)
- def _importHist(self, version, pkgarch, checksum, value):
- val = None
- data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
- (version, pkgarch, checksum))
- row = data.fetchone()
- if row is not None:
- val=row[0]
- else:
- #no value found, try to insert
- try:
- self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table),
- (version, pkgarch, checksum, value))
- except sqlite3.IntegrityError as exc:
- logger.error(str(exc))
- self.dirty = True
- data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
- (version, pkgarch, checksum))
- row = data.fetchone()
- if row is not None:
- val = row[0]
- return val
- def _importNohist(self, version, pkgarch, checksum, value):
- try:
- #try to insert
- self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table),
- (version, pkgarch, checksum,value))
- except sqlite3.IntegrityError as exc:
- #already have the record, try to update
- try:
- self._execute("UPDATE %s SET value=? WHERE version=? AND pkgarch=? AND checksum=? AND value<?"
- % (self.table),
- (value,version,pkgarch,checksum,value))
- except sqlite3.IntegrityError as exc:
- logger.error(str(exc))
- self.dirty = True
- data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=? AND value>=?;" % self.table,
- (version,pkgarch,checksum,value))
- row=data.fetchone()
- if row is not None:
- return row[0]
- else:
- return None
- def importone(self, version, pkgarch, checksum, value):
- if self.nohist:
- return self._importNohist(version, pkgarch, checksum, value)
- else:
- return self._importHist(version, pkgarch, checksum, value)
- def export(self, version, pkgarch, checksum, colinfo):
- metainfo = {}
- #column info
- if colinfo:
- metainfo['tbl_name'] = self.table
- metainfo['core_ver'] = prserv.__version__
- metainfo['col_info'] = []
- data = self._execute("PRAGMA table_info(%s);" % self.table)
- for row in data:
- col = {}
- col['name'] = row['name']
- col['type'] = row['type']
- col['notnull'] = row['notnull']
- col['dflt_value'] = row['dflt_value']
- col['pk'] = row['pk']
- metainfo['col_info'].append(col)
- #data info
- datainfo = []
- if self.nohist:
- sqlstmt = "SELECT T1.version, T1.pkgarch, T1.checksum, T1.value FROM %s as T1, \
- (SELECT version,pkgarch,max(value) as maxvalue FROM %s GROUP BY version,pkgarch) as T2 \
- WHERE T1.version=T2.version AND T1.pkgarch=T2.pkgarch AND T1.value=T2.maxvalue " % (self.table, self.table)
- else:
- sqlstmt = "SELECT * FROM %s as T1 WHERE 1=1 " % self.table
- sqlarg = []
- where = ""
- if version:
- where += "AND T1.version=? "
- sqlarg.append(str(version))
- if pkgarch:
- where += "AND T1.pkgarch=? "
- sqlarg.append(str(pkgarch))
- if checksum:
- where += "AND T1.checksum=? "
- sqlarg.append(str(checksum))
- sqlstmt += where + ";"
- if len(sqlarg):
- data = self._execute(sqlstmt, tuple(sqlarg))
- else:
- data = self._execute(sqlstmt)
- for row in data:
- if row['version']:
- col = {}
- col['version'] = row['version']
- col['pkgarch'] = row['pkgarch']
- col['checksum'] = row['checksum']
- col['value'] = row['value']
- datainfo.append(col)
- return (metainfo, datainfo)
- def dump_db(self, fd):
- writeCount = 0
- for line in self.conn.iterdump():
- writeCount = writeCount + len(line) + 1
- fd.write(line)
- fd.write('\n')
- return writeCount
- class PRData(object):
- """Object representing the PR database"""
- def __init__(self, filename, nohist=True):
- self.filename=os.path.abspath(filename)
- self.nohist=nohist
- #build directory hierarchy
- try:
- os.makedirs(os.path.dirname(self.filename))
- except OSError as e:
- if e.errno != errno.EEXIST:
- raise e
- self.connection=sqlite3.connect(self.filename, isolation_level="EXCLUSIVE", check_same_thread = False)
- self.connection.row_factory=sqlite3.Row
- self.connection.execute("pragma synchronous = off;")
- self.connection.execute("PRAGMA journal_mode = MEMORY;")
- self._tables={}
- def disconnect(self):
- self.connection.close()
- def __getitem__(self,tblname):
- if not isinstance(tblname, str):
- raise TypeError("tblname argument must be a string, not '%s'" %
- type(tblname))
- if tblname in self._tables:
- return self._tables[tblname]
- else:
- tableobj = self._tables[tblname] = PRTable(self.connection, tblname, self.nohist)
- return tableobj
- def __delitem__(self, tblname):
- if tblname in self._tables:
- del self._tables[tblname]
- logger.info("drop table %s" % (tblname))
- self.connection.execute("DROP TABLE IF EXISTS %s;" % tblname)
|