|
@@ -1,6 +1,6 @@
|
|
|
# tinfoil: a simple wrapper around cooker for bitbake-based command-line utilities
|
|
|
#
|
|
|
-# Copyright (C) 2012 Intel Corporation
|
|
|
+# Copyright (C) 2012-2016 Intel Corporation
|
|
|
# Copyright (C) 2011 Mentor Graphics Corporation
|
|
|
#
|
|
|
# This program is free software; you can redistribute it and/or modify
|
|
@@ -17,47 +17,172 @@
|
|
|
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
|
|
|
|
import logging
|
|
|
-import warnings
|
|
|
import os
|
|
|
import sys
|
|
|
+import atexit
|
|
|
+import re
|
|
|
+from collections import OrderedDict, defaultdict
|
|
|
|
|
|
import bb.cache
|
|
|
import bb.cooker
|
|
|
import bb.providers
|
|
|
import bb.utils
|
|
|
-from bb.cooker import state, BBCooker, CookerFeatures
|
|
|
+import bb.command
|
|
|
from bb.cookerdata import CookerConfiguration, ConfigParameters
|
|
|
+from bb.main import setup_bitbake, BitBakeConfigParameters, BBMainException
|
|
|
import bb.fetch2
|
|
|
|
|
|
+
|
|
|
+# We need this in order to shut down the connection to the bitbake server,
|
|
|
+# otherwise the process will never properly exit
|
|
|
+_server_connections = []
|
|
|
+def _terminate_connections():
|
|
|
+ for connection in _server_connections:
|
|
|
+ connection.terminate()
|
|
|
+atexit.register(_terminate_connections)
|
|
|
+
|
|
|
+class TinfoilUIException(Exception):
|
|
|
+ """Exception raised when the UI returns non-zero from its main function"""
|
|
|
+ def __init__(self, returncode):
|
|
|
+ self.returncode = returncode
|
|
|
+ def __repr__(self):
|
|
|
+ return 'UI module main returned %d' % self.returncode
|
|
|
+
|
|
|
+class TinfoilCommandFailed(Exception):
|
|
|
+ """Exception raised when run_command fails"""
|
|
|
+
|
|
|
+class TinfoilDataStoreConnector:
|
|
|
+
|
|
|
+ def __init__(self, tinfoil, dsindex):
|
|
|
+ self.tinfoil = tinfoil
|
|
|
+ self.dsindex = dsindex
|
|
|
+ def getVar(self, name):
|
|
|
+ value = self.tinfoil.run_command('dataStoreConnectorFindVar', self.dsindex, name)
|
|
|
+ if isinstance(value, dict):
|
|
|
+ if '_connector_origtype' in value:
|
|
|
+ value['_content'] = self.tinfoil._reconvert_type(value['_content'], value['_connector_origtype'])
|
|
|
+ del value['_connector_origtype']
|
|
|
+
|
|
|
+ return value
|
|
|
+ def getKeys(self):
|
|
|
+ return set(self.tinfoil.run_command('dataStoreConnectorGetKeys', self.dsindex))
|
|
|
+ def getVarHistory(self, name):
|
|
|
+ return self.tinfoil.run_command('dataStoreConnectorGetVarHistory', self.dsindex, name)
|
|
|
+ def expandPythonRef(self, varname, expr):
|
|
|
+ ret = self.tinfoil.run_command('dataStoreConnectorExpandPythonRef', self.dsindex, varname, expr)
|
|
|
+ return ret
|
|
|
+ def setVar(self, varname, value):
|
|
|
+ if self.dsindex is None:
|
|
|
+ self.tinfoil.run_command('setVariable', varname, value)
|
|
|
+ else:
|
|
|
+ # Not currently implemented - indicate that setting should
|
|
|
+ # be redirected to local side
|
|
|
+ return True
|
|
|
+
|
|
|
+class TinfoilCookerAdapter:
|
|
|
+ """
|
|
|
+ Provide an adapter for existing code that expects to access a cooker object via Tinfoil,
|
|
|
+ since now Tinfoil is on the client side it no longer has direct access.
|
|
|
+ """
|
|
|
+
|
|
|
+ class TinfoilCookerCollectionAdapter:
|
|
|
+ """ cooker.collection adapter """
|
|
|
+ def __init__(self, tinfoil):
|
|
|
+ self.tinfoil = tinfoil
|
|
|
+ def get_file_appends(self, fn):
|
|
|
+ return self.tinfoil.run_command('getFileAppends', fn)
|
|
|
+ def __getattr__(self, name):
|
|
|
+ if name == 'overlayed':
|
|
|
+ return self.tinfoil.get_overlayed_recipes()
|
|
|
+ elif name == 'bbappends':
|
|
|
+ return self.tinfoil.run_command('getAllAppends')
|
|
|
+ else:
|
|
|
+ raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
|
|
|
+
|
|
|
+ class TinfoilRecipeCacheAdapter:
|
|
|
+ """ cooker.recipecache adapter """
|
|
|
+ def __init__(self, tinfoil):
|
|
|
+ self.tinfoil = tinfoil
|
|
|
+ self._cache = {}
|
|
|
+
|
|
|
+ def get_pkg_pn_fn(self):
|
|
|
+ pkg_pn = defaultdict(list, self.tinfoil.run_command('getRecipes') or [])
|
|
|
+ pkg_fn = {}
|
|
|
+ for pn, fnlist in pkg_pn.items():
|
|
|
+ for fn in fnlist:
|
|
|
+ pkg_fn[fn] = pn
|
|
|
+ self._cache['pkg_pn'] = pkg_pn
|
|
|
+ self._cache['pkg_fn'] = pkg_fn
|
|
|
+
|
|
|
+ def __getattr__(self, name):
|
|
|
+ # Grab these only when they are requested since they aren't always used
|
|
|
+ if name in self._cache:
|
|
|
+ return self._cache[name]
|
|
|
+ elif name == 'pkg_pn':
|
|
|
+ self.get_pkg_pn_fn()
|
|
|
+ return self._cache[name]
|
|
|
+ elif name == 'pkg_fn':
|
|
|
+ self.get_pkg_pn_fn()
|
|
|
+ return self._cache[name]
|
|
|
+ elif name == 'deps':
|
|
|
+ attrvalue = defaultdict(list, self.tinfoil.run_command('getRecipeDepends') or [])
|
|
|
+ elif name == 'rundeps':
|
|
|
+ attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeDepends') or [])
|
|
|
+ elif name == 'runrecs':
|
|
|
+ attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeRecommends') or [])
|
|
|
+ elif name == 'pkg_pepvpr':
|
|
|
+ attrvalue = self.tinfoil.run_command('getRecipeVersions') or {}
|
|
|
+ elif name == 'inherits':
|
|
|
+ attrvalue = self.tinfoil.run_command('getRecipeInherits') or {}
|
|
|
+ elif name == 'bbfile_priority':
|
|
|
+ attrvalue = self.tinfoil.run_command('getBbFilePriority') or {}
|
|
|
+ elif name == 'pkg_dp':
|
|
|
+ attrvalue = self.tinfoil.run_command('getDefaultPreference') or {}
|
|
|
+ else:
|
|
|
+ raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
|
|
|
+
|
|
|
+ self._cache[name] = attrvalue
|
|
|
+ return attrvalue
|
|
|
+
|
|
|
+ def __init__(self, tinfoil):
|
|
|
+ self.tinfoil = tinfoil
|
|
|
+ self.collection = self.TinfoilCookerCollectionAdapter(tinfoil)
|
|
|
+ self.recipecaches = {}
|
|
|
+ # FIXME all machines
|
|
|
+ self.recipecaches[''] = self.TinfoilRecipeCacheAdapter(tinfoil)
|
|
|
+ self._cache = {}
|
|
|
+ def __getattr__(self, name):
|
|
|
+ # Grab these only when they are requested since they aren't always used
|
|
|
+ if name in self._cache:
|
|
|
+ return self._cache[name]
|
|
|
+ elif name == 'skiplist':
|
|
|
+ attrvalue = self.tinfoil.get_skipped_recipes()
|
|
|
+ elif name == 'bbfile_config_priorities':
|
|
|
+ ret = self.tinfoil.run_command('getLayerPriorities')
|
|
|
+ bbfile_config_priorities = []
|
|
|
+ for collection, pattern, regex, pri in ret:
|
|
|
+ bbfile_config_priorities.append((collection, pattern, re.compile(regex), pri))
|
|
|
+
|
|
|
+ attrvalue = bbfile_config_priorities
|
|
|
+ else:
|
|
|
+ raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
|
|
|
+
|
|
|
+ self._cache[name] = attrvalue
|
|
|
+ return attrvalue
|
|
|
+
|
|
|
+ def findBestProvider(self, pn):
|
|
|
+ return self.tinfoil.find_best_provider(pn)
|
|
|
+
|
|
|
+
|
|
|
class Tinfoil:
|
|
|
- def __init__(self, output=sys.stdout, tracking=False):
|
|
|
- # Needed to avoid deprecation warnings with python 2.6
|
|
|
- warnings.filterwarnings("ignore", category=DeprecationWarning)
|
|
|
|
|
|
- # Set up logging
|
|
|
+ def __init__(self, output=sys.stdout, tracking=False):
|
|
|
self.logger = logging.getLogger('BitBake')
|
|
|
- self._log_hdlr = logging.StreamHandler(output)
|
|
|
- bb.msg.addDefaultlogFilter(self._log_hdlr)
|
|
|
- format = bb.msg.BBLogFormatter("%(levelname)s: %(message)s")
|
|
|
- if output.isatty():
|
|
|
- format.enable_color()
|
|
|
- self._log_hdlr.setFormatter(format)
|
|
|
- self.logger.addHandler(self._log_hdlr)
|
|
|
-
|
|
|
- self.config = CookerConfiguration()
|
|
|
- configparams = TinfoilConfigParameters(parse_only=True)
|
|
|
- self.config.setConfigParameters(configparams)
|
|
|
- self.config.setServerRegIdleCallback(self.register_idle_function)
|
|
|
- features = []
|
|
|
- if tracking:
|
|
|
- features.append(CookerFeatures.BASEDATASTORE_TRACKING)
|
|
|
- self.cooker = BBCooker(self.config, features)
|
|
|
- self.config_data = self.cooker.data
|
|
|
- bb.providers.logger.setLevel(logging.ERROR)
|
|
|
- self.cooker_data = None
|
|
|
-
|
|
|
- def register_idle_function(self, function, data):
|
|
|
- pass
|
|
|
+ self.config_data = None
|
|
|
+ self.cooker = None
|
|
|
+ self.tracking = tracking
|
|
|
+ self.ui_module = None
|
|
|
+ self.server_connection = None
|
|
|
|
|
|
def __enter__(self):
|
|
|
return self
|
|
@@ -65,30 +190,120 @@ class Tinfoil:
|
|
|
def __exit__(self, type, value, traceback):
|
|
|
self.shutdown()
|
|
|
|
|
|
- def parseRecipes(self):
|
|
|
- sys.stderr.write("Parsing recipes..")
|
|
|
- self.logger.setLevel(logging.WARNING)
|
|
|
+ def prepare(self, config_only=False, config_params=None, quiet=0):
|
|
|
+ if self.tracking:
|
|
|
+ extrafeatures = [bb.cooker.CookerFeatures.BASEDATASTORE_TRACKING]
|
|
|
+ else:
|
|
|
+ extrafeatures = []
|
|
|
|
|
|
- try:
|
|
|
- while self.cooker.state in (state.initial, state.parsing):
|
|
|
- self.cooker.updateCache()
|
|
|
- except KeyboardInterrupt:
|
|
|
- self.cooker.shutdown()
|
|
|
- self.cooker.updateCache()
|
|
|
- sys.exit(2)
|
|
|
+ if not config_params:
|
|
|
+ config_params = TinfoilConfigParameters(config_only=config_only, quiet=quiet)
|
|
|
|
|
|
- self.logger.setLevel(logging.INFO)
|
|
|
- sys.stderr.write("done.\n")
|
|
|
+ cookerconfig = CookerConfiguration()
|
|
|
+ cookerconfig.setConfigParameters(config_params)
|
|
|
|
|
|
- self.cooker_data = self.cooker.recipecaches['']
|
|
|
+ server, self.server_connection, ui_module = setup_bitbake(config_params,
|
|
|
+ cookerconfig,
|
|
|
+ extrafeatures)
|
|
|
|
|
|
- def prepare(self, config_only = False):
|
|
|
- if not self.cooker_data:
|
|
|
+ self.ui_module = ui_module
|
|
|
+
|
|
|
+ if self.server_connection:
|
|
|
+ _server_connections.append(self.server_connection)
|
|
|
if config_only:
|
|
|
- self.cooker.parseConfiguration()
|
|
|
- self.cooker_data = self.cooker.recipecaches['']
|
|
|
+ config_params.updateToServer(self.server_connection.connection, os.environ.copy())
|
|
|
+ self.run_command('parseConfiguration')
|
|
|
else:
|
|
|
- self.parseRecipes()
|
|
|
+ self.run_actions(config_params)
|
|
|
+
|
|
|
+ self.config_data = bb.data.init()
|
|
|
+ connector = TinfoilDataStoreConnector(self, None)
|
|
|
+ self.config_data.setVar('_remote_data', connector)
|
|
|
+ self.cooker = TinfoilCookerAdapter(self)
|
|
|
+ self.cooker_data = self.cooker.recipecaches['']
|
|
|
+ else:
|
|
|
+ raise Exception('Failed to start bitbake server')
|
|
|
+
|
|
|
+ def run_actions(self, config_params):
|
|
|
+ """
|
|
|
+ Run the actions specified in config_params through the UI.
|
|
|
+ """
|
|
|
+ ret = self.ui_module.main(self.server_connection.connection, self.server_connection.events, config_params)
|
|
|
+ if ret:
|
|
|
+ raise TinfoilUIException(ret)
|
|
|
+
|
|
|
+ def parseRecipes(self):
|
|
|
+ """
|
|
|
+ Force a parse of all recipes. Normally you should specify
|
|
|
+ config_only=False when calling prepare() instead of using this
|
|
|
+ function; this function is designed for situations where you need
|
|
|
+ to initialise Tinfoil and use it with config_only=True first and
|
|
|
+ then conditionally call this function to parse recipes later.
|
|
|
+ """
|
|
|
+ config_params = TinfoilConfigParameters(config_only=False)
|
|
|
+ self.run_actions(config_params)
|
|
|
+
|
|
|
+ def run_command(self, command, *params):
|
|
|
+ """
|
|
|
+ Run a command on the server (as implemented in bb.command).
|
|
|
+ Note that there are two types of command - synchronous and
|
|
|
+ asynchronous; in order to receive the results of asynchronous
|
|
|
+ commands you will need to set an appropriate event mask
|
|
|
+ using set_event_mask() and listen for the result using
|
|
|
+ wait_event() - with the correct event mask you'll at least get
|
|
|
+ bb.command.CommandCompleted and possibly other events before
|
|
|
+ that depending on the command.
|
|
|
+ """
|
|
|
+ if not self.server_connection:
|
|
|
+ raise Exception('Not connected to server (did you call .prepare()?)')
|
|
|
+
|
|
|
+ commandline = [command]
|
|
|
+ if params:
|
|
|
+ commandline.extend(params)
|
|
|
+ result = self.server_connection.connection.runCommand(commandline)
|
|
|
+ if result[1]:
|
|
|
+ raise TinfoilCommandFailed(result[1])
|
|
|
+ return result[0]
|
|
|
+
|
|
|
+ def set_event_mask(self, eventlist):
|
|
|
+ """Set the event mask which will be applied within wait_event()"""
|
|
|
+ if not self.server_connection:
|
|
|
+ raise Exception('Not connected to server (did you call .prepare()?)')
|
|
|
+ llevel, debug_domains = bb.msg.constructLogOptions()
|
|
|
+ ret = self.run_command('setEventMask', self.server_connection.connection.getEventHandle(), llevel, debug_domains, eventlist)
|
|
|
+ if not ret:
|
|
|
+ raise Exception('setEventMask failed')
|
|
|
+
|
|
|
+ def wait_event(self, timeout=0):
|
|
|
+ """
|
|
|
+ Wait for an event from the server for the specified time.
|
|
|
+ A timeout of 0 means don't wait if there are no events in the queue.
|
|
|
+ Returns the next event in the queue or None if the timeout was
|
|
|
+ reached. Note that in order to recieve any events you will
|
|
|
+ first need to set the internal event mask using set_event_mask()
|
|
|
+ (otherwise whatever event mask the UI set up will be in effect).
|
|
|
+ """
|
|
|
+ if not self.server_connection:
|
|
|
+ raise Exception('Not connected to server (did you call .prepare()?)')
|
|
|
+ return self.server_connection.events.waitEvent(timeout)
|
|
|
+
|
|
|
+ def get_overlayed_recipes(self):
|
|
|
+ return defaultdict(list, self.run_command('getOverlayedRecipes'))
|
|
|
+
|
|
|
+ def get_skipped_recipes(self):
|
|
|
+ return OrderedDict(self.run_command('getSkippedRecipes'))
|
|
|
+
|
|
|
+ def get_all_providers(self):
|
|
|
+ return defaultdict(list, self.run_command('allProviders'))
|
|
|
+
|
|
|
+ def find_providers(self):
|
|
|
+ return self.run_command('findProviders')
|
|
|
+
|
|
|
+ def find_best_provider(self, pn):
|
|
|
+ return self.run_command('findBestProvider', pn)
|
|
|
+
|
|
|
+ def get_runtime_providers(self, rdep):
|
|
|
+ return self.run_command('getRuntimeProviders', rdep)
|
|
|
|
|
|
def parse_recipe_file(self, fn, appends=True, appendlist=None, config_data=None):
|
|
|
"""
|
|
@@ -126,22 +341,72 @@ class Tinfoil:
|
|
|
envdata = parser.loadDataFull(fn, appendfiles)
|
|
|
return envdata
|
|
|
|
|
|
+ def build_file(self, buildfile, task):
|
|
|
+ """
|
|
|
+ Runs the specified task for just a single recipe (i.e. no dependencies).
|
|
|
+ This is equivalent to bitbake -b.
|
|
|
+ """
|
|
|
+ return self.run_command('buildFile', buildfile, task)
|
|
|
+
|
|
|
def shutdown(self):
|
|
|
- self.cooker.shutdown(force=True)
|
|
|
- self.cooker.post_serve()
|
|
|
- self.cooker.unlockBitbake()
|
|
|
- self.logger.removeHandler(self._log_hdlr)
|
|
|
+ if self.server_connection:
|
|
|
+ self.run_command('clientComplete')
|
|
|
+ _server_connections.remove(self.server_connection)
|
|
|
+ bb.event.ui_queue = []
|
|
|
+ self.server_connection.terminate()
|
|
|
+ self.server_connection = None
|
|
|
|
|
|
-class TinfoilConfigParameters(ConfigParameters):
|
|
|
+ def _reconvert_type(self, obj, origtypename):
|
|
|
+ """
|
|
|
+ Convert an object back to the right type, in the case
|
|
|
+ that marshalling has changed it (especially with xmlrpc)
|
|
|
+ """
|
|
|
+ supported_types = {
|
|
|
+ 'set': set,
|
|
|
+ 'DataStoreConnectionHandle': bb.command.DataStoreConnectionHandle,
|
|
|
+ }
|
|
|
|
|
|
- def __init__(self, **options):
|
|
|
+ origtype = supported_types.get(origtypename, None)
|
|
|
+ if origtype is None:
|
|
|
+ raise Exception('Unsupported type "%s"' % origtypename)
|
|
|
+ if type(obj) == origtype:
|
|
|
+ newobj = obj
|
|
|
+ elif isinstance(obj, dict):
|
|
|
+ # New style class
|
|
|
+ newobj = origtype()
|
|
|
+ for k,v in obj.items():
|
|
|
+ setattr(newobj, k, v)
|
|
|
+ else:
|
|
|
+ # Assume we can coerce the type
|
|
|
+ newobj = origtype(obj)
|
|
|
+
|
|
|
+ if isinstance(newobj, bb.command.DataStoreConnectionHandle):
|
|
|
+ connector = TinfoilDataStoreConnector(self, newobj.dsindex)
|
|
|
+ newobj = bb.data.init()
|
|
|
+ newobj.setVar('_remote_data', connector)
|
|
|
+
|
|
|
+ return newobj
|
|
|
+
|
|
|
+
|
|
|
+class TinfoilConfigParameters(BitBakeConfigParameters):
|
|
|
+
|
|
|
+ def __init__(self, config_only, **options):
|
|
|
self.initial_options = options
|
|
|
- super(TinfoilConfigParameters, self).__init__()
|
|
|
+ # Apply some sane defaults
|
|
|
+ if not 'parse_only' in options:
|
|
|
+ self.initial_options['parse_only'] = not config_only
|
|
|
+ #if not 'status_only' in options:
|
|
|
+ # self.initial_options['status_only'] = config_only
|
|
|
+ if not 'ui' in options:
|
|
|
+ self.initial_options['ui'] = 'knotty'
|
|
|
+ if not 'argv' in options:
|
|
|
+ self.initial_options['argv'] = []
|
|
|
|
|
|
- def parseCommandLine(self, argv=sys.argv):
|
|
|
- class DummyOptions:
|
|
|
- def __init__(self, initial_options):
|
|
|
- for key, val in initial_options.items():
|
|
|
- setattr(self, key, val)
|
|
|
+ super(TinfoilConfigParameters, self).__init__()
|
|
|
|
|
|
- return DummyOptions(self.initial_options), None
|
|
|
+ def parseCommandLine(self, argv=None):
|
|
|
+ # We don't want any parameters parsed from the command line
|
|
|
+ opts = super(TinfoilConfigParameters, self).parseCommandLine([])
|
|
|
+ for key, val in self.initial_options.items():
|
|
|
+ setattr(opts[0], key, val)
|
|
|
+ return opts
|