tinfoil.py 39 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902
  1. # tinfoil: a simple wrapper around cooker for bitbake-based command-line utilities
  2. #
  3. # Copyright (C) 2012-2017 Intel Corporation
  4. # Copyright (C) 2011 Mentor Graphics Corporation
  5. # Copyright (C) 2006-2012 Richard Purdie
  6. #
  7. # SPDX-License-Identifier: GPL-2.0-only
  8. #
  9. # This program is free software; you can redistribute it and/or modify
  10. # it under the terms of the GNU General Public License version 2 as
  11. # published by the Free Software Foundation.
  12. #
  13. # This program is distributed in the hope that it will be useful,
  14. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16. # GNU General Public License for more details.
  17. #
  18. # You should have received a copy of the GNU General Public License along
  19. # with this program; if not, write to the Free Software Foundation, Inc.,
  20. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  21. import logging
  22. import os
  23. import sys
  24. import atexit
  25. import re
  26. from collections import OrderedDict, defaultdict
  27. import bb.cache
  28. import bb.cooker
  29. import bb.providers
  30. import bb.taskdata
  31. import bb.utils
  32. import bb.command
  33. import bb.remotedata
  34. from bb.cookerdata import CookerConfiguration, ConfigParameters
  35. from bb.main import setup_bitbake, BitBakeConfigParameters, BBMainException
  36. import bb.fetch2
  37. # We need this in order to shut down the connection to the bitbake server,
  38. # otherwise the process will never properly exit
  39. _server_connections = []
  40. def _terminate_connections():
  41. for connection in _server_connections:
  42. connection.terminate()
  43. atexit.register(_terminate_connections)
  44. class TinfoilUIException(Exception):
  45. """Exception raised when the UI returns non-zero from its main function"""
  46. def __init__(self, returncode):
  47. self.returncode = returncode
  48. def __repr__(self):
  49. return 'UI module main returned %d' % self.returncode
  50. class TinfoilCommandFailed(Exception):
  51. """Exception raised when run_command fails"""
  52. class TinfoilDataStoreConnector:
  53. """Connector object used to enable access to datastore objects via tinfoil"""
  54. def __init__(self, tinfoil, dsindex):
  55. self.tinfoil = tinfoil
  56. self.dsindex = dsindex
  57. def getVar(self, name):
  58. value = self.tinfoil.run_command('dataStoreConnectorFindVar', self.dsindex, name)
  59. overrides = None
  60. if isinstance(value, dict):
  61. if '_connector_origtype' in value:
  62. value['_content'] = self.tinfoil._reconvert_type(value['_content'], value['_connector_origtype'])
  63. del value['_connector_origtype']
  64. if '_connector_overrides' in value:
  65. overrides = value['_connector_overrides']
  66. del value['_connector_overrides']
  67. return value, overrides
  68. def getKeys(self):
  69. return set(self.tinfoil.run_command('dataStoreConnectorGetKeys', self.dsindex))
  70. def getVarHistory(self, name):
  71. return self.tinfoil.run_command('dataStoreConnectorGetVarHistory', self.dsindex, name)
  72. def expandPythonRef(self, varname, expr, d):
  73. ds = bb.remotedata.RemoteDatastores.transmit_datastore(d)
  74. ret = self.tinfoil.run_command('dataStoreConnectorExpandPythonRef', ds, varname, expr)
  75. return ret
  76. def setVar(self, varname, value):
  77. if self.dsindex is None:
  78. self.tinfoil.run_command('setVariable', varname, value)
  79. else:
  80. # Not currently implemented - indicate that setting should
  81. # be redirected to local side
  82. return True
  83. def setVarFlag(self, varname, flagname, value):
  84. if self.dsindex is None:
  85. self.tinfoil.run_command('dataStoreConnectorSetVarFlag', self.dsindex, varname, flagname, value)
  86. else:
  87. # Not currently implemented - indicate that setting should
  88. # be redirected to local side
  89. return True
  90. def delVar(self, varname):
  91. if self.dsindex is None:
  92. self.tinfoil.run_command('dataStoreConnectorDelVar', self.dsindex, varname)
  93. else:
  94. # Not currently implemented - indicate that setting should
  95. # be redirected to local side
  96. return True
  97. def delVarFlag(self, varname, flagname):
  98. if self.dsindex is None:
  99. self.tinfoil.run_command('dataStoreConnectorDelVar', self.dsindex, varname, flagname)
  100. else:
  101. # Not currently implemented - indicate that setting should
  102. # be redirected to local side
  103. return True
  104. def renameVar(self, name, newname):
  105. if self.dsindex is None:
  106. self.tinfoil.run_command('dataStoreConnectorRenameVar', self.dsindex, name, newname)
  107. else:
  108. # Not currently implemented - indicate that setting should
  109. # be redirected to local side
  110. return True
  111. class TinfoilCookerAdapter:
  112. """
  113. Provide an adapter for existing code that expects to access a cooker object via Tinfoil,
  114. since now Tinfoil is on the client side it no longer has direct access.
  115. """
  116. class TinfoilCookerCollectionAdapter:
  117. """ cooker.collection adapter """
  118. def __init__(self, tinfoil):
  119. self.tinfoil = tinfoil
  120. def get_file_appends(self, fn):
  121. return self.tinfoil.get_file_appends(fn)
  122. def __getattr__(self, name):
  123. if name == 'overlayed':
  124. return self.tinfoil.get_overlayed_recipes()
  125. elif name == 'bbappends':
  126. return self.tinfoil.run_command('getAllAppends')
  127. else:
  128. raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
  129. class TinfoilRecipeCacheAdapter:
  130. """ cooker.recipecache adapter """
  131. def __init__(self, tinfoil):
  132. self.tinfoil = tinfoil
  133. self._cache = {}
  134. def get_pkg_pn_fn(self):
  135. pkg_pn = defaultdict(list, self.tinfoil.run_command('getRecipes') or [])
  136. pkg_fn = {}
  137. for pn, fnlist in pkg_pn.items():
  138. for fn in fnlist:
  139. pkg_fn[fn] = pn
  140. self._cache['pkg_pn'] = pkg_pn
  141. self._cache['pkg_fn'] = pkg_fn
  142. def __getattr__(self, name):
  143. # Grab these only when they are requested since they aren't always used
  144. if name in self._cache:
  145. return self._cache[name]
  146. elif name == 'pkg_pn':
  147. self.get_pkg_pn_fn()
  148. return self._cache[name]
  149. elif name == 'pkg_fn':
  150. self.get_pkg_pn_fn()
  151. return self._cache[name]
  152. elif name == 'deps':
  153. attrvalue = defaultdict(list, self.tinfoil.run_command('getRecipeDepends') or [])
  154. elif name == 'rundeps':
  155. attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeDepends') or [])
  156. elif name == 'runrecs':
  157. attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeRecommends') or [])
  158. elif name == 'pkg_pepvpr':
  159. attrvalue = self.tinfoil.run_command('getRecipeVersions') or {}
  160. elif name == 'inherits':
  161. attrvalue = self.tinfoil.run_command('getRecipeInherits') or {}
  162. elif name == 'bbfile_priority':
  163. attrvalue = self.tinfoil.run_command('getBbFilePriority') or {}
  164. elif name == 'pkg_dp':
  165. attrvalue = self.tinfoil.run_command('getDefaultPreference') or {}
  166. elif name == 'fn_provides':
  167. attrvalue = self.tinfoil.run_command('getRecipeProvides') or {}
  168. elif name == 'packages':
  169. attrvalue = self.tinfoil.run_command('getRecipePackages') or {}
  170. elif name == 'packages_dynamic':
  171. attrvalue = self.tinfoil.run_command('getRecipePackagesDynamic') or {}
  172. elif name == 'rproviders':
  173. attrvalue = self.tinfoil.run_command('getRProviders') or {}
  174. else:
  175. raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
  176. self._cache[name] = attrvalue
  177. return attrvalue
  178. def __init__(self, tinfoil):
  179. self.tinfoil = tinfoil
  180. self.collection = self.TinfoilCookerCollectionAdapter(tinfoil)
  181. self.recipecaches = {}
  182. # FIXME all machines
  183. self.recipecaches[''] = self.TinfoilRecipeCacheAdapter(tinfoil)
  184. self._cache = {}
  185. def __getattr__(self, name):
  186. # Grab these only when they are requested since they aren't always used
  187. if name in self._cache:
  188. return self._cache[name]
  189. elif name == 'skiplist':
  190. attrvalue = self.tinfoil.get_skipped_recipes()
  191. elif name == 'bbfile_config_priorities':
  192. ret = self.tinfoil.run_command('getLayerPriorities')
  193. bbfile_config_priorities = []
  194. for collection, pattern, regex, pri in ret:
  195. bbfile_config_priorities.append((collection, pattern, re.compile(regex), pri))
  196. attrvalue = bbfile_config_priorities
  197. else:
  198. raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
  199. self._cache[name] = attrvalue
  200. return attrvalue
  201. def findBestProvider(self, pn):
  202. return self.tinfoil.find_best_provider(pn)
  203. class TinfoilRecipeInfo:
  204. """
  205. Provides a convenient representation of the cached information for a single recipe.
  206. Some attributes are set on construction, others are read on-demand (which internally
  207. may result in a remote procedure call to the bitbake server the first time).
  208. Note that only information which is cached is available through this object - if
  209. you need other variable values you will need to parse the recipe using
  210. Tinfoil.parse_recipe().
  211. """
  212. def __init__(self, recipecache, d, pn, fn, fns):
  213. self._recipecache = recipecache
  214. self._d = d
  215. self.pn = pn
  216. self.fn = fn
  217. self.fns = fns
  218. self.inherit_files = recipecache.inherits[fn]
  219. self.depends = recipecache.deps[fn]
  220. (self.pe, self.pv, self.pr) = recipecache.pkg_pepvpr[fn]
  221. self._cached_packages = None
  222. self._cached_rprovides = None
  223. self._cached_packages_dynamic = None
  224. def __getattr__(self, name):
  225. if name == 'alternates':
  226. return [x for x in self.fns if x != self.fn]
  227. elif name == 'rdepends':
  228. return self._recipecache.rundeps[self.fn]
  229. elif name == 'rrecommends':
  230. return self._recipecache.runrecs[self.fn]
  231. elif name == 'provides':
  232. return self._recipecache.fn_provides[self.fn]
  233. elif name == 'packages':
  234. if self._cached_packages is None:
  235. self._cached_packages = []
  236. for pkg, fns in self._recipecache.packages.items():
  237. if self.fn in fns:
  238. self._cached_packages.append(pkg)
  239. return self._cached_packages
  240. elif name == 'packages_dynamic':
  241. if self._cached_packages_dynamic is None:
  242. self._cached_packages_dynamic = []
  243. for pkg, fns in self._recipecache.packages_dynamic.items():
  244. if self.fn in fns:
  245. self._cached_packages_dynamic.append(pkg)
  246. return self._cached_packages_dynamic
  247. elif name == 'rprovides':
  248. if self._cached_rprovides is None:
  249. self._cached_rprovides = []
  250. for pkg, fns in self._recipecache.rproviders.items():
  251. if self.fn in fns:
  252. self._cached_rprovides.append(pkg)
  253. return self._cached_rprovides
  254. else:
  255. raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
  256. def inherits(self, only_recipe=False):
  257. """
  258. Get the inherited classes for a recipe. Returns the class names only.
  259. Parameters:
  260. only_recipe: True to return only the classes inherited by the recipe
  261. itself, False to return all classes inherited within
  262. the context for the recipe (which includes globally
  263. inherited classes).
  264. """
  265. if only_recipe:
  266. global_inherit = [x for x in (self._d.getVar('BBINCLUDED') or '').split() if x.endswith('.bbclass')]
  267. else:
  268. global_inherit = []
  269. for clsfile in self.inherit_files:
  270. if only_recipe and clsfile in global_inherit:
  271. continue
  272. clsname = os.path.splitext(os.path.basename(clsfile))[0]
  273. yield clsname
  274. def __str__(self):
  275. return '%s' % self.pn
  276. class Tinfoil:
  277. """
  278. Tinfoil - an API for scripts and utilities to query
  279. BitBake internals and perform build operations.
  280. """
  281. def __init__(self, output=sys.stdout, tracking=False, setup_logging=True):
  282. """
  283. Create a new tinfoil object.
  284. Parameters:
  285. output: specifies where console output should be sent. Defaults
  286. to sys.stdout.
  287. tracking: True to enable variable history tracking, False to
  288. disable it (default). Enabling this has a minor
  289. performance impact so typically it isn't enabled
  290. unless you need to query variable history.
  291. setup_logging: True to setup a logger so that things like
  292. bb.warn() will work immediately and timeout warnings
  293. are visible; False to let BitBake do this itself.
  294. """
  295. self.logger = logging.getLogger('BitBake')
  296. self.config_data = None
  297. self.cooker = None
  298. self.tracking = tracking
  299. self.ui_module = None
  300. self.server_connection = None
  301. self.recipes_parsed = False
  302. self.quiet = 0
  303. self.oldhandlers = self.logger.handlers[:]
  304. if setup_logging:
  305. # This is the *client-side* logger, nothing to do with
  306. # logging messages from the server
  307. bb.msg.logger_create('BitBake', output)
  308. self.localhandlers = []
  309. for handler in self.logger.handlers:
  310. if handler not in self.oldhandlers:
  311. self.localhandlers.append(handler)
  312. def __enter__(self):
  313. return self
  314. def __exit__(self, type, value, traceback):
  315. self.shutdown()
  316. def prepare(self, config_only=False, config_params=None, quiet=0, extra_features=None):
  317. """
  318. Prepares the underlying BitBake system to be used via tinfoil.
  319. This function must be called prior to calling any of the other
  320. functions in the API.
  321. NOTE: if you call prepare() you must absolutely call shutdown()
  322. before your code terminates. You can use a "with" block to ensure
  323. this happens e.g.
  324. with bb.tinfoil.Tinfoil() as tinfoil:
  325. tinfoil.prepare()
  326. ...
  327. Parameters:
  328. config_only: True to read only the configuration and not load
  329. the cache / parse recipes. This is useful if you just
  330. want to query the value of a variable at the global
  331. level or you want to do anything else that doesn't
  332. involve knowing anything about the recipes in the
  333. current configuration. False loads the cache / parses
  334. recipes.
  335. config_params: optionally specify your own configuration
  336. parameters. If not specified an instance of
  337. TinfoilConfigParameters will be created internally.
  338. quiet: quiet level controlling console output - equivalent
  339. to bitbake's -q/--quiet option. Default of 0 gives
  340. the same output level as normal bitbake execution.
  341. extra_features: extra features to be added to the feature
  342. set requested from the server. See
  343. CookerFeatures._feature_list for possible
  344. features.
  345. """
  346. self.quiet = quiet
  347. if self.tracking:
  348. extrafeatures = [bb.cooker.CookerFeatures.BASEDATASTORE_TRACKING]
  349. else:
  350. extrafeatures = []
  351. if extra_features:
  352. extrafeatures += extra_features
  353. if not config_params:
  354. config_params = TinfoilConfigParameters(config_only=config_only, quiet=quiet)
  355. cookerconfig = CookerConfiguration()
  356. cookerconfig.setConfigParameters(config_params)
  357. if not config_only:
  358. # Disable local loggers because the UI module is going to set up its own
  359. for handler in self.localhandlers:
  360. self.logger.handlers.remove(handler)
  361. self.localhandlers = []
  362. self.server_connection, ui_module = setup_bitbake(config_params,
  363. cookerconfig,
  364. extrafeatures)
  365. self.ui_module = ui_module
  366. # Ensure the path to bitbake's bin directory is in PATH so that things like
  367. # bitbake-worker can be run (usually this is the case, but it doesn't have to be)
  368. path = os.getenv('PATH').split(':')
  369. bitbakebinpath = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', 'bin'))
  370. for entry in path:
  371. if entry.endswith(os.sep):
  372. entry = entry[:-1]
  373. if os.path.abspath(entry) == bitbakebinpath:
  374. break
  375. else:
  376. path.insert(0, bitbakebinpath)
  377. os.environ['PATH'] = ':'.join(path)
  378. if self.server_connection:
  379. _server_connections.append(self.server_connection)
  380. if config_only:
  381. config_params.updateToServer(self.server_connection.connection, os.environ.copy())
  382. self.run_command('parseConfiguration')
  383. else:
  384. self.run_actions(config_params)
  385. self.recipes_parsed = True
  386. self.config_data = bb.data.init()
  387. connector = TinfoilDataStoreConnector(self, None)
  388. self.config_data.setVar('_remote_data', connector)
  389. self.cooker = TinfoilCookerAdapter(self)
  390. self.cooker_data = self.cooker.recipecaches['']
  391. else:
  392. raise Exception('Failed to start bitbake server')
  393. def run_actions(self, config_params):
  394. """
  395. Run the actions specified in config_params through the UI.
  396. """
  397. ret = self.ui_module.main(self.server_connection.connection, self.server_connection.events, config_params)
  398. if ret:
  399. raise TinfoilUIException(ret)
  400. def parseRecipes(self):
  401. """
  402. Legacy function - use parse_recipes() instead.
  403. """
  404. self.parse_recipes()
  405. def parse_recipes(self):
  406. """
  407. Load information on all recipes. Normally you should specify
  408. config_only=False when calling prepare() instead of using this
  409. function; this function is designed for situations where you need
  410. to initialise Tinfoil and use it with config_only=True first and
  411. then conditionally call this function to parse recipes later.
  412. """
  413. config_params = TinfoilConfigParameters(config_only=False)
  414. self.run_actions(config_params)
  415. self.recipes_parsed = True
  416. def run_command(self, command, *params):
  417. """
  418. Run a command on the server (as implemented in bb.command).
  419. Note that there are two types of command - synchronous and
  420. asynchronous; in order to receive the results of asynchronous
  421. commands you will need to set an appropriate event mask
  422. using set_event_mask() and listen for the result using
  423. wait_event() - with the correct event mask you'll at least get
  424. bb.command.CommandCompleted and possibly other events before
  425. that depending on the command.
  426. """
  427. if not self.server_connection:
  428. raise Exception('Not connected to server (did you call .prepare()?)')
  429. commandline = [command]
  430. if params:
  431. commandline.extend(params)
  432. result = self.server_connection.connection.runCommand(commandline)
  433. if result[1]:
  434. raise TinfoilCommandFailed(result[1])
  435. return result[0]
  436. def set_event_mask(self, eventlist):
  437. """Set the event mask which will be applied within wait_event()"""
  438. if not self.server_connection:
  439. raise Exception('Not connected to server (did you call .prepare()?)')
  440. llevel, debug_domains = bb.msg.constructLogOptions()
  441. ret = self.run_command('setEventMask', self.server_connection.connection.getEventHandle(), llevel, debug_domains, eventlist)
  442. if not ret:
  443. raise Exception('setEventMask failed')
  444. def wait_event(self, timeout=0):
  445. """
  446. Wait for an event from the server for the specified time.
  447. A timeout of 0 means don't wait if there are no events in the queue.
  448. Returns the next event in the queue or None if the timeout was
  449. reached. Note that in order to recieve any events you will
  450. first need to set the internal event mask using set_event_mask()
  451. (otherwise whatever event mask the UI set up will be in effect).
  452. """
  453. if not self.server_connection:
  454. raise Exception('Not connected to server (did you call .prepare()?)')
  455. return self.server_connection.events.waitEvent(timeout)
  456. def get_overlayed_recipes(self):
  457. """
  458. Find recipes which are overlayed (i.e. where recipes exist in multiple layers)
  459. """
  460. return defaultdict(list, self.run_command('getOverlayedRecipes'))
  461. def get_skipped_recipes(self):
  462. """
  463. Find recipes which were skipped (i.e. SkipRecipe was raised
  464. during parsing).
  465. """
  466. return OrderedDict(self.run_command('getSkippedRecipes'))
  467. def get_all_providers(self):
  468. return defaultdict(list, self.run_command('allProviders'))
  469. def find_providers(self):
  470. return self.run_command('findProviders')
  471. def find_best_provider(self, pn):
  472. return self.run_command('findBestProvider', pn)
  473. def get_runtime_providers(self, rdep):
  474. return self.run_command('getRuntimeProviders', rdep)
  475. def get_recipe_file(self, pn):
  476. """
  477. Get the file name for the specified recipe/target. Raises
  478. bb.providers.NoProvider if there is no match or the recipe was
  479. skipped.
  480. """
  481. best = self.find_best_provider(pn)
  482. if not best or (len(best) > 3 and not best[3]):
  483. skiplist = self.get_skipped_recipes()
  484. taskdata = bb.taskdata.TaskData(None, skiplist=skiplist)
  485. skipreasons = taskdata.get_reasons(pn)
  486. if skipreasons:
  487. raise bb.providers.NoProvider('%s is unavailable:\n %s' % (pn, ' \n'.join(skipreasons)))
  488. else:
  489. raise bb.providers.NoProvider('Unable to find any recipe file matching "%s"' % pn)
  490. return best[3]
  491. def get_file_appends(self, fn):
  492. """
  493. Find the bbappends for a recipe file
  494. """
  495. return self.run_command('getFileAppends', fn)
  496. def all_recipes(self, mc='', sort=True):
  497. """
  498. Enable iterating over all recipes in the current configuration.
  499. Returns an iterator over TinfoilRecipeInfo objects created on demand.
  500. Parameters:
  501. mc: The multiconfig, default of '' uses the main configuration.
  502. sort: True to sort recipes alphabetically (default), False otherwise
  503. """
  504. recipecache = self.cooker.recipecaches[mc]
  505. if sort:
  506. recipes = sorted(recipecache.pkg_pn.items())
  507. else:
  508. recipes = recipecache.pkg_pn.items()
  509. for pn, fns in recipes:
  510. prov = self.find_best_provider(pn)
  511. recipe = TinfoilRecipeInfo(recipecache,
  512. self.config_data,
  513. pn=pn,
  514. fn=prov[3],
  515. fns=fns)
  516. yield recipe
  517. def all_recipe_files(self, mc='', variants=True, preferred_only=False):
  518. """
  519. Enable iterating over all recipe files in the current configuration.
  520. Returns an iterator over file paths.
  521. Parameters:
  522. mc: The multiconfig, default of '' uses the main configuration.
  523. variants: True to include variants of recipes created through
  524. BBCLASSEXTEND (default) or False to exclude them
  525. preferred_only: True to include only the preferred recipe where
  526. multiple exist providing the same PN, False to list
  527. all recipes
  528. """
  529. recipecache = self.cooker.recipecaches[mc]
  530. if preferred_only:
  531. files = []
  532. for pn in recipecache.pkg_pn.keys():
  533. prov = self.find_best_provider(pn)
  534. files.append(prov[3])
  535. else:
  536. files = recipecache.pkg_fn.keys()
  537. for fn in sorted(files):
  538. if not variants and fn.startswith('virtual:'):
  539. continue
  540. yield fn
  541. def get_recipe_info(self, pn, mc=''):
  542. """
  543. Get information on a specific recipe in the current configuration by name (PN).
  544. Returns a TinfoilRecipeInfo object created on demand.
  545. Parameters:
  546. mc: The multiconfig, default of '' uses the main configuration.
  547. """
  548. recipecache = self.cooker.recipecaches[mc]
  549. prov = self.find_best_provider(pn)
  550. fn = prov[3]
  551. if fn:
  552. actual_pn = recipecache.pkg_fn[fn]
  553. recipe = TinfoilRecipeInfo(recipecache,
  554. self.config_data,
  555. pn=actual_pn,
  556. fn=fn,
  557. fns=recipecache.pkg_pn[actual_pn])
  558. return recipe
  559. else:
  560. return None
  561. def parse_recipe(self, pn):
  562. """
  563. Parse the specified recipe and return a datastore object
  564. representing the environment for the recipe.
  565. """
  566. fn = self.get_recipe_file(pn)
  567. return self.parse_recipe_file(fn)
  568. def parse_recipe_file(self, fn, appends=True, appendlist=None, config_data=None):
  569. """
  570. Parse the specified recipe file (with or without bbappends)
  571. and return a datastore object representing the environment
  572. for the recipe.
  573. Parameters:
  574. fn: recipe file to parse - can be a file path or virtual
  575. specification
  576. appends: True to apply bbappends, False otherwise
  577. appendlist: optional list of bbappend files to apply, if you
  578. want to filter them
  579. config_data: custom config datastore to use. NOTE: if you
  580. specify config_data then you cannot use a virtual
  581. specification for fn.
  582. """
  583. if self.tracking:
  584. # Enable history tracking just for the parse operation
  585. self.run_command('enableDataTracking')
  586. try:
  587. if appends and appendlist == []:
  588. appends = False
  589. if config_data:
  590. dctr = bb.remotedata.RemoteDatastores.transmit_datastore(config_data)
  591. dscon = self.run_command('parseRecipeFile', fn, appends, appendlist, dctr)
  592. else:
  593. dscon = self.run_command('parseRecipeFile', fn, appends, appendlist)
  594. if dscon:
  595. return self._reconvert_type(dscon, 'DataStoreConnectionHandle')
  596. else:
  597. return None
  598. finally:
  599. if self.tracking:
  600. self.run_command('disableDataTracking')
  601. def build_file(self, buildfile, task, internal=True):
  602. """
  603. Runs the specified task for just a single recipe (i.e. no dependencies).
  604. This is equivalent to bitbake -b, except with the default internal=True
  605. no warning about dependencies will be produced, normal info messages
  606. from the runqueue will be silenced and BuildInit, BuildStarted and
  607. BuildCompleted events will not be fired.
  608. """
  609. return self.run_command('buildFile', buildfile, task, internal)
  610. def build_targets(self, targets, task=None, handle_events=True, extra_events=None, event_callback=None):
  611. """
  612. Builds the specified targets. This is equivalent to a normal invocation
  613. of bitbake. Has built-in event handling which is enabled by default and
  614. can be extended if needed.
  615. Parameters:
  616. targets:
  617. One or more targets to build. Can be a list or a
  618. space-separated string.
  619. task:
  620. The task to run; if None then the value of BB_DEFAULT_TASK
  621. will be used. Default None.
  622. handle_events:
  623. True to handle events in a similar way to normal bitbake
  624. invocation with knotty; False to return immediately (on the
  625. assumption that the caller will handle the events instead).
  626. Default True.
  627. extra_events:
  628. An optional list of events to add to the event mask (if
  629. handle_events=True). If you add events here you also need
  630. to specify a callback function in event_callback that will
  631. handle the additional events. Default None.
  632. event_callback:
  633. An optional function taking a single parameter which
  634. will be called first upon receiving any event (if
  635. handle_events=True) so that the caller can override or
  636. extend the event handling. Default None.
  637. """
  638. if isinstance(targets, str):
  639. targets = targets.split()
  640. if not task:
  641. task = self.config_data.getVar('BB_DEFAULT_TASK')
  642. if handle_events:
  643. # A reasonable set of default events matching up with those we handle below
  644. eventmask = [
  645. 'bb.event.BuildStarted',
  646. 'bb.event.BuildCompleted',
  647. 'logging.LogRecord',
  648. 'bb.event.NoProvider',
  649. 'bb.command.CommandCompleted',
  650. 'bb.command.CommandFailed',
  651. 'bb.build.TaskStarted',
  652. 'bb.build.TaskFailed',
  653. 'bb.build.TaskSucceeded',
  654. 'bb.build.TaskFailedSilent',
  655. 'bb.build.TaskProgress',
  656. 'bb.runqueue.runQueueTaskStarted',
  657. 'bb.runqueue.sceneQueueTaskStarted',
  658. 'bb.event.ProcessStarted',
  659. 'bb.event.ProcessProgress',
  660. 'bb.event.ProcessFinished',
  661. ]
  662. if extra_events:
  663. eventmask.extend(extra_events)
  664. ret = self.set_event_mask(eventmask)
  665. includelogs = self.config_data.getVar('BBINCLUDELOGS')
  666. loglines = self.config_data.getVar('BBINCLUDELOGS_LINES')
  667. ret = self.run_command('buildTargets', targets, task)
  668. if handle_events:
  669. result = False
  670. # Borrowed from knotty, instead somewhat hackily we use the helper
  671. # as the object to store "shutdown" on
  672. helper = bb.ui.uihelper.BBUIHelper()
  673. # We set up logging optionally in the constructor so now we need to
  674. # grab the handlers to pass to TerminalFilter
  675. console = None
  676. errconsole = None
  677. for handler in self.logger.handlers:
  678. if isinstance(handler, logging.StreamHandler):
  679. if handler.stream == sys.stdout:
  680. console = handler
  681. elif handler.stream == sys.stderr:
  682. errconsole = handler
  683. format_str = "%(levelname)s: %(message)s"
  684. format = bb.msg.BBLogFormatter(format_str)
  685. helper.shutdown = 0
  686. parseprogress = None
  687. termfilter = bb.ui.knotty.TerminalFilter(helper, helper, console, errconsole, format, quiet=self.quiet)
  688. try:
  689. while True:
  690. try:
  691. event = self.wait_event(0.25)
  692. if event:
  693. if event_callback and event_callback(event):
  694. continue
  695. if helper.eventHandler(event):
  696. if isinstance(event, bb.build.TaskFailedSilent):
  697. logger.warning("Logfile for failed setscene task is %s" % event.logfile)
  698. elif isinstance(event, bb.build.TaskFailed):
  699. bb.ui.knotty.print_event_log(event, includelogs, loglines, termfilter)
  700. continue
  701. if isinstance(event, bb.event.ProcessStarted):
  702. if self.quiet > 1:
  703. continue
  704. parseprogress = bb.ui.knotty.new_progress(event.processname, event.total)
  705. parseprogress.start(False)
  706. continue
  707. if isinstance(event, bb.event.ProcessProgress):
  708. if self.quiet > 1:
  709. continue
  710. if parseprogress:
  711. parseprogress.update(event.progress)
  712. else:
  713. bb.warn("Got ProcessProgress event for someting that never started?")
  714. continue
  715. if isinstance(event, bb.event.ProcessFinished):
  716. if self.quiet > 1:
  717. continue
  718. if parseprogress:
  719. parseprogress.finish()
  720. parseprogress = None
  721. continue
  722. if isinstance(event, bb.command.CommandCompleted):
  723. result = True
  724. break
  725. if isinstance(event, bb.command.CommandFailed):
  726. self.logger.error(str(event))
  727. result = False
  728. break
  729. if isinstance(event, logging.LogRecord):
  730. if event.taskpid == 0 or event.levelno > logging.INFO:
  731. self.logger.handle(event)
  732. continue
  733. if isinstance(event, bb.event.NoProvider):
  734. self.logger.error(str(event))
  735. result = False
  736. break
  737. elif helper.shutdown > 1:
  738. break
  739. termfilter.updateFooter()
  740. except KeyboardInterrupt:
  741. termfilter.clearFooter()
  742. if helper.shutdown == 1:
  743. print("\nSecond Keyboard Interrupt, stopping...\n")
  744. ret = self.run_command("stateForceShutdown")
  745. if ret and ret[2]:
  746. self.logger.error("Unable to cleanly stop: %s" % ret[2])
  747. elif helper.shutdown == 0:
  748. print("\nKeyboard Interrupt, closing down...\n")
  749. interrupted = True
  750. ret = self.run_command("stateShutdown")
  751. if ret and ret[2]:
  752. self.logger.error("Unable to cleanly shutdown: %s" % ret[2])
  753. helper.shutdown = helper.shutdown + 1
  754. termfilter.clearFooter()
  755. finally:
  756. termfilter.finish()
  757. if helper.failed_tasks:
  758. result = False
  759. return result
  760. else:
  761. return ret
  762. def shutdown(self):
  763. """
  764. Shut down tinfoil. Disconnects from the server and gracefully
  765. releases any associated resources. You must call this function if
  766. prepare() has been called, or use a with... block when you create
  767. the tinfoil object which will ensure that it gets called.
  768. """
  769. if self.server_connection:
  770. self.run_command('clientComplete')
  771. _server_connections.remove(self.server_connection)
  772. bb.event.ui_queue = []
  773. self.server_connection.terminate()
  774. self.server_connection = None
  775. # Restore logging handlers to how it looked when we started
  776. if self.oldhandlers:
  777. for handler in self.logger.handlers:
  778. if handler not in self.oldhandlers:
  779. self.logger.handlers.remove(handler)
  780. def _reconvert_type(self, obj, origtypename):
  781. """
  782. Convert an object back to the right type, in the case
  783. that marshalling has changed it (especially with xmlrpc)
  784. """
  785. supported_types = {
  786. 'set': set,
  787. 'DataStoreConnectionHandle': bb.command.DataStoreConnectionHandle,
  788. }
  789. origtype = supported_types.get(origtypename, None)
  790. if origtype is None:
  791. raise Exception('Unsupported type "%s"' % origtypename)
  792. if type(obj) == origtype:
  793. newobj = obj
  794. elif isinstance(obj, dict):
  795. # New style class
  796. newobj = origtype()
  797. for k,v in obj.items():
  798. setattr(newobj, k, v)
  799. else:
  800. # Assume we can coerce the type
  801. newobj = origtype(obj)
  802. if isinstance(newobj, bb.command.DataStoreConnectionHandle):
  803. connector = TinfoilDataStoreConnector(self, newobj.dsindex)
  804. newobj = bb.data.init()
  805. newobj.setVar('_remote_data', connector)
  806. return newobj
  807. class TinfoilConfigParameters(BitBakeConfigParameters):
  808. def __init__(self, config_only, **options):
  809. self.initial_options = options
  810. # Apply some sane defaults
  811. if not 'parse_only' in options:
  812. self.initial_options['parse_only'] = not config_only
  813. #if not 'status_only' in options:
  814. # self.initial_options['status_only'] = config_only
  815. if not 'ui' in options:
  816. self.initial_options['ui'] = 'knotty'
  817. if not 'argv' in options:
  818. self.initial_options['argv'] = []
  819. super(TinfoilConfigParameters, self).__init__()
  820. def parseCommandLine(self, argv=None):
  821. # We don't want any parameters parsed from the command line
  822. opts = super(TinfoilConfigParameters, self).parseCommandLine([])
  823. for key, val in self.initial_options.items():
  824. setattr(opts[0], key, val)
  825. return opts