cookerdata.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440
  1. #!/usr/bin/env python
  2. # ex:ts=4:sw=4:sts=4:et
  3. # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
  4. #
  5. # Copyright (C) 2003, 2004 Chris Larson
  6. # Copyright (C) 2003, 2004 Phil Blundell
  7. # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
  8. # Copyright (C) 2005 Holger Hans Peter Freyther
  9. # Copyright (C) 2005 ROAD GmbH
  10. # Copyright (C) 2006 Richard Purdie
  11. #
  12. # SPDX-License-Identifier: GPL-2.0-only
  13. #
  14. # This program is free software; you can redistribute it and/or modify
  15. # it under the terms of the GNU General Public License version 2 as
  16. # published by the Free Software Foundation.
  17. #
  18. # This program is distributed in the hope that it will be useful,
  19. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  20. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  21. # GNU General Public License for more details.
  22. #
  23. # You should have received a copy of the GNU General Public License along
  24. # with this program; if not, write to the Free Software Foundation, Inc.,
  25. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  26. import logging
  27. import os
  28. import re
  29. import sys
  30. from functools import wraps
  31. import bb
  32. from bb import data
  33. import bb.parse
  34. logger = logging.getLogger("BitBake")
  35. parselog = logging.getLogger("BitBake.Parsing")
  36. class ConfigParameters(object):
  37. def __init__(self, argv=sys.argv):
  38. self.options, targets = self.parseCommandLine(argv)
  39. self.environment = self.parseEnvironment()
  40. self.options.pkgs_to_build = targets or []
  41. for key, val in self.options.__dict__.items():
  42. setattr(self, key, val)
  43. def parseCommandLine(self, argv=sys.argv):
  44. raise Exception("Caller must implement commandline option parsing")
  45. def parseEnvironment(self):
  46. return os.environ.copy()
  47. def updateFromServer(self, server):
  48. if not self.options.cmd:
  49. defaulttask, error = server.runCommand(["getVariable", "BB_DEFAULT_TASK"])
  50. if error:
  51. raise Exception("Unable to get the value of BB_DEFAULT_TASK from the server: %s" % error)
  52. self.options.cmd = defaulttask or "build"
  53. _, error = server.runCommand(["setConfig", "cmd", self.options.cmd])
  54. if error:
  55. raise Exception("Unable to set configuration option 'cmd' on the server: %s" % error)
  56. if not self.options.pkgs_to_build:
  57. bbpkgs, error = server.runCommand(["getVariable", "BBTARGETS"])
  58. if error:
  59. raise Exception("Unable to get the value of BBTARGETS from the server: %s" % error)
  60. if bbpkgs:
  61. self.options.pkgs_to_build.extend(bbpkgs.split())
  62. def updateToServer(self, server, environment):
  63. options = {}
  64. for o in ["abort", "force", "invalidate_stamp",
  65. "verbose", "debug", "dry_run", "dump_signatures",
  66. "debug_domains", "extra_assume_provided", "profile",
  67. "prefile", "postfile", "server_timeout"]:
  68. options[o] = getattr(self.options, o)
  69. ret, error = server.runCommand(["updateConfig", options, environment, sys.argv])
  70. if error:
  71. raise Exception("Unable to update the server configuration with local parameters: %s" % error)
  72. def parseActions(self):
  73. # Parse any commandline into actions
  74. action = {'action':None, 'msg':None}
  75. if self.options.show_environment:
  76. if 'world' in self.options.pkgs_to_build:
  77. action['msg'] = "'world' is not a valid target for --environment."
  78. elif 'universe' in self.options.pkgs_to_build:
  79. action['msg'] = "'universe' is not a valid target for --environment."
  80. elif len(self.options.pkgs_to_build) > 1:
  81. action['msg'] = "Only one target can be used with the --environment option."
  82. elif self.options.buildfile and len(self.options.pkgs_to_build) > 0:
  83. action['msg'] = "No target should be used with the --environment and --buildfile options."
  84. elif len(self.options.pkgs_to_build) > 0:
  85. action['action'] = ["showEnvironmentTarget", self.options.pkgs_to_build]
  86. else:
  87. action['action'] = ["showEnvironment", self.options.buildfile]
  88. elif self.options.buildfile is not None:
  89. action['action'] = ["buildFile", self.options.buildfile, self.options.cmd]
  90. elif self.options.revisions_changed:
  91. action['action'] = ["compareRevisions"]
  92. elif self.options.show_versions:
  93. action['action'] = ["showVersions"]
  94. elif self.options.parse_only:
  95. action['action'] = ["parseFiles"]
  96. elif self.options.dot_graph:
  97. if self.options.pkgs_to_build:
  98. action['action'] = ["generateDotGraph", self.options.pkgs_to_build, self.options.cmd]
  99. else:
  100. action['msg'] = "Please specify a package name for dependency graph generation."
  101. else:
  102. if self.options.pkgs_to_build:
  103. action['action'] = ["buildTargets", self.options.pkgs_to_build, self.options.cmd]
  104. else:
  105. #action['msg'] = "Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information."
  106. action = None
  107. self.options.initialaction = action
  108. return action
  109. class CookerConfiguration(object):
  110. """
  111. Manages build options and configurations for one run
  112. """
  113. def __init__(self):
  114. self.debug_domains = []
  115. self.extra_assume_provided = []
  116. self.prefile = []
  117. self.postfile = []
  118. self.debug = 0
  119. self.cmd = None
  120. self.abort = True
  121. self.force = False
  122. self.profile = False
  123. self.nosetscene = False
  124. self.setsceneonly = False
  125. self.invalidate_stamp = False
  126. self.dump_signatures = []
  127. self.dry_run = False
  128. self.tracking = False
  129. self.xmlrpcinterface = []
  130. self.server_timeout = None
  131. self.writeeventlog = False
  132. self.server_only = False
  133. self.limited_deps = False
  134. self.runall = []
  135. self.runonly = []
  136. self.env = {}
  137. def setConfigParameters(self, parameters):
  138. for key in self.__dict__.keys():
  139. if key in parameters.options.__dict__:
  140. setattr(self, key, parameters.options.__dict__[key])
  141. self.env = parameters.environment.copy()
  142. def setServerRegIdleCallback(self, srcb):
  143. self.server_register_idlecallback = srcb
  144. def __getstate__(self):
  145. state = {}
  146. for key in self.__dict__.keys():
  147. if key == "server_register_idlecallback":
  148. state[key] = None
  149. else:
  150. state[key] = getattr(self, key)
  151. return state
  152. def __setstate__(self,state):
  153. for k in state:
  154. setattr(self, k, state[k])
  155. def catch_parse_error(func):
  156. """Exception handling bits for our parsing"""
  157. @wraps(func)
  158. def wrapped(fn, *args):
  159. try:
  160. return func(fn, *args)
  161. except IOError as exc:
  162. import traceback
  163. parselog.critical(traceback.format_exc())
  164. parselog.critical("Unable to parse %s: %s" % (fn, exc))
  165. sys.exit(1)
  166. except bb.data_smart.ExpansionError as exc:
  167. import traceback
  168. bbdir = os.path.dirname(__file__) + os.sep
  169. exc_class, exc, tb = sys.exc_info()
  170. for tb in iter(lambda: tb.tb_next, None):
  171. # Skip frames in bitbake itself, we only want the metadata
  172. fn, _, _, _ = traceback.extract_tb(tb, 1)[0]
  173. if not fn.startswith(bbdir):
  174. break
  175. parselog.critical("Unable to parse %s" % fn, exc_info=(exc_class, exc, tb))
  176. sys.exit(1)
  177. except bb.parse.ParseError as exc:
  178. parselog.critical(str(exc))
  179. sys.exit(1)
  180. return wrapped
  181. @catch_parse_error
  182. def parse_config_file(fn, data, include=True):
  183. return bb.parse.handle(fn, data, include)
  184. @catch_parse_error
  185. def _inherit(bbclass, data):
  186. bb.parse.BBHandler.inherit(bbclass, "configuration INHERITs", 0, data)
  187. return data
  188. def findConfigFile(configfile, data):
  189. search = []
  190. bbpath = data.getVar("BBPATH")
  191. if bbpath:
  192. for i in bbpath.split(":"):
  193. search.append(os.path.join(i, "conf", configfile))
  194. path = os.getcwd()
  195. while path != "/":
  196. search.append(os.path.join(path, "conf", configfile))
  197. path, _ = os.path.split(path)
  198. for i in search:
  199. if os.path.exists(i):
  200. return i
  201. return None
  202. #
  203. # We search for a conf/bblayers.conf under an entry in BBPATH or in cwd working
  204. # up to /. If that fails, we search for a conf/bitbake.conf in BBPATH.
  205. #
  206. def findTopdir():
  207. d = bb.data.init()
  208. bbpath = None
  209. if 'BBPATH' in os.environ:
  210. bbpath = os.environ['BBPATH']
  211. d.setVar('BBPATH', bbpath)
  212. layerconf = findConfigFile("bblayers.conf", d)
  213. if layerconf:
  214. return os.path.dirname(os.path.dirname(layerconf))
  215. if bbpath:
  216. bitbakeconf = bb.utils.which(bbpath, "conf/bitbake.conf")
  217. if bitbakeconf:
  218. return os.path.dirname(os.path.dirname(bitbakeconf))
  219. return None
  220. class CookerDataBuilder(object):
  221. def __init__(self, cookercfg, worker = False):
  222. self.prefiles = cookercfg.prefile
  223. self.postfiles = cookercfg.postfile
  224. self.tracking = cookercfg.tracking
  225. bb.utils.set_context(bb.utils.clean_context())
  226. bb.event.set_class_handlers(bb.event.clean_class_handlers())
  227. self.basedata = bb.data.init()
  228. if self.tracking:
  229. self.basedata.enableTracking()
  230. # Keep a datastore of the initial environment variables and their
  231. # values from when BitBake was launched to enable child processes
  232. # to use environment variables which have been cleaned from the
  233. # BitBake processes env
  234. self.savedenv = bb.data.init()
  235. for k in cookercfg.env:
  236. self.savedenv.setVar(k, cookercfg.env[k])
  237. filtered_keys = bb.utils.approved_variables()
  238. bb.data.inheritFromOS(self.basedata, self.savedenv, filtered_keys)
  239. self.basedata.setVar("BB_ORIGENV", self.savedenv)
  240. if worker:
  241. self.basedata.setVar("BB_WORKERCONTEXT", "1")
  242. self.data = self.basedata
  243. self.mcdata = {}
  244. def parseBaseConfiguration(self):
  245. try:
  246. bb.parse.init_parser(self.basedata)
  247. self.data = self.parseConfigurationFiles(self.prefiles, self.postfiles)
  248. if self.data.getVar("BB_WORKERCONTEXT", False) is None:
  249. bb.fetch.fetcher_init(self.data)
  250. bb.codeparser.parser_cache_init(self.data)
  251. bb.event.fire(bb.event.ConfigParsed(), self.data)
  252. reparse_cnt = 0
  253. while self.data.getVar("BB_INVALIDCONF", False) is True:
  254. if reparse_cnt > 20:
  255. logger.error("Configuration has been re-parsed over 20 times, "
  256. "breaking out of the loop...")
  257. raise Exception("Too deep config re-parse loop. Check locations where "
  258. "BB_INVALIDCONF is being set (ConfigParsed event handlers)")
  259. self.data.setVar("BB_INVALIDCONF", False)
  260. self.data = self.parseConfigurationFiles(self.prefiles, self.postfiles)
  261. reparse_cnt += 1
  262. bb.event.fire(bb.event.ConfigParsed(), self.data)
  263. bb.parse.init_parser(self.data)
  264. self.data_hash = self.data.get_hash()
  265. self.mcdata[''] = self.data
  266. multiconfig = (self.data.getVar("BBMULTICONFIG") or "").split()
  267. for config in multiconfig:
  268. mcdata = self.parseConfigurationFiles(self.prefiles, self.postfiles, config)
  269. bb.event.fire(bb.event.ConfigParsed(), mcdata)
  270. self.mcdata[config] = mcdata
  271. if multiconfig:
  272. bb.event.fire(bb.event.MultiConfigParsed(self.mcdata), self.data)
  273. except (SyntaxError, bb.BBHandledException):
  274. raise bb.BBHandledException
  275. except bb.data_smart.ExpansionError as e:
  276. logger.error(str(e))
  277. raise bb.BBHandledException
  278. except Exception:
  279. logger.exception("Error parsing configuration files")
  280. raise bb.BBHandledException
  281. # Create a copy so we can reset at a later date when UIs disconnect
  282. self.origdata = self.data
  283. self.data = bb.data.createCopy(self.origdata)
  284. self.mcdata[''] = self.data
  285. def reset(self):
  286. # We may not have run parseBaseConfiguration() yet
  287. if not hasattr(self, 'origdata'):
  288. return
  289. self.data = bb.data.createCopy(self.origdata)
  290. self.mcdata[''] = self.data
  291. def _findLayerConf(self, data):
  292. return findConfigFile("bblayers.conf", data)
  293. def parseConfigurationFiles(self, prefiles, postfiles, mc = "default"):
  294. data = bb.data.createCopy(self.basedata)
  295. data.setVar("BB_CURRENT_MC", mc)
  296. # Parse files for loading *before* bitbake.conf and any includes
  297. for f in prefiles:
  298. data = parse_config_file(f, data)
  299. layerconf = self._findLayerConf(data)
  300. if layerconf:
  301. parselog.debug(2, "Found bblayers.conf (%s)", layerconf)
  302. # By definition bblayers.conf is in conf/ of TOPDIR.
  303. # We may have been called with cwd somewhere else so reset TOPDIR
  304. data.setVar("TOPDIR", os.path.dirname(os.path.dirname(layerconf)))
  305. data = parse_config_file(layerconf, data)
  306. layers = (data.getVar('BBLAYERS') or "").split()
  307. data = bb.data.createCopy(data)
  308. approved = bb.utils.approved_variables()
  309. for layer in layers:
  310. if not os.path.isdir(layer):
  311. parselog.critical("Layer directory '%s' does not exist! "
  312. "Please check BBLAYERS in %s" % (layer, layerconf))
  313. sys.exit(1)
  314. parselog.debug(2, "Adding layer %s", layer)
  315. if 'HOME' in approved and '~' in layer:
  316. layer = os.path.expanduser(layer)
  317. if layer.endswith('/'):
  318. layer = layer.rstrip('/')
  319. data.setVar('LAYERDIR', layer)
  320. data.setVar('LAYERDIR_RE', re.escape(layer))
  321. data = parse_config_file(os.path.join(layer, "conf", "layer.conf"), data)
  322. data.expandVarref('LAYERDIR')
  323. data.expandVarref('LAYERDIR_RE')
  324. data.delVar('LAYERDIR_RE')
  325. data.delVar('LAYERDIR')
  326. bbfiles_dynamic = (data.getVar('BBFILES_DYNAMIC') or "").split()
  327. collections = (data.getVar('BBFILE_COLLECTIONS') or "").split()
  328. invalid = []
  329. for entry in bbfiles_dynamic:
  330. parts = entry.split(":", 1)
  331. if len(parts) != 2:
  332. invalid.append(entry)
  333. continue
  334. l, f = parts
  335. if l in collections:
  336. data.appendVar("BBFILES", " " + f)
  337. if invalid:
  338. bb.fatal("BBFILES_DYNAMIC entries must be of the form <collection name>:<filename pattern>, not:\n %s" % "\n ".join(invalid))
  339. layerseries = set((data.getVar("LAYERSERIES_CORENAMES") or "").split())
  340. collections_tmp = collections[:]
  341. for c in collections:
  342. collections_tmp.remove(c)
  343. if c in collections_tmp:
  344. bb.fatal("Found duplicated BBFILE_COLLECTIONS '%s', check bblayers.conf or layer.conf to fix it." % c)
  345. compat = set((data.getVar("LAYERSERIES_COMPAT_%s" % c) or "").split())
  346. if compat and not (compat & layerseries):
  347. bb.fatal("Layer %s is not compatible with the core layer which only supports these series: %s (layer is compatible with %s)"
  348. % (c, " ".join(layerseries), " ".join(compat)))
  349. elif not compat and not data.getVar("BB_WORKERCONTEXT"):
  350. bb.warn("Layer %s should set LAYERSERIES_COMPAT_%s in its conf/layer.conf file to list the core layer names it is compatible with." % (c, c))
  351. if not data.getVar("BBPATH"):
  352. msg = "The BBPATH variable is not set"
  353. if not layerconf:
  354. msg += (" and bitbake did not find a conf/bblayers.conf file in"
  355. " the expected location.\nMaybe you accidentally"
  356. " invoked bitbake from the wrong directory?")
  357. raise SystemExit(msg)
  358. data = parse_config_file(os.path.join("conf", "bitbake.conf"), data)
  359. # Parse files for loading *after* bitbake.conf and any includes
  360. for p in postfiles:
  361. data = parse_config_file(p, data)
  362. # Handle any INHERITs and inherit the base class
  363. bbclasses = ["base"] + (data.getVar('INHERIT') or "").split()
  364. for bbclass in bbclasses:
  365. data = _inherit(bbclass, data)
  366. # Nomally we only register event handlers at the end of parsing .bb files
  367. # We register any handlers we've found so far here...
  368. for var in data.getVar('__BBHANDLERS', False) or []:
  369. handlerfn = data.getVarFlag(var, "filename", False)
  370. if not handlerfn:
  371. parselog.critical("Undefined event handler function '%s'" % var)
  372. sys.exit(1)
  373. handlerln = int(data.getVarFlag(var, "lineno", False))
  374. bb.event.register(var, data.getVar(var, False), (data.getVarFlag(var, "eventmask") or "").split(), handlerfn, handlerln)
  375. data.setVar('BBINCLUDED',bb.parse.get_file_depends(data))
  376. return data