main.py 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512
  1. #!/usr/bin/env python
  2. # ex:ts=4:sw=4:sts=4:et
  3. # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
  4. #
  5. # Copyright (C) 2003, 2004 Chris Larson
  6. # Copyright (C) 2003, 2004 Phil Blundell
  7. # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
  8. # Copyright (C) 2005 Holger Hans Peter Freyther
  9. # Copyright (C) 2005 ROAD GmbH
  10. # Copyright (C) 2006 Richard Purdie
  11. #
  12. # SPDX-License-Identifier: GPL-2.0-only
  13. #
  14. # This program is free software; you can redistribute it and/or modify
  15. # it under the terms of the GNU General Public License version 2 as
  16. # published by the Free Software Foundation.
  17. #
  18. # This program is distributed in the hope that it will be useful,
  19. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  20. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  21. # GNU General Public License for more details.
  22. #
  23. # You should have received a copy of the GNU General Public License along
  24. # with this program; if not, write to the Free Software Foundation, Inc.,
  25. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  26. import os
  27. import sys
  28. import logging
  29. import optparse
  30. import warnings
  31. import fcntl
  32. import time
  33. import traceback
  34. import bb
  35. from bb import event
  36. import bb.msg
  37. from bb import cooker
  38. from bb import ui
  39. from bb import server
  40. from bb import cookerdata
  41. import bb.server.process
  42. import bb.server.xmlrpcclient
  43. logger = logging.getLogger("BitBake")
  44. class BBMainException(Exception):
  45. pass
  46. class BBMainFatal(bb.BBHandledException):
  47. pass
  48. def present_options(optionlist):
  49. if len(optionlist) > 1:
  50. return ' or '.join([', '.join(optionlist[:-1]), optionlist[-1]])
  51. else:
  52. return optionlist[0]
  53. class BitbakeHelpFormatter(optparse.IndentedHelpFormatter):
  54. def format_option(self, option):
  55. # We need to do this here rather than in the text we supply to
  56. # add_option() because we don't want to call list_extension_modules()
  57. # on every execution (since it imports all of the modules)
  58. # Note also that we modify option.help rather than the returned text
  59. # - this is so that we don't have to re-format the text ourselves
  60. if option.dest == 'ui':
  61. valid_uis = list_extension_modules(bb.ui, 'main')
  62. option.help = option.help.replace('@CHOICES@', present_options(valid_uis))
  63. return optparse.IndentedHelpFormatter.format_option(self, option)
  64. def list_extension_modules(pkg, checkattr):
  65. """
  66. Lists extension modules in a specific Python package
  67. (e.g. UIs, servers). NOTE: Calling this function will import all of the
  68. submodules of the specified module in order to check for the specified
  69. attribute; this can have unusual side-effects. As a result, this should
  70. only be called when displaying help text or error messages.
  71. Parameters:
  72. pkg: previously imported Python package to list
  73. checkattr: attribute to look for in module to determine if it's valid
  74. as the type of extension you are looking for
  75. """
  76. import pkgutil
  77. pkgdir = os.path.dirname(pkg.__file__)
  78. modules = []
  79. for _, modulename, _ in pkgutil.iter_modules([pkgdir]):
  80. if os.path.isdir(os.path.join(pkgdir, modulename)):
  81. # ignore directories
  82. continue
  83. try:
  84. module = __import__(pkg.__name__, fromlist=[modulename])
  85. except:
  86. # If we can't import it, it's not valid
  87. continue
  88. module_if = getattr(module, modulename)
  89. if getattr(module_if, 'hidden_extension', False):
  90. continue
  91. if not checkattr or hasattr(module_if, checkattr):
  92. modules.append(modulename)
  93. return modules
  94. def import_extension_module(pkg, modulename, checkattr):
  95. try:
  96. # Dynamically load the UI based on the ui name. Although we
  97. # suggest a fixed set this allows you to have flexibility in which
  98. # ones are available.
  99. module = __import__(pkg.__name__, fromlist=[modulename])
  100. return getattr(module, modulename)
  101. except AttributeError:
  102. modules = present_options(list_extension_modules(pkg, checkattr))
  103. raise BBMainException('FATAL: Unable to import extension module "%s" from %s. '
  104. 'Valid extension modules: %s' % (modulename, pkg.__name__, modules))
  105. # Display bitbake/OE warnings via the BitBake.Warnings logger, ignoring others"""
  106. warnlog = logging.getLogger("BitBake.Warnings")
  107. _warnings_showwarning = warnings.showwarning
  108. def _showwarning(message, category, filename, lineno, file=None, line=None):
  109. if file is not None:
  110. if _warnings_showwarning is not None:
  111. _warnings_showwarning(message, category, filename, lineno, file, line)
  112. else:
  113. s = warnings.formatwarning(message, category, filename, lineno)
  114. warnlog.warning(s)
  115. warnings.showwarning = _showwarning
  116. warnings.filterwarnings("ignore")
  117. warnings.filterwarnings("default", module="(<string>$|(oe|bb)\.)")
  118. warnings.filterwarnings("ignore", category=PendingDeprecationWarning)
  119. warnings.filterwarnings("ignore", category=ImportWarning)
  120. warnings.filterwarnings("ignore", category=DeprecationWarning, module="<string>$")
  121. warnings.filterwarnings("ignore", message="With-statements now directly support multiple context managers")
  122. class BitBakeConfigParameters(cookerdata.ConfigParameters):
  123. def parseCommandLine(self, argv=sys.argv):
  124. parser = optparse.OptionParser(
  125. formatter=BitbakeHelpFormatter(),
  126. version="BitBake Build Tool Core version %s" % bb.__version__,
  127. usage="""%prog [options] [recipename/target recipe:do_task ...]
  128. Executes the specified task (default is 'build') for a given set of target recipes (.bb files).
  129. It is assumed there is a conf/bblayers.conf available in cwd or in BBPATH which
  130. will provide the layer, BBFILES and other configuration information.""")
  131. parser.add_option("-b", "--buildfile", action="store", dest="buildfile", default=None,
  132. help="Execute tasks from a specific .bb recipe directly. WARNING: Does "
  133. "not handle any dependencies from other recipes.")
  134. parser.add_option("-k", "--continue", action="store_false", dest="abort", default=True,
  135. help="Continue as much as possible after an error. While the target that "
  136. "failed and anything depending on it cannot be built, as much as "
  137. "possible will be built before stopping.")
  138. parser.add_option("-f", "--force", action="store_true", dest="force", default=False,
  139. help="Force the specified targets/task to run (invalidating any "
  140. "existing stamp file).")
  141. parser.add_option("-c", "--cmd", action="store", dest="cmd",
  142. help="Specify the task to execute. The exact options available "
  143. "depend on the metadata. Some examples might be 'compile'"
  144. " or 'populate_sysroot' or 'listtasks' may give a list of "
  145. "the tasks available.")
  146. parser.add_option("-C", "--clear-stamp", action="store", dest="invalidate_stamp",
  147. help="Invalidate the stamp for the specified task such as 'compile' "
  148. "and then run the default task for the specified target(s).")
  149. parser.add_option("-r", "--read", action="append", dest="prefile", default=[],
  150. help="Read the specified file before bitbake.conf.")
  151. parser.add_option("-R", "--postread", action="append", dest="postfile", default=[],
  152. help="Read the specified file after bitbake.conf.")
  153. parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False,
  154. help="Enable tracing of shell tasks (with 'set -x'). "
  155. "Also print bb.note(...) messages to stdout (in "
  156. "addition to writing them to ${T}/log.do_<task>).")
  157. parser.add_option("-D", "--debug", action="count", dest="debug", default=0,
  158. help="Increase the debug level. You can specify this "
  159. "more than once. -D sets the debug level to 1, "
  160. "where only bb.debug(1, ...) messages are printed "
  161. "to stdout; -DD sets the debug level to 2, where "
  162. "both bb.debug(1, ...) and bb.debug(2, ...) "
  163. "messages are printed; etc. Without -D, no debug "
  164. "messages are printed. Note that -D only affects "
  165. "output to stdout. All debug messages are written "
  166. "to ${T}/log.do_taskname, regardless of the debug "
  167. "level.")
  168. parser.add_option("-q", "--quiet", action="count", dest="quiet", default=0,
  169. help="Output less log message data to the terminal. You can specify this more than once.")
  170. parser.add_option("-n", "--dry-run", action="store_true", dest="dry_run", default=False,
  171. help="Don't execute, just go through the motions.")
  172. parser.add_option("-S", "--dump-signatures", action="append", dest="dump_signatures",
  173. default=[], metavar="SIGNATURE_HANDLER",
  174. help="Dump out the signature construction information, with no task "
  175. "execution. The SIGNATURE_HANDLER parameter is passed to the "
  176. "handler. Two common values are none and printdiff but the handler "
  177. "may define more/less. none means only dump the signature, printdiff"
  178. " means compare the dumped signature with the cached one.")
  179. parser.add_option("-p", "--parse-only", action="store_true",
  180. dest="parse_only", default=False,
  181. help="Quit after parsing the BB recipes.")
  182. parser.add_option("-s", "--show-versions", action="store_true",
  183. dest="show_versions", default=False,
  184. help="Show current and preferred versions of all recipes.")
  185. parser.add_option("-e", "--environment", action="store_true",
  186. dest="show_environment", default=False,
  187. help="Show the global or per-recipe environment complete with information"
  188. " about where variables were set/changed.")
  189. parser.add_option("-g", "--graphviz", action="store_true", dest="dot_graph", default=False,
  190. help="Save dependency tree information for the specified "
  191. "targets in the dot syntax.")
  192. parser.add_option("-I", "--ignore-deps", action="append",
  193. dest="extra_assume_provided", default=[],
  194. help="Assume these dependencies don't exist and are already provided "
  195. "(equivalent to ASSUME_PROVIDED). Useful to make dependency "
  196. "graphs more appealing")
  197. parser.add_option("-l", "--log-domains", action="append", dest="debug_domains", default=[],
  198. help="Show debug logging for the specified logging domains")
  199. parser.add_option("-P", "--profile", action="store_true", dest="profile", default=False,
  200. help="Profile the command and save reports.")
  201. # @CHOICES@ is substituted out by BitbakeHelpFormatter above
  202. parser.add_option("-u", "--ui", action="store", dest="ui",
  203. default=os.environ.get('BITBAKE_UI', 'knotty'),
  204. help="The user interface to use (@CHOICES@ - default %default).")
  205. parser.add_option("", "--token", action="store", dest="xmlrpctoken",
  206. default=os.environ.get("BBTOKEN"),
  207. help="Specify the connection token to be used when connecting "
  208. "to a remote server.")
  209. parser.add_option("", "--revisions-changed", action="store_true",
  210. dest="revisions_changed", default=False,
  211. help="Set the exit code depending on whether upstream floating "
  212. "revisions have changed or not.")
  213. parser.add_option("", "--server-only", action="store_true",
  214. dest="server_only", default=False,
  215. help="Run bitbake without a UI, only starting a server "
  216. "(cooker) process.")
  217. parser.add_option("-B", "--bind", action="store", dest="bind", default=False,
  218. help="The name/address for the bitbake xmlrpc server to bind to.")
  219. parser.add_option("-T", "--idle-timeout", type=float, dest="server_timeout",
  220. default=os.getenv("BB_SERVER_TIMEOUT"),
  221. help="Set timeout to unload bitbake server due to inactivity, "
  222. "set to -1 means no unload, "
  223. "default: Environment variable BB_SERVER_TIMEOUT.")
  224. parser.add_option("", "--no-setscene", action="store_true",
  225. dest="nosetscene", default=False,
  226. help="Do not run any setscene tasks. sstate will be ignored and "
  227. "everything needed, built.")
  228. parser.add_option("", "--setscene-only", action="store_true",
  229. dest="setsceneonly", default=False,
  230. help="Only run setscene tasks, don't run any real tasks.")
  231. parser.add_option("", "--remote-server", action="store", dest="remote_server",
  232. default=os.environ.get("BBSERVER"),
  233. help="Connect to the specified server.")
  234. parser.add_option("-m", "--kill-server", action="store_true",
  235. dest="kill_server", default=False,
  236. help="Terminate any running bitbake server.")
  237. parser.add_option("", "--observe-only", action="store_true",
  238. dest="observe_only", default=False,
  239. help="Connect to a server as an observing-only client.")
  240. parser.add_option("", "--status-only", action="store_true",
  241. dest="status_only", default=False,
  242. help="Check the status of the remote bitbake server.")
  243. parser.add_option("-w", "--write-log", action="store", dest="writeeventlog",
  244. default=os.environ.get("BBEVENTLOG"),
  245. help="Writes the event log of the build to a bitbake event json file. "
  246. "Use '' (empty string) to assign the name automatically.")
  247. parser.add_option("", "--runall", action="append", dest="runall",
  248. help="Run the specified task for any recipe in the taskgraph of the specified target (even if it wouldn't otherwise have run).")
  249. parser.add_option("", "--runonly", action="append", dest="runonly",
  250. help="Run only the specified task within the taskgraph of the specified targets (and any task dependencies those tasks may have).")
  251. options, targets = parser.parse_args(argv)
  252. if options.quiet and options.verbose:
  253. parser.error("options --quiet and --verbose are mutually exclusive")
  254. if options.quiet and options.debug:
  255. parser.error("options --quiet and --debug are mutually exclusive")
  256. # use configuration files from environment variables
  257. if "BBPRECONF" in os.environ:
  258. options.prefile.append(os.environ["BBPRECONF"])
  259. if "BBPOSTCONF" in os.environ:
  260. options.postfile.append(os.environ["BBPOSTCONF"])
  261. # fill in proper log name if not supplied
  262. if options.writeeventlog is not None and len(options.writeeventlog) == 0:
  263. from datetime import datetime
  264. eventlog = "bitbake_eventlog_%s.json" % datetime.now().strftime("%Y%m%d%H%M%S")
  265. options.writeeventlog = eventlog
  266. if options.bind:
  267. try:
  268. #Checking that the port is a number and is a ':' delimited value
  269. (host, port) = options.bind.split(':')
  270. port = int(port)
  271. except (ValueError,IndexError):
  272. raise BBMainException("FATAL: Malformed host:port bind parameter")
  273. options.xmlrpcinterface = (host, port)
  274. else:
  275. options.xmlrpcinterface = (None, 0)
  276. return options, targets[1:]
  277. def bitbake_main(configParams, configuration):
  278. # Python multiprocessing requires /dev/shm on Linux
  279. if sys.platform.startswith('linux') and not os.access('/dev/shm', os.W_OK | os.X_OK):
  280. raise BBMainException("FATAL: /dev/shm does not exist or is not writable")
  281. # Unbuffer stdout to avoid log truncation in the event
  282. # of an unorderly exit as well as to provide timely
  283. # updates to log files for use with tail
  284. try:
  285. if sys.stdout.name == '<stdout>':
  286. # Reopen with O_SYNC (unbuffered)
  287. fl = fcntl.fcntl(sys.stdout.fileno(), fcntl.F_GETFL)
  288. fl |= os.O_SYNC
  289. fcntl.fcntl(sys.stdout.fileno(), fcntl.F_SETFL, fl)
  290. except:
  291. pass
  292. configuration.setConfigParameters(configParams)
  293. if configParams.server_only and configParams.remote_server:
  294. raise BBMainException("FATAL: The '--server-only' option conflicts with %s.\n" %
  295. ("the BBSERVER environment variable" if "BBSERVER" in os.environ \
  296. else "the '--remote-server' option"))
  297. if configParams.observe_only and not (configParams.remote_server or configParams.bind):
  298. raise BBMainException("FATAL: '--observe-only' can only be used by UI clients "
  299. "connecting to a server.\n")
  300. if "BBDEBUG" in os.environ:
  301. level = int(os.environ["BBDEBUG"])
  302. if level > configuration.debug:
  303. configuration.debug = level
  304. bb.msg.init_msgconfig(configParams.verbose, configuration.debug,
  305. configuration.debug_domains)
  306. server_connection, ui_module = setup_bitbake(configParams, configuration)
  307. # No server connection
  308. if server_connection is None:
  309. if configParams.status_only:
  310. return 1
  311. if configParams.kill_server:
  312. return 0
  313. if not configParams.server_only:
  314. if configParams.status_only:
  315. server_connection.terminate()
  316. return 0
  317. try:
  318. for event in bb.event.ui_queue:
  319. server_connection.events.queue_event(event)
  320. bb.event.ui_queue = []
  321. return ui_module.main(server_connection.connection, server_connection.events,
  322. configParams)
  323. finally:
  324. server_connection.terminate()
  325. else:
  326. return 0
  327. return 1
  328. def setup_bitbake(configParams, configuration, extrafeatures=None):
  329. # Ensure logging messages get sent to the UI as events
  330. handler = bb.event.LogHandler()
  331. if not configParams.status_only:
  332. # In status only mode there are no logs and no UI
  333. logger.addHandler(handler)
  334. if configParams.server_only:
  335. featureset = []
  336. ui_module = None
  337. else:
  338. ui_module = import_extension_module(bb.ui, configParams.ui, 'main')
  339. # Collect the feature set for the UI
  340. featureset = getattr(ui_module, "featureSet", [])
  341. if extrafeatures:
  342. for feature in extrafeatures:
  343. if not feature in featureset:
  344. featureset.append(feature)
  345. server_connection = None
  346. # Clear away any spurious environment variables while we stoke up the cooker
  347. # (done after import_extension_module() above since for example import gi triggers env var usage)
  348. cleanedvars = bb.utils.clean_environment()
  349. if configParams.remote_server:
  350. # Connect to a remote XMLRPC server
  351. server_connection = bb.server.xmlrpcclient.connectXMLRPC(configParams.remote_server, featureset,
  352. configParams.observe_only, configParams.xmlrpctoken)
  353. else:
  354. retries = 8
  355. while retries:
  356. try:
  357. topdir, lock = lockBitbake()
  358. sockname = topdir + "/bitbake.sock"
  359. if lock:
  360. if configParams.status_only or configParams.kill_server:
  361. logger.info("bitbake server is not running.")
  362. lock.close()
  363. return None, None
  364. # we start a server with a given configuration
  365. logger.info("Starting bitbake server...")
  366. # Clear the event queue since we already displayed messages
  367. bb.event.ui_queue = []
  368. server = bb.server.process.BitBakeServer(lock, sockname, configuration, featureset)
  369. else:
  370. logger.info("Reconnecting to bitbake server...")
  371. if not os.path.exists(sockname):
  372. logger.info("Previous bitbake instance shutting down?, waiting to retry...")
  373. i = 0
  374. lock = None
  375. # Wait for 5s or until we can get the lock
  376. while not lock and i < 50:
  377. time.sleep(0.1)
  378. _, lock = lockBitbake()
  379. i += 1
  380. if lock:
  381. bb.utils.unlockfile(lock)
  382. raise bb.server.process.ProcessTimeout("Bitbake still shutting down as socket exists but no lock?")
  383. if not configParams.server_only:
  384. try:
  385. server_connection = bb.server.process.connectProcessServer(sockname, featureset)
  386. except EOFError:
  387. # The server may have been shutting down but not closed the socket yet. If that happened,
  388. # ignore it.
  389. pass
  390. if server_connection or configParams.server_only:
  391. break
  392. except BBMainFatal:
  393. raise
  394. except (Exception, bb.server.process.ProcessTimeout) as e:
  395. if not retries:
  396. raise
  397. retries -= 1
  398. tryno = 8 - retries
  399. if isinstance(e, (bb.server.process.ProcessTimeout, BrokenPipeError)):
  400. logger.info("Retrying server connection (#%d)..." % tryno)
  401. else:
  402. logger.info("Retrying server connection (#%d)... (%s)" % (tryno, traceback.format_exc()))
  403. if not retries:
  404. bb.fatal("Unable to connect to bitbake server, or start one")
  405. if retries < 5:
  406. time.sleep(5)
  407. if configParams.kill_server:
  408. server_connection.connection.terminateServer()
  409. server_connection.terminate()
  410. bb.event.ui_queue = []
  411. logger.info("Terminated bitbake server.")
  412. return None, None
  413. # Restore the environment in case the UI needs it
  414. for k in cleanedvars:
  415. os.environ[k] = cleanedvars[k]
  416. logger.removeHandler(handler)
  417. return server_connection, ui_module
  418. def lockBitbake():
  419. topdir = bb.cookerdata.findTopdir()
  420. if not topdir:
  421. bb.error("Unable to find conf/bblayers.conf or conf/bitbake.conf. BBAPTH is unset and/or not in a build directory?")
  422. raise BBMainFatal
  423. lockfile = topdir + "/bitbake.lock"
  424. return topdir, bb.utils.lockfile(lockfile, False, False)