knotty.py 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601
  1. #
  2. # BitBake (No)TTY UI Implementation
  3. #
  4. # Handling output to TTYs or files (no TTY)
  5. #
  6. # Copyright (C) 2006-2012 Richard Purdie
  7. #
  8. # This program is free software; you can redistribute it and/or modify
  9. # it under the terms of the GNU General Public License version 2 as
  10. # published by the Free Software Foundation.
  11. #
  12. # This program is distributed in the hope that it will be useful,
  13. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15. # GNU General Public License for more details.
  16. #
  17. # You should have received a copy of the GNU General Public License along
  18. # with this program; if not, write to the Free Software Foundation, Inc.,
  19. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  20. from __future__ import division
  21. import os
  22. import sys
  23. import xmlrpc.client as xmlrpclib
  24. import logging
  25. import progressbar
  26. import signal
  27. import bb.msg
  28. import time
  29. import fcntl
  30. import struct
  31. import copy
  32. import atexit
  33. from bb.ui import uihelper
  34. featureSet = [bb.cooker.CookerFeatures.SEND_SANITYEVENTS]
  35. logger = logging.getLogger("BitBake")
  36. interactive = sys.stdout.isatty()
  37. class BBProgress(progressbar.ProgressBar):
  38. def __init__(self, msg, maxval):
  39. self.msg = msg
  40. widgets = [progressbar.Percentage(), ' ', progressbar.Bar(), ' ',
  41. progressbar.ETA()]
  42. try:
  43. self._resize_default = signal.getsignal(signal.SIGWINCH)
  44. except:
  45. self._resize_default = None
  46. progressbar.ProgressBar.__init__(self, maxval, [self.msg + ": "] + widgets, fd=sys.stdout)
  47. def _handle_resize(self, signum, frame):
  48. progressbar.ProgressBar._handle_resize(self, signum, frame)
  49. if self._resize_default:
  50. self._resize_default(signum, frame)
  51. def finish(self):
  52. progressbar.ProgressBar.finish(self)
  53. if self._resize_default:
  54. signal.signal(signal.SIGWINCH, self._resize_default)
  55. class NonInteractiveProgress(object):
  56. fobj = sys.stdout
  57. def __init__(self, msg, maxval):
  58. self.msg = msg
  59. self.maxval = maxval
  60. def start(self):
  61. self.fobj.write("%s..." % self.msg)
  62. self.fobj.flush()
  63. return self
  64. def update(self, value):
  65. pass
  66. def finish(self):
  67. self.fobj.write("done.\n")
  68. self.fobj.flush()
  69. def new_progress(msg, maxval):
  70. if interactive:
  71. return BBProgress(msg, maxval)
  72. else:
  73. return NonInteractiveProgress(msg, maxval)
  74. def pluralise(singular, plural, qty):
  75. if(qty == 1):
  76. return singular % qty
  77. else:
  78. return plural % qty
  79. class InteractConsoleLogFilter(logging.Filter):
  80. def __init__(self, tf, format):
  81. self.tf = tf
  82. self.format = format
  83. def filter(self, record):
  84. if record.levelno == self.format.NOTE and (record.msg.startswith("Running") or record.msg.startswith("recipe ")):
  85. return False
  86. self.tf.clearFooter()
  87. return True
  88. class TerminalFilter(object):
  89. rows = 25
  90. columns = 80
  91. def sigwinch_handle(self, signum, frame):
  92. self.rows, self.columns = self.getTerminalColumns()
  93. if self._sigwinch_default:
  94. self._sigwinch_default(signum, frame)
  95. def getTerminalColumns(self):
  96. def ioctl_GWINSZ(fd):
  97. try:
  98. cr = struct.unpack('hh', fcntl.ioctl(fd, self.termios.TIOCGWINSZ, '1234'))
  99. except:
  100. return None
  101. return cr
  102. cr = ioctl_GWINSZ(sys.stdout.fileno())
  103. if not cr:
  104. try:
  105. fd = os.open(os.ctermid(), os.O_RDONLY)
  106. cr = ioctl_GWINSZ(fd)
  107. os.close(fd)
  108. except:
  109. pass
  110. if not cr:
  111. try:
  112. cr = (env['LINES'], env['COLUMNS'])
  113. except:
  114. cr = (25, 80)
  115. return cr
  116. def __init__(self, main, helper, console, errconsole, format):
  117. self.main = main
  118. self.helper = helper
  119. self.cuu = None
  120. self.stdinbackup = None
  121. self.interactive = sys.stdout.isatty()
  122. self.footer_present = False
  123. self.lastpids = []
  124. if not self.interactive:
  125. return
  126. try:
  127. import curses
  128. except ImportError:
  129. sys.exit("FATAL: The knotty ui could not load the required curses python module.")
  130. import termios
  131. self.curses = curses
  132. self.termios = termios
  133. try:
  134. fd = sys.stdin.fileno()
  135. self.stdinbackup = termios.tcgetattr(fd)
  136. new = copy.deepcopy(self.stdinbackup)
  137. new[3] = new[3] & ~termios.ECHO
  138. termios.tcsetattr(fd, termios.TCSADRAIN, new)
  139. curses.setupterm()
  140. if curses.tigetnum("colors") > 2:
  141. format.enable_color()
  142. self.ed = curses.tigetstr("ed")
  143. if self.ed:
  144. self.cuu = curses.tigetstr("cuu")
  145. try:
  146. self._sigwinch_default = signal.getsignal(signal.SIGWINCH)
  147. signal.signal(signal.SIGWINCH, self.sigwinch_handle)
  148. except:
  149. pass
  150. self.rows, self.columns = self.getTerminalColumns()
  151. except:
  152. self.cuu = None
  153. if not self.cuu:
  154. self.interactive = False
  155. bb.note("Unable to use interactive mode for this terminal, using fallback")
  156. return
  157. console.addFilter(InteractConsoleLogFilter(self, format))
  158. errconsole.addFilter(InteractConsoleLogFilter(self, format))
  159. def clearFooter(self):
  160. if self.footer_present:
  161. lines = self.footer_present
  162. sys.stdout.buffer.write(self.curses.tparm(self.cuu, lines))
  163. sys.stdout.buffer.write(self.curses.tparm(self.ed))
  164. sys.stdout.flush()
  165. self.footer_present = False
  166. def updateFooter(self):
  167. if not self.cuu:
  168. return
  169. activetasks = self.helper.running_tasks
  170. failedtasks = self.helper.failed_tasks
  171. runningpids = self.helper.running_pids
  172. if self.footer_present and (self.lastcount == self.helper.tasknumber_current) and (self.lastpids == runningpids):
  173. return
  174. if self.footer_present:
  175. self.clearFooter()
  176. if (not self.helper.tasknumber_total or self.helper.tasknumber_current == self.helper.tasknumber_total) and not len(activetasks):
  177. return
  178. tasks = []
  179. for t in runningpids:
  180. tasks.append("%s (pid %s)" % (activetasks[t]["title"], t))
  181. if self.main.shutdown:
  182. content = "Waiting for %s running tasks to finish:" % len(activetasks)
  183. elif not len(activetasks):
  184. content = "No currently running tasks (%s of %s)" % (self.helper.tasknumber_current, self.helper.tasknumber_total)
  185. else:
  186. content = "Currently %s running tasks (%s of %s):" % (len(activetasks), self.helper.tasknumber_current, self.helper.tasknumber_total)
  187. print(content)
  188. lines = 1 + int(len(content) / (self.columns + 1))
  189. for tasknum, task in enumerate(tasks[:(self.rows - 2)]):
  190. content = "%s: %s" % (tasknum, task)
  191. print(content)
  192. lines = lines + 1 + int(len(content) / (self.columns + 1))
  193. self.footer_present = lines
  194. self.lastpids = runningpids[:]
  195. self.lastcount = self.helper.tasknumber_current
  196. def finish(self):
  197. if self.stdinbackup:
  198. fd = sys.stdin.fileno()
  199. self.termios.tcsetattr(fd, self.termios.TCSADRAIN, self.stdinbackup)
  200. def _log_settings_from_server(server):
  201. # Get values of variables which control our output
  202. includelogs, error = server.runCommand(["getVariable", "BBINCLUDELOGS"])
  203. if error:
  204. logger.error("Unable to get the value of BBINCLUDELOGS variable: %s" % error)
  205. raise BaseException(error)
  206. loglines, error = server.runCommand(["getVariable", "BBINCLUDELOGS_LINES"])
  207. if error:
  208. logger.error("Unable to get the value of BBINCLUDELOGS_LINES variable: %s" % error)
  209. raise BaseException(error)
  210. consolelogfile, error = server.runCommand(["getSetVariable", "BB_CONSOLELOG"])
  211. if error:
  212. logger.error("Unable to get the value of BB_CONSOLELOG variable: %s" % error)
  213. raise BaseException(error)
  214. return includelogs, loglines, consolelogfile
  215. _evt_list = [ "bb.runqueue.runQueueExitWait", "bb.event.LogExecTTY", "logging.LogRecord",
  216. "bb.build.TaskFailed", "bb.build.TaskBase", "bb.event.ParseStarted",
  217. "bb.event.ParseProgress", "bb.event.ParseCompleted", "bb.event.CacheLoadStarted",
  218. "bb.event.CacheLoadProgress", "bb.event.CacheLoadCompleted", "bb.command.CommandFailed",
  219. "bb.command.CommandExit", "bb.command.CommandCompleted", "bb.cooker.CookerExit",
  220. "bb.event.MultipleProviders", "bb.event.NoProvider", "bb.runqueue.sceneQueueTaskStarted",
  221. "bb.runqueue.runQueueTaskStarted", "bb.runqueue.runQueueTaskFailed", "bb.runqueue.sceneQueueTaskFailed",
  222. "bb.event.BuildBase", "bb.build.TaskStarted", "bb.build.TaskSucceeded", "bb.build.TaskFailedSilent"]
  223. def main(server, eventHandler, params, tf = TerminalFilter):
  224. includelogs, loglines, consolelogfile = _log_settings_from_server(server)
  225. if sys.stdin.isatty() and sys.stdout.isatty():
  226. log_exec_tty = True
  227. else:
  228. log_exec_tty = False
  229. helper = uihelper.BBUIHelper()
  230. console = logging.StreamHandler(sys.stdout)
  231. errconsole = logging.StreamHandler(sys.stderr)
  232. format_str = "%(levelname)s: %(message)s"
  233. format = bb.msg.BBLogFormatter(format_str)
  234. bb.msg.addDefaultlogFilter(console, bb.msg.BBLogFilterStdOut)
  235. bb.msg.addDefaultlogFilter(errconsole, bb.msg.BBLogFilterStdErr)
  236. console.setFormatter(format)
  237. errconsole.setFormatter(format)
  238. logger.addHandler(console)
  239. logger.addHandler(errconsole)
  240. bb.utils.set_process_name("KnottyUI")
  241. if params.options.remote_server and params.options.kill_server:
  242. server.terminateServer()
  243. return
  244. consolelog = None
  245. if consolelogfile and not params.options.show_environment and not params.options.show_versions:
  246. bb.utils.mkdirhier(os.path.dirname(consolelogfile))
  247. conlogformat = bb.msg.BBLogFormatter(format_str)
  248. consolelog = logging.FileHandler(consolelogfile)
  249. bb.msg.addDefaultlogFilter(consolelog)
  250. consolelog.setFormatter(conlogformat)
  251. logger.addHandler(consolelog)
  252. llevel, debug_domains = bb.msg.constructLogOptions()
  253. server.runCommand(["setEventMask", server.getEventHandle(), llevel, debug_domains, _evt_list])
  254. universe = False
  255. if not params.observe_only:
  256. params.updateFromServer(server)
  257. params.updateToServer(server, os.environ.copy())
  258. cmdline = params.parseActions()
  259. if not cmdline:
  260. print("Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.")
  261. return 1
  262. if 'msg' in cmdline and cmdline['msg']:
  263. logger.error(cmdline['msg'])
  264. return 1
  265. if cmdline['action'][0] == "buildTargets" and "universe" in cmdline['action'][1]:
  266. universe = True
  267. ret, error = server.runCommand(cmdline['action'])
  268. if error:
  269. logger.error("Command '%s' failed: %s" % (cmdline, error))
  270. return 1
  271. elif ret != True:
  272. logger.error("Command '%s' failed: returned %s" % (cmdline, ret))
  273. return 1
  274. parseprogress = None
  275. cacheprogress = None
  276. main.shutdown = 0
  277. interrupted = False
  278. return_value = 0
  279. errors = 0
  280. warnings = 0
  281. taskfailures = []
  282. termfilter = tf(main, helper, console, errconsole, format)
  283. atexit.register(termfilter.finish)
  284. while True:
  285. try:
  286. event = eventHandler.waitEvent(0)
  287. if event is None:
  288. if main.shutdown > 1:
  289. break
  290. termfilter.updateFooter()
  291. event = eventHandler.waitEvent(0.25)
  292. if event is None:
  293. continue
  294. helper.eventHandler(event)
  295. if isinstance(event, bb.runqueue.runQueueExitWait):
  296. if not main.shutdown:
  297. main.shutdown = 1
  298. continue
  299. if isinstance(event, bb.event.LogExecTTY):
  300. if log_exec_tty:
  301. tries = event.retries
  302. while tries:
  303. print("Trying to run: %s" % event.prog)
  304. if os.system(event.prog) == 0:
  305. break
  306. time.sleep(event.sleep_delay)
  307. tries -= 1
  308. if tries:
  309. continue
  310. logger.warning(event.msg)
  311. continue
  312. if isinstance(event, logging.LogRecord):
  313. if event.levelno >= format.ERROR:
  314. errors = errors + 1
  315. return_value = 1
  316. elif event.levelno == format.WARNING:
  317. warnings = warnings + 1
  318. if event.taskpid != 0:
  319. # For "normal" logging conditions, don't show note logs from tasks
  320. # but do show them if the user has changed the default log level to
  321. # include verbose/debug messages
  322. if event.levelno <= format.NOTE and (event.levelno < llevel or (event.levelno == format.NOTE and llevel != format.VERBOSE)):
  323. continue
  324. # Prefix task messages with recipe/task
  325. if event.taskpid in helper.running_tasks:
  326. taskinfo = helper.running_tasks[event.taskpid]
  327. event.msg = taskinfo['title'] + ': ' + event.msg
  328. if hasattr(event, 'fn'):
  329. event.msg = event.fn + ': ' + event.msg
  330. logger.handle(event)
  331. continue
  332. if isinstance(event, bb.build.TaskFailedSilent):
  333. logger.warning("Logfile for failed setscene task is %s" % event.logfile)
  334. continue
  335. if isinstance(event, bb.build.TaskFailed):
  336. return_value = 1
  337. logfile = event.logfile
  338. if logfile and os.path.exists(logfile):
  339. termfilter.clearFooter()
  340. bb.error("Logfile of failure stored in: %s" % logfile)
  341. if includelogs and not event.errprinted:
  342. print("Log data follows:")
  343. f = open(logfile, "r")
  344. lines = []
  345. while True:
  346. l = f.readline()
  347. if l == '':
  348. break
  349. l = l.rstrip()
  350. if loglines:
  351. lines.append(' | %s' % l)
  352. if len(lines) > int(loglines):
  353. lines.pop(0)
  354. else:
  355. print('| %s' % l)
  356. f.close()
  357. if lines:
  358. for line in lines:
  359. print(line)
  360. if isinstance(event, bb.build.TaskBase):
  361. logger.info(event._message)
  362. continue
  363. if isinstance(event, bb.event.ParseStarted):
  364. if event.total == 0:
  365. continue
  366. parseprogress = new_progress("Parsing recipes", event.total).start()
  367. continue
  368. if isinstance(event, bb.event.ParseProgress):
  369. parseprogress.update(event.current)
  370. continue
  371. if isinstance(event, bb.event.ParseCompleted):
  372. if not parseprogress:
  373. continue
  374. parseprogress.finish()
  375. print(("Parsing of %d .bb files complete (%d cached, %d parsed). %d targets, %d skipped, %d masked, %d errors."
  376. % ( event.total, event.cached, event.parsed, event.virtuals, event.skipped, event.masked, event.errors)))
  377. continue
  378. if isinstance(event, bb.event.CacheLoadStarted):
  379. cacheprogress = new_progress("Loading cache", event.total).start()
  380. continue
  381. if isinstance(event, bb.event.CacheLoadProgress):
  382. cacheprogress.update(event.current)
  383. continue
  384. if isinstance(event, bb.event.CacheLoadCompleted):
  385. cacheprogress.finish()
  386. print("Loaded %d entries from dependency cache." % event.num_entries)
  387. continue
  388. if isinstance(event, bb.command.CommandFailed):
  389. return_value = event.exitcode
  390. if event.error:
  391. errors = errors + 1
  392. logger.error("Command execution failed: %s", event.error)
  393. main.shutdown = 2
  394. continue
  395. if isinstance(event, bb.command.CommandExit):
  396. if not return_value:
  397. return_value = event.exitcode
  398. continue
  399. if isinstance(event, (bb.command.CommandCompleted, bb.cooker.CookerExit)):
  400. main.shutdown = 2
  401. continue
  402. if isinstance(event, bb.event.MultipleProviders):
  403. logger.info("multiple providers are available for %s%s (%s)", event._is_runtime and "runtime " or "",
  404. event._item,
  405. ", ".join(event._candidates))
  406. rtime = ""
  407. if event._is_runtime:
  408. rtime = "R"
  409. logger.info("consider defining a PREFERRED_%sPROVIDER entry to match %s" % (rtime, event._item))
  410. continue
  411. if isinstance(event, bb.event.NoProvider):
  412. if event._runtime:
  413. r = "R"
  414. else:
  415. r = ""
  416. extra = ''
  417. if not event._reasons:
  418. if event._close_matches:
  419. extra = ". Close matches:\n %s" % '\n '.join(event._close_matches)
  420. # For universe builds, only show these as warnings, not errors
  421. h = logger.warning
  422. if not universe:
  423. return_value = 1
  424. errors = errors + 1
  425. h = logger.error
  426. if event._dependees:
  427. h("Nothing %sPROVIDES '%s' (but %s %sDEPENDS on or otherwise requires it)%s", r, event._item, ", ".join(event._dependees), r, extra)
  428. else:
  429. h("Nothing %sPROVIDES '%s'%s", r, event._item, extra)
  430. if event._reasons:
  431. for reason in event._reasons:
  432. h("%s", reason)
  433. continue
  434. if isinstance(event, bb.runqueue.sceneQueueTaskStarted):
  435. logger.info("Running setscene task %d of %d (%s)" % (event.stats.completed + event.stats.active + event.stats.failed + 1, event.stats.total, event.taskstring))
  436. continue
  437. if isinstance(event, bb.runqueue.runQueueTaskStarted):
  438. if event.noexec:
  439. tasktype = 'noexec task'
  440. else:
  441. tasktype = 'task'
  442. logger.info("Running %s %s of %s (ID: %s, %s)",
  443. tasktype,
  444. event.stats.completed + event.stats.active +
  445. event.stats.failed + 1,
  446. event.stats.total, event.taskid, event.taskstring)
  447. continue
  448. if isinstance(event, bb.runqueue.runQueueTaskFailed):
  449. return_value = 1
  450. taskfailures.append(event.taskstring)
  451. logger.error("Task %s (%s) failed with exit code '%s'",
  452. event.taskid, event.taskstring, event.exitcode)
  453. continue
  454. if isinstance(event, bb.runqueue.sceneQueueTaskFailed):
  455. logger.warning("Setscene task %s (%s) failed with exit code '%s' - real task will be run instead",
  456. event.taskid, event.taskstring, event.exitcode)
  457. continue
  458. if isinstance(event, bb.event.DepTreeGenerated):
  459. continue
  460. # ignore
  461. if isinstance(event, (bb.event.BuildBase,
  462. bb.event.MetadataEvent,
  463. bb.event.StampUpdate,
  464. bb.event.ConfigParsed,
  465. bb.event.RecipeParsed,
  466. bb.event.RecipePreFinalise,
  467. bb.runqueue.runQueueEvent,
  468. bb.event.OperationStarted,
  469. bb.event.OperationCompleted,
  470. bb.event.OperationProgress,
  471. bb.event.DiskFull)):
  472. continue
  473. logger.error("Unknown event: %s", event)
  474. except EnvironmentError as ioerror:
  475. termfilter.clearFooter()
  476. # ignore interrupted io
  477. if ioerror.args[0] == 4:
  478. continue
  479. sys.stderr.write(str(ioerror))
  480. if not params.observe_only:
  481. _, error = server.runCommand(["stateForceShutdown"])
  482. main.shutdown = 2
  483. except KeyboardInterrupt:
  484. termfilter.clearFooter()
  485. if params.observe_only:
  486. print("\nKeyboard Interrupt, exiting observer...")
  487. main.shutdown = 2
  488. if not params.observe_only and main.shutdown == 1:
  489. print("\nSecond Keyboard Interrupt, stopping...\n")
  490. _, error = server.runCommand(["stateForceShutdown"])
  491. if error:
  492. logger.error("Unable to cleanly stop: %s" % error)
  493. if not params.observe_only and main.shutdown == 0:
  494. print("\nKeyboard Interrupt, closing down...\n")
  495. interrupted = True
  496. _, error = server.runCommand(["stateShutdown"])
  497. if error:
  498. logger.error("Unable to cleanly shutdown: %s" % error)
  499. main.shutdown = main.shutdown + 1
  500. pass
  501. except Exception as e:
  502. import traceback
  503. sys.stderr.write(traceback.format_exc())
  504. if not params.observe_only:
  505. _, error = server.runCommand(["stateForceShutdown"])
  506. main.shutdown = 2
  507. return_value = 1
  508. try:
  509. termfilter.clearFooter()
  510. summary = ""
  511. if taskfailures:
  512. summary += pluralise("\nSummary: %s task failed:",
  513. "\nSummary: %s tasks failed:", len(taskfailures))
  514. for failure in taskfailures:
  515. summary += "\n %s" % failure
  516. if warnings:
  517. summary += pluralise("\nSummary: There was %s WARNING message shown.",
  518. "\nSummary: There were %s WARNING messages shown.", warnings)
  519. if return_value and errors:
  520. summary += pluralise("\nSummary: There was %s ERROR message shown, returning a non-zero exit code.",
  521. "\nSummary: There were %s ERROR messages shown, returning a non-zero exit code.", errors)
  522. if summary:
  523. print(summary)
  524. if interrupted:
  525. print("Execution was interrupted, returning a non-zero exit code.")
  526. if return_value == 0:
  527. return_value = 1
  528. except IOError as e:
  529. import errno
  530. if e.errno == errno.EPIPE:
  531. pass
  532. if consolelog:
  533. logger.removeHandler(consolelog)
  534. consolelog.close()
  535. return return_value