build.py 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906
  1. # ex:ts=4:sw=4:sts=4:et
  2. # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
  3. #
  4. # BitBake 'Build' implementation
  5. #
  6. # Core code for function execution and task handling in the
  7. # BitBake build tools.
  8. #
  9. # Copyright (C) 2003, 2004 Chris Larson
  10. #
  11. # Based on Gentoo's portage.py.
  12. #
  13. # SPDX-License-Identifier: GPL-2.0-only
  14. #
  15. # Based on functions from the base bb module, Copyright 2003 Holger Schurig
  16. import os
  17. import sys
  18. import logging
  19. import shlex
  20. import glob
  21. import time
  22. import stat
  23. import bb
  24. import bb.msg
  25. import bb.process
  26. import bb.progress
  27. from bb import data, event, utils
  28. bblogger = logging.getLogger('BitBake')
  29. logger = logging.getLogger('BitBake.Build')
  30. __mtime_cache = {}
  31. def cached_mtime_noerror(f):
  32. if f not in __mtime_cache:
  33. try:
  34. __mtime_cache[f] = os.stat(f)[stat.ST_MTIME]
  35. except OSError:
  36. return 0
  37. return __mtime_cache[f]
  38. def reset_cache():
  39. global __mtime_cache
  40. __mtime_cache = {}
  41. # When we execute a Python function, we'd like certain things
  42. # in all namespaces, hence we add them to __builtins__.
  43. # If we do not do this and use the exec globals, they will
  44. # not be available to subfunctions.
  45. if hasattr(__builtins__, '__setitem__'):
  46. builtins = __builtins__
  47. else:
  48. builtins = __builtins__.__dict__
  49. builtins['bb'] = bb
  50. builtins['os'] = os
  51. class FuncFailed(Exception):
  52. def __init__(self, name = None, logfile = None):
  53. self.logfile = logfile
  54. self.name = name
  55. if name:
  56. self.msg = 'Function failed: %s' % name
  57. else:
  58. self.msg = "Function failed"
  59. def __str__(self):
  60. if self.logfile and os.path.exists(self.logfile):
  61. msg = ("%s (log file is located at %s)" %
  62. (self.msg, self.logfile))
  63. else:
  64. msg = self.msg
  65. return msg
  66. class TaskBase(event.Event):
  67. """Base class for task events"""
  68. def __init__(self, t, logfile, d):
  69. self._task = t
  70. self._package = d.getVar("PF")
  71. self._mc = d.getVar("BB_CURRENT_MC")
  72. self.taskfile = d.getVar("FILE")
  73. self.taskname = self._task
  74. self.logfile = logfile
  75. self.time = time.time()
  76. event.Event.__init__(self)
  77. self._message = "recipe %s: task %s: %s" % (d.getVar("PF"), t, self.getDisplayName())
  78. def getTask(self):
  79. return self._task
  80. def setTask(self, task):
  81. self._task = task
  82. def getDisplayName(self):
  83. return bb.event.getName(self)[4:]
  84. task = property(getTask, setTask, None, "task property")
  85. class TaskStarted(TaskBase):
  86. """Task execution started"""
  87. def __init__(self, t, logfile, taskflags, d):
  88. super(TaskStarted, self).__init__(t, logfile, d)
  89. self.taskflags = taskflags
  90. class TaskSucceeded(TaskBase):
  91. """Task execution completed"""
  92. class TaskFailed(TaskBase):
  93. """Task execution failed"""
  94. def __init__(self, task, logfile, metadata, errprinted = False):
  95. self.errprinted = errprinted
  96. super(TaskFailed, self).__init__(task, logfile, metadata)
  97. class TaskFailedSilent(TaskBase):
  98. """Task execution failed (silently)"""
  99. def getDisplayName(self):
  100. # Don't need to tell the user it was silent
  101. return "Failed"
  102. class TaskInvalid(TaskBase):
  103. def __init__(self, task, metadata):
  104. super(TaskInvalid, self).__init__(task, None, metadata)
  105. self._message = "No such task '%s'" % task
  106. class TaskProgress(event.Event):
  107. """
  108. Task made some progress that could be reported to the user, usually in
  109. the form of a progress bar or similar.
  110. NOTE: this class does not inherit from TaskBase since it doesn't need
  111. to - it's fired within the task context itself, so we don't have any of
  112. the context information that you do in the case of the other events.
  113. The event PID can be used to determine which task it came from.
  114. The progress value is normally 0-100, but can also be negative
  115. indicating that progress has been made but we aren't able to determine
  116. how much.
  117. The rate is optional, this is simply an extra string to display to the
  118. user if specified.
  119. """
  120. def __init__(self, progress, rate=None):
  121. self.progress = progress
  122. self.rate = rate
  123. event.Event.__init__(self)
  124. class LogTee(object):
  125. def __init__(self, logger, outfile):
  126. self.outfile = outfile
  127. self.logger = logger
  128. self.name = self.outfile.name
  129. def write(self, string):
  130. self.logger.plain(string)
  131. self.outfile.write(string)
  132. def __enter__(self):
  133. self.outfile.__enter__()
  134. return self
  135. def __exit__(self, *excinfo):
  136. self.outfile.__exit__(*excinfo)
  137. def __repr__(self):
  138. return '<LogTee {0}>'.format(self.name)
  139. def flush(self):
  140. self.outfile.flush()
  141. #
  142. # pythonexception allows the python exceptions generated to be raised
  143. # as the real exceptions (not FuncFailed) and without a backtrace at the
  144. # origin of the failure.
  145. #
  146. def exec_func(func, d, dirs = None, pythonexception=False):
  147. """Execute a BB 'function'"""
  148. try:
  149. oldcwd = os.getcwd()
  150. except:
  151. oldcwd = None
  152. flags = d.getVarFlags(func)
  153. cleandirs = flags.get('cleandirs') if flags else None
  154. if cleandirs:
  155. for cdir in d.expand(cleandirs).split():
  156. bb.utils.remove(cdir, True)
  157. bb.utils.mkdirhier(cdir)
  158. if flags and dirs is None:
  159. dirs = flags.get('dirs')
  160. if dirs:
  161. dirs = d.expand(dirs).split()
  162. if dirs:
  163. for adir in dirs:
  164. bb.utils.mkdirhier(adir)
  165. adir = dirs[-1]
  166. else:
  167. adir = None
  168. body = d.getVar(func, False)
  169. if not body:
  170. if body is None:
  171. logger.warning("Function %s doesn't exist", func)
  172. return
  173. ispython = flags.get('python')
  174. lockflag = flags.get('lockfiles')
  175. if lockflag:
  176. lockfiles = [f for f in d.expand(lockflag).split()]
  177. else:
  178. lockfiles = None
  179. tempdir = d.getVar('T')
  180. # or func allows items to be executed outside of the normal
  181. # task set, such as buildhistory
  182. task = d.getVar('BB_RUNTASK') or func
  183. if task == func:
  184. taskfunc = task
  185. else:
  186. taskfunc = "%s.%s" % (task, func)
  187. runfmt = d.getVar('BB_RUNFMT') or "run.{func}.{pid}"
  188. runfn = runfmt.format(taskfunc=taskfunc, task=task, func=func, pid=os.getpid())
  189. runfile = os.path.join(tempdir, runfn)
  190. bb.utils.mkdirhier(os.path.dirname(runfile))
  191. # Setup the courtesy link to the runfn, only for tasks
  192. # we create the link 'just' before the run script is created
  193. # if we create it after, and if the run script fails, then the
  194. # link won't be created as an exception would be fired.
  195. if task == func:
  196. runlink = os.path.join(tempdir, 'run.{0}'.format(task))
  197. if runlink:
  198. bb.utils.remove(runlink)
  199. try:
  200. os.symlink(runfn, runlink)
  201. except OSError:
  202. pass
  203. with bb.utils.fileslocked(lockfiles):
  204. if ispython:
  205. exec_func_python(func, d, runfile, cwd=adir, pythonexception=pythonexception)
  206. else:
  207. exec_func_shell(func, d, runfile, cwd=adir)
  208. try:
  209. curcwd = os.getcwd()
  210. except:
  211. curcwd = None
  212. if oldcwd and curcwd != oldcwd:
  213. try:
  214. bb.warn("Task %s changed cwd to %s" % (func, curcwd))
  215. os.chdir(oldcwd)
  216. except:
  217. pass
  218. _functionfmt = """
  219. {function}(d)
  220. """
  221. logformatter = bb.msg.BBLogFormatter("%(levelname)s: %(message)s")
  222. def exec_func_python(func, d, runfile, cwd=None, pythonexception=False):
  223. """Execute a python BB 'function'"""
  224. code = _functionfmt.format(function=func)
  225. bb.utils.mkdirhier(os.path.dirname(runfile))
  226. with open(runfile, 'w') as script:
  227. bb.data.emit_func_python(func, script, d)
  228. if cwd:
  229. try:
  230. olddir = os.getcwd()
  231. except OSError as e:
  232. bb.warn("%s: Cannot get cwd: %s" % (func, e))
  233. olddir = None
  234. os.chdir(cwd)
  235. bb.debug(2, "Executing python function %s" % func)
  236. try:
  237. text = "def %s(d):\n%s" % (func, d.getVar(func, False))
  238. fn = d.getVarFlag(func, "filename", False)
  239. lineno = int(d.getVarFlag(func, "lineno", False))
  240. bb.methodpool.insert_method(func, text, fn, lineno - 1)
  241. comp = utils.better_compile(code, func, "exec_python_func() autogenerated")
  242. utils.better_exec(comp, {"d": d}, code, "exec_python_func() autogenerated", pythonexception=pythonexception)
  243. except (bb.parse.SkipRecipe, bb.build.FuncFailed):
  244. raise
  245. except Exception as e:
  246. if pythonexception:
  247. raise
  248. logger.error(str(e))
  249. raise FuncFailed(func, None)
  250. finally:
  251. bb.debug(2, "Python function %s finished" % func)
  252. if cwd and olddir:
  253. try:
  254. os.chdir(olddir)
  255. except OSError as e:
  256. bb.warn("%s: Cannot restore cwd %s: %s" % (func, olddir, e))
  257. def shell_trap_code():
  258. return '''#!/bin/sh\n
  259. # Emit a useful diagnostic if something fails:
  260. bb_exit_handler() {
  261. ret=$?
  262. case $ret in
  263. 0) ;;
  264. *) case $BASH_VERSION in
  265. "") echo "WARNING: exit code $ret from a shell command.";;
  266. *) echo "WARNING: ${BASH_SOURCE[0]}:${BASH_LINENO[0]} exit $ret from '$BASH_COMMAND'";;
  267. esac
  268. exit $ret
  269. esac
  270. }
  271. trap 'bb_exit_handler' 0
  272. set -e
  273. '''
  274. def exec_func_shell(func, d, runfile, cwd=None):
  275. """Execute a shell function from the metadata
  276. Note on directory behavior. The 'dirs' varflag should contain a list
  277. of the directories you need created prior to execution. The last
  278. item in the list is where we will chdir/cd to.
  279. """
  280. # Don't let the emitted shell script override PWD
  281. d.delVarFlag('PWD', 'export')
  282. with open(runfile, 'w') as script:
  283. script.write(shell_trap_code())
  284. bb.data.emit_func(func, script, d)
  285. if bb.msg.loggerVerboseLogs:
  286. script.write("set -x\n")
  287. if cwd:
  288. script.write("cd '%s'\n" % cwd)
  289. script.write("%s\n" % func)
  290. script.write('''
  291. # cleanup
  292. ret=$?
  293. trap '' 0
  294. exit $ret
  295. ''')
  296. os.chmod(runfile, 0o775)
  297. cmd = runfile
  298. if d.getVarFlag(func, 'fakeroot', False):
  299. fakerootcmd = d.getVar('FAKEROOT')
  300. if fakerootcmd:
  301. cmd = [fakerootcmd, runfile]
  302. if bb.msg.loggerDefaultVerbose:
  303. logfile = LogTee(logger, sys.stdout)
  304. else:
  305. logfile = sys.stdout
  306. progress = d.getVarFlag(func, 'progress')
  307. if progress:
  308. if progress == 'percent':
  309. # Use default regex
  310. logfile = bb.progress.BasicProgressHandler(d, outfile=logfile)
  311. elif progress.startswith('percent:'):
  312. # Use specified regex
  313. logfile = bb.progress.BasicProgressHandler(d, regex=progress.split(':', 1)[1], outfile=logfile)
  314. elif progress.startswith('outof:'):
  315. # Use specified regex
  316. logfile = bb.progress.OutOfProgressHandler(d, regex=progress.split(':', 1)[1], outfile=logfile)
  317. else:
  318. bb.warn('%s: invalid task progress varflag value "%s", ignoring' % (func, progress))
  319. fifobuffer = bytearray()
  320. def readfifo(data):
  321. nonlocal fifobuffer
  322. fifobuffer.extend(data)
  323. while fifobuffer:
  324. message, token, nextmsg = fifobuffer.partition(b"\00")
  325. if token:
  326. splitval = message.split(b' ', 1)
  327. cmd = splitval[0].decode("utf-8")
  328. if len(splitval) > 1:
  329. value = splitval[1].decode("utf-8")
  330. else:
  331. value = ''
  332. if cmd == 'bbplain':
  333. bb.plain(value)
  334. elif cmd == 'bbnote':
  335. bb.note(value)
  336. elif cmd == 'bbverbnote':
  337. bb.verbnote(value)
  338. elif cmd == 'bbwarn':
  339. bb.warn(value)
  340. elif cmd == 'bberror':
  341. bb.error(value)
  342. elif cmd == 'bbfatal':
  343. # The caller will call exit themselves, so bb.error() is
  344. # what we want here rather than bb.fatal()
  345. bb.error(value)
  346. elif cmd == 'bbfatal_log':
  347. bb.error(value, forcelog=True)
  348. elif cmd == 'bbdebug':
  349. splitval = value.split(' ', 1)
  350. level = int(splitval[0])
  351. value = splitval[1]
  352. bb.debug(level, value)
  353. else:
  354. bb.warn("Unrecognised command '%s' on FIFO" % cmd)
  355. fifobuffer = nextmsg
  356. else:
  357. break
  358. tempdir = d.getVar('T')
  359. fifopath = os.path.join(tempdir, 'fifo.%s' % os.getpid())
  360. if os.path.exists(fifopath):
  361. os.unlink(fifopath)
  362. os.mkfifo(fifopath)
  363. with open(fifopath, 'r+b', buffering=0) as fifo:
  364. try:
  365. bb.debug(2, "Executing shell function %s" % func)
  366. try:
  367. with open(os.devnull, 'r+') as stdin:
  368. bb.process.run(cmd, shell=False, stdin=stdin, log=logfile, extrafiles=[(fifo,readfifo)])
  369. except bb.process.CmdError:
  370. logfn = d.getVar('BB_LOGFILE')
  371. raise FuncFailed(func, logfn)
  372. finally:
  373. os.unlink(fifopath)
  374. bb.debug(2, "Shell function %s finished" % func)
  375. def _task_data(fn, task, d):
  376. localdata = bb.data.createCopy(d)
  377. localdata.setVar('BB_FILENAME', fn)
  378. localdata.setVar('BB_CURRENTTASK', task[3:])
  379. localdata.setVar('OVERRIDES', 'task-%s:%s' %
  380. (task[3:].replace('_', '-'), d.getVar('OVERRIDES', False)))
  381. localdata.finalize()
  382. bb.data.expandKeys(localdata)
  383. return localdata
  384. def _exec_task(fn, task, d, quieterr):
  385. """Execute a BB 'task'
  386. Execution of a task involves a bit more setup than executing a function,
  387. running it with its own local metadata, and with some useful variables set.
  388. """
  389. if not d.getVarFlag(task, 'task', False):
  390. event.fire(TaskInvalid(task, d), d)
  391. logger.error("No such task: %s" % task)
  392. return 1
  393. logger.debug(1, "Executing task %s", task)
  394. localdata = _task_data(fn, task, d)
  395. tempdir = localdata.getVar('T')
  396. if not tempdir:
  397. bb.fatal("T variable not set, unable to build")
  398. # Change nice level if we're asked to
  399. nice = localdata.getVar("BB_TASK_NICE_LEVEL")
  400. if nice:
  401. curnice = os.nice(0)
  402. nice = int(nice) - curnice
  403. newnice = os.nice(nice)
  404. logger.debug(1, "Renice to %s " % newnice)
  405. ionice = localdata.getVar("BB_TASK_IONICE_LEVEL")
  406. if ionice:
  407. try:
  408. cls, prio = ionice.split(".", 1)
  409. bb.utils.ioprio_set(os.getpid(), int(cls), int(prio))
  410. except:
  411. bb.warn("Invalid ionice level %s" % ionice)
  412. bb.utils.mkdirhier(tempdir)
  413. # Determine the logfile to generate
  414. logfmt = localdata.getVar('BB_LOGFMT') or 'log.{task}.{pid}'
  415. logbase = logfmt.format(task=task, pid=os.getpid())
  416. # Document the order of the tasks...
  417. logorder = os.path.join(tempdir, 'log.task_order')
  418. try:
  419. with open(logorder, 'a') as logorderfile:
  420. logorderfile.write('{0} ({1}): {2}\n'.format(task, os.getpid(), logbase))
  421. except OSError:
  422. logger.exception("Opening log file '%s'", logorder)
  423. pass
  424. # Setup the courtesy link to the logfn
  425. loglink = os.path.join(tempdir, 'log.{0}'.format(task))
  426. logfn = os.path.join(tempdir, logbase)
  427. if loglink:
  428. bb.utils.remove(loglink)
  429. try:
  430. os.symlink(logbase, loglink)
  431. except OSError:
  432. pass
  433. prefuncs = localdata.getVarFlag(task, 'prefuncs', expand=True)
  434. postfuncs = localdata.getVarFlag(task, 'postfuncs', expand=True)
  435. class ErrorCheckHandler(logging.Handler):
  436. def __init__(self):
  437. self.triggered = False
  438. logging.Handler.__init__(self, logging.ERROR)
  439. def emit(self, record):
  440. if getattr(record, 'forcelog', False):
  441. self.triggered = False
  442. else:
  443. self.triggered = True
  444. # Handle logfiles
  445. try:
  446. bb.utils.mkdirhier(os.path.dirname(logfn))
  447. logfile = open(logfn, 'w')
  448. except OSError:
  449. logger.exception("Opening log file '%s'", logfn)
  450. pass
  451. # Dup the existing fds so we dont lose them
  452. osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()]
  453. oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()]
  454. ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()]
  455. # Replace those fds with our own
  456. with open('/dev/null', 'r') as si:
  457. os.dup2(si.fileno(), osi[1])
  458. os.dup2(logfile.fileno(), oso[1])
  459. os.dup2(logfile.fileno(), ose[1])
  460. # Ensure Python logging goes to the logfile
  461. handler = logging.StreamHandler(logfile)
  462. handler.setFormatter(logformatter)
  463. # Always enable full debug output into task logfiles
  464. handler.setLevel(logging.DEBUG - 2)
  465. bblogger.addHandler(handler)
  466. errchk = ErrorCheckHandler()
  467. bblogger.addHandler(errchk)
  468. localdata.setVar('BB_LOGFILE', logfn)
  469. localdata.setVar('BB_RUNTASK', task)
  470. localdata.setVar('BB_TASK_LOGGER', bblogger)
  471. flags = localdata.getVarFlags(task)
  472. try:
  473. try:
  474. event.fire(TaskStarted(task, logfn, flags, localdata), localdata)
  475. except (bb.BBHandledException, SystemExit):
  476. return 1
  477. except FuncFailed as exc:
  478. logger.error(str(exc))
  479. return 1
  480. try:
  481. for func in (prefuncs or '').split():
  482. exec_func(func, localdata)
  483. exec_func(task, localdata)
  484. for func in (postfuncs or '').split():
  485. exec_func(func, localdata)
  486. except FuncFailed as exc:
  487. if quieterr:
  488. event.fire(TaskFailedSilent(task, logfn, localdata), localdata)
  489. else:
  490. errprinted = errchk.triggered
  491. logger.error(str(exc))
  492. event.fire(TaskFailed(task, logfn, localdata, errprinted), localdata)
  493. return 1
  494. except bb.BBHandledException:
  495. event.fire(TaskFailed(task, logfn, localdata, True), localdata)
  496. return 1
  497. finally:
  498. sys.stdout.flush()
  499. sys.stderr.flush()
  500. bblogger.removeHandler(handler)
  501. # Restore the backup fds
  502. os.dup2(osi[0], osi[1])
  503. os.dup2(oso[0], oso[1])
  504. os.dup2(ose[0], ose[1])
  505. # Close the backup fds
  506. os.close(osi[0])
  507. os.close(oso[0])
  508. os.close(ose[0])
  509. logfile.close()
  510. if os.path.exists(logfn) and os.path.getsize(logfn) == 0:
  511. logger.debug(2, "Zero size logfn %s, removing", logfn)
  512. bb.utils.remove(logfn)
  513. bb.utils.remove(loglink)
  514. event.fire(TaskSucceeded(task, logfn, localdata), localdata)
  515. if not localdata.getVarFlag(task, 'nostamp', False) and not localdata.getVarFlag(task, 'selfstamp', False):
  516. make_stamp(task, localdata)
  517. return 0
  518. def exec_task(fn, task, d, profile = False):
  519. try:
  520. quieterr = False
  521. if d.getVarFlag(task, "quieterrors", False) is not None:
  522. quieterr = True
  523. if profile:
  524. profname = "profile-%s.log" % (d.getVar("PN") + "-" + task)
  525. try:
  526. import cProfile as profile
  527. except:
  528. import profile
  529. prof = profile.Profile()
  530. ret = profile.Profile.runcall(prof, _exec_task, fn, task, d, quieterr)
  531. prof.dump_stats(profname)
  532. bb.utils.process_profilelog(profname)
  533. return ret
  534. else:
  535. return _exec_task(fn, task, d, quieterr)
  536. except Exception:
  537. from traceback import format_exc
  538. if not quieterr:
  539. logger.error("Build of %s failed" % (task))
  540. logger.error(format_exc())
  541. failedevent = TaskFailed(task, None, d, True)
  542. event.fire(failedevent, d)
  543. return 1
  544. def stamp_internal(taskname, d, file_name, baseonly=False, noextra=False):
  545. """
  546. Internal stamp helper function
  547. Makes sure the stamp directory exists
  548. Returns the stamp path+filename
  549. In the bitbake core, d can be a CacheData and file_name will be set.
  550. When called in task context, d will be a data store, file_name will not be set
  551. """
  552. taskflagname = taskname
  553. if taskname.endswith("_setscene") and taskname != "do_setscene":
  554. taskflagname = taskname.replace("_setscene", "")
  555. if file_name:
  556. stamp = d.stamp[file_name]
  557. extrainfo = d.stamp_extrainfo[file_name].get(taskflagname) or ""
  558. else:
  559. stamp = d.getVar('STAMP')
  560. file_name = d.getVar('BB_FILENAME')
  561. extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info') or ""
  562. if baseonly:
  563. return stamp
  564. if noextra:
  565. extrainfo = ""
  566. if not stamp:
  567. return
  568. stamp = bb.parse.siggen.stampfile(stamp, file_name, taskname, extrainfo)
  569. stampdir = os.path.dirname(stamp)
  570. if cached_mtime_noerror(stampdir) == 0:
  571. bb.utils.mkdirhier(stampdir)
  572. return stamp
  573. def stamp_cleanmask_internal(taskname, d, file_name):
  574. """
  575. Internal stamp helper function to generate stamp cleaning mask
  576. Returns the stamp path+filename
  577. In the bitbake core, d can be a CacheData and file_name will be set.
  578. When called in task context, d will be a data store, file_name will not be set
  579. """
  580. taskflagname = taskname
  581. if taskname.endswith("_setscene") and taskname != "do_setscene":
  582. taskflagname = taskname.replace("_setscene", "")
  583. if file_name:
  584. stamp = d.stampclean[file_name]
  585. extrainfo = d.stamp_extrainfo[file_name].get(taskflagname) or ""
  586. else:
  587. stamp = d.getVar('STAMPCLEAN')
  588. file_name = d.getVar('BB_FILENAME')
  589. extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info') or ""
  590. if not stamp:
  591. return []
  592. cleanmask = bb.parse.siggen.stampcleanmask(stamp, file_name, taskname, extrainfo)
  593. return [cleanmask, cleanmask.replace(taskflagname, taskflagname + "_setscene")]
  594. def make_stamp(task, d, file_name = None):
  595. """
  596. Creates/updates a stamp for a given task
  597. (d can be a data dict or dataCache)
  598. """
  599. cleanmask = stamp_cleanmask_internal(task, d, file_name)
  600. for mask in cleanmask:
  601. for name in glob.glob(mask):
  602. # Preserve sigdata files in the stamps directory
  603. if "sigdata" in name or "sigbasedata" in name:
  604. continue
  605. # Preserve taint files in the stamps directory
  606. if name.endswith('.taint'):
  607. continue
  608. os.unlink(name)
  609. stamp = stamp_internal(task, d, file_name)
  610. # Remove the file and recreate to force timestamp
  611. # change on broken NFS filesystems
  612. if stamp:
  613. bb.utils.remove(stamp)
  614. open(stamp, "w").close()
  615. # If we're in task context, write out a signature file for each task
  616. # as it completes
  617. if not task.endswith("_setscene") and task != "do_setscene" and not file_name:
  618. stampbase = stamp_internal(task, d, None, True)
  619. file_name = d.getVar('BB_FILENAME')
  620. bb.parse.siggen.dump_sigtask(file_name, task, stampbase, True)
  621. def del_stamp(task, d, file_name = None):
  622. """
  623. Removes a stamp for a given task
  624. (d can be a data dict or dataCache)
  625. """
  626. stamp = stamp_internal(task, d, file_name)
  627. bb.utils.remove(stamp)
  628. def write_taint(task, d, file_name = None):
  629. """
  630. Creates a "taint" file which will force the specified task and its
  631. dependents to be re-run the next time by influencing the value of its
  632. taskhash.
  633. (d can be a data dict or dataCache)
  634. """
  635. import uuid
  636. if file_name:
  637. taintfn = d.stamp[file_name] + '.' + task + '.taint'
  638. else:
  639. taintfn = d.getVar('STAMP') + '.' + task + '.taint'
  640. bb.utils.mkdirhier(os.path.dirname(taintfn))
  641. # The specific content of the taint file is not really important,
  642. # we just need it to be random, so a random UUID is used
  643. with open(taintfn, 'w') as taintf:
  644. taintf.write(str(uuid.uuid4()))
  645. def stampfile(taskname, d, file_name = None, noextra=False):
  646. """
  647. Return the stamp for a given task
  648. (d can be a data dict or dataCache)
  649. """
  650. return stamp_internal(taskname, d, file_name, noextra=noextra)
  651. def add_tasks(tasklist, d):
  652. task_deps = d.getVar('_task_deps', False)
  653. if not task_deps:
  654. task_deps = {}
  655. if not 'tasks' in task_deps:
  656. task_deps['tasks'] = []
  657. if not 'parents' in task_deps:
  658. task_deps['parents'] = {}
  659. for task in tasklist:
  660. task = d.expand(task)
  661. d.setVarFlag(task, 'task', 1)
  662. if not task in task_deps['tasks']:
  663. task_deps['tasks'].append(task)
  664. flags = d.getVarFlags(task)
  665. def getTask(name):
  666. if not name in task_deps:
  667. task_deps[name] = {}
  668. if name in flags:
  669. deptask = d.expand(flags[name])
  670. task_deps[name][task] = deptask
  671. getTask('mcdepends')
  672. getTask('depends')
  673. getTask('rdepends')
  674. getTask('deptask')
  675. getTask('rdeptask')
  676. getTask('recrdeptask')
  677. getTask('recideptask')
  678. getTask('nostamp')
  679. getTask('fakeroot')
  680. getTask('noexec')
  681. getTask('umask')
  682. task_deps['parents'][task] = []
  683. if 'deps' in flags:
  684. for dep in flags['deps']:
  685. # Check and warn for "addtask task after foo" while foo does not exist
  686. #if not dep in tasklist:
  687. # bb.warn('%s: dependent task %s for %s does not exist' % (d.getVar('PN'), dep, task))
  688. dep = d.expand(dep)
  689. task_deps['parents'][task].append(dep)
  690. # don't assume holding a reference
  691. d.setVar('_task_deps', task_deps)
  692. def addtask(task, before, after, d):
  693. if task[:3] != "do_":
  694. task = "do_" + task
  695. d.setVarFlag(task, "task", 1)
  696. bbtasks = d.getVar('__BBTASKS', False) or []
  697. if task not in bbtasks:
  698. bbtasks.append(task)
  699. d.setVar('__BBTASKS', bbtasks)
  700. existing = d.getVarFlag(task, "deps", False) or []
  701. if after is not None:
  702. # set up deps for function
  703. for entry in after.split():
  704. if entry not in existing:
  705. existing.append(entry)
  706. d.setVarFlag(task, "deps", existing)
  707. if before is not None:
  708. # set up things that depend on this func
  709. for entry in before.split():
  710. existing = d.getVarFlag(entry, "deps", False) or []
  711. if task not in existing:
  712. d.setVarFlag(entry, "deps", [task] + existing)
  713. def deltask(task, d):
  714. if task[:3] != "do_":
  715. task = "do_" + task
  716. bbtasks = d.getVar('__BBTASKS', False) or []
  717. if task in bbtasks:
  718. bbtasks.remove(task)
  719. d.delVarFlag(task, 'task')
  720. d.setVar('__BBTASKS', bbtasks)
  721. d.delVarFlag(task, 'deps')
  722. for bbtask in d.getVar('__BBTASKS', False) or []:
  723. deps = d.getVarFlag(bbtask, 'deps', False) or []
  724. if task in deps:
  725. deps.remove(task)
  726. d.setVarFlag(bbtask, 'deps', deps)
  727. def preceedtask(task, with_recrdeptasks, d):
  728. """
  729. Returns a set of tasks in the current recipe which were specified as
  730. precondition by the task itself ("after") or which listed themselves
  731. as precondition ("before"). Preceeding tasks specified via the
  732. "recrdeptask" are included in the result only if requested. Beware
  733. that this may lead to the task itself being listed.
  734. """
  735. preceed = set()
  736. # Ignore tasks which don't exist
  737. tasks = d.getVar('__BBTASKS', False)
  738. if task not in tasks:
  739. return preceed
  740. preceed.update(d.getVarFlag(task, 'deps') or [])
  741. if with_recrdeptasks:
  742. recrdeptask = d.getVarFlag(task, 'recrdeptask')
  743. if recrdeptask:
  744. preceed.update(recrdeptask.split())
  745. return preceed
  746. def tasksbetween(task_start, task_end, d):
  747. """
  748. Return the list of tasks between two tasks in the current recipe,
  749. where task_start is to start at and task_end is the task to end at
  750. (and task_end has a dependency chain back to task_start).
  751. """
  752. outtasks = []
  753. tasks = list(filter(lambda k: d.getVarFlag(k, "task"), d.keys()))
  754. def follow_chain(task, endtask, chain=None):
  755. if not chain:
  756. chain = []
  757. chain.append(task)
  758. for othertask in tasks:
  759. if othertask == task:
  760. continue
  761. if task == endtask:
  762. for ctask in chain:
  763. if ctask not in outtasks:
  764. outtasks.append(ctask)
  765. else:
  766. deps = d.getVarFlag(othertask, 'deps', False)
  767. if task in deps:
  768. follow_chain(othertask, endtask, chain)
  769. chain.pop()
  770. follow_chain(task_start, task_end)
  771. return outtasks