build.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445
  1. #!/usr/bin/env python
  2. # ex:ts=4:sw=4:sts=4:et
  3. # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
  4. #
  5. # BitBake 'Build' implementation
  6. #
  7. # Core code for function execution and task handling in the
  8. # BitBake build tools.
  9. #
  10. # Copyright (C) 2003, 2004 Chris Larson
  11. #
  12. # Based on Gentoo's portage.py.
  13. #
  14. # This program is free software; you can redistribute it and/or modify
  15. # it under the terms of the GNU General Public License version 2 as
  16. # published by the Free Software Foundation.
  17. #
  18. # This program is distributed in the hope that it will be useful,
  19. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  20. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  21. # GNU General Public License for more details.
  22. #
  23. # You should have received a copy of the GNU General Public License along
  24. # with this program; if not, write to the Free Software Foundation, Inc.,
  25. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  26. #
  27. #Based on functions from the base bb module, Copyright 2003 Holger Schurig
  28. from bb import data, fetch, event, mkdirhier, utils
  29. import bb, os
  30. # events
  31. class FuncFailed(Exception):
  32. """Executed function failed"""
  33. class EventException(Exception):
  34. """Exception which is associated with an Event."""
  35. def __init__(self, msg, event):
  36. self.args = msg, event
  37. class TaskBase(event.Event):
  38. """Base class for task events"""
  39. def __init__(self, t, d ):
  40. self._task = t
  41. event.Event.__init__(self, d)
  42. def getTask(self):
  43. return self._task
  44. def setTask(self, task):
  45. self._task = task
  46. task = property(getTask, setTask, None, "task property")
  47. class TaskStarted(TaskBase):
  48. """Task execution started"""
  49. class TaskSucceeded(TaskBase):
  50. """Task execution completed"""
  51. class TaskFailed(TaskBase):
  52. """Task execution failed"""
  53. class InvalidTask(TaskBase):
  54. """Invalid Task"""
  55. # functions
  56. def exec_func(func, d, dirs = None):
  57. """Execute an BB 'function'"""
  58. body = data.getVar(func, d)
  59. if not body:
  60. return
  61. if not dirs:
  62. dirs = (data.getVarFlag(func, 'dirs', d) or "").split()
  63. for adir in dirs:
  64. adir = data.expand(adir, d)
  65. mkdirhier(adir)
  66. if len(dirs) > 0:
  67. adir = dirs[-1]
  68. else:
  69. adir = data.getVar('B', d, 1)
  70. adir = data.expand(adir, d)
  71. try:
  72. prevdir = os.getcwd()
  73. except OSError:
  74. prevdir = data.expand('${TOPDIR}', d)
  75. if adir and os.access(adir, os.F_OK):
  76. os.chdir(adir)
  77. if data.getVarFlag(func, "python", d):
  78. exec_func_python(func, d)
  79. else:
  80. exec_func_shell(func, d)
  81. if os.path.exists(prevdir):
  82. os.chdir(prevdir)
  83. def exec_func_python(func, d):
  84. """Execute a python BB 'function'"""
  85. import re, os
  86. tmp = "def " + func + "():\n%s" % data.getVar(func, d)
  87. tmp += '\n' + func + '()'
  88. comp = utils.better_compile(tmp, func, bb.data.getVar('FILE', d, 1) )
  89. prevdir = os.getcwd()
  90. g = {} # globals
  91. g['bb'] = bb
  92. g['os'] = os
  93. g['d'] = d
  94. utils.better_exec(comp,g,tmp, bb.data.getVar('FILE',d,1))
  95. if os.path.exists(prevdir):
  96. os.chdir(prevdir)
  97. def exec_func_shell(func, d):
  98. """Execute a shell BB 'function' Returns true if execution was successful.
  99. For this, it creates a bash shell script in the tmp dectory, writes the local
  100. data into it and finally executes. The output of the shell will end in a log file and stdout.
  101. Note on directory behavior. The 'dirs' varflag should contain a list
  102. of the directories you need created prior to execution. The last
  103. item in the list is where we will chdir/cd to.
  104. """
  105. import sys
  106. deps = data.getVarFlag(func, 'deps', d)
  107. check = data.getVarFlag(func, 'check', d)
  108. interact = data.getVarFlag(func, 'interactive', d)
  109. if check in globals():
  110. if globals()[check](func, deps):
  111. return
  112. global logfile
  113. t = data.getVar('T', d, 1)
  114. if not t:
  115. return 0
  116. mkdirhier(t)
  117. logfile = "%s/log.%s.%s" % (t, func, str(os.getpid()))
  118. runfile = "%s/run.%s.%s" % (t, func, str(os.getpid()))
  119. f = open(runfile, "w")
  120. f.write("#!/bin/sh -e\n")
  121. if bb.msg.debug_level['default'] > 0: f.write("set -x\n")
  122. data.emit_env(f, d)
  123. f.write("cd %s\n" % os.getcwd())
  124. if func: f.write("%s\n" % func)
  125. f.close()
  126. os.chmod(runfile, 0775)
  127. if not func:
  128. bb.msg.error(bb.msg.domain.Build, "Function not specified")
  129. raise FuncFailed()
  130. # open logs
  131. si = file('/dev/null', 'r')
  132. try:
  133. if bb.msg.debug_level['default'] > 0:
  134. so = os.popen("tee \"%s\"" % logfile, "w")
  135. else:
  136. so = file(logfile, 'w')
  137. except OSError, e:
  138. bb.msg.error(bb.msg.domain.Build, "opening log file: %s" % e)
  139. pass
  140. se = so
  141. if not interact:
  142. # dup the existing fds so we dont lose them
  143. osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()]
  144. oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()]
  145. ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()]
  146. # replace those fds with our own
  147. os.dup2(si.fileno(), osi[1])
  148. os.dup2(so.fileno(), oso[1])
  149. os.dup2(se.fileno(), ose[1])
  150. # execute function
  151. prevdir = os.getcwd()
  152. if data.getVarFlag(func, "fakeroot", d):
  153. maybe_fakeroot = "PATH=\"%s\" fakeroot " % bb.data.getVar("PATH", d, 1)
  154. else:
  155. maybe_fakeroot = ''
  156. ret = os.system('%ssh -e %s' % (maybe_fakeroot, runfile))
  157. try:
  158. os.chdir(prevdir)
  159. except:
  160. pass
  161. if not interact:
  162. # restore the backups
  163. os.dup2(osi[0], osi[1])
  164. os.dup2(oso[0], oso[1])
  165. os.dup2(ose[0], ose[1])
  166. # close our logs
  167. si.close()
  168. so.close()
  169. se.close()
  170. # close the backup fds
  171. os.close(osi[0])
  172. os.close(oso[0])
  173. os.close(ose[0])
  174. if ret==0:
  175. if bb.msg.debug_level['default'] > 0:
  176. os.remove(runfile)
  177. # os.remove(logfile)
  178. return
  179. else:
  180. bb.msg.error(bb.msg.domain.Build, "function %s failed" % func)
  181. if data.getVar("BBINCLUDELOGS", d):
  182. bb.msg.error(bb.msg.domain.Build, "log data follows (%s)" % logfile)
  183. f = open(logfile, "r")
  184. while True:
  185. l = f.readline()
  186. if l == '':
  187. break
  188. l = l.rstrip()
  189. print '| %s' % l
  190. f.close()
  191. else:
  192. bb.msg.error(bb.msg.domain.Build, "see log in %s" % logfile)
  193. raise FuncFailed( logfile )
  194. def exec_task(task, d):
  195. """Execute an BB 'task'
  196. The primary difference between executing a task versus executing
  197. a function is that a task exists in the task digraph, and therefore
  198. has dependencies amongst other tasks."""
  199. # check if the task is in the graph..
  200. task_graph = data.getVar('_task_graph', d)
  201. if not task_graph:
  202. task_graph = bb.digraph()
  203. data.setVar('_task_graph', task_graph, d)
  204. task_cache = data.getVar('_task_cache', d)
  205. if not task_cache:
  206. task_cache = []
  207. data.setVar('_task_cache', task_cache, d)
  208. if not task_graph.hasnode(task):
  209. raise EventException("Missing node in task graph", InvalidTask(task, d))
  210. # check whether this task needs executing..
  211. if not data.getVarFlag(task, 'force', d):
  212. if stamp_is_current(task, d):
  213. return 1
  214. # follow digraph path up, then execute our way back down
  215. def execute(graph, item):
  216. if data.getVarFlag(item, 'task', d):
  217. if item in task_cache:
  218. return 1
  219. if task != item:
  220. # deeper than toplevel, exec w/ deps
  221. exec_task(item, d)
  222. return 1
  223. try:
  224. bb.msg.debug(1, bb.msg.domain.Build, "Executing task %s" % item)
  225. old_overrides = data.getVar('OVERRIDES', d, 0)
  226. localdata = data.createCopy(d)
  227. data.setVar('OVERRIDES', 'task_%s:%s' % (item, old_overrides), localdata)
  228. data.update_data(localdata)
  229. event.fire(TaskStarted(item, localdata))
  230. exec_func(item, localdata)
  231. event.fire(TaskSucceeded(item, localdata))
  232. task_cache.append(item)
  233. data.setVar('_task_cache', task_cache, d)
  234. except FuncFailed, reason:
  235. bb.msg.note(1, bb.msg.domain.Build, "Task failed: %s" % reason )
  236. failedevent = TaskFailed(item, d)
  237. event.fire(failedevent)
  238. raise EventException("Function failed in task: %s" % reason, failedevent)
  239. if data.getVarFlag(task, 'dontrundeps', d):
  240. execute(None, task)
  241. else:
  242. task_graph.walkdown(task, execute)
  243. # make stamp, or cause event and raise exception
  244. if not data.getVarFlag(task, 'nostamp', d):
  245. mkstamp(task, d)
  246. def stamp_is_current_cache(dataCache, file_name, task, checkdeps = 1):
  247. """
  248. Check status of a given task's stamp.
  249. Returns 0 if it is not current and needs updating.
  250. Same as stamp_is_current but works against the dataCache instead of d
  251. """
  252. task_graph = dataCache.task_queues[file_name]
  253. if not dataCache.stamp[file_name]:
  254. return 0
  255. stampfile = "%s.%s" % (dataCache.stamp[file_name], task)
  256. if not os.access(stampfile, os.F_OK):
  257. return 0
  258. if checkdeps == 0:
  259. return 1
  260. import stat
  261. tasktime = os.stat(stampfile)[stat.ST_MTIME]
  262. _deps = []
  263. def checkStamp(graph, task):
  264. # check for existance
  265. if 'nostamp' in dataCache.task_deps[file_name] and task in dataCache.task_deps[file_name]['nostamp']:
  266. return 1
  267. if not stamp_is_current_cache(dataCache, file_name, task, 0):
  268. return 0
  269. depfile = "%s.%s" % (dataCache.stamp[file_name], task)
  270. deptime = os.stat(depfile)[stat.ST_MTIME]
  271. if deptime > tasktime:
  272. return 0
  273. return 1
  274. return task_graph.walkdown(task, checkStamp)
  275. def stamp_is_current(task, d, checkdeps = 1):
  276. """
  277. Check status of a given task's stamp.
  278. Returns 0 if it is not current and needs updating.
  279. """
  280. task_graph = data.getVar('_task_graph', d)
  281. if not task_graph:
  282. task_graph = bb.digraph()
  283. data.setVar('_task_graph', task_graph, d)
  284. stamp = data.getVar('STAMP', d)
  285. if not stamp:
  286. return 0
  287. stampfile = "%s.%s" % (data.expand(stamp, d), task)
  288. if not os.access(stampfile, os.F_OK):
  289. return 0
  290. if checkdeps == 0:
  291. return 1
  292. import stat
  293. tasktime = os.stat(stampfile)[stat.ST_MTIME]
  294. _deps = []
  295. def checkStamp(graph, task):
  296. # check for existance
  297. if data.getVarFlag(task, 'nostamp', d):
  298. return 1
  299. if not stamp_is_current(task, d, 0):
  300. return 0
  301. depfile = "%s.%s" % (data.expand(stamp, d), task)
  302. deptime = os.stat(depfile)[stat.ST_MTIME]
  303. if deptime > tasktime:
  304. return 0
  305. return 1
  306. return task_graph.walkdown(task, checkStamp)
  307. def md5_is_current(task):
  308. """Check if a md5 file for a given task is current"""
  309. def mkstamp(task, d):
  310. """Creates/updates a stamp for a given task"""
  311. stamp = data.getVar('STAMP', d)
  312. if not stamp:
  313. return
  314. stamp = "%s.%s" % (data.expand(stamp, d), task)
  315. mkdirhier(os.path.dirname(stamp))
  316. # Remove the file and recreate to force timestamp
  317. # change on broken NFS filesystems
  318. if os.access(stamp, os.F_OK):
  319. os.remove(stamp)
  320. f = open(stamp, "w")
  321. f.close()
  322. def add_task(task, deps, d):
  323. task_graph = data.getVar('_task_graph', d)
  324. if not task_graph:
  325. task_graph = bb.digraph()
  326. data.setVarFlag(task, 'task', 1, d)
  327. task_graph.addnode(task, None)
  328. for dep in deps:
  329. if not task_graph.hasnode(dep):
  330. task_graph.addnode(dep, None)
  331. task_graph.addnode(task, dep)
  332. # don't assume holding a reference
  333. data.setVar('_task_graph', task_graph, d)
  334. task_deps = data.getVar('_task_deps', d)
  335. if not task_deps:
  336. task_deps = {}
  337. def getTask(name):
  338. deptask = data.getVarFlag(task, name, d)
  339. if deptask:
  340. if not name in task_deps:
  341. task_deps[name] = {}
  342. task_deps[name][task] = deptask
  343. getTask('deptask')
  344. getTask('rdeptask')
  345. getTask('recrdeptask')
  346. getTask('nostamp')
  347. data.setVar('_task_deps', task_deps, d)
  348. def remove_task(task, kill, d):
  349. """Remove an BB 'task'.
  350. If kill is 1, also remove tasks that depend on this task."""
  351. task_graph = data.getVar('_task_graph', d)
  352. if not task_graph:
  353. task_graph = bb.digraph()
  354. if not task_graph.hasnode(task):
  355. return
  356. data.delVarFlag(task, 'task', d)
  357. ref = 1
  358. if kill == 1:
  359. ref = 2
  360. task_graph.delnode(task, ref)
  361. data.setVar('_task_graph', task_graph, d)
  362. def task_exists(task, d):
  363. task_graph = data.getVar('_task_graph', d)
  364. if not task_graph:
  365. task_graph = bb.digraph()
  366. data.setVar('_task_graph', task_graph, d)
  367. return task_graph.hasnode(task)