build.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388
  1. # ex:ts=4:sw=4:sts=4:et
  2. # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
  3. #
  4. # BitBake 'Build' implementation
  5. #
  6. # Core code for function execution and task handling in the
  7. # BitBake build tools.
  8. #
  9. # Copyright (C) 2003, 2004 Chris Larson
  10. #
  11. # Based on Gentoo's portage.py.
  12. #
  13. # This program is free software; you can redistribute it and/or modify
  14. # it under the terms of the GNU General Public License version 2 as
  15. # published by the Free Software Foundation.
  16. #
  17. # This program is distributed in the hope that it will be useful,
  18. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  19. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  20. # GNU General Public License for more details.
  21. #
  22. # You should have received a copy of the GNU General Public License along
  23. # with this program; if not, write to the Free Software Foundation, Inc.,
  24. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  25. #
  26. #Based on functions from the base bb module, Copyright 2003 Holger Schurig
  27. from bb import data, event, mkdirhier, utils
  28. import bb, os, sys
  29. import bb.utils
  30. # When we execute a python function we'd like certain things
  31. # in all namespaces, hence we add them to __builtins__
  32. # If we do not do this and use the exec globals, they will
  33. # not be available to subfunctions.
  34. __builtins__['bb'] = bb
  35. __builtins__['os'] = os
  36. # events
  37. class FuncFailed(Exception):
  38. """
  39. Executed function failed
  40. First parameter a message
  41. Second paramter is a logfile (optional)
  42. """
  43. class EventException(Exception):
  44. """Exception which is associated with an Event."""
  45. def __init__(self, msg, event):
  46. self.args = msg, event
  47. class TaskBase(event.Event):
  48. """Base class for task events"""
  49. def __init__(self, t, d ):
  50. self._task = t
  51. self._package = bb.data.getVar("PF", d, 1)
  52. event.Event.__init__(self)
  53. self._message = "package %s: task %s: %s" % (bb.data.getVar("PF", d, 1), t, bb.event.getName(self)[4:])
  54. def getTask(self):
  55. return self._task
  56. def setTask(self, task):
  57. self._task = task
  58. task = property(getTask, setTask, None, "task property")
  59. class TaskStarted(TaskBase):
  60. """Task execution started"""
  61. class TaskSucceeded(TaskBase):
  62. """Task execution completed"""
  63. class TaskFailed(TaskBase):
  64. """Task execution failed"""
  65. def __init__(self, msg, logfile, t, d ):
  66. self.logfile = logfile
  67. self.msg = msg
  68. TaskBase.__init__(self, t, d)
  69. class InvalidTask(TaskBase):
  70. """Invalid Task"""
  71. # functions
  72. def exec_func(func, d, dirs = None):
  73. """Execute an BB 'function'"""
  74. body = data.getVar(func, d)
  75. if not body:
  76. return
  77. flags = data.getVarFlags(func, d)
  78. for item in ['deps', 'check', 'interactive', 'python', 'cleandirs', 'dirs', 'lockfiles', 'fakeroot']:
  79. if not item in flags:
  80. flags[item] = None
  81. ispython = flags['python']
  82. cleandirs = flags['cleandirs']
  83. if cleandirs:
  84. for cdir in data.expand(cleandirs, d).split():
  85. os.system("rm -rf %s" % cdir)
  86. if dirs is None:
  87. dirs = flags['dirs']
  88. if dirs:
  89. dirs = data.expand(dirs, d).split()
  90. if dirs:
  91. for adir in dirs:
  92. bb.utils.mkdirhier(adir)
  93. adir = dirs[-1]
  94. else:
  95. adir = data.getVar('B', d, 1)
  96. # Save current directory
  97. try:
  98. prevdir = os.getcwd()
  99. except OSError:
  100. prevdir = data.getVar('TOPDIR', d, True)
  101. # Setup logfiles
  102. t = data.getVar('T', d, 1)
  103. if not t:
  104. raise SystemExit("T variable not set, unable to build")
  105. bb.utils.mkdirhier(t)
  106. logfile = "%s/log.%s.%s" % (t, func, str(os.getpid()))
  107. runfile = "%s/run.%s.%s" % (t, func, str(os.getpid()))
  108. # Change to correct directory (if specified)
  109. if adir and os.access(adir, os.F_OK):
  110. os.chdir(adir)
  111. # Handle logfiles
  112. si = file('/dev/null', 'r')
  113. try:
  114. if bb.msg.debug_level['default'] > 0 or ispython:
  115. so = os.popen("tee \"%s\"" % logfile, "w")
  116. else:
  117. so = file(logfile, 'w')
  118. except OSError as e:
  119. bb.msg.error(bb.msg.domain.Build, "opening log file: %s" % e)
  120. pass
  121. se = so
  122. # Dup the existing fds so we dont lose them
  123. osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()]
  124. oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()]
  125. ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()]
  126. # Replace those fds with our own
  127. os.dup2(si.fileno(), osi[1])
  128. os.dup2(so.fileno(), oso[1])
  129. os.dup2(se.fileno(), ose[1])
  130. locks = []
  131. lockfiles = flags['lockfiles']
  132. if lockfiles:
  133. for lock in data.expand(lockfiles, d).split():
  134. locks.append(bb.utils.lockfile(lock))
  135. try:
  136. # Run the function
  137. if ispython:
  138. exec_func_python(func, d, runfile, logfile)
  139. else:
  140. exec_func_shell(func, d, runfile, logfile, flags)
  141. # Restore original directory
  142. try:
  143. os.chdir(prevdir)
  144. except:
  145. pass
  146. finally:
  147. # Unlock any lockfiles
  148. for lock in locks:
  149. bb.utils.unlockfile(lock)
  150. # Restore the backup fds
  151. os.dup2(osi[0], osi[1])
  152. os.dup2(oso[0], oso[1])
  153. os.dup2(ose[0], ose[1])
  154. # Close our logs
  155. si.close()
  156. so.close()
  157. se.close()
  158. if os.path.exists(logfile) and os.path.getsize(logfile) == 0:
  159. bb.msg.debug(2, bb.msg.domain.Build, "Zero size logfile %s, removing" % logfile)
  160. os.remove(logfile)
  161. # Close the backup fds
  162. os.close(osi[0])
  163. os.close(oso[0])
  164. os.close(ose[0])
  165. def exec_func_python(func, d, runfile, logfile):
  166. """Execute a python BB 'function'"""
  167. bbfile = bb.data.getVar('FILE', d, 1)
  168. tmp = "def " + func + "(d):\n%s" % data.getVar(func, d)
  169. tmp += '\n' + func + '(d)'
  170. f = open(runfile, "w")
  171. f.write(tmp)
  172. comp = utils.better_compile(tmp, func, bbfile)
  173. try:
  174. utils.better_exec(comp, {"d": d}, tmp, bbfile)
  175. except:
  176. (t, value, tb) = sys.exc_info()
  177. if t in [bb.parse.SkipPackage, bb.build.FuncFailed]:
  178. raise
  179. raise FuncFailed("Function %s failed" % func, logfile)
  180. def exec_func_shell(func, d, runfile, logfile, flags):
  181. """Execute a shell BB 'function' Returns true if execution was successful.
  182. For this, it creates a bash shell script in the tmp dectory, writes the local
  183. data into it and finally executes. The output of the shell will end in a log file and stdout.
  184. Note on directory behavior. The 'dirs' varflag should contain a list
  185. of the directories you need created prior to execution. The last
  186. item in the list is where we will chdir/cd to.
  187. """
  188. deps = flags['deps']
  189. check = flags['check']
  190. if check in globals():
  191. if globals()[check](func, deps):
  192. return
  193. f = open(runfile, "w")
  194. f.write("#!/bin/sh -e\n")
  195. if bb.msg.debug_level['default'] > 0: f.write("set -x\n")
  196. data.emit_env(f, d)
  197. f.write("cd %s\n" % os.getcwd())
  198. if func: f.write("%s\n" % func)
  199. f.close()
  200. os.chmod(runfile, 0775)
  201. if not func:
  202. raise FuncFailed("Function not specified for exec_func_shell")
  203. # execute function
  204. if flags['fakeroot']:
  205. maybe_fakeroot = "PATH=\"%s\" %s " % (bb.data.getVar("PATH", d, 1), bb.data.getVar("FAKEROOT", d, 1) or "fakeroot")
  206. else:
  207. maybe_fakeroot = ''
  208. lang_environment = "LC_ALL=C "
  209. ret = os.system('%s%ssh -e %s' % (lang_environment, maybe_fakeroot, runfile))
  210. if ret == 0:
  211. return
  212. raise FuncFailed("function %s failed" % func, logfile)
  213. def exec_task(task, d):
  214. """Execute an BB 'task'
  215. The primary difference between executing a task versus executing
  216. a function is that a task exists in the task digraph, and therefore
  217. has dependencies amongst other tasks."""
  218. # Check whther this is a valid task
  219. if not data.getVarFlag(task, 'task', d):
  220. raise EventException("No such task", InvalidTask(task, d))
  221. try:
  222. bb.msg.debug(1, bb.msg.domain.Build, "Executing task %s" % task)
  223. old_overrides = data.getVar('OVERRIDES', d, 0)
  224. localdata = data.createCopy(d)
  225. data.setVar('OVERRIDES', 'task-%s:%s' % (task[3:], old_overrides), localdata)
  226. data.update_data(localdata)
  227. data.expandKeys(localdata)
  228. event.fire(TaskStarted(task, localdata), localdata)
  229. exec_func(task, localdata)
  230. event.fire(TaskSucceeded(task, localdata), localdata)
  231. except FuncFailed as message:
  232. # Try to extract the optional logfile
  233. try:
  234. (msg, logfile) = message
  235. except:
  236. logfile = None
  237. msg = message
  238. bb.msg.note(1, bb.msg.domain.Build, "Task failed: %s" % message )
  239. failedevent = TaskFailed(msg, logfile, task, d)
  240. event.fire(failedevent, d)
  241. raise EventException("Function failed in task: %s" % message, failedevent)
  242. # make stamp, or cause event and raise exception
  243. if not data.getVarFlag(task, 'nostamp', d) and not data.getVarFlag(task, 'selfstamp', d):
  244. make_stamp(task, d)
  245. def extract_stamp(d, fn):
  246. """
  247. Extracts stamp format which is either a data dictionary (fn unset)
  248. or a dataCache entry (fn set).
  249. """
  250. if fn:
  251. return d.stamp[fn]
  252. return data.getVar('STAMP', d, 1)
  253. def stamp_internal(task, d, file_name):
  254. """
  255. Internal stamp helper function
  256. Removes any stamp for the given task
  257. Makes sure the stamp directory exists
  258. Returns the stamp path+filename
  259. """
  260. stamp = extract_stamp(d, file_name)
  261. if not stamp:
  262. return
  263. stamp = "%s.%s" % (stamp, task)
  264. bb.utils.mkdirhier(os.path.dirname(stamp))
  265. # Remove the file and recreate to force timestamp
  266. # change on broken NFS filesystems
  267. if os.access(stamp, os.F_OK):
  268. os.remove(stamp)
  269. return stamp
  270. def make_stamp(task, d, file_name = None):
  271. """
  272. Creates/updates a stamp for a given task
  273. (d can be a data dict or dataCache)
  274. """
  275. stamp = stamp_internal(task, d, file_name)
  276. if stamp:
  277. f = open(stamp, "w")
  278. f.close()
  279. def del_stamp(task, d, file_name = None):
  280. """
  281. Removes a stamp for a given task
  282. (d can be a data dict or dataCache)
  283. """
  284. stamp_internal(task, d, file_name)
  285. def add_tasks(tasklist, d):
  286. task_deps = data.getVar('_task_deps', d)
  287. if not task_deps:
  288. task_deps = {}
  289. if not 'tasks' in task_deps:
  290. task_deps['tasks'] = []
  291. if not 'parents' in task_deps:
  292. task_deps['parents'] = {}
  293. for task in tasklist:
  294. task = data.expand(task, d)
  295. data.setVarFlag(task, 'task', 1, d)
  296. if not task in task_deps['tasks']:
  297. task_deps['tasks'].append(task)
  298. flags = data.getVarFlags(task, d)
  299. def getTask(name):
  300. if not name in task_deps:
  301. task_deps[name] = {}
  302. if name in flags:
  303. deptask = data.expand(flags[name], d)
  304. task_deps[name][task] = deptask
  305. getTask('depends')
  306. getTask('deptask')
  307. getTask('rdeptask')
  308. getTask('recrdeptask')
  309. getTask('nostamp')
  310. task_deps['parents'][task] = []
  311. for dep in flags['deps']:
  312. dep = data.expand(dep, d)
  313. task_deps['parents'][task].append(dep)
  314. # don't assume holding a reference
  315. data.setVar('_task_deps', task_deps, d)
  316. def remove_task(task, kill, d):
  317. """Remove an BB 'task'.
  318. If kill is 1, also remove tasks that depend on this task."""
  319. data.delVarFlag(task, 'task', d)