cooker.py 39 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004
  1. #!/usr/bin/env python
  2. # ex:ts=4:sw=4:sts=4:et
  3. # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
  4. #
  5. # Copyright (C) 2003, 2004 Chris Larson
  6. # Copyright (C) 2003, 2004 Phil Blundell
  7. # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
  8. # Copyright (C) 2005 Holger Hans Peter Freyther
  9. # Copyright (C) 2005 ROAD GmbH
  10. # Copyright (C) 2006 - 2007 Richard Purdie
  11. #
  12. # This program is free software; you can redistribute it and/or modify
  13. # it under the terms of the GNU General Public License version 2 as
  14. # published by the Free Software Foundation.
  15. #
  16. # This program is distributed in the hope that it will be useful,
  17. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  18. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  19. # GNU General Public License for more details.
  20. #
  21. # You should have received a copy of the GNU General Public License along
  22. # with this program; if not, write to the Free Software Foundation, Inc.,
  23. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  24. from __future__ import print_function
  25. import sys, os, glob, os.path, re, time
  26. import sre_constants
  27. from cStringIO import StringIO
  28. from contextlib import closing
  29. import bb
  30. from bb import utils, data, parse, event, cache, providers, taskdata, command, runqueue
  31. class MultipleMatches(Exception):
  32. """
  33. Exception raised when multiple file matches are found
  34. """
  35. class ParsingErrorsFound(Exception):
  36. """
  37. Exception raised when parsing errors are found
  38. """
  39. class NothingToBuild(Exception):
  40. """
  41. Exception raised when there is nothing to build
  42. """
  43. # Different states cooker can be in
  44. cookerClean = 1
  45. cookerParsing = 2
  46. cookerParsed = 3
  47. # Different action states the cooker can be in
  48. cookerRun = 1 # Cooker is running normally
  49. cookerShutdown = 2 # Active tasks should be brought to a controlled stop
  50. cookerStop = 3 # Stop, now!
  51. #============================================================================#
  52. # BBCooker
  53. #============================================================================#
  54. class BBCooker:
  55. """
  56. Manages one bitbake build run
  57. """
  58. def __init__(self, configuration, server):
  59. self.status = None
  60. self.cache = None
  61. self.bb_cache = None
  62. self.server = server.BitBakeServer(self)
  63. self.configuration = configuration
  64. self.configuration.data = bb.data.init()
  65. bb.data.inheritFromOS(self.configuration.data)
  66. self.parseConfigurationFiles(self.configuration.file)
  67. if not self.configuration.cmd:
  68. self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data, True) or "build"
  69. bbpkgs = bb.data.getVar('BBPKGS', self.configuration.data, True)
  70. if bbpkgs and len(self.configuration.pkgs_to_build) == 0:
  71. self.configuration.pkgs_to_build.extend(bbpkgs.split())
  72. #
  73. # Special updated configuration we use for firing events
  74. #
  75. self.configuration.event_data = bb.data.createCopy(self.configuration.data)
  76. bb.data.update_data(self.configuration.event_data)
  77. # TOSTOP must not be set or our children will hang when they output
  78. fd = sys.stdout.fileno()
  79. if os.isatty(fd):
  80. import termios
  81. tcattr = termios.tcgetattr(fd)
  82. if tcattr[3] & termios.TOSTOP:
  83. bb.msg.note(1, bb.msg.domain.Build, "The terminal had the TOSTOP bit set, clearing...")
  84. tcattr[3] = tcattr[3] & ~termios.TOSTOP
  85. termios.tcsetattr(fd, termios.TCSANOW, tcattr)
  86. self.command = bb.command.Command(self)
  87. self.cookerState = cookerClean
  88. self.cookerAction = cookerRun
  89. def parseConfiguration(self):
  90. # Change nice level if we're asked to
  91. nice = bb.data.getVar("BB_NICE_LEVEL", self.configuration.data, True)
  92. if nice:
  93. curnice = os.nice(0)
  94. nice = int(nice) - curnice
  95. bb.msg.note(2, bb.msg.domain.Build, "Renice to %s " % os.nice(nice))
  96. def parseCommandLine(self):
  97. # Parse any commandline into actions
  98. if self.configuration.show_environment:
  99. self.commandlineAction = None
  100. if 'world' in self.configuration.pkgs_to_build:
  101. bb.msg.error(bb.msg.domain.Build, "'world' is not a valid target for --environment.")
  102. elif len(self.configuration.pkgs_to_build) > 1:
  103. bb.msg.error(bb.msg.domain.Build, "Only one target can be used with the --environment option.")
  104. elif self.configuration.buildfile and len(self.configuration.pkgs_to_build) > 0:
  105. bb.msg.error(bb.msg.domain.Build, "No target should be used with the --environment and --buildfile options.")
  106. elif len(self.configuration.pkgs_to_build) > 0:
  107. self.commandlineAction = ["showEnvironmentTarget", self.configuration.pkgs_to_build]
  108. else:
  109. self.commandlineAction = ["showEnvironment", self.configuration.buildfile]
  110. elif self.configuration.buildfile is not None:
  111. self.commandlineAction = ["buildFile", self.configuration.buildfile, self.configuration.cmd]
  112. elif self.configuration.revisions_changed:
  113. self.commandlineAction = ["compareRevisions"]
  114. elif self.configuration.show_versions:
  115. self.commandlineAction = ["showVersions"]
  116. elif self.configuration.parse_only:
  117. self.commandlineAction = ["parseFiles"]
  118. elif self.configuration.dot_graph:
  119. if self.configuration.pkgs_to_build:
  120. self.commandlineAction = ["generateDotGraph", self.configuration.pkgs_to_build, self.configuration.cmd]
  121. else:
  122. self.commandlineAction = None
  123. bb.msg.error(bb.msg.domain.Build, "Please specify a package name for dependency graph generation.")
  124. else:
  125. if self.configuration.pkgs_to_build:
  126. self.commandlineAction = ["buildTargets", self.configuration.pkgs_to_build, self.configuration.cmd]
  127. else:
  128. self.commandlineAction = None
  129. bb.msg.error(bb.msg.domain.Build, "Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.")
  130. def runCommands(self, server, data, abort):
  131. """
  132. Run any queued asynchronous command
  133. This is done by the idle handler so it runs in true context rather than
  134. tied to any UI.
  135. """
  136. return self.command.runAsyncCommand()
  137. def showVersions(self):
  138. # Need files parsed
  139. self.updateCache()
  140. pkg_pn = self.status.pkg_pn
  141. preferred_versions = {}
  142. latest_versions = {}
  143. # Sort by priority
  144. for pn in pkg_pn:
  145. (last_ver, last_file, pref_ver, pref_file) = bb.providers.findBestProvider(pn, self.configuration.data, self.status)
  146. preferred_versions[pn] = (pref_ver, pref_file)
  147. latest_versions[pn] = (last_ver, last_file)
  148. bb.msg.plain("%-35s %25s %25s" % ("Package Name", "Latest Version", "Preferred Version"))
  149. bb.msg.plain("%-35s %25s %25s\n" % ("============", "==============", "================="))
  150. for p in sorted(pkg_pn):
  151. pref = preferred_versions[p]
  152. latest = latest_versions[p]
  153. prefstr = pref[0][0] + ":" + pref[0][1] + '-' + pref[0][2]
  154. lateststr = latest[0][0] + ":" + latest[0][1] + "-" + latest[0][2]
  155. if pref == latest:
  156. prefstr = ""
  157. bb.msg.plain("%-35s %25s %25s" % (p, lateststr, prefstr))
  158. def compareRevisions(self):
  159. ret = bb.fetch.fetcher_compare_revisons(self.configuration.data)
  160. bb.event.fire(bb.command.CookerCommandSetExitCode(ret), self.configuration.event_data)
  161. def showEnvironment(self, buildfile = None, pkgs_to_build = []):
  162. """
  163. Show the outer or per-package environment
  164. """
  165. fn = None
  166. envdata = None
  167. if buildfile:
  168. self.cb = None
  169. self.bb_cache = bb.cache.init(self)
  170. fn = self.matchFile(buildfile)
  171. elif len(pkgs_to_build) == 1:
  172. self.updateCache()
  173. localdata = data.createCopy(self.configuration.data)
  174. bb.data.update_data(localdata)
  175. bb.data.expandKeys(localdata)
  176. taskdata = bb.taskdata.TaskData(self.configuration.abort)
  177. taskdata.add_provider(localdata, self.status, pkgs_to_build[0])
  178. taskdata.add_unresolved(localdata, self.status)
  179. targetid = taskdata.getbuild_id(pkgs_to_build[0])
  180. fnid = taskdata.build_targets[targetid][0]
  181. fn = taskdata.fn_index[fnid]
  182. else:
  183. envdata = self.configuration.data
  184. if fn:
  185. try:
  186. envdata = self.bb_cache.loadDataFull(fn, self.get_file_appends(fn), self.configuration.data)
  187. except IOError as e:
  188. bb.msg.error(bb.msg.domain.Parsing, "Unable to read %s: %s" % (fn, e))
  189. raise
  190. except Exception as e:
  191. bb.msg.error(bb.msg.domain.Parsing, "%s" % e)
  192. raise
  193. # emit variables and shell functions
  194. try:
  195. data.update_data(envdata)
  196. with closing(StringIO()) as env:
  197. data.emit_env(env, envdata, True)
  198. bb.msg.plain(env.getvalue())
  199. except Exception as e:
  200. bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
  201. # emit the metadata which isnt valid shell
  202. data.expandKeys(envdata)
  203. for e in envdata.keys():
  204. if data.getVarFlag( e, 'python', envdata ):
  205. bb.msg.plain("\npython %s () {\n%s}\n" % (e, data.getVar(e, envdata, 1)))
  206. def generateDepTreeData(self, pkgs_to_build, task):
  207. """
  208. Create a dependency tree of pkgs_to_build, returning the data.
  209. """
  210. # Need files parsed
  211. self.updateCache()
  212. # If we are told to do the None task then query the default task
  213. if (task == None):
  214. task = self.configuration.cmd
  215. pkgs_to_build = self.checkPackages(pkgs_to_build)
  216. localdata = data.createCopy(self.configuration.data)
  217. bb.data.update_data(localdata)
  218. bb.data.expandKeys(localdata)
  219. taskdata = bb.taskdata.TaskData(self.configuration.abort)
  220. runlist = []
  221. for k in pkgs_to_build:
  222. taskdata.add_provider(localdata, self.status, k)
  223. runlist.append([k, "do_%s" % task])
  224. taskdata.add_unresolved(localdata, self.status)
  225. rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
  226. rq.prepare_runqueue()
  227. seen_fnids = []
  228. depend_tree = {}
  229. depend_tree["depends"] = {}
  230. depend_tree["tdepends"] = {}
  231. depend_tree["pn"] = {}
  232. depend_tree["rdepends-pn"] = {}
  233. depend_tree["packages"] = {}
  234. depend_tree["rdepends-pkg"] = {}
  235. depend_tree["rrecs-pkg"] = {}
  236. for task in range(len(rq.runq_fnid)):
  237. taskname = rq.runq_task[task]
  238. fnid = rq.runq_fnid[task]
  239. fn = taskdata.fn_index[fnid]
  240. pn = self.status.pkg_fn[fn]
  241. version = "%s:%s-%s" % self.status.pkg_pepvpr[fn]
  242. if pn not in depend_tree["pn"]:
  243. depend_tree["pn"][pn] = {}
  244. depend_tree["pn"][pn]["filename"] = fn
  245. depend_tree["pn"][pn]["version"] = version
  246. for dep in rq.runq_depends[task]:
  247. depfn = taskdata.fn_index[rq.runq_fnid[dep]]
  248. deppn = self.status.pkg_fn[depfn]
  249. dotname = "%s.%s" % (pn, rq.runq_task[task])
  250. if not dotname in depend_tree["tdepends"]:
  251. depend_tree["tdepends"][dotname] = []
  252. depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, rq.runq_task[dep]))
  253. if fnid not in seen_fnids:
  254. seen_fnids.append(fnid)
  255. packages = []
  256. depend_tree["depends"][pn] = []
  257. for dep in taskdata.depids[fnid]:
  258. depend_tree["depends"][pn].append(taskdata.build_names_index[dep])
  259. depend_tree["rdepends-pn"][pn] = []
  260. for rdep in taskdata.rdepids[fnid]:
  261. depend_tree["rdepends-pn"][pn].append(taskdata.run_names_index[rdep])
  262. rdepends = self.status.rundeps[fn]
  263. for package in rdepends:
  264. depend_tree["rdepends-pkg"][package] = []
  265. for rdepend in rdepends[package]:
  266. depend_tree["rdepends-pkg"][package].append(rdepend)
  267. packages.append(package)
  268. rrecs = self.status.runrecs[fn]
  269. for package in rrecs:
  270. depend_tree["rrecs-pkg"][package] = []
  271. for rdepend in rrecs[package]:
  272. depend_tree["rrecs-pkg"][package].append(rdepend)
  273. if not package in packages:
  274. packages.append(package)
  275. for package in packages:
  276. if package not in depend_tree["packages"]:
  277. depend_tree["packages"][package] = {}
  278. depend_tree["packages"][package]["pn"] = pn
  279. depend_tree["packages"][package]["filename"] = fn
  280. depend_tree["packages"][package]["version"] = version
  281. return depend_tree
  282. def generateDepTreeEvent(self, pkgs_to_build, task):
  283. """
  284. Create a task dependency graph of pkgs_to_build.
  285. Generate an event with the result
  286. """
  287. depgraph = self.generateDepTreeData(pkgs_to_build, task)
  288. bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.configuration.data)
  289. def generateDotGraphFiles(self, pkgs_to_build, task):
  290. """
  291. Create a task dependency graph of pkgs_to_build.
  292. Save the result to a set of .dot files.
  293. """
  294. depgraph = self.generateDepTreeData(pkgs_to_build, task)
  295. # Prints a flattened form of package-depends below where subpackages of a package are merged into the main pn
  296. depends_file = file('pn-depends.dot', 'w' )
  297. print("digraph depends {", file=depends_file)
  298. for pn in depgraph["pn"]:
  299. fn = depgraph["pn"][pn]["filename"]
  300. version = depgraph["pn"][pn]["version"]
  301. print('"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn), file=depends_file)
  302. for pn in depgraph["depends"]:
  303. for depend in depgraph["depends"][pn]:
  304. print('"%s" -> "%s"' % (pn, depend), file=depends_file)
  305. for pn in depgraph["rdepends-pn"]:
  306. for rdepend in depgraph["rdepends-pn"][pn]:
  307. print('"%s" -> "%s" [style=dashed]' % (pn, rdepend), file=depends_file)
  308. print("}", file=depends_file)
  309. bb.msg.plain("PN dependencies saved to 'pn-depends.dot'")
  310. depends_file = file('package-depends.dot', 'w' )
  311. print("digraph depends {", file=depends_file)
  312. for package in depgraph["packages"]:
  313. pn = depgraph["packages"][package]["pn"]
  314. fn = depgraph["packages"][package]["filename"]
  315. version = depgraph["packages"][package]["version"]
  316. if package == pn:
  317. print('"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn), file=depends_file)
  318. else:
  319. print('"%s" [label="%s(%s) %s\\n%s"]' % (package, package, pn, version, fn), file=depends_file)
  320. for depend in depgraph["depends"][pn]:
  321. print('"%s" -> "%s"' % (package, depend), file=depends_file)
  322. for package in depgraph["rdepends-pkg"]:
  323. for rdepend in depgraph["rdepends-pkg"][package]:
  324. print('"%s" -> "%s" [style=dashed]' % (package, rdepend), file=depends_file)
  325. for package in depgraph["rrecs-pkg"]:
  326. for rdepend in depgraph["rrecs-pkg"][package]:
  327. print('"%s" -> "%s" [style=dashed]' % (package, rdepend), file=depends_file)
  328. print("}", file=depends_file)
  329. bb.msg.plain("Package dependencies saved to 'package-depends.dot'")
  330. tdepends_file = file('task-depends.dot', 'w' )
  331. print("digraph depends {", file=tdepends_file)
  332. for task in depgraph["tdepends"]:
  333. (pn, taskname) = task.rsplit(".", 1)
  334. fn = depgraph["pn"][pn]["filename"]
  335. version = depgraph["pn"][pn]["version"]
  336. print('"%s.%s" [label="%s %s\\n%s\\n%s"]' % (pn, taskname, pn, taskname, version, fn), file=tdepends_file)
  337. for dep in depgraph["tdepends"][task]:
  338. print('"%s" -> "%s"' % (task, dep), file=tdepends_file)
  339. print("}", file=tdepends_file)
  340. bb.msg.plain("Task dependencies saved to 'task-depends.dot'")
  341. def buildDepgraph( self ):
  342. all_depends = self.status.all_depends
  343. pn_provides = self.status.pn_provides
  344. localdata = data.createCopy(self.configuration.data)
  345. bb.data.update_data(localdata)
  346. bb.data.expandKeys(localdata)
  347. matched = set()
  348. def calc_bbfile_priority(filename):
  349. for _, _, regex, pri in self.status.bbfile_config_priorities:
  350. if regex.match(filename):
  351. if not regex in matched:
  352. matched.add(regex)
  353. return pri
  354. return 0
  355. # Handle PREFERRED_PROVIDERS
  356. for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, 1) or "").split():
  357. try:
  358. (providee, provider) = p.split(':')
  359. except:
  360. bb.msg.fatal(bb.msg.domain.Provider, "Malformed option in PREFERRED_PROVIDERS variable: %s" % p)
  361. continue
  362. if providee in self.status.preferred and self.status.preferred[providee] != provider:
  363. bb.msg.error(bb.msg.domain.Provider, "conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.status.preferred[providee]))
  364. self.status.preferred[providee] = provider
  365. # Calculate priorities for each file
  366. for p in self.status.pkg_fn:
  367. self.status.bbfile_priority[p] = calc_bbfile_priority(p)
  368. for collection, pattern, regex, _ in self.status.bbfile_config_priorities:
  369. if not regex in matched:
  370. bb.msg.warn(bb.msg.domain.Provider, "No bb files matched BBFILE_PATTERN_%s '%s'" %
  371. (collection, pattern))
  372. def buildWorldTargetList(self):
  373. """
  374. Build package list for "bitbake world"
  375. """
  376. all_depends = self.status.all_depends
  377. pn_provides = self.status.pn_provides
  378. bb.msg.debug(1, bb.msg.domain.Parsing, "collating packages for \"world\"")
  379. for f in self.status.possible_world:
  380. terminal = True
  381. pn = self.status.pkg_fn[f]
  382. for p in pn_provides[pn]:
  383. if p.startswith('virtual/'):
  384. bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to %s provider starting with virtual/" % (f, p))
  385. terminal = False
  386. break
  387. for pf in self.status.providers[p]:
  388. if self.status.pkg_fn[pf] != pn:
  389. bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to both us and %s providing %s" % (f, pf, p))
  390. terminal = False
  391. break
  392. if terminal:
  393. self.status.world_target.add(pn)
  394. # drop reference count now
  395. self.status.possible_world = None
  396. self.status.all_depends = None
  397. def interactiveMode( self ):
  398. """Drop off into a shell"""
  399. try:
  400. from bb import shell
  401. except ImportError as details:
  402. bb.msg.fatal(bb.msg.domain.Parsing, "Sorry, shell not available (%s)" % details )
  403. else:
  404. shell.start( self )
  405. def _findLayerConf(self):
  406. path = os.getcwd()
  407. while path != "/":
  408. bblayers = os.path.join(path, "conf", "bblayers.conf")
  409. if os.path.exists(bblayers):
  410. return bblayers
  411. path, _ = os.path.split(path)
  412. def parseConfigurationFiles(self, files):
  413. try:
  414. data = self.configuration.data
  415. for f in files:
  416. data = bb.parse.handle(f, data)
  417. layerconf = self._findLayerConf()
  418. if layerconf:
  419. bb.msg.debug(2, bb.msg.domain.Parsing, "Found bblayers.conf (%s)" % layerconf)
  420. data = bb.parse.handle(layerconf, data)
  421. layers = (bb.data.getVar('BBLAYERS', data, True) or "").split()
  422. data = bb.data.createCopy(data)
  423. for layer in layers:
  424. bb.msg.debug(2, bb.msg.domain.Parsing, "Adding layer %s" % layer)
  425. bb.data.setVar('LAYERDIR', layer, data)
  426. data = bb.parse.handle(os.path.join(layer, "conf", "layer.conf"), data)
  427. # XXX: Hack, relies on the local keys of the datasmart
  428. # instance being stored in the 'dict' attribute and makes
  429. # assumptions about how variable expansion works, but
  430. # there's no better way to force an expansion of a single
  431. # variable across the datastore today, and this at least
  432. # lets us reference LAYERDIR without having to immediately
  433. # eval all our variables that use it.
  434. for key in data.dict:
  435. if key != "_data":
  436. value = data.getVar(key, False)
  437. if value and "${LAYERDIR}" in value:
  438. data.setVar(key, value.replace("${LAYERDIR}", layer))
  439. bb.data.delVar('LAYERDIR', data)
  440. if not data.getVar("BBPATH", True):
  441. bb.fatal("The BBPATH variable is not set")
  442. data = bb.parse.handle(os.path.join("conf", "bitbake.conf"), data)
  443. self.configuration.data = data
  444. # Handle any INHERITs and inherit the base class
  445. inherits = ["base"] + (bb.data.getVar('INHERIT', self.configuration.data, True ) or "").split()
  446. for inherit in inherits:
  447. self.configuration.data = bb.parse.handle(os.path.join('classes', '%s.bbclass' % inherit), self.configuration.data, True )
  448. # Nomally we only register event handlers at the end of parsing .bb files
  449. # We register any handlers we've found so far here...
  450. for var in bb.data.getVar('__BBHANDLERS', self.configuration.data) or []:
  451. bb.event.register(var, bb.data.getVar(var, self.configuration.data))
  452. bb.fetch.fetcher_init(self.configuration.data)
  453. bb.event.fire(bb.event.ConfigParsed(), self.configuration.data)
  454. except IOError as e:
  455. bb.msg.fatal(bb.msg.domain.Parsing, "Error when parsing %s: %s" % (files, str(e)))
  456. except bb.parse.ParseError as details:
  457. bb.msg.fatal(bb.msg.domain.Parsing, "Unable to parse %s (%s)" % (files, details) )
  458. def handleCollections( self, collections ):
  459. """Handle collections"""
  460. if collections:
  461. collection_list = collections.split()
  462. for c in collection_list:
  463. regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, self.configuration.data, 1)
  464. if regex == None:
  465. bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s not defined" % c)
  466. continue
  467. priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, self.configuration.data, 1)
  468. if priority == None:
  469. bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PRIORITY_%s not defined" % c)
  470. continue
  471. try:
  472. cre = re.compile(regex)
  473. except re.error:
  474. bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
  475. continue
  476. try:
  477. pri = int(priority)
  478. self.status.bbfile_config_priorities.append((c, regex, cre, pri))
  479. except ValueError:
  480. bb.msg.error(bb.msg.domain.Parsing, "invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
  481. def buildSetVars(self):
  482. """
  483. Setup any variables needed before starting a build
  484. """
  485. if not bb.data.getVar("BUILDNAME", self.configuration.data):
  486. bb.data.setVar("BUILDNAME", time.strftime('%Y%m%d%H%M'), self.configuration.data)
  487. bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S', time.gmtime()), self.configuration.data)
  488. def matchFiles(self, buildfile):
  489. """
  490. Find the .bb files which match the expression in 'buildfile'.
  491. """
  492. bf = os.path.abspath(buildfile)
  493. try:
  494. os.stat(bf)
  495. return [bf]
  496. except OSError:
  497. (filelist, masked) = self.collect_bbfiles()
  498. regexp = re.compile(buildfile)
  499. matches = []
  500. for f in filelist:
  501. if regexp.search(f) and os.path.isfile(f):
  502. bf = f
  503. matches.append(f)
  504. return matches
  505. def matchFile(self, buildfile):
  506. """
  507. Find the .bb file which matches the expression in 'buildfile'.
  508. Raise an error if multiple files
  509. """
  510. matches = self.matchFiles(buildfile)
  511. if len(matches) != 1:
  512. bb.msg.error(bb.msg.domain.Parsing, "Unable to match %s (%s matches found):" % (buildfile, len(matches)))
  513. for f in matches:
  514. bb.msg.error(bb.msg.domain.Parsing, " %s" % f)
  515. raise MultipleMatches
  516. return matches[0]
  517. def buildFile(self, buildfile, task):
  518. """
  519. Build the file matching regexp buildfile
  520. """
  521. # Parse the configuration here. We need to do it explicitly here since
  522. # buildFile() doesn't use the cache
  523. self.parseConfiguration()
  524. # If we are told to do the None task then query the default task
  525. if (task == None):
  526. task = self.configuration.cmd
  527. self.bb_cache = bb.cache.init(self)
  528. self.status = bb.cache.CacheData()
  529. (fn, cls) = self.bb_cache.virtualfn2realfn(buildfile)
  530. buildfile = self.matchFile(fn)
  531. fn = self.bb_cache.realfn2virtual(buildfile, cls)
  532. self.buildSetVars()
  533. # Load data into the cache for fn and parse the loaded cache data
  534. the_data = self.bb_cache.loadDataFull(fn, self.get_file_appends(fn), self.configuration.data)
  535. self.bb_cache.setData(fn, buildfile, the_data)
  536. self.bb_cache.handle_data(fn, self.status)
  537. # Tweak some variables
  538. item = self.bb_cache.getVar('PN', fn, True)
  539. self.status.ignored_dependencies = set()
  540. self.status.bbfile_priority[fn] = 1
  541. # Remove external dependencies
  542. self.status.task_deps[fn]['depends'] = {}
  543. self.status.deps[fn] = []
  544. self.status.rundeps[fn] = []
  545. self.status.runrecs[fn] = []
  546. # Remove stamp for target if force mode active
  547. if self.configuration.force:
  548. bb.msg.note(2, bb.msg.domain.RunQueue, "Remove stamp %s, %s" % (task, fn))
  549. bb.build.del_stamp('do_%s' % task, self.status, fn)
  550. # Setup taskdata structure
  551. taskdata = bb.taskdata.TaskData(self.configuration.abort)
  552. taskdata.add_provider(self.configuration.data, self.status, item)
  553. buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
  554. bb.event.fire(bb.event.BuildStarted(buildname, [item]), self.configuration.event_data)
  555. # Execute the runqueue
  556. runlist = [[item, "do_%s" % task]]
  557. rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
  558. def buildFileIdle(server, rq, abort):
  559. if abort or self.cookerAction == cookerStop:
  560. rq.finish_runqueue(True)
  561. elif self.cookerAction == cookerShutdown:
  562. rq.finish_runqueue(False)
  563. failures = 0
  564. try:
  565. retval = rq.execute_runqueue()
  566. except runqueue.TaskFailure as exc:
  567. for fnid in exc.args:
  568. bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid])
  569. failures = failures + 1
  570. retval = False
  571. if not retval:
  572. bb.event.fire(bb.event.BuildCompleted(buildname, item, failures), self.configuration.event_data)
  573. self.command.finishAsyncCommand()
  574. return False
  575. return 0.5
  576. self.server.register_idle_function(buildFileIdle, rq)
  577. def buildTargets(self, targets, task):
  578. """
  579. Attempt to build the targets specified
  580. """
  581. # Need files parsed
  582. self.updateCache()
  583. # If we are told to do the NULL task then query the default task
  584. if (task == None):
  585. task = self.configuration.cmd
  586. targets = self.checkPackages(targets)
  587. def buildTargetsIdle(server, rq, abort):
  588. if abort or self.cookerAction == cookerStop:
  589. rq.finish_runqueue(True)
  590. elif self.cookerAction == cookerShutdown:
  591. rq.finish_runqueue(False)
  592. failures = 0
  593. try:
  594. retval = rq.execute_runqueue()
  595. except runqueue.TaskFailure as exc:
  596. for fnid in exc.args:
  597. bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid])
  598. failures = failures + 1
  599. retval = False
  600. if not retval:
  601. bb.event.fire(bb.event.BuildCompleted(buildname, targets, failures), self.configuration.event_data)
  602. self.command.finishAsyncCommand()
  603. return None
  604. return 0.5
  605. self.buildSetVars()
  606. buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
  607. bb.event.fire(bb.event.BuildStarted(buildname, targets), self.configuration.event_data)
  608. localdata = data.createCopy(self.configuration.data)
  609. bb.data.update_data(localdata)
  610. bb.data.expandKeys(localdata)
  611. taskdata = bb.taskdata.TaskData(self.configuration.abort)
  612. runlist = []
  613. for k in targets:
  614. taskdata.add_provider(localdata, self.status, k)
  615. runlist.append([k, "do_%s" % task])
  616. taskdata.add_unresolved(localdata, self.status)
  617. rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
  618. self.server.register_idle_function(buildTargetsIdle, rq)
  619. def updateCache(self):
  620. if self.cookerState == cookerParsed:
  621. return
  622. if self.cookerState != cookerParsing:
  623. self.parseConfiguration ()
  624. # Import Psyco if available and not disabled
  625. import platform
  626. if platform.machine() in ['i386', 'i486', 'i586', 'i686']:
  627. if not self.configuration.disable_psyco:
  628. try:
  629. import psyco
  630. except ImportError:
  631. bb.msg.note(1, bb.msg.domain.Collection, "Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
  632. else:
  633. psyco.bind( CookerParser.parse_next )
  634. else:
  635. bb.msg.note(1, bb.msg.domain.Collection, "You have disabled Psyco. This decreases performance.")
  636. self.status = bb.cache.CacheData()
  637. ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
  638. self.status.ignored_dependencies = set(ignore.split())
  639. for dep in self.configuration.extra_assume_provided:
  640. self.status.ignored_dependencies.add(dep)
  641. self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
  642. (filelist, masked) = self.collect_bbfiles()
  643. bb.data.renameVar("__depends", "__base_depends", self.configuration.data)
  644. self.parser = CookerParser(self, filelist, masked)
  645. self.cookerState = cookerParsing
  646. if not self.parser.parse_next():
  647. bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete")
  648. self.buildDepgraph()
  649. self.cookerState = cookerParsed
  650. return None
  651. return True
  652. def checkPackages(self, pkgs_to_build):
  653. if len(pkgs_to_build) == 0:
  654. raise NothingToBuild
  655. if 'world' in pkgs_to_build:
  656. self.buildWorldTargetList()
  657. pkgs_to_build.remove('world')
  658. for t in self.status.world_target:
  659. pkgs_to_build.append(t)
  660. return pkgs_to_build
  661. def get_bbfiles( self, path = os.getcwd() ):
  662. """Get list of default .bb files by reading out the current directory"""
  663. contents = os.listdir(path)
  664. bbfiles = []
  665. for f in contents:
  666. (root, ext) = os.path.splitext(f)
  667. if ext == ".bb":
  668. bbfiles.append(os.path.abspath(os.path.join(os.getcwd(), f)))
  669. return bbfiles
  670. def find_bbfiles( self, path ):
  671. """Find all the .bb and .bbappend files in a directory"""
  672. from os.path import join
  673. found = []
  674. for dir, dirs, files in os.walk(path):
  675. for ignored in ('SCCS', 'CVS', '.svn'):
  676. if ignored in dirs:
  677. dirs.remove(ignored)
  678. found += [join(dir, f) for f in files if (f.endswith('.bb') or f.endswith('.bbappend'))]
  679. return found
  680. def collect_bbfiles( self ):
  681. """Collect all available .bb build files"""
  682. parsed, cached, skipped, masked = 0, 0, 0, 0
  683. self.bb_cache = bb.cache.init(self)
  684. bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
  685. files = (data.getVar( "BBFILES", self.configuration.data, 1 ) or "").split()
  686. data.setVar("BBFILES", " ".join(files), self.configuration.data)
  687. if not len(files):
  688. files = self.get_bbfiles()
  689. if not len(files):
  690. bb.msg.error(bb.msg.domain.Collection, "no recipe files to build, check your BBPATH and BBFILES?")
  691. bb.event.fire(CookerExit(), self.configuration.event_data)
  692. newfiles = set()
  693. for f in files:
  694. if os.path.isdir(f):
  695. dirfiles = self.find_bbfiles(f)
  696. newfiles.update(dirfiles)
  697. else:
  698. globbed = glob.glob(f)
  699. if not globbed and os.path.exists(f):
  700. globbed = [f]
  701. newfiles.update(globbed)
  702. bbmask = bb.data.getVar('BBMASK', self.configuration.data, 1)
  703. if bbmask:
  704. try:
  705. bbmask_compiled = re.compile(bbmask)
  706. except sre_constants.error:
  707. bb.msg.fatal(bb.msg.domain.Collection, "BBMASK is not a valid regular expression.")
  708. bbfiles = []
  709. bbappend = []
  710. for f in newfiles:
  711. if bbmask and bbmask_compiled.search(f):
  712. bb.msg.debug(1, bb.msg.domain.Collection, "skipping masked file %s" % f)
  713. masked += 1
  714. continue
  715. if f.endswith('.bb'):
  716. bbfiles.append(f)
  717. elif f.endswith('.bbappend'):
  718. bbappend.append(f)
  719. else:
  720. bb.msg.note(1, bb.msg.domain.Collection, "File %s of unknown filetype in BBFILES? Ignorning..." % f)
  721. # Build a list of .bbappend files for each .bb file
  722. self.appendlist = {}
  723. for f in bbappend:
  724. base = os.path.basename(f).replace('.bbappend', '.bb')
  725. if not base in self.appendlist:
  726. self.appendlist[base] = []
  727. self.appendlist[base].append(f)
  728. return (bbfiles, masked)
  729. def get_file_appends(self, fn):
  730. """
  731. Returns a list of .bbappend files to apply to fn
  732. NB: collect_files() must have been called prior to this
  733. """
  734. f = os.path.basename(fn)
  735. if f in self.appendlist:
  736. return self.appendlist[f]
  737. return []
  738. def serve(self):
  739. # Empty the environment. The environment will be populated as
  740. # necessary from the data store.
  741. bb.utils.empty_environment()
  742. if self.configuration.profile:
  743. try:
  744. import cProfile as profile
  745. except:
  746. import profile
  747. profile.runctx("self.server.serve_forever()", globals(), locals(), "profile.log")
  748. # Redirect stdout to capture profile information
  749. pout = open('profile.log.processed', 'w')
  750. so = sys.stdout.fileno()
  751. os.dup2(pout.fileno(), so)
  752. import pstats
  753. p = pstats.Stats('profile.log')
  754. p.sort_stats('time')
  755. p.print_stats()
  756. p.print_callers()
  757. p.sort_stats('cumulative')
  758. p.print_stats()
  759. os.dup2(so, pout.fileno())
  760. pout.flush()
  761. pout.close()
  762. else:
  763. self.server.serve_forever()
  764. bb.event.fire(CookerExit(), self.configuration.event_data)
  765. class CookerExit(bb.event.Event):
  766. """
  767. Notify clients of the Cooker shutdown
  768. """
  769. def __init__(self):
  770. bb.event.Event.__init__(self)
  771. class CookerParser:
  772. def __init__(self, cooker, filelist, masked):
  773. # Internal data
  774. self.filelist = filelist
  775. self.cooker = cooker
  776. # Accounting statistics
  777. self.parsed = 0
  778. self.cached = 0
  779. self.error = 0
  780. self.masked = masked
  781. self.total = len(filelist)
  782. self.skipped = 0
  783. self.virtuals = 0
  784. # Pointer to the next file to parse
  785. self.pointer = 0
  786. def parse_next(self):
  787. cooker = self.cooker
  788. if self.pointer < len(self.filelist):
  789. f = self.filelist[self.pointer]
  790. try:
  791. fromCache, skipped, virtuals = cooker.bb_cache.loadData(f, cooker.get_file_appends(f), cooker.configuration.data, cooker.status)
  792. if fromCache:
  793. self.cached += 1
  794. else:
  795. self.parsed += 1
  796. self.skipped += skipped
  797. self.virtuals += virtuals
  798. except IOError as e:
  799. self.error += 1
  800. cooker.bb_cache.remove(f)
  801. bb.msg.error(bb.msg.domain.Collection, "opening %s: %s" % (f, e))
  802. pass
  803. except KeyboardInterrupt:
  804. cooker.bb_cache.remove(f)
  805. cooker.bb_cache.sync()
  806. raise
  807. except Exception as e:
  808. self.error += 1
  809. cooker.bb_cache.remove(f)
  810. bb.msg.error(bb.msg.domain.Collection, "%s while parsing %s" % (e, f))
  811. except:
  812. cooker.bb_cache.remove(f)
  813. raise
  814. finally:
  815. bb.event.fire(bb.event.ParseProgress(self.cached, self.parsed, self.skipped, self.masked, self.virtuals, self.error, self.total), cooker.configuration.event_data)
  816. self.pointer += 1
  817. if self.pointer >= self.total:
  818. cooker.bb_cache.sync()
  819. if self.error > 0:
  820. raise ParsingErrorsFound
  821. return False
  822. return True