taskdata.py 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596
  1. #!/usr/bin/env python
  2. # ex:ts=4:sw=4:sts=4:et
  3. # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
  4. """
  5. BitBake 'TaskData' implementation
  6. Task data collection and handling
  7. """
  8. # Copyright (C) 2006 Richard Purdie
  9. #
  10. # SPDX-License-Identifier: GPL-2.0-only
  11. #
  12. # This program is free software; you can redistribute it and/or modify
  13. # it under the terms of the GNU General Public License version 2 as
  14. # published by the Free Software Foundation.
  15. #
  16. # This program is distributed in the hope that it will be useful,
  17. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  18. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  19. # GNU General Public License for more details.
  20. #
  21. # You should have received a copy of the GNU General Public License along
  22. # with this program; if not, write to the Free Software Foundation, Inc.,
  23. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  24. import logging
  25. import re
  26. import bb
  27. logger = logging.getLogger("BitBake.TaskData")
  28. def re_match_strings(target, strings):
  29. """
  30. Whether or not the string 'target' matches
  31. any one string of the strings which can be regular expression string
  32. """
  33. return any(name == target or re.match(name, target)
  34. for name in strings)
  35. class TaskEntry:
  36. def __init__(self):
  37. self.tdepends = []
  38. self.idepends = []
  39. self.irdepends = []
  40. class TaskData:
  41. """
  42. BitBake Task Data implementation
  43. """
  44. def __init__(self, abort = True, skiplist = None, allowincomplete = False):
  45. self.build_targets = {}
  46. self.run_targets = {}
  47. self.external_targets = []
  48. self.seenfns = []
  49. self.taskentries = {}
  50. self.depids = {}
  51. self.rdepids = {}
  52. self.consider_msgs_cache = []
  53. self.failed_deps = []
  54. self.failed_rdeps = []
  55. self.failed_fns = []
  56. self.abort = abort
  57. self.allowincomplete = allowincomplete
  58. self.skiplist = skiplist
  59. self.mcdepends = []
  60. def add_tasks(self, fn, dataCache):
  61. """
  62. Add tasks for a given fn to the database
  63. """
  64. task_deps = dataCache.task_deps[fn]
  65. if fn in self.failed_fns:
  66. bb.msg.fatal("TaskData", "Trying to re-add a failed file? Something is broken...")
  67. # Check if we've already seen this fn
  68. if fn in self.seenfns:
  69. return
  70. self.seenfns.append(fn)
  71. self.add_extra_deps(fn, dataCache)
  72. def add_mcdepends(task):
  73. for dep in task_deps['mcdepends'][task].split():
  74. if len(dep.split(':')) != 5:
  75. bb.msg.fatal("TaskData", "Error for %s:%s[%s], multiconfig dependency %s does not contain exactly four ':' characters.\n Task '%s' should be specified in the form 'multiconfig:fromMC:toMC:packagename:task'" % (fn, task, 'mcdepends', dep, 'mcdepends'))
  76. if dep not in self.mcdepends:
  77. self.mcdepends.append(dep)
  78. # Common code for dep_name/depends = 'depends'/idepends and 'rdepends'/irdepends
  79. def handle_deps(task, dep_name, depends, seen):
  80. if dep_name in task_deps and task in task_deps[dep_name]:
  81. ids = []
  82. for dep in task_deps[dep_name][task].split():
  83. if dep:
  84. parts = dep.split(":")
  85. if len(parts) != 2:
  86. bb.msg.fatal("TaskData", "Error for %s:%s[%s], dependency %s in '%s' does not contain exactly one ':' character.\n Task '%s' should be specified in the form 'packagename:task'" % (fn, task, dep_name, dep, task_deps[dep_name][task], dep_name))
  87. ids.append((parts[0], parts[1]))
  88. seen(parts[0])
  89. depends.extend(ids)
  90. for task in task_deps['tasks']:
  91. tid = "%s:%s" % (fn, task)
  92. self.taskentries[tid] = TaskEntry()
  93. # Work out task dependencies
  94. parentids = []
  95. for dep in task_deps['parents'][task]:
  96. if dep not in task_deps['tasks']:
  97. bb.debug(2, "Not adding dependency of %s on %s since %s does not exist" % (task, dep, dep))
  98. continue
  99. parentid = "%s:%s" % (fn, dep)
  100. parentids.append(parentid)
  101. self.taskentries[tid].tdepends.extend(parentids)
  102. # Touch all intertask dependencies
  103. handle_deps(task, 'depends', self.taskentries[tid].idepends, self.seen_build_target)
  104. handle_deps(task, 'rdepends', self.taskentries[tid].irdepends, self.seen_run_target)
  105. if 'mcdepends' in task_deps and task in task_deps['mcdepends']:
  106. add_mcdepends(task)
  107. # Work out build dependencies
  108. if not fn in self.depids:
  109. dependids = set()
  110. for depend in dataCache.deps[fn]:
  111. dependids.add(depend)
  112. self.depids[fn] = list(dependids)
  113. logger.debug(2, "Added dependencies %s for %s", str(dataCache.deps[fn]), fn)
  114. # Work out runtime dependencies
  115. if not fn in self.rdepids:
  116. rdependids = set()
  117. rdepends = dataCache.rundeps[fn]
  118. rrecs = dataCache.runrecs[fn]
  119. rdependlist = []
  120. rreclist = []
  121. for package in rdepends:
  122. for rdepend in rdepends[package]:
  123. rdependlist.append(rdepend)
  124. rdependids.add(rdepend)
  125. for package in rrecs:
  126. for rdepend in rrecs[package]:
  127. rreclist.append(rdepend)
  128. rdependids.add(rdepend)
  129. if rdependlist:
  130. logger.debug(2, "Added runtime dependencies %s for %s", str(rdependlist), fn)
  131. if rreclist:
  132. logger.debug(2, "Added runtime recommendations %s for %s", str(rreclist), fn)
  133. self.rdepids[fn] = list(rdependids)
  134. for dep in self.depids[fn]:
  135. self.seen_build_target(dep)
  136. if dep in self.failed_deps:
  137. self.fail_fn(fn)
  138. return
  139. for dep in self.rdepids[fn]:
  140. self.seen_run_target(dep)
  141. if dep in self.failed_rdeps:
  142. self.fail_fn(fn)
  143. return
  144. def add_extra_deps(self, fn, dataCache):
  145. func = dataCache.extradepsfunc.get(fn, None)
  146. if func:
  147. bb.providers.buildWorldTargetList(dataCache)
  148. pn = dataCache.pkg_fn[fn]
  149. params = {'deps': dataCache.deps[fn],
  150. 'world_target': dataCache.world_target,
  151. 'pkg_pn': dataCache.pkg_pn,
  152. 'self_pn': pn}
  153. funcname = '_%s_calculate_extra_depends' % pn.replace('-', '_')
  154. paramlist = ','.join(params.keys())
  155. func = 'def %s(%s):\n%s\n\n%s(%s)' % (funcname, paramlist, func, funcname, paramlist)
  156. bb.utils.better_exec(func, params)
  157. def have_build_target(self, target):
  158. """
  159. Have we a build target matching this name?
  160. """
  161. if target in self.build_targets and self.build_targets[target]:
  162. return True
  163. return False
  164. def have_runtime_target(self, target):
  165. """
  166. Have we a runtime target matching this name?
  167. """
  168. if target in self.run_targets and self.run_targets[target]:
  169. return True
  170. return False
  171. def seen_build_target(self, name):
  172. """
  173. Maintain a list of build targets
  174. """
  175. if name not in self.build_targets:
  176. self.build_targets[name] = []
  177. def add_build_target(self, fn, item):
  178. """
  179. Add a build target.
  180. If already present, append the provider fn to the list
  181. """
  182. if item in self.build_targets:
  183. if fn in self.build_targets[item]:
  184. return
  185. self.build_targets[item].append(fn)
  186. return
  187. self.build_targets[item] = [fn]
  188. def seen_run_target(self, name):
  189. """
  190. Maintain a list of runtime build targets
  191. """
  192. if name not in self.run_targets:
  193. self.run_targets[name] = []
  194. def add_runtime_target(self, fn, item):
  195. """
  196. Add a runtime target.
  197. If already present, append the provider fn to the list
  198. """
  199. if item in self.run_targets:
  200. if fn in self.run_targets[item]:
  201. return
  202. self.run_targets[item].append(fn)
  203. return
  204. self.run_targets[item] = [fn]
  205. def mark_external_target(self, target):
  206. """
  207. Mark a build target as being externally requested
  208. """
  209. if target not in self.external_targets:
  210. self.external_targets.append(target)
  211. def get_unresolved_build_targets(self, dataCache):
  212. """
  213. Return a list of build targets who's providers
  214. are unknown.
  215. """
  216. unresolved = []
  217. for target in self.build_targets:
  218. if re_match_strings(target, dataCache.ignored_dependencies):
  219. continue
  220. if target in self.failed_deps:
  221. continue
  222. if not self.build_targets[target]:
  223. unresolved.append(target)
  224. return unresolved
  225. def get_unresolved_run_targets(self, dataCache):
  226. """
  227. Return a list of runtime targets who's providers
  228. are unknown.
  229. """
  230. unresolved = []
  231. for target in self.run_targets:
  232. if re_match_strings(target, dataCache.ignored_dependencies):
  233. continue
  234. if target in self.failed_rdeps:
  235. continue
  236. if not self.run_targets[target]:
  237. unresolved.append(target)
  238. return unresolved
  239. def get_provider(self, item):
  240. """
  241. Return a list of providers of item
  242. """
  243. return self.build_targets[item]
  244. def get_dependees(self, item):
  245. """
  246. Return a list of targets which depend on item
  247. """
  248. dependees = []
  249. for fn in self.depids:
  250. if item in self.depids[fn]:
  251. dependees.append(fn)
  252. return dependees
  253. def get_rdependees(self, item):
  254. """
  255. Return a list of targets which depend on runtime item
  256. """
  257. dependees = []
  258. for fn in self.rdepids:
  259. if item in self.rdepids[fn]:
  260. dependees.append(fn)
  261. return dependees
  262. def get_reasons(self, item, runtime=False):
  263. """
  264. Get the reason(s) for an item not being provided, if any
  265. """
  266. reasons = []
  267. if self.skiplist:
  268. for fn in self.skiplist:
  269. skipitem = self.skiplist[fn]
  270. if skipitem.pn == item:
  271. reasons.append("%s was skipped: %s" % (skipitem.pn, skipitem.skipreason))
  272. elif runtime and item in skipitem.rprovides:
  273. reasons.append("%s RPROVIDES %s but was skipped: %s" % (skipitem.pn, item, skipitem.skipreason))
  274. elif not runtime and item in skipitem.provides:
  275. reasons.append("%s PROVIDES %s but was skipped: %s" % (skipitem.pn, item, skipitem.skipreason))
  276. return reasons
  277. def get_close_matches(self, item, provider_list):
  278. import difflib
  279. if self.skiplist:
  280. skipped = []
  281. for fn in self.skiplist:
  282. skipped.append(self.skiplist[fn].pn)
  283. full_list = provider_list + skipped
  284. else:
  285. full_list = provider_list
  286. return difflib.get_close_matches(item, full_list, cutoff=0.7)
  287. def add_provider(self, cfgData, dataCache, item):
  288. try:
  289. self.add_provider_internal(cfgData, dataCache, item)
  290. except bb.providers.NoProvider:
  291. if self.abort:
  292. raise
  293. self.remove_buildtarget(item)
  294. self.mark_external_target(item)
  295. def add_provider_internal(self, cfgData, dataCache, item):
  296. """
  297. Add the providers of item to the task data
  298. Mark entries were specifically added externally as against dependencies
  299. added internally during dependency resolution
  300. """
  301. if re_match_strings(item, dataCache.ignored_dependencies):
  302. return
  303. if not item in dataCache.providers:
  304. close_matches = self.get_close_matches(item, list(dataCache.providers.keys()))
  305. # Is it in RuntimeProviders ?
  306. all_p = bb.providers.getRuntimeProviders(dataCache, item)
  307. for fn in all_p:
  308. new = dataCache.pkg_fn[fn] + " RPROVIDES " + item
  309. if new not in close_matches:
  310. close_matches.append(new)
  311. bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees(item), reasons=self.get_reasons(item), close_matches=close_matches), cfgData)
  312. raise bb.providers.NoProvider(item)
  313. if self.have_build_target(item):
  314. return
  315. all_p = dataCache.providers[item]
  316. eligible, foundUnique = bb.providers.filterProviders(all_p, item, cfgData, dataCache)
  317. eligible = [p for p in eligible if not p in self.failed_fns]
  318. if not eligible:
  319. bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees(item), reasons=["No eligible PROVIDERs exist for '%s'" % item]), cfgData)
  320. raise bb.providers.NoProvider(item)
  321. if len(eligible) > 1 and foundUnique == False:
  322. if item not in self.consider_msgs_cache:
  323. providers_list = []
  324. for fn in eligible:
  325. providers_list.append(dataCache.pkg_fn[fn])
  326. bb.event.fire(bb.event.MultipleProviders(item, providers_list), cfgData)
  327. self.consider_msgs_cache.append(item)
  328. for fn in eligible:
  329. if fn in self.failed_fns:
  330. continue
  331. logger.debug(2, "adding %s to satisfy %s", fn, item)
  332. self.add_build_target(fn, item)
  333. self.add_tasks(fn, dataCache)
  334. #item = dataCache.pkg_fn[fn]
  335. def add_rprovider(self, cfgData, dataCache, item):
  336. """
  337. Add the runtime providers of item to the task data
  338. (takes item names from RDEPENDS/PACKAGES namespace)
  339. """
  340. if re_match_strings(item, dataCache.ignored_dependencies):
  341. return
  342. if self.have_runtime_target(item):
  343. return
  344. all_p = bb.providers.getRuntimeProviders(dataCache, item)
  345. if not all_p:
  346. bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees(item), reasons=self.get_reasons(item, True)), cfgData)
  347. raise bb.providers.NoRProvider(item)
  348. eligible, numberPreferred = bb.providers.filterProvidersRunTime(all_p, item, cfgData, dataCache)
  349. eligible = [p for p in eligible if not p in self.failed_fns]
  350. if not eligible:
  351. bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees(item), reasons=["No eligible RPROVIDERs exist for '%s'" % item]), cfgData)
  352. raise bb.providers.NoRProvider(item)
  353. if len(eligible) > 1 and numberPreferred == 0:
  354. if item not in self.consider_msgs_cache:
  355. providers_list = []
  356. for fn in eligible:
  357. providers_list.append(dataCache.pkg_fn[fn])
  358. bb.event.fire(bb.event.MultipleProviders(item, providers_list, runtime=True), cfgData)
  359. self.consider_msgs_cache.append(item)
  360. if numberPreferred > 1:
  361. if item not in self.consider_msgs_cache:
  362. providers_list = []
  363. for fn in eligible:
  364. providers_list.append(dataCache.pkg_fn[fn])
  365. bb.event.fire(bb.event.MultipleProviders(item, providers_list, runtime=True), cfgData)
  366. self.consider_msgs_cache.append(item)
  367. raise bb.providers.MultipleRProvider(item)
  368. # run through the list until we find one that we can build
  369. for fn in eligible:
  370. if fn in self.failed_fns:
  371. continue
  372. logger.debug(2, "adding '%s' to satisfy runtime '%s'", fn, item)
  373. self.add_runtime_target(fn, item)
  374. self.add_tasks(fn, dataCache)
  375. def fail_fn(self, fn, missing_list=None):
  376. """
  377. Mark a file as failed (unbuildable)
  378. Remove any references from build and runtime provider lists
  379. missing_list, A list of missing requirements for this target
  380. """
  381. if fn in self.failed_fns:
  382. return
  383. if not missing_list:
  384. missing_list = []
  385. logger.debug(1, "File '%s' is unbuildable, removing...", fn)
  386. self.failed_fns.append(fn)
  387. for target in self.build_targets:
  388. if fn in self.build_targets[target]:
  389. self.build_targets[target].remove(fn)
  390. if len(self.build_targets[target]) == 0:
  391. self.remove_buildtarget(target, missing_list)
  392. for target in self.run_targets:
  393. if fn in self.run_targets[target]:
  394. self.run_targets[target].remove(fn)
  395. if len(self.run_targets[target]) == 0:
  396. self.remove_runtarget(target, missing_list)
  397. def remove_buildtarget(self, target, missing_list=None):
  398. """
  399. Mark a build target as failed (unbuildable)
  400. Trigger removal of any files that have this as a dependency
  401. """
  402. if not missing_list:
  403. missing_list = [target]
  404. else:
  405. missing_list = [target] + missing_list
  406. logger.verbose("Target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", target, missing_list)
  407. self.failed_deps.append(target)
  408. dependees = self.get_dependees(target)
  409. for fn in dependees:
  410. self.fail_fn(fn, missing_list)
  411. for tid in self.taskentries:
  412. for (idepend, idependtask) in self.taskentries[tid].idepends:
  413. if idepend == target:
  414. fn = tid.rsplit(":",1)[0]
  415. self.fail_fn(fn, missing_list)
  416. if self.abort and target in self.external_targets:
  417. logger.error("Required build target '%s' has no buildable providers.\nMissing or unbuildable dependency chain was: %s", target, missing_list)
  418. raise bb.providers.NoProvider(target)
  419. def remove_runtarget(self, target, missing_list=None):
  420. """
  421. Mark a run target as failed (unbuildable)
  422. Trigger removal of any files that have this as a dependency
  423. """
  424. if not missing_list:
  425. missing_list = [target]
  426. else:
  427. missing_list = [target] + missing_list
  428. logger.info("Runtime target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", target, missing_list)
  429. self.failed_rdeps.append(target)
  430. dependees = self.get_rdependees(target)
  431. for fn in dependees:
  432. self.fail_fn(fn, missing_list)
  433. for tid in self.taskentries:
  434. for (idepend, idependtask) in self.taskentries[tid].irdepends:
  435. if idepend == target:
  436. fn = tid.rsplit(":",1)[0]
  437. self.fail_fn(fn, missing_list)
  438. def add_unresolved(self, cfgData, dataCache):
  439. """
  440. Resolve all unresolved build and runtime targets
  441. """
  442. logger.info("Resolving any missing task queue dependencies")
  443. while True:
  444. added = 0
  445. for target in self.get_unresolved_build_targets(dataCache):
  446. try:
  447. self.add_provider_internal(cfgData, dataCache, target)
  448. added = added + 1
  449. except bb.providers.NoProvider:
  450. if self.abort and target in self.external_targets and not self.allowincomplete:
  451. raise
  452. if not self.allowincomplete:
  453. self.remove_buildtarget(target)
  454. for target in self.get_unresolved_run_targets(dataCache):
  455. try:
  456. self.add_rprovider(cfgData, dataCache, target)
  457. added = added + 1
  458. except (bb.providers.NoRProvider, bb.providers.MultipleRProvider):
  459. self.remove_runtarget(target)
  460. logger.debug(1, "Resolved " + str(added) + " extra dependencies")
  461. if added == 0:
  462. break
  463. # self.dump_data()
  464. def get_providermap(self, prefix=None):
  465. provmap = {}
  466. for name in self.build_targets:
  467. if prefix and not name.startswith(prefix):
  468. continue
  469. if self.have_build_target(name):
  470. provider = self.get_provider(name)
  471. if provider:
  472. provmap[name] = provider[0]
  473. return provmap
  474. def get_mcdepends(self):
  475. return self.mcdepends
  476. def dump_data(self):
  477. """
  478. Dump some debug information on the internal data structures
  479. """
  480. logger.debug(3, "build_names:")
  481. logger.debug(3, ", ".join(self.build_targets))
  482. logger.debug(3, "run_names:")
  483. logger.debug(3, ", ".join(self.run_targets))
  484. logger.debug(3, "build_targets:")
  485. for target in self.build_targets:
  486. targets = "None"
  487. if target in self.build_targets:
  488. targets = self.build_targets[target]
  489. logger.debug(3, " %s: %s", target, targets)
  490. logger.debug(3, "run_targets:")
  491. for target in self.run_targets:
  492. targets = "None"
  493. if target in self.run_targets:
  494. targets = self.run_targets[target]
  495. logger.debug(3, " %s: %s", target, targets)
  496. logger.debug(3, "tasks:")
  497. for tid in self.taskentries:
  498. logger.debug(3, " %s: %s %s %s",
  499. tid,
  500. self.taskentries[tid].idepends,
  501. self.taskentries[tid].irdepends,
  502. self.taskentries[tid].tdepends)
  503. logger.debug(3, "dependency ids (per fn):")
  504. for fn in self.depids:
  505. logger.debug(3, " %s: %s", fn, self.depids[fn])
  506. logger.debug(3, "runtime dependency ids (per fn):")
  507. for fn in self.rdepids:
  508. logger.debug(3, " %s: %s", fn, self.rdepids[fn])