taskdata.py 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584
  1. #!/usr/bin/env python
  2. # ex:ts=4:sw=4:sts=4:et
  3. # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
  4. """
  5. BitBake 'TaskData' implementation
  6. Task data collection and handling
  7. """
  8. # Copyright (C) 2006 Richard Purdie
  9. #
  10. # SPDX-License-Identifier: GPL-2.0-only
  11. #
  12. import logging
  13. import re
  14. import bb
  15. logger = logging.getLogger("BitBake.TaskData")
  16. def re_match_strings(target, strings):
  17. """
  18. Whether or not the string 'target' matches
  19. any one string of the strings which can be regular expression string
  20. """
  21. return any(name == target or re.match(name, target)
  22. for name in strings)
  23. class TaskEntry:
  24. def __init__(self):
  25. self.tdepends = []
  26. self.idepends = []
  27. self.irdepends = []
  28. class TaskData:
  29. """
  30. BitBake Task Data implementation
  31. """
  32. def __init__(self, abort = True, skiplist = None, allowincomplete = False):
  33. self.build_targets = {}
  34. self.run_targets = {}
  35. self.external_targets = []
  36. self.seenfns = []
  37. self.taskentries = {}
  38. self.depids = {}
  39. self.rdepids = {}
  40. self.consider_msgs_cache = []
  41. self.failed_deps = []
  42. self.failed_rdeps = []
  43. self.failed_fns = []
  44. self.abort = abort
  45. self.allowincomplete = allowincomplete
  46. self.skiplist = skiplist
  47. self.mcdepends = []
  48. def add_tasks(self, fn, dataCache):
  49. """
  50. Add tasks for a given fn to the database
  51. """
  52. task_deps = dataCache.task_deps[fn]
  53. if fn in self.failed_fns:
  54. bb.msg.fatal("TaskData", "Trying to re-add a failed file? Something is broken...")
  55. # Check if we've already seen this fn
  56. if fn in self.seenfns:
  57. return
  58. self.seenfns.append(fn)
  59. self.add_extra_deps(fn, dataCache)
  60. def add_mcdepends(task):
  61. for dep in task_deps['mcdepends'][task].split():
  62. if len(dep.split(':')) != 5:
  63. bb.msg.fatal("TaskData", "Error for %s:%s[%s], multiconfig dependency %s does not contain exactly four ':' characters.\n Task '%s' should be specified in the form 'multiconfig:fromMC:toMC:packagename:task'" % (fn, task, 'mcdepends', dep, 'mcdepends'))
  64. if dep not in self.mcdepends:
  65. self.mcdepends.append(dep)
  66. # Common code for dep_name/depends = 'depends'/idepends and 'rdepends'/irdepends
  67. def handle_deps(task, dep_name, depends, seen):
  68. if dep_name in task_deps and task in task_deps[dep_name]:
  69. ids = []
  70. for dep in task_deps[dep_name][task].split():
  71. if dep:
  72. parts = dep.split(":")
  73. if len(parts) != 2:
  74. bb.msg.fatal("TaskData", "Error for %s:%s[%s], dependency %s in '%s' does not contain exactly one ':' character.\n Task '%s' should be specified in the form 'packagename:task'" % (fn, task, dep_name, dep, task_deps[dep_name][task], dep_name))
  75. ids.append((parts[0], parts[1]))
  76. seen(parts[0])
  77. depends.extend(ids)
  78. for task in task_deps['tasks']:
  79. tid = "%s:%s" % (fn, task)
  80. self.taskentries[tid] = TaskEntry()
  81. # Work out task dependencies
  82. parentids = []
  83. for dep in task_deps['parents'][task]:
  84. if dep not in task_deps['tasks']:
  85. bb.debug(2, "Not adding dependency of %s on %s since %s does not exist" % (task, dep, dep))
  86. continue
  87. parentid = "%s:%s" % (fn, dep)
  88. parentids.append(parentid)
  89. self.taskentries[tid].tdepends.extend(parentids)
  90. # Touch all intertask dependencies
  91. handle_deps(task, 'depends', self.taskentries[tid].idepends, self.seen_build_target)
  92. handle_deps(task, 'rdepends', self.taskentries[tid].irdepends, self.seen_run_target)
  93. if 'mcdepends' in task_deps and task in task_deps['mcdepends']:
  94. add_mcdepends(task)
  95. # Work out build dependencies
  96. if not fn in self.depids:
  97. dependids = set()
  98. for depend in dataCache.deps[fn]:
  99. dependids.add(depend)
  100. self.depids[fn] = list(dependids)
  101. logger.debug(2, "Added dependencies %s for %s", str(dataCache.deps[fn]), fn)
  102. # Work out runtime dependencies
  103. if not fn in self.rdepids:
  104. rdependids = set()
  105. rdepends = dataCache.rundeps[fn]
  106. rrecs = dataCache.runrecs[fn]
  107. rdependlist = []
  108. rreclist = []
  109. for package in rdepends:
  110. for rdepend in rdepends[package]:
  111. rdependlist.append(rdepend)
  112. rdependids.add(rdepend)
  113. for package in rrecs:
  114. for rdepend in rrecs[package]:
  115. rreclist.append(rdepend)
  116. rdependids.add(rdepend)
  117. if rdependlist:
  118. logger.debug(2, "Added runtime dependencies %s for %s", str(rdependlist), fn)
  119. if rreclist:
  120. logger.debug(2, "Added runtime recommendations %s for %s", str(rreclist), fn)
  121. self.rdepids[fn] = list(rdependids)
  122. for dep in self.depids[fn]:
  123. self.seen_build_target(dep)
  124. if dep in self.failed_deps:
  125. self.fail_fn(fn)
  126. return
  127. for dep in self.rdepids[fn]:
  128. self.seen_run_target(dep)
  129. if dep in self.failed_rdeps:
  130. self.fail_fn(fn)
  131. return
  132. def add_extra_deps(self, fn, dataCache):
  133. func = dataCache.extradepsfunc.get(fn, None)
  134. if func:
  135. bb.providers.buildWorldTargetList(dataCache)
  136. pn = dataCache.pkg_fn[fn]
  137. params = {'deps': dataCache.deps[fn],
  138. 'world_target': dataCache.world_target,
  139. 'pkg_pn': dataCache.pkg_pn,
  140. 'self_pn': pn}
  141. funcname = '_%s_calculate_extra_depends' % pn.replace('-', '_')
  142. paramlist = ','.join(params.keys())
  143. func = 'def %s(%s):\n%s\n\n%s(%s)' % (funcname, paramlist, func, funcname, paramlist)
  144. bb.utils.better_exec(func, params)
  145. def have_build_target(self, target):
  146. """
  147. Have we a build target matching this name?
  148. """
  149. if target in self.build_targets and self.build_targets[target]:
  150. return True
  151. return False
  152. def have_runtime_target(self, target):
  153. """
  154. Have we a runtime target matching this name?
  155. """
  156. if target in self.run_targets and self.run_targets[target]:
  157. return True
  158. return False
  159. def seen_build_target(self, name):
  160. """
  161. Maintain a list of build targets
  162. """
  163. if name not in self.build_targets:
  164. self.build_targets[name] = []
  165. def add_build_target(self, fn, item):
  166. """
  167. Add a build target.
  168. If already present, append the provider fn to the list
  169. """
  170. if item in self.build_targets:
  171. if fn in self.build_targets[item]:
  172. return
  173. self.build_targets[item].append(fn)
  174. return
  175. self.build_targets[item] = [fn]
  176. def seen_run_target(self, name):
  177. """
  178. Maintain a list of runtime build targets
  179. """
  180. if name not in self.run_targets:
  181. self.run_targets[name] = []
  182. def add_runtime_target(self, fn, item):
  183. """
  184. Add a runtime target.
  185. If already present, append the provider fn to the list
  186. """
  187. if item in self.run_targets:
  188. if fn in self.run_targets[item]:
  189. return
  190. self.run_targets[item].append(fn)
  191. return
  192. self.run_targets[item] = [fn]
  193. def mark_external_target(self, target):
  194. """
  195. Mark a build target as being externally requested
  196. """
  197. if target not in self.external_targets:
  198. self.external_targets.append(target)
  199. def get_unresolved_build_targets(self, dataCache):
  200. """
  201. Return a list of build targets who's providers
  202. are unknown.
  203. """
  204. unresolved = []
  205. for target in self.build_targets:
  206. if re_match_strings(target, dataCache.ignored_dependencies):
  207. continue
  208. if target in self.failed_deps:
  209. continue
  210. if not self.build_targets[target]:
  211. unresolved.append(target)
  212. return unresolved
  213. def get_unresolved_run_targets(self, dataCache):
  214. """
  215. Return a list of runtime targets who's providers
  216. are unknown.
  217. """
  218. unresolved = []
  219. for target in self.run_targets:
  220. if re_match_strings(target, dataCache.ignored_dependencies):
  221. continue
  222. if target in self.failed_rdeps:
  223. continue
  224. if not self.run_targets[target]:
  225. unresolved.append(target)
  226. return unresolved
  227. def get_provider(self, item):
  228. """
  229. Return a list of providers of item
  230. """
  231. return self.build_targets[item]
  232. def get_dependees(self, item):
  233. """
  234. Return a list of targets which depend on item
  235. """
  236. dependees = []
  237. for fn in self.depids:
  238. if item in self.depids[fn]:
  239. dependees.append(fn)
  240. return dependees
  241. def get_rdependees(self, item):
  242. """
  243. Return a list of targets which depend on runtime item
  244. """
  245. dependees = []
  246. for fn in self.rdepids:
  247. if item in self.rdepids[fn]:
  248. dependees.append(fn)
  249. return dependees
  250. def get_reasons(self, item, runtime=False):
  251. """
  252. Get the reason(s) for an item not being provided, if any
  253. """
  254. reasons = []
  255. if self.skiplist:
  256. for fn in self.skiplist:
  257. skipitem = self.skiplist[fn]
  258. if skipitem.pn == item:
  259. reasons.append("%s was skipped: %s" % (skipitem.pn, skipitem.skipreason))
  260. elif runtime and item in skipitem.rprovides:
  261. reasons.append("%s RPROVIDES %s but was skipped: %s" % (skipitem.pn, item, skipitem.skipreason))
  262. elif not runtime and item in skipitem.provides:
  263. reasons.append("%s PROVIDES %s but was skipped: %s" % (skipitem.pn, item, skipitem.skipreason))
  264. return reasons
  265. def get_close_matches(self, item, provider_list):
  266. import difflib
  267. if self.skiplist:
  268. skipped = []
  269. for fn in self.skiplist:
  270. skipped.append(self.skiplist[fn].pn)
  271. full_list = provider_list + skipped
  272. else:
  273. full_list = provider_list
  274. return difflib.get_close_matches(item, full_list, cutoff=0.7)
  275. def add_provider(self, cfgData, dataCache, item):
  276. try:
  277. self.add_provider_internal(cfgData, dataCache, item)
  278. except bb.providers.NoProvider:
  279. if self.abort:
  280. raise
  281. self.remove_buildtarget(item)
  282. self.mark_external_target(item)
  283. def add_provider_internal(self, cfgData, dataCache, item):
  284. """
  285. Add the providers of item to the task data
  286. Mark entries were specifically added externally as against dependencies
  287. added internally during dependency resolution
  288. """
  289. if re_match_strings(item, dataCache.ignored_dependencies):
  290. return
  291. if not item in dataCache.providers:
  292. close_matches = self.get_close_matches(item, list(dataCache.providers.keys()))
  293. # Is it in RuntimeProviders ?
  294. all_p = bb.providers.getRuntimeProviders(dataCache, item)
  295. for fn in all_p:
  296. new = dataCache.pkg_fn[fn] + " RPROVIDES " + item
  297. if new not in close_matches:
  298. close_matches.append(new)
  299. bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees(item), reasons=self.get_reasons(item), close_matches=close_matches), cfgData)
  300. raise bb.providers.NoProvider(item)
  301. if self.have_build_target(item):
  302. return
  303. all_p = dataCache.providers[item]
  304. eligible, foundUnique = bb.providers.filterProviders(all_p, item, cfgData, dataCache)
  305. eligible = [p for p in eligible if not p in self.failed_fns]
  306. if not eligible:
  307. bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees(item), reasons=["No eligible PROVIDERs exist for '%s'" % item]), cfgData)
  308. raise bb.providers.NoProvider(item)
  309. if len(eligible) > 1 and foundUnique == False:
  310. if item not in self.consider_msgs_cache:
  311. providers_list = []
  312. for fn in eligible:
  313. providers_list.append(dataCache.pkg_fn[fn])
  314. bb.event.fire(bb.event.MultipleProviders(item, providers_list), cfgData)
  315. self.consider_msgs_cache.append(item)
  316. for fn in eligible:
  317. if fn in self.failed_fns:
  318. continue
  319. logger.debug(2, "adding %s to satisfy %s", fn, item)
  320. self.add_build_target(fn, item)
  321. self.add_tasks(fn, dataCache)
  322. #item = dataCache.pkg_fn[fn]
  323. def add_rprovider(self, cfgData, dataCache, item):
  324. """
  325. Add the runtime providers of item to the task data
  326. (takes item names from RDEPENDS/PACKAGES namespace)
  327. """
  328. if re_match_strings(item, dataCache.ignored_dependencies):
  329. return
  330. if self.have_runtime_target(item):
  331. return
  332. all_p = bb.providers.getRuntimeProviders(dataCache, item)
  333. if not all_p:
  334. bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees(item), reasons=self.get_reasons(item, True)), cfgData)
  335. raise bb.providers.NoRProvider(item)
  336. eligible, numberPreferred = bb.providers.filterProvidersRunTime(all_p, item, cfgData, dataCache)
  337. eligible = [p for p in eligible if not p in self.failed_fns]
  338. if not eligible:
  339. bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees(item), reasons=["No eligible RPROVIDERs exist for '%s'" % item]), cfgData)
  340. raise bb.providers.NoRProvider(item)
  341. if len(eligible) > 1 and numberPreferred == 0:
  342. if item not in self.consider_msgs_cache:
  343. providers_list = []
  344. for fn in eligible:
  345. providers_list.append(dataCache.pkg_fn[fn])
  346. bb.event.fire(bb.event.MultipleProviders(item, providers_list, runtime=True), cfgData)
  347. self.consider_msgs_cache.append(item)
  348. if numberPreferred > 1:
  349. if item not in self.consider_msgs_cache:
  350. providers_list = []
  351. for fn in eligible:
  352. providers_list.append(dataCache.pkg_fn[fn])
  353. bb.event.fire(bb.event.MultipleProviders(item, providers_list, runtime=True), cfgData)
  354. self.consider_msgs_cache.append(item)
  355. raise bb.providers.MultipleRProvider(item)
  356. # run through the list until we find one that we can build
  357. for fn in eligible:
  358. if fn in self.failed_fns:
  359. continue
  360. logger.debug(2, "adding '%s' to satisfy runtime '%s'", fn, item)
  361. self.add_runtime_target(fn, item)
  362. self.add_tasks(fn, dataCache)
  363. def fail_fn(self, fn, missing_list=None):
  364. """
  365. Mark a file as failed (unbuildable)
  366. Remove any references from build and runtime provider lists
  367. missing_list, A list of missing requirements for this target
  368. """
  369. if fn in self.failed_fns:
  370. return
  371. if not missing_list:
  372. missing_list = []
  373. logger.debug(1, "File '%s' is unbuildable, removing...", fn)
  374. self.failed_fns.append(fn)
  375. for target in self.build_targets:
  376. if fn in self.build_targets[target]:
  377. self.build_targets[target].remove(fn)
  378. if len(self.build_targets[target]) == 0:
  379. self.remove_buildtarget(target, missing_list)
  380. for target in self.run_targets:
  381. if fn in self.run_targets[target]:
  382. self.run_targets[target].remove(fn)
  383. if len(self.run_targets[target]) == 0:
  384. self.remove_runtarget(target, missing_list)
  385. def remove_buildtarget(self, target, missing_list=None):
  386. """
  387. Mark a build target as failed (unbuildable)
  388. Trigger removal of any files that have this as a dependency
  389. """
  390. if not missing_list:
  391. missing_list = [target]
  392. else:
  393. missing_list = [target] + missing_list
  394. logger.verbose("Target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", target, missing_list)
  395. self.failed_deps.append(target)
  396. dependees = self.get_dependees(target)
  397. for fn in dependees:
  398. self.fail_fn(fn, missing_list)
  399. for tid in self.taskentries:
  400. for (idepend, idependtask) in self.taskentries[tid].idepends:
  401. if idepend == target:
  402. fn = tid.rsplit(":",1)[0]
  403. self.fail_fn(fn, missing_list)
  404. if self.abort and target in self.external_targets:
  405. logger.error("Required build target '%s' has no buildable providers.\nMissing or unbuildable dependency chain was: %s", target, missing_list)
  406. raise bb.providers.NoProvider(target)
  407. def remove_runtarget(self, target, missing_list=None):
  408. """
  409. Mark a run target as failed (unbuildable)
  410. Trigger removal of any files that have this as a dependency
  411. """
  412. if not missing_list:
  413. missing_list = [target]
  414. else:
  415. missing_list = [target] + missing_list
  416. logger.info("Runtime target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", target, missing_list)
  417. self.failed_rdeps.append(target)
  418. dependees = self.get_rdependees(target)
  419. for fn in dependees:
  420. self.fail_fn(fn, missing_list)
  421. for tid in self.taskentries:
  422. for (idepend, idependtask) in self.taskentries[tid].irdepends:
  423. if idepend == target:
  424. fn = tid.rsplit(":",1)[0]
  425. self.fail_fn(fn, missing_list)
  426. def add_unresolved(self, cfgData, dataCache):
  427. """
  428. Resolve all unresolved build and runtime targets
  429. """
  430. logger.info("Resolving any missing task queue dependencies")
  431. while True:
  432. added = 0
  433. for target in self.get_unresolved_build_targets(dataCache):
  434. try:
  435. self.add_provider_internal(cfgData, dataCache, target)
  436. added = added + 1
  437. except bb.providers.NoProvider:
  438. if self.abort and target in self.external_targets and not self.allowincomplete:
  439. raise
  440. if not self.allowincomplete:
  441. self.remove_buildtarget(target)
  442. for target in self.get_unresolved_run_targets(dataCache):
  443. try:
  444. self.add_rprovider(cfgData, dataCache, target)
  445. added = added + 1
  446. except (bb.providers.NoRProvider, bb.providers.MultipleRProvider):
  447. self.remove_runtarget(target)
  448. logger.debug(1, "Resolved " + str(added) + " extra dependencies")
  449. if added == 0:
  450. break
  451. # self.dump_data()
  452. def get_providermap(self, prefix=None):
  453. provmap = {}
  454. for name in self.build_targets:
  455. if prefix and not name.startswith(prefix):
  456. continue
  457. if self.have_build_target(name):
  458. provider = self.get_provider(name)
  459. if provider:
  460. provmap[name] = provider[0]
  461. return provmap
  462. def get_mcdepends(self):
  463. return self.mcdepends
  464. def dump_data(self):
  465. """
  466. Dump some debug information on the internal data structures
  467. """
  468. logger.debug(3, "build_names:")
  469. logger.debug(3, ", ".join(self.build_targets))
  470. logger.debug(3, "run_names:")
  471. logger.debug(3, ", ".join(self.run_targets))
  472. logger.debug(3, "build_targets:")
  473. for target in self.build_targets:
  474. targets = "None"
  475. if target in self.build_targets:
  476. targets = self.build_targets[target]
  477. logger.debug(3, " %s: %s", target, targets)
  478. logger.debug(3, "run_targets:")
  479. for target in self.run_targets:
  480. targets = "None"
  481. if target in self.run_targets:
  482. targets = self.run_targets[target]
  483. logger.debug(3, " %s: %s", target, targets)
  484. logger.debug(3, "tasks:")
  485. for tid in self.taskentries:
  486. logger.debug(3, " %s: %s %s %s",
  487. tid,
  488. self.taskentries[tid].idepends,
  489. self.taskentries[tid].irdepends,
  490. self.taskentries[tid].tdepends)
  491. logger.debug(3, "dependency ids (per fn):")
  492. for fn in self.depids:
  493. logger.debug(3, " %s: %s", fn, self.depids[fn])
  494. logger.debug(3, "runtime dependency ids (per fn):")
  495. for fn in self.rdepids:
  496. logger.debug(3, " %s: %s", fn, self.rdepids[fn])