taskdata.py 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581
  1. """
  2. BitBake 'TaskData' implementation
  3. Task data collection and handling
  4. """
  5. # Copyright (C) 2006 Richard Purdie
  6. #
  7. # SPDX-License-Identifier: GPL-2.0-only
  8. #
  9. import logging
  10. import re
  11. import bb
  12. logger = logging.getLogger("BitBake.TaskData")
  13. def re_match_strings(target, strings):
  14. """
  15. Whether or not the string 'target' matches
  16. any one string of the strings which can be regular expression string
  17. """
  18. return any(name == target or re.match(name, target)
  19. for name in strings)
  20. class TaskEntry:
  21. def __init__(self):
  22. self.tdepends = []
  23. self.idepends = []
  24. self.irdepends = []
  25. class TaskData:
  26. """
  27. BitBake Task Data implementation
  28. """
  29. def __init__(self, abort = True, skiplist = None, allowincomplete = False):
  30. self.build_targets = {}
  31. self.run_targets = {}
  32. self.external_targets = []
  33. self.seenfns = []
  34. self.taskentries = {}
  35. self.depids = {}
  36. self.rdepids = {}
  37. self.consider_msgs_cache = []
  38. self.failed_deps = []
  39. self.failed_rdeps = []
  40. self.failed_fns = []
  41. self.abort = abort
  42. self.allowincomplete = allowincomplete
  43. self.skiplist = skiplist
  44. self.mcdepends = []
  45. def add_tasks(self, fn, dataCache):
  46. """
  47. Add tasks for a given fn to the database
  48. """
  49. task_deps = dataCache.task_deps[fn]
  50. if fn in self.failed_fns:
  51. bb.msg.fatal("TaskData", "Trying to re-add a failed file? Something is broken...")
  52. # Check if we've already seen this fn
  53. if fn in self.seenfns:
  54. return
  55. self.seenfns.append(fn)
  56. self.add_extra_deps(fn, dataCache)
  57. def add_mcdepends(task):
  58. for dep in task_deps['mcdepends'][task].split():
  59. if len(dep.split(':')) != 5:
  60. bb.msg.fatal("TaskData", "Error for %s:%s[%s], multiconfig dependency %s does not contain exactly four ':' characters.\n Task '%s' should be specified in the form 'mc:fromMC:toMC:packagename:task'" % (fn, task, 'mcdepends', dep, 'mcdepends'))
  61. if dep not in self.mcdepends:
  62. self.mcdepends.append(dep)
  63. # Common code for dep_name/depends = 'depends'/idepends and 'rdepends'/irdepends
  64. def handle_deps(task, dep_name, depends, seen):
  65. if dep_name in task_deps and task in task_deps[dep_name]:
  66. ids = []
  67. for dep in task_deps[dep_name][task].split():
  68. if dep:
  69. parts = dep.split(":")
  70. if len(parts) != 2:
  71. bb.msg.fatal("TaskData", "Error for %s:%s[%s], dependency %s in '%s' does not contain exactly one ':' character.\n Task '%s' should be specified in the form 'packagename:task'" % (fn, task, dep_name, dep, task_deps[dep_name][task], dep_name))
  72. ids.append((parts[0], parts[1]))
  73. seen(parts[0])
  74. depends.extend(ids)
  75. for task in task_deps['tasks']:
  76. tid = "%s:%s" % (fn, task)
  77. self.taskentries[tid] = TaskEntry()
  78. # Work out task dependencies
  79. parentids = []
  80. for dep in task_deps['parents'][task]:
  81. if dep not in task_deps['tasks']:
  82. bb.debug(2, "Not adding dependency of %s on %s since %s does not exist" % (task, dep, dep))
  83. continue
  84. parentid = "%s:%s" % (fn, dep)
  85. parentids.append(parentid)
  86. self.taskentries[tid].tdepends.extend(parentids)
  87. # Touch all intertask dependencies
  88. handle_deps(task, 'depends', self.taskentries[tid].idepends, self.seen_build_target)
  89. handle_deps(task, 'rdepends', self.taskentries[tid].irdepends, self.seen_run_target)
  90. if 'mcdepends' in task_deps and task in task_deps['mcdepends']:
  91. add_mcdepends(task)
  92. # Work out build dependencies
  93. if not fn in self.depids:
  94. dependids = set()
  95. for depend in dataCache.deps[fn]:
  96. dependids.add(depend)
  97. self.depids[fn] = list(dependids)
  98. logger.debug(2, "Added dependencies %s for %s", str(dataCache.deps[fn]), fn)
  99. # Work out runtime dependencies
  100. if not fn in self.rdepids:
  101. rdependids = set()
  102. rdepends = dataCache.rundeps[fn]
  103. rrecs = dataCache.runrecs[fn]
  104. rdependlist = []
  105. rreclist = []
  106. for package in rdepends:
  107. for rdepend in rdepends[package]:
  108. rdependlist.append(rdepend)
  109. rdependids.add(rdepend)
  110. for package in rrecs:
  111. for rdepend in rrecs[package]:
  112. rreclist.append(rdepend)
  113. rdependids.add(rdepend)
  114. if rdependlist:
  115. logger.debug(2, "Added runtime dependencies %s for %s", str(rdependlist), fn)
  116. if rreclist:
  117. logger.debug(2, "Added runtime recommendations %s for %s", str(rreclist), fn)
  118. self.rdepids[fn] = list(rdependids)
  119. for dep in self.depids[fn]:
  120. self.seen_build_target(dep)
  121. if dep in self.failed_deps:
  122. self.fail_fn(fn)
  123. return
  124. for dep in self.rdepids[fn]:
  125. self.seen_run_target(dep)
  126. if dep in self.failed_rdeps:
  127. self.fail_fn(fn)
  128. return
  129. def add_extra_deps(self, fn, dataCache):
  130. func = dataCache.extradepsfunc.get(fn, None)
  131. if func:
  132. bb.providers.buildWorldTargetList(dataCache)
  133. pn = dataCache.pkg_fn[fn]
  134. params = {'deps': dataCache.deps[fn],
  135. 'world_target': dataCache.world_target,
  136. 'pkg_pn': dataCache.pkg_pn,
  137. 'self_pn': pn}
  138. funcname = '_%s_calculate_extra_depends' % pn.replace('-', '_')
  139. paramlist = ','.join(params.keys())
  140. func = 'def %s(%s):\n%s\n\n%s(%s)' % (funcname, paramlist, func, funcname, paramlist)
  141. bb.utils.better_exec(func, params)
  142. def have_build_target(self, target):
  143. """
  144. Have we a build target matching this name?
  145. """
  146. if target in self.build_targets and self.build_targets[target]:
  147. return True
  148. return False
  149. def have_runtime_target(self, target):
  150. """
  151. Have we a runtime target matching this name?
  152. """
  153. if target in self.run_targets and self.run_targets[target]:
  154. return True
  155. return False
  156. def seen_build_target(self, name):
  157. """
  158. Maintain a list of build targets
  159. """
  160. if name not in self.build_targets:
  161. self.build_targets[name] = []
  162. def add_build_target(self, fn, item):
  163. """
  164. Add a build target.
  165. If already present, append the provider fn to the list
  166. """
  167. if item in self.build_targets:
  168. if fn in self.build_targets[item]:
  169. return
  170. self.build_targets[item].append(fn)
  171. return
  172. self.build_targets[item] = [fn]
  173. def seen_run_target(self, name):
  174. """
  175. Maintain a list of runtime build targets
  176. """
  177. if name not in self.run_targets:
  178. self.run_targets[name] = []
  179. def add_runtime_target(self, fn, item):
  180. """
  181. Add a runtime target.
  182. If already present, append the provider fn to the list
  183. """
  184. if item in self.run_targets:
  185. if fn in self.run_targets[item]:
  186. return
  187. self.run_targets[item].append(fn)
  188. return
  189. self.run_targets[item] = [fn]
  190. def mark_external_target(self, target):
  191. """
  192. Mark a build target as being externally requested
  193. """
  194. if target not in self.external_targets:
  195. self.external_targets.append(target)
  196. def get_unresolved_build_targets(self, dataCache):
  197. """
  198. Return a list of build targets who's providers
  199. are unknown.
  200. """
  201. unresolved = []
  202. for target in self.build_targets:
  203. if re_match_strings(target, dataCache.ignored_dependencies):
  204. continue
  205. if target in self.failed_deps:
  206. continue
  207. if not self.build_targets[target]:
  208. unresolved.append(target)
  209. return unresolved
  210. def get_unresolved_run_targets(self, dataCache):
  211. """
  212. Return a list of runtime targets who's providers
  213. are unknown.
  214. """
  215. unresolved = []
  216. for target in self.run_targets:
  217. if re_match_strings(target, dataCache.ignored_dependencies):
  218. continue
  219. if target in self.failed_rdeps:
  220. continue
  221. if not self.run_targets[target]:
  222. unresolved.append(target)
  223. return unresolved
  224. def get_provider(self, item):
  225. """
  226. Return a list of providers of item
  227. """
  228. return self.build_targets[item]
  229. def get_dependees(self, item):
  230. """
  231. Return a list of targets which depend on item
  232. """
  233. dependees = []
  234. for fn in self.depids:
  235. if item in self.depids[fn]:
  236. dependees.append(fn)
  237. return dependees
  238. def get_rdependees(self, item):
  239. """
  240. Return a list of targets which depend on runtime item
  241. """
  242. dependees = []
  243. for fn in self.rdepids:
  244. if item in self.rdepids[fn]:
  245. dependees.append(fn)
  246. return dependees
  247. def get_reasons(self, item, runtime=False):
  248. """
  249. Get the reason(s) for an item not being provided, if any
  250. """
  251. reasons = []
  252. if self.skiplist:
  253. for fn in self.skiplist:
  254. skipitem = self.skiplist[fn]
  255. if skipitem.pn == item:
  256. reasons.append("%s was skipped: %s" % (skipitem.pn, skipitem.skipreason))
  257. elif runtime and item in skipitem.rprovides:
  258. reasons.append("%s RPROVIDES %s but was skipped: %s" % (skipitem.pn, item, skipitem.skipreason))
  259. elif not runtime and item in skipitem.provides:
  260. reasons.append("%s PROVIDES %s but was skipped: %s" % (skipitem.pn, item, skipitem.skipreason))
  261. return reasons
  262. def get_close_matches(self, item, provider_list):
  263. import difflib
  264. if self.skiplist:
  265. skipped = []
  266. for fn in self.skiplist:
  267. skipped.append(self.skiplist[fn].pn)
  268. full_list = provider_list + skipped
  269. else:
  270. full_list = provider_list
  271. return difflib.get_close_matches(item, full_list, cutoff=0.7)
  272. def add_provider(self, cfgData, dataCache, item):
  273. try:
  274. self.add_provider_internal(cfgData, dataCache, item)
  275. except bb.providers.NoProvider:
  276. if self.abort:
  277. raise
  278. self.remove_buildtarget(item)
  279. self.mark_external_target(item)
  280. def add_provider_internal(self, cfgData, dataCache, item):
  281. """
  282. Add the providers of item to the task data
  283. Mark entries were specifically added externally as against dependencies
  284. added internally during dependency resolution
  285. """
  286. if re_match_strings(item, dataCache.ignored_dependencies):
  287. return
  288. if not item in dataCache.providers:
  289. close_matches = self.get_close_matches(item, list(dataCache.providers.keys()))
  290. # Is it in RuntimeProviders ?
  291. all_p = bb.providers.getRuntimeProviders(dataCache, item)
  292. for fn in all_p:
  293. new = dataCache.pkg_fn[fn] + " RPROVIDES " + item
  294. if new not in close_matches:
  295. close_matches.append(new)
  296. bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees(item), reasons=self.get_reasons(item), close_matches=close_matches), cfgData)
  297. raise bb.providers.NoProvider(item)
  298. if self.have_build_target(item):
  299. return
  300. all_p = dataCache.providers[item]
  301. eligible, foundUnique = bb.providers.filterProviders(all_p, item, cfgData, dataCache)
  302. eligible = [p for p in eligible if not p in self.failed_fns]
  303. if not eligible:
  304. bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees(item), reasons=["No eligible PROVIDERs exist for '%s'" % item]), cfgData)
  305. raise bb.providers.NoProvider(item)
  306. if len(eligible) > 1 and not foundUnique:
  307. if item not in self.consider_msgs_cache:
  308. providers_list = []
  309. for fn in eligible:
  310. providers_list.append(dataCache.pkg_fn[fn])
  311. bb.event.fire(bb.event.MultipleProviders(item, providers_list), cfgData)
  312. self.consider_msgs_cache.append(item)
  313. for fn in eligible:
  314. if fn in self.failed_fns:
  315. continue
  316. logger.debug(2, "adding %s to satisfy %s", fn, item)
  317. self.add_build_target(fn, item)
  318. self.add_tasks(fn, dataCache)
  319. #item = dataCache.pkg_fn[fn]
  320. def add_rprovider(self, cfgData, dataCache, item):
  321. """
  322. Add the runtime providers of item to the task data
  323. (takes item names from RDEPENDS/PACKAGES namespace)
  324. """
  325. if re_match_strings(item, dataCache.ignored_dependencies):
  326. return
  327. if self.have_runtime_target(item):
  328. return
  329. all_p = bb.providers.getRuntimeProviders(dataCache, item)
  330. if not all_p:
  331. bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees(item), reasons=self.get_reasons(item, True)), cfgData)
  332. raise bb.providers.NoRProvider(item)
  333. eligible, numberPreferred = bb.providers.filterProvidersRunTime(all_p, item, cfgData, dataCache)
  334. eligible = [p for p in eligible if not p in self.failed_fns]
  335. if not eligible:
  336. bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees(item), reasons=["No eligible RPROVIDERs exist for '%s'" % item]), cfgData)
  337. raise bb.providers.NoRProvider(item)
  338. if len(eligible) > 1 and numberPreferred == 0:
  339. if item not in self.consider_msgs_cache:
  340. providers_list = []
  341. for fn in eligible:
  342. providers_list.append(dataCache.pkg_fn[fn])
  343. bb.event.fire(bb.event.MultipleProviders(item, providers_list, runtime=True), cfgData)
  344. self.consider_msgs_cache.append(item)
  345. if numberPreferred > 1:
  346. if item not in self.consider_msgs_cache:
  347. providers_list = []
  348. for fn in eligible:
  349. providers_list.append(dataCache.pkg_fn[fn])
  350. bb.event.fire(bb.event.MultipleProviders(item, providers_list, runtime=True), cfgData)
  351. self.consider_msgs_cache.append(item)
  352. raise bb.providers.MultipleRProvider(item)
  353. # run through the list until we find one that we can build
  354. for fn in eligible:
  355. if fn in self.failed_fns:
  356. continue
  357. logger.debug(2, "adding '%s' to satisfy runtime '%s'", fn, item)
  358. self.add_runtime_target(fn, item)
  359. self.add_tasks(fn, dataCache)
  360. def fail_fn(self, fn, missing_list=None):
  361. """
  362. Mark a file as failed (unbuildable)
  363. Remove any references from build and runtime provider lists
  364. missing_list, A list of missing requirements for this target
  365. """
  366. if fn in self.failed_fns:
  367. return
  368. if not missing_list:
  369. missing_list = []
  370. logger.debug(1, "File '%s' is unbuildable, removing...", fn)
  371. self.failed_fns.append(fn)
  372. for target in self.build_targets:
  373. if fn in self.build_targets[target]:
  374. self.build_targets[target].remove(fn)
  375. if len(self.build_targets[target]) == 0:
  376. self.remove_buildtarget(target, missing_list)
  377. for target in self.run_targets:
  378. if fn in self.run_targets[target]:
  379. self.run_targets[target].remove(fn)
  380. if len(self.run_targets[target]) == 0:
  381. self.remove_runtarget(target, missing_list)
  382. def remove_buildtarget(self, target, missing_list=None):
  383. """
  384. Mark a build target as failed (unbuildable)
  385. Trigger removal of any files that have this as a dependency
  386. """
  387. if not missing_list:
  388. missing_list = [target]
  389. else:
  390. missing_list = [target] + missing_list
  391. logger.verbose("Target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", target, missing_list)
  392. self.failed_deps.append(target)
  393. dependees = self.get_dependees(target)
  394. for fn in dependees:
  395. self.fail_fn(fn, missing_list)
  396. for tid in self.taskentries:
  397. for (idepend, idependtask) in self.taskentries[tid].idepends:
  398. if idepend == target:
  399. fn = tid.rsplit(":",1)[0]
  400. self.fail_fn(fn, missing_list)
  401. if self.abort and target in self.external_targets:
  402. logger.error("Required build target '%s' has no buildable providers.\nMissing or unbuildable dependency chain was: %s", target, missing_list)
  403. raise bb.providers.NoProvider(target)
  404. def remove_runtarget(self, target, missing_list=None):
  405. """
  406. Mark a run target as failed (unbuildable)
  407. Trigger removal of any files that have this as a dependency
  408. """
  409. if not missing_list:
  410. missing_list = [target]
  411. else:
  412. missing_list = [target] + missing_list
  413. logger.info("Runtime target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", target, missing_list)
  414. self.failed_rdeps.append(target)
  415. dependees = self.get_rdependees(target)
  416. for fn in dependees:
  417. self.fail_fn(fn, missing_list)
  418. for tid in self.taskentries:
  419. for (idepend, idependtask) in self.taskentries[tid].irdepends:
  420. if idepend == target:
  421. fn = tid.rsplit(":",1)[0]
  422. self.fail_fn(fn, missing_list)
  423. def add_unresolved(self, cfgData, dataCache):
  424. """
  425. Resolve all unresolved build and runtime targets
  426. """
  427. logger.info("Resolving any missing task queue dependencies")
  428. while True:
  429. added = 0
  430. for target in self.get_unresolved_build_targets(dataCache):
  431. try:
  432. self.add_provider_internal(cfgData, dataCache, target)
  433. added = added + 1
  434. except bb.providers.NoProvider:
  435. if self.abort and target in self.external_targets and not self.allowincomplete:
  436. raise
  437. if not self.allowincomplete:
  438. self.remove_buildtarget(target)
  439. for target in self.get_unresolved_run_targets(dataCache):
  440. try:
  441. self.add_rprovider(cfgData, dataCache, target)
  442. added = added + 1
  443. except (bb.providers.NoRProvider, bb.providers.MultipleRProvider):
  444. self.remove_runtarget(target)
  445. logger.debug(1, "Resolved " + str(added) + " extra dependencies")
  446. if added == 0:
  447. break
  448. # self.dump_data()
  449. def get_providermap(self, prefix=None):
  450. provmap = {}
  451. for name in self.build_targets:
  452. if prefix and not name.startswith(prefix):
  453. continue
  454. if self.have_build_target(name):
  455. provider = self.get_provider(name)
  456. if provider:
  457. provmap[name] = provider[0]
  458. return provmap
  459. def get_mcdepends(self):
  460. return self.mcdepends
  461. def dump_data(self):
  462. """
  463. Dump some debug information on the internal data structures
  464. """
  465. logger.debug(3, "build_names:")
  466. logger.debug(3, ", ".join(self.build_targets))
  467. logger.debug(3, "run_names:")
  468. logger.debug(3, ", ".join(self.run_targets))
  469. logger.debug(3, "build_targets:")
  470. for target in self.build_targets:
  471. targets = "None"
  472. if target in self.build_targets:
  473. targets = self.build_targets[target]
  474. logger.debug(3, " %s: %s", target, targets)
  475. logger.debug(3, "run_targets:")
  476. for target in self.run_targets:
  477. targets = "None"
  478. if target in self.run_targets:
  479. targets = self.run_targets[target]
  480. logger.debug(3, " %s: %s", target, targets)
  481. logger.debug(3, "tasks:")
  482. for tid in self.taskentries:
  483. logger.debug(3, " %s: %s %s %s",
  484. tid,
  485. self.taskentries[tid].idepends,
  486. self.taskentries[tid].irdepends,
  487. self.taskentries[tid].tdepends)
  488. logger.debug(3, "dependency ids (per fn):")
  489. for fn in self.depids:
  490. logger.debug(3, " %s: %s", fn, self.depids[fn])
  491. logger.debug(3, "runtime dependency ids (per fn):")
  492. for fn in self.rdepids:
  493. logger.debug(3, " %s: %s", fn, self.rdepids[fn])