123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596 |
- #!/usr/bin/env python
- # ex:ts=4:sw=4:sts=4:et
- # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
- """
- BitBake 'TaskData' implementation
- Task data collection and handling
- """
- # Copyright (C) 2006 Richard Purdie
- #
- # SPDX-License-Identifier: GPL-2.0-only
- #
- # This program is free software; you can redistribute it and/or modify
- # it under the terms of the GNU General Public License version 2 as
- # published by the Free Software Foundation.
- #
- # This program is distributed in the hope that it will be useful,
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- # GNU General Public License for more details.
- #
- # You should have received a copy of the GNU General Public License along
- # with this program; if not, write to the Free Software Foundation, Inc.,
- # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- import logging
- import re
- import bb
- logger = logging.getLogger("BitBake.TaskData")
- def re_match_strings(target, strings):
- """
- Whether or not the string 'target' matches
- any one string of the strings which can be regular expression string
- """
- return any(name == target or re.match(name, target)
- for name in strings)
- class TaskEntry:
- def __init__(self):
- self.tdepends = []
- self.idepends = []
- self.irdepends = []
- class TaskData:
- """
- BitBake Task Data implementation
- """
- def __init__(self, abort = True, skiplist = None, allowincomplete = False):
- self.build_targets = {}
- self.run_targets = {}
- self.external_targets = []
- self.seenfns = []
- self.taskentries = {}
- self.depids = {}
- self.rdepids = {}
- self.consider_msgs_cache = []
- self.failed_deps = []
- self.failed_rdeps = []
- self.failed_fns = []
- self.abort = abort
- self.allowincomplete = allowincomplete
- self.skiplist = skiplist
- self.mcdepends = []
- def add_tasks(self, fn, dataCache):
- """
- Add tasks for a given fn to the database
- """
- task_deps = dataCache.task_deps[fn]
- if fn in self.failed_fns:
- bb.msg.fatal("TaskData", "Trying to re-add a failed file? Something is broken...")
- # Check if we've already seen this fn
- if fn in self.seenfns:
- return
- self.seenfns.append(fn)
- self.add_extra_deps(fn, dataCache)
- def add_mcdepends(task):
- for dep in task_deps['mcdepends'][task].split():
- if len(dep.split(':')) != 5:
- bb.msg.fatal("TaskData", "Error for %s:%s[%s], multiconfig dependency %s does not contain exactly four ':' characters.\n Task '%s' should be specified in the form 'multiconfig:fromMC:toMC:packagename:task'" % (fn, task, 'mcdepends', dep, 'mcdepends'))
- if dep not in self.mcdepends:
- self.mcdepends.append(dep)
- # Common code for dep_name/depends = 'depends'/idepends and 'rdepends'/irdepends
- def handle_deps(task, dep_name, depends, seen):
- if dep_name in task_deps and task in task_deps[dep_name]:
- ids = []
- for dep in task_deps[dep_name][task].split():
- if dep:
- parts = dep.split(":")
- if len(parts) != 2:
- bb.msg.fatal("TaskData", "Error for %s:%s[%s], dependency %s in '%s' does not contain exactly one ':' character.\n Task '%s' should be specified in the form 'packagename:task'" % (fn, task, dep_name, dep, task_deps[dep_name][task], dep_name))
- ids.append((parts[0], parts[1]))
- seen(parts[0])
- depends.extend(ids)
- for task in task_deps['tasks']:
- tid = "%s:%s" % (fn, task)
- self.taskentries[tid] = TaskEntry()
- # Work out task dependencies
- parentids = []
- for dep in task_deps['parents'][task]:
- if dep not in task_deps['tasks']:
- bb.debug(2, "Not adding dependency of %s on %s since %s does not exist" % (task, dep, dep))
- continue
- parentid = "%s:%s" % (fn, dep)
- parentids.append(parentid)
- self.taskentries[tid].tdepends.extend(parentids)
- # Touch all intertask dependencies
- handle_deps(task, 'depends', self.taskentries[tid].idepends, self.seen_build_target)
- handle_deps(task, 'rdepends', self.taskentries[tid].irdepends, self.seen_run_target)
- if 'mcdepends' in task_deps and task in task_deps['mcdepends']:
- add_mcdepends(task)
- # Work out build dependencies
- if not fn in self.depids:
- dependids = set()
- for depend in dataCache.deps[fn]:
- dependids.add(depend)
- self.depids[fn] = list(dependids)
- logger.debug(2, "Added dependencies %s for %s", str(dataCache.deps[fn]), fn)
- # Work out runtime dependencies
- if not fn in self.rdepids:
- rdependids = set()
- rdepends = dataCache.rundeps[fn]
- rrecs = dataCache.runrecs[fn]
- rdependlist = []
- rreclist = []
- for package in rdepends:
- for rdepend in rdepends[package]:
- rdependlist.append(rdepend)
- rdependids.add(rdepend)
- for package in rrecs:
- for rdepend in rrecs[package]:
- rreclist.append(rdepend)
- rdependids.add(rdepend)
- if rdependlist:
- logger.debug(2, "Added runtime dependencies %s for %s", str(rdependlist), fn)
- if rreclist:
- logger.debug(2, "Added runtime recommendations %s for %s", str(rreclist), fn)
- self.rdepids[fn] = list(rdependids)
- for dep in self.depids[fn]:
- self.seen_build_target(dep)
- if dep in self.failed_deps:
- self.fail_fn(fn)
- return
- for dep in self.rdepids[fn]:
- self.seen_run_target(dep)
- if dep in self.failed_rdeps:
- self.fail_fn(fn)
- return
- def add_extra_deps(self, fn, dataCache):
- func = dataCache.extradepsfunc.get(fn, None)
- if func:
- bb.providers.buildWorldTargetList(dataCache)
- pn = dataCache.pkg_fn[fn]
- params = {'deps': dataCache.deps[fn],
- 'world_target': dataCache.world_target,
- 'pkg_pn': dataCache.pkg_pn,
- 'self_pn': pn}
- funcname = '_%s_calculate_extra_depends' % pn.replace('-', '_')
- paramlist = ','.join(params.keys())
- func = 'def %s(%s):\n%s\n\n%s(%s)' % (funcname, paramlist, func, funcname, paramlist)
- bb.utils.better_exec(func, params)
- def have_build_target(self, target):
- """
- Have we a build target matching this name?
- """
- if target in self.build_targets and self.build_targets[target]:
- return True
- return False
- def have_runtime_target(self, target):
- """
- Have we a runtime target matching this name?
- """
- if target in self.run_targets and self.run_targets[target]:
- return True
- return False
- def seen_build_target(self, name):
- """
- Maintain a list of build targets
- """
- if name not in self.build_targets:
- self.build_targets[name] = []
- def add_build_target(self, fn, item):
- """
- Add a build target.
- If already present, append the provider fn to the list
- """
- if item in self.build_targets:
- if fn in self.build_targets[item]:
- return
- self.build_targets[item].append(fn)
- return
- self.build_targets[item] = [fn]
- def seen_run_target(self, name):
- """
- Maintain a list of runtime build targets
- """
- if name not in self.run_targets:
- self.run_targets[name] = []
- def add_runtime_target(self, fn, item):
- """
- Add a runtime target.
- If already present, append the provider fn to the list
- """
- if item in self.run_targets:
- if fn in self.run_targets[item]:
- return
- self.run_targets[item].append(fn)
- return
- self.run_targets[item] = [fn]
- def mark_external_target(self, target):
- """
- Mark a build target as being externally requested
- """
- if target not in self.external_targets:
- self.external_targets.append(target)
- def get_unresolved_build_targets(self, dataCache):
- """
- Return a list of build targets who's providers
- are unknown.
- """
- unresolved = []
- for target in self.build_targets:
- if re_match_strings(target, dataCache.ignored_dependencies):
- continue
- if target in self.failed_deps:
- continue
- if not self.build_targets[target]:
- unresolved.append(target)
- return unresolved
- def get_unresolved_run_targets(self, dataCache):
- """
- Return a list of runtime targets who's providers
- are unknown.
- """
- unresolved = []
- for target in self.run_targets:
- if re_match_strings(target, dataCache.ignored_dependencies):
- continue
- if target in self.failed_rdeps:
- continue
- if not self.run_targets[target]:
- unresolved.append(target)
- return unresolved
- def get_provider(self, item):
- """
- Return a list of providers of item
- """
- return self.build_targets[item]
- def get_dependees(self, item):
- """
- Return a list of targets which depend on item
- """
- dependees = []
- for fn in self.depids:
- if item in self.depids[fn]:
- dependees.append(fn)
- return dependees
- def get_rdependees(self, item):
- """
- Return a list of targets which depend on runtime item
- """
- dependees = []
- for fn in self.rdepids:
- if item in self.rdepids[fn]:
- dependees.append(fn)
- return dependees
- def get_reasons(self, item, runtime=False):
- """
- Get the reason(s) for an item not being provided, if any
- """
- reasons = []
- if self.skiplist:
- for fn in self.skiplist:
- skipitem = self.skiplist[fn]
- if skipitem.pn == item:
- reasons.append("%s was skipped: %s" % (skipitem.pn, skipitem.skipreason))
- elif runtime and item in skipitem.rprovides:
- reasons.append("%s RPROVIDES %s but was skipped: %s" % (skipitem.pn, item, skipitem.skipreason))
- elif not runtime and item in skipitem.provides:
- reasons.append("%s PROVIDES %s but was skipped: %s" % (skipitem.pn, item, skipitem.skipreason))
- return reasons
- def get_close_matches(self, item, provider_list):
- import difflib
- if self.skiplist:
- skipped = []
- for fn in self.skiplist:
- skipped.append(self.skiplist[fn].pn)
- full_list = provider_list + skipped
- else:
- full_list = provider_list
- return difflib.get_close_matches(item, full_list, cutoff=0.7)
- def add_provider(self, cfgData, dataCache, item):
- try:
- self.add_provider_internal(cfgData, dataCache, item)
- except bb.providers.NoProvider:
- if self.abort:
- raise
- self.remove_buildtarget(item)
- self.mark_external_target(item)
- def add_provider_internal(self, cfgData, dataCache, item):
- """
- Add the providers of item to the task data
- Mark entries were specifically added externally as against dependencies
- added internally during dependency resolution
- """
- if re_match_strings(item, dataCache.ignored_dependencies):
- return
- if not item in dataCache.providers:
- close_matches = self.get_close_matches(item, list(dataCache.providers.keys()))
- # Is it in RuntimeProviders ?
- all_p = bb.providers.getRuntimeProviders(dataCache, item)
- for fn in all_p:
- new = dataCache.pkg_fn[fn] + " RPROVIDES " + item
- if new not in close_matches:
- close_matches.append(new)
- bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees(item), reasons=self.get_reasons(item), close_matches=close_matches), cfgData)
- raise bb.providers.NoProvider(item)
- if self.have_build_target(item):
- return
- all_p = dataCache.providers[item]
- eligible, foundUnique = bb.providers.filterProviders(all_p, item, cfgData, dataCache)
- eligible = [p for p in eligible if not p in self.failed_fns]
- if not eligible:
- bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees(item), reasons=["No eligible PROVIDERs exist for '%s'" % item]), cfgData)
- raise bb.providers.NoProvider(item)
- if len(eligible) > 1 and foundUnique == False:
- if item not in self.consider_msgs_cache:
- providers_list = []
- for fn in eligible:
- providers_list.append(dataCache.pkg_fn[fn])
- bb.event.fire(bb.event.MultipleProviders(item, providers_list), cfgData)
- self.consider_msgs_cache.append(item)
- for fn in eligible:
- if fn in self.failed_fns:
- continue
- logger.debug(2, "adding %s to satisfy %s", fn, item)
- self.add_build_target(fn, item)
- self.add_tasks(fn, dataCache)
- #item = dataCache.pkg_fn[fn]
- def add_rprovider(self, cfgData, dataCache, item):
- """
- Add the runtime providers of item to the task data
- (takes item names from RDEPENDS/PACKAGES namespace)
- """
- if re_match_strings(item, dataCache.ignored_dependencies):
- return
- if self.have_runtime_target(item):
- return
- all_p = bb.providers.getRuntimeProviders(dataCache, item)
- if not all_p:
- bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees(item), reasons=self.get_reasons(item, True)), cfgData)
- raise bb.providers.NoRProvider(item)
- eligible, numberPreferred = bb.providers.filterProvidersRunTime(all_p, item, cfgData, dataCache)
- eligible = [p for p in eligible if not p in self.failed_fns]
- if not eligible:
- bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees(item), reasons=["No eligible RPROVIDERs exist for '%s'" % item]), cfgData)
- raise bb.providers.NoRProvider(item)
- if len(eligible) > 1 and numberPreferred == 0:
- if item not in self.consider_msgs_cache:
- providers_list = []
- for fn in eligible:
- providers_list.append(dataCache.pkg_fn[fn])
- bb.event.fire(bb.event.MultipleProviders(item, providers_list, runtime=True), cfgData)
- self.consider_msgs_cache.append(item)
- if numberPreferred > 1:
- if item not in self.consider_msgs_cache:
- providers_list = []
- for fn in eligible:
- providers_list.append(dataCache.pkg_fn[fn])
- bb.event.fire(bb.event.MultipleProviders(item, providers_list, runtime=True), cfgData)
- self.consider_msgs_cache.append(item)
- raise bb.providers.MultipleRProvider(item)
- # run through the list until we find one that we can build
- for fn in eligible:
- if fn in self.failed_fns:
- continue
- logger.debug(2, "adding '%s' to satisfy runtime '%s'", fn, item)
- self.add_runtime_target(fn, item)
- self.add_tasks(fn, dataCache)
- def fail_fn(self, fn, missing_list=None):
- """
- Mark a file as failed (unbuildable)
- Remove any references from build and runtime provider lists
- missing_list, A list of missing requirements for this target
- """
- if fn in self.failed_fns:
- return
- if not missing_list:
- missing_list = []
- logger.debug(1, "File '%s' is unbuildable, removing...", fn)
- self.failed_fns.append(fn)
- for target in self.build_targets:
- if fn in self.build_targets[target]:
- self.build_targets[target].remove(fn)
- if len(self.build_targets[target]) == 0:
- self.remove_buildtarget(target, missing_list)
- for target in self.run_targets:
- if fn in self.run_targets[target]:
- self.run_targets[target].remove(fn)
- if len(self.run_targets[target]) == 0:
- self.remove_runtarget(target, missing_list)
- def remove_buildtarget(self, target, missing_list=None):
- """
- Mark a build target as failed (unbuildable)
- Trigger removal of any files that have this as a dependency
- """
- if not missing_list:
- missing_list = [target]
- else:
- missing_list = [target] + missing_list
- logger.verbose("Target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", target, missing_list)
- self.failed_deps.append(target)
- dependees = self.get_dependees(target)
- for fn in dependees:
- self.fail_fn(fn, missing_list)
- for tid in self.taskentries:
- for (idepend, idependtask) in self.taskentries[tid].idepends:
- if idepend == target:
- fn = tid.rsplit(":",1)[0]
- self.fail_fn(fn, missing_list)
- if self.abort and target in self.external_targets:
- logger.error("Required build target '%s' has no buildable providers.\nMissing or unbuildable dependency chain was: %s", target, missing_list)
- raise bb.providers.NoProvider(target)
- def remove_runtarget(self, target, missing_list=None):
- """
- Mark a run target as failed (unbuildable)
- Trigger removal of any files that have this as a dependency
- """
- if not missing_list:
- missing_list = [target]
- else:
- missing_list = [target] + missing_list
- logger.info("Runtime target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", target, missing_list)
- self.failed_rdeps.append(target)
- dependees = self.get_rdependees(target)
- for fn in dependees:
- self.fail_fn(fn, missing_list)
- for tid in self.taskentries:
- for (idepend, idependtask) in self.taskentries[tid].irdepends:
- if idepend == target:
- fn = tid.rsplit(":",1)[0]
- self.fail_fn(fn, missing_list)
- def add_unresolved(self, cfgData, dataCache):
- """
- Resolve all unresolved build and runtime targets
- """
- logger.info("Resolving any missing task queue dependencies")
- while True:
- added = 0
- for target in self.get_unresolved_build_targets(dataCache):
- try:
- self.add_provider_internal(cfgData, dataCache, target)
- added = added + 1
- except bb.providers.NoProvider:
- if self.abort and target in self.external_targets and not self.allowincomplete:
- raise
- if not self.allowincomplete:
- self.remove_buildtarget(target)
- for target in self.get_unresolved_run_targets(dataCache):
- try:
- self.add_rprovider(cfgData, dataCache, target)
- added = added + 1
- except (bb.providers.NoRProvider, bb.providers.MultipleRProvider):
- self.remove_runtarget(target)
- logger.debug(1, "Resolved " + str(added) + " extra dependencies")
- if added == 0:
- break
- # self.dump_data()
- def get_providermap(self, prefix=None):
- provmap = {}
- for name in self.build_targets:
- if prefix and not name.startswith(prefix):
- continue
- if self.have_build_target(name):
- provider = self.get_provider(name)
- if provider:
- provmap[name] = provider[0]
- return provmap
- def get_mcdepends(self):
- return self.mcdepends
- def dump_data(self):
- """
- Dump some debug information on the internal data structures
- """
- logger.debug(3, "build_names:")
- logger.debug(3, ", ".join(self.build_targets))
- logger.debug(3, "run_names:")
- logger.debug(3, ", ".join(self.run_targets))
- logger.debug(3, "build_targets:")
- for target in self.build_targets:
- targets = "None"
- if target in self.build_targets:
- targets = self.build_targets[target]
- logger.debug(3, " %s: %s", target, targets)
- logger.debug(3, "run_targets:")
- for target in self.run_targets:
- targets = "None"
- if target in self.run_targets:
- targets = self.run_targets[target]
- logger.debug(3, " %s: %s", target, targets)
- logger.debug(3, "tasks:")
- for tid in self.taskentries:
- logger.debug(3, " %s: %s %s %s",
- tid,
- self.taskentries[tid].idepends,
- self.taskentries[tid].irdepends,
- self.taskentries[tid].tdepends)
- logger.debug(3, "dependency ids (per fn):")
- for fn in self.depids:
- logger.debug(3, " %s: %s", fn, self.depids[fn])
- logger.debug(3, "runtime dependency ids (per fn):")
- for fn in self.rdepids:
- logger.debug(3, " %s: %s", fn, self.rdepids[fn])
|