|
@@ -525,7 +525,7 @@ class BBCooker:
|
|
|
self.parseConfiguration()
|
|
|
|
|
|
fn, cls, mc = bb.cache.virtualfn2realfn(buildfile)
|
|
|
- fn = self.matchFile(fn)
|
|
|
+ fn = self.matchFile(fn, mc)
|
|
|
fn = bb.cache.realfn2virtual(fn, cls, mc)
|
|
|
elif len(pkgs_to_build) == 1:
|
|
|
mc = mc_base(pkgs_to_build[0])
|
|
@@ -542,7 +542,7 @@ class BBCooker:
|
|
|
if fn:
|
|
|
try:
|
|
|
bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array)
|
|
|
- envdata = bb_cache.loadDataFull(fn, self.collection.get_file_appends(fn))
|
|
|
+ envdata = bb_cache.loadDataFull(fn, self.collections[mc].get_file_appends(fn))
|
|
|
except Exception as e:
|
|
|
parselog.exception("Unable to read %s", fn)
|
|
|
raise
|
|
@@ -929,26 +929,33 @@ class BBCooker:
|
|
|
logger.info("Task dependencies saved to 'task-depends.dot'")
|
|
|
|
|
|
def show_appends_with_no_recipes(self):
|
|
|
+ appends_without_recipes = {}
|
|
|
# Determine which bbappends haven't been applied
|
|
|
-
|
|
|
- # First get list of recipes, including skipped
|
|
|
- recipefns = list(self.recipecaches[''].pkg_fn.keys())
|
|
|
- recipefns.extend(self.skiplist.keys())
|
|
|
-
|
|
|
- # Work out list of bbappends that have been applied
|
|
|
- applied_appends = []
|
|
|
- for fn in recipefns:
|
|
|
- applied_appends.extend(self.collection.get_file_appends(fn))
|
|
|
-
|
|
|
- appends_without_recipes = []
|
|
|
- for _, appendfn in self.collection.bbappends:
|
|
|
- if not appendfn in applied_appends:
|
|
|
- appends_without_recipes.append(appendfn)
|
|
|
-
|
|
|
- if appends_without_recipes:
|
|
|
- msg = 'No recipes available for:\n %s' % '\n '.join(appends_without_recipes)
|
|
|
- warn_only = self.data.getVar("BB_DANGLINGAPPENDS_WARNONLY", \
|
|
|
- False) or "no"
|
|
|
+ for mc in self.multiconfigs:
|
|
|
+ # First get list of recipes, including skipped
|
|
|
+ recipefns = list(self.recipecaches[mc].pkg_fn.keys())
|
|
|
+ recipefns.extend(self.skiplist.keys())
|
|
|
+
|
|
|
+ # Work out list of bbappends that have been applied
|
|
|
+ applied_appends = []
|
|
|
+ for fn in recipefns:
|
|
|
+ applied_appends.extend(self.collections[mc].get_file_appends(fn))
|
|
|
+
|
|
|
+ appends_without_recipes[mc] = []
|
|
|
+ for _, appendfn in self.collections[mc].bbappends:
|
|
|
+ if not appendfn in applied_appends:
|
|
|
+ appends_without_recipes[mc].append(appendfn)
|
|
|
+
|
|
|
+ msgs = []
|
|
|
+ for mc in sorted(appends_without_recipes.keys()):
|
|
|
+ if appends_without_recipes[mc]:
|
|
|
+ msgs.append('No recipes in %s available for:\n %s' % (mc if mc else 'default',
|
|
|
+ '\n '.join(appends_without_recipes[mc])))
|
|
|
+
|
|
|
+ if msgs:
|
|
|
+ msg = "\n".join(msgs)
|
|
|
+ warn_only = self.databuilder.mcdata[mc].getVar("BB_DANGLINGAPPENDS_WARNONLY", \
|
|
|
+ False) or "no"
|
|
|
if warn_only.lower() in ("1", "yes", "true"):
|
|
|
bb.warn(msg)
|
|
|
else:
|
|
@@ -1249,15 +1256,15 @@ class BBCooker:
|
|
|
if siggen_cache:
|
|
|
bb.parse.siggen.checksum_cache.mtime_cache.clear()
|
|
|
|
|
|
- def matchFiles(self, bf):
|
|
|
+ def matchFiles(self, bf, mc=''):
|
|
|
"""
|
|
|
Find the .bb files which match the expression in 'buildfile'.
|
|
|
"""
|
|
|
if bf.startswith("/") or bf.startswith("../"):
|
|
|
bf = os.path.abspath(bf)
|
|
|
|
|
|
- self.collection = CookerCollectFiles(self.bbfile_config_priorities)
|
|
|
- filelist, masked, searchdirs = self.collection.collect_bbfiles(self.data, self.data)
|
|
|
+ self.collections = {mc: CookerCollectFiles(self.bbfile_config_priorities, mc)}
|
|
|
+ filelist, masked, searchdirs = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc])
|
|
|
try:
|
|
|
os.stat(bf)
|
|
|
bf = os.path.abspath(bf)
|
|
@@ -1270,12 +1277,12 @@ class BBCooker:
|
|
|
matches.append(f)
|
|
|
return matches
|
|
|
|
|
|
- def matchFile(self, buildfile):
|
|
|
+ def matchFile(self, buildfile, mc=''):
|
|
|
"""
|
|
|
Find the .bb file which matches the expression in 'buildfile'.
|
|
|
Raise an error if multiple files
|
|
|
"""
|
|
|
- matches = self.matchFiles(buildfile)
|
|
|
+ matches = self.matchFiles(buildfile, mc)
|
|
|
if len(matches) != 1:
|
|
|
if matches:
|
|
|
msg = "Unable to match '%s' to a specific recipe file - %s matches found:" % (buildfile, len(matches))
|
|
@@ -1316,14 +1323,14 @@ class BBCooker:
|
|
|
task = "do_%s" % task
|
|
|
|
|
|
fn, cls, mc = bb.cache.virtualfn2realfn(buildfile)
|
|
|
- fn = self.matchFile(fn)
|
|
|
+ fn = self.matchFile(fn, mc)
|
|
|
|
|
|
self.buildSetVars()
|
|
|
self.reset_mtime_caches()
|
|
|
|
|
|
bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array)
|
|
|
|
|
|
- infos = bb_cache.parse(fn, self.collection.get_file_appends(fn))
|
|
|
+ infos = bb_cache.parse(fn, self.collections[mc].get_file_appends(fn))
|
|
|
infos = dict(infos)
|
|
|
|
|
|
fn = bb.cache.realfn2virtual(fn, cls, mc)
|
|
@@ -1552,14 +1559,24 @@ class BBCooker:
|
|
|
for dep in self.configuration.extra_assume_provided:
|
|
|
self.recipecaches[mc].ignored_dependencies.add(dep)
|
|
|
|
|
|
- self.collection = CookerCollectFiles(self.bbfile_config_priorities)
|
|
|
- (filelist, masked, searchdirs) = self.collection.collect_bbfiles(self.data, self.data)
|
|
|
+ self.collections = {}
|
|
|
+
|
|
|
+ mcfilelist = {}
|
|
|
+ total_masked = 0
|
|
|
+ searchdirs = set()
|
|
|
+ for mc in self.multiconfigs:
|
|
|
+ self.collections[mc] = CookerCollectFiles(self.bbfile_config_priorities, mc)
|
|
|
+ (filelist, masked, search) = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc])
|
|
|
+
|
|
|
+ mcfilelist[mc] = filelist
|
|
|
+ total_masked += masked
|
|
|
+ searchdirs |= set(search)
|
|
|
|
|
|
# Add inotify watches for directories searched for bb/bbappend files
|
|
|
for dirent in searchdirs:
|
|
|
self.add_filewatch([[dirent]], dirs=True)
|
|
|
|
|
|
- self.parser = CookerParser(self, filelist, masked)
|
|
|
+ self.parser = CookerParser(self, mcfilelist, total_masked)
|
|
|
self.parsecache_valid = True
|
|
|
|
|
|
self.state = state.parsing
|
|
@@ -1571,7 +1588,7 @@ class BBCooker:
|
|
|
self.show_appends_with_no_recipes()
|
|
|
self.handlePrefProviders()
|
|
|
for mc in self.multiconfigs:
|
|
|
- self.recipecaches[mc].bbfile_priority = self.collection.collection_priorities(self.recipecaches[mc].pkg_fn, self.data)
|
|
|
+ self.recipecaches[mc].bbfile_priority = self.collections[mc].collection_priorities(self.recipecaches[mc].pkg_fn, self.data)
|
|
|
self.state = state.running
|
|
|
|
|
|
# Send an event listing all stamps reachable after parsing
|
|
@@ -1679,7 +1696,8 @@ class CookerExit(bb.event.Event):
|
|
|
|
|
|
|
|
|
class CookerCollectFiles(object):
|
|
|
- def __init__(self, priorities):
|
|
|
+ def __init__(self, priorities, mc=''):
|
|
|
+ self.mc = mc
|
|
|
self.bbappends = []
|
|
|
# Priorities is a list of tupples, with the second element as the pattern.
|
|
|
# We need to sort the list with the longest pattern first, and so on to
|
|
@@ -1846,7 +1864,7 @@ class CookerCollectFiles(object):
|
|
|
(bbappend, filename) = b
|
|
|
if (bbappend == f) or ('%' in bbappend and bbappend.startswith(f[:bbappend.index('%')])):
|
|
|
filelist.append(filename)
|
|
|
- return filelist
|
|
|
+ return tuple(filelist)
|
|
|
|
|
|
def collection_priorities(self, pkgfns, d):
|
|
|
|
|
@@ -1882,7 +1900,8 @@ class CookerCollectFiles(object):
|
|
|
for collection, pattern, regex, _ in self.bbfile_config_priorities:
|
|
|
if regex in unmatched:
|
|
|
if d.getVar('BBFILE_PATTERN_IGNORE_EMPTY_%s' % collection) != '1':
|
|
|
- collectlog.warning("No bb files matched BBFILE_PATTERN_%s '%s'" % (collection, pattern))
|
|
|
+ collectlog.warning("No bb files in %s matched BBFILE_PATTERN_%s '%s'" % (self.mc if self.mc else 'default',
|
|
|
+ collection, pattern))
|
|
|
|
|
|
return priorities
|
|
|
|
|
@@ -1978,8 +1997,8 @@ class Parser(multiprocessing.Process):
|
|
|
bb.event.LogHandler.filter = origfilter
|
|
|
|
|
|
class CookerParser(object):
|
|
|
- def __init__(self, cooker, filelist, masked):
|
|
|
- self.filelist = filelist
|
|
|
+ def __init__(self, cooker, mcfilelist, masked):
|
|
|
+ self.mcfilelist = mcfilelist
|
|
|
self.cooker = cooker
|
|
|
self.cfgdata = cooker.data
|
|
|
self.cfghash = cooker.data_hash
|
|
@@ -1993,25 +2012,27 @@ class CookerParser(object):
|
|
|
|
|
|
self.skipped = 0
|
|
|
self.virtuals = 0
|
|
|
- self.total = len(filelist)
|
|
|
|
|
|
self.current = 0
|
|
|
self.process_names = []
|
|
|
|
|
|
self.bb_cache = bb.cache.Cache(self.cfgbuilder, self.cfghash, cooker.caches_array)
|
|
|
- self.fromcache = []
|
|
|
- self.willparse = []
|
|
|
- for filename in self.filelist:
|
|
|
- appends = self.cooker.collection.get_file_appends(filename)
|
|
|
- if not self.bb_cache.cacheValid(filename, appends):
|
|
|
- self.willparse.append((filename, appends))
|
|
|
- else:
|
|
|
- self.fromcache.append((filename, appends))
|
|
|
- self.toparse = self.total - len(self.fromcache)
|
|
|
+ self.fromcache = set()
|
|
|
+ self.willparse = set()
|
|
|
+ for mc in self.cooker.multiconfigs:
|
|
|
+ for filename in self.mcfilelist[mc]:
|
|
|
+ appends = self.cooker.collections[mc].get_file_appends(filename)
|
|
|
+ if not self.bb_cache.cacheValid(filename, appends):
|
|
|
+ self.willparse.add((filename, appends))
|
|
|
+ else:
|
|
|
+ self.fromcache.add((filename, appends))
|
|
|
+
|
|
|
+ self.total = len(self.fromcache) + len(self.willparse)
|
|
|
+ self.toparse = len(self.willparse)
|
|
|
self.progress_chunk = int(max(self.toparse / 100, 1))
|
|
|
|
|
|
self.num_processes = min(int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS") or
|
|
|
- multiprocessing.cpu_count()), len(self.willparse))
|
|
|
+ multiprocessing.cpu_count()), self.toparse)
|
|
|
|
|
|
self.start()
|
|
|
self.haveshutdown = False
|
|
@@ -2032,7 +2053,7 @@ class CookerParser(object):
|
|
|
|
|
|
def chunkify(lst,n):
|
|
|
return [lst[i::n] for i in range(n)]
|
|
|
- self.jobs = chunkify(self.willparse, self.num_processes)
|
|
|
+ self.jobs = chunkify(list(self.willparse), self.num_processes)
|
|
|
|
|
|
for i in range(0, self.num_processes):
|
|
|
parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, init, self.cooker.configuration.profile)
|
|
@@ -2095,9 +2116,9 @@ class CookerParser(object):
|
|
|
print("Processed parsing statistics saved to %s" % (pout))
|
|
|
|
|
|
def load_cached(self):
|
|
|
- for filename, appends in self.fromcache:
|
|
|
- cached, infos = self.bb_cache.load(filename, appends)
|
|
|
- yield not cached, infos
|
|
|
+ for mc, filename, appends in self.fromcache:
|
|
|
+ cached, infos = self.bb_cache.load(mc, filename, appends)
|
|
|
+ yield not cached, mc, infos
|
|
|
|
|
|
def parse_generator(self):
|
|
|
while True:
|
|
@@ -2119,7 +2140,7 @@ class CookerParser(object):
|
|
|
result = []
|
|
|
parsed = None
|
|
|
try:
|
|
|
- parsed, result = next(self.results)
|
|
|
+ parsed, mc, result = next(self.results)
|
|
|
except StopIteration:
|
|
|
self.shutdown()
|
|
|
return False
|
|
@@ -2181,7 +2202,11 @@ class CookerParser(object):
|
|
|
return True
|
|
|
|
|
|
def reparse(self, filename):
|
|
|
- infos = self.bb_cache.parse(filename, self.cooker.collection.get_file_appends(filename))
|
|
|
- for vfn, info_array in infos:
|
|
|
- (fn, cls, mc) = bb.cache.virtualfn2realfn(vfn)
|
|
|
- self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array)
|
|
|
+ to_reparse = set()
|
|
|
+ for mc in self.cooker.multiconfigs:
|
|
|
+ to_reparse.add((mc, filename, self.cooker.collections[mc].get_file_appends(filename)))
|
|
|
+
|
|
|
+ for mc, filename, appends in to_reparse:
|
|
|
+ infos = self.bb_cache.parse(filename, appends)
|
|
|
+ for vfn, info_array in infos:
|
|
|
+ self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array)
|