Browse Source

bitbake: cooker: Split file collections per multiconfig

Splits the cooker to track a collection per multiconfig instead of a
single collection for all multiconfigs. Practically speaking, this
allows each multiconfigs to each have different BBMASKs that apply to it
instead of each one using the mask specified in the base configuration.

Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Joshua Watt 4 years ago
parent
commit
dd6d8eca20
7 changed files with 148 additions and 91 deletions
  1. 1 1
      lib/bb/cache.py
  2. 29 8
      lib/bb/command.py
  3. 82 57
      lib/bb/cooker.py
  4. 6 5
      lib/bb/runqueue.py
  5. 13 11
      lib/bb/tinfoil.py
  6. 15 7
      lib/bblayers/action.py
  7. 2 2
      lib/bblayers/query.py

+ 1 - 1
lib/bb/cache.py

@@ -623,7 +623,7 @@ class Cache(NoCache):
                         self.remove(fn)
                         self.remove(fn)
                         return False
                         return False
 
 
-        if appends != info_array[0].appends:
+        if tuple(appends) != tuple(info_array[0].appends):
             logger.debug(2, "Cache: appends for %s changed", fn)
             logger.debug(2, "Cache: appends for %s changed", fn)
             logger.debug(2, "%s to %s" % (str(appends), str(info_array[0].appends)))
             logger.debug(2, "%s to %s" % (str(appends), str(info_array[0].appends)))
             self.remove(fn)
             self.remove(fn)

+ 29 - 8
lib/bb/command.py

@@ -232,7 +232,11 @@ class CommandsSync:
 
 
     def matchFile(self, command, params):
     def matchFile(self, command, params):
         fMatch = params[0]
         fMatch = params[0]
-        return command.cooker.matchFile(fMatch)
+        try:
+            mc = params[0]
+        except IndexError:
+            mc = ''
+        return command.cooker.matchFile(fMatch, mc)
     matchFile.needconfig = False
     matchFile.needconfig = False
 
 
     def getUIHandlerNum(self, command, params):
     def getUIHandlerNum(self, command, params):
@@ -395,22 +399,38 @@ class CommandsSync:
     def getSkippedRecipes(self, command, params):
     def getSkippedRecipes(self, command, params):
         # Return list sorted by reverse priority order
         # Return list sorted by reverse priority order
         import bb.cache
         import bb.cache
-        skipdict = OrderedDict(sorted(command.cooker.skiplist.items(),
-                                      key=lambda x: (-command.cooker.collection.calc_bbfile_priority(bb.cache.virtualfn2realfn(x[0])[0]), x[0])))
+        def sortkey(x):
+            vfn, _ = x
+            realfn, _, mc = bb.cache.virtualfn2realfn(vfn)
+            return (-command.cooker.collections[mc].calc_bbfile_priority(realfn), vfn)
+
+        skipdict = OrderedDict(sorted(command.cooker.skiplist.items(), key=sortkey))
         return list(skipdict.items())
         return list(skipdict.items())
     getSkippedRecipes.readonly = True
     getSkippedRecipes.readonly = True
 
 
     def getOverlayedRecipes(self, command, params):
     def getOverlayedRecipes(self, command, params):
-        return list(command.cooker.collection.overlayed.items())
+        try:
+            mc = params[0]
+        except IndexError:
+            mc = ''
+        return list(command.cooker.collections[mc].overlayed.items())
     getOverlayedRecipes.readonly = True
     getOverlayedRecipes.readonly = True
 
 
     def getFileAppends(self, command, params):
     def getFileAppends(self, command, params):
         fn = params[0]
         fn = params[0]
-        return command.cooker.collection.get_file_appends(fn)
+        try:
+            mc = params[1]
+        except IndexError:
+            mc = ''
+        return command.cooker.collections[mc].get_file_appends(fn)
     getFileAppends.readonly = True
     getFileAppends.readonly = True
 
 
     def getAllAppends(self, command, params):
     def getAllAppends(self, command, params):
-        return command.cooker.collection.bbappends
+        try:
+            mc = params[0]
+        except IndexError:
+            mc = ''
+        return command.cooker.collections[mc].bbappends
     getAllAppends.readonly = True
     getAllAppends.readonly = True
 
 
     def findProviders(self, command, params):
     def findProviders(self, command, params):
@@ -496,6 +516,7 @@ class CommandsSync:
         for the recipe.
         for the recipe.
         """
         """
         fn = params[0]
         fn = params[0]
+        mc = bb.runqueue.mc_from_tid(fn)
         appends = params[1]
         appends = params[1]
         appendlist = params[2]
         appendlist = params[2]
         if len(params) > 3:
         if len(params) > 3:
@@ -507,7 +528,7 @@ class CommandsSync:
             if appendlist is not None:
             if appendlist is not None:
                 appendfiles = appendlist
                 appendfiles = appendlist
             else:
             else:
-                appendfiles = command.cooker.collection.get_file_appends(fn)
+                appendfiles = command.cooker.collections[mc].get_file_appends(fn)
         else:
         else:
             appendfiles = []
             appendfiles = []
         # We are calling bb.cache locally here rather than on the server,
         # We are calling bb.cache locally here rather than on the server,
@@ -517,7 +538,7 @@ class CommandsSync:
         if config_data:
         if config_data:
             # We have to use a different function here if we're passing in a datastore
             # We have to use a different function here if we're passing in a datastore
             # NOTE: we took a copy above, so we don't do it here again
             # NOTE: we took a copy above, so we don't do it here again
-            envdata = bb.cache.parse_recipe(config_data, fn, appendfiles)['']
+            envdata = bb.cache.parse_recipe(config_data, fn, appendfiles, mc)['']
         else:
         else:
             # Use the standard path
             # Use the standard path
             parser = bb.cache.NoCache(command.cooker.databuilder)
             parser = bb.cache.NoCache(command.cooker.databuilder)

+ 82 - 57
lib/bb/cooker.py

@@ -525,7 +525,7 @@ class BBCooker:
             self.parseConfiguration()
             self.parseConfiguration()
 
 
             fn, cls, mc = bb.cache.virtualfn2realfn(buildfile)
             fn, cls, mc = bb.cache.virtualfn2realfn(buildfile)
-            fn = self.matchFile(fn)
+            fn = self.matchFile(fn, mc)
             fn = bb.cache.realfn2virtual(fn, cls, mc)
             fn = bb.cache.realfn2virtual(fn, cls, mc)
         elif len(pkgs_to_build) == 1:
         elif len(pkgs_to_build) == 1:
             mc = mc_base(pkgs_to_build[0])
             mc = mc_base(pkgs_to_build[0])
@@ -542,7 +542,7 @@ class BBCooker:
         if fn:
         if fn:
             try:
             try:
                 bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array)
                 bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array)
-                envdata = bb_cache.loadDataFull(fn, self.collection.get_file_appends(fn))
+                envdata = bb_cache.loadDataFull(fn, self.collections[mc].get_file_appends(fn))
             except Exception as e:
             except Exception as e:
                 parselog.exception("Unable to read %s", fn)
                 parselog.exception("Unable to read %s", fn)
                 raise
                 raise
@@ -929,26 +929,33 @@ class BBCooker:
         logger.info("Task dependencies saved to 'task-depends.dot'")
         logger.info("Task dependencies saved to 'task-depends.dot'")
 
 
     def show_appends_with_no_recipes(self):
     def show_appends_with_no_recipes(self):
+        appends_without_recipes = {}
         # Determine which bbappends haven't been applied
         # Determine which bbappends haven't been applied
-
-        # First get list of recipes, including skipped
-        recipefns = list(self.recipecaches[''].pkg_fn.keys())
-        recipefns.extend(self.skiplist.keys())
-
-        # Work out list of bbappends that have been applied
-        applied_appends = []
-        for fn in recipefns:
-            applied_appends.extend(self.collection.get_file_appends(fn))
-
-        appends_without_recipes = []
-        for _, appendfn in self.collection.bbappends:
-            if not appendfn in applied_appends:
-                appends_without_recipes.append(appendfn)
-
-        if appends_without_recipes:
-            msg = 'No recipes available for:\n  %s' % '\n  '.join(appends_without_recipes)
-            warn_only = self.data.getVar("BB_DANGLINGAPPENDS_WARNONLY", \
-                 False) or "no"
+        for mc in self.multiconfigs:
+            # First get list of recipes, including skipped
+            recipefns = list(self.recipecaches[mc].pkg_fn.keys())
+            recipefns.extend(self.skiplist.keys())
+
+            # Work out list of bbappends that have been applied
+            applied_appends = []
+            for fn in recipefns:
+                applied_appends.extend(self.collections[mc].get_file_appends(fn))
+
+            appends_without_recipes[mc] = []
+            for _, appendfn in self.collections[mc].bbappends:
+                if not appendfn in applied_appends:
+                    appends_without_recipes[mc].append(appendfn)
+
+        msgs = []
+        for mc in sorted(appends_without_recipes.keys()):
+            if appends_without_recipes[mc]:
+                msgs.append('No recipes in %s available for:\n  %s' % (mc if mc else 'default',
+                                                                        '\n  '.join(appends_without_recipes[mc])))
+
+        if msgs:
+            msg = "\n".join(msgs)
+            warn_only = self.databuilder.mcdata[mc].getVar("BB_DANGLINGAPPENDS_WARNONLY", \
+                False) or "no"
             if warn_only.lower() in ("1", "yes", "true"):
             if warn_only.lower() in ("1", "yes", "true"):
                 bb.warn(msg)
                 bb.warn(msg)
             else:
             else:
@@ -1249,15 +1256,15 @@ class BBCooker:
         if siggen_cache:
         if siggen_cache:
             bb.parse.siggen.checksum_cache.mtime_cache.clear()
             bb.parse.siggen.checksum_cache.mtime_cache.clear()
 
 
-    def matchFiles(self, bf):
+    def matchFiles(self, bf, mc=''):
         """
         """
         Find the .bb files which match the expression in 'buildfile'.
         Find the .bb files which match the expression in 'buildfile'.
         """
         """
         if bf.startswith("/") or bf.startswith("../"):
         if bf.startswith("/") or bf.startswith("../"):
             bf = os.path.abspath(bf)
             bf = os.path.abspath(bf)
 
 
-        self.collection = CookerCollectFiles(self.bbfile_config_priorities)
-        filelist, masked, searchdirs = self.collection.collect_bbfiles(self.data, self.data)
+        self.collections = {mc: CookerCollectFiles(self.bbfile_config_priorities, mc)}
+        filelist, masked, searchdirs = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc])
         try:
         try:
             os.stat(bf)
             os.stat(bf)
             bf = os.path.abspath(bf)
             bf = os.path.abspath(bf)
@@ -1270,12 +1277,12 @@ class BBCooker:
                     matches.append(f)
                     matches.append(f)
             return matches
             return matches
 
 
-    def matchFile(self, buildfile):
+    def matchFile(self, buildfile, mc=''):
         """
         """
         Find the .bb file which matches the expression in 'buildfile'.
         Find the .bb file which matches the expression in 'buildfile'.
         Raise an error if multiple files
         Raise an error if multiple files
         """
         """
-        matches = self.matchFiles(buildfile)
+        matches = self.matchFiles(buildfile, mc)
         if len(matches) != 1:
         if len(matches) != 1:
             if matches:
             if matches:
                 msg = "Unable to match '%s' to a specific recipe file - %s matches found:" % (buildfile, len(matches))
                 msg = "Unable to match '%s' to a specific recipe file - %s matches found:" % (buildfile, len(matches))
@@ -1316,14 +1323,14 @@ class BBCooker:
             task = "do_%s" % task
             task = "do_%s" % task
 
 
         fn, cls, mc = bb.cache.virtualfn2realfn(buildfile)
         fn, cls, mc = bb.cache.virtualfn2realfn(buildfile)
-        fn = self.matchFile(fn)
+        fn = self.matchFile(fn, mc)
 
 
         self.buildSetVars()
         self.buildSetVars()
         self.reset_mtime_caches()
         self.reset_mtime_caches()
 
 
         bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array)
         bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array)
 
 
-        infos = bb_cache.parse(fn, self.collection.get_file_appends(fn))
+        infos = bb_cache.parse(fn, self.collections[mc].get_file_appends(fn))
         infos = dict(infos)
         infos = dict(infos)
 
 
         fn = bb.cache.realfn2virtual(fn, cls, mc)
         fn = bb.cache.realfn2virtual(fn, cls, mc)
@@ -1552,14 +1559,24 @@ class BBCooker:
                 for dep in self.configuration.extra_assume_provided:
                 for dep in self.configuration.extra_assume_provided:
                     self.recipecaches[mc].ignored_dependencies.add(dep)
                     self.recipecaches[mc].ignored_dependencies.add(dep)
 
 
-            self.collection = CookerCollectFiles(self.bbfile_config_priorities)
-            (filelist, masked, searchdirs) = self.collection.collect_bbfiles(self.data, self.data)
+            self.collections = {}
+
+            mcfilelist = {}
+            total_masked = 0
+            searchdirs = set()
+            for mc in self.multiconfigs:
+                self.collections[mc] = CookerCollectFiles(self.bbfile_config_priorities, mc)
+                (filelist, masked, search) = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc])
+
+                mcfilelist[mc] = filelist
+                total_masked += masked
+                searchdirs |= set(search)
 
 
             # Add inotify watches for directories searched for bb/bbappend files
             # Add inotify watches for directories searched for bb/bbappend files
             for dirent in searchdirs:
             for dirent in searchdirs:
                 self.add_filewatch([[dirent]], dirs=True)
                 self.add_filewatch([[dirent]], dirs=True)
 
 
-            self.parser = CookerParser(self, filelist, masked)
+            self.parser = CookerParser(self, mcfilelist, total_masked)
             self.parsecache_valid = True
             self.parsecache_valid = True
 
 
         self.state = state.parsing
         self.state = state.parsing
@@ -1571,7 +1588,7 @@ class BBCooker:
             self.show_appends_with_no_recipes()
             self.show_appends_with_no_recipes()
             self.handlePrefProviders()
             self.handlePrefProviders()
             for mc in self.multiconfigs:
             for mc in self.multiconfigs:
-                self.recipecaches[mc].bbfile_priority = self.collection.collection_priorities(self.recipecaches[mc].pkg_fn, self.data)
+                self.recipecaches[mc].bbfile_priority = self.collections[mc].collection_priorities(self.recipecaches[mc].pkg_fn, self.data)
             self.state = state.running
             self.state = state.running
 
 
             # Send an event listing all stamps reachable after parsing
             # Send an event listing all stamps reachable after parsing
@@ -1679,7 +1696,8 @@ class CookerExit(bb.event.Event):
 
 
 
 
 class CookerCollectFiles(object):
 class CookerCollectFiles(object):
-    def __init__(self, priorities):
+    def __init__(self, priorities, mc=''):
+        self.mc = mc
         self.bbappends = []
         self.bbappends = []
         # Priorities is a list of tupples, with the second element as the pattern.
         # Priorities is a list of tupples, with the second element as the pattern.
         # We need to sort the list with the longest pattern first, and so on to
         # We need to sort the list with the longest pattern first, and so on to
@@ -1846,7 +1864,7 @@ class CookerCollectFiles(object):
             (bbappend, filename) = b
             (bbappend, filename) = b
             if (bbappend == f) or ('%' in bbappend and bbappend.startswith(f[:bbappend.index('%')])):
             if (bbappend == f) or ('%' in bbappend and bbappend.startswith(f[:bbappend.index('%')])):
                 filelist.append(filename)
                 filelist.append(filename)
-        return filelist
+        return tuple(filelist)
 
 
     def collection_priorities(self, pkgfns, d):
     def collection_priorities(self, pkgfns, d):
 
 
@@ -1882,7 +1900,8 @@ class CookerCollectFiles(object):
         for collection, pattern, regex, _ in self.bbfile_config_priorities:
         for collection, pattern, regex, _ in self.bbfile_config_priorities:
             if regex in unmatched:
             if regex in unmatched:
                 if d.getVar('BBFILE_PATTERN_IGNORE_EMPTY_%s' % collection) != '1':
                 if d.getVar('BBFILE_PATTERN_IGNORE_EMPTY_%s' % collection) != '1':
-                    collectlog.warning("No bb files matched BBFILE_PATTERN_%s '%s'" % (collection, pattern))
+                    collectlog.warning("No bb files in %s matched BBFILE_PATTERN_%s '%s'" % (self.mc if self.mc else 'default',
+                                                                                             collection, pattern))
 
 
         return priorities
         return priorities
 
 
@@ -1978,8 +1997,8 @@ class Parser(multiprocessing.Process):
             bb.event.LogHandler.filter = origfilter
             bb.event.LogHandler.filter = origfilter
 
 
 class CookerParser(object):
 class CookerParser(object):
-    def __init__(self, cooker, filelist, masked):
-        self.filelist = filelist
+    def __init__(self, cooker, mcfilelist, masked):
+        self.mcfilelist = mcfilelist
         self.cooker = cooker
         self.cooker = cooker
         self.cfgdata = cooker.data
         self.cfgdata = cooker.data
         self.cfghash = cooker.data_hash
         self.cfghash = cooker.data_hash
@@ -1993,25 +2012,27 @@ class CookerParser(object):
 
 
         self.skipped = 0
         self.skipped = 0
         self.virtuals = 0
         self.virtuals = 0
-        self.total = len(filelist)
 
 
         self.current = 0
         self.current = 0
         self.process_names = []
         self.process_names = []
 
 
         self.bb_cache = bb.cache.Cache(self.cfgbuilder, self.cfghash, cooker.caches_array)
         self.bb_cache = bb.cache.Cache(self.cfgbuilder, self.cfghash, cooker.caches_array)
-        self.fromcache = []
-        self.willparse = []
-        for filename in self.filelist:
-            appends = self.cooker.collection.get_file_appends(filename)
-            if not self.bb_cache.cacheValid(filename, appends):
-                self.willparse.append((filename, appends))
-            else:
-                self.fromcache.append((filename, appends))
-        self.toparse = self.total - len(self.fromcache)
+        self.fromcache = set()
+        self.willparse = set()
+        for mc in self.cooker.multiconfigs:
+            for filename in self.mcfilelist[mc]:
+                appends = self.cooker.collections[mc].get_file_appends(filename)
+                if not self.bb_cache.cacheValid(filename, appends):
+                    self.willparse.add((filename, appends))
+                else:
+                    self.fromcache.add((filename, appends))
+
+        self.total = len(self.fromcache) + len(self.willparse)
+        self.toparse = len(self.willparse)
         self.progress_chunk = int(max(self.toparse / 100, 1))
         self.progress_chunk = int(max(self.toparse / 100, 1))
 
 
         self.num_processes = min(int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS") or
         self.num_processes = min(int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS") or
-                                 multiprocessing.cpu_count()), len(self.willparse))
+                                 multiprocessing.cpu_count()), self.toparse)
 
 
         self.start()
         self.start()
         self.haveshutdown = False
         self.haveshutdown = False
@@ -2032,7 +2053,7 @@ class CookerParser(object):
 
 
             def chunkify(lst,n):
             def chunkify(lst,n):
                 return [lst[i::n] for i in range(n)]
                 return [lst[i::n] for i in range(n)]
-            self.jobs = chunkify(self.willparse, self.num_processes)
+            self.jobs = chunkify(list(self.willparse), self.num_processes)
 
 
             for i in range(0, self.num_processes):
             for i in range(0, self.num_processes):
                 parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, init, self.cooker.configuration.profile)
                 parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, init, self.cooker.configuration.profile)
@@ -2095,9 +2116,9 @@ class CookerParser(object):
             print("Processed parsing statistics saved to %s" % (pout))
             print("Processed parsing statistics saved to %s" % (pout))
 
 
     def load_cached(self):
     def load_cached(self):
-        for filename, appends in self.fromcache:
-            cached, infos = self.bb_cache.load(filename, appends)
-            yield not cached, infos
+        for mc, filename, appends in self.fromcache:
+            cached, infos = self.bb_cache.load(mc, filename, appends)
+            yield not cached, mc, infos
 
 
     def parse_generator(self):
     def parse_generator(self):
         while True:
         while True:
@@ -2119,7 +2140,7 @@ class CookerParser(object):
         result = []
         result = []
         parsed = None
         parsed = None
         try:
         try:
-            parsed, result = next(self.results)
+            parsed, mc, result = next(self.results)
         except StopIteration:
         except StopIteration:
             self.shutdown()
             self.shutdown()
             return False
             return False
@@ -2181,7 +2202,11 @@ class CookerParser(object):
         return True
         return True
 
 
     def reparse(self, filename):
     def reparse(self, filename):
-        infos = self.bb_cache.parse(filename, self.cooker.collection.get_file_appends(filename))
-        for vfn, info_array in infos:
-            (fn, cls, mc) = bb.cache.virtualfn2realfn(vfn)
-            self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array)
+        to_reparse = set()
+        for mc in self.cooker.multiconfigs:
+            to_reparse.add((mc, filename, self.cooker.collections[mc].get_file_appends(filename)))
+
+        for mc, filename, appends in to_reparse:
+            infos = self.bb_cache.parse(filename, appends)
+            for vfn, info_array in infos:
+                self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array)

+ 6 - 5
lib/bb/runqueue.py

@@ -1557,7 +1557,8 @@ class RunQueue:
 
 
     def rq_dump_sigfn(self, fn, options):
     def rq_dump_sigfn(self, fn, options):
         bb_cache = bb.cache.NoCache(self.cooker.databuilder)
         bb_cache = bb.cache.NoCache(self.cooker.databuilder)
-        the_data = bb_cache.loadDataFull(fn, self.cooker.collection.get_file_appends(fn))
+        mc = bb.runqueue.mc_from_tid(fn)
+        the_data = bb_cache.loadDataFull(fn, self.cooker.collections[mc].get_file_appends(fn))
         siggen = bb.parse.siggen
         siggen = bb.parse.siggen
         dataCaches = self.rqdata.dataCaches
         dataCaches = self.rqdata.dataCaches
         siggen.dump_sigfn(fn, dataCaches, options)
         siggen.dump_sigfn(fn, dataCaches, options)
@@ -2042,10 +2043,10 @@ class RunQueueExecute:
             if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run:
             if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run:
                 if not mc in self.rq.fakeworker:
                 if not mc in self.rq.fakeworker:
                     self.rq.start_fakeworker(self, mc)
                     self.rq.start_fakeworker(self, mc)
-                self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>")
+                self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>")
                 self.rq.fakeworker[mc].process.stdin.flush()
                 self.rq.fakeworker[mc].process.stdin.flush()
             else:
             else:
-                self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>")
+                self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>")
                 self.rq.worker[mc].process.stdin.flush()
                 self.rq.worker[mc].process.stdin.flush()
 
 
             self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
             self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
@@ -2129,10 +2130,10 @@ class RunQueueExecute:
                         self.rq.state = runQueueFailed
                         self.rq.state = runQueueFailed
                         self.stats.taskFailed()
                         self.stats.taskFailed()
                         return True
                         return True
-                self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>")
+                self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>")
                 self.rq.fakeworker[mc].process.stdin.flush()
                 self.rq.fakeworker[mc].process.stdin.flush()
             else:
             else:
-                self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>")
+                self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>")
                 self.rq.worker[mc].process.stdin.flush()
                 self.rq.worker[mc].process.stdin.flush()
 
 
             self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
             self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)

+ 13 - 11
lib/bb/tinfoil.py

@@ -117,15 +117,16 @@ class TinfoilCookerAdapter:
 
 
     class TinfoilCookerCollectionAdapter:
     class TinfoilCookerCollectionAdapter:
         """ cooker.collection adapter """
         """ cooker.collection adapter """
-        def __init__(self, tinfoil):
+        def __init__(self, tinfoil, mc=''):
             self.tinfoil = tinfoil
             self.tinfoil = tinfoil
+            self.mc = mc
         def get_file_appends(self, fn):
         def get_file_appends(self, fn):
-            return self.tinfoil.get_file_appends(fn)
+            return self.tinfoil.get_file_appends(fn, self.mc)
         def __getattr__(self, name):
         def __getattr__(self, name):
             if name == 'overlayed':
             if name == 'overlayed':
-                return self.tinfoil.get_overlayed_recipes()
+                return self.tinfoil.get_overlayed_recipes(self.mc)
             elif name == 'bbappends':
             elif name == 'bbappends':
-                return self.tinfoil.run_command('getAllAppends')
+                return self.tinfoil.run_command('getAllAppends', self.mc)
             else:
             else:
                 raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
                 raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
 
 
@@ -185,10 +186,11 @@ class TinfoilCookerAdapter:
 
 
     def __init__(self, tinfoil):
     def __init__(self, tinfoil):
         self.tinfoil = tinfoil
         self.tinfoil = tinfoil
-        self.collection = self.TinfoilCookerCollectionAdapter(tinfoil)
+        self.multiconfigs = [''] + (tinfoil.config_data.getVar('BBMULTICONFIG') or '').split()
+        self.collections = {}
         self.recipecaches = {}
         self.recipecaches = {}
-        self.recipecaches[''] = self.TinfoilRecipeCacheAdapter(tinfoil)
-        for mc in (tinfoil.config_data.getVar('BBMULTICONFIG') or '').split():
+        for mc in self.multiconfigs:
+            self.collections[mc] = self.TinfoilCookerCollectionAdapter(tinfoil, mc)
             self.recipecaches[mc] = self.TinfoilRecipeCacheAdapter(tinfoil, mc)
             self.recipecaches[mc] = self.TinfoilRecipeCacheAdapter(tinfoil, mc)
         self._cache = {}
         self._cache = {}
     def __getattr__(self, name):
     def __getattr__(self, name):
@@ -492,11 +494,11 @@ class Tinfoil:
             raise Exception('Not connected to server (did you call .prepare()?)')
             raise Exception('Not connected to server (did you call .prepare()?)')
         return self.server_connection.events.waitEvent(timeout)
         return self.server_connection.events.waitEvent(timeout)
 
 
-    def get_overlayed_recipes(self):
+    def get_overlayed_recipes(self, mc=''):
         """
         """
         Find recipes which are overlayed (i.e. where recipes exist in multiple layers)
         Find recipes which are overlayed (i.e. where recipes exist in multiple layers)
         """
         """
-        return defaultdict(list, self.run_command('getOverlayedRecipes'))
+        return defaultdict(list, self.run_command('getOverlayedRecipes', mc))
 
 
     def get_skipped_recipes(self):
     def get_skipped_recipes(self):
         """
         """
@@ -534,11 +536,11 @@ class Tinfoil:
                 raise bb.providers.NoProvider('Unable to find any recipe file matching "%s"' % pn)
                 raise bb.providers.NoProvider('Unable to find any recipe file matching "%s"' % pn)
         return best[3]
         return best[3]
 
 
-    def get_file_appends(self, fn):
+    def get_file_appends(self, fn, mc=''):
         """
         """
         Find the bbappends for a recipe file
         Find the bbappends for a recipe file
         """
         """
-        return self.run_command('getFileAppends', fn)
+        return self.run_command('getFileAppends', fn, mc)
 
 
     def all_recipes(self, mc='', sort=True):
     def all_recipes(self, mc='', sort=True):
         """
         """

+ 15 - 7
lib/bblayers/action.py

@@ -143,11 +143,12 @@ build results (as the layer priority order has effectively changed).
 
 
         applied_appends = []
         applied_appends = []
         for layer in layers:
         for layer in layers:
-            overlayed = []
-            for f in self.tinfoil.cooker.collection.overlayed.keys():
-                for of in self.tinfoil.cooker.collection.overlayed[f]:
-                    if of.startswith(layer):
-                        overlayed.append(of)
+            overlayed = set()
+            for mc in self.tinfoil.cooker.multiconfigs:
+                for f in self.tinfoil.cooker.collections[mc].overlayed.keys():
+                    for of in self.tinfoil.cooker.collections[mc].overlayed[f]:
+                        if of.startswith(layer):
+                            overlayed.add(of)
 
 
             logger.plain('Copying files from %s...' % layer )
             logger.plain('Copying files from %s...' % layer )
             for root, dirs, files in os.walk(layer):
             for root, dirs, files in os.walk(layer):
@@ -174,14 +175,21 @@ build results (as the layer priority order has effectively changed).
                                     logger.warning('Overwriting file %s', fdest)
                                     logger.warning('Overwriting file %s', fdest)
                             bb.utils.copyfile(f1full, fdest)
                             bb.utils.copyfile(f1full, fdest)
                             if ext == '.bb':
                             if ext == '.bb':
-                                for append in self.tinfoil.cooker.collection.get_file_appends(f1full):
+                                appends = set()
+                                for mc in self.tinfoil.cooker.multiconfigs:
+                                    appends |= set(self.tinfoil.cooker.collections[mc].get_file_appends(f1full))
+                                for append in appends:
                                     if layer_path_match(append):
                                     if layer_path_match(append):
                                         logger.plain('  Applying append %s to %s' % (append, fdest))
                                         logger.plain('  Applying append %s to %s' % (append, fdest))
                                         self.apply_append(append, fdest)
                                         self.apply_append(append, fdest)
                                         applied_appends.append(append)
                                         applied_appends.append(append)
 
 
         # Take care of when some layers are excluded and yet we have included bbappends for those recipes
         # Take care of when some layers are excluded and yet we have included bbappends for those recipes
-        for b in self.tinfoil.cooker.collection.bbappends:
+        bbappends = set()
+        for mc in self.tinfoil.cooker.multiconfigs:
+            bbappends |= set(self.tinfoil.cooker.collections[mc].bbappends)
+
+        for b in bbappends:
             (recipename, appendname) = b
             (recipename, appendname) = b
             if appendname not in applied_appends:
             if appendname not in applied_appends:
                 first_append = None
                 first_append = None

+ 2 - 2
lib/bblayers/query.py

@@ -320,12 +320,12 @@ Lists recipes with the bbappends that apply to them as subitems.
     def get_appends_for_files(self, filenames):
     def get_appends_for_files(self, filenames):
         appended, notappended = [], []
         appended, notappended = [], []
         for filename in filenames:
         for filename in filenames:
-            _, cls, _ = bb.cache.virtualfn2realfn(filename)
+            _, cls, mc = bb.cache.virtualfn2realfn(filename)
             if cls:
             if cls:
                 continue
                 continue
 
 
             basename = os.path.basename(filename)
             basename = os.path.basename(filename)
-            appends = self.tinfoil.cooker.collection.get_file_appends(basename)
+            appends = self.tinfoil.cooker.collections[mc].get_file_appends(basename)
             if appends:
             if appends:
                 appended.append((basename, list(appends)))
                 appended.append((basename, list(appends)))
             else:
             else: