Put extra requested fields into different cache files

index d083c51..f9d2e5f 100644
@@ -187,7 +187,10 @@ class Cache(object):
     BitBake Cache implementation
     """
 
-    def __init__(self, data):
+    def __init__(self, data, extracaches=None):       
+        # Pass extracaches information into Cache constructor
+        # It will be used in later for deciding whether
+        # we need separate cache file dump/load support
         self.cachedir = bb.data.getVar("CACHE", data, True)
         self.clean = set()
         self.checked = set()

@@ -196,6 +199,11 @@ class Cache(object):
         self.data_fn = None
         self.cacheclean = True
 
+        # Below three variables are used for loading extra cache
+        self.extra_depends_cache = {}        
+        self.extracaches = extracaches
+        self.extra_cachefile = None
+
         if self.cachedir in [None, '']:
             self.has_cache = False
             logger.info("Not using a cache. "

@@ -205,6 +213,11 @@ class Cache(object):
         self.has_cache = True
         self.cachefile = os.path.join(self.cachedir, "bb_cache.dat")
 
+        # if extracaches is not None, we need to load extra cache files
+        if self.extracaches:
+            self.extra_cachefile = os.path.join(self.cachedir,
+                        "bb_extracache_%s.dat" %self.extracaches)
+
         logger.debug(1, "Using cache in '%s'", self.cachedir)
         bb.utils.mkdirhier(self.cachedir)
 

@@ -302,7 +315,7 @@ class Cache(object):
         return bb_data[virtual]
 
     @classmethod
-    def parse(cls, filename, appends, configdata):
+    def parse(cls, filename, appends, configdata, extracaches=None):
         """Parse the specified filename, returning the recipe information"""
         infos = []
         datastores = cls.load_bbfile(filename, appends, configdata)

@@ -565,8 +578,9 @@ class CacheData(object):
     The data structures we compile from the cached data
     """
 
-    def __init__(self):
+    def __init__(self, extracaches):
         # Direct cache variables
+        self.extracaches = extracaches
         self.providers = defaultdict(list)
         self.rproviders = defaultdict(list)
         self.packages = defaultdict(list)

index a1cd4d7..eaf5923 100644
@@ -71,6 +71,10 @@ class BBCooker:
             self.server = server.BitBakeServer(self)
 
         self.configuration = configuration
+        # Cooker can get the bitbake mode information from configuration,
+        # we need to log down this information and pass it to Cache for
+        # extra cache file implementation
+        self.extracaches = configuration.ui
 
         self.configuration.data = bb.data.init()
 

@@ -713,9 +717,10 @@ class BBCooker:
 
         self.buildSetVars()
 
-        self.status = bb.cache.CacheData()
+        self.status = bb.cache.CacheData(self.extracaches)
         infos = bb.cache.Cache.parse(fn, self.get_file_appends(fn), \
-                                     self.configuration.data)
+                                     self.configuration.data, \
+                                     self.extracaches)
         infos = dict(infos)
 
         fn = bb.cache.Cache.realfn2virtual(buildfile, cls)

@@ -859,7 +864,7 @@ class BBCooker:
                 else:
                     collectlog.info("You have disabled Psyco. This decreases performance.")
 
-            self.status = bb.cache.CacheData()
+            self.status = bb.cache.CacheData(self.extracaches)
 
             ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
             self.status.ignored_dependencies = set(ignore.split())

index f9d2e5f..82712ec 100644
@@ -83,7 +83,7 @@ recipe_fields = (
 )
 
 
-class RecipeInfo(namedtuple('RecipeInfo', recipe_fields)):
+class RecipeRetrieve():
     __slots__ = ()
 
     @classmethod

@@ -117,6 +117,7 @@ class RecipeInfo(namedtuple('RecipeInfo', recipe_fields)):
     def getvar(cls, var, metadata):
         return metadata.getVar(var, True) or ''
 
+class RecipeInfo(namedtuple('RecipeInfo', recipe_fields), RecipeRetrieve):
     @classmethod
     def make_optional(cls, default=None, **kwargs):
         """Construct the namedtuple from the specified keyword arguments,

index 82712ec..7dc518f 100644
@@ -118,6 +118,8 @@ class RecipeRetrieve():
         return metadata.getVar(var, True) or ''
 
 class RecipeInfo(namedtuple('RecipeInfo', recipe_fields), RecipeRetrieve):
+    # Please note: fields are the static class member
+    # in namedtuple class RecipeInfo
     @classmethod
     def make_optional(cls, default=None, **kwargs):
         """Construct the namedtuple from the specified keyword arguments,

new file mode 100644
index 0000000..11bb493
@@ -0,0 +1,29 @@
+from collections import namedtuple
+from bb.cache import RecipeRetrieve
+
+# This three fields are only requested by Image Creator
+# hob_recipe_fields, HobRecipeInfo Definition are both
+# introduced by Image Creator. If a user want to introduce
+# more fields, we need to to the same thing as below:
+# 1. introduce extra recipe fields
+# 2. Define your extra named tuple class
+# 3. Add extra data loading methods in the else part of
+#    the Factory Class.
+hob_extra_recipe_fields = (
+    'summary',
+    'license',
+    'section'
+)
+
+HobExtraRecipeInfo = namedtuple('HobExtraRecipeInfo', hob_extra_recipe_fields)
+    
+class ExtraRecipeInfoFactory(RecipeRetrieve):
+    @classmethod
+    def from_metadata(cls, extracaches, metadata):
+        if extracaches == "hob":
+            return HobExtraRecipeInfo(
+                summary          = cls.getvar('SUMMARY', metadata),
+                license          = cls.getvar('LICENSE', metadata),
+                section          = cls.getvar('SECTION', metadata),
+            )
+

index 7dc518f..f25bafb 100644
@@ -75,9 +75,6 @@ recipe_fields = (
     'basetaskhashes',
     'hashfilename',
     'inherits',
-    'summary',
-    'license',
-    'section',
     'fakerootenv',
     'fakerootdirs'
 )

@@ -117,6 +114,7 @@ class RecipeRetrieve():
     def getvar(cls, var, metadata):
         return metadata.getVar(var, True) or ''
 
+from bb.extra_cache import ExtraRecipeInfoFactory
 class RecipeInfo(namedtuple('RecipeInfo', recipe_fields), RecipeRetrieve):
     # Please note: fields are the static class member
     # in namedtuple class RecipeInfo

@@ -177,9 +175,6 @@ class RecipeInfo(namedtuple('RecipeInfo', recipe_fields), RecipeRetrieve):
             rdepends_pkg     = cls.pkgvar('RDEPENDS', packages, metadata),
             rrecommends_pkg  = cls.pkgvar('RRECOMMENDS', packages, metadata),
             inherits         = cls.getvar('__inherit_cache', metadata),
-            summary          = cls.getvar('SUMMARY', metadata),
-            license          = cls.getvar('LICENSE', metadata),
-            section          = cls.getvar('SECTION', metadata),
             fakerootenv      = cls.getvar('FAKEROOTENV', metadata),
             fakerootdirs     = cls.getvar('FAKEROOTDIRS', metadata),
         )

@@ -232,11 +227,21 @@ class Cache(object):
         old_mtimes = [old_mtime for _, old_mtime in deps]
         old_mtimes.append(newest_mtime)
         newest_mtime = max(old_mtimes)
+        
+        # We need load extra cache
+        if extracaches:
+            if bb.parse.cached_mtime_noerror(self.cachefile) >= newest_mtime \
+                    and bb.parse.cached_mtime_noerror(self.extra_cachefile) \
+                            >= newest_mtime:
+                self.load_cachefile()
+            elif os.path.isfile(self.cachefile):
+                logger.info("Out of date cache found, rebuilding...")
+        else:
+            if bb.parse.cached_mtime_noerror(self.cachefile) >= newest_mtime:
+                self.load_cachefile()
+            elif os.path.isfile(self.cachefile):
+                logger.info("Out of date cache found, rebuilding...") 
 
-        if bb.parse.cached_mtime_noerror(self.cachefile) >= newest_mtime:
-            self.load_cachefile()
-        elif os.path.isfile(self.cachefile):
-            logger.info("Out of date cache found, rebuilding...")
 
     def load_cachefile(self):
         with open(self.cachefile, "rb") as cachefile:

@@ -256,9 +261,36 @@ class Cache(object):
                 return
 
             cachesize = os.fstat(cachefile.fileno()).st_size
-            bb.event.fire(bb.event.CacheLoadStarted(cachesize), self.data)
 
             previous_percent = 0
+            current_progress = 0
+            extra_current_progress = 0
+
+            if self.extracaches:
+                with open(self.extra_cachefile, "rb") as extra_cachefile:
+                    cachesize += os.fstat(extra_cachefile.fileno()).st_size
+                    pickled_extra = pickle.Unpickler(extra_cachefile)
+                    bb.event.fire(
+                            bb.event.CacheLoadStarted(cachesize), self.data)
+                    # Refresh Progress Bar                    
+                    while extra_cachefile:
+                        try:
+                            key = pickled_extra.load()
+                            value = pickled_extra.load()
+                        except Exception:
+                            break
+                        self.extra_depends_cache[key] = value
+                        current_progress = extra_cachefile.tell()
+                        current_percent = 100 * current_progress / cachesize
+                        if current_percent > previous_percent:
+                            previous_percent = current_percent
+                            bb.event.fire(
+                                bb.event.CacheLoadProgress(current_progress),
+                                          self.data)
+                extra_current_progress = current_progress
+            else:
+                bb.event.fire(bb.event.CacheLoadStarted(cachesize), self.data)
+
             while cachefile:
                 try:
                     key = pickled.load()

@@ -269,13 +301,15 @@ class Cache(object):
                 self.depends_cache[key] = value
 
                 # only fire events on even percentage boundaries
-                current_progress = cachefile.tell()
+                current_progress = cachefile.tell() + extra_current_progress
                 current_percent = 100 * current_progress / cachesize
                 if current_percent > previous_percent:
                     previous_percent = current_percent
                     bb.event.fire(bb.event.CacheLoadProgress(current_progress),
                                   self.data)
 
+            # printed cache loaded entry size need no increase since extra fields
+            # belong to the same one entry
             bb.event.fire(bb.event.CacheLoadCompleted(cachesize,
                                                       len(self.depends_cache)),
                           self.data)

@@ -321,6 +355,7 @@ class Cache(object):
     def parse(cls, filename, appends, configdata, extracaches=None):
         """Parse the specified filename, returning the recipe information"""
         infos = []
+        extra_infos = []
         datastores = cls.load_bbfile(filename, appends, configdata)
         depends = set()
         for variant, data in sorted(datastores.iteritems(),

@@ -332,7 +367,11 @@ class Cache(object):
                 data.setVar("__depends", depends)
             info = RecipeInfo.from_metadata(filename, data)
             infos.append((virtualfn, info))
-        return infos
+            if extracaches:
+                extra_info = \
+                    ExtraRecipeInfoFactory.from_metadata(extracaches, data)
+                extra_infos.append((virtualfn, extra_info))
+        return infos, extra_infos
 
     def load(self, filename, appends, configdata):
         """Obtain the recipe information for the specified filename,

@@ -345,15 +384,20 @@ class Cache(object):
         cached = self.cacheValid(filename)
         if cached:
             infos = []
+            extra_infos= []
             info = self.depends_cache[filename]
-            for variant in info.variants:
+            for i in range(0, len(info.variants)):
+                variant = info.variants[i]
                 virtualfn = self.realfn2virtual(filename, variant)
                 infos.append((virtualfn, self.depends_cache[virtualfn]))
+                if self.extracaches:
+                    extra_infos.append((virtualfn, \
+                            self.extra_depends_cache[virtualfn]))
         else:
             logger.debug(1, "Parsing %s", filename)
-            return self.parse(filename, appends, configdata)
+            return self.parse(filename, appends, configdata, self.extracaches)
 
-        return cached, infos
+        return cached, infos, extra_infos
 
     def loadData(self, fn, appends, cfgData, cacheData):
         """Load the recipe info for the specified filename,

@@ -361,13 +405,17 @@ class Cache(object):
         the recipe information to the supplied CacheData instance."""
         skipped, virtuals = 0, 0
 
-        cached, infos = self.load(fn, appends, cfgData)
-        for virtualfn, info in infos:
+        cached, infos, extra_infos = self.load(fn, appends, cfgData)
+        for i in range (0, len(infos)):
+            (virtualfn, info) = infos[i]
+            if self.extracaches:
+                extra_info = extra_infos[i]
             if info.skipped:
                 logger.debug(1, "Skipping %s", virtualfn)
                 skipped += 1
             else:
-                self.add_info(virtualfn, info, cacheData, not cached)
+                self.add_info(virtualfn, info, extra_info, \
+                            cacheData, not cached)
                 virtuals += 1
 
         return cached, skipped, virtuals

@@ -399,7 +447,8 @@ class Cache(object):
         self.checked.add(fn)
 
         # File isn't in depends_cache
-        if not fn in self.depends_cache:
+        if (not fn in self.depends_cache) or (self.extracaches \
+                       and (not fn in self.extra_depends_cache)):
             logger.debug(2, "Cache: %s is not cached", fn)
             return False
 

@@ -440,7 +489,9 @@ class Cache(object):
         for cls in info.variants:
             virtualfn = self.realfn2virtual(fn, cls)
             self.clean.add(virtualfn)
-            if virtualfn not in self.depends_cache:
+            if (virtualfn not in self.depends_cache) or \
+                         (self.extracaches \
+                         and virtualfn not in self.extra_depends_cache):
                 logger.debug(2, "Cache: %s is not cached", virtualfn)
                 invalid = True
 

@@ -467,6 +518,12 @@ class Cache(object):
         if fn in self.depends_cache:
             logger.debug(1, "Removing %s from cache", fn)
             del self.depends_cache[fn]
+        # when deleting, we need to maintain the consistency
+        # of the two independent cache
+        if self.extracaches and fn in self.extra_depends_cache:
+            logger.debug(1, "Removing %s from ui_extra_cache", fn)
+            del self.extra_depends_cache[fn]
+
         if fn in self.clean:
             logger.debug(1, "Marking %s as unclean", fn)
             self.clean.remove(fn)

@@ -492,15 +549,26 @@ class Cache(object):
                 pickler.dump(key)
                 pickler.dump(value)
 
+        # Sync back the extra cache fields into the separate cache file
+        if self.extracaches:
+            with open(self.extra_cachefile, "wb") as extra_cachefile:
+                extra_pickler = pickle.Pickler(extra_cachefile, \
+                                               pickle.HIGHEST_PROTOCOL)
+                for key, value in self.extra_depends_cache.iteritems():
+                    extra_pickler.dump(key)
+                    extra_pickler.dump(value)
+
         del self.depends_cache
+        del self.extra_depends_cache
 
     @staticmethod
     def mtime(cachefile):
         return bb.parse.cached_mtime_noerror(cachefile)
 
-    def add_info(self, filename, info, cacheData, parsed=None):
+    def add_info(self, filename, info, extra_info, cacheData, parsed=None):
         if not info.skipped:
             cacheData.add_from_recipeinfo(filename, info)
+            cacheData.add_from_extra_recipeinfo(filename, extra_info)
 
         if not self.has_cache:
             return

@@ -509,15 +577,20 @@ class Cache(object):
             if parsed:
                 self.cacheclean = False
             self.depends_cache[filename] = info
+            if self.extracaches:
+                self.extra_depends_cache[filename] = extra_info
 
     def add(self, file_name, data, cacheData, parsed=None):
         """
         Save data we need into the cache
         """
-
         realfn = self.virtualfn2realfn(file_name)[0]
+        extra_info = None
         info = RecipeInfo.from_metadata(realfn, data)
-        self.add_info(file_name, info, cacheData, parsed)
+        if self.extracaches:
+            extra_info = \
+                ExtraRecipeInfoFactory.from_metadata(self.extracaches, data)
+        self.add_info(file_name, info, extra_info, cacheData, parsed)
 
     @staticmethod
     def load_bbfile(bbfile, appends, config):

@@ -609,11 +682,13 @@ class CacheData(object):
         self.basetaskhash = {}
         self.hashfn = {}
         self.inherits = {}
+        self.fakerootenv = {}
+        self.fakerootdirs = {}
+
+# Extra cache fields
         self.summary = {}
         self.license = {}
         self.section = {}
-        self.fakerootenv = {}
-        self.fakerootdirs = {}
 
         # Indirect Cache variables (set elsewhere)
         self.ignored_dependencies = []

@@ -621,6 +696,12 @@ class CacheData(object):
         self.bbfile_priority = {}
         self.bbfile_config_priorities = []
 
+    def add_from_extra_recipeinfo(self, fn, extra_info):
+        if self.extracaches:
+            self.summary[fn] = extra_info.summary
+            self.license[fn] = extra_info.license
+            self.section[fn] = extra_info.section
+
     def add_from_recipeinfo(self, fn, info):
         self.task_deps[fn] = info.task_deps
         self.pkg_fn[fn] = info.pn

@@ -679,8 +760,5 @@ class CacheData(object):
             self.basetaskhash[identifier] = taskhash
 
         self.inherits[fn] = info.inherits
-        self.summary[fn] = info.summary
-        self.license[fn] = info.license
-        self.section[fn] = info.section
         self.fakerootenv[fn] = info.fakerootenv
         self.fakerootdirs[fn] = info.fakerootdirs

100 lines not on this page · raw message