update_data() removal patchset
cover update_data() removal patchset
Removing the need for finalization or update_data is something we've idly thought about for a long time. Basically, the whole idea of a specific finalization phase to the data store causes a lof of issues and horrible corner cases. For example, should bitbake call update_data, then expandKeys, or the other way around? Or should we keep calling update_data until nothing changes?
Through careful thought of some operations and working around the corner case problems, we've avoided most of the issues however there are some things we keep observing:
* We keep needing "expanded" data stores in different parts of the system alongside their unexpanded versions
* Few people understand why/when exactly they should call update_data()
* Its near impossible to set PREFERRED_PROVIDERS of recipes which are dynamically named (gcc-cross-${TARGET_ARCH}) along with overrides. This is something meta-darwin/mingw/baremetal could really use.
The alternative is a datastore which dynamically adjusts when anything changes so that the work is done at getVar time rather than precaching things at update_data time.
What is missing from the branch as yet is removal of the "expanded" data stores from use within bitbake and starting to remove the update_data calls (which are now just empty calls). This can happen once the code is merged/stablising.
The following changes since commit f883cf240266ee7be2cbd8971a8164cf4df9e372:
cooker: Improve DATE/TIME handling (2015-07-08 00:06:01 +0100)
are available in the git repository at:
git://git.openembedded.org/bitbake noupdatedata
http://cgit.openembedded.org/cgit.cgi/bitbake/log/?h=noupdatedata
Richard Purdie (16):
data_smart: Defer append/prepend handling
data_smart: Remove need for update_data calls
data_smart: Improve override handling
data_smart: Fix cache clearance problems
data_smart: Cache overrides and fix data store
data_smart: Move override handling to getVar parse/ast/data_smart: Add parsing flag to getVar/setVar
data_smart: VariableHistory: Ignore override duplicates and overlap
with CoW functions
data_smart: Seperate out the override cache
data_smart: Tweak OVERRIDES value cache for performance
data_smart: Improve override recursion handling
data_smart: Cache set(self.overrides)
data_smart: Fix keys() missing overrides
data_smart: Fix data expansion cache issues
data_smart: Fix appendVar/prependVar
tests/data: Add new data tests
lib/bb/data_smart.py | 337 ++++++++++++++++++++++++++++++-------------------- lib/bb/parse/ast.py | 14 +-- lib/bb/tests/data.py | 47 +++++++ lib/bb/tests/parse.py | 46 +++++++ 4 files changed, 306 insertions(+), 138 deletions(-)
Signature · 1 line
2.1.0
PATCH 1/16 data_smart: Defer append/prepend handling
Signed-off-by Richard Purdie <[email protected]>
--- lib/bb/data_smart.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-)
lib/bb/data_smart.py +22 -5
@@ -429,12 +429,13 @@ class DataSmart(MutableMapping): continue if op == "_append": - sval = self.getVar(append, False) or "" - sval += a - self.setVar(append, sval) + apps = self.getVarFlag(append, "_appendactive", False) or [] + apps.extend([a]) + self.setVarFlag(append, "_appendactive", apps, ignore=True) elif op == "_prepend": - sval = a + (self.getVar(append, False) or "") - self.setVar(append, sval) + prepends = self.getVarFlag(append, "_prependactive", False) or [] + prepends.extend([a]) + self.setVarFlag(append, "_prependactive", prepends, ignore=True) elif op == "_remove": removes = self.getVarFlag(append, "_removeactive", False) or [] removes.extend(a.split())
@@ -507,6 +508,11 @@ class DataSmart(MutableMapping): if not var in self.dict: self._makeShadowCopy(var) + if "_appendactive" in self.dict[var]: + del self.dict[var]["_appendactive"] + if "_prependactive" in self.dict[var]: + del self.dict[var]["_prependactive"] + # more cookies for the cookie monster if '_' in var: self._setvar_update_overrides(var)
@@ -612,6 +618,17 @@ class DataSmart(MutableMapping): value = copy.copy(local_var[flag]) elif flag == "_content" and "_defaultval" in local_var and not noweakdefault: value = copy.copy(local_var["_defaultval"]) + + if flag == "_content" and local_var is not None and "_appendactive" in local_var: + if not value: + value = "" + for r in local_var["_appendactive"]: + value = value + r + if flag == "_content" and local_var is not None and "_prependactive" in local_var: + if not value: + value = "" + for r in local_var["_prependactive"]: + value = r + value if expand and value: # Only getvar (flag == _content) hits the expand cache cachename = None
Signature · 1 line
2.1.0
PATCH 2/16 data_smart: Remove need for update_data calls
Move the update_data functionality into internal data store operations so the main finalize (update_data) call is a nop.
To make this work we need to call the internal finalization function whenever OVERRIDES is changed to ensure values get updated correctly.
This has performance issues but the subsequant patches look into this.
Signed-off-by Richard Purdie <[email protected]>
--- lib/bb/data_smart.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-)
lib/bb/data_smart.py +15 -4
@@ -298,6 +298,7 @@ class VariableHistory(object): class DataSmart(MutableMapping): def __init__(self, special = None, seen = None ): self.dict = {} + self.overrides = [] if special is None: special = COWDictBase.copy()
@@ -354,11 +355,13 @@ class DataSmart(MutableMapping): def expand(self, s, varname = None): return self.expandWithRefs(s, varname).value - def finalize(self, parent = False): + return + + def internal_finalize(self, parent = False): """Performs final steps upon the datastore, including application of overrides""" - overrides = (self.getVar("OVERRIDES", True) or "").split(":") or [] + self.overrides = (self.getVar("OVERRIDES", True) or "").split(":") or [] finalize_caller = { 'op': 'finalize', }
@@ -383,7 +386,7 @@ class DataSmart(MutableMapping): # We only want to report finalization once per variable overridden. finalizes_reported = {} - for o in overrides: + for o in self.overrides: # calculate '_'+override l = len(o) + 1
@@ -417,12 +420,15 @@ class DataSmart(MutableMapping): if op in self._special_values: appends = self._special_values[op] or [] for append in appends: + self.handle_special_values(append, op) + + def handle_special_values(self, append, op): keep = [] for (a, o) in self.getVarFlag(append, op) or []: match = True if o: for o2 in o.split("_"): - if not o2 in overrides: + if not o2 in self.overrides: match = False if not match: keep.append((a ,o))
@@ -502,6 +508,7 @@ class DataSmart(MutableMapping): except KeyError: self._special_values[keyword] = set() self._special_values[keyword].add(base) + self.handle_special_values(base, keyword) return
@@ -521,6 +528,9 @@ class DataSmart(MutableMapping): self.dict[var]["_content"] = value self.varhistory.record(**loginfo) + if var == "OVERRIDES": + self.internal_finalize(True) + def _setvar_update_overrides(self, var): # aka pay the cookie monster override = var[var.rfind('_')+1:]
@@ -733,6 +743,7 @@ class DataSmart(MutableMapping): # we really want this to be a DataSmart... data = DataSmart(seen=self._seen_overrides.copy(), special=self._special_values.copy()) data.dict["_data"] = self.dict + data.overrides = copy.copy(self.overrides) data.varhistory = self.varhistory.copy() data.varhistory.datasmart = data data.inchistory = self.inchistory.copy()
Signature · 1 line
2.1.0
PATCH 3/16 data_smart: Improve override handling
In reply to Richard PurdieRather than the heavy lifting in internal_finalize, move the bulk of the functionality to getVar and rely on a new internal replaces variable to keep track of any mappings that are needed. This removes the need for the COW _special_values cache.
This change in functionality also implies we need to track any changes not only to OVERRIDES but also any variable which OVERIDES depends upon. Add code to handle this.
Explicitly list FILE as a value OVERRIDES depends upon since it doesn't automatically get detected and is of key importance to OVERRIDES, otherwise PN doesn't update when FILE changes.
Signed-off-by Richard Purdie <[email protected]>
--- lib/bb/data_smart.py | 156 ++++++++++++++++++++------------------------------- 1 file changed, 60 insertions(+), 96 deletions(-)
lib/bb/data_smart.py +60 -96
@@ -296,12 +296,9 @@ class VariableHistory(object): self.variables[var] = [] class DataSmart(MutableMapping): - def __init__(self, special = None, seen = None ): + def __init__(self, seen = None): self.dict = {} - self.overrides = [] - if special is None: - special = COWDictBase.copy() if seen is None: seen = COWDictBase.copy()
@@ -310,11 +307,13 @@ class DataSmart(MutableMapping): self._tracking = False # cookie monster tribute - self._special_values = special self._seen_overrides = seen self.expand_cache = {} + self.overridevars = set(["OVERRIDES", "FILE"]) + self.replaces = {} + def enableTracking(self): self._tracking = True
@@ -361,11 +360,7 @@ class DataSmart(MutableMapping): def internal_finalize(self, parent = False): """Performs final steps upon the datastore, including application of overrides""" - self.overrides = (self.getVar("OVERRIDES", True) or "").split(":") or [] - finalize_caller = { - 'op': 'finalize', - } - infer_caller_details(finalize_caller, parent = parent, varval = False) + overrides = (self.getVar("OVERRIDES", True) or "").split(":") or [] # # Well let us see what breaks here. We used to iterate
@@ -383,10 +378,9 @@ class DataSmart(MutableMapping): # information for later. # - # We only want to report finalization once per variable overridden. - finalizes_reported = {} + self.replaces = {} - for o in self.overrides: + for o in overrides: # calculate '_'+override l = len(o) + 1
@@ -397,61 +391,7 @@ class DataSmart(MutableMapping): vars = self._seen_overrides[o].copy() for var in vars: name = var[:-l] - try: - # Report only once, even if multiple changes. - if name not in finalizes_reported: - finalizes_reported[name] = True - finalize_caller['variable'] = name - finalize_caller['detail'] = 'was: ' + str(self.getVar(name, False)) - self.varhistory.record(**finalize_caller) - # Copy history of the override over. - for event in self.varhistory.variable(var): - loginfo = event.copy() - loginfo['variable'] = name - loginfo['op'] = 'override[%s]:%s' % (o, loginfo['op']) - self.varhistory.record(**loginfo) - self.setVar(name, self.getVar(var, False), op = 'finalize', file = 'override[%s]' % o, line = '') - self.delVar(var) - except Exception: - logger.info("Untracked delVar") - - # now on to the appends and prepends, and stashing the removes - for op in __setvar_keyword__: - if op in self._special_values: - appends = self._special_values[op] or [] - for append in appends: - self.handle_special_values(append, op) - - def handle_special_values(self, append, op): - keep = [] - for (a, o) in self.getVarFlag(append, op) or []: - match = True - if o: - for o2 in o.split("_"): - if not o2 in self.overrides: - match = False - if not match: - keep.append((a ,o)) - continue - - if op == "_append": - apps = self.getVarFlag(append, "_appendactive", False) or [] - apps.extend([a]) - self.setVarFlag(append, "_appendactive", apps, ignore=True) - elif op == "_prepend": - prepends = self.getVarFlag(append, "_prependactive", False) or [] - prepends.extend([a]) - self.setVarFlag(append, "_prependactive", prepends, ignore=True) - elif op == "_remove": - removes = self.getVarFlag(append, "_removeactive", False) or [] - removes.extend(a.split()) - self.setVarFlag(append, "_removeactive", removes, ignore=True) - - # We save overrides that may be applied at some later stage - if keep: - self.setVarFlag(append, op, keep, ignore=True) - else: - self.delVarFlag(append, op, ignore=True) + self.replaces[name] = var def initVar(self, var): self.expand_cache = {}
@@ -503,22 +443,19 @@ class DataSmart(MutableMapping): self.varhistory.record(**loginfo) # todo make sure keyword is not __doc__ or __module__ # pay the cookie monster - try: - self._special_values[keyword].add(base) - except KeyError: - self._special_values[keyword] = set() - self._special_values[keyword].add(base) - self.handle_special_values(base, keyword) + if base in self.overridevars: + self.overridevars.update(self.expandWithRefs(value, var).references) + self.internal_finalize(True) return if not var in self.dict: self._makeShadowCopy(var) - if "_appendactive" in self.dict[var]: - del self.dict[var]["_appendactive"] - if "_prependactive" in self.dict[var]: - del self.dict[var]["_prependactive"] + if "_append" in self.dict[var]: + del self.dict[var]["_append"] + if "_prepend" in self.dict[var]: + del self.dict[var]["_prepend"] # more cookies for the cookie monster if '_' in var:
@@ -528,7 +465,8 @@ class DataSmart(MutableMapping): self.dict[var]["_content"] = value self.varhistory.record(**loginfo) - if var == "OVERRIDES": + if var in self.overridevars: + self.overridevars.update(self.expandWithRefs(value, var).references) self.internal_finalize(True) def _setvar_update_overrides(self, var):
@@ -568,10 +506,6 @@ class DataSmart(MutableMapping): dest.extend(src) self.setVarFlag(newkey, i, dest, ignore=True) - if i in self._special_values and key in self._special_values[i]: - self._special_values[i].remove(key) - self._special_values[i].add(newkey) - loginfo['variable'] = key loginfo['op'] = 'rename (to)' loginfo['detail'] = newkey
@@ -623,22 +557,42 @@ class DataSmart(MutableMapping): def getVarFlag(self, var, flag, expand=False, noweakdefault=False): local_var = self._findVar(var) value = None - if local_var is not None: + + if flag == "_content" and var in self.replaces: + value = self.getVar(self.replaces[var]) + + if local_var is not None and value is None: if flag in local_var: value = copy.copy(local_var[flag]) elif flag == "_content" and "_defaultval" in local_var and not noweakdefault: value = copy.copy(local_var["_defaultval"]) - if flag == "_content" and local_var is not None and "_appendactive" in local_var: + if flag == "_content" and local_var is not None and "_append" in local_var: if not value: value = "" - for r in local_var["_appendactive"]: - value = value + r - if flag == "_content" and local_var is not None and "_prependactive" in local_var: + for (r, o) in local_var["_append"]: + match = True + if o: + overrides = (self.getVar("OVERRIDES", True) or "").split(":") or [] + for o2 in o.split("_"): + if not o2 in overrides: + match = False + if match: + value = value + r + + if flag == "_content" and local_var is not None and "_prepend" in local_var: if not value: value = "" - for r in local_var["_prependactive"]: - value = r + value + for (r, o) in local_var["_prepend"]: + match = True + if o: + overrides = (self.getVar("OVERRIDES", True) or "").split(":") or [] + for o2 in o.split("_"): + if not o2 in overrides: + match = False + if match: + value = r + value + if expand and value: # Only getvar (flag == _content) hits the expand cache cachename = None
@@ -647,9 +601,19 @@ class DataSmart(MutableMapping): else: cachename = var + "[" + flag + "]" value = self.expand(value, cachename) - if value and flag == "_content" and local_var is not None and "_removeactive" in local_var: - removes = [self.expand(r).split() for r in local_var["_removeactive"]] - removes = reduce(lambda a, b: a+b, removes, []) + + if value and flag == "_content" and local_var is not None and "_remove" in local_var: + removes = [] + for (r, o) in local_var["_remove"]: + match = True + if o: + overrides = (self.getVar("OVERRIDES", True) or "").split(":") or [] + for o2 in o.split("_"): + if not o2 in overrides: + match = False + if match: + removes.extend(self.expand(r).split()) + filtered = filter(lambda v: v not in removes, value.split()) value = " ".join(filtered)
@@ -735,21 +699,21 @@ class DataSmart(MutableMapping): else: del self.dict[var] - def createCopy(self): """ Create a copy of self by setting _data to self """ # we really want this to be a DataSmart... - data = DataSmart(seen=self._seen_overrides.copy(), special=self._special_values.copy()) + data = DataSmart(seen=self._seen_overrides.copy()) data.dict["_data"] = self.dict - data.overrides = copy.copy(self.overrides) data.varhistory = self.varhistory.copy() data.varhistory.datasmart = data data.inchistory = self.inchistory.copy() data._tracking = self._tracking + data.overridevars = copy.copy(self.overridevars) + return data def expandVarref(self, variable, parents=False):
Signature · 1 line
2.1.0
PATCH 4/16 data_smart: Fix cache clearance problems
In reply to Richard PurdieThese write operations should clear the expand cache since they can influence returned variable values but currently do not. Fix this.
Signed-off-by Richard Purdie <[email protected]>
--- lib/bb/data_smart.py | 4 ++++ 1 file changed, 4 insertions(+)
lib/bb/data_smart.py +4 -0
@@ -536,6 +536,7 @@ class DataSmart(MutableMapping): self._seen_overrides[override].remove(var) def setVarFlag(self, var, flag, value, **loginfo): + self.expand_cache = {} if 'op' not in loginfo: loginfo['op'] = "set" loginfo['flag'] = flag
@@ -624,6 +625,7 @@ class DataSmart(MutableMapping): return value def delVarFlag(self, var, flag, **loginfo): + self.expand_cache = {} local_var = self._findVar(var) if not local_var: return
@@ -653,6 +655,7 @@ class DataSmart(MutableMapping): self.setVarFlag(var, flag, newvalue, ignore=True) def setVarFlags(self, var, flags, **loginfo): + self.expand_cache = {} infer_caller_details(loginfo) if not var in self.dict: self._makeShadowCopy(var)
@@ -682,6 +685,7 @@ class DataSmart(MutableMapping): def delVarFlags(self, var, **loginfo): + self.expand_cache = {} if not var in self.dict: self._makeShadowCopy(var)
Signature · 1 line
2.1.0
PATCH 5/16 data_smart: Cache overrides and fix data store
In reply to Richard PurdieRather than repeatedly expanding the value of OVERRIDES, cache the value and update it when things change.
There were also some bugs introduced in the replaces functionality which this approach fixes by ensuring the replaces data is updated at the correct points.
Signed-off-by Richard Purdie <[email protected]>
--- lib/bb/data_smart.py | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-)
lib/bb/data_smart.py +27 -8
@@ -311,6 +311,7 @@ class DataSmart(MutableMapping): self.expand_cache = {} + self.overrides = [] self.overridevars = set(["OVERRIDES", "FILE"]) self.replaces = {}
@@ -360,7 +361,7 @@ class DataSmart(MutableMapping): def internal_finalize(self, parent = False): """Performs final steps upon the datastore, including application of overrides""" - overrides = (self.getVar("OVERRIDES", True) or "").split(":") or [] + self.overrides = (self.getVar("OVERRIDES", True) or "").split(":") or [] # # Well let us see what breaks here. We used to iterate
@@ -380,7 +381,7 @@ class DataSmart(MutableMapping): self.replaces = {} - for o in overrides: + for o in self.overrides: # calculate '_'+override l = len(o) + 1
@@ -393,6 +394,18 @@ class DataSmart(MutableMapping): name = var[:-l] self.replaces[name] = var + def internal_finalize2(self, var, o): + """Performs final steps upon the datastore, including application of overrides""" + + l = len(o) + 1 + name = var[:-l] + + if name in self.replaces: + del self.replaces[name] + for o in self.overrides: + if o in self._seen_overrides and name + "_" + o in self._seen_overrides[o]: + self.replaces[name] = name + "_" + o + def initVar(self, var): self.expand_cache = {} if not var in self.dict:
@@ -457,6 +470,12 @@ class DataSmart(MutableMapping): if "_prepend" in self.dict[var]: del self.dict[var]["_prepend"] + if var in self.replaces: + del self.replaces[var] + for o in self.overrides: + if o in self._seen_overrides and var + "_" + o in self._seen_overrides[o]: + self.delVar(var + "_" + o) + # more cookies for the cookie monster if '_' in var: self._setvar_update_overrides(var)
@@ -477,6 +496,7 @@ class DataSmart(MutableMapping): if override not in self._seen_overrides: self._seen_overrides[override] = set() self._seen_overrides[override].add( var ) + self.internal_finalize2(var, override) override = None if "_" in shortvar: override = var[shortvar.rfind('_')+1:]
@@ -534,6 +554,7 @@ class DataSmart(MutableMapping): override = var[var.rfind('_')+1:] if override and override in self._seen_overrides and var in self._seen_overrides[override]: self._seen_overrides[override].remove(var) + self.internal_finalize2(var, override) def setVarFlag(self, var, flag, value, **loginfo): self.expand_cache = {}
@@ -574,9 +595,8 @@ class DataSmart(MutableMapping): for (r, o) in local_var["_append"]: match = True if o: - overrides = (self.getVar("OVERRIDES", True) or "").split(":") or [] for o2 in o.split("_"): - if not o2 in overrides: + if not o2 in self.overrides: match = False if match: value = value + r
@@ -587,9 +607,8 @@ class DataSmart(MutableMapping): for (r, o) in local_var["_prepend"]: match = True if o: - overrides = (self.getVar("OVERRIDES", True) or "").split(":") or [] for o2 in o.split("_"): - if not o2 in overrides: + if not o2 in self.overrides: match = False if match: value = r + value
@@ -608,9 +627,8 @@ class DataSmart(MutableMapping): for (r, o) in local_var["_remove"]: match = True if o: - overrides = (self.getVar("OVERRIDES", True) or "").split(":") or [] for o2 in o.split("_"): - if not o2 in overrides: + if not o2 in self.overrides: match = False if match: removes.extend(self.expand(r).split())
@@ -716,6 +734,7 @@ class DataSmart(MutableMapping): data._tracking = self._tracking + data.overrides = copy.copy(self.overrides) data.overridevars = copy.copy(self.overridevars) return data
Signature · 1 line
2.1.0
PATCH 6/16 data_smart: Move override handling to getVar
In reply to Richard PurdieCompeltely remove the replaces functionality and move all overrides handling to getVar time. We can move the cookie cache into a hidden flag within the variables themselves.
This removes the need for any of the internal_finalize steps.
This obsolete the need for the _seen_overrides COW cache.
Signed-off-by Richard Purdie <[email protected]>
--- lib/bb/data_smart.py | 131 ++++++++++++++++++++++++--------------------------- 1 file changed, 61 insertions(+), 70 deletions(-)
lib/bb/data_smart.py +61 -70
@@ -296,24 +296,18 @@ class VariableHistory(object): self.variables[var] = [] class DataSmart(MutableMapping): - def __init__(self, seen = None): + def __init__(self): self.dict = {} - if seen is None: - seen = COWDictBase.copy() - self.inchistory = IncludeHistory() self.varhistory = VariableHistory(self) self._tracking = False - # cookie monster tribute - self._seen_overrides = seen - self.expand_cache = {} + # cookie monster tribute self.overrides = [] self.overridevars = set(["OVERRIDES", "FILE"]) - self.replaces = {} def enableTracking(self): self._tracking = True
@@ -363,49 +357,6 @@ class DataSmart(MutableMapping): self.overrides = (self.getVar("OVERRIDES", True) or "").split(":") or [] - # - # Well let us see what breaks here. We used to iterate - # over each variable and apply the override and then - # do the line expanding. - # If we have bad luck - which we will have - the keys - # where in some order that is so important for this - # method which we don't have anymore. - # Anyway we will fix that and write test cases this - # time. - - # - # First we apply all overrides - # Then we will handle _append and _prepend and store the _remove - # information for later. - # - - self.replaces = {} - - for o in self.overrides: - # calculate '_'+override - l = len(o) + 1 - - # see if one should even try - if o not in self._seen_overrides: - continue - - vars = self._seen_overrides[o].copy() - for var in vars: - name = var[:-l] - self.replaces[name] = var - - def internal_finalize2(self, var, o): - """Performs final steps upon the datastore, including application of overrides""" - - l = len(o) + 1 - name = var[:-l] - - if name in self.replaces: - del self.replaces[name] - for o in self.overrides: - if o in self._seen_overrides and name + "_" + o in self._seen_overrides[o]: - self.replaces[name] = name + "_" + o - def initVar(self, var): self.expand_cache = {} if not var in self.dict:
@@ -457,6 +408,11 @@ class DataSmart(MutableMapping): # todo make sure keyword is not __doc__ or __module__ # pay the cookie monster + # more cookies for the cookie monster + if '_' in var: + self._setvar_update_overrides(base, **loginfo) + + if base in self.overridevars: self.overridevars.update(self.expandWithRefs(value, var).references) self.internal_finalize(True)
@@ -469,16 +425,12 @@ class DataSmart(MutableMapping): del self.dict[var]["_append"] if "_prepend" in self.dict[var]: del self.dict[var]["_prepend"] - - if var in self.replaces: - del self.replaces[var] - for o in self.overrides: - if o in self._seen_overrides and var + "_" + o in self._seen_overrides[o]: - self.delVar(var + "_" + o) + if "_overrides" in self.dict[var]: + del self.dict[var]["_overrides"] # more cookies for the cookie monster if '_' in var: - self._setvar_update_overrides(var) + self._setvar_update_overrides(var, **loginfo) # setting var self.dict[var]["_content"] = value
@@ -488,19 +440,29 @@ class DataSmart(MutableMapping): self.overridevars.update(self.expandWithRefs(value, var).references) self.internal_finalize(True) - def _setvar_update_overrides(self, var): + def _setvar_update_overrides(self, var, **loginfo): # aka pay the cookie monster override = var[var.rfind('_')+1:] shortvar = var[:var.rfind('_')] while override: - if override not in self._seen_overrides: - self._seen_overrides[override] = set() - self._seen_overrides[override].add( var ) - self.internal_finalize2(var, override) + l = self.getVarFlag(shortvar, "_overrides") or [] + if [var, override] not in l: + l.append([var, override]) + self.setVarFlag(shortvar, "_overrides", l, ignore=True) + for event in self.varhistory.variable(var): + if 'flag' in loginfo and not loginfo['flag'].startswith("_"): + continue + loginfo = event.copy() + loginfo['variable'] = shortvar + loginfo['op'] = 'override[%s]:%s' % (override, loginfo['op']) + self.varhistory.record(**loginfo) + override = None if "_" in shortvar: override = var[shortvar.rfind('_')+1:] shortvar = var[:shortvar.rfind('_')] + if len(shortvar) == 0: + override = None def getVar(self, var, expand=False, noweakdefault=False): return self.getVarFlag(var, "_content", expand, noweakdefault)
@@ -526,6 +488,9 @@ class DataSmart(MutableMapping): dest.extend(src) self.setVarFlag(newkey, i, dest, ignore=True) + if '_' in newkey and val is None: + self._setvar_update_overrides(newkey, **loginfo) + loginfo['variable'] = key loginfo['op'] = 'rename (to)' loginfo['detail'] = newkey
@@ -552,9 +517,13 @@ class DataSmart(MutableMapping): self.dict[var] = {} if '_' in var: override = var[var.rfind('_')+1:] - if override and override in self._seen_overrides and var in self._seen_overrides[override]: - self._seen_overrides[override].remove(var) - self.internal_finalize2(var, override) + shortvar = var[:var.rfind('_')] + l = self.getVarFlag(shortvar, "_overrides") or [] + try: + l.remove([var, override]) + except ValueError as e: + pass + self.setVarFlag(shortvar, "_overrides", l, ignore=True) def setVarFlag(self, var, flag, value, **loginfo): self.expand_cache = {}
@@ -567,7 +536,7 @@ class DataSmart(MutableMapping): self.dict[var][flag] = value if flag == "_defaultval" and '_' in var: - self._setvar_update_overrides(var) + self._setvar_update_overrides(var, **loginfo) if flag == "unexport" or flag == "export": if not "__exportlist" in self.dict:
@@ -580,8 +549,30 @@ class DataSmart(MutableMapping): local_var = self._findVar(var) value = None - if flag == "_content" and var in self.replaces: - value = self.getVar(self.replaces[var]) + if flag == "_content" and local_var is not None and "_overrides" in local_var: + match = False + active = {} + for (r, o) in local_var["_overrides"]: + # What about double overrides both with "_" in the name? + if o in self.overrides: + active[o] = r + elif set(o.split("_")).issubset(set(self.overrides)): + active[o] = r + mod = True + while mod: + mod = False + for o in self.overrides: + for a in active.copy(): + if a.endswith("_" + o): + t = active[a] + del active[a] + active[a.replace("_" + o, "")] = t + mod = True + elif a == o: + match = active[a] + del active[a] + if match: + value = self.getVar(match) if local_var is not None and value is None: if flag in local_var:
@@ -726,7 +717,7 @@ class DataSmart(MutableMapping): Create a copy of self by setting _data to self """ # we really want this to be a DataSmart... - data = DataSmart(seen=self._seen_overrides.copy()) + data = DataSmart() data.dict["_data"] = self.dict data.varhistory = self.varhistory.copy() data.varhistory.datasmart = data
Signature · 1 line
2.1.0
PATCH 7/16 parse/ast/data_smart: Add parsing flag to getVar/setVar
In reply to Richard PurdieWhen parsing we find problems if we clear prepends/appends when setting variables during the initial parsing phases. Later, we actively want to do this (in what would be post finalisation previously).
To handle this, pass a parsing flag to the operations to control the correct behaviour for the context.
Signed-off-by Richard Purdie <[email protected]>
--- lib/bb/data_smart.py | 36 +++++++++++++++++++++--------------- lib/bb/parse/ast.py | 14 +++++++------- 2 files changed, 28 insertions(+), 22 deletions(-)
lib/bb/data_smart.py +21 -15
@@ -386,6 +386,10 @@ class DataSmart(MutableMapping): def setVar(self, var, value, **loginfo): #print("var=" + str(var) + " val=" + str(value)) + parsing=False + if 'parsing' in loginfo: + parsing=True + if 'op' not in loginfo: loginfo['op'] = "set" self.expand_cache = {}
@@ -421,12 +425,13 @@ class DataSmart(MutableMapping): if not var in self.dict: self._makeShadowCopy(var) - if "_append" in self.dict[var]: - del self.dict[var]["_append"] - if "_prepend" in self.dict[var]: - del self.dict[var]["_prepend"] - if "_overrides" in self.dict[var]: - del self.dict[var]["_overrides"] + if not parsing: + if "_append" in self.dict[var]: + del self.dict[var]["_append"] + if "_prepend" in self.dict[var]: + del self.dict[var]["_prepend"] + if "_overrides" in self.dict[var]: + del self.dict[var]["_overrides"] # more cookies for the cookie monster if '_' in var:
@@ -464,20 +469,20 @@ class DataSmart(MutableMapping): if len(shortvar) == 0: override = None - def getVar(self, var, expand=False, noweakdefault=False): - return self.getVarFlag(var, "_content", expand, noweakdefault) + def getVar(self, var, expand=False, noweakdefault=False, parsing=False): + return self.getVarFlag(var, "_content", expand, noweakdefault, parsing) def renameVar(self, key, newkey, **loginfo): """ Rename the variable key to newkey """ - val = self.getVar(key, 0) + val = self.getVar(key, 0, parsing=True) if val is not None: loginfo['variable'] = newkey loginfo['op'] = 'rename from %s' % key loginfo['detail'] = val self.varhistory.record(**loginfo) - self.setVar(newkey, val, ignore=True) + self.setVar(newkey, val, ignore=True, parsing=True) for i in (__setvar_keyword__): src = self.getVarFlag(key, i)
@@ -545,11 +550,10 @@ class DataSmart(MutableMapping): self.dict["__exportlist"]["_content"] = set() self.dict["__exportlist"]["_content"].add(var) - def getVarFlag(self, var, flag, expand=False, noweakdefault=False): + def getVarFlag(self, var, flag, expand=False, noweakdefault=False, parsing=False): local_var = self._findVar(var) value = None - - if flag == "_content" and local_var is not None and "_overrides" in local_var: + if flag == "_content" and local_var is not None and "_overrides" in local_var and not parsing: match = False active = {} for (r, o) in local_var["_overrides"]:
@@ -580,7 +584,8 @@ class DataSmart(MutableMapping): elif flag == "_content" and "_defaultval" in local_var and not noweakdefault: value = copy.copy(local_var["_defaultval"]) - if flag == "_content" and local_var is not None and "_append" in local_var: + + if flag == "_content" and local_var is not None and "_append" in local_var and not parsing: if not value: value = "" for (r, o) in local_var["_append"]:
@@ -592,10 +597,11 @@ class DataSmart(MutableMapping): if match: value = value + r - if flag == "_content" and local_var is not None and "_prepend" in local_var: + if flag == "_content" and local_var is not None and "_prepend" in local_var and not parsing: if not value: value = "" for (r, o) in local_var["_prepend"]: + match = True if o: for o2 in o.split("_"):
lib/bb/parse/ast.py +7 -7
@@ -85,7 +85,7 @@ class DataNode(AstNode): if 'flag' in self.groupd and self.groupd['flag'] != None: return data.getVarFlag(key, self.groupd['flag'], noweakdefault=True) else: - return data.getVar(key, False, noweakdefault=True) + return data.getVar(key, False, noweakdefault=True, parsing=True) def eval(self, data): groupd = self.groupd
@@ -136,7 +136,7 @@ class DataNode(AstNode): if flag: data.setVarFlag(key, flag, val, **loginfo) else: - data.setVar(key, val, **loginfo) + data.setVar(key, val, parsing=True, **loginfo) class MethodNode(AstNode): tr_tbl = string.maketrans('/.+-@%&', '_______')
@@ -155,10 +155,10 @@ class MethodNode(AstNode): anonfuncs = data.getVar('__BBANONFUNCS', False) or [] anonfuncs.append(funcname) data.setVar('__BBANONFUNCS', anonfuncs) - data.setVar(funcname, text) + data.setVar(funcname, text, parsing=True) else: data.setVarFlag(self.func_name, "func", 1) - data.setVar(self.func_name, text) + data.setVar(self.func_name, text, parsing=True) class PythonMethodNode(AstNode): def __init__(self, filename, lineno, function, modulename, body):
@@ -175,7 +175,7 @@ class PythonMethodNode(AstNode): bb.methodpool.insert_method(self.modulename, text, self.filename) data.setVarFlag(self.function, "func", 1) data.setVarFlag(self.function, "python", 1) - data.setVar(self.function, text) + data.setVar(self.function, text, parsing=True) class MethodFlagsNode(AstNode): def __init__(self, filename, lineno, key, m):
@@ -224,11 +224,11 @@ class ExportFuncsNode(AstNode): data.setVarFlag(calledfunc, flag, data.getVarFlag(func, flag)) if data.getVarFlag(calledfunc, "python"): - data.setVar(func, " bb.build.exec_func('" + calledfunc + "', d)\n") + data.setVar(func, " bb.build.exec_func('" + calledfunc + "', d)\n", parsing=True) else: if "-" in self.classname: bb.fatal("The classname %s contains a dash character and is calling an sh function %s using EXPORT_FUNCTIONS. Since a dash is illegal in sh function names, this cannot work, please rename the class or don't use EXPORT_FUNCTIONS." % (self.classname, calledfunc)) - data.setVar(func, " " + calledfunc + "\n") + data.setVar(func, " " + calledfunc + "\n", parsing=True) data.setVarFlag(func, 'export_func', '1') class AddTaskNode(AstNode):
Signature · 1 line
2.1.0
PATCH 8/16 data_smart: VariableHistory: Ignore override duplicates and overlap with CoW functions
In reply to Richard PurdieTrying to look up a variable called 'copy' in COW is problematic due to internal implmentation details, at least avoid tracebacks from this.
Also don't duplicate override history (which is an atrifact of changed override behviour) as otherwise the bitbake -e output is convoluted.
Signed-off-by Richard Purdie <[email protected]>
--- lib/bb/data_smart.py | 5 +++++ 1 file changed, 5 insertions(+)
lib/bb/data_smart.py +5 -0
@@ -231,6 +231,10 @@ class VariableHistory(object): if var not in self.variables: self.variables[var] = [] + if not isinstance(self.variables[var], list): + return + if 'nodups' in loginfo and loginfo in self.variables[var]: + return self.variables[var].append(loginfo.copy()) def variable(self, var):
@@ -460,6 +464,7 @@ class DataSmart(MutableMapping): loginfo = event.copy() loginfo['variable'] = shortvar loginfo['op'] = 'override[%s]:%s' % (override, loginfo['op']) + loginfo['nodups'] = True self.varhistory.record(**loginfo) override = None
Signature · 1 line
2.1.0
PATCH 9/16 data_smart: Seperate out the override cache
In reply to Richard PurdieUsing an internal flag for override mapping turns out to be slower than is optimal, not least as we don't want the keys list to list variables that have no value other than a potential override expansion.
Maintinaing a seperate cache is therefore more optimal from a speed perspective.
Signed-off-by Richard Purdie <[email protected]>
--- lib/bb/data_smart.py | 55 +++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 41 insertions(+), 14 deletions(-)
lib/bb/data_smart.py +41 -14
@@ -310,6 +310,8 @@ class DataSmart(MutableMapping): self.expand_cache = {} # cookie monster tribute + self.overridedata = {} + self.overrides = [] self.overridevars = set(["OVERRIDES", "FILE"])
@@ -434,8 +436,18 @@ class DataSmart(MutableMapping): del self.dict[var]["_append"] if "_prepend" in self.dict[var]: del self.dict[var]["_prepend"] - if "_overrides" in self.dict[var]: - del self.dict[var]["_overrides"] + if var in self.overridedata: + active = [] + self.need_overrides() + for (r, o) in self.overridedata[var]: + if o in self.overrides: + active.append(r) + elif "_" in o: + if set(o.split("_")).issubset(set(self.overrides)): + active.append(r) + for a in active: + self.delVar(a) + del self.overridedata[var] # more cookies for the cookie monster if '_' in var:
@@ -454,10 +466,10 @@ class DataSmart(MutableMapping): override = var[var.rfind('_')+1:] shortvar = var[:var.rfind('_')] while override: - l = self.getVarFlag(shortvar, "_overrides") or [] - if [var, override] not in l: - l.append([var, override]) - self.setVarFlag(shortvar, "_overrides", l, ignore=True) + if shortvar not in self.overridedata: + self.overridedata[shortvar] = [] + if [var, override] not in self.overridedata[shortvar]: + self.overridedata[shortvar].append([var, override]) for event in self.varhistory.variable(var): if 'flag' in loginfo and not loginfo['flag'].startswith("_"): continue
@@ -498,6 +510,12 @@ class DataSmart(MutableMapping): dest.extend(src) self.setVarFlag(newkey, i, dest, ignore=True) + if key in self.overridedata: + self.overridedata[newkey] = [] + for (v, o) in self.overridedata[key]: + self.overridedata[newkey].append([v.replace(key, newkey), o]) + self.renameVar(v, v.replace(key, newkey)) + if '_' in newkey and val is None: self._setvar_update_overrides(newkey, **loginfo)
@@ -525,15 +543,23 @@ class DataSmart(MutableMapping): self.varhistory.record(**loginfo) self.expand_cache = {} self.dict[var] = {} + if var in self.overridedata: + del self.overridedata[var] if '_' in var: override = var[var.rfind('_')+1:] shortvar = var[:var.rfind('_')] - l = self.getVarFlag(shortvar, "_overrides") or [] - try: - l.remove([var, override]) - except ValueError as e: - pass - self.setVarFlag(shortvar, "_overrides", l, ignore=True) + while override: + try: + if shortvar in self.overridedata: + self.overridedata[shortvar].remove([var, override]) + except ValueError as e: + pass + override = None + if "_" in shortvar: + override = var[shortvar.rfind('_')+1:] + shortvar = var[:shortvar.rfind('_')] + if len(shortvar) == 0: + override = None def setVarFlag(self, var, flag, value, **loginfo): self.expand_cache = {}
@@ -558,10 +584,10 @@ class DataSmart(MutableMapping): def getVarFlag(self, var, flag, expand=False, noweakdefault=False, parsing=False): local_var = self._findVar(var) value = None - if flag == "_content" and local_var is not None and "_overrides" in local_var and not parsing: + if flag == "_content" and var in self.overridedata and not parsing: match = False active = {} - for (r, o) in local_var["_overrides"]: + for (r, o) in self.overridedata[var]: # What about double overrides both with "_" in the name? if o in self.overrides: active[o] = r
@@ -738,6 +764,7 @@ class DataSmart(MutableMapping): data.overrides = copy.copy(self.overrides) data.overridevars = copy.copy(self.overridevars) + data.overridedata = copy.copy(self.overridedata) return data
Signature · 1 line
2.1.0
PATCH 10/16 data_smart: Tweak OVERRIDES value cache for performance
In reply to Richard PurdieUpdating the value of OVERRIDES whenever it changes turns out to be extremely expensve/pointless. Instead, clear its value and re-establish the value when we're going to use it.
This gives significant speed back.
Signed-off-by Richard Purdie <[email protected]>
--- lib/bb/data_smart.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-)
lib/bb/data_smart.py +12 -4
@@ -311,8 +311,7 @@ class DataSmart(MutableMapping): # cookie monster tribute self.overridedata = {} - - self.overrides = [] + self.overrides = None self.overridevars = set(["OVERRIDES", "FILE"]) def enableTracking(self):
@@ -360,8 +359,13 @@ class DataSmart(MutableMapping): def internal_finalize(self, parent = False): """Performs final steps upon the datastore, including application of overrides""" + self.overrides = None - self.overrides = (self.getVar("OVERRIDES", True) or "").split(":") or [] + def need_overrides(self): + if self.overrides is None: + self.overrides = [] + self.overrides = (self.getVar("OVERRIDES", True) or "").split(":") or [] + self.expand_cache = {} def initVar(self, var): self.expand_cache = {}
@@ -587,6 +591,7 @@ class DataSmart(MutableMapping): if flag == "_content" and var in self.overridedata and not parsing: match = False active = {} + self.need_overrides() for (r, o) in self.overridedata[var]: # What about double overrides both with "_" in the name? if o in self.overrides:
@@ -619,6 +624,7 @@ class DataSmart(MutableMapping): if flag == "_content" and local_var is not None and "_append" in local_var and not parsing: if not value: value = "" + self.need_overrides() for (r, o) in local_var["_append"]: match = True if o:
@@ -631,6 +637,7 @@ class DataSmart(MutableMapping): if flag == "_content" and local_var is not None and "_prepend" in local_var and not parsing: if not value: value = "" + self.need_overrides() for (r, o) in local_var["_prepend"]: match = True
@@ -652,6 +659,7 @@ class DataSmart(MutableMapping): if value and flag == "_content" and local_var is not None and "_remove" in local_var: removes = [] + self.need_overrides() for (r, o) in local_var["_remove"]: match = True if o:
@@ -762,7 +770,7 @@ class DataSmart(MutableMapping): data._tracking = self._tracking - data.overrides = copy.copy(self.overrides) + data.overrides = None data.overridevars = copy.copy(self.overridevars) data.overridedata = copy.copy(self.overridedata)
Signature · 1 line
2.1.0
PATCH 11/16 data_smart: Improve override recursion handling
In reply to Richard PurdieWhen expanding OVERRIDES, its possible someone might try and override a variable that is used in OVERRIDES. This could lead to infinite recursion. Add in guards against this.
Signed-off-by Richard Purdie <[email protected]>
--- lib/bb/data_smart.py | 6 ++++++ 1 file changed, 6 insertions(+)
lib/bb/data_smart.py +6 -0
@@ -313,6 +313,7 @@ class DataSmart(MutableMapping): self.overridedata = {} self.overrides = None self.overridevars = set(["OVERRIDES", "FILE"]) + self.inoverride = False def enableTracking(self): self._tracking = True
@@ -363,8 +364,13 @@ class DataSmart(MutableMapping): def need_overrides(self): if self.overrides is None: + if self.inoverride: + return + self.inoverride = True + # Can end up here recursively so setup dummy values self.overrides = [] self.overrides = (self.getVar("OVERRIDES", True) or "").split(":") or [] + self.inoverride = False self.expand_cache = {} def initVar(self, var):
Signature · 1 line
2.1.0
PATCH 12/16 data_smart: Cache set(self.overrides)
In reply to Richard PurdieThis performs better than continually regeneratiing the set().
Also only use set comparisions when its necessary to save some overhead as issubset() is slower in the single item case.
Signed-off-by Richard Purdie <[email protected]>
--- lib/bb/data_smart.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-)
lib/bb/data_smart.py +9 -5
@@ -369,7 +369,9 @@ class DataSmart(MutableMapping): self.inoverride = True # Can end up here recursively so setup dummy values self.overrides = [] + self.overridesset = set() self.overrides = (self.getVar("OVERRIDES", True) or "").split(":") or [] + self.overridesset = set(self.overrides) self.inoverride = False self.expand_cache = {}
@@ -450,10 +452,10 @@ class DataSmart(MutableMapping): active = [] self.need_overrides() for (r, o) in self.overridedata[var]: - if o in self.overrides: + if o in self.overridesset: active.append(r) elif "_" in o: - if set(o.split("_")).issubset(set(self.overrides)): + if set(o.split("_")).issubset(self.overridesset): active.append(r) for a in active: self.delVar(a)
@@ -600,10 +602,12 @@ class DataSmart(MutableMapping): self.need_overrides() for (r, o) in self.overridedata[var]: # What about double overrides both with "_" in the name? - if o in self.overrides: - active[o] = r - elif set(o.split("_")).issubset(set(self.overrides)): + if o in self.overridesset: active[o] = r + elif "_" in o: + if set(o.split("_")).issubset(self.overridesset): + active[o] = r + mod = True while mod: mod = False
Signature · 1 line
2.1.0
PATCH 13/16 data_smart: Fix keys() missing overrides
In reply to Richard Purdied.keys() was not returning items which had no base content but the variable came into existance through overrides. We have to process self.overridedata to find these other variables.
Signed-off-by Richard Purdie <[email protected]>
--- lib/bb/data_smart.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+)
lib/bb/data_smart.py +15 -0
@@ -809,6 +809,7 @@ class DataSmart(MutableMapping): def __iter__(self): deleted = set() + overrides = set() def keylist(d): klist = set() for key in d:
@@ -816,6 +817,8 @@ class DataSmart(MutableMapping): continue if key in deleted: continue + if key in overrides: + continue if not d[key]: deleted.add(key) continue
@@ -826,9 +829,21 @@ class DataSmart(MutableMapping): return klist + self.need_overrides() + for var in self.overridedata: + for (r, o) in self.overridedata[var]: + if o in self.overridesset: + overrides.add(var) + elif "_" in o: + if set(o.split("_")).issubset(self.overridesset): + overrides.add(var) + for k in keylist(self.dict): yield k + for k in overrides: + yield k + def __len__(self): return len(frozenset(self))
Signature · 1 line
2.1.0
PATCH 14/16 data_smart: Fix data expansion cache issues
In reply to Richard PurdieThe expand cache can be cleared in the middle of getVar now due to the use of operations like delVar. We therefore need to check if variables are in the cache before accessing in case it was cleared.
Signed-off-by Richard Purdie <[email protected]>
--- lib/bb/data_smart.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
lib/bb/data_smart.py +1 -1
@@ -682,7 +682,7 @@ class DataSmart(MutableMapping): filtered = filter(lambda v: v not in removes, value.split()) value = " ".join(filtered) - if expand: + if expand and var in self.expand_cache: # We need to ensure the expand cache has the correct value # flag == "_content" here self.expand_cache[var].value = value
Signature · 1 line
2.1.0
PATCH 15/16 data_smart: Fix appendVar/prependVar
In reply to Richard PurdieNow that overrides get expanded 'on the fly', change appendVar and prependVar to work using _append and _prepend, else we'd have to re-implement pieces of getVar and the timing of expansions becomes problematic.
Using _append/_prepend equivalence gives the behaviour users likley expect from these functions.
Signed-off-by Richard Purdie <[email protected]>
--- lib/bb/data_smart.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-)
lib/bb/data_smart.py +2 -4
@@ -540,14 +540,12 @@ class DataSmart(MutableMapping): def appendVar(self, var, value, **loginfo): loginfo['op'] = 'append' self.varhistory.record(**loginfo) - newvalue = (self.getVar(var, False) or "") + value - self.setVar(var, newvalue, ignore=True) + self.setVar(var + "_append", value, ignore=True, parsing=True) def prependVar(self, var, value, **loginfo): loginfo['op'] = 'prepend' self.varhistory.record(**loginfo) - newvalue = value + (self.getVar(var, False) or "") - self.setVar(var, newvalue, ignore=True) + self.setVar(var + "_prepend", value, ignore=True, parsing=True) def delVar(self, var, **loginfo): loginfo['detail'] = ""
Signature · 1 line
2.1.0
PATCH 16/16 tests/data: Add new data tests
In reply to Richard PurdieAdd a variety of tests which were found to be useful when working on the data store recently.
Signed-off-by Richard Purdie <[email protected]>
--- lib/bb/tests/data.py | 47 +++++++++++++++++++++++++++++++++++++++++++++++ lib/bb/tests/parse.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+)
lib/bb/tests/data.py +47 -0
@@ -270,6 +270,13 @@ class TestConcatOverride(unittest.TestCase): bb.data.update_data(self.d) self.assertEqual(self.d.getVar("TEST", True), "foo:val:val2:bar") + def test_append_unset(self): + self.d.setVar("TEST_prepend", "${FOO}:") + self.d.setVar("TEST_append", ":val2") + self.d.setVar("TEST_append", ":${BAR}") + bb.data.update_data(self.d) + self.assertEqual(self.d.getVar("TEST", True), "foo::val2:bar") + def test_remove(self): self.d.setVar("TEST", "${VAL} ${BAR}") self.d.setVar("TEST_remove", "val")
@@ -318,12 +325,52 @@ class TestOverrides(unittest.TestCase): bb.data.update_data(self.d) self.assertEqual(self.d.getVar("TEST", True), "testvalue2") + def test_one_override_unset(self): + self.d.setVar("TEST2_bar", "testvalue2") + bb.data.update_data(self.d) + self.assertEqual(self.d.getVar("TEST2", True), "testvalue2") + self.assertItemsEqual(self.d.keys(), ['TEST', 'TEST2', 'OVERRIDES', 'TEST2_bar']) + def test_multiple_override(self): self.d.setVar("TEST_bar", "testvalue2") self.d.setVar("TEST_local", "testvalue3") self.d.setVar("TEST_foo", "testvalue4") bb.data.update_data(self.d) self.assertEqual(self.d.getVar("TEST", True), "testvalue3") + self.assertItemsEqual(self.d.keys(), ['TEST', 'TEST_foo', 'OVERRIDES', 'TEST_bar', 'TEST_local']) + + def test_multiple_combined_overrides(self): + self.d.setVar("TEST_local_foo_bar", "testvalue3") + bb.data.update_data(self.d) + self.assertEqual(self.d.getVar("TEST", True), "testvalue3") + + def test_multiple_overrides_unset(self): + self.d.setVar("TEST2_local_foo_bar", "testvalue3") + bb.data.update_data(self.d) + self.assertEqual(self.d.getVar("TEST2", True), "testvalue3") + + def test_keyexpansion_override(self): + self.d.setVar("LOCAL", "local") + self.d.setVar("TEST_bar", "testvalue2") + self.d.setVar("TEST_${LOCAL}", "testvalue3") + self.d.setVar("TEST_foo", "testvalue4") + bb.data.update_data(self.d) + bb.data.expandKeys(self.d) + self.assertEqual(self.d.getVar("TEST", True), "testvalue3") + + def test_rename_override(self): + self.d.setVar("ALTERNATIVE_ncurses-tools_class-target", "a") + self.d.setVar("OVERRIDES", "class-target") + bb.data.update_data(self.d) + self.d.renameVar("ALTERNATIVE_ncurses-tools", "ALTERNATIVE_lib32-ncurses-tools") + self.assertEqual(self.d.getVar("ALTERNATIVE_lib32-ncurses-tools", True), "a") + + def test_underscore_override(self): + self.d.setVar("TEST_bar", "testvalue2") + self.d.setVar("TEST_some_val", "testvalue3") + self.d.setVar("TEST_foo", "testvalue4") + self.d.setVar("OVERRIDES", "foo:bar:some_val") + self.assertEqual(self.d.getVar("TEST", True), "testvalue3") class TestKeyExpansion(unittest.TestCase): def setUp(self):
lib/bb/tests/parse.py +46 -0
49 lines not on this page · raw message
Signature · 1 line
2.1.0