bitbake: optimize file parsing speed
cover bitbake: optimize file parsing speed
Hi Richard,
This RFC/pull request aims to speed up the bb file parsing speed. Please help to review it, thanks!
We observed that the ??= operator costs a lot of cycles when parsing bb files, this patchset implement the ??= in getVar() function. If the normal getVar returns None, it will check the defaultval flag and return its value.
Besides, 0001 patch also fixes a bug that if foo_OVERA[bar] = "x" and foo is never set as a variable directly, the override is never expanded.
I slightly modified 0001 patch compared with the previous discussion version. If simply copy the setVar logic into setVarFlag, I found that variable "do_install_append_virtclass-native" will both exist in self._special_values and self._seen_overrides, this is because when setting "content" for "do_install_append_virtclass-native", the variable will be recorded in self._special_values, and when setting "func" flag for "do_install_append_virtclass-native", the variable will be recorded in self._seen_overrides. This version of patch could fix the issue.
Pull URL: git://git.pokylinux.org/poky-contrib.git
Branch: dxu4/parsefile
Browse: http://git.pokylinux.org/cgit.cgi/poky-contrib/log/?h=dxu4/parsefile
Thanks,
Dongxiao Xu <[email protected]>
---
Dongxiao Xu (2):
bitbake: Make setVar a special case of setVarFlag
bitbake: Change the way to implement ??= operator
bitbake/lib/bb/data_smart.py | 63 ++++++++++++++++++++--------------------- bitbake/lib/bb/parse/ast.py | 7 ---- 2 files changed, 31 insertions(+), 39 deletions(-)
PATCH 1/2 bitbake: Make setVar a special case of setVarFlag
Move logic in setVar into setVarFlag, and make setVar a special case of setVarFlag.
This can fix the bug that, if we assign foo_OVERA[bar] = "x" and foo is never set as a variable directly, the override is never expanded in original code.
--- bitbake/lib/bb/data_smart.py | 60 +++++++++++++++++++---------------------- 1 files changed, 28 insertions(+), 32 deletions(-)
bitbake/lib/bb/data_smart.py +28 -32
@@ -205,38 +205,7 @@ class DataSmart: self.initVar(var) def setVar(self, var, value): - self.expand_cache = {} - match = __setvar_regexp__.match(var) - if match and match.group("keyword") in __setvar_keyword__: - base = match.group('base') - keyword = match.group("keyword") - override = match.group('add') - l = self.getVarFlag(base, keyword) or [] - l.append([value, override]) - self.setVarFlag(base, keyword, l) - - # todo make sure keyword is not __doc__ or __module__ - # pay the cookie monster - try: - self._special_values[keyword].add( base ) - except KeyError: - self._special_values[keyword] = set() - self._special_values[keyword].add( base ) - - return - - if not var in self.dict: - self._makeShadowCopy(var) - - # more cookies for the cookie monster - if '_' in var: - override = var[var.rfind('_')+1:] - if override not in self._seen_overrides: - self._seen_overrides[override] = set() - self._seen_overrides[override].add( var ) - - # setting var - self.dict[var]["content"] = value + self.setVarFlag(var, "content", value) def getVar(self, var, exp): value = self.getVarFlag(var, "content")
@@ -273,6 +242,33 @@ class DataSmart: self.dict[var] = {} def setVarFlag(self, var, flag, flagvalue): + self.expand_cache = {} + match = __setvar_regexp__.match(var) + if match and match.group("keyword") in __setvar_keyword__: + base = match.group('base') + keyword = match.group("keyword") + # todo make sure keyword is not __doc__ or __module__ + # pay the cookie monster + try: + self._special_values[keyword].add( base ) + except KeyError: + self._special_values[keyword] = set() + self._special_values[keyword].add( base ) + + if flag == "content": + override = match.group('add') + l = self.getVarFlag(base, keyword) or [] + l.append([flagvalue, override]) + self.setVarFlag(base, keyword, l) + return + + # more cookies for the cookie monster + elif '_' in var: + override = var[var.rfind('_')+1:] + if override not in self._seen_overrides: + self._seen_overrides[override] = set() + self._seen_overrides[override].add( var ) + if not var in self.dict: self._makeShadowCopy(var) self.dict[var][flag] = flagvalue
Signature · 1 line
1.6.3.3
PATCH 2/2 bitbake: Change the way to implement ??= operator
In reply to Dongxiao XuPreviously we implement ??= operator while doing finalize(), which hurts performance due to the large index of keys. This new method will return getVarFlag("defaultval") if getVar returns None. This patch brings about 15% performance gain when parsing bb files.
--- bitbake/lib/bb/data_smart.py | 3 +++ bitbake/lib/bb/parse/ast.py | 7 ------- 2 files changed, 3 insertions(+), 7 deletions(-)
bitbake/lib/bb/data_smart.py +3 -0
@@ -210,6 +210,9 @@ class DataSmart: def getVar(self, var, exp): value = self.getVarFlag(var, "content") + if value is None: + value = self.getVarFlag(var, "defaultval") + if exp and value: return self.expand(value, var) return value
bitbake/lib/bb/parse/ast.py +0 -7
@@ -109,9 +109,6 @@ class DataNode(AstNode): if 'flag' in groupd and groupd['flag'] != None: bb.data.setVarFlag(key, groupd['flag'], val, data) elif groupd["lazyques"]: - assigned = bb.data.getVar("__lazy_assigned", data) or [] - assigned.append(key) - bb.data.setVar("__lazy_assigned", assigned, data) bb.data.setVarFlag(key, "defaultval", val, data) else: bb.data.setVar(key, val, data)
@@ -301,10 +298,6 @@ def handleInherit(statements, m): statements.append(InheritNode(m.group(1))) def finalize(fn, d, variant = None): - for lazykey in bb.data.getVar("__lazy_assigned", d) or (): - if bb.data.getVar(lazykey, d) is None: - val = bb.data.getVarFlag(lazykey, "defaultval", d) - bb.data.setVar(lazykey, val, d) bb.data.expandKeys(d) bb.data.update_data(d)
Signature · 1 line
1.6.3.3