data_smart: Improve performance for VariableHistory
cover data_smart: Improve performance for VariableHistory
From: Robert Yang <[email protected]>
The following changes since commit 7c1de3118fef96fa899ae5365f80ea8ccb1a5876:
create-spdx-3.0/populate_sdk_base: Add SDK_CLASSES inherit mechanism to fix tarball SPDX manifests (2024-07-26 12:28:42 +0100)
are available in the Git repository at:
https://github.com/robertlinux/yocto rbt/data_smart
https://github.com/robertlinux/yocto/tree/rbt/data_smart
Robert Yang (1):
data_smart: Improve performance for VariableHistory
bitbake/lib/bb/data_smart.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-)
Signature · 1 line
2.25.1
PATCH 1/1 data_smart: Improve performance for VariableHistory
From: Robert Yang <[email protected]>
Fixed:
- BBMULTICONFIG = "qemux86-64 qemuarm64" and more than 70 layers in BBLAYERS
$ bitbake -p -P Check profile.log.processed, the record() cost more than 20 seconds, it is less than 1 second when multiconfig is not enabled, and there would be the following error when more muticonfigs are enabled:
Timeout while waiting for a reply from the bitbake server
Don't change the type of loginfo['detail'] or re-assign it can make record() back to less than 1 second, this won't affect COW since loginfo is a mutable type.
The time mainly affected by two factors:
1) The number of enabled layers, nearly 1 second added per layer when the number is larger than 50.
2) The global var such as USER_CLASSES, about 1 ~ 2 seconds added per layer when the layers number is larger than 50.
--- bitbake/lib/bb/data_smart.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-)
bitbake/lib/bb/data_smart.py +3 -5
@@ -272,12 +272,9 @@ class VariableHistory(object): return if 'op' not in loginfo or not loginfo['op']: loginfo['op'] = 'set' - if 'detail' in loginfo: - loginfo['detail'] = str(loginfo['detail']) if 'variable' not in loginfo or 'file' not in loginfo: raise ValueError("record() missing variable or file.") var = loginfo['variable'] - if var not in self.variables: self.variables[var] = [] if not isinstance(self.variables[var], list):
@@ -336,7 +333,8 @@ class VariableHistory(object): flag = '[%s] ' % (event['flag']) else: flag = '' - o.write("# %s %s:%s%s\n# %s\"%s\"\n" % (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n# ', event['detail']))) + o.write("# %s %s:%s%s\n# %s\"%s\"\n" % \ + (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n# ', str(event['detail'])))) if len(history) > 1: o.write("# pre-expansion value:\n") o.write('# "%s"\n' % (commentVal))
@@ -390,7 +388,7 @@ class VariableHistory(object): if isset and event['op'] == 'set?': continue isset = True - items = d.expand(event['detail']).split() + items = d.expand(str(event['detail'])).split() for item in items: # This is a little crude but is belt-and-braces to avoid us # having to handle every possible operation type specifically
Signature · 1 line
2.25.1
Quoted 62 lines · Robert Yang via lists.openembedded.org
From: Robert Yang <[email protected]>
Fixed:
- BBMULTICONFIG = "qemux86-64 qemuarm64" and more than 70 layers in BBLAYERS$ bitbake -p -P Check profile.log.processed, the record() cost more than 20 seconds, it is less than 1 second when multiconfig is not enabled, and there would be the following error when more muticonfigs are enabled:Timeout while waiting for a reply from the bitbake server
Don't change the type of loginfo['detail'] or re-assign it can make
record()back to less than 1 second, this won't affect COW since loginfo is a mutable type.The time mainly affected by two factors:
1) The number of enabled layers, nearly 1 second added per layer when the number is larger than 50.2) The global var such as
USER_CLASSES, about 1 ~ 2 seconds added per layer when the layers number is larger than 50.Signed-off-by Robert Yang <[email protected]>---
bitbake/lib/bb/data_smart.py | 8 +++-----1 file changed, 3 insertions(+), 5 deletions(-)bitbake/lib/bb/data_smart.py +0
-0@@ -272,12 +272,9 @@ class VariableHistory(object):return
if 'op' not in loginfo or not loginfo['op']:
loginfo['op'] = 'set'
- if 'detail' in loginfo:
- loginfo['detail'] =str(loginfo['detail'])if 'variable' not in loginfo or 'file' not in loginfo:
raiseValueError("record() missing variable or file.")var = loginfo['variable']
-
if var not in self.variables:
self.variables[var] = []
if notisinstance(self.variables[var], list):bitbake/lib/bb/data_smart.py +0
-0@@ -336,7 +333,8 @@ class VariableHistory(object): flag = '[%s] ' % (event['flag'])else:
flag = ''
- o.write("# %s %s:%s%s\n# %s\"%s\"\n" % (event['op'], event['file'], event['line'],display_func, flag,re.sub('\n', '\n# ', event['detail']))) + o.write("# %s %s:%s%s\n# %s\"%s\"\n" % \
+ (event['op'], event['file'], event['line'],display_func, flag,re.sub('\n', '\n# ', str(event['detail']))))
if len(history) > 1:
o.write("# pre-expansion value:\n")
o.write('# "%s"\n' % (commentVal))bitbake/lib/bb/data_smart.py +0
-0@@ -390,7 +388,7 @@ class VariableHistory(object):if isset and event['op'] == 'set?':
continue
isset = True
- items = d.expand(event['detail']).split()
+ items = d.expand(str(event['detail'])).split()
for item in items:
# This is a little crude but is belt-and-braces to avoid us
Quoted 2 lines
# having to handle every possible operation type specifically
This looks like a good find, thanks for digging into it! I'm a little surprised this causes as much of a problem and was a bit worried about unintended side effects but I didn't find any in testing so it has merged, thanks!
Cheers,
Richard