utils.py: fix remove obsolete dependencies
A message that is not in the archive (sent off-list or not yet received): [email protected]
PATCH 1/5 utils.py: fix remove obsolete dependencies
The obsolete dependency is the one which is in database but not in conf/layer.conf anymore. The old code had a problem for newly created layerbranch, the new layerbranch has no dependencies, so no need remove. And it had a side effect was that when need_remove was cleaned up, it would be set again in the next for loop, thus might wrongly remove dependencies. This patch can fix the problem.
--- layerindex/utils.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-)
layerindex/utils.py +2 -4
@@ -122,7 +122,7 @@ def _add_dependency(var, name, layerbranch, config_data, logger=None, required=T logger.debug('Error parsing %s_%s for %s\n%s' % (var, var_name, layer_name, str(vse))) return - need_remove = None + need_remove = LayerDependency.objects.filter(layerbranch=layerbranch).filter(required=required) for dep, ver_list in list(dep_dict.items()): ver_str = None if ver_list:
@@ -142,9 +142,7 @@ def _add_dependency(var, name, layerbranch, config_data, logger=None, required=T continue # Preparing to remove obsolete ones - if not need_remove: - need_remove = LayerDependency.objects.filter(layerbranch=layerbranch).filter(required=required).exclude(dependency=dep_layer) - else: + if need_remove: need_remove = need_remove.exclude(dependency=dep_layer) # Skip existing entries.
Signature · 1 line
2.7.4
Hi Paul,
Do you have any comments, please ?
// Robert
Quoted 37 lines · Robert Yang
The obsolete dependency is the one which is in database but not in
conf/layer.confanymore. The old code had a problem for newly created layerbranch, the new layerbranch has no dependencies, so no need remove. And it had a side effect was that whenneed_removewas cleaned up, it would be set again in the next for loop, thus might wrongly remove dependencies. This patch can fix the problem.Signed-off-by Robert Yang <[email protected]>--- layerindex/utils.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-)layerindex/utils.py +2
-4@@ -122,7 +122,7 @@ def _add_dependency(var, name, layerbranch, config_data, logger=None, required=T logger.debug('Error parsing %s_%s for %s\n%s' % (var, var_name, layer_name, str(vse))) return - need_remove = None + need_remove = LayerDependency.objects.filter(layerbranch=layerbranch).filter(required=required) for dep, ver_list in list(dep_dict.items()): ver_str = None if ver_list:@@ -142,9 +142,7 @@ def _add_dependency(var, name, layerbranch, config_data, logger=None, required=T continue # Preparing to remove obsolete ones - if not need_remove: - need_remove = LayerDependency.objects.filter(layerbranch=layerbranch).filter(required=required).exclude(dependency=dep_layer) - else: + if need_remove: need_remove = need_remove.exclude(dependency=dep_layer) # Skip existing entries.
PATCH 2/5 update.py: make sure oe-core is fetched
In reply to a message not in the archiveFixed:
$ ./update.py -l foo -b new_branch INFO: Fetching remote repository foo DEBUG: run cmd 'git fetch -p' in 'foo' [snip] DEBUG: run cmd 'git checkout origin/new_branch' in oe-core ERROR: error: pathspec 'origin/new_branch' did not match any file(s) known to git.
The "new_branch" is newly created, it doesn't exist in local repo since it isn't fetched, it only fetches the layer specified by -l, so only the foo layer is fetched. This patch fixes problem.
--- layerindex/update.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-)
layerindex/update.py +7 -1
@@ -257,8 +257,14 @@ def main(): bitbakepath = os.path.join(fetchdir, 'bitbake') if not options.nofetch: + # Make sure oe-core is fetched since recipe parsing requires it + layerquery_core = LayerItem.objects.filter(classic=False).filter(name=settings.CORE_LAYER_NAME) + if layerquery_core in layerquery: + layerquery_fetch = list(layerquery) + else: + layerquery_fetch = list(layerquery) + list(layerquery_core) # Fetch latest metadata from repositories - for layer in layerquery: + for layer in layerquery_fetch: # Handle multiple layers in a single repo urldir = layer.get_fetch_dir() repodir = os.path.join(fetchdir, urldir)
Signature · 1 line
2.7.4
PATCH 3/5 update.py: fetch repos parallelly
In reply to a message not in the archiveThis can save a lot of time, here is my testing data when PARALLEL_JOBS is 10, this is the fetch time only, I hacked it to stop when the fetch is done to get the data (124 layers):
$ update.py -b <branch> Before: 2m30 Now: 16s
--- layerindex/update.py | 53 +++++++++++++++++++++++++++++++++------------------- settings.py | 3 +++ 2 files changed, 37 insertions(+), 19 deletions(-)
layerindex/update.py +34 -19
@@ -20,6 +20,7 @@ from distutils.version import LooseVersion import utils import operator import re +import multiprocessing import warnings warnings.filterwarnings("ignore", category=DeprecationWarning)
@@ -127,6 +128,18 @@ def update_actual_branch(layerquery, fetchdir, branch, options, update_bitbake, for s in to_save: s.save() +def fetch_repo(vcs_url, repodir, urldir, fetchdir, layer_name): + logger.info("Fetching remote repository %s" % vcs_url) + try: + if not os.path.exists(repodir): + utils.runcmd("git clone %s %s" % (vcs_url, urldir), fetchdir, logger=logger, printerr=False) + else: + utils.runcmd("git fetch -p", repodir, logger=logger, printerr=False) + return (vcs_url, None) + except subprocess.CalledProcessError as e: + logger.error("Fetch of layer %s failed: %s" % (layer_name, e.output)) + return (vcs_url, e.output) + def main(): if LooseVersion(git.__version__) < '0.3.1': logger.error("Version of GitPython is too old, please install GitPython (python-git) 0.3.1 or later in order to use this script")
@@ -232,6 +245,9 @@ def main(): if not os.path.exists(fetchdir): os.makedirs(fetchdir) + + allrepos = {} + fetchedresult = [] fetchedrepos = [] failedrepos = {}
@@ -268,30 +284,29 @@ def main(): # Handle multiple layers in a single repo urldir = layer.get_fetch_dir() repodir = os.path.join(fetchdir, urldir) - if not (layer.vcs_url in fetchedrepos or layer.vcs_url in failedrepos): - logger.info("Fetching remote repository %s" % layer.vcs_url) - out = None - try: - if not os.path.exists(repodir): - out = utils.runcmd("git clone %s %s" % (layer.vcs_url, urldir), fetchdir, logger=logger, printerr=False) - else: - out = utils.runcmd("git fetch -p", repodir, logger=logger, printerr=False) - except subprocess.CalledProcessError as e: - logger.error("Fetch of layer %s failed: %s" % (layer.name, e.output)) - failedrepos[layer.vcs_url] = e.output - continue - fetchedrepos.append(layer.vcs_url) + if layer.vcs_url not in allrepos: + allrepos[layer.vcs_url] = (repodir, urldir, fetchdir, layer.name) + # Add bitbake + allrepos[settings.BITBAKE_REPO_URL] = (bitbakepath, "bitbake", fetchdir, "bitbake") + # Parallel fetching + pool = multiprocessing.Pool(int(settings.PARALLEL_JOBS)) + for url in allrepos: + fetchedresult.append(pool.apply_async(fetch_repo, \ + (url, allrepos[url][0], allrepos[url][1], allrepos[url][2], allrepos[url][3],))) + pool.close() + pool.join() + + for url in fetchedresult[:]: + # The format is (url, error), the error is None when succeed. + if url.get()[1]: + failedrepos[url.get()[0]] = url.get()[1] + else: + fetchedrepos.append(url.get()[0]) if not (fetchedrepos or update_bitbake): logger.error("No repositories could be fetched, exiting") sys.exit(1) - logger.info("Fetching bitbake from remote repository %s" % settings.BITBAKE_REPO_URL) - if not os.path.exists(bitbakepath): - out = utils.runcmd("git clone %s %s" % (settings.BITBAKE_REPO_URL, 'bitbake'), fetchdir, logger=logger) - else: - out = utils.runcmd("git fetch -p", bitbakepath, logger=logger) - if options.actual_branch: update_actual_branch(layerquery, fetchdir, branches[0], options, update_bitbake, bitbakepath) return
settings.py +3 -0
@@ -228,3 +228,6 @@ SUBMIT_EMAIL_SUBJECT = 'OE Layerindex layer submission' # RabbitMQ settings RABBIT_BROKER = 'amqp://' RABBIT_BACKEND = 'rpc://' + +# Used for fetching repo +PARALLEL_JOBS = "4"
Signature · 1 line
2.7.4
PATCH 4/5 update_layer.py: use it as a module
In reply to a message not in the archiveIt had been split to a separate tool and use subprocess to run it to avoid tinfoil issues (e.g., can't be shutdown correctly in old version), but the utils.setup_django() cost a lot of time which made it a little slow, use mulitprocessing.Process() to run it as function can avoid both issues (tinfoil and slow issues).
And remove bitbake modules when swith branch since it is may not be compatible between branches.
This can save about 5 minutes in my testing when "update.py -b <branch>", with parallel fetch and this, we can reduce the time from more than 9mins to less than 3 mins.
--- layerindex/update.py | 115 +++++++++++++++++++-------------------------- layerindex/update_layer.py | 103 +++++++++------------------------------- 2 files changed, 70 insertions(+), 148 deletions(-)
layerindex/update.py +49 -66
@@ -21,6 +21,9 @@ import utils import operator import re import multiprocessing +import update_layer +import copy +import imp import warnings warnings.filterwarnings("ignore", category=DeprecationWarning)
@@ -34,61 +37,15 @@ except ImportError: logger.error("Please install PythonGit 0.3.1 or later in order to use this script") sys.exit(1) - -def reenable_sigint(): - signal.signal(signal.SIGINT, signal.SIG_DFL) - -def run_command_interruptible(cmd): - """ - Run a command with output displayed on the console, but ensure any Ctrl+C is - processed only by the child process. - """ - signal.signal(signal.SIGINT, signal.SIG_IGN) +def call_update_layer(options, queue): + ret = 1 + output = "" try: - process = subprocess.Popen( - cmd, cwd=os.path.dirname(sys.argv[0]), shell=True, preexec_fn=reenable_sigint, stdout=subprocess.PIPE, stderr=subprocess.STDOUT - ) - - reader = codecs.getreader('utf-8')(process.stdout, errors='surrogateescape') - buf = '' - while True: - out = reader.read(1, 1) - if out: - sys.stdout.write(out) - sys.stdout.flush() - buf += out - elif out == '' and process.poll() != None: - break - + ret, output = update_layer.main(options, queue) + except Exception: + raise finally: - signal.signal(signal.SIGINT, signal.SIG_DFL) - return process.returncode, buf - - -def prepare_update_layer_command(options, branch, layer, initial=False): - """Prepare the update_layer.py command line""" - if branch.update_environment: - cmdprefix = branch.update_environment.get_command() - else: - cmdprefix = 'python3' - cmd = '%s update_layer.py -l %s -b %s' % (cmdprefix, layer.name, branch.name) - if options.reload: - cmd += ' --reload' - if options.fullreload: - cmd += ' --fullreload' - if options.nocheckout: - cmd += ' --nocheckout' - if options.dryrun: - cmd += ' -n' - if initial: - cmd += ' -i' - if options.loglevel == logging.DEBUG: - cmd += ' -d' - elif options.loglevel == logging.ERROR: - cmd += ' -q' - if options.keep_temp: - cmd += ' --keep-temp' - return cmd + queue.put([ret, output]) def update_actual_branch(layerquery, fetchdir, branch, options, update_bitbake, bitbakepath): """Update actual branch for layers and bitbake in database"""
@@ -266,6 +223,7 @@ def main(): try: lockfn = os.path.join(fetchdir, "layerindex.lock") lockfile = utils.lock_file(lockfn) + update_log_path = update_layer.update_log_path if not lockfile: logger.error("Layer index lock timeout expired") sys.exit(1)
@@ -315,8 +273,15 @@ def main(): # We now do this by calling out to a separate script; doing otherwise turned out to be # unreliable due to leaking memory (we're using bitbake internals in a manner in which # they never get used during normal operation). + update_layer_options = copy.copy(options) last_rev = {} for branch in branches: + # The bitbake modules are not compatible between branches, so + # remove them, they will be imported again when needed. + for name, module in sys.modules.copy().items(): + if '__file__' in module.__dict__ and module.__file__.startswith(bitbakepath): + logger.debug('Removing module %s' % name) + del sys.modules[name] # If layer_A depends(or recommends) on layer_B, add layer_B before layer_A deps_dict_all = {} layerquery_sorted = []
@@ -333,16 +298,22 @@ def main(): collections_done.add((layerbranch.collection, layerbranch.version)) for layer in layerquery: - cmd = prepare_update_layer_command(options, branchobj, layer, initial=True) - logger.debug('Running layer update command: %s' % cmd) - ret, output = run_command_interruptible(cmd) - logger.debug('output: %s' % output) - if ret != 0: + logger.info('Getting initial data for layer %s', layer.name) + update_layer_options.initial = True + update_layer_options.layers = layer.name + update_layer_options.branch = branch + queue = multiprocessing.Queue() + p = multiprocessing.Process(target=call_update_layer, args=(update_layer_options, queue,)) + p.start() + ret, initial_data = queue.get() + p.join() + logger.debug('initial_data: %s' % initial_data) + if ret != 0 or not initial_data: continue - col = re.search("^BBFILE_COLLECTIONS = \"(.*)\"", output, re.M).group(1) or '' - ver = re.search("^LAYERVERSION = \"(.*)\"", output, re.M).group(1) or '' - deps = re.search("^LAYERDEPENDS = \"(.*)\"", output, re.M).group(1) or '' - recs = re.search("^LAYERRECOMMENDS = \"(.*)\"", output, re.M).group(1) or '' + col = initial_data[0] + ver = initial_data[1] + deps = initial_data[2] + recs = initial_data[3] deps_dict = utils.explode_dep_versions2(bitbakepath, deps + ' ' + recs) if len(deps_dict) == 0:
@@ -382,6 +353,7 @@ def main(): logger.error("Known collections: %s" % collections_done) sys.exit(1) + update_layer_options.initial = False for layer in layerquery_sorted: layerupdate = LayerUpdate() layerupdate.update = update
@@ -399,10 +371,18 @@ def main(): layerupdate.save() continue - cmd = prepare_update_layer_command(options, branchobj, layer) - logger.debug('Running layer update command: %s' % cmd) + logger.info('Updating layer %s on branch %s' % (layer.name, branch)) + update_layer_options.layers = layer.name + update_layer_options.branch = branch + # Refresh update log + with open(update_log_path, 'w') as f: + pass + queue = multiprocessing.Queue() + p = multiprocessing.Process(target=call_update_layer, args=(update_layer_options, queue,)) + p.start() + ret, output = queue.get() + p.join() layerupdate.started = datetime.now() - ret, output = run_command_interruptible(cmd) layerupdate.finished = datetime.now() # We need to get layerbranch here because it might not have existed until
@@ -412,7 +392,8 @@ def main(): if layerbranch: last_rev[layerbranch] = layerbranch.vcs_last_rev layerupdate.layerbranch = layerbranch - layerupdate.log = output + with open(update_log_path, 'r') as f: + layerupdate.log = f.read() if not options.dryrun: layerupdate.save()
@@ -421,6 +402,8 @@ def main(): break finally: utils.unlock_file(lockfile) + if os.path.exists(update_log_path): + os.remove(update_log_path) finally: update.log = ''.join(listhandler.read())
layerindex/update_layer.py +21 -82
188 lines not on this page · raw message
Signature · 1 line
2.7.4
Re: [layerindex-web][PATCH 4/5] update_layer.py: use it as a module
Hi Robert,
Quoted 113 lines · NZDT Robert Yang
It had been split to a separate tool and use subprocess to run it to avoid tinfoil issues (e.g., can't be shutdown correctly in old version), but the
utils.setup_django()cost a lot of time which made it a little slow, usemulitprocessing.Process()to run it as function can avoid both issues (tinfoil and slow issues).And remove bitbake modules when swith branch since it is may not be compatible between branches.
This can save about 5 minutes in my testing when "
update.py-b <branch>", with parallel fetch and this, we can reduce the time from more than 9mins to less than 3 mins.Signed-off-by Robert Yang <[email protected]>--- layerindex/update.py | 115 +++++++++++++++++++-------------------------- layerindex/update_layer.py | 103 +++++++++------------------------------- 2 files changed, 70 insertions(+), 148 deletions(-)layerindex/update.py +11
-53@@ -21,6 +21,9 @@ import utils import operator import re import multiprocessing +import update_layer +import copy +import imp import warnings warnings.filterwarnings("ignore", category=DeprecationWarning)@@ -34,61 +37,15 @@ except ImportError: logger.error("Please install PythonGit 0.3.1 or later in order to use this script") sys.exit(1) - -def reenable_sigint(): - signal.signal(signal.SIGINT, signal.SIG_DFL) - -def run_command_interruptible(cmd): - """ - Run a command with output displayed on the console, but ensure any Ctrl+C is - processed only by the child process. - """ - signal.signal(signal.SIGINT, signal.SIG_IGN) +def call_update_layer(options, queue): + ret = 1 + output = "" try: - process = subprocess.Popen( - cmd, cwd=os.path.dirname(sys.argv[0]), shell=True, preexec_fn=reenable_sigint, stdout=subprocess.PIPE, stderr=subprocess.STDOUT - ) - - reader = codecs.getreader('utf-8')(process.stdout, errors='surrogateescape') - buf = '' - while True: - out = reader.read(1, 1) - if out: - sys.stdout.write(out) - sys.stdout.flush() - buf += out - elif out == '' and process.poll() != None: - break - + ret, output = update_layer.main(options, queue) + except Exception: + raise finally: - signal.signal(signal.SIGINT, signal.SIG_DFL) - return process.returncode, buf - - -def prepare_update_layer_command(options, branch, layer, initial=False): - """Prepare the update_layer.py command line""" - if branch.update_environment: - cmdprefix = branch.update_environment.get_command() - else: - cmdprefix = 'python3' - cmd = '%s update_layer.py -l %s -b %s' % (cmdprefix, layer.name, branch.name) - if options.reload: - cmd += ' --reload' - if options.fullreload: - cmd += ' --fullreload' - if options.nocheckout: - cmd += ' --nocheckout' - if options.dryrun: - cmd += ' -n' - if initial: - cmd += ' -i' - if options.loglevel == logging.DEBUG: - cmd += ' -d' - elif options.loglevel == logging.ERROR: - cmd += ' -q' - if options.keep_temp: - cmd += ' --keep-temp' - return cmd + queue.put([ret, output]) def update_actual_branch(layerquery, fetchdir, branch, options, update_bitbake, bitbakepath): """Update actual branch for layers and bitbake in database"""@@ -266,6 +223,7 @@ def main(): try: lockfn = os.path.join(fetchdir, "layerindex.lock") lockfile = utils.lock_file(lockfn) + update_log_path = update_layer.update_log_path if not lockfile: logger.error("Layer index lock timeout expired") sys.exit(1)@@ -315,8 +273,15 @@ def main(): # We now do this by calling out to a separate script; doing otherwise turned out to be # unreliable due to leaking memory (we're using bitbake internals in a manner in which
layerindex/update.py def main(): · line 284
Quoted 10 lines
layerindex/update.py +7
-0# they never get used during normal operation). + update_layer_options = copy.copy(options) last_rev = {} for branch in branches: + # The bitbake modules are not compatible between branches, so + # remove them, they will be imported again when needed. + for name, module in sys.modules.copy().items(): + if '__file__' in module.__dict__ and module.__file__.startswith(bitbakepath): + logger.debug('Removing module %s' % name) + del sys.modules[name]
This part worried me, so I did some quick searching and came across this:
http://justus.science/blog/2015/04/19/sys.modules-is-dangerous.html
Relying upon removing old modules from memory just seems fraught with danger to me. I'd really rather not go back to debugging the contamination issues we dealt with earlier for the sake of saving a few minutes.
Another problem with this is it breaks supporting python 2 and python 3 in the same parse - in the OE layer index we actually still enable parsing back to fido (1.8). I honestly don't know how many updates those branches actually see, and we should probably disable parsing for fido and perhaps jethro, but I suspect at least krogoth (2.1) still gets some usage. Until we disable parsing of all of the python 2 based branches we need the ability to handle both versions and that requires update_layer.py to be run separately.
I do agree in general that performance of the update process could be improved - one particularly poor area is where we collect the variable values for each layer, that part is very slow and we're doing it regardless of whether the layer has changed or not which seems wasteful. That would be my recommendation for where to look next (assuming you have time).
Cheers,
Paul
Signature · 2 lines
Paul Eggleton Intel Open Source Technology Centre
Re: [layerindex-web][PATCH 4/5] update_layer.py: use it as a module
Hi Paul,
Thank you very much, and please see my comments inline.
Quoted 143 lines · Paul Eggleton
Hi Robert,
Quoted 113 lines · NZDT Robert Yang
It had been split to a separate tool and use subprocess to run it to avoid tinfoil issues (e.g., can't be shutdown correctly in old version), but the
utils.setup_django()cost a lot of time which made it a little slow, usemulitprocessing.Process()to run it as function can avoid both issues (tinfoil and slow issues).And remove bitbake modules when swith branch since it is may not be compatible between branches.
This can save about 5 minutes in my testing when "
update.py-b <branch>", with parallel fetch and this, we can reduce the time from more than 9mins to less than 3 mins.Signed-off-by Robert Yang <[email protected]>--- layerindex/update.py | 115 +++++++++++++++++++-------------------------- layerindex/update_layer.py | 103 +++++++++------------------------------- 2 files changed, 70 insertions(+), 148 deletions(-)layerindex/update.py +11
-53@@ -21,6 +21,9 @@ import utils import operator import re import multiprocessing +import update_layer +import copy +import imp import warnings warnings.filterwarnings("ignore", category=DeprecationWarning)@@ -34,61 +37,15 @@ except ImportError: logger.error("Please install PythonGit 0.3.1 or later in order to use this script") sys.exit(1) - -def reenable_sigint(): - signal.signal(signal.SIGINT, signal.SIG_DFL) - -def run_command_interruptible(cmd): - """ - Run a command with output displayed on the console, but ensure any Ctrl+C is - processed only by the child process. - """ - signal.signal(signal.SIGINT, signal.SIG_IGN) +def call_update_layer(options, queue): + ret = 1 + output = "" try: - process = subprocess.Popen( - cmd, cwd=os.path.dirname(sys.argv[0]), shell=True, preexec_fn=reenable_sigint, stdout=subprocess.PIPE, stderr=subprocess.STDOUT - ) - - reader = codecs.getreader('utf-8')(process.stdout, errors='surrogateescape') - buf = '' - while True: - out = reader.read(1, 1) - if out: - sys.stdout.write(out) - sys.stdout.flush() - buf += out - elif out == '' and process.poll() != None: - break - + ret, output = update_layer.main(options, queue) + except Exception: + raise finally: - signal.signal(signal.SIGINT, signal.SIG_DFL) - return process.returncode, buf - - -def prepare_update_layer_command(options, branch, layer, initial=False): - """Prepare the update_layer.py command line""" - if branch.update_environment: - cmdprefix = branch.update_environment.get_command() - else: - cmdprefix = 'python3' - cmd = '%s update_layer.py -l %s -b %s' % (cmdprefix, layer.name, branch.name) - if options.reload: - cmd += ' --reload' - if options.fullreload: - cmd += ' --fullreload' - if options.nocheckout: - cmd += ' --nocheckout' - if options.dryrun: - cmd += ' -n' - if initial: - cmd += ' -i' - if options.loglevel == logging.DEBUG: - cmd += ' -d' - elif options.loglevel == logging.ERROR: - cmd += ' -q' - if options.keep_temp: - cmd += ' --keep-temp' - return cmd + queue.put([ret, output]) def update_actual_branch(layerquery, fetchdir, branch, options, update_bitbake, bitbakepath): """Update actual branch for layers and bitbake in database"""@@ -266,6 +223,7 @@ def main(): try: lockfn = os.path.join(fetchdir, "layerindex.lock") lockfile = utils.lock_file(lockfn) + update_log_path = update_layer.update_log_path if not lockfile: logger.error("Layer index lock timeout expired") sys.exit(1)@@ -315,8 +273,15 @@ def main(): # We now do this by calling out to a separate script; doing otherwise turned out to be # unreliable due to leaking memory (we're using bitbake internals in a manner in whichlayerindex/update.py def main(): · line 284
Quoted 10 lines
layerindex/update.py +7
-0# they never get used during normal operation). + update_layer_options = copy.copy(options) last_rev = {} for branch in branches: + # The bitbake modules are not compatible between branches, so + # remove them, they will be imported again when needed. + for name, module in sys.modules.copy().items(): + if '__file__' in module.__dict__ and module.__file__.startswith(bitbakepath): + logger.debug('Removing module %s' % name) + del sys.modules[name]This part worried me, so I did some quick searching and came across this:
http://justus.science/blog/2015/04/19/sys.modules-is-dangerous.htmlRelying upon removing old modules from memory just seems fraught with danger to me. I'd really rather not go back to debugging the contamination issues we dealt with earlier for the sake of saving a few minutes.
Another problem with this is it breaks supporting python 2 and python 3 in the same parse - in the OE layer index we actually still enable parsing back to fido (1.8). I honestly don't know how many updates those branches actually see, and we should probably disable parsing for fido and perhaps jethro, but I suspect at least krogoth (2.1) still gets some usage. Until we disable parsing of all of the python 2 based branches we need the ability to handle both versions and that requires
update_layer.pyto be run separately.
Quoted 5 lines
I do agree in general that performance of the update process could be improved - one particularly poor area is where we collect the variable values for each layer, that part is very slow and we're doing it regardless of whether the layer has changed or not which seems wasteful. That would be my recommendation for where to look next (assuming you have time).
I understand that python2 is a problem. I've sent patches to oe-core to improve parsing time. And I will look at your recommendations.
// Robert
Quoted 4 lines
Cheers,
Paul
PATCH 5/5 update_layer.py: only call init_parser when needed
In reply to a message not in the archiveThe tinoil is not needed in cases like the layer is already up-to-date or the layer is invalid, so only init it when needed.
This can save about 1min when run "update.py -b <branch>" (124 layers).
--- layerindex/update_layer.py | 45 +++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 22 deletions(-)
layerindex/update_layer.py +23 -22
@@ -244,22 +244,8 @@ def main(options, queue): logger.info("Skipping update of layer %s - branch %s doesn't exist" % (layer.name, branchdesc)) return (1, '') - try: - (tinfoil, tempdir) = recipeparse.init_parser(settings, branch, bitbakepath, nocheckout=options.nocheckout, logger=logger) - except recipeparse.RecipeParseError as e: - logger.error(str(e)) - return(1, '') - logger.debug('Using temp directory %s' % tempdir) - # Clear the default value of SUMMARY so that we can use DESCRIPTION instead if it hasn't been set - tinfoil.config_data.setVar('SUMMARY', '') - # Clear the default value of DESCRIPTION so that we can see where it's not set - tinfoil.config_data.setVar('DESCRIPTION', '') - # Clear the default value of HOMEPAGE ('unknown') - tinfoil.config_data.setVar('HOMEPAGE', '') - # Set a blank value for LICENSE so that it doesn't cause the parser to die (e.g. with meta-ti - - # why won't they just fix that?!) - tinfoil.config_data.setVar('LICENSE', '') - + tinfoil = None + tempdir = None try: with transaction.atomic(): newbranch = False
@@ -323,6 +309,20 @@ def main(options, queue): return (1, '') logger.info("Collecting data for layer %s on branch %s" % (layer.name, branchdesc)) + try: + (tinfoil, tempdir) = recipeparse.init_parser(settings, branch, bitbakepath, nocheckout=options.nocheckout, logger=logger) + except recipeparse.RecipeParseError as e: + logger.error(str(e)) + return(1, '') + # Clear the default value of SUMMARY so that we can use DESCRIPTION instead if it hasn't been set + tinfoil.config_data.setVar('SUMMARY', '') + # Clear the default value of DESCRIPTION so that we can see where it's not set + tinfoil.config_data.setVar('DESCRIPTION', '') + # Clear the default value of HOMEPAGE ('unknown') + tinfoil.config_data.setVar('HOMEPAGE', '') + # Set a blank value for LICENSE so that it doesn't cause the parser to die (e.g. with meta-ti - + # why won't they just fix that?!) + tinfoil.config_data.setVar('LICENSE', '') layerconfparser = layerconfparse.LayerConfParse(logger=logger, tinfoil=tinfoil) layer_config_data = layerconfparser.parse_layer(layerdir)
@@ -687,12 +687,13 @@ def main(options, queue): import traceback traceback.print_exc() finally: - if LooseVersion(bb.__version__) > LooseVersion("1.27"): + if tinfoil and (LooseVersion(bb.__version__) > LooseVersion("1.27")): tinfoil.shutdown() - if options.keep_temp: - logger.debug('Preserving temp directory %s' % tempdir) - else: - logger.debug('Deleting temp directory') - shutil.rmtree(tempdir) + if tempdir: + if options.keep_temp: + logger.debug('Preserving temp directory %s' % tempdir) + else: + logger.debug('Deleting temp directory') + shutil.rmtree(tempdir) return(0, '')
Signature · 1 line
2.7.4