populate_sdk_ext: Avoid copying and producing .pyc files
PATCH 1/1 populate_sdk_ext: Avoid copying and producing .pyc files
Since pyc cache files are really system specific, no real reason to copy or generate them during the eSDK build process. Also generating them has the possibility of re-using inodes that pseudo may have been tracking, leading a build failure.
--- meta/classes/populate_sdk_ext.bbclass | 4 +++- meta/lib/oe/copy_buildsystem.py | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-)
meta/classes/populate_sdk_ext.bbclass +3 -1
@@ -251,7 +251,9 @@ python copy_buildsystem () { # Create a layer for new recipes / appends bbpath = d.getVar('BBPATH') - bb.process.run(['devtool', '--bbpath', bbpath, '--basepath', baseoutpath, 'create-workspace', '--create-only', os.path.join(baseoutpath, 'workspace')]) + env = os.environ.copy() + env['PYTHONDONTWRITEBYTECODE'] = '1' + bb.process.run(['devtool', '--bbpath', bbpath, '--basepath', baseoutpath, 'create-workspace', '--create-only', os.path.join(baseoutpath, 'workspace')], env=env) # Create bblayers.conf bb.utils.mkdirhier(baseoutpath + '/conf')
meta/lib/oe/copy_buildsystem.py +3 -3
@@ -20,7 +20,7 @@ def _smart_copy(src, dest): mode = os.stat(src).st_mode if stat.S_ISDIR(mode): bb.utils.mkdirhier(dest) - cmd = "tar --exclude='.git' --xattrs --xattrs-include='*' -chf - -C %s -p . \ + cmd = "tar --exclude='.git' --exclude='__pycache__' --xattrs --xattrs-include='*' -chf - -C %s -p . \ | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dest) subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) else:
@@ -259,7 +259,7 @@ def create_locked_sstate_cache(lockedsigs, input_sstate_cache, output_sstate_cac bb.note('Generating sstate-cache...') nativelsbstring = d.getVar('NATIVELSBSTRING') - bb.process.run("gen-lockedsig-cache %s %s %s %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache, nativelsbstring, filterfile or '')) + bb.process.run("PYTHONDONTWRITEBYTECODE=1 gen-lockedsig-cache %s %s %s %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache, nativelsbstring, filterfile or '')) if fixedlsbstring and nativelsbstring != fixedlsbstring: nativedir = output_sstate_cache + '/' + nativelsbstring if os.path.isdir(nativedir):
@@ -286,7 +286,7 @@ def check_sstate_task_list(d, targets, filteroutfile, cmdprefix='', cwd=None, lo logparam = '-l %s' % logfile else: logparam = '' - cmd = "%sBB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 oe-check-sstate %s -s -o %s %s" % (cmdprefix, targets, filteroutfile, logparam) + cmd = "%sPYTHONDONTWRITEBYTECODE=1 BB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 oe-check-sstate %s -s -o %s %s" % (cmdprefix, targets, filteroutfile, logparam) env = dict(d.getVar('BB_ORIGENV', False)) env.pop('BUILDDIR', '') env.pop('BBPATH', '')
Signature · 1 line
2.17.1
Wait, I have to say neither reason is convincing. Pyc files are not system-specific, they are the same on all systems. And pseudo issues should be dealt with by fixing the issue, not working around the symptoms, no?
Alex
Quoted 80 lines · Mark Hatle
Since pyc cache files are really system specific, no real reason to copy or generate them during the eSDK build process. Also generating them has the possibility of re-using inodes that pseudo may have been tracking, leading a build failure.
Signed-off-by Mark Hatle <[email protected]>--- meta/classes/populate_sdk_ext.bbclass | 4 +++- meta/lib/oe/copy_buildsystem.py | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-)meta/classes/populate_sdk_ext.bbclass +3
-1@@ -251,7 +251,9 @@ python copy_buildsystem () { # Create a layer for new recipes / appends bbpath = d.getVar('BBPATH') - bb.process.run(['devtool', '--bbpath', bbpath, '--basepath', baseoutpath, 'create-workspace', '--create-only', os.path.join(baseoutpath, 'workspace')]) + env = os.environ.copy() + env['PYTHONDONTWRITEBYTECODE'] = '1' + bb.process.run(['devtool', '--bbpath', bbpath, '--basepath', baseoutpath, 'create-workspace', '--create-only', os.path.join(baseoutpath,'workspace')], env=env)
# Create bblayers.conf
bb.utils.mkdirhier(baseoutpath + '/conf')meta/lib/oe/copy_buildsystem.py +1
-3@@ -20,7 +20,7 @@ def _smart_copy(src, dest): mode = os.stat(src).st_mode if stat.S_ISDIR(mode): bb.utils.mkdirhier(dest) - cmd = "tar --exclude='.git' --xattrs --xattrs-include='*' -chf - -C %s -p . \ + cmd = "tar --exclude='.git' --exclude='__pycache__' --xattrs --xattrs-include='*' -chf - -C %s -p . \ | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dest) subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) else:@@ -259,7 +259,7 @@ def create_locked_sstate_cache(lockedsigs,
input_sstate_cache,output_sstate_cac
bb.note('Generating sstate-cache...')nativelsbstring =
d.getVar('NATIVELSBSTRING')
- bb.process.run("gen-lockedsig-cache %s %s %s %s %s" % (lockedsigs,input_sstate_cache,output_sstate_cache, nativelsbstring, filterfile or ''))
+ bb.process.run("PYTHONDONTWRITEBYTECODE=1 gen-lockedsig-cache %s %s
%s %s %s" % (lockedsigs,input_sstate_cache,output_sstate_cache, nativelsbstring, filterfile or ''))
if fixedlsbstring and nativelsbstring != fixedlsbstring:
nativedir =output_sstate_cache+ '/' + nativelsbstring ifos.path.isdir(nativedir):meta/lib/oe/copy_buildsystem.py +0
-0@@ -286,7 +286,7 @@ def check_sstate_task_list(d, targets, filteroutfile,cmdprefix='', cwd=None, lo
logparam = '-l %s' % logfile
else:
logparam = ''meta/lib/oe/copy_buildsystem.py +1
-2- cmd = "%sBB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 oe-check-sstate %s -s -o %s %s" % (cmdprefix, targets, filteroutfile, logparam) + cmd = "%sPYTHONDONTWRITEBYTECODE=1 BB_SETSCENE_ENFORCE=1PSEUDO_DISABLED=1 oe-check-sstate %s -s -o %s %s" % (cmdprefix, targets, filteroutfile, logparam)
env = dict(d.getVar('BB_ORIGENV', False)) env.pop('BUILDDIR', '') env.pop('BBPATH', '')--
2.17.1
Quoted 3 lines · Alexander Kanavin
Wait, I have to say neither reason is convincing. Pyc files are not system-specific, they are the same on all systems. And pseudo issues should be dealt with by fixing the issue, not working around the symptoms, no?
These are pyc files that are generated by the host system python. As such, they contain versioning information of the particular host they were constructed on.
The next user (of the eSDK) would regenerate the pyc files, assuming their python was a different version/configuration anyway.
The pseudo issue is simply caused by the generation of these files outside of the pseudo context. The three places modified need to run without pseudo control in order to setup the bitbake (host) environment properly, as well as verify that the previous files copied/setup are infact correct.
So we're not working around symptoms.. we are indeed fixing the underlying issues in the implementation. It just happened to be that I was able to reproduce the build failure reliably for a time vs it just going away on it's own.
--Mark
Quoted 88 lines
Alex
On Thu, 25 Mar 2021 at 23:44, Mark Hatle <[email protected] <mailto:[email protected]>> wrote:
Since pyc cache files are really system specific, no real reason to copy or generate them during the eSDK build process. Also generating them has the possibility of re-using inodes that pseudo may have been tracking, leading a build failure.
Signed-off-by Mark Hatle <[email protected]<mailto:[email protected]>>
---
meta/classes/populate_sdk_ext.bbclass | 4 +++- meta/lib/oe/copy_buildsystem.py | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-)diff --git a/meta/classes/populate_sdk_ext.bbclass b/meta/classes/populate_sdk_ext.bbclass index
9112ab6c5e..14689ec6ac100644
--- a/meta/classes/populate_sdk_ext.bbclass +++ b/meta/classes/populate_sdk_ext.bbclass @@ -251,7 +251,9 @@ pythoncopy_buildsystem() {# Create a layer for new recipes / appends
bbpath =d.getVar('BBPATH')
- bb.process.run(['devtool', '--bbpath', bbpath, '--basepath', baseoutpath, 'create-workspace', '--create-only',os.path.join(baseoutpath, 'workspace')])
+ env =os.environ.copy()
+ env['PYTHONDONTWRITEBYTECODE'] = '1'
+ bb.process.run(['devtool', '--bbpath', bbpath, '--basepath', baseoutpath, 'create-workspace', '--create-only',os.path.join(baseoutpath, 'workspace')], env=env)# Create bblayers.conf
bb.utils.mkdirhier(baseoutpath + '/conf')
diff --gita/meta/lib/oe/copy_buildsystem.pyb/meta/lib/oe/copy_buildsystem.pyindex31a84f5b06..d97bf9d1b9100644
---a/meta/lib/oe/copy_buildsystem.py
+++b/meta/lib/oe/copy_buildsystem.py
@@ -20,7 +20,7 @@ def_smart_copy(src, dest):
mode = os.stat(src).st_mode
ifstat.S_ISDIR(mode):
bb.utils.mkdirhier(dest)
- cmd = "tar --exclude='.git' --xattrs --xattrs-include='*' -chf - -C %s -p . \
+ cmd = "tar --exclude='.git' --exclude='__pycache__' --xattrs --xattrs-include='*' -chf - -C %s -p . \
| tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dest)subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)else:
@@ -259,7 +259,7 @@ def create_locked_sstate_cache(lockedsigs,input_sstate_cache,output_sstate_cac
bb.note('Generating sstate-cache...')nativelsbstring =
d.getVar('NATIVELSBSTRING')
- bb.process.run("gen-lockedsig-cache %s %s %s %s %s" % (lockedsigs,input_sstate_cache,output_sstate_cache, nativelsbstring, filterfile or '')) + bb.process.run("PYTHONDONTWRITEBYTECODE=1 gen-lockedsig-cache %s %s %s %s %s" % (lockedsigs,input_sstate_cache,output_sstate_cache, nativelsbstring, filterfile or ''))
if fixedlsbstring and nativelsbstring != fixedlsbstring:
nativedir =output_sstate_cache+ '/' + nativelsbstring ifos.path.isdir(nativedir):
@@ -286,7 +286,7 @@ def check_sstate_task_list(d, targets, filteroutfile, cmdprefix='', cwd=None, lo
logparam = '-l %s' % logfile
else:
logparam = ''
- cmd = "%sBB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 oe-check-sstate %s -s -o %s %s" % (cmdprefix, targets, filteroutfile, logparam)
+ cmd = "%sPYTHONDONTWRITEBYTECODE=1 BB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 oe-check-sstate %s -s -o %s %s" % (cmdprefix, targets, filteroutfile, logparam)
env =dict(d.getVar('BB_ORIGENV', False))env.pop('BUILDDIR', '')
env.pop('BBPATH', '')
--
2.17.1