Skip to contentSkip to conversation

populate_sdk_ext: Avoid copying and producing .pyc files

PATCH In review Mark Hatle 2 participants 3 messages Mar 25, 2021 – Mar 26, 2021

lore
Share & tools
lore

Mark Hatle

author
#0 lore raw

PATCH 1/1 populate_sdk_ext: Avoid copying and producing .pyc files

Since pyc cache files are really system specific, no real reason to copy or generate them during the eSDK build process. Also generating them has the possibility of re-using inodes that pseudo may have been tracking, leading a build failure.

---
 meta/classes/populate_sdk_ext.bbclass | 4 +++-
 meta/lib/oe/copy_buildsystem.py       | 6 +++---
 2 files changed, 6 insertions(+), 4 deletions(-)
meta/classes/populate_sdk_ext.bbclass +3 -1
index 9112ab6c5e..14689ec6ac 100644
@@ -251,7 +251,9 @@ python copy_buildsystem () {
 
     # Create a layer for new recipes / appends
     bbpath = d.getVar('BBPATH')
-    bb.process.run(['devtool', '--bbpath', bbpath, '--basepath', baseoutpath, 'create-workspace', '--create-only', os.path.join(baseoutpath, 'workspace')])
+    env = os.environ.copy()
+    env['PYTHONDONTWRITEBYTECODE'] = '1'
+    bb.process.run(['devtool', '--bbpath', bbpath, '--basepath', baseoutpath, 'create-workspace', '--create-only', os.path.join(baseoutpath, 'workspace')], env=env)
 
     # Create bblayers.conf
     bb.utils.mkdirhier(baseoutpath + '/conf')
meta/lib/oe/copy_buildsystem.py +3 -3
index 31a84f5b06..d97bf9d1b9 100644
@@ -20,7 +20,7 @@ def _smart_copy(src, dest):
     mode = os.stat(src).st_mode
     if stat.S_ISDIR(mode):
         bb.utils.mkdirhier(dest)
-        cmd = "tar --exclude='.git' --xattrs --xattrs-include='*' -chf - -C %s -p . \
+        cmd = "tar --exclude='.git' --exclude='__pycache__' --xattrs --xattrs-include='*' -chf - -C %s -p . \
         | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dest)
         subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
     else:
@@ -259,7 +259,7 @@ def create_locked_sstate_cache(lockedsigs, input_sstate_cache, output_sstate_cac
     bb.note('Generating sstate-cache...')
 
     nativelsbstring = d.getVar('NATIVELSBSTRING')
-    bb.process.run("gen-lockedsig-cache %s %s %s %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache, nativelsbstring, filterfile or ''))
+    bb.process.run("PYTHONDONTWRITEBYTECODE=1 gen-lockedsig-cache %s %s %s %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache, nativelsbstring, filterfile or ''))
     if fixedlsbstring and nativelsbstring != fixedlsbstring:
         nativedir = output_sstate_cache + '/' + nativelsbstring
         if os.path.isdir(nativedir):
@@ -286,7 +286,7 @@ def check_sstate_task_list(d, targets, filteroutfile, cmdprefix='', cwd=None, lo
         logparam = '-l %s' % logfile
     else:
         logparam = ''
-    cmd = "%sBB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 oe-check-sstate %s -s -o %s %s" % (cmdprefix, targets, filteroutfile, logparam)
+    cmd = "%sPYTHONDONTWRITEBYTECODE=1 BB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 oe-check-sstate %s -s -o %s %s" % (cmdprefix, targets, filteroutfile, logparam)
     env = dict(d.getVar('BB_ORIGENV', False))
     env.pop('BUILDDIR', '')
     env.pop('BBPATH', '')
Signature · 1 line
2.17.1
#1 lore raw

Wait, I have to say neither reason is convincing. Pyc files are not system-specific, they are the same on all systems. And pseudo issues should be dealt with by fixing the issue, not working around the symptoms, no?

Alex

Quoted 80 lines · Mark Hatle

Since pyc cache files are really system specific, no real reason to copy or generate them during the eSDK build process. Also generating them has the possibility of re-using inodes that pseudo may have been tracking, leading a build failure.

---
 meta/classes/populate_sdk_ext.bbclass | 4 +++-
 meta/lib/oe/copy_buildsystem.py       | 6 +++---
 2 files changed, 6 insertions(+), 4 deletions(-)
meta/classes/populate_sdk_ext.bbclass +3 -1
index 9112ab6c5e..14689ec6ac 100644
@@ -251,7 +251,9 @@ python copy_buildsystem () {
 
     # Create a layer for new recipes / appends
     bbpath = d.getVar('BBPATH')
-    bb.process.run(['devtool', '--bbpath', bbpath, '--basepath',
 baseoutpath, 'create-workspace', '--create-only', os.path.join(baseoutpath,
 'workspace')])
+    env = os.environ.copy()
+    env['PYTHONDONTWRITEBYTECODE'] = '1'
+    bb.process.run(['devtool', '--bbpath', bbpath, '--basepath',
 baseoutpath, 'create-workspace', '--create-only', os.path.join(baseoutpath,

'workspace')], env=env)

# Create bblayers.conf
bb.utils.mkdirhier(baseoutpath + '/conf')

meta/lib/oe/copy_buildsystem.py +1 -3
index 31a84f5b06..d97bf9d1b9 100644
@@ -20,7 +20,7 @@ def _smart_copy(src, dest):
     mode = os.stat(src).st_mode
     if stat.S_ISDIR(mode):
         bb.utils.mkdirhier(dest)
-        cmd = "tar --exclude='.git' --xattrs --xattrs-include='*' -chf -
-C %s -p . \
+        cmd = "tar --exclude='.git' --exclude='__pycache__' --xattrs
--xattrs-include='*' -chf - -C %s -p . \
         | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dest)
         subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
     else:
@@ -259,7 +259,7 @@ def create_locked_sstate_cache(lockedsigs,

input_sstate_cache, output_sstate_cac
bb.note('Generating sstate-cache...')

nativelsbstring = d.getVar('NATIVELSBSTRING')
- bb.process.run("gen-lockedsig-cache %s %s %s %s %s" % (lockedsigs,
input_sstate_cache, output_sstate_cache, nativelsbstring, filterfile or ''))
+ bb.process.run("PYTHONDONTWRITEBYTECODE=1 gen-lockedsig-cache %s %s
%s %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache, nativelsbstring, filterfile or ''))
if fixedlsbstring and nativelsbstring != fixedlsbstring:
nativedir = output_sstate_cache + '/' + nativelsbstring if os.path.isdir(nativedir):

meta/lib/oe/copy_buildsystem.py +0 -0
@@ -286,7 +286,7 @@ def check_sstate_task_list(d, targets, filteroutfile,

cmdprefix='', cwd=None, lo
logparam = '-l %s' % logfile
else:
logparam = ''

meta/lib/oe/copy_buildsystem.py +1 -2
-    cmd = "%sBB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 oe-check-sstate %s
-s -o %s %s" % (cmdprefix, targets, filteroutfile, logparam)
+    cmd = "%sPYTHONDONTWRITEBYTECODE=1 BB_SETSCENE_ENFORCE=1

PSEUDO_DISABLED=1 oe-check-sstate %s -s -o %s %s" % (cmdprefix, targets, filteroutfile, logparam)

     env = dict(d.getVar('BB_ORIGENV', False))
     env.pop('BUILDDIR', '')
     env.pop('BBPATH', '')

--
2.17.1

author
#2 lore raw
Quoted 3 lines · Alexander Kanavin

Wait, I have to say neither reason is convincing. Pyc files are not system-specific, they are the same on all systems. And pseudo issues should be dealt with by fixing the issue, not working around the symptoms, no?

These are pyc files that are generated by the host system python. As such, they contain versioning information of the particular host they were constructed on.

The next user (of the eSDK) would regenerate the pyc files, assuming their python was a different version/configuration anyway.

The pseudo issue is simply caused by the generation of these files outside of the pseudo context. The three places modified need to run without pseudo control in order to setup the bitbake (host) environment properly, as well as verify that the previous files copied/setup are infact correct.

So we're not working around symptoms.. we are indeed fixing the underlying issues in the implementation. It just happened to be that I was able to reproduce the build failure reliably for a time vs it just going away on it's own.

--Mark

Quoted 88 lines

Alex

On Thu, 25 Mar 2021 at 23:44, Mark Hatle <[email protected] <mailto:[email protected]>> wrote:

Since pyc cache files are really system specific, no real reason to copy or generate them during the eSDK build process.  Also generating them has the possibility of re-using inodes that pseudo may have been tracking, leading a build failure.

<mailto:[email protected]>>
---
 meta/classes/populate_sdk_ext.bbclass | 4 +++- meta/lib/oe/copy_buildsystem.py       | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/meta/classes/populate_sdk_ext.bbclass b/meta/classes/populate_sdk_ext.bbclass index 9112ab6c5e..14689ec6ac 100644
--- a/meta/classes/populate_sdk_ext.bbclass +++ b/meta/classes/populate_sdk_ext.bbclass @@ -251,7 +251,9 @@ python copy_buildsystem () {

     # Create a layer for new recipes / appends
     bbpath = d.getVar('BBPATH')
-    bb.process.run(['devtool', '--bbpath', bbpath, '--basepath', baseoutpath, 'create-workspace', '--create-only', os.path.join(baseoutpath, 'workspace')])
+    env = os.environ.copy()
+    env['PYTHONDONTWRITEBYTECODE'] = '1'
+    bb.process.run(['devtool', '--bbpath', bbpath, '--basepath', baseoutpath, 'create-workspace', '--create-only', os.path.join(baseoutpath, 'workspace')], env=env)

     # Create bblayers.conf
     bb.utils.mkdirhier(baseoutpath + '/conf')
diff --git a/meta/lib/oe/copy_buildsystem.py b/meta/lib/oe/copy_buildsystem.py index 31a84f5b06..d97bf9d1b9 100644
--- a/meta/lib/oe/copy_buildsystem.py
+++ b/meta/lib/oe/copy_buildsystem.py
@@ -20,7 +20,7 @@ def _smart_copy(src, dest):
     mode = os.stat(src).st_mode
     if stat.S_ISDIR(mode):
         bb.utils.mkdirhier(dest)
-        cmd = "tar --exclude='.git' --xattrs --xattrs-include='*' -chf - -C %s -p . \
+        cmd = "tar --exclude='.git' --exclude='__pycache__' --xattrs --xattrs-include='*' -chf - -C %s -p . \
         | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dest) subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) else:
@@ -259,7 +259,7 @@ def create_locked_sstate_cache(lockedsigs, input_sstate_cache, output_sstate_cac
     bb.note('Generating sstate-cache...')

     nativelsbstring = d.getVar('NATIVELSBSTRING')
-    bb.process.run("gen-lockedsig-cache %s %s %s %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache, nativelsbstring, filterfile or '')) +    bb.process.run("PYTHONDONTWRITEBYTECODE=1 gen-lockedsig-cache %s %s %s %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache, nativelsbstring, filterfile or ''))
     if fixedlsbstring and nativelsbstring != fixedlsbstring:
         nativedir = output_sstate_cache + '/' + nativelsbstring if os.path.isdir(nativedir):
@@ -286,7 +286,7 @@ def check_sstate_task_list(d, targets, filteroutfile, cmdprefix='', cwd=None, lo
         logparam = '-l %s' % logfile
     else:
         logparam = ''
-    cmd = "%sBB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 oe-check-sstate %s -s -o %s %s" % (cmdprefix, targets, filteroutfile, logparam)
+    cmd = "%sPYTHONDONTWRITEBYTECODE=1 BB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 oe-check-sstate %s -s -o %s %s" % (cmdprefix, targets, filteroutfile, logparam)
     env = dict(d.getVar('BB_ORIGENV', False)) env.pop('BUILDDIR', '')
     env.pop('BBPATH', '')
--
2.17.1