Received: from kernel.crashing.org (kernel.crashing.org [76.164.61.194]) by mx.groups.io with SMTP id smtpd.web10.13521.1616769019478576482 for ; Fri, 26 Mar 2021 07:30:19 -0700 Authentication-Results: mx.groups.io; dkim=missing; spf=permerror, err=syntax error for token: (domain: kernel.crashing.org, ip: 76.164.61.194, mailfrom: mark.hatle@kernel.crashing.org) Received: from Marks-MacBook-Pro-16.local ([76.164.61.198]) (authenticated bits=0) by kernel.crashing.org (8.14.7/8.14.7) with ESMTP id 12QEUDfJ009701 (version=TLSv1/SSLv3 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128 verify=NO); Fri, 26 Mar 2021 09:30:14 -0500 Subject: Re: [OE-core] [PATCH] populate_sdk_ext: Avoid copying and producing .pyc files To: Alexander Kanavin Cc: OE-core References: <20210325224402.145099-1-mark.hatle@kernel.crashing.org> From: "Mark Hatle" Message-ID: <4ea9e38d-d62d-2898-363b-b68d675e3190@kernel.crashing.org> Date: Fri, 26 Mar 2021 09:30:13 -0500 User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:78.0) Gecko/20100101 Thunderbird/78.8.1 MIME-Version: 1.0 In-Reply-To: X-MIME-Autoconverted: from 8bit to quoted-printable by kernel.crashing.org id 12QEUDfJ009701 Content-Type: text/plain; charset=utf-8 Content-Language: en-US Content-Transfer-Encoding: quoted-printable On 3/26/21 12:54 AM, Alexander Kanavin wrote: > Wait, I have to say neither reason is convincing. Pyc files are not > system-specific, they are the same on all systems. And pseudo issues sho= uld be > dealt with by fixing the issue, not working around the symptoms, no? These are pyc files that are generated by the host system python. As such= , they contain versioning information of the particular host they were constructe= d on. The next user (of the eSDK) would regenerate the pyc files, assuming their python was a different version/configuration anyway. The pseudo issue is simply caused by the generation of these files outside= of the pseudo context. The three places modified need to run without pseudo control in order to setup the bitbake (host) environment properly, as well= as verify that the previous files copied/setup are infact correct. So we're not working around symptoms.. we are indeed fixing the underlying issues in the implementation. It just happened to be that I was able to reproduce the build failure reliably for a time vs it just going away on i= t's own. --Mark > Alex >=20 > On Thu, 25 Mar 2021 at 23:44, Mark Hatle > wrote: >=20 > Since pyc cache files are really system specific, no real reason to = copy or > generate them during the eSDK build process.=C2=A0 Also generating t= hem has the > possibility of re-using inodes that pseudo may have been tracking, l= eading > a build failure. >=20 > Signed-off-by: Mark Hatle > > --- > =C2=A0meta/classes/populate_sdk_ext.bbclass | 4 +++- > =C2=A0meta/lib/oe/copy_buildsystem.py=C2=A0 =C2=A0 =C2=A0 =C2=A0| 6 = +++--- > =C2=A02 files changed, 6 insertions(+), 4 deletions(-) >=20 > diff --git a/meta/classes/populate_sdk_ext.bbclass > b/meta/classes/populate_sdk_ext.bbclass > index 9112ab6c5e..14689ec6ac 100644 > --- a/meta/classes/populate_sdk_ext.bbclass > +++ b/meta/classes/populate_sdk_ext.bbclass > @@ -251,7 +251,9 @@ python copy_buildsystem () { >=20 > =C2=A0 =C2=A0 =C2=A0# Create a layer for new recipes / appends > =C2=A0 =C2=A0 =C2=A0bbpath =3D d.getVar('BBPATH') > -=C2=A0 =C2=A0 bb.process.run(['devtool', '--bbpath', bbpath, '--bas= epath', > baseoutpath, 'create-workspace', '--create-only', os.path.join(baseo= utpath, > 'workspace')]) > +=C2=A0 =C2=A0 env =3D os.environ.copy() > +=C2=A0 =C2=A0 env['PYTHONDONTWRITEBYTECODE'] =3D '1' > +=C2=A0 =C2=A0 bb.process.run(['devtool', '--bbpath', bbpath, '--bas= epath', > baseoutpath, 'create-workspace', '--create-only', os.path.join(baseo= utpath, > 'workspace')], env=3Denv) >=20 > =C2=A0 =C2=A0 =C2=A0# Create bblayers.conf > =C2=A0 =C2=A0 =C2=A0bb.utils.mkdirhier(baseoutpath + '/conf') > diff --git a/meta/lib/oe/copy_buildsystem.py b/meta/lib/oe/copy_buil= dsystem.py > index 31a84f5b06..d97bf9d1b9 100644 > --- a/meta/lib/oe/copy_buildsystem.py > +++ b/meta/lib/oe/copy_buildsystem.py > @@ -20,7 +20,7 @@ def _smart_copy(src, dest): > =C2=A0 =C2=A0 =C2=A0mode =3D os.stat(src).st_mode > =C2=A0 =C2=A0 =C2=A0if stat.S_ISDIR(mode): > =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0bb.utils.mkdirhier(dest) > -=C2=A0 =C2=A0 =C2=A0 =C2=A0 cmd =3D "tar --exclude=3D'.git' --xattr= s --xattrs-include=3D'*' -chf - -C > %s -p . \ > +=C2=A0 =C2=A0 =C2=A0 =C2=A0 cmd =3D "tar --exclude=3D'.git' --exclu= de=3D'__pycache__' --xattrs > --xattrs-include=3D'*' -chf - -C %s -p . \ > =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0| tar --xattrs --xattrs-include=3D= '*' -xf - -C %s" % (src, dest) > =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0subprocess.check_output(cmd, shell= = =3DTrue, stderr=3Dsubprocess.STDOUT) > =C2=A0 =C2=A0 =C2=A0else: > @@ -259,7 +259,7 @@ def create_locked_sstate_cache(lockedsigs, > input_sstate_cache, output_sstate_cac > =C2=A0 =C2=A0 =C2=A0bb.note('Generating sstate-cache...') >=20 > =C2=A0 =C2=A0 =C2=A0nativelsbstring =3D d.getVar('NATIVELSBSTRING') > -=C2=A0 =C2=A0 bb.process.run("gen-lockedsig-cache %s %s %s %s %s" %= (lockedsigs, > input_sstate_cache, output_sstate_cache, nativelsbstring, filterfile= or '')) > +=C2=A0 =C2=A0 bb.process.run("PYTHONDONTWRITEBYTECODE=3D1 gen-locke= dsig-cache %s %s %s > %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache, > nativelsbstring, filterfile or '')) > =C2=A0 =C2=A0 =C2=A0if fixedlsbstring and nativelsbstring !=3D fixed= lsbstring: > =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0nativedir =3D output_sstate_cache = + '/' + nativelsbstring > =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0if os.path.isdir(nativedir): > @@ -286,7 +286,7 @@ def check_sstate_task_list(d, targets, filterout= file, > cmdprefix=3D'', cwd=3DNone, lo > =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0logparam =3D '-l %s' % logfile > =C2=A0 =C2=A0 =C2=A0else: > =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0logparam =3D '' > -=C2=A0 =C2=A0 cmd =3D "%sBB_SETSCENE_ENFORCE=3D1 PSEUDO_DISABLED=3D= 1 oe-check-sstate %s -s > -o %s %s" % (cmdprefix, targets, filteroutfile, logparam) > +=C2=A0 =C2=A0 cmd =3D "%sPYTHONDONTWRITEBYTECODE=3D1 BB_SETSCENE_EN= FORCE=3D1 > PSEUDO_DISABLED=3D1 oe-check-sstate %s -s -o %s %s" % (cmdprefix, ta= rgets, > filteroutfile, logparam) > =C2=A0 =C2=A0 =C2=A0env =3D dict(d.getVar('BB_ORIGENV', False)) > =C2=A0 =C2=A0 =C2=A0env.pop('BUILDDIR', '') > =C2=A0 =C2=A0 =C2=A0env.pop('BBPATH', '') > --=20 > 2.17.1 >=20 >=20 >=20 >=20 >=20 >=20 >=20 >=20