Support symbolic links in paths in PSEUDO_IGNORE_PATHS
v3 cover Support symbolic links in paths in PSEUDO_IGNORE_PATHS
The changes in this patch series aim to rectify a problem with pseudo and its support for ignoring paths. When pseudo compares a path to the paths specified in PSEUDO_IGNORE_PATHS, it uses a path that has been canonicalized. However, it does not canonicalize the paths in PSEUDO_IGNORE_PATHS itself and unless they are canonicalized, they will not match as intended. Thus the paths in PSEUDO_IGNORE_PATHS needs to be canonicalized. These patches do that by adding a new function oe.path.canonicalize(), which is then used when passing ${PSEUDO_IGNORE_PATHS} to pseudo.
There is also one patch that adds two patches to pseudo to clean up pseudo_client_ignore_path_chroot(), and they also plug a memory leak. The patches were brought about as I initially intended to do the canonicalization in pseudo itself in this function.
I have not tested the change to wic as we do not use it, though I do not expect it to be problematic.
PATCHv2:
* Removed some unrelated changes in the second patch.
PATCHv3:
* Handle an edge case in pseudo_client_ignore_path_chroot() in case PSEUDO_IGNORE_PATHS contains ",," or leading/trailing commas.
* Introduce oe.path.canonicalize() and use it instead of adding PSEUDO_IGNORE_REAL_PATHS as was previously done.
* Add similar code to wic.
//Peter
The following changes since commit 0ac99625bf2c8a19d4ee746dc13e9c6b2e1956e2:
python3-numpy: Add ptest (2020-11-30 14:42:22 +0000)
are available in the Git repository at:
git://push.yoctoproject.org/poky-contrib pkj/pseudo-ignore
Peter Kjellerstedt (5):
pseudo: Simplify pseudo_client_ignore_path_chroot() bitbake.conf: Add all layers (from BBLAYERS) to PSEUDO_IGNORE_PATHS lib/oe/path: Add canonicalize()
bitbake.conf: Canonicalize paths in PSEUDO_IGNORE_PATHS
wic: Pass canonicalized paths in PSEUDO_IGNORE_PATHS
meta/conf/bitbake.conf | 6 +- meta/lib/oe/path.py | 12 ++++ ...ssen-indentation-of-pseudo_client_ig.patch | 69 +++++++++++++++++++ ...mplify-pseudo_client_ignore_path_chr.patch | 50 ++++++++++++++ meta/recipes-devtools/pseudo/pseudo_git.bb | 2 + scripts/lib/wic/partition.py | 6 +- 6 files changed, 141 insertions(+), 4 deletions(-) create mode 100644 meta/recipes-devtools/pseudo/files/0002-pseudo_client-Lessen-indentation-of-pseudo_client_ig.patch create mode 100644 meta/recipes-devtools/pseudo/files/0003-pseudo_client-Simplify-pseudo_client_ignore_path_chr.patch
PATCH v3 1/5 pseudo: Simplify pseudo_client_ignore_path_chroot()
This also plugs a memory leak in pseudo_client_ignore_path_chroot().
--- ...ssen-indentation-of-pseudo_client_ig.patch | 69 +++++++++++++++++++ ...mplify-pseudo_client_ignore_path_chr.patch | 50 ++++++++++++++ meta/recipes-devtools/pseudo/pseudo_git.bb | 2 + 3 files changed, 121 insertions(+) create mode 100644 meta/recipes-devtools/pseudo/files/0002-pseudo_client-Lessen-indentation-of-pseudo_client_ig.patch create mode 100644 meta/recipes-devtools/pseudo/files/0003-pseudo_client-Simplify-pseudo_client_ignore_path_chr.patch
meta/recipes-devtools/pseudo/files/0002-pseudo_client-Lessen-indentation-of-pseudo_client_ig.patch new file +69 -0
@@ -0,0 +1,69 @@ +From 28c760542eecd7c5b35ea88aa2b14d62afbda961 Mon Sep 17 00:00:00 2001 +From: Peter Kjellerstedt <[email protected]> +Date: Sat, 21 Nov 2020 17:22:38 +0100 +Subject: [PATCH] pseudo_client: Lessen indentation of + pseudo_client_ignore_path_chroot() + +Change-Id: I739b18efb7a95ce2d2d907d0faf7df539ab9af1c +--- + pseudo_client.c | 45 +++++++++++++++++++++++++-------------------- + 1 file changed, 25 insertions(+), 20 deletions(-) + +diff --git a/pseudo_client.c b/pseudo_client.c +index 116d926..a8bc3dc 100644 +--- a/pseudo_client.c ++++ b/pseudo_client.c +@@ -1527,28 +1527,33 @@ int pseudo_client_ignore_fd(int fd) { + + int pseudo_client_ignore_path_chroot(const char *path, int ignore_chroot) { + char *env; +- if (path) { +- if (ignore_chroot && pseudo_chroot && strncmp(path, pseudo_chroot, pseudo_chroot_len) == 0) +- return 0; +- env = pseudo_get_value("PSEUDO_IGNORE_PATHS"); +- if (env) { +- char *p = env; +- while (*p) { +- char *next = strchr(p, ','); +- if (!next) +- next = strchr(p, '\0'); +- if ((next - p) && !strncmp(path, p, next - p)) { +- pseudo_debug(PDBGF_PATH | PDBGF_VERBOSE, "ignoring path: '%s'\n", path); +- return 1; +- } +- if (next && *next != '\0') +- p = next+1; +- else +- break; +- } +- free(env); ++ ++ if (!path) ++ return 0; ++ ++ if (ignore_chroot && pseudo_chroot && strncmp(path, pseudo_chroot, pseudo_chroot_len) == 0) ++ return 0; ++ ++ env = pseudo_get_value("PSEUDO_IGNORE_PATHS"); ++ if (!env) ++ return 0; ++ ++ char *p = env; ++ while (*p) { ++ char *next = strchr(p, ','); ++ if (!next) ++ next = strchr(p, '\0'); ++ if ((next - p) && !strncmp(path, p, next - p)) { ++ pseudo_debug(PDBGF_PATH | PDBGF_VERBOSE, "ignoring path: '%s'\n", path); ++ return 1; + } ++ if (next && *next != '\0') ++ p = next+1; ++ else ++ break; + } ++ free(env); ++ + return 0; + } +
meta/recipes-devtools/pseudo/files/0003-pseudo_client-Simplify-pseudo_client_ignore_path_chr.patch new file +50 -0
50 lines not on this page · raw message
meta/recipes-devtools/pseudo/pseudo_git.bb +2 -0
@@ -4,6 +4,8 @@ SRC_URI = "git://git.yoctoproject.org/pseudo;branch=oe-core \ file://0001-configure-Prune-PIE-flags.patch \ file://fallback-passwd \ file://fallback-group \ + file://0002-pseudo_client-Lessen-indentation-of-pseudo_client_ig.patch \ + file://0003-pseudo_client-Simplify-pseudo_client_ignore_path_chr.patch \ " SRCREV = "cca0d7f15b7197095cd587420d31b187620c3093"
PATCH v3 2/5 bitbake.conf: Add all layers (from BBLAYERS) to PSEUDO_IGNORE_PATHS
In reply to Peter KjellerstedtInstead of ignoring ${COREBASE}/meta in PSEUDO_IGNORE_PATHS (which may or may not ignore all layers depending on how they are named and placed under ${COREBASE}), ignore all layers.
--- meta/conf/bitbake.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
meta/conf/bitbake.conf +1 -1
@@ -685,7 +685,7 @@ SRC_URI = "" PSEUDO_LOCALSTATEDIR ?= "${WORKDIR}/pseudo/" PSEUDO_PASSWD ?= "${STAGING_DIR_TARGET}:${PSEUDO_SYSROOT}" PSEUDO_SYSROOT = "${COMPONENTS_DIR}/${BUILD_ARCH}/pseudo-native" -PSEUDO_IGNORE_PATHS = "/usr/,/etc/,/lib,/dev/,${T},${WORKDIR}/recipe-sysroot,${SSTATE_DIR},${STAMPS_DIR},${WORKDIR}/pkgdata-sysroot,${TMPDIR}/sstate-control,${DEPLOY_DIR},${WORKDIR}/deploy-,${TMPDIR}/buildstats,${WORKDIR}/sstate-build-package_,${WORKDIR}/sstate-install-package_,${WORKDIR}/sstate-build-image_complete,${TMPDIR}/sysroots-components,${BUILDHISTORY_DIR},${TMPDIR}/pkgdata,${TOPDIR}/cache,${COREBASE}/scripts,${COREBASE}/meta,${CCACHE_DIR}" +PSEUDO_IGNORE_PATHS = "/usr/,/etc/,/lib,/dev/,${T},${WORKDIR}/recipe-sysroot,${SSTATE_DIR},${STAMPS_DIR},${WORKDIR}/pkgdata-sysroot,${TMPDIR}/sstate-control,${DEPLOY_DIR},${WORKDIR}/deploy-,${TMPDIR}/buildstats,${WORKDIR}/sstate-build-package_,${WORKDIR}/sstate-install-package_,${WORKDIR}/sstate-build-image_complete,${TMPDIR}/sysroots-components,${BUILDHISTORY_DIR},${TMPDIR}/pkgdata,${TOPDIR}/cache,${COREBASE}/scripts,${@','.join(d.getVar('BBLAYERS').split())},${CCACHE_DIR}" export PSEUDO_DISABLED = "1" #export PSEUDO_PREFIX = "${STAGING_DIR_NATIVE}${prefix_native}"
Re: [OE-core] [PATCHv3 2/5] bitbake.conf: Add all layers (from BBLAYERS) to PSEUDO_IGNORE_PATHS
Quoted 15 lines · Peter Kjellerstedt
Instead of ignoring ${COREBASE}/meta in
PSEUDO_IGNORE_PATHS(which may or may not ignore all layers depending on how they are named and placed under ${COREBASE}), ignore all layers.Signed-off-by Peter Kjellerstedt <[email protected]>--- meta/conf/bitbake.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)meta/conf/bitbake.conf +0
-0
meta/conf/bitbake.conf SRC_URI = "" · line 688
Quoted 5 lines
meta/conf/bitbake.conf +1
-1@@ -685,7 +685,7 @@ SRC_URI = "" PSEUDO_LOCALSTATEDIR ?= "${WORKDIR}/pseudo/" PSEUDO_PASSWD ?= "${STAGING_DIR_TARGET}:${PSEUDO_SYSROOT}" PSEUDO_SYSROOT = "${COMPONENTS_DIR}/${BUILD_ARCH}/pseudo-native" -PSEUDO_IGNORE_PATHS = "/usr/,/etc/,/lib,/dev/,${T},${WORKDIR}/recipe-sysroot,${SSTATE_DIR},${STAMPS_DIR},${WORKDIR}/pkgdata-sysroot,${TMPDIR}/sstate-control,${DEPLOY_DIR},${WORKDIR}/deploy-,${TMPDIR}/buildstats,${WORKDIR}/sstate-build-package_,${WORKDIR}/sstate-install-package_,${WORKDIR}/sstate-build-image_complete,${TMPDIR}/sysroots-components,${BUILDHISTORY_DIR},${TMPDIR}/pkgdata,${TOPDIR}/cache,${COREBASE}/scripts,${COREBASE}/meta,${CCACHE_DIR}" +PSEUDO_IGNORE_PATHS = "/usr/,/etc/,/lib,/dev/,${T},${WORKDIR}/recipe-sysroot,${SSTATE_DIR},${STAMPS_DIR},${WORKDIR}/pkgdata-sysroot,${TMPDIR}/sstate-control,${DEPLOY_DIR},${WORKDIR}/deploy-,${TMPDIR}/buildstats,${WORKDIR}/sstate-build-package_,${WORKDIR}/sstate-install-package_,${WORKDIR}/sstate-build-image_complete,${TMPDIR}/sysroots-components,${BUILDHISTORY_DIR},${TMPDIR}/pkgdata,${TOPDIR}/cache,${COREBASE}/scripts,${@','.join(d.getVar('BBLAYERS').split())},${CCACHE_DIR}"
This has now landed in master and unfortunately it breaks the way I run builds. I often end up building inside a layer directory. For example the one that failed this morning has:
BBLAYERS ?= " \
/home/pbarker/Projects/Yocto/meta-linux-mainline \
/home/pbarker/Projects/Yocto/meta-linux-mainline/layers/poky/meta \
/home/pbarker/Projects/Yocto/meta-linux-mainline/layers/poky/meta-poky"TOPDIR is "/home/pbarker/Projects/Yocto/meta-linux-mainline/build" and TMPDIR is "${TOPDIR}/tmp". So everything under TMPDIR is now ignored by pseudo and various do_install/do_package tasks fail.
I don't think this will just affect me as this is a common pattern, especially when using something like kas to setup builds.
Before we revert this I'd like to understand the problem this patch is trying to fix. Is there a specific reason why you need to ignore all layer directories? I'm not 100% sure why "${COREBASE}/meta" is ignored anyway though I guess it is to reduce pseudo db size and so speed up builds.
Thanks,
Signature · 2 lines
Paul Barker Konsulko Group
Re: [OE-core] [PATCHv3 2/5] bitbake.conf: Add all layers (from BBLAYERS) to PSEUDO_IGNORE_PATHS
Quoted 37 lines · Paul Barker
Quoted 15 lines · Peter Kjellerstedt
Instead of ignoring ${COREBASE}/meta in
PSEUDO_IGNORE_PATHS(which may or may not ignore all layers depending on how they are named and placed under ${COREBASE}), ignore all layers.Signed-off-by Peter Kjellerstedt <[email protected]>--- meta/conf/bitbake.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)meta/conf/bitbake.conf +0
-0meta/conf/bitbake.conf SRC_URI = "" · line 688
Quoted 5 lines
meta/conf/bitbake.conf +1
-1@@ -685,7 +685,7 @@ SRC_URI = "" PSEUDO_LOCALSTATEDIR ?= "${WORKDIR}/pseudo/" PSEUDO_PASSWD ?= "${STAGING_DIR_TARGET}:${PSEUDO_SYSROOT}" PSEUDO_SYSROOT = "${COMPONENTS_DIR}/${BUILD_ARCH}/pseudo-native" -PSEUDO_IGNORE_PATHS = "/usr/,/etc/,/lib,/dev/,${T},${WORKDIR}/recipe-sysroot,${SSTATE_DIR},${STAMPS_DIR},${WORKDIR}/pkgdata-sysroot,${TMPDIR}/sstate-control,${DEPLOY_DIR},${WORKDIR}/deploy-,${TMPDIR}/buildstats,${WORKDIR}/sstate-build-package_,${WORKDIR}/sstate-install-package_,${WORKDIR}/sstate-build-image_complete,${TMPDIR}/sysroots-components,${BUILDHISTORY_DIR},${TMPDIR}/pkgdata,${TOPDIR}/cache,${COREBASE}/scripts,${COREBASE}/meta,${CCACHE_DIR}" +PSEUDO_IGNORE_PATHS = "/usr/,/etc/,/lib,/dev/,${T},${WORKDIR}/recipe-sysroot,${SSTATE_DIR},${STAMPS_DIR},${WORKDIR}/pkgdata-sysroot,${TMPDIR}/sstate-control,${DEPLOY_DIR},${WORKDIR}/deploy-,${TMPDIR}/buildstats,${WORKDIR}/sstate-build-package_,${WORKDIR}/sstate-install-package_,${WORKDIR}/sstate-build-image_complete,${TMPDIR}/sysroots-components,${BUILDHISTORY_DIR},${TMPDIR}/pkgdata,${TOPDIR}/cache,${COREBASE}/scripts,${@','.join(d.getVar('BBLAYERS').split())},${CCACHE_DIR}"This has now landed in master and unfortunately it breaks the way I run builds. I often end up building inside a layer directory. For example the one that failed this morning has:
BBLAYERS ?= " \ /home/pbarker/Projects/Yocto/meta-linux-mainline \ /home/pbarker/Projects/Yocto/meta-linux-mainline/layers/poky/meta \ /home/pbarker/Projects/Yocto/meta-linux-mainline/layers/poky/meta-poky"TOPDIR is "
/home/pbarker/Projects/Yocto/meta-linux-mainline/build" and TMPDIR is "${TOPDIR}/tmp". So everything under TMPDIR is now ignored by pseudo and various do_install/do_package tasks fail.
Quoted 8 lines
I don't think this will just affect me as this is a common pattern, especially when using something like kas to setup builds.
Before we revert this I'd like to understand the problem this patch is trying to fix. Is there a specific reason why you need to ignore all layer directories? I'm not 100% sure why "${COREBASE}/meta" is ignored anyway though I guess it is to reduce pseudo db size and so speed up builds.
To follow up some more: The entries in PSEUDO_IGNORE_PATHS are treated as string prefixes within pseudo. So if "/home/pbarker/Projects/Yocto/meta-linux-mainline" is added to the ignore list it will exclude not just "/home/pbarker/Projects/Yocto/meta-linux-mainline/build" but also ""/home/pbarker/Projects/Yocto/meta-linux-mainline-build".
I wonder if some more of those entries should have trailing slashes.
Signature · 2 lines
Paul Barker Konsulko Group
Re: [OE-core] [PATCHv3 2/5] bitbake.conf: Add all layers (from BBLAYERS) to PSEUDO_IGNORE_PATHS
Quoted 11 lines · Paul Barker
On Mon, 7 Dec 2020 at 10:29, Paul Barker <[email protected]> wrote:
To follow up some more: The entries inPSEUDO_IGNORE_PATHSare treated
as string prefixes within pseudo. So if "/home/pbarker/Projects/Yocto/meta-linux-mainline" is added to the ignore list it will exclude not just "/home/pbarker/Projects/Yocto/meta-linux-mainline/build" but also ""/home/pbarker/Projects/Yocto/meta-linux-mainline-build".I wonder if some more of those entries should have trailing slashes.
In most (all?) cases it was very deliberate FWIW...
Cheers,
Richard
Re: [OE-core] [PATCHv3 2/5] bitbake.conf: Add all layers (from BBLAYERS) to PSEUDO_IGNORE_PATHS
Quoted 12 lines · Richard Purdie
Quoted 10 lines · Paul Barker
On Mon, 7 Dec 2020 at 10:29, Paul Barker <[email protected]> wrote:
To follow up some more: The entries inPSEUDO_IGNORE_PATHSare treated
as string prefixes within pseudo. So if "/home/pbarker/Projects/Yocto/meta-linux-mainline" is added to the ignore list it will exclude not just "/home/pbarker/Projects/Yocto/meta-linux-mainline/build" but also ""/home/pbarker/Projects/Yocto/meta-linux-mainline-build".
Quoted 3 lines
Quoted 1 line
I wonder if some more of those entries should have trailing slashes.
In most (all?) cases it was very deliberate FWIW...
That does make sense. Ignoring "${COREBASE}/meta" will also ignore most layers unpacked or cloned within the poky directory as their names start with "meta". However that does miss layers if you use a different directory structure which is what Peter's patch addresses (though I'm still not sure if there's an actual build failure with some layers which it is intended to fix or if it's just to make things consistent).
The issue comes when you clone a layer as the top-level of your working tree and build within that. That's how I work with meta-sancloud & meta-linux-mainline. It's also what happens if you build using the kas config in meta-raspberrypi. So it's not uncommon.
Investigating why the layer directories are being ignored I found this commit added the ignore of "${COREBASE}/meta":
commit e0cb6dd689a362d8433caa14cc5a9fdd5eb44923
Author: Richard Purdie <[email protected]>
Date: Wed Oct 7 23:08:45 2020 +0100
bitbake.conf: Extend PSEUDO_IGNORE_PATHS to ${COREBASE}/meta
Unfortunately, .pyc files can be generated in meta/lib/oewhich corrupt the pseudo
database so we need to extend the ignore list to cover this as well.
Could we instead ignore "${LAYERDIR}/lib" for each layer?
An alternative would be to detect the case where TOPDIR or TMPDIR is beneath an entry in BBLAYERS and handle that as a special case.
Thanks,
Signature · 2 lines
Paul Barker Konsulko Group
Re: [OE-core] [PATCHv3 2/5] bitbake.conf: Add all layers (from BBLAYERS) to PSEUDO_IGNORE_PATHS
Quoted 48 lines · Paul Barker
Quoted 12 lines · Richard Purdie
Quoted 11 lines · Paul Barker
On Mon, 7 Dec 2020 at 10:29, Paul Barker <[email protected]> wrote:
To follow up some more: The entries inPSEUDO_IGNORE_PATHSare treated
as string prefixes within pseudo. So if "/home/pbarker/Projects/Yocto/meta-linux-mainline" is added to the
ignore list it will exclude not just "/home/pbarker/Projects/Yocto/meta-linux-mainline/build" but also ""/home/pbarker/Projects/Yocto/meta-linux-mainline-build".Quoted 4 lines
Quoted 2 lines
I wonder if some more of those entries should have trailing slashes.
In most (all?) cases it was very deliberate FWIW...
That does make sense. Ignoring "${COREBASE}/meta" will also ignore most layers unpacked or cloned within the poky directory as their names start with "meta". However that does miss layers if you use a different directory structure which is what Peter's patch addresses (though I'm still not sure if there's an actual build failure with some layers which it is intended to fix or if it's just to make things
consistent).The issue comes when you clone a layer as the top-level of your working tree and build within that. That's how I work with meta-sancloud & meta-linux-mainline. It's also what happens if you build using the kas config in meta-raspberrypi. So it's not uncommon.
Investigating why the layer directories are being ignored I found this
commit added the ignore of "${COREBASE}/meta":commit e0cb6dd689a362d8433caa14cc5a9fdd5eb44923 Author: Richard Purdie <[email protected]> Date: Wed Oct 7 23:08:45 2020 +0100 bitbake.conf: Extend PSEUDO_IGNORE_PATHS to ${COREBASE}/meta Unfortunately, .pyc files can be generated in meta/lib/oewhich corrupt the pseudo
database so we need to extend the ignore list to cover this
as well.
Quoted 7 lines
Signed-off-by: Richard Purdie <
[email protected]>Could we instead ignore "${LAYERDIR}/lib" for each layer?
An alternative would be to detect the case where TOPDIR or TMPDIR is beneath an entry in BBLAYERS and handle that as a special case.
I think we'll have to look at alternatives, yes. Thanks for reminding me this was for the pyc files, I'd forgotten the exact reason.
The reason I added meta was because at least in core we have multiple python lib directories (there are also things in scripts/).
Peter: Which files were you having a problem with in your layers?
We may need to go back to requiring layers to set this up correctly themselves where needed?
Cheers,
Richard
Re: [OE-core] [PATCHv3 2/5] bitbake.conf: Add all layers (from BBLAYERS) to PSEUDO_IGNORE_PATHS
In reply to Paul BarkerQuoted 48 lines · Paul Barker
Quoted 12 lines · Richard Purdie
Quoted 11 lines · Paul Barker
On Mon, 7 Dec 2020 at 10:29, Paul Barker <[email protected]> wrote:
To follow up some more: The entries inPSEUDO_IGNORE_PATHSare treated
as string prefixes within pseudo. So if "/home/pbarker/Projects/Yocto/meta-linux-mainline" is added to the
ignore list it will exclude not just "/home/pbarker/Projects/Yocto/meta-linux-mainline/build" but also ""/home/pbarker/Projects/Yocto/meta-linux-mainline-build".Quoted 4 lines
Quoted 2 lines
I wonder if some more of those entries should have trailing slashes.
In most (all?) cases it was very deliberate FWIW...
That does make sense. Ignoring "${COREBASE}/meta" will also ignore most layers unpacked or cloned within the poky directory as their names start with "meta". However that does miss layers if you use a different directory structure which is what Peter's patch addresses (though I'm still not sure if there's an actual build failure with some layers which it is intended to fix or if it's just to make things
consistent).The issue comes when you clone a layer as the top-level of your working tree and build within that. That's how I work with meta-sancloud & meta-linux-mainline. It's also what happens if you build using the kas config in meta-raspberrypi. So it's not uncommon.
Investigating why the layer directories are being ignored I found this
commit added the ignore of "${COREBASE}/meta":commit e0cb6dd689a362d8433caa14cc5a9fdd5eb44923 Author: Richard Purdie <[email protected]> Date: Wed Oct 7 23:08:45 2020 +0100 bitbake.conf: Extend PSEUDO_IGNORE_PATHS to ${COREBASE}/meta Unfortunately, .pyc files can be generated in meta/lib/oewhich corrupt the pseudo
database so we need to extend the ignore list to cover this
as well.
Quoted 7 lines
Signed-off-by: Richard Purdie <
[email protected]>Could we instead ignore "${LAYERDIR}/lib" for each layer?
An alternative would be to detect the case where TOPDIR or TMPDIR is beneath an entry in BBLAYERS and handle that as a special case.
I'm wondering if we should just set:
sys.dont_write_bytecode=True
since the number of places we'd get significant speed from this is limited. We probably only need to do this in any pseudo using code paths?
Cheers,
Richard
PATCH v3 3/5 lib/oe/path: Add canonicalize()
In reply to Peter Kjellerstedtoe.path.canonicalize() is used to canonicalize paths (i.e., remove symbolic links and "..", and make them absolute). It takes a string with paths separated by commas, and returns the canonicalized path in the same format.
--- meta/lib/oe/path.py | 12 ++++++++++++ 1 file changed, 12 insertions(+)
meta/lib/oe/path.py +12 -0
@@ -320,3 +320,15 @@ def which_wild(pathname, path=None, mode=os.F_OK, *, reverse=False, candidates=F return files +def canonicalize(paths, sep=','): + """Given a string with paths (separated by commas by default), expand + each path using os.path.realpath() and return the resulting paths as a + string (separated using the same separator a the original string). + """ + # Ignore paths containing "$" as they are assumed to be unexpanded bitbake + # variables. Normally they would be ignored, e.g., when passing the paths + # through the shell they would expand to empty strings. However, when they + # are passed through os.path.realpath(), it will cause them to be prefixed + # with the absolute path to the current directory and thus not be empty + # anymore. + return sep.join(os.path.realpath(path) for path in (paths or '').split(sep) if "$" not in path)
PATCH v3 4/5 bitbake.conf: Canonicalize paths in PSEUDO_IGNORE_PATHS
In reply to Peter KjellerstedtUse oe.path.canonicalize() to canonicalize the paths in PSEUDO_IGNORE_PATHS before passing them to pseudo. This is needed since pseudo will compare them to paths that are canonicalized.
--- meta/conf/bitbake.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
meta/conf/bitbake.conf +2 -2
@@ -691,9 +691,9 @@ export PSEUDO_DISABLED = "1" #export PSEUDO_PREFIX = "${STAGING_DIR_NATIVE}${prefix_native}" #export PSEUDO_BINDIR = "${STAGING_DIR_NATIVE}${bindir_native}" #export PSEUDO_LIBDIR = "${STAGING_DIR_NATIVE}$PSEUDOBINDIR/../lib/pseudo/lib -FAKEROOTBASEENV = "PSEUDO_BINDIR=${PSEUDO_SYSROOT}${bindir_native} PSEUDO_LIBDIR=${PSEUDO_SYSROOT}${prefix_native}/lib/pseudo/lib PSEUDO_PREFIX=${PSEUDO_SYSROOT}${prefix_native} PSEUDO_IGNORE_PATHS=${PSEUDO_IGNORE_PATHS} PSEUDO_DISABLED=1" +FAKEROOTBASEENV = "PSEUDO_BINDIR=${PSEUDO_SYSROOT}${bindir_native} PSEUDO_LIBDIR=${PSEUDO_SYSROOT}${prefix_native}/lib/pseudo/lib PSEUDO_PREFIX=${PSEUDO_SYSROOT}${prefix_native} PSEUDO_IGNORE_PATHS=${@oe.path.canonicalize(d.getVar('PSEUDO_IGNORE_PATHS'))} PSEUDO_DISABLED=1" FAKEROOTCMD = "${PSEUDO_SYSROOT}${bindir_native}/pseudo" -FAKEROOTENV = "PSEUDO_PREFIX=${PSEUDO_SYSROOT}${prefix_native} PSEUDO_LOCALSTATEDIR=${PSEUDO_LOCALSTATEDIR} PSEUDO_PASSWD=${PSEUDO_PASSWD} PSEUDO_NOSYMLINKEXP=1 PSEUDO_IGNORE_PATHS=${PSEUDO_IGNORE_PATHS} PSEUDO_DISABLED=0" +FAKEROOTENV = "PSEUDO_PREFIX=${PSEUDO_SYSROOT}${prefix_native} PSEUDO_LOCALSTATEDIR=${PSEUDO_LOCALSTATEDIR} PSEUDO_PASSWD=${PSEUDO_PASSWD} PSEUDO_NOSYMLINKEXP=1 PSEUDO_IGNORE_PATHS=${@oe.path.canonicalize(d.getVar('PSEUDO_IGNORE_PATHS'))} PSEUDO_DISABLED=0" FAKEROOTNOENV = "PSEUDO_UNLOAD=1" FAKEROOTDIRS = "${PSEUDO_LOCALSTATEDIR}" PREFERRED_PROVIDER_virtual/fakeroot-native ?= "pseudo-native"
PATCH v3 5/5 wic: Pass canonicalized paths in PSEUDO_IGNORE_PATHS
In reply to Peter Kjellerstedt--- scripts/lib/wic/partition.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-)
scripts/lib/wic/partition.py +5 -1
@@ -207,11 +207,15 @@ class Partition(): p_prefix = os.environ.get("PSEUDO_PREFIX", "%s/usr" % native_sysroot) if (pseudo_dir): + # Canonicalize the ignore paths. This corresponds to + # calling oe.path.canonicalize(), which is used in bitbake.conf. + ignore_paths = [rootfs] + (get_bitbake_var("PSEUDO_IGNORE_PATHS") or "").split(",") + ignore_paths = ",".join(os.path.realpath(path) for path in ignore_paths if "$" not in path) pseudo = "export PSEUDO_PREFIX=%s;" % p_prefix pseudo += "export PSEUDO_LOCALSTATEDIR=%s;" % pseudo_dir pseudo += "export PSEUDO_PASSWD=%s;" % rootfs_dir pseudo += "export PSEUDO_NOSYMLINKEXP=1;" - pseudo += "export PSEUDO_IGNORE_PATHS=%s;" % (rootfs + "," + (get_bitbake_var("PSEUDO_IGNORE_PATHS") or "")) + pseudo += "export PSEUDO_IGNORE_PATHS=%s;" % ignore_paths pseudo += "%s " % get_bitbake_var("FAKEROOTCMD") else: pseudo = None