sstate: allow specifying indirect dependencies to exclude from sysroot
PATCH 1/4 sstate: allow specifying indirect dependencies to exclude from sysroot
From: André Draszik <[email protected]>
Currently, a dependency on any -native recipe will pull in all dependencies of that -native recipe in the recipe sysroot. This behaviour might not always be wanted, e.g. when that -native recipe depends on build-tools that are not relevant for the current recipe.
This change adds a SSTATE_EXCLUDEDEPS_SYSROOT variable, which will be evaluated for such recursive dependencies to be excluded. The idea is similar to
http://lists.openembedded.org/pipermail/openembedded-core/2018-January/146324.html
except that the list is not hard-coded anymore.
SSTATE_EXCLUDEDEPS_SYSROOT is evaluated as two regular expressions of recipe and dependency to ignore, e.g. in the above flex-native / bison-native use-case, one would specify
SSTATE_EXCLUDEDEPS_SYSROOT = ".*->(flex|bison)-native"
in layer.conf.
The existing special handling of "-initial" as well as "base-passwd" and "shadow-sysroot" could also be streamlined:
SSTATE_EXCLUDEDEPS_SYSROOT += "\
.*->.*-initial.* \
.*(base-passwd|shadow-sysroot)->.* \
"Another anticipated user is meta-java, where certain newer JDKs can only be bootstrapped (built) using older JDKs, but it doesn't make much sense to copy all those older JDKs and their own build tools (ant, etc.) into the sysroot of recipes wanting to be built using the newer JDK (only), e.g.:
SSTATE_EXCLUDEDEPS_SYSROOT += "\
openjdk-8-native->(ant-native|attr-native|coreutils-native|icedtea7-native|libxslt-native|make-native|openssl-native|zip-native|unzip-native) \
"--- meta/classes/sstate.bbclass | 13 +++++++++++++ 1 file changed, 13 insertions(+)
meta/classes/sstate.bbclass +13 -0
@@ -897,6 +897,7 @@ def setscene_depvalid(task, taskdependees, notneeded, d, log=None): # task is included in taskdependees too # Return - False - We need this dependency # - True - We can skip this dependency + import re def logit(msg, log): if log is not None:
@@ -957,6 +958,18 @@ def setscene_depvalid(task, taskdependees, notneeded, d, log=None): # Nothing need depend on libc-initial/gcc-cross-initial if "-initial" in taskdependees[task][0]: continue + # Allow excluding certain recursive dependencies. If a recipe needs it should add a + # specific dependency itself, rather than relying on one of its dependees to pull + # them in. + # See also http://lists.openembedded.org/pipermail/openembedded-core/2018-January/146324.html + not_needed = False + for excl in (d.getVar('SSTATE_EXCLUDEDEPS_SYSROOT') or "").split(): + if re.match(excl.split('->', 1)[0], taskdependees[dep][0]): + if re.match(excl.split('->', 1)[1], taskdependees[task][0]): + not_needed = True + break + if not_needed: + continue # For meta-extsdk-toolchain we want all sysroot dependencies if taskdependees[dep][0] == 'meta-extsdk-toolchain': return False
Signature · 1 line
2.17.0
PATCH 2/4 sstate: use SSTATE_EXCLUDEDEPS_SYSROOT for skipping *-initial recipes
From: André Draszik <[email protected]>
Use the newly introduced SSTATE_EXCLUDEDEPS_SYSROOT for specifying the *-initial recipes to be excluded from a recipe sysroot.
--- meta/classes/sstate.bbclass | 3 --- meta/conf/layer.conf | 5 +++++ 2 files changed, 5 insertions(+), 3 deletions(-)
meta/classes/sstate.bbclass +0 -3
@@ -955,9 +955,6 @@ def setscene_depvalid(task, taskdependees, notneeded, d, log=None): # base-passwd/shadow-sysroot don't need their dependencies if taskdependees[dep][0].endswith(("base-passwd", "shadow-sysroot")): continue - # Nothing need depend on libc-initial/gcc-cross-initial - if "-initial" in taskdependees[task][0]: - continue # Allow excluding certain recursive dependencies. If a recipe needs it should add a # specific dependency itself, rather than relying on one of its dependees to pull # them in.
meta/conf/layer.conf +5 -0
@@ -78,5 +78,10 @@ SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS += " \ weston-init->kbd \ " +# Nothing needs to depend on libc-initial/gcc-cross-initial +SSTATE_EXCLUDEDEPS_SYSROOT += "\ + .*->.*-initial.* \ +" + # We need to keep bitbake tools in PATH PATH := "${@os.path.dirname(bb.utils.which(d.getVar('PATH'),'bitbake'))}:${HOSTTOOLS_DIR}"
Signature · 1 line
2.17.0
PATCH 3/4 sstate: use SSTATE_EXCLUDEDEPS_SYSROOT for skipping base-passwd|shadow-sysroot recipes
In reply to André DraszikFrom: André Draszik <[email protected]>
Use the newly introduced SSTATE_EXCLUDEDEPS_SYSROOT for specifying the base-passwd|shadow-sysroot recipes to be excluded from a recipe sysroot.
--- meta/classes/sstate.bbclass | 3 --- meta/conf/layer.conf | 2 ++ 2 files changed, 2 insertions(+), 3 deletions(-)
meta/classes/sstate.bbclass +0 -3
@@ -952,9 +952,6 @@ def setscene_depvalid(task, taskdependees, notneeded, d, log=None): # Consider sysroot depending on sysroot tasks if taskdependees[task][1] == 'do_populate_sysroot' and taskdependees[dep][1] == 'do_populate_sysroot': - # base-passwd/shadow-sysroot don't need their dependencies - if taskdependees[dep][0].endswith(("base-passwd", "shadow-sysroot")): - continue # Allow excluding certain recursive dependencies. If a recipe needs it should add a # specific dependency itself, rather than relying on one of its dependees to pull # them in.
meta/conf/layer.conf +2 -0
@@ -79,8 +79,10 @@ SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS += " \ " # Nothing needs to depend on libc-initial/gcc-cross-initial +# base-passwd/shadow-sysroot don't need their dependencies SSTATE_EXCLUDEDEPS_SYSROOT += "\ .*->.*-initial.* \ + .*(base-passwd|shadow-sysroot)->.* \ " # We need to keep bitbake tools in PATH
Signature · 1 line
2.17.0
PATCH 4/4 sstate: avoid indirect bison/flex-native dependencies (via SSTATE_EXCLUDEDEPS_SYSROOT)
In reply to André DraszikFrom: André Draszik <[email protected]>
Avoid adding flex-native or bison-native to the sysroot without a specific dependency in the recipe. This means indirect dependencies (e.g. X -> Y -> binutils-cross -> flex-native) no longer meet the dependency incidentally. This improves determinism and avoids build failures when people switch to external toolchains.
Based on an idea by Richard Purdie:
http://lists.openembedded.org/pipermail/openembedded-core/2018-January/146324.html
--- meta/conf/layer.conf | 6 ++++++ 1 file changed, 6 insertions(+)
meta/conf/layer.conf +6 -0
@@ -78,6 +78,12 @@ SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS += " \ weston-init->kbd \ " +# Avoid adding flex-native or bison-native to the sysroot without a specific +# dependency in the recipe. This means indirect dependencies +# (e.g. X -> Y -> binutils-cross -> flex-native) no longer meet the +# dependency incidentally. This improves determinism and avoids build +# failures when people switch to external toolchains. +SSTATE_EXCLUDEDEPS_SYSROOT += ".*->(flex|bison)-native" # Nothing needs to depend on libc-initial/gcc-cross-initial # base-passwd/shadow-sysroot don't need their dependencies SSTATE_EXCLUDEDEPS_SYSROOT += "\
Signature · 1 line
2.17.0
Re: [PATCH 4/4] sstate: avoid indirect bison/flex-native dependencies (via SSTATE_EXCLUDEDEPS_SYSROOT)
Quoted 9 lines · André Draszik
From: André Draszik <[email protected]>
Avoid adding flex-native or bison-native to the sysroot without a specific
dependency in the recipe. This means indirect dependencies (e.g. X -> Y -> binutils-cross -> flex-native) no longer meet the dependency incidentally. This improves determinism and avoids build failures when people switch to external toolchains.
Quoted 8 lines
Based on an idea by Richard Purdie:
http://lists.openembedded.org/pipermail/openembedded-core/2018-Ja
nuary/146324.htmlSigned-off-by André Draszik <[email protected]>---
meta/conf/layer.conf | 6 ++++++1 file changed, 6 insertions(+)
The trouble was that the patch above of mine doesn't actually work and breaks things from what I remember, some issue showed up in testing on our autobuilder. I don't remember exactly what it breaks unfortunately :(.
Cheers,
Richard
Re: [PATCH 4/4] sstate: avoid indirect bison/flex-native dependencies (via SSTATE_EXCLUDEDEPS_SYSROOT)
Quoted 19 lines · Richard Purdie
Quoted 9 lines · André Draszik
From: André Draszik <[email protected]>
Avoid adding flex-native or bison-native to the sysroot without a specific
dependency in the recipe. This means indirect dependencies (e.g. X -> Y -> binutils-cross -> flex-native) no longer meet the dependency incidentally. This improves determinism and avoids build failures when people switch to external toolchains.Quoted 8 lines
Based on an idea by Richard Purdie:
http://lists.openembedded.org/pipermail/openembedded-core/2018-Ja
nuary/146324.htmlSigned-off-by André Draszik <[email protected]>--- meta/conf/layer.conf | 6 ++++++ 1 file changed, 6 insertions(+)
Quoted 4 lines
The trouble was that the patch above of mine doesn't actually work and breaks things from what I remember, some issue showed up in testing on our autobuilder. I don't remember exactly what it breaks unfortunately :(.
Well, I did test this series with the images I build, and nothing broke, except for one missing dependency in meta-oe, for which I've sent a patch already.
I think your patch placed the skipping in the wrong place? Maybe it was intentional, but I have it in a different place, and I don't return (True) on a match, but continue iterating over the remaining items.
In any case, if there are other breakages caused by this, the autobuilder would at least have a log message, so I can fix the patch hopefully... :-)
Cheers,
Andre'
Quoted 33 lines · André Draszik
meta/classes/sstate.bbclass +0
-0@@ -897,6 +897,7 @@ def setscene_depvalid(task, taskdependees,notneeded, d, log=None):
# task is included in taskdependees too # Return - False - We need this dependency # - True - We can skip this dependency + import re def logit(msg, log): if log is not None:meta/classes/sstate.bbclass +0
-0@@ -957,6 +958,18 @@ def setscene_depvalid(task, taskdependees,notneeded, d, log=None):
# Nothing need depend on libc-initial/gcc-cross-initial if "-initial" in taskdependees[task][0]:
continue
+ # Allow excluding certain recursive dependencies. If a recipe needs it should add a
+ # specific dependency itself, rather than relying on one of its dependees to pull
+ # them in.
+ # See also http://lists.openembedded.org/pipermail/opene
mbedded-core/2018-January/146324.html
+ not_needed = False
+ for excl in (d.getVar('SSTATE_EXCLUDEDEPS_SYSROOT')or "").split():
+ ifre.match(excl.split('->', 1)[0], taskdependees[dep][0]):
+ ifre.match(excl.split('->', 1)[1], taskdependees[task][0]):
Quoted 8 lines
+ not_needed = True
+ break
+ ifnot_needed:
+ continue
# For meta-extsdk-toolchain we want all sysroot dependencies
if taskdependees[dep][0] == 'meta-extsdk-toolchain':
return False
Have you looked at the performance impact of this change? It looks like it will be compiling the regexp each time inside a tight loop which gets called once per task dependency which will show up significantly on the performance chart.
We already have a "*" syntax used in SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS and I'd rather use that syntax here to avoid the use of re if we can help it. Could you see if that is possible please? That would also keep the syntax compatible.
Cheers,
Richard
Quoted 39 lines · Richard Purdie
Quoted 38 lines · André Draszik
meta/classes/sstate.bbclass +0
-0@@ -897,6 +897,7 @@ def setscene_depvalid(task, taskdependees,notneeded, d, log=None):
# task is included in taskdependees too # Return - False - We need this dependency # - True - We can skip this dependencymeta/classes/sstate.bbclass +1
-0+ import re def logit(msg, log): if log is not None:@@ -957,6 +958,18 @@ def setscene_depvalid(task, taskdependees,notneeded, d, log=None):
# Nothing need depend on libc-initial/gcc-cross-initial if "-initial" in taskdependees[task][0]:
continue
+ # Allow excluding certain recursive dependencies. If a
recipe needs it should add a
+ # specific dependency itself, rather than relying on one
of its dependees to pullmeta/classes/sstate.bbclass +2
-0+ # them in. + # See also http://lists.openembedded.org/pipermail/openembedded-core/2018-January/146324.html
meta/classes/sstate.bbclass +2
-0+ not_needed = False + for excl in (d.getVar('SSTATE_EXCLUDEDEPS_SYSROOT') or"").split():
+ if re.match(excl.split('->', 1)[0],
taskdependees[dep][0]):
+ if re.match(excl.split('->', 1)[1],
taskdependees[task][0]):meta/classes/sstate.bbclass +4
-0+ not_needed = True + break + if not_needed: + continue # For meta-extsdk-toolchain we want all sysroot
Quoted 8 lines
Quoted 3 lines
dependencies
if taskdependees[dep][0] == 'meta-extsdk-toolchain': return FalseHave you looked at the performance impact of this change? It looks like it will be compiling the regexp each time inside a tight loop which gets called once per task dependency which will show up significantly on the performance chart.
True, and no, I haven't. Is there a good way to compare the performance with and without this change other than using 'time'?
Quoted 4 lines
We already have a "*" syntax used in
SIGGEN_EXCLUDE_SAFE_RECIPE_DEPSand I'd rather use that syntax here to avoid the use of re if we can help it. Could you see if that is possible please? That would also keep the syntax compatible.
OK.
Cheers,
Andre'
Quoted 9 lines · André Draszik
Quoted 6 lines
Have you looked at the performance impact of this change? It looks like
it will be compiling the regexp each time inside a tight loop which gets called once per task dependency which will show up significantly
on the performance chart.True, and no, I haven't. Is there a good way to compare the performance with
and without this change other than using 'time'?
I'd try something like "bitbake <image> -P -n" which should generate a profile output for the dry run of building the image. That would not build anything but would include all the dependency calculations (and unfortunately all the fork overheadin this case).
Quoted 9 lines
Quoted 8 lines
We already have a "*" syntax used in
SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS
and I'd rather use that syntax here to avoid the use of re if we can
help it. Could you see if that is possible please? That would also keep
the syntax compatible.OK.
I think it uses glob btw which is much lighter weight than re.
Cheers,
Richard
Quoted 39 lines · Richard Purdie
Quoted 38 lines · André Draszik
meta/classes/sstate.bbclass +0
-0@@ -897,6 +897,7 @@ def setscene_depvalid(task, taskdependees,notneeded, d, log=None):
# task is included in taskdependees too # Return - False - We need this dependency # - True - We can skip this dependencymeta/classes/sstate.bbclass +1
-0+ import re def logit(msg, log): if log is not None:@@ -957,6 +958,18 @@ def setscene_depvalid(task, taskdependees,notneeded, d, log=None):
# Nothing need depend on libc-initial/gcc-cross-initial if "-initial" in taskdependees[task][0]:
continue
+ # Allow excluding certain recursive dependencies. If a
recipe needs it should add a
+ # specific dependency itself, rather than relying on one
of its dependees to pullmeta/classes/sstate.bbclass +2
-0+ # them in. + # See also http://lists.openembedded.org/pipermail/openembedded-core/2018-January/146324.html
meta/classes/sstate.bbclass +2
-0+ not_needed = False + for excl in (d.getVar('SSTATE_EXCLUDEDEPS_SYSROOT') or"").split():
+ if re.match(excl.split('->', 1)[0],
taskdependees[dep][0]):
+ if re.match(excl.split('->', 1)[1],
taskdependees[task][0]):meta/classes/sstate.bbclass +4
-0+ not_needed = True + break + if not_needed: + continue # For meta-extsdk-toolchain we want all sysroot
Quoted 8 lines
Quoted 3 lines
dependencies
if taskdependees[dep][0] == 'meta-extsdk-toolchain': return FalseHave you looked at the performance impact of this change? It looks like it will be compiling the regexp each time inside a tight loop which gets called once per task dependency which will show up significantly on the performance chart.
I guess I'll have to do some more tests or so, but the change doesn't seem to affect performance noticeably using the following regex grep -wE '(match|setscene_depvalid)' profile.log.processed:
before my change:
ncalls tottime percall cumtime percall filename:lineno(function)
472862 0.096 0.000 0.096 0.000 {method 'match' of '_sre.SRE_Pattern' objects}
150956 0.040 0.000 0.106 0.000 /usr/lib/python3.6/re.py:169(match)
112 0.004 0.000 0.044 0.000 .../poky/meta/classes/sstate.bbclass:895(setscene_depvalid)
4602 0.001 0.000 0.001 0.000 /usr/lib/python3.6/sre_parse.py:248(match)after my change:
ncalls tottime percall cumtime percall filename:lineno(function)
473139 0.095 0.000 0.095 0.000 {method 'match' of '_sre.SRE_Pattern' objects}
151114 0.042 0.000 0.108 0.000 /usr/lib/python3.6/re.py:169(match)
112 0.003 0.000 0.040 0.000 .../poky/meta/classes/sstate.bbclass:895(setscene_depvalid)
4602 0.001 0.000 0.001 0.000 /usr/lib/python3.6/sre_parse.py:248(match)(it reports executing 1082 + 3981 tasks)
Maybe I'm looking at the wrong lines...
Cheers,
Andre'
Quoted 46 lines · André Draszik
From: André Draszik <[email protected]>
Currently, a dependency on any -native recipe will pull in all dependencies of that -native recipe in the recipe sysroot. This behaviour might not always be wanted, e.g. when that -native recipe depends on build-tools that are not relevant for the current recipe.
This change adds a
SSTATE_EXCLUDEDEPS_SYSROOTvariable, which will be evaluated for such recursive dependencies to be excluded. The idea is similar to
http://lists.openembedded.org/pipermail/openembedded-core/2018-Jan
uary/146324.html
except that the list is not hard-coded anymore.
SSTATE_EXCLUDEDEPS_SYSROOTis evaluated as two regular expressions of recipe and dependency to ignore, e.g. in the above flex-native / bison-native use-case, one would specifySSTATE_EXCLUDEDEPS_SYSROOT = ".*->(flex|bison)-native"in layer.conf.
The existing special handling of "-initial" as well as "base-passwd" and "shadow-sysroot" could also be streamlined:
SSTATE_EXCLUDEDEPS_SYSROOT += "\ .*->.*-initial.* \ .*(base-passwd|shadow-sysroot)->.* \ "Another anticipated user is meta-java, where certain newer JDKs can only be bootstrapped (built) using older JDKs, but it doesn't make much sense to copy all those older JDKs and their own build tools (ant, etc.) into the sysroot of recipes wanting to be built using the newer JDK (only), e.g.:
SSTATE_EXCLUDEDEPS_SYSROOT += "\ openjdk-8-native->(ant-native|attr-native|coreutils- native|icedtea7-native|libxslt-native|make-native|openssl-native|zip- native|unzip-native) \ "
Quoted 5 lines
Signed-off-by André Draszik <[email protected]>--- meta/classes/sstate.bbclass | 13 +++++++++++++ 1 file changed, 13 insertions(+)
I have some reservations about this series as I've previously mentioned but I had a try at testing it on the autobuilder:
https://autobuilder.yocto.io/builders/build-appliance/builds/1059/steps/BuildImages_1/logs/stdio
Basically binutils-cross is bust with errors like:
| x86_64-poky-linux-ar: error while loading shared libraries: libfl.so.2: cannot open shared object file: No such file or directory | make[5]: *** [libaudio_resampler_sse.la] Error 127
There are a ton of other failures, I've not waded through them but I'm guessing there are similar issues.
Cheers,
Richard
Quoted 53 lines · Richard Purdie
Quoted 46 lines · André Draszik
From: André Draszik <[email protected]>
Currently, a dependency on any -native recipe will pull in all dependencies of that -native recipe in the recipe sysroot. This behaviour might not always be wanted, e.g. when that -native recipe depends on build-tools that are not relevant for the current recipe.
This change adds a
SSTATE_EXCLUDEDEPS_SYSROOTvariable, which will be evaluated for such recursive dependencies to be excluded. The idea is similar to
http://lists.openembedded.org/pipermail/openembedded-core/2018-Jan
uary/146324.html
except that the list is not hard-coded anymore.
SSTATE_EXCLUDEDEPS_SYSROOTis evaluated as two regular expressions of recipe and dependency to ignore, e.g. in the above flex-native / bison-native use-case, one would specifySSTATE_EXCLUDEDEPS_SYSROOT = ".*->(flex|bison)-native"in layer.conf.
The existing special handling of "-initial" as well as "base-passwd" and "shadow-sysroot" could also be streamlined:
SSTATE_EXCLUDEDEPS_SYSROOT += "\ .*->.*-initial.* \ .*(base-passwd|shadow-sysroot)->.* \ "Another anticipated user is meta-java, where certain newer JDKs can only be bootstrapped (built) using older JDKs, but it doesn't make much sense to copy all those older JDKs and their own build tools (ant, etc.) into the sysroot of recipes wanting to be built using the newer JDK (only), e.g.:
SSTATE_EXCLUDEDEPS_SYSROOT += "\ openjdk-8-native->(ant-native|attr-native|coreutils- native|icedtea7-native|libxslt-native|make-native|openssl-native|zip- native|unzip-native) \ "Quoted 5 lines
Signed-off-by André Draszik <[email protected]>--- meta/classes/sstate.bbclass | 13 +++++++++++++ 1 file changed, 13 insertions(+)
Quoted 5 lines
I have some reservations about this series as I've previously mentioned but I had a try at testing it on the autobuilder:
https://autobuilder.yocto.io/builders/build-appliance/builds/1059/steps/Bu
ildImages_1/logs/stdio
Thank you Richard.
I should maybe explain my ultimate goal here:
meta-java is completely self-hosted, which is nice. But to be able to compile OpenJDK v8, various other Java compilers need to be compiled first. This of course means that all those different Java compilers/VMs/runtimes and tools needed to compile them end up in each Java recipe's sysroot.
While in theory this is fine, and all of them can be installed in parallel, as everything resides below subdirectories in /usr/lib/jvm/... and usr/bin/java normally is a symlink to the default chosen via update- alternatives, this doesn't work in yocto for -native recipes.
In particular this is annoying, because 'usr/bin/java' in the recipe sysroot is a *very* old version of java, with no possibility of control.
This has caused me a bit of headache, and if it was possible to avoid all the recursive OpenJDK-8 build time dependencies then:
* usr/bin/java could disappear (for -native)
* tools inherited from the OpenJDK-8 build can be avoided during other recipe builds
In other words, as a follow-up to this series here I have a local meta-java change to meta-java's layer.conf:
SSTATE_EXCLUDEDEPS_SYSROOT += "\
openjdk-8-native->(ant-native|attr-native|coreutils-native|icedtea7-native|libxslt-native|make-native|openssl-native|zip-native|unzip-native) \"
This really is a similar situation to *-initial treatment in sstate.bbclass, but for Java / OpenJDK.
So rather than having to add knowledge about Java / OpenJDK into sstate.bbclass, I need a way to inject that instead.
Given the above, would you be happy to consider a series where the infrastructure is put in place, but without the flex-native and bison-native changes? I.e. patches 1 through 3, but not patch 4 of this series? (With your previous review comments addressed, of course).
Quoted 8 lines
Basically binutils-cross is bust with errors like:
Quoted 3 lines
x86_64-poky-linux-ar: error while loading shared libraries: libfl.so.2:
cannot open shared object file: No such file or directory make[5]: *** [libaudio_resampler_sse.la] Error 127There are a ton of other failures, I've not waded through them but I'm guessing there are similar issues.
I am not sure why I didn't see these in my builds before submitting, but now I do, too.
It looks like the cross binutils-native (or more specifically cross -ar) dynamically links against libflex-native. So the last patch in this series is of course responsible for that.
Comparing to my debian, ar isn't dynamically linked against libflex, I wonder why yocto does it, or why debian doesn't.
In any case my builds succeed now by dropping the flex-native from patch 4 of this series. Alternative would be to switch to static linking for libflex. But see above.
Cheers,
Andre'
Quoted 76 lines · André Draszik
I should maybe explain my ultimate goal here:
meta-java is completely self-hosted, which is nice. But to be able to compile OpenJDK v8, various other Java compilers need to be compiled first.
This of course means that all those different Java compilers/VMs/runtimes
and tools needed to compile them end up in each Java recipe's sysroot.While in theory this is fine, and all of them can be installed in parallel,
as everything resides below subdirectories in/usr/lib/jvm/... andusr/bin/javanormally is a symlink to the default chosen via update- alternatives, this doesn't work in yocto for -native recipes.In particular this is annoying, because '
usr/bin/java' in the recipe sysroot
is a *very* old version of java, with no possibility of control.This has caused me a bit of headache, and if it was possible to avoid all
the recursive OpenJDK-8 build time dependencies then:
*usr/bin/javacould disappear (for -native)
* tools inherited from the OpenJDK-8 build can be avoided during other
recipe buildsIn other words, as a follow-up to this series here I have a local meta-java
change to meta-java's layer.conf:SSTATE_EXCLUDEDEPS_SYSROOT += "\ openjdk-8-native->(ant-native|attr-native|coreutils-native|icedtea7-native|libxslt-native|make-native|openssl-native|zip- native|unzip-native) \
"This really is a similar situation to *-initial treatment in sstate.bbclass,
but for Java / OpenJDK.So rather than having to add knowledge about Java / OpenJDK into sstate.bbclass, I need a way to inject that instead.
Given the above, would you be happy to consider a series where the infrastructure is put in place, but without the flex-native and bison-native
changes? I.e. patches 1 through 3, but not patch 4 of this series? (With
your previous review comments addressed, of course).Quoted 10 lines
Basically binutils-cross is bust with errors like:
Quoted 4 lines
x86_64-poky-linux-ar: error while loading shared libraries:
libfl.so.2:
cannot open shared object file: No such file or directory make[5]: *** [libaudio_resampler_sse.la] Error 127There are a ton of other failures, I've not waded through them but I'm
guessing there are similar issues.I am not sure why I didn't see these in my builds before submitting, but now
I do, too.It looks like the cross binutils-native (or more specifically cross -ar)
dynamically links against libflex-native. So the last patch in this series
is of course responsible for that.
Quoted 8 lines
Comparing to my debian, ar isn't dynamically linked against libflex, I
wonder why yocto does it, or why debian doesn't.In any case my builds succeed now by dropping the flex-native from patch 4
of this series. Alternative would be to switch to static linking for libflex. But see above.
The above is good info thanks. I'm not against the series, I originally thought you wanted to optimise out flex-native and the test results showed it didn't work so I was waiting on your response to that as the series didn't build as it stood. I'd mentioned there were problems but couldn't remember the specifics, you pushed me to "prove" it, I gave you the errors.
The flex-native change doesn't work due to the binutils dependency and may not work in general for anything linking to libfl. It would be interesting to investigate why we have that linkage and if we can avoid it. (Note that bison-native may work as it doesn't use a library iirc so we could split that patch into two?).
I continue to be worried about the regular expression usage. I believe modern python does some caching behind the scenes which helps but tight loops with regexps in have caused us pain in the past, particularly recompiling the expression each time and I worry about new API which ties us to this. Is it possible to use glob?
I'm not against the patch series, we just need to drop the flex-native change (maybe adding a special case for binutils if we can figure out the linking) and figure out if we can use a simpler mechanism than regexps or at least stop me worrying about performance. I'd need to spend more time than I have right now to figure out how to properly benchmark that function but as I said we have been bitten before :(.
Cheers,
Richard
Quoted 40 lines · André Draszik
From: André Draszik <[email protected]>
Currently, a dependency on any -native recipe will pull in all dependencies of that -native recipe in the recipe sysroot. This behaviour might not always be wanted, e.g. when that -native recipe depends on build-tools that are not relevant for the current recipe.
This change adds a
SSTATE_EXCLUDEDEPS_SYSROOTvariable, which will be evaluated for such recursive dependencies to be excluded. The idea is similar to
http://lists.openembedded.org/pipermail/openembedded-core/2018-Jan
uary/146324.html
except that the list is not hard-coded anymore.
SSTATE_EXCLUDEDEPS_SYSROOTis evaluated as two regular expressions of recipe and dependency to ignore, e.g. in the above flex-native / bison-native use-case, one would specifySSTATE_EXCLUDEDEPS_SYSROOT = ".*->(flex|bison)-native"in layer.conf.
The existing special handling of "-initial" as well as "base-passwd" and "shadow-sysroot" could also be streamlined:
SSTATE_EXCLUDEDEPS_SYSROOT += "\ .*->.*-initial.* \ .*(base-passwd|shadow-sysroot)->.* \ "Another anticipated user is meta-java, where certain newer JDKs can only be bootstrapped (built) using older JDKs, but it doesn't make much sense to copy all those older JDKs and their own build tools (ant, etc.) into the sysroot of recipes wanting to be built using the newer JDK (only), e.g.:
Quoted 7 lines
SSTATE_EXCLUDEDEPS_SYSROOT += "\ openjdk-8-native->(ant-native|attr-native|coreutils- native|icedtea7-native|libxslt-native|make-native|openssl-native|zip- native|unzip-native) \ "Signed-off-by André Draszik <[email protected]>
I ended up doing a bit more digging on the potential performance of this patch. In a build it only really has impact when the setscene functions are being considered by runqueue. If your build has already built, it won't show up on performance charts.
I ended up separating out a standalone test which I've included below. It tests some data I took from a build (bitbake bash so a simple one) against three possible options:
a) the code as it stands (re)
b) re.compile() optimised re
c) fnmatch
The results of running it on my machine were:
0.7498898960002407
0.1507232149997435
1.2697166610014392
So clearly fnmatch (i.e. glob) is a worse idea and re is not internally caching. The code isn't being executed as much as I thought it would be which is good although in a larger build, this could still become more significant.
One option may be to use re but to compile SSTATE_EXCLUDEDEPS_SYSROOT into another variable once, then use the other variable and the precompiled regexps...
Cheers,
Richard
#!/usr/bin/env python3
import timeit
import re
import fnmatch
regexp = ".*->bison-native .*->.*-initial.* .*(base-passwd|shadow-sysroot)->.* " regexp2 = []
for excl in regexp.split():
regexp2.append( (re.compile(excl.split('->', 1)[0]), re.compile(excl.split('->', 1)[1])))data = [
('bash', 'glibc-locale'),
('dpkg-native', 'perl-native'),
('glibc-locale', 'cross-localedef-native'),
('bash', 'gettext-native'),
('dpkg-native', 'gettext-native'),
('bash', 'ncurses'),
('bash', 'libtool-cross'),
('ncurses', 'libtool-cross'),
('bash', 'opkg-utils'),
('ncurses', 'gcc-runtime'),
('gcc-runtime', 'libgcc'),
('gcc-runtime', 'gcc-cross-i586'),
('libgcc', 'gcc-cross-i586'),
('bash', 'gcc-cross-i586'),
('opkg-utils', 'gcc-cross-i586'),
('ncurses', 'gcc-cross-i586'),
('ncurses', 'glibc'),
('rpm-native', 'python3-native'),
('rpm-native', 'libarchive-native'),
('rpm-native', 'nss-native'),
('rpm-native', 'dbus-native'),
('rpm-native', 'file-native'),
('rpm-native', 'db-native'),
('rpm-native', 'elfutils-native'),
('rpm-native', 'popt-native'),
('libarchive-native', 'e2fsprogs-native'),
('dbus-native', 'autoconf-archive-native'),
('dbus-native', 'expat-native'),
('glibc', 'gperf-native'),
('glibc', 'glibc-initial'),
('glibc', 'make-native'),
('glibc', 'libgcc-initial'),
('glibc', 'gcc-cross-initial-i586'),
('glibc', 'linux-libc-headers'),
('linux-libc-headers', 'binutils-cross-i586'),
('gcc-runtime', 'binutils-cross-i586'),
('glibc', 'binutils-cross-i586'),
('libgcc', 'binutils-cross-i586'),
('glibc-locale', 'binutils-cross-i586'),
('bash', 'binutils-cross-i586'),
('opkg-utils', 'binutils-cross-i586'),
('ncurses', 'binutils-cross-i586'),
('bash', 'bison-native'),
('glibc', 'bison-native'),
('linux-libc-headers', 'unifdef-native'),
('nss-native', 'nspr-native'),
('e2fsprogs-native', 'util-linux-native'),
('e2fsprogs-native', 'attr-native'),
('nss-native', 'sqlite3-native'),
('rpm-native', 'bzip2-native'),
('python3-native', 'gdbm-native'),
('python3-native', 'openssl-native'),
('python3-native', 'readline-native'),
('file-native', 'zlib-native'),
('util-linux-native', 'ncurses-native'),
('libarchive-native', 'lzo-native'),
('dbus-native', 'pkgconfig-native'),
('rpm-native', 'gettext-minimal-native'),
('dbus-native', 'libtool-native'),
('dbus-native', 'automake-native'),
('automake-native', 'autoconf-native'),
('autoconf-native', 'm4-native'),
('dbus-native', 'gnu-config-native'),
('automake-native', 'texinfo-dummy-native'),
('opkg-utils-native', 'quilt-native')
]def test():
for task, task1 in data:
for excl in regexp.split():
re.match(excl.split('->', 1)[0], task) re.match(excl.split('->', 1)[1], task1)
def test1():
for task, task1 in data:
for excl in regexp2:
excl[0].match(task)
excl[1].match(task1)
def test2():
regexp = "*->bison-native *->*-initial* *base-passwd->* shadow-sysroot->*" for task, task1 in data:
for excl in regexp.split():
fnmatch.fnmatch(excl, task)
fnmatch.fnmatch(excl, task1)
print(timeit.timeit("test()", setup="from __main__ import test",number=1000, globals=globals())) print(timeit.timeit("test1()", setup="from __main__ import test",number=1000, globals=globals())) print(timeit.timeit("test2()", setup="from __main__ import test",number=1000, globals=globals()))