Skip to contentSkip to conversation

sstate.bbclass: make sure changes to SSTATE_SCAN_FILES are not ignored

PATCH In review Alexander Kanavin 4 participants 19 messages Jan 23, 2019 – Jan 25, 2019

lore
Share & tools
lore
#0 lore raw

PATCH 1/10 sstate.bbclass: make sure changes to SSTATE_SCAN_FILES are not ignored

When changing the SSTATE_SCAN_FILES variable in a recipe it doesn't cause a rebuild, so if there's a sstate-cache available with "bad" sstate data in it that will still be used even though the recipe is updated to address this.

[YOCTO #13144]

---
 meta/classes/sstate.bbclass | 1 +
 1 file changed, 1 insertion(+)
meta/classes/sstate.bbclass +1 -0
index 2f0bbd2d7df..6f51d9c1879 100644
@@ -62,6 +62,7 @@ SSTATE_ARCHS = " \
 SSTATE_MANMACH ?= "${SSTATE_PKGARCH}"
 
 SSTATECREATEFUNCS = "sstate_hardcode_path"
+SSTATECREATEFUNCS[vardeps] = "SSTATE_SCAN_FILES"
 SSTATEPOSTCREATEFUNCS = ""
 SSTATEPREINSTFUNCS = ""
 SSTATEPOSTUNPACKFUNCS = "sstate_hardcode_path_unpack"
Signature · 1 line
2.17.1
#1 lore raw

PATCH 2/10 fontcache: fix postinst for nativesdk case

Both installing the binary into the correct place, and passing that place to postinst_intercept were missing.

---
 meta/classes/fontcache.bbclass                        | 1 +
 meta/recipes-graphics/fontconfig/fontconfig_2.12.6.bb | 6 ++++++
 2 files changed, 7 insertions(+)
meta/classes/fontcache.bbclass +1 -0
index f71a754a4dd..13f9df1592f 100644
@@ -20,6 +20,7 @@ if [ -n "$D" ] ; then
 	$INTERCEPT_DIR/postinst_intercept update_font_cache ${PKG} mlprefix=${MLPREFIX} binprefix=${MLPREFIX} \
 		'bindir="${bindir}"' \
 		'libdir="${libdir}"' \
+                'libexecdir="${libexecdir}"' \
 		'base_libdir="${base_libdir}"' \
 		'fontconfigcachedir="${FONTCONFIG_CACHE_DIR}"' \
 		'fontconfigcacheparams="${FONTCONFIG_CACHE_PARAMS}"' \
meta/recipes-graphics/fontconfig/fontconfig_2.12.6.bb +6 -0
index 6128d5e2ae2..311c6454a6e 100644
@@ -41,6 +41,12 @@ do_install_append_class-target() {
     ln ${D}${bindir}/fc-cache ${D}${libexecdir}/${MLPREFIX}fc-cache
 }
 
+do_install_append_class-nativesdk() {
+    # duplicate fc-cache for postinstall script
+    mkdir -p ${D}${libexecdir}
+    ln ${D}${bindir}/fc-cache ${D}${libexecdir}/${MLPREFIX}fc-cache
+}
+
 PACKAGES =+ "fontconfig-utils"
 FILES_${PN} =+ "${datadir}/xml/*"
 FILES_fontconfig-utils = "${bindir}/* ${libexecdir}/*"
Signature · 1 line
2.17.1
#2 lore raw

PATCH 3/10 qemuwrapper-cross: check qemu usermode only when building a target package

In reply to Alexander Kanavin

When building nativesdk- package, MACHINE_FEATURES do not apply as they are specified only for target machines, not ones hosting the sdk.

---
 meta/recipes-devtools/qemu/qemuwrapper-cross_1.0.bb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
meta/recipes-devtools/qemu/qemuwrapper-cross_1.0.bb +1 -1
index 06f15617df5..a0448a18039 100644
@@ -20,7 +20,7 @@ do_install () {
 #!/bin/sh
 set -x
 
-if [ ${@bb.utils.contains('MACHINE_FEATURES', 'qemu-usermode', 'True', 'False', d)} = False ]; then
+if [ ${@bb.utils.contains('MACHINE_FEATURES', 'qemu-usermode', 'True', 'False', d)} = False -a "${PN}" != "nativesdk-qemuwrapper-cross" ]; then
 	echo "qemuwrapper: qemu usermode is not supported"
 	exit 1
 fi
Signature · 1 line
2.17.1
#3 lore raw

PATCH 4/10 lib/oe/package_manager: turn postinst_intercept warnings into failures for nativesdk

In reply to Alexander Kanavin

The few cases where they failed should be now all fixed. The only allowed exception is when building mingw32 SDKs, as there is currently no support for running postinst_intercepts through wine.

---
 meta/lib/oe/package_manager.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)
meta/lib/oe/package_manager.py +6 -1
index 1087144d47f..f26f597d03d 100644
@@ -439,6 +439,11 @@ class PackageManager(object, metaclass=ABCMeta):
                 self._postpone_to_first_boot(script_full)
                 continue
 
+            if populate_sdk == 'host' and self.d.getVar('SDK_OS') == 'mingw32':
+                bb.warn("The postinstall intercept hook '%s' could not be executed due to missing wine support, details in %s/log.do_%s"
+                                % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK')))
+                continue
+
             bb.note("> Executing %s intercept ..." % script)
 
             try:
@@ -447,7 +452,7 @@ class PackageManager(object, metaclass=ABCMeta):
             except subprocess.CalledProcessError as e:
                 bb.note("Exit code %d. Output:\n%s" % (e.returncode, e.output.decode("utf-8")))
                 if populate_sdk == 'host':
-                    bb.warn("The postinstall intercept hook '%s' failed, details in %s/log.do_%s" % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK')))
+                    bb.fatal("The postinstall intercept hook '%s' failed, details in %s/log.do_%s" % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK')))
                 elif populate_sdk == 'target':
                     if "qemuwrapper: qemu usermode is not supported" in e.output.decode("utf-8"):
                         bb.warn("The postinstall intercept hook '%s' could not be executed due to missing qemu usermode support, details in %s/log.do_%s"
Signature · 1 line
2.17.1
#4 lore raw

Re: [PATCH 04/10] lib/oe/package_manager: turn postinst_intercept warnings into failures for nativesdk

Quoted 26 lines · Alexander Kanavin

The few cases where they failed should be now all fixed. The only allowed exception is when building mingw32 SDKs, as there is currently no support for running postinst_intercepts through wine.

---
 meta/lib/oe/package_manager.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)
meta/lib/oe/package_manager.py +5 -0
index 1087144d47f..f26f597d03d 100644
@@ -439,6 +439,11 @@ class PackageManager(object, metaclass=ABCMeta):
                 self._postpone_to_first_boot(script_full)
                 continue
 
+            if populate_sdk == 'host' and self.d.getVar('SDK_OS') == 'mingw32':
+                bb.warn("The postinstall intercept hook '%s' could not be executed due to missing wine support, details in %s/log.do_%s"
+                                % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK')))
+                continue
+
             bb.note("> Executing %s intercept ..." % script)
 
             try:

meta/lib/oe/package_manager.py class PackageManager(object, metaclass=ABCMeta): · line 458

Quoted 8 lines
meta/lib/oe/package_manager.py +1 -1
@@ -447,7 +452,7 @@ class PackageManager(object, metaclass=ABCMeta):
             except subprocess.CalledProcessError as e:
                 bb.note("Exit code %d. Output:\n%s" % (e.returncode, e.output.decode("utf-8")))
                 if populate_sdk == 'host':
-                    bb.warn("The postinstall intercept hook '%s' failed, details in %s/log.do_%s" % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK')))
+                    bb.fatal("The postinstall intercept hook '%s' failed, details in %s/log.do_%s" % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK')))
                 elif populate_sdk == 'target':
                     if "qemuwrapper: qemu usermode is not supported" in e.output.decode("utf-8"):
                         bb.warn("The postinstall intercept hook '%s' could not be executed due to missing qemu usermode support, details in %s/log.do_%s"

I'm ok with this but could we change the bb.warn to something less important like a note?

The reason I say this is that I'd like to get the autobuilder to the point where its warning free. This warning is the only reason meta- mingw shows warnings now. Either we fix them or if we can't, they probably aren't warnings.

What should we do here?

(also cc'd Joshua)

Cheers,

Richard

#5 lore raw

Re: [PATCH 04/10] lib/oe/package_manager: turn postinst_intercept warnings into failures for nativesdk

Quoted 52 lines · On Thu, Jan 24, 2019, 4:43 PM Richard Purdie <…
Quoted 2 lines · Alexander Kanavin

The few cases where they failed should be now all fixed. The only allowed exception is when building mingw32 SDKs, as there is currently no

support for running

Review

Quoted 8 lines

postinst_intercepts through wine.

---
 meta/lib/oe/package_manager.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)
(quoted diff) +0 -0

b/meta/lib/oe/package_manager.py

meta/lib/oe/package_manager.py class PackageManager(object, metaclass=ABCMeta): · line 442

Quoted 8 lines

index 1087144d47f..f26f597d03d 100644

meta/lib/oe/package_manager.py +1 -0
@@ -439,6 +439,11 @@ class PackageManager(object, metaclass=ABCMeta):
                 self._postpone_to_first_boot(script_full)
                 continue
 
+            if populate_sdk == 'host' and self.d.getVar('SDK_OS') ==

'mingw32':

meta/lib/oe/package_manager.py class PackageManager(object, metaclass=ABCMeta): · line 443

Quoted 1 line
meta/lib/oe/package_manager.py +1 -0
+                bb.warn("The postinstall intercept hook '%s' could not

be executed due to missing wine support, details in %s/log.do_%s"

meta/lib/oe/package_manager.py class PackageManager(object, metaclass=ABCMeta): · line 444

Quoted 1 line
meta/lib/oe/package_manager.py +1 -0
+                                % (script, self.d.getVar('T'),

self.d.getVar('BB_CURRENTTASK')))

meta/lib/oe/package_manager.py class PackageManager(object, metaclass=ABCMeta): · line 453

Quoted 8 lines
meta/lib/oe/package_manager.py +2 -0
+                continue
+
             bb.note("> Executing %s intercept ..." % script)
 
             try:
@@ -447,7 +452,7 @@ class PackageManager(object, metaclass=ABCMeta):
             except subprocess.CalledProcessError as e:
                 bb.note("Exit code %d. Output:\n%s" % (e.returncode,

e.output.decode("utf-8")))

meta/lib/oe/package_manager.py class PackageManager(object, metaclass=ABCMeta): · line 450

Quoted 2 lines
meta/lib/oe/package_manager.py +0 -1
                 if populate_sdk == 'host':
-                    bb.warn("The postinstall intercept hook '%s'

failed, details in %s/log.do_%s" % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK')))

meta/lib/oe/package_manager.py class PackageManager(object, metaclass=ABCMeta): · line 455

Quoted 1 line
meta/lib/oe/package_manager.py +1 -0
+                    bb.fatal("The postinstall intercept hook '%s'

failed, details in %s/log.do_%s" % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK')))

meta/lib/oe/package_manager.py class PackageManager(object, metaclass=ABCMeta): · line 457

Quoted 2 lines
meta/lib/oe/package_manager.py +0 -0
                 elif populate_sdk == 'target':
                     if "qemuwrapper: qemu usermode is not supported" in

e.output.decode("utf-8"):

meta/lib/oe/package_manager.py class PackageManager(object, metaclass=ABCMeta): · line 458

Quoted 1 line
meta/lib/oe/package_manager.py +0 -0
                         bb.warn("The postinstall intercept hook '%s'

could not be executed due to missing qemu usermode support, details in %s/log.do_%s"

I'm ok with this but could we change the bb.warn to something less important like a note?

Quoted 9 lines

The reason I say this is that I'd like to get the autobuilder to the point where its warning free. This warning is the only reason meta- mingw shows warnings now. Either we fix them or if we can't, they probably aren't warnings.

What should we do here?

(also cc'd Joshua)

I think the one that fails currently is updating the GIO module cache. I'm not sure if this is even necessary for mingw (my guess is no, since this has never worked and no one has complained). We probably just need to disable it in mingw, but when I looked very briefly a while ago, I didn't see a straight forward way of doing that.

See https://bugzilla.yoctoproject.org/show_bug.cgi?id=12962

Quoted 5 lines

Cheers,

Richard

#6 lore raw

Re: [PATCH 04/10] lib/oe/package_manager: turn postinst_intercept warnings into failures for nativesdk

There is a similar issue in multilib for target packages (a warning because qemu usermode support is missing):
https://autobuilder.yoctoproject.org/typhoon/#/builders/44/builds/214/steps/7/logs/warnings

I'm as well not sure how to handle this:
1. Keep these two warnings as warnings so users see them in their local builds, but
add some kind of string pattern to AB so it excludes missing qemu
support and missing wine support warnings.

2. Turn them into notes, which means they will go to the logs only, and users are not going to see them ever. On the other hand, the postinsts tend to
create things like font caches and similar, which should not generally be needed at compile time (which is what SDKs are for).

I'm leaning towards option two.

Alex

Quoted 63 lines · Joshua Watt
Quoted 40 lines · On Thu, Jan 24, 2019, 4:43 PM Richard Purdie…
Quoted 26 lines · Alexander Kanavin

The few cases where they failed should be now all fixed. The only allowed exception is when building mingw32 SDKs, as there is currently no support for running postinst_intercepts through wine.

---
 meta/lib/oe/package_manager.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)
meta/lib/oe/package_manager.py +5 -0
index 1087144d47f..f26f597d03d 100644
@@ -439,6 +439,11 @@ class PackageManager(object, metaclass=ABCMeta):
                 self._postpone_to_first_boot(script_full)
                 continue
 
+            if populate_sdk == 'host' and self.d.getVar('SDK_OS') == 'mingw32':
+                bb.warn("The postinstall intercept hook '%s' could not be executed due to missing wine support, details in %s/log.do_%s"
+                                % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK')))
+                continue
+
             bb.note("> Executing %s intercept ..." % script)
 
             try:

meta/lib/oe/package_manager.py class PackageManager(object, metaclass=ABCMeta): · line 458

Quoted 8 lines
meta/lib/oe/package_manager.py +1 -1
@@ -447,7 +452,7 @@ class PackageManager(object, metaclass=ABCMeta):
             except subprocess.CalledProcessError as e:
                 bb.note("Exit code %d. Output:\n%s" % (e.returncode, e.output.decode("utf-8")))
                 if populate_sdk == 'host':
-                    bb.warn("The postinstall intercept hook '%s' failed, details in %s/log.do_%s" % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK')))
+                    bb.fatal("The postinstall intercept hook '%s' failed, details in %s/log.do_%s" % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK')))
                 elif populate_sdk == 'target':
                     if "qemuwrapper: qemu usermode is not supported" in e.output.decode("utf-8"):
                         bb.warn("The postinstall intercept hook '%s' could not be executed due to missing qemu usermode support, details in %s/log.do_%s"

I'm ok with this but could we change the bb.warn to something less important like a note?

Quoted 8 lines

The reason I say this is that I'd like to get the autobuilder to the point where its warning free. This warning is the only reason meta- mingw shows warnings now. Either we fix them or if we can't, they probably aren't warnings.

What should we do here?

(also cc'd Joshua)

I think the one that fails currently is updating the GIO module cache. I'm not sure if this is even necessary for mingw (my guess is no, since this has never worked and no one has complained). We probably just need to disable it in mingw, but when I looked very briefly a while ago, I didn't see a straight forward way of doing that.

See https://bugzilla.yoctoproject.org/show_bug.cgi?id=12962

Quoted 5 lines

Cheers,

Richard

#7 lore raw

Re: [PATCH 04/10] lib/oe/package_manager: turn postinst_intercept warnings into failures for nativesdk

Quoted 10 lines · Alexander Kanavin

There is a similar issue in multilib for target packages (a warning because qemu usermode support is missing):
https://autobuilder.yoctoproject.org/typhoon/#/builders/44/builds/214/steps/7/logs/warnings

I'm as well not sure how to handle this:
1. Keep these two warnings as warnings so users see them in their local builds, but
add some kind of string pattern to AB so it excludes missing qemu
support and missing wine support warnings.

Quoted 8 lines

2. Turn them into notes, which means they will go to the logs only, and users are not going to see them ever. On the other hand, the postinsts tend to
create things like font caches and similar, which should not generally
be needed at compile time (which is what SDKs are for).

I'm leaning towards option two.

I was leaning toward keeping them warnings and then suppressing them in the cases where we know they don't work and are unnecessary (perhaps with a variable like POSTINST_INTERCEPT_${PN} = "0" ?)

That way if a user enables a package where it's not going to run and we haven't previously evaluated it to be OK, they will at least see the warning. I naively think this might help reduce the number of bugs filed, or at least make them easier to triage :)

Quoted 90 lines

Alex

Quoted 85 lines · Joshua Watt
Quoted 64 lines · On Thu, Jan 24, 2019, 4:43 PM Richard Purdie <…
Quoted 39 lines · Alexander Kanavin

The few cases where they failed should be now all fixed. The only allowed
exception is when building mingw32 SDKs, as there is currently no support for running
postinst_intercepts through wine.

---
 meta/lib/oe/package_manager.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)
meta/lib/oe/package_manager.py +0 -0
index 1087144d47f..f26f597d03d 100644
@@ -439,6 +439,11 @@ class PackageManager(object,

metaclass=ABCMeta):
self._postpone_to_first_boot(script_full) continue

+ if populate_sdk == 'host' and
self.d.getVar('SDK_OS') == 'mingw32':
+ bb.warn("The postinstall intercept hook '%s'
could not be executed due to missing wine support, details in %s/log.do_%s"
+ % (script, self.d.getVar('T'),
self.d.getVar('BB_CURRENTTASK')))

meta/lib/oe/package_manager.py +2 -0
+                continue
+
             bb.note("> Executing %s intercept ..." % script)
 
             try:
@@ -447,7 +452,7 @@ class PackageManager(object,

metaclass=ABCMeta):
except subprocess.CalledProcessError as e:
bb.note("Exit code %d. Output:\n%s" %
(e.returncode, e.output.decode("utf-8")))

                 if populate_sdk == 'host':
Quoted 12 lines

- bb.warn("The postinstall intercept hook
'%s' failed, details in %s/log.do_%s" % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK')))
+ bb.fatal("The postinstall intercept hook
'%s' failed, details in %s/log.do_%s" % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK')))
elif populate_sdk == 'target':
if "qemuwrapper: qemu usermode is not
supported" in e.output.decode("utf-8"):
bb.warn("The postinstall intercept
hook '%s' could not be executed due to missing qemu usermode support, details in %s/log.do_%s"

I'm ok with this but could we change the bb.warn to something less
important like a note?

The reason I say this is that I'd like to get the autobuilder to the
point where its warning free. This warning is the only reason meta-
mingw shows warnings now. Either we fix them or if we can't, they probably aren't warnings.

Quoted 3 lines

What should we do here?

(also cc'd Joshua)

I think the one that fails currently is updating the GIO module cache. I'm not sure if this is even necessary for mingw (my guess is no, since this has never worked and no one has complained). We probably just need to disable it in mingw, but when I looked very briefly a while ago, I didn't see a straight forward way of doing that.

See https://bugzilla.yoctoproject.org/show_bug.cgi?id=12962

Quoted 4 lines

Cheers,

Richard

Signature · 1 line
Joshua Watt <[email protected]>
#8 lore raw

Re: [PATCH 04/10] lib/oe/package_manager: turn postinst_intercept warnings into failures for nativesdk

Quoted 20 lines · Joshua Watt
Quoted 10 lines · Alexander Kanavin

There is a similar issue in multilib for target packages (a warning because qemu usermode support is missing):
https://autobuilder.yoctoproject.org/typhoon/#/builders/44/builds/214/steps/7/logs/warnings

I'm as well not sure how to handle this:
1. Keep these two warnings as warnings so users see them in their local builds, but
add some kind of string pattern to AB so it excludes missing qemu
support and missing wine support warnings.

Quoted 8 lines

2. Turn them into notes, which means they will go to the logs only, and users are not going to see them ever. On the other hand, the postinsts tend to
create things like font caches and similar, which should not generally
be needed at compile time (which is what SDKs are for).

I'm leaning towards option two.

Quoted 8 lines

I was leaning toward keeping them warnings and then suppressing them in the cases where we know they don't work and are unnecessary (perhaps with a variable like POSTINST_INTERCEPT_${PN} = "0" ?)

That way if a user enables a package where it's not going to run and we haven't previously evaluated it to be OK, they will at least see the warning. I naively think this might help reduce the number of bugs filed, or at least make them easier to triage :)

I think this may be safer...

Cheers,

Richard

#9 lore raw

Re: [PATCH 04/10] lib/oe/package_manager: turn postinst_intercept warnings into failures for nativesdk

Quoted 23 lines · Richard Purdie
Quoted 23 lines
Quoted 10 lines

There is a similar issue in multilib for target packages (a warning because qemu usermode support is missing):
https://autobuilder.yoctoproject.org/typhoon/#/builders/44/builds/214/steps/7/logs/warnings

I'm as well not sure how to handle this:
1. Keep these two warnings as warnings so users see them in their local builds, but
add some kind of string pattern to AB so it excludes missing qemu
support and missing wine support warnings.

Quoted 8 lines

2. Turn them into notes, which means they will go to the logs only, and users are not going to see them ever. On the other hand, the postinsts tend to
create things like font caches and similar, which should not generally
be needed at compile time (which is what SDKs are for).

I'm leaning towards option two.

I was leaning toward keeping them warnings and then suppressing them in the cases where we know they don't work and are unnecessary (perhaps with a variable like POSTINST_INTERCEPT_${PN} = "0" ?)

Quoted 6 lines
Quoted 4 lines

That way if a user enables a package where it's not going to run and we haven't previously evaluated it to be OK, they will at least see the warning. I naively think this might help reduce the number of bugs filed, or at least make them easier to triage :)

I think this may be safer...

The postinst_intercepts are not executed per package (and accordingly can't be suppressed in that manner), they are run as a set of scripts available in scripts/postinst-intercepts/ during "populate_sdk" phase for an image. I have sent a patch where both specific failures are turned from warning to note (mingw, populating the target sysroot for nativesdk) - both should be actually okay to suppress, as
a) mingw case never worked and has not been a problem as Joshua says;
b) target nativesdk sysroot should not be a problem either, as it will be used just to build stuff, in a way similar to target Yocto builds, where those intercept scripts are not executed either, at least not until we populate a rootfs for the final image - and this has not been an issue for anyone. The scripts all create some kind of 'caches' which are only needed at runtime really.

Hope this helps,
Alex

#10 lore raw

Re: [PATCH 04/10] lib/oe/package_manager: turn postinst_intercept warnings into failures for nativesdk

Quoted 42 lines · Alexander Kanavin
Quoted 38 lines · Richard Purdie
Quoted 30 lines
Quoted 22 lines

There is a similar issue in multilib for target packages (a warning
because qemu usermode support is missing):
https://autobuilder.yoctoproject.org/typhoon/#/builders/44/builds/214/steps/7/logs/warnings

I'm as well not sure how to handle this:
1. Keep these two warnings as warnings so users see them in their
local builds, but
add some kind of string pattern to AB so it excludes missing
qemu
support and missing wine support warnings.

2. Turn them into notes, which means they will go to the logs only,
and users are not going to see them ever. On the other hand, the
postinsts tend to
create things like font caches and similar, which should not generally
be needed at compile time (which is what SDKs are for).

Quoted 1 line

I'm leaning towards option two.

I was leaning toward keeping them warnings and then suppressing them in
the cases where we know they don't work and are unnecessary (perhaps
with a variable like POSTINST_INTERCEPT_${PN} = "0" ?)

Quoted 7 lines

That way if a user enables a package where it's not going to run and we
haven't previously evaluated it to be OK, they will at least see the
warning. I naively think this might help reduce the number of bugs
filed, or at least make them easier to triage :)

Quoted 1 line

I think this may be safer...

Quoted 15 lines

The postinst_intercepts are not executed per package (and accordingly can't be suppressed in that manner), they are run as a set of scripts available in scripts/postinst-intercepts/ during "populate_sdk" phase for an image. I have sent a patch where both specific failures are turned from warning to note (mingw, populating the target sysroot for nativesdk) - both should be actually okay to suppress, as
a) mingw case never worked and has not been a problem as Joshua says;
b) target nativesdk sysroot should not be a problem either, as it will
be used just to build stuff, in a way similar to target Yocto builds, where those intercept scripts are not executed either, at least not until we populate a rootfs for the final image - and this has not been
an issue for anyone. The scripts all create some kind of 'caches' which are only needed at runtime really.

I probably should have been a little more specific: I was thinking that POSTINST_INTERCEPT_${PN} = "0" would prevent the creation of the scripts at packaging time, not try and suppress them at rootfs creation time. The idea would be to add that to relevant recipes with a appropriate override in a .bbappend in meta-mingw. I'm not familiar enough with multilib; is there a an appropriate override that could be use there?

I think this would also have the advantage that mingw32 specific logic in package_manager.py could be removed; we'll just suppress the postinst intercept scripts in meta-mingw and any that aren't will generate the typical failure warning.

Quoted 3 lines

Hope this helps,
Alex

Signature · 1 line
Joshua Watt <[email protected]>
#11 lore raw

Re: [PATCH 04/10] lib/oe/package_manager: turn postinst_intercept warnings into failures for nativesdk

Quoted 8 lines · Joshua Watt

I probably should have been a little more specific: I was thinking that POSTINST_INTERCEPT_${PN} = "0" would prevent the creation of the scripts at packaging time, not try and suppress them at rootfs creation time. The idea would be to add that to relevant recipes with a appropriate override in a .bbappend in meta-mingw. I'm not familiar enough with multilib; is there a an appropriate override that could be use there?

Quoted 4 lines

I think this would also have the advantage that mingw32 specific logic in package_manager.py could be removed; we'll just suppress the postinst intercept scripts in meta-mingw and any that aren't will generate the typical failure warning.

The problem with multilib case is that the same target package is used for populating the target sysroot in the nativesdk, and for populating the target image rootfs. In the former case, postinst_intercept should be suppressed, in the latter, it should not. But we won't know that until populating the sdk, or creating the image. I still think that changing the warnings to notes should not be harmful; the scripts all create caches of some kind, which are generally unneeded at build time. Or we would run into issues with plain yocto builds already, as we don't create those caches either.

Alex

Alex

#12 lore raw

PATCH 5/10 cmake: do not look into native sysroot in the nativesdk environment

In reply to Alexander Kanavin

I am not sure why we do this in the first place, but it is causing cmake to erroneously pick up items from the native sysroot when building for the target and the target item is missing, for example:

https://autobuilder.yoctoproject.org/typhoon/#/builders/59/builds/198/steps/7/logs/step2c

Note that for executable programs this variable is not referred to, as set by
set( CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER )
in the same file.

---
 meta/recipes-devtools/cmake/cmake/OEToolchainConfig.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
meta/recipes-devtools/cmake/cmake/OEToolchainConfig.cmake +1 -1
index 8a0fb4cb12e..398069eef28 100644
@@ -5,7 +5,7 @@ set( CMAKE_ASM_FLAGS ${CMAKE_C_FLAGS} CACHE STRING "" FORCE )
 set( CMAKE_LDFLAGS_FLAGS ${CMAKE_CXX_FLAGS} CACHE STRING "" FORCE )
 set( CMAKE_SYSROOT $ENV{OECORE_TARGET_SYSROOT} )
 
-set( CMAKE_FIND_ROOT_PATH $ENV{OECORE_TARGET_SYSROOT} $ENV{OECORE_NATIVE_SYSROOT} )
+set( CMAKE_FIND_ROOT_PATH $ENV{OECORE_TARGET_SYSROOT} )
 set( CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER )
 set( CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY )
 set( CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY )
Signature · 1 line
2.17.1
#13 lore raw

PATCH 6/10 testimage.bbclass: add support for passing runqemu params

In reply to Alexander Kanavin

This is particularly useful when setting up GL tests.

---
 meta/classes/testimage.bbclass    | 4 +++-
 meta/lib/oeqa/core/target/qemu.py | 4 ++--
 2 files changed, 5 insertions(+), 3 deletions(-)
meta/classes/testimage.bbclass +3 -1
index cb8c12accee..ff1c53b93e1 100644
@@ -32,6 +32,7 @@ TESTIMAGE_AUTO ??= "0"
 # Booting is handled by this class, and it's not a test in itself.
 # TEST_QEMUBOOT_TIMEOUT can be used to set the maximum time in seconds the launch code will wait for the login prompt.
 # TEST_QEMUPARAMS can be used to pass extra parameters to qemu, e.g. "-m 1024" for setting the amount of ram to 1 GB.
+# TEST_RUNQEMUPARAMS can be used to pass extra parameters to runqemu, e.g. "gl" to enable OpenGL acceleration.
 
 TEST_LOG_DIR ?= "${WORKDIR}/testimage"
 
@@ -65,6 +66,7 @@ TEST_SUITES ?= "${DEFAULT_TEST_SUITES}"
 TEST_QEMUBOOT_TIMEOUT ?= "1000"
 TEST_TARGET ?= "qemu"
 TEST_QEMUPARAMS ?= ""
+TEST_RUNQEMUPARAMS ?= ""
 
 TESTIMAGEDEPENDS = ""
 TESTIMAGEDEPENDS_append_qemuall = " qemu-native:do_populate_sysroot qemu-helper-native:do_populate_sysroot qemu-helper-native:do_addto_recipe_sysroot"
@@ -294,7 +296,7 @@ def testimage_main(d):
     try:
         # We need to check if runqemu ends unexpectedly
         # or if the worker send us a SIGTERM
-        tc.target.start(params=d.getVar("TEST_QEMUPARAMS"))
+        tc.target.start(params=d.getVar("TEST_QEMUPARAMS"), runqemuparams=d.getVar("TEST_RUNQEMUPARAMS"))
         results = tc.runTests()
     except (RuntimeError, BlockingIOError) as err:
         if isinstance(err, RuntimeError):
meta/lib/oeqa/core/target/qemu.py +2 -2
index f47fd7468ed..7a161a32319 100644
@@ -33,11 +33,11 @@ class OEQemuTarget(OESSHTarget):
                                  use_kvm=kvm, use_slirp=slirp, dump_dir=dump_dir,
                                  dump_host_cmds=dump_host_cmds, logger=logger)
 
-    def start(self, params=None, extra_bootparams=None):
+    def start(self, params=None, extra_bootparams=None, runqemuparams=''):
         if self.use_slirp and not self.server_ip:
             self.logger.error("Could not start qemu with slirp without server ip - provide 'TEST_SERVER_IP'")
             raise RuntimeError("FAILED to start qemu - check the task log and the boot log")
-        if self.runner.start(params, extra_bootparams=extra_bootparams):
+        if self.runner.start(params, extra_bootparams=extra_bootparams, runqemuparams=runqemuparams):
             self.ip = self.runner.ip
             if self.use_slirp:
                 target_ip_port = self.runner.ip.split(':')
Signature · 1 line
2.17.1
#14 lore raw

PATCH 7/10 kmscube: update to latest commit, switch over to meson

In reply to Alexander Kanavin
---
 meta/recipes-graphics/kmscube/kmscube_git.bb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
meta/recipes-graphics/kmscube/kmscube_git.bb +2 -2
index 46aeeb01795..c6cb1842158 100644
@@ -6,13 +6,13 @@ DEPENDS = "virtual/libgles2 virtual/egl libdrm gstreamer1.0 gstreamer1.0-plugins
 
 LIC_FILES_CHKSUM = "file://kmscube.c;beginline=1;endline=23;md5=8b309d4ee67b7315ff7381270dd631fb"
 
-SRCREV = "9dcce71e603616ee7a54707e932f962cdf8fb20a"
+SRCREV = "d8da3dcfdfe33ee525cf562e928a5266ac69843c"
 SRC_URI = "git://anongit.freedesktop.org/mesa/kmscube;branch=master;protocol=git \
     file://detect-gst_bo_map-_unmap-and-use-it-or-avoid-it.patch"
 UPSTREAM_CHECK_COMMITS = "1"
 
 S = "${WORKDIR}/git"
 
-inherit autotools pkgconfig distro_features_check
+inherit meson pkgconfig distro_features_check
 
 REQUIRED_DISTRO_FEATURES = "opengl"
Signature · 1 line
2.17.1
#15 lore raw

PATCH 8/10 kmscube: make gstreamer dependency optional

In reply to Alexander Kanavin

This in particular saves build times for virgl oe-selftest.

---
 meta/recipes-graphics/kmscube/kmscube_git.bb | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)
meta/recipes-graphics/kmscube/kmscube_git.bb +4 -1
index c6cb1842158..b2dd6b30b0e 100644
@@ -2,7 +2,7 @@ DESCRIPTION = "Demo application to showcase 3D graphics using kms and gbm"
 HOMEPAGE = "https://cgit.freedesktop.org/mesa/kmscube/"
 LICENSE = "MIT"
 SECTION = "graphics"
-DEPENDS = "virtual/libgles2 virtual/egl libdrm gstreamer1.0 gstreamer1.0-plugins-base"
+DEPENDS = "virtual/libgles2 virtual/egl libdrm"
 
 LIC_FILES_CHKSUM = "file://kmscube.c;beginline=1;endline=23;md5=8b309d4ee67b7315ff7381270dd631fb"
 
@@ -16,3 +16,6 @@ S = "${WORKDIR}/git"
 inherit meson pkgconfig distro_features_check
 
 REQUIRED_DISTRO_FEATURES = "opengl"
+
+PACKAGECONFIG ??= ""
+PACKAGECONFIG[gstreamer] = "-Dgstreamer=enabled,-Dgstreamer=disabled,gstreamer1.0 gstreamer1.0-plugins-base"
Signature · 1 line
2.17.1
#16 lore raw

PATCH 9/10 lib/oe/reciputils.py: parallelize upstream version checks

In reply to Alexander Kanavin

Previously this was done via bitbake tasks, and when this was rewritten to a for loop, performance sufered significantly:
from 90 seconds to about 12 minutes for oe-core. This change restores the previous run time, and makes it possible to perform such checks with command line utilities in an interactive way.

Implementation note: we have to create a copy of the recipe data, as Tinfoil API can't be used from multiple threads and only allows one process to access the data at a time.

---
 meta/lib/oe/recipeutils.py | 77 +++++++++++++++++++++++++++-----------
 1 file changed, 56 insertions(+), 21 deletions(-)
meta/lib/oe/recipeutils.py +56 -21
index 39d3de4bb1f..92c0f65257f 100644
@@ -1020,8 +1020,54 @@ def get_recipe_upstream_version(rd):
 
     return ru
 
+def _get_recipe_upgrade_status(data):
+    uv = get_recipe_upstream_version(data)
+
+    pn = data.getVar('PN')
+    cur_ver = uv['current_version']
+
+    upstream_version_unknown = data.getVar('UPSTREAM_VERSION_UNKNOWN')
+    if not uv['version']:
+        status = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN"
+    else:
+        cmp = vercmp_string(uv['current_version'], uv['version'])
+        if cmp == -1:
+            status = "UPDATE" if not upstream_version_unknown else "KNOWN_BROKEN"
+        elif cmp == 0:
+            status = "MATCH" if not upstream_version_unknown else "KNOWN_BROKEN"
+        else:
+            status = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN"
+
+    next_ver = uv['version'] if uv['version'] else "N/A"
+    revision = uv['revision'] if uv['revision'] else "N/A"
+    maintainer = data.getVar('RECIPE_MAINTAINER')
+    no_upgrade_reason = data.getVar('RECIPE_NO_UPDATE_REASON')
+
+    return (pn, status, cur_ver, next_ver, maintainer, revision, no_upgrade_reason)
+
 def get_recipe_upgrade_status(recipes=None):
     pkgs_list = []
+    data_copy_list = []
+    copy_vars = ('SRC_URI',
+                 'PV',
+                 'GITDIR',
+                 'DL_DIR',
+                 'PN',
+                 'CACHE',
+                 'PERSISTENT_DIR',
+                 'BB_URI_HEADREVS',
+                 'UPSTREAM_CHECK_COMMITS',
+                 'UPSTREAM_CHECK_GITTAGREGEX',
+                 'UPSTREAM_CHECK_REGEX',
+                 'UPSTREAM_CHECK_URI',
+                 'UPSTREAM_VERSION_UNKNOWN',
+                 'RECIPE_MAINTAINER',
+                 'RECIPE_NO_UPDATE_REASON',
+                 'RECIPE_UPSTREAM_VERSION',
+                 'RECIPE_UPSTREAM_DATE',
+                 'CHECK_DATE',
+            )
+
     with bb.tinfoil.Tinfoil() as tinfoil:
         tinfoil.prepare(config_only=False)
 
@@ -1043,28 +1089,17 @@ def get_recipe_upgrade_status(recipes=None):
                 bb.note(" Skip package %s as upstream check unreliable" % pn)
                 continue
 
-            uv = get_recipe_upstream_version(data)
-
-            pn = data.getVar('PN')
-            cur_ver = uv['current_version']
-
-            upstream_version_unknown = data.getVar('UPSTREAM_VERSION_UNKNOWN')
-            if not uv['version']:
-                status = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN"
-            else:
-                cmp = vercmp_string(uv['current_version'], uv['version'])
-                if cmp == -1:
-                    status = "UPDATE" if not upstream_version_unknown else "KNOWN_BROKEN"
-                elif cmp == 0:
-                    status = "MATCH" if not upstream_version_unknown else "KNOWN_BROKEN"
-                else:
-                    status = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN"
+            data_copy = bb.data.init()
+            for var in copy_vars:
+                data_copy.setVar(var, data.getVar(var))
+            for k in data:
+                if k.startswith('SRCREV'):
+                    data_copy.setVar(k, data.getVar(k))
 
-            next_ver = uv['version'] if uv['version'] else "N/A"
-            revision = uv['revision'] if uv['revision'] else "N/A"
-            maintainer = data.getVar('RECIPE_MAINTAINER')
-            no_upgrade_reason = data.getVar('RECIPE_NO_UPDATE_REASON')
+            data_copy_list.append(data_copy)
 
-            pkgs_list.append((pn, status, cur_ver, next_ver, maintainer, revision, no_upgrade_reason))
+    from concurrent.futures import ProcessPoolExecutor
+    with ProcessPoolExecutor(max_workers=utils.cpu_count()) as executor:
+        pkgs_list = executor.map(_get_recipe_upgrade_status, data_copy_list)
 
     return pkgs_list
Signature · 1 line
2.17.1
#17 lore raw

PATCH 10/10 devtool: add a command to print an overall list of recipes that can be updated

In reply to Alexander Kanavin

A sample portion of the output:

$ devtool check-upgrade-status
...
NOTE: acpid                     2.0.30          2.0.31          Ross Burton <[email protected]>
NOTE: u-boot-fw-utils           2018.11         2019.01         Marek Vasut <[email protected]> d3689267f92c5956e09cc7d1baa4700141662bff
NOTE: u-boot-tools              2018.11         2019.01         Marek Vasut <[email protected]> d3689267f92c5956e09cc7d1baa4700141662bff
NOTE: u-boot                    2018.11         2019.01         Marek Vasut <[email protected]> d3689267f92c5956e09cc7d1baa4700141662bff
NOTE: bind                      9.11.5          9.13.5          Armin Kuster <[email protected]>  cannot be updated due to: 9.11 is LTS 2021
NOTE: iproute2                  4.19.0          4.20.0          Changhyeok Bae <[email protected]>
NOTE: ofono                     1.25            1.27            Ross Burton <[email protected]>
NOTE: wpa-supplicant            2.6             2.7             Changhyeok Bae <[email protected]>
NOTE: base-passwd               3.5.29          3.5.45          Anuj Mittal <[email protected]>  cannot be updated due to: Version 3.5.38 requires cdebconf for update-passwd utility
NOTE: busybox                   1.29.2          1.30.0          Andrej Valek <[email protected]>
NOTE: dbus-test                 1.12.10         1.12.12         Chen Qi <[email protected]>
NOTE: dbus                      1.12.10         1.12.12         Chen Qi <[email protected]>
NOTE: glib-2.0                  2.58.0          2.58.3          Anuj Mittal <[email protected]>
NOTE: glib-networking           2.54.1          2.58.0          Anuj Mittal <[email protected]>
...
---
 scripts/lib/devtool/upgrade.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)
scripts/lib/devtool/upgrade.py +20 -0
index 202007793b2..42a4c0e4bff 100644
@@ -600,6 +600,20 @@ def latest_version(args, config, basepath, workspace):
         tinfoil.shutdown()
     return 0
 
+def check_upgrade_status(args, config, basepath, workspace):
+    if not args.recipe:
+        logger.info("Checking the upstream status for all recipes may take a few minutes")
+    results = oe.recipeutils.get_recipe_upgrade_status(args.recipe)
+    for result in results:
+        # pn, update_status, current, latest, maintainer, latest_commit, no_update_reason
+        if result[1] != 'MATCH':
+            logger.info("{:25} {:15} {:15} {} {} {}".format(   result[0],
+                                                               result[2],
+                                                               result[1] if result[1] != 'UPDATE' else (result[3] if not result[3].endswith("new-commits-available") else "new commits"),
+                                                               result[4],
+                                                               result[5] if result[5] != 'N/A' else "",
+                                                               "cannot be updated due to: %s" %(result[6]) if result[6] else ""))
+
 def register_commands(subparsers, context):
     """Register devtool subcommands from this plugin"""
 
@@ -627,3 +641,9 @@ def register_commands(subparsers, context):
                                                   group='info')
     parser_latest_version.add_argument('recipename', help='Name of recipe to query (just name - no version, path or extension)')
     parser_latest_version.set_defaults(func=latest_version)
+
+    parser_check_upgrade_status = subparsers.add_parser('check-upgrade-status', help="Report upgradability for multiple (or all) recipes",
+                                                        description="Prints a table of recipes together with versions currently provided by recipes, and latest upstream versions, when there is a later version available",
+                                                        group='info')
+    parser_check_upgrade_status.add_argument('recipe', help='Name of the recipe to report (omit to report upgrade info for all recipes)', nargs='*')
+    parser_check_upgrade_status.set_defaults(func=check_upgrade_status)
Signature · 1 line
2.17.1
#18 lore raw

Re: [PATCH 10/10] devtool: add a command to print an overall list of recipes that can be updated

Quoted 18 lines · Alexander Kanavin

A sample portion of the output:

$ devtool check-upgrade-status
...
NOTE: acpid                     2.0.30          2.0.31          Ross Burton <[email protected]>
NOTE: u-boot-fw-utils           2018.11         2019.01         Marek Vasut <[email protected]> d3689267f92c5956e09cc7d1baa4700141662bff
NOTE: u-boot-tools              2018.11         2019.01         Marek Vasut <[email protected]> d3689267f92c5956e09cc7d1baa4700141662bff
NOTE: u-boot                    2018.11         2019.01         Marek Vasut <[email protected]> d3689267f92c5956e09cc7d1baa4700141662bff
NOTE: bind                      9.11.5          9.13.5          Armin Kuster <[email protected]>  cannot be updated due to: 9.11 is LTS 2021
NOTE: iproute2                  4.19.0          4.20.0          Changhyeok Bae <[email protected]>
NOTE: ofono                     1.25            1.27            Ross Burton <[email protected]>
NOTE: wpa-supplicant            2.6             2.7             Changhyeok Bae <[email protected]>
NOTE: base-passwd               3.5.29          3.5.45          Anuj Mittal <[email protected]>  cannot be updated due to: Version 3.5.38 requires cdebconf for update-passwd utility
NOTE: busybox                   1.29.2          1.30.0          Andrej Valek <[email protected]>
NOTE: dbus-test                 1.12.10         1.12.12         Chen Qi <[email protected]>
NOTE: dbus                      1.12.10         1.12.12         Chen Qi <[email protected]>
NOTE: glib-2.0                  2.58.0          2.58.3          Anuj Mittal <[email protected]>
NOTE: glib-networking           2.54.1          2.58.0          Anuj Mittal <[email protected]>

this looks great.

thanks.

- armin

Quoted 42 lines

...

---
 scripts/lib/devtool/upgrade.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)
scripts/lib/devtool/upgrade.py +20 -0
index 202007793b2..42a4c0e4bff 100644
@@ -600,6 +600,20 @@ def latest_version(args, config, basepath, workspace):
         tinfoil.shutdown()
     return 0
 
+def check_upgrade_status(args, config, basepath, workspace):
+    if not args.recipe:
+        logger.info("Checking the upstream status for all recipes may take a few minutes")
+    results = oe.recipeutils.get_recipe_upgrade_status(args.recipe)
+    for result in results:
+        # pn, update_status, current, latest, maintainer, latest_commit, no_update_reason
+        if result[1] != 'MATCH':
+            logger.info("{:25} {:15} {:15} {} {} {}".format(   result[0],
+                                                               result[2],
+                                                               result[1] if result[1] != 'UPDATE' else (result[3] if not result[3].endswith("new-commits-available") else "new commits"),
+                                                               result[4],
+                                                               result[5] if result[5] != 'N/A' else "",
+                                                               "cannot be updated due to: %s" %(result[6]) if result[6] else ""))
+
 def register_commands(subparsers, context):
     """Register devtool subcommands from this plugin"""
 
@@ -627,3 +641,9 @@ def register_commands(subparsers, context):
                                                   group='info')
     parser_latest_version.add_argument('recipename', help='Name of recipe to query (just name - no version, path or extension)')
     parser_latest_version.set_defaults(func=latest_version)
+
+    parser_check_upgrade_status = subparsers.add_parser('check-upgrade-status', help="Report upgradability for multiple (or all) recipes",
+                                                        description="Prints a table of recipes together with versions currently provided by recipes, and latest upstream versions, when there is a later version available",
+                                                        group='info')
+    parser_check_upgrade_status.add_argument('recipe', help='Name of the recipe to report (omit to report upgrade info for all recipes)', nargs='*')
+    parser_check_upgrade_status.set_defaults(func=check_upgrade_status)