Skip to contentSkip to conversation

python3: split python target configuration into own class

PATCH In review [email protected] 2 participants 8 messages Dec 1, 2020 1 message not in the archive

lore
Share & tools
lore
#0 lore raw

PATCH 1/5 python3: split python target configuration into own class

From: Alexander Kanavin <[email protected]>

Setting _PYTHON_SYSCONFIGDATA_NAME in python3native class globally was problematic as it was leaking into host python environment, which was causing tracebacks depending on host distro and action (typically anything involving importing sysconfig module).

The new class sets the variable only in specific tasks where it is needed, and should be inherited explicitly:
- use python3native to run scripts with native python
- use python3targetconfig to run scripts with native python
if those scripts need to access target config data (such
as correct installation directories). This also adds a dependency on target python, so should be used carefully to avoid lengthening builds.

(From OE-Core rev: 5a118d4e7985fa88f04c3611f8db813f0dafce75)

(cherry picked from commit c9f7ed88a946078c27fa1a62d54d6310fad9d37a)
---
 meta/classes/python3native.bbclass       |  2 --
 meta/classes/python3targetconfig.bbclass | 15 +++++++++++++++
 2 files changed, 15 insertions(+), 2 deletions(-)
 create mode 100644 meta/classes/python3targetconfig.bbclass
meta/classes/python3native.bbclass +0 -2
index d98fb4c758..2e3a88c126 100644
@@ -17,8 +17,6 @@ export STAGING_LIBDIR
 export PYTHON_LIBRARY="${STAGING_LIBDIR}/lib${PYTHON_DIR}${PYTHON_ABI}.so"
 export PYTHON_INCLUDE_DIR="${STAGING_INCDIR}/${PYTHON_DIR}${PYTHON_ABI}"
 
-export _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata"
-
 # suppress host user's site-packages dirs.
 export PYTHONNOUSERSITE = "1"
 
meta/classes/python3targetconfig.bbclass new file +15 -0
new file mode 100644
index 0000000000..640d0c97b6
@@ -0,0 +1,15 @@
+inherit python3native
+
+DEPENDS_append = " python3"
+
+do_configure_prepend() {
+        export _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata"
+}
+
+do_compile_prepend() {
+        export _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata"
+}
+
+do_install_prepend() {
+        export _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata"
+}
Signature · 1 line
2.29.2
#1 lore raw

PATCH 2/5 python3-pycairo: use python3targetconfig

From: Alexander Kanavin <[email protected]>

(From OE-Core rev: dadf001c85938b831def8da5851a40dc0977e3d0)

(cherry picked from commit 5077e7abbacf639f63bd444517d1d59a27d502a1)
---
 meta/recipes-devtools/python/python3-pycairo_1.19.0.bb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
meta/recipes-devtools/python/python3-pycairo_1.19.0.bb +1 -1
index 8f60834c17..8452d7fa9f 100644
@@ -18,7 +18,7 @@ SRC_URI[sha256sum] = "4f5ba9374a46c98729dd3727d993f5e17ed0286fd6738ed464fe4efa06
 
 S = "${WORKDIR}/pycairo-${PV}"
 
-inherit meson pkgconfig
+inherit meson pkgconfig python3targetconfig
 
 CFLAGS += "-fPIC"
 
Signature · 1 line
2.29.2
#2 lore raw

PATCH 3/5 distutils3-base.bbclass: use python3targetconfig

In reply to [email protected]

From: Alexander Kanavin <[email protected]>

(From OE-Core rev: 9c8f666097802cb594a759989edcf01603a22df3)

(cherry picked from commit 3ec7f79ad2687381300010392c3320cca0dc638e)
---
 meta/classes/distutils3-base.bbclass | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
meta/classes/distutils3-base.bbclass +1 -1
index 7dbf07ac4b..a277d1c7bc 100644
@@ -1,5 +1,5 @@
 DEPENDS  += "${@["${PYTHON_PN}-native ${PYTHON_PN}", ""][(d.getVar('PACKAGES') == '')]}"
 RDEPENDS_${PN} += "${@['', '${PYTHON_PN}-core']['${CLASSOVERRIDE}' == 'class-target']}"
 
-inherit distutils-common-base python3native
+inherit distutils-common-base python3native python3targetconfig
 
Signature · 1 line
2.29.2
#3 lore raw

PATCH 4/5 meta: drop _PYTHON_SYSCONFIGDATA_NAME hacks

In reply to [email protected]

From: Alexander Kanavin <[email protected]>

(From OE-Core rev: d3a81dd0e72a3495bfc7cc969c2bb806b666023d)

(cherry picked from commit acfdab8daa3a67ebabc65fb7e152cc3a476b4333)
---
 meta/classes/scons.bbclass          | 3 ---
 meta/lib/oe/prservice.py            | 4 ----
 meta/recipes-core/glib-2.0/glib.inc | 4 ----
 meta/recipes-graphics/mesa/mesa.inc | 5 -----
 4 files changed, 16 deletions(-)
meta/classes/scons.bbclass +0 -3
index 6b171ca8df..4f3ae502ef 100644
@@ -5,7 +5,6 @@ DEPENDS += "python3-scons-native"
 EXTRA_OESCONS ?= ""
 
 do_configure() {
-	unset _PYTHON_SYSCONFIGDATA_NAME
 	if [ -n "${CONFIGURESTAMPFILE}" ]; then
 		if [ -e "${CONFIGURESTAMPFILE}" -a "`cat ${CONFIGURESTAMPFILE}`" != "${BB_TASKHASH}" -a "${CLEANBROKEN}" != "1" ]; then
 			${STAGING_BINDIR_NATIVE}/scons --clean PREFIX=${prefix} prefix=${prefix} ${EXTRA_OESCONS}
@@ -17,13 +16,11 @@ do_configure() {
 }
 
 scons_do_compile() {
-	unset _PYTHON_SYSCONFIGDATA_NAME
 	${STAGING_BINDIR_NATIVE}/scons ${PARALLEL_MAKE} PREFIX=${prefix} prefix=${prefix} ${EXTRA_OESCONS} || \
 	die "scons build execution failed."
 }
 
 scons_do_install() {
-	unset _PYTHON_SYSCONFIGDATA_NAME
 	${STAGING_BINDIR_NATIVE}/scons install_root=${D}${prefix} PREFIX=${prefix} prefix=${prefix} ${EXTRA_OESCONS} install || \
 	die "scons install execution failed."
 }
meta/lib/oe/prservice.py +0 -4
index 2d3c9c7e50..fcdbe66c19 100644
@@ -3,10 +3,6 @@
 #
 
 def prserv_make_conn(d, check = False):
-    # Otherwise this fails when called from recipes which e.g. inherit python3native (which sets _PYTHON_SYSCONFIGDATA_NAME) with:
-    # No module named '_sysconfigdata'
-    if '_PYTHON_SYSCONFIGDATA_NAME' in os.environ:
-        del os.environ['_PYTHON_SYSCONFIGDATA_NAME']
     import prserv.serv
     host_params = list([_f for _f in (d.getVar("PRSERV_HOST") or '').split(':') if _f])
     try:
meta/recipes-core/glib-2.0/glib.inc +0 -4
index 7ebed0e5fd..c3ddf18387 100644
@@ -32,10 +32,6 @@ inherit meson gettext gtk-doc pkgconfig ptest-gnome upstream-version-is-even bas
 
 GTKDOC_MESON_OPTION = "gtk_doc"
 
-# This avoids the need to depend on target python3, which in case of mingw is not even possible.
-# meson's python configuration pokes into python3 configuration, so this provides the native config to it.
-unset _PYTHON_SYSCONFIGDATA_NAME
-
 S = "${WORKDIR}/glib-${PV}"
 
 PACKAGECONFIG ??= "system-pcre libmount \
meta/recipes-graphics/mesa/mesa.inc +0 -5
index b7ef496fdc..a1bf878b1a 100644
@@ -26,11 +26,6 @@ PROVIDES = " \
 
 inherit meson pkgconfig python3native gettext features_check
 
-# Unset these to stop python trying to report the target Python setup
-_PYTHON_SYSCONFIGDATA_NAME[unexport] = "1"
-STAGING_INCDIR[unexport] = "1"
-STAGING_LIBDIR[unexport] = "1"
-
 BBCLASSEXTEND = "native nativesdk"
 
 ANY_OF_DISTRO_FEATURES_class-target = "opengl vulkan"
Signature · 1 line
2.29.2
#4 lore raw

PATCH 5/5 gpgme: use python3targetconfig

In reply to [email protected]

From: Alexander Kanavin <[email protected]>

(From OE-Core rev: 38ecb83c444406b5157712d87aef3bbb320b45ec)

(cherry picked from commit de2a5404d3ef1a5dda7c5bb66c5a338883e249e6)
---
 meta/recipes-support/gpgme/gpgme_1.13.1.bb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
meta/recipes-support/gpgme/gpgme_1.13.1.bb +1 -1
index b51534351d..6e945d3165 100644
@@ -49,7 +49,7 @@ DEFAULT_LANGUAGES_class-target = "cpp"
 LANGUAGES ?= "${DEFAULT_LANGUAGES} python"
 
 PYTHON_INHERIT = "${@bb.utils.contains('PACKAGECONFIG', 'python2', 'pythonnative', '', d)}"
-PYTHON_INHERIT .= "${@bb.utils.contains('PACKAGECONFIG', 'python3', 'python3native', '', d)}"
+PYTHON_INHERIT .= "${@bb.utils.contains('PACKAGECONFIG', 'python3', 'python3native python3targetconfig', '', d)}"
 
 EXTRA_OECONF += '--enable-languages="${LANGUAGES}" \
                  --disable-gpgconf-test \
Signature · 1 line
2.29.2
#5 lore raw
In reply to [email protected]

Not sure if this is worth backport (it's very useful, but requires changes in other layers - and these changes aren't backwards compatible so they would need to be applied only in dunfell and not gatesgarth branches of these layers or the same patchset would need to land in oe-core/gatesgarth first).

But you're definitely missing this fix from master:
https://git.openembedded.org/openembedded-core/commit/?id=c99bb79087e74a967286469e1d8888a546ebec83

Quoted 69 lines · Ernst Sjöstrand

From: Alexander Kanavin <[email protected]>

Setting _PYTHON_SYSCONFIGDATA_NAME in python3native class globally was problematic as it was leaking into host python environment, which was causing tracebacks depending on host distro and action (typically anything involving importing sysconfig module).

The new class sets the variable only in specific tasks where it is needed, and should be inherited explicitly:
- use python3native to run scripts with native python
- use python3targetconfig to run scripts with native python
if those scripts need to access target config data (such
as correct installation directories). This also adds a dependency on target python, so should be used carefully to avoid lengthening builds.

(From OE-Core rev: 5a118d4e7985fa88f04c3611f8db813f0dafce75)

(cherry picked from commit c9f7ed88a946078c27fa1a62d54d6310fad9d37a)
---
 meta/classes/python3native.bbclass       |  2 --
 meta/classes/python3targetconfig.bbclass | 15 +++++++++++++++
 2 files changed, 15 insertions(+), 2 deletions(-)
 create mode 100644 meta/classes/python3targetconfig.bbclass
meta/classes/python3native.bbclass +0 -2
index d98fb4c758..2e3a88c126 100644
@@ -17,8 +17,6 @@ export STAGING_LIBDIR
 export PYTHON_LIBRARY="${STAGING_LIBDIR}/lib${PYTHON_DIR}${PYTHON_ABI}.so"
 export PYTHON_INCLUDE_DIR="${STAGING_INCDIR}/${PYTHON_DIR}${PYTHON_ABI}"
 
-export _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata"
-
 # suppress host user's site-packages dirs.
 export PYTHONNOUSERSITE = "1"
 
meta/classes/python3targetconfig.bbclass new file +15 -1
new file mode 100644
index 0000000000..640d0c97b6
@@ -0,0 +1,15 @@
+inherit python3native
+
+DEPENDS_append = " python3"
+
+do_configure_prepend() {
+        export _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata"
+}
+
+do_compile_prepend() {
+        export _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata"
+}
+
+do_install_prepend() {
+        export _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata"
+}
--

2.29.2

#7 lore raw
In reply to a message not in the archive

Also all these e-mails belong to openembedded-core ML not bitbake-devel.

Quoted 85 lines · Martin Jansa via lists.openembedded.org

Not sure if this is worth backport (it's very useful, but requires changes in other layers - and these changes aren't backwards compatible so they would need to be applied only in dunfell and not gatesgarth branches of these layers or the same patchset would need to land in oe-core/gatesgarth first).

But you're definitely missing this fix from master:

https://git.openembedded.org/openembedded-core/commit/?id=c99bb79087e74a967286469e1d8888a546ebec83

Quoted 70 lines · Ernst Sjöstrand

From: Alexander Kanavin <[email protected]>

Setting _PYTHON_SYSCONFIGDATA_NAME in python3native class globally was problematic as it was leaking into host python environment, which was causing tracebacks depending on host distro and action (typically anything involving importing sysconfig module).

The new class sets the variable only in specific tasks where it is needed, and should be inherited explicitly:
- use python3native to run scripts with native python
- use python3targetconfig to run scripts with native python
if those scripts need to access target config data (such
as correct installation directories). This also adds a dependency on target python, so should be used carefully to avoid lengthening builds.

(From OE-Core rev: 5a118d4e7985fa88f04c3611f8db813f0dafce75)

(cherry picked from commit c9f7ed88a946078c27fa1a62d54d6310fad9d37a)
---
 meta/classes/python3native.bbclass       |  2 --
 meta/classes/python3targetconfig.bbclass | 15 +++++++++++++++
 2 files changed, 15 insertions(+), 2 deletions(-)
 create mode 100644 meta/classes/python3targetconfig.bbclass
meta/classes/python3native.bbclass +0 -0
index d98fb4c758..2e3a88c126 100644
@@ -17,8 +17,6 @@ export STAGING_LIBDIR
 export

PYTHON_LIBRARY="${STAGING_LIBDIR}/lib${PYTHON_DIR}${PYTHON_ABI}.so"
export PYTHON_INCLUDE_DIR="${STAGING_INCDIR}/${PYTHON_DIR}${PYTHON_ABI}"

meta/classes/python3native.bbclass +0 -2
-export _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata"
-
 # suppress host user's site-packages dirs.
 export PYTHONNOUSERSITE = "1"
meta/classes/python3targetconfig.bbclass new file +15 -1
new file mode 100644
index 0000000000..640d0c97b6
@@ -0,0 +1,15 @@
+inherit python3native
+
+DEPENDS_append = " python3"
+
+do_configure_prepend() {
+        export _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata"
+}
+
+do_compile_prepend() {
+        export _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata"
+}
+
+do_install_prepend() {
+        export _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata"
+}
--

2.29.2

#8 lore raw

Yeah I was preparing for both lists and got it mixed up. :-) There's another patch that goes to the bitbake list.
(bitbake: lib/bb/fetch2/__init__.py: drop _PYTHON_SYSCONFIGDATA_NAME unsetting)

Ah the patch you mentioned is a bit newer than my initial local backport.

The reason for the backport is that many of the initial reports were for Dunfell, and that Dunfell will be LTS.

Regards
//Ernst

Den tis 1 dec. 2020 kl 17:32 skrev Martin Jansa <[email protected]>:

Quoted 92 lines

Also all these e-mails belong to openembedded-core ML not bitbake-devel.

Quoted 87 lines · Martin Jansa via lists.openembedded.org

Not sure if this is worth backport (it's very useful, but requires changes in other layers - and these changes aren't backwards compatible so they would need to be applied only in dunfell and not gatesgarth branches of these layers or the same patchset would need to land in oe-core/gatesgarth first).

But you're definitely missing this fix from master:

https://git.openembedded.org/openembedded-core/commit/?id=c99bb79087e74a967286469e1d8888a546ebec83

Quoted 72 lines · Ernst Sjöstrand

From: Alexander Kanavin <[email protected]>

Setting _PYTHON_SYSCONFIGDATA_NAME in python3native class globally was problematic as it was leaking into host python environment, which was causing tracebacks depending on host distro and action (typically anything involving importing sysconfig module).

The new class sets the variable only in specific tasks where it is needed,
and should be inherited explicitly:
- use python3native to run scripts with native python
- use python3targetconfig to run scripts with native python if those scripts need to access target config data (such as correct installation directories). This also adds a dependency on target python, so should be used carefully to avoid lengthening builds.

(From OE-Core rev: 5a118d4e7985fa88f04c3611f8db813f0dafce75)

(cherry picked from commit c9f7ed88a946078c27fa1a62d54d6310fad9d37a)
---
 meta/classes/python3native.bbclass       |  2 --
 meta/classes/python3targetconfig.bbclass | 15 +++++++++++++++
 2 files changed, 15 insertions(+), 2 deletions(-)
 create mode 100644 meta/classes/python3targetconfig.bbclass
meta/classes/python3native.bbclass +0 -0
index d98fb4c758..2e3a88c126 100644
@@ -17,8 +17,6 @@ export STAGING_LIBDIR
 export

PYTHON_LIBRARY="${STAGING_LIBDIR}/lib${PYTHON_DIR}${PYTHON_ABI}.so"
export PYTHON_INCLUDE_DIR="${STAGING_INCDIR}/${PYTHON_DIR}${PYTHON_ABI}"

meta/classes/python3native.bbclass +0 -2
-export _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata"
-
 # suppress host user's site-packages dirs.
 export PYTHONNOUSERSITE = "1"
meta/classes/python3targetconfig.bbclass new file +15 -1
new file mode 100644
index 0000000000..640d0c97b6
@@ -0,0 +1,15 @@
+inherit python3native
+
+DEPENDS_append = " python3"
+
+do_configure_prepend() {
+        export _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata"
+}
+
+do_compile_prepend() {
+        export _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata"
+}
+
+do_install_prepend() {
+        export _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata"
+}
--

2.29.2