Skip to contentSkip to conversation

python3: split python target configuration into own class

PATCH In review Alexander Kanavin 4 participants 8 messages Nov 13, 2020 – Nov 17, 2020

lore
Share & tools
lore
#0 lore raw

PATCH 1/5 python3: split python target configuration into own class

Setting _PYTHON_SYSCONFIGDATA_NAME in python3native class globally was problematic as it was leaking into host python environment, which was causing tracebacks depending on host distro and action (typically anything involving importing sysconfig module).

The new class sets the variable only in specific tasks where it is needed, and should be inherited explicitly:
- use python3native to run scripts with native python
- use python3targetconfig to run scripts with native python
if those scripts need to access target config data (such
as correct installation directories). This also adds a dependency on target python, so should be used carefully to avoid lengthening builds.

---
 meta/classes/python3native.bbclass       |  2 --
 meta/classes/python3targetconfig.bbclass | 15 +++++++++++++++
 2 files changed, 15 insertions(+), 2 deletions(-)
 create mode 100644 meta/classes/python3targetconfig.bbclass
meta/classes/python3native.bbclass +0 -2
index d98fb4c758..2e3a88c126 100644
@@ -17,8 +17,6 @@ export STAGING_LIBDIR
 export PYTHON_LIBRARY="${STAGING_LIBDIR}/lib${PYTHON_DIR}${PYTHON_ABI}.so"
 export PYTHON_INCLUDE_DIR="${STAGING_INCDIR}/${PYTHON_DIR}${PYTHON_ABI}"
 
-export _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata"
-
 # suppress host user's site-packages dirs.
 export PYTHONNOUSERSITE = "1"
 
meta/classes/python3targetconfig.bbclass new file +15 -0
new file mode 100644
index 0000000000..640d0c97b6
@@ -0,0 +1,15 @@
+inherit python3native
+
+DEPENDS_append = " python3"
+
+do_configure_prepend() {
+        export _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata"
+}
+
+do_compile_prepend() {
+        export _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata"
+}
+
+do_install_prepend() {
+        export _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata"
+}
Signature · 1 line
2.29.2
#1 lore raw

PATCH 2/5 python3-pycairo: use python3targetconfig

---
 meta/recipes-devtools/python/python3-pycairo_1.20.0.bb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
meta/recipes-devtools/python/python3-pycairo_1.20.0.bb +1 -1
index 37faabcdfb..8987b7a428 100644
@@ -18,7 +18,7 @@ SRC_URI[sha256sum] = "5695a10cb7f9ae0d01f665b56602a845b0a8cb17e2123bfece10c2e585
 
 S = "${WORKDIR}/pycairo-${PV}"
 
-inherit meson pkgconfig
+inherit meson pkgconfig python3targetconfig
 
 CFLAGS += "-fPIC"
 
Signature · 1 line
2.29.2
#2 lore raw

PATCH 3/5 distutils3-base.bbclass: use python3targetconfig

In reply to Alexander Kanavin
---
 meta/classes/distutils3-base.bbclass | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
meta/classes/distutils3-base.bbclass +1 -1
index 7dbf07ac4b..a277d1c7bc 100644
@@ -1,5 +1,5 @@
 DEPENDS  += "${@["${PYTHON_PN}-native ${PYTHON_PN}", ""][(d.getVar('PACKAGES') == '')]}"
 RDEPENDS_${PN} += "${@['', '${PYTHON_PN}-core']['${CLASSOVERRIDE}' == 'class-target']}"
 
-inherit distutils-common-base python3native
+inherit distutils-common-base python3native python3targetconfig
 
Signature · 1 line
2.29.2
#3 lore raw

Re: [OE-core] [PATCH 3/5] distutils3-base.bbclass: use python3targetconfig

this is causing issues see [1]. I have sent a potential fix too.

http://jenkins.nas-admin.org/view/OE/job/oe_world_workspace-compare-signatures/995/console

Quoted 23 lines · Alexander Kanavin
---
 meta/classes/distutils3-base.bbclass | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
meta/classes/distutils3-base.bbclass +1 -2
index 7dbf07ac4b..a277d1c7bc 100644
@@ -1,5 +1,5 @@
 DEPENDS  += "${@["${PYTHON_PN}-native ${PYTHON_PN}", ""][(d.getVar('PACKAGES') == '')]}"
 RDEPENDS_${PN} += "${@['', '${PYTHON_PN}-core']['${CLASSOVERRIDE}' == 'class-target']}"
 
-inherit distutils-common-base python3native
+inherit distutils-common-base python3native python3targetconfig
 
--

2.29.2

#4 lore raw

PATCH 4/5 meta: drop _PYTHON_SYSCONFIGDATA_NAME hacks

In reply to Alexander Kanavin
---
 meta/classes/scons.bbclass          | 3 ---
 meta/lib/oe/prservice.py            | 4 ----
 meta/recipes-core/glib-2.0/glib.inc | 4 ----
 meta/recipes-graphics/mesa/mesa.inc | 5 -----
 4 files changed, 16 deletions(-)
meta/classes/scons.bbclass +0 -3
index 6b171ca8df..4f3ae502ef 100644
@@ -5,7 +5,6 @@ DEPENDS += "python3-scons-native"
 EXTRA_OESCONS ?= ""
 
 do_configure() {
-	unset _PYTHON_SYSCONFIGDATA_NAME
 	if [ -n "${CONFIGURESTAMPFILE}" ]; then
 		if [ -e "${CONFIGURESTAMPFILE}" -a "`cat ${CONFIGURESTAMPFILE}`" != "${BB_TASKHASH}" -a "${CLEANBROKEN}" != "1" ]; then
 			${STAGING_BINDIR_NATIVE}/scons --clean PREFIX=${prefix} prefix=${prefix} ${EXTRA_OESCONS}
@@ -17,13 +16,11 @@ do_configure() {
 }
 
 scons_do_compile() {
-	unset _PYTHON_SYSCONFIGDATA_NAME
 	${STAGING_BINDIR_NATIVE}/scons ${PARALLEL_MAKE} PREFIX=${prefix} prefix=${prefix} ${EXTRA_OESCONS} || \
 	die "scons build execution failed."
 }
 
 scons_do_install() {
-	unset _PYTHON_SYSCONFIGDATA_NAME
 	${STAGING_BINDIR_NATIVE}/scons install_root=${D}${prefix} PREFIX=${prefix} prefix=${prefix} ${EXTRA_OESCONS} install || \
 	die "scons install execution failed."
 }
meta/lib/oe/prservice.py +0 -4
index 2d3c9c7e50..fcdbe66c19 100644
@@ -3,10 +3,6 @@
 #
 
 def prserv_make_conn(d, check = False):
-    # Otherwise this fails when called from recipes which e.g. inherit python3native (which sets _PYTHON_SYSCONFIGDATA_NAME) with:
-    # No module named '_sysconfigdata'
-    if '_PYTHON_SYSCONFIGDATA_NAME' in os.environ:
-        del os.environ['_PYTHON_SYSCONFIGDATA_NAME']
     import prserv.serv
     host_params = list([_f for _f in (d.getVar("PRSERV_HOST") or '').split(':') if _f])
     try:
meta/recipes-core/glib-2.0/glib.inc +0 -4
index a0055d81b0..06f4ae137e 100644
@@ -34,10 +34,6 @@ DEPENDS_append_class-target = "${@' gtk-doc' if d.getVar('GTKDOC_ENABLED') == 'T
 
 GTKDOC_MESON_OPTION = "gtk_doc"
 
-# This avoids the need to depend on target python3, which in case of mingw is not even possible.
-# meson's python configuration pokes into python3 configuration, so this provides the native config to it.
-unset _PYTHON_SYSCONFIGDATA_NAME
-
 S = "${WORKDIR}/glib-${PV}"
 
 PACKAGECONFIG ??= "system-pcre libmount \
meta/recipes-graphics/mesa/mesa.inc +0 -5
index ca593b8b1f..a6652b0ddb 100644
@@ -49,11 +49,6 @@ PROVIDES = " \
 
 inherit meson pkgconfig python3native gettext features_check
 
-# Unset these to stop python trying to report the target Python setup
-_PYTHON_SYSCONFIGDATA_NAME[unexport] = "1"
-STAGING_INCDIR[unexport] = "1"
-STAGING_LIBDIR[unexport] = "1"
-
 BBCLASSEXTEND = "native nativesdk"
 
 ANY_OF_DISTRO_FEATURES_class-target = "opengl vulkan"
Signature · 1 line
2.29.2
#5 lore raw

PATCH 5/5 gpgme: use python3targetconfig

In reply to Alexander Kanavin
---
 meta/recipes-support/gpgme/gpgme_1.14.0.bb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
meta/recipes-support/gpgme/gpgme_1.14.0.bb +1 -1
index 9fa8212808..fb7215381c 100644
@@ -48,7 +48,7 @@ DEFAULT_LANGUAGES_class-target = "cpp"
 LANGUAGES ?= "${DEFAULT_LANGUAGES} python"
 
 PYTHON_INHERIT = "${@bb.utils.contains('PACKAGECONFIG', 'python2', 'pythonnative', '', d)}"
-PYTHON_INHERIT .= "${@bb.utils.contains('PACKAGECONFIG', 'python3', 'python3native', '', d)}"
+PYTHON_INHERIT .= "${@bb.utils.contains('PACKAGECONFIG', 'python3', 'python3native python3targetconfig', '', d)}"
 
 EXTRA_OECONF += '--enable-languages="${LANGUAGES}" \
                  --disable-gpgconf-test \
Signature · 1 line
2.29.2
#6 lore raw
In reply to Alexander Kanavin
Quoted 14 lines · Alexander Kanavin

Setting _PYTHON_SYSCONFIGDATA_NAME in python3native class globally was problematic as it was leaking into host python environment, which was causing tracebacks depending on host distro and action (typically anything involving importing sysconfig module).

The new class sets the variable only in specific tasks where it is needed, and should be inherited explicitly:
- use python3native to run scripts with native python
- use python3targetconfig to run scripts with native python
if those scripts need to access target config data (such
as correct installation directories). This also adds a dependency on target python, so should be used carefully to avoid lengthening builds.

Quoted 1 line

This fixes the devtool modify u-boot failure for me on Arch.

Alistair

Quoted 46 lines
---
 meta/classes/python3native.bbclass       |  2 --
 meta/classes/python3targetconfig.bbclass | 15 +++++++++++++++
 2 files changed, 15 insertions(+), 2 deletions(-)
 create mode 100644 meta/classes/python3targetconfig.bbclass
meta/classes/python3native.bbclass +0 -2
index d98fb4c758..2e3a88c126 100644
@@ -17,8 +17,6 @@ export STAGING_LIBDIR
 export PYTHON_LIBRARY="${STAGING_LIBDIR}/lib${PYTHON_DIR}${PYTHON_ABI}.so"
 export PYTHON_INCLUDE_DIR="${STAGING_INCDIR}/${PYTHON_DIR}${PYTHON_ABI}"
 
-export _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata"
-
 # suppress host user's site-packages dirs.
 export PYTHONNOUSERSITE = "1"
 
meta/classes/python3targetconfig.bbclass new file +15 -1
new file mode 100644
index 0000000000..640d0c97b6
@@ -0,0 +1,15 @@
+inherit python3native
+
+DEPENDS_append = " python3"
+
+do_configure_prepend() {
+        export _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata"
+}
+
+do_compile_prepend() {
+        export _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata"
+}
+
+do_install_prepend() {
+        export _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata"
+}
--

2.29.2

#7 lore raw
In reply to Alexander Kanavin
Quoted 5 lines · Alexander Kanavin

Setting _PYTHON_SYSCONFIGDATA_NAME in python3native class globally was problematic as it was leaking into host python environment, which was causing tracebacks depending on host distro and action (typically anything involving importing sysconfig module).

Quoted 7 lines

The new class sets the variable only in specific tasks where it is needed, and should be inherited explicitly:
- use python3native to run scripts with native python
- use python3targetconfig to run scripts with native python
if those scripts need to access target config data (such
as correct installation directories). This also adds a dependency on target python, so should be used carefully to avoid lengthening builds.

cool.  I wonder if this fixes this open defect.
https://bugzilla.yoctoproject.org/show_bug.cgi?id=14076

Quoted 45 lines
---
 meta/classes/python3native.bbclass       |  2 --
 meta/classes/python3targetconfig.bbclass | 15 +++++++++++++++
 2 files changed, 15 insertions(+), 2 deletions(-)
 create mode 100644 meta/classes/python3targetconfig.bbclass
meta/classes/python3native.bbclass +0 -2
index d98fb4c758..2e3a88c126 100644
@@ -17,8 +17,6 @@ export STAGING_LIBDIR
 export PYTHON_LIBRARY="${STAGING_LIBDIR}/lib${PYTHON_DIR}${PYTHON_ABI}.so"
 export PYTHON_INCLUDE_DIR="${STAGING_INCDIR}/${PYTHON_DIR}${PYTHON_ABI}"
 
-export _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata"
-
 # suppress host user's site-packages dirs.
 export PYTHONNOUSERSITE = "1"
 
meta/classes/python3targetconfig.bbclass new file +15 -0
new file mode 100644
index 0000000000..640d0c97b6
@@ -0,0 +1,15 @@
+inherit python3native
+
+DEPENDS_append = " python3"
+
+do_configure_prepend() {
+        export _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata"
+}
+
+do_compile_prepend() {
+        export _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata"
+}
+
+do_install_prepend() {
+        export _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata"
+}