Skip to contentSkip to conversation

python3: support setting RPROVIDES in the manifest

PATCH New Ross Burton 1 participant 4 messages Nov 20

lore
Share & tools
lore
#0 lore raw

PATCH v3 1/4 python3: support setting RPROVIDES in the manifest

Allow modules to have a 'rprovides' key, that sets RPROVIDES for the generated package.

---
 .../python/python3/create_manifest3.py        | 23 ++++++++++++-------
 .../recipes-devtools/python/python3_3.15.2.bb |  9 ++++++++
 2 files changed, 24 insertions(+), 8 deletions(-)
meta/recipes-devtools/python/python3/create_manifest3.py +15 -8
index 9ce04fdd2aa..8ba51a2a0c3 100644
@@ -229,9 +229,20 @@ for pypkg in old_manifest:
 # will RDEPEND on it
 
 for pypkg in old_manifest:
+    # Handle special cases, we assume that when they were manually added
+    # to the manifest we knew what we were doing.
+    special_packages = ['misc', 'modules', 'dev', 'tests']
+    if pypkg in special_packages or 'staticdev' in pypkg:
+        print_indent('Passing %s package directly' % pypkg, 2)
+        new_manifest[pypkg] = old_manifest[pypkg]
+        if 'rprovides' not in new_manifest[pypkg]:
+            new_manifest[pypkg]['rprovides'] = []
+        continue
+
     # Use an empty dict as data structure to hold data for each package and fill it up
     new_manifest[pypkg] = {}
     new_manifest[pypkg]['summary'] = old_manifest[pypkg]['summary']
+    new_manifest[pypkg]['rprovides'] = old_manifest[pypkg].get('rprovides', [])
     new_manifest[pypkg]['rdepends'] = []
     if "skip_depends" in old_manifest[pypkg]:
         new_manifest[pypkg]['skip_depends'] = old_manifest[pypkg]['skip_depends']
@@ -248,14 +259,6 @@ for pypkg in old_manifest:
     print('Handling package %s' % pypkg)
     print('--------------------------')
 
-    # Handle special cases, we assume that when they were manually added 
-    # to the manifest we knew what we were doing.
-    special_packages = ['misc', 'modules', 'dev', 'tests']
-    if pypkg in special_packages or 'staticdev' in pypkg:
-        print_indent('Passing %s package directly' % pypkg, 2)
-        new_manifest[pypkg] = old_manifest[pypkg]
-        continue
-
     for filedep in old_manifest[pypkg]['files']:
         # We already handled core on the first pass, we can ignore it now
         if pypkg == 'core':
@@ -440,6 +443,10 @@ for pypkg in new_manifest:
     new_manifest[pypkg]['files'].sort()
     new_manifest[pypkg]['cached'].sort()
     new_manifest[pypkg]['rdepends'].sort()
+    new_manifest[pypkg]['rprovides'].sort()
+    # Remove empty sections
+    if not new_manifest[pypkg]['rprovides']:
+        new_manifest[pypkg].pop('rprovides')
 
 # Create the manifest from the data structure that was built
 with open('python3-manifest.json.new','w') as outfile:
meta/recipes-devtools/python/python3_3.15.2.bb +9 -0
index bfc783c9227..13d10face62 100644
@@ -428,6 +428,10 @@ python(){
         pypackage = pn + '-' + key + '-native'
         if pypackage not in rprovides:
               rprovides.append(pypackage)
+        for subkey in python_manifest[key].get('rprovides', ()):
+            pypackage = pn + '-' + subkey + '-native'
+            if pypackage not in rprovides:
+                rprovides.append(pypackage)
 
     d.setVar('RPROVIDES:class-native', ' '.join(rprovides))
 
@@ -461,6 +465,11 @@ python(){
                 value=value.split('-', 1)[1]
             d.appendVar('RDEPENDS:' + pypackage, ' ' + pn + '-' + value)
 
+        for value in python_manifest[key].get('rprovides', ()):
+            if '${PN}' in value:
+                value=value.split('-', 1)[1]
+            d.appendVar('RPROVIDES:' + pypackage, ' ' + pn + '-' + value)
+
         for value in python_manifest[key].get('rrecommends', ()):
             if '${PN}' in value:
                 value=value.split('-', 1)[1]
Signature · 1 line
2.43.0
#1 lore raw

PATCH v3 2/4 python3: set RPROVIDES for fcntl and pkgutil instead of empty packages

The -fcntl and -pkgutil packages were previously separate but merged into -core as they became mandatory dependencies.

The -fcntl package had a file in the manifest, but this was already packaged into -core so the package was actually empty.

Instead of generating empty packages, we can now just RPROVIDE the old name.

[1] pkgutil in oe-core e6ab9f16b92 ("python3: update to 3.8.1"),

    fcntl in oe-core 0b49c9aa312 ("python3: update 3.12.6 -> 3.13.0")
---
 .../python/python3/python3-manifest.json      | 22 ++++---------------
 .../recipes-devtools/python/python3_3.15.2.bb |  2 --
 2 files changed, 4 insertions(+), 20 deletions(-)
meta/recipes-devtools/python/python3/python3-manifest.json +4 -18
index 5f9bfbdf0e7..7a7f7db9b20 100644
@@ -185,6 +185,10 @@
     },
     "core": {
         "summary": "Python interpreter and core modules",
+        "rprovides": [
+            "fcntl",
+            "pkgutil"
+        ],
         "rdepends": [],
         "skip_depends": {
             "core": [
@@ -634,14 +638,6 @@
         ],
         "cached": []
     },
-    "fcntl": {
-        "summary": "Python's fcntl interface",
-        "rdepends": [
-            "core"
-        ],
-        "files": [],
-        "cached": []
-    },
     "gdbm": {
         "summary": "Python GNU database support",
         "rdepends": [
@@ -810,7 +806,6 @@
             "doctest",
             "email",
             "ensurepip",
-            "fcntl",
             "html",
             "idle",
             "image",
@@ -826,7 +821,6 @@
             "netserver",
             "numbers",
             "pickle",
-            "pkgutil",
             "plistlib",
             "pprint",
             "profile",
@@ -963,14 +957,6 @@
             "${libdir}/python${PYTHON_MAJMIN}/__pycache__/shelve.*.pyc"
         ]
     },
-    "pkgutil": {
-        "summary": "Python package extension utility support",
-        "rdepends": [
-            "core"
-        ],
-        "files": [],
-        "cached": []
-    },
     "plistlib": {
         "summary": "Generate and parse Mac OS X .plist files",
         "rdepends": [
meta/recipes-devtools/python/python3_3.15.2.bb +0 -2
index 13d10face62..61eabc07b54 100644
@@ -480,9 +480,7 @@ python(){
     # Prepending so to avoid python-misc getting everything
     packages = newpackages + packages
     d.setVar('PACKAGES', ' '.join(packages))
-    d.setVar('ALLOW_EMPTY:${PN}-fcntl', '1')
     d.setVar('ALLOW_EMPTY:${PN}-modules', '1')
-    d.setVar('ALLOW_EMPTY:${PN}-pkgutil', '1')
 
     if "pgo" in d.getVar("PACKAGECONFIG").split() and not bb.utils.contains('MACHINE_FEATURES', 'qemu-usermode', True, False, d):
         bb.fatal("pgo cannot be enabled as there is no qemu-usermode support for this architecture/machine")
Signature · 1 line
2.43.0
#2 lore raw

PATCH v3 3/4 python3: improve generation of _sysconfigdata.py

In reply to Ross Burton

Improve the regexes used to match so they are not assuming specific whitespace, and use a glob instead of find to look for the input file.

---
 meta/recipes-devtools/python/python3_3.15.2.bb | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)
meta/recipes-devtools/python/python3_3.15.2.bb +8 -8
index 61eabc07b54..388c7b82e40 100644
@@ -221,14 +221,14 @@ do_install:append() {
         rm -f ${D}${libdir}/python${PYTHON_MAJMIN}/__pycache__/_sysconfigdata*.cpython*
 
         mkdir -p ${D}${libdir}/python-sysconfigdata
-        sysconfigfile=`find ${D} -name _sysconfig*.py`
-        sed -i  \
-                -e "s,^ 'LIBDIR'.*, 'LIBDIR': '${STAGING_LIBDIR}'\,,g" \
-                -e "s,^ 'INCLUDEDIR'.*, 'INCLUDEDIR': '${STAGING_INCDIR}'\,,g" \
-                -e "s,^ 'CONFINCLUDEDIR'.*, 'CONFINCLUDEDIR': '${STAGING_INCDIR}'\,,g" \
-                -e "s,^ 'INCLUDEPY'.*, 'INCLUDEPY': '${STAGING_INCDIR}/python${PYTHON_MAJMIN}'\,,g" \
-                -e "s,^ 'CONFINCLUDEPY'.*, 'CONFINCLUDEPY': '${STAGING_INCDIR}/python${PYTHON_MAJMIN}'\,,g" \
-                -e "s,^ 'BUILD_GNU_TYPE'.*, 'BUILD_GNU_TYPE': '${HOST_SYS}'\,,g" \
+        sysconfigfile="${D}${libdir}/python${PYTHON_MAJMIN}/_sysconfig*.py"
+        sed -i \
+                -e "s,'LIBDIR':.*,'LIBDIR': '${STAGING_LIBDIR}'\,,g" \
+                -e "s,'INCLUDEDIR':.*,'INCLUDEDIR': '${STAGING_INCDIR}'\,,g" \
+                -e "s,'CONFINCLUDEDIR':.*,'CONFINCLUDEDIR': '${STAGING_INCDIR}'\,,g" \
+                -e "s,'INCLUDEPY':.*,'INCLUDEPY': '${STAGING_INCDIR}/python${PYTHON_MAJMIN}'\,,g" \
+                -e "s,'CONFINCLUDEPY':.*,'CONFINCLUDEPY': '${STAGING_INCDIR}/python${PYTHON_MAJMIN}'\,,g" \
+                -e "s,'BUILD_GNU_TYPE':.*,'BUILD_GNU_TYPE': '${HOST_SYS}'\,,g" \
                 -e "s,--build=${BUILD_ARCH},--build=${HOST_ARCH},g" \
                 -e "s,build_alias=${BUILD_ARCH},build_alias=${HOST_ARCH},g" \
                 -e "s,${B},/build/path/unavailable/,g" \
Signature · 1 line
2.43.0
#3 lore raw

PATCH v3 4/4 python3: remove obsolete reformatting of _sysconfigdata*.py

In reply to Ross Burton

The python3 recipe currently reformats the _sysconfigdata*.py files after they are modified to remove build paths, as the files may be wrapped differently based on the length of strings[1].

However, this wrapping was due to the use of pprint.pprint(), which since Python 3.13 is no longer used to generate the sysconfigdata[2].

As such, remove the reformatting as it is no longer needed.

[1] oe-core ec8a2b310d5 ("python3: Reformat sysconfig")
[2] python 5ecedbd2669 ("gh-106789: avoid importing pprint from sysconfig (#106790)")

---
 .../python/python3/reformat_sysconfig.py      | 21 -------------------
 .../recipes-devtools/python/python3_3.15.2.bb |  8 +------
 2 files changed, 1 insertion(+), 28 deletions(-)
 delete mode 100644 meta/recipes-devtools/python/python3/reformat_sysconfig.py
meta/recipes-devtools/python/python3/reformat_sysconfig.py deleted +0 -21
deleted file mode 100644
index 5e2b12879d9..00000000000
@@ -1,21 +0,0 @@
-#! /usr/bin/env python3
-#
-# SPDX-License-Identifier: MIT
-#
-# Copyright 2019 by Garmin Ltd. or its subsidiaries
-#
-# A script to reformat python sysconfig
-
-import sys
-import pprint
-l = {}
-g = {}
-with open(sys.argv[1], 'r') as f:
-    exec(f.read(), g, l)
-
-with open(sys.argv[1], 'w') as f:
-    for k in sorted(l.keys()):
-        f.write('%s = ' % k)
-        pprint.pprint(l[k], stream=f, width=1)
-        f.write('\n')
-
meta/recipes-devtools/python/python3_3.15.2.bb +1 -7
index 388c7b82e40..da59f0e8bb2 100644
@@ -12,7 +12,6 @@ SRC_URI = "http://www.python.org/ftp/python/${PV}/Python-${PV}.tar.xz \
            file://get_module_deps3.py \
            file://python3-manifest.json \
            file://check_build_completeness.py \
-           file://reformat_sysconfig.py \
            file://0001-Makefile.pre-use-qemu-wrapper-when-gathering-profile.patch \
            file://0001-test_locale.py-correct-the-test-output-format.patch \
            file://makerace.patch \
@@ -215,9 +214,6 @@ do_install:append:class-native() {
 }
 
 do_install:append() {
-        for c in ${D}/${libdir}/python${PYTHON_MAJMIN}/_sysconfigdata*.py; do
-            python3 ${UNPACKDIR}/reformat_sysconfig.py $c
-        done
         rm -f ${D}${libdir}/python${PYTHON_MAJMIN}/__pycache__/_sysconfigdata*.cpython*
 
         mkdir -p ${D}${libdir}/python-sysconfigdata
@@ -381,10 +377,8 @@ py_package_preprocess () {
                 ${PKGD}/${libdir}/python${PYTHON_MAJMIN}/_sysconfig_vars*.json \
                 ${PKGD}/${bindir}/python${PYTHON_MAJMIN}-config
 
-        # Reformat and recompile _sysconfigdata after modifying it so that it remains
-        # reproducible
+        # Recompile _sysconfigdata after modifying it
         for c in ${PKGD}/${libdir}/python${PYTHON_MAJMIN}/_sysconfigdata*.py; do
-            python3 ${UNPACKDIR}/reformat_sysconfig.py $c
             python3 -mcompileall $c -s ${PKGD} -p /
         done
 
Signature · 1 line
2.43.0