python3: Improve logging, syntax and update deprecated modules to create_manifest
PATCH 1/2 python3: Improve logging, syntax and update deprecated modules to create_manifest
From: Alejandro Enedino Hernandez Samaniego <[email protected]>
The imp module has een deprecated by upstream python, drop its usage
(imp.get_tag) in favor of sys.implementation.cache_tag.
Avoid incorrectly getting dependencies for running script and multiprocessing module.
Improve logging behavior of the create_manifest task:
- Use indentation.
- Logs on temp directory.
- Use a proper debug flag.
- Standarize syntax.
--- .../python/python3/create_manifest3.py | 47 +++++++----- .../python/python3/get_module_deps3.py | 72 +++++++++++-------- 2 files changed, 71 insertions(+), 48 deletions(-)
meta/recipes-devtools/python/python3/create_manifest3.py +29 -18
@@ -36,7 +36,7 @@ # Tha method to handle cached files does not work when a module includes a folder which # itself contains the pycache folder, gladly this is almost never the case. # -# Author: Alejandro Enedino Hernandez Samaniego "aehs29" <aehs29 at gmail dot com> +# Author: Alejandro Enedino Hernandez Samaniego <alejandro at enedino dot org> import sys
@@ -45,6 +45,11 @@ import json import os import collections +if '-d' in sys.argv: + debugFlag = '-d' +else: + debugFlag = '' + # Get python version from ${PYTHON_MAJMIN} pyversion = str(sys.argv[1])
@@ -84,6 +89,12 @@ def prepend_comments(comments, json_manifest): manifest.seek(0, 0) manifest.write(comments + json_contents) +def print_indent(msg, offset): + for l in msg.splitlines(): + msg = ' ' * offset + l + print(msg) + + # Read existing JSON manifest with open('python3-manifest.json') as manifest: # The JSON format doesn't allow comments so we hack the call to keep the comments using a marker
@@ -99,7 +110,7 @@ with open('python3-manifest.json') as manifest: # Not exactly the same so it should not be a function # -print ('Getting dependencies for package: core') +print_indent('Getting dependencies for package: core', 0) # This special call gets the core dependencies and
@@ -109,7 +120,7 @@ print ('Getting dependencies for package: core') # on the new core package, they will still find them # even when checking the old_manifest -output = subprocess.check_output([sys.executable, 'get_module_deps3.py', 'python-core-package']).decode('utf8') +output = subprocess.check_output([sys.executable, 'get_module_deps3.py', 'python-core-package', '%s' % debugFlag]).decode('utf8') for coredep in output.split(): coredep = coredep.replace(pyversion,'${PYTHON_MAJMIN}') if isCached(coredep):
@@ -149,17 +160,16 @@ for filedep in old_manifest['core']['files']: # Get actual module name , shouldnt be affected by libdir/bindir, etc. pymodule = os.path.splitext(os.path.basename(os.path.normpath(filedep)))[0] - # We now know that were dealing with a python module, so we can import it # and check what its dependencies are. # We launch a separate task for each module for deterministic behavior. # Each module will only import what is necessary for it to work in specific. # The output of each task will contain each module's dependencies - print ('Getting dependencies for module: %s' % pymodule) - output = subprocess.check_output([sys.executable, 'get_module_deps3.py', '%s' % pymodule]).decode('utf8') - print ('The following dependencies were found for module %s:\n' % pymodule) - print (output) + print_indent('Getting dependencies for module: %s' % pymodule, 2) + output = subprocess.check_output([sys.executable, 'get_module_deps3.py', '%s' % pymodule, '%s' % debugFlag]).decode('utf8') + print_indent('The following dependencies were found for module %s:\n' % pymodule, 4) + print_indent(output, 6) for pymodule_dep in output.split():
@@ -178,12 +188,13 @@ for filedep in old_manifest['core']['files']: # all others will use this a base. +print('\n\nChecking for directories...\n') # To improve the script speed, we check which packages contain directories # since we will be looping through (only) those later. for pypkg in old_manifest: for filedep in old_manifest[pypkg]['files']: if isFolder(filedep): - print ('%s is a folder' % filedep) + print_indent('%s is a directory' % filedep, 2) if pypkg not in hasfolders: hasfolders.append(pypkg) if filedep not in allfolders:
@@ -221,14 +232,14 @@ for pypkg in old_manifest: print('\n') print('--------------------------') - print ('Handling package %s' % pypkg) + print('Handling package %s' % pypkg) print('--------------------------') # Handle special cases, we assume that when they were manually added # to the manifest we knew what we were doing. special_packages = ['misc', 'modules', 'dev', 'tests'] if pypkg in special_packages or 'staticdev' in pypkg: - print('Passing %s package directly' % pypkg) + print_indent('Passing %s package directly' % pypkg, 2) new_manifest[pypkg] = old_manifest[pypkg] continue
@@ -259,7 +270,7 @@ for pypkg in old_manifest: # Get actual module name , shouldnt be affected by libdir/bindir, etc. # We need to check if the imported module comes from another (e.g. sqlite3.dump) - path,pymodule = os.path.split(filedep) + path, pymodule = os.path.split(filedep) path = os.path.basename(path) pymodule = os.path.splitext(os.path.basename(pymodule))[0]
@@ -279,10 +290,10 @@ for pypkg in old_manifest: # Each module will only import what is necessary for it to work in specific. # The output of each task will contain each module's dependencies - print ('\nGetting dependencies for module: %s' % pymodule) - output = subprocess.check_output([sys.executable, 'get_module_deps3.py', '%s' % pymodule]).decode('utf8') - print ('The following dependencies were found for module %s:\n' % pymodule) - print (output) + print_indent('\nGetting dependencies for module: %s' % pymodule, 2) + output = subprocess.check_output([sys.executable, 'get_module_deps3.py', '%s' % pymodule, '%s' % debugFlag]).decode('utf8') + print_indent('The following dependencies were found for module %s:\n' % pymodule, 4) + print_indent(output, 6) reportFILES = [] reportRDEPS = []
@@ -325,7 +336,7 @@ for pypkg in old_manifest: # print('Checking folder %s on package %s' % (pymodule_dep,pypkg_with_folder)) for folder_dep in old_manifest[pypkg_with_folder]['files'] or folder_dep in old_manifest[pypkg_with_folder]['cached']: if folder_dep == folder: - print ('%s folder found in %s' % (folder, pypkg_with_folder)) + print ('%s directory found in %s' % (folder, pypkg_with_folder)) folderFound = True if pypkg_with_folder not in new_manifest[pypkg]['rdepends'] and pypkg_with_folder != pypkg: new_manifest[pypkg]['rdepends'].append(pypkg_with_folder)
@@ -424,7 +435,7 @@ prepend_comments(comments,'python3-manifest.json.new') if (repeated): error_msg = '\n\nERROR:\n' - error_msg += 'The following files are repeated (contained in more than one package),\n' + error_msg += 'The following files were found in more than one package),\n' error_msg += 'this is likely to happen when new files are introduced after an upgrade,\n' error_msg += 'please check which package should get it,\n modify the manifest accordingly and re-run the create_manifest task:\n' error_msg += '\n'.join(repeated)
meta/recipes-devtools/python/python3/get_module_deps3.py +42 -30
167 lines not on this page · raw message
Signature · 1 line
2.25.1
PATCH 2/2 python3: Upgrade 3.9.2 -> 3.9.4
From: Alejandro Enedino Hernandez Samaniego <[email protected]>
- Rebased patch 0001-test_locale.py-correct-the-test-output-format Maintainer needs to sign CLA and resubmit
- configure now explicitly requires autoconf-archive to be present
---
...le.py-correct-the-test-output-format.patch | 22 +++++++++----------
.../{python3_3.9.2.bb => python3_3.9.4.bb} | 4 ++--
2 files changed, 13 insertions(+), 13 deletions(-)
rename meta/recipes-devtools/python/{python3_3.9.2.bb => python3_3.9.4.bb} (99%)meta/recipes-devtools/python/python3/0001-test_locale.py-correct-the-test-output-format.patch +11 -11
@@ -23,24 +23,24 @@ Before this patch: Upstream-Status: Submitted [https://github.com/python/cpython/pull/15132] + +Rebased for 3.9.4, still not accepted upstream Signed-off-by: Alejandro Hernandez <[email protected]> + Signed-off-by: Mingli Yu <[email protected]> --- Lib/test/test_locale.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py -index e2c2178..558d63c 100644 ---- a/Lib/test/test_locale.py -+++ b/Lib/test/test_locale.py -@@ -527,7 +527,7 @@ class TestMiscellaneous(unittest.TestCase): +Index: Python-3.9.4/Lib/test/test_locale.py +=================================================================== +--- Python-3.9.4.orig/Lib/test/test_locale.py ++++ Python-3.9.4/Lib/test/test_locale.py +@@ -562,7 +562,7 @@ class TestMiscellaneous(unittest.TestCas self.skipTest('test needs Turkish locale') loc = locale.getlocale(locale.LC_CTYPE) if verbose: - print('testing with %a' % (loc,), end=' ', flush=True) + print('testing with %a...' % (loc,), end=' ', flush=True) - locale.setlocale(locale.LC_CTYPE, loc) - self.assertEqual(loc, locale.getlocale(locale.LC_CTYPE)) - --- -2.7.4 - + try: + locale.setlocale(locale.LC_CTYPE, loc) + except locale.Error as exc:
meta/recipes-devtools/python/python3_3.9.4.bb renamed from meta/recipes-devtools/python/python3_3.9.2.bb +2 -2
@@ -38,7 +38,7 @@ SRC_URI_append_class-native = " \ file://12-distutils-prefix-is-inside-staging-area.patch \ file://0001-Don-t-search-system-for-headers-libraries.patch \ " -SRC_URI[sha256sum] = "3c2034c54f811448f516668dce09d24008a0716c3a794dd8639b5388cbde247d" +SRC_URI[sha256sum] = "4b0e6644a76f8df864ae24ac500a51bbf68bd098f6a173e27d3b61cdca9aa134" # exclude pre-releases for both python 2.x and 3.x UPSTREAM_CHECK_REGEX = "[Pp]ython-(?P<pver>\d+(\.\d+)+).tar"
@@ -69,7 +69,7 @@ ALTERNATIVE_LINK_NAME[python3-config] = "${bindir}/python${PYTHON_MAJMIN}-config ALTERNATIVE_TARGET[python3-config] = "${bindir}/python${PYTHON_MAJMIN}-config-${MULTILIB_SUFFIX}" -DEPENDS = "bzip2-replacement-native libffi bzip2 openssl sqlite3 zlib virtual/libintl xz virtual/crypt util-linux libtirpc libnsl2" +DEPENDS = "bzip2-replacement-native libffi bzip2 openssl sqlite3 zlib virtual/libintl xz virtual/crypt util-linux libtirpc libnsl2 autoconf-archive-native" DEPENDS_append_class-target = " python3-native" DEPENDS_append_class-nativesdk = " python3-native"
Signature · 1 line
2.25.1