Skip to contentSkip to conversation

python3: Improve logging, syntax and update deprecated modules to create_manifest

PATCH New Alejandro Hernandez Samaniego 1 participant 2 messages Apr 17, 2021

lore
Share & tools
lore
#0 lore raw

PATCH 1/2 python3: Improve logging, syntax and update deprecated modules to create_manifest

From: Alejandro Enedino Hernandez Samaniego <[email protected]>

The imp module has een deprecated by upstream python, drop its usage
(imp.get_tag) in favor of sys.implementation.cache_tag.

Avoid incorrectly getting dependencies for running script and multiprocessing module.

Improve logging behavior of the create_manifest task:
- Use indentation.
- Logs on temp directory.
- Use a proper debug flag.
- Standarize syntax.

---
 .../python/python3/create_manifest3.py        | 47 +++++++-----
 .../python/python3/get_module_deps3.py        | 72 +++++++++++--------
 2 files changed, 71 insertions(+), 48 deletions(-)
meta/recipes-devtools/python/python3/create_manifest3.py +29 -18
index 4da02a2991..045240ea0b 100644
@@ -36,7 +36,7 @@
 # Tha method to handle cached files does not work when a module includes a folder which
 # itself contains the pycache folder, gladly this is almost never the case.
 #
-# Author: Alejandro Enedino Hernandez Samaniego "aehs29" <aehs29 at gmail dot com>
+# Author: Alejandro Enedino Hernandez Samaniego <alejandro at enedino dot org>
 
 
 import sys
@@ -45,6 +45,11 @@ import json
 import os
 import collections
 
+if '-d' in sys.argv:
+    debugFlag = '-d'
+else:
+    debugFlag = ''
+
 # Get python version from ${PYTHON_MAJMIN}
 pyversion = str(sys.argv[1])
 
@@ -84,6 +89,12 @@ def prepend_comments(comments, json_manifest):
         manifest.seek(0, 0)
         manifest.write(comments + json_contents)
 
+def print_indent(msg, offset):
+    for l in msg.splitlines():
+        msg = ' ' * offset + l
+        print(msg)
+
+
 # Read existing JSON manifest
 with open('python3-manifest.json') as manifest:
     # The JSON format doesn't allow comments so we hack the call to keep the comments using a marker
@@ -99,7 +110,7 @@ with open('python3-manifest.json') as manifest:
 # Not exactly the same so it should not be a function
 #
 
-print ('Getting dependencies for package: core')
+print_indent('Getting dependencies for package: core', 0)
 
 
 # This special call gets the core dependencies and
@@ -109,7 +120,7 @@ print ('Getting dependencies for package: core')
 # on the new core package, they will still find them
 # even when checking the old_manifest
 
-output = subprocess.check_output([sys.executable, 'get_module_deps3.py', 'python-core-package']).decode('utf8')
+output = subprocess.check_output([sys.executable, 'get_module_deps3.py', 'python-core-package', '%s' % debugFlag]).decode('utf8')
 for coredep in output.split():
     coredep = coredep.replace(pyversion,'${PYTHON_MAJMIN}')
     if isCached(coredep):
@@ -149,17 +160,16 @@ for filedep in old_manifest['core']['files']:
     # Get actual module name , shouldnt be affected by libdir/bindir, etc.
     pymodule = os.path.splitext(os.path.basename(os.path.normpath(filedep)))[0]
 
-
     # We now know that were dealing with a python module, so we can import it
     # and check what its dependencies are.
     # We launch a separate task for each module for deterministic behavior.
     # Each module will only import what is necessary for it to work in specific.
     # The output of each task will contain each module's dependencies
 
-    print ('Getting dependencies for module: %s' % pymodule)
-    output = subprocess.check_output([sys.executable, 'get_module_deps3.py', '%s' % pymodule]).decode('utf8')
-    print ('The following dependencies were found for module %s:\n' % pymodule)
-    print (output)
+    print_indent('Getting dependencies for module: %s' % pymodule, 2)
+    output = subprocess.check_output([sys.executable, 'get_module_deps3.py', '%s' % pymodule, '%s' % debugFlag]).decode('utf8')
+    print_indent('The following dependencies were found for module %s:\n' % pymodule, 4)
+    print_indent(output, 6)
 
 
     for pymodule_dep in output.split():
@@ -178,12 +188,13 @@ for filedep in old_manifest['core']['files']:
 # all others will use this a base.
 
 
+print('\n\nChecking for directories...\n')
 # To improve the script speed, we check which packages contain directories
 # since we will be looping through (only) those later.
 for pypkg in old_manifest:
     for filedep in old_manifest[pypkg]['files']:
         if isFolder(filedep):
-            print ('%s is a folder' % filedep)
+            print_indent('%s is a directory' % filedep, 2)
             if pypkg not in hasfolders:
                 hasfolders.append(pypkg)
             if filedep not in allfolders:
@@ -221,14 +232,14 @@ for pypkg in old_manifest:
 
     print('\n')
     print('--------------------------')
-    print ('Handling package %s' % pypkg)
+    print('Handling package %s' % pypkg)
     print('--------------------------')
 
     # Handle special cases, we assume that when they were manually added 
     # to the manifest we knew what we were doing.
     special_packages = ['misc', 'modules', 'dev', 'tests']
     if pypkg in special_packages or 'staticdev' in pypkg:
-        print('Passing %s package directly' % pypkg)
+        print_indent('Passing %s package directly' % pypkg, 2)
         new_manifest[pypkg] = old_manifest[pypkg]
         continue
 
@@ -259,7 +270,7 @@ for pypkg in old_manifest:
 
         # Get actual module name , shouldnt be affected by libdir/bindir, etc.
         # We need to check if the imported module comes from another (e.g. sqlite3.dump)
-        path,pymodule = os.path.split(filedep)
+        path, pymodule = os.path.split(filedep)
         path = os.path.basename(path)
         pymodule = os.path.splitext(os.path.basename(pymodule))[0]
 
@@ -279,10 +290,10 @@ for pypkg in old_manifest:
         # Each module will only import what is necessary for it to work in specific.
         # The output of each task will contain each module's dependencies
 
-        print ('\nGetting dependencies for module: %s' % pymodule)
-        output = subprocess.check_output([sys.executable, 'get_module_deps3.py', '%s' % pymodule]).decode('utf8')
-        print ('The following dependencies were found for module %s:\n' % pymodule)
-        print (output)
+        print_indent('\nGetting dependencies for module: %s' % pymodule, 2)
+        output = subprocess.check_output([sys.executable, 'get_module_deps3.py', '%s' % pymodule, '%s' % debugFlag]).decode('utf8')
+        print_indent('The following dependencies were found for module %s:\n' % pymodule, 4)
+        print_indent(output, 6)
 
         reportFILES = []
         reportRDEPS = []
@@ -325,7 +336,7 @@ for pypkg in old_manifest:
                             # print('Checking folder %s on package %s' % (pymodule_dep,pypkg_with_folder))
                             for folder_dep in old_manifest[pypkg_with_folder]['files'] or folder_dep in old_manifest[pypkg_with_folder]['cached']:
                                 if folder_dep == folder:
-                                    print ('%s folder found in %s' % (folder, pypkg_with_folder))
+                                    print ('%s directory found in %s' % (folder, pypkg_with_folder))
                                     folderFound = True
                                     if pypkg_with_folder not in new_manifest[pypkg]['rdepends'] and pypkg_with_folder != pypkg:
                                         new_manifest[pypkg]['rdepends'].append(pypkg_with_folder)
@@ -424,7 +435,7 @@ prepend_comments(comments,'python3-manifest.json.new')
 
 if (repeated):
     error_msg = '\n\nERROR:\n'
-    error_msg += 'The following files are repeated (contained in more than one package),\n'
+    error_msg += 'The following files were found in more than one package),\n'
     error_msg += 'this is likely to happen when new files are introduced after an upgrade,\n'
     error_msg += 'please check which package should get it,\n modify the manifest accordingly and re-run the create_manifest task:\n'
     error_msg += '\n'.join(repeated)
meta/recipes-devtools/python/python3/get_module_deps3.py +42 -30

167 lines not on this page · raw message

Signature · 1 line
2.25.1
#1 lore raw

PATCH 2/2 python3: Upgrade 3.9.2 -> 3.9.4

From: Alejandro Enedino Hernandez Samaniego <[email protected]>

- Rebased patch 0001-test_locale.py-correct-the-test-output-format Maintainer needs to sign CLA and resubmit
- configure now explicitly requires autoconf-archive to be present

---
 ...le.py-correct-the-test-output-format.patch | 22 +++++++++----------
 .../{python3_3.9.2.bb => python3_3.9.4.bb}    |  4 ++--
 2 files changed, 13 insertions(+), 13 deletions(-)
 rename meta/recipes-devtools/python/{python3_3.9.2.bb => python3_3.9.4.bb} (99%)
meta/recipes-devtools/python/python3/0001-test_locale.py-correct-the-test-output-format.patch +11 -11
index 35b7e0c480..c3d1e06d00 100644
@@ -23,24 +23,24 @@ Before this patch:
 
 Upstream-Status: Submitted [https://github.com/python/cpython/pull/15132]
 
+
+Rebased for 3.9.4, still not accepted upstream Signed-off-by: Alejandro Hernandez <[email protected]>
+
 Signed-off-by: Mingli Yu <[email protected]>
 ---
  Lib/test/test_locale.py | 2 +-
  1 file changed, 1 insertion(+), 1 deletion(-)
 
-diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py
-index e2c2178..558d63c 100644
---- a/Lib/test/test_locale.py
-+++ b/Lib/test/test_locale.py
-@@ -527,7 +527,7 @@ class TestMiscellaneous(unittest.TestCase):
+Index: Python-3.9.4/Lib/test/test_locale.py
+===================================================================
+--- Python-3.9.4.orig/Lib/test/test_locale.py
++++ Python-3.9.4/Lib/test/test_locale.py
+@@ -562,7 +562,7 @@ class TestMiscellaneous(unittest.TestCas
              self.skipTest('test needs Turkish locale')
          loc = locale.getlocale(locale.LC_CTYPE)
          if verbose:
 -            print('testing with %a' % (loc,), end=' ', flush=True)
 +            print('testing with %a...' % (loc,), end=' ', flush=True)
-         locale.setlocale(locale.LC_CTYPE, loc)
-         self.assertEqual(loc, locale.getlocale(locale.LC_CTYPE))
- 
--- 
-2.7.4
-
+         try:
+             locale.setlocale(locale.LC_CTYPE, loc)
+         except locale.Error as exc:
meta/recipes-devtools/python/python3_3.9.4.bb renamed from meta/recipes-devtools/python/python3_3.9.2.bb +2 -2
similarity index 99%
rename from meta/recipes-devtools/python/python3_3.9.2.bb
rename to meta/recipes-devtools/python/python3_3.9.4.bb
index fd1172335a..cb371ceed7 100644
@@ -38,7 +38,7 @@ SRC_URI_append_class-native = " \
            file://12-distutils-prefix-is-inside-staging-area.patch \
            file://0001-Don-t-search-system-for-headers-libraries.patch \
            "
-SRC_URI[sha256sum] = "3c2034c54f811448f516668dce09d24008a0716c3a794dd8639b5388cbde247d"
+SRC_URI[sha256sum] = "4b0e6644a76f8df864ae24ac500a51bbf68bd098f6a173e27d3b61cdca9aa134"
 
 # exclude pre-releases for both python 2.x and 3.x
 UPSTREAM_CHECK_REGEX = "[Pp]ython-(?P<pver>\d+(\.\d+)+).tar"
@@ -69,7 +69,7 @@ ALTERNATIVE_LINK_NAME[python3-config] = "${bindir}/python${PYTHON_MAJMIN}-config
 ALTERNATIVE_TARGET[python3-config] = "${bindir}/python${PYTHON_MAJMIN}-config-${MULTILIB_SUFFIX}"
 
 
-DEPENDS = "bzip2-replacement-native libffi bzip2 openssl sqlite3 zlib virtual/libintl xz virtual/crypt util-linux libtirpc libnsl2"
+DEPENDS = "bzip2-replacement-native libffi bzip2 openssl sqlite3 zlib virtual/libintl xz virtual/crypt util-linux libtirpc libnsl2 autoconf-archive-native"
 DEPENDS_append_class-target = " python3-native"
 DEPENDS_append_class-nativesdk = " python3-native"
 
Signature · 1 line
2.25.1