python3: Improve handling of python3 manifest generation
PATCH v2 1/2 python3: Improve handling of python3 manifest generation
Specifically cover detection of modules within a python package that do not import anything within their __init__.py. This is at least the case with the xmlrpc package which is only used via its modules xmlrpc.server and xmlrpc.client. Other important corner cases include ctypes.utils which depends on some modules not used by ctypes.
This is implemented by generally assuming that importing all the modules of a package (aka *.py within a package, excluding _*.py) will provide enough information.
Also due to this change some modules import sysconfig, resulting in sysconfigdata being imported. Handle the conversion of its path to a wildcard based on the platform dependent name being replaced.
---
Changes in v2:
- Remove duplicate importlib.import_module()
- Only treat imports which are "__init__.py" files as python packages for which to import child modules for
- Add additional patch to this series for updating the manifest due to the updated dependency logic and other minor fixes
--- .../python/python3/get_module_deps3.py | 20 ++++++++++++++++++-- .../python/python3/python3-manifest.json | 2 +- 2 files changed, 19 insertions(+), 3 deletions(-)
meta/recipes-devtools/python/python3/get_module_deps3.py +18 -2
@@ -9,6 +9,7 @@ debug=False import sys +import os # We can get a list of the modules which are currently required to run python # so we run python-core and get its modules, we then import what we need
@@ -48,8 +49,19 @@ current_module = str(sys.argv[1]).rstrip() if(debug==True): log = open('log_%s' % current_module,'w') log.write('Module %s generated the following dependencies:\n' % current_module) -try: - importlib.import_module('%s' % current_module) +try: + m = importlib.import_module(current_module) + # handle python packages which may not include all modules in the __init__ + if os.path.basename(m.__file__) == "__init__.py": + modulepath = os.path.dirname(m.__file__) + for i in os.listdir(modulepath): + if i.startswith("_") or not(i.endswith(".py")): + continue + submodule = "{}.{}".format(current_module, i[:-3]) + try: + importlib.import_module(submodule) + except: + pass # ignore all import or other exceptions raised during import except ImportError as e: if (debug==True): log.write('Module was not found')
@@ -107,6 +119,8 @@ for item in dif: dep_path = dep_path.replace(soabi,'*') print (dep_path) continue + if "_sysconfigdata" in dep_path: + dep_path = dep_path.replace(sysconfig._get_sysconfigdata_name(), "_sysconfigdata*") if (debug==True): log.write(dep_path+'\n')
@@ -140,6 +154,8 @@ for item in dif: log.write(cached) cached = fix_path(cached) cached = cached.replace(cpython_tag,'*') + if "_sysconfigdata" in cached: + cached = cached.replace(sysconfig._get_sysconfigdata_name(), "_sysconfigdata*") print (cached) if debug==True:
meta/recipes-devtools/python/python3/python3-manifest.json +1 -1
@@ -324,7 +324,7 @@ "${libdir}/python${PYTHON_MAJMIN}/__pycache__/_compression.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/_markupbase.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/_sitebuiltins.*.pyc", - "${libdir}/python${PYTHON_MAJMIN}/__pycache__/_sysconfigdata.*.pyc", + "${libdir}/python${PYTHON_MAJMIN}/__pycache__/_sysconfigdata*.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/_weakrefset.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/abc.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/argparse.*.pyc",
---
2.28.0
PATCH v2 2/2 python3-manifest.json: Updates
- Regenerate using create_manifest
- Fix up some indentation compared to generated
- Add "secrets" module into netclient package (introduced in 3.6)
- Move "urllib" python package into core package, it is used more commonly than just netclient (e.g. email, xml, mimetypes, pydoc)
- Update compression package dependencies due to some modules moving into core package
- Update dependencies due to improvements to get_module_deps handling modules of a python package (e.g. ctypes.utils)
--- .../python/python3/python3-manifest.json | 66 ++++++++++++++++------ 1 file changed, 48 insertions(+), 18 deletions(-)
meta/recipes-devtools/python/python3/python3-manifest.json +48 -18
@@ -285,7 +285,7 @@ "${libdir}/python${PYTHON_MAJMIN}/operator.py", "${libdir}/python${PYTHON_MAJMIN}/optparse.py", "${libdir}/python${PYTHON_MAJMIN}/os.py", - "${libdir}/python${PYTHON_MAJMIN}/pathlib.py", + "${libdir}/python${PYTHON_MAJMIN}/pathlib.py", "${libdir}/python${PYTHON_MAJMIN}/pkgutil.py", "${libdir}/python${PYTHON_MAJMIN}/platform.py", "${libdir}/python${PYTHON_MAJMIN}/posixpath.py",
@@ -313,6 +313,8 @@ "${libdir}/python${PYTHON_MAJMIN}/tokenize.py", "${libdir}/python${PYTHON_MAJMIN}/traceback.py", "${libdir}/python${PYTHON_MAJMIN}/types.py", + "${libdir}/python${PYTHON_MAJMIN}/urllib", + "${libdir}/python${PYTHON_MAJMIN}/urllib/parse.py", "${libdir}/python${PYTHON_MAJMIN}/warnings.py", "${libdir}/python${PYTHON_MAJMIN}/weakref.py", "${prefix}/lib/python${PYTHON_MAJMIN}/config*/*[!.a]"
@@ -359,7 +361,7 @@ "${libdir}/python${PYTHON_MAJMIN}/__pycache__/operator.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/optparse.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/os.*.pyc", - "${libdir}/python${PYTHON_MAJMIN}/__pycache__/pathlib.*.pyc", + "${libdir}/python${PYTHON_MAJMIN}/__pycache__/pathlib.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/pkgutil.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/platform.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/posixpath.*.pyc",
@@ -397,7 +399,9 @@ "${libdir}/python${PYTHON_MAJMIN}/importlib/__pycache__", "${libdir}/python${PYTHON_MAJMIN}/importlib/__pycache__/abc.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/importlib/__pycache__/machinery.*.pyc", - "${libdir}/python${PYTHON_MAJMIN}/importlib/__pycache__/util.*.pyc" + "${libdir}/python${PYTHON_MAJMIN}/importlib/__pycache__/util.*.pyc", + "${libdir}/python${PYTHON_MAJMIN}/urllib/__pycache__", + "${libdir}/python${PYTHON_MAJMIN}/urllib/__pycache__/parse.*.pyc" ] }, "crypt": {
@@ -427,7 +431,10 @@ "ctypes": { "summary": "Python C types support", "rdepends": [ - "core" + "core", + "crypt", + "io", + "math" ], "files": [ "${libdir}/python${PYTHON_MAJMIN}/ctypes",
@@ -537,7 +544,10 @@ "distutils": { "summary": "Python Distribution Utilities", "rdepends": [ - "core" + "compression", + "core", + "email", + "stringold" ], "files": [ "${libdir}/python${PYTHON_MAJMIN}/distutils"
@@ -548,7 +558,6 @@ "summary": "Python framework for running examples in docstrings", "rdepends": [ "asyncio", - "compression", "core", "debugger", "difflib",
@@ -577,7 +586,9 @@ "datetime", "io", "math", - "netclient" + "mime", + "netclient", + "stringold" ], "files": [ "${libdir}/python${PYTHON_MAJMIN}/email",
@@ -648,7 +659,6 @@ "io": { "summary": "Python low-level I/O", "rdepends": [ - "compression", "core", "crypt", "math",
@@ -690,7 +700,11 @@ "summary": "Python logging support", "rdepends": [ "core", - "stringold" + "io", + "netserver", + "pickle", + "stringold", + "threading" ], "files": [ "${libdir}/python${PYTHON_MAJMIN}/logging"
@@ -824,11 +838,18 @@ "summary": "Python multiprocessing support", "rdepends": [ "core", + "crypt", + "ctypes", "io", - "pickle" + "math", + "mmap", + "netclient", + "pickle", + "threading" ], "files": [ "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/_multiprocessing.*.so", + "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/_posixshmem.*.so", "${libdir}/python${PYTHON_MAJMIN}/multiprocessing" ], "cached": []
@@ -855,10 +876,9 @@ "${libdir}/python${PYTHON_MAJMIN}/mimetypes.py", "${libdir}/python${PYTHON_MAJMIN}/nntplib.py", "${libdir}/python${PYTHON_MAJMIN}/poplib.py", + "${libdir}/python${PYTHON_MAJMIN}/secrets.py", "${libdir}/python${PYTHON_MAJMIN}/smtplib.py", "${libdir}/python${PYTHON_MAJMIN}/telnetlib.py", - "${libdir}/python${PYTHON_MAJMIN}/urllib", - "${libdir}/python${PYTHON_MAJMIN}/urllib/__pycache__", "${libdir}/python${PYTHON_MAJMIN}/uuid.py" ], "cached": [
@@ -868,6 +888,7 @@ "${libdir}/python${PYTHON_MAJMIN}/__pycache__/mimetypes.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/nntplib.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/poplib.*.pyc", + "${libdir}/python${PYTHON_MAJMIN}/__pycache__/secrets.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/smtplib.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/telnetlib.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/uuid.*.pyc"
@@ -876,7 +897,6 @@ "netserver": { "summary": "Python Internet Protocol servers", "rdepends": [ - "compression", "core", "crypt", "datetime",
@@ -992,8 +1012,7 @@ "pydoc": { "summary": "Python interactive help support", "rdepends": [ - "core", - "netclient" + "core" ], "files": [ "${bindir}/pydoc*",
@@ -1017,7 +1036,6 @@ "shell": { "summary": "Python shell-like functionality", "rdepends": [ - "compression", "core", "stringold" ],
@@ -1150,7 +1168,6 @@ "summary": "Python unit testing framework", "rdepends": [ "asyncio", - "compression", "core", "difflib", "io",
@@ -1185,7 +1202,6 @@ "venv": { "summary": "Provides support for creating lightweight virtual environments with their own site directories, optionally isolated from system site directories.", "rdepends": [ - "compression", "core", "logging", "stringold"
@@ -1211,7 +1227,21 @@ "xmlrpc": { "summary": "Python XML-RPC support", "rdepends": [ + "compression", "core", + "crypt", + "datetime", + "email", + "fcntl", + "html", + "io", + "math", + "mime", + "netclient", + "netserver", + "numbers", + "pydoc", + "stringold", "xml" ], "files": [
---
2.28.0