Go module dependencies
cover Go module dependencies
From: Christian Lindeberg <[email protected]>
Here's a stack that together with another stack for meta-oe will try to close the gap between the go-mod-update-modules class and the two gomod fetcher based recipes in meta-oe.
Christian Lindeberg (3):
oe/license_finder: Add find_licenses_up function go-mod-update-modules.bbclass: Update license finding recipetool/create_go: Tidy up a bit
.../go-mod-update-modules.bbclass | 30 ++++---- meta/lib/oe/license_finder.py | 27 ++++++- scripts/lib/recipetool/create_go.py | 70 +++++++++---------- 3 files changed, 72 insertions(+), 55 deletions(-)
Signature · 1 line
2.39.5
PATCH 1/3 oe/license_finder: Add find_licenses_up function
From: Christian Lindeberg <[email protected]>
Add a function for finding licenses in a directory or upwards but not above a top directory.
--- meta/lib/oe/license_finder.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-)
meta/lib/oe/license_finder.py +26 -1
@@ -120,14 +120,34 @@ def _crunch_license(licfile): return md5val -def find_license_files(srctree, first_only=False): +def find_license_files(srctree, first_only=False, bottom=""): """ Search srctree for files that look like they could be licenses. If first_only is True, only return the first file found. + If bottom is not empty, start at bottom and continue upwards to the top. """ licspecs = ['*LICEN[CS]E*', 'COPYING*', '*[Ll]icense*', 'LEGAL*', '[Ll]egal*', '*GPL*', 'README.lic*', 'COPYRIGHT*', '[Cc]opyright*', 'e[dp]l-v10'] skip_extensions = (".html", ".js", ".json", ".svg", ".ts", ".go", ".sh") licfiles = [] + if bottom: + srcdir = bottom + while srcdir.startswith(srctree): + files = [] + with os.scandir(srcdir) as it: + for entry in it: + if entry.is_file(): + files.append(entry.name) + for name in sorted(files): + if name.endswith(skip_extensions): + continue + for spec in licspecs: + if fnmatch.fnmatch(name, spec): + licfiles.append(os.path.join(srcdir, name)) + if first_only: + return licfiles + srcdir = os.path.dirname(srcdir) + return licfiles + for root, dirs, files in os.walk(srctree): # Sort files so that LICENSE is before LICENSE.subcomponent, which is # meaningful if first_only is set.
@@ -177,3 +197,8 @@ def find_licenses(srctree, d, first_only=False, extra_hashes={}): # FIXME should we grab at least one source file with a license header and add that too? return licenses + + +def find_licenses_up(srcdir, topdir, d, first_only=False, extra_hashes={}): + licfiles = find_license_files(topdir, first_only, srcdir) + return match_licenses(licfiles, topdir, d, extra_hashes)
Signature · 1 line
2.39.5
PATCH 2/3 go-mod-update-modules.bbclass: Update license finding
In reply to Christian LindebergFrom: Christian Lindeberg <[email protected]>
Use ${GO_INSTALL} when listing package dependencies.
Look for licenses for each package dependency continuing upwards, but not above the module root, until some license is found.
--- .../go-mod-update-modules.bbclass | 30 ++++++++----------- 1 file changed, 12 insertions(+), 18 deletions(-)
meta/classes-recipe/go-mod-update-modules.bbclass +12 -18
@@ -15,7 +15,7 @@ do_update_modules[network] = "1" python do_update_modules() { import subprocess, tempfile, json, re, urllib.parse from oe.license import tidy_licenses - from oe.license_finder import find_licenses + from oe.license_finder import find_licenses_up def unescape_path(path): """Unescape capital letters using exclamation points."""
@@ -47,12 +47,10 @@ python do_update_modules() { """ env = dict(os.environ, GOMODCACHE=mod_cache_dir) - source = d.expand("${UNPACKDIR}/${GO_SRCURI_DESTSUFFIX}") - output = subprocess.check_output(("go", "mod", "edit", "-json"), cwd=source, env=env, text=True) - go_mod = json.loads(output) - - output = subprocess.check_output(("go", "list", "-json=Dir,Module", "-deps", f"{go_mod['Module']['Path']}/..."), cwd=source, env=env, text=True) + go_install = d.getVar("GO_INSTALL").split() + output = subprocess.check_output(("go", "list", "-json=Dir,Module", "-deps", *go_install), + cwd=source, env=env, text=True) # # Licenses
@@ -66,26 +64,22 @@ python do_update_modules() { # Very frustrating that the json parser in python can't repeatedly # parse from a stream. pkgs = json.loads('[' + output.replace('}\n{', '},\n{') + ']') + # Collect licenses for the dependencies. - licenses = set() - lic_files_chksum = [] lic_files = {} - for pkg in pkgs: - mod = pkg.get('Module', None) - if not mod or mod.get('Main', False): - continue - - mod_dir = mod['Dir'] - - if not mod_dir.startswith(mod_cache_dir): + pkg_dir = pkg['Dir'] + if not pkg_dir.startswith(mod_cache_dir): continue + mod_dir = pkg['Module']['Dir'] path = os.path.relpath(mod_dir, mod_cache_dir) - for license_name, license_file, license_md5 in find_licenses(mod['Dir'], d, first_only=True, extra_hashes=extra_hashes): - lic_files[os.path.join(path, license_file)] = (license_name, license_md5) + for name, file, md5 in find_licenses_up(pkg_dir, mod_dir, d, first_only=True, extra_hashes=extra_hashes): + lic_files[os.path.join(path, file)] = (name, md5) + licenses = set() + lic_files_chksum = [] for lic_file in lic_files: license_name, license_md5 = lic_files[lic_file] if license_name == "Unknown":
Signature · 1 line
2.39.5
PATCH 3/3 recipetool/create_go: Tidy up a bit
In reply to Christian LindebergFrom: Christian Lindeberg <[email protected]>
There is no need for a temporary Go module cache after moving generation of module dependency include files to go-mod-update-modules.bbclass.
--- scripts/lib/recipetool/create_go.py | 70 ++++++++++++++--------------- 1 file changed, 34 insertions(+), 36 deletions(-)
scripts/lib/recipetool/create_go.py +34 -36
@@ -32,7 +32,6 @@ def tinfoil_init(instance): tinfoil = instance - class GoRecipeHandler(RecipeHandler): """Class to handle the go recipe creation"""
@@ -85,41 +84,40 @@ class GoRecipeHandler(RecipeHandler): classes.append("go-mod-update-modules") extravalues["run_tasks"] = "update_modules" - with tempfile.TemporaryDirectory(prefix="go-mod-") as tmp_mod_dir: - env = dict(os.environ) - env["PATH"] += f":{go_bindir}" - env['GOMODCACHE'] = tmp_mod_dir - - stdout = subprocess.check_output(["go", "mod", "edit", "-json"], cwd=srctree, env=env, text=True) - go_mod = json.loads(stdout) - go_import = re.sub(r'/v([0-9]+)$', '', go_mod['Module']['Path']) - - localfilesdir = tempfile.mkdtemp(prefix='recipetool-go-') - extravalues.setdefault('extrafiles', {}) - - # Write the stub ${BPN}-licenses.inc and ${BPN}-go-mods.inc files - basename = "{pn}-licenses.inc" - filename = os.path.join(localfilesdir, basename) - with open(filename, "w") as f: - f.write("# FROM RECIPETOOL\n") - extravalues['extrafiles'][f"../{basename}"] = filename - - basename = "{pn}-go-mods.inc" - filename = os.path.join(localfilesdir, basename) - with open(filename, "w") as f: - f.write("# FROM RECIPETOOL\n") - extravalues['extrafiles'][f"../{basename}"] = filename - - # Do generic license handling - d = bb.data.createCopy(tinfoil.config_data) - handle_license_vars(srctree, lines_before, handled, extravalues, d) - self.__rewrite_lic_vars(lines_before) - - self.__rewrite_src_uri(lines_before) - - lines_before.append('require ${BPN}-licenses.inc') - lines_before.append('require ${BPN}-go-mods.inc') - lines_before.append(f'GO_IMPORT = "{go_import}"') + env = dict(os.environ) + env["PATH"] += f":{go_bindir}" + + stdout = subprocess.check_output(("go", "mod", "edit", "-json"), + cwd=srctree, env=env, text=True) + go_mod = json.loads(stdout) + go_import = re.sub(r'/v([0-9]+)$', '', go_mod['Module']['Path']) + + localfilesdir = tempfile.mkdtemp(prefix='recipetool-go-') + extravalues.setdefault('extrafiles', {}) + + # Write the stub ${BPN}-licenses.inc and ${BPN}-go-mods.inc files + basename = "{pn}-licenses.inc" + filename = os.path.join(localfilesdir, basename) + with open(filename, "w") as f: + f.write("# FROM RECIPETOOL\n") + extravalues['extrafiles'][f"../{basename}"] = filename + + basename = "{pn}-go-mods.inc" + filename = os.path.join(localfilesdir, basename) + with open(filename, "w") as f: + f.write("# FROM RECIPETOOL\n") + extravalues['extrafiles'][f"../{basename}"] = filename + + # Do generic license handling + d = bb.data.createCopy(tinfoil.config_data) + handle_license_vars(srctree, lines_before, handled, extravalues, d) + self.__rewrite_lic_vars(lines_before) + + self.__rewrite_src_uri(lines_before) + + lines_before.append('require ${BPN}-licenses.inc') + lines_before.append('require ${BPN}-go-mods.inc') + lines_before.append(f'GO_IMPORT = "{go_import}"') def __update_lines_before(self, updated, newlines, lines_before): if updated:
Signature · 1 line
2.39.5