recipetool create improvements
v2 cover recipetool create improvements
Changes since v1:
* Set PKGV and SUMMARY per package for split npm packages so they get the values appropriate to the module they contain. PKGV is particularly important so that the image manifests have the appropriate version for each module.
* Create shrinkwrap/lockdown files next to npm recipes and add references to them within the recipe
* Don't fail if license value in an npm module's package.json isn't a simple string (more complicated license definitions are allowed according to the npm documentation).
The following changes since commit 0d02159c8d66bb136f7da2c10fda7d1a57f40cec:
image.bbclass: fix incomplete .rootfs customization (2016-03-07 22:01:16 +0000)
are available in the git repository at:
git://git.openembedded.org/openembedded-core-contrib paule/recipetool3
http://cgit.openembedded.org/cgit.cgi/openembedded-core-contrib/log/?h=paule/recipetool3
Paul Eggleton (8):
recipetool: create: be more tolerant of spacing in configure.ac
recipetool: create: improve mapping for autotools program macros
recipetool: create: match *LICENSE* as a license file
recipetool: create: add license file crunching
recipetool: create: split npm module dependencies into packages
recipetool: create: check if npm available if npm:// URL specified
recipetool: create: support creation of additional files by plugins
recipetool: create: shrinkwrap and lockdown npm modules
meta/classes/npm.bbclass | 20 ++++ meta/lib/oe/package.py | 32 +++++++ scripts/lib/devtool/standard.py | 6 +- scripts/lib/recipetool/create.py | 146 +++++++++++++++++++++++++++++- scripts/lib/recipetool/create_buildsys.py | 54 ++++++++--- scripts/lib/recipetool/create_npm.py | 114 ++++++++++++++++++++++- 6 files changed, 349 insertions(+), 23 deletions(-)
Signature · 1 line
2.5.0
PATCH v2 1/8 recipetool: create: be more tolerant of spacing in configure.ac
Allow for whitespace in appropriate places, and ensure we match all whitespace chars not just the space character.
(This fixes extracting dependencies from tmux's configure.ac, for example.)
--- scripts/lib/recipetool/create_buildsys.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-)
scripts/lib/recipetool/create_buildsys.py +8 -8
@@ -413,15 +413,15 @@ class AutotoolsRecipeHandler(RecipeHandler): progclassmap = {'gconftool-2': 'gconf', 'pkg-config': 'pkgconfig'} - pkg_re = re.compile('PKG_CHECK_MODULES\(\[?[a-zA-Z0-9_]*\]?, *\[?([^,\]]*)\]?[),].*') - pkgce_re = re.compile('PKG_CHECK_EXISTS\(\[?([^,\]]*)\]?[),].*') - lib_re = re.compile('AC_CHECK_LIB\(\[?([^,\]]*)\]?,.*') - libx_re = re.compile('AX_CHECK_LIBRARY\(\[?[^,\]]*\]?, *\[?([^,\]]*)\]?, *\[?([a-zA-Z0-9-]*)\]?,.*') - progs_re = re.compile('_PROGS?\(\[?[a-zA-Z0-9_]*\]?, \[?([^,\]]*)\]?[),].*') + pkg_re = re.compile('PKG_CHECK_MODULES\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*') + pkgce_re = re.compile('PKG_CHECK_EXISTS\(\s*\[?([^,\]]*)\]?[),].*') + lib_re = re.compile('AC_CHECK_LIB\(\s*\[?([^,\]]*)\]?,.*') + libx_re = re.compile('AX_CHECK_LIBRARY\(\s*\[?[^,\]]*\]?,\s*\[?([^,\]]*)\]?,\s*\[?([a-zA-Z0-9-]*)\]?,.*') + progs_re = re.compile('_PROGS?\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*') dep_re = re.compile('([^ ><=]+)( [<>=]+ [^ ><=]+)?') - ac_init_re = re.compile('AC_INIT\(([^,]+), *([^,]+)[,)].*') - am_init_re = re.compile('AM_INIT_AUTOMAKE\(([^,]+), *([^,]+)[,)].*') - define_re = re.compile(' *(m4_)?define\(([^,]+), *([^,]+)\)') + ac_init_re = re.compile('AC_INIT\(\s*([^,]+),\s*([^,]+)[,)].*') + am_init_re = re.compile('AM_INIT_AUTOMAKE\(\s*([^,]+),\s*([^,]+)[,)].*') + define_re = re.compile('\s*(m4_)?define\(\s*([^,]+),\s*([^,]+)\)') defines = {} def subst_defines(value):
Signature · 1 line
2.5.0
PATCH v2 2/8 recipetool: create: improve mapping for autotools program macros
In reply to Paul EggletonMake the following improvements to mapping items specified in AC_CHECK_PROG, AC_PATH_PROG and AX_WITH_PROG to recipes/classes:
* Produce a map of native recipe -> binary for all binaries currently in STAGING_BINDIR_NATIVE and use this when mapping items
* Add some more entries to the class map
* Ignore autotools binaries since they are covered by the inherit of autotools
* Ignore coreutils-native since that would almost always be a bogus dependency
--- scripts/lib/recipetool/create.py | 22 +++++++++++++++++-- scripts/lib/recipetool/create_buildsys.py | 36 +++++++++++++++++++++++++------ 2 files changed, 50 insertions(+), 8 deletions(-)
scripts/lib/recipetool/create.py +20 -2
@@ -1,6 +1,6 @@ # Recipe creation tool - create command plugin # -# Copyright (C) 2014-2015 Intel Corporation +# Copyright (C) 2014-2016 Intel Corporation # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as
@@ -44,6 +44,7 @@ class RecipeHandler(object): recipelibmap = {} recipeheadermap = {} recipecmakefilemap = {} + recipebinmap = {} @staticmethod def load_libmap(d):
@@ -122,6 +123,23 @@ class RecipeHandler(object): RecipeHandler.recipecmakefilemap[fn] = pn @staticmethod + def load_binmap(d): + '''Build up native binary->recipe mapping''' + if RecipeHandler.recipebinmap: + return + sstate_manifests = d.getVar('SSTATE_MANIFESTS', True) + staging_bindir_native = d.getVar('STAGING_BINDIR_NATIVE', True) + build_arch = d.getVar('BUILD_ARCH', True) + fileprefix = 'manifest-%s-' % build_arch + for fn in glob.glob(os.path.join(sstate_manifests, '%s*-native.populate_sysroot' % fileprefix)): + with open(fn, 'r') as f: + pn = os.path.basename(fn).rsplit('.', 1)[0][len(fileprefix):] + for line in f: + if line.startswith(staging_bindir_native): + prog = os.path.basename(line.rstrip()) + RecipeHandler.recipebinmap[prog] = pn + + @staticmethod def checkfiles(path, speclist, recursive=False): results = [] if recursive:
@@ -143,7 +161,7 @@ class RecipeHandler(object): RecipeHandler.load_libmap(d) ignorelibs = ['socket'] - ignoredeps = ['gcc-runtime', 'glibc', 'uclibc', 'musl', 'tar-native', 'binutils-native'] + ignoredeps = ['gcc-runtime', 'glibc', 'uclibc', 'musl', 'tar-native', 'binutils-native', 'coreutils-native'] unmappedpc = [] pcdeps = list(set(pcdeps))
scripts/lib/recipetool/create_buildsys.py +30 -6
@@ -1,6 +1,6 @@ # Recipe creation tool - create command build system handlers # -# Copyright (C) 2014 Intel Corporation +# Copyright (C) 2014-2016 Intel Corporation # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as
@@ -404,14 +404,34 @@ class AutotoolsRecipeHandler(RecipeHandler): values = {} inherits = [] - # FIXME this mapping is very thin + # Hardcoded map, we also use a dynamic one based on what's in the sysroot progmap = {'flex': 'flex-native', 'bison': 'bison-native', 'm4': 'm4-native', 'tar': 'tar-native', - 'ar': 'binutils-native'} + 'ar': 'binutils-native', + 'ranlib': 'binutils-native', + 'ld': 'binutils-native', + 'strip': 'binutils-native', + 'libtool': '', + 'autoconf': '', + 'autoheader': '', + 'automake': '', + 'uname': '', + 'rm': '', + 'cp': '', + 'mv': '', + 'find': '', + 'awk': '', + 'sed': '', + } progclassmap = {'gconftool-2': 'gconf', - 'pkg-config': 'pkgconfig'} + 'pkg-config': 'pkgconfig', + 'python': 'pythonnative', + 'python3': 'python3native', + 'perl': 'perlnative', + 'makeinfo': 'texinfo', + } pkg_re = re.compile('PKG_CHECK_MODULES\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*') pkgce_re = re.compile('PKG_CHECK_EXISTS\(\s*\[?([^,\]]*)\]?[),].*')
@@ -462,6 +482,8 @@ class AutotoolsRecipeHandler(RecipeHandler): deps = [] unmapped = [] + RecipeHandler.load_binmap(tinfoil.config_data) + def process_macro(keyword, value): for handler in handlers: if handler.process_macro(srctree, keyword, value, process_value, libdeps, pcdeps, deps, outlines, inherits, values):
@@ -498,10 +520,12 @@ class AutotoolsRecipeHandler(RecipeHandler): if progclass: inherits.append(progclass) else: - progdep = progmap.get(prog, None) + progdep = RecipeHandler.recipebinmap.get(prog, None) + if not progdep: + progdep = progmap.get(prog, None) if progdep: deps.append(progdep) - else: + elif progdep is None: if not prog.startswith('$'): unmapped.append(prog) elif keyword == 'AC_CHECK_LIB':
Signature · 1 line
2.5.0
PATCH v2 3/8 recipetool: create: match *LICENSE* as a license file
In reply to Paul EggletonFor example, this picks up a file named MIT-LICENSE.txt.
--- scripts/lib/recipetool/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
scripts/lib/recipetool/create.py +1 -1
@@ -722,7 +722,7 @@ def guess_license(srctree): md5sums = get_license_md5sums(tinfoil.config_data) licenses = [] - licspecs = ['LICENSE*', 'COPYING*', '*[Ll]icense*', 'LICENCE*', 'LEGAL*', '[Ll]egal*', '*GPL*', 'README.lic*', 'COPYRIGHT*', '[Cc]opyright*'] + licspecs = ['*LICEN[CS]E*', 'COPYING*', '*[Ll]icense*', 'LEGAL*', '[Ll]egal*', '*GPL*', 'README.lic*', 'COPYRIGHT*', '[Cc]opyright*'] licfiles = [] for root, dirs, files in os.walk(srctree): for fn in files:
Signature · 1 line
2.5.0
PATCH v2 4/8 recipetool: create: add license file crunching
In reply to Paul EggletonMatching license texts directly to md5sums only goes so far. Some licenses make the copyright statement an intrinsic part of the license statement (e.g. MIT) which of course varies between projects. Also, people often seem to take standard license texts such as GPLv2 and reformat them cosmetically - re-wrapping lines at a different width or changing quoting styles are seemingly popular examples. In order to match license files to their actual licenses more effectively, "crunch" out these elements before comparing to an md5sum. (The existing plain md5sum matching has been left in since it's a shortcut, and our list of crunched md5sums isn't a complete replacement for it.)
As always, this code isn't providing any guarantees (legal or otherwise) that it will always get the license correct - as indicated by the accompanying comments the LICENSE values it writes out to the recipe are indicative and you should verify them yourself by looking at the documentation supplied from upstream for the software being built if you have any concerns.
--- scripts/lib/recipetool/create.py | 77 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-)
scripts/lib/recipetool/create.py +76 -1
@@ -25,6 +25,7 @@ import json import logging import scriptutils import urlparse +import hashlib logger = logging.getLogger('recipetool')
@@ -717,6 +718,76 @@ def get_license_md5sums(d, static_only=False): md5sums['54c7042be62e169199200bc6477f04d1'] = 'BSD-3-Clause' return md5sums +def crunch_license(licfile): + ''' + Remove non-material text from a license file and then check + its md5sum against a known list. This works well for licenses + which contain a copyright statement, but is also a useful way + to handle people's insistence upon reformatting the license text + slightly (with no material difference to the text of the + license). + ''' + + import oe.utils + + # Note: these are carefully constructed! + license_title_re = re.compile('^\(?(#+ *)?(The )?.{1,10} [Ll]icen[sc]e( \(.{1,10}\))?\)?:?$') + license_statement_re = re.compile('^This (project|software) is( free software)? released under the .{1,10} [Ll]icen[sc]e:?$') + copyright_re = re.compile('^(#+)? *Copyright .*$') + + crunched_md5sums = {} + # The following two were gleaned from the "forever" npm package + crunched_md5sums['0a97f8e4cbaf889d6fa51f84b89a79f6'] = 'ISC' + crunched_md5sums['eecf6429523cbc9693547cf2db790b5c'] = 'MIT' + # https://github.com/vasi/pixz/blob/master/LICENSE + crunched_md5sums['2f03392b40bbe663597b5bd3cc5ebdb9'] = 'BSD-2-Clause' + # https://github.com/waffle-gl/waffle/blob/master/LICENSE.txt + crunched_md5sums['e72e5dfef0b1a4ca8a3d26a60587db66'] = 'BSD-2-Clause' + # https://github.com/spigwitmer/fakeds1963s/blob/master/LICENSE + crunched_md5sums['8be76ac6d191671f347ee4916baa637e'] = 'GPLv2' + # https://github.com/datto/dattobd/blob/master/COPYING + # http://git.savannah.gnu.org/cgit/freetype/freetype2.git/tree/docs/GPLv2.TXT + crunched_md5sums['1d65c5ad4bf6489f85f4812bf08ae73d'] = 'GPLv2' + # http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt + # http://git.neil.brown.name/?p=mdadm.git;a=blob;f=COPYING;h=d159169d1050894d3ea3b98e1c965c4058208fe1;hb=HEAD + crunched_md5sums['fb530f66a7a89ce920f0e912b5b66d4b'] = 'GPLv2' + # https://github.com/gkos/nrf24/blob/master/COPYING + crunched_md5sums['7b6aaa4daeafdfa6ed5443fd2684581b'] = 'GPLv2' + # https://github.com/josch09/resetusb/blob/master/COPYING + crunched_md5sums['8b8ac1d631a4d220342e83bcf1a1fbc3'] = 'GPLv3' + # https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.LGPLv2.1 + crunched_md5sums['2ea316ed973ae176e502e2297b574bb3'] = 'LGPLv2.1' + # unixODBC-2.3.4 COPYING + crunched_md5sums['1daebd9491d1e8426900b4fa5a422814'] = 'LGPLv2.1' + # https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.LGPLv3 + crunched_md5sums['2ebfb3bb49b9a48a075cc1425e7f4129'] = 'LGPLv3' + lictext = [] + with open(licfile, 'r') as f: + for line in f: + # Drop opening statements + if copyright_re.match(line): + continue + elif license_title_re.match(line): + continue + elif license_statement_re.match(line): + continue + # Squash spaces, and replace smart quotes, double quotes + # and backticks with single quotes + line = oe.utils.squashspaces(line.strip()).decode("utf-8") + line = line.replace(u"\u2018", "'").replace(u"\u2019", "'").replace(u"\u201c","'").replace(u"\u201d", "'").replace('"', '\'').replace('`', '\'') + if line: + lictext.append(line) + + m = hashlib.md5() + try: + m.update(' '.join(lictext)) + md5val = m.hexdigest() + except UnicodeEncodeError: + md5val = None + lictext = '' + license = crunched_md5sums.get(md5val, None) + return license, md5val, lictext + def guess_license(srctree): import bb md5sums = get_license_md5sums(tinfoil.config_data)
@@ -733,7 +804,11 @@ def guess_license(srctree): licfiles.append(fullpath) for licfile in licfiles: md5value = bb.utils.md5_file(licfile) - license = md5sums.get(md5value, 'Unknown') + license = md5sums.get(md5value, None) + if not license: + license, crunched_md5, lictext = crunch_license(licfile) + if not license: + license = 'Unknown' licenses.append((license, os.path.relpath(licfile, srctree), md5value)) # FIXME should we grab at least one source file with a license header and add that too?
Signature · 1 line
2.5.0
PATCH v2 5/8 recipetool: create: split npm module dependencies into packages
In reply to Paul EggletonRather than rolling all of an npm module's dependencies into the same package, split them into one module per package, setting the SUMMARY and PKGV values from the package.json file for each package. Additionally, mark each package with the appropriate license using the license scanning we already do, falling back to the license stated in the package.json file for the module if unknown. All of this is mostly in aid of ensuring all modules and their licenses now show up in the manifests for the image.
Additionally we set the main LICENSE value more concretely once we've calculated the per-package licenses, since we have more information at that point.
--- meta/classes/npm.bbclass | 20 +++++++++++++ meta/lib/oe/package.py | 32 ++++++++++++++++++++ scripts/lib/recipetool/create.py | 28 ++++++++++++++++++ scripts/lib/recipetool/create_npm.py | 57 ++++++++++++++++++++++++++++++++++-- 4 files changed, 134 insertions(+), 3 deletions(-)
meta/classes/npm.bbclass +20 -0
@@ -18,6 +18,26 @@ npm_do_install() { cp -a ${S}/* ${D}${libdir}/node_modules/${PN}/ --no-preserve=ownership } +python populate_packages_prepend () { + instdir = d.expand('${D}${libdir}/node_modules/${PN}') + extrapackages = oe.package.npm_split_package_dirs(instdir) + pkgnames = extrapackages.keys() + d.prependVar('PACKAGES', '%s ' % ' '.join(pkgnames)) + for pkgname in pkgnames: + pkgrelpath, pdata = extrapackages[pkgname] + pkgpath = '${libdir}/node_modules/${PN}/' + pkgrelpath + expanded_pkgname = d.expand(pkgname) + d.setVar('FILES_%s' % expanded_pkgname, pkgpath) + if pdata: + version = pdata.get('version', None) + if version: + d.setVar('PKGV_%s' % expanded_pkgname, version.encode("utf8")) + description = pdata.get('description', None) + if description: + d.setVar('SUMMARY_%s' % expanded_pkgname, description.replace(u"\u2018", "'").replace(u"\u2019", "'").encode("utf8")) + d.appendVar('RDEPENDS_%s' % d.getVar('PN', True), ' %s' % ' '.join(pkgnames)) +} + FILES_${PN} += " \ ${libdir}/node_modules/${PN} \ "
meta/lib/oe/package.py +32 -0
@@ -123,3 +123,35 @@ def read_shlib_providers(d): shlib_provider[s[0]] = {} shlib_provider[s[0]][s[1]] = (dep_pkg, s[2]) return shlib_provider + + +def npm_split_package_dirs(pkgdir): + """ + Work out the packages fetched and unpacked by BitBake's npm fetcher + Returns a dict of packagename -> (relpath, package.json) ordered + such that it is suitable for use in PACKAGES and FILES + """ + from collections import OrderedDict + import json + packages = {} + for root, dirs, files in os.walk(pkgdir): + if os.path.basename(root) == 'node_modules': + for dn in dirs: + relpth = os.path.relpath(os.path.join(root, dn), pkgdir) + pkgitems = ['${PN}'] + for pathitem in relpth.split('/'): + if pathitem == 'node_modules': + continue + pkgitems.append(pathitem) + pkgname = '-'.join(pkgitems) + pkgfile = os.path.join(root, dn, 'package.json') + data = None + if os.path.exists(pkgfile): + with open(pkgfile, 'r') as f: + data = json.loads(f.read()) + packages[pkgname] = (relpth, data) + # We want the main package for a module sorted *after* its subpackages + # (so that it doesn't otherwise steal the files for the subpackage), so + # this is a cheap way to do that whilst still having an otherwise + # alphabetical sort + return OrderedDict((key, packages[key]) for key in sorted(packages, key=lambda pkg: pkg + '~'))
scripts/lib/recipetool/create.py +28 -0
@@ -544,6 +544,7 @@ def create_recipe(args): # Apply the handlers handled = [] + handled.append(('license', licvalues)) if args.binary: classes.append('bin_package')
@@ -815,6 +816,33 @@ def guess_license(srctree): return licenses +def split_pkg_licenses(licvalues, packages, outlines, fallback_licenses=None, pn='${PN}'): + """ + Given a list of (license, path, md5sum) as returned by guess_license(), + a dict of package name to path mappings, write out a set of + package-specific LICENSE values. + """ + pkglicenses = {pn: []} + for license, licpath, _ in licvalues: + for pkgname, pkgpath in packages.iteritems(): + if licpath.startswith(pkgpath + '/'): + if pkgname in pkglicenses: + pkglicenses[pkgname].append(license) + else: + pkglicenses[pkgname] = [license] + break + else: + # Accumulate on the main package + pkglicenses[pn].append(license) + outlicenses = {} + for pkgname in packages: + license = ' '.join(list(set(pkglicenses.get(pkgname, ['Unknown'])))) + if license == 'Unknown' and pkgname in fallback_licenses: + license = fallback_licenses[pkgname] + outlines.append('LICENSE_%s = "%s"' % (pkgname, license)) + outlicenses[pkgname] = license.split() + return outlicenses + def read_pkgconfig_provides(d): pkgdatadir = d.getVar('PKGDATA_DIR', True) pkgmap = {}
scripts/lib/recipetool/create_npm.py +54 -3
80 lines not on this page · raw message
Signature · 1 line
2.5.0
PATCH v2 6/8 recipetool: create: check if npm available if npm:// URL specified
In reply to Paul EggletonIf the user specifies an npm:// URL then the fetcher needs npm to be available to run, so check if it's available early rather than failing later.
--- scripts/lib/recipetool/create.py | 6 ++++++ 1 file changed, 6 insertions(+)
scripts/lib/recipetool/create.py +6 -0
@@ -355,6 +355,12 @@ def create_recipe(args): srcuri = rev_re.sub('', srcuri) tempsrc = tempfile.mkdtemp(prefix='recipetool-') srctree = tempsrc + if fetchuri.startswith('npm://'): + # Check if npm is available + npm = bb.utils.which(tinfoil.config_data.getVar('PATH', True), 'npm') + if not npm: + logger.error('npm:// URL requested but npm is not available - you need to either build nodejs-native or install npm using your package manager') + sys.exit(1) logger.info('Fetching %s...' % srcuri) try: checksums = scriptutils.fetch_uri(tinfoil.config_data, fetchuri, srctree, srcrev)
Signature · 1 line
2.5.0
PATCH v2 7/8 recipetool: create: support creation of additional files by plugins
In reply to Paul EggletonAllow plugins to create additional files to go alongside the recipe. The plugins don't know what the output filename is going to be, so they need to put the files in a temporary location and add them to an "extrafiles" dict within extravalues where the destination filename is the key and the temporary path is the value.
devtool add was also extended to ensure these files get moved in and preserved upon reset if they've been edited by the user.
--- scripts/lib/devtool/standard.py | 6 +++++- scripts/lib/recipetool/create.py | 11 +++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-)
scripts/lib/devtool/standard.py +5 -1
@@ -189,6 +189,9 @@ def add(args, config, basepath, workspace): raise DevtoolError('Couldn\'t find source tree created by recipetool') bb.utils.mkdirhier(recipedir) shutil.move(recipes[0], recipefile) + # Move any additional files created by recipetool + for fn in os.listdir(tempdir): + shutil.move(os.path.join(tempdir, fn), recipedir) else: raise DevtoolError('Command \'%s\' did not create any recipe file:\n%s' % (e.command, e.stdout)) attic_recipe = os.path.join(config.workspace_path, 'attic', recipename, os.path.basename(recipefile))
@@ -199,7 +202,8 @@ def add(args, config, basepath, workspace): shutil.rmtree(tmpsrcdir) shutil.rmtree(tempdir) - _add_md5(config, recipename, recipefile) + for fn in os.listdir(recipedir): + _add_md5(config, recipename, os.path.join(recipedir, fn)) if args.fetch and not args.no_git: setup_git_repo(srctree, args.version, 'devtool')
scripts/lib/recipetool/create.py +11 -0
@@ -560,6 +560,8 @@ def create_recipe(args): for handler in handlers: handler.process(srctree_use, classes, lines_before, lines_after, handled, extravalues) + extrafiles = extravalues.pop('extrafiles', {}) + if not realpv: realpv = extravalues.get('PV', None) if realpv:
@@ -601,6 +603,15 @@ def create_recipe(args): logger.error('Output file %s already exists' % outfile) sys.exit(1) + # Move any extra files the plugins created to a directory next to the recipe + if outfile == '-': + extraoutdir = pn + else: + extraoutdir = os.path.join(os.path.dirname(outfile), pn) + bb.utils.mkdirhier(extraoutdir) + for destfn, extrafile in extrafiles.iteritems(): + shutil.move(extrafile, os.path.join(extraoutdir, destfn)) + lines = lines_before lines_before = [] skipblank = True
Signature · 1 line
2.5.0
PATCH v2 8/8 recipetool: create: shrinkwrap and lockdown npm modules
In reply to Paul Eggleton"npm shrinkwrap" creates a file that ensures that the exact same versions get fetched the next time the recipe is built. lockdown is similar but also includes sha1sums of the modules thus validating they haven't changed between builds. These ensure that the build is reproducible.
Fixes [YOCTO #9225].
--- scripts/lib/recipetool/create_npm.py | 57 ++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+)
scripts/lib/recipetool/create_npm.py +57 -0
@@ -15,14 +15,27 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import os import logging +import subprocess +import tempfile +import shutil import json from recipetool.create import RecipeHandler, split_pkg_licenses logger = logging.getLogger('recipetool') +tinfoil = None + +def tinfoil_init(instance): + global tinfoil + tinfoil = instance + + class NpmRecipeHandler(RecipeHandler): + lockdownpath = None + def _handle_license(self, data): ''' Handle the license value from an npm package.json file
@@ -34,7 +47,44 @@ class NpmRecipeHandler(RecipeHandler): license = license.get('type', None) return None + def _shrinkwrap(self, srctree, localfilesdir, extravalues, lines_before): + try: + runenv = dict(os.environ, PATH=tinfoil.config_data.getVar('PATH', True)) + bb.process.run('npm shrinkwrap', cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True) + except bb.process.ExecutionError as e: + logger.warn('npm shrinkwrap failed:\n%s' % e.stdout) + return + + tmpfile = os.path.join(localfilesdir, 'npm-shrinkwrap.json') + shutil.move(os.path.join(srctree, 'npm-shrinkwrap.json'), tmpfile) + extravalues.setdefault('extrafiles', {}) + extravalues['extrafiles']['npm-shrinkwrap.json'] = tmpfile + lines_before.append('NPM_SHRINKWRAP := "${THISDIR}/${PN}/npm-shrinkwrap.json"') + + def _lockdown(self, srctree, localfilesdir, extravalues, lines_before): + runenv = dict(os.environ, PATH=tinfoil.config_data.getVar('PATH', True)) + if not NpmRecipeHandler.lockdownpath: + NpmRecipeHandler.lockdownpath = tempfile.mkdtemp('recipetool-npm-lockdown') + bb.process.run('npm install lockdown --prefix %s' % NpmRecipeHandler.lockdownpath, + cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True) + relockbin = os.path.join(NpmRecipeHandler.lockdownpath, 'node_modules', 'lockdown', 'relock.js') + if not os.path.exists(relockbin): + logger.warn('Could not find relock.js within lockdown directory; skipping lockdown') + return + try: + bb.process.run('node %s' % relockbin, cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True) + except bb.process.ExecutionError as e: + logger.warn('lockdown-relock failed:\n%s' % e.stdout) + return + + tmpfile = os.path.join(localfilesdir, 'lockdown.json') + shutil.move(os.path.join(srctree, 'lockdown.json'), tmpfile) + extravalues.setdefault('extrafiles', {}) + extravalues['extrafiles']['lockdown.json'] = tmpfile + lines_before.append('NPM_LOCKDOWN := "${THISDIR}/${PN}/lockdown.json"') + def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): + import bb.utils import oe from collections import OrderedDict
@@ -58,6 +108,13 @@ class NpmRecipeHandler(RecipeHandler): if 'homepage' in data: lines_before.append('HOMEPAGE = "%s"' % data['homepage']) + # Shrinkwrap + localfilesdir = tempfile.mkdtemp(prefix='recipetool-npm') + self._shrinkwrap(srctree, localfilesdir, extravalues, lines_before) + + # Lockdown + self._lockdown(srctree, localfilesdir, extravalues, lines_before) + # Split each npm module out to is own package npmpackages = oe.package.npm_split_package_dirs(srctree) for item in handled:
Signature · 1 line
2.5.0