Skip to contentSkip to conversation

recipetool create improvements

PATCH New Paul Eggleton 1 participant 9 messages Mar 9, 2016

lore
Share & tools
lore
#0 lore raw

v2 cover recipetool create improvements

Changes since v1:
* Set PKGV and SUMMARY per package for split npm packages so they get the values appropriate to the module they contain. PKGV is particularly important so that the image manifests have the appropriate version for each module.
* Create shrinkwrap/lockdown files next to npm recipes and add references to them within the recipe
* Don't fail if license value in an npm module's package.json isn't a simple string (more complicated license definitions are allowed according to the npm documentation).

The following changes since commit 0d02159c8d66bb136f7da2c10fda7d1a57f40cec:

image.bbclass: fix incomplete .rootfs customization (2016-03-07 22:01:16 +0000)

are available in the git repository at:

git://git.openembedded.org/openembedded-core-contrib paule/recipetool3
http://cgit.openembedded.org/cgit.cgi/openembedded-core-contrib/log/?h=paule/recipetool3

Paul Eggleton (8):
recipetool: create: be more tolerant of spacing in configure.ac
recipetool: create: improve mapping for autotools program macros
recipetool: create: match *LICENSE* as a license file
recipetool: create: add license file crunching
recipetool: create: split npm module dependencies into packages
recipetool: create: check if npm available if npm:// URL specified
recipetool: create: support creation of additional files by plugins
recipetool: create: shrinkwrap and lockdown npm modules

 meta/classes/npm.bbclass                  |  20 ++++
 meta/lib/oe/package.py                    |  32 +++++++
 scripts/lib/devtool/standard.py           |   6 +-
 scripts/lib/recipetool/create.py          | 146 +++++++++++++++++++++++++++++-
 scripts/lib/recipetool/create_buildsys.py |  54 ++++++++---
 scripts/lib/recipetool/create_npm.py      | 114 ++++++++++++++++++++++-
 6 files changed, 349 insertions(+), 23 deletions(-)
Signature · 1 line
2.5.0
#1 lore raw

PATCH v2 1/8 recipetool: create: be more tolerant of spacing in configure.ac

Allow for whitespace in appropriate places, and ensure we match all whitespace chars not just the space character.

(This fixes extracting dependencies from tmux's configure.ac, for example.)

---
 scripts/lib/recipetool/create_buildsys.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)
scripts/lib/recipetool/create_buildsys.py +8 -8
index 0228269..909743b 100644
@@ -413,15 +413,15 @@ class AutotoolsRecipeHandler(RecipeHandler):
         progclassmap = {'gconftool-2': 'gconf',
                 'pkg-config': 'pkgconfig'}
 
-        pkg_re = re.compile('PKG_CHECK_MODULES\(\[?[a-zA-Z0-9_]*\]?, *\[?([^,\]]*)\]?[),].*')
-        pkgce_re = re.compile('PKG_CHECK_EXISTS\(\[?([^,\]]*)\]?[),].*')
-        lib_re = re.compile('AC_CHECK_LIB\(\[?([^,\]]*)\]?,.*')
-        libx_re = re.compile('AX_CHECK_LIBRARY\(\[?[^,\]]*\]?, *\[?([^,\]]*)\]?, *\[?([a-zA-Z0-9-]*)\]?,.*')
-        progs_re = re.compile('_PROGS?\(\[?[a-zA-Z0-9_]*\]?, \[?([^,\]]*)\]?[),].*')
+        pkg_re = re.compile('PKG_CHECK_MODULES\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*')
+        pkgce_re = re.compile('PKG_CHECK_EXISTS\(\s*\[?([^,\]]*)\]?[),].*')
+        lib_re = re.compile('AC_CHECK_LIB\(\s*\[?([^,\]]*)\]?,.*')
+        libx_re = re.compile('AX_CHECK_LIBRARY\(\s*\[?[^,\]]*\]?,\s*\[?([^,\]]*)\]?,\s*\[?([a-zA-Z0-9-]*)\]?,.*')
+        progs_re = re.compile('_PROGS?\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*')
         dep_re = re.compile('([^ ><=]+)( [<>=]+ [^ ><=]+)?')
-        ac_init_re = re.compile('AC_INIT\(([^,]+), *([^,]+)[,)].*')
-        am_init_re = re.compile('AM_INIT_AUTOMAKE\(([^,]+), *([^,]+)[,)].*')
-        define_re = re.compile(' *(m4_)?define\(([^,]+), *([^,]+)\)')
+        ac_init_re = re.compile('AC_INIT\(\s*([^,]+),\s*([^,]+)[,)].*')
+        am_init_re = re.compile('AM_INIT_AUTOMAKE\(\s*([^,]+),\s*([^,]+)[,)].*')
+        define_re = re.compile('\s*(m4_)?define\(\s*([^,]+),\s*([^,]+)\)')
 
         defines = {}
         def subst_defines(value):
Signature · 1 line
2.5.0
#2 lore raw

PATCH v2 2/8 recipetool: create: improve mapping for autotools program macros

In reply to Paul Eggleton

Make the following improvements to mapping items specified in AC_CHECK_PROG, AC_PATH_PROG and AX_WITH_PROG to recipes/classes:

* Produce a map of native recipe -> binary for all binaries currently in STAGING_BINDIR_NATIVE and use this when mapping items
* Add some more entries to the class map
* Ignore autotools binaries since they are covered by the inherit of autotools
* Ignore coreutils-native since that would almost always be a bogus dependency

---
 scripts/lib/recipetool/create.py          | 22 +++++++++++++++++--
 scripts/lib/recipetool/create_buildsys.py | 36 +++++++++++++++++++++++++------
 2 files changed, 50 insertions(+), 8 deletions(-)
scripts/lib/recipetool/create.py +20 -2
index 7560cdf..a77c191 100644
@@ -1,6 +1,6 @@
 # Recipe creation tool - create command plugin
 #
-# Copyright (C) 2014-2015 Intel Corporation
+# Copyright (C) 2014-2016 Intel Corporation
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -44,6 +44,7 @@ class RecipeHandler(object):
     recipelibmap = {}
     recipeheadermap = {}
     recipecmakefilemap = {}
+    recipebinmap = {}
 
     @staticmethod
     def load_libmap(d):
@@ -122,6 +123,23 @@ class RecipeHandler(object):
                         RecipeHandler.recipecmakefilemap[fn] = pn
 
     @staticmethod
+    def load_binmap(d):
+        '''Build up native binary->recipe mapping'''
+        if RecipeHandler.recipebinmap:
+            return
+        sstate_manifests = d.getVar('SSTATE_MANIFESTS', True)
+        staging_bindir_native = d.getVar('STAGING_BINDIR_NATIVE', True)
+        build_arch = d.getVar('BUILD_ARCH', True)
+        fileprefix = 'manifest-%s-' % build_arch
+        for fn in glob.glob(os.path.join(sstate_manifests, '%s*-native.populate_sysroot' % fileprefix)):
+            with open(fn, 'r') as f:
+                pn = os.path.basename(fn).rsplit('.', 1)[0][len(fileprefix):]
+                for line in f:
+                    if line.startswith(staging_bindir_native):
+                        prog = os.path.basename(line.rstrip())
+                        RecipeHandler.recipebinmap[prog] = pn
+
+    @staticmethod
     def checkfiles(path, speclist, recursive=False):
         results = []
         if recursive:
@@ -143,7 +161,7 @@ class RecipeHandler(object):
             RecipeHandler.load_libmap(d)
 
         ignorelibs = ['socket']
-        ignoredeps = ['gcc-runtime', 'glibc', 'uclibc', 'musl', 'tar-native', 'binutils-native']
+        ignoredeps = ['gcc-runtime', 'glibc', 'uclibc', 'musl', 'tar-native', 'binutils-native', 'coreutils-native']
 
         unmappedpc = []
         pcdeps = list(set(pcdeps))
scripts/lib/recipetool/create_buildsys.py +30 -6
index 909743b..f84ec3d 100644
@@ -1,6 +1,6 @@
 # Recipe creation tool - create command build system handlers
 #
-# Copyright (C) 2014 Intel Corporation
+# Copyright (C) 2014-2016 Intel Corporation
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -404,14 +404,34 @@ class AutotoolsRecipeHandler(RecipeHandler):
         values = {}
         inherits = []
 
-        # FIXME this mapping is very thin
+        # Hardcoded map, we also use a dynamic one based on what's in the sysroot
         progmap = {'flex': 'flex-native',
                 'bison': 'bison-native',
                 'm4': 'm4-native',
                 'tar': 'tar-native',
-                'ar': 'binutils-native'}
+                'ar': 'binutils-native',
+                'ranlib': 'binutils-native',
+                'ld': 'binutils-native',
+                'strip': 'binutils-native',
+                'libtool': '',
+                'autoconf': '',
+                'autoheader': '',
+                'automake': '',
+                'uname': '',
+                'rm': '',
+                'cp': '',
+                'mv': '',
+                'find': '',
+                'awk': '',
+                'sed': '',
+                }
         progclassmap = {'gconftool-2': 'gconf',
-                'pkg-config': 'pkgconfig'}
+                'pkg-config': 'pkgconfig',
+                'python': 'pythonnative',
+                'python3': 'python3native',
+                'perl': 'perlnative',
+                'makeinfo': 'texinfo',
+                }
 
         pkg_re = re.compile('PKG_CHECK_MODULES\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*')
         pkgce_re = re.compile('PKG_CHECK_EXISTS\(\s*\[?([^,\]]*)\]?[),].*')
@@ -462,6 +482,8 @@ class AutotoolsRecipeHandler(RecipeHandler):
         deps = []
         unmapped = []
 
+        RecipeHandler.load_binmap(tinfoil.config_data)
+
         def process_macro(keyword, value):
             for handler in handlers:
                 if handler.process_macro(srctree, keyword, value, process_value, libdeps, pcdeps, deps, outlines, inherits, values):
@@ -498,10 +520,12 @@ class AutotoolsRecipeHandler(RecipeHandler):
                         if progclass:
                             inherits.append(progclass)
                         else:
-                            progdep = progmap.get(prog, None)
+                            progdep = RecipeHandler.recipebinmap.get(prog, None)
+                            if not progdep:
+                                progdep = progmap.get(prog, None)
                             if progdep:
                                 deps.append(progdep)
-                            else:
+                            elif progdep is None:
                                 if not prog.startswith('$'):
                                     unmapped.append(prog)
             elif keyword == 'AC_CHECK_LIB':
Signature · 1 line
2.5.0
#3 lore raw

PATCH v2 3/8 recipetool: create: match *LICENSE* as a license file

In reply to Paul Eggleton

For example, this picks up a file named MIT-LICENSE.txt.

---
 scripts/lib/recipetool/create.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
scripts/lib/recipetool/create.py +1 -1
index a77c191..def2eea 100644
@@ -722,7 +722,7 @@ def guess_license(srctree):
     md5sums = get_license_md5sums(tinfoil.config_data)
 
     licenses = []
-    licspecs = ['LICENSE*', 'COPYING*', '*[Ll]icense*', 'LICENCE*', 'LEGAL*', '[Ll]egal*', '*GPL*', 'README.lic*', 'COPYRIGHT*', '[Cc]opyright*']
+    licspecs = ['*LICEN[CS]E*', 'COPYING*', '*[Ll]icense*', 'LEGAL*', '[Ll]egal*', '*GPL*', 'README.lic*', 'COPYRIGHT*', '[Cc]opyright*']
     licfiles = []
     for root, dirs, files in os.walk(srctree):
         for fn in files:
Signature · 1 line
2.5.0
#4 lore raw

PATCH v2 4/8 recipetool: create: add license file crunching

In reply to Paul Eggleton

Matching license texts directly to md5sums only goes so far. Some licenses make the copyright statement an intrinsic part of the license statement (e.g. MIT) which of course varies between projects. Also, people often seem to take standard license texts such as GPLv2 and reformat them cosmetically - re-wrapping lines at a different width or changing quoting styles are seemingly popular examples. In order to match license files to their actual licenses more effectively, "crunch" out these elements before comparing to an md5sum. (The existing plain md5sum matching has been left in since it's a shortcut, and our list of crunched md5sums isn't a complete replacement for it.)

As always, this code isn't providing any guarantees (legal or otherwise) that it will always get the license correct - as indicated by the accompanying comments the LICENSE values it writes out to the recipe are indicative and you should verify them yourself by looking at the documentation supplied from upstream for the software being built if you have any concerns.

---
 scripts/lib/recipetool/create.py | 77 +++++++++++++++++++++++++++++++++++++++-
 1 file changed, 76 insertions(+), 1 deletion(-)
scripts/lib/recipetool/create.py +76 -1
index def2eea..718f2aa 100644
@@ -25,6 +25,7 @@ import json
 import logging
 import scriptutils
 import urlparse
+import hashlib
 
 logger = logging.getLogger('recipetool')
 
@@ -717,6 +718,76 @@ def get_license_md5sums(d, static_only=False):
     md5sums['54c7042be62e169199200bc6477f04d1'] = 'BSD-3-Clause'
     return md5sums
 
+def crunch_license(licfile):
+    '''
+    Remove non-material text from a license file and then check
+    its md5sum against a known list. This works well for licenses
+    which contain a copyright statement, but is also a useful way
+    to handle people's insistence upon reformatting the license text
+    slightly (with no material difference to the text of the
+    license).
+    '''
+
+    import oe.utils
+
+    # Note: these are carefully constructed!
+    license_title_re = re.compile('^\(?(#+ *)?(The )?.{1,10} [Ll]icen[sc]e( \(.{1,10}\))?\)?:?$')
+    license_statement_re = re.compile('^This (project|software) is( free software)? released under the .{1,10} [Ll]icen[sc]e:?$')
+    copyright_re = re.compile('^(#+)? *Copyright .*$')
+
+    crunched_md5sums = {}
+    # The following two were gleaned from the "forever" npm package
+    crunched_md5sums['0a97f8e4cbaf889d6fa51f84b89a79f6'] = 'ISC'
+    crunched_md5sums['eecf6429523cbc9693547cf2db790b5c'] = 'MIT'
+    # https://github.com/vasi/pixz/blob/master/LICENSE
+    crunched_md5sums['2f03392b40bbe663597b5bd3cc5ebdb9'] = 'BSD-2-Clause'
+    # https://github.com/waffle-gl/waffle/blob/master/LICENSE.txt
+    crunched_md5sums['e72e5dfef0b1a4ca8a3d26a60587db66'] = 'BSD-2-Clause'
+    # https://github.com/spigwitmer/fakeds1963s/blob/master/LICENSE
+    crunched_md5sums['8be76ac6d191671f347ee4916baa637e'] = 'GPLv2'
+    # https://github.com/datto/dattobd/blob/master/COPYING
+    # http://git.savannah.gnu.org/cgit/freetype/freetype2.git/tree/docs/GPLv2.TXT
+    crunched_md5sums['1d65c5ad4bf6489f85f4812bf08ae73d'] = 'GPLv2'
+    # http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt
+    # http://git.neil.brown.name/?p=mdadm.git;a=blob;f=COPYING;h=d159169d1050894d3ea3b98e1c965c4058208fe1;hb=HEAD
+    crunched_md5sums['fb530f66a7a89ce920f0e912b5b66d4b'] = 'GPLv2'
+    # https://github.com/gkos/nrf24/blob/master/COPYING
+    crunched_md5sums['7b6aaa4daeafdfa6ed5443fd2684581b'] = 'GPLv2'
+    # https://github.com/josch09/resetusb/blob/master/COPYING
+    crunched_md5sums['8b8ac1d631a4d220342e83bcf1a1fbc3'] = 'GPLv3'
+    # https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.LGPLv2.1
+    crunched_md5sums['2ea316ed973ae176e502e2297b574bb3'] = 'LGPLv2.1'
+    # unixODBC-2.3.4 COPYING
+    crunched_md5sums['1daebd9491d1e8426900b4fa5a422814'] = 'LGPLv2.1'
+    # https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.LGPLv3
+    crunched_md5sums['2ebfb3bb49b9a48a075cc1425e7f4129'] = 'LGPLv3'
+    lictext = []
+    with open(licfile, 'r') as f:
+        for line in f:
+            # Drop opening statements
+            if copyright_re.match(line):
+                continue
+            elif license_title_re.match(line):
+                continue
+            elif license_statement_re.match(line):
+                continue
+            # Squash spaces, and replace smart quotes, double quotes
+            # and backticks with single quotes
+            line = oe.utils.squashspaces(line.strip()).decode("utf-8")
+            line = line.replace(u"\u2018", "'").replace(u"\u2019", "'").replace(u"\u201c","'").replace(u"\u201d", "'").replace('"', '\'').replace('`', '\'')
+            if line:
+                lictext.append(line)
+
+    m = hashlib.md5()
+    try:
+        m.update(' '.join(lictext))
+        md5val = m.hexdigest()
+    except UnicodeEncodeError:
+        md5val = None
+        lictext = ''
+    license = crunched_md5sums.get(md5val, None)
+    return license, md5val, lictext
+
 def guess_license(srctree):
     import bb
     md5sums = get_license_md5sums(tinfoil.config_data)
@@ -733,7 +804,11 @@ def guess_license(srctree):
                         licfiles.append(fullpath)
     for licfile in licfiles:
         md5value = bb.utils.md5_file(licfile)
-        license = md5sums.get(md5value, 'Unknown')
+        license = md5sums.get(md5value, None)
+        if not license:
+            license, crunched_md5, lictext = crunch_license(licfile)
+            if not license:
+                license = 'Unknown'
         licenses.append((license, os.path.relpath(licfile, srctree), md5value))
 
     # FIXME should we grab at least one source file with a license header and add that too?
Signature · 1 line
2.5.0
#5 lore raw

PATCH v2 5/8 recipetool: create: split npm module dependencies into packages

In reply to Paul Eggleton

Rather than rolling all of an npm module's dependencies into the same package, split them into one module per package, setting the SUMMARY and PKGV values from the package.json file for each package. Additionally, mark each package with the appropriate license using the license scanning we already do, falling back to the license stated in the package.json file for the module if unknown. All of this is mostly in aid of ensuring all modules and their licenses now show up in the manifests for the image.

Additionally we set the main LICENSE value more concretely once we've calculated the per-package licenses, since we have more information at that point.

---
 meta/classes/npm.bbclass             | 20 +++++++++++++
 meta/lib/oe/package.py               | 32 ++++++++++++++++++++
 scripts/lib/recipetool/create.py     | 28 ++++++++++++++++++
 scripts/lib/recipetool/create_npm.py | 57 ++++++++++++++++++++++++++++++++++--
 4 files changed, 134 insertions(+), 3 deletions(-)
meta/classes/npm.bbclass +20 -0
index be76056..b5db99d 100644
@@ -18,6 +18,26 @@ npm_do_install() {
 	cp -a ${S}/* ${D}${libdir}/node_modules/${PN}/ --no-preserve=ownership
 }
 
+python populate_packages_prepend () {
+    instdir = d.expand('${D}${libdir}/node_modules/${PN}')
+    extrapackages = oe.package.npm_split_package_dirs(instdir)
+    pkgnames = extrapackages.keys()
+    d.prependVar('PACKAGES', '%s ' % ' '.join(pkgnames))
+    for pkgname in pkgnames:
+        pkgrelpath, pdata = extrapackages[pkgname]
+        pkgpath = '${libdir}/node_modules/${PN}/' + pkgrelpath
+        expanded_pkgname = d.expand(pkgname)
+        d.setVar('FILES_%s' % expanded_pkgname, pkgpath)
+        if pdata:
+            version = pdata.get('version', None)
+            if version:
+                d.setVar('PKGV_%s' % expanded_pkgname, version.encode("utf8"))
+            description = pdata.get('description', None)
+            if description:
+                d.setVar('SUMMARY_%s' % expanded_pkgname, description.replace(u"\u2018", "'").replace(u"\u2019", "'").encode("utf8"))
+    d.appendVar('RDEPENDS_%s' % d.getVar('PN', True), ' %s' % ' '.join(pkgnames))
+}
+
 FILES_${PN} += " \
     ${libdir}/node_modules/${PN} \
 "
meta/lib/oe/package.py +32 -0
index f176446..dea443d 100644
@@ -123,3 +123,35 @@ def read_shlib_providers(d):
                         shlib_provider[s[0]] = {}
                     shlib_provider[s[0]][s[1]] = (dep_pkg, s[2])
     return shlib_provider
+
+
+def npm_split_package_dirs(pkgdir):
+    """
+    Work out the packages fetched and unpacked by BitBake's npm fetcher
+    Returns a dict of packagename -> (relpath, package.json) ordered
+    such that it is suitable for use in PACKAGES and FILES
+    """
+    from collections import OrderedDict
+    import json
+    packages = {}
+    for root, dirs, files in os.walk(pkgdir):
+        if os.path.basename(root) == 'node_modules':
+            for dn in dirs:
+                relpth = os.path.relpath(os.path.join(root, dn), pkgdir)
+                pkgitems = ['${PN}']
+                for pathitem in relpth.split('/'):
+                    if pathitem == 'node_modules':
+                        continue
+                    pkgitems.append(pathitem)
+                pkgname = '-'.join(pkgitems)
+                pkgfile = os.path.join(root, dn, 'package.json')
+                data = None
+                if os.path.exists(pkgfile):
+                    with open(pkgfile, 'r') as f:
+                        data = json.loads(f.read())
+                packages[pkgname] = (relpth, data)
+    # We want the main package for a module sorted *after* its subpackages
+    # (so that it doesn't otherwise steal the files for the subpackage), so
+    # this is a cheap way to do that whilst still having an otherwise
+    # alphabetical sort
+    return OrderedDict((key, packages[key]) for key in sorted(packages, key=lambda pkg: pkg + '~'))
scripts/lib/recipetool/create.py +28 -0
index 718f2aa..43c0784 100644
@@ -544,6 +544,7 @@ def create_recipe(args):
 
     # Apply the handlers
     handled = []
+    handled.append(('license', licvalues))
 
     if args.binary:
         classes.append('bin_package')
@@ -815,6 +816,33 @@ def guess_license(srctree):
 
     return licenses
 
+def split_pkg_licenses(licvalues, packages, outlines, fallback_licenses=None, pn='${PN}'):
+    """
+    Given a list of (license, path, md5sum) as returned by guess_license(),
+    a dict of package name to path mappings, write out a set of
+    package-specific LICENSE values.
+    """
+    pkglicenses = {pn: []}
+    for license, licpath, _ in licvalues:
+        for pkgname, pkgpath in packages.iteritems():
+            if licpath.startswith(pkgpath + '/'):
+                if pkgname in pkglicenses:
+                    pkglicenses[pkgname].append(license)
+                else:
+                    pkglicenses[pkgname] = [license]
+                break
+        else:
+            # Accumulate on the main package
+            pkglicenses[pn].append(license)
+    outlicenses = {}
+    for pkgname in packages:
+        license = ' '.join(list(set(pkglicenses.get(pkgname, ['Unknown']))))
+        if license == 'Unknown' and pkgname in fallback_licenses:
+            license = fallback_licenses[pkgname]
+        outlines.append('LICENSE_%s = "%s"' % (pkgname, license))
+        outlicenses[pkgname] = license.split()
+    return outlicenses
+
 def read_pkgconfig_provides(d):
     pkgdatadir = d.getVar('PKGDATA_DIR', True)
     pkgmap = {}
scripts/lib/recipetool/create_npm.py +54 -3

80 lines not on this page · raw message

Signature · 1 line
2.5.0
#6 lore raw

PATCH v2 6/8 recipetool: create: check if npm available if npm:// URL specified

In reply to Paul Eggleton

If the user specifies an npm:// URL then the fetcher needs npm to be available to run, so check if it's available early rather than failing later.

---
 scripts/lib/recipetool/create.py | 6 ++++++
 1 file changed, 6 insertions(+)
scripts/lib/recipetool/create.py +6 -0
index 43c0784..1d48e36 100644
@@ -355,6 +355,12 @@ def create_recipe(args):
             srcuri = rev_re.sub('', srcuri)
         tempsrc = tempfile.mkdtemp(prefix='recipetool-')
         srctree = tempsrc
+        if fetchuri.startswith('npm://'):
+            # Check if npm is available
+            npm = bb.utils.which(tinfoil.config_data.getVar('PATH', True), 'npm')
+            if not npm:
+                logger.error('npm:// URL requested but npm is not available - you need to either build nodejs-native or install npm using your package manager')
+                sys.exit(1)
         logger.info('Fetching %s...' % srcuri)
         try:
             checksums = scriptutils.fetch_uri(tinfoil.config_data, fetchuri, srctree, srcrev)
Signature · 1 line
2.5.0
#7 lore raw

PATCH v2 7/8 recipetool: create: support creation of additional files by plugins

In reply to Paul Eggleton

Allow plugins to create additional files to go alongside the recipe. The plugins don't know what the output filename is going to be, so they need to put the files in a temporary location and add them to an "extrafiles" dict within extravalues where the destination filename is the key and the temporary path is the value.

devtool add was also extended to ensure these files get moved in and preserved upon reset if they've been edited by the user.

---
 scripts/lib/devtool/standard.py  |  6 +++++-
 scripts/lib/recipetool/create.py | 11 +++++++++++
 2 files changed, 16 insertions(+), 1 deletion(-)
scripts/lib/devtool/standard.py +5 -1
index 5f83a91..b344001 100644
@@ -189,6 +189,9 @@ def add(args, config, basepath, workspace):
                     raise DevtoolError('Couldn\'t find source tree created by recipetool')
             bb.utils.mkdirhier(recipedir)
             shutil.move(recipes[0], recipefile)
+            # Move any additional files created by recipetool
+            for fn in os.listdir(tempdir):
+                shutil.move(os.path.join(tempdir, fn), recipedir)
         else:
             raise DevtoolError('Command \'%s\' did not create any recipe file:\n%s' % (e.command, e.stdout))
         attic_recipe = os.path.join(config.workspace_path, 'attic', recipename, os.path.basename(recipefile))
@@ -199,7 +202,8 @@ def add(args, config, basepath, workspace):
             shutil.rmtree(tmpsrcdir)
         shutil.rmtree(tempdir)
 
-    _add_md5(config, recipename, recipefile)
+    for fn in os.listdir(recipedir):
+        _add_md5(config, recipename, os.path.join(recipedir, fn))
 
     if args.fetch and not args.no_git:
         setup_git_repo(srctree, args.version, 'devtool')
scripts/lib/recipetool/create.py +11 -0
index 1d48e36..1649e40 100644
@@ -560,6 +560,8 @@ def create_recipe(args):
     for handler in handlers:
         handler.process(srctree_use, classes, lines_before, lines_after, handled, extravalues)
 
+    extrafiles = extravalues.pop('extrafiles', {})
+
     if not realpv:
         realpv = extravalues.get('PV', None)
         if realpv:
@@ -601,6 +603,15 @@ def create_recipe(args):
                 logger.error('Output file %s already exists' % outfile)
                 sys.exit(1)
 
+    # Move any extra files the plugins created to a directory next to the recipe
+    if outfile == '-':
+        extraoutdir = pn
+    else:
+        extraoutdir = os.path.join(os.path.dirname(outfile), pn)
+    bb.utils.mkdirhier(extraoutdir)
+    for destfn, extrafile in extrafiles.iteritems():
+        shutil.move(extrafile, os.path.join(extraoutdir, destfn))
+
     lines = lines_before
     lines_before = []
     skipblank = True
Signature · 1 line
2.5.0
#8 lore raw

PATCH v2 8/8 recipetool: create: shrinkwrap and lockdown npm modules

In reply to Paul Eggleton

"npm shrinkwrap" creates a file that ensures that the exact same versions get fetched the next time the recipe is built. lockdown is similar but also includes sha1sums of the modules thus validating they haven't changed between builds. These ensure that the build is reproducible.

Fixes [YOCTO #9225].

---
 scripts/lib/recipetool/create_npm.py | 57 ++++++++++++++++++++++++++++++++++++
 1 file changed, 57 insertions(+)
scripts/lib/recipetool/create_npm.py +57 -0
index 4bf6cae..b3ffcdb 100644
@@ -15,14 +15,27 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
+import os
 import logging
+import subprocess
+import tempfile
+import shutil
 import json
 from recipetool.create import RecipeHandler, split_pkg_licenses
 
 logger = logging.getLogger('recipetool')
 
 
+tinfoil = None
+
+def tinfoil_init(instance):
+    global tinfoil
+    tinfoil = instance
+
+
 class NpmRecipeHandler(RecipeHandler):
+    lockdownpath = None
+
     def _handle_license(self, data):
         '''
         Handle the license value from an npm package.json file
@@ -34,7 +47,44 @@ class NpmRecipeHandler(RecipeHandler):
                 license = license.get('type', None)
         return None
 
+    def _shrinkwrap(self, srctree, localfilesdir, extravalues, lines_before):
+        try:
+            runenv = dict(os.environ, PATH=tinfoil.config_data.getVar('PATH', True))
+            bb.process.run('npm shrinkwrap', cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
+        except bb.process.ExecutionError as e:
+            logger.warn('npm shrinkwrap failed:\n%s' % e.stdout)
+            return
+
+        tmpfile = os.path.join(localfilesdir, 'npm-shrinkwrap.json')
+        shutil.move(os.path.join(srctree, 'npm-shrinkwrap.json'), tmpfile)
+        extravalues.setdefault('extrafiles', {})
+        extravalues['extrafiles']['npm-shrinkwrap.json'] = tmpfile
+        lines_before.append('NPM_SHRINKWRAP := "${THISDIR}/${PN}/npm-shrinkwrap.json"')
+
+    def _lockdown(self, srctree, localfilesdir, extravalues, lines_before):
+        runenv = dict(os.environ, PATH=tinfoil.config_data.getVar('PATH', True))
+        if not NpmRecipeHandler.lockdownpath:
+            NpmRecipeHandler.lockdownpath = tempfile.mkdtemp('recipetool-npm-lockdown')
+            bb.process.run('npm install lockdown --prefix %s' % NpmRecipeHandler.lockdownpath,
+                           cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
+        relockbin = os.path.join(NpmRecipeHandler.lockdownpath, 'node_modules', 'lockdown', 'relock.js')
+        if not os.path.exists(relockbin):
+            logger.warn('Could not find relock.js within lockdown directory; skipping lockdown')
+            return
+        try:
+            bb.process.run('node %s' % relockbin, cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
+        except bb.process.ExecutionError as e:
+            logger.warn('lockdown-relock failed:\n%s' % e.stdout)
+            return
+
+        tmpfile = os.path.join(localfilesdir, 'lockdown.json')
+        shutil.move(os.path.join(srctree, 'lockdown.json'), tmpfile)
+        extravalues.setdefault('extrafiles', {})
+        extravalues['extrafiles']['lockdown.json'] = tmpfile
+        lines_before.append('NPM_LOCKDOWN := "${THISDIR}/${PN}/lockdown.json"')
+
     def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
+        import bb.utils
         import oe
         from collections import OrderedDict
 
@@ -58,6 +108,13 @@ class NpmRecipeHandler(RecipeHandler):
                 if 'homepage' in data:
                     lines_before.append('HOMEPAGE = "%s"' % data['homepage'])
 
+                # Shrinkwrap
+                localfilesdir = tempfile.mkdtemp(prefix='recipetool-npm')
+                self._shrinkwrap(srctree, localfilesdir, extravalues, lines_before)
+
+                # Lockdown
+                self._lockdown(srctree, localfilesdir, extravalues, lines_before)
+
                 # Split each npm module out to is own package
                 npmpackages = oe.package.npm_split_package_dirs(srctree)
                 for item in handled:
Signature · 1 line
2.5.0