Skip to contentSkip to conversation

recipetool create improvements

PATCH New Paul Eggleton 1 participant 5 messages Feb 29, 2016

lore
Share & tools
lore
#0 lore raw

cover recipetool create improvements

Support for creating recipes for kernels, out-of-tree kernel modules, and npm modules.

The following changes since commit 390bad905537820f49add855c95d726b5b55c8fa:

sanity: Fix int verses string reference (2016-02-28 22:54:06 +0000)

are available in the git repository at:

git://git.openembedded.org/openembedded-core-contrib paule/recipetool2
http://cgit.openembedded.org/cgit.cgi/openembedded-core-contrib/log/?h=paule/recipetool2

Paul Eggleton (4):
classes/module: allow substitution of the modules_install target name
recipetool: create: add support for out-of-tree kernel modules
recipetool: create: add basic support for generating linux kernel recipes
recipetool: create: add basic support for new npm fetcher/class

 meta/classes/module.bbclass             |   4 +-
 scripts/lib/recipetool/create_kernel.py |  99 +++++++++++++++++++++
 scripts/lib/recipetool/create_kmod.py   | 152 ++++++++++++++++++++++++++++++++
 scripts/lib/recipetool/create_npm.py    |  48 ++++++++++
 4 files changed, 302 insertions(+), 1 deletion(-)
 create mode 100644 scripts/lib/recipetool/create_kernel.py
 create mode 100644 scripts/lib/recipetool/create_kmod.py
 create mode 100644 scripts/lib/recipetool/create_npm.py
Signature · 1 line
2.5.0
#1 lore raw

PATCH 1/4 classes/module: allow substitution of the modules_install target name

Quite a few external kernel modules I've found floating around don't have a modules_install target, but they do have an install target that basically differs only in name. To make it easier to build these just make this a MODULES_INSTALL_TARGET variable that you can set from the recipe - the alternative would be copy-and-paste the do_install definition from this class which is potentially fragile.

---
 meta/classes/module.bbclass | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
meta/classes/module.bbclass +3 -1
index 0952c0c..01c9309 100644
@@ -6,6 +6,8 @@ do_make_scripts[depends] += "virtual/kernel:do_shared_workdir"
 
 EXTRA_OEMAKE += "KERNEL_SRC=${STAGING_KERNEL_DIR}"
 
+MODULES_INSTALL_TARGET ?= "modules_install"
+
 module_do_compile() {
 	unset CFLAGS CPPFLAGS CXXFLAGS LDFLAGS
 	oe_runmake KERNEL_PATH=${STAGING_KERNEL_DIR}   \
@@ -21,7 +23,7 @@ module_do_install() {
 	oe_runmake DEPMOD=echo INSTALL_MOD_PATH="${D}" \
 	           CC="${KERNEL_CC}" LD="${KERNEL_LD}" \
 	           O=${STAGING_KERNEL_BUILDDIR} \
-	           modules_install
+	           ${MODULES_INSTALL_TARGET}
 }
 
 EXPORT_FUNCTIONS do_compile do_install
Signature · 1 line
2.5.0
#2 lore raw

PATCH 2/4 recipetool: create: add support for out-of-tree kernel modules

In reply to Paul Eggleton

Detect kernel modules by looking for #include <linux/module.h>, and handle the various styles of Makefile that appear to be used. I was able to use this code to successfully build a number of external kernel modules I found.

Implements [YOCTO #8982].

---
 scripts/lib/recipetool/create_kmod.py | 152 ++++++++++++++++++++++++++++++++++
 1 file changed, 152 insertions(+)
 create mode 100644 scripts/lib/recipetool/create_kmod.py
scripts/lib/recipetool/create_kmod.py new file +152 -0
new file mode 100644
index 0000000..fe39edb
@@ -0,0 +1,152 @@
+# Recipe creation tool - kernel module support plugin
+#
+# Copyright (C) 2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import re
+import logging
+from recipetool.create import RecipeHandler, read_pkgconfig_provides, validate_pv
+
+logger = logging.getLogger('recipetool')
+
+tinfoil = None
+
+def tinfoil_init(instance):
+    global tinfoil
+    tinfoil = instance
+
+
+class KernelModuleRecipeHandler(RecipeHandler):
+    def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
+        import bb.process
+        if 'buildsystem' in handled:
+            return False
+
+        module_inc_re = re.compile(r'^#include\s+<linux/module.h>$')
+        makefiles = []
+        is_module = False
+
+        makefiles = []
+
+        files = RecipeHandler.checkfiles(srctree, ['*.c', '*.h'], recursive=True)
+        if files:
+            for cfile in files:
+                # Look in same dir or parent for Makefile
+                for makefile in [os.path.join(os.path.dirname(cfile), 'Makefile'), os.path.join(os.path.dirname(os.path.dirname(cfile)), 'Makefile')]:
+                    if makefile in makefiles:
+                        break
+                    else:
+                        if os.path.exists(makefile):
+                            makefiles.append(makefile)
+                            break
+                else:
+                    continue
+                with open(cfile, 'r') as f:
+                    for line in f:
+                        if module_inc_re.match(line.strip()):
+                            is_module = True
+                            break
+                if is_module:
+                    break
+
+        if is_module:
+            classes.append('module')
+            handled.append('buildsystem')
+            # module.bbclass and the classes it inherits do most of the hard
+            # work, but we need to tweak it slightly depending on what the
+            # Makefile does (and there is a range of those)
+            # Check the makefile for the appropriate install target
+            install_lines = []
+            compile_lines = []
+            in_install = False
+            in_compile = False
+            install_target = None
+            with open(makefile, 'r') as f:
+                for line in f:
+                    if line.startswith('install:'):
+                        if not install_lines:
+                            in_install = True
+                            install_target = 'install'
+                    elif line.startswith('modules_install:'):
+                        install_lines = []
+                        in_install = True
+                        install_target = 'modules_install'
+                    elif line.startswith('modules:'):
+                        compile_lines = []
+                        in_compile = True
+                    elif line.startswith(('all:', 'default:')):
+                        if not compile_lines:
+                            in_compile = True
+                    elif line:
+                        if line[0] == '\t':
+                            if in_install:
+                                install_lines.append(line)
+                            elif in_compile:
+                                compile_lines.append(line)
+                        elif ':' in line:
+                            in_install = False
+                            in_compile = False
+
+            def check_target(lines, install):
+                kdirpath = ''
+                manual_install = False
+                for line in lines:
+                    splitline = line.split()
+                    if splitline[0] in ['make', 'gmake', '$(MAKE)']:
+                        if '-C' in splitline:
+                            idx = splitline.index('-C') + 1
+                            if idx < len(splitline):
+                                kdirpath = splitline[idx]
+                                break
+                    elif install and splitline[0] == 'install':
+                        if '.ko' in line:
+                            manual_install = True
+                return kdirpath, manual_install
+
+            kdirpath = None
+            manual_install = False
+            if install_lines:
+                kdirpath, manual_install = check_target(install_lines, install=True)
+            if compile_lines and not kdirpath:
+                kdirpath, _ = check_target(compile_lines, install=False)
+
+            if manual_install or not install_lines:
+                lines_after.append('EXTRA_OEMAKE_append_task-install = " -C ${STAGING_KERNEL_DIR} M=${S}"')
+            elif install_target and install_target != 'modules_install':
+                lines_after.append('MODULES_INSTALL_TARGET = "install"')
+
+            warnmsg = None
+            kdirvar = None
+            if kdirpath:
+                res = re.match(r'\$\(([^$)]+)\)', kdirpath)
+                if res:
+                    kdirvar = res.group(1)
+                    if kdirvar != 'KERNEL_SRC':
+                        lines_after.append('EXTRA_OEMAKE += "%s=${STAGING_KERNEL_DIR}"' % kdirvar)
+                elif kdirpath.startswith('/lib/'):
+                    warnmsg = 'Kernel path in install makefile is hardcoded - you will need to patch the makefile'
+            if not kdirvar and not warnmsg:
+                warnmsg = 'Unable to find means of passing kernel path into install makefile - if kernel path is hardcoded you will need to patch the makefile'
+            if warnmsg:
+                warnmsg += '. Note that the variable KERNEL_SRC will be passed in as the kernel source path.'
+                logger.warn(warnmsg)
+                lines_after.append('# %s' % warnmsg)
+
+            return True
+
+        return False
+
+def register_recipe_handlers(handlers):
+    handlers.append((KernelModuleRecipeHandler(), 15))
Signature · 1 line
2.5.0
#3 lore raw

PATCH 3/4 recipetool: create: add basic support for generating linux kernel recipes

In reply to Paul Eggleton

Add support for detecting a Linux kernel source tree and generating a basic kernel recipe using meta-skeleton's linux-yocto-custom recipe as a base.

Implements [YOCTO #8981].

---
 scripts/lib/recipetool/create_kernel.py | 99 +++++++++++++++++++++++++++++++++
 1 file changed, 99 insertions(+)
 create mode 100644 scripts/lib/recipetool/create_kernel.py
scripts/lib/recipetool/create_kernel.py new file +99 -0
new file mode 100644
index 0000000..c6e86bd
@@ -0,0 +1,99 @@
+# Recipe creation tool - kernel support plugin
+#
+# Copyright (C) 2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import re
+import logging
+from recipetool.create import RecipeHandler, read_pkgconfig_provides, validate_pv
+
+logger = logging.getLogger('recipetool')
+
+tinfoil = None
+
+def tinfoil_init(instance):
+    global tinfoil
+    tinfoil = instance
+
+
+class KernelRecipeHandler(RecipeHandler):
+    def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
+        import bb.process
+        if 'buildsystem' in handled:
+            return False
+
+        for tell in ['arch', 'firmware', 'Kbuild', 'Kconfig']:
+            if not os.path.exists(os.path.join(srctree, tell)):
+                return False
+
+        handled.append('buildsystem')
+        del lines_after[:]
+        del classes[:]
+        template = os.path.join(tinfoil.config_data.getVar('COREBASE', True), 'meta-skeleton', 'recipes-kernel', 'linux', 'linux-yocto-custom.bb')
+        def handle_var(varname, origvalue, op, newlines):
+            if varname in ['SRCREV', 'SRCREV_machine']:
+                while newlines[-1].startswith('#'):
+                    del newlines[-1]
+                try:
+                    stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree, shell=True)
+                except bb.process.ExecutionError as e:
+                    stdout = None
+                if stdout:
+                    return stdout.strip(), op, 0, True
+            elif varname == 'LINUX_VERSION':
+                makefile = os.path.join(srctree, 'Makefile')
+                if os.path.exists(makefile):
+                    kversion = -1
+                    kpatchlevel = -1
+                    ksublevel = -1
+                    kextraversion = ''
+                    with open(makefile, 'r') as f:
+                        for i, line in enumerate(f):
+                            if i > 10:
+                                break
+                            if line.startswith('VERSION ='):
+                                kversion = int(line.split('=')[1].strip())
+                            elif line.startswith('PATCHLEVEL ='):
+                                kpatchlevel = int(line.split('=')[1].strip())
+                            elif line.startswith('SUBLEVEL ='):
+                                ksublevel = int(line.split('=')[1].strip())
+                            elif line.startswith('EXTRAVERSION ='):
+                                kextraversion = line.split('=')[1].strip()
+                    version = ''
+                    if kversion > -1 and kpatchlevel > -1:
+                        version = '%d.%d' % (kversion, kpatchlevel)
+                        if ksublevel > -1:
+                            version += '.%d' % ksublevel
+                        version += kextraversion
+                    if version:
+                        return version, op, 0, True
+            elif varname == 'SRC_URI':
+                while newlines[-1].startswith('#'):
+                    del newlines[-1]
+            elif varname == 'COMPATIBLE_MACHINE':
+                while newlines[-1].startswith('#'):
+                    del newlines[-1]
+                machine = tinfoil.config_data.getVar('MACHINE', True)
+                return machine, op, 0, True
+            return origvalue, op, 0, True
+        with open(template, 'r') as f:
+            varlist = ['SRCREV', 'SRCREV_machine', 'SRC_URI', 'LINUX_VERSION', 'COMPATIBLE_MACHINE']
+            (_, newlines) = bb.utils.edit_metadata(f, varlist, handle_var)
+        lines_before[:] = [line.rstrip('\n') for line in newlines]
+
+        return True
+
+def register_recipe_handlers(handlers):
+    handlers.append((KernelRecipeHandler(), 100))
Signature · 1 line
2.5.0
#4 lore raw

PATCH 4/4 recipetool: create: add basic support for new npm fetcher/class

In reply to Paul Eggleton

Add detection for npm modules and support for extracting the name and version from package.json as is usually part of an npm module contents.

Note: this will likely only produce a buildable recipe if you use an npm:// URL; simply pointing to a node.js source repository isn't going to fetch the module's dependencies. It also doesn't set up the shrinkwrap/lockdown automatically, so there is some room for improvement later.

Implements [YOCTO #8690].

---
 scripts/lib/recipetool/create_npm.py | 48 ++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)
 create mode 100644 scripts/lib/recipetool/create_npm.py
scripts/lib/recipetool/create_npm.py new file +48 -0
new file mode 100644
index 0000000..0e33cc9
@@ -0,0 +1,48 @@
+# Recipe creation tool - node.js NPM module support plugin
+#
+# Copyright (C) 2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import logging
+import json
+from recipetool.create import RecipeHandler
+
+logger = logging.getLogger('recipetool')
+
+
+class NpmRecipeHandler(RecipeHandler):
+    def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
+        if 'buildsystem' in handled:
+            return False
+
+        files = RecipeHandler.checkfiles(srctree, ['package.json'])
+        if files:
+            with open(files[0], 'r') as f:
+                data = json.loads(f.read())
+            if 'name' in data and 'version' in data:
+                extravalues['PN'] = data['name']
+                extravalues['PV'] = data['version']
+                classes.append('npm')
+                handled.append('buildsystem')
+                if 'description' in data:
+                    lines_before.append('SUMMARY = "%s"' % data['description'])
+                if 'homepage' in data:
+                    lines_before.append('HOMEPAGE = "%s"' % data['homepage'])
+                return True
+
+        return False
+
+def register_recipe_handlers(handlers):
+    handlers.append((NpmRecipeHandler(), 60))
Signature · 1 line
2.5.0