Skip to contentSkip to conversation

devtool/recipetool fixes

PATCH New Paul Eggleton 1 participant 17 messages Mar 16, 2015

lore
Share & tools
lore
#0 lore raw

cover devtool/recipetool fixes

Some fixes and minor improvements for devtool/recipetool and extensions to the oe-selftest tests for both.

The following changes since commit fb29441216435b9bae47ca9cd42db5a6b1fe77d8:

oeqa/parselogs: Skip hda opcode errors (2015-03-12 12:49:22 +0000)

are available in the git repository at:

git://git.openembedded.org/openembedded-core-contrib paule/devtool3
http://cgit.openembedded.org/cgit.cgi/openembedded-core-contrib/log/?h=paule/devtool3

Paul Eggleton (16):
devtool: deploy-target: fix deploying to previously deployed machine
devtool: name command line parsers appropriately
devtool: reset: add ability to reset entire workspace
devtool: modify: get correct initial revision from previously extracted source tree
devtool: modify/extract: prevent usage with incompatible recipes
devtool: deploy-target: allow disabling host key checking
devtool: deploy-target: detect and error if D is empty
devtool: deploy-target: add an option to disable quiet mode classes/externalsrc: fix for recipes that fetch local files
devtool: deploy-target: add dry-run option
gitignore: exclude meta-selftest, drop meta-hob
oe-selftest: support getting unexported variable values
oe-selftest: add a test for devtool deploy-target
oe-selftest: add a test for devtool modify on recipes fetching local files
recipetool: fix duplicate licenses being picked up
oe-selftest: add missing dependency to test_recipetool_create_git

 .gitignore                                         |   2 +-
 .../recipes-test/images/oe-selftest-image.bb       |   9 ++
 meta/classes/externalsrc.bbclass                   |  14 +-
 meta/lib/oeqa/selftest/devtool.py                  | 145 ++++++++++++++++++-
 meta/lib/oeqa/utils/commands.py                    |   7 +
 scripts/lib/devtool/deploy.py                      |  56 ++++++--
 scripts/lib/devtool/standard.py                    | 159 +++++++++++++--------
 scripts/lib/recipetool/create.py                   |   4 +-
 8 files changed, 327 insertions(+), 69 deletions(-)
 create mode 100644 meta-selftest/recipes-test/images/oe-selftest-image.bb
Signature · 1 line
1.9.3
#1 lore raw

PATCH 1/16 devtool: deploy-target: fix deploying to previously deployed machine

* Pass correct arguments to undeploy() function
* If an error occurs during undeploy(), exit instead of continuing

---
 scripts/lib/devtool/deploy.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
scripts/lib/devtool/deploy.py +3 -1
index bd23e95..896b618 100644
@@ -44,7 +44,9 @@ def deploy(args, config, basepath, workspace):
     deploy_file = os.path.join(deploy_dir, args.recipename + '.list')
 
     if os.path.exists(deploy_file):
-        undeploy(args)
+        if undeploy(args, config, basepath, workspace):
+            # Error already shown
+            return -1
 
     stdout, stderr = exec_build_env_command(config.init_path, basepath, 'bitbake -e %s' % args.recipename, shell=True)
     recipe_outdir = re.search(r'^D="(.*)"', stdout, re.MULTILINE).group(1)
Signature · 1 line
1.9.3
#2 lore raw

PATCH 2/16 devtool: name command line parsers appropriately

In reply to Paul Eggleton

No functional changes, just use a unique name for each parser.

---
 scripts/lib/devtool/standard.py | 44 ++++++++++++++++++++---------------------
 1 file changed, 22 insertions(+), 22 deletions(-)
scripts/lib/devtool/standard.py +22 -22
index cabf3fe..435878c 100644
@@ -1,6 +1,6 @@
 # Development tool - standard commands plugin
 #
-# Copyright (C) 2014 Intel Corporation
+# Copyright (C) 2014-2015 Intel Corporation
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -602,33 +602,33 @@ def register_commands(subparsers, context):
     parser_add.add_argument('--version', '-V', help='Version to use within recipe (PV)')
     parser_add.set_defaults(func=add)
 
-    parser_add = subparsers.add_parser('modify', help='Modify the source for an existing recipe',
+    parser_modify = subparsers.add_parser('modify', help='Modify the source for an existing recipe',
                                        description='Enables modifying the source for an existing recipe',
                                        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-    parser_add.add_argument('recipename', help='Name for recipe to edit')
-    parser_add.add_argument('srctree', help='Path to external source tree')
-    parser_add.add_argument('--wildcard', '-w', action="store_true", help='Use wildcard for unversioned bbappend')
-    parser_add.add_argument('--extract', '-x', action="store_true", help='Extract source as well')
-    parser_add.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
-    parser_add.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (only when using -x)')
-    parser_add.set_defaults(func=modify)
-
-    parser_add = subparsers.add_parser('extract', help='Extract the source for an existing recipe',
+    parser_modify.add_argument('recipename', help='Name for recipe to edit')
+    parser_modify.add_argument('srctree', help='Path to external source tree')
+    parser_modify.add_argument('--wildcard', '-w', action="store_true", help='Use wildcard for unversioned bbappend')
+    parser_modify.add_argument('--extract', '-x', action="store_true", help='Extract source as well')
+    parser_modify.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
+    parser_modify.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (only when using -x)')
+    parser_modify.set_defaults(func=modify)
+
+    parser_extract = subparsers.add_parser('extract', help='Extract the source for an existing recipe',
                                        description='Extracts the source for an existing recipe',
                                        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-    parser_add.add_argument('recipename', help='Name for recipe to extract the source for')
-    parser_add.add_argument('srctree', help='Path to where to extract the source tree')
-    parser_add.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout')
-    parser_add.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
-    parser_add.set_defaults(func=extract)
+    parser_extract.add_argument('recipename', help='Name for recipe to extract the source for')
+    parser_extract.add_argument('srctree', help='Path to where to extract the source tree')
+    parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout')
+    parser_extract.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
+    parser_extract.set_defaults(func=extract)
 
-    parser_add = subparsers.add_parser('update-recipe', help='Apply changes from external source tree to recipe',
+    parser_update_recipe = subparsers.add_parser('update-recipe', help='Apply changes from external source tree to recipe',
                                        description='Applies changes from external source tree to a recipe (updating/adding/removing patches as necessary, or by updating SRCREV)')
-    parser_add.add_argument('recipename', help='Name of recipe to update')
-    parser_add.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE')
-    parser_add.add_argument('--initial-rev', help='Starting revision for patches')
-    parser_add.add_argument('--no-remove', '-n', action="store_true", help='Don\'t remove patches, only add or update')
-    parser_add.set_defaults(func=update_recipe)
+    parser_update_recipe.add_argument('recipename', help='Name of recipe to update')
+    parser_update_recipe.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE')
+    parser_update_recipe.add_argument('--initial-rev', help='Starting revision for patches')
+    parser_update_recipe.add_argument('--no-remove', '-n', action="store_true", help='Don\'t remove patches, only add or update')
+    parser_update_recipe.set_defaults(func=update_recipe)
 
     parser_status = subparsers.add_parser('status', help='Show workspace status',
                                           description='Lists recipes currently in your workspace and the paths to their respective external source trees',
Signature · 1 line
1.9.3
#3 lore raw

PATCH 3/16 devtool: reset: add ability to reset entire workspace

In reply to Paul Eggleton

Add a -a/--all option to allow you to quickly reset all recipes in your workspace.

---
 meta/lib/oeqa/selftest/devtool.py | 29 ++++++++++++++++++++++
 scripts/lib/devtool/standard.py   | 52 +++++++++++++++++++++++++--------------
 2 files changed, 62 insertions(+), 19 deletions(-)
meta/lib/oeqa/selftest/devtool.py +29 -0
index 932d6b9..a8c339a 100644
@@ -384,3 +384,32 @@ class DevtoolTests(oeSelfTest):
         result = runCmd('devtool extract remake %s' % tempdir)
         self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile.am')), 'Extracted source could not be found')
         self.assertTrue(os.path.isdir(os.path.join(tempdir, '.git')), 'git repository for external source tree not found')
+
+    def test_devtool_reset_all(self):
+        # Check preconditions
+        workspacedir = os.path.join(self.builddir, 'workspace')
+        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
+        self.track_for_cleanup(tempdir)
+        self.track_for_cleanup(workspacedir)
+        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
+        testrecipe1 = 'mdadm'
+        testrecipe2 = 'cronie'
+        result = runCmd('devtool modify -x %s %s' % (testrecipe1, os.path.join(tempdir, testrecipe1)))
+        result = runCmd('devtool modify -x %s %s' % (testrecipe2, os.path.join(tempdir, testrecipe2)))
+        result = runCmd('devtool build %s' % testrecipe1)
+        result = runCmd('devtool build %s' % testrecipe2)
+        stampprefix1 = get_bb_var('STAMP', testrecipe1)
+        self.assertTrue(stampprefix1, 'Unable to get STAMP value for recipe %s' % testrecipe1)
+        stampprefix2 = get_bb_var('STAMP', testrecipe2)
+        self.assertTrue(stampprefix2, 'Unable to get STAMP value for recipe %s' % testrecipe2)
+        result = runCmd('devtool reset -a')
+        self.assertIn(testrecipe1, result.output)
+        self.assertIn(testrecipe2, result.output)
+        result = runCmd('devtool status')
+        self.assertNotIn(testrecipe1, result.output)
+        self.assertNotIn(testrecipe2, result.output)
+        matches1 = glob.glob(stampprefix1 + '*')
+        self.assertFalse(matches1, 'Stamp files exist for recipe %s that should have been cleaned' % testrecipe1)
+        matches2 = glob.glob(stampprefix2 + '*')
+        self.assertFalse(matches2, 'Stamp files exist for recipe %s that should have been cleaned' % testrecipe2)
scripts/lib/devtool/standard.py +33 -19

68 lines not on this page · raw message

Signature · 1 line
1.9.3
#4 lore raw

PATCH 4/16 devtool: modify: get correct initial revision from previously extracted source tree

In reply to Paul Eggleton

If you point devtool modify to a source tree previously created by devtool modify or devtool extract, then we need to try to pick up the correct initial revision so that devtool update-recipe knows where to start looking for commits that match up with patches in the recipe.

---
 scripts/lib/devtool/standard.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)
scripts/lib/devtool/standard.py +13 -2
index 32fb3b6..f9369ee 100644
@@ -326,8 +326,19 @@ def modify(args, config, basepath, workspace):
         commits = stdout.split()
     else:
         if os.path.exists(os.path.join(args.srctree, '.git')):
-            (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=args.srctree)
-            initial_rev = stdout.rstrip()
+            # Check if it's a tree previously extracted by us
+            try:
+                (stdout, _) = bb.process.run('git branch --contains devtool-base', cwd=args.srctree)
+            except bb.process.ExecutionError:
+                stdout = ''
+            for line in stdout.splitlines():
+                if line.startswith('*'):
+                    (stdout, _) = bb.process.run('git rev-parse devtool-base', cwd=args.srctree)
+                    initial_rev = stdout.rstrip()
+            if not initial_rev:
+                # Otherwise, just grab the head revision
+                (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=args.srctree)
+                initial_rev = stdout.rstrip()
 
     # Check that recipe isn't using a shared workdir
     s = rd.getVar('S', True)
Signature · 1 line
1.9.3
#5 lore raw

PATCH 5/16 devtool: modify/extract: prevent usage with incompatible recipes

In reply to Paul Eggleton

Consolidate code for checking compatible recipes and consider meta and packagegroup recipes as well as package-index and gcc-source to be incompatible.

---
 meta/lib/oeqa/selftest/devtool.py | 24 ++++++++++++++++++++
 scripts/lib/devtool/standard.py   | 48 ++++++++++++++++++++++++++++-----------
 2 files changed, 59 insertions(+), 13 deletions(-)
meta/lib/oeqa/selftest/devtool.py +24 -0
index a8c339a..2574e4b 100644
@@ -227,6 +227,30 @@ class DevtoolTests(oeSelfTest):
         matches = glob.glob(stampprefix + '*')
         self.assertFalse(matches, 'Stamp files exist for recipe mdadm that should have been cleaned')
 
+    def test_devtool_modify_invalid(self):
+        # Check preconditions
+        workspacedir = os.path.join(self.builddir, 'workspace')
+        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+        # Try modifying some recipes
+        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
+        self.track_for_cleanup(tempdir)
+        self.track_for_cleanup(workspacedir)
+        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
+
+        testrecipes = 'perf gcc-source kernel-devsrc package-index core-image-minimal meta-toolchain packagegroup-core-sdk meta-ide-support'.split()
+        for testrecipe in testrecipes:
+            # Check it's a valid recipe
+            bitbake('%s -e' % testrecipe)
+            # devtool extract should fail
+            result = runCmd('devtool extract %s %s' % (testrecipe, os.path.join(tempdir, testrecipe)), ignore_status=True)
+            self.assertNotEqual(result.status, 0, 'devtool extract on %s should have failed' % testrecipe)
+            self.assertNotIn('Fetching ', result.output, 'devtool extract on %s should have errored out before trying to fetch' % testrecipe)
+            self.assertIn('ERROR: ', result.output, 'devtool extract on %s should have given an ERROR' % testrecipe)
+            # devtool modify should fail
+            result = runCmd('devtool modify %s -x %s' % (testrecipe, os.path.join(tempdir, testrecipe)), ignore_status=True)
+            self.assertNotEqual(result.status, 0, 'devtool modify on %s should have failed' % testrecipe)
+            self.assertIn('ERROR: ', result.output, 'devtool modify on %s should have given an ERROR' % testrecipe)
+
     def test_devtool_modify_git(self):
         # Check preconditions
         workspacedir = os.path.join(self.builddir, 'workspace')
scripts/lib/devtool/standard.py +35 -13

66 lines not on this page · raw message

Signature · 1 line
1.9.3
#6 lore raw

PATCH 6/16 devtool: deploy-target: allow disabling host key checking

In reply to Paul Eggleton

If you're testing with multiple images/devices that have the same IP address / hostname then it can be annoying to deal with host key mismatches all of the time. As a MITM attack is unlikely in the local test environment, provide a command line option to pass the appropriate options to scp/ssh to disable the host key checking.

Note: if you wish to apply this permanently, the best way is to do it through your ssh configuration e.g. by adding the following to your ~/.ssh/config:

Host 192.168.7.2
UserKnownHostsFile=/dev/null
StrictHostKeyChecking no

---
 scripts/lib/devtool/deploy.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)
scripts/lib/devtool/deploy.py +13 -4
index 896b618..d232d31 100644
@@ -1,6 +1,6 @@
 # Development tool - deploy/undeploy command plugin
 #
-# Copyright (C) 2014 Intel Corporation
+# Copyright (C) 2014-2015 Intel Corporation
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -50,7 +50,10 @@ def deploy(args, config, basepath, workspace):
 
     stdout, stderr = exec_build_env_command(config.init_path, basepath, 'bitbake -e %s' % args.recipename, shell=True)
     recipe_outdir = re.search(r'^D="(.*)"', stdout, re.MULTILINE).group(1)
-    ret = subprocess.call('scp -qr %s/* %s:%s' % (recipe_outdir, args.target, destdir), shell=True)
+    extraoptions = ''
+    if args.no_host_check:
+        extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
+    ret = subprocess.call('scp -qr %s %s/* %s:%s' % (extraoptions, recipe_outdir, args.target, destdir), shell=True)
     if ret != 0:
         return ret
 
@@ -77,12 +80,16 @@ def undeploy(args, config, basepath, workspace):
          logger.error('%s has not been deployed' % args.recipename)
          return -1
 
-    ret = subprocess.call("scp -q %s %s:/tmp" % (deploy_file, args.target), shell=True)
+    extraoptions = ''
+    if args.no_host_check:
+        extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
+
+    ret = subprocess.call("scp -q %s %s %s:/tmp" % (extraoptions, deploy_file, args.target), shell=True)
     if ret != 0:
         logger.error('Failed to copy %s to %s' % (deploy, args.target))
         return -1
 
-    ret = subprocess.call("ssh %s 'xargs -n1 rm -f </tmp/%s'" % (args.target, os.path.basename(deploy_file)), shell=True)
+    ret = subprocess.call("ssh %s %s 'xargs -n1 rm -f </tmp/%s'" % (extraoptions, args.target, os.path.basename(deploy_file)), shell=True)
     if ret == 0:
         logger.info('Successfully undeployed %s' % args.recipename)
         os.remove(deploy_file)
@@ -94,9 +101,11 @@ def register_commands(subparsers, context):
     parser_deploy = subparsers.add_parser('deploy-target', help='Deploy recipe output files to live target machine')
     parser_deploy.add_argument('recipename', help='Recipe to deploy')
     parser_deploy.add_argument('target', help='Live target machine running an ssh server: user@hostname[:destdir]')
+    parser_deploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true')
     parser_deploy.set_defaults(func=deploy)
 
     parser_undeploy = subparsers.add_parser('undeploy-target', help='Undeploy recipe output files in live target machine')
     parser_undeploy.add_argument('recipename', help='Recipe to undeploy')
     parser_undeploy.add_argument('target', help='Live target machine running an ssh server: user@hostname')
+    parser_undeploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true')
     parser_undeploy.set_defaults(func=undeploy)
Signature · 1 line
1.9.3
#7 lore raw

PATCH 7/16 devtool: deploy-target: detect and error if D is empty

In reply to Paul Eggleton

If you haven't built the recipe yet or if the output directory (${D}) is empty, then we should tell the user rather than have scp error out.

---
 scripts/lib/devtool/deploy.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)
scripts/lib/devtool/deploy.py +6 -2
index d232d31..68edb98 100644
@@ -43,13 +43,17 @@ def deploy(args, config, basepath, workspace):
     deploy_dir = os.path.join(basepath, 'target_deploy', args.target)
     deploy_file = os.path.join(deploy_dir, args.recipename + '.list')
 
+    stdout, stderr = exec_build_env_command(config.init_path, basepath, 'bitbake -e %s' % args.recipename, shell=True)
+    recipe_outdir = re.search(r'^D="(.*)"', stdout, re.MULTILINE).group(1)
+    if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir):
+        logger.error('No files to deploy - have you built the %s recipe? If so, the install step has not installed any files.' % args.recipename)
+        return -1
+
     if os.path.exists(deploy_file):
         if undeploy(args, config, basepath, workspace):
             # Error already shown
             return -1
 
-    stdout, stderr = exec_build_env_command(config.init_path, basepath, 'bitbake -e %s' % args.recipename, shell=True)
-    recipe_outdir = re.search(r'^D="(.*)"', stdout, re.MULTILINE).group(1)
     extraoptions = ''
     if args.no_host_check:
         extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
Signature · 1 line
1.9.3
#8 lore raw

PATCH 8/16 devtool: deploy-target: add an option to disable quiet mode

In reply to Paul Eggleton

The -q option to scp does stop the progress being shown, which is mostly superfluous, however it also stops errors from ssh being shown - if there's a problem, you'll just get "lost connection" which really isn't that helpful. As a compromise, add a -s/--show-status option and advertise this when the command fails.

---
 scripts/lib/devtool/deploy.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)
scripts/lib/devtool/deploy.py +12 -3
index 68edb98..c152ac0 100644
@@ -57,8 +57,11 @@ def deploy(args, config, basepath, workspace):
     extraoptions = ''
     if args.no_host_check:
         extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
-    ret = subprocess.call('scp -qr %s %s/* %s:%s' % (extraoptions, recipe_outdir, args.target, destdir), shell=True)
+    if not args.show_status:
+        extraoptions += ' -q'
+    ret = subprocess.call('scp -r %s %s/* %s:%s' % (extraoptions, recipe_outdir, args.target, destdir), shell=True)
     if ret != 0:
+        logger.error('Deploy failed - rerun with -s to get a complete error message')
         return ret
 
     logger.info('Successfully deployed %s' % recipe_outdir)
@@ -87,16 +90,20 @@ def undeploy(args, config, basepath, workspace):
     extraoptions = ''
     if args.no_host_check:
         extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
+    if not args.show_status:
+        extraoptions += ' -q'
 
-    ret = subprocess.call("scp -q %s %s %s:/tmp" % (extraoptions, deploy_file, args.target), shell=True)
+    ret = subprocess.call("scp %s %s %s:/tmp" % (extraoptions, deploy_file, args.target), shell=True)
     if ret != 0:
-        logger.error('Failed to copy %s to %s' % (deploy, args.target))
+        logger.error('Failed to copy file list to %s - rerun with -s to get a complete error message' % args.target)
         return -1
 
     ret = subprocess.call("ssh %s %s 'xargs -n1 rm -f </tmp/%s'" % (extraoptions, args.target, os.path.basename(deploy_file)), shell=True)
     if ret == 0:
         logger.info('Successfully undeployed %s' % args.recipename)
         os.remove(deploy_file)
+    else:
+        logger.error('Undeploy failed - rerun with -s to get a complete error message')
 
     return ret
 
@@ -106,10 +113,12 @@ def register_commands(subparsers, context):
     parser_deploy.add_argument('recipename', help='Recipe to deploy')
     parser_deploy.add_argument('target', help='Live target machine running an ssh server: user@hostname[:destdir]')
     parser_deploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true')
+    parser_deploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true')
     parser_deploy.set_defaults(func=deploy)
 
     parser_undeploy = subparsers.add_parser('undeploy-target', help='Undeploy recipe output files in live target machine')
     parser_undeploy.add_argument('recipename', help='Recipe to undeploy')
     parser_undeploy.add_argument('target', help='Live target machine running an ssh server: user@hostname')
     parser_undeploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true')
+    parser_undeploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true')
     parser_undeploy.set_defaults(func=undeploy)
Signature · 1 line
1.9.3
#9 lore raw

PATCH 9/16 classes/externalsrc: fix for recipes that fetch local files

In reply to Paul Eggleton

If SRC_URI contains local files (file:// references) these will almost certainly be required at some point during the build process, so we need to actually fetch these to ${WORKDIR} as we would normally.

---
 meta/classes/externalsrc.bbclass | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)
meta/classes/externalsrc.bbclass +13 -1
index 25f0be9..75bdb7a 100644
@@ -35,7 +35,13 @@ python () {
             d.setVar('B', externalsrcbuild)
         else:
             d.setVar('B', '${WORKDIR}/${BPN}-${PV}/')
-        d.setVar('SRC_URI', '')
+
+        srcuri = (d.getVar('SRC_URI', True) or '').split()
+        local_srcuri = []
+        for uri in srcuri:
+            if uri.startswith('file://'):
+                local_srcuri.append(uri)
+        d.setVar('SRC_URI', ' '.join(local_srcuri))
 
         if '{SRCPV}' in d.getVar('PV', False):
             # Dummy value because the default function can't be called with blank SRC_URI
@@ -65,7 +71,13 @@ python () {
                 if setvalue:
                     d.setVarFlag(task, 'cleandirs', ' '.join(cleandirs))
 
+        fetch_tasks = ['do_fetch', 'do_unpack']
+        # If we deltask do_patch, there's no dependency to ensure do_unpack gets run, so add one
+        d.appendVarFlag('do_configure', 'deps', ['do_unpack'])
+
         for task in d.getVar("SRCTREECOVEREDTASKS", True).split():
+            if local_srcuri and task in fetch_tasks:
+                continue
             bb.build.deltask(task, d)
 
         d.prependVarFlag('do_compile', 'prefuncs', "externalsrc_compile_prefunc ")
Signature · 1 line
1.9.3
#10 lore raw

PATCH 10/16 devtool: deploy-target: add dry-run option

In reply to Paul Eggleton

Add a dry-run option to the deploy-target and undeploy-target subcommands so you can see the list of files to be deployed or un-deployed before actually carrying out the operation.

---
 scripts/lib/devtool/deploy.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
scripts/lib/devtool/deploy.py +16 -0
index c152ac0..f016b23 100644
@@ -49,6 +49,13 @@ def deploy(args, config, basepath, workspace):
         logger.error('No files to deploy - have you built the %s recipe? If so, the install step has not installed any files.' % args.recipename)
         return -1
 
+    if args.dry_run:
+        print('Files to be deployed for %s on target %s:' % (args.recipename, args.target))
+        for root, dirs, files in os.walk(recipe_outdir):
+            for fn in files:
+                print('  %s' % os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn))
+        return 0
+
     if os.path.exists(deploy_file):
         if undeploy(args, config, basepath, workspace):
             # Error already shown
@@ -87,6 +94,13 @@ def undeploy(args, config, basepath, workspace):
          logger.error('%s has not been deployed' % args.recipename)
          return -1
 
+    if args.dry_run:
+        print('Previously deployed files to be un-deployed for %s on target %s:' % (args.recipename, args.target))
+        with open(deploy_file, 'r') as f:
+            for line in f:
+                print('  %s' % line.rstrip())
+        return 0
+
     extraoptions = ''
     if args.no_host_check:
         extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
@@ -114,6 +128,7 @@ def register_commands(subparsers, context):
     parser_deploy.add_argument('target', help='Live target machine running an ssh server: user@hostname[:destdir]')
     parser_deploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true')
     parser_deploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true')
+    parser_deploy.add_argument('-n', '--dry-run', help='List files to be deployed only', action='store_true')
     parser_deploy.set_defaults(func=deploy)
 
     parser_undeploy = subparsers.add_parser('undeploy-target', help='Undeploy recipe output files in live target machine')
@@ -121,4 +136,5 @@ def register_commands(subparsers, context):
     parser_undeploy.add_argument('target', help='Live target machine running an ssh server: user@hostname')
     parser_undeploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true')
     parser_undeploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true')
+    parser_undeploy.add_argument('-n', '--dry-run', help='List files to be undeployed only', action='store_true')
     parser_undeploy.set_defaults(func=undeploy)
Signature · 1 line
1.9.3
#11 lore raw

PATCH 11/16 gitignore: exclude meta-selftest, drop meta-hob

In reply to Paul Eggleton

* We don't want to ignore changes in the meta-selftest directory
* While I'm at it, meta-hob has been gone for a while now, so drop the reference to it here.

---
 .gitignore | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
.gitignore +1 -1
index 6ece6b8..e80a2fd 100644
@@ -9,7 +9,7 @@ scripts/oe-git-proxy-socks
 sources/
 meta-*/
 !meta-skeleton
-!meta-hob
+!meta-selftest
 hob-image-*.bb
 *.swp
 *.orig
Signature · 1 line
1.9.3
#12 lore raw

PATCH 12/16 oe-selftest: support getting unexported variable values

In reply to Paul Eggleton

Allow get_bb_var() to work with unexported variable values such as MACHINE - the workaround is a little crude but should suffice for now.

---
 meta/lib/oeqa/utils/commands.py | 7 +++++++
 1 file changed, 7 insertions(+)
meta/lib/oeqa/utils/commands.py +7 -0
index 5b601d9..e8a467f 100644
@@ -137,11 +137,18 @@ def get_bb_env(target=None, postconfig=None):
 def get_bb_var(var, target=None, postconfig=None):
     val = None
     bbenv = get_bb_env(target, postconfig=postconfig)
+    lastline = None
     for line in bbenv.splitlines():
         if line.startswith(var + "=") or line.startswith("export " + var + "="):
             val = line.split('=')[1]
             val = val.strip('\"')
             break
+        elif line.startswith("unset " + var):
+            # Handle [unexport] variables
+            if lastline.startswith('#   "'):
+                val = lastline.split('\"')[1]
+                break
+        lastline = line
     return val
 
 def get_test_layer():
Signature · 1 line
1.9.3
#13 lore raw

PATCH 13/16 oe-selftest: add a test for devtool deploy-target

In reply to Paul Eggleton

Whilst this test would seemingly be better placed as a runtime test, unfortunately the runtime tests run under bitbake and you can't run devtool within bitbake (since devtool needs to run bitbake itself). Additionally we are testing build-time functionality as well, so really this has to be done as an oe-selftest test.

This test does have a few perhaps unusual requirements in order to run:
* pexpect is installed
* MACHINE is set to one of the qemu machines
* runqemu tap devices have been set up

---
 .../recipes-test/images/oe-selftest-image.bb       |  9 ++++
 meta/lib/oeqa/selftest/devtool.py                  | 56 ++++++++++++++++++++++
 2 files changed, 65 insertions(+)
 create mode 100644 meta-selftest/recipes-test/images/oe-selftest-image.bb
meta-selftest/recipes-test/images/oe-selftest-image.bb new file +9 -0
new file mode 100644
index 0000000..f17094c
@@ -0,0 +1,9 @@
+SUMMARY = "An image used during oe-selftest tests"
+
+IMAGE_INSTALL = "packagegroup-core-boot ${ROOTFS_PKGMANAGE_BOOTSTRAP} dropbear"
+IMAGE_FEATURES = "debug-tweaks"
+
+IMAGE_LINGUAS = " "
+
+inherit core-image
+
meta/lib/oeqa/selftest/devtool.py +56 -0

59 lines not on this page · raw message

Signature · 1 line
1.9.3
#14 lore raw

PATCH 14/16 oe-selftest: add a test for devtool modify on recipes fetching local files

In reply to Paul Eggleton

With the change to externalsrc we can now handle these, so add a test to ensure they can be built.

---
 meta/lib/oeqa/selftest/devtool.py | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)
meta/lib/oeqa/selftest/devtool.py +32 -0
index 1caf0f0..869fecf 100644
@@ -284,6 +284,38 @@ class DevtoolTests(oeSelfTest):
         # Try building
         bitbake(testrecipe)
 
+    def test_devtool_modify_localfiles(self):
+        # Check preconditions
+        workspacedir = os.path.join(self.builddir, 'workspace')
+        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+        testrecipe = 'lighttpd'
+        src_uri = (get_bb_var('SRC_URI', testrecipe) or '').split()
+        foundlocal = False
+        for item in src_uri:
+            if item.startswith('file://') and '.patch' not in item:
+                foundlocal = True
+                break
+        self.assertTrue(foundlocal, 'This test expects the %s recipe to fetch local files and it seems that it no longer does' % testrecipe)
+        # Clean up anything in the workdir/sysroot/sstate cache
+        bitbake('%s -c cleansstate' % testrecipe)
+        # Try modifying a recipe
+        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
+        self.track_for_cleanup(tempdir)
+        self.track_for_cleanup(workspacedir)
+        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
+        self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe)
+        result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))
+        self.assertTrue(os.path.exists(os.path.join(tempdir, 'configure.ac')), 'Extracted source could not be found')
+        self.assertTrue(os.path.exists(os.path.join(workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created')
+        matches = glob.glob(os.path.join(workspacedir, 'appends', '%s_*.bbappend' % testrecipe))
+        self.assertTrue(matches, 'bbappend not created')
+        # Test devtool status
+        result = runCmd('devtool status')
+        self.assertIn(testrecipe, result.output)
+        self.assertIn(tempdir, result.output)
+        # Try building
+        bitbake(testrecipe)
+
     def test_devtool_update_recipe(self):
         # Check preconditions
         workspacedir = os.path.join(self.builddir, 'workspace')
Signature · 1 line
1.9.3
#15 lore raw

PATCH 15/16 recipetool: fix duplicate licenses being picked up

In reply to Paul Eggleton

If a license file matched more than one of the specifications (e.g. COPYING.GPL) then it was being added to LIC_FILES_CHKSUM more than once.

---
 scripts/lib/recipetool/create.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
scripts/lib/recipetool/create.py +3 -1
index ae599cb..1c71b24 100644
@@ -310,7 +310,9 @@ def guess_license(srctree):
         for fn in files:
             for spec in licspecs:
                 if fnmatch.fnmatch(fn, spec):
-                    licfiles.append(os.path.join(root, fn))
+                    fullpath = os.path.join(root, fn)
+                    if not fullpath in licfiles:
+                        licfiles.append(fullpath)
     for licfile in licfiles:
         md5value = bb.utils.md5_file(licfile)
         license = md5sums.get(md5value, 'Unknown')
Signature · 1 line
1.9.3
#16 lore raw

PATCH 16/16 oe-selftest: add missing dependency to test_recipetool_create_git

In reply to Paul Eggleton

libmatchbox links to libjpeg if it is present so just explicitly build it and then check it appears in DEPENDS.

---
 meta/lib/oeqa/selftest/devtool.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
meta/lib/oeqa/selftest/devtool.py +2 -2
index 869fecf..dc1cf21 100644
@@ -67,7 +67,7 @@ class DevtoolTests(oeSelfTest):
 
     def test_recipetool_create_git(self):
         # Ensure we have the right data in shlibs/pkgdata
-        bitbake('libpng pango libx11 libxext')
+        bitbake('libpng pango libx11 libxext jpeg')
         # Try adding a recipe
         tempdir = tempfile.mkdtemp(prefix='devtoolqa')
         self.track_for_cleanup(tempdir)
@@ -83,7 +83,7 @@ class DevtoolTests(oeSelfTest):
         checkvars['S'] = '${WORKDIR}/git'
         checkvars['PV'] = '1.0+git${SRCPV}'
         checkvars['SRC_URI'] = srcuri
-        checkvars['DEPENDS'] = 'libpng pango libx11 libxext'
+        checkvars['DEPENDS'] = 'libpng pango libx11 libxext jpeg'
         inherits = []
         with open(recipefile, 'r') as f:
             for line in f:
Signature · 1 line
1.9.3