Several fixes around recipetool appendsrcfile(s) and oe.recipeutils.bbappend_recipe
v3 cover Several fixes around recipetool appendsrcfile(s) and oe.recipeutils.bbappend_recipe
Hi all,
I was trying to use recipetool appendsrcfile to add a file to a recipe and I noticed several issues:
* -m is not correctly supported
* recipetool appendsrfile(s) are missing a usefull dry-run mode
* appendsrc function relies on oe.recipeutils.bbappend_recipe but
duplicates some code: it constructs itself the new src_uri entry
although oe.recipeutils.bbappend_recipe is already doing it
* we are lacking a mode to patch the recipe itself
So this series tries to fix the issues above, fix the selftest accordingly and add new test for -m and for "patch mode" (update recipe) and also add a way to specify the name of the file to add (in oe.recipeutils.bbappend_recipe)
Improvements from v2:
- rebase on latest master fix conflict and update the way to add newname in bbappend_recipe()
Improvements from v1:
- in bbappend_recipe function remove old entry if an entry with different parameters already exists to avoid duplicate entries with different parameters
Link to v1: https://lists.openembedded.org/g/openembedded-core/message/191544
Link to v2: https://lists.openembedded.org/g/openembedded-core/message/191744
Julien Stephan (10):
recipetool: appendsrcfile(s): add dry-run mode
recipeutils: bbappend_recipe: fix undefined variable
recipeutils: bbappend_recipe: fix docstring
recipeutils: bbappend_recipe: add a way to specify the name of the
file to add
recipeutils: bbappend_recipe: remove old srcuri entry if parameters
are different
recipetool: appendsrcfile(s): use params instead of extraline
recipeutils: bbappend_recipe: allow to patch the recipe itself
recipetool: appendsrcfile(s): add a mode to update the recipe itself oeqa/selftest/recipetool: appendsrfile: add test for machine oeqa/selftest/recipetool: appendsrc: add test for update mode
meta/lib/oe/recipeutils.py | 57 +++++++++++---- meta/lib/oeqa/selftest/cases/recipetool.py | 83 +++++++++++++++++++--- scripts/lib/recipetool/append.py | 63 +++++++++++----- 3 files changed, 164 insertions(+), 39 deletions(-)
Signature · 1 line
2.42.0
PATCH v3 1/10 recipetool: appendsrcfile(s): add dry-run mode
Add dry-run mode for recipetool appendsrcfile and appendsrcfiles, with if necessary, a diff of changes
--- scripts/lib/recipetool/append.py | 33 +++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-)
scripts/lib/recipetool/append.py +32 -1
@@ -18,6 +18,7 @@ import shutil import scriptutils import errno from collections import defaultdict +import difflib logger = logging.getLogger('recipetool')
@@ -355,7 +356,35 @@ def appendsrc(args, files, rd, extralines=None): extralines.append('SRC_URI += {0}'.format(source_uri)) copyfiles[newfile] = {'path' : srcfile} - oe.recipeutils.bbappend_recipe(rd, args.destlayer, copyfiles, None, wildcardver=args.wildcard_version, machine=args.machine, extralines=extralines) + dry_run_output = None + dry_run_outdir = None + if args.dry_run: + import tempfile + dry_run_output = tempfile.TemporaryDirectory(prefix='devtool') + dry_run_outdir = dry_run_output.name + + appendfile, _ = oe.recipeutils.bbappend_recipe(rd, args.destlayer, copyfiles, None, wildcardver=args.wildcard_version, machine=args.machine, extralines=extralines, redirect_output=dry_run_outdir) + if args.dry_run: + output = '' + appendfilename = os.path.basename(appendfile) + newappendfile = appendfile + if appendfile and os.path.exists(appendfile): + with open(appendfile, 'r') as f: + oldlines = f.readlines() + else: + appendfile = '/dev/null' + oldlines = [] + + with open(os.path.join(dry_run_outdir, appendfilename), 'r') as f: + newlines = f.readlines() + diff = difflib.unified_diff(oldlines, newlines, appendfile, newappendfile) + difflines = list(diff) + if difflines: + output += ''.join(difflines) + if output: + logger.info('Diff of changed files:\n%s' % output) + else: + logger.info('No changed files') tinfoil.modified_files() def appendsrcfiles(parser, args):
@@ -436,6 +465,7 @@ def register_commands(subparsers): help='Create/update a bbappend to add or replace source files', description='Creates a bbappend (or updates an existing one) to add or replace the specified file in the recipe sources, either those in WORKDIR or those in the source tree. This command lets you specify multiple files with a destination directory, so cannot specify the destination filename. See the `appendsrcfile` command for the other behavior.') parser.add_argument('-D', '--destdir', help='Destination directory (relative to S or WORKDIR, defaults to ".")', default='', type=destination_path) + parser.add_argument('-n', '--dry-run', help='Dry run mode', action='store_true') parser.add_argument('files', nargs='+', metavar='FILE', help='File(s) to be added to the recipe sources (WORKDIR or S)', type=existing_path) parser.set_defaults(func=lambda a: appendsrcfiles(parser, a), parserecipes=True)
@@ -443,6 +473,7 @@ def register_commands(subparsers): parents=[common_src], help='Create/update a bbappend to add or replace a source file', description='Creates a bbappend (or updates an existing one) to add or replace the specified files in the recipe sources, either those in WORKDIR or those in the source tree. This command lets you specify the destination filename, not just destination directory, but only works for one file. See the `appendsrcfiles` command for the other behavior.') + parser.add_argument('-n', '--dry-run', help='Dry run mode', action='store_true') parser.add_argument('file', metavar='FILE', help='File to be added to the recipe sources (WORKDIR or S)', type=existing_path) parser.add_argument('destfile', metavar='DESTFILE', nargs='?', help='Destination path (relative to S or WORKDIR, optional)', type=destination_path) parser.set_defaults(func=lambda a: appendsrcfile(parser, a), parserecipes=True)
Signature · 1 line
2.42.0
PATCH v3 2/10 recipeutils: bbappend_recipe: fix undefined variable
In reply to Julien StephanIn the case get_bbappend_path returns None (could not find the layer containing the recipe) the error message tries to print the recipefile, but it is not defined. Fix it.
--- meta/lib/oe/recipeutils.py | 1 + 1 file changed, 1 insertion(+)
meta/lib/oe/recipeutils.py +1 -0
@@ -702,6 +702,7 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, if not removevalues: removevalues = {} + recipefile = rd.getVar('FILE') # Determine how the bbappend should be named appendpath, pathok = get_bbappend_path(rd, destlayerdir, wildcardver) if not appendpath:
Signature · 1 line
2.42.0
PATCH v3 3/10 recipeutils: bbappend_recipe: fix docstring
In reply to Julien StephanAdd missing patchdir key in docstring description for srcfiles parameters.
Also fix typo: value --> key in srcfile docstring description
--- meta/lib/oe/recipeutils.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-)
meta/lib/oe/recipeutils.py +5 -3
@@ -671,10 +671,12 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, rd: data dictionary for the recipe destlayerdir: base directory of the layer to place the bbappend in (subdirectory path from there will be determined automatically) - srcfiles: dict of source files to add to SRC_URI, where the value + srcfiles: dict of source files to add to SRC_URI, where the key is the full path to the file to be added, and the value is a - dict with 'path' key containing the original filename as it - would appear in SRC_URI or None if it isn't already present. + dict with following optional keys: + path: the original filename as it would appear in SRC_URI + or None if it isn't already present. + patchdir: the patchdir parameter You may pass None for this parameter if you simply want to specify your own content via the extralines parameter. install: dict mapping entries in srcfiles to a tuple of two elements:
Signature · 1 line
2.42.0
PATCH v3 4/10 recipeutils: bbappend_recipe: add a way to specify the name of the file to add
In reply to Julien Stephanbbappend_recipe can take a dict of source files to add to SRC_URI where the key is the full path to the file to be added and the value is a dict
Add a new optionnal entry "newname" to specify the name of the newly added file
--- meta/lib/oe/recipeutils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-)
meta/lib/oe/recipeutils.py +6 -1
@@ -677,6 +677,8 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, path: the original filename as it would appear in SRC_URI or None if it isn't already present. patchdir: the patchdir parameter + newname: the name to give to the new added file. None to use + the default value: basename(path) You may pass None for this parameter if you simply want to specify your own content via the extralines parameter. install: dict mapping entries in srcfiles to a tuple of two elements:
@@ -769,7 +771,10 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, for i, (newfile, param) in enumerate(srcfiles.items()): srcurientry = None if not 'path' in param or not param['path']: - srcfile = os.path.basename(newfile) + if 'newname' in param and param['newname']: + srcfile = param['newname'] + else: + srcfile = os.path.basename(newfile) srcurientry = 'file://%s' % srcfile if params and params[i]: srcurientry = '%s;%s' % (srcurientry, ';'.join('%s=%s' % (k,v) for k,v in params[i].items()))
Signature · 1 line
2.42.0
PATCH v3 5/10 recipeutils: bbappend_recipe: remove old srcuri entry if parameters are different
In reply to Julien StephanCurrently we do not add a new src_ury entry if the entry already exists AND the parameters are the same.
I believe that when an entry already exist with different parameters, we should remove it and add the new entry otherwise we end up with two entries with different parameters
--- meta/lib/oe/recipeutils.py | 8 ++++++++ 1 file changed, 8 insertions(+)
meta/lib/oe/recipeutils.py +8 -0
@@ -776,14 +776,22 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, else: srcfile = os.path.basename(newfile) srcurientry = 'file://%s' % srcfile + oldentry = None + for uri in rd.getVar('SRC_URI').split(): + if srcurientry in uri: + oldentry = uri if params and params[i]: srcurientry = '%s;%s' % (srcurientry, ';'.join('%s=%s' % (k,v) for k,v in params[i].items())) # Double-check it's not there already # FIXME do we care if the entry is added by another bbappend that might go away? if not srcurientry in rd.getVar('SRC_URI').split(): if machine: + if oldentry: + appendline('SRC_URI:remove%s' % appendoverride, '=', ' ' + oldentry) appendline('SRC_URI:append%s' % appendoverride, '=', ' ' + srcurientry) else: + if oldentry: + appendline('SRC_URI:remove', '=', oldentry) appendline('SRC_URI', '+=', srcurientry) param['path'] = srcfile else:
Signature · 1 line
2.42.0
PATCH v3 6/10 recipetool: appendsrcfile(s): use params instead of extraline
In reply to Julien Stephanappendsrc function relies on oe.recipeutils.bbappend_recipe to copy files and add the corresponding entries in SRC_URI.
Currently, appendsrc function build itself the new SRC_URI entry to add the correct subdir param, and gives it using the extralines parameter. This has 2 drawbacks:
- oe.recipeutils.bbappend_recipe can already do this if we specify the correct params, so we have duplicate code
- the duplicated code is not fully functional: for example, it doesn't take into account the -m/--machine parameter
So fix this by not using extralines but give correctly formatted params.
Also remove the check for already existing entries as oe.recipeutils.bbappend_recipe already implement it
The new bbappend file now have the SRC_URI entry after the FILESEXTRAPATHS so fix the selftest.
Update test_recipetool_appendsrcfile_existing_in_src_uri_diff_params test because recipetool appendsrcfiles used to not add new src_uri entry if the entry already exist even with different parameters while oe.recipeutils.bbappend_recipe adds it if parameters are different (and remove the old entry)
--- meta/lib/oeqa/selftest/cases/recipetool.py | 21 ++++++++++------- scripts/lib/recipetool/append.py | 26 +++++++++------------- 2 files changed, 23 insertions(+), 24 deletions(-)
meta/lib/oeqa/selftest/cases/recipetool.py +13 -8
@@ -1054,9 +1054,9 @@ class RecipetoolAppendsrcBase(RecipetoolBase): for uri in src_uri: p = urllib.parse.urlparse(uri) if p.scheme == 'file': - return p.netloc + p.path + return p.netloc + p.path, uri - def _test_appendsrcfile(self, testrecipe, filename=None, destdir=None, has_src_uri=True, srcdir=None, newfile=None, options=''): + def _test_appendsrcfile(self, testrecipe, filename=None, destdir=None, has_src_uri=True, srcdir=None, newfile=None, remove=None, options=''): if newfile is None: newfile = self.testfile
@@ -1083,12 +1083,18 @@ class RecipetoolAppendsrcBase(RecipetoolBase): expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', '\n'] + + if remove: + for entry in remove: + expectedlines.extend(['SRC_URI:remove = "%s"\n' % entry, + '\n']) + if has_src_uri: uri = 'file://%s' % filename if expected_subdir: uri += ';subdir=%s' % expected_subdir - expectedlines[0:0] = ['SRC_URI += "%s"\n' % uri, - '\n'] + expectedlines.extend(['SRC_URI += "%s"\n' % uri, + '\n']) return self._try_recipetool_appendsrcfile(testrecipe, newfile, destpath, options, expectedlines, [filename])
@@ -1143,18 +1149,17 @@ class RecipetoolAppendsrcTests(RecipetoolAppendsrcBase): def test_recipetool_appendsrcfile_existing_in_src_uri(self): testrecipe = 'base-files' - filepath = self._get_first_file_uri(testrecipe) + filepath,_ = self._get_first_file_uri(testrecipe) self.assertTrue(filepath, 'Unable to test, no file:// uri found in SRC_URI for %s' % testrecipe) self._test_appendsrcfile(testrecipe, filepath, has_src_uri=False) def test_recipetool_appendsrcfile_existing_in_src_uri_diff_params(self): testrecipe = 'base-files' subdir = 'tmp' - filepath = self._get_first_file_uri(testrecipe) + filepath, srcuri_entry = self._get_first_file_uri(testrecipe) self.assertTrue(filepath, 'Unable to test, no file:// uri found in SRC_URI for %s' % testrecipe) - output = self._test_appendsrcfile(testrecipe, filepath, subdir, has_src_uri=False) - self.assertTrue(any('with different parameters' in l for l in output)) + self._test_appendsrcfile(testrecipe, filepath, subdir, remove=[srcuri_entry]) def test_recipetool_appendsrcfile_replace_file_srcdir(self): testrecipe = 'bash'
scripts/lib/recipetool/append.py +10 -16
@@ -300,7 +300,9 @@ def appendfile(args): if st.st_mode & stat.S_IXUSR: perms = '0755' install = {args.newfile: (args.targetpath, perms)} - oe.recipeutils.bbappend_recipe(rd, args.destlayer, {args.newfile: {'path' : sourcepath}}, install, wildcardver=args.wildcard_version, machine=args.machine) + if sourcepath: + sourcepath = os.path.basename(sourcepath) + oe.recipeutils.bbappend_recipe(rd, args.destlayer, {args.newfile: {'newname' : sourcepath}}, install, wildcardver=args.wildcard_version, machine=args.machine) tinfoil.modified_files() return 0 else:
@@ -329,6 +331,7 @@ def appendsrc(args, files, rd, extralines=None): copyfiles = {} extralines = extralines or [] + params = [] for newfile, srcfile in files.items(): src_destdir = os.path.dirname(srcfile) if not args.use_workdir:
@@ -339,22 +342,12 @@ def appendsrc(args, files, rd, extralines=None): src_destdir = os.path.join(os.path.relpath(srcdir, workdir), src_destdir) src_destdir = os.path.normpath(src_destdir) - source_uri = 'file://{0}'.format(os.path.basename(srcfile)) if src_destdir and src_destdir != '.': - source_uri += ';subdir={0}'.format(src_destdir) - - simple = bb.fetch.URI(source_uri) - simple.params = {} - simple_str = str(simple) - if simple_str in simplified: - existing = simplified[simple_str] - if source_uri != existing: - logger.warning('{0!r} is already in SRC_URI, with different parameters: {1!r}, not adding'.format(source_uri, existing)) - else: - logger.warning('{0!r} is already in SRC_URI, not adding'.format(source_uri)) + params.append({'subdir': src_destdir}) else: - extralines.append('SRC_URI += {0}'.format(source_uri)) - copyfiles[newfile] = {'path' : srcfile} + params.append({}) + + copyfiles[newfile] = {'newname' : os.path.basename(srcfile)} dry_run_output = None dry_run_outdir = None
@@ -363,7 +356,8 @@ def appendsrc(args, files, rd, extralines=None): dry_run_output = tempfile.TemporaryDirectory(prefix='devtool') dry_run_outdir = dry_run_output.name - appendfile, _ = oe.recipeutils.bbappend_recipe(rd, args.destlayer, copyfiles, None, wildcardver=args.wildcard_version, machine=args.machine, extralines=extralines, redirect_output=dry_run_outdir) + appendfile, _ = oe.recipeutils.bbappend_recipe(rd, args.destlayer, copyfiles, None, wildcardver=args.wildcard_version, machine=args.machine, extralines=extralines, params=params, + redirect_output=dry_run_outdir) if args.dry_run: output = '' appendfilename = os.path.basename(appendfile)
Signature · 1 line
2.42.0
PATCH v3 7/10 recipeutils: bbappend_recipe: allow to patch the recipe itself
In reply to Julien StephanAdd a new parameter update_original_recipe to allow to patch a recipe instead of creating/updating a bbappend
--- meta/lib/oe/recipeutils.py | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-)
meta/lib/oe/recipeutils.py +25 -10
@@ -664,7 +664,7 @@ def get_bbappend_path(d, destlayerdir, wildcardver=False): return (appendpath, pathok) -def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, machine=None, extralines=None, removevalues=None, redirect_output=None, params=None): +def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, machine=None, extralines=None, removevalues=None, redirect_output=None, params=None, update_original_recipe=False): """ Writes a bbappend file for a recipe Parameters:
@@ -701,19 +701,29 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, params: Parameters to use when adding entries to SRC_URI. If specified, should be a list of dicts with the same length as srcfiles. + update_original_recipe: + Force to update the original recipe instead of creating/updating + a bbapend. destlayerdir must contain the original recipe """ if not removevalues: removevalues = {} recipefile = rd.getVar('FILE') - # Determine how the bbappend should be named - appendpath, pathok = get_bbappend_path(rd, destlayerdir, wildcardver) - if not appendpath: - bb.error('Unable to determine layer directory containing %s' % recipefile) - return (None, None) - if not pathok: - bb.warn('Unable to determine correct subdirectory path for bbappend file - check that what %s adds to BBFILES also matches .bbappend files. Using %s for now, but until you fix this the bbappend will not be applied.' % (os.path.join(destlayerdir, 'conf', 'layer.conf'), os.path.dirname(appendpath))) + if update_original_recipe: + if destlayerdir not in recipefile: + bb.error("destlayerdir %s doesn't contain the original recipe (%s), cannot update it" % (destlayerdir, recipefile)) + return (None, None) + + appendpath = recipefile + else: + # Determine how the bbappend should be named + appendpath, pathok = get_bbappend_path(rd, destlayerdir, wildcardver) + if not appendpath: + bb.error('Unable to determine layer directory containing %s' % recipefile) + return (None, None) + if not pathok: + bb.warn('Unable to determine correct subdirectory path for bbappend file - check that what %s adds to BBFILES also matches .bbappend files. Using %s for now, but until you fix this the bbappend will not be applied.' % (os.path.join(destlayerdir, 'conf', 'layer.conf'), os.path.dirname(appendpath))) appenddir = os.path.dirname(appendpath) if not redirect_output:
@@ -758,7 +768,7 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, bbappendlines.append((varname, op, value)) destsubdir = rd.getVar('PN') - if srcfiles: + if not update_original_recipe and srcfiles: bbappendlines.append(('FILESEXTRAPATHS:prepend', ':=', '${THISDIR}/${PN}:')) appendoverride = ''
@@ -791,7 +801,10 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, appendline('SRC_URI:append%s' % appendoverride, '=', ' ' + srcurientry) else: if oldentry: - appendline('SRC_URI:remove', '=', oldentry) + if update_original_recipe: + removevalues['SRC_URI'] = oldentry + else: + appendline('SRC_URI:remove', '=', oldentry) appendline('SRC_URI', '+=', srcurientry) param['path'] = srcfile else:
@@ -816,6 +829,8 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, # multiple times per operation when we're handling overrides) if os.path.exists(appendpath) and not os.path.exists(outfile): shutil.copy2(appendpath, outfile) + elif update_original_recipe: + outfile = recipefile else: bb.note('Writing append file %s' % appendpath) outfile = appendpath
Signature · 1 line
2.42.0
PATCH v3 8/10 recipetool: appendsrcfile(s): add a mode to update the recipe itself
In reply to Julien StephanAdd a new option -u/--update-recipe to update the recipe itself instead of creating/updating a bbappend file
--- scripts/lib/recipetool/append.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-)
scripts/lib/recipetool/append.py +5 -1
@@ -357,7 +357,9 @@ def appendsrc(args, files, rd, extralines=None): dry_run_outdir = dry_run_output.name appendfile, _ = oe.recipeutils.bbappend_recipe(rd, args.destlayer, copyfiles, None, wildcardver=args.wildcard_version, machine=args.machine, extralines=extralines, params=params, - redirect_output=dry_run_outdir) + redirect_output=dry_run_outdir, update_original_recipe=args.update_recipe) + if not appendfile: + return if args.dry_run: output = '' appendfilename = os.path.basename(appendfile)
@@ -459,6 +461,7 @@ def register_commands(subparsers): help='Create/update a bbappend to add or replace source files', description='Creates a bbappend (or updates an existing one) to add or replace the specified file in the recipe sources, either those in WORKDIR or those in the source tree. This command lets you specify multiple files with a destination directory, so cannot specify the destination filename. See the `appendsrcfile` command for the other behavior.') parser.add_argument('-D', '--destdir', help='Destination directory (relative to S or WORKDIR, defaults to ".")', default='', type=destination_path) + parser.add_argument('-u', '--update-recipe', help='Update recipe instead of creating (or updating) a bbapend file. DESTLAYER must contains the recipe to update', action='store_true') parser.add_argument('-n', '--dry-run', help='Dry run mode', action='store_true') parser.add_argument('files', nargs='+', metavar='FILE', help='File(s) to be added to the recipe sources (WORKDIR or S)', type=existing_path) parser.set_defaults(func=lambda a: appendsrcfiles(parser, a), parserecipes=True)
@@ -467,6 +470,7 @@ def register_commands(subparsers): parents=[common_src], help='Create/update a bbappend to add or replace a source file', description='Creates a bbappend (or updates an existing one) to add or replace the specified files in the recipe sources, either those in WORKDIR or those in the source tree. This command lets you specify the destination filename, not just destination directory, but only works for one file. See the `appendsrcfiles` command for the other behavior.') + parser.add_argument('-u', '--update-recipe', help='Update recipe instead of creating (or updating) a bbapend file. DESTLAYER must contains the recipe to update', action='store_true') parser.add_argument('-n', '--dry-run', help='Dry run mode', action='store_true') parser.add_argument('file', metavar='FILE', help='File to be added to the recipe sources (WORKDIR or S)', type=existing_path) parser.add_argument('destfile', metavar='DESTFILE', nargs='?', help='Destination path (relative to S or WORKDIR, optional)', type=destination_path)
Signature · 1 line
2.42.0
PATCH v3 9/10 oeqa/selftest/recipetool: appendsrfile: add test for machine
In reply to Julien StephanAdd a new test for machine specific bbappend override
--- meta/lib/oeqa/selftest/cases/recipetool.py | 44 +++++++++++++++++++--- 1 file changed, 38 insertions(+), 6 deletions(-)
meta/lib/oeqa/selftest/cases/recipetool.py +38 -6
@@ -1056,7 +1056,7 @@ class RecipetoolAppendsrcBase(RecipetoolBase): if p.scheme == 'file': return p.netloc + p.path, uri - def _test_appendsrcfile(self, testrecipe, filename=None, destdir=None, has_src_uri=True, srcdir=None, newfile=None, remove=None, options=''): + def _test_appendsrcfile(self, testrecipe, filename=None, destdir=None, has_src_uri=True, srcdir=None, newfile=None, remove=None, machine=None , options=''): if newfile is None: newfile = self.testfile
@@ -1084,17 +1084,39 @@ class RecipetoolAppendsrcBase(RecipetoolBase): expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', '\n'] + override = "" + if machine: + options += ' -m %s' % machine + override = ':append:%s' % machine + expectedlines.extend(['PACKAGE_ARCH = "${MACHINE_ARCH}"\n', + '\n']) + if remove: for entry in remove: - expectedlines.extend(['SRC_URI:remove = "%s"\n' % entry, + if machine: + entry_remove_line = 'SRC_URI:remove:%s = " %s"\n' % (machine, entry) + else: + entry_remove_line = 'SRC_URI:remove = "%s"\n' % entry + + expectedlines.extend([entry_remove_line, '\n']) if has_src_uri: uri = 'file://%s' % filename if expected_subdir: uri += ';subdir=%s' % expected_subdir - expectedlines.extend(['SRC_URI += "%s"\n' % uri, - '\n']) + if machine: + src_uri_line = 'SRC_URI%s = " %s"\n' % (override, uri) + else: + src_uri_line = 'SRC_URI += "%s"\n' % uri + + expectedlines.extend([src_uri_line, '\n']) + + with open("/tmp/tmp.txt", "w") as file: + print(expectedlines, file=file) + + if machine: + filename = '%s/%s' % (machine, filename) return self._try_recipetool_appendsrcfile(testrecipe, newfile, destpath, options, expectedlines, [filename])
@@ -1153,13 +1175,23 @@ class RecipetoolAppendsrcTests(RecipetoolAppendsrcBase): self.assertTrue(filepath, 'Unable to test, no file:// uri found in SRC_URI for %s' % testrecipe) self._test_appendsrcfile(testrecipe, filepath, has_src_uri=False) - def test_recipetool_appendsrcfile_existing_in_src_uri_diff_params(self): + def test_recipetool_appendsrcfile_existing_in_src_uri_diff_params(self, machine=None): testrecipe = 'base-files' subdir = 'tmp' filepath, srcuri_entry = self._get_first_file_uri(testrecipe) self.assertTrue(filepath, 'Unable to test, no file:// uri found in SRC_URI for %s' % testrecipe) - self._test_appendsrcfile(testrecipe, filepath, subdir, remove=[srcuri_entry]) + self._test_appendsrcfile(testrecipe, filepath, subdir, machine=machine, remove=[srcuri_entry]) + + def test_recipetool_appendsrcfile_machine(self): + # A very basic test + self._test_appendsrcfile('base-files', 'a-file', machine='mymachine') + + # Force cleaning the output of previous test + self.tearDownLocal() + + # A more complex test: existing entry in src_uri with different param + self.test_recipetool_appendsrcfile_existing_in_src_uri_diff_params(machine='mymachine') def test_recipetool_appendsrcfile_replace_file_srcdir(self): testrecipe = 'bash'
Signature · 1 line
2.42.0
Re: [OE-core] [PATCH v3 09/10] oeqa/selftest/recipetool: appendsrfile: add test for machine
Quoted 71 lines · Julien Stephan
Add a new test for machine specific bbappend override
Signed-off-by Julien Stephan <[email protected]>--- meta/lib/oeqa/selftest/cases/recipetool.py | 44 +++++++++++++++++++--- 1 file changed, 38 insertions(+), 6 deletions(-)meta/lib/oeqa/selftest/cases/recipetool.py +27
-5@@ -1056,7 +1056,7 @@ class RecipetoolAppendsrcBase(RecipetoolBase): if p.scheme == 'file': return p.netloc + p.path, uri - def _test_appendsrcfile(self, testrecipe, filename=None, destdir=None, has_src_uri=True, srcdir=None, newfile=None, remove=None, options=''): + def _test_appendsrcfile(self, testrecipe, filename=None, destdir=None, has_src_uri=True, srcdir=None, newfile=None, remove=None, machine=None , options=''): if newfile is None: newfile = self.testfile@@ -1084,17 +1084,39 @@ class RecipetoolAppendsrcBase(RecipetoolBase): expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', '\n'] + override = "" + if machine: + options += ' -m %s' % machine + override = ':append:%s' % machine + expectedlines.extend(['PACKAGE_ARCH = "${MACHINE_ARCH}"\n', + '\n']) + if remove: for entry in remove: - expectedlines.extend(['SRC_URI:remove = "%s"\n' % entry, + if machine: + entry_remove_line = 'SRC_URI:remove:%s = " %s"\n' % (machine, entry) + else: + entry_remove_line = 'SRC_URI:remove = "%s"\n' % entry + + expectedlines.extend([entry_remove_line, '\n']) if has_src_uri: uri = 'file://%s' % filename if expected_subdir: uri += ';subdir=%s' % expected_subdir - expectedlines.extend(['SRC_URI += "%s"\n' % uri, - '\n']) + if machine: + src_uri_line = 'SRC_URI%s = " %s"\n' % (override, uri) + else: + src_uri_line = 'SRC_URI += "%s"\n' % uri + + expectedlines.extend([src_uri_line, '\n']) + + with open("/tmp/tmp.txt", "w") as file: + print(expectedlines, file=file) + + if machine: + filename = '%s/%s' % (machine, filename) return self._try_recipetool_appendsrcfile(testrecipe, newfile, destpath, options, expectedlines, [filename])@@ -1153,13 +1175,23 @@ class RecipetoolAppendsrcTests(RecipetoolAppendsrcBase): self.assertTrue(filepath, 'Unable to test, no file:// uri found in SRC_URI for %s' % testrecipe) self._test_appendsrcfile(testrecipe, filepath, has_src_uri=False) - def test_recipetool_appendsrcfile_existing_in_src_uri_diff_params(self): + def test_recipetool_appendsrcfile_existing_in_src_uri_diff_params(self, machine=None): testrecipe = 'base-files' subdir = 'tmp'
meta/lib/oeqa/selftest/cases/recipetool.py class RecipetoolAppendsrcTests(RecipetoolAppendsrcBase): · line 1189
Quoted 10 lines
meta/lib/oeqa/selftest/cases/recipetool.py +6
-1filepath, srcuri_entry = self._get_first_file_uri(testrecipe) self.assertTrue(filepath, 'Unable to test, no file:// uri found in SRC_URI for %s' % testrecipe) - self._test_appendsrcfile(testrecipe, filepath, subdir, remove=[srcuri_entry]) + self._test_appendsrcfile(testrecipe, filepath, subdir, machine=machine, remove=[srcuri_entry]) + + def test_recipetool_appendsrcfile_machine(self): + # A very basic test + self._test_appendsrcfile('base-files', 'a-file', machine='mymachine') +
I think this is causing problems with other selftests, e.g. this set of failures here:
https://autobuilder.yoctoproject.org/typhoon/#/builders/127/builds/2559/steps/14/logs/stdio
with errors like:
ERROR: When reparsing /home/pokybuild/yocto-worker/oe-selftest-armhost/build/meta/recipes-core/base-files/base-files_3.0.14.bb:do_fetch, the basehash value changed from b720db5f2039a9a72917b5d400acd7f874d37014fec362112a44f90504ba79cd to 1b5f5a1ec1552e4a1e5798b2a1c23c062be5054be287d79c98e0e29da780c3b0. The metadata is not deterministic and this needs to be fixed.
ERROR: The following commands may help:
ERROR: $ bitbake base-files -cdo_fetch -Snone
ERROR: Then:
ERROR: $ bitbake base-files -cdo_fetch -Sprintdiff
since this test is running on the same metadata as the other tests are using. We need to keep modifications isolated from the other tests that could be running.
Cheers,
Richard
PATCH v3 10/10 oeqa/selftest/recipetool: appendsrc: add test for update mode
In reply to Julien Stephanadd a basic test for testing the update mode of recipetool appendsrcfile(s)
--- meta/lib/oeqa/selftest/cases/recipetool.py | 28 ++++++++++++++++++++++ 1 file changed, 28 insertions(+)
meta/lib/oeqa/selftest/cases/recipetool.py +28 -0
@@ -1193,6 +1193,34 @@ class RecipetoolAppendsrcTests(RecipetoolAppendsrcBase): # A more complex test: existing entry in src_uri with different param self.test_recipetool_appendsrcfile_existing_in_src_uri_diff_params(machine='mymachine') + def test_recipetool_appendsrcfile_update_recipe_basic(self): + testrecipe = "base-files" + recipefile = get_bb_var('FILE', testrecipe) + result = runCmd('bitbake-layers show-layers') + layerrecipe = None + for line in result.output.splitlines()[3:]: + with open("/tmp/juju.txt", "a") as file: + layer = line.split()[1] + print(layer, file=file) + if layer in recipefile: + layerrecipe = layer + break + self.assertTrue(layerrecipe, 'Unable to find the layer containing %s' % testrecipe) + cmd = 'recipetool appendsrcfile -u %s %s %s' % (layerrecipe, testrecipe, self.testfile) + result = runCmd(cmd) + self.assertNotIn('Traceback', result.output) + self.add_command_to_tearDown('cd %s; rm -f %s/%s; git checkout .' % (os.path.dirname(recipefile), testrecipe, os.path.basename(self.testfile))) + + expected_status = [(' M', '.*/%s$' % os.path.basename(recipefile)), + ('??', '.*/%s/%s/%s$' % (testrecipe, testrecipe, os.path.basename(self.testfile)))] + self._check_repo_status(os.path.dirname(recipefile), expected_status) + result = runCmd('git diff %s' % os.path.basename(recipefile), cwd=os.path.dirname(recipefile)) + removelines = [] + addlines = [ + 'file://%s \\\\' % os.path.basename(self.testfile), + ] + self._check_diff(result.output, addlines, removelines) + def test_recipetool_appendsrcfile_replace_file_srcdir(self): testrecipe = 'bash' filepath = 'Makefile.in'
Signature · 1 line
2.42.0