Skip to contentSkip to conversation

recipetool improvements

PATCH New Paul Eggleton 1 participant 8 messages May 29, 2016

lore
Share & tools
lore
#0 lore raw

cover recipetool improvements

Make the following improvements to recipetool create (and by extension, devtool add):
* Improve creation of recipes for binary deb/ipk/rpm package files with -b
* General supporting improvements to patch_recipe_* functions
* Set a fixed SRCREV when pointing to a git repo by default rather than using ${AUTOREV}

The following changes since commit bb4ead9b7b1400c37a72d148d9775bdf4210ec37:

linux-yocto/4.4: integrate v4.4.11 (2016-05-25 07:49:55 +0100)

are available in the git repository at:

git://git.openembedded.org/openembedded-core-contrib paule/recipetool-fixes2-oe
http://cgit.openembedded.org/cgit.cgi/openembedded-core-contrib/log/?h=paule/recipetool-fixes2-oe

Paul Eggleton (7):
lib/oe/recipeutils: split out patch_recipe_lines() lib/oe/recipeutils: patch_recipe_lines: allow omitting trailing newlines lib/oe/recipeutils: fix insertion of variable values
recipetool: create: support extracting SUMMARY and HOMEPAGE
recipetool: create: extract variable values from .deb/.ipk/.rpm
recipetool: create: use ${BP} for subdir for binary packages recipetool / devtool: set a fixed SRCREV by default when fetching from git

 meta/lib/oe/recipeutils.py                |  40 ++++++--
 meta/lib/oeqa/selftest/devtool.py         |   6 +-
 scripts/lib/devtool/standard.py           |   3 +
 scripts/lib/recipetool/create.py          | 165 +++++++++++++++++++++++-------
 scripts/lib/recipetool/create_buildsys.py |  37 ++++---
 scripts/lib/recipetool/create_npm.py      |   4 +-
 6 files changed, 193 insertions(+), 62 deletions(-)
Signature · 1 line
2.5.5
#1 lore raw

PATCH 1/7 lib/oe/recipeutils: split out patch_recipe_lines()

Split out a function from patch_recipe_file() that takes just the lines as input so we can edit recipe lines in memory. This will be used within recipetool to ensure we insert new values in the right place.

---
 meta/lib/oe/recipeutils.py | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)
meta/lib/oe/recipeutils.py +18 -5
index 6c7adb5..ef82755 100644
@@ -158,9 +158,8 @@ def split_var_value(value, assignment=True):
     return outlist
 
 
-def patch_recipe_file(fn, values, patch=False, relpath=''):
-    """Update or insert variable values into a recipe file (assuming you
-       have already identified the exact file you want to update.)
+def patch_recipe_lines(fromlines, values):
+    """Update or insert variable values into lines from a recipe.
        Note that some manual inspection/intervention may be required
        since this cannot handle all situations.
     """
@@ -247,8 +246,7 @@ def patch_recipe_file(fn, values, patch=False, relpath=''):
 
     # First run - establish which values we want to set are already in the file
     varlist = [re.escape(item) for item in values.keys()]
-    with open(fn, 'r') as f:
-        changed, fromlines = bb.utils.edit_metadata(f, varlist, patch_recipe_varfunc)
+    bb.utils.edit_metadata(fromlines, varlist, patch_recipe_varfunc)
     # Second run - actually set everything
     modifying = True
     varlist.extend(recipe_progression_restrs)
@@ -260,6 +258,21 @@ def patch_recipe_file(fn, values, patch=False, relpath=''):
         for k in remainingnames.keys():
             outputvalue(k, tolines)
 
+    return changed, tolines
+
+
+def patch_recipe_file(fn, values, patch=False, relpath=''):
+    """Update or insert variable values into a recipe file (assuming you
+       have already identified the exact file you want to update.)
+       Note that some manual inspection/intervention may be required
+       since this cannot handle all situations.
+    """
+
+    with open(fn, 'r') as f:
+        fromlines = f.readlines()
+
+    _, tolines = patch_recipe_lines(fromlines, values)
+
     if patch:
         relfn = os.path.relpath(fn, relpath)
         diff = difflib.unified_diff(fromlines, tolines, 'a/%s' % relfn, 'b/%s' % relfn)
Signature · 1 line
2.5.5
#2 lore raw

PATCH 2/7 lib/oe/recipeutils: patch_recipe_lines: allow omitting trailing newlines

In reply to Paul Eggleton

This function was assuming that what you wanted was that output lines had trailing newline characters. If you're just outputting each line verbatim to a text file then that's fine, but sometimes you start with the assumption that the lines don't have trailing newlines; thus we shouldn't allow for the possibility that the caller doesn't want them and add a parameter to control it.

---
 meta/lib/oe/recipeutils.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)
meta/lib/oe/recipeutils.py +11 -6
index ef82755..b9f6ada 100644
@@ -158,7 +158,7 @@ def split_var_value(value, assignment=True):
     return outlist
 
 
-def patch_recipe_lines(fromlines, values):
+def patch_recipe_lines(fromlines, values, trailing_newline=True):
     """Update or insert variable values into lines from a recipe.
        Note that some manual inspection/intervention may be required
        since this cannot handle all situations.
@@ -166,6 +166,11 @@ def patch_recipe_lines(fromlines, values):
 
     import bb.utils
 
+    if trailing_newline:
+        newline = '\n'
+    else:
+        newline = ''
+
     recipe_progression_res = []
     recipe_progression_restrs = []
     for item in recipe_progression:
@@ -196,7 +201,7 @@ def patch_recipe_lines(fromlines, values):
     def outputvalue(name, lines, rewindcomments=False):
         if values[name] is None:
             return
-        rawtext = '%s = "%s"\n' % (name, values[name])
+        rawtext = '%s = "%s"%s' % (name, values[name], newline)
         addlines = []
         if name in nowrap_vars:
             addlines.append(rawtext)
@@ -204,19 +209,19 @@ def patch_recipe_lines(fromlines, values):
             splitvalue = split_var_value(values[name], assignment=False)
             if len(splitvalue) > 1:
                 linesplit = ' \\\n' + (' ' * (len(name) + 4))
-                addlines.append('%s = "%s%s"\n' % (name, linesplit.join(splitvalue), linesplit))
+                addlines.append('%s = "%s%s"%s' % (name, linesplit.join(splitvalue), linesplit, newline))
             else:
                 addlines.append(rawtext)
         else:
             wrapped = textwrap.wrap(rawtext)
             for wrapline in wrapped[:-1]:
-                addlines.append('%s \\\n' % wrapline)
-            addlines.append('%s\n' % wrapped[-1])
+                addlines.append('%s \\%s' % (wrapline, newline))
+            addlines.append('%s%s' % (wrapped[-1], newline))
         if rewindcomments:
             # Ensure we insert the lines before any leading comments
             # (that we'd want to ensure remain leading the next value)
             for i, ln in reversed(list(enumerate(lines))):
-                if ln[0] != '#':
+                if not ln.startswith('#'):
                     lines[i+1:i+1] = addlines
                     break
             else:
Signature · 1 line
2.5.5
#3 lore raw

PATCH 3/7 lib/oe/recipeutils: fix insertion of variable values

In reply to Paul Eggleton

Add some more variables in appropriate places in recipe_progression such that the patch_recipe_* functions are able to insert variables in the right place within a recipe.

---
 meta/lib/oe/recipeutils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
meta/lib/oe/recipeutils.py +1 -1
index b9f6ada..3e17873 100644
@@ -19,7 +19,7 @@ from collections import OrderedDict, defaultdict
 
 
 # Help us to find places to insert values
-recipe_progression = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRCREV', 'SRC_URI', 'S', 'do_fetch()', 'do_unpack()', 'do_patch()', 'EXTRA_OECONF', 'do_configure()', 'EXTRA_OEMAKE', 'do_compile()', 'do_install()', 'do_populate_sysroot()', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'do_package()', 'do_deploy()']
+recipe_progression = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LICENSE_FLAGS', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRCREV', 'SRCPV', 'SRC_URI', 'S', 'do_fetch()', 'do_unpack()', 'do_patch()', 'EXTRA_OECONF', 'EXTRA_OECMAKE', 'EXTRA_OESCONS', 'do_configure()', 'EXTRA_OEMAKE', 'do_compile()', 'do_install()', 'do_populate_sysroot()', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'populate_packages()', 'do_package()', 'do_deploy()']
 # Variables that sometimes are a bit long but shouldn't be wrapped
 nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER', 'SRC_URI[md5sum]', 'SRC_URI[sha256sum]']
 list_vars = ['SRC_URI', 'LIC_FILES_CHKSUM']
Signature · 1 line
2.5.5
#4 lore raw

PATCH 4/7 recipetool: create: support extracting SUMMARY and HOMEPAGE

In reply to Paul Eggleton

Allow plugins to set any variable value through the extravalues dict, and use this to support extracting SUMMARY and HOMEPAGE values from spec files included with the source; additionally translate "License:" to a comment next to the LICENSE field (we have our own logic for setting LICENSE, but it will often be useful to see what the spec file says if one is present).

Also use the same mechanism for setting the same variables for node.js modules; this was already supported but wasn't inserting the settings in the appropriate place in the file which this will now do.

---
 scripts/lib/recipetool/create.py          | 49 +++++++++++++++++--------------
 scripts/lib/recipetool/create_buildsys.py | 37 +++++++++++++++--------
 scripts/lib/recipetool/create_npm.py      |  4 +--
 3 files changed, 54 insertions(+), 36 deletions(-)
scripts/lib/recipetool/create.py +27 -22
index aade40b..66c881a 100644
@@ -331,6 +331,7 @@ def create_recipe(args):
     import bb.process
     import tempfile
     import shutil
+    import oe.recipeutils
 
     pkgarch = ""
     if args.machine:
@@ -429,7 +430,8 @@ def create_recipe(args):
     lines_before.append('# Recipe created by %s' % os.path.basename(sys.argv[0]))
     lines_before.append('# This is the basis of a recipe and may need further editing in order to be fully functional.')
     lines_before.append('# (Feel free to remove these comments when editing.)')
-    lines_before.append('#')
+    # We need a blank line here so that patch_recipe_lines can rewind before the LICENSE comments
+    lines_before.append('')
 
     licvalues = guess_license(srctree_use)
     lic_files_chksum = []
@@ -561,28 +563,28 @@ def create_recipe(args):
         handler.process(srctree_use, classes, lines_before, lines_after, handled, extravalues)
 
     extrafiles = extravalues.pop('extrafiles', {})
+    extra_pn = extravalues.pop('PN', None)
+    extra_pv = extravalues.pop('PV', None)
 
-    if not realpv:
-        realpv = extravalues.get('PV', None)
-        if realpv:
-            if not validate_pv(realpv):
-                realpv = None
-            else:
-                realpv = realpv.lower().split()[0]
-                if '_' in realpv:
-                    realpv = realpv.replace('_', '-')
-    if not pn:
-        pn = extravalues.get('PN', None)
-        if pn:
-            if pn.startswith('GNU '):
-                pn = pn[4:]
-            if ' ' in pn:
-                # Probably a descriptive identifier rather than a proper name
-                pn = None
-            else:
-                pn = pn.lower()
-                if '_' in pn:
-                    pn = pn.replace('_', '-')
+    if extra_pv and not realpv:
+        realpv = extra_pv
+        if not validate_pv(realpv):
+            realpv = None
+        else:
+            realpv = realpv.lower().split()[0]
+            if '_' in realpv:
+                realpv = realpv.replace('_', '-')
+    if extra_pn and not pn:
+        pn = extra_pn
+        if pn.startswith('GNU '):
+            pn = pn[4:]
+        if ' ' in pn:
+            # Probably a descriptive identifier rather than a proper name
+            pn = None
+        else:
+            pn = pn.lower()
+            if '_' in pn:
+                pn = pn.replace('_', '-')
 
     if not outfile:
         if not pn:
@@ -662,6 +664,9 @@ def create_recipe(args):
         outlines.append('')
     outlines.extend(lines_after)
 
+    if extravalues:
+        _, outlines = oe.recipeutils.patch_recipe_lines(outlines, extravalues, trailing_newline=False)
+
     if args.extract_to:
         scriptutils.git_convert_standalone_clone(srctree)
         if os.path.isdir(args.extract_to):
scripts/lib/recipetool/create_buildsys.py +25 -12
index f84ec3d..37d161e 100644
@@ -830,22 +830,35 @@ class SpecFileRecipeHandler(RecipeHandler):
         if 'PV' in extravalues and 'PN' in extravalues:
             return
         filelist = RecipeHandler.checkfiles(srctree, ['*.spec'], recursive=True)
-        pn = None
-        pv = None
+        valuemap = {'Name': 'PN',
+                    'Version': 'PV',
+                    'Summary': 'SUMMARY',
+                    'Url': 'HOMEPAGE',
+                    'License': 'LICENSE'}
+        foundvalues = {}
         for fileitem in filelist:
             linecount = 0
             with open(fileitem, 'r') as f:
                 for line in f:
-                    if line.startswith('Name:') and not pn:
-                        pn = line.split(':')[1].strip()
-                    if line.startswith('Version:') and not pv:
-                        pv = line.split(':')[1].strip()
-            if pv or pn:
-                if pv and not 'PV' in extravalues and validate_pv(pv):
-                    extravalues['PV'] = pv
-                if pn and not 'PN' in extravalues:
-                    extravalues['PN'] = pn
-                break
+                    for value, varname in valuemap.iteritems():
+                        if line.startswith(value + ':') and not varname in foundvalues:
+                            foundvalues[varname] = line.split(':', 1)[1].strip()
+                            break
+                    if len(foundvalues) == len(valuemap):
+                        break
+        if 'PV' in foundvalues:
+            if not validate_pv(foundvalues['PV']):
+                del foundvalues['PV']
+        license = foundvalues.pop('LICENSE', None)
+        if license:
+            liccomment = '# NOTE: spec file indicates the license may be "%s"' % license
+            for i, line in enumerate(lines_before):
+                if line.startswith('LICENSE ='):
+                    lines_before.insert(i, liccomment)
+                    break
+            else:
+                lines_before.append(liccomment)
+        extravalues.update(foundvalues)
 
 def register_recipe_handlers(handlers):
     # Set priorities with some gaps so that other plugins can insert
scripts/lib/recipetool/create_npm.py +2 -2
index cc4fb42..ffbcb49 100644
@@ -104,9 +104,9 @@ class NpmRecipeHandler(RecipeHandler):
                 classes.append('npm')
                 handled.append('buildsystem')
                 if 'description' in data:
-                    lines_before.append('SUMMARY = "%s"' % data['description'])
+                    extravalues['SUMMARY'] = data['description']
                 if 'homepage' in data:
-                    lines_before.append('HOMEPAGE = "%s"' % data['homepage'])
+                    extravalues['HOMEPAGE'] = data['homepage']
 
                 # Shrinkwrap
                 localfilesdir = tempfile.mkdtemp(prefix='recipetool-npm')
Signature · 1 line
2.5.5
#5 lore raw

PATCH 5/7 recipetool: create: extract variable values from .deb/.ipk/.rpm

In reply to Paul Eggleton

Extract the metadata from package files and use it to set variable values in the recipe (including recipe name and version, LICENSE, SUMMARY, DESCRIPTION, SECTION and HOMEPAGE). For LICENSE we take care not to step on any value determined by our license file scan; if there is one we simply add a comment above the LICENSE setting so the user can resolve it.

---
 scripts/lib/recipetool/create.py | 105 +++++++++++++++++++++++++++++++++++----
 1 file changed, 94 insertions(+), 11 deletions(-)
scripts/lib/recipetool/create.py +94 -11
index 66c881a..ad26189 100644
@@ -261,7 +261,11 @@ def determine_from_filename(srcfile):
         namepart = srcfile.split('.tar.')[0].lower()
     else:
         namepart = os.path.splitext(srcfile)[0].lower()
-    splitval = namepart.rsplit('_', 1)
+    if is_package(srcfile):
+        # Force getting the value from the package metadata
+        return None, None
+    else:
+        splitval = namepart.rsplit('_', 1)
     if len(splitval) == 1:
         splitval = namepart.rsplit('-', 1)
     pn = splitval[0].replace('_', '-')
@@ -327,6 +331,13 @@ def reformat_git_uri(uri):
             return 'git://%s;protocol=%s%s' % (res.group(2), res.group(1), res.group(4) or '')
     return uri
 
+def is_package(url):
+    '''Check if a URL points to a package'''
+    checkurl = url.split(';', 1)[0]
+    if checkurl.endswith(('.deb', '.ipk', '.rpm', '.srpm')):
+        return True
+    return False
+
 def create_recipe(args):
     import bb.process
     import tempfile
@@ -337,6 +348,7 @@ def create_recipe(args):
     if args.machine:
         pkgarch = "${MACHINE_ARCH}"
 
+    extravalues = {}
     checksums = (None, None)
     tempsrc = ''
     srcsubdir = ''
@@ -382,6 +394,33 @@ def create_recipe(args):
                     if '<html' in f.read(100).lower():
                         logger.error('Fetching "%s" returned a single HTML page - check the URL is correct and functional' % fetchuri)
                         sys.exit(1)
+
+        if is_package(fetchuri):
+            tmpfdir = tempfile.mkdtemp(prefix='recipetool-')
+            try:
+                pkgfile = None
+                try:
+                    fileuri = fetchuri + ';unpack=0'
+                    scriptutils.fetch_uri(tinfoil.config_data, fileuri, tmpfdir, srcrev)
+                    for root, _, files in os.walk(tmpfdir):
+                        for f in files:
+                            pkgfile = os.path.join(root, f)
+                            break
+                except bb.fetch2.BBFetchException as e:
+                    logger.warn('Second fetch to get metadata failed: %s' % str(e).rstrip())
+
+                if pkgfile:
+                    if pkgfile.endswith(('.deb', '.ipk')):
+                        stdout, _ = bb.process.run('ar x %s control.tar.gz' % pkgfile, cwd=tmpfdir)
+                        stdout, _ = bb.process.run('tar xf control.tar.gz ./control', cwd=tmpfdir)
+                        values = convert_debian(tmpfdir)
+                        extravalues.update(values)
+                    elif pkgfile.endswith(('.rpm', '.srpm')):
+                        stdout, _ = bb.process.run('rpm -qp --xml %s > pkginfo.xml' % pkgfile, cwd=tmpfdir)
+                        values = convert_rpm_xml(os.path.join(tmpfdir, 'pkginfo.xml'))
+                        extravalues.update(values)
+            finally:
+                shutil.rmtree(tmpfdir)
     else:
         # Assume we're pointing to an existing source tree
         if args.extract_to:
@@ -458,6 +497,13 @@ def create_recipe(args):
         lines_before.append('# will not be in most cases) you must specify the correct value before using this')
         lines_before.append('# recipe for anything other than initial testing/development!')
         licenses = ['CLOSED']
+    pkg_license = extravalues.pop('LICENSE', None)
+    if pkg_license:
+        if licenses == ['Unknown']:
+            lines_before.append('# NOTE: The following LICENSE value was determined from the original package metadata')
+            licenses = [pkg_license]
+        else:
+            lines_before.append('# NOTE: Original package metadata indicates license is: %s' % pkg_license)
     lines_before.append('LICENSE = "%s"' % ' '.join(licenses))
     lines_before.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n                    '.join(lic_files_chksum))
     lines_before.append('')
@@ -558,7 +604,6 @@ def create_recipe(args):
         classes.append('bin_package')
         handled.append('buildsystem')
 
-    extravalues = {}
     for handler in handlers:
         handler.process(srctree_use, classes, lines_before, lines_after, handled, extravalues)
 
@@ -665,6 +710,9 @@ def create_recipe(args):
     outlines.extend(lines_after)
 
     if extravalues:
+        if 'LICENSE' in extravalues and not licvalues:
+            # Don't blow away 'CLOSED' value that comments say we set
+            del extravalues['LICENSE']
         _, outlines = oe.recipeutils.patch_recipe_lines(outlines, extravalues, trailing_newline=False)
 
     if args.extract_to:
@@ -913,6 +961,12 @@ def convert_pkginfo(pkginfofile):
     return values
 
 def convert_debian(debpath):
+    value_map = {'Package': 'PN',
+                 'Version': 'PV',
+                 'Section': 'SECTION',
+                 'License': 'LICENSE',
+                 'Homepage': 'HOMEPAGE'}
+
     # FIXME extend this mapping - perhaps use distro_alias.inc?
     depmap = {'libz-dev': 'zlib'}
 
@@ -922,34 +976,63 @@ def convert_debian(debpath):
         indesc = False
         for line in f:
             if indesc:
-                if line.strip():
+                if line.startswith(' '):
                     if line.startswith(' This package contains'):
                         indesc = False
                     else:
-                        values['DESCRIPTION'] += ' ' + line.strip()
+                        if 'DESCRIPTION' in values:
+                            values['DESCRIPTION'] += ' ' + line.strip()
+                        else:
+                            values['DESCRIPTION'] = line.strip()
                 else:
                     indesc = False
-            else:
+            if not indesc:
                 splitline = line.split(':', 1)
-                key = line[0]
-                value = line[1]
+                if len(splitline) < 2:
+                    continue
+                key = splitline[0]
+                value = splitline[1].strip()
                 if key == 'Build-Depends':
                     for dep in value.split(','):
                         dep = dep.split()[0]
                         mapped = depmap.get(dep, '')
                         if mapped:
                             depends.append(mapped)
-                elif key == 'Section':
-                    values['SECTION'] = value
                 elif key == 'Description':
                     values['SUMMARY'] = value
                     indesc = True
+                else:
+                    varname = value_map.get(key, None)
+                    if varname:
+                        values[varname] = value
 
-    if depends:
-        values['DEPENDS'] = ' '.join(depends)
+    #if depends:
+    #    values['DEPENDS'] = ' '.join(depends)
 
     return values
 
+def convert_rpm_xml(xmlfile):
+    '''Converts the output from rpm -qp --xml to a set of variable values'''
+    import xml.etree.ElementTree as ElementTree
+    rpmtag_map = {'Name': 'PN',
+                  'Version': 'PV',
+                  'Summary': 'SUMMARY',
+                  'Description': 'DESCRIPTION',
+                  'License': 'LICENSE',
+                  'Url': 'HOMEPAGE'}
+
+    values = {}
+    tree = ElementTree.parse(xmlfile)
+    root = tree.getroot()
+    for child in root:
+        if child.tag == 'rpmTag':
+            name = child.attrib.get('name', None)
+            if name:
+                varname = rpmtag_map.get(name, None)
+                if varname:
+                    values[varname] = child[0].text
+    return values
+
 
 def register_commands(subparsers):
     parser_create = subparsers.add_parser('create',
Signature · 1 line
2.5.5
#6 lore raw

PATCH 6/7 recipetool: create: use ${BP} for subdir for binary packages

In reply to Paul Eggleton

If we use ${BP} for the subdirectory, the default value of S will work rather than having to have an ugly value derived from the package file name in both places. This does mean that we have to assume the default though (we can't just let the normal logic work because the value of BP is the default until later on, so the replacement doesn't work).

---
 scripts/lib/recipetool/create.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)
scripts/lib/recipetool/create.py +4 -2
index ad26189..1899a0d 100644
@@ -359,7 +359,7 @@ def create_recipe(args):
         if args.binary:
             # Assume the archive contains the directory structure verbatim
             # so we need to extract to a subdirectory
-            fetchuri += ';subdir=%s' % os.path.splitext(os.path.basename(urlparse.urlsplit(fetchuri).path))[0]
+            fetchuri += ';subdir=${BP}'
         srcuri = fetchuri
         rev_re = re.compile(';rev=([^;]+)')
         res = rev_re.search(srcuri)
@@ -566,7 +566,9 @@ def create_recipe(args):
         lines_before.append('SRCREV = "%s"' % srcrev)
     lines_before.append('')
 
-    if srcsubdir:
+    if srcsubdir and not args.binary:
+        # (for binary packages we explicitly specify subdir= when fetching to
+        # match the default value of S, so we don't need to set it in that case)
         lines_before.append('S = "${WORKDIR}/%s"' % srcsubdir)
         lines_before.append('')
 
Signature · 1 line
2.5.5
#7 lore raw

PATCH 7/7 recipetool / devtool: set a fixed SRCREV by default when fetching from git

In reply to Paul Eggleton

If fetching source from a git repository, typically within OpenEmbedded we encourage setting SRCREV to a fixed revision, so change to do that by default and add a -a/--autorev option to use "${AUTOREV}" instead.

---
 meta/lib/oeqa/selftest/devtool.py | 6 ++++--
 scripts/lib/devtool/standard.py   | 3 +++
 scripts/lib/recipetool/create.py  | 5 +++++
 3 files changed, 12 insertions(+), 2 deletions(-)
meta/lib/oeqa/selftest/devtool.py +4 -2
index 132a73d..b64f9b3 100644
@@ -207,12 +207,14 @@ class DevtoolTests(DevtoolBase):
         tempdir = tempfile.mkdtemp(prefix='devtoolqa')
         self.track_for_cleanup(tempdir)
         pn = 'dbus-wait'
+        srcrev = '6cc6077a36fe2648a5f993fe7c16c9632f946517'
         # We choose an https:// git URL here to check rewriting the URL works
         url = 'https://git.yoctoproject.org/git/dbus-wait'
         # Force fetching to "noname" subdir so we verify we're picking up the name from autoconf
         # instead of the directory name
         result = runCmd('git clone %s noname' % url, cwd=tempdir)
         srcdir = os.path.join(tempdir, 'noname')
+        result = runCmd('git reset --hard %s' % srcrev, cwd=srcdir)
         self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure.ac')), 'Unable to find configure script in source directory')
         # Test devtool add
         self.track_for_cleanup(self.workspacedir)
@@ -235,7 +237,7 @@ class DevtoolTests(DevtoolBase):
         checkvars['S'] = '${WORKDIR}/git'
         checkvars['PV'] = '0.1+git${SRCPV}'
         checkvars['SRC_URI'] = 'git://git.yoctoproject.org/git/dbus-wait;protocol=https'
-        checkvars['SRCREV'] = '${AUTOREV}'
+        checkvars['SRCREV'] = srcrev
         checkvars['DEPENDS'] = set(['dbus'])
         self._test_recipe_contents(recipefile, checkvars, [])
 
@@ -345,7 +347,7 @@ class DevtoolTests(DevtoolBase):
         self.track_for_cleanup(self.workspacedir)
         self.add_command_to_tearDown('bitbake -c cleansstate %s' % testrecipe)
         self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
-        result = runCmd('devtool add %s %s -f %s' % (testrecipe, srcdir, url))
+        result = runCmd('devtool add %s %s -a -f %s' % (testrecipe, srcdir, url))
         self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created: %s' % result.output)
         self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure.ac')), 'Unable to find configure.ac in source directory')
         # Test devtool status
scripts/lib/devtool/standard.py +3 -0
index 77a82d5..3be3214 100644
@@ -144,6 +144,8 @@ def add(args, config, basepath, workspace):
         extracmdopts += ' --also-native'
     if args.src_subdir:
         extracmdopts += ' --src-subdir "%s"' % args.src_subdir
+    if args.autorev:
+        extracmdopts += ' -a'
 
     tempdir = tempfile.mkdtemp(prefix='devtool')
     try:
@@ -1390,6 +1392,7 @@ def register_commands(subparsers, context):
     parser_add.add_argument('--fetch', '-f', help='Fetch the specified URI and extract it to create the source tree (deprecated - pass as positional argument instead)', metavar='URI')
     parser_add.add_argument('--version', '-V', help='Version to use within recipe (PV)')
     parser_add.add_argument('--no-git', '-g', help='If fetching source, do not set up source tree as a git repository', action="store_true")
+    parser_add.add_argument('--autorev', '-a', help='When fetching from a git repository, set SRCREV in the recipe to a floating revision instead of fixed', action="store_true")
     parser_add.add_argument('--binary', '-b', help='Treat the source tree as something that should be installed verbatim (no compilation, same directory structure). Useful with binary packages e.g. RPMs.', action='store_true')
     parser_add.add_argument('--also-native', help='Also add native variant (i.e. support building recipe for the build host as well as the target machine)', action='store_true')
     parser_add.add_argument('--src-subdir', help='Specify subdirectory within source tree to use', metavar='SUBDIR')
scripts/lib/recipetool/create.py +5 -0
index 1899a0d..4a59363 100644
@@ -563,6 +563,10 @@ def create_recipe(args):
         lines_before.append('')
         lines_before.append('# Modify these as desired')
         lines_before.append('PV = "%s+git${SRCPV}"' % (realpv or '1.0'))
+        if not args.autorev and srcrev == '${AUTOREV}':
+            if os.path.exists(os.path.join(srctree, '.git')):
+                (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree)
+            srcrev = stdout.rstrip()
         lines_before.append('SRCREV = "%s"' % srcrev)
     lines_before.append('')
 
@@ -1049,5 +1053,6 @@ def register_commands(subparsers):
     parser_create.add_argument('-b', '--binary', help='Treat the source tree as something that should be installed verbatim (no compilation, same directory structure)', action='store_true')
     parser_create.add_argument('--also-native', help='Also add native variant (i.e. support building recipe for the build host as well as the target machine)', action='store_true')
     parser_create.add_argument('--src-subdir', help='Specify subdirectory within source tree to use', metavar='SUBDIR')
+    parser_create.add_argument('-a', '--autorev', help='When fetching from a git repository, set SRCREV in the recipe to a floating revision instead of fixed', action="store_true")
     parser_create.set_defaults(func=create_recipe)
 
Signature · 1 line
2.5.5