A bunch of fixes for devtool update-recipe

index 4523048..1511641 100644
@@ -332,10 +332,11 @@ def _git_ls_tree(repodir, treeish='HEAD', recursive=False):
         cmd.append('-r')
     out, _ = bb.process.run(cmd, cwd=repodir)
     ret = {}
-    for line in out.split('\0'):
-        if line:
-            split = line.split(None, 4)
-            ret[split[3]] = split[0:3]
+    if out:
+        for line in out.split('\0'):
+            if line:
+                split = line.split(None, 4)
+                ret[split[3]] = split[0:3]
     return ret
 
 def _git_exclude_path(srctree, path):

new file mode 100644
index 0000000..28ff49e
@@ -0,0 +1,6 @@
+LICENSE = "CLOSED"
+INHIBIT_DEFAULT_DEPS = "1"
+
+SRC_URI = "file://file1 \
+           file://file2"
+

new file mode 100644
index 0000000..f4bdcfc
@@ -0,0 +1 @@
+The first file

new file mode 100644
index 0000000..a7e2414
@@ -0,0 +1 @@
+The second file

index 46f5a0b..f6226c1 100644
@@ -930,6 +930,24 @@ class DevtoolTests(DevtoolBase):
                            ('??', '.*/0001-Add-new-file.patch$')]
         self._check_repo_status(os.path.dirname(recipefile), expected_status)
 
+    def test_devtool_update_recipe_local_files_3(self):
+        # First, modify the recipe
+        testrecipe = 'devtool-test-localonly'
+        recipefile = get_bb_var('FILE', testrecipe)
+        src_uri = get_bb_var('SRC_URI', testrecipe)
+        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
+        self.track_for_cleanup(tempdir)
+        self.track_for_cleanup(self.workspacedir)
+        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
+        # (don't bother with cleaning the recipe on teardown, we won't be building it)
+        result = runCmd('devtool modify %s' % testrecipe)
+        # Modify one file
+        runCmd('echo "Another line" >> file2', cwd=os.path.join(self.workspacedir, 'sources', testrecipe, 'oe-local-files'))
+        self.add_command_to_tearDown('cd %s; rm %s/*; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, testrecipe, os.path.basename(recipefile)))
+        result = runCmd('devtool update-recipe %s' % testrecipe)
+        expected_status = [(' M', '.*/%s/file2$' % testrecipe)]
+        self._check_repo_status(os.path.dirname(recipefile), expected_status)
+
     @testcase(1163)
     def test_devtool_extract(self):
         tempdir = tempfile.mkdtemp(prefix='devtoolqa')

index 0332f10..dbefd28 100644
@@ -428,6 +428,7 @@ class GitApplyTree(PatchTree):
     def extractPatches(tree, startcommit, outdir, paths=None):
         import tempfile
         import shutil
+        import re
         tempdir = tempfile.mkdtemp(prefix='oepatch')
         try:
             shellcmd = ["git", "format-patch", startcommit, "-o", tempdir]

@@ -443,10 +444,13 @@ class GitApplyTree(PatchTree):
                         try:
                             with open(srcfile, 'r', encoding=encoding) as f:
                                 for line in f:
-                                    if line.startswith(GitApplyTree.patch_line_prefix):
+                                    checkline = line
+                                    if checkline.startswith('Subject: '):
+                                        checkline = re.sub(r'\[.+?\]\s*', '', checkline[9:])
+                                    if checkline.startswith(GitApplyTree.patch_line_prefix):
                                         outfile = line.split()[-1].strip()
                                         continue
-                                    if line.startswith(GitApplyTree.ignore_commit_prefix):
+                                    if checkline.startswith(GitApplyTree.ignore_commit_prefix):
                                         continue
                                     patchlines.append(line)
                         except UnicodeDecodeError:

new file mode 100644
index 0000000..719a5f1
@@ -0,0 +1,16 @@
+LICENSE = "GPLv2+"
+LIC_FILES_CHKSUM = "file://COPYING;md5=94d55d512a9ba36caa9b7df079bae19f"
+
+DEPENDS = "libxres libxext virtual/libx11 ncurses"
+
+SRC_URI = "http://downloads.yoctoproject.org/releases/xrestop/xrestop-0.4.tar.gz \
+           file://readme.patch.gz \
+           "
+
+S = "${WORKDIR}/xrestop-0.4"
+
+SRC_URI[md5sum] = "d8a54596cbaf037e62b80c4585a3ca9b"
+SRC_URI[sha256sum] = "67c2fc94a7ecedbaae0d1837e82e93d1d98f4a6d759828860e552119af3ce257"
+
+inherit autotools pkgconfig
+

new file mode 100644
index 0000000000000000000000000000000000000000..4752492ccd661e2c360d8fe2ca4dcef61dea89ce
GIT binary patch

index 1f6927b..2c1f58c 100644
@@ -10,13 +10,13 @@ PATCH_GIT_USER_EMAIL ?= "oe.patch@oe"
 
 inherit terminal
 
-def src_patches(d, all = False ):
+def src_patches(d, all=False, expand=True):
     workdir = d.getVar('WORKDIR', True)
     fetch = bb.fetch2.Fetch([], d)
     patches = []
     sources = []
     for url in fetch.urls:
-        local = patch_path(url, fetch, workdir)
+        local = patch_path(url, fetch, workdir, expand)
         if not local:
             if all:
                 local = fetch.localpath(url)

@@ -55,13 +55,14 @@ def src_patches(d, all = False ):
 
     return patches
 
-def patch_path(url, fetch, workdir):
+def patch_path(url, fetch, workdir, expand=True):
     """Return the local path of a patch, or None if this isn't a patch"""
 
     local = fetch.localpath(url)
     base, ext = os.path.splitext(os.path.basename(local))
     if ext in ('.gz', '.bz2', '.Z'):
-        local = os.path.join(workdir, base)
+        if expand:
+            local = os.path.join(workdir, base)
         ext = os.path.splitext(base)[1]
 
     urldata = fetch.ud[url]

index 6caae5f..ab4177a 100644
@@ -397,7 +397,7 @@ def get_recipe_local_files(d, patches=False):
     for uri in uris:
         if fetch.ud[uri].type == 'file':
             if (not patches and
-                    bb.utils.exec_flat_python_func('patch_path', uri, fetch, '')):
+                    bb.utils.exec_flat_python_func('patch_path', uri, fetch, '', expand=False)):
                 continue
             # Skip files that are referenced by absolute path
             fname = fetch.ud[uri].basepath

@@ -418,7 +418,7 @@ def get_recipe_patches(d):
     patchfiles = []
     # Execute src_patches() defined in patch.bbclass - this works since that class
     # is inherited globally
-    patches = bb.utils.exec_flat_python_func('src_patches', d)
+    patches = bb.utils.exec_flat_python_func('src_patches', d, expand=False)
     for patch in patches:
         _, _, local, _, _, parm = bb.fetch.decodeurl(patch)
         patchfiles.append(local)

@@ -437,7 +437,7 @@ def get_recipe_patched_files(d):
     import oe.patch
     # Execute src_patches() defined in patch.bbclass - this works since that class
     # is inherited globally
-    patches = bb.utils.exec_flat_python_func('src_patches', d)
+    patches = bb.utils.exec_flat_python_func('src_patches', d, expand=False)
     patchedfiles = {}
     for patch in patches:
         _, _, patchfile, _, _, parm = bb.fetch.decodeurl(patch)

index f6226c1..92dc5e5 100644
@@ -948,6 +948,30 @@ class DevtoolTests(DevtoolBase):
         expected_status = [(' M', '.*/%s/file2$' % testrecipe)]
         self._check_repo_status(os.path.dirname(recipefile), expected_status)
 
+    def test_devtool_update_recipe_local_patch_gz(self):
+        # First, modify the recipe
+        testrecipe = 'devtool-test-patch-gz'
+        recipefile = get_bb_var('FILE', testrecipe)
+        src_uri = get_bb_var('SRC_URI', testrecipe)
+        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
+        self.track_for_cleanup(tempdir)
+        self.track_for_cleanup(self.workspacedir)
+        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
+        # (don't bother with cleaning the recipe on teardown, we won't be building it)
+        result = runCmd('devtool modify %s' % testrecipe)
+        # Modify one file
+        srctree = os.path.join(self.workspacedir, 'sources', testrecipe)
+        runCmd('echo "Another line" >> README', cwd=srctree)
+        runCmd('git commit -a --amend --no-edit', cwd=srctree)
+        self.add_command_to_tearDown('cd %s; rm %s/*; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, testrecipe, os.path.basename(recipefile)))
+        result = runCmd('devtool update-recipe %s' % testrecipe)
+        expected_status = [(' M', '.*/%s/readme.patch.gz$' % testrecipe)]
+        self._check_repo_status(os.path.dirname(recipefile), expected_status)
+        patch_gz = os.path.join(os.path.dirname(recipefile), testrecipe, 'readme.patch.gz')
+        result = runCmd('file %s' % patch_gz)
+        if 'gzip compressed data' not in result.output:
+            self.fail('New patch file is not gzipped - file reports:\n%s' % result.output)
+
     @testcase(1163)
     def test_devtool_extract(self):
         tempdir = tempfile.mkdtemp(prefix='devtoolqa')

82 lines not on this page · raw message

index af0d467..34de7bd 100644
@@ -1104,6 +1104,15 @@ def _remove_file_entries(srcuri, filelist):
                 break
     return entries, remaining
 
+def _replace_srcuri_entry(srcuri, filename, newentry):
+    """Replace entry corresponding to specified file with a new entry"""
+    basename = os.path.basename(filename)
+    for i in range(len(srcuri)):
+        if os.path.basename(srcuri[i].split(';')[0]) == basename:
+            srcuri.pop(i)
+            srcuri.insert(i, newentry)
+            break
+
 def _remove_source_files(append, files, destpath):
     """Unlink existing patch files"""
     for path in files:

@@ -1424,6 +1433,10 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
         raise DevtoolError('Unable to find initial revision - please specify '
                            'it with --initial-rev')
 
+    dl_dir = rd.getVar('DL_DIR', True)
+    if not dl_dir.endswith('/'):
+        dl_dir += '/'
+
     tempdir = tempfile.mkdtemp(prefix='devtool')
     try:
         local_files_dir = tempfile.mkdtemp(dir=tempdir)

@@ -1468,6 +1481,7 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
                 logger.info('No patches or local source files needed updating')
         else:
             # Update existing files
+            files_dir = _determine_files_dir(rd)
             for basepath, path in upd_f.items():
                 logger.info('Updating file %s' % basepath)
                 if os.path.isabs(basepath):

@@ -1479,11 +1493,19 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
                 updatefiles = True
             for basepath, path in upd_p.items():
                 patchfn = os.path.join(patches_dir, basepath)
-                logger.info('Updating patch %s' % basepath)
+                if os.path.dirname(path) + '/' == dl_dir:
+                    # This is a a downloaded patch file - we now need to
+                    # replace the entry in SRC_URI with our local version
+                    logger.info('Replacing remote patch %s with updated local version' % basepath)
+                    path = os.path.join(files_dir, basepath)
+                    _replace_srcuri_entry(srcuri, basepath, 'file://%s' % basepath)
+                    updaterecipe = True
+                else:
+                    logger.info('Updating patch %s' % basepath)
+                logger.debug('Moving new patch %s to %s' % (patchfn, path))
                 _move_file(patchfn, path)
                 updatefiles = True
             # Add any new files
-            files_dir = _determine_files_dir(rd)
             for basepath, path in new_f.items():
                 logger.info('Adding new file %s' % basepath)
                 _move_file(os.path.join(local_files_dir, basepath),

index ab4177a..1589feb 100644
@@ -389,10 +389,15 @@ def copy_recipe_files(d, tgt_dir, whole_dir=False, download=True):
     return copied, remotes
 
 
-def get_recipe_local_files(d, patches=False):
+def get_recipe_local_files(d, patches=False, archives=False):
     """Get a list of local files in SRC_URI within a recipe."""
     uris = (d.getVar('SRC_URI', True) or "").split()
     fetch = bb.fetch2.Fetch(uris, d)
+    # FIXME this list should be factored out somewhere else (such as the
+    # fetcher) though note that this only encompasses actual container formats
+    # i.e. that can contain multiple files as opposed to those that only
+    # contain a compressed stream (i.e. .tar.gz as opposed to just .gz)
+    archive_exts = ['.tar', '.tgz', '.tar.gz', '.tar.Z', '.tbz', '.tbz2', '.tar.bz2', '.tar.xz', '.tar.lz', '.zip', '.jar', '.rpm', '.srpm', '.deb', '.ipk', '.tar.7z', '.7z']
     ret = {}
     for uri in uris:
         if fetch.ud[uri].type == 'file':

@@ -409,7 +414,14 @@ def get_recipe_local_files(d, patches=False):
                 if os.path.isabs(subdir):
                     continue
                 fname = os.path.join(subdir, fname)
-            ret[fname] = fetch.localpath(uri)
+            localpath = fetch.localpath(uri)
+            if not archives:
+                # Ignore archives that will be unpacked
+                if localpath.endswith(tuple(archive_exts)):
+                    unpack = fetch.ud[uri].parm.get('unpack', True)
+                    if unpack:
+                        continue
+            ret[fname] = localpath
     return ret
 
 

new file mode 100644
index 0000000..1a163f8
@@ -0,0 +1,7 @@
+LICENSE = "CLOSED"
+INHIBIT_DEFAULT_DEPS = "1"
+
+SRC_URI = "file://devtool-test-subdir.tar.gz \
+           file://testfile;subdir=${BPN}"
+
+S = "${WORKDIR}/${BPN}"

new file mode 100644
index 0000000000000000000000000000000000000000..3d44f803cb996f47ddb9566a6a2c278bd2fd38a7
GIT binary patch

new file mode 100644
index 0000000..12b519c
@@ -0,0 +1 @@
+Modified version

index 92dc5e5..2a08721 100644
@@ -972,6 +972,31 @@ class DevtoolTests(DevtoolBase):
         if 'gzip compressed data' not in result.output:
             self.fail('New patch file is not gzipped - file reports:\n%s' % result.output)
 
+    def test_devtool_update_recipe_local_files_subdir(self):
+        # Try devtool extract on a recipe that has a file with subdir= set in
+        # SRC_URI such that it overwrites a file that was in an archive that
+        # was also in SRC_URI
+        # First, modify the recipe
+        testrecipe = 'devtool-test-subdir'
+        recipefile = get_bb_var('FILE', testrecipe)
+        src_uri = get_bb_var('SRC_URI', testrecipe)
+        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
+        self.track_for_cleanup(tempdir)
+        self.track_for_cleanup(self.workspacedir)
+        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
+        # (don't bother with cleaning the recipe on teardown, we won't be building it)
+        result = runCmd('devtool modify %s' % testrecipe)
+        testfile = os.path.join(self.workspacedir, 'sources', testrecipe, 'testfile')
+        self.assertTrue(os.path.exists(testfile), 'Extracted source could not be found')
+        with open(testfile, 'r') as f:
+            contents = f.read().rstrip()
+        self.assertEqual(contents, 'Modified version', 'File has apparently not been overwritten as it should have been')
+        # Test devtool update-recipe without modifying any files
+        self.add_command_to_tearDown('cd %s; rm %s/*; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, testrecipe, os.path.basename(recipefile)))
+        result = runCmd('devtool update-recipe %s' % testrecipe)
+        expected_status = []
+        self._check_repo_status(os.path.dirname(recipefile), expected_status)
+
     @testcase(1163)
     def test_devtool_extract(self):
         tempdir = tempfile.mkdtemp(prefix='devtoolqa')