Update new fetcher with unpack parm
cover Update new fetcher with unpack parm
Introduce the unpack param to the new fetcher2. This allows packages to disable the unpack of given files, even if they would normally be unpacked automatically.
The existing "unpack" param for SRPM handling was changed to "extract".
I did change the cp to a tar action. This would ensure that any files that are hardlinks are preserved once copied. I'm not sure this is really such a big deal in the end.. (note, the hard links are local to the set of files.. so the set in ${WORKDIR} are not linked to the set in the recipe.
Pull URL: git://git.pokylinux.org/poky-contrib.git
Branch: mhatle/fetcher
Browse: http://git.pokylinux.org/cgit.cgi/poky-contrib/log/?h=mhatle/fetcher
Thanks,
Mark Hatle <[email protected]>
---
Mark Hatle (1):
fetch2: unpack revision
bitbake/lib/bb/fetch2/__init__.py | 92 ++++++++++++++++++------------- bitbake/lib/bb/utils.py | 12 ++++ meta/recipes-devtools/rpm/rpm_5.4.0.bb | 2 +- 3 files changed, 66 insertions(+), 40 deletions(-)
Signature · 1 line
1.7.3.4
PATCH 1/1 fetch2: unpack revision
Revise the unpack function to have a way to disable the unpack. This is based on the work from "Andreas Oberritter <[email protected]>", see
http://cgit.openembedded.net/cgit.cgi/openembedded/commit/?id=2bdfe8519eda8067845019a699acdf19a21ba380
In addition, the to_boolean function comes from the work of "Chris Larson <[email protected]>", see
http://cgit.openembedded.net/cgit.cgi/openembedded/commit/?id=900cc29b603691eb3a077cb660545ead3715ed54
--- bitbake/lib/bb/fetch2/__init__.py | 92 ++++++++++++++++++------------- bitbake/lib/bb/utils.py | 12 ++++ meta/recipes-devtools/rpm/rpm_5.4.0.bb | 2 +- 3 files changed, 66 insertions(+), 40 deletions(-)
bitbake/lib/bb/fetch2/__init__.py +53 -39
@@ -637,6 +637,13 @@ class FetchMethod(object): import subprocess iterate = False file = urldata.localpath + + try: + unpack = bb.utils.to_boolean(urldata.parm.get('unpack'), True) + except ValueError, exc: + bb.fatal("Invalid value for 'unpack' parameter for %s: %s" % + (file, urldata.parm.get('unpack'))) + dots = file.split(".") if dots[-1] in ['gz', 'bz2', 'Z']: efile = os.path.join(bb.data.getVar('WORKDIR', data, True),os.path.basename('.'.join(dots[0:-1])))
@@ -644,34 +651,41 @@ class FetchMethod(object): efile = file cmd = None - if file.endswith('.tar'): - cmd = 'tar x --no-same-owner -f %s' % file - elif file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'): - cmd = 'tar xz --no-same-owner -f %s' % file - elif file.endswith('.tbz') or file.endswith('.tbz2') or file.endswith('.tar.bz2'): - cmd = 'bzip2 -dc %s | tar x --no-same-owner -f -' % file - elif file.endswith('.gz') or file.endswith('.Z') or file.endswith('.z'): - cmd = 'gzip -dc %s > %s' % (file, efile) - elif file.endswith('.bz2'): - cmd = 'bzip2 -dc %s > %s' % (file, efile) - elif file.endswith('.tar.xz'): - cmd = 'xz -dc %s | tar x --no-same-owner -f -' % file - elif file.endswith('.xz'): - cmd = 'xz -dc %s > %s' % (file, efile) - elif file.endswith('.zip') or file.endswith('.jar'): - cmd = 'unzip -q -o' - if 'dos' in urldata.parm: - cmd = '%s -a' % cmd - cmd = "%s '%s'" % (cmd, file) - elif file.endswith('.src.rpm') or file.endswith('.srpm'): - if 'unpack' in urldata.parm: - unpack_file = ("%s" % urldata.parm['unpack']) - cmd = 'rpm2cpio.sh %s | cpio -i %s' % (file, unpack_file) - iterate = True - iterate_file = unpack_file - else: - cmd = 'rpm2cpio.sh %s | cpio -i' % (file) - else: + if unpack: + if file.endswith('.tar'): + cmd = 'tar x --no-same-owner -f %s' % file + elif file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'): + cmd = 'tar xz --no-same-owner -f %s' % file + elif file.endswith('.tbz') or file.endswith('.tbz2') or file.endswith('.tar.bz2'): + cmd = 'bzip2 -dc %s | tar x --no-same-owner -f -' % file + elif file.endswith('.gz') or file.endswith('.Z') or file.endswith('.z'): + cmd = 'gzip -dc %s > %s' % (file, efile) + elif file.endswith('.bz2'): + cmd = 'bzip2 -dc %s > %s' % (file, efile) + elif file.endswith('.tar.xz'): + cmd = 'xz -dc %s | tar x --no-same-owner -f -' % file + elif file.endswith('.xz'): + cmd = 'xz -dc %s > %s' % (file, efile) + elif file.endswith('.zip') or file.endswith('.jar'): + try: + dos = bb.utils.to_boolean(urldata.parm.get('dos'), False) + except ValueError, exc: + bb.fatal("Invalid value for 'dos' parameter for %s: %s" % + (file, urldata.parm.get('dos'))) + cmd = 'unzip -q -o' + if dos: + cmd = '%s -a' % cmd + cmd = "%s '%s'" % (cmd, file) + elif file.endswith('.src.rpm') or file.endswith('.srpm'): + if 'extract' in urldata.parm: + unpack_file = urldata.parm.get('extract') + cmd = 'rpm2cpio.sh %s | cpio -i %s' % (file, unpack_file) + iterate = True + iterate_file = unpack_file + else: + cmd = 'rpm2cpio.sh %s | cpio -i' % (file) + + if not unpack or not cmd: # If file == dest, then avoid any copies, as we already put the file into dest! dest = os.path.join(rootdir, os.path.basename(file)) if (file != dest) and not (os.path.exists(dest) and os.path.samefile(file, dest)):
@@ -685,17 +699,17 @@ class FetchMethod(object): destdir = "." elif not os.access("%s/%s" % (rootdir, destdir), os.F_OK): os.makedirs("%s/%s" % (rootdir, destdir)) - cmd = 'cp -pPR %s %s/%s/' % (file, rootdir, destdir) + #cmd = 'cp -pPR %s %s/%s/' % (file, rootdir, destdir) + cmd = 'tar -cf - -C "%d" -ps . | tar -xf - -C "%s/%s/"' % (file, rootdir, destdir) else: - if not 'patch' in urldata.parm: - # The "destdir" handling was specifically done for FILESPATH - # items. So, only do so for file:// entries. - if urldata.type == "file" and urldata.path.find("/") != -1: - destdir = urldata.path.rsplit("/", 1)[0] - else: - destdir = "." - bb.mkdirhier("%s/%s" % (rootdir, destdir)) - cmd = 'cp %s %s/%s/' % (file, rootdir, destdir) + # The "destdir" handling was specifically done for FILESPATH + # items. So, only do so for file:// entries. + if urldata.type == "file" and urldata.path.find("/") != -1: + destdir = urldata.path.rsplit("/", 1)[0] + else: + destdir = "." + bb.mkdirhier("%s/%s" % (rootdir, destdir)) + cmd = 'cp %s %s/%s/' % (file, rootdir, destdir) if not cmd: return
@@ -704,7 +718,7 @@ class FetchMethod(object): save_cwd = os.getcwd(); os.chdir(rootdir) if 'subdir' in urldata.parm: - newdir = ("%s/%s" % (rootdir, urldata.parm['subdir'])) + newdir = ("%s/%s" % (rootdir, urldata.parm.get('subdir'))) bb.mkdirhier(newdir) os.chdir(newdir)
bitbake/lib/bb/utils.py +12 -0
@@ -831,3 +831,15 @@ def init_logger(logger, verbose, debug, debug_domains): if debug_domains: bb.msg.set_debug_domains(debug_domains) + +def to_boolean(string, default=None): + if not string: + return default + + normalized = string.lower() + if normalized in ("y", "yes", "1", "true"): + return True + elif normalized in ("n", "no", "0", "false"): + return False + else: + raise ValueError("Invalid value for to_boolean: %s" % string)
meta/recipes-devtools/rpm/rpm_5.4.0.bb +1 -1
@@ -47,7 +47,7 @@ PR = "r11" # rpm2cpio is a shell script, which is part of the rpm src.rpm. It is needed # in order to extract the distribution SRPM into a format we can extract... -SRC_URI = "http://www.rpm5.org/files/rpm/rpm-5.4/rpm-5.4.0-0.20101229.src.rpm;unpack=rpm-5.4.0.tar.gz \ +SRC_URI = "http://www.rpm5.org/files/rpm/rpm-5.4/rpm-5.4.0-0.20101229.src.rpm;extract=rpm-5.4.0.tar.gz \ file://perfile_rpmdeps.sh \ file://rpm-autogen.patch \ file://rpm-libsql-fix.patch \
Signature · 1 line
1.7.3.4
Richard Purdie
AppliedQuoted 12 lines · Mark Hatle
Introduce the unpack param to the new fetcher2. This allows packages to disable the unpack of given files, even if they would normally be unpacked automatically.
The existing "unpack" param for SRPM handling was changed to "extract".
I did change the cp to a tar action. This would ensure that any files that are hardlinks are preserved once copied. I'm not sure this is really such a big deal in the end.. (note, the hard links are local to the set of files.. so the set in ${WORKDIR} are not linked to the set in the recipe.
Quoted 6 lines
Pull URL:
git://git.pokylinux.org/poky-contrib.git
Branch: mhatle/fetcher
Browse: http://git.pokylinux.org/cgit.cgi/poky-contrib/log/?h=mhatle/fetcherThanks,
Mark Hatle <[email protected]>
Merged into master, thanks.
Richard