Skip to contentSkip to conversation

do_unpack, do_patch: shift some responsibility around, clean things up

PATCH In review Chris Larson 3 participants 4 messages Sep 1, 2010 – Sep 2, 2010

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 do_unpack, do_patch: shift some responsibility around, clean things up

From: Chris Larson <[email protected]>

- Consolidate 'is this file a patch' logic
- Move unpack functions from classes into oe.patch
- Move the unpacking message printing into do_unpack
- Move the destination directory determination into do_unpack
- Use subprocess's ability to pass in PATH and cwd rather than mangling the cmd
- Use shutil.copy2/copytree for ordinary file "unpack"
- Use the existing urldata from bb.fetch.init rather than re-decodeurl'ing the urls
- Make handling of globs in url paths explicit rather than implicit, calling oe_unpack on each one, so showing an unpacking message to the user for each globbed file, rather than the entirety

---
 classes/base.bbclass  |  161 +++++++++++++++++--------------------------------
 classes/patch.bbclass |   35 ++++-------
 lib/oe/unpack.py      |   95 +++++++++++++++++++++++++++++
 3 files changed, 165 insertions(+), 126 deletions(-)
 create mode 100644 lib/oe/unpack.py
classes/base.bbclass +55 -104
index 438327c..299e875 100644
@@ -163,117 +163,68 @@ python base_do_fetch() {
 			raise bb.build.FuncFailed("Checksum of '%s' failed" % uri)
 }
 
-def oe_unpack_file(file, data, url = None):
-	import subprocess
-	if not url:
-		url = "file://%s" % file
-	dots = file.split(".")
-	if dots[-1] in ['gz', 'bz2', 'Z']:
-		efile = os.path.join(bb.data.getVar('WORKDIR', data, 1),os.path.basename('.'.join(dots[0:-1])))
-	else:
-		efile = file
-	cmd = None
-	(type, host, path, user, pswd, parm) = bb.decodeurl(url)
-	if file.endswith('.tar'):
-		cmd = 'tar x --no-same-owner -f %s' % file
-	elif file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'):
-		cmd = 'tar xz --no-same-owner -f %s' % file
-	elif file.endswith('.tbz') or file.endswith('.tbz2') or file.endswith('.tar.bz2'):
-		cmd = 'bzip2 -dc %s | tar x --no-same-owner -f -' % file
-	elif file.endswith('.gz') or file.endswith('.Z') or file.endswith('.z'):
-		cmd = 'gzip -dc %s > %s' % (file, efile)
-	elif file.endswith('.bz2'):
-		cmd = 'bzip2 -dc %s > %s' % (file, efile)
-	elif file.endswith('.tar.xz'):
-		cmd = 'xz -dc %s | tar x --no-same-owner -f -' % file
-	elif file.endswith('.xz'):
-		cmd = 'xz -dc %s > %s' % (file, efile)
-	elif file.endswith('.zip') or file.endswith('.jar'):
-		cmd = 'unzip -q -o'
-		if 'dos' in parm:
-			cmd = '%s -a' % cmd
-		cmd = "%s '%s'" % (cmd, file)
-	elif (type == "file" and file.endswith('.patch') or file.endswith('.diff')) and parm.get('apply') != 'no':
-	# patch and diff files are special and need not be copied to workdir
-		cmd = ""
-	elif os.path.isdir(file):
-		destdir = "."
-		filespath = bb.data.getVar("FILESPATH", data, 1).split(":")
-		for fp in filespath:
-			if file[0:len(fp)] == fp:
-				destdir = file[len(fp):file.rfind('/')]
-				destdir = destdir.strip('/')
-				if len(destdir) < 1:
-					destdir = "."
-				elif not os.access("%s/%s" % (os.getcwd(), destdir), os.F_OK):
-					os.makedirs("%s/%s" % (os.getcwd(), destdir))
-				break
-
-		cmd = 'cp -pPR %s %s/%s/' % (file, os.getcwd(), destdir)
-	else:
-		if not 'patch' in parm and parm.get('apply') != 'yes':
-			# The "destdir" handling was specifically done for FILESPATH
-			# items.  So, only do so for file:// entries.
-			if type == "file":
-				if not host:
-					dest = os.path.dirname(path) or "."
-				else:
-				# this case is for backward compatiblity with older version
-				# of bitbake which do not have the fix
-				# http://cgit.openembedded.org/cgit.cgi/bitbake/commit/?id=ca257adc587bb0937ea76d8b32b654fdbf4192b8
-				# this should not be needed once all releases of bitbake has this fix
-				# applied/backported
-					dest = host + os.path.dirname(path) or "."
-			else:
-				dest = "."
-			bb.mkdirhier("%s" % oe.path.join(os.getcwd(),dest))
-			cmd = 'cp %s %s' % (file, oe.path.join(os.getcwd(), dest))
-	if not cmd:
-		return True
-	if not host:
-		dest = oe.path.join(os.getcwd(), path)
-	else:
-		dest = oe.path.join(os.getcwd(), oe.path.join(host, path))
-	if os.path.exists(dest):
-		if os.path.samefile(file, dest):
-			return True
-	# Change to subdir before executing command
-	save_cwd = os.getcwd();
-	if 'subdir' in parm:
-		newdir = ("%s/%s" % (os.getcwd(), parm['subdir']))
-		bb.mkdirhier(newdir)
-		os.chdir(newdir)
-
-	cmd = "PATH=\"%s\" %s" % (bb.data.getVar('PATH', data, 1), cmd)
-	bb.note("Unpacking %s to %s/" % (base_path_out(file, data), base_path_out(os.getcwd(), data)))
-	ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True)
-
-	os.chdir(save_cwd)
+def oe_unpack(d, local, urldata):
+    from oe.unpack import unpack_file, is_patch, UnpackError
+    if is_patch(local, urldata.parm):
+        return
 
-	return ret == 0
+    subdirs = []
+    if "subdir" in urldata.parm:
+        subdirs.append(urldata.parm["subdir"])
+
+    if urldata.type == "file":
+        if not urldata.host:
+            urlpath = urldata.path
+        else:
+            urlpath = oe.path.join(urldata.host, urldata.path)
+
+        if not os.path.isabs(urlpath):
+            subdirs.append(os.path.dirname(urlpath))
+
+    workdir = d.getVar("WORKDIR", True)
+    if subdirs:
+        destdir = oe.path.join(workdir, *subdirs)
+        bb.mkdirhier(destdir)
+    else:
+        destdir = workdir
+    dos = urldata.parm.get("dos")
+
+    bb.note("Unpacking %s to %s/" % (base_path_out(local, d),
+                                     base_path_out(destdir, d)))
+    try:
+        unpack_file(local, destdir, env={"PATH": d.getVar("PATH", True)}, dos=dos)
+    except UnpackError, exc:
+        bb.fatal(str(exc))
 
 addtask unpack after do_fetch
 do_unpack[dirs] = "${WORKDIR}"
 python base_do_unpack() {
-	import re
+    from glob import glob
+
+    srcurldata = bb.fetch.init(d.getVar("SRC_URI", True).split(), d, True)
+    filespath = d.getVar("FILESPATH", True).split(":")
+
+    for url, urldata in srcurldata.iteritems():
+        if urldata.type == "file" and "*" in urldata.path:
+            # The fetch code doesn't know how to handle globs, so
+            # we need to handle the local bits ourselves
+            for path in filespath:
+                srcdir = oe.path.join(path, urldata.host,
+                                      os.path.dirname(urldata.path))
+                if os.path.exists(srcdir):
+                    break
+            else:
+                bb.fatal("Unable to locate files for %s" % url)
 
-	localdata = bb.data.createCopy(d)
-	bb.data.update_data(localdata)
+            for filename in glob(oe.path.join(srcdir,
+                                              os.path.basename(urldata.path))):
+                oe_unpack(d, filename, urldata)
+        else:
+            local = urldata.localpath
+            if not local:
+                raise bb.build.FuncFailed('Unable to locate local file for %s' % url)
 
-	src_uri = bb.data.getVar('SRC_URI', localdata, True)
-	if not src_uri:
-		return
-	for url in src_uri.split():
-		try:
-			local = bb.data.expand(bb.fetch.localpath(url, localdata), localdata)
-		except bb.MalformedUrl, e:
-			raise bb.build.FuncFailed('Unable to generate local path for malformed uri: %s' % e)
-		if not local:
-			raise bb.build.FuncFailed('Unable to locate local file for %s' % url)
-		local = os.path.realpath(local)
-		ret = oe_unpack_file(local, localdata, url)
-		if not ret:
-			raise bb.build.FuncFailed()
+            oe_unpack(d, local, urldata)
 }
 
 addhandler base_eventhandler
classes/patch.bbclass +14 -21
index 227d7c9..58b931f 100644
@@ -7,6 +7,7 @@ PATCHDEPENDENCY = "${PATCHTOOL}-native:do_populate_sysroot"
 
 python patch_do_patch() {
 	import oe.patch
+	import oe.unpack
 
 	src_uri = (bb.data.getVar('SRC_URI', d, 1) or '').split()
 	if not src_uri:
@@ -34,32 +35,24 @@ python patch_do_patch() {
 
 	classes = {}
 
+	src_uri = d.getVar("SRC_URI", True).split()
+	srcurldata = bb.fetch.init(src_uri, d, True)
 	workdir = bb.data.getVar('WORKDIR', d, 1)
-	for url in src_uri:
-		(type, host, path, user, pswd, parm) = bb.decodeurl(url)
+	for url in d.getVar("SRC_URI", True).split():
+		urldata = srcurldata[url]
 
-		local = None
-		base, ext = os.path.splitext(os.path.basename(path))
+		local = urldata.localpath
+		if not local:
+			raise bb.build.FuncFailed('Unable to locate local file for %s' % url)
+
+		base, ext = os.path.splitext(os.path.basename(local))
 		if ext in ('.gz', '.bz2', '.Z'):
-			local = os.path.join(workdir, base)
-			ext = os.path.splitext(base)[1]
-
-		if "apply" in parm:
-			apply = parm["apply"]
-			if apply != "yes":
-				if apply != "no":
-					bb.msg.warn(None, "Unsupported value '%s' for 'apply' url param in '%s', please use 'yes' or 'no'" % (apply, url))
-				continue
-		elif "patch" in parm:
-			bb.msg.warn(None, "Deprecated usage of 'patch' url param in '%s', please use 'apply={yes,no}'" % url)
-		elif ext not in (".diff", ".patch"):
+			local = oe.path.join(workdir, base)
+
+		if not oe.unpack.is_patch(local, urldata.parm):
 			continue
 
-		if not local:
-			bb.fetch.init([url],d)
-			url = bb.encodeurl((type, host, path, user, pswd, []))
-			local = os.path.join('/', bb.fetch.localpath(url, d))
-		local = bb.data.expand(local, d)
+		parm = urldata.parm
 
 		if "striplevel" in parm:
 			striplevel = parm["striplevel"]
lib/oe/unpack.py new file +95 -0

95 lines not on this page · raw message

Signature · 1 line
1.7.2.1.66.g0d0ba
#1 lore raw
Quoted 1 line · Chris Larson

- Move unpack functions from classes into oe.patch

Why do I never notice the typos until after sending it to the list? This should be oe.unpack, of course.

Signature · 5 lines
Christopher Larson
clarson at kergoth dot com
Founder - BitBake, OpenEmbedded, OpenZaurus
Maintainer - Tslib
Senior Software Engineer, Mentor Graphics
Acked-by
#2 lore raw
Quoted 7 lines · Chris Larson
Quoted 1 line · Chris Larson

- Move unpack functions from classes into oe.patch

Why do I never notice the typos until after sending it to the list?  This should be oe.unpack, of course.

Looks good.

Quoted 11 lines

--
Christopher Larson
clarson at kergoth dot com
Founder - BitBake, OpenEmbedded, OpenZaurus
Maintainer - Tslib
Senior Software Engineer, Mentor Graphics
_______________________________________________
Openembedded-devel mailing list
[email protected]
http://lists.linuxtogo.org/cgi-bin/mailman/listinfo/openembedded-devel

Acked-by
#3 lore raw
Quoted 12 lines · Khem Raj
Quoted 6 lines · Chris Larson
Quoted 1 line · Chris Larson

- Move unpack functions from classes into oe.patch

Why do I never notice the typos until after sending it to the list? This should be oe.unpack, of course.

Looks good.

Put this past a -c patchall -k world and then reproduced the failures w/o the patch (by hand, hopefully didn't miss anything).

Signature · 2 lines
Tom Rini
Mentor Graphics Corporation