package.bbclass: Optimise the per file rpm handling

Currently a process was being forked off for each individual file this class wanted to inspect with rpmdeps. This converts it to use rpmdeps-oecore which allows batch processing of these dependencies.

For do_package for perl, this reduced the time by about 1 minute (33%).

Signed-off-by Richard Purdie<[email protected]>

---

meta/classes/package.bbclass +1 -1

index 348c13c..dfabcf8 100644
@@ -1125,7 +1125,7 @@ if [ x"$D" = "x" ]; then
  fi
  }
 
-RPMDEPS = "${STAGING_LIBDIR_NATIVE}/rpm/bin/rpmdeps --macros ${STAGING_LIBDIR_NATIVE}/rpm/macros --define '_rpmfc_magic_path ${STAGING_DIR_NATIVE}${datadir_native}/misc/magic.mgc' --rpmpopt ${STAGING_LIBDIR_NATIVE}/rpm/rpmpopt"
+RPMDEPS = "${STAGING_LIBDIR_NATIVE}/rpm/bin/rpmdeps-oecore --macros ${STAGING_LIBDIR_NATIVE}/rpm/macros --define '_rpmfc_magic_path ${STAGING_DIR_NATIVE}${datadir_native}/misc/magic.mgc' --rpmpopt ${STAGING_LIBDIR_NATIVE}/rpm/rpmpopt"
 
  # Collect perfile run-time dependency metadata
  # Output:

meta/classes/package.bbclass +30 -18

  	pkgdest = d.getVar('PKGDEST', True)
  	packages = d.getVar('PACKAGES', True)

@@ -1145,39 +1145,49 @@ python package_do_filedeps() {
  	r = re.compile(r'[<>=]+ +[^ ]*')
 
  	# Quick routine to process the results of the rpmdeps call...
-	def process_deps(pipe, pkg, f, provides_files, requires_files):
-		provides = []
-		requires = []
-		file = f.replace(pkgdest + "/" + pkg, "")
-		file = file.replace("@", "@at@")
-		file = file.replace(" ", "@space@")
-		file = file.replace("\t", "@tab@")
-		file = file.replace("[", "@openbrace@")
-		file = file.replace("]", "@closebrace@")
-		file = file.replace("_", "@underscore@")
+	def process_deps(pipe, pkg, provides_files, requires_files):
+		provides = {}
+		requires = {}
 
  		for line in pipe:
+			f = line.split(" ", 1)[0].strip()
+			line = line.split(" ", 1)[1].strip()
+
  			if line.startswith("Requires:"):
  				i = requires
  			elif line.startswith("Provides:"):
  				i = provides
  			else:
  				continue
+
+			file = f.replace(pkgdest + "/" + pkg, "")
+			file = file.replace("@", "@at@")
+			file = file.replace(" ", "@space@")
+			file = file.replace("\t", "@tab@")
+			file = file.replace("[", "@openbrace@")
+			file = file.replace("]", "@closebrace@")
+			file = file.replace("_", "@underscore@")
  			value = line.split(":", 1)[1].strip()
  			value = r.sub(r'(\g<0>)', value)
+
  			if value.startswith("rpmlib("):
  				continue
-			i.append(value)
+			if file not in i:
+				i[file] = []
+			i[file].append(value)
 
-		if len(provides)>  0:
+		for file in provides:
  			provides_files.append(file)
  			key = "FILERPROVIDES_" + file + "_" + pkg
-			d.setVar(key, " ".join(provides))
+			d.setVar(key, " ".join(provides[file]))
 
-		if len(requires)>  0:
+		for file in requires:
  			requires_files.append(file)
  			key = "FILERDEPENDS_" + file + "_" + pkg
-			d.setVar(key, " ".join(requires))
+			d.setVar(key, " ".join(requires[file]))
+
+	def chunks(files, n):
+		return [files[i:i+n] for i in range(0, len(files), n)]
 
  	# Determine dependencies
  	for pkg in packages.split():

@@ -1186,13 +1196,15 @@ python package_do_filedeps() {
 
  		provides_files = []
  		requires_files = []
+		rpfiles = []
  		for root, dirs, files in os.walk(pkgdest + "/" + pkg):
  			for file in files:
-				f = os.path.join(root, file)
+				rpfiles.append(os.path.join(root, file))
 
-				dep_pipe = os.popen(rpmdeps + " --provides --requires -v " + f)
+		for files in chunks(rpfiles, 100):
+			dep_pipe = os.popen(rpmdeps + " " + " ".join(files))
 
-				process_deps(dep_pipe, pkg, f, provides_files, requires_files)
+			process_deps(dep_pipe, pkg, provides_files, requires_files)
 
  		d.setVar("FILERDEPENDSFLIST_" + pkg, " ".join(requires_files))
  		d.setVar("FILERPROVIDESFLIST_" + pkg, " ".join(provides_files))

meta/recipes-core/busybox/busybox.inc +2 -2

index f35779d..5fb436e 100644
@@ -216,9 +216,9 @@ python package_do_filedeps_append () {
  	# Load/backup original set
  	filerprovides = d.getVar('FILERPROVIDES_%s_%s' % (f_busybox, pkg), True) or ""
 
-	dep_pipe = os.popen('sed -e "s,^,Provides: ," %s/%s%s' % (pkgdest, pkg, f_busybox_links))
+	dep_pipe = os.popen('sed -e "s,^,%s/%s%s Provides: ," %s/%s%s' % (pkgdest, pkg, f_busybox, pkgdest, pkg, f_busybox_links))
 
-	process_deps(dep_pipe, pkg, "%s/%s%s" % (pkgdest, pkg, f_busybox), provides_files, requires_files)
+	process_deps(dep_pipe, pkg, provides_files, requires_files)
 
  	# Add the new set
  	filerprovides += d.getVar('FILERPROVIDES_%s_%s' % (f_busybox, pkg), True) or ""

_______________________________________________
Openembedded-core mailing list
[email protected]
http://lists.linuxtogo.org/cgi-bin/mailman/listinfo/openembedded-core

Quoted 5 lines · Richard Purdie

Currently a process was being forked off for each individual file this class wanted to inspect with rpmdeps. This converts it to use rpmdeps-oecore which allows batch processing of these dependencies.

For do_package for perl, this reduced the time by about 1 minute (33%).

Awesome! Sounds like this addresses #1718

http://bugzilla.pokylinux.org/show_bug.cgi?id=1718

meta/classes/package.bbclass RPMDEPS = "${STAGING_LIBDIR_NATIVE}/rpm/bin/rpmdeps --macros ${STAGING_LIBDIR_NA · line 1139

Quoted 6 lines

meta/classes/package.bbclass +1 -1

@@ -1136,7 +1136,7 @@ RPMDEPS = "${STAGING_LIBDIR_NATIVE}/rpm/bin/rpmdeps --macros ${STAGING_LIBDIR_NA
  #  FILERDEPENDS_filepath_pkg - per file dep
 
  python package_do_filedeps() {
-	import os, re
+	import re, subprocess

Was this accidentally left in? I don't see any calls to subprocess?