Skip to contentSkip to conversation

package_rpm.bbclass: Optimise per file dependency handling

PATCH In review Richard Purdie 2 participants 2 messages Mar 2, 2013 – Mar 4, 2013

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 package_rpm.bbclass: Optimise per file dependency handling

Currently the process for injecting the per file rpm dependencies into rpmbuild is painfully slow. Its done through the repeated execution of a script which has to return the correct value in each case. This continual execution means the CPU usage of rpmbuild is low.

This patch allows the option of collapsing the per file dependencies to a per package basis and injecting them through the .spec file. This removes the execution overhead and allows rpmbuild to run at 100% of cpu.

Ultimately it would be nice to inject the per file dependencies through the .spec file however that is not currently possible.

Since few people use the per file dependency information, this patch goes for the faster approach. It can be enabled if anyone needs it although I'd mention that its being used to us as this code may well go away in the future if nobody complains.

---

meta/classes/package_rpm.bbclass +104 -65
index 697bb36..3ac379d 100644
@@ -8,6 +8,10 @@ RPMBUILD="rpmbuild"
 PKGWRITEDIRRPM = "${WORKDIR}/deploy-rpms"
 PKGWRITEDIRSRPM = "${DEPLOY_DIR}/sources/deploy-srpm"
 
+# Maintaining the perfile dependencies has singificant overhead when writing the 
+# packages. When set, this value merges them for efficiency.
+MERGEPERFILEDEPS = "1"
+
 #
 # Update the packages indexes ${DEPLOY_DIR_RPM}
 #
@@ -460,6 +464,78 @@ EOF
 	fi
 }
 
+# Construct per file dependencies file
+def write_rpm_perfiledata(srcname, d):
+    workdir = d.getVar('WORKDIR', True)
+    packages = d.getVar('PACKAGES', True)
+    pkgd = d.getVar('PKGD', True)
+
+    def dump_filerdeps(varname, outfile, d):
+        outfile.write("#!/usr/bin/env python\n\n")
+        outfile.write("# Dependency table\n")
+        outfile.write('deps = {\n')
+        for pkg in packages.split():
+            dependsflist_key = 'FILE' + varname + 'FLIST' + "_" + pkg
+            dependsflist = (d.getVar(dependsflist_key, True) or "")
+            for dfile in dependsflist.split():
+                key = "FILE" + varname + "_" + dfile + "_" + pkg
+                depends_dict = bb.utils.explode_dep_versions(d.getVar(key, True) or "")
+                file = dfile.replace("@underscore@", "_")
+                file = file.replace("@closebrace@", "]")
+                file = file.replace("@openbrace@", "[")
+                file = file.replace("@tab@", "\t")
+                file = file.replace("@space@", " ")
+                file = file.replace("@at@", "@")
+                outfile.write('"' + pkgd + file + '" : "')
+                for dep in depends_dict:
+                    ver = depends_dict[dep]
+                    if dep and ver:
+                        ver = ver.replace("(","")
+                        ver = ver.replace(")","")
+                        outfile.write(dep + " " + ver + " ")
+                    else:
+                        outfile.write(dep + " ")
+                outfile.write('",\n')
+        outfile.write('}\n\n')
+        outfile.write("import sys\n")
+        outfile.write("while 1:\n")
+        outfile.write("\tline = sys.stdin.readline().strip()\n")
+        outfile.write("\tif not line:\n")
+        outfile.write("\t\tsys.exit(0)\n")
+        outfile.write("\tif line in deps:\n")
+        outfile.write("\t\tprint(deps[line] + '\\n')\n")
+
+    # OE-core dependencies a.k.a. RPM requires
+    outdepends = workdir + "/" + srcname + ".requires"
+
+    try:
+        from __builtin__ import file
+        dependsfile = file(outdepends, 'w')
+    except OSError:
+        raise bb.build.FuncFailed("unable to open spec file for writing.")
+
+    dump_filerdeps('RDEPENDS', dependsfile, d)
+
+    dependsfile.close()
+    os.chmod(outdepends, 0755)
+
+    # OE-core / RPM Provides
+    outprovides = workdir + "/" + srcname + ".provides"
+
+    try:
+        from __builtin__ import file
+        providesfile = file(outprovides, 'w')
+    except OSError:
+        raise bb.build.FuncFailed("unable to open spec file for writing.")
+
+    dump_filerdeps('RPROVIDES', providesfile, d)
+
+    providesfile.close()
+    os.chmod(outprovides, 0755)
+
+    return (outdepends, outprovides)
+
+
 python write_specfile () {
     import textwrap
     import oe.packagedata
@@ -576,6 +652,17 @@ python write_specfile () {
         scr = scr[:pos] + 'if [ "$1" = "0" ] ; then\n' + scr[pos:] + '\nfi'
         return scr
 
+    def get_perfile(varname, pkg, d):
+        deps = []
+        dependsflist_key = 'FILE' + varname + 'FLIST' + "_" + pkg
+        dependsflist = (d.getVar(dependsflist_key, True) or "")
+        for dfile in dependsflist.split():
+            key = "FILE" + varname + "_" + dfile + "_" + pkg
+            depends = d.getVar(key, True)
+            if depends:
+                deps.append(depends)
+        return " ".join(deps)
+
     packages = d.getVar('PACKAGES', True)
     if not packages or packages == '':
         bb.debug(1, "No packages; nothing to do")
@@ -626,6 +713,8 @@ python write_specfile () {
     spec_files_top = []
     spec_files_bottom = []
 
+    perfiledeps = (d.getVar("MERGEPERFILEDEPS", True) or "0") == "0"
+
     for pkg in packages.split():
         localdata = bb.data.createCopy(d)
 
@@ -679,6 +768,12 @@ python write_specfile () {
         splitrprerm    = localdata.getVar('pkg_prerm', True)
         splitrpostrm   = localdata.getVar('pkg_postrm', True)
 
+
+        if not perfiledeps:
+            # Add in summary of per file dependencies
+            splitrdepends = splitrdepends + " " + get_perfile('RDEPENDS', pkg, d)
+            splitrprovides = splitrprovides + " " + get_perfile('RPROVIDES', pkg, d)
+
         # Gather special src/first package data
         if srcname == splitname:
             srcrdepends    = splitrdepends
@@ -977,69 +1072,9 @@ python do_package_rpm () {
     d.setVar('OUTSPECFILE', outspecfile)
     bb.build.exec_func('write_specfile', d)
 
-    # Construct per file dependencies file
-    def dump_filerdeps(varname, outfile, d):
-        outfile.write("#!/usr/bin/env python\n\n")
-        outfile.write("# Dependency table\n")
-        outfile.write('deps = {\n')
-        for pkg in packages.split():
-            dependsflist_key = 'FILE' + varname + 'FLIST' + "_" + pkg
-            dependsflist = (d.getVar(dependsflist_key, True) or "")
-            for dfile in dependsflist.split():
-                key = "FILE" + varname + "_" + dfile + "_" + pkg
-                depends_dict = bb.utils.explode_dep_versions(d.getVar(key, True) or "")
-                file = dfile.replace("@underscore@", "_")
-                file = file.replace("@closebrace@", "]")
-                file = file.replace("@openbrace@", "[")
-                file = file.replace("@tab@", "\t")
-                file = file.replace("@space@", " ")
-                file = file.replace("@at@", "@")
-                outfile.write('"' + pkgd + file + '" : "')
-                for dep in depends_dict:
-                    ver = depends_dict[dep]
-                    if dep and ver:
-                        ver = ver.replace("(","")
-                        ver = ver.replace(")","")
-                        outfile.write(dep + " " + ver + " ")
-                    else:
-                        outfile.write(dep + " ")
-                outfile.write('",\n')
-        outfile.write('}\n\n')
-        outfile.write("import sys\n")
-        outfile.write("while 1:\n")
-        outfile.write("\tline = sys.stdin.readline().strip()\n")
-        outfile.write("\tif not line:\n")
-        outfile.write("\t\tsys.exit(0)\n")
-        outfile.write("\tif line in deps:\n")
-        outfile.write("\t\tprint(deps[line] + '\\n')\n")
-
-    # OE-core dependencies a.k.a. RPM requires
-    outdepends = workdir + "/" + srcname + ".requires"
-
-    try:
-        from __builtin__ import file
-        dependsfile = file(outdepends, 'w')
-    except OSError:
-        raise bb.build.FuncFailed("unable to open spec file for writing.")
-
-    dump_filerdeps('RDEPENDS', dependsfile, d)
-
-    dependsfile.close()
-    os.chmod(outdepends, 0755)
-
-    # OE-core / RPM Provides
-    outprovides = workdir + "/" + srcname + ".provides"
-
-    try:
-        from __builtin__ import file
-        providesfile = file(outprovides, 'w')
-    except OSError:
-        raise bb.build.FuncFailed("unable to open spec file for writing.")
-
-    dump_filerdeps('RPROVIDES', providesfile, d)
-
-    providesfile.close()
-    os.chmod(outprovides, 0755)
+    perfiledeps = (d.getVar("MERGEPERFILEDEPS", True) or "0") == "0"
+    if perfiledeps:
+        outdepends, outprovides = write_rpm_perfiledata(srcname, d)
 
     # Setup the rpmbuild arguments...
     rpmbuild = d.getVar('RPMBUILD', True)
@@ -1062,8 +1097,12 @@ python do_package_rpm () {
     cmd = cmd + " --define '_topdir " + workdir + "' --define '_rpmdir " + pkgwritedir + "'"
     cmd = cmd + " --define '_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm'"
     cmd = cmd + " --define '_use_internal_dependency_generator 0'"
-    cmd = cmd + " --define '__find_requires " + outdepends + "'"
-    cmd = cmd + " --define '__find_provides " + outprovides + "'"
+    if perfiledeps:
+        cmd = cmd + " --define '__find_requires " + outdepends + "'"
+        cmd = cmd + " --define '__find_provides " + outprovides + "'"
+    else:
+        cmd = cmd + " --define '__find_requires %{nil}'"
+        cmd = cmd + " --define '__find_provides %{nil}'"
     cmd = cmd + " --define '_unpackaged_files_terminate_build 0'"
     cmd = cmd + " --define 'debug_package %{nil}'"
     cmd = cmd + " --define '_rpmfc_magic_path " + magicfile + "'"
#1 lore raw
Quoted 24 lines · Richard Purdie

Currently the process for injecting the per file rpm dependencies into rpmbuild is painfully slow. Its done through the repeated execution of a script which has to return the correct value in each case. This continual execution means the CPU usage of rpmbuild is low.

This patch allows the option of collapsing the per file dependencies to a per package basis and injecting them through the .spec file. This removes the execution overhead and allows rpmbuild to run at 100% of cpu.

Ultimately it would be nice to inject the per file dependencies through the .spec file however that is not currently possible.

Since few people use the per file dependency information, this patch goes for the faster approach. It can be enabled if anyone needs it although I'd mention that its being used to us as this code may well go away in the future if nobody complains.

---

meta/classes/package_rpm.bbclass +0 -0
index 697bb36..3ac379d 100644

meta/classes/package_rpm.bbclass RPMBUILD="rpmbuild" · line 13

Quoted 6 lines
meta/classes/package_rpm.bbclass +3 -0
@@ -8,6 +8,10 @@ RPMBUILD="rpmbuild"
  PKGWRITEDIRRPM = "${WORKDIR}/deploy-rpms"
  PKGWRITEDIRSRPM = "${DEPLOY_DIR}/sources/deploy-srpm"
 
+# Maintaining the perfile dependencies has singificant overhead when writing the
+# packages. When set, this value merges them for efficiency.
+MERGEPERFILEDEPS = "1"

Can this be a "?=" so it can be easily overridden?

Quoted 88 lines
meta/classes/package_rpm.bbclass +74 -0
+
  #
  # Update the packages indexes ${DEPLOY_DIR_RPM}
  #
@@ -460,6 +464,78 @@ EOF
  	fi
  }
 
+# Construct per file dependencies file
+def write_rpm_perfiledata(srcname, d):
+    workdir = d.getVar('WORKDIR', True)
+    packages = d.getVar('PACKAGES', True)
+    pkgd = d.getVar('PKGD', True)
+
+    def dump_filerdeps(varname, outfile, d):
+        outfile.write("#!/usr/bin/env python\n\n")
+        outfile.write("# Dependency table\n")
+        outfile.write('deps = {\n')
+        for pkg in packages.split():
+            dependsflist_key = 'FILE' + varname + 'FLIST' + "_" + pkg
+            dependsflist = (d.getVar(dependsflist_key, True) or "")
+            for dfile in dependsflist.split():
+                key = "FILE" + varname + "_" + dfile + "_" + pkg
+                depends_dict = bb.utils.explode_dep_versions(d.getVar(key, True) or "")
+                file = dfile.replace("@underscore@", "_")
+                file = file.replace("@closebrace@", "]")
+                file = file.replace("@openbrace@", "[")
+                file = file.replace("@tab@", "\t")
+                file = file.replace("@space@", " ")
+                file = file.replace("@at@", "@")
+                outfile.write('"' + pkgd + file + '" : "')
+                for dep in depends_dict:
+                    ver = depends_dict[dep]
+                    if dep and ver:
+                        ver = ver.replace("(","")
+                        ver = ver.replace(")","")
+                        outfile.write(dep + " " + ver + " ")
+                    else:
+                        outfile.write(dep + " ")
+                outfile.write('",\n')
+        outfile.write('}\n\n')
+        outfile.write("import sys\n")
+        outfile.write("while 1:\n")
+        outfile.write("\tline = sys.stdin.readline().strip()\n")
+        outfile.write("\tif not line:\n")
+        outfile.write("\t\tsys.exit(0)\n")
+        outfile.write("\tif line in deps:\n")
+        outfile.write("\t\tprint(deps[line] + '\\n')\n")
+
+    # OE-core dependencies a.k.a. RPM requires
+    outdepends = workdir + "/" + srcname + ".requires"
+
+    try:
+        from __builtin__ import file
+        dependsfile = file(outdepends, 'w')
+    except OSError:
+        raise bb.build.FuncFailed("unable to open spec file for writing.")
+
+    dump_filerdeps('RDEPENDS', dependsfile, d)
+
+    dependsfile.close()
+    os.chmod(outdepends, 0755)
+
+    # OE-core / RPM Provides
+    outprovides = workdir + "/" + srcname + ".provides"
+
+    try:
+        from __builtin__ import file
+        providesfile = file(outprovides, 'w')
+    except OSError:
+        raise bb.build.FuncFailed("unable to open spec file for writing.")
+
+    dump_filerdeps('RPROVIDES', providesfile, d)
+
+    providesfile.close()
+    os.chmod(outprovides, 0755)
+
+    return (outdepends, outprovides)
+
+
  python write_specfile () {
      import textwrap
      import oe.packagedata
@@ -576,6 +652,17 @@ python write_specfile () {
          scr = scr[:pos] + 'if [ "$1" = "0" ] ; then\n' + scr[pos:] + '\nfi'
          return scr
 
+    def get_perfile(varname, pkg, d):

meta/classes/package_rpm.bbclass python write_specfile () { · line 665

Quoted 10 lines
meta/classes/package_rpm.bbclass +10 -0
+        deps = []
+        dependsflist_key = 'FILE' + varname + 'FLIST' + "_" + pkg
+        dependsflist = (d.getVar(dependsflist_key, True) or "")
+        for dfile in dependsflist.split():
+            key = "FILE" + varname + "_" + dfile + "_" + pkg
+            depends = d.getVar(key, True)
+            if depends:
+                deps.append(depends)
+        return " ".join(deps)
+

The above doesn't avoid duplicate entries. Is this a concern? (RPM doesn't care, it'll do it itself... but it may be slower due to this.)

Quoted 115 lines
meta/classes/package_rpm.bbclass +17 -65
      packages = d.getVar('PACKAGES', True)
      if not packages or packages == '':
          bb.debug(1, "No packages; nothing to do")
@@ -626,6 +713,8 @@ python write_specfile () {
      spec_files_top = []
      spec_files_bottom = []
 
+    perfiledeps = (d.getVar("MERGEPERFILEDEPS", True) or "0") == "0"
+
      for pkg in packages.split():
          localdata = bb.data.createCopy(d)
 
@@ -679,6 +768,12 @@ python write_specfile () {
          splitrprerm    = localdata.getVar('pkg_prerm', True)
          splitrpostrm   = localdata.getVar('pkg_postrm', True)
 
+
+        if not perfiledeps:
+            # Add in summary of per file dependencies
+            splitrdepends = splitrdepends + " " + get_perfile('RDEPENDS', pkg, d)
+            splitrprovides = splitrprovides + " " + get_perfile('RPROVIDES', pkg, d)
+
          # Gather special src/first package data
          if srcname == splitname:
              srcrdepends    = splitrdepends
@@ -977,69 +1072,9 @@ python do_package_rpm () {
      d.setVar('OUTSPECFILE', outspecfile)
      bb.build.exec_func('write_specfile', d)
 
-    # Construct per file dependencies file
-    def dump_filerdeps(varname, outfile, d):
-        outfile.write("#!/usr/bin/env python\n\n")
-        outfile.write("# Dependency table\n")
-        outfile.write('deps = {\n')
-        for pkg in packages.split():
-            dependsflist_key = 'FILE' + varname + 'FLIST' + "_" + pkg
-            dependsflist = (d.getVar(dependsflist_key, True) or "")
-            for dfile in dependsflist.split():
-                key = "FILE" + varname + "_" + dfile + "_" + pkg
-                depends_dict = bb.utils.explode_dep_versions(d.getVar(key, True) or "")
-                file = dfile.replace("@underscore@", "_")
-                file = file.replace("@closebrace@", "]")
-                file = file.replace("@openbrace@", "[")
-                file = file.replace("@tab@", "\t")
-                file = file.replace("@space@", " ")
-                file = file.replace("@at@", "@")
-                outfile.write('"' + pkgd + file + '" : "')
-                for dep in depends_dict:
-                    ver = depends_dict[dep]
-                    if dep and ver:
-                        ver = ver.replace("(","")
-                        ver = ver.replace(")","")
-                        outfile.write(dep + " " + ver + " ")
-                    else:
-                        outfile.write(dep + " ")
-                outfile.write('",\n')
-        outfile.write('}\n\n')
-        outfile.write("import sys\n")
-        outfile.write("while 1:\n")
-        outfile.write("\tline = sys.stdin.readline().strip()\n")
-        outfile.write("\tif not line:\n")
-        outfile.write("\t\tsys.exit(0)\n")
-        outfile.write("\tif line in deps:\n")
-        outfile.write("\t\tprint(deps[line] + '\\n')\n")
-
-    # OE-core dependencies a.k.a. RPM requires
-    outdepends = workdir + "/" + srcname + ".requires"
-
-    try:
-        from __builtin__ import file
-        dependsfile = file(outdepends, 'w')
-    except OSError:
-        raise bb.build.FuncFailed("unable to open spec file for writing.")
-
-    dump_filerdeps('RDEPENDS', dependsfile, d)
-
-    dependsfile.close()
-    os.chmod(outdepends, 0755)
-
-    # OE-core / RPM Provides
-    outprovides = workdir + "/" + srcname + ".provides"
-
-    try:
-        from __builtin__ import file
-        providesfile = file(outprovides, 'w')
-    except OSError:
-        raise bb.build.FuncFailed("unable to open spec file for writing.")
-
-    dump_filerdeps('RPROVIDES', providesfile, d)
-
-    providesfile.close()
-    os.chmod(outprovides, 0755)
+    perfiledeps = (d.getVar("MERGEPERFILEDEPS", True) or "0") == "0"
+    if perfiledeps:
+        outdepends, outprovides = write_rpm_perfiledata(srcname, d)
 
      # Setup the rpmbuild arguments...
      rpmbuild = d.getVar('RPMBUILD', True)
@@ -1062,8 +1097,12 @@ python do_package_rpm () {
      cmd = cmd + " --define '_topdir " + workdir + "' --define '_rpmdir " + pkgwritedir + "'"
      cmd = cmd + " --define '_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm'"
      cmd = cmd + " --define '_use_internal_dependency_generator 0'"
-    cmd = cmd + " --define '__find_requires " + outdepends + "'"
-    cmd = cmd + " --define '__find_provides " + outprovides + "'"
+    if perfiledeps:
+        cmd = cmd + " --define '__find_requires " + outdepends + "'"
+        cmd = cmd + " --define '__find_provides " + outprovides + "'"
+    else:
+        cmd = cmd + " --define '__find_requires %{nil}'"
+        cmd = cmd + " --define '__find_provides %{nil}'"
      cmd = cmd + " --define '_unpackaged_files_terminate_build 0'"
      cmd = cmd + " --define 'debug_package %{nil}'"
      cmd = cmd + " --define '_rpmfc_magic_path " + magicfile + "'"