Skip to contentSkip to conversation

buildhistory: Label packages providing per-file dependencies in depends.dot

PATCH Superseded Andres Beltran 3 participants 3 messages Aug 18, 2021 – Aug 27, 2021

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 buildhistory: Label packages providing per-file dependencies in depends.dot

Currently, depends.dot includes per-file dependencies but not the packages providing those files. This makes it hard to obtain all package dependencies by just looking at depends.dot.

Parse the RPROVIDES and FILERPROVIDES fields from pkgdata to map each of their values to the package providing the component. Include runtime packages as dependencies in depends.dot, together with the component provided by the package as a label.

---
 meta/classes/buildhistory.bbclass |  4 +++-
 meta/lib/oe/utils.py              | 29 +++++++++++++++++++++++++++--
 2 files changed, 30 insertions(+), 3 deletions(-)
meta/classes/buildhistory.bbclass +3 -1
index 8a1359acbe..9ec48a879a 100644
@@ -438,7 +438,7 @@ def buildhistory_list_installed(d, rootfs_type="image"):
         output_file_full = os.path.join(d.getVar('WORKDIR'), output_file)
 
         with open(output_file_full, 'w') as output:
-            output.write(format_pkg_list(pkgs, output_type))
+            output.write(format_pkg_list(pkgs, output_type, d.getVar('PKGDATA_DIR')))
 
 python buildhistory_list_installed_image() {
     buildhistory_list_installed(d)
@@ -479,6 +479,8 @@ buildhistory_get_installed() {
 	       -e 's:|: -> :' \
 	       -e 's:"\[REC\]":[style=dotted]:' \
 	       -e 's:"\([<>=]\+\)" "\([^"]*\)":[label="\1 \2"]:' \
+	       -e 's:"\([*]\+\)" "\([^"]*\)":[label="\2"]:' \
+	       -e 's:"\[RPROVIDES\]":[style=dashed]:' \
 		$1/depends.tmp
 	# Add header, sorted and de-duped contents and footer and then delete the temp file
 	printf "digraph depends {\n    node [shape=plaintext]\n" > $1/depends.dot
meta/lib/oe/utils.py +27 -2
index 83d298906b..0f3dfd644d 100644
@@ -345,7 +345,26 @@ def squashspaces(string):
     import re
     return re.sub(r"\s+", " ", string).strip()
 
-def format_pkg_list(pkg_dict, ret_format=None):
+def rprovides_map(pkgdata_dir, pkg_dict):
+    # Map file -> pkg provider
+    rprov_map = {}
+
+    for pkg in pkg_dict:
+        with open(os.path.join(pkgdata_dir, 'runtime-reverse', pkg)) as f:
+            for line in f:
+                if line.startswith('RPROVIDES') or line.startswith('FILERPROVIDES'):
+                    # List all components provided by pkg.
+                    # Exclude version strings, i.e. those starting with (
+                    provides = [x for x in line.split()[1:] if not x.startswith('(')]
+                    for prov in provides:
+                        if prov in rprov_map:
+                            rprov_map[prov].append(pkg)
+                        else:
+                            rprov_map[prov] = [pkg]
+
+    return rprov_map
+
+def format_pkg_list(pkg_dict, ret_format=None, pkgdata_dir=None):
     output = []
 
     if ret_format == "arch":
@@ -358,9 +377,15 @@ def format_pkg_list(pkg_dict, ret_format=None):
         for pkg in sorted(pkg_dict):
             output.append("%s %s %s" % (pkg, pkg_dict[pkg]["arch"], pkg_dict[pkg]["ver"]))
     elif ret_format == "deps":
+        rprov_map = rprovides_map(pkgdata_dir, pkg_dict)
         for pkg in sorted(pkg_dict):
             for dep in pkg_dict[pkg]["deps"]:
-                output.append("%s|%s" % (pkg, dep))
+                if dep in rprov_map:
+                    # There could be multiple providers within the image
+                    for pkg_provider in rprov_map[dep]:
+                        output.append("%s|%s * %s [RPROVIDES]" % (pkg, pkg_provider, dep))
+                else:
+                    output.append("%s|%s" % (pkg, dep))
     else:
         for pkg in sorted(pkg_dict):
             output.append(pkg)
Signature · 1 line
2.17.1
Acked-by on v1 1/1
#1 lore raw
Quoted 10 lines · On Thursday, 19 August 2021 05:17:26 NZST Andres Beltran wrote:

Currently, depends.dot includes per-file dependencies but not the packages providing those files. This makes it hard to obtain all package dependencies by just looking at depends.dot.

Parse the RPROVIDES and FILERPROVIDES fields from pkgdata to map each of their values to the package providing the component. Include runtime packages as dependencies in depends.dot, together with the component provided by the package as a label.

(FWIW)

Cheers
Paul

#2 lore raw
In reply to Andres Beltran
Quoted 76 lines · Andres Beltran

Currently, depends.dot includes per-file dependencies but not the packages providing those files. This makes it hard to obtain all package dependencies by just looking at depends.dot.

Parse the RPROVIDES and FILERPROVIDES fields from pkgdata to map each of their values to the package providing the component. Include runtime packages as dependencies in depends.dot, together with the component provided by the package as a label.

---
 meta/classes/buildhistory.bbclass |  4 +++-
 meta/lib/oe/utils.py              | 29 +++++++++++++++++++++++++++--
 2 files changed, 30 insertions(+), 3 deletions(-)
meta/classes/buildhistory.bbclass +3 -1
index 8a1359acbe..9ec48a879a 100644
@@ -438,7 +438,7 @@ def buildhistory_list_installed(d, rootfs_type="image"):
         output_file_full = os.path.join(d.getVar('WORKDIR'), output_file)
 
         with open(output_file_full, 'w') as output:
-            output.write(format_pkg_list(pkgs, output_type))
+            output.write(format_pkg_list(pkgs, output_type, d.getVar('PKGDATA_DIR')))
 
 python buildhistory_list_installed_image() {
     buildhistory_list_installed(d)
@@ -479,6 +479,8 @@ buildhistory_get_installed() {
 	       -e 's:|: -> :' \
 	       -e 's:"\[REC\]":[style=dotted]:' \
 	       -e 's:"\([<>=]\+\)" "\([^"]*\)":[label="\1 \2"]:' \
+	       -e 's:"\([*]\+\)" "\([^"]*\)":[label="\2"]:' \
+	       -e 's:"\[RPROVIDES\]":[style=dashed]:' \
 		$1/depends.tmp
 	# Add header, sorted and de-duped contents and footer and then delete the temp file
 	printf "digraph depends {\n    node [shape=plaintext]\n" > $1/depends.dot
meta/lib/oe/utils.py +21 -1
index 83d298906b..0f3dfd644d 100644
@@ -345,7 +345,26 @@ def squashspaces(string):
     import re
     return re.sub(r"\s+", " ", string).strip()
 
-def format_pkg_list(pkg_dict, ret_format=None):
+def rprovides_map(pkgdata_dir, pkg_dict):
+    # Map file -> pkg provider
+    rprov_map = {}
+
+    for pkg in pkg_dict:
+        with open(os.path.join(pkgdata_dir, 'runtime-reverse', pkg)) as f:
+            for line in f:
+                if line.startswith('RPROVIDES') or line.startswith('FILERPROVIDES'):
+                    # List all components provided by pkg.
+                    # Exclude version strings, i.e. those starting with (
+                    provides = [x for x in line.split()[1:] if not x.startswith('(')]
+                    for prov in provides:
+                        if prov in rprov_map:
+                            rprov_map[prov].append(pkg)
+                        else:
+                            rprov_map[prov] = [pkg]
+
+    return rprov_map
+
+def format_pkg_list(pkg_dict, ret_format=None, pkgdata_dir=None):
     output = []
 
     if ret_format == "arch":
@@ -358,9 +377,15 @@ def format_pkg_list(pkg_dict, ret_format=None):
         for pkg in sorted(pkg_dict):
             output.append("%s %s %s" % (pkg, pkg_dict[pkg]["arch"], pkg_dict[pkg]["ver"]))
     elif ret_format == "deps":
+        rprov_map = rprovides_map(pkgdata_dir, pkg_dict)
         for pkg in sorted(pkg_dict):
             for dep in pkg_dict[pkg]["deps"]:

meta/lib/oe/utils.py def format_pkg_list(pkg_dict, ret_format=None): · line 391

Quoted 10 lines
meta/lib/oe/utils.py +6 -1
-                output.append("%s|%s" % (pkg, dep))
+                if dep in rprov_map:
+                    # There could be multiple providers within the image
+                    for pkg_provider in rprov_map[dep]:
+                        output.append("%s|%s * %s [RPROVIDES]" % (pkg, pkg_provider, dep))
+                else:
+                    output.append("%s|%s" % (pkg, dep))
     else:
         for pkg in sorted(pkg_dict):
             output.append(pkg)

This seemed to cause failures when generating SDKs:

https://autobuilder.yoctoproject.org/typhoon/#/builders/53/builds/3914
https://autobuilder.yoctoproject.org/typhoon/#/builders/42/builds/3895
https://autobuilder.yoctoproject.org/typhoon/#/builders/74/builds/3882
https://autobuilder.yoctoproject.org/typhoon/#/builders/63/builds/3875
https://autobuilder.yoctoproject.org/typhoon/#/builders/59/builds/3884
https://autobuilder.yoctoproject.org/typhoon/#/builders/73/builds/3880

Cheers,

Richard