support/graphs: speedup graph-depends (branch yem/graphs-from-make-2)

index db7f06ddbe..dd3e20ca57 100644
@@ -859,6 +859,7 @@ $(1)-show-build-order: $$(patsubst %,%-show-build-order,$$($(2)_FINAL_ALL_DEPEND
 	$$(info $(1))
 
 $(1)-show-dependency-tree: $$(patsubst %,%-show-dependency-tree,$$($(2)_FINAL_ALL_DEPENDENCIES))
+	@:
 	$$(info $(1): $(4) $$(if $$($(2)_IS_VIRTUAL),virtual,$$($(2)_DL_VERSION)))
 	$$(info $(1) -> $$(foreach d,$$($(2)_FINAL_ALL_DEPENDENCIES),$$(d)))
 

@@ -1079,6 +1080,7 @@ DL_TOOLS_DEPENDENCIES += $$(call extractor-dependency,$$($(2)_SOURCE))
 	$(1)-external-deps \
 	$(1)-extract \
 	$(1)-graph-depends \
+	$(1)-graph-rdepends \
 	$(1)-install \
 	$(1)-install-host \
 	$(1)-install-images \

@@ -1091,6 +1093,7 @@ DL_TOOLS_DEPENDENCIES += $$(call extractor-dependency,$$($(2)_SOURCE))
 	$(1)-reconfigure \
 	$(1)-reinstall \
 	$(1)-rsync \
+	$(1)-show-dependency-tree \
 	$(1)-show-depends \
 	$(1)-show-version \
 	$(1)-source

index 29134c8237..7ed28440bb 100755
@@ -216,13 +216,17 @@ def check_circular_deps(deps):
 
 # This functions trims down the dependency list of all packages.
 # It applies in sequence all the dependency-elimination methods.
-def remove_extra_deps(deps, rootpkg, transitive):
-    for pkg in list(deps.keys()):
-        if not pkg == rootpkg:
-            for d in get_mandatory_deps(pkg, deps):
-                if d not in deps[rootpkg]:
-                    deps[rootpkg].append(d)
-            deps[pkg] = remove_mandatory_deps(pkg, deps)
+def remove_extra_deps(deps, rootpkg, transitive, arrow_dir):
+    # For the direct dependencies, find and eliminate mandatory
+    # deps, and add them to the root package. Don't do it for a
+    # reverse graph, because mandatory deps are only direct deps.
+    if arrow_dir == "forward":
+        for pkg in list(deps.keys()):
+            if not pkg == rootpkg:
+                for d in get_mandatory_deps(pkg, deps):
+                    if d not in deps[rootpkg]:
+                        deps[rootpkg].append(d)
+                deps[pkg] = remove_mandatory_deps(pkg, deps)
     for pkg in list(deps.keys()):
         if not transitive or pkg == rootpkg:
             deps[pkg] = remove_transitive_deps(pkg, deps)

@@ -420,7 +424,7 @@ def main():
     if check_only:
         sys.exit(0)
 
-    dict_deps = remove_extra_deps(dict_deps, rootpkg, args.transitive)
+    dict_deps = remove_extra_deps(dict_deps, rootpkg, args.transitive, arrow_dir)
     dict_version = brpkgutil.get_version([pkg for pkg in allpkgs
                                           if pkg != "all" and not pkg.startswith("root")])
 

index e70d525353..9ceda71b7f 100644
@@ -1,67 +1,50 @@
 # Copyright (C) 2010-2013 Thomas Petazzoni <[email protected]>
+# Copyright (C) 2019 Yann E. MORIN <[email protected]>
 
 import logging
-import sys
+import os
 import subprocess
+from collections import defaultdict
 
 
-# Execute the "make <pkg>-show-version" command to get the version of a given
-# list of packages, and return the version formatted as a Python dictionary.
-def get_version(pkgs):
-    logging.info("Getting version for %s" % pkgs)
-    cmd = ["make", "-s", "--no-print-directory"]
-    for pkg in pkgs:
-        cmd.append("%s-show-version" % pkg)
-    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, universal_newlines=True)
-    output = p.communicate()[0]
-    if p.returncode != 0:
-        logging.error("Error getting version %s" % pkgs)
-        sys.exit(1)
-    output = output.split("\n")
-    if len(output) != len(pkgs) + 1:
-        logging.error("Error getting version")
-        sys.exit(1)
-    version = {}
-    for i in range(0, len(pkgs)):
-        pkg = pkgs[i]
-        version[pkg] = output[i]
-    return version
+# This function returns a tuple of four dictionaries, all using package
+# names as keys:
+# - a dictionary which values are the lists of packages that are the
+#   dependencies of the package used as key;
+# - a dictionary which values are the lists of packages that are the
+#   reverse dependencies of the package used as key;
+# - a dictionary which values are the type of the package used as key;
+# - a dictionary which values are the version of the package used as key,
+#   'virtual' for a virtual package, or the empty string for a rootfs.
+def get_dependency_tree():
+    logging.info("Getting dependency tree...")
 
+    deps = defaultdict(list)
+    rdeps = defaultdict(list)
+    types = {}
+    versions = {}
 
-def _get_depends(pkgs, rule):
-    logging.info("Getting dependencies for %s" % pkgs)
-    cmd = ["make", "-s", "--no-print-directory"]
-    for pkg in pkgs:
-        cmd.append("%s-%s" % (pkg, rule))
-    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, universal_newlines=True)
-    output = p.communicate()[0]
-    if p.returncode != 0:
-        logging.error("Error getting dependencies %s\n" % pkgs)
-        sys.exit(1)
-    output = output.split("\n")
-    if len(output) != len(pkgs) + 1:
-        logging.error("Error getting dependencies")
-        sys.exit(1)
-    deps = {}
-    for i in range(0, len(pkgs)):
-        pkg = pkgs[i]
-        pkg_deps = output[i].split(" ")
-        if pkg_deps == ['']:
-            deps[pkg] = []
+    # Special case for the 'all' top-level fake package
+    deps['all'] = []
+    types['all'] = 'target'
+    versions['all'] = ''
+
+    cmd = ["make", "-s", "--no-print-directory", "show-dependency-tree"]
+    with open(os.devnull, 'wb') as devnull:
+        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=devnull, universal_newlines=True)
+        output = p.communicate()[0]
+
+    for l in output.splitlines():
+        if " -> " in l:
+            pkg = l.split(" -> ")[0]
+            deps[pkg] += l.split(" -> ")[1].split()
+            for p in l.split(" -> ")[1].split():
+                rdeps[p].append(pkg)
         else:
-            deps[pkg] = pkg_deps
-    return deps
+            pkg, type_version = l.split(": ", 1)
+            t, v = "{} -".format(type_version).split(None, 2)[:2]
+            deps['all'].append(pkg)
+            types[pkg] = t
+            versions[pkg] = v
 
-
-# Execute the "make <pkg>-show-depends" command to get the list of
-# dependencies of a given list of packages, and return the list of
-# dependencies formatted as a Python dictionary.
-def get_depends(pkgs):
-    return _get_depends(pkgs, 'show-depends')
-
-
-# Execute the "make <pkg>-show-rdepends" command to get the list of
-# reverse dependencies of a given list of packages, and return the
-# list of dependencies formatted as a Python dictionary.
-def get_rdepends(pkgs):
-    return _get_depends(pkgs, 'show-rdepends')
+    return (deps, rdeps, types, versions)

210 lines not on this page · raw message