Generating a graph of the size installed by each package

Currently, all the installation work of the toolchain-external package is done during the install-staging step. However, in order to be able to properly collect the size added by each package to the target filesystem, we need to make sure that toolchain-external installs its files to $(TARGET_DIR) during the install-target step.

Signed-off-by Thomas Petazzoni <[email protected]>

---
 toolchain/toolchain-external/toolchain-external.mk | 36 +++++++++++++++++-----
 1 file changed, 29 insertions(+), 7 deletions(-)

toolchain/toolchain-external/toolchain-external.mk +1 -3

index c73cc4a..45926cf 100644
@@ -551,7 +551,7 @@ endif
 #                       considered when searching libraries for copy
 #                       to the target filesystem.
 
-define TOOLCHAIN_EXTERNAL_INSTALL_CORE
+define TOOLCHAIN_EXTERNAL_INSTALL_TARGET_LIBS
 	$(Q)SYSROOT_DIR="$(call toolchain_find_sysroot,$(TOOLCHAIN_EXTERNAL_CC))" ; \
 	if test -z "$${SYSROOT_DIR}" ; then \
 		@echo "External toolchain doesn't support --sysroot. Cannot use." ; \

@@ -576,8 +576,6 @@ define TOOLCHAIN_EXTERNAL_INSTALL_CORE
 			$(call copy_toolchain_lib_root,$${ARCH_SYSROOT_DIR},$${SUPPORT_LIB_DIR},$${ARCH_LIB_DIR},$$libs,/usr/lib); \
 		done ; \
 	fi ; \
-	$(call MESSAGE,"Copying external toolchain sysroot to staging...") ; \
-	$(call copy_toolchain_sysroot,$${SYSROOT_DIR},$${ARCH_SYSROOT_DIR},$${ARCH_SUBDIR},$${ARCH_LIB_DIR},$${SUPPORT_LIB_DIR}) ; \
 	if test "$(BR2_TOOLCHAIN_EXTERNAL_GDB_SERVER_COPY)" = "y"; then \
 		$(call MESSAGE,"Copying gdbserver") ; \
 		gdbserver_found=0 ; \

toolchain/toolchain-external/toolchain-external.mk +5 -0

+	$(Q)SYSROOT_DIR="$(call toolchain_find_sysroot,$(TOOLCHAIN_EXTERNAL_CC))" ; \
+	if test -z "$${SYSROOT_DIR}" ; then \
+		@echo "External toolchain doesn't support --sysroot. Cannot use." ; \
+		exit 1 ; \
+	fi ; \

toolchain/toolchain-external/toolchain-external.mk +12 -4

+	$(call MESSAGE,"Copying external toolchain sysroot to staging...") ; \
+	$(call copy_toolchain_sysroot,$${SYSROOT_DIR},$${ARCH_SYSROOT_DIR},$${ARCH_SUBDIR},$${ARCH_LIB_DIR},$${SUPPORT_LIB_DIR})
+endef
+
 # Special installation target used on the Blackfin architecture when
 # FDPIC is not the primary binary format being used, but the user has
 # nonetheless requested the installation of the FDPIC libraries to the

@@ -685,15 +703,19 @@ define TOOLCHAIN_EXTERNAL_INSTALL_GDBINIT
 	fi
 endef
 
+define TOOLCHAIN_EXTERNAL_INSTALL_STAGING_CMDS
+	$(TOOLCHAIN_EXTERNAL_INSTALL_SYSROOT_LIBS)
+	$(TOOLCHAIN_EXTERNAL_INSTALL_WRAPPER)
+	$(TOOLCHAIN_EXTERNAL_INSTALL_GDBINIT)
+endef
+
 # Even though we're installing things in both the staging, the host
 # and the target directory, we do everything within the
 # install-staging step, arbitrarily.
-define TOOLCHAIN_EXTERNAL_INSTALL_STAGING_CMDS
-	$(TOOLCHAIN_EXTERNAL_INSTALL_CORE)
+define TOOLCHAIN_EXTERNAL_INSTALL_TARGET_CMDS
+	$(TOOLCHAIN_EXTERNAL_INSTALL_TARGET_LIBS)
 	$(TOOLCHAIN_EXTERNAL_INSTALL_BFIN_FDPIC)
 	$(TOOLCHAIN_EXTERNAL_INSTALL_BFIN_FLAT)
-	$(TOOLCHAIN_EXTERNAL_INSTALL_WRAPPER)
-	$(TOOLCHAIN_EXTERNAL_INSTALL_GDBINIT)
 endef
 
 $(eval $(generic-package))

Signature · 6 lines

2.0.0

_______________________________________________
buildroot mailing list
buildroot at busybox.net
http://lists.busybox.net/mailman/listinfo/buildroot

Quoted 1 line

+define TOOLCHAIN_EXTERNAL_INSTALL_SYSROOT_LIBS

Maybe the code from here...

Quoted 10 lines

toolchain/toolchain-external/toolchain-external.mk +10 -0

+	$(Q)SYSROOT_DIR="$(call toolchain_find_sysroot,$(TOOLCHAIN_EXTERNAL_CC))" ; \
+	if test -z "$${SYSROOT_DIR}" ; then \
+		@echo "External toolchain doesn't support --sysroot. Cannot use." ; \
+		exit 1 ; \
+	fi ; \
+	ARCH_SYSROOT_DIR="$(call toolchain_find_sysroot,$(TOOLCHAIN_EXTERNAL_CC) $(TOOLCHAIN_EXTERNAL_CFLAGS))" ; \
+	ARCH_LIB_DIR="$(call toolchain_find_libdir,$(TOOLCHAIN_EXTERNAL_CC) $(TOOLCHAIN_EXTERNAL_CFLAGS))" ; \
+	SUPPORT_LIB_DIR="" ; \
+	if test `find $${ARCH_SYSROOT_DIR} -name 'libstdc++.a' | wc -l` -eq 0 ; then \
+		LIBSTDCPP_A_LOCATION=$$(LANG=C $(TOOLCHAIN_EXTERNAL_CC) $(TOOLCHAIN_EXTERNAL_CFLAGS) -print-file-name=libstdc++.a) ; \

toolchain/toolchain-external/toolchain-external.mk define TOOLCHAIN_EXTERNAL_INSTALL_CORE · line 611

Quoted 5 lines

toolchain/toolchain-external/toolchain-external.mk +5 -0

+		if [ -e "$${LIBSTDCPP_A_LOCATION}" ]; then \
+			SUPPORT_LIB_DIR=`readlink -f $${LIBSTDCPP_A_LOCATION} | sed -r -e 's:libstdc\+\+\.a::'` ; \
+		fi ; \
+	fi ; \
+	ARCH_SUBDIR=`echo $${ARCH_SYSROOT_DIR} | sed -r -e "s:^$${SYSROOT_DIR}(.*)/$$:\1:"` ; \

... to here could be moved to a common function, so it can be shared between the staging and target functions?

Quoted 14 lines

Quoted 1 line

+define TOOLCHAIN_EXTERNAL_INSTALL_SYSROOT_LIBS

Maybe the code from here...

Quoted 10 lines

toolchain/toolchain-external/toolchain-external.mk +10 -0

+	$(Q)SYSROOT_DIR="$(call toolchain_find_sysroot,$(TOOLCHAIN_EXTERNAL_CC))" ; \
+	if test -z "$${SYSROOT_DIR}" ; then \
+		@echo "External toolchain doesn't support --sysroot. Cannot use." ; \
+		exit 1 ; \
+	fi ; \
+	ARCH_SYSROOT_DIR="$(call toolchain_find_sysroot,$(TOOLCHAIN_EXTERNAL_CC) $(TOOLCHAIN_EXTERNAL_CFLAGS))" ; \
+	ARCH_LIB_DIR="$(call toolchain_find_libdir,$(TOOLCHAIN_EXTERNAL_CC) $(TOOLCHAIN_EXTERNAL_CFLAGS))" ; \
+	SUPPORT_LIB_DIR="" ; \
+	if test `find $${ARCH_SYSROOT_DIR} -name 'libstdc++.a' | wc -l` -eq 0 ; then \
+		LIBSTDCPP_A_LOCATION=$$(LANG=C $(TOOLCHAIN_EXTERNAL_CC) $(TOOLCHAIN_EXTERNAL_CFLAGS) -print-file-name=libstdc++.a) ; \

Quoted 8 lines

toolchain/toolchain-external/toolchain-external.mk define TOOLCHAIN_EXTERNAL_INSTALL_CORE · line 611

Quoted 5 lines

toolchain/toolchain-external/toolchain-external.mk +5 -0

+		if [ -e "$${LIBSTDCPP_A_LOCATION}" ]; then \
+			SUPPORT_LIB_DIR=`readlink -f $${LIBSTDCPP_A_LOCATION} | sed -r -e 's:libstdc\+\+\.a::'` ; \
+		fi ; \
+	fi ; \
+	ARCH_SUBDIR=`echo $${ARCH_SYSROOT_DIR} | sed -r -e "s:^$${SYSROOT_DIR}(.*)/$$:\1:"` ; \

... to here could be moved to a common function, so it can be shared between the staging and target functions?

How do you suggest this to be done? The problem is that we need those variables to be defined within the shell block that follows. I don't see any easy way to factorize that. Or maybe I should just take this opportunity, and move some of this crap into a helper shell script, which will avoid these horrible shell blocks with lots of quoting and backslashes.

This patch adds a global instrumentation hook that collects the list of files installed in $(TARGET_DIR) by each package, and stores this list into a file called $(BUILD_DIR)/<pkgname>.filelist. It can later be used to determine the size contribution of each package to the target root filesystem.

The only limitation is that if a file is installed by a package A, and then overriden by a file from package B, the file will only be listed in $(BUILD_DIR)/A.filelist as it is the first time we will see the file.

Signed-off-by Thomas Petazzoni <[email protected]>

---
 package/pkg-generic.mk | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

package/pkg-generic.mk +24 -0

index 5116ed9..069653e 100644
@@ -55,6 +55,30 @@ define step_time
 endef
 GLOBAL_INSTRUMENTATION_HOOKS += step_time
 
+# Package size steps
+define step_pkg_size_start
+	echo "PKG SIZE START $(1)"
+	(cd $(TARGET_DIR) ; find . -type f) | sort > \
+		$(BUILD_DIR)/$(1).tmp_filelist_before
+endef
+
+define step_pkg_size_end
+	echo "PKG SIZE END $(1)"
+	(cd $(TARGET_DIR); find . -type f) | sort > \
+		$(BUILD_DIR)/$(1).tmp_filelist_after
+	diff -u $(BUILD_DIR)/$(1).tmp_filelist_before $(BUILD_DIR)/$(1).tmp_filelist_after | \
+		grep '^\+\./' | sed 's%^\+%%' > $(BUILD_DIR)/$(1).filelist
+	$(RM) -f $(BUILD_DIR)/$(1).tmp_filelist_before \
+		$(BUILD_DIR)/$(1).tmp_filelist_after
+endef
+
+define step_pkg_size
+	$(if $(filter install-target,$(2)),\
+		$(if $(filter start,$(1)),$(call step_pkg_size_start,$(3))) \
+		$(if $(filter end,$(1)),$(call step_pkg_size_end,$(3))))
+endef
+GLOBAL_INSTRUMENTATION_HOOKS += step_pkg_size
+
 # User-supplied script
 define step_user
 	@$(foreach user_hook, $(BR2_INSTRUMENTATION_SCRIPTS), \

Quoted 5 lines · Thomas Petazzoni

This patch adds a global instrumentation hook that collects the list of files installed in $(TARGET_DIR) by each package, and stores this list into a file called $(BUILD_DIR)/<pkgname>.filelist. It can later be used to determine the size contribution of each package to the target root filesystem.

How does this play with parallel build? Is install-target guaranteed to run sequentially for each package?

This patch adds a global instrumentation hook that collects the list of files installed in $(TARGET_DIR) by each package, and stores this list into a file called $(BUILD_DIR)/<pkgname>.filelist. It can later be used to determine the size contribution of each package to the target root filesystem.

The only limitation is that if a file is installed by a package A, and then overriden by a file from package B, the file will only be listed in $(BUILD_DIR)/A.filelist as it is the first time we will see the file.

Signed-off-by Thomas Petazzoni <[email protected]>

---
 package/pkg-generic.mk | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

package/pkg-generic.mk +0 -0

index 5116ed9..069653e 100644

package/pkg-generic.mk +19 -0

+endef
+
+define step_pkg_size_end
+	echo "PKG SIZE END $(1)"
+	(cd $(TARGET_DIR); find . -type f) | sort > \
+		$(BUILD_DIR)/$(1).tmp_filelist_after
+	diff -u $(BUILD_DIR)/$(1).tmp_filelist_before $(BUILD_DIR)/$(1).tmp_filelist_after | \
+		grep '^\+\./' | sed 's%^\+%%' > $(BUILD_DIR)/$(1).filelist
+	$(RM) -f $(BUILD_DIR)/$(1).tmp_filelist_before \
+		$(BUILD_DIR)/$(1).tmp_filelist_after
+endef
+
+define step_pkg_size
+	$(if $(filter install-target,$(2)),\
+		$(if $(filter start,$(1)),$(call step_pkg_size_start,$(3))) \
+		$(if $(filter end,$(1)),$(call step_pkg_size_end,$(3))))
+endef
+GLOBAL_INSTRUMENTATION_HOOKS += step_pkg_size
+
 # User-supplied script
 define step_user
 	@$(foreach user_hook, $(BR2_INSTRUMENTATION_SCRIPTS), \

Signature · 6 lines

2.0.0

_______________________________________________
buildroot mailing list
buildroot at busybox.net
http://lists.busybox.net/mailman/listinfo/buildroot

Thomas, All,

On 2014-06-07 23:46 +0200, Thomas Petazzoni spake thusly:

Quoted 10 lines

This patch adds a global instrumentation hook that collects the list of files installed in $(TARGET_DIR) by each package, and stores this list into a file called $(BUILD_DIR)/<pkgname>.filelist. It can later be used to determine the size contribution of each package to the target root filesystem.

The only limitation is that if a file is installed by a package A, and then overriden by a file from package B, the file will only be listed in $(BUILD_DIR)/A.filelist as it is the first time we will see the file.

If we really wanted to account for the realy package, we'd have to somehow notice that a pacakge did change the content of a file.

So, we would need to run sha1sum on all the files in the pre-step and the post step. Any differing line would mean a new file, or a changed file.

Quoted 14 lines

Thomas, All,

On 2014-06-07 23:46 +0200, Thomas Petazzoni spake thusly:

Quoted 10 lines

This patch adds a global instrumentation hook that collects the list of files installed in $(TARGET_DIR) by each package, and stores this list into a file called $(BUILD_DIR)/<pkgname>.filelist. It can later be used to determine the size contribution of each package to the target root filesystem.

The only limitation is that if a file is installed by a package A, and then overriden by a file from package B, the file will only be listed in $(BUILD_DIR)/A.filelist as it is the first time we will see the file.

Quoted 6 lines

If we really wanted to account for the realy package, we'd have to somehow notice that a pacakge did change the content of a file.

So, we would need to run sha1sum on all the files in the pre-step and the post step. Any differing line would mean a new file, or a changed file.

I did something similar in the past. I used inotify to follow modification done on TARGET_DIR. It was fast and detect overwritten files.

Handling build interruption was a little tricky, but results was corrects.

J?r?me, All,

On 2014-06-10 18:20 +0200, J?r?me Pouiller spake thusly:

Quoted 18 lines · On Tuesday 10 June 2014 00:02:41 Yann E. MORIN wrote:

Quoted 17 lines

Thomas, All,

On 2014-06-07 23:46 +0200, Thomas Petazzoni spake thusly:

Quoted 10 lines

This patch adds a global instrumentation hook that collects the list of files installed in $(TARGET_DIR) by each package, and stores this list into a file called $(BUILD_DIR)/<pkgname>.filelist. It can later be used to determine the size contribution of each package to the target root filesystem.

The only limitation is that if a file is installed by a package A, and then overriden by a file from package B, the file will only be listed in $(BUILD_DIR)/A.filelist as it is the first time we will see the file.

If we really wanted to account for the realy package, we'd have to somehow notice that a pacakge did change the content of a file.

Quoted 6 lines

Quoted 3 lines

So, we would need to run sha1sum on all the files in the pre-step and the post step. Any differing line would mean a new file, or a changed file.

I did something similar in the past. I used inotify to follow modification done on TARGET_DIR. It was fast and detect overwritten files.

I've been playing with inotify too, and it is not really up to the task. I was able to overload the "wait queue" very easily, and missed some events (sometimes a lot of them), when the build was creating/changing a lot of files in rapid-fire.

I was able to overcome the "wait-queue" limitation by increasing the number of queueable events, but that requires root access, but not everyone can be root on their machine (think shared build farms), and the defaults are quite low.

Quoted 1 line

Handling build interruption was a little tricky, but results was corrects.

That's also an issue I see, and I don't think we want to go into too complex a setting.

This patch adds a global instrumentation hook that collects the list of files installed in $(TARGET_DIR) by each package, and stores this list into a file called $(BUILD_DIR)/<pkgname>.filelist. It can later be used to determine the size contribution of each package to the target root filesystem.

The only limitation is that if a file is installed by a package A, and then overriden by a file from package B, the file will only be listed in $(BUILD_DIR)/A.filelist as it is the first time we will see the file.

Signed-off-by Thomas Petazzoni <[email protected]>

---
 package/pkg-generic.mk | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

package/pkg-generic.mk +13 -0

index 5116ed9..069653e 100644
@@ -55,6 +55,30 @@ define step_time
 endef
 GLOBAL_INSTRUMENTATION_HOOKS += step_time
 
+# Package size steps
+define step_pkg_size_start
+	echo "PKG SIZE START $(1)"
+	(cd $(TARGET_DIR) ; find . -type f) | sort > \
+		$(BUILD_DIR)/$(1).tmp_filelist_before
+endef
+
+define step_pkg_size_end
+	echo "PKG SIZE END $(1)"
+	(cd $(TARGET_DIR); find . -type f) | sort > \
+		$(BUILD_DIR)/$(1).tmp_filelist_after
+	diff -u $(BUILD_DIR)/$(1).tmp_filelist_before $(BUILD_DIR)/$(1).tmp_filelist_after | \
+		grep '^\+\./' | sed 's%^\+%%' > $(BUILD_DIR)/$(1).filelist

package/pkg-generic.mk +1 -0

+
 # User-supplied script
 define step_user
 	@$(foreach user_hook, $(BR2_INSTRUMENTATION_SCRIPTS), \

Quoted 19 lines

package/pkg-generic.mk +19 -0

+# Package size steps
+define step_pkg_size_start
+	echo "PKG SIZE START $(1)"
+	(cd $(TARGET_DIR) ; find . -type f) | sort > \
+		$(BUILD_DIR)/$(1).tmp_filelist_before
+endef
+
+define step_pkg_size_end
+	echo "PKG SIZE END $(1)"
+	(cd $(TARGET_DIR); find . -type f) | sort > \
+		$(BUILD_DIR)/$(1).tmp_filelist_after
+	diff -u $(BUILD_DIR)/$(1).tmp_filelist_before $(BUILD_DIR)/$(1).tmp_filelist_after | \
+		grep '^\+\./' | sed 's%^\+%%' > $(BUILD_DIR)/$(1).filelist
+	$(RM) -f $(BUILD_DIR)/$(1).tmp_filelist_before \
+		$(BUILD_DIR)/$(1).tmp_filelist_after
+endef
+
+define step_pkg_size
+	$(if $(filter install-target,$(2)),\

package/pkg-generic.mk define step_time · line 80

Quoted 4 lines

package/pkg-generic.mk +4 -0

+		$(if $(filter start,$(1)),$(call step_pkg_size_start,$(3))) \
+		$(if $(filter end,$(1)),$(call step_pkg_size_end,$(3))))
+endef
+GLOBAL_INSTRUMENTATION_HOOKS += step_pkg_size

Since these instrumentation steps are relatively expensive (especially for
large builds with many small packages), I would prefer to only enable this after setting some environment variable or config option or similar.

Quoted 43 lines

This patch adds a global instrumentation hook that collects the list of files installed in $(TARGET_DIR) by each package, and stores this list into a file called $(BUILD_DIR)/<pkgname>.filelist. It can later be used to determine the size contribution of each package to the target root filesystem.

The only limitation is that if a file is installed by a package A, and then overriden by a file from package B, the file will only be listed in $(BUILD_DIR)/A.filelist as it is the first time we will see the file.

Signed-off-by Thomas Petazzoni <[email protected]>

---
 package/pkg-generic.mk | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

package/pkg-generic.mk +19 -0

index 5116ed9..069653e 100644
@@ -55,6 +55,30 @@ define step_time
 endef
 GLOBAL_INSTRUMENTATION_HOOKS += step_time
 
+# Package size steps
+define step_pkg_size_start
+	echo "PKG SIZE START $(1)"
+	(cd $(TARGET_DIR) ; find . -type f) | sort > \
+		$(BUILD_DIR)/$(1).tmp_filelist_before
+endef
+
+define step_pkg_size_end
+	echo "PKG SIZE END $(1)"
+	(cd $(TARGET_DIR); find . -type f) | sort > \
+		$(BUILD_DIR)/$(1).tmp_filelist_after
+	diff -u $(BUILD_DIR)/$(1).tmp_filelist_before $(BUILD_DIR)/$(1).tmp_filelist_after | \
+		grep '^\+\./' | sed 's%^\+%%' > $(BUILD_DIR)/$(1).filelist
+	$(RM) -f $(BUILD_DIR)/$(1).tmp_filelist_before \
+		$(BUILD_DIR)/$(1).tmp_filelist_after
+endef
+
+define step_pkg_size
+	$(if $(filter install-target,$(2)),\

package/pkg-generic.mk define step_time · line 80

Quoted 4 lines

package/pkg-generic.mk +4 -0

+		$(if $(filter start,$(1)),$(call step_pkg_size_start,$(3))) \
+		$(if $(filter end,$(1)),$(call step_pkg_size_end,$(3))))
+endef
+GLOBAL_INSTRUMENTATION_HOOKS += step_pkg_size

Since these instrumentation steps are relatively expensive (especially for
large builds with many small packages), I would prefer to only enable this after setting some environment variable or config option or similar.

This new script uses the data collected by the step_pkg_size instrumentation hook to generate a pie chart of the size contribution of each package to the target root filesystem. To achieve this, it looks at each file in $(TARGET_DIR), and using the <pkgname>.filelist information collected by the step_pkg_size hook, it determines to which package the file belongs. It is therefore able to give the size installed by each package.

Signed-off-by Thomas Petazzoni <[email protected]>

---
 support/scripts/graph-size | 164 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 164 insertions(+)
 create mode 100755 support/scripts/graph-size

support/scripts/graph-size new file +164 -0

new file mode 100755
index 0000000..5f7fe58
@@ -0,0 +1,164 @@
+#!/usr/bin/env python
+
+# Copyright (C) 2014 by Thomas Petazzoni <[email protected]>
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+# This script draws a pie chart of the size used by each package in
+# the target filesystem.
+
+import sys
+import os.path
+import argparse
+import matplotlib.font_manager as fm
+import matplotlib.pyplot as plt
+
+colors = ['#e60004', '#009836', '#2e1d86', '#ffed00',
+          '#0068b5', '#f28e00', '#940084', '#97c000']
+
+#
+# This function parses one .filelist file (which lists the files
+# installed by a particular package), and returns a Python list of the
+# file paths installed by the package.
+#
+# pkgf: path to the .filelist file
+#
+def handle_pkg(pkgf):
+    files = []
+    with open(pkgf) as f:
+        for l in f.readlines():
+            files.append(l.strip().replace("./", ""))
+    return files
+
+#
+# This function returns the list of files present in the skeleton,
+# with the exception of the .empty files. It is used to create a fake
+# entry in the dictionary of files installed by each package, emulated
+# the presence of a skeleton package.
+#
+def handle_skeleton():
+    skeleton_files = []
+    for root, _, files in os.walk("system/skeleton"):
+        for f in files:
+            if f == ".empty":
+                continue
+            frelpath = os.path.relpath(os.path.join(root, f), "system/skeleton")
+            skeleton_files.append(frelpath)
+    return skeleton_files
+
+#
+# This function returns a dict where each key is the name of a
+# package, and the value is a list of the files installed by this
+# package.
+#
+# builddir: path to the Buildroot output directory
+#
+def build_package_dict(builddir):
+    pkgdict = {}
+    # Parse all the .filelist files generated by the package
+    # installation process
+    filelist = os.listdir(os.path.join(builddir, "build"))
+    for file in filelist:
+        (_, ext) = os.path.splitext(file)
+        if ext != ".filelist":
+            continue
+        pkgname = file.replace(".filelist", "")
+        pkgdict[pkgname] = handle_pkg(os.path.join(builddir, "build", file))
+    # Add a special fake entry for the files installed by the skeleton
+    pkgdict['skeleton'] = handle_skeleton()
+    return pkgdict
+
+#
+# This function looks into the 'pkgdict' dictionary (as generated by
+# build_package_dict) to see which package has installed the file 'f',
+# and returns the name of that package.
+#
+def find_file(pkgdict, f):
+    for (p, flist) in pkgdict.iteritems():
+        if f in flist:
+            return p
+    return None
+
+#
+# This function build a dictionary that contains the name of a package
+# as key, and the size of the files installed by this package as the
+# value.
+#
+def build_package_size(builddir):
+    pkgdict = build_package_dict(builddir)
+    pkgsize = {}
+
+    for root, _, files in os.walk(os.path.join(builddir, "target")):
+        for f in files:
+            fpath = os.path.join(root, f)
+            if os.path.islink(fpath):
+                continue
+            frelpath = os.path.relpath(fpath, os.path.join(builddir, "target"))
+            pkg = find_file(pkgdict, frelpath)
+            if pkg is None:
+                print "WARNING: %s is not part of any package" % frelpath
+                pkg = "unknown"
+            if pkg in pkgsize:
+                pkgsize[pkg] += os.path.getsize(fpath)
+            else:
+                pkgsize[pkg] = os.path.getsize(fpath)
+
+    return pkgsize
+
+#
+# Given a dict returned by build_package_size(), this function
+# generates a pie chart of the size installed by each package.
+#
+def draw_graph(pkgsize, outputf):
+    total = 0
+    for (p, sz) in pkgsize.iteritems():
+        total += sz
+    labels = []
+    values = []
+    other_value = 0
+    for (p, sz) in pkgsize.iteritems():
+        if sz < (total * 0.01):
+            other_value += sz
+        else:
+            labels.append(p)
+            values.append(sz)
+    labels.append("Other")
+    values.append(other_value)
+
+    plt.figure()
+    patches, texts, autotexts = plt.pie(values, labels=labels,
+                                        autopct='%1.1f%%', shadow=True,
+                                        colors=colors)
+    # Reduce text size
+    proptease = fm.FontProperties()
+    proptease.set_size('xx-small')
+    plt.setp(autotexts, fontproperties=proptease)
+    plt.setp(texts, fontproperties=proptease)
+
+    plt.title('Size per package')
+    plt.savefig(outputf)
+
+parser = argparse.ArgumentParser(description='Draw build time graphs')
+
+parser.add_argument("--builddir", '-i', metavar="BUILDDIR",
+                    help="Buildroot output directory")
+parser.add_argument("--output", '-o', metavar="OUTPUT", required=True,
+                    help="Output file (.pdf or .png extension)")
+args = parser.parse_args()
+
+
+ps = build_package_size(args.builddir)
+draw_graph(ps, args.output)
+

Signature · 6 lines

2.0.0

_______________________________________________
buildroot mailing list
buildroot at busybox.net
http://lists.busybox.net/mailman/listinfo/buildroot

Like we have graph-build and graph-depends, this commit implements a graph-size target to generate the corresponding graph.

Signed-off-by Thomas Petazzoni <[email protected]>

---
 Makefile | 6 ++++++
 1 file changed, 6 insertions(+)

Makefile +6 -0

index 0b4264a..fee6b46 100644
@@ -677,6 +677,11 @@ graph-depends:
 	|tee $(O)/graphs/$(@).dot \
 	|dot -T$(BR_GRAPH_OUT) -o $(O)/graphs/$(@).$(BR_GRAPH_OUT)
 
+graph-size:
+	@$(INSTALL) -d $(O)/graphs
+	@cd "$(TOPDIR)"; \
+	./support/scripts/graph-size --builddir $(O) --output $(O)/graphs/$(@).$(BR_GRAPH_OUT)
+
 else # ifeq ($(BR2_HAVE_DOT_CONFIG),y)
 
 all: menuconfig

@@ -892,6 +897,7 @@ endif
 	@echo '  manual-epub            - build manual in ePub'
 	@echo '  graph-build            - generate graphs of the build times'
 	@echo '  graph-depends          - generate graph of the dependency tree'
+	@echo '  graph-size             - generate graph of the filesystem size'
 	@echo
 	@echo 'Miscellaneous:'
 	@echo '  source                 - download all sources needed for offline-build'

Signature · 6 lines

2.0.0

_______________________________________________
buildroot mailing list
buildroot at busybox.net
http://lists.busybox.net/mailman/listinfo/buildroot

Hello,

I gave a training this week, and one of the question I had was how to analyze the size of the things that are present on the root filesystem. And I thought that Buildroot was lacking a tool to help with this. Therefore, the following set of commits implement a script that generates a pie chart of the size contribution of each package to the target root filesystem.

To see an example of the generated pie chart, see:

http://free-electrons.com/~thomas/pub/buildroot/graph-size.pdf

The implementation consists in adding a global instrumentation hook that registers which files are installed by each package. A limitation of the current implementation is that when a file is installed by a package A and then overriden by package B, the mechanism will assume the file was installed by package A. Suggestions to welcome on how to solve this in a reasonably simple way.

Quoted 6 lines

The implementation consists in adding a global instrumentation hook that registers which files are installed by each package. A limitation of the current implementation is that when a file is installed by a package A and then overriden by package B, the mechanism will assume the file was installed by package A. Suggestions to welcome on how to solve this in a reasonably simple way.

Can you not record the timestamp the first time you see the file and see if it has changed at the end. I appreciate a bash script to do this is going to be pretty hard.

Hello,

I gave a training this week, and one of the question I had was how to analyze the size of the things that are present on the root filesystem. And I thought that Buildroot was lacking a tool to help with this. Therefore, the following set of commits implement a script that generates a pie chart of the size contribution of each package to the target root filesystem.

To see an example of the generated pie chart, see:

http://free-electrons.com/~thomas/pub/buildroot/graph-size.pdf

The implementation consists in adding a global instrumentation hook that registers which files are installed by each package. A limitation of the current implementation is that when a file is installed by a package A and then overriden by package B, the mechanism will assume the file was installed by package A. Suggestions to welcome on how to solve this in a reasonably simple way.

Thomas Petazzoni wrote:

Quoted 10 lines

To see an example of the generated pie chart, see:

http://free-electrons.com/~thomas/pub/buildroot/graph-size.pdf

The implementation consists in adding a global instrumentation hook that registers which files are installed by each package. A limitation of the current implementation is that when a file is installed by a package A and then overriden by package B, the mechanism will assume the file was installed by package A. Suggestions to welcome on how to solve this in a reasonably simple way.

It's very well possible that I'm missing something, but I don't get why you need to save the list of all installed files.

Can't you just save the whole rootfs size before and after installation? It can be simply computer by 'du -bs $(TARGET_DIR)', and it's way easier to parse later. It would also take into account the change of size for overwritten files, for free.

Of course your approach collects more information, but I don't see these extra info used in the final graph.

Quoted 6 lines

The implementation consists in adding a global instrumentation hook that registers which files are installed by each package. A limitation of the current implementation is that when a file is installed by a package A and then overriden by package B, the mechanism will assume the file was installed by package A. Suggestions to welcome on how to solve this in a reasonably simple way.

It's very well possible that I'm missing something, but I don't get why you need to save the list of all installed files.

Can't you just save the whole rootfs size before and after installation? It can be simply computer by 'du -bs $(TARGET_DIR)', and it's way easier to parse later. It would also take into account the change of size for overwritten files, for free.

Of course your approach collects more information, but I don't see these extra info used in the final graph.

Quoted 2 lines

Of course your approach collects more information, but I don't see these extra info used in the final graph.

The reason is explained a bit further in that cover letter: we only do
stripping and removing of redundant stuff in the finalize step, so calculating the size before that doesn't make much sense.

The alternative of repeating the finalize step after each package doesn't sound
very attractive either.

On 24/06/14 15:05, Luca Ceresoli wrote:
[snip]

Quoted 15 lines

Quoted 6 lines

The implementation consists in adding a global instrumentation hook that registers which files are installed by each package. A limitation of the current implementation is that when a file is installed by a package A and then overriden by package B, the mechanism will assume the file was installed by package A. Suggestions to welcome on how to solve this in a reasonably simple way.

It's very well possible that I'm missing something, but I don't get why you need to save the list of all installed files.

Can't you just save the whole rootfs size before and after installation? It can be simply computer by 'du -bs $(TARGET_DIR)', and it's way easier to parse later. It would also take into account the change of size for overwritten files, for free.

Quoted 2 lines

Of course your approach collects more information, but I don't see these extra info used in the final graph.

The reason is explained a bit further in that cover letter: we only do
stripping and removing of redundant stuff in the finalize step, so calculating the size before that doesn't make much sense.

Quoted 3 lines

The reason is explained a bit further in that cover letter: we only do
stripping and removing of redundant stuff in the finalize step, so calculating the size before that doesn't make much sense.

Aaah, yes, stripping! Of course! I was sure there is a good reason...