Speedup LTP compile time by ~20% on multi-machine
v2 cover Speedup LTP compile time by ~20% on multi-machine
The following results show how this optimization moves compile times really far from what we have now. With a single threaded build, LTP would take approximately 12 minutes to compile. With a multi-threaded build on 16 CPUs, this drops down to ~46 seconds.
time make -j16
Executed in 45.96 secs fish external usr time 519.11 secs 0.24 millis 519.11 secs sys time 167.13 secs 1.08 millis 167.13 secs
---
Changes in v2:
- fix build race condition
- Link to v1: https://lore.kernel.org/[email protected]
---
Andrea Cervesato (3):
build: Fix library prerequisites in testcases.mk
build: Build shared libs once on subtree make
build: Parallelize subdirectory recursion in generic_trunk_target
Makefile | 2 ++ include/mk/generic_trunk_target.inc | 58 ++++++++++++++++++++++++++++++++----- include/mk/testcases.mk | 14 +++++---- testcases/kernel/Makefile | 3 ++ testcases/kernel/include/lib.mk | 4 +-- testcases/network/rpc/Makefile | 3 ++ 6 files changed, 69 insertions(+), 15 deletions(-)
---
base-commit: 121876b3da3ff59b0dd17b4798b2e4699ba431d1
change-id: 20261020-build_system_optimize-6cd0343f08e1
Best regards,
Signature · 5 lines
Andrea Cervesato <[email protected]> -- Mailing list info: https://lists.linux.it/listinfo/ltp
PATCH v2 1/3 build: Fix library prerequisites in testcases.mk
From: Andrea Cervesato <[email protected]>
Fix LTPLIBS_FILES to match the actual libltp<name>.a archive names and only mark them .PHONY when MAKELEVEL == 0 so recursive leaf sub-makes do not redundantly rebuild them in parallel.
Also make directory prerequisites order-only so directory mtime updates do not trigger unnecessary archive rebuilds.
--- include/mk/testcases.mk | 14 ++++++++------ testcases/kernel/include/lib.mk | 4 ++-- 2 files changed, 10 insertions(+), 8 deletions(-)
include/mk/testcases.mk +8 -6
@@ -12,11 +12,11 @@ LIBLTP_DIR := $(abs_top_builddir)/lib LIBLTP := $(LIBLTP_DIR)/libltp.a -$(APICMDS_DIR)/tst_kvercmp: $(APICMDS_DIR) - $(MAKE) -C "$^" -f "$(abs_top_srcdir)/tools/apicmds/Makefile" all +$(APICMDS_DIR)/tst_kvercmp: | $(APICMDS_DIR) + $(MAKE) -C "$(APICMDS_DIR)" -f "$(abs_top_srcdir)/tools/apicmds/Makefile" all -$(LIBLTP): $(LIBLTP_DIR) - $(MAKE) -C "$^" -f "$(abs_top_srcdir)/lib/Makefile" all +$(LIBLTP): | $(LIBLTP_DIR) + $(MAKE) -C "$(LIBLTP_DIR)" -f "$(abs_top_srcdir)/lib/Makefile" all MAKE_DEPS := $(LIBLTP)
@@ -27,13 +27,15 @@ LDLIBS += -lltp ifdef LTPLIBS LTPLIBS_DIRS = $(addprefix $(abs_top_builddir)/libs/, $(LTPLIBS)) -LTPLIBS_FILES = $(addsuffix .a, $(addprefix $(abs_top_builddir)/libs/, $(foreach LIB,$(LTPLIBS),$(LIB)/lib$(LIB)))) +LTPLIBS_FILES = $(addsuffix .a, $(addprefix $(abs_top_builddir)/libs/, $(foreach LIB,$(LTPLIBS),$(LIB)/libltp$(LIB)))) MAKE_DEPS += $(LTPLIBS_FILES) +ifeq ($(MAKELEVEL),0) .PHONY: $(LTPLIBS_FILES) +endif -$(LTPLIBS_FILES): $(LTPLIBS_DIRS) +$(LTPLIBS_FILES): | $(LTPLIBS_DIRS) $(LTPLIBS_FILES): %: ifdef VERBOSE
testcases/kernel/include/lib.mk +2 -2
@@ -29,8 +29,8 @@ LDFLAGS += -L$(LIBKERNTEST_DIR) $(LIBKERNTEST_DIR): mkdir -p "$@" -$(LIBKERNTEST): $(LIBKERNTEST_DIR) - $(MAKE) -C $^ -f "$(LIBKERNTEST_SRCDIR)/Makefile" all +$(LIBKERNTEST): | $(LIBKERNTEST_DIR) + $(MAKE) -C "$(LIBKERNTEST_DIR)" -f "$(LIBKERNTEST_SRCDIR)/Makefile" all MAKE_DEPS += $(LIBKERNTEST)
Signature · 5 lines
2.51.0 -- Mailing list info: https://lists.linux.it/listinfo/ltp
Re: [LTP] build: Fix library prerequisites in testcases.mk
Hi Andrea,
Quoted 1 line · Andrea Cervesato
[PATCH 1/3] build: Fix library prerequisites in testcases.mk
--- [PATCH 2/3] ---
include/mk/generic_trunk_target.inc
Quoted 2 lines
include/mk/generic_trunk_target.inc +1
-0+MAKE_DEPS += $(abs_top_builddir)/lib/libltp.a $(abs_top_builddir)/libs-all
Medium: testcases/lib already has a per-library ujson prerequisite, so when ujson needs rebuilding, make -j4 -C testcases/lib tst_run_shell launches that builder alongside libs-all's ujson builder. Both write the same objects and libujson.a concurrently, allowing a corrupt archive or a failed link. Serialize or deduplicate these prerequisite paths so each library has only one builder.
Verdict - Needs revision
---
Note:
The agent can sometimes produce false positives although often its findings are genuine. If you find issues with the review, please comment this email or ignore the suggestions.
Regards,
LTP AI Reviewer
Signature · 1 line
Mailing list info: https://lists.linux.it/listinfo/ltp
Re: [LTP] build: Fix library prerequisites in testcases.mk
Quoted 7 lines
Hi Andrea,
Quoted 1 line · Andrea Cervesato
[PATCH 1/3] build: Fix library prerequisites in testcases.mk
--- [PATCH 2/3] ---
Quoted 9 lines
include/mk/generic_trunk_target.inc
Quoted 2 lines
include/mk/generic_trunk_target.inc +1
-0+MAKE_DEPS += $(abs_top_builddir)/lib/libltp.a $(abs_top_builddir)/libs-allMedium: testcases/lib already has a per-library ujson prerequisite, so when ujson needs rebuilding, make -j4 -C testcases/lib
tst_run_shelllaunches that builder alongside libs-all's ujson builder. Both write the same objects and libujson.a concurrently, allowing a corrupt archive or a failed link. Serialize or deduplicate these prerequisite paths so each library has only one builder.
Valid, I will fix it
--
Andrea Cervesato
SUSE QE Automation Engineer Linux
[email protected]
Signature · 1 line
Mailing list info: https://lists.linux.it/listinfo/ltp
PATCH v2 2/3 build: Build shared libs once on subtree make
In reply to Andrea CervesatoFrom: Andrea Cervesato <[email protected]>
When make is invoked directly in a trunk subdirectory (MAKELEVEL == 0), build shared prerequisites (libltp.a, libs, and libkerntest.a) once via MAKE_DEPS before launching parallel subdirectory builds.
--- include/mk/generic_trunk_target.inc | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)
include/mk/generic_trunk_target.inc +23 -0
@@ -36,6 +36,29 @@ RECURSIVE_TARGETS ?= all install check $(eval $(get_make_dirs)) +ifeq ($(MAKELEVEL),0) +ifneq ($(filter $(abs_top_builddir)/testcases%,$(CURDIR)),) +MAKE_DEPS += $(abs_top_builddir)/lib/libltp.a $(abs_top_builddir)/libs-all + +$(abs_top_builddir)/lib/libltp.a: + mkdir -m 00755 -p "$(abs_top_builddir)/lib" + $(MAKE) -C "$(abs_top_builddir)/lib" -f "$(abs_top_srcdir)/lib/Makefile" all + +.PHONY: $(abs_top_builddir)/libs-all +$(abs_top_builddir)/libs-all: + mkdir -m 00755 -p "$(abs_top_builddir)/libs" + $(MAKE) -C "$(abs_top_builddir)/libs" -f "$(abs_top_srcdir)/libs/Makefile" all +endif + +ifneq ($(filter $(abs_top_builddir)/testcases/kernel/%,$(CURDIR)),) +MAKE_DEPS += $(abs_top_builddir)/testcases/kernel/lib/libkerntest.a + +$(abs_top_builddir)/testcases/kernel/lib/libkerntest.a: + mkdir -m 00755 -p "$(abs_top_builddir)/testcases/kernel/lib" + $(MAKE) -C "$(abs_top_builddir)/testcases/kernel/lib" -f "$(abs_top_srcdir)/testcases/kernel/lib/Makefile" all +endif +endif + .PHONY: $(RECURSIVE_TARGETS) $(addprefix trunk-,$(RECURSIVE_TARGETS)) $(SUBDIRS): %:
Signature · 5 lines
2.51.0 -- Mailing list info: https://lists.linux.it/listinfo/ltp
PATCH v2 3/3 build: Parallelize subdirectory recursion in generic_trunk_target
In reply to Andrea CervesatoFrom: Andrea Cervesato <[email protected]>
Currently, generic_trunk_target traverses $(SUBDIRS) using a serial shell 'for' loop. This serializes directory traversal and prevents 'make -jN' from building sibling directories in parallel.
Replace the shell loop with target-based recursion:
1. Map each subdirectory to a phony target: <dir>-all, <dir>-install, and <dir>-check.
2. Make 'all', 'install', and 'check' depend on these targets so GNU Make can dispatch them concurrently across available job slots.
3. In each recipe, strip the suffix with $(patsubst %-<action>,%,$@) to extract the directory name and invoke $(MAKE) -C <dir>.
4. Add explicit ordering dependencies where subdirectories produce libraries required by sibling directories (tools before testcases, kernel/lib before syscalls and mem, rpc-tirpc before basic_tests).
This cuts full build time by over 20% on multi-core systems.
--- Makefile | 2 ++ include/mk/generic_trunk_target.inc | 35 ++++++++++++++++++++++++++++------- testcases/kernel/Makefile | 3 +++ testcases/network/rpc/Makefile | 3 +++ 4 files changed, 36 insertions(+), 7 deletions(-)
Makefile +2 -0
@@ -63,6 +63,8 @@ CHECK_TARGETS := $(addsuffix -check,testcases lib) all: $(addsuffix -all,$(COMMON_TARGETS)) Version $(MAKE_TARGETS): lib-all libs-all +# tools contains apicmds/tst_kvercmp required by some testcases +testcases-all: tools-all .PHONY: include-all include-install include-install: $(top_builddir)/include/config.h include/mk/config.mk include-all
include/mk/generic_trunk_target.inc +28 -7
@@ -106,15 +106,36 @@ ifeq ($(strip $(SUBDIRS)),) $(warning CURDIR is: $(CURDIR)) $(error SUBDIRS empty -- did you want generic_leaf_target instead?) else -$(RECURSIVE_TARGETS): %: | $(SUBDIRS) +# Recurse into subdirectories in parallel instead of using a serial shell loop. +# Each subdirectory is mapped to a phony target (<dir>-all, <dir>-install, etc.). +# GNU Make can then schedule sibling directories concurrently under -jN. +all: $(addsuffix -all,$(SUBDIRS)) +install: $(addsuffix -install,$(SUBDIRS)) +check: $(addsuffix -check,$(SUBDIRS)) + +.PHONY: $(addsuffix -all,$(SUBDIRS)) $(addsuffix -install,$(SUBDIRS)) $(addsuffix -check,$(SUBDIRS)) + +# For each target (e.g. 'open-all'), strip the suffix to get the directory name, +# build trunk-local targets first, and ensure the subdirectory exists. +$(addsuffix -all,$(SUBDIRS)): trunk-all | $(SUBDIRS) $(MAKE_DEPS) ifdef VERBOSE - @set -e; for dir in $(SUBDIRS); do \ - $(MAKE) -C $$dir -f "$(abs_srcdir)/$$dir/Makefile" $@; \ - done + $(MAKE) -C "$(patsubst %-all,%,$@)" -f "$(abs_srcdir)/$(patsubst %-all,%,$@)/Makefile" all else - @set -e; for dir in $(SUBDIRS); do \ - $(MAKE) --no-print-directory -C $$dir -f "$(abs_srcdir)/$$dir/Makefile" $@; \ - done + @$(MAKE) --no-print-directory -C "$(patsubst %-all,%,$@)" -f "$(abs_srcdir)/$(patsubst %-all,%,$@)/Makefile" all +endif + +$(addsuffix -install,$(SUBDIRS)): trunk-install | $(SUBDIRS) $(MAKE_DEPS) +ifdef VERBOSE + $(MAKE) -C "$(patsubst %-install,%,$@)" -f "$(abs_srcdir)/$(patsubst %-install,%,$@)/Makefile" install +else + @$(MAKE) --no-print-directory -C "$(patsubst %-install,%,$@)" -f "$(abs_srcdir)/$(patsubst %-install,%,$@)/Makefile" install +endif + +$(addsuffix -check,$(SUBDIRS)): trunk-check | $(SUBDIRS) $(MAKE_DEPS) +ifdef VERBOSE + $(MAKE) -C "$(patsubst %-check,%,$@)" -f "$(abs_srcdir)/$(patsubst %-check,%,$@)/Makefile" check +else + @$(MAKE) --no-print-directory -C "$(patsubst %-check,%,$@)" -f "$(abs_srcdir)/$(patsubst %-check,%,$@)/Makefile" check endif endif
testcases/kernel/Makefile +3 -0
@@ -54,3 +54,6 @@ FILTER_OUT_DIRS += containers controllers device-drivers fs io ipc mem \ endif include $(top_srcdir)/include/mk/generic_trunk_target.mk + +# libkerntest.a in lib/ must be built before syscalls and mem consumers +syscalls-all mem-all: lib-all
testcases/network/rpc/Makefile +3 -0
@@ -9,3 +9,6 @@ include $(top_srcdir)/include/mk/env_pre.mk INSTALL_TARGETS := *.sh include $(top_srcdir)/include/mk/generic_trunk_target.mk + +# librpc-tirpc.a in rpc-tirpc/ must be built before basic_tests +basic_tests-all: rpc-tirpc-all
Signature · 5 lines
2.51.0 -- Mailing list info: https://lists.linux.it/listinfo/ltp