automatically determine job concurrency

PATCH New Nathan Lynch 5 participants 18 messages Jun 12, 2012 – Jun 24, 2012

Share & tools

Nathan Lynch

author

2012-06-12 02:09 UTC #0 lore raw

cover automatically determine job concurrency

This series implements the option of "autodetecting" the number of jobs to run concurrently (make -jN), using the number of reported CPUs when BR2_JLEVEL is set to 0.

Nathan Lynch (7):
introduce PARALLEL_JOBS build system variable
boost: change uses of BR2_JLEVEL to PARALLEL_JOBS
cmake: change use of BR2_JLEVEL to PARALLEL_JOBS
midori: change use of BR2_JLEVEL to PARALLEL_JOBS
qt: change use of BR2_JLEVEL to PARALLEL_JOBS toolchain/crosstool-ng: change use of BR2_JLEVEL to PARALLEL_JOBS automatically set PARALLEL_JOBS when BR2_JLEVEL is 0

 Config.in                                        |    8 +++++---
 package/Makefile.in                              |    8 +++++++-
 package/boost/boost.mk                           |    4 ++--
 package/cmake/cmake.mk                           |    2 +-
 package/midori/midori.mk                         |    2 +-
 package/qt/qt.mk                                 |    2 +-
 toolchain/toolchain-crosstool-ng/crosstool-ng.mk |    2 +-
 7 files changed, 18 insertions(+), 10 deletions(-)

Signature · 1 line

1.7.6.5

Nathan Lynch

author

2012-06-12 02:09 UTC #1 lore raw

PATCH 1/7 introduce PARALLEL_JOBS build system variable

BR2_JLEVEL currently is expanded directly in $(MAKE), and used in invocations of other build software (e.g. ct-ng). However, we are going to allow "0" to be a meaningful value for BR2_JLEVEL, which won't work for these uses. Given that it is not permissible to modify BR2_-prefixed variables in Makefiles, we need an intermediate variable.

Define PARALLEL_JOBS to $(BR2_JLEVEL), and use the former in MAKE's definition. Uses of BR2_JLEVEL throughout the rest of the tree to be adjusted similarly in follow-up patches.

Signed-off-by Nathan Lynch <[email protected]>

---
 package/Makefile.in |    4 +++-
 1 files changed, 3 insertions(+), 1 deletions(-)

package/Makefile.in +3 -1

index df7042c..9177a1b 100644
@@ -6,8 +6,10 @@ HOSTMAKE=$(MAKE)
 endif
 HOSTMAKE :=$(shell which $(HOSTMAKE) || type -p $(HOSTMAKE) || echo make)
 
+PARALLEL_JOBS:=$(BR2_JLEVEL)
+
 MAKE1:=$(HOSTMAKE) -j1
-MAKE:=$(HOSTMAKE) -j$(BR2_JLEVEL)
+MAKE:=$(HOSTMAKE) -j$(PARALLEL_JOBS)
 
 # Compute GNU_TARGET_NAME and REAL_GNU_TARGET_NAME
 GNU_TARGET_NAME=$(ARCH)-linux

Signature · 1 line

1.7.6.5

Nathan Lynch

author

2012-06-12 02:09 UTC #2 lore raw

PATCH 2/7 boost: change uses of BR2_JLEVEL to PARALLEL_JOBS

In reply to Nathan Lynch

Signed-off-by Nathan Lynch <[email protected]>

---
 package/boost/boost.mk |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)

package/boost/boost.mk +2 -2

index 3667964..abdf3a2 100644
@@ -70,7 +70,7 @@ define BOOST_CONFIGURE_CMDS
 endef
 
 define BOOST_INSTALL_TARGET_CMDS
-	(cd $(@D) && ./b2 -j$(BR2_JLEVEL) -q -d+2 \
+	(cd $(@D) && ./b2 -j$(PARALLEL_JOBS) -q -d+2 \
 	--user-config=$(@D)/user-config.jam \
 	$(BOOST_OPT) \
 	--prefix=$(TARGET_DIR)/usr \

@@ -78,7 +78,7 @@ define BOOST_INSTALL_TARGET_CMDS
 endef
 
 define BOOST_INSTALL_STAGING_CMDS
-	(cd $(@D) && ./bjam -j$(BR2_JLEVEL) -d+2 \
+	(cd $(@D) && ./bjam -j$(PARALLEL_JOBS) -d+2 \
 	--user-config=$(@D)/user-config.jam \
 	$(BOOST_OPT) \
 	--prefix=$(STAGING_DIR)/usr \

Signature · 1 line

1.7.6.5

Nathan Lynch

author

2012-06-12 02:09 UTC #3 lore raw

PATCH 3/7 cmake: change use of BR2_JLEVEL to PARALLEL_JOBS

In reply to Nathan Lynch

Signed-off-by Nathan Lynch <[email protected]>

---
 package/cmake/cmake.mk |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

package/cmake/cmake.mk +1 -1

index af88eea..af6c53c 100644
@@ -5,7 +5,7 @@ define HOST_CMAKE_CONFIGURE_CMDS
  (cd $(@D); \
 	LDFLAGS="$(HOST_LDFLAGS)" \
 	CFLAGS="$(HOST_CFLAGS)" \
-	./bootstrap --prefix=$(HOST_DIR)/usr --parallel=$(BR2_JLEVEL) \
+	./bootstrap --prefix=$(HOST_DIR)/usr --parallel=$(PARALLEL_JOBS) \
  )
 endef

Signature · 1 line

1.7.6.5

Nathan Lynch

author

2012-06-12 02:09 UTC #4 lore raw

PATCH 4/7 midori: change use of BR2_JLEVEL to PARALLEL_JOBS

In reply to Nathan Lynch

Signed-off-by Nathan Lynch <[email protected]>

---
 package/midori/midori.mk |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

package/midori/midori.mk +1 -1

index 0b3439c..aa376a2 100644
@@ -34,7 +34,7 @@ define MIDORI_CONFIGURE_CMDS
 endef
 
 define MIDORI_BUILD_CMDS
-       (cd $(@D); ./waf build -j $(BR2_JLEVEL))
+       (cd $(@D); ./waf build -j $(PARALLEL_JOBS))
 endef
 
 define MIDORI_INSTALL_TARGET_CMDS

Signature · 1 line

1.7.6.5

Nathan Lynch

author

2012-06-12 02:09 UTC #5 lore raw

PATCH 5/7 qt: change use of BR2_JLEVEL to PARALLEL_JOBS

In reply to Nathan Lynch

Signed-off-by Nathan Lynch <[email protected]>

---
 package/qt/qt.mk |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

package/qt/qt.mk +1 -1

index b751696..ff9c1bf 100644
@@ -488,7 +488,7 @@ define QT_CONFIGURE_CMDS
 		PKG_CONFIG_SYSROOT_DIR="$(STAGING_DIR)" \
 		PKG_CONFIG="$(PKG_CONFIG_HOST_BINARY)" \
 		PKG_CONFIG_PATH="$(STAGING_DIR)/usr/lib/pkgconfig:$(PKG_CONFIG_PATH)" \
-		MAKEFLAGS="$(MAKEFLAGS) -j$(BR2_JLEVEL)" ./configure \
+		MAKEFLAGS="$(MAKEFLAGS) -j$(PARALLEL_JOBS)" ./configure \
 		$(if $(VERBOSE),-verbose,-silent) \
 		-force-pkg-config \
 		$(QT_CONFIGURE_OPTS) \

Signature · 1 line

1.7.6.5

Nathan Lynch

author

2012-06-12 02:09 UTC #6 lore raw

PATCH 6/7 toolchain/crosstool-ng: change use of BR2_JLEVEL to PARALLEL_JOBS

In reply to Nathan Lynch

Signed-off-by Nathan Lynch <[email protected]>

---
 toolchain/toolchain-crosstool-ng/crosstool-ng.mk |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

toolchain/toolchain-crosstool-ng/crosstool-ng.mk +1 -1

index 25d2a16..c3a8a01 100644
@@ -81,7 +81,7 @@ $(STAMP_DIR)/ct-ng-toolchain-installed: $(STAMP_DIR)/ct-ng-toolchain-built
 #       depending on the selected C library. Those deps are added later
 
 $(STAMP_DIR)/ct-ng-toolchain-built: $(CTNG_DIR)/.config
-	$(Q)$(call ctng,build.$(BR2_JLEVEL))
+	$(Q)$(call ctng,build.$(PARALLEL_JOBS))
 	$(Q)printf "\n"
 	$(Q)touch $@

Signature · 1 line

1.7.6.5

Nathan Lynch

author

2012-06-12 02:09 UTC #7 lore raw

PATCH 7/7 automatically set PARALLEL_JOBS when BR2_JLEVEL is 0

In reply to Nathan Lynch

When BR2_JLEVEL is 0, set PARALLEL_JOBS to the number of CPUs detected. This allows one to more or less fully utilize the host system without manually tuning the configuration.

Also make 0 the default value for BR2_JLEVEL.

Signed-off-by Nathan Lynch <[email protected]>

---
 Config.in           |    8 +++++---
 package/Makefile.in |    4 ++++
 2 files changed, 9 insertions(+), 3 deletions(-)

Config.in +5 -3

index 95c2e8c..3745255 100644
@@ -178,10 +178,12 @@ config BR2_DEBIAN_MIRROR
 endmenu
 
 config BR2_JLEVEL
-	int "Number of jobs to run simultaneously"
-	default "2"
+	int "Number of jobs to run simultaneously (0 for auto)"
+	default "0"
 	help
-	  Number of jobs to run simultaneously
+	  Number of jobs to run simultaneously.  If 0, determine
+	  automatically according to number of CPUs on the host
+	  system.
 
 config BR2_CCACHE
 	bool "Enable compiler cache"

package/Makefile.in +4 -0

index 9177a1b..a9d2771 100644
@@ -6,7 +6,11 @@ HOSTMAKE=$(MAKE)
 endif
 HOSTMAKE :=$(shell which $(HOSTMAKE) || type -p $(HOSTMAKE) || echo make)
 
+ifeq ($(BR2_JLEVEL),0)
+PARALLEL_JOBS:=$(shell getconf _NPROCESSORS_ONLN 2>/dev/null || echo 1)
+else
 PARALLEL_JOBS:=$(BR2_JLEVEL)
+endif
 
 MAKE1:=$(HOSTMAKE) -j1
 MAKE:=$(HOSTMAKE) -j$(PARALLEL_JOBS)

Signature · 1 line

1.7.6.5

Arnout Vandecappelle

2012-06-13 23:27 UTC #8 lore raw

PATCH 7/7 automatically set PARALLEL_JOBS when BR2_JLEVEL is 0

Quoted 3 lines · Nathan Lynch

When BR2_JLEVEL is 0, set PARALLEL_JOBS to the number of CPUs detected. This allows one to more or less fully utilize the host system without manually tuning the configuration.

In my experience, you need 2 times the number of CPUs to fully utilize
the processor because most compilations have long iowait latencies.

Otherwise, looks good.

Regards,
Arnout

Signature · 6 lines

Arnout Vandecappelle                               arnout at mind be
Senior Embedded Software Architect                 +32-16-286540
Essensium/Mind                                     http://www.mind.be
G.Geenslaan 9, 3001 Leuven, Belgium                BE 872 984 063 RPR Leuven
LinkedIn profile: http://www.linkedin.com/in/arnoutvandecappelle
GPG fingerprint:  7CB5 E4CC 6C2E EFD4 6E3D A754 F963 ECAB 2450 2F1F

Nathan Lynch

author

2012-06-13 23:44 UTC #9 lore raw

PATCH 7/7 automatically set PARALLEL_JOBS when BR2_JLEVEL is 0

Quoted 7 lines · Arnout Vandecappelle

Quoted 3 lines · Nathan Lynch
When BR2_JLEVEL is 0, set PARALLEL_JOBS to the number of CPUs detected. This allows one to more or less fully utilize the host system without manually tuning the configuration.
In my experience, you need 2 times the number of CPUs to fully utilize
the processor because most compilations have long iowait latencies.

Yeah. I'm happy to make this change, assuming others agree?

Thanks for having a look!

Peter Korsgaard

2012-06-14 06:07 UTC #10 lore raw

PATCH 7/7 automatically set PARALLEL_JOBS when BR2_JLEVEL is 0

Quoted 4 lines

Quoted 4 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
"Nathan" == Nathan Lynch <[email protected]> writes:
In my experience, you need 2 times the number of CPUs to fully utilize the processor because most compilations have long iowait latencies.

Nathan> Yeah. I'm happy to make this change, assuming others agree?

I was going to say something like that as well, so yes please.

Signature · 1 line

Bye, Peter Korsgaard

Nathan Lynch

author

2012-06-14 06:16 UTC #11 lore raw

PATCH 7/7 automatically set PARALLEL_JOBS when BR2_JLEVEL is 0

Quoted 8 lines · Peter Korsgaard

Quoted 4 lines
Quoted 4 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
"Nathan" == Nathan Lynch <[email protected]> writes:
In my experience, you need 2 times the number of CPUs to fully utilize the processor because most compilations have long iowait latencies.
Nathan> Yeah. I'm happy to make this change, assuming others agree?
I was going to say something like that as well, so yes please.

Certainly. Is an update of patch #7 acceptable or shall I re-send the series?

Peter Korsgaard

2012-06-14 08:56 UTC #12 lore raw

PATCH 7/7 automatically set PARALLEL_JOBS when BR2_JLEVEL is 0

Quoted 1 line

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
"Nathan" == Nathan Lynch <[email protected]> writes:

Quoted 5 lines · Peter Korsgaard

Quoted 5 lines
Quoted 4 lines
Quoted 4 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
"Nathan" == Nathan Lynch <[email protected]> writes:
In my experience, you need 2 times the number of CPUs to fully utilize the processor because most compilations have long iowait latencies.

Nathan> Yeah. I'm happy to make this change, assuming others agree?

Quoted 2 lines

Quoted 2 lines
I was going to say something like that as well, so yes please.

Nathan> Certainly. Is an update of patch #7 acceptable or shall I Nathan> re-send the series?

Just #7 (with a clearly marked [PATCHv2] in subject) is fine if that's the only change since last time.

Signature · 1 line

Bye, Peter Korsgaard

Richard Braun

2012-06-14 08:33 UTC #13 lore raw

PATCH 7/7 automatically set PARALLEL_JOBS when BR2_JLEVEL is 0

In reply to Arnout Vandecappelle

Quoted 2 lines · Arnout Vandecappelle

In my experience, you need 2 times the number of CPUs to fully utilize
the processor because most compilations have long iowait latencies.

Unless you have little memory, this is no longer true, and as some compilations can eat a lot of memory, doubling their number reduces the amount of available page cache, potentially increasing the amount of I/O. Using the number of available processors should be just fine.

Signature · 1 line

Richard Braun

Nathan Lynch

author

2012-06-14 15:02 UTC #14 lore raw

PATCH v2 7/7 automatically set PARALLEL_JOBS when BR2_JLEVEL is 0

In reply to Nathan Lynch

When BR2_JLEVEL is 0, set PARALLEL_JOBS to double the number of CPUs detected. This allows one to more or less fully utilize the host system without manually tuning the configuration.

Also make 0 the default value for BR2_JLEVEL.

Signed-off-by Nathan Lynch <[email protected]>

---

v2: Updated to use double number of detected CPUs per Arnout and Peter.

 Config.in           |    8 +++++---
 package/Makefile.in |    5 +++++
 2 files changed, 10 insertions(+), 3 deletions(-)

Config.in +5 -3

index 95c2e8c..3745255 100644
@@ -178,10 +178,12 @@ config BR2_DEBIAN_MIRROR
 endmenu
 
 config BR2_JLEVEL
-	int "Number of jobs to run simultaneously"
-	default "2"
+	int "Number of jobs to run simultaneously (0 for auto)"
+	default "0"
 	help
-	  Number of jobs to run simultaneously
+	  Number of jobs to run simultaneously.  If 0, determine
+	  automatically according to number of CPUs on the host
+	  system.
 
 config BR2_CCACHE
 	bool "Enable compiler cache"

package/Makefile.in +5 -0

index 9177a1b..95e0ae6 100644
@@ -6,7 +6,12 @@ HOSTMAKE=$(MAKE)
 endif
 HOSTMAKE :=$(shell which $(HOSTMAKE) || type -p $(HOSTMAKE) || echo make)
 
+ifeq ($(BR2_JLEVEL),0)
+PARALLEL_JOBS:=$(shell echo \
+	$$((2 * `getconf _NPROCESSORS_ONLN 2>/dev/null || echo 1`)))
+else
 PARALLEL_JOBS:=$(BR2_JLEVEL)
+endif
 
 MAKE1:=$(HOSTMAKE) -j1
 MAKE:=$(HOSTMAKE) -j$(PARALLEL_JOBS)

Signature · 1 line

1.7.10.2

Baruch Siach

2012-06-14 18:09 UTC #15 lore raw

PATCH v2 7/7 automatically set PARALLEL_JOBS when BR2_JLEVEL is 0

Hi Nathan,

Quoted 8 lines · Nathan Lynch

When BR2_JLEVEL is 0, set PARALLEL_JOBS to double the number of CPUs detected. This allows one to more or less fully utilize the host system without manually tuning the configuration.
Also make 0 the default value for BR2_JLEVEL.
Signed-off-by Nathan Lynch <[email protected]>
---

[snip]

package/Makefile.in HOSTMAKE=$(MAKE) · line 14

Quoted 6 lines

package/Makefile.in +5 -0

+ifeq ($(BR2_JLEVEL),0)
+PARALLEL_JOBS:=$(shell echo \
+	$$((2 * `getconf _NPROCESSORS_ONLN 2>/dev/null || echo 1`)))
+else
 PARALLEL_JOBS:=$(BR2_JLEVEL)
+endif

IMO, there should be a comment here explaining the magic (2*CPUs) number.

baruch

Signature · 3 lines

     http://baruch.siach.name/blog/                  ~. .~   Tk Open Systems
=}------------------------------------------------ooO--U--Ooo------------{=
   - baruch at tkos.co.il - tel: +972.2.679.5364, http://www.tkos.co.il -

Nathan Lynch

author

2012-06-16 19:37 UTC #16 lore raw

PATCH v3 7/7 automatically set PARALLEL_JOBS when BR2_JLEVEL is 0

In reply to Nathan Lynch

When BR2_JLEVEL is 0, set PARALLEL_JOBS to double the number of CPUs detected. This allows one to more or less fully utilize the host system without manually tuning the configuration.

Also make 0 the default value for BR2_JLEVEL.

Signed-off-by Nathan Lynch <[email protected]>

---

v3: Added comment to PARALLEL_JOBS calculation per Baruch.

v2: Updated to use double number of detected CPUs per Arnout and Peter.

 Config.in           |    8 +++++---
 package/Makefile.in |    8 ++++++++
 2 files changed, 13 insertions(+), 3 deletions(-)

Config.in +5 -3

index 95c2e8c..3745255 100644
@@ -178,10 +178,12 @@ config BR2_DEBIAN_MIRROR
 endmenu
 
 config BR2_JLEVEL
-	int "Number of jobs to run simultaneously"
-	default "2"
+	int "Number of jobs to run simultaneously (0 for auto)"
+	default "0"
 	help
-	  Number of jobs to run simultaneously
+	  Number of jobs to run simultaneously.  If 0, determine
+	  automatically according to number of CPUs on the host
+	  system.
 
 config BR2_CCACHE
 	bool "Enable compiler cache"

package/Makefile.in +8 -0

index 9177a1b..ffcb9b6 100644
@@ -6,7 +6,15 @@ HOSTMAKE=$(MAKE)
 endif
 HOSTMAKE :=$(shell which $(HOSTMAKE) || type -p $(HOSTMAKE) || echo make)
 
+# If BR2_LEVEL is 0, scale the maximum concurrency with the number of
+# CPUs.  A coefficient of 2 is used in order to keep processors busy
+# while waiting on I/O.
+ifeq ($(BR2_JLEVEL),0)
+PARALLEL_JOBS:=$(shell echo \
+	$$((2 * `getconf _NPROCESSORS_ONLN 2>/dev/null || echo 1`)))
+else
 PARALLEL_JOBS:=$(BR2_JLEVEL)
+endif
 
 MAKE1:=$(HOSTMAKE) -j1
 MAKE:=$(HOSTMAKE) -j$(PARALLEL_JOBS)

Signature · 1 line

1.7.10.2

Peter Korsgaard

2012-06-24 09:13 UTC #17 lore raw

cover automatically determine job concurrency

In reply to Nathan Lynch

Quoted 1 line

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
"Nathan" == Nathan Lynch <[email protected]> writes:

Nathan> This series implements the option of "autodetecting" the number of Nathan> jobs to run concurrently (make -jN), using the number of reported CPUs Nathan> when BR2_JLEVEL is set to 0.

Committed series, thanks!

Signature · 1 line

Bye, Peter Korsgaard