Introduce housekeeping subsystem

The housekeeping is currently driven by nohz_full where any CPU that is not in the nohz_full range is considered as a housekeeper. This is a design mistake because nohz is just a detail among all the existing isolation features. Nohz shouldn't imply anything else than tick related things.

We rather want to drive all the isolation features from the housekeeping subsystem which is responsible for all the work that can be either affined (unpinned workqueues, timers, kthreads, ...) or offloaded (scheduler tick, ...).

Let's start with a boot option to define the housekeepers. We should be able to further enhance that through cpusets.

Signed-off-by Frederic Weisbecker <[email protected]>

Cc Chris Metcalf <[email protected]>

Cc Rik van Riel <[email protected]>

Cc Peter Zijlstra <[email protected]>

Cc Thomas Gleixner <[email protected]>

Cc Mike Galbraith <[email protected]>

Cc Ingo Molnar <[email protected]>

Cc Christoph Lameter <[email protected]>

Cc Paul E. McKenney <[email protected]>

Cc Wanpeng Li <[email protected]>

Cc Luiz Capitulino <[email protected]>

---
 include/linux/housekeeping.h |  2 --
 init/main.c                  |  2 --
 kernel/housekeeping.c        | 22 ++++++++++------------
 3 files changed, 10 insertions(+), 16 deletions(-)

include/linux/housekeeping.h +0 -2

index 320cc2b..ba769c8 100644
@@ -11,7 +11,6 @@ extern int housekeeping_any_cpu(void);
 extern const struct cpumask *housekeeping_cpumask(void);
 extern void housekeeping_affine(struct task_struct *t);
 extern bool housekeeping_test_cpu(int cpu);
-extern void __init housekeeping_init(void);
 
 #else
 

@@ -26,7 +25,6 @@ static inline const struct cpumask *housekeeping_cpumask(void)
 }
 
 static inline void housekeeping_affine(struct task_struct *t) { }
-static inline void housekeeping_init(void) { }
 #endif /* CONFIG_NO_HZ_FULL */
 
 static inline bool housekeeping_cpu(int cpu)

init/main.c +0 -2

index 9904a1e..9789ab7 100644
@@ -46,7 +46,6 @@
 #include <linux/cgroup.h>
 #include <linux/efi.h>
 #include <linux/tick.h>
-#include <linux/housekeeping.h>
 #include <linux/interrupt.h>
 #include <linux/taskstats_kern.h>
 #include <linux/delayacct.h>

@@ -608,7 +607,6 @@ asmlinkage __visible void __init start_kernel(void)
 	early_irq_init();
 	init_IRQ();
 	tick_init();
-	housekeeping_init();
 	rcu_init_nohz();
 	init_timers();
 	hrtimers_init();

kernel/housekeeping.c +7 -10

index f8be7e6..a54765d 100644
@@ -45,23 +45,21 @@ bool housekeeping_test_cpu(int cpu)
 	return true;
 }
 
-void __init housekeeping_init(void)
+/* Parse the boot-time housekeeping CPU list from the kernel parameters. */
+static int __init housekeeping_setup(char *str)
 {
-	if (!tick_nohz_full_enabled())
-		return;
-
-	if (!alloc_cpumask_var(&housekeeping_mask, GFP_KERNEL)) {
-		WARN(1, "NO_HZ: Can't allocate not-full dynticks cpumask\n");
-		cpumask_clear(tick_nohz_full_mask);
-		tick_nohz_full_running = false;
-		return;
+	alloc_bootmem_cpumask_var(&housekeeping_mask);
+	if (cpulist_parse(str, housekeeping_mask) < 0) {
+		pr_warn("Housekeeping: Incorrect cpumask\n");
+		free_bootmem_cpumask_var(housekeeping_mask);
+		return 1;
 	}
 
-	cpumask_andnot(housekeeping_mask,

Quoted 24 lines · On Fri, 21 Jul 2017 15:21:28 +0200 Frederic Weisbecker <[email protected]>…

kernel/housekeeping.c +7 -12

-void __init housekeeping_init(void)
+/* Parse the boot-time housekeeping CPU list from the kernel parameters. */
+static int __init housekeeping_setup(char *str)
 {
-	if (!tick_nohz_full_enabled())
-		return;
-
-	if (!alloc_cpumask_var(&housekeeping_mask, GFP_KERNEL)) {
-		WARN(1, "NO_HZ: Can't allocate not-full dynticks cpumask\n");
-		cpumask_clear(tick_nohz_full_mask);
-		tick_nohz_full_running = false;
-		return;
+	alloc_bootmem_cpumask_var(&housekeeping_mask);
+	if (cpulist_parse(str, housekeeping_mask) < 0) {
+		pr_warn("Housekeeping: Incorrect cpumask\n");
+		free_bootmem_cpumask_var(housekeeping_mask);
+		return 1;
 	}
 
-	cpumask_andnot(housekeeping_mask,
-		       cpu_possible_mask, tick_nohz_full_mask);
-
 	static_branch_enable(&housekeeping_overriden);

kernel/housekeeping.c bool housekeeping_test_cpu(int cpu) · line 65

Quoted 6 lines

kernel/housekeeping.c +3 -0

 	/* We need at least one CPU to handle housekeeping work */
 	WARN_ON_ONCE(cpumask_empty(housekeeping_mask));
+
+	return 1;
 }
+__setup("housekeeping=", housekeeping_setup);

Am I right that from now on nohz_full= users will also have to specify housekeeping= in order to get nohz_full working? If that's correct, then won't this patch break nohz_full for existing setups?

Also, I just give this series a try and got this:

[    0.000000] Kernel command line: BOOT_IMAGE=/vmlinuz-4.13.0-rc4+ root=/dev/mapper/rhel_virtlab508-root ro crashkernel=auto rd.lvm.lv=rhel_virtlab508/root rd.lvm.lv=rhel_virtlab508/swap console=ttyS1,115200 LANG=en_US.UTF-8 housekeeping=0,2,4,6,8,10,12,14,1 isolcpus=15 nohz_full=15 intel_pstate=disable
[    0.000000] static_key_slow_inc used before call to jump_label_init
[    0.000000] ------------[ cut here ]------------
[    0.000000] WARNING: CPU: 0 PID: 0 at kernel/jump_label.c:108 static_key_slow_inc+0x86/0xa0

Quoted 26 lines · Luiz Capitulino

Quoted 24 lines · On Fri, 21 Jul 2017 15:21:28 +0200 Frederic Weisbecker <[email protected]>…

kernel/housekeeping.c +7 -12

-void __init housekeeping_init(void)
+/* Parse the boot-time housekeeping CPU list from the kernel parameters. */
+static int __init housekeeping_setup(char *str)
 {
-	if (!tick_nohz_full_enabled())
-		return;
-
-	if (!alloc_cpumask_var(&housekeeping_mask, GFP_KERNEL)) {
-		WARN(1, "NO_HZ: Can't allocate not-full dynticks cpumask\n");
-		cpumask_clear(tick_nohz_full_mask);
-		tick_nohz_full_running = false;
-		return;
+	alloc_bootmem_cpumask_var(&housekeeping_mask);
+	if (cpulist_parse(str, housekeeping_mask) < 0) {
+		pr_warn("Housekeeping: Incorrect cpumask\n");
+		free_bootmem_cpumask_var(housekeeping_mask);
+		return 1;
 	}
 
-	cpumask_andnot(housekeeping_mask,
-		       cpu_possible_mask, tick_nohz_full_mask);
-
 	static_branch_enable(&housekeeping_overriden);

Quoted 11 lines

kernel/housekeeping.c bool housekeeping_test_cpu(int cpu) · line 65

Quoted 6 lines

kernel/housekeeping.c +3 -0

 	/* We need at least one CPU to handle housekeeping work */
 	WARN_ON_ONCE(cpumask_empty(housekeeping_mask));
+
+	return 1;
 }
+__setup("housekeeping=", housekeeping_setup);  

Am I right that from now on nohz_full= users will also have to specify housekeeping= in order to get nohz_full working? If that's correct, then won't this patch break nohz_full for existing setups?

nohz_full= will still work but will only imply tick stop. A few isolation details that were enabled by nohz_full= won't be handled anymore such as:
unbound timers affinity, watchdog disablement, rcu threads affinity, sched idle load balancing... Those are now handled by housekeeping=

So yes in a sense, this can break some setup that assume nohz_full= does more than stopping the tick.

Perhaps I should remove the nohz_full= parameter altogether and let nohz_full controlled by housekeeping= only. How much can kernel parameters be considered as kernel ABIs?

Also I'm wondering if "housekeeping=" is a clear name for users. "isolation=" or "cpu_isolation=" would be better and more obvious. Housekeeping based naming would only be internal implementation detail. And deactivating the tick through "cpu_isolation=" would be clearer than if we did through "housekeeping=".

Of course the problem is that we already have "isolcpus=". But re-implementing isolcpus on top of housekeeping might be a good idea. I believe that the current implementation on top of NULL domains isn't much beloved. A less controversial implementation might even allow us to control it though cpusets.

Quoted 5 lines · On Sat, 12 Aug 2017 16:10:06 +0200 Frederic Weisbecker <[email protected]>…

Quoted 4 lines

Am I right that from now on nohz_full= users will also have to specify housekeeping= in order to get nohz_full working? If that's correct, then won't this patch break nohz_full for existing setups?

Quoted 7 lines

nohz_full= will still work but will only imply tick stop. A few isolation details that were enabled by nohz_full= won't be handled anymore such as:
unbound timers affinity, watchdog disablement, rcu threads affinity, sched idle load balancing... Those are now handled by housekeeping=

So yes in a sense, this can break some setup that assume nohz_full= does more than stopping the tick.

Yes, the problem is that this is how it has always worked. Also, the breakage will be very subtle and hard to debug.

Quoted 3 lines

Perhaps I should remove the nohz_full= parameter altogether and let nohz_full controlled by housekeeping= only. How much can kernel parameters be considered as kernel ABIs?

That's a very good question, I don't have an answer for that.

Quoted 5 lines

Also I'm wondering if "housekeeping=" is a clear name for users. "isolation=" or "cpu_isolation=" would be better and more obvious. Housekeeping based naming would only be internal implementation detail. And deactivating the tick through "cpu_isolation=" would be clearer than if we did through "housekeeping=".

That's exactly my thinking while I was reviewing the series!

Quoted 4 lines

Of course the problem is that we already have "isolcpus=". But re-implementing isolcpus on top of housekeeping might be a good idea. I believe that the current implementation on top of NULL domains isn't much beloved. A less controversial implementation might even allow us to control it though cpusets.

You're completely right. Some people don't use isolcpus= because it disables load balancing and that may be a problem for setups where tasks are pinned to a set of CPUs where the number of tasks is greater than the number of CPUs. However, for the cases where you have a single task pinned to a CPU, having load balancing taking place adds an extra latency (I won't remember how much, but I guess it was more than 10us).

If there's a way to "disable" load balancing from user-space, say with cpusets, then I think we should keep the isolated CPUs attached to a domain as you suggest.

Another detail about isolcpus= is that it doesn't isolate the CPU from kernel threads. That is, unpinned kernel threads are allowed to run on CPUs not isolated with isolcpus=. We might consider changing that for a new isolation option.

I know that there are many arguments against isolcpus= and some people advice using cpusets. The problem with that advice is that isolcpus= goes a bit beyond isolating a CPU from user-space tasks. One additional thing is does for example, is pinning the kernel_init() thread to housekeeping CPUs. This is key, because that thread will create timers at early boot that will pin themselves to the CPU they run.

Finally, I'm wondering how all this will fit together with TASK_ISOLATION. One of the questions I ask myself is: can/should the things TASK_ISOLATION does be done by a kernel command-line parameter instead? Or should we try to come up with a list of global things to control (eg. the tick, kernel thread affinity, etc) and per-task controls?

Quoted 4 lines

Quoted 2 lines

Perhaps I should remove the nohz_full= parameter altogether and let nohz_full controlled by housekeeping= only. How much can kernel parameters be considered as kernel ABIs?

That's a very good question, I don't have an answer for that.

That said, "nohz_full=" never implied too much isolation features so far, and those have often changed over time, as in RCU. I think unbound timer affinity is the most important one.

Perhaps we can keep "nohz_full=1-15" as an alias for a future "cpu_isolation=nohz,1-15" and at least imply unbound timer affinity with it.

Quoted 12 lines

Quoted 4 lines

Also I'm wondering if "housekeeping=" is a clear name for users. "isolation=" or "cpu_isolation=" would be better and more obvious. Housekeeping based naming would only be internal implementation detail. And deactivating the tick through "cpu_isolation=" would be clearer than if we did through "housekeeping=".

That's exactly my thinking while I was reviewing the series!

Quoted 4 lines

Of course the problem is that we already have "isolcpus=". But re-implementing isolcpus on top of housekeeping might be a good idea. I believe that the current implementation on top of NULL domains isn't much beloved. A less controversial implementation might even allow us to control it though cpusets.

Quoted 7 lines

You're completely right. Some people don't use isolcpus= because it disables load balancing and that may be a problem for setups where tasks are pinned to a set of CPUs where the number of tasks is greater than the number of CPUs. However, for the cases where you have a single task pinned to a CPU, having load balancing taking place adds an extra latency (I won't remember how much, but I guess it was more than 10us).

What is the source of the load balancing inducing such latency when a single task is affine to a CPU? If this is idle load balancing, it is now affine to housekeepers. If this is task wakeup then it's suprising because select_task_rq() is optimized toward single CPU affinity.

Is there another source I'm overlooking?

Quoted 3 lines

If there's a way to "disable" load balancing from user-space, say with cpusets, then I think we should keep the isolated CPUs attached to a domain as you suggest.

I'm not sure such a solution would be accepted. The most sensible way to disable load balancing is still to tune the affinity of tasks. If there is an off-case overhead with load balancing (ie: when no more than one task is affine to that CPU) then we should solve that with a fast path.

Quoted 4 lines

Another detail about isolcpus= is that it doesn't isolate the CPU from kernel threads. That is, unpinned kernel threads are allowed to run on CPUs not isolated with isolcpus=. We might consider changing that for a new isolation option.

You mean unpinned kernel threads are allowed to run on isolcpus, right?

That definetly can be solved.

Quoted 7 lines

I know that there are many arguments against isolcpus= and some people advice using cpusets. The problem with that advice is that isolcpus= goes a bit beyond isolating a CPU from user-space tasks. One additional thing is does for example, is pinning the kernel_init() thread to housekeeping CPUs. This is key, because that thread will create timers at early boot that will pin themselves to the CPU they run.

Right, but also unbound timers are affine to housekeepers, we needed that for nohz_full.

Quoted 5 lines

Finally, I'm wondering how all this will fit together with TASK_ISOLATION. One of the questions I ask myself is: can/should the things TASK_ISOLATION does be done by a kernel command-line parameter instead? Or should we try to come up with a list of global things to control (eg. the tick, kernel thread affinity, etc) and per-task controls?

So I've been thinking a lot about that lately. I told Chris that TASK_ISOLATION shouldn't be a CPU feature but a task feature. Then I realized that it doesn't work either, my bad :-) In the end I think that the most part of it must be a CPU
property: nohz, task isolation, timers and workqueue affinity, etc... Then what's left for the per task thing is to tell it when it is unexpectingly interrupted by noise.

Therefore I think most of the isolation features should be controlled by command line and cpusets (through a new cpuset subsystem maybe) then TASK_ISOLATION through prtcl() for the noise monitoring.

On Mon, 14 Aug 2017 19:01:09 +0200

Quoted 4 lines

What is the source of the load balancing inducing such latency when a single task is affine to a CPU? If this is idle load balancing, it is now affine to housekeepers. If this is task wakeup then it's suprising because select_task_rq() is optimized toward single CPU affinity.

I guess it was idle load balancing, but I don't remember because this was a few years ago. I think this might be reproducible without using isolcpus=. I'll give it a try shortly and let you know.

Quoted 7 lines · Luiz Capitulino

On Mon, 14 Aug 2017 19:01:09 +0200

Quoted 4 lines

What is the source of the load balancing inducing such latency when a single task is affine to a CPU? If this is idle load balancing, it is now affine to housekeepers. If this is task wakeup then it's suprising because select_task_rq() is optimized toward single CPU affinity.

Quoted 3 lines

I guess it was idle load balancing, but I don't remember because this was a few years ago. I think this might be reproducible without using isolcpus=. I'll give it a try shortly and let you know.

idle_balance() can swamp other noise by a couple orders of magnitude,

Quoted 8 lines · Mike Galbraith

Quoted 7 lines · Luiz Capitulino

On Mon, 14 Aug 2017 19:01:09 +0200

Quoted 4 lines

What is the source of the load balancing inducing such latency when a single task is affine to a CPU? If this is idle load balancing, it is now affine to housekeepers. If this is task wakeup then it's suprising because select_task_rq() is optimized toward single CPU affinity.

Quoted 5 lines

Quoted 3 lines

I guess it was idle load balancing, but I don't remember because this was a few years ago. I think this might be reproducible without using isolcpus=. I'll give it a try shortly and let you know.

idle_balance() can swamp other noise by a couple orders of magnitude,

Ah I missed that one. Is there any way we can also lower the overhead there?

Quoted 13 lines · Frederic Weisbecker

Quoted 12 lines · Mike Galbraith

Quoted 10 lines · Luiz Capitulino

On Mon, 14 Aug 2017 19:01:09 +0200

Quoted 4 lines

What is the source of the load balancing inducing such latency when a single task is affine to a CPU? If this is idle load balancing, it is now affine to housekeepers. If this is task wakeup then it's suprising because select_task_rq() is optimized toward single CPU affinity.

I guess it was idle load balancing, but I don't remember because this was a few years ago. I think this might be reproducible without using isolcpus=. I'll give it a try shortly and let you know.

Quoted 3 lines

Quoted 1 line

idle_balance() can swamp other noise by a couple orders of magnitude,

Ah I missed that one. Is there any way we can also lower the overhead there?

Why?  HPC proggies won't benefit from a partially filled pothole any more that a ~zero ground clearance formula 1 car would.  The pothole intolerant either isolate, killing (most) LB, or they meet a wall.

Don't the HPC guys just disable idle_balance(), or am I out of date again?

Quoted 1 line · Paul E. McKenney

Don't the HPC guys just disable idle_balance(), or am I out of date again?

Ummm.. Why does idle management matter when your goal is to keep all processor busy working at maximum throughput?

Quoted 6 lines · Christopher Lameter

Quoted 1 line · Paul E. McKenney

Don't the HPC guys just disable idle_balance(), or am I out of date again?

Ummm.. Why does idle management matter when your goal is to keep all processor busy working at maximum throughput?

If you _never_ idle, you never have to worry about it.  Is 100% CPU until the end of time all there is to HPC?

Quoted 8 lines · Mike Galbraith

Quoted 6 lines · Christopher Lameter

Quoted 1 line · Paul E. McKenney

Don't the HPC guys just disable idle_balance(), or am I out of date again?

Ummm.. Why does idle management matter when your goal is to keep all processor busy working at maximum throughput?

Quoted 2 lines

If you _never_ idle, you never have to worry about it.  Is 100% CPU until the end of time all there is to HPC?

Most of the time that is true for HPC loads. They may also go through a I/O throughput constrained processing phase or synchronization phase where idle activity occurs.

Quoted 16 lines · Mike Galbraith

Quoted 15 lines · Frederic Weisbecker

Quoted 13 lines · Mike Galbraith

Quoted 10 lines · Luiz Capitulino

On Mon, 14 Aug 2017 19:01:09 +0200

Quoted 4 lines

What is the source of the load balancing inducing such latency when a single task is affine to a CPU? If this is idle load balancing, it is now affine to housekeepers. If this is task wakeup then it's suprising because select_task_rq() is optimized toward single CPU affinity.

I guess it was idle load balancing, but I don't remember because this was a few years ago. I think this might be reproducible without using isolcpus=. I'll give it a try shortly and let you know.

idle_balance() can swamp other noise by a couple orders of magnitude,

Quoted 5 lines

Quoted 1 line

Ah I missed that one. Is there any way we can also lower the overhead there?

Why?  HPC proggies won't benefit from a partially filled pothole any more that a ~zero ground clearance formula 1 car would.  The pothole intolerant either isolate, killing (most) LB, or they meet a wall.

Don't the HPC guys just disable idle_balance(), or am I out of date again?

I'm leaving for two weeks so this is food for thoughts in the meantime :)

We have a design issue with nohz_full: it drives the isolation features through the *housekeeping*() functions: kthreads, unpinned timers, watchdog, ...

But things should work the other way around because the tick is just an isolation feature among others.

So we need a housekeeping subsystem to drive all these isolation features, including nohz full in a later iteration. For now this is a basic draft. In the long run this subsystem should also drive the tick offloading (remove residual 1Hz) and all unbound kthreads.

git://git.kernel.org/pub/scm/linux/kernel/git/frederic/linux-dynticks.git
nohz/0hz

HEAD: 68e3af1de5db228bf6c2a5e721bce59a02cfc4e1

Quoted 14 lines · Frederic Weisbecker

I'm leaving for two weeks so this is food for thoughts in the meantime :)

We have a design issue with nohz_full: it drives the isolation features through the *housekeeping*() functions: kthreads, unpinned timers, watchdog, ...

But things should work the other way around because the tick is just an isolation feature among others.

So we need a housekeeping subsystem to drive all these isolation features, including nohz full in a later iteration. For now this is a basic draft. In the long run this subsystem should also drive the tick offloading (remove residual 1Hz) and all unbound kthreads.

Quoted 4 lines

git://git.kernel.org/pub/scm/linux/kernel/git/frederic/linux-dynticks.git
nohz/0hz

HEAD: 68e3af1de5db228bf6c2a5e721bce59a02cfc4e1

For the series:

Reviewed-by Chris Metcalf <[email protected]>

I spotted a few typos that you should grep for and fix for your next version:
"watchog", "Lets/lets" instead of "Let's/let's", "overriden" (should have two d's).

The new housekeeping=MASK boot option seems like it might make it a little irritating to specify nohz_full=MASK as well. I guess if setting NO_HZ_FULL_ALL
implied "all but housekeeping", it becomes a reasonably tidy solution. To make
this work right you might have to make the housekeeping option early_param instead so its value is available early enough.

But perhaps I should add a new NO_HZ_FULL_BUT_HOUSEKEEPING option. Otherwise we'll change the meaning of NO_HZ_FULL_ALL way too much, to the point that its default behaviour will be the exact opposite of the current one: by default every CPU is housekeeping, so NO_HZ_FULL_ALL would have no effect anymore if we don't set housekeeping boot option.

Also I plan to add a housekeeping option to offload the residual 1Hz tick from nohz_full CPUs. So having "housekeeping=0,tick_offload" would make CPU 0 the housekeeper, make the other CPUs nohz_full and handle their 1hz tick from CPU 0.

Quoted 5 lines · Frederic Weisbecker

But perhaps I should add a new NO_HZ_FULL_BUT_HOUSEKEEPING option. Otherwise we'll change the meaning of NO_HZ_FULL_ALL way too much, to the point that its default behaviour will be the exact opposite of the current one: by default every CPU is housekeeping, so NO_HZ_FULL_ALL would have no effect anymore if we don't set housekeeping boot option.

Maybe a CONFIG_HOUSEKEEPING_BOOT_ONLY as a way to restrict housekeeping by default to just the boot cpu. In conjunction with NOHZ_FULL_ALL you would then get the expected semantics.

Quoted 7 lines · Chris Metcalf

Quoted 5 lines · Frederic Weisbecker

But perhaps I should add a new NO_HZ_FULL_BUT_HOUSEKEEPING option. Otherwise we'll change the meaning of NO_HZ_FULL_ALL way too much, to the point that its default behaviour will be the exact opposite of the current one: by default every CPU is housekeeping, so NO_HZ_FULL_ALL would have no effect anymore if we don't set housekeeping boot option.

Quoted 3 lines

Maybe a CONFIG_HOUSEKEEPING_BOOT_ONLY as a way to restrict housekeeping by default to just the boot cpu. In conjunction with NOHZ_FULL_ALL you would then get the expected semantics.

A big box with only the boot cpu for housekeeping is likely screwed.

Personally, I think NOHZ_FULL_ALL should just die.

Quoted 1 line

Personally, I think NOHZ_FULL_ALL should just die.

Yeah, although it's still useful for automatic boot testing to detect issues with nohz_full on.

Quoted 9 lines · Chris Metcalf

Quoted 5 lines · Frederic Weisbecker

But perhaps I should add a new NO_HZ_FULL_BUT_HOUSEKEEPING option. Otherwise we'll change the meaning of NO_HZ_FULL_ALL way too much, to the point that its default behaviour will be the exact opposite of the current one: by default every CPU is housekeeping, so NO_HZ_FULL_ALL would have no effect anymore if we don't set housekeeping boot option.

Maybe a CONFIG_HOUSEKEEPING_BOOT_ONLY as a way to restrict housekeeping by default to just the boot cpu. In conjunction with NOHZ_FULL_ALL you would then get the expected semantics.

A big box with only the boot cpu for housekeeping is likely screwed.

Quoted 5 lines

Quoted 4 lines

Maybe a CONFIG_HOUSEKEEPING_BOOT_ONLY as a way to restrict housekeeping by default to just the boot cpu. In conjunction with NOHZ_FULL_ALL you would
then get the expected semantics.

A big box with only the boot cpu for housekeeping is likely screwed.

Fair point - this kind of configuration would be primarily useful for dedicated systems that were running a high-traffic-rate networking application on many cores, for example. In this mode you don't end up putting a lot of burden on the housekeeping core. In any case, probably not worth adding an additional kernel config for.

Ah, Chris since you are here: What is happening with the dataplane patches?