isolation: 1Hz residual tick offloading v3

When a CPU runs in full dynticks mode, a 1Hz tick remains in order to keep the scheduler stats alive. However this residual tick is a burden for bare metal tasks that can't stand any interruption at all, or want to minimize them.

Adding the boot parameter "isolcpus=nohz_offload" will now outsource these scheduler ticks to the global workqueue so that a housekeeping CPU handles that tick remotely.

Note it's still up to the user to affine the global workqueues to the housekeeping CPUs through /sys/devices/virtual/workqueue/cpumask or domains isolation.

Signed-off-by Frederic Weisbecker <[email protected]>

Cc Chris Metcalf <[email protected]>

Cc Christoph Lameter <[email protected]>

Cc Luiz Capitulino <[email protected]>

Cc Mike Galbraith <[email protected]>

Cc Paul E. McKenney <[email protected]>

Cc Peter Zijlstra <[email protected]>

Cc Rik van Riel <[email protected]>

Cc Thomas Gleixner <[email protected]>

Cc Wanpeng Li <[email protected]>

Cc Ingo Molnar <[email protected]>

---
 kernel/sched/core.c      | 88 ++++++++++++++++++++++++++++++++++++++++++++++--
 kernel/sched/isolation.c |  4 +++
 kernel/sched/sched.h     |  2 ++
 3 files changed, 91 insertions(+), 3 deletions(-)

kernel/sched/core.c +24 -3

index d72d0e9..b964890 100644
@@ -3052,9 +3052,14 @@ void scheduler_tick(void)
  */
 u64 scheduler_tick_max_deferment(void)
 {
-	struct rq *rq = this_rq();
-	unsigned long next, now = READ_ONCE(jiffies);
+	struct rq *rq;
+	unsigned long next, now;
 
+	if (!housekeeping_cpu(smp_processor_id(), HK_FLAG_TICK_SCHED))
+		return ktime_to_ns(KTIME_MAX);
+
+	rq = this_rq();
+	now = READ_ONCE(jiffies);
 	next = rq->last_sched_tick + HZ;
 
 	if (time_before_eq(next, now))

@@ -3062,7 +3067,82 @@ u64 scheduler_tick_max_deferment(void)
 
 	return jiffies_to_nsecs(next - now);
 }
-#endif
+
+struct tick_work {
+	int			cpu;
+	struct delayed_work	work;
+};
+
+static struct tick_work __percpu *tick_work_cpu;
+
+static void sched_tick_remote(struct work_struct *work)
+{
+	struct delayed_work *dwork = to_delayed_work(work);
+	struct tick_work *twork = container_of(dwork, struct tick_work, work);
+	int cpu = twork->cpu;
+	struct rq *rq = cpu_rq(cpu);
+	struct rq_flags rf;
+
+	/*

kernel/sched/core.c +51 -0

+
+	queue_delayed_work(system_unbound_wq, dwork, HZ);
+}
+
+static void sched_tick_start(int cpu)
+{
+	struct tick_work *twork;
+
+	if (housekeeping_cpu(cpu, HK_FLAG_TICK_SCHED))
+		return;
+
+	WARN_ON_ONCE(!tick_work_cpu);
+
+	twork = per_cpu_ptr(tick_work_cpu, cpu);
+	twork->cpu = cpu;
+	INIT_DELAYED_WORK(&twork->work, sched_tick_remote);
+	queue_delayed_work(system_unbound_wq, &twork->work, HZ);
+}
+
+#ifdef CONFIG_HOTPLUG_CPU
+static void sched_tick_stop(int cpu)
+{
+	struct tick_work *twork;
+
+	if (housekeeping_cpu(cpu, HK_FLAG_TICK_SCHED))
+		return;
+
+	WARN_ON_ONCE(!tick_work_cpu);
+
+	twork = per_cpu_ptr(tick_work_cpu, cpu);
+	cancel_delayed_work_sync(&twork->work);
+}
+#endif /* CONFIG_HOTPLUG_CPU */
+
+int __init sched_tick_offload_init(void)
+{
+	tick_work_cpu = alloc_percpu(struct tick_work);
+	if (!tick_work_cpu) {
+		pr_err("Can't allocate remote tick struct\n");
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+#else
+static void sched_tick_start(int cpu) { }
+static void sched_tick_stop(int cpu) { }
+#endif /* CONFIG_NO_HZ_FULL */
 
 #if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
 				defined(CONFIG_PREEMPT_TRACER))

@@ -5713,6 +5793,7 @@ int sched_cpu_starting(unsigned int cpu)
 {
 	set_cpu_rq_start_time(cpu);
 	sched_rq_cpu_starting(cpu);
+	sched_tick_start(cpu);
 	return 0;
 }
 

@@ -5724,6 +5805,7 @@ int sched_cpu_dying(unsigned int cpu)
 
 	/* Handle pending wakeups and then migrate everything off */
 	sched_ttwu_pending();
+	sched_tick_stop(cpu);
 
 	rq_lock_irqsave(rq, &rf);
 	if (rq->rd) {

kernel/sched/isolation.c +4 -0

index 264ddcd..c5e7e90a 100644
@@ -12,6 +12,7 @@
 #include <linux/kernel.h>
 #include <linux/static_key.h>
 #include <linux/ctype.h>
+#include "sched.h"
 
 DEFINE_STATIC_KEY_FALSE(housekeeping_overriden);
 EXPORT_SYMBOL_GPL(housekeeping_overriden);

@@ -60,6 +61,9 @@ void __init housekeeping_init(void)
 
 	static_branch_enable(&housekeeping_overriden);
 
+	if (housekeeping_flags & HK_FLAG_TICK_SCHED)
+		sched_tick_offload_init();
+
 	/* We need at least one CPU to handle housekeeping work */
 	WARN_ON_ONCE(cpumask_empty(housekeeping_mask));
 }

kernel/sched/sched.h +2 -0

index b19552a2..5a3b82c 100644
@@ -1587,6 +1587,7 @@ extern void post_init_entity_util_avg(struct sched_entity *se);
 
 #ifdef CONFIG_NO_HZ_FULL
 extern bool sched_can_stop_tick(struct rq *rq);
+extern int __init sched_tick_offload_init(void);
 
 /*
  * Tick may be needed by tasks in the runqueue depending on their policy and

@@ -1611,6 +1612,7 @@ static inline void sched_update_tick_dependency(struct rq *rq)
 		tick_nohz_dep_set_cpu(cpu, TICK_DEP_BIT_SCHED);
 }
 #else
+static inline int sched_tick_offload_init(void) { return 0; }
 static inline void sched_update_tick_dependency(struct rq *rq) { }
 #endif
 

Quoted 78 lines · On Thu, 4 Jan 2018 05:25:36 +0100 Frederic Weisbecker <[email protected]>…

When a CPU runs in full dynticks mode, a 1Hz tick remains in order to keep the scheduler stats alive. However this residual tick is a burden for bare metal tasks that can't stand any interruption at all, or want to minimize them.

Adding the boot parameter "isolcpus=nohz_offload" will now outsource these scheduler ticks to the global workqueue so that a housekeeping CPU handles that tick remotely.

Note it's still up to the user to affine the global workqueues to the housekeeping CPUs through /sys/devices/virtual/workqueue/cpumask or domains isolation.

Signed-off-by Frederic Weisbecker <[email protected]>

Cc Chris Metcalf <[email protected]>

Cc Christoph Lameter <[email protected]>

Cc Luiz Capitulino <[email protected]>

Cc Mike Galbraith <[email protected]>

Cc Paul E. McKenney <[email protected]>

Cc Peter Zijlstra <[email protected]>

Cc Rik van Riel <[email protected]>

Cc Thomas Gleixner <[email protected]>

Cc Wanpeng Li <[email protected]>

Cc Ingo Molnar <[email protected]>

---
 kernel/sched/core.c      | 88 ++++++++++++++++++++++++++++++++++++++++++++++--
 kernel/sched/isolation.c |  4 +++
 kernel/sched/sched.h     |  2 ++
 3 files changed, 91 insertions(+), 3 deletions(-)

kernel/sched/core.c +29 -3

index d72d0e9..b964890 100644
@@ -3052,9 +3052,14 @@ void scheduler_tick(void)
  */
 u64 scheduler_tick_max_deferment(void)
 {
-	struct rq *rq = this_rq();
-	unsigned long next, now = READ_ONCE(jiffies);
+	struct rq *rq;
+	unsigned long next, now;
 
+	if (!housekeeping_cpu(smp_processor_id(), HK_FLAG_TICK_SCHED))
+		return ktime_to_ns(KTIME_MAX);
+
+	rq = this_rq();
+	now = READ_ONCE(jiffies);
 	next = rq->last_sched_tick + HZ;
 
 	if (time_before_eq(next, now))

@@ -3062,7 +3067,82 @@ u64 scheduler_tick_max_deferment(void)
 
 	return jiffies_to_nsecs(next - now);
 }
-#endif
+
+struct tick_work {
+	int			cpu;
+	struct delayed_work	work;
+};
+
+static struct tick_work __percpu *tick_work_cpu;
+
+static void sched_tick_remote(struct work_struct *work)
+{
+	struct delayed_work *dwork = to_delayed_work(work);
+	struct tick_work *twork = container_of(dwork, struct tick_work, work);
+	int cpu = twork->cpu;
+	struct rq *rq = cpu_rq(cpu);
+	struct rq_flags rf;
+
+	/*
+	 * Handle the tick only if it appears the remote CPU is running
+	 * in full dynticks mode. The check is racy by nature, but
+	 * missing a tick or having one too much is no big deal.
+	 */
+	if (!idle_cpu(cpu) && tick_nohz_tick_stopped_cpu(cpu)) {

kernel/sched/core.c u64 scheduler_tick_max_deferment(void) · line 3096

Quoted 5 lines

kernel/sched/core.c +5 -0

+		rq_lock_irq(rq, &rf);
+		update_rq_clock(rq);
+		rq->curr->sched_class->task_tick(rq, rq->curr, 0);
+		rq_unlock_irq(rq, &rf);
+	}

OK, so this executes task_tick() remotely. What about account_process_tick()? Don't we need it as well?

In particular, when I run a hog application on a nohz_full core configured with tick offload, I can see in top that the CPU usage goes from 100% to idle for a few seconds every couple of seconds. Could this be related?

Also, in my testing I'm sometimes seeing the tick. Sometimes at 10 or 20 seconds interval. Is this expected? I'll dig deeper next week.

Quoted 81 lines · Luiz Capitulino

Quoted 78 lines · On Thu, 4 Jan 2018 05:25:36 +0100 Frederic Weisbecker <[email protected]>…

When a CPU runs in full dynticks mode, a 1Hz tick remains in order to keep the scheduler stats alive. However this residual tick is a burden for bare metal tasks that can't stand any interruption at all, or want to minimize them.

Adding the boot parameter "isolcpus=nohz_offload" will now outsource these scheduler ticks to the global workqueue so that a housekeeping CPU handles that tick remotely.

Note it's still up to the user to affine the global workqueues to the housekeeping CPUs through /sys/devices/virtual/workqueue/cpumask or domains isolation.

Signed-off-by Frederic Weisbecker <[email protected]>

Cc Chris Metcalf <[email protected]>

Cc Christoph Lameter <[email protected]>

Cc Luiz Capitulino <[email protected]>

Cc Mike Galbraith <[email protected]>

Cc Paul E. McKenney <[email protected]>

Cc Peter Zijlstra <[email protected]>

Cc Rik van Riel <[email protected]>

Cc Thomas Gleixner <[email protected]>

Cc Wanpeng Li <[email protected]>

Cc Ingo Molnar <[email protected]>

---
 kernel/sched/core.c      | 88 ++++++++++++++++++++++++++++++++++++++++++++++--
 kernel/sched/isolation.c |  4 +++
 kernel/sched/sched.h     |  2 ++
 3 files changed, 91 insertions(+), 3 deletions(-)

kernel/sched/core.c +29 -3

index d72d0e9..b964890 100644
@@ -3052,9 +3052,14 @@ void scheduler_tick(void)
  */
 u64 scheduler_tick_max_deferment(void)
 {
-	struct rq *rq = this_rq();
-	unsigned long next, now = READ_ONCE(jiffies);
+	struct rq *rq;
+	unsigned long next, now;
 
+	if (!housekeeping_cpu(smp_processor_id(), HK_FLAG_TICK_SCHED))
+		return ktime_to_ns(KTIME_MAX);
+
+	rq = this_rq();
+	now = READ_ONCE(jiffies);
 	next = rq->last_sched_tick + HZ;
 
 	if (time_before_eq(next, now))

@@ -3062,7 +3067,82 @@ u64 scheduler_tick_max_deferment(void)
 
 	return jiffies_to_nsecs(next - now);
 }
-#endif
+
+struct tick_work {
+	int			cpu;
+	struct delayed_work	work;
+};
+
+static struct tick_work __percpu *tick_work_cpu;
+
+static void sched_tick_remote(struct work_struct *work)
+{
+	struct delayed_work *dwork = to_delayed_work(work);
+	struct tick_work *twork = container_of(dwork, struct tick_work, work);
+	int cpu = twork->cpu;
+	struct rq *rq = cpu_rq(cpu);
+	struct rq_flags rf;
+
+	/*
+	 * Handle the tick only if it appears the remote CPU is running
+	 * in full dynticks mode. The check is racy by nature, but
+	 * missing a tick or having one too much is no big deal.
+	 */
+	if (!idle_cpu(cpu) && tick_nohz_tick_stopped_cpu(cpu)) {

Quoted 8 lines

kernel/sched/core.c u64 scheduler_tick_max_deferment(void) · line 3096

Quoted 5 lines

kernel/sched/core.c +5 -0

+		rq_lock_irq(rq, &rf);
+		update_rq_clock(rq);
+		rq->curr->sched_class->task_tick(rq, rq->curr, 0);
+		rq_unlock_irq(rq, &rf);
+	}  

OK, so this executes task_tick() remotely. What about account_process_tick()? Don't we need it as well?

Nope, tasks in nohz_full mode have their special accounting that doesn't rely on the tick.

Quoted 6 lines

In particular, when I run a hog application on a nohz_full core configured with tick offload, I can see in top that the CPU usage goes from 100% to idle for a few seconds every couple of seconds. Could this be related?

Also, in my testing I'm sometimes seeing the tick. Sometimes at 10 or 20 seconds interval. Is this expected? I'll dig deeper next week.

That's expected, see the changelog: the offload is not affine by default. You need to either also isolate the domains:

isolcpus=nohz_offload,domain

or tweak the workqueue cpumask through:

/sys/devices/virtual/workqueue/cpumask

Thanks.

Ingo,

Please pull the sched/0hz branch that can be found at:

git://git.kernel.org/pub/scm/linux/kernel/git/frederic/linux-dynticks.git
sched/0hz

HEAD: 9e932b2cc707209febd130978a5eb9f4a943a3f4

--
Now that scheduler_tick() has become resilient towards the absence of ticks, current->sched_class->task_tick() is the last piece that needs at least 1Hz tick to keep scheduler stats alive.

This patchset adds a flag to the isolcpus boot option to offload the residual 1Hz tick. This way the nohz_full CPUs don't have anymore tick (assuming nothing else requires it) as their residual 1Hz tick is offloaded to the housekeepers.

For quick testing, say on CPUs 1-7:

	"isolcpus=nohz_offload,domain,1-7"

Thanks,
Frederic
---

Frederic Weisbecker (5):
sched: Rename init_rq_hrtick to hrtick_rq_init sched/isolation: Add scheduler tick offloading interface
nohz: Allow to check if remote CPU tick is stopped sched/isolation: Residual 1Hz scheduler tick offload sched/isolation: Document "nohz_offload" flag

 Documentation/admin-guide/kernel-parameters.txt |  7 +-
 include/linux/sched/isolation.h                 |  3 +-
 include/linux/tick.h                            |  2 +
 kernel/sched/core.c                             | 94 +++++++++++++++++++++++--
 kernel/sched/isolation.c                        | 10 +++
 kernel/sched/sched.h                            |  2 +
 kernel/time/tick-sched.c                        |  7 ++
 7 files changed, 117 insertions(+), 8 deletions(-)

Quoted 21 lines · On Thu, 4 Jan 2018 05:25:32 +0100 Frederic Weisbecker <[email protected]>…

Ingo,

Please pull the sched/0hz branch that can be found at:

git://git.kernel.org/pub/scm/linux/kernel/git/frederic/linux-dynticks.git
sched/0hz

HEAD: 9e932b2cc707209febd130978a5eb9f4a943a3f4

--
Now that scheduler_tick() has become resilient towards the absence of ticks, current->sched_class->task_tick() is the last piece that needs at least 1Hz tick to keep scheduler stats alive.

This patchset adds a flag to the isolcpus boot option to offload the residual 1Hz tick. This way the nohz_full CPUs don't have anymore tick (assuming nothing else requires it) as their residual 1Hz tick is offloaded to the housekeepers.

For quick testing, say on CPUs 1-7:

Quoted 1 line

	"isolcpus=nohz_offload,domain,1-7"

Sorry for being very late to this series, but I've a few comments to make (one right now and others in individual patches).

Why are extending isolcpus= given that it's a deprecated interface? Some people have already moved away from isolcpus= now, but with this new feature they will be forced back to using it.

What about just adding the new functionality to nohz_full=? That is, no new options, just make the tick go away since this has always been what nohz_full= was intended to do?

Quoted 24 lines · Luiz Capitulino

Quoted 21 lines · On Thu, 4 Jan 2018 05:25:32 +0100 Frederic Weisbecker <[email protected]>…

Ingo,

Please pull the sched/0hz branch that can be found at:

git://git.kernel.org/pub/scm/linux/kernel/git/frederic/linux-dynticks.git
sched/0hz

HEAD: 9e932b2cc707209febd130978a5eb9f4a943a3f4

--
Now that scheduler_tick() has become resilient towards the absence of ticks, current->sched_class->task_tick() is the last piece that needs at least 1Hz tick to keep scheduler stats alive.

This patchset adds a flag to the isolcpus boot option to offload the residual 1Hz tick. This way the nohz_full CPUs don't have anymore tick (assuming nothing else requires it) as their residual 1Hz tick is offloaded to the housekeepers.

For quick testing, say on CPUs 1-7:

Quoted 8 lines

Quoted 1 line

	"isolcpus=nohz_offload,domain,1-7"  

Sorry for being very late to this series, but I've a few comments to make (one right now and others in individual patches).

Why are extending isolcpus= given that it's a deprecated interface? Some people have already moved away from isolcpus= now, but with this new feature they will be forced back to using it.

I tried to remove isolcpus or at least change the way it works so that its effects are reversible (ie: affine the init task instead of isolating domains) but that got nacked due to the behaviour's expectations for userspace.

That's when I realized that kernel parameters are like userspace ABIs, they can't be removed easily whether we deprecate them or not.

Also I needed to be able to control the various isolation features, and nohz_full is the wrong place to do that as nohz_full is really just an isolation feature like the others, nohz_full= should really just imply full dynticks and not watchdog, workqueue or tilegx NAPI isolation...

So isolcpus= is now the place where we control the isolation features and nohz is one of them.

The complain about isolcpus is the immutable result. I'm thinking about making it modifiable to cpuset but I only see two possible solutions:

- Make the root cpuset modifiable
- Create a directory called "isolcpus" visible on the first cpuset mount and move all processes there.

Quoted 3 lines

What about just adding the new functionality to nohz_full=? That is, no new options, just make the tick go away since this has always been what nohz_full= was intended to do?

We can, or have isolcpus=nohz to do it, as both do almost the same.

But I'm afraid about the overhead for people used to nohz_full= once they upgrade their kernels and see those workqueues once per second.

We can still affine those workqueues (in fact the whole unbound workqueue mask) outside the nohz_full range. Still current users may be surprised about that new overhead on housekeeping CPUs...

Quoted 2 lines · On Tue, 16 Jan 2018 16:41:00 +0100 Frederic Weisbecker <[email protected]>…

So isolcpus= is now the place where we control the isolation features and nohz is one of them.

That's the part I'm not very sure about. We've been advising users to move away from isolcpus= when possible, but this very wanted nohz_offload feature will force everyone back to using isolcpus= again.

I have the impression this series is trying to solve two problems:

1. How (and where) we control the various isolation features in the kernel

2. Where we add the control for the tick offload feature

I think item 1 is too complex to solve right now. IMHO, this series should focus on item 2. And regarding item 2, I think we have two choices to make:

1. Make tick offload a first class citizen by making it default to nohz_full=. If there are regressions, we handle them

2. Add a new option to nohz_full=, like nohz_full=tick_offload

As an avid user of nohz_full I'm dying to see option 1 happening, but I'm not totally sure what the consequences can be.

Another idea is to add CONFIG_NOHZ_TICK_OFFLOAD as an experimental feature.

Quoted 4 lines

The complain about isolcpus is the immutable result. I'm thinking about making it modifiable to cpuset but I only see two possible solutions:

Quoted 3 lines

- Make the root cpuset modifiable
- Create a directory called "isolcpus" visible on the first cpuset mount and move all processes there.

So, if we move the control of the tick offload to nohz_full= itself, we can completely ditch any isolcpus= change in this series.

I think this should give you a great relief :)

Quoted 8 lines · Luiz Capitulino

Quoted 2 lines · On Tue, 16 Jan 2018 16:41:00 +0100 Frederic Weisbecker <[email protected]>…

So isolcpus= is now the place where we control the isolation features and nohz is one of them.

That's the part I'm not very sure about. We've been advising users to move away from isolcpus= when possible, but this very wanted nohz_offload feature will force everyone back to using isolcpus= again.

Note "isolcpus=nohz" only implies nohz. You need to add "domain" to get the behaviour that you've been advising users against. We are simply reusing a kernel parameter that was abandoned to now control the isolation features that were disorganized and opaque behind nohz.

Quoted 5 lines

I have the impression this series is trying to solve two problems:

1. How (and where) we control the various isolation features in the kernel

No, that has already been done in the previous merge window. We have a dedicated isolation subsystem now (kernel/sched/isolation.c) and
an interface to control all these isolation features that were abusively implied by nohz. The initial plan was to introduce "cpu_isolation=" but it looked too much like "isolcpus=". Then in fact, why not using "isolcpus=" and give it a second life. And there we are.

Quoted 21 lines · On Tue, 16 Jan 2018 23:51:29 +0100 Frederic Weisbecker <[email protected]>…

Quoted 8 lines · Luiz Capitulino

Quoted 2 lines · On Tue, 16 Jan 2018 16:41:00 +0100 Frederic Weisbecker <[email protected]>…

So isolcpus= is now the place where we control the isolation features and nohz is one of them.

That's the part I'm not very sure about. We've been advising users to move away from isolcpus= when possible, but this very wanted nohz_offload feature will force everyone back to using isolcpus= again.

Note "isolcpus=nohz" only implies nohz. You need to add "domain" to get the behaviour that you've been advising users against. We are simply reusing a kernel parameter that was abandoned to now control the isolation features that were disorganized and opaque behind nohz.

Quoted 5 lines

I have the impression this series is trying to solve two problems:

1. How (and where) we control the various isolation features in the kernel

Quoted 6 lines

No, that has already been done in the previous merge window. We have a dedicated isolation subsystem now (kernel/sched/isolation.c) and
an interface to control all these isolation features that were abusively implied by nohz. The initial plan was to introduce "cpu_isolation=" but it looked too much like "isolcpus=". Then in fact, why not using "isolcpus=" and give it a second life. And there we are.

OK, I get it now. But then series has to un-deprecate isolcpus= otherwise it doesn't make sense to use it.

Quoted 31 lines · Luiz Capitulino

Quoted 21 lines · On Tue, 16 Jan 2018 23:51:29 +0100 Frederic Weisbecker <[email protected]>…

Quoted 8 lines · Luiz Capitulino

Quoted 2 lines · On Tue, 16 Jan 2018 16:41:00 +0100 Frederic Weisbecker <[email protected]>…

So isolcpus= is now the place where we control the isolation features and nohz is one of them.

That's the part I'm not very sure about. We've been advising users to move away from isolcpus= when possible, but this very wanted nohz_offload feature will force everyone back to using isolcpus= again.

Note "isolcpus=nohz" only implies nohz. You need to add "domain" to get the behaviour that you've been advising users against. We are simply reusing a kernel parameter that was abandoned to now control the isolation features that were disorganized and opaque behind nohz.

Quoted 5 lines

I have the impression this series is trying to solve two problems:

1. How (and where) we control the various isolation features in the kernel

Quoted 6 lines

No, that has already been done in the previous merge window. We have a dedicated isolation subsystem now (kernel/sched/isolation.c) and
an interface to control all these isolation features that were abusively implied by nohz. The initial plan was to introduce "cpu_isolation=" but it looked too much like "isolcpus=". Then in fact, why not using "isolcpus=" and give it a second life. And there we are.

Quoted 2 lines

OK, I get it now. But then series has to un-deprecate isolcpus= otherwise it doesn't make sense to use it.

Good point. Also I think you convinced me toward just applying that tick offload on the existing nohz kernel parameter right away, that is, to both existing "nohz_full=" and "isolcpus=nohz".

After all that tick offload is an implementation detail.

Like you said if people complain about a regression, we can still fix it with a new option. But eventually I doubt this will be needed.

I'll respin with that.

Quoted 3 lines · Luiz Capitulino

Why are extending isolcpus= given that it's a deprecated interface? Some people have already moved away from isolcpus= now, but with this new feature they will be forced back to using it.

I tried to remove isolcpus or at least change the way it works so that its effects are reversible (ie: affine the init task instead of isolating domains) but that got nacked due to the behaviour's expectations for userspace.

Quoted 6 lines · Frederic Weisbecker

Quoted 3 lines · Luiz Capitulino

Why are extending isolcpus= given that it's a deprecated interface? Some people have already moved away from isolcpus= now, but with this new feature they will be forced back to using it.

Quoted 3 lines

I tried to remove isolcpus or at least change the way it works so that its effects are reversible (ie: affine the init task instead of isolating domains) but that got nacked due to the behaviour's expectations for userspace.

So we paint ourselves into a static corner forever more, despite every bit of this being all about "properties of sets of cpus", ie precisely what cpusets was born to do.  That's sad, dynamic wasn't that far away.

Quoted 3 lines

I tried to remove isolcpus or at least change the way it works so that its effects are reversible (ie: affine the init task instead of isolating domains) but that got nacked due to the behaviour's expectations for userspace.

So we paint ourselves into a static corner forever more, despite every bit of this being all about "properties of sets of cpus", ie precisely what cpusets was born to do.  That's sad, dynamic wasn't that far away.

Quoted 4 lines · Mike Galbraith

Quoted 3 lines

I tried to remove isolcpus or at least change the way it works so that its effects are reversible (ie: affine the init task instead of isolating domains) but that got nacked due to the behaviour's expectations for userspace.

Quoted 3 lines

So we paint ourselves into a static corner forever more, despite every bit of this being all about "properties of sets of cpus", ie precisely what cpusets was born to do.  That's sad, dynamic wasn't that far away.

cpusets was born in order to isolate applications to sets of processors. The properties of sets of cpus was not on the horizon when SGI started this.

We have sets of cpus associated with affinity masks in the form of bitmaps etc etc which is much more lightweight than having slug around the cgroup overhead everywhere.

A simple bitmask is much better if you have to control detailed system behavior for each core and are planning each processes role because you need to make full use of the harware resources available.

Domain connectivity very much is a property of a set of CPUs, a rather important one, and one managed by cpusets.  NOHZ_FULL is a property of a set of cpus, thus a most excellent fit.  Other things are as well.

Quoted 3 lines

We have sets of cpus associated with affinity masks in the form of bitmaps etc etc which is much more lightweight than having slug around the cgroup overhead everywhere.

What does everywhere mean, set creation time?

Quoted 3 lines

A simple bitmask is much better if you have to control detailed system behavior for each core and are planning each processes role because you need to make full use of the harware resources available.

If you live in a static world, maybe.

I like the flexibility of being able to configure on the fly.  One tiny
example: for a high performance aircraft manufacturer, having military simulation background, I know that simulators frequently have to be ready to go at the drop of a hat, so I twiddled cpusets to let them flip their extra fancy video game (80 cores, real controls/avionics... "game over, insert one gold bar to continue" kind of fancy) from low power idle to full bore hard realtime with one poke to a cpuset file.

Static may be fine for some, for others, dynamic is much better.

Quoted 3 lines · Mike Galbraith

Domain connectivity very much is a property of a set of CPUs, a rather important one, and one managed by cpusets.  NOHZ_FULL is a property of a set of cpus, thus a most excellent fit.  Other things are as well.

Not sure to what domain refers to in this context.

Quoted 5 lines

Quoted 3 lines

We have sets of cpus associated with affinity masks in the form of bitmaps etc etc which is much more lightweight than having slug around the cgroup overhead everywhere.

What does everywhere mean, set creation time?

You would need to create multiple cgroups to create what you want. Those will "inherit" characteristics from higher levels etc etc. It gets needlessly complicated and difficult to debug if something goes work.

Quoted 5 lines

Quoted 3 lines

A simple bitmask is much better if you have to control detailed system behavior for each core and are planning each processes role because you need to make full use of the harware resources available.

If you live in a static world, maybe.

Why would that be restricted to a static world?

Quoted 8 lines

I like the flexibility of being able to configure on the fly.  One tiny
example: for a high performance aircraft manufacturer, having military simulation background, I know that simulators frequently have to be ready to go at the drop of a hat, so I twiddled cpusets to let them flip their extra fancy video game (80 cores, real controls/avionics... "game over, insert one gold bar to continue" kind of fancy) from low power idle to full bore hard realtime with one poke to a cpuset file.

Quoted 1 line

Static may be fine for some, for others, dynamic is much better.

The problem is that I may be flipping a flag in a cpuset to enable something but some other cpuset somewhere in the complex hieracy does something different that causes a conflict.

The directness to control is
lost. Instead there is the fog of complexity created by the cgroups that have various plugins and whatnot.