nohz: Move nohz kick out of scheduler IPI, v3

We are going to extend irq work to support remote queuing.

So lets add a cpu argument to arch_irq_work_raise(). The architectures willing to support that must then provide the backend to raise irq work IPIs remotely.

Initial support is provided for x86 and ARM since they are easily extended. The other archs that overwrite arch_irq_work_raise() seem to use local clock interrupts and therefore need deeper rewrite of their irq work support to implement remote raising.

Cc Andrew Morton <[email protected]>

Cc Benjamin Herrenschmidt <[email protected]>

Cc David S. Miller <[email protected]>

Cc Ingo Molnar <[email protected]>

Cc Kevin Hilman <[email protected]>

Cc Paul E. McKenney <[email protected]>

Cc Paul Mackerras <[email protected]>

Cc Peter Zijlstra <[email protected]>

Cc Russell King <[email protected]>

Cc Thomas Gleixner <[email protected]>

Cc Viresh Kumar <[email protected]>

Signed-off-by Frederic Weisbecker <[email protected]>

---
 arch/Kconfig               | 12 ++++++++++++
 arch/alpha/kernel/time.c   |  3 ++-
 arch/arm/Kconfig           |  1 +
 arch/arm/kernel/smp.c      |  4 ++--
 arch/powerpc/kernel/time.c |  3 ++-
 arch/sparc/kernel/pcr.c    |  3 ++-
 arch/x86/Kconfig           |  1 +
 arch/x86/kernel/irq_work.c | 10 ++--------
 kernel/irq_work.c          |  4 ++--
 9 files changed, 26 insertions(+), 15 deletions(-)

arch/Kconfig +12 -0

index 97ff872..3a38356 100644
@@ -472,6 +472,18 @@ config HAVE_IRQ_EXIT_ON_IRQ_STACK
 	  This spares a stack switch and improves cache usage on softirq
 	  processing.
 
+config HAVE_IRQ_WORK_IPI
+	bool
+	help
+	  Architecture supports raising irq work interrupts both locally and
+	  remotely. Without this capability, we can only trigger local irq works
+	  loosely handled by the generic timer tick with the bad implications
+	  coming along: the irq work is subject to HZ latency and it runs under
+	  the tick random locking scenario (possibly holding hrtimer lock).
+
+	  This capability is required on configs running with a very minimized
+	  tick rate like full dynticks.
+
 #
 # ABI hall of shame
 #

arch/alpha/kernel/time.c +2 -1

index ee39cee..2ff0c61 100644
@@ -60,8 +60,9 @@ DEFINE_PER_CPU(u8, irq_work_pending);
 #define test_irq_work_pending()      __get_cpu_var(irq_work_pending)
 #define clear_irq_work_pending()     __get_cpu_var(irq_work_pending) = 0
 
-void arch_irq_work_raise(void)
+void arch_irq_work_raise(int cpu)
 {
+	WARN_ON_ONCE(cpu != smp_processor_id());
 	set_irq_work_pending_flag();
 }
 

arch/arm/Kconfig +1 -0

index db3c541..7edce21 100644
@@ -46,6 +46,7 @@ config ARM
 	select HAVE_HW_BREAKPOINT if (PERF_EVENTS && (CPU_V6 || CPU_V6K || CPU_V7))
 	select HAVE_IDE if PCI || ISA || PCMCIA
 	select HAVE_IRQ_TIME_ACCOUNTING
+	select HAVE_IRQ_WORK_IPI
 	select HAVE_KERNEL_GZIP
 	select HAVE_KERNEL_LZ4
 	select HAVE_KERNEL_LZMA

arch/arm/kernel/smp.c +2 -2

index 7c4fada..042a800 100644
@@ -454,10 +454,10 @@ void arch_send_call_function_single_ipi(int cpu)
 }
 
 #ifdef CONFIG_IRQ_WORK
-void arch_irq_work_raise(void)
+void arch_irq_work_raise(int cpu)
 {
 	if (is_smp())
-		smp_cross_call(cpumask_of(smp_processor_id()), IPI_IRQ_WORK);
+		smp_cross_call(cpumask_of(cpu), IPI_IRQ_WORK);
 }
 #endif
 

arch/powerpc/kernel/time.c +2 -1

index 122a580..4de25f4 100644
@@ -464,9 +464,10 @@ DEFINE_PER_CPU(u8, irq_work_pending);
 
 #endif /* 32 vs 64 bit */
 
-void arch_irq_work_raise(void)
+void arch_irq_work_raise(int cpu)
 {
 	preempt_disable();
+	WARN_ON_ONCE(cpu != smp_processor_id());
 	set_irq_work_pending_flag();
 	set_dec(1);
 	preempt_enable();

arch/sparc/kernel/pcr.c +2 -1

index 269af58..0e5bfd9 100644
@@ -43,8 +43,9 @@ void __irq_entry deferred_pcr_work_irq(int irq, struct pt_regs *regs)
 	set_irq_regs(old_regs);
 }
 
-void arch_irq_work_raise(void)
+void arch_irq_work_raise(int cpu)
 {
+	WARN_ON_ONCE(cpu != smp_processor_id());
 	set_softint(1 << PIL_DEFERRED_PCR_WORK);
 }
 

arch/x86/Kconfig +1 -0

index 25d2c6f..b06f3fd 100644
@@ -130,6 +130,7 @@ config X86
 	select HAVE_CC_STACKPROTECTOR
 	select GENERIC_CPU_AUTOPROBE
 	select HAVE_ARCH_AUDITSYSCALL
+	select HAVE_IRQ_WORK_IPI
 
 config INSTRUCTION_DECODER
 	def_bool y

arch/x86/kernel/irq_work.c +2 -8

index 1de84e3..500ec1f 100644
@@ -38,13 +38,7 @@ __visible void smp_trace_irq_work_interrupt(struct pt_regs *regs)
 	exiting_irq();
 }
 
-void arch_irq_work_raise(void)
+void arch_irq_work_raise(int cpu)
 {
-#ifdef CONFIG_X86_LOCAL_APIC
-	if (!cpu_has_apic)
-		return;
-
-	apic->send_IPI_self(IRQ_WORK_VECTOR);
-	apic_wait_icr_idle();
-#endif
+	apic->send_IPI_mask(cpumask_of(cpu), IRQ_WORK_VECTOR);
 }

kernel/irq_work.c +2 -2

index a82170e..2559383 100644
@@ -48,7 +48,7 @@ static bool irq_work_claim(struct irq_work *work)
 	return true;
 }
 
-void __weak arch_irq_work_raise(void)
+void __weak arch_irq_work_raise(int cpu)
 {
 	/*
 	 * Lame architectures will get the timer tick callback

@@ -79,7 +79,7 @@ bool irq_work_queue(struct irq_work *work)
 	 */
 	if (!(work->flags & IRQ_WORK_LAZY) || tick_nohz_tick_stopped()) {
 		if (!this_cpu_cmpxchg(irq_work_raised, 0, 1))
-			arch_irq_work_raise();
+			arch_irq_work_raise(smp_processor_id());
 	}
 
 	preempt_enable();

Quoted 10 lines · Frederic Weisbecker

We are going to extend irq work to support remote queuing.

So lets add a cpu argument to arch_irq_work_raise(). The architectures willing to support that must then provide the backend to raise irq work IPIs remotely.

Initial support is provided for x86 and ARM since they are easily extended. The other archs that overwrite arch_irq_work_raise() seem to use local clock interrupts and therefore need deeper rewrite of their irq work support to implement remote raising.

Well, looks like it's time to turn it into an IPI... It gets a bit more tricky because whether whacking the interrupt controller is safe to do from an NMI is safe or not might depend on that irq controller implementation...

It looks like XICS and MPIC should be safe though, so at least we should be able to cover ppc64, but I'll leave ppc32 alone.

Quoted 12 lines · Benjamin Herrenschmidt

Quoted 10 lines · Frederic Weisbecker

We are going to extend irq work to support remote queuing.

So lets add a cpu argument to arch_irq_work_raise(). The architectures willing to support that must then provide the backend to raise irq work IPIs remotely.

Initial support is provided for x86 and ARM since they are easily extended. The other archs that overwrite arch_irq_work_raise() seem to use local clock interrupts and therefore need deeper rewrite of their irq work support to implement remote raising.

Quoted 7 lines

Well, looks like it's time to turn it into an IPI... It gets a bit more tricky because whether whacking the interrupt controller is safe to do from an NMI is safe or not might depend on that irq controller implementation...

It looks like XICS and MPIC should be safe though, so at least we should be able to cover ppc64, but I'll leave ppc32 alone.

Correction... that's actually a bit more tricky. We might need an MMIO to trigger the IPI. That means potentially having to take a hash miss, and we certainly can't do that at NMI time at the moment.

We *could* hard disable interrupts (which blocks our NMIs since they arent't real NMIs, they are just a way to bypass our soft-disable state for perf interrupts) for hash_page, but that still makes me somewhat nervous.

Another option would be to add an ioremap flag of some description to be able to install bolted hash entries. (It already does so if called early enough during boot, so it might actually just work by accident but that's an undebuggable horror show waiting to happen if we ever change that).

So needs a bit more thinking on our side.

We are going to extend irq work to support remote queuing.

So lets add a cpu argument to arch_irq_work_raise(). The architectures willing to support that must then provide the backend to raise irq work IPIs remotely.

Initial support is provided for x86 and ARM since they are easily extended. The other archs that overwrite arch_irq_work_raise() seem to use local clock interrupts and therefore need deeper rewrite of their irq work support to implement remote raising.

Signed-off-by Frederic Weisbecker <[email protected]>

Quoted 11 lines · Frederic Weisbecker

We are going to extend irq work to support remote queuing.

So lets add a cpu argument to arch_irq_work_raise(). The architectures willing to support that must then provide the backend to raise irq work IPIs remotely.

Initial support is provided for x86 and ARM since they are easily extended. The other archs that overwrite arch_irq_work_raise() seem to use local clock interrupts and therefore need deeper rewrite of their irq work support to implement remote raising.

Quoted 1 line

Signed-off-by Frederic Weisbecker <[email protected]>

Why not borrow the smp_call_function IPI for the remote bits? We could limit the 'safe from NMI' to the local works. And we validate this by putting a WARN_ON(in_nmi()) in irq_work_queue_on().

At some later stage we could look at maybe merging the smp_function_call and irq_work into a single thing, where we have the irq_work interface as async and the smp_function_call() as sync interface.

But for now a quick 'hack' would be to call __irq_work_run() from generic_smp_call_function_single_interrupt().

That would leave arch_irq_work_raise() as the special NMI safe local IPI hook.

Quoted 13 lines · Peter Zijlstra

Quoted 11 lines · Frederic Weisbecker

We are going to extend irq work to support remote queuing.

So lets add a cpu argument to arch_irq_work_raise(). The architectures willing to support that must then provide the backend to raise irq work IPIs remotely.

Initial support is provided for x86 and ARM since they are easily extended. The other archs that overwrite arch_irq_work_raise() seem to use local clock interrupts and therefore need deeper rewrite of their irq work support to implement remote raising.

Quoted 5 lines

Quoted 1 line

Signed-off-by Frederic Weisbecker <[email protected]>

Why not borrow the smp_call_function IPI for the remote bits? We could limit the 'safe from NMI' to the local works. And we validate this by putting a WARN_ON(in_nmi()) in irq_work_queue_on().

Right, but although I don't need it to be safe from NMI, I need it to be callable concurrently and when irqs are disabled.

So we can't use smp_call_function_single() for that. But we can use the async version in which case we must keep the irq work claim. But that's about the same than smp_queue_function_single() we had previously and we are back with our csd_lock issue.

Quoted 23 lines · Frederic Weisbecker

Quoted 13 lines · Peter Zijlstra

Quoted 11 lines · Frederic Weisbecker

We are going to extend irq work to support remote queuing.

So lets add a cpu argument to arch_irq_work_raise(). The architectures willing to support that must then provide the backend to raise irq work IPIs remotely.

Initial support is provided for x86 and ARM since they are easily extended. The other archs that overwrite arch_irq_work_raise() seem to use local clock interrupts and therefore need deeper rewrite of their irq work support to implement remote raising.

Quoted 5 lines

Quoted 1 line

Signed-off-by Frederic Weisbecker <[email protected]>

Why not borrow the smp_call_function IPI for the remote bits? We could limit the 'safe from NMI' to the local works. And we validate this by putting a WARN_ON(in_nmi()) in irq_work_queue_on().

Right, but although I don't need it to be safe from NMI, I need it to be callable concurrently and when irqs are disabled.

Quoted 4 lines

So we can't use smp_call_function_single() for that. But we can use the async version in which case we must keep the irq work claim. But that's about the same than smp_queue_function_single() we had previously and we are back with our csd_lock issue.

Who said anything about using smp_call_function_single()?

---

kernel/irq_work.c +2 -1

index a82170e2fa78..2fc9d8ece05a 100644
@@ -61,7 +61,8 @@ void __weak arch_irq_work_raise(void)
  *
  * Can be re-enqueued while the callback is still in progress.
  */
-bool irq_work_queue(struct irq_work *work)
+static __always_inline bool
+__irq_work_queue_on(struct irq_work *work, int cpu)
 {
 	/* Only queue if not already pending */
 	if (!irq_work_claim(work))

@@ -78,16 +79,31 @@ bool irq_work_queue(struct irq_work *work)
 	 * for the next tick.

kernel/irq_work.c +11 -0

 	}
 
 	preempt_enable();
 
 	return true;
 }
+
+bool irq_work_queue(struct irq_work *work)
+{
+	return __irq_work_queue_on(work, -1);
+}
 EXPORT_SYMBOL_GPL(irq_work_queue);
 
+bool irq_work_queue_on(struct irq_work *work, int cpu)
+{
+	WARN_ON_ONCE(in_nmi());
+	return __irq_work_queue_on(work, cpu);
+}
+
 bool irq_work_needs_cpu(void)
 {
 	struct llist_head *this_list;

kernel/smp.c +0 -0

index 06d574e42c72..0fd53963c4fb 100644