Skip to contentSkip to conversation

smp: irq_work / smp_call_function rework

PATCH In review Peter Zijlstra 3 participants 24 messages Oct 28, 2020 – Oct 29, 2020

Subsystems (21)
lore
Share & tools
lore
author maintainer
#12 lore raw

Re: [PATCH v3 5/6] irq_work: Provide irq_work_queue_remote()

In reply to Frederic Weisbecker
Quoted 6 lines · Frederic Weisbecker
Quoted 3 lines · Peter Zijlstra

While the traditional irq_work relies on the ability to self-IPI, it makes sense to provide an unconditional irq_work_queue_remote() interface.

We may need a reason as well here.

Rewrote it like so.
---

While the traditional irq_work relies on the ability to self-IPI, the remote irq_work only requires generic SMP bits and can thus be made available unconditionally.

author maintainer
#3 lore raw

PATCH v3 2/6 smp: Cleanup smp_call_function*()

In reply to Peter Zijlstra

Get rid of the __call_single_node union and cleanup the API a little to avoid external code relying on the structure layout as much.

---
 arch/mips/kernel/process.c                      |    5 +-
 arch/mips/kernel/smp.c                          |   25 ++----------
 arch/s390/pci/pci_irq.c                         |    4 -
 arch/x86/kernel/cpuid.c                         |    7 +--
 arch/x86/lib/msr-smp.c                          |    7 +--
 block/blk-mq.c                                  |    4 -
 drivers/cpuidle/coupled.c                       |    3 -
 drivers/net/ethernet/cavium/liquidio/lio_core.c |    9 ----
 include/linux/smp.h                             |   19 ++++-----
 kernel/debug/debug_core.c                       |    6 +-
 kernel/sched/core.c                             |   12 -----
 kernel/smp.c                                    |   50 ++++++++++++------------
 net/core/dev.c                                  |    3 -
 13 files changed, 60 insertions(+), 94 deletions(-)
arch/mips/kernel/process.c +3 -2
@@ -702,7 +702,6 @@ unsigned long arch_align_stack(unsigned
 	return sp & ALMASK;
 }
 
-static DEFINE_PER_CPU(call_single_data_t, backtrace_csd);
 static struct cpumask backtrace_csd_busy;
 
 static void handle_backtrace(void *info)
@@ -711,6 +710,9 @@ static void handle_backtrace(void *info)
 	cpumask_clear_cpu(smp_processor_id(), &backtrace_csd_busy);
 }
 
+static DEFINE_PER_CPU(call_single_data_t, backtrace_csd) =
+	CSD_INIT(handle_backtrace, NULL);
+
 static void raise_backtrace(cpumask_t *mask)
 {
 	call_single_data_t *csd;
@@ -730,7 +732,6 @@ static void raise_backtrace(cpumask_t *m
 		}
 
 		csd = &per_cpu(backtrace_csd, cpu);
-		csd->func = handle_backtrace;
 		smp_call_function_single_async(cpu, csd);
 	}
 }
arch/mips/kernel/smp.c +6 -19
@@ -687,36 +687,23 @@ EXPORT_SYMBOL(flush_tlb_one);
 
 #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
 
-static DEFINE_PER_CPU(call_single_data_t, tick_broadcast_csd);
-
-void tick_broadcast(const struct cpumask *mask)
-{
-	call_single_data_t *csd;
-	int cpu;
-
-	for_each_cpu(cpu, mask) {
-		csd = &per_cpu(tick_broadcast_csd, cpu);
-		smp_call_function_single_async(cpu, csd);
-	}
-}
-
 static void tick_broadcast_callee(void *info)
 {
 	tick_receive_broadcast();
 }
 
-static int __init tick_broadcast_init(void)
+static DEFINE_PER_CPU(call_single_data_t, tick_broadcast_csd) =
+	CSD_INIT(tick_broadcast_callee, NULL);
+
+void tick_broadcast(const struct cpumask *mask)
 {
 	call_single_data_t *csd;
 	int cpu;
 
-	for (cpu = 0; cpu < NR_CPUS; cpu++) {
+	for_each_cpu(cpu, mask) {
 		csd = &per_cpu(tick_broadcast_csd, cpu);
-		csd->func = tick_broadcast_callee;
+		smp_call_function_single_async(cpu, csd);
 	}
-
-	return 0;
 }
-early_initcall(tick_broadcast_init);
 
 #endif /* CONFIG_GENERIC_CLOCKEVENTS_BROADCAST */
arch/s390/pci/pci_irq.c +1 -3
@@ -178,9 +178,7 @@ static void zpci_handle_fallback_irq(voi
 		if (atomic_inc_return(&cpu_data->scheduled) > 1)
 			continue;
 
-		cpu_data->csd.func = zpci_handle_remote_irq;
-		cpu_data->csd.info = &cpu_data->scheduled;
-		cpu_data->csd.flags = 0;
+		INIT_CSD(&cpu_data->csd, zpci_handle_remote_irq, &cpu_data->scheduled);
 		smp_call_function_single_async(cpu, &cpu_data->csd);
 	}
 }
arch/x86/kernel/cpuid.c +3 -4
@@ -74,10 +74,9 @@ static ssize_t cpuid_read(struct file *f
 
 	init_completion(&cmd.done);
 	for (; count; count -= 16) {
-		call_single_data_t csd = {
-			.func = cpuid_smp_cpuid,
-			.info = &cmd,
-		};
+		call_single_data_t csd;
+
+		INIT_CSD(&csd, cpuid_smp_cpuid, &cmd);
 
 		cmd.regs.eax = pos;
 		cmd.regs.ecx = pos >> 32;
arch/x86/lib/msr-smp.c +3 -4
@@ -169,12 +169,11 @@ static void __wrmsr_safe_on_cpu(void *in
 int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h)
 {
 	struct msr_info_completion rv;
-	call_single_data_t csd = {
-		.func	= __rdmsr_safe_on_cpu,
-		.info	= &rv,
-	};
+	call_single_data_t csd;
 	int err;
 
+	INIT_CSD(&csd, __rdmsr_safe_on_cpu, &rv);
+
 	memset(&rv, 0, sizeof(rv));
 	init_completion(&rv.done);
 	rv.msr.msr_no = msr_no;
block/blk-mq.c +1 -3
@@ -671,9 +671,7 @@ bool blk_mq_complete_request_remote(stru
 		return false;
 
 	if (blk_mq_complete_need_ipi(rq)) {
-		rq->csd.func = __blk_mq_complete_request_remote;
-		rq->csd.info = rq;
-		rq->csd.flags = 0;
+		INIT_CSD(&rq->csd, __blk_mq_complete_request_remote, rq);
 		smp_call_function_single_async(rq->mq_ctx->cpu, &rq->csd);
 	} else {
 		if (rq->q->nr_hw_queues > 1)
drivers/cpuidle/coupled.c +1 -2
@@ -674,8 +674,7 @@ int cpuidle_coupled_register_device(stru
 	coupled->refcnt++;
 
 	csd = &per_cpu(cpuidle_coupled_poke_cb, dev->cpu);
-	csd->func = cpuidle_coupled_handle_poke;
-	csd->info = (void *)(unsigned long)dev->cpu;
+	INIT_CSD(csd, cpuidle_coupled_handle_poke, (void *)(unsigned long)dev->cpu);
 
 	return 0;
 }
drivers/net/ethernet/cavium/liquidio/lio_core.c +2 -7
@@ -729,13 +729,8 @@ static void liquidio_napi_drv_callback(v
 	    droq->cpu_id == this_cpu) {
 		napi_schedule_irqoff(&droq->napi);
 	} else {
-		call_single_data_t *csd = &droq->csd;
-
-		csd->func = napi_schedule_wrapper;
-		csd->info = &droq->napi;
-		csd->flags = 0;
-
-		smp_call_function_single_async(droq->cpu_id, csd);
+		INIT_CSD(&droq->csd, napi_schedule_wrapper, &droq->napi);
+		smp_call_function_single_async(droq->cpu_id, &droq->csd);
 	}
 }
 
include/linux/smp.h +9 -10
@@ -21,24 +21,23 @@ typedef bool (*smp_cond_func_t)(int cpu,
  * structure shares (partial) layout with struct irq_work
  */
 struct __call_single_data {
-	union {
-		struct __call_single_node node;
-		struct {
-			struct llist_node llist;
-			unsigned int flags;
-#ifdef CONFIG_64BIT
-			u16 src, dst;
-#endif
-		};
-	};
+	struct __call_single_node node;
 	smp_call_func_t func;
 	void *info;
 };
 
+#define CSD_INIT(_func, _info) \
+	(struct __call_single_data){ .func = (_func), .info = (_info), }
+
 /* Use __aligned() to avoid to use 2 cache lines for 1 csd */
 typedef struct __call_single_data call_single_data_t
 	__aligned(sizeof(struct __call_single_data));
 
+#define INIT_CSD(_csd, _func, _info)		\
+do {						\
+	*(_csd) = CSD_INIT((_func), (_info));	\
+} while (0)
+
 /*
  * Enqueue a llist_node on the call_single_queue; be very careful, read
  * flush_smp_call_function_queue() in detail.
kernel/debug/debug_core.c +3 -3
@@ -225,8 +225,6 @@ NOKPROBE_SYMBOL(kgdb_skipexception);
  * Default (weak) implementation for kgdb_roundup_cpus
  */
 
-static DEFINE_PER_CPU(call_single_data_t, kgdb_roundup_csd);
-
 void __weak kgdb_call_nmi_hook(void *ignored)
 {
 	/*
@@ -241,6 +239,9 @@ void __weak kgdb_call_nmi_hook(void *ign
 }
 NOKPROBE_SYMBOL(kgdb_call_nmi_hook);
 
+static DEFINE_PER_CPU(call_single_data_t, kgdb_roundup_csd) =
+	CSD_INIT(kgdb_call_nmi_hook, NULL);
+
 void __weak kgdb_roundup_cpus(void)
 {
 	call_single_data_t *csd;
@@ -267,7 +268,6 @@ void __weak kgdb_roundup_cpus(void)
 			continue;
 		kgdb_info[cpu].rounding_up = true;
 
-		csd->func = kgdb_call_nmi_hook;
 		ret = smp_call_function_single_async(cpu, csd);
 		if (ret)
 			kgdb_info[cpu].rounding_up = false;
kernel/sched/core.c +2 -10
@@ -320,14 +320,6 @@ void update_rq_clock(struct rq *rq)
 	update_rq_clock_task(rq, delta);
 }
 
-static inline void
-rq_csd_init(struct rq *rq, call_single_data_t *csd, smp_call_func_t func)
-{
-	csd->flags = 0;
-	csd->func = func;
-	csd->info = rq;
-}
-
 #ifdef CONFIG_SCHED_HRTICK
 /*
  * Use HR-timers to deliver accurate preemption points.
@@ -428,7 +420,7 @@ void hrtick_start(struct rq *rq, u64 del
 static void hrtick_rq_init(struct rq *rq)
 {
 #ifdef CONFIG_SMP
-	rq_csd_init(rq, &rq->hrtick_csd, __hrtick_start);
+	INIT_CSD(&rq->hrtick_csd, __hrtick_start, rq);
 #endif
 	hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
 	rq->hrtick_timer.function = hrtick;
@@ -7188,7 +7180,7 @@ void __init sched_init(void)
 		rq->last_blocked_load_update_tick = jiffies;
 		atomic_set(&rq->nohz_flags, 0);
 
-		rq_csd_init(rq, &rq->nohz_csd, nohz_csd_func);
+		INIT_CSD(&rq->nohz_csd, nohz_csd_func, rq);
 #endif
 #endif /* CONFIG_SMP */
 		hrtick_rq_init(rq);
kernel/smp.c +25 -25

164 lines not on this page · raw message

net/core/dev.c +1 -2
@@ -11165,8 +11165,7 @@ static int __init net_dev_init(void)
 		INIT_LIST_HEAD(&sd->poll_list);
 		sd->output_queue_tailp = &sd->output_queue;
 #ifdef CONFIG_RPS
-		sd->csd.func = rps_trigger_softirq;
-		sd->csd.info = sd;
+		INIT_CSD(&sd->csd, rps_trigger_softirq, sd);
 		sd->cpu = i;
 #endif
 
author maintainer
#1 lore raw

PATCH v3 1/6 irq_work: Cleanup

In reply to Peter Zijlstra

Get rid of the __call_single_node union and clean up the API a little to avoid external code relying on the structure layout as much.

---
 drivers/gpu/drm/i915/i915_request.c |    4 ++--
 include/linux/irq_work.h            |   33 +++++++++++++++++++++------------
 include/linux/irqflags.h            |    4 ++--
 kernel/bpf/stackmap.c               |    2 +-
 kernel/irq_work.c                   |   18 +++++++++---------
 kernel/printk/printk.c              |    6 ++----
 kernel/rcu/tree.c                   |    3 +--
 kernel/time/tick-sched.c            |    6 ++----
 kernel/trace/bpf_trace.c            |    2 +-
 9 files changed, 41 insertions(+), 37 deletions(-)
drivers/gpu/drm/i915/i915_request.c +2 -2
@@ -197,7 +197,7 @@ __notify_execute_cb(struct i915_request
 
 	llist_for_each_entry_safe(cb, cn,
 				  llist_del_all(&rq->execute_cb),
-				  work.llnode)
+				  work.node.llist)
 		fn(&cb->work);
 }
 
@@ -460,7 +460,7 @@ __await_execution(struct i915_request *r
 	 * callback first, then checking the ACTIVE bit, we serialise with
 	 * the completed/retired request.
 	 */
-	if (llist_add(&cb->work.llnode, &signal->execute_cb)) {
+	if (llist_add(&cb->work.node.llist, &signal->execute_cb)) {
 		if (i915_request_is_active(signal) ||
 		    __request_in_flight(signal))
 			__notify_execute_cb_imm(signal);
include/linux/irq_work.h +21 -12
@@ -14,28 +14,37 @@
  */
 
 struct irq_work {
-	union {
-		struct __call_single_node node;
-		struct {
-			struct llist_node llnode;
-			atomic_t flags;
-		};
-	};
+	struct __call_single_node node;
 	void (*func)(struct irq_work *);
 };
 
+#define __IRQ_WORK_INIT(_func, _flags) (struct irq_work){	\
+	.node = { .u_flags = (_flags), },			\
+	.func = (_func),					\
+}
+
+#define IRQ_WORK_INIT(_func) __IRQ_WORK_INIT(_func, 0)
+#define IRQ_WORK_INIT_LAZY(_func) __IRQ_WORK_INIT(_func, IRQ_WORK_LAZY)
+#define IRQ_WORK_INIT_HARD(_func) __IRQ_WORK_INIT(_func, IRQ_WORK_HARD_IRQ)
+
+#define DEFINE_IRQ_WORK(name, _f)				\
+	struct irq_work name = IRQ_WORK_INIT(_f)
+
 static inline
 void init_irq_work(struct irq_work *work, void (*func)(struct irq_work *))
 {
-	atomic_set(&work->flags, 0);
-	work->func = func;
+	*work = IRQ_WORK_INIT(func);
 }
 
-#define DEFINE_IRQ_WORK(name, _f) struct irq_work name = {	\
-		.flags = ATOMIC_INIT(0),			\
-		.func  = (_f)					\
+static inline bool irq_work_is_pending(struct irq_work *work)
+{
+	return atomic_read(&work->node.a_flags) & IRQ_WORK_PENDING;
 }
 
+static inline bool irq_work_is_busy(struct irq_work *work)
+{
+	return atomic_read(&work->node.a_flags) & IRQ_WORK_BUSY;
+}
 
 bool irq_work_queue(struct irq_work *work);
 bool irq_work_queue_on(struct irq_work *work, int cpu);
include/linux/irqflags.h +2 -2
@@ -109,12 +109,12 @@ do {						\
 
 # define lockdep_irq_work_enter(__work)					\
 	  do {								\
-		  if (!(atomic_read(&__work->flags) & IRQ_WORK_HARD_IRQ))\
+		  if (!(atomic_read(&__work->node.a_flags) & IRQ_WORK_HARD_IRQ))\
 			current->irq_config = 1;			\
 	  } while (0)
 # define lockdep_irq_work_exit(__work)					\
 	  do {								\
-		  if (!(atomic_read(&__work->flags) & IRQ_WORK_HARD_IRQ))\
+		  if (!(atomic_read(&__work->node.a_flags) & IRQ_WORK_HARD_IRQ))\
 			current->irq_config = 0;			\
 	  } while (0)
 
kernel/bpf/stackmap.c +1 -1
@@ -298,7 +298,7 @@ static void stack_map_get_build_id_offse
 	if (irqs_disabled()) {
 		if (!IS_ENABLED(CONFIG_PREEMPT_RT)) {
 			work = this_cpu_ptr(&up_read_work);
-			if (atomic_read(&work->irq_work.flags) & IRQ_WORK_BUSY) {
+			if (irq_work_is_busy(&work->irq_work)) {
 				/* cannot queue more up_read, fallback */
 				irq_work_busy = true;
 			}
kernel/irq_work.c +9 -9
@@ -31,7 +31,7 @@ static bool irq_work_claim(struct irq_wo
 {
 	int oflags;
 
-	oflags = atomic_fetch_or(IRQ_WORK_CLAIMED | CSD_TYPE_IRQ_WORK, &work->flags);
+	oflags = atomic_fetch_or(IRQ_WORK_CLAIMED | CSD_TYPE_IRQ_WORK, &work->node.a_flags);
 	/*
 	 * If the work is already pending, no need to raise the IPI.
 	 * The pairing atomic_fetch_andnot() in irq_work_run() makes sure
@@ -53,12 +53,12 @@ void __weak arch_irq_work_raise(void)
 static void __irq_work_queue_local(struct irq_work *work)
 {
 	/* If the work is "lazy", handle it from next tick if any */
-	if (atomic_read(&work->flags) & IRQ_WORK_LAZY) {
-		if (llist_add(&work->llnode, this_cpu_ptr(&lazy_list)) &&
+	if (atomic_read(&work->node.a_flags) & IRQ_WORK_LAZY) {
+		if (llist_add(&work->node.llist, this_cpu_ptr(&lazy_list)) &&
 		    tick_nohz_tick_stopped())
 			arch_irq_work_raise();
 	} else {
-		if (llist_add(&work->llnode, this_cpu_ptr(&raised_list)))
+		if (llist_add(&work->node.llist, this_cpu_ptr(&raised_list)))
 			arch_irq_work_raise();
 	}
 }
@@ -102,7 +102,7 @@ bool irq_work_queue_on(struct irq_work *
 	if (cpu != smp_processor_id()) {
 		/* Arch remote IPI send/receive backend aren't NMI safe */
 		WARN_ON_ONCE(in_nmi());
-		__smp_call_single_queue(cpu, &work->llnode);
+		__smp_call_single_queue(cpu, &work->node.llist);
 	} else {
 		__irq_work_queue_local(work);
 	}
@@ -142,7 +142,7 @@ void irq_work_single(void *arg)
 	 * to claim that work don't rely on us to handle their data
 	 * while we are in the middle of the func.
 	 */
-	flags = atomic_fetch_andnot(IRQ_WORK_PENDING, &work->flags);
+	flags = atomic_fetch_andnot(IRQ_WORK_PENDING, &work->node.a_flags);
 
 	lockdep_irq_work_enter(work);
 	work->func(work);
@@ -152,7 +152,7 @@ void irq_work_single(void *arg)
 	 * no-one else claimed it meanwhile.
 	 */
 	flags &= ~IRQ_WORK_PENDING;
-	(void)atomic_cmpxchg(&work->flags, flags, flags & ~IRQ_WORK_BUSY);
+	(void)atomic_cmpxchg(&work->node.a_flags, flags, flags & ~IRQ_WORK_BUSY);
 }
 
 static void irq_work_run_list(struct llist_head *list)
@@ -166,7 +166,7 @@ static void irq_work_run_list(struct lli
 		return;
 
 	llnode = llist_del_all(list);
-	llist_for_each_entry_safe(work, tmp, llnode, llnode)
+	llist_for_each_entry_safe(work, tmp, llnode, node.llist)
 		irq_work_single(work);
 }
 
@@ -198,7 +198,7 @@ void irq_work_sync(struct irq_work *work
 {
 	lockdep_assert_irqs_enabled();
 
-	while (atomic_read(&work->flags) & IRQ_WORK_BUSY)
+	while (irq_work_is_busy(work))
 		cpu_relax();
 }
 EXPORT_SYMBOL_GPL(irq_work_sync);
kernel/printk/printk.c +2 -4
@@ -3025,10 +3025,8 @@ static void wake_up_klogd_work_func(stru
 		wake_up_interruptible(&log_wait);
 }
 
-static DEFINE_PER_CPU(struct irq_work, wake_up_klogd_work) = {
-	.func = wake_up_klogd_work_func,
-	.flags = ATOMIC_INIT(IRQ_WORK_LAZY),
-};
+static DEFINE_PER_CPU(struct irq_work, wake_up_klogd_work) =
+	IRQ_WORK_INIT_LAZY(wake_up_klogd_work_func);
 
 void wake_up_klogd(void)
 {
kernel/rcu/tree.c +1 -2
@@ -1311,8 +1311,6 @@ static int rcu_implicit_dynticks_qs(stru
 		if (IS_ENABLED(CONFIG_IRQ_WORK) &&
 		    !rdp->rcu_iw_pending && rdp->rcu_iw_gp_seq != rnp->gp_seq &&
 		    (rnp->ffmask & rdp->grpmask)) {
-			init_irq_work(&rdp->rcu_iw, rcu_iw_handler);
-			atomic_set(&rdp->rcu_iw.flags, IRQ_WORK_HARD_IRQ);
 			rdp->rcu_iw_pending = true;
 			rdp->rcu_iw_gp_seq = rnp->gp_seq;
 			irq_work_queue_on(&rdp->rcu_iw, rdp->cpu);
@@ -3964,6 +3962,7 @@ int rcutree_prepare_cpu(unsigned int cpu
 	rdp->cpu_no_qs.b.norm = true;
 	rdp->core_needs_qs = false;
 	rdp->rcu_iw_pending = false;
+	rdp->rcu_iw = IRQ_WORK_INIT_HARD(rcu_iw_handler);
 	rdp->rcu_iw_gp_seq = rdp->gp_seq - 1;
 	trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuonl"));
 	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
kernel/time/tick-sched.c +2 -4
@@ -243,10 +243,8 @@ static void nohz_full_kick_func(struct i
 	/* Empty, the tick restart happens on tick_nohz_irq_exit() */
 }
 
-static DEFINE_PER_CPU(struct irq_work, nohz_full_kick_work) = {
-	.func = nohz_full_kick_func,
-	.flags = ATOMIC_INIT(IRQ_WORK_HARD_IRQ),
-};
+static DEFINE_PER_CPU(struct irq_work, nohz_full_kick_work) =
+	IRQ_WORK_INIT_HARD(nohz_full_kick_func);
 
 /*
  * Kick this CPU if it's full dynticks in order to force it to
kernel/trace/bpf_trace.c +1 -1
@@ -1086,7 +1086,7 @@ static int bpf_send_signal_common(u32 si
 			return -EINVAL;
 
 		work = this_cpu_ptr(&send_signal_work);
-		if (atomic_read(&work->irq_work.flags) & IRQ_WORK_BUSY)
+		if (irq_work_is_busy(&work->irq_work))
 			return -EBUSY;
 
 		/* Add the current task, which is the target of sending signal,
author maintainer
#10 lore raw

PATCH v3 5/6 irq_work: Provide irq_work_queue_remote()

In reply to Peter Zijlstra

While the traditional irq_work relies on the ability to self-IPI, it makes sense to provide an unconditional irq_work_queue_remote() interface.

---
 include/linux/irq_work.h |   17 ++++--
 kernel/irq_work.c        |  129 ++++++++++++++++++++++++++++-------------------
 kernel/rcu/tree.c        |    5 +
 3 files changed, 94 insertions(+), 57 deletions(-)
include/linux/irq_work.h +12 -5
@@ -46,22 +46,29 @@ static inline bool irq_work_is_busy(stru
 	return atomic_read(&work->node.a_flags) & IRQ_WORK_BUSY;
 }
 
+#ifdef CONFIG_IRQ_WORK
+
 bool irq_work_queue(struct irq_work *work);
 bool irq_work_queue_on(struct irq_work *work, int cpu);
 
 void irq_work_tick(void);
 void irq_work_sync(struct irq_work *work);
 
-#ifdef CONFIG_IRQ_WORK
 #include <asm/irq_work.h>
 
 void irq_work_run(void);
 bool irq_work_needs_cpu(void);
-void irq_work_single(void *arg);
-#else
-static inline bool irq_work_needs_cpu(void) { return false; }
+
+#else /* !CONFIG_IRQ_WORK */
+
 static inline void irq_work_run(void) { }
-static inline void irq_work_single(void *arg) { }
+static inline bool irq_work_needs_cpu(void) { return false; }
+
+#endif /* CONFIG_IRQ_WORK */
+
+#ifdef CONFIG_SMP
+extern int irq_work_queue_remote(int cpu, struct irq_work *work);
+extern void irq_work_single(void *arg);
 #endif
 
 #endif /* _LINUX_IRQ_WORK_H */
kernel/irq_work.c +79 -50

157 lines not on this page · raw message

kernel/rcu/tree.c +3 -2
@@ -1308,13 +1308,14 @@ static int rcu_implicit_dynticks_qs(stru
 			resched_cpu(rdp->cpu);
 			WRITE_ONCE(rdp->last_fqs_resched, jiffies);
 		}
-		if (IS_ENABLED(CONFIG_IRQ_WORK) &&
-		    !rdp->rcu_iw_pending && rdp->rcu_iw_gp_seq != rnp->gp_seq &&
+#ifdef CONFIG_IRQ_WORK
+		if (!rdp->rcu_iw_pending && rdp->rcu_iw_gp_seq != rnp->gp_seq &&
 		    (rnp->ffmask & rdp->grpmask)) {
 			rdp->rcu_iw_pending = true;
 			rdp->rcu_iw_gp_seq = rnp->gp_seq;
 			irq_work_queue_on(&rdp->rcu_iw, rdp->cpu);
 		}
+#endif
 	}
 
 	return 0;
author maintainer
#17 lore raw

Re: [RFC][PATCH v3 6/6] rcu/tree: Use irq_work_queue_remote()

In reply to Peter Zijlstra
Quoted 11 lines · Peter Zijlstra
Quoted 5 lines · Peter Zijlstra

AFAICT we only need/use irq_work_queue_on() on remote CPUs, since we can directly access local state. So avoid the IRQ_WORK dependency and use the unconditionally available irq_work_queue_remote().

This survives a number of TREE01 runs.

OK, Paul mentioned on IRC that while it is extremely unlikely, this code does not indeed guarantee it will not try to IPI self.

I'll try again.

This is the best I could come up with.. :/

---
Subject: rcu/tree: Use irq_work_queue_remote()
From: Peter Zijlstra <[email protected]>
Date: Wed Oct 28 11:53:40 CET 2020

All sites that consume rcu_iw_gp_seq seem to have rcu_node lock held, so setting it probably should too. Also the effect of self-IPI here would be setting rcu_iw_gp_seq to the value we just set it to (pointless) and clearing rcu_iw_pending, which we just set, so don't set it.

Passes TREE01.

---
 kernel/rcu/tree.c |   10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)
kernel/rcu/tree.c +6 -4
@@ -1308,14 +1308,16 @@ static int rcu_implicit_dynticks_qs(stru
 			resched_cpu(rdp->cpu);
 			WRITE_ONCE(rdp->last_fqs_resched, jiffies);
 		}
-#ifdef CONFIG_IRQ_WORK
+		raw_spin_lock_rcu_node(rnp);
 		if (!rdp->rcu_iw_pending && rdp->rcu_iw_gp_seq != rnp->gp_seq &&
 		    (rnp->ffmask & rdp->grpmask)) {
-			rdp->rcu_iw_pending = true;
 			rdp->rcu_iw_gp_seq = rnp->gp_seq;
-			irq_work_queue_on(&rdp->rcu_iw, rdp->cpu);
+			if (likely(rdp->cpu != smp_processor_id())) {
+				rdp->rcu_iw_pending = true;
+				irq_work_queue_remote(rdp->cpu, &rdp->rcu_iw);
+			}
 		}
-#endif
+		raw_spin_unlock_rcu_node(rnp);
 	}
 
 	return 0;
author maintainer
#16 lore raw

Re: [RFC][PATCH v3 6/6] rcu/tree: Use irq_work_queue_remote()

In reply to Peter Zijlstra
Quoted 5 lines · Peter Zijlstra

AFAICT we only need/use irq_work_queue_on() on remote CPUs, since we can directly access local state. So avoid the IRQ_WORK dependency and use the unconditionally available irq_work_queue_remote().

This survives a number of TREE01 runs.

OK, Paul mentioned on IRC that while it is extremely unlikely, this code does not indeed guarantee it will not try to IPI self.

I'll try again.

author maintainer
#15 lore raw

RFC PATCH v3 6/6 rcu/tree: Use irq_work_queue_remote()

In reply to Peter Zijlstra

AFAICT we only need/use irq_work_queue_on() on remote CPUs, since we can directly access local state. So avoid the IRQ_WORK dependency and use the unconditionally available irq_work_queue_remote().

This survives a number of TREE01 runs.

---
 kernel/rcu/tree.c |    4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)
kernel/rcu/tree.c +1 -3
@@ -1308,14 +1308,12 @@ static int rcu_implicit_dynticks_qs(stru
 			resched_cpu(rdp->cpu);
 			WRITE_ONCE(rdp->last_fqs_resched, jiffies);
 		}
-#ifdef CONFIG_IRQ_WORK
 		if (!rdp->rcu_iw_pending && rdp->rcu_iw_gp_seq != rnp->gp_seq &&
 		    (rnp->ffmask & rdp->grpmask)) {
 			rdp->rcu_iw_pending = true;
 			rdp->rcu_iw_gp_seq = rnp->gp_seq;
-			irq_work_queue_on(&rdp->rcu_iw, rdp->cpu);
+			irq_work_queue_remote(rdp->cpu, &rdp->rcu_iw);
 		}
-#endif
 	}
 
 	return 0;
#11 lore raw

Re: [PATCH v3 5/6] irq_work: Provide irq_work_queue_remote()

In reply to Peter Zijlstra
Quoted 3 lines · Peter Zijlstra

While the traditional irq_work relies on the ability to self-IPI, it makes sense to provide an unconditional irq_work_queue_remote() interface.

We may need a reason as well here.

kernel/rcu/tree.c static int rcu_implicit_dynticks_qs(stru · line 1312

Quoted 10 lines
kernel/rcu/tree.c +2 -2
@@ -1308,13 +1308,14 @@ static int rcu_implicit_dynticks_qs(stru
 			resched_cpu(rdp->cpu);
 			WRITE_ONCE(rdp->last_fqs_resched, jiffies);
 		}
-		if (IS_ENABLED(CONFIG_IRQ_WORK) &&
-		    !rdp->rcu_iw_pending && rdp->rcu_iw_gp_seq != rnp->gp_seq &&
+#ifdef CONFIG_IRQ_WORK
+		if (!rdp->rcu_iw_pending && rdp->rcu_iw_gp_seq != rnp->gp_seq &&

If it's unconditional on SMP, I expect it to be unconditional on rcutree.

Also this chunk seems unrelated to this patch.

Quoted 11 lines
kernel/rcu/tree.c +1 -0
 		    (rnp->ffmask & rdp->grpmask)) {
 			rdp->rcu_iw_pending = true;
 			rdp->rcu_iw_gp_seq = rnp->gp_seq;
 			irq_work_queue_on(&rdp->rcu_iw, rdp->cpu);
 		}
+#endif
 	}
 
 	return 0;
maintainer Reviewed-by on v3 1/6
#2 lore raw

Re: [PATCH v3 1/6] irq_work: Cleanup

In reply to Peter Zijlstra

include/linux/irq_work.h line 22

Quoted 2 lines · Peter Zijlstra
include/linux/irq_work.h +2 -0
+#define __IRQ_WORK_INIT(_func, _flags) (struct irq_work){	\
+	.node = { .u_flags = (_flags), },			\

I guess, just for the sake of being conservative:

+ .node = { .a_flags = ATOMIC_INIT(_flags), },

Review

Quoted 2 lines
(quoted diff) +2 -0
+	.func = (_func),					\
+}

Thanks.

#13 lore raw

Re: [PATCH v3 5/6] irq_work: Provide irq_work_queue_remote()

In reply to Peter Zijlstra
Quoted 6 lines · Peter Zijlstra
Quoted 5 lines · Frederic Weisbecker
Quoted 3 lines · Peter Zijlstra

While the traditional irq_work relies on the ability to self-IPI, it makes sense to provide an unconditional irq_work_queue_remote() interface.

Quoted 8 lines
Quoted 1 line

We may need a reason as well here.

Rewrote it like so.
---

While the traditional irq_work relies on the ability to self-IPI, the remote irq_work only requires generic SMP bits and can thus be made available unconditionally.

Thanks!

maintainer
#18 lore raw

Re: [RFC][PATCH v3 6/6] rcu/tree: Use irq_work_queue_remote()

In reply to Peter Zijlstra
Quoted 36 lines · Peter Zijlstra
Quoted 11 lines · Peter Zijlstra
Quoted 5 lines · Peter Zijlstra

AFAICT we only need/use irq_work_queue_on() on remote CPUs, since we can directly access local state. So avoid the IRQ_WORK dependency and use the unconditionally available irq_work_queue_remote().

This survives a number of TREE01 runs.

OK, Paul mentioned on IRC that while it is extremely unlikely, this code does not indeed guarantee it will not try to IPI self.

I'll try again.

This is the best I could come up with.. :/

---
Subject: rcu/tree: Use irq_work_queue_remote()
From: Peter Zijlstra <[email protected]>
Date: Wed Oct 28 11:53:40 CET 2020

All sites that consume rcu_iw_gp_seq seem to have rcu_node lock held, so setting it probably should too. Also the effect of self-IPI here would be setting rcu_iw_gp_seq to the value we just set it to (pointless) and clearing rcu_iw_pending, which we just set, so don't set it.

Passes TREE01.

---
 kernel/rcu/tree.c |   10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

kernel/rcu/tree.c static int rcu_implicit_dynticks_qs(stru · line 1311

Quoted 5 lines
kernel/rcu/tree.c +1 -1
@@ -1308,14 +1308,16 @@ static int rcu_implicit_dynticks_qs(stru
 			resched_cpu(rdp->cpu);
 			WRITE_ONCE(rdp->last_fqs_resched, jiffies);
 		}
-#ifdef CONFIG_IRQ_WORK
+		raw_spin_lock_rcu_node(rnp);

The caller of rcu_implicit_dynticks_qs() already holds this lock. Please see the force_qs_rnp() function and its second call site, to which rcu_implicit_dynticks_qs() is passed as an argument.

But other than that, this does look plausible. And getting rid of that #ifdef is worth something. ;-)

Thanx, Paul

Quoted 15 lines
kernel/rcu/tree.c +5 -3
 		if (!rdp->rcu_iw_pending && rdp->rcu_iw_gp_seq != rnp->gp_seq &&
 		    (rnp->ffmask & rdp->grpmask)) {
-			rdp->rcu_iw_pending = true;
 			rdp->rcu_iw_gp_seq = rnp->gp_seq;
-			irq_work_queue_on(&rdp->rcu_iw, rdp->cpu);
+			if (likely(rdp->cpu != smp_processor_id())) {
+				rdp->rcu_iw_pending = true;
+				irq_work_queue_remote(rdp->cpu, &rdp->rcu_iw);
+			}
 		}
-#endif
+		raw_spin_unlock_rcu_node(rnp);
 	}
 
 	return 0;
author maintainer
#7 lore raw

PATCH v3 4/6 irq_work: Unconditionally build on SMP

In reply to Peter Zijlstra
---
 kernel/Makefile   |    1 +
 kernel/irq_work.c |    3 +++
 2 files changed, 4 insertions(+)
kernel/Makefile +1 -0
@@ -105,6 +105,7 @@ obj-$(CONFIG_TRACE_CLOCK) += trace/
 obj-$(CONFIG_RING_BUFFER) += trace/
 obj-$(CONFIG_TRACEPOINTS) += trace/
 obj-$(CONFIG_IRQ_WORK) += irq_work.o
+obj-$(CONFIG_SMP) += irq_work.o
 obj-$(CONFIG_CPU_PM) += cpu_pm.o
 obj-$(CONFIG_BPF) += bpf/
 obj-$(CONFIG_KCSAN) += kcsan/
kernel/irq_work.c +3 -0
@@ -20,6 +20,7 @@
 #include <linux/smp.h>
 #include <asm/processor.h>
 
+#ifdef CONFIG_IRQ_WORK
 
 static DEFINE_PER_CPU(struct llist_head, raised_list);
 static DEFINE_PER_CPU(struct llist_head, lazy_list);
@@ -212,3 +213,5 @@ void irq_work_sync(struct irq_work *work
 		cpu_relax();
 }
 EXPORT_SYMBOL_GPL(irq_work_sync);
+
+#endif /* CONFIG_IRQ_WORK */
author maintainer
#9 lore raw

Re: [PATCH v3 4/6] irq_work: Unconditionally build on SMP

In reply to Frederic Weisbecker
Quoted 3 lines · Frederic Weisbecker

On Wed, Oct 28, 2020 at 12:07:11PM +0100, Peter Zijlstra wrote:

This may need a changelog :-)

Like so then?
---

Only (the local) irq_work_queue() relies on arch self-IPI bits, prepare to have the remote queueing available on all SMP.

author maintainer
#14 lore raw

Re: [PATCH v3 5/6] irq_work: Provide irq_work_queue_remote()

In reply to Frederic Weisbecker
Quoted 6 lines · Frederic Weisbecker
Quoted 3 lines · Peter Zijlstra

While the traditional irq_work relies on the ability to self-IPI, it makes sense to provide an unconditional irq_work_queue_remote() interface.

We may need a reason as well here.

Well, it doesn't rely on arch self-IPI code. The remote irq_work bits are generic SMP code.

Quoted 13 lines

kernel/rcu/tree.c static int rcu_implicit_dynticks_qs(stru · line 1312

Quoted 10 lines
kernel/rcu/tree.c +2 -2
@@ -1308,13 +1308,14 @@ static int rcu_implicit_dynticks_qs(stru
 			resched_cpu(rdp->cpu);
 			WRITE_ONCE(rdp->last_fqs_resched, jiffies);
 		}
-		if (IS_ENABLED(CONFIG_IRQ_WORK) &&
-		    !rdp->rcu_iw_pending && rdp->rcu_iw_gp_seq != rnp->gp_seq &&
+#ifdef CONFIG_IRQ_WORK
+		if (!rdp->rcu_iw_pending && rdp->rcu_iw_gp_seq != rnp->gp_seq &&

If it's unconditional on SMP, I expect it to be unconditional on rcutree.

Quoted 1 line

Also this chunk seems unrelated to this patch.

This hunk is due to irq_work_queue_on() no longer existing for CONFIG_IRQ_WORK and hence breaking the compile with that IS_ENABLED() crud.

That is, this changes IS_ENABLED() for a proper #ifdef.

Quoted 11 lines
Quoted 11 lines
kernel/rcu/tree.c +1 -0
 		    (rnp->ffmask & rdp->grpmask)) {
 			rdp->rcu_iw_pending = true;
 			rdp->rcu_iw_gp_seq = rnp->gp_seq;
 			irq_work_queue_on(&rdp->rcu_iw, rdp->cpu);
 		}
+#endif
 	}
 
 	return 0;
#8 lore raw

Re: [PATCH v3 4/6] irq_work: Unconditionally build on SMP

In reply to Peter Zijlstra

On Wed, Oct 28, 2020 at 12:07:11PM +0100, Peter Zijlstra wrote:

This may need a changelog :-)

Quoted 35 lines
---
 kernel/Makefile   |    1 +
 kernel/irq_work.c |    3 +++
 2 files changed, 4 insertions(+)
kernel/Makefile +1 -0
@@ -105,6 +105,7 @@ obj-$(CONFIG_TRACE_CLOCK) += trace/
 obj-$(CONFIG_RING_BUFFER) += trace/
 obj-$(CONFIG_TRACEPOINTS) += trace/
 obj-$(CONFIG_IRQ_WORK) += irq_work.o
+obj-$(CONFIG_SMP) += irq_work.o
 obj-$(CONFIG_CPU_PM) += cpu_pm.o
 obj-$(CONFIG_BPF) += bpf/
 obj-$(CONFIG_KCSAN) += kcsan/
kernel/irq_work.c +3 -0
@@ -20,6 +20,7 @@
 #include <linux/smp.h>
 #include <asm/processor.h>
 
+#ifdef CONFIG_IRQ_WORK
 
 static DEFINE_PER_CPU(struct llist_head, raised_list);
 static DEFINE_PER_CPU(struct llist_head, lazy_list);
@@ -212,3 +213,5 @@ void irq_work_sync(struct irq_work *work
 		cpu_relax();
 }
 EXPORT_SYMBOL_GPL(irq_work_sync);
+
+#endif /* CONFIG_IRQ_WORK */
author maintainer
#5 lore raw

PATCH v3 3/6 irq_work: Optimize irq_work_single()

In reply to Peter Zijlstra

Trade one atomic op for a full memory barrier.

---
 include/linux/irqflags.h |    8 ++++----
 kernel/irq_work.c        |   29 +++++++++++++++++------------
 2 files changed, 21 insertions(+), 16 deletions(-)
include/linux/irqflags.h +4 -4
@@ -88,14 +88,14 @@ do {						\
 		  current->irq_config = 0;			\
 	  } while (0)
 
-# define lockdep_irq_work_enter(__work)					\
+# define lockdep_irq_work_enter(_flags)					\
 	  do {								\
-		  if (!(atomic_read(&__work->node.a_flags) & IRQ_WORK_HARD_IRQ))\
+		  if (!((_flags) & IRQ_WORK_HARD_IRQ))			\
 			current->irq_config = 1;			\
 	  } while (0)
-# define lockdep_irq_work_exit(__work)					\
+# define lockdep_irq_work_exit(_flags)					\
 	  do {								\
-		  if (!(atomic_read(&__work->node.a_flags) & IRQ_WORK_HARD_IRQ))\
+		  if (!((_flags) & IRQ_WORK_HARD_IRQ))			\
 			current->irq_config = 0;			\
 	  } while (0)
 
kernel/irq_work.c +17 -12
@@ -34,7 +34,7 @@ static bool irq_work_claim(struct irq_wo
 	oflags = atomic_fetch_or(IRQ_WORK_CLAIMED | CSD_TYPE_IRQ_WORK, &work->node.a_flags);
 	/*
 	 * If the work is already pending, no need to raise the IPI.
-	 * The pairing atomic_fetch_andnot() in irq_work_run() makes sure
+	 * The pairing smp_mb() in irq_work_single() makes sure
 	 * everything we did before is visible.
 	 */
 	if (oflags & IRQ_WORK_PENDING)
@@ -136,22 +136,27 @@ void irq_work_single(void *arg)
 	int flags;
 
 	/*
-	 * Clear the PENDING bit, after this point the @work
-	 * can be re-used.
-	 * Make it immediately visible so that other CPUs trying
-	 * to claim that work don't rely on us to handle their data
-	 * while we are in the middle of the func.
+	 * Clear the PENDING bit, after this point the @work can be re-used.
+	 * The PENDING bit acts as a lock, and we own it, so we can clear it
+	 * without atomic ops.
 	 */
-	flags = atomic_fetch_andnot(IRQ_WORK_PENDING, &work->node.a_flags);
+	flags = atomic_read(&work->node.a_flags);
+	flags &= ~IRQ_WORK_PENDING;
+	atomic_set(&work->node.a_flags, flags);
+
+	/*
+	 * See irq_work_claim().
+	 */
+	smp_mb();
 
-	lockdep_irq_work_enter(work);
+	lockdep_irq_work_enter(flags);
 	work->func(work);
-	lockdep_irq_work_exit(work);
+	lockdep_irq_work_exit(flags);
+
 	/*
-	 * Clear the BUSY bit and return to the free state if
-	 * no-one else claimed it meanwhile.
+	 * Clear the BUSY bit, if set, and return to the free state if no-one
+	 * else claimed it meanwhile.
 	 */
-	flags &= ~IRQ_WORK_PENDING;
 	(void)atomic_cmpxchg(&work->node.a_flags, flags, flags & ~IRQ_WORK_BUSY);
 }
 

Peter Zijlstra

author maintainer
#0 lore raw

v3 cover smp: irq_work / smp_call_function rework

Hi,

These are the non-controversial irq_work / smp_call_function integration / cleanup patches.

There's one RFC patch to use the new irq_work_queue_remote() which seems to survive rcutorture.

Please consider.

author maintainer
#19 lore raw

Re: [RFC][PATCH v3 6/6] rcu/tree: Use irq_work_queue_remote()

In reply to Paul E. McKenney
Quoted 7 lines · Paul E. McKenney

Review

Quoted 2 lines · Peter Zijlstra
(quoted diff) +1 -1
-#ifdef CONFIG_IRQ_WORK
+		raw_spin_lock_rcu_node(rnp);

The caller of rcu_implicit_dynticks_qs() already holds this lock. Please see the force_qs_rnp() function and its second call site, to which rcu_implicit_dynticks_qs() is passed as an argument.

Like this then.

---
Subject: rcu/tree: Use irq_work_queue_remote()
From: Peter Zijlstra <[email protected]>
Date: Wed Oct 28 11:53:40 CET 2020

The effect of an self-IPI here would be setting rcu_iw_gp_seq to the value we just set it to (pointless) and clearing rcu_iw_pending, which we just set, so don't set it.

---
 kernel/rcu/tree.c |   10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)
kernel/rcu/tree.c +6 -4
@@ -1204,6 +1204,8 @@ static int rcu_implicit_dynticks_qs(stru
 	bool *ruqp;
 	struct rcu_node *rnp = rdp->mynode;
 
+	raw_lockdep_assert_held_rcu_node(rnp);
+
 	/*
 	 * If the CPU passed through or entered a dynticks idle phase with
 	 * no active irq/NMI handlers, then we can safely pretend that the CPU
@@ -1308,14 +1310,14 @@ static int rcu_implicit_dynticks_qs(stru
 			resched_cpu(rdp->cpu);
 			WRITE_ONCE(rdp->last_fqs_resched, jiffies);
 		}
-#ifdef CONFIG_IRQ_WORK
 		if (!rdp->rcu_iw_pending && rdp->rcu_iw_gp_seq != rnp->gp_seq &&
 		    (rnp->ffmask & rdp->grpmask)) {
-			rdp->rcu_iw_pending = true;
 			rdp->rcu_iw_gp_seq = rnp->gp_seq;
-			irq_work_queue_on(&rdp->rcu_iw, rdp->cpu);
+			if (likely(rdp->cpu != smp_processor_id())) {
+				rdp->rcu_iw_pending = true;
+				irq_work_queue_remote(rdp->cpu, &rdp->rcu_iw);
+			}
 		}
-#endif
 	}
 
 	return 0;
author maintainer
#21 lore raw

Re: [RFC][PATCH v3 6/6] rcu/tree: Use irq_work_queue_remote()

In reply to Paul E. McKenney
Quoted 32 lines · Paul E. McKenney
Quoted 20 lines · Peter Zijlstra

Subject: rcu/tree: Use irq_work_queue_remote()
From: Peter Zijlstra <[email protected]>
Date: Wed Oct 28 11:53:40 CET 2020

All sites that consume rcu_iw_gp_seq seem to have rcu_node lock held, so setting it probably should too. Also the effect of self-IPI here would be setting rcu_iw_gp_seq to the value we just set it to (pointless) and clearing rcu_iw_pending, which we just set, so don't set it.

Passes TREE01.

---
 kernel/rcu/tree.c |   10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

kernel/rcu/tree.c static int rcu_implicit_dynticks_qs(stru · line 1311

Quoted 5 lines
kernel/rcu/tree.c +1 -1
@@ -1308,14 +1308,16 @@ static int rcu_implicit_dynticks_qs(stru
 			resched_cpu(rdp->cpu);
 			WRITE_ONCE(rdp->last_fqs_resched, jiffies);
 		}
-#ifdef CONFIG_IRQ_WORK
+		raw_spin_lock_rcu_node(rnp);

The caller of rcu_implicit_dynticks_qs() already holds this lock. Please see the force_qs_rnp() function and its second call site, to which rcu_implicit_dynticks_qs() is passed as an argument.

Quoted 2 lines

But other than that, this does look plausible. And getting rid of that #ifdef is worth something. ;-)

Dang, clearly TREE01 didn't actually hit any of this code :/ Is there another test I should be running?

maintainer
#22 lore raw

Re: [RFC][PATCH v3 6/6] rcu/tree: Use irq_work_queue_remote()

Quoted 33 lines · Peter Zijlstra
Quoted 32 lines · Paul E. McKenney
Quoted 20 lines · Peter Zijlstra

Subject: rcu/tree: Use irq_work_queue_remote()
From: Peter Zijlstra <[email protected]>
Date: Wed Oct 28 11:53:40 CET 2020

All sites that consume rcu_iw_gp_seq seem to have rcu_node lock held, so setting it probably should too. Also the effect of self-IPI here would be setting rcu_iw_gp_seq to the value we just set it to (pointless) and clearing rcu_iw_pending, which we just set, so don't set it.

Passes TREE01.

---
 kernel/rcu/tree.c |   10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

kernel/rcu/tree.c static int rcu_implicit_dynticks_qs(stru · line 1311

Quoted 5 lines
kernel/rcu/tree.c +1 -1
@@ -1308,14 +1308,16 @@ static int rcu_implicit_dynticks_qs(stru
 			resched_cpu(rdp->cpu);
 			WRITE_ONCE(rdp->last_fqs_resched, jiffies);
 		}
-#ifdef CONFIG_IRQ_WORK
+		raw_spin_lock_rcu_node(rnp);

The caller of rcu_implicit_dynticks_qs() already holds this lock. Please see the force_qs_rnp() function and its second call site, to which rcu_implicit_dynticks_qs() is passed as an argument.

Quoted 5 lines
Quoted 2 lines

But other than that, this does look plausible. And getting rid of that #ifdef is worth something. ;-)

Dang, clearly TREE01 didn't actually hit any of this code :/ Is there another test I should be running?

TREE01 is fine, but you have to tell rcutorture to actually generate an RCU CPU stall warning. Like this for a 25-second stall with interrupts disabled:

tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration 3 --configs "10*TREE04" --bootargs "rcutorture.stall_cpu=25 rcutorture.stall_cpu_irqsoff=1" --trust-make

And like this to stall with interrupts enabled:

tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration 3 --configs "10*TREE04" --bootargs "rcutorture.stall_cpu=25" --trust-make

I use TREE04 instead of TREE01 because I have recently seen the pattern you are looking for in a similar configuration. But any of the TREE scenarios should work.

And you need a bit of luck. The CPU that rcutorture forces to stall needs to be blocking the current grace period at the beginning of that stall, or, alternatively, another grace period needs to start while the CPU is stalled. But the first command (with stall_cpu_irqsoff=1) will sometimes get you something like this:

rcu: INFO: rcu_sched detected stalls on CPUs/tasks:

rcu: 3-...0: (1 GPs behind) idle=156/1/0x4000000000000000 softirq=493/494 fqs=4570 last_accelerate: 0000/eaad dyntick_enabled: 0
(detected by 7, t=21003 jiffies, g=693, q=35687)

The zero in "...0" on the second line indicates that the RCU CPU stall warning found that the stalled CPU had its interrupts disabled for the last portion of that stall.

This is probabilistic. Much depends on the state of both RCU and rcutorture at the time that the stall starts. I got the above results once in 20 attempts, with other attempts having variously prevented RCU's grace-period kthread from running, gotten the irq-work executed just before the stall was reported, and so on.

Of course, to test your change, you also need the grace-period kthread to migrate to the stalled CPU just after interrupts are enabled. For this, you need something like an 11-second stall plus something to move the grace-period kthread at the right (wrong?) time. Or just run the above commands in a loop on a system with ample storage over night or some such. I see about 70MB of storage per run, so disk size shouldn't be too much of a problem.

If you do wish to muck with RCU and rcutorture to migrate RCU's grace-period kthread, look for uses of stall_cpu_irqsoff in kernel/rcu/rcutorture.c on the one hand and rcu_gp_kthread() along with the functions that it calls in kernel/rcu/tree.c on the other. For the latter, maybe force_qs_rnp() would be a good choice.

Thanx, Paul

maintainer
#20 lore raw

Re: [RFC][PATCH v3 6/6] rcu/tree: Use irq_work_queue_remote()

In reply to Peter Zijlstra
Quoted 10 lines · Peter Zijlstra
Quoted 7 lines · Paul E. McKenney

Review

Quoted 2 lines · Peter Zijlstra
(quoted diff) +1 -1
-#ifdef CONFIG_IRQ_WORK
+		raw_spin_lock_rcu_node(rnp);

The caller of rcu_implicit_dynticks_qs() already holds this lock. Please see the force_qs_rnp() function and its second call site, to which rcu_implicit_dynticks_qs() is passed as an argument.

Like this then.

This does look plausible! But I am sure that rcutorture will also have an opinion. ;-)

Thanx, Paul

Quoted 44 lines

---
Subject: rcu/tree: Use irq_work_queue_remote()
From: Peter Zijlstra <[email protected]>
Date: Wed Oct 28 11:53:40 CET 2020

The effect of an self-IPI here would be setting rcu_iw_gp_seq to the value we just set it to (pointless) and clearing rcu_iw_pending, which we just set, so don't set it.

---
 kernel/rcu/tree.c |   10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)
kernel/rcu/tree.c +6 -4
@@ -1204,6 +1204,8 @@ static int rcu_implicit_dynticks_qs(stru
 	bool *ruqp;
 	struct rcu_node *rnp = rdp->mynode;
 
+	raw_lockdep_assert_held_rcu_node(rnp);
+
 	/*
 	 * If the CPU passed through or entered a dynticks idle phase with
 	 * no active irq/NMI handlers, then we can safely pretend that the CPU
@@ -1308,14 +1310,14 @@ static int rcu_implicit_dynticks_qs(stru
 			resched_cpu(rdp->cpu);
 			WRITE_ONCE(rdp->last_fqs_resched, jiffies);
 		}
-#ifdef CONFIG_IRQ_WORK
 		if (!rdp->rcu_iw_pending && rdp->rcu_iw_gp_seq != rnp->gp_seq &&
 		    (rnp->ffmask & rdp->grpmask)) {
-			rdp->rcu_iw_pending = true;
 			rdp->rcu_iw_gp_seq = rnp->gp_seq;
-			irq_work_queue_on(&rdp->rcu_iw, rdp->cpu);
+			if (likely(rdp->cpu != smp_processor_id())) {
+				rdp->rcu_iw_pending = true;
+				irq_work_queue_remote(rdp->cpu, &rdp->rcu_iw);
+			}
 		}
-#endif
 	}
 
 	return 0;
author maintainer
#23 lore raw

Re: [RFC][PATCH v3 6/6] rcu/tree: Use irq_work_queue_remote()

In reply to Paul E. McKenney
Quoted 11 lines · Paul E. McKenney
Quoted 2 lines · Peter Zijlstra

Dang, clearly TREE01 didn't actually hit any of this code :/ Is there another test I should be running?

TREE01 is fine, but you have to tell rcutorture to actually generate an RCU CPU stall warning. Like this for a 25-second stall with interrupts disabled:

tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration 3 --configs "10*TREE04" --bootargs "rcutorture.stall_cpu=25 rcutorture.stall_cpu_irqsoff=1" --trust-make

Quoted 7 lines

Of course, to test your change, you also need the grace-period kthread to migrate to the stalled CPU just after interrupts are enabled. For this, you need something like an 11-second stall plus something to move the grace-period kthread at the right (wrong?) time. Or just run the above commands in a loop on a system with ample storage over night or some such. I see about 70MB of storage per run, so disk size shouldn't be too much of a problem.

Thanks!, I'll make the above run over night in a loop.