net: provide a way to delegate processing a softirq to ksoftirqd
Subsystems (2)
PATCH 1/2 net: provide a way to delegate processing a softirq to ksoftirqd
If the NET_RX uses up all of his budget it moves the following NAPI invocations into the ksoftirqd. On -RT it does not do so. Instead it rises the NET_RX softirq in its current context again.
In order to get closer to mainline's behaviour this patch provides __raise_softirq_irqoff_ksoft() which raises the softirq in the ksoftird.
--- include/linux/interrupt.h | 8 ++++++++ kernel/softirq.c | 13 +++++++++++++ net/core/dev.c | 2 +- 3 files changed, 22 insertions(+), 1 deletion(-)
include/linux/interrupt.h +8 -0
@@ -465,6 +465,14 @@ extern void thread_do_softirq(void);extern void open_softirq (int nr,void (*action)(struct softirq_action *));extern void softirq_init (void );extern void __raise_softirq_irqoff (unsigned int nr); +#ifdef CONFIG_PREEMPT_RT_FULL +extern void __raise_softirq_irqoff_ksoft (unsigned int nr); +#else +static inline void __raise_softirq_irqoff_ksoft (unsigned int nr) +{ +__raise_softirq_irqoff (nr); +} +#endif extern void raise_softirq_irqoff (unsigned int nr);extern void raise_softirq (unsigned int nr);
kernel/softirq.c +13 -0
@@ -641,6 +641,19 @@ void __raise_softirq_irqoff(unsigned int nr) }/* +* Same as __raise_softirq_irqoff() but will process them in ksoftirqd +*/ +void __raise_softirq_irqoff_ksoft (unsigned int nr) +{ +if (WARN_ON_ONCE (!__this_cpu_read (ksoftirqd))) +return ; +trace_softirq_raise (nr); +or_softirq_pending (1UL << nr); +__this_cpu_read (ksoftirqd)->softirqs_raised |= (1U << nr); +wakeup_softirqd (); +} + +/* * This function must run with irqs disabled! */ void raise_softirq_irqoff (unsigned int nr)
net/core/dev.c +1 -1
@@ -4955,7 +4955,7 @@ static void net_rx_action(struct softirq_action *h)list_splice_tail (&repoll, &list);list_splice (&list, &sd->poll_list);if (!list_empty (&sd->poll_list)) -__raise_softirq_irqoff (NET_RX_SOFTIRQ ); +__raise_softirq_irqoff_ksoft (NET_RX_SOFTIRQ );net_rps_action_and_irq_enable (sd); }
Signature · 1 line
2.7.0.rc3
PATCH 2/2 softirq: split timer softirqs out of ksoftirqd
The softirqd runs in -RT with SCHED_FIFO (prio 1) and deals mostly with timer wakeup which can not happen in hardirq context. The prio has been risen from the normal SCHED_OTHER so the timer wakeup does not happen too late.
With enough networking load it is possible that the system never goes idle and schedules ksoftirqd and everything else with a higher priority. One of the tasks left behind is one of RCU's threads and so we see stalls and eventually run out of memory.
This patch moves the TIMER and HRTIMER softirqs out of the ksoftirqd thread into its own ktimersoftd. The former can now run SCHED_OTHER (same as mainline) and the latter at SCHED_FIFO due to the wakeups.
Quoted 1 line
From networking point of view: The NAPI callback runs after the network
interrupt thread completes. If its run time takes too long the NAPI code itself schedules the ksoftirqd. Here in the thread it can run at SCHED_OTHER priority and it won't defer RCU anymore.
--- kernel/softirq.c | 107 +++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 89 insertions(+), 18 deletions(-)
kernel/softirq.c +89 -18
@@ -58,6 +58,10 @@ EXPORT_SYMBOL(irq_stat); static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp; DEFINE_PER_CPU(struct task_struct *, ksoftirqd); +#ifdef CONFIG_PREEMPT_RT_FULL +#define TIMER_SOFTIRQS ((1 << TIMER_SOFTIRQ) | (1 << HRTIMER_SOFTIRQ)) +DEFINE_PER_CPU(struct task_struct *, ktimer_softirqd); +#endif const char *const softirq_to_name[NR_SOFTIRQS ] = {"HI" ,"TIMER" ,"NET_TX" ,"NET_RX" ,"BLOCK" ,"BLOCK_IOPOLL" ,
@@ -171,6 +175,17 @@ static void wakeup_softirqd(void)wake_up_process (tsk); } +#ifdef CONFIG_PREEMPT_RT_FULL +static void wakeup_timer_softirqd (void ) +{ +/* Interrupts are disabled: no need to stop preemption */ +struct task_struct *tsk =__this_cpu_read (ktimer_softirqd); + +if (tsk && tsk->state !=TASK_RUNNING ) +wake_up_process (tsk); +} +#endif +static void handle_softirq (unsigned int vec_nr) {struct softirq_action *h = softirq_vec + vec_nr;
@@ -473,7 +488,6 @@ void __raise_softirq_irqoff(unsigned int nr)static inline void local_bh_disable_nort (void ) {local_bh_disable (); }static inline void _local_bh_enable_nort (void ) {_local_bh_enable (); }static void ksoftirqd_set_sched_params (unsigned int cpu) { } -static void ksoftirqd_clr_sched_params (unsigned int cpu,bool online) { }#else /* !PREEMPT_RT_FULL */
@@ -618,8 +632,12 @@ void thread_do_softirq(void)static void do_raise_softirq_irqoff (unsigned int nr) { +unsigned int mask; + + mask =1UL << nr; +trace_softirq_raise (nr); -or_softirq_pending (1UL << nr); +or_softirq_pending (mask);/* * If we are not in a hard interrupt and inside a bh disabled
@@ -628,16 +646,30 @@ static void do_raise_softirq_irqoff(unsigned int nr)* delegate it to ksoftirqd. */ if (!in_irq () && current->softirq_nestcnt) - current->softirqs_raised |= (1U << nr); -else if (__this_cpu_read (ksoftirqd)) -__this_cpu_read (ksoftirqd)->softirqs_raised |= (1U << nr); + current->softirqs_raised |= mask; +else if (!__this_cpu_read (ksoftirqd) || !__this_cpu_read (ktimer_softirqd)) +return ; + +if (mask &TIMER_SOFTIRQS ) +__this_cpu_read (ktimer_softirqd)->softirqs_raised |= mask; +else +__this_cpu_read (ksoftirqd)->softirqs_raised |= mask; } +static void wakeup_proper_softirq (unsigned int nr) +{ +if ((1UL << nr) &TIMER_SOFTIRQS ) +wakeup_timer_softirqd (); +else +wakeup_softirqd (); +} + +void __raise_softirq_irqoff (unsigned int nr) {do_raise_softirq_irqoff (nr);if (!in_irq () && !current->softirq_nestcnt) -wakeup_softirqd (); +wakeup_proper_softirq (nr); }/*
@@ -645,12 +677,20 @@ void __raise_softirq_irqoff(unsigned int nr)*/ void __raise_softirq_irqoff_ksoft (unsigned int nr) { -if (WARN_ON_ONCE (!__this_cpu_read (ksoftirqd))) +unsigned int mask; + +if (WARN_ON_ONCE (!__this_cpu_read (ksoftirqd) || + !__this_cpu_read (ktimer_softirqd)))return ; + mask =1UL << nr; +trace_softirq_raise (nr); -or_softirq_pending (1UL << nr); -__this_cpu_read (ksoftirqd)->softirqs_raised |= (1U << nr); -wakeup_softirqd (); +or_softirq_pending (mask); +if (mask &TIMER_SOFTIRQS ) +__this_cpu_read (ktimer_softirqd)->softirqs_raised |= mask; +else +__this_cpu_read (ksoftirqd)->softirqs_raised |= mask; +wakeup_proper_softirq (nr); }/*
@@ -676,7 +716,7 @@ void raise_softirq_irqoff(unsigned int nr)* raise a WARN() if the condition is met. */ if (!current->softirq_nestcnt) -wakeup_softirqd (); +wakeup_proper_softirq (nr); }static inline int ksoftirqd_softirq_pending (void )
@@ -689,22 +729,37 @@ static inline void _local_bh_enable_nort(void) { }static inline void ksoftirqd_set_sched_params (unsigned int cpu) { -struct sched_param param = { .sched_priority =1 }; - -sched_setscheduler (current,SCHED_FIFO , ¶m); -/* Take over all pending softirqs when starting */ +/* Take over all but timer pending softirqs when starting */ local_irq_disable (); - current->softirqs_raised =local_softirq_pending (); + current->softirqs_raised =local_softirq_pending () & ~TIMER_SOFTIRQS ;local_irq_enable (); } -static inline void ksoftirqd_clr_sched_params (unsigned int cpu,bool online) +static inline void ktimer_softirqd_set_sched_params (unsigned int cpu) +{ +struct sched_param param = { .sched_priority =1 }; + +sched_setscheduler (current,SCHED_FIFO , ¶m); + +/* Take over timer pending softirqs when starting */ +local_irq_disable (); + current->softirqs_raised =local_softirq_pending () &TIMER_SOFTIRQS ; +local_irq_enable (); +} + +static inline void ktimer_softirqd_clr_sched_params (unsigned int cpu, +bool online) {struct sched_param param = { .sched_priority =0 };sched_setscheduler (current,SCHED_NORMAL , ¶m); } +static int ktimer_softirqd_should_run (unsigned int cpu) +{ +return current->softirqs_raised; +} +#endif /* PREEMPT_RT_FULL */ /* * Enter an interrupt context.
@@ -754,6 +809,9 @@ static inline void invoke_softirq(void)if (__this_cpu_read (ksoftirqd) &&__this_cpu_read (ksoftirqd)->softirqs_raised)wakeup_softirqd (); +if (__this_cpu_read (ktimer_softirqd) && +__this_cpu_read (ktimer_softirqd)->softirqs_raised) +wakeup_timer_softirqd ();local_irq_restore (flags);#endif }
@@ -1186,17 +1244,30 @@ static struct notifier_block cpu_nfb = {static struct smp_hotplug_thread softirq_threads = { .store = &ksoftirqd, .setup = ksoftirqd_set_sched_params, - .cleanup = ksoftirqd_clr_sched_params, .thread_should_run = ksoftirqd_should_run, .thread_fn = run_ksoftirqd, .thread_comm ="ksoftirqd/%u" , }; +#ifdef CONFIG_PREEMPT_RT_FULL +static struct smp_hotplug_thread softirq_timer_threads = { + .store = &ktimer_softirqd, + .setup = ktimer_softirqd_set_sched_params, + .cleanup = ktimer_softirqd_clr_sched_params, + .thread_should_run = ktimer_softirqd_should_run, + .thread_fn = run_ksoftirqd, + .thread_comm ="ktimersoftd/%u" , +}; +#endif +static __init int spawn_ksoftirqd (void ) {register_cpu_notifier (&cpu_nfb);BUG_ON (smpboot_register_percpu_thread (&softirq_threads)); +#ifdef CONFIG_PREEMPT_RT_FULL +BUG_ON (smpboot_register_percpu_thread (&softirq_timer_threads)); +#endif return 0 ; }
Signature · 1 line
2.7.0.rc3
Re: [PATCH RT 2/2] softirq: split timer softirqs out of ksoftirqd
I've been running with this patch on ARM (dra7 SOC, which is a dual-core A15) with a 4.1.18-rt17 kernel for a while without any noticeable problems. Boosting the ktimersoftd processes on both cores to a higher priority fixed some timing problems my userspace application was seeing too.
The userspace application in question has a repeating timerfd in an epoll and spends most of its time in blocking epoll_wait calls. Under system load (most from other lower-RT-priority userspace processes), it was seeing several-ms delays between the timerfd interrupt happening and ksoftirqd running the HRTIMER softirq to wake it up. That was with ksoftirqd running as SCHED_OTHER, which is what it defaulted to. The priority inversion between the userspace processes is gone after making ktimersoftd the same priority as the highest-priority application.