Skip to contentSkip to conversation

rcu: Add per-CPU rcuc task info to RCU CPU stall warnings

PATCH Superseded Zqiang 3 participants 6 messages Jan 24, 2022 – Jan 25, 2022 RCU

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore

Zqiang

author
#0 lore raw

PATCH v2 1/1 rcu: Add per-CPU rcuc task info to RCU CPU stall warnings

When the 'use_softirq' be set zero, all RCU_SOFTIRQ processing be moved to per-CPU rcuc kthreads, if the rcuc kthreads is being starved, quiescent state can not report in time. the RCU stall may be triggered. this commit adds a stack trace of this CPU and dump rcuc kthreads stack to help analyze what prevents rcuc kthreads from running.

---
v1->v2:
Avoid print anything when CPU is offline or idle.

 kernel/rcu/tree.c        |  3 +++
 kernel/rcu/tree.h        |  1 +
 kernel/rcu/tree_plugin.h |  3 +++
 kernel/rcu/tree_stall.h  | 36 ++++++++++++++++++++++++++++++++++++
 4 files changed, 43 insertions(+)
kernel/rcu/tree.c +3 -0
index a4c25a6283b0..e3fc31a0f546 100644
@@ -2850,10 +2850,12 @@ static void rcu_cpu_kthread(unsigned int cpu)
 {
 	unsigned int *statusp = this_cpu_ptr(&rcu_data.rcu_cpu_kthread_status);
 	char work, *workp = this_cpu_ptr(&rcu_data.rcu_cpu_has_work);
+	unsigned long *j = this_cpu_ptr(&rcu_data.rcuc_activity);
 	int spincnt;
 
 	trace_rcu_utilization(TPS("Start CPU kthread@rcu_run"));
 	for (spincnt = 0; spincnt < 10; spincnt++) {
+		WRITE_ONCE(*j, jiffies);
 		local_bh_disable();
 		*statusp = RCU_KTHREAD_RUNNING;
 		local_irq_disable();
@@ -2874,6 +2876,7 @@ static void rcu_cpu_kthread(unsigned int cpu)
 	schedule_timeout_idle(2);
 	trace_rcu_utilization(TPS("End CPU kthread@rcu_yield"));
 	*statusp = RCU_KTHREAD_WAITING;
+	WRITE_ONCE(*j, jiffies);
 }
 
 static struct smp_hotplug_thread rcu_cpu_thread_spec = {
kernel/rcu/tree.h +1 -0
index 486fc901bd08..4e0fdebb62e8 100644
@@ -237,6 +237,7 @@ struct rcu_data {
 					/* rcuc per-CPU kthread or NULL. */
 	unsigned int rcu_cpu_kthread_status;
 	char rcu_cpu_has_work;
+	unsigned long rcuc_activity;
 
 	/* 7) Diagnostic data, including RCU CPU stall warnings. */
 	unsigned int softirq_snap;	/* Snapshot of softirq activity. */
kernel/rcu/tree_plugin.h +3 -0
index c5b45c2f68a1..327bbfd79cc6 100644
@@ -996,12 +996,15 @@ dump_blkd_tasks(struct rcu_node *rnp, int ncheck)
  */
 static void rcu_cpu_kthread_setup(unsigned int cpu)
 {
+	struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
 #ifdef CONFIG_RCU_BOOST
 	struct sched_param sp;
 
 	sp.sched_priority = kthread_prio;
 	sched_setscheduler_nocheck(current, SCHED_FIFO, &sp);
 #endif /* #ifdef CONFIG_RCU_BOOST */
+
+	WRITE_ONCE(rdp->rcuc_activity, jiffies);
 }
 
 #ifdef CONFIG_RCU_BOOST
kernel/rcu/tree_stall.h +36 -0
index 21bebf7c9030..739a2d87ea5e 100644
@@ -379,6 +379,15 @@ static bool rcu_is_gp_kthread_starving(unsigned long *jp)
 	return j > 2 * HZ;
 }
 
+static bool rcu_is_rcuc_kthread_starving(struct rcu_data *rdp, unsigned long *jp)
+{
+	unsigned long j = jiffies - READ_ONCE(rdp->rcuc_activity);
+
+	if (jp)
+		*jp = j;
+	return j > 2 * HZ;
+}
+
 /*
  * Print out diagnostic information for the specified stalled CPU.
  *
@@ -430,6 +439,30 @@ static void print_cpu_stall_info(int cpu)
 	       falsepositive ? " (false positive?)" : "");
 }
 
+static void rcuc_kthread_dump(struct rcu_data *rdp)
+{
+	int cpu;
+	unsigned long j;
+	struct task_struct *rcuc = rdp->rcu_cpu_kthread_task;
+
+	if (rcu_is_rcuc_kthread_starving(rdp, &j)) {
+		cpu = rcuc ? task_cpu(rcuc) : -1;
+
+		if (rcuc) {
+			pr_err("%s kthread starved for %ld jiffies, stack dump:\n",
+									rcuc->comm, j);
+			sched_show_task(rcuc);
+			if (cpu >= 0) {
+				if (cpu_online(cpu) && !idle_cpu(cpu)) {
+					pr_err("Dump current CPU stack:\n");
+					if (!trigger_single_cpu_backtrace(cpu))
+						dump_cpu_task(cpu);
+				}
+			}
+		}
+	}
+}
+
 /* Complain about starvation of grace-period kthread.  */
 static void rcu_check_gp_kthread_starvation(void)
 {
@@ -601,6 +634,9 @@ static void print_cpu_stall(unsigned long gps)
 	rcu_check_gp_kthread_expired_fqs_timer();
 	rcu_check_gp_kthread_starvation();
 
+	if (!use_softirq)
+		rcuc_kthread_dump(rdp);
+
 	rcu_dump_cpu_stacks();
 
 	raw_spin_lock_irqsave_rcu_node(rnp, flags);
Signature · 1 line
2.25.1
#1 lore raw

Hi Zqiang,

On Sat, 22 Jan 2022 02:30:00 +0800, Zqiang <[email protected]> wrote:

```
> +static void rcuc_kthread_dump(struct rcu_data *rdp)
> +{
> +        int cpu;
> +        unsigned long j;
> +        struct task_struct *rcuc = rdp->rcu_cpu_kthread_task;
> +
> +        if (rcu_is_rcuc_kthread_starving(rdp, &j)) {
> +                cpu = rcuc ? task_cpu(rcuc) : -1;
> +
> +                if (rcuc) {
> +                        pr_err("%s kthread starved for %ld jiffies, stack dump:\n",
> + rcuc->comm, j);
> +                        sched_show_task(rcuc);
> +                        if (cpu >= 0) {
> +                                if (cpu_online(cpu) && !idle_cpu(cpu)) {
> +                                        pr_err("Dump current CPU stack:\n");
> +                                        if (!trigger_single_cpu_backtrace(cpu))
> + dump_cpu_task(cpu);
> +                                }
> +                        }
> +                }
> +        }
> +}
```

1) We can reduce the nested if with an early return after checking rcu_is_rcuc_kthread_starving().

2) This ternary operator doesn't make sense:

  cpu = rcuc ? task_cpu(rcuc) : -1;

If rcuc is NULL, then the "if (rcuc)" block will never be executed, and cpu variable won't be used, why should we perform a conditional with ternary to assign -1 here?

3) We can use an early return as well for the if (rcuc) to avoid more nested if.

FWIW, this one makes more sense:

```
static void rcuc_kthread_dump(struct rcu_data *rdp)
{
    int cpu;
    unsigned long j;
    struct task_struct *rcuc;

    if (!rcu_is_rcuc_kthread_starving(rdp, &j))
        return;

    rcuc = rdp->rcu_cpu_kthread_task;
    if (!rcuc)
        return;

    pr_err("%s kthread starved for %ld jiffies, stack dump:\n", rcuc->comm, j);
    sched_show_task(rcuc);
    cpu = task_cpu(rcuc);
    if (cpu_online(cpu) && !idle_cpu(cpu)) {
        pr_err("Dump current CPU stack:\n");
        if (!trigger_single_cpu_backtrace(cpu))
            dump_cpu_task(cpu);
    }
}
```

Thank you!

Signature · 1 line
Ammar Faizi
#2 lore raw
In reply to Zqiang

[ I resend and fix my reply, my previous reply seems to have an
issue with the "Date" ]

Hi Zqiang,

On Mon, 24 Jan 2022 18:36:37 +0800, Zqiang <[email protected]> wrote:> +static void rcuc_kthread_dump(struct rcu_data *rdp)

Quoted 12 lines
kernel/rcu/tree_stall.h +12 -0
+{
+	int cpu;
+	unsigned long j;
+	struct task_struct *rcuc = rdp->rcu_cpu_kthread_task;
+
+	if (rcu_is_rcuc_kthread_starving(rdp, &j)) {
+		cpu = rcuc ? task_cpu(rcuc) : -1;
+
+		if (rcuc) {
+			pr_err("%s kthread starved for %ld jiffies, stack dump:\n",
+									rcuc->comm, j);
+			sched_show_task(rcuc);

kernel/rcu/tree_stall.h static void print_cpu_stall_info(int cpu) · line 464

Quoted 10 lines
kernel/rcu/tree_stall.h +10 -0
+			if (cpu >= 0) {
+				if (cpu_online(cpu) && !idle_cpu(cpu)) {
+					pr_err("Dump current CPU stack:\n");
+					if (!trigger_single_cpu_backtrace(cpu))
+						dump_cpu_task(cpu);
+				}
+			}
+		}
+	}
+}

1) We can reduce the nested if with an early return after checking rcu_is_rcuc_kthread_starving().

2) This ternary operator doesn't make sense:

cpu = rcuc ? task_cpu(rcuc) : -1;

If rcuc is NULL, then the "if (rcuc)" block will never be executed, and cpu variable won't be used, why should we perform a conditional with ternary to assign -1 here?

3) We can use an early return as well for the if (rcuc) to avoid more nested if.

FWIW, this one makes more sense:

```
static void rcuc_kthread_dump(struct rcu_data *rdp)
{
	 int cpu;
	 unsigned long j;
	 struct task_struct *rcuc;

	 if (!rcu_is_rcuc_kthread_starving(rdp, &j))
		 return;

	 rcuc = rdp->rcu_cpu_kthread_task;
	 if (!rcuc)
		 return;

	 pr_err("%s kthread starved for %ld jiffies, stack dump:\n", rcuc->comm, j);
	 sched_show_task(rcuc);
	 cpu = task_cpu(rcuc);
	 if (cpu_online(cpu) && !idle_cpu(cpu)) {
		 pr_err("Dump current CPU stack:\n");
		 if (!trigger_single_cpu_backtrace(cpu))
			 dump_cpu_task(cpu);
	 }
}
```

Thank you!

Signature · 1 line
Ammar Faizi
maintainer
#3 lore raw
Quoted 47 lines · Ammar Faizi

[ I resend and fix my reply, my previous reply seems to have an
issue with the "Date" ]

Hi Zqiang,

On Mon, 24 Jan 2022 18:36:37 +0800, Zqiang <[email protected]> wrote:> +static void rcuc_kthread_dump(struct rcu_data *rdp)

Quoted 12 lines
kernel/rcu/tree_stall.h +12 -0
+{
+	int cpu;
+	unsigned long j;
+	struct task_struct *rcuc = rdp->rcu_cpu_kthread_task;
+
+	if (rcu_is_rcuc_kthread_starving(rdp, &j)) {
+		cpu = rcuc ? task_cpu(rcuc) : -1;
+
+		if (rcuc) {
+			pr_err("%s kthread starved for %ld jiffies, stack dump:\n",
+									rcuc->comm, j);
+			sched_show_task(rcuc);

kernel/rcu/tree_stall.h static void print_cpu_stall_info(int cpu) · line 464

Quoted 10 lines
kernel/rcu/tree_stall.h +10 -0
+			if (cpu >= 0) {
+				if (cpu_online(cpu) && !idle_cpu(cpu)) {
+					pr_err("Dump current CPU stack:\n");
+					if (!trigger_single_cpu_backtrace(cpu))
+						dump_cpu_task(cpu);
+				}
+			}
+		}
+	}
+}

1) We can reduce the nested if with an early return after checking rcu_is_rcuc_kthread_starving().

2) This ternary operator doesn't make sense:

cpu = rcuc ? task_cpu(rcuc) : -1;

If rcuc is NULL, then the "if (rcuc)" block will never be executed, and cpu variable won't be used, why should we perform a conditional with ternary to assign -1 here?

3) We can use an early return as well for the if (rcuc) to avoid more nested if.

FWIW, this one makes more sense:
```
static void rcuc_kthread_dump(struct rcu_data *rdp)

Quoted 13 lines
{
	 int cpu;
	 unsigned long j;
	 struct task_struct *rcuc;

	 if (!rcu_is_rcuc_kthread_starving(rdp, &j))
		 return;

	 rcuc = rdp->rcu_cpu_kthread_task;
	 if (!rcuc)
		 return;

	 pr_err("%s kthread starved for %ld jiffies, stack dump:\n", rcuc->comm, j);

Thank you for looking this over and for the great feedback, Ammar!

I am also wondering why the above message should be printed when the corresponding CPU is offline or idle. Why not move the above pr_err() line down to replace the pr_err() line below?

Thanx, Paul

Quoted 14 lines
	 sched_show_task(rcuc);
	 cpu = task_cpu(rcuc);
	 if (cpu_online(cpu) && !idle_cpu(cpu)) {
		 pr_err("Dump current CPU stack:\n");
		 if (!trigger_single_cpu_backtrace(cpu))
			 dump_cpu_task(cpu);
	 }
}

```

Thank you!

Signature · 1 line
Ammar Faizi
#4 lore raw
Quoted 19 lines · Paul E. McKenney
Quoted 4 lines · Ammar Faizi

[snip...]
FWIW, this one makes more sense:
```
static void rcuc_kthread_dump(struct rcu_data *rdp)

Quoted 13 lines
{
	 int cpu;
	 unsigned long j;
	 struct task_struct *rcuc;

	 if (!rcu_is_rcuc_kthread_starving(rdp, &j))
		 return;

	 rcuc = rdp->rcu_cpu_kthread_task;
	 if (!rcuc)
		 return;

	 pr_err("%s kthread starved for %ld jiffies, stack dump:\n", rcuc->comm, j);
Quoted 7 lines

Thank you for looking this over and for the great feedback, Ammar!

I am also wondering why the above message should be printed when the corresponding CPU is offline or idle. Why not move the above pr_err() line down to replace the pr_err() line below?

Thanx, Paul

Hi Paul, Thank you for the review. Agree with that. Hopefully this one looks better (untested):

```
static void rcuc_kthread_dump(struct rcu_data *rdp)
{
	int cpu;
	unsigned long j;
	struct task_struct *rcuc;

	rcuc = rdp->rcu_cpu_kthread_task;
	if (!rcuc)
		return;

	cpu = task_cpu(rcuc);
	if (cpu_is_offline(cpu) || idle_cpu(cpu))
		return;

	if (!rcu_is_rcuc_kthread_starving(rdp, &j))
		return;

	pr_err("%s kthread starved for %ld jiffies\n", rcuc->comm, j);
	sched_show_task(rcuc);
	if (!trigger_single_cpu_backtrace(cpu))
		dump_cpu_task(cpu);
}
```

Recall that dump_cpu_task looks like this:

```
void dump_cpu_task(int cpu)
{
	pr_info("Task dump for CPU %d:\n", cpu);
	sched_show_task(cpu_curr(cpu));
}
```

which already tells us it's a dump, so "stack dump" in the pr_err() can be omitted. Any comment, Zqiang?

Signature · 1 line
Ammar Faizi
#5 lore raw
Quoted 19 lines · Paul E. McKenney
Quoted 16 lines · Ammar Faizi

[snip...]
FWIW, this one makes more sense:
```

static void rcuc_kthread_dump(struct rcu_data *rdp) {
	 int cpu;
	 unsigned long j;
	 struct task_struct *rcuc;

	 if (!rcu_is_rcuc_kthread_starving(rdp, &j))
		 return;

	 rcuc = rdp->rcu_cpu_kthread_task;
	 if (!rcuc)
		 return;

	 pr_err("%s kthread starved for %ld jiffies, stack dump:\n", 
Quoted 1 line

rcuc->comm, j);

Quoted 43 lines

Thank you for looking this over and for the great feedback, Ammar!

I am also wondering why the above message should be printed when the corresponding CPU is offline or idle. Why not move the above pr_err() line down to replace the pr_err() line below?

Thanx, Paul

Quoted 35 lines

Hi Paul, Thank you for the review. Agree with that. Hopefully this one looks better (untested):

```
static void rcuc_kthread_dump(struct rcu_data *rdp) {
	int cpu;
	unsigned long j;
	struct task_struct *rcuc;

	rcuc = rdp->rcu_cpu_kthread_task;
	if (!rcuc)
		return;

	cpu = task_cpu(rcuc);
	if (cpu_is_offline(cpu) || idle_cpu(cpu))
		return;

	if (!rcu_is_rcuc_kthread_starving(rdp, &j))
		return;

	pr_err("%s kthread starved for %ld jiffies\n", rcuc->comm, j);
	sched_show_task(rcuc);
	if (!trigger_single_cpu_backtrace(cpu))
		dump_cpu_task(cpu);
}
```

Recall that dump_cpu_task looks like this:

```
void dump_cpu_task(int cpu)
{
	pr_info("Task dump for CPU %d:\n", cpu);
	sched_show_task(cpu_curr(cpu));
}
```

which already tells us it's a dump, so "stack dump" in the pr_err() can be omitted. Any comment, Zqiang?

Thanks Ammar, this look like more compact, I wiil resend.

Thanks
Zqiang

Quoted 3 lines
Quoted 3 lines
Quoted 1 line
Quoted 1 line

--
Ammar Faizi