sched/deadline: Fix priority inheritance with multiple scheduling classes
Juri Lelli
maintainerPATCH v2 1/1 sched/deadline: Fix priority inheritance with multiple scheduling classes
Glenn reported that "an application [he developed produces] a BUG in deadline.c when a SCHED_DEADLINE task contends with CFS tasks on nested PTHREAD_PRIO_INHERIT mutexes. I believe the bug is triggered when a CFS task that was boosted by a SCHED_DEADLINE task boosts another CFS task (nested priority inheritance).
------------[ cut here ]------------ kernel BUG at kernel/sched/deadline.c:1462! invalid opcode: 0000 [#1] PREEMPT SMP CPU: 12 PID: 19171 Comm: dl_boost_bug Tainted: ... Hardware name: ... RIP: 0010:enqueue_task_dl+0x335/0x910 Code: ... RSP: 0018:ffffc9000c2bbc68 EFLAGS: 00010002 RAX: 0000000000000009 RBX: ffff888c0af94c00 RCX: ffffffff81e12500 RDX: 000000000000002e RSI: ffff888c0af94c00 RDI: ffff888c10b22600 RBP: ffffc9000c2bbd08 R08: 0000000000000009 R09: 0000000000000078 R10: ffffffff81e12440 R11: ffffffff81e1236c R12: ffff888bc8932600 R13: ffff888c0af94eb8 R14: ffff888c10b22600 R15: ffff888bc8932600 FS: 00007fa58ac55700(0000) GS:ffff888c10b00000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 00007fa58b523230 CR3: 0000000bf44ab003 CR4: 00000000007606e0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 PKRU: 55555554 Call Trace: ? intel_pstate_update_util_hwp+0x13/0x170 rt_mutex_setprio+0x1cc/0x4b0 task_blocks_on_rt_mutex+0x225/0x260 rt_spin_lock_slowlock_locked+0xab/0x2d0 rt_spin_lock_slowlock+0x50/0x80 hrtimer_grab_expiry_lock+0x20/0x30 hrtimer_cancel+0x13/0x30 do_nanosleep+0xa0/0x150 hrtimer_nanosleep+0xe1/0x230 ? __hrtimer_init_sleeper+0x60/0x60 __x64_sys_nanosleep+0x8d/0xa0 do_syscall_64+0x4a/0x100 entry_SYSCALL_64_after_hwframe+0x49/0xbe RIP: 0033:0x7fa58b52330d ... ---[ end trace 0000000000000002 ]—
He also provided a simple reproducer creating the situation below:
So the execution order of locking steps are the following (N1 and N2 are non-deadline tasks. D1 is a deadline task. M1 and M2 are mutexes that are enabled * with priority inheritance.)
Time moves forward as this timeline goes down:
N1 N2 D1 | | | | | | Lock(M1) | | | | | | Lock(M2) | | | | | | Lock(M2) | | | | Lock(M1) | | (!!bug triggered!) |
Daniel reported a similar situation as well, by just letting ksoftirqd run with DEADLINE (and eventually block on a mutex).
Problem is that boosted entities (Priority Inheritance) use static DEADLINE parameters of the top priority waiter. However, there might be cases where top waiter could be a non-DEADLINE entity that is currently boosted by a DEADLINE entity from a different lock chain (i.e., nested priority chains involving entities of non-DEADLINE classes). In this case, top waiter static DEADLINE parameters could be null (initialized to 0 at fork()) and replenish_dl_entity() would hit a BUG().
Fix this by keeping track of the original donor and using its parameters when a task is boosted.
---
v1->v2: Replace dl_boosted with inline funcion (Valentin)
v1: [email protected]
--- include/linux/sched.h | 10 ++++- kernel/sched/core.c | 11 ++--- kernel/sched/deadline.c | 97 ++++++++++++++++++++++------------------- 3 files changed, 68 insertions(+), 50 deletions(-)
include/linux/sched.h +9 -1
@@ -552,7 +552,6 @@ struct sched_dl_entity {* overruns. */ unsigned int dl_throttled : 1; - unsigned int dl_boosted : 1; unsigned int dl_yielded : 1; unsigned int dl_non_contending : 1; unsigned int dl_overrun : 1;
@@ -571,6 +570,15 @@ struct sched_dl_entity {* time. */ struct hrtimer inactive_timer; + +#ifdef CONFIG_RT_MUTEXES +/* +* Priority Inheritance. When a DEADLINE scheduling entity is boosted +* pi_se points to the donor, otherwise points to the dl_se it belongs +* to (the original one/itself). +*/ +struct sched_dl_entity *pi_se; +#endif };#ifdef CONFIG_UCLAMP_TASK
kernel/sched/core.c +6 -5
@@ -5405,20 +5405,21 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task)if (!dl_prio (p->normal_prio) || (pi_task &&dl_prio (pi_task->prio) &&dl_entity_preempt (&pi_task->dl, &p->dl))) { - p->dl.dl_boosted =1 ; + p->dl.pi_se = pi_task->dl.pi_se; queue_flag |=ENQUEUE_REPLENISH ; - }else - p->dl.dl_boosted =0 ; + }else { + p->dl.pi_se = &p->dl; + } p->sched_class = &dl_sched_class; }else if (rt_prio (prio)) {if (dl_prio (oldprio)) - p->dl.dl_boosted =0 ; + p->dl.pi_se = &p->dl;if (oldprio < prio) queue_flag |=ENQUEUE_HEAD ; p->sched_class = &rt_sched_class; }else {if (dl_prio (oldprio)) - p->dl.dl_boosted =0 ; + p->dl.pi_se = &p->dl;if (rt_prio (oldprio)) p->rt.timeout =0 ; p->sched_class = &fair_sched_class;
kernel/sched/deadline.c +53 -44
233 lines not on this page · raw message
Signature · 1 line
2.28.0
Quoted 74 lines · Juri Lelli
Glenn reported that "an application [he developed produces] a BUG in
deadline.cwhen aSCHED_DEADLINEtask contends with CFS tasks on nestedPTHREAD_PRIO_INHERITmutexes. I believe the bug is triggered when a CFS task that was boosted by aSCHED_DEADLINEtask boosts another CFS task (nested priority inheritance).------------[ cut here ]------------ kernel BUG at kernel/sched/deadline.c:1462! invalid opcode: 0000 [#1] PREEMPT SMP CPU: 12 PID: 19171 Comm: dl_boost_bug Tainted: ... Hardware name: ... RIP: 0010:enqueue_task_dl+0x335/0x910 Code: ... RSP: 0018:ffffc9000c2bbc68 EFLAGS: 00010002 RAX: 0000000000000009 RBX: ffff888c0af94c00 RCX: ffffffff81e12500 RDX: 000000000000002e RSI: ffff888c0af94c00 RDI: ffff888c10b22600 RBP: ffffc9000c2bbd08 R08: 0000000000000009 R09: 0000000000000078 R10: ffffffff81e12440 R11: ffffffff81e1236c R12: ffff888bc8932600 R13: ffff888c0af94eb8 R14: ffff888c10b22600 R15: ffff888bc8932600 FS: 00007fa58ac55700(0000) GS:ffff888c10b00000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 00007fa58b523230 CR3: 0000000bf44ab003 CR4: 00000000007606e0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 PKRU: 55555554 Call Trace: ? intel_pstate_update_util_hwp+0x13/0x170 rt_mutex_setprio+0x1cc/0x4b0 task_blocks_on_rt_mutex+0x225/0x260 rt_spin_lock_slowlock_locked+0xab/0x2d0 rt_spin_lock_slowlock+0x50/0x80 hrtimer_grab_expiry_lock+0x20/0x30 hrtimer_cancel+0x13/0x30 do_nanosleep+0xa0/0x150 hrtimer_nanosleep+0xe1/0x230 ? __hrtimer_init_sleeper+0x60/0x60 __x64_sys_nanosleep+0x8d/0xa0 do_syscall_64+0x4a/0x100 entry_SYSCALL_64_after_hwframe+0x49/0xbe RIP: 0033:0x7fa58b52330d ... ---[ end trace 0000000000000002 ]—He also provided a simple reproducer creating the situation below:
So the execution order of locking steps are the following (N1 and N2 are non-deadline tasks. D1 is a deadline task. M1 and M2 are mutexes that are enabled * with priority inheritance.)
Time moves forward as this timeline goes down:
N1 N2 D1 | | | | | | Lock(M1) | | | | | | Lock(M2) | | | | | | Lock(M2) | | | | Lock(M1) | | (!!bug triggered!) |Daniel reported a similar situation as well, by just letting ksoftirqd run with DEADLINE (and eventually block on a mutex).
Problem is that boosted entities (Priority Inheritance) use static DEADLINE parameters of the top priority waiter. However, there might be cases where top waiter could be a non-DEADLINE entity that is currently boosted by a DEADLINE entity from a different lock chain (i.e., nested priority chains involving entities of non-DEADLINE classes). In this case, top waiter static DEADLINE parameters could be null (initialized to 0 at
fork()) andreplenish_dl_entity()would hit aBUG().
Quoted 6 lines
Fix this by keeping track of the original donor and using its parameters when a task is boosted.
Reported-by Glenn Elliott <[email protected]>Reported-by Daniel Bristot de Oliveira <[email protected]>Signed-off-by Juri Lelli <[email protected]>
Thanks!
-- Daniel
The following commit has been merged into the sched/urgent branch of tip:
The following commit has been merged into the sched/urgent branch of tip:
Commit-ID: 2279f540ea7d05f22d2f0c4224319330228586bc Gitweb: https://git.kernel.org/tip/2279f540ea7d05f22d2f0c4224319330228586bc Author: Juri Lelli <[email protected]> AuthorDate: Tue, 17 Nov 2020 07:14:32 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 17 Nov 2020 13:15:28 +01:00
sched/deadline: Fix priority inheritance with multiple scheduling classes
Glenn reported that "an application [he developed produces] a BUG in deadline.c when a SCHED_DEADLINE task contends with CFS tasks on nested PTHREAD_PRIO_INHERIT mutexes. I believe the bug is triggered when a CFS task that was boosted by a SCHED_DEADLINE task boosts another CFS task (nested priority inheritance).
------------[ cut here ]------------ kernel BUG at kernel/sched/deadline.c:1462! invalid opcode: 0000 [#1] PREEMPT SMP CPU: 12 PID: 19171 Comm: dl_boost_bug Tainted: ... Hardware name: ... RIP: 0010:enqueue_task_dl+0x335/0x910 Code: ... RSP: 0018:ffffc9000c2bbc68 EFLAGS: 00010002 RAX: 0000000000000009 RBX: ffff888c0af94c00 RCX: ffffffff81e12500 RDX: 000000000000002e RSI: ffff888c0af94c00 RDI: ffff888c10b22600 RBP: ffffc9000c2bbd08 R08: 0000000000000009 R09: 0000000000000078 R10: ffffffff81e12440 R11: ffffffff81e1236c R12: ffff888bc8932600 R13: ffff888c0af94eb8 R14: ffff888c10b22600 R15: ffff888bc8932600 FS: 00007fa58ac55700(0000) GS:ffff888c10b00000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 00007fa58b523230 CR3: 0000000bf44ab003 CR4: 00000000007606e0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 PKRU: 55555554 Call Trace: ? intel_pstate_update_util_hwp+0x13/0x170 rt_mutex_setprio+0x1cc/0x4b0 task_blocks_on_rt_mutex+0x225/0x260 rt_spin_lock_slowlock_locked+0xab/0x2d0 rt_spin_lock_slowlock+0x50/0x80 hrtimer_grab_expiry_lock+0x20/0x30 hrtimer_cancel+0x13/0x30 do_nanosleep+0xa0/0x150 hrtimer_nanosleep+0xe1/0x230 ? __hrtimer_init_sleeper+0x60/0x60 __x64_sys_nanosleep+0x8d/0xa0 do_syscall_64+0x4a/0x100 entry_SYSCALL_64_after_hwframe+0x49/0xbe RIP: 0033:0x7fa58b52330d ... ---[ end trace 0000000000000002 ]—
He also provided a simple reproducer creating the situation below:
So the execution order of locking steps are the following (N1 and N2 are non-deadline tasks. D1 is a deadline task. M1 and M2 are mutexes that are enabled * with priority inheritance.)
Time moves forward as this timeline goes down:
N1 N2 D1 | | | | | | Lock(M1) | | | | | | Lock(M2) | | | | | | Lock(M2) | | | | Lock(M1) | | (!!bug triggered!) |
Daniel reported a similar situation as well, by just letting ksoftirqd run with DEADLINE (and eventually block on a mutex).
Problem is that boosted entities (Priority Inheritance) use static DEADLINE parameters of the top priority waiter. However, there might be cases where top waiter could be a non-DEADLINE entity that is currently boosted by a DEADLINE entity from a different lock chain (i.e., nested priority chains involving entities of non-DEADLINE classes). In this case, top waiter static DEADLINE parameters could be null (initialized to 0 at fork()) and replenish_dl_entity() would hit a BUG().
Fix this by keeping track of the original donor and using its parameters when a task is boosted.
--- include/linux/sched.h | 10 +++- kernel/sched/core.c | 11 ++--- kernel/sched/deadline.c | 97 +++++++++++++++++++++------------------- 3 files changed, 68 insertions(+), 50 deletions(-)
include/linux/sched.h +9 -1
@@ -551,7 +551,6 @@ struct sched_dl_entity {* overruns. */ unsigned int dl_throttled : 1; - unsigned int dl_boosted : 1; unsigned int dl_yielded : 1; unsigned int dl_non_contending : 1; unsigned int dl_overrun : 1;
@@ -570,6 +569,15 @@ struct sched_dl_entity {* time. */ struct hrtimer inactive_timer; + +#ifdef CONFIG_RT_MUTEXES +/* +* Priority Inheritance. When a DEADLINE scheduling entity is boosted +* pi_se points to the donor, otherwise points to the dl_se it belongs +* to (the original one/itself). +*/ +struct sched_dl_entity *pi_se; +#endif };#ifdef CONFIG_UCLAMP_TASK
kernel/sched/core.c +6 -5
@@ -4912,20 +4912,21 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task)if (!dl_prio (p->normal_prio) || (pi_task &&dl_prio (pi_task->prio) &&dl_entity_preempt (&pi_task->dl, &p->dl))) { - p->dl.dl_boosted =1 ; + p->dl.pi_se = pi_task->dl.pi_se; queue_flag |=ENQUEUE_REPLENISH ; - }else - p->dl.dl_boosted =0 ; + }else { + p->dl.pi_se = &p->dl; + } p->sched_class = &dl_sched_class; }else if (rt_prio (prio)) {if (dl_prio (oldprio)) - p->dl.dl_boosted =0 ; + p->dl.pi_se = &p->dl;if (oldprio < prio) queue_flag |=ENQUEUE_HEAD ; p->sched_class = &rt_sched_class; }else {if (dl_prio (oldprio)) - p->dl.dl_boosted =0 ; + p->dl.pi_se = &p->dl;if (rt_prio (oldprio)) p->rt.timeout =0 ; p->sched_class = &fair_sched_class;
kernel/sched/deadline.c +53 -44
233 lines not on this page · raw message