futex-pi: fix possible pi_lock deadlock
PATCH 1/1 futex-pi: fix possible pi_lock deadlock
From: Ingo Molnar <[email protected]>
The lock validator detected a possible deadlock between tasklist lock and task->pi_lock.
Prevent the deadlock by disabling interrupts across pi_lock operations.
kernel/futex.c | 28 ++++++++++++------------ kernel/rtmutex.c | 63 ++++++++++++++++++++++++++++++------------------------- kernel/sched.c | 45 ++++++++++++++++++++++++++++++--------- 3 files changed, 84 insertions(+), 52 deletions(-)
kernel/sched.c +35 -10
@@ -360,6 +360,25 @@ static inline void finish_lock_switch(ru#endif /* __ARCH_WANT_UNLOCKED_CTXSW */ /* +* __task_rq_lock - lock the runqueue a given task resides on. +* Must be called interrupts disabled. +*/ +static inline runqueue_t *__task_rq_lock (task_t *p) + __acquires(rq->lock) +{ +struct runqueue *rq; + +repeat_lock_task: + rq =task_rq (p); +spin_lock (&rq->lock); +if (unlikely (rq !=task_rq (p))) { +spin_unlock (&rq->lock); + goto repeat_lock_task; + } +return rq; +} + +/* * task_rq_lock - lock the runqueue a given task resides on and disable * interrupts. Note the ordering: we can safely lookup the task_rq without * explicitly disabling preemption.
@@ -380,6 +399,12 @@ repeat_lock_task:return rq; } +static inline void __task_rq_unlock (runqueue_t *rq) + __releases(rq->lock) +{ +spin_unlock (&rq->lock); +} +static inline void task_rq_unlock (runqueue_t *rq,unsigned long *flags) __releases(rq->lock) {
@@ -4016,17 +4041,17 @@ recheck:* make sure no PI-waiters arrive (or leave) while we are * changing the priority of the task: */ -spin_lock (&p->pi_lock); +spin_lock_irqsave (&p->pi_lock, flags);/* * To be able to change p->policy safely, the apropriate * runqueue lock must be held. */ - rq =task_rq_lock (p, &flags); + rq =__task_rq_lock (p);/* recheck policy now with rq lock held */ if (unlikely (oldpolicy !=-1 && oldpolicy != p->policy)) { policy = oldpolicy =-1 ; -task_rq_unlock (rq, &flags); -spin_unlock (&p->pi_lock); +__task_rq_unlock (rq); +spin_unlock_irqrestore (&p->pi_lock, flags); goto recheck; } array = p->array;
@@ -4047,8 +4072,8 @@ recheck: }else if (TASK_PREEMPTS_CURR (p, rq))resched_task (rq->curr); } -task_rq_unlock (rq, &flags); -spin_unlock (&p->pi_lock); +__task_rq_unlock (rq); +spin_unlock_irqrestore (&p->pi_lock, flags);return 0 ; }
@@ -6680,8 +6705,8 @@ void normalize_rt_tasks(void)if (!rt_task (p))continue ; -spin_lock (&p->pi_lock); - rq =task_rq_lock (p, &flags); +spin_lock_irqsave (&p->pi_lock, flags); + rq =__task_rq_lock (p); array = p->array;if (array)
@@ -6692,8 +6717,8 @@ void normalize_rt_tasks(void)resched_task (rq->curr); } -task_rq_unlock (rq, &flags); -spin_unlock (&p->pi_lock); +__task_rq_unlock (rq); +spin_unlock_irqrestore (&p->pi_lock, flags); }read_unlock_irq (&tasklist_lock); }
kernel/futex.c +14 -14
@@ -358,9 +358,9 @@ static void free_pi_state(struct futex_pWARN_ON (!pi_state->owner);WARN_ON (!rt_mutex_is_locked (&pi_state->pi_mutex)); -spin_lock (&pi_state->owner->pi_lock); +spin_lock_irq (&pi_state->owner->pi_lock);list_del_init (&pi_state->list); -spin_unlock (&pi_state->owner->pi_lock); +spin_unlock_irq (&pi_state->owner->pi_lock);rt_mutex_proxy_unlock (&pi_state->pi_mutex, pi_state->owner);
@@ -422,18 +422,18 @@ void exit_pi_state_list(struct task_stru* pi_state_list anymore, but we have to be careful * versus waiters unqueueing themselfs */ -spin_lock (&curr->pi_lock); +spin_lock_irq (&curr->pi_lock);while (!list_empty (head)) { next = head->next; pi_state = list_entry(next, struct futex_pi_state, list); key = pi_state->key; -spin_unlock (&curr->pi_lock); +spin_unlock_irq (&curr->pi_lock); hb =hash_futex (&key);spin_lock (&hb->lock); -spin_lock (&curr->pi_lock); +spin_lock_irq (&curr->pi_lock);if (head->next != next) {spin_unlock (&hb->lock);continue ;
@@ -444,15 +444,15 @@ void exit_pi_state_list(struct task_struWARN_ON (pi_state->owner != curr); pi_state->owner =NULL ; -spin_unlock (&curr->pi_lock); +spin_unlock_irq (&curr->pi_lock);rt_mutex_unlock (&pi_state->pi_mutex);spin_unlock (&hb->lock); -spin_lock (&curr->pi_lock); +spin_lock_irq (&curr->pi_lock); } -spin_unlock (&curr->pi_lock); +spin_unlock_irq (&curr->pi_lock); } static int
@@ -500,10 +500,10 @@ lookup_pi_state(u32 uval, struct futex_h/* Store the key for possible exit cleanups: */ pi_state->key = me->key; -spin_lock (&p->pi_lock); +spin_lock_irq (&p->pi_lock);list_add (&pi_state->list, &p->pi_state_list); pi_state->owner = p; -spin_unlock (&p->pi_lock); +spin_unlock_irq (&p->pi_lock);put_task_struct (p);
@@ -1213,17 +1213,17 @@ static int do_futex_lock_pi(u32 __user */* Owner died? */ if (q.pi_state->owner !=NULL ) { -spin_lock (&q.pi_state->owner->pi_lock); +spin_lock_irq (&q.pi_state->owner->pi_lock);list_del_init (&q.pi_state->list); -spin_unlock (&q.pi_state->owner->pi_lock); +spin_unlock_irq (&q.pi_state->owner->pi_lock); }else newtid |=FUTEX_OWNER_DIED ; q.pi_state->owner = current; -spin_lock (¤t->pi_lock); +spin_lock_irq (¤t->pi_lock);list_add (&q.pi_state->list, ¤t->pi_state_list); -spin_unlock (¤t->pi_lock); +spin_unlock_irq (¤t->pi_lock);/* Unqueue and drop the lock */ unqueue_me_pi (&q, hb);
kernel/rtmutex.c +35 -28
214 lines not on this page · raw message
Quoted 5 lines · Thomas Gleixner
From: Ingo Molnar <[email protected]>
The lock validator detected a possible deadlock between tasklist lock and
task->pi_lock. Prevent the deadlock by disabling interrupts acrosspi_lockoperations.
some background about how this bug sneaked in: we have pi_lock as an irq-safe lock in the -rt tree but we didnt realize that it was needed in the pi-futex implementation too. The race window is pretty narrow there so it didnt trigger during testing. (the window is wider in -rt)
Ingo