futex: Fix might_sleep() warning in futex_pivot_pending()
Peter Zijlstra
maintainerPATCH 1/1 futex: Fix might_sleep() warning in futex_pivot_pending()
A younger me put a WARN in might_sleep() to warn about nested sleep loops. This younger me also build a wait-loop variant that can deal with it. This wait-loop variant doesn't have all the fancy wrappers, since it isn't used much. It also lacks wait-bit support.
Add the wait-bit support and use it to fix the nested wait issue.
--- include/linux/wait.h | 1 + include/linux/wait_bit.h | 1 + kernel/futex/core.c | 27 ++++++++++++++++++++++++++- kernel/sched/wait.c | 15 +++++++++++++++ kernel/sched/wait_bit.c | 15 ++++++++++++--- 5 files changed, 55 insertions(+), 4 deletions(-)
include/linux/wait.h +1 -0
@@ -1228,6 +1228,7 @@ long prepare_to_wait_event(struct wait_qvoid finish_wait (struct wait_queue_head *wq_head,struct wait_queue_entry *wq_entry);long wait_woken (struct wait_queue_entry *wq_entry,unsigned mode,long timeout);int woken_wake_function (struct wait_queue_entry *wq_entry,unsigned mode,int sync,void *key); +int woken_wake_bit_function (struct wait_queue_entry *wq_entry,unsigned mode,int sync,void *key);int autoremove_wake_function (struct wait_queue_entry *wq_entry,unsigned mode,int sync,void *key);#define DEFINE_WAIT_FUNC (name, function) \
include/linux/wait_bit.h +1 -0
@@ -32,6 +32,7 @@ int out_of_line_wait_on_bit_timeout(unsiint out_of_line_wait_on_bit_lock (unsigned long *word,int ,wait_bit_action_f *action,unsigned int mode);struct wait_queue_head *bit_waitqueue (unsigned long *word,int bit);extern void __init wait_bit_init (void ); +extern struct wait_bit_key *__var_wake_key (struct wait_queue_entry *wq_entry,void *arg);int wake_bit_function (struct wait_queue_entry *wq_entry,unsigned mode,int sync,void *key);
kernel/futex/core.c +26 -1
@@ -46,6 +46,7 @@#include <linux/slab.h> #include <linux/vmalloc.h> #include <linux/kmemleak.h> +#include <linux/wait_bit.h> #include <vdso/futex.h>
@@ -1886,11 +1887,35 @@ static int futex_hash_allocate(unsignedfutex_hash_bucket_init (&fph->queues[i]);if (custom) { +struct wait_bit_queue_entry __wbq_entry; +struct wait_queue_head *__wq_head; +/* * Only let prctl() wait / retry; don't unduly delay clone(). */ again: -wait_var_event (mm,futex_pivot_pending (mm)); + __wq_head =__var_waitqueue (mm); +init_wait_var_entry (&__wbq_entry, mm,0 ); + __wbq_entry.wq_entry.func = woken_wake_bit_function; +add_wait_queue (__wq_head, &__wbq_entry.wq_entry); + +/* +* add_wait_queue() futex_ref_put() +* MB (this) MB (implied) +* futex_pivot_pending() wake_up_var() +* waitqueue_active() +* +* Notably, it must not be possible to see +* !futex_pivot_pending() && !waitqueue_active(). +*/ +smp_mb (); + +while (!futex_pivot_pending (mm) && +wait_woken (&__wbq_entry.wq_entry,TASK_UNINTERRUPTIBLE , +MAX_SCHEDULE_TIMEOUT )) +/* empty */ ; + +remove_wait_queue (__wq_head, &__wbq_entry.wq_entry); } scoped_guard(mutex, &mm->futex.phash.lock) {
kernel/sched/wait.c +15 -0
@@ -5,6 +5,7 @@* (C) 2004 Nadia Yvette Chambers, Oracle */ #include "sched.h" +#include <linux/wait_bit.h> void __init_waitqueue_head (struct wait_queue_head *wq_head,const char *name,struct lock_class_key *key) {
@@ -463,3 +464,17 @@ int woken_wake_function(struct wait_queureturn default_wake_function (wq_entry, mode, sync, key); }EXPORT_SYMBOL (woken_wake_function); + +int woken_wake_bit_function (struct wait_queue_entry *wq_entry,unsigned mode,int sync,void *arg) +{ +struct wait_bit_key *key =__var_wake_key (wq_entry, arg); +if (!key) +return 0 ; + +/* Pairs with the smp_store_mb() in wait_woken(). */ +smp_mb ();/* C */ + wq_entry->flags |=WQ_FLAG_WOKEN ; + +return default_wake_function (wq_entry, mode, sync, key); +} +EXPORT_SYMBOL (woken_wake_bit_function);
kernel/sched/wait_bit.c +12 -3
@@ -167,9 +167,7 @@ wait_queue_head_t *__var_waitqueue(void }EXPORT_SYMBOL (__var_waitqueue); -static int -var_wake_function (struct wait_queue_entry *wq_entry,unsigned int mode, -int sync,void *arg) +struct wait_bit_key *__var_wake_key (struct wait_queue_entry *wq_entry,void *arg) {struct wait_bit_key *key = arg; struct wait_bit_queue_entry *wbq_entry =
@@ -177,6 +175,17 @@ var_wake_function(struct wait_queue_entrif (wbq_entry->key.flags != key->flags || wbq_entry->key.bit_nr != key->bit_nr) +return NULL ; + +return key; +} + +static int +var_wake_function (struct wait_queue_entry *wq_entry,unsigned int mode, +int sync,void *arg) +{ +struct wait_bit_key *key =__var_wake_key (wq_entry, arg); +if (!key)return 0 ;return autoremove_wake_function (wq_entry, mode, sync, key);
The following commit has been merged into the locking/urgent branch of tip:
The following commit has been merged into the locking/urgent branch of tip:
Commit-ID: d8aa5dd97944a72d4a9e3cc79bb80fcac7d6e829 Gitweb: https://git.kernel.org/tip/d8aa5dd97944a72d4a9e3cc79bb80fcac7d6e829 Author: Peter Zijlstra <[email protected]> AuthorDate: Thu, 20 Aug 2026 09:49:27 +02:00 Committer: Thomas Gleixner <[email protected]> CommitterDate: Thu, 20 Aug 2026 18:26:42 +02:00
futex: Fix might_sleep() warning in futex_pivot_pending()
A younger me put a WARN in might_sleep() to warn about nested sleep loops. This younger me also build a wait-loop variant that can deal with it. This wait-loop variant doesn't have all the fancy wrappers, since it isn't used much. It also lacks wait-bit support.
Add the wait-bit support and use it to fix the nested wait issue.
--- include/linux/wait.h | 1 + include/linux/wait_bit.h | 1 + kernel/futex/core.c | 27 ++++++++++++++++++++++++++- kernel/sched/wait.c | 15 +++++++++++++++ kernel/sched/wait_bit.c | 14 +++++++++++--- 5 files changed, 54 insertions(+), 4 deletions(-)
include/linux/wait.h +1 -0
@@ -1228,6 +1228,7 @@ long prepare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_envoid finish_wait (struct wait_queue_head *wq_head,struct wait_queue_entry *wq_entry);long wait_woken (struct wait_queue_entry *wq_entry,unsigned mode,long timeout);int woken_wake_function (struct wait_queue_entry *wq_entry,unsigned mode,int sync,void *key); +int woken_wake_bit_function (struct wait_queue_entry *wq_entry,unsigned mode,int sync,void *key);int autoremove_wake_function (struct wait_queue_entry *wq_entry,unsigned mode,int sync,void *key);#define DEFINE_WAIT_FUNC (name, function) \
include/linux/wait_bit.h +1 -0
@@ -32,6 +32,7 @@ int out_of_line_wait_on_bit_timeout(unsigned long *word, int, wait_bit_action_fint out_of_line_wait_on_bit_lock (unsigned long *word,int ,wait_bit_action_f *action,unsigned int mode);struct wait_queue_head *bit_waitqueue (unsigned long *word,int bit);extern void __init wait_bit_init (void ); +extern struct wait_bit_key *__var_wake_key (struct wait_queue_entry *wq_entry,void *arg);int wake_bit_function (struct wait_queue_entry *wq_entry,unsigned mode,int sync,void *key);
kernel/futex/core.c +26 -1
@@ -45,6 +45,7 @@#include <linux/rseq.h> #include <linux/slab.h> #include <linux/vmalloc.h> +#include <linux/wait_bit.h> #include <vdso/futex.h>
@@ -1884,11 +1885,35 @@ static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags)futex_hash_bucket_init (&fph->queues[i]);if (custom) { +struct wait_bit_queue_entry __wbq_entry; +struct wait_queue_head *__wq_head; +/* * Only let prctl() wait / retry; don't unduly delay clone(). */ again: -wait_var_event (mm,futex_pivot_pending (mm)); + __wq_head =__var_waitqueue (mm); +init_wait_var_entry (&__wbq_entry, mm,0 ); + __wbq_entry.wq_entry.func = woken_wake_bit_function; +add_wait_queue (__wq_head, &__wbq_entry.wq_entry); + +/* +* add_wait_queue() futex_ref_put() +* MB (this) MB (implied) +* futex_pivot_pending() wake_up_var() +* waitqueue_active() +* +* Notably, it must not be possible to see +* !futex_pivot_pending() && !waitqueue_active(). +*/ +smp_mb (); + +while (!futex_pivot_pending (mm) && +wait_woken (&__wbq_entry.wq_entry,TASK_UNINTERRUPTIBLE , +MAX_SCHEDULE_TIMEOUT )) +/* empty */ ; + +remove_wait_queue (__wq_head, &__wbq_entry.wq_entry); } scoped_guard(mutex, &mm->futex.phash.lock) {
kernel/sched/wait.c +15 -0
@@ -5,6 +5,7 @@* (C) 2004 Nadia Yvette Chambers, Oracle */ #include "sched.h" +#include <linux/wait_bit.h> void __init_waitqueue_head (struct wait_queue_head *wq_head,const char *name,struct lock_class_key *key) {
@@ -463,3 +464,17 @@ int woken_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int syreturn default_wake_function (wq_entry, mode, sync, key); }EXPORT_SYMBOL (woken_wake_function); + +int woken_wake_bit_function (struct wait_queue_entry *wq_entry,unsigned mode,int sync,void *arg) +{ +struct wait_bit_key *key =__var_wake_key (wq_entry, arg); +if (!key) +return 0 ; + +/* Pairs with the smp_store_mb() in wait_woken(). */ +smp_mb ();/* C */ + wq_entry->flags |=WQ_FLAG_WOKEN ; + +return default_wake_function (wq_entry, mode, sync, key); +} +EXPORT_SYMBOL (woken_wake_bit_function);
kernel/sched/wait_bit.c +11 -3
@@ -167,9 +167,7 @@ wait_queue_head_t *__var_waitqueue(void *p) }EXPORT_SYMBOL (__var_waitqueue); -static int -var_wake_function (struct wait_queue_entry *wq_entry,unsigned int mode, -int sync,void *arg) +struct wait_bit_key *__var_wake_key (struct wait_queue_entry *wq_entry,void *arg) {struct wait_bit_key *key = arg; struct wait_bit_queue_entry *wbq_entry =
@@ -177,6 +175,16 @@ var_wake_function(struct wait_queue_entry *wq_entry, unsigned int mode,if (wbq_entry->key.flags != key->flags || wbq_entry->key.bit_nr != key->bit_nr) +return NULL ; + +return key; +} + +static int var_wake_function (struct wait_queue_entry *wq_entry,unsigned int mode, +int sync,void *arg) +{ +struct wait_bit_key *key =__var_wake_key (wq_entry, arg); +if (!key)return 0 ;return autoremove_wake_function (wq_entry, mode, sync, key);