FAILED: patch "[PATCH] wait_on_bit: add an acquire memory barrier" failed to apply to 5.4-stable tree
Subsystems (8)
[email protected]
maintainerThe patch below does not apply to the 5.4-stable tree.
If someone wants it applied there, or to any other stable or longterm tree, then please email the backport, including the original git commit id to <[email protected]>.
thanks,
greg k-h
------------------ original commit in Linus's tree ------------------
From 8238b4579866b7c1bb99883cfe102a43db5506ff Mon Sep 17 00:00:00 2001
From: Mikulas Patocka <[email protected]>
Date: Fri, 26 Aug 2022 09:17:08 -0400
Subject: [PATCH] wait_on_bit: add an acquire memory barrier
There are several places in the kernel where wait_on_bit is not followed by a memory barrier (for example, in drivers/md/dm-bufio.c:new_read).
On architectures with weak memory ordering, it may happen that memory accesses that follow wait_on_bit are reordered before wait_on_bit and they may return invalid data.
Fix this class of bugs by introducing a new function "test_bit_acquire" that works like test_bit, but has acquire memory ordering semantics.
Signed-off-by Mikulas Patocka <[email protected]>
Acked-by Will Deacon <[email protected]>
Signed-off-by Linus Torvalds <[email protected]>
Documentation/atomic_bitops.txt +4 -6
@@ -58,13 +58,11 @@ Like with atomic_t, the rule of thumb is: - RMW operations that have a return value are fully ordered. - - RMW operations that are conditional are unordered on FAILURE, - otherwise the above rules apply. In the case of test_and_set_bit_lock(), - if the bit in memory is unchanged by the operation then it is deemed to have - failed. + - RMW operations that are conditional are fully ordered. -Except for a successful test_and_set_bit_lock() which has ACQUIRE semantics and -clear_bit_unlock() which has RELEASE semantics. +Except for a successful test_and_set_bit_lock() which has ACQUIRE semantics, +clear_bit_unlock() which has RELEASE semantics and test_bit_acquire which has +ACQUIRE semantics. Since a platform only has a single means of achieving atomic operations the same barriers as for atomic_t are used, see atomic_t.txt.
arch/x86/include/asm/bitops.h +21 -0
@@ -207,6 +207,20 @@ static __always_inline bool constant_test_bit(long nr, const volatile unsigned l (addr[nr >> _BITOPS_LONG_SHIFT])) != 0; } +static __always_inline bool constant_test_bit_acquire(long nr, const volatile unsigned long *addr) +{ +bool oldbit; + + asm volatile("testb %2,%1" + CC_SET(nz) + : CC_OUT(nz) (oldbit) + : "m" (((unsigned char *)addr)[nr >> 3]), + "i" (1 << (nr & 7)) + :"memory"); + +return oldbit; +} + static __always_inline bool variable_test_bit(long nr, volatile const unsigned long *addr) {bool oldbit;
@@ -226,6 +240,13 @@ arch_test_bit(unsigned long nr, const volatile unsigned long *addr)variable_test_bit (nr, addr); } +static __always_inline bool +arch_test_bit_acquire (unsigned long nr,const volatile unsigned long *addr) +{ +return __builtin_constant_p (nr) ?constant_test_bit_acquire (nr, addr) : +variable_test_bit (nr, addr); +} +/** * __ffs - find first set bit in word * @word: The word to search
include/asm-generic/bitops/generic-non-atomic.h +14 -0
@@ -4,6 +4,7 @@#define __ASM_GENERIC_BITOPS_GENERIC_NON_ATOMIC_H#include <linux/bits.h> +#include <asm/barrier.h> #ifndef _LINUX_BITOPS_H#error only <linux/bitops.h> can be included directly
@@ -127,6 +128,18 @@ generic_test_bit(unsigned long nr, const volatile unsigned long *addr)return 1UL & (addr[BIT_WORD (nr)] >> (nr & (BITS_PER_LONG -1 ))); } +/** +* generic_test_bit_acquire - Determine, with acquire semantics, whether a bit is set +* @nr: bit number to test +* @addr: Address to start counting from +*/ +static __always_inline bool +generic_test_bit_acquire (unsigned long nr,const volatile unsigned long *addr) +{ +unsigned long *p = ((unsigned long *)addr) +BIT_WORD (nr); +return 1UL & (smp_load_acquire (p) >> (nr & (BITS_PER_LONG -1 ))); +} +/* * const_*() definitions provide good compile-time optimizations when * the passed arguments can be resolved at compile time.
@@ -137,6 +150,7 @@ generic_test_bit(unsigned long nr, const volatile unsigned long *addr)#define const___test_and_set_bit generic___test_and_set_bit#define const___test_and_clear_bit generic___test_and_clear_bit#define const___test_and_change_bit generic___test_and_change_bit +#define const_test_bit_acquire generic_test_bit_acquire/** * const_test_bit - Determine whether a bit is set
include/asm-generic/bitops/instrumented-non-atomic.h +12 -0
@@ -142,4 +142,16 @@ _test_bit(unsigned long nr, const volatile unsigned long *addr) return arch_test_bit(nr, addr); } +/** +* _test_bit_acquire - Determine, with acquire semantics, whether a bit is set +* @nr: bit number to test +* @addr: Address to start counting from +*/ +static __always_inline bool +_test_bit_acquire (unsigned long nr,const volatile unsigned long *addr) +{ +instrument_atomic_read (addr +BIT_WORD (nr),sizeof (long )); +return arch_test_bit_acquire (nr, addr); +} +#endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H */
include/asm-generic/bitops/non-atomic.h +1 -0
@@ -13,6 +13,7 @@#define arch___test_and_change_bit generic___test_and_change_bit#define arch_test_bit generic_test_bit +#define arch_test_bit_acquire generic_test_bit_acquire#include <asm-generic/bitops/non-instrumented-non-atomic.h>
include/asm-generic/bitops/non-instrumented-non-atomic.h +1 -0
@@ -12,5 +12,6 @@#define ___test_and_change_bit arch___test_and_change_bit#define _test_bit arch_test_bit +#define _test_bit_acquire arch_test_bit_acquire#endif /* __ASM_GENERIC_BITOPS_NON_INSTRUMENTED_NON_ATOMIC_H */
include/linux/bitops.h +1 -0
@@ -59,6 +59,7 @@ extern unsigned long __sw_hweight64(__u64 w);#define __test_and_clear_bit (nr, addr) bitop(___test_and_clear_bit, nr, addr)#define __test_and_change_bit (nr, addr) bitop(___test_and_change_bit, nr, addr)#define test_bit (nr, addr) bitop(_test_bit, nr, addr) +#define test_bit_acquire (nr, addr) bitop(_test_bit_acquire, nr, addr)/* * Include this here because some architectures need generic_ffs/fls in
include/linux/buffer_head.h +1 -1
@@ -156,7 +156,7 @@ static __always_inline int buffer_uptodate(const struct buffer_head *bh)* make it consistent with folio_test_uptodate * pairs with smp_mb__before_atomic in set_buffer_uptodate */ -return (smp_load_acquire (&bh->b_state) & (1UL << BH_Uptodate)) !=0 ; +return test_bit_acquire (BH_Uptodate, &bh->b_state); }#define bh_offset (bh) ((unsigned long)(bh)->b_data & ~PAGE_MASK)
include/linux/wait_bit.h +4 -4
@@ -71,7 +71,7 @@ static inline int wait_on_bit(unsigned long *word, int bit, unsigned mode) {might_sleep (); -if (!test_bit (bit, word)) +if (!test_bit_acquire (bit, word))return 0 ; return out_of_line_wait_on_bit(word, bit, bit_wait,
@@ -96,7 +96,7 @@ static inline int wait_on_bit_io(unsigned long *word, int bit, unsigned mode) {might_sleep (); -if (!test_bit (bit, word)) +if (!test_bit_acquire (bit, word))return 0 ; return out_of_line_wait_on_bit(word, bit, bit_wait_io,
@@ -123,7 +123,7 @@ wait_on_bit_timeout(unsigned long *word, int bit, unsigned mode, unsigned long timeout) {might_sleep (); -if (!test_bit (bit, word)) +if (!test_bit_acquire (bit, word))return 0 ; return out_of_line_wait_on_bit_timeout(word, bit, bit_wait_timeout,
@@ -151,7 +151,7 @@ wait_on_bit_action(unsigned long *word, int bit, wait_bit_action_f *action, unsigned mode) {might_sleep (); -if (!test_bit (bit, word)) +if (!test_bit_acquire (bit, word))return 0 ;return out_of_line_wait_on_bit (word, bit, action, mode); }
kernel/sched/wait_bit.c +1 -1
@@ -47,7 +47,7 @@ __wait_on_bit(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_prepare_to_wait (wq_head, &wbq_entry->wq_entry, mode);if (test_bit (wbq_entry->key.bit_nr, wbq_entry->key.flags)) ret = (*action)(&wbq_entry->key, mode); - }while (test_bit (wbq_entry->key.bit_nr, wbq_entry->key.flags) && !ret); + }while (test_bit_acquire (wbq_entry->key.bit_nr, wbq_entry->key.flags) && !ret);finish_wait (wq_head, &wbq_entry->wq_entry);