bpf: sockmap, fix preempt_rt splat when using raw_spin_lock_t
Subsystems (5)
John Fastabend
maintainerPATCH 1/1 bpf: sockmap, fix preempt_rt splat when using raw_spin_lock_t
Sockmap and sockhash maps are a collection of psocks that are objects representing a socket plus a set of metadata needed to manage the BPF programs associated with the socket. These maps use the stab->lock to protect from concurrent operations on the maps, e.g. trying to insert to objects into the array at the same time in the same slot. Additionally, a sockhash map has a bucket lock to protect iteration and insert/delete into the hash entry.
Each psock has a psock->link which is a linked list of all the maps that a psock is attached to. This allows a psock (socket) to be included in multiple sockmap and sockhash maps. This linked list is protected the psock->link_lock.
They _must_ be nested correctly to avoid deadlock,
lock(stab->lock)
: do BPF map operations and psock insert/delete lock(psock->link_lock)
: add map to psock linked list of maps
unlock(psock->link_lock)
unlock(stab->lock)
For non PREEMPT_RT kernels both raw_spin_lock_t and spin_lock_t are guaranteed to not sleep. But, with PREEMPT_RT kernels the spin_lock_t variants may sleep. In the current code we have many patterns like this,
rcu_critical_section:
raw_spin_lock(stab->lock)
spin_lock(psock->link_lock) <- may sleep ouch
spin_unlock(psock->link_lock)
raw_spin_unlock(stab->lock) rcu_critical_section
Nesting spin_lock() inside a raw_spin_lock() violates locking rules for PREEMPT_RT kernels. And additionally we do alloc(GFP_ATOMICS) inside the stab->lock, but those might sleep on PREEMPT_RT kernels. The result is splats like this,
./test_progs -t sockmap_basic
[ 33.344330] bpf_testmod: loading out-of-tree module taints kernel.
[ 33.441933]
[ 33.442089] =============================
[ 33.442421] [ BUG: Invalid wait context ]
[ 33.442763] 6.5.0-rc5-01731-gec0ded2e0282 #4958 Tainted: G O
[ 33.443320] -----------------------------
[ 33.443624] test_progs/2073 is trying to lock:
[ 33.443960] ffff888102a1c290 (&psock->link_lock){....}-{3:3}, at: sock_map_update_common+0x2c2/0x3d0
[ 33.444636] other info that might help us debug this:
[ 33.444991] context-{5:5}
[ 33.445183] 3 locks held by test_progs/2073:
[ 33.445498] #0: ffff88811a208d30 (sk_lock-AF_INET){+.+.}-{0:0}, at: sock_map_update_elem_sys+0xff/0x330
[ 33.446159] #1: ffffffff842539e0 (rcu_read_lock){....}-{1:3}, at: sock_map_update_elem_sys+0xf5/0x330
[ 33.446809] #2: ffff88810d687240 (&stab->lock){+...}-{2:2}, at: sock_map_update_common+0x177/0x3d0
[ 33.447445] stack backtrace:
[ 33.447655] CPU: 10 PIDTo fix observe we can't readily remove the allocations (for that we would need to use/create something similar to bpf_map_alloc). So convert raw_spin_lock_t to spin_lock_t. We note that sock_map_update that would trigger the allocate and potential sleep is only allowed through sys_bpf ops and via sock_ops which precludes hw interrupts and low level atomic sections in RT preempt kernel. On non RT preempt kernel there are no changes here and spin locks sections and alloc(GFP_ATOMIC) are still not sleepable.
--- net/core/sock_map.c | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-)
net/core/sock_map.c +18 -18
@@ -18,7 +18,7 @@ struct bpf_stab {struct bpf_map map;struct sock **sks;struct sk_psock_progs progs; -raw_spinlock_t lock; +spinlock_t lock; };#define SOCK_CREATE_FLAG_MASK \
@@ -44,7 +44,7 @@ static struct bpf_map *sock_map_alloc(union bpf_attr *attr)return ERR_PTR (-ENOMEM );bpf_map_init_from_attr (&stab->map, attr); -raw_spin_lock_init (&stab->lock); +spin_lock_init (&stab->lock); stab->sks = bpf_map_area_alloc((u64) stab->map.max_entries * sizeof(struct sock *),
@@ -411,7 +411,7 @@ static int __sock_map_delete(struct bpf_stab *stab, struct sock *sk_test,struct sock *sk;int err =0 ; -raw_spin_lock_bh (&stab->lock); +spin_lock_bh (&stab->lock); sk = *psk;if (!sk_test || sk_test == sk) sk =xchg (psk,NULL );
@@ -421,7 +421,7 @@ static int __sock_map_delete(struct bpf_stab *stab, struct sock *sk_test,else err = -EINVAL ; -raw_spin_unlock_bh (&stab->lock); +spin_unlock_bh (&stab->lock);return err; }
@@ -487,7 +487,7 @@ static int sock_map_update_common(struct bpf_map *map, u32 idx, psock =sk_psock (sk);WARN_ON_ONCE (!psock); -raw_spin_lock_bh (&stab->lock); +spin_lock_bh (&stab->lock); osk = stab->sks[idx];if (osk && flags ==BPF_NOEXIST ) { ret = -EEXIST ;
@@ -501,10 +501,10 @@ static int sock_map_update_common(struct bpf_map *map, u32 idx, stab->sks[idx] = sk;if (osk)sock_map_unref (osk, &stab->sks[idx]); -raw_spin_unlock_bh (&stab->lock); +spin_unlock_bh (&stab->lock);return 0 ; out_unlock: -raw_spin_unlock_bh (&stab->lock); +spin_unlock_bh (&stab->lock);if (psock)sk_psock_put (sk, psock); out_free:
@@ -835,7 +835,7 @@ struct bpf_shtab_elem {struct bpf_shtab_bucket {struct hlist_head head; -raw_spinlock_t lock; +spinlock_t lock; };struct bpf_shtab {
@@ -910,7 +910,7 @@ static void sock_hash_delete_from_link(struct bpf_map *map, struct sock *sk,* is okay since it's going away only after RCU grace period. * However, we need to check whether it's still present. */ -raw_spin_lock_bh (&bucket->lock); +spin_lock_bh (&bucket->lock); elem_probe =sock_hash_lookup_elem_raw (&bucket->head, elem->hash, elem->key, map->key_size);if (elem_probe && elem_probe == elem) {
@@ -918,7 +918,7 @@ static void sock_hash_delete_from_link(struct bpf_map *map, struct sock *sk,sock_map_unref (elem->sk, elem);sock_hash_free_elem (htab, elem); } -raw_spin_unlock_bh (&bucket->lock); +spin_unlock_bh (&bucket->lock); }static long sock_hash_delete_elem (struct bpf_map *map,void *key)
@@ -932,7 +932,7 @@ static long sock_hash_delete_elem(struct bpf_map *map, void *key) hash =sock_hash_bucket_hash (key, key_size); bucket =sock_hash_select_bucket (htab, hash); -raw_spin_lock_bh (&bucket->lock); +spin_lock_bh (&bucket->lock); elem =sock_hash_lookup_elem_raw (&bucket->head, hash, key, key_size);if (elem) {hlist_del_rcu (&elem->node);
@@ -940,7 +940,7 @@ static long sock_hash_delete_elem(struct bpf_map *map, void *key)sock_hash_free_elem (htab, elem); ret =0 ; } -raw_spin_unlock_bh (&bucket->lock); +spin_unlock_bh (&bucket->lock);return ret; }
@@ -1000,7 +1000,7 @@ static int sock_hash_update_common(struct bpf_map *map, void *key, hash =sock_hash_bucket_hash (key, key_size); bucket =sock_hash_select_bucket (htab, hash); -raw_spin_lock_bh (&bucket->lock); +spin_lock_bh (&bucket->lock); elem =sock_hash_lookup_elem_raw (&bucket->head, hash, key, key_size);if (elem && flags ==BPF_NOEXIST ) { ret = -EEXIST ;
@@ -1026,10 +1026,10 @@ static int sock_hash_update_common(struct bpf_map *map, void *key,sock_map_unref (elem->sk, elem);sock_hash_free_elem (htab, elem); } -raw_spin_unlock_bh (&bucket->lock); +spin_unlock_bh (&bucket->lock);return 0 ; out_unlock: -raw_spin_unlock_bh (&bucket->lock); +spin_unlock_bh (&bucket->lock);sk_psock_put (sk, psock); out_free:sk_psock_free_link (link);
@@ -1115,7 +1115,7 @@ static struct bpf_map *sock_hash_alloc(union bpf_attr *attr)for (i =0 ; i < htab->buckets_num; i++) {INIT_HLIST_HEAD (&htab->buckets[i].head); -raw_spin_lock_init (&htab->buckets[i].lock); +spin_lock_init (&htab->buckets[i].lock); }return &htab->map;
@@ -1147,11 +1147,11 @@ static void sock_hash_free(struct bpf_map *map)* exists, psock exists and holds a ref to socket. That * lets us to grab a socket ref too. */ -raw_spin_lock_bh (&bucket->lock); +spin_lock_bh (&bucket->lock);hlist_for_each_entry (elem, &bucket->head, node)sock_hold (elem->sk);hlist_move_list (&bucket->head, &unlink_list); -raw_spin_unlock_bh (&bucket->lock); +spin_unlock_bh (&bucket->lock);/* Process removed entries out of atomic context to * block for socket lock before deleting the psock's
Signature · 1 line
2.33.0
Daniel Borkmann
maintainerOn 8/30/23 7:35 AM, John Fastabend wrote:
[...]
Quoted 8 lines
net/core/sock_map.c | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-)net/core/sock_map.c +0
-0
net/core/sock_map.c struct bpf_stab { · line 22
Quoted 6 lines
net/core/sock_map.c +1
-1@@ -18,7 +18,7 @@ struct bpf_stab {struct bpf_map map;struct sock **sks;struct sk_psock_progs progs; -raw_spinlock_t lock; +spinlock_t lock; };
Looks good to me, and I agree that it's better to take this direction rather than converting both to raw_spinlock_t if there is no good reason for it. Thanks for looking into it, applied!
This patch was applied to bpf/bpf.git (master) by Daniel Borkmann <[email protected]>:
Hello:
This patch was applied to bpf/bpf.git (master)
by Daniel Borkmann <[email protected]>:
Quoted 10 lines · On Tue, 29 Aug 2023 22:35:17 -0700 you wrote:
Sockmap and sockhash maps are a collection of psocks that are objects representing a socket plus a set of metadata needed to manage the BPF programs associated with the socket. These maps use the
stab->lockto protect from concurrent operations on the maps, e.g. trying to insert to objects into the array at the same time in the same slot. Additionally, a sockhash map has a bucket lock to protect iteration and insert/delete into the hash entry.[...]
Here is the summary with links:
- [bpf] bpf: sockmap, fix preempt_rt splat when using raw_spin_lock_t
https://git.kernel.org/bpf/bpf/c/35d2b7ffffc1
You are awesome, thank you!
Signature · 2 lines
Deet-doot-dot, I am a bot. https://korg.docs.kernel.org/patchwork/pwbot.html