bpf: support cloning sk storage on accept()
Subsystems (7)
v2 cover bpf: support cloning sk storage on accept()
Currently there is no way to propagate sk storage from the listener socket to a newly accepted one. Consider the following use case:
fd = socket();
setsockopt(fd, SOL_IP, IP_TOS,...);
/* ^^^ setsockopt BPF program triggers here and saves something
* into sk storage of the listener.
*/
listen(fd, ...);
while (client = accept(fd)) {
/* At this point all association between listener
* socket and newly accepted one is gone. New
* socket will not have any sk storage attached.
*/
}Let's add new BPF_F_CLONE flag that can be specified when creating a socket storage map. This new flag indicates that map contents should be cloned when the socket is cloned.
v2:
* remove spinlocks around selem_link_map/sk (Martin KaFai Lau)
* BPF_F_CLONE on a map, not selem (Martin KaFai Lau)
* hold a map while cloning (Martin KaFai Lau)
* use BTF maps in selftests (Yonghong Song)
* do proper cleanup selftests; don't call close(-1) (Yonghong Song)
* export bpf_map_inc_not_zero
Stanislav Fomichev (4):
bpf: export bpf_map_inc_not_zero
bpf: support cloning sk storage on accept()
bpf: sync bpf.h to tools/
selftests/bpf: add sockopt clone/inheritance test
include/linux/bpf.h | 2 + include/net/bpf_sk_storage.h | 10 + include/uapi/linux/bpf.h | 3 + kernel/bpf/syscall.c | 16 +- net/core/bpf_sk_storage.c | 100 ++++++- net/core/sock.c | 9 +- tools/include/uapi/linux/bpf.h | 3 + tools/testing/selftests/bpf/.gitignore | 1 + tools/testing/selftests/bpf/Makefile | 3 +- .../selftests/bpf/progs/sockopt_inherit.c | 97 +++++++ .../selftests/bpf/test_sockopt_inherit.c | 253 ++++++++++++++++++ 11 files changed, 487 insertions(+), 10 deletions(-) create mode 100644 tools/testing/selftests/bpf/progs/sockopt_inherit.c create mode 100644 tools/testing/selftests/bpf/test_sockopt_inherit.c
Signature · 1 line
2.23.0.rc1.153.gdeed80330f-goog
PATCH v2 1/4 bpf: export bpf_map_inc_not_zero
Rename existing bpf_map_inc_not_zero to __bpf_map_inc_not_zero to indicate that it's caller's responsibility to do proper locking. Create and export bpf_map_inc_not_zero wrapper that properly locks map_idr_lock. Will be used in the next commit to hold a map while cloning a socket.
--- include/linux/bpf.h | 2 ++ kernel/bpf/syscall.c | 16 +++++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-)
include/linux/bpf.h +2 -0
@@ -647,6 +647,8 @@ void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock);struct bpf_map *bpf_map_get_with_uref (u32 ufd);struct bpf_map *__bpf_map_get (struct fd f);struct bpf_map *__must_check bpf_map_inc (struct bpf_map *map,bool uref); +struct bpf_map *__must_check bpf_map_inc_not_zero (struct bpf_map *map, +bool uref);void bpf_map_put_with_uref (struct bpf_map *map);void bpf_map_put (struct bpf_map *map);int bpf_map_charge_memlock (struct bpf_map *map,u32 pages);
kernel/bpf/syscall.c +13 -3
@@ -683,8 +683,8 @@ struct bpf_map *bpf_map_get_with_uref(u32 ufd) }/* map_idr_lock should have been held */ -static struct bpf_map *bpf_map_inc_not_zero (struct bpf_map *map, -bool uref) +static struct bpf_map *__bpf_map_inc_not_zero (struct bpf_map *map, +bool uref) {int refold;
@@ -704,6 +704,16 @@ static struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map,return map; } +struct bpf_map *bpf_map_inc_not_zero (struct bpf_map *map,bool uref) +{ +spin_lock_bh (&map_idr_lock); + map =__bpf_map_inc_not_zero (map, uref); +spin_unlock_bh (&map_idr_lock); + +return map; +} +EXPORT_SYMBOL_GPL (bpf_map_inc_not_zero); +int __weak bpf_stackmap_copy (struct bpf_map *map,void *key,void *value) {return -ENOTSUPP ;
@@ -2177,7 +2187,7 @@ static int bpf_map_get_fd_by_id(const union bpf_attr *attr)spin_lock_bh (&map_idr_lock); map =idr_find (&map_idr, id);if (map) - map =bpf_map_inc_not_zero (map, true); + map =__bpf_map_inc_not_zero (map, true);else map =ERR_PTR (-ENOENT );spin_unlock_bh (&map_idr_lock);
Signature · 1 line
2.23.0.rc1.153.gdeed80330f-goog
Re: [PATCH bpf-next v2 1/4] bpf: export bpf_map_inc_not_zero
Quoted 9 lines · Stanislav Fomichev
Rename existing
bpf_map_inc_not_zeroto__bpf_map_inc_not_zeroto indicate that it's caller's responsibility to do proper locking. Create and exportbpf_map_inc_not_zerowrapper that properly locksmap_idr_lock. Will be used in the next commit to hold a map while cloning a socket.Cc Martin KaFai Lau <[email protected]>Cc Yonghong Song <[email protected]>Signed-off-by Stanislav Fomichev <[email protected]>
PATCH v2 2/4 bpf: support cloning sk storage on accept()
In reply to Stanislav FomichevAdd new helper bpf_sk_storage_clone which optionally clones sk storage and call it from sk_clone_lock.
--- include/net/bpf_sk_storage.h | 10 ++++ include/uapi/linux/bpf.h | 3 ++ net/core/bpf_sk_storage.c | 100 +++++++++++++++++++++++++++++++++-- net/core/sock.c | 9 ++-- 4 files changed, 116 insertions(+), 6 deletions(-)
include/net/bpf_sk_storage.h +10 -0
@@ -10,4 +10,14 @@ void bpf_sk_storage_free(struct sock *sk);extern const struct bpf_func_proto bpf_sk_storage_get_proto;extern const struct bpf_func_proto bpf_sk_storage_delete_proto; +#ifdef CONFIG_BPF_SYSCALL +int bpf_sk_storage_clone (const struct sock *sk,struct sock *newsk); +#else +static inline int bpf_sk_storage_clone (const struct sock *sk, +struct sock *newsk) +{ +return 0 ; +} +#endif +#endif /* _BPF_SK_STORAGE_H */
include/uapi/linux/bpf.h +3 -0
@@ -337,6 +337,9 @@ enum bpf_attach_type {#define BPF_F_RDONLY_PROG (1U << 7)#define BPF_F_WRONLY_PROG (1U << 8) +/* Clone map from listener for newly accepted socket */ +#define BPF_F_CLONE (1U << 9) +/* flags for BPF_PROG_QUERY */ #define BPF_F_QUERY_EFFECTIVE (1U << 0)
net/core/bpf_sk_storage.c +97 -3
136 lines not on this page · raw message
net/core/sock.c +6 -3
@@ -1851,9 +1851,12 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority) goto out; }RCU_INIT_POINTER (newsk->sk_reuseport_cb,NULL ); -#ifdef CONFIG_BPF_SYSCALL -RCU_INIT_POINTER (newsk->sk_bpf_storage,NULL ); -#endif + +if (bpf_sk_storage_clone (sk, newsk)) { +sk_free_unlock_clone (newsk); + newsk =NULL ; + goto out; + } newsk->sk_err =0 ; newsk->sk_err_soft =0 ;
Signature · 1 line
2.23.0.rc1.153.gdeed80330f-goog
Quoted 6 lines · Stanislav Fomichev
Add new helper
bpf_sk_storage_clonewhich optionally clones sk storage and call it fromsk_clone_lock.Cc Martin KaFai Lau <[email protected]>Cc Yonghong Song <[email protected]>Signed-off-by Stanislav Fomichev <[email protected]>
Daniel Borkmann
maintainerQuoted 6 lines · Stanislav Fomichev
Add new helper
bpf_sk_storage_clonewhich optionally clones sk storage and call it fromsk_clone_lock.Cc Martin KaFai Lau <[email protected]>Cc Yonghong Song <[email protected]>Signed-off-by Stanislav Fomichev <[email protected]>
[...]
Quoted 42 lines
net/core/bpf_sk_storage.c +42
-0+int bpf_sk_storage_clone (const struct sock *sk,struct sock *newsk) +{ +struct bpf_sk_storage *new_sk_storage =NULL ; +struct bpf_sk_storage *sk_storage; +struct bpf_sk_storage_elem *selem; +int ret; + +RCU_INIT_POINTER (newsk->sk_bpf_storage,NULL ); + +rcu_read_lock (); + sk_storage =rcu_dereference (sk->sk_bpf_storage); + +if (!sk_storage ||hlist_empty (&sk_storage->list)) + goto out; + +hlist_for_each_entry_rcu (selem, &sk_storage->list, snode) { +struct bpf_sk_storage_elem *copy_selem; +struct bpf_sk_storage_map *smap; +struct bpf_map *map; +int refold; + + smap =rcu_dereference (SDATA (selem)->smap); +if (!(smap->map.map_flags &BPF_F_CLONE )) +continue ; + + map =bpf_map_inc_not_zero (&smap->map, false); +if (IS_ERR (map)) +continue ; + + copy_selem =bpf_sk_storage_clone_elem (newsk, smap, selem); +if (!copy_selem) { + ret = -ENOMEM ; +bpf_map_put (map); + goto err; + } + +if (new_sk_storage) { +selem_link_map (smap, copy_selem); +__selem_link_sk (new_sk_storage, copy_selem); + }else { + ret =sk_storage_alloc (newsk, smap, copy_selem); +if (ret) {
net/core/bpf_sk_storage.c static int bpf_fd_sk_storage_delete_elem(struct bpf_map *map, void *key) · line 822
Quoted 10 lines
net/core/bpf_sk_storage.c +10
-0+kfree (copy_selem); +atomic_sub (smap->elem_size, + &newsk->sk_omem_alloc); +bpf_map_put (map); + goto err; + } + + new_sk_storage =rcu_dereference (copy_selem->sk_storage); + } +bpf_map_put (map);
The map get/put combination /under/ RCU read lock seems a bit odd to me, could you exactly describe the race that this would be preventing?
net/core/bpf_sk_storage.c static int bpf_fd_sk_storage_delete_elem(struct bpf_map *map, void *key) · line 834
Quoted 12 lines
net/core/bpf_sk_storage.c +12
-0+ } + +out: +rcu_read_unlock (); +return 0 ; + +err: +rcu_read_unlock (); + +bpf_sk_storage_free (newsk); +return ret; +}
Thanks,
Daniel
Quoted 55 lines · Daniel Borkmann
Quoted 6 lines · Stanislav Fomichev
Add new helper
bpf_sk_storage_clonewhich optionally clones sk storage and call it fromsk_clone_lock.Cc Martin KaFai Lau <[email protected]>Cc Yonghong Song <[email protected]>Signed-off-by Stanislav Fomichev <[email protected]>[...]
Quoted 47 lines
net/core/bpf_sk_storage.c +47
-0+int bpf_sk_storage_clone (const struct sock *sk,struct sock *newsk) +{ +struct bpf_sk_storage *new_sk_storage =NULL ; +struct bpf_sk_storage *sk_storage; +struct bpf_sk_storage_elem *selem; +int ret; + +RCU_INIT_POINTER (newsk->sk_bpf_storage,NULL ); + +rcu_read_lock (); + sk_storage =rcu_dereference (sk->sk_bpf_storage); + +if (!sk_storage ||hlist_empty (&sk_storage->list)) + goto out; + +hlist_for_each_entry_rcu (selem, &sk_storage->list, snode) { +struct bpf_sk_storage_elem *copy_selem; +struct bpf_sk_storage_map *smap; +struct bpf_map *map; +int refold; + + smap =rcu_dereference (SDATA (selem)->smap); +if (!(smap->map.map_flags &BPF_F_CLONE )) +continue ; + + map =bpf_map_inc_not_zero (&smap->map, false); +if (IS_ERR (map)) +continue ; + + copy_selem =bpf_sk_storage_clone_elem (newsk, smap, selem); +if (!copy_selem) { + ret = -ENOMEM ; +bpf_map_put (map); + goto err; + } + +if (new_sk_storage) { +selem_link_map (smap, copy_selem); +__selem_link_sk (new_sk_storage, copy_selem); + }else { + ret =sk_storage_alloc (newsk, smap, copy_selem); +if (ret) { +kfree (copy_selem); +atomic_sub (smap->elem_size, + &newsk->sk_omem_alloc); +bpf_map_put (map); + goto err;
Quoted 8 lines
net/core/bpf_sk_storage.c static int bpf_fd_sk_storage_delete_elem(struct bpf_map *map, void *key) · line 822
Quoted 5 lines
net/core/bpf_sk_storage.c +5
-0+ } + + new_sk_storage =rcu_dereference (copy_selem->sk_storage); + } +bpf_map_put (map);The map get/put combination /under/ RCU read lock seems a bit odd to me, could you exactly describe the race that this would be preventing?
There is a race between sk storage release and sk storage clone. bpf_sk_storage_map_free uses synchronize_rcu to wait for all existing users to finish and the new ones are prevented via map's refcnt being zero; we need to do something like that for the clone. Martin suggested to use bpf_map_inc_not_zero/bpf_map_put. If I read everythin correctly, I think without map_inc/map_put we get the following race:
CPU0 CPU1
bpf_map_put
bpf_sk_storage_map_free(smap)
synchronize_rcu
// no more users via bpf or
// syscall, but clone
// can still happen
for each (bucket)
selem_unlink
selem_unlink_map(smap)
// adding anything at
// this point to the
// bucket will leak
rcu_read_lock
tcp_v4_rcv
tcp_v4_do_rcv
// sk is lockless TCP_LISTEN
tcp_v4_cookie_check
tcp_v4_syn_recv_sock
bpf_sk_storage_clone
rcu_dereference(sk->sk_bpf_storage)
selem_link_map(smap, copy)
// adding new element to the
// map -> leak
rcu_read_unlock
selem_unlink_sk
sk->sk_bpf_storage = NULL
synchronize_rcuDaniel Borkmann
maintainerQuoted 99 lines · Stanislav Fomichev
Quoted 55 lines · Daniel Borkmann
Quoted 6 lines · Stanislav Fomichev
Add new helper
bpf_sk_storage_clonewhich optionally clones sk storage and call it fromsk_clone_lock.Cc Martin KaFai Lau <[email protected]>Cc Yonghong Song <[email protected]>Signed-off-by Stanislav Fomichev <[email protected]>[...]
Quoted 47 lines
net/core/bpf_sk_storage.c +47
-0+int bpf_sk_storage_clone (const struct sock *sk,struct sock *newsk) +{ +struct bpf_sk_storage *new_sk_storage =NULL ; +struct bpf_sk_storage *sk_storage; +struct bpf_sk_storage_elem *selem; +int ret; + +RCU_INIT_POINTER (newsk->sk_bpf_storage,NULL ); + +rcu_read_lock (); + sk_storage =rcu_dereference (sk->sk_bpf_storage); + +if (!sk_storage ||hlist_empty (&sk_storage->list)) + goto out; + +hlist_for_each_entry_rcu (selem, &sk_storage->list, snode) { +struct bpf_sk_storage_elem *copy_selem; +struct bpf_sk_storage_map *smap; +struct bpf_map *map; +int refold; + + smap =rcu_dereference (SDATA (selem)->smap); +if (!(smap->map.map_flags &BPF_F_CLONE )) +continue ; + + map =bpf_map_inc_not_zero (&smap->map, false); +if (IS_ERR (map)) +continue ; + + copy_selem =bpf_sk_storage_clone_elem (newsk, smap, selem); +if (!copy_selem) { + ret = -ENOMEM ; +bpf_map_put (map); + goto err; + } + +if (new_sk_storage) { +selem_link_map (smap, copy_selem); +__selem_link_sk (new_sk_storage, copy_selem); + }else { + ret =sk_storage_alloc (newsk, smap, copy_selem); +if (ret) { +kfree (copy_selem); +atomic_sub (smap->elem_size, + &newsk->sk_omem_alloc); +bpf_map_put (map); + goto err;Quoted 8 lines
net/core/bpf_sk_storage.c static int bpf_fd_sk_storage_delete_elem(struct bpf_map *map, void *key) · line 822
Quoted 5 lines
net/core/bpf_sk_storage.c +5
-0+ } + + new_sk_storage =rcu_dereference (copy_selem->sk_storage); + } +bpf_map_put (map);The map get/put combination /under/ RCU read lock seems a bit odd to me, could you exactly describe the race that this would be preventing?
There is a race between sk storage release and sk storage clone.
bpf_sk_storage_map_freeusessynchronize_rcuto wait for all existing users to finish and the new ones are prevented via map's refcnt being zero; we need to do something like that for the clone. Martin suggested to use bpf_map_inc_not_zero/bpf_map_put. If I read everythin correctly, I think without map_inc/map_put we get the following race:CPU0 CPU1
bpf_map_put
bpf_sk_storage_map_free(smap)synchronize_rcu // no more users via bpf or // syscall, but clone // can still happen for each (bucket) selem_unlink selem_unlink_map(smap) // adding anything at // this point to the // bucket will leak rcu_read_lock tcp_v4_rcv tcp_v4_do_rcv // sk is lockless TCP_LISTEN tcp_v4_cookie_check tcp_v4_syn_recv_sock bpf_sk_storage_clone rcu_dereference(sk->sk_bpf_storage) selem_link_map(smap, copy)
Quoted 9 lines
// adding new element to the
// map -> leak
rcu_read_unlock
selem_unlink_sk
sk->sk_bpf_storage = NULL
synchronize_rcuMakes sense, thanks for clarifying. Perhaps a small comment on top of the bpf_map_inc_not_zero() would be great as well, so it's immediately clear also from this location when reading the code why this is done.
Thanks,
Daniel
Quoted 107 lines · Daniel Borkmann
Quoted 106 lines · Stanislav Fomichev
Quoted 55 lines · Daniel Borkmann
Quoted 6 lines · Stanislav Fomichev
Add new helper
bpf_sk_storage_clonewhich optionally clones sk storage and call it fromsk_clone_lock.Cc Martin KaFai Lau <[email protected]>Cc Yonghong Song <[email protected]>Signed-off-by Stanislav Fomichev <[email protected]>[...]
Quoted 47 lines
net/core/bpf_sk_storage.c +47
-0+int bpf_sk_storage_clone (const struct sock *sk,struct sock *newsk) +{ +struct bpf_sk_storage *new_sk_storage =NULL ; +struct bpf_sk_storage *sk_storage; +struct bpf_sk_storage_elem *selem; +int ret; + +RCU_INIT_POINTER (newsk->sk_bpf_storage,NULL ); + +rcu_read_lock (); + sk_storage =rcu_dereference (sk->sk_bpf_storage); + +if (!sk_storage ||hlist_empty (&sk_storage->list)) + goto out; + +hlist_for_each_entry_rcu (selem, &sk_storage->list, snode) { +struct bpf_sk_storage_elem *copy_selem; +struct bpf_sk_storage_map *smap; +struct bpf_map *map; +int refold; + + smap =rcu_dereference (SDATA (selem)->smap); +if (!(smap->map.map_flags &BPF_F_CLONE )) +continue ; + + map =bpf_map_inc_not_zero (&smap->map, false); +if (IS_ERR (map)) +continue ; + + copy_selem =bpf_sk_storage_clone_elem (newsk, smap, selem); +if (!copy_selem) { + ret = -ENOMEM ; +bpf_map_put (map); + goto err; + } + +if (new_sk_storage) { +selem_link_map (smap, copy_selem); +__selem_link_sk (new_sk_storage, copy_selem); + }else { + ret =sk_storage_alloc (newsk, smap, copy_selem); +if (ret) { +kfree (copy_selem); +atomic_sub (smap->elem_size, + &newsk->sk_omem_alloc); +bpf_map_put (map); + goto err;Quoted 8 lines
net/core/bpf_sk_storage.c static int bpf_fd_sk_storage_delete_elem(struct bpf_map *map, void *key) · line 822
Quoted 5 lines
net/core/bpf_sk_storage.c +5
-0+ } + + new_sk_storage =rcu_dereference (copy_selem->sk_storage); + } +bpf_map_put (map);The map get/put combination /under/ RCU read lock seems a bit odd to me, could you exactly describe the race that this would be preventing?
There is a race between sk storage release and sk storage clone.
bpf_sk_storage_map_freeusessynchronize_rcuto wait for all existing users to finish and the new ones are prevented via map's refcnt being zero; we need to do something like that for the clone. Martin suggested to use bpf_map_inc_not_zero/bpf_map_put. If I read everythin correctly, I think without map_inc/map_put we get the following race:CPU0 CPU1
bpf_map_put
bpf_sk_storage_map_free(smap)synchronize_rcu // no more users via bpf or // syscall, but clone // can still happen for each (bucket) selem_unlink selem_unlink_map(smap) // adding anything at // this point to the // bucket will leak rcu_read_lock tcp_v4_rcv tcp_v4_do_rcv // sk is lockless TCP_LISTEN tcp_v4_cookie_check tcp_v4_syn_recv_sock bpf_sk_storage_clone rcu_dereference(sk->sk_bpf_storage) selem_link_map(smap, copy) // adding new element to the // map -> leak rcu_read_unlock selem_unlink_sk sk->sk_bpf_storage = NULL
Quoted 6 lines
Quoted 2 lines
synchronize_rcuMakes sense, thanks for clarifying. Perhaps a small comment on top of the
bpf_map_inc_not_zero()would be great as well, so it's immediately clear also from this location when reading the code why this is done.
Sure, no problem, will have something similar to what I have before synchronize_rcu in bpf_sk_storage_map_free.
Quoted 2 lines
Thanks,
Daniel
Quoted 2 lines · Stanislav Fomichev
Add new helper
bpf_sk_storage_clonewhich optionally clones sk storage and call it fromsk_clone_lock.
Thanks for v2. Sorry for the delay. I am traveling.
Quoted 76 lines
Cc Martin KaFai Lau <[email protected]>Cc Yonghong Song <[email protected]>Signed-off-by Stanislav Fomichev <[email protected]>--- include/net/bpf_sk_storage.h | 10 ++++ include/uapi/linux/bpf.h | 3 ++ net/core/bpf_sk_storage.c | 100 +++++++++++++++++++++++++++++++++-- net/core/sock.c | 9 ++-- 4 files changed, 116 insertions(+), 6 deletions(-)include/net/bpf_sk_storage.h +10
-0@@ -10,4 +10,14 @@ void bpf_sk_storage_free(struct sock *sk);extern const struct bpf_func_proto bpf_sk_storage_get_proto;extern const struct bpf_func_proto bpf_sk_storage_delete_proto; +#ifdef CONFIG_BPF_SYSCALL +int bpf_sk_storage_clone (const struct sock *sk,struct sock *newsk); +#else +static inline int bpf_sk_storage_clone (const struct sock *sk, +struct sock *newsk) +{ +return 0 ; +} +#endif +#endif /* _BPF_SK_STORAGE_H */ include/uapi/linux/bpf.h +3
-0@@ -337,6 +337,9 @@ enum bpf_attach_type {#define BPF_F_RDONLY_PROG (1U << 7)#define BPF_F_WRONLY_PROG (1U << 8) +/* Clone map from listener for newly accepted socket */ +#define BPF_F_CLONE (1U << 9) +/* flags for BPF_PROG_QUERY */ #define BPF_F_QUERY_EFFECTIVE (1U << 0)net/core/bpf_sk_storage.c +4
-2@@ -12,6 +12,9 @@static atomic_t cache_idx; +#define SK_STORAGE_CREATE_FLAG_MASK \ + (BPF_F_NO_PREALLOC | BPF_F_CLONE) +struct bucket {struct hlist_head list;raw_spinlock_t lock;@@ -209,7 +212,6 @@ static void selem_unlink_sk(struct bpf_sk_storage_elem *selem)kfree_rcu (sk_storage, rcu); } -/* sk_storage->lock must be held and sk_storage->list cannot be empty */ static void __selem_link_sk (struct bpf_sk_storage *sk_storage,struct bpf_sk_storage_elem *selem) {@@ -509,7 +511,7 @@ static int sk_storage_delete(struct sock *sk, struct bpf_map *map)return 0 ; } -/* Called by __sk_destruct() */ +/* Called by __sk_destruct() & bpf_sk_storage_clone() */ void bpf_sk_storage_free (struct sock *sk) {struct bpf_sk_storage_elem *selem;
net/core/bpf_sk_storage.c static void bpf_sk_storage_map_free(struct bpf_map *map) · line 566
Quoted 8 lines
net/core/bpf_sk_storage.c +5
-0@@ -557,6 +559,11 @@ static void bpf_sk_storage_map_free(struct bpf_map *map) smap = (struct bpf_sk_storage_map *)map; +/* Note that this map might be concurrently cloned from +* bpf_sk_storage_clone. Wait for any existing bpf_sk_storage_clone +* RCU read section to finish before proceeding. New RCU +* read sections should be prevented via bpf_map_inc_not_zero. +*/
Thanks!
net/core/bpf_sk_storage.c static void bpf_sk_storage_map_free(struct bpf_map *map) · line 612
Quoted 10 lines
net/core/bpf_sk_storage.c +2
-1synchronize_rcu ();/* bpf prog and the userspace can no longer access this map @@ -601,7 +608,8 @@ static void bpf_sk_storage_map_free(struct bpf_map *map)static int bpf_sk_storage_map_alloc_check (union bpf_attr *attr) { - if (attr->map_flags != BPF_F_NO_PREALLOC || attr->max_entries || + if (attr->map_flags & ~SK_STORAGE_CREATE_FLAG_MASK || + attr->max_entries ||
I think "!(attr->map_flags & BPF_F_NO_PREALLOC)" should also be needed.
Quoted 80 lines
net/core/bpf_sk_storage.c +73
-0attr->key_size !=sizeof (int ) || !attr->value_size ||/* Enforce BTF for userspace sk dumping */ !attr->btf_key_type_id || !attr->btf_value_type_id)@@ -739,6 +747,92 @@ static int bpf_fd_sk_storage_delete_elem(struct bpf_map *map, void *key)return err; } +static struct bpf_sk_storage_elem * +bpf_sk_storage_clone_elem (struct sock *newsk, +struct bpf_sk_storage_map *smap, +struct bpf_sk_storage_elem *selem) +{ +struct bpf_sk_storage_elem *copy_selem; + + copy_selem =selem_alloc (smap, newsk,NULL , true); +if (!copy_selem) +return NULL ; + +if (map_value_has_spin_lock (&smap->map)) +copy_map_value_locked (&smap->map,SDATA (copy_selem)->data, +SDATA (selem)->data, true); +else +copy_map_value (&smap->map,SDATA (copy_selem)->data, +SDATA (selem)->data); + +return copy_selem; +} + +int bpf_sk_storage_clone (const struct sock *sk,struct sock *newsk) +{ +struct bpf_sk_storage *new_sk_storage =NULL ; +struct bpf_sk_storage *sk_storage; +struct bpf_sk_storage_elem *selem; +int ret; + +RCU_INIT_POINTER (newsk->sk_bpf_storage,NULL ); + +rcu_read_lock (); + sk_storage =rcu_dereference (sk->sk_bpf_storage); + +if (!sk_storage ||hlist_empty (&sk_storage->list)) + goto out; + +hlist_for_each_entry_rcu (selem, &sk_storage->list, snode) { +struct bpf_sk_storage_elem *copy_selem; +struct bpf_sk_storage_map *smap; +struct bpf_map *map; +int refold; + + smap =rcu_dereference (SDATA (selem)->smap); +if (!(smap->map.map_flags &BPF_F_CLONE )) +continue ; + + map =bpf_map_inc_not_zero (&smap->map, false); +if (IS_ERR (map)) +continue ; + + copy_selem =bpf_sk_storage_clone_elem (newsk, smap, selem); +if (!copy_selem) { + ret = -ENOMEM ; +bpf_map_put (map); + goto err; + } + +if (new_sk_storage) { +selem_link_map (smap, copy_selem); +__selem_link_sk (new_sk_storage, copy_selem); + }else { + ret =sk_storage_alloc (newsk, smap, copy_selem); +if (ret) { +kfree (copy_selem); +atomic_sub (smap->elem_size, + &newsk->sk_omem_alloc); +bpf_map_put (map); + goto err; + } + + new_sk_storage =rcu_dereference (copy_selem->sk_storage); + } +bpf_map_put (map);
net/core/bpf_sk_storage.c static int bpf_fd_sk_storage_delete_elem(struct bpf_map *map, void *key) · line 832
Quoted 10 lines
net/core/bpf_sk_storage.c +10
-0+ } + +out: +rcu_read_unlock (); +return 0 ; + +err: +rcu_read_unlock (); + +bpf_sk_storage_free (newsk);
The later sk_free_unlock_clone(newsk) should eventually call bpf_sk_storage_free(newsk) also?
Others LGTM.
Quoted 29 lines
net/core/bpf_sk_storage.c +3
-0+return ret; +} + BPF_CALL_4(bpf_sk_storage_get, struct bpf_map *, map, struct sock *, sk, void *, value, u64, flags) {net/core/sock.c +6
-3@@ -1851,9 +1851,12 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority) goto out; }RCU_INIT_POINTER (newsk->sk_reuseport_cb,NULL ); -#ifdef CONFIG_BPF_SYSCALL -RCU_INIT_POINTER (newsk->sk_bpf_storage,NULL ); -#endif + +if (bpf_sk_storage_clone (sk, newsk)) { +sk_free_unlock_clone (newsk); + newsk =NULL ; + goto out; + } newsk->sk_err =0 ; newsk->sk_err_soft =0 ;Signature · 1 line
2.23.0.rc1.153.gdeed80330f-goog
Quoted 4 lines · Martin Lau
Quoted 2 lines · Stanislav Fomichev
Add new helper
bpf_sk_storage_clonewhich optionally clones sk storage and call it fromsk_clone_lock.Thanks for v2. Sorry for the delay. I am traveling.
No worries, take your time, if you're OOO feel free to do another round when you get back, not urgent.
Quoted 91 lines
Quoted 76 lines
Cc Martin KaFai Lau <[email protected]>Cc Yonghong Song <[email protected]>Signed-off-by Stanislav Fomichev <[email protected]>--- include/net/bpf_sk_storage.h | 10 ++++ include/uapi/linux/bpf.h | 3 ++ net/core/bpf_sk_storage.c | 100 +++++++++++++++++++++++++++++++++-- net/core/sock.c | 9 ++-- 4 files changed, 116 insertions(+), 6 deletions(-)include/net/bpf_sk_storage.h +10
-0@@ -10,4 +10,14 @@ void bpf_sk_storage_free(struct sock *sk);extern const struct bpf_func_proto bpf_sk_storage_get_proto;extern const struct bpf_func_proto bpf_sk_storage_delete_proto; +#ifdef CONFIG_BPF_SYSCALL +int bpf_sk_storage_clone (const struct sock *sk,struct sock *newsk); +#else +static inline int bpf_sk_storage_clone (const struct sock *sk, +struct sock *newsk) +{ +return 0 ; +} +#endif +#endif /* _BPF_SK_STORAGE_H */ include/uapi/linux/bpf.h +3
-0@@ -337,6 +337,9 @@ enum bpf_attach_type {#define BPF_F_RDONLY_PROG (1U << 7)#define BPF_F_WRONLY_PROG (1U << 8) +/* Clone map from listener for newly accepted socket */ +#define BPF_F_CLONE (1U << 9) +/* flags for BPF_PROG_QUERY */ #define BPF_F_QUERY_EFFECTIVE (1U << 0)net/core/bpf_sk_storage.c +4
-2@@ -12,6 +12,9 @@static atomic_t cache_idx; +#define SK_STORAGE_CREATE_FLAG_MASK \ + (BPF_F_NO_PREALLOC | BPF_F_CLONE) +struct bucket {struct hlist_head list;raw_spinlock_t lock;@@ -209,7 +212,6 @@ static void selem_unlink_sk(struct bpf_sk_storage_elem *selem)kfree_rcu (sk_storage, rcu); } -/* sk_storage->lock must be held and sk_storage->list cannot be empty */ static void __selem_link_sk (struct bpf_sk_storage *sk_storage,struct bpf_sk_storage_elem *selem) {@@ -509,7 +511,7 @@ static int sk_storage_delete(struct sock *sk, struct bpf_map *map)return 0 ; } -/* Called by __sk_destruct() */ +/* Called by __sk_destruct() & bpf_sk_storage_clone() */ void bpf_sk_storage_free (struct sock *sk) {struct bpf_sk_storage_elem *selem;net/core/bpf_sk_storage.c static void bpf_sk_storage_map_free(struct bpf_map *map) · line 566
Quoted 8 lines
net/core/bpf_sk_storage.c +5
-0@@ -557,6 +559,11 @@ static void bpf_sk_storage_map_free(struct bpf_map *map) smap = (struct bpf_sk_storage_map *)map; +/* Note that this map might be concurrently cloned from +* bpf_sk_storage_clone. Wait for any existing bpf_sk_storage_clone +* RCU read section to finish before proceeding. New RCU +* read sections should be prevented via bpf_map_inc_not_zero. +*/ Thanks!
Quoted 5 lines
net/core/bpf_sk_storage.c +0
-0synchronize_rcu ();/* bpf prog and the userspace can no longer access this map @@ -601,7 +608,8 @@ static void bpf_sk_storage_map_free(struct bpf_map *map)
Quoted 6 lines
net/core/bpf_sk_storage.c static void bpf_sk_storage_map_free(struct bpf_map *map) · line 612
Quoted 5 lines
net/core/bpf_sk_storage.c +2
-1static int bpf_sk_storage_map_alloc_check (union bpf_attr *attr) { - if (attr->map_flags != BPF_F_NO_PREALLOC || attr->max_entries || + if (attr->map_flags & ~SK_STORAGE_CREATE_FLAG_MASK || + attr->max_entries ||I think "!(attr->map_flags &
BPF_F_NO_PREALLOC)" should also be needed.
Makes sense, we always want to have BPF_F_NO_PREALLOC set. Will add, thanks!
Quoted 84 lines
Quoted 84 lines
net/core/bpf_sk_storage.c +77
-0attr->key_size !=sizeof (int ) || !attr->value_size ||/* Enforce BTF for userspace sk dumping */ !attr->btf_key_type_id || !attr->btf_value_type_id)@@ -739,6 +747,92 @@ static int bpf_fd_sk_storage_delete_elem(struct bpf_map *map, void *key)return err; } +static struct bpf_sk_storage_elem * +bpf_sk_storage_clone_elem (struct sock *newsk, +struct bpf_sk_storage_map *smap, +struct bpf_sk_storage_elem *selem) +{ +struct bpf_sk_storage_elem *copy_selem; + + copy_selem =selem_alloc (smap, newsk,NULL , true); +if (!copy_selem) +return NULL ; + +if (map_value_has_spin_lock (&smap->map)) +copy_map_value_locked (&smap->map,SDATA (copy_selem)->data, +SDATA (selem)->data, true); +else +copy_map_value (&smap->map,SDATA (copy_selem)->data, +SDATA (selem)->data); + +return copy_selem; +} + +int bpf_sk_storage_clone (const struct sock *sk,struct sock *newsk) +{ +struct bpf_sk_storage *new_sk_storage =NULL ; +struct bpf_sk_storage *sk_storage; +struct bpf_sk_storage_elem *selem; +int ret; + +RCU_INIT_POINTER (newsk->sk_bpf_storage,NULL ); + +rcu_read_lock (); + sk_storage =rcu_dereference (sk->sk_bpf_storage); + +if (!sk_storage ||hlist_empty (&sk_storage->list)) + goto out; + +hlist_for_each_entry_rcu (selem, &sk_storage->list, snode) { +struct bpf_sk_storage_elem *copy_selem; +struct bpf_sk_storage_map *smap; +struct bpf_map *map; +int refold; + + smap =rcu_dereference (SDATA (selem)->smap); +if (!(smap->map.map_flags &BPF_F_CLONE )) +continue ; + + map =bpf_map_inc_not_zero (&smap->map, false); +if (IS_ERR (map)) +continue ; + + copy_selem =bpf_sk_storage_clone_elem (newsk, smap, selem); +if (!copy_selem) { + ret = -ENOMEM ; +bpf_map_put (map); + goto err; + } + +if (new_sk_storage) { +selem_link_map (smap, copy_selem); +__selem_link_sk (new_sk_storage, copy_selem); + }else { + ret =sk_storage_alloc (newsk, smap, copy_selem); +if (ret) { +kfree (copy_selem); +atomic_sub (smap->elem_size, + &newsk->sk_omem_alloc); +bpf_map_put (map); + goto err; + } + + new_sk_storage =rcu_dereference (copy_selem->sk_storage); + } +bpf_map_put (map); + } + +out: +rcu_read_unlock ();
Quoted 8 lines
net/core/bpf_sk_storage.c static int bpf_fd_sk_storage_delete_elem(struct bpf_map *map, void *key) · line 832
Quoted 6 lines
net/core/bpf_sk_storage.c +6
-0+return 0 ; + +err: +rcu_read_unlock (); + +bpf_sk_storage_free (newsk);The later
sk_free_unlock_clone(newsk)should eventually callbpf_sk_storage_free(newsk)also?
Hm, good point, I can drop it from here and rely on sk_free_unlock_clone to call __sk_destruct/bpf_sk_storage_free. That probably deserves a comment though :-)
Quoted 1 line
Others LGTM.
Thank you for a review!
Quoted 29 lines
Quoted 29 lines
net/core/bpf_sk_storage.c +3
-0+return ret; +} + BPF_CALL_4(bpf_sk_storage_get, struct bpf_map *, map, struct sock *, sk, void *, value, u64, flags) {net/core/sock.c +6
-3@@ -1851,9 +1851,12 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority) goto out; }RCU_INIT_POINTER (newsk->sk_reuseport_cb,NULL ); -#ifdef CONFIG_BPF_SYSCALL -RCU_INIT_POINTER (newsk->sk_bpf_storage,NULL ); -#endif + +if (bpf_sk_storage_clone (sk, newsk)) { +sk_free_unlock_clone (newsk); + newsk =NULL ; + goto out; + } newsk->sk_err =0 ; newsk->sk_err_soft =0 ;Signature · 1 line
2.23.0.rc1.153.gdeed80330f-goog
PATCH v2 3/4 bpf: sync bpf.h to tools/
In reply to Stanislav FomichevSync new sk storage clone flag.
--- tools/include/uapi/linux/bpf.h | 3 +++ 1 file changed, 3 insertions(+)
tools/include/uapi/linux/bpf.h +3 -0
@@ -337,6 +337,9 @@ enum bpf_attach_type {#define BPF_F_RDONLY_PROG (1U << 7)#define BPF_F_WRONLY_PROG (1U << 8) +/* Clone map from listener for newly accepted socket */ +#define BPF_F_CLONE (1U << 9) +/* flags for BPF_PROG_QUERY */ #define BPF_F_QUERY_EFFECTIVE (1U << 0)
Signature · 1 line
2.23.0.rc1.153.gdeed80330f-goog
PATCH v2 4/4 selftests/bpf: add sockopt clone/inheritance test
In reply to Stanislav FomichevAdd a test that calls setsockopt on the listener socket which triggers BPF program. This BPF program writes to the sk storage and sets clone flag. Make sure that sk storage is cloned for a newly accepted connection.
We have two cloned maps in the tests to make sure we hit both cases in bpf_sk_storage_clone: first element (sk_storage_alloc) and non-first element(s) (selem_link_map).
--- tools/testing/selftests/bpf/.gitignore | 1 + tools/testing/selftests/bpf/Makefile | 3 +- .../selftests/bpf/progs/sockopt_inherit.c | 97 +++++++ .../selftests/bpf/test_sockopt_inherit.c | 253 ++++++++++++++++++ 4 files changed, 353 insertions(+), 1 deletion(-) create mode 100644 tools/testing/selftests/bpf/progs/sockopt_inherit.c create mode 100644 tools/testing/selftests/bpf/test_sockopt_inherit.c
tools/testing/selftests/bpf/.gitignore +1 -0
@@ -42,4 +42,5 @@ xdping test_sockopt test_sockopt_sk test_sockopt_multi +test_sockopt_inherit test_tcp_rtt
tools/testing/selftests/bpf/Makefile +2 -1
@@ -29,7 +29,7 @@ TEST_GEN_PROGS = test_verifier test_tag test_maps test_lru_map test_lpm_map test test_cgroup_storage test_select_reuseport test_section_names \ test_netcnt test_tcpnotify_user test_sock_fields test_sysctl test_hashmap \ test_btf_dump test_cgroup_attach xdping test_sockopt test_sockopt_sk \ - test_sockopt_multi test_tcp_rtt + test_sockopt_multi test_sockopt_inherit test_tcp_rtt BPF_OBJ_FILES = $(patsubst %.c,%.o, $(notdir $(wildcard progs/*.c))) TEST_GEN_FILES = $(BPF_OBJ_FILES)
@@ -110,6 +110,7 @@ $(OUTPUT)/test_cgroup_attach: cgroup_helpers.c $(OUTPUT)/test_sockopt: cgroup_helpers.c $(OUTPUT)/test_sockopt_sk: cgroup_helpers.c $(OUTPUT)/test_sockopt_multi: cgroup_helpers.c +$(OUTPUT)/test_sockopt_inherit: cgroup_helpers.c $(OUTPUT)/test_tcp_rtt: cgroup_helpers.c .PHONY: force
tools/testing/selftests/bpf/progs/sockopt_inherit.c new file +97 -0
97 lines not on this page · raw message
tools/testing/selftests/bpf/test_sockopt_inherit.c new file +253 -0
253 lines not on this page · raw message
Signature · 1 line
2.23.0.rc1.153.gdeed80330f-goog
Re: [PATCH bpf-next v2 4/4] selftests/bpf: add sockopt clone/inheritance test
Quoted 5 lines · Stanislav Fomichev
Add a test that calls setsockopt on the listener socket which triggers BPF program. This BPF program writes to the sk storage and sets clone flag. Make sure that sk storage is cloned for a newly accepted connection.
Quoted 7 lines
We have two cloned maps in the tests to make sure we hit both cases in
bpf_sk_storage_clone: first element (sk_storage_alloc) and non-first element(s) (selem_link_map).Cc Martin KaFai Lau <[email protected]>Cc Yonghong Song <[email protected]>Signed-off-by Stanislav Fomichev <[email protected]>