bpf: support cloning sk storage on accept()
Subsystems (7)
v4 cover bpf: support cloning sk storage on accept()
Currently there is no way to propagate sk storage from the listener socket to a newly accepted one. Consider the following use case:
fd = socket();
setsockopt(fd, SOL_IP, IP_TOS,...);
/* ^^^ setsockopt BPF program triggers here and saves something
* into sk storage of the listener.
*/
listen(fd, ...);
while (client = accept(fd)) {
/* At this point all association between listener
* socket and newly accepted one is gone. New
* socket will not have any sk storage attached.
*/
}Let's add new BPF_F_CLONE flag that can be specified when creating a socket storage map. This new flag indicates that map contents should be cloned when the socket is cloned.
v4:
* drop 'goto err' in bpf_sk_storage_clone (Yonghong Song)
* add comment about race with bpf_sk_storage_map_free to the bpf_sk_storage_clone side as well (Daniel Borkmann)
v3:
* make sure BPF_F_NO_PREALLOC is always present when creating a map (Martin KaFai Lau)
* don't call bpf_sk_storage_free explicitly, rely on sk_free_unlock_clone to do the cleanup (Martin KaFai Lau)
v2:
* remove spinlocks around selem_link_map/sk (Martin KaFai Lau)
* BPF_F_CLONE on a map, not selem (Martin KaFai Lau)
* hold a map while cloning (Martin KaFai Lau)
* use BTF maps in selftests (Yonghong Song)
* do proper cleanup selftests; don't call close(-1) (Yonghong Song)
* export bpf_map_inc_not_zero
Stanislav Fomichev (4):
bpf: export bpf_map_inc_not_zero
bpf: support cloning sk storage on accept()
bpf: sync bpf.h to tools/
selftests/bpf: add sockopt clone/inheritance test
include/linux/bpf.h | 2 + include/net/bpf_sk_storage.h | 10 + include/uapi/linux/bpf.h | 3 + kernel/bpf/syscall.c | 16 +- net/core/bpf_sk_storage.c | 104 ++++++- net/core/sock.c | 9 +- tools/include/uapi/linux/bpf.h | 3 + tools/testing/selftests/bpf/.gitignore | 1 + tools/testing/selftests/bpf/Makefile | 3 +- .../selftests/bpf/progs/sockopt_inherit.c | 97 +++++++ .../selftests/bpf/test_sockopt_inherit.c | 253 ++++++++++++++++++ 11 files changed, 491 insertions(+), 10 deletions(-) create mode 100644 tools/testing/selftests/bpf/progs/sockopt_inherit.c create mode 100644 tools/testing/selftests/bpf/test_sockopt_inherit.c
Signature · 1 line
2.23.0.rc1.153.gdeed80330f-goog
PATCH v4 1/4 bpf: export bpf_map_inc_not_zero
Rename existing bpf_map_inc_not_zero to __bpf_map_inc_not_zero to indicate that it's caller's responsibility to do proper locking. Create and export bpf_map_inc_not_zero wrapper that properly locks map_idr_lock. Will be used in the next commit to hold a map while cloning a socket.
--- include/linux/bpf.h | 2 ++ kernel/bpf/syscall.c | 16 +++++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-)
include/linux/bpf.h +2 -0
@@ -647,6 +647,8 @@ void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock);struct bpf_map *bpf_map_get_with_uref (u32 ufd);struct bpf_map *__bpf_map_get (struct fd f);struct bpf_map *__must_check bpf_map_inc (struct bpf_map *map,bool uref); +struct bpf_map *__must_check bpf_map_inc_not_zero (struct bpf_map *map, +bool uref);void bpf_map_put_with_uref (struct bpf_map *map);void bpf_map_put (struct bpf_map *map);int bpf_map_charge_memlock (struct bpf_map *map,u32 pages);
kernel/bpf/syscall.c +13 -3
@@ -683,8 +683,8 @@ struct bpf_map *bpf_map_get_with_uref(u32 ufd) }/* map_idr_lock should have been held */ -static struct bpf_map *bpf_map_inc_not_zero (struct bpf_map *map, -bool uref) +static struct bpf_map *__bpf_map_inc_not_zero (struct bpf_map *map, +bool uref) {int refold;
@@ -704,6 +704,16 @@ static struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map,return map; } +struct bpf_map *bpf_map_inc_not_zero (struct bpf_map *map,bool uref) +{ +spin_lock_bh (&map_idr_lock); + map =__bpf_map_inc_not_zero (map, uref); +spin_unlock_bh (&map_idr_lock); + +return map; +} +EXPORT_SYMBOL_GPL (bpf_map_inc_not_zero); +int __weak bpf_stackmap_copy (struct bpf_map *map,void *key,void *value) {return -ENOTSUPP ;
@@ -2177,7 +2187,7 @@ static int bpf_map_get_fd_by_id(const union bpf_attr *attr)spin_lock_bh (&map_idr_lock); map =idr_find (&map_idr, id);if (map) - map =bpf_map_inc_not_zero (map, true); + map =__bpf_map_inc_not_zero (map, true);else map =ERR_PTR (-ENOENT );spin_unlock_bh (&map_idr_lock);
Signature · 1 line
2.23.0.rc1.153.gdeed80330f-goog
PATCH v4 2/4 bpf: support cloning sk storage on accept()
In reply to Stanislav FomichevAdd new helper bpf_sk_storage_clone which optionally clones sk storage and call it from sk_clone_lock.
--- include/net/bpf_sk_storage.h | 10 ++++ include/uapi/linux/bpf.h | 3 + net/core/bpf_sk_storage.c | 104 ++++++++++++++++++++++++++++++++++- net/core/sock.c | 9 ++- 4 files changed, 120 insertions(+), 6 deletions(-)
include/net/bpf_sk_storage.h +10 -0
@@ -10,4 +10,14 @@ void bpf_sk_storage_free(struct sock *sk);extern const struct bpf_func_proto bpf_sk_storage_get_proto;extern const struct bpf_func_proto bpf_sk_storage_delete_proto; +#ifdef CONFIG_BPF_SYSCALL +int bpf_sk_storage_clone (const struct sock *sk,struct sock *newsk); +#else +static inline int bpf_sk_storage_clone (const struct sock *sk, +struct sock *newsk) +{ +return 0 ; +} +#endif +#endif /* _BPF_SK_STORAGE_H */
include/uapi/linux/bpf.h +3 -0
@@ -337,6 +337,9 @@ enum bpf_attach_type {#define BPF_F_RDONLY_PROG (1U << 7)#define BPF_F_WRONLY_PROG (1U << 8) +/* Clone map from listener for newly accepted socket */ +#define BPF_F_CLONE (1U << 9) +/* flags for BPF_PROG_QUERY */ #define BPF_F_QUERY_EFFECTIVE (1U << 0)
net/core/bpf_sk_storage.c +101 -3
140 lines not on this page · raw message
net/core/sock.c +6 -3
@@ -1851,9 +1851,12 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority) goto out; }RCU_INIT_POINTER (newsk->sk_reuseport_cb,NULL ); -#ifdef CONFIG_BPF_SYSCALL -RCU_INIT_POINTER (newsk->sk_bpf_storage,NULL ); -#endif + +if (bpf_sk_storage_clone (sk, newsk)) { +sk_free_unlock_clone (newsk); + newsk =NULL ; + goto out; + } newsk->sk_err =0 ; newsk->sk_err_soft =0 ;
Signature · 1 line
2.23.0.rc1.153.gdeed80330f-goog
PATCH v4 3/4 bpf: sync bpf.h to tools/
In reply to Stanislav FomichevSync new sk storage clone flag.
--- tools/include/uapi/linux/bpf.h | 3 +++ 1 file changed, 3 insertions(+)
tools/include/uapi/linux/bpf.h +3 -0
@@ -337,6 +337,9 @@ enum bpf_attach_type {#define BPF_F_RDONLY_PROG (1U << 7)#define BPF_F_WRONLY_PROG (1U << 8) +/* Clone map from listener for newly accepted socket */ +#define BPF_F_CLONE (1U << 9) +/* flags for BPF_PROG_QUERY */ #define BPF_F_QUERY_EFFECTIVE (1U << 0)
Signature · 1 line
2.23.0.rc1.153.gdeed80330f-goog
PATCH v4 4/4 selftests/bpf: add sockopt clone/inheritance test
In reply to Stanislav FomichevAdd a test that calls setsockopt on the listener socket which triggers BPF program. This BPF program writes to the sk storage and sets clone flag. Make sure that sk storage is cloned for a newly accepted connection.
We have two cloned maps in the tests to make sure we hit both cases in bpf_sk_storage_clone: first element (sk_storage_alloc) and non-first element(s) (selem_link_map).
--- tools/testing/selftests/bpf/.gitignore | 1 + tools/testing/selftests/bpf/Makefile | 3 +- .../selftests/bpf/progs/sockopt_inherit.c | 97 +++++++ .../selftests/bpf/test_sockopt_inherit.c | 253 ++++++++++++++++++ 4 files changed, 353 insertions(+), 1 deletion(-) create mode 100644 tools/testing/selftests/bpf/progs/sockopt_inherit.c create mode 100644 tools/testing/selftests/bpf/test_sockopt_inherit.c
tools/testing/selftests/bpf/.gitignore +1 -0
@@ -42,4 +42,5 @@ xdping test_sockopt test_sockopt_sk test_sockopt_multi +test_sockopt_inherit test_tcp_rtt
tools/testing/selftests/bpf/Makefile +2 -1
@@ -29,7 +29,7 @@ TEST_GEN_PROGS = test_verifier test_tag test_maps test_lru_map test_lpm_map test test_cgroup_storage test_select_reuseport test_section_names \ test_netcnt test_tcpnotify_user test_sock_fields test_sysctl test_hashmap \ test_btf_dump test_cgroup_attach xdping test_sockopt test_sockopt_sk \ - test_sockopt_multi test_tcp_rtt + test_sockopt_multi test_sockopt_inherit test_tcp_rtt BPF_OBJ_FILES = $(patsubst %.c,%.o, $(notdir $(wildcard progs/*.c))) TEST_GEN_FILES = $(BPF_OBJ_FILES)
@@ -111,6 +111,7 @@ $(OUTPUT)/test_cgroup_attach: cgroup_helpers.c $(OUTPUT)/test_sockopt: cgroup_helpers.c $(OUTPUT)/test_sockopt_sk: cgroup_helpers.c $(OUTPUT)/test_sockopt_multi: cgroup_helpers.c +$(OUTPUT)/test_sockopt_inherit: cgroup_helpers.c $(OUTPUT)/test_tcp_rtt: cgroup_helpers.c .PHONY: force
tools/testing/selftests/bpf/progs/sockopt_inherit.c new file +97 -0
97 lines not on this page · raw message
tools/testing/selftests/bpf/test_sockopt_inherit.c new file +253 -0
253 lines not on this page · raw message
Signature · 1 line
2.23.0.rc1.153.gdeed80330f-goog
Quoted 60 lines · Stanislav Fomichev
Currently there is no way to propagate sk storage from the listener socket to a newly accepted one. Consider the following use case:
fd = socket(); setsockopt(fd, SOL_IP, IP_TOS,...); /* ^^^ setsockopt BPF program triggers here and saves something * into sk storage of the listener. */ listen(fd, ...); while (client = accept(fd)) { /* At this point all association between listener * socket and newly accepted one is gone. New * socket will not have any sk storage attached. */ }Let's add new
BPF_F_CLONEflag that can be specified when creating a socket storage map. This new flag indicates that map contents should be cloned when the socket is cloned.v4:
* drop 'goto err' inbpf_sk_storage_clone(Yonghong Song)
* add comment about race withbpf_sk_storage_map_freeto thebpf_sk_storage_cloneside as well (Daniel Borkmann)v3:
* make sureBPF_F_NO_PREALLOCis always present when creating a map (Martin KaFai Lau)
* don't callbpf_sk_storage_freeexplicitly, rely onsk_free_unlock_cloneto do the cleanup (Martin KaFai Lau)v2:
* remove spinlocks around selem_link_map/sk (Martin KaFai Lau)
*BPF_F_CLONEon a map, not selem (Martin KaFai Lau)
* hold a map while cloning (Martin KaFai Lau)
* use BTF maps in selftests (Yonghong Song)
* do proper cleanup selftests; don't call close(-1) (Yonghong Song)
* exportbpf_map_inc_not_zeroCc Martin KaFai Lau <[email protected]>Cc Yonghong Song <[email protected]>Stanislav Fomichev (4):
bpf: exportbpf_map_inc_not_zero
bpf: support cloning sk storage onaccept()
bpf: syncbpf.htotools/
selftests/bpf: add sockopt clone/inheritance testinclude/linux/bpf.h | 2 + include/net/bpf_sk_storage.h | 10 + include/uapi/linux/bpf.h | 3 + kernel/bpf/syscall.c | 16 +- net/core/bpf_sk_storage.c | 104 ++++++- net/core/sock.c | 9 +- tools/include/uapi/linux/bpf.h | 3 + tools/testing/selftests/bpf/.gitignore | 1 + tools/testing/selftests/bpf/Makefile | 3 +- .../selftests/bpf/progs/sockopt_inherit.c | 97 +++++++ .../selftests/bpf/test_sockopt_inherit.c | 253 ++++++++++++++++++ 11 files changed, 491 insertions(+), 10 deletions(-)
Quoted 3 lines
create mode 100644
tools/testing/selftests/bpf/progs/sockopt_inherit.ccreate mode 100644tools/testing/selftests/bpf/test_sockopt_inherit.c
Applied, thanks!