Allow updating sockmap / sockhash from BPF
Subsystems (9)
v2 cover Allow updating sockmap / sockhash from BPF
We're currently building a control plane for our BPF socket dispatch work. As part of that, we have a need to create a copy of an existing sockhash, to allow us to change the keys. I previously proposed allowing privileged userspace to look up sockets, which doesn't work due to security concerns (see [1]).
In follow up discussions during BPF office hours we identified bpf_iter as a possible solution: instead of accessing sockets from user space we can iterate the source sockhash, and insert the values into a new map. Enabling this requires two pieces: the ability to iterate sockmap and sockhash, as well as being able to call map_update_elem from BPF.
This patch set implements the latter: it's now possible to update sockmap from BPF context. As a next step, we can implement bpf_iter for sockmap.
Changes in v2:
- Fix warning in patch #2 (Jakub K)
- Renamed override_map_arg_type (John)
- Only allow updating sockmap from known safe contexts (John)
- Use __s64 for sockmap updates from user space (Yonghong)
- Various small test fixes around test macros and such (Yonghong)
Thank your for your reviews!
1: https://lore.kernel.org/bpf/[email protected]/
Lorenz Bauer (6):
net: sk_msg: simplify sk_psock initialization
bpf: sockmap: merge sockmap and sockhash update functions
bpf: sockmap: call sock_map_update_elem directly
bpf: override the meaning of ARG_PTR_TO_MAP_VALUE for sockmap and
sockhash
bpf: sockmap: allow update from BPF
selftests: bpf: test sockmap update from BPF
include/linux/bpf.h | 7 + include/linux/skmsg.h | 17 --- kernel/bpf/syscall.c | 5 +- kernel/bpf/verifier.c | 78 +++++++++- net/core/skmsg.c | 34 ++++- net/core/sock_map.c | 137 ++++++++---------- net/ipv4/tcp_bpf.c | 13 +- net/ipv4/udp_bpf.c | 9 +- .../selftests/bpf/prog_tests/sockmap_basic.c | 71 +++++++++ .../selftests/bpf/progs/test_sockmap_copy.c | 48 ++++++ 10 files changed, 301 insertions(+), 118 deletions(-) create mode 100644 tools/testing/selftests/bpf/progs/test_sockmap_copy.c
Signature · 1 line
2.25.1
PATCH v2 1/6 net: sk_msg: simplify sk_psock initialization
Initializing psock->sk_proto and other saved callbacks is only done in sk_psock_update_proto, after sk_psock_init has returned. The logic for this is difficult to follow, and needlessly complex.
Instead, initialize psock->sk_proto whenever we allocate a new psock. Additionally, assert the following invariants:
* The SK has no ULP: ULP does it's own finagling of sk->sk_prot
* sk_user_data is unused: we need it to store sk_psock
Protect our access to sk_user_data with sk_callback_lock, which is what other users like reuseport arrays, etc. do.
The result is that an sk_psock is always fully initialized, and that psock->sk_proto is always the "original" struct proto. The latter allows us to use psock->sk_proto when initializing IPv6 TCP / UDP callbacks for sockmap.
--- include/linux/skmsg.h | 17 ----------------- net/core/skmsg.c | 34 ++++++++++++++++++++++++++++------ net/core/sock_map.c | 14 ++++---------- net/ipv4/tcp_bpf.c | 13 +++++-------- net/ipv4/udp_bpf.c | 9 ++++----- 5 files changed, 41 insertions(+), 46 deletions(-)
include/linux/skmsg.h +0 -17
@@ -340,23 +340,6 @@ static inline void sk_psock_update_proto(struct sock *sk,struct sk_psock *psock, struct proto *ops) { -/* Initialize saved callbacks and original proto only once, since this -* function may be called multiple times for a psock, e.g. when -* psock->progs.msg_parser is updated. -* -* Since we've not installed the new proto, psock is not yet in use and -* we can initialize it without synchronization. -*/ -if (!psock->sk_proto) { -struct proto *orig =READ_ONCE (sk->sk_prot); - - psock->saved_unhash = orig->unhash; - psock->saved_close = orig->close; - psock->saved_write_space = sk->sk_write_space; - - psock->sk_proto = orig; - } -/* Pairs with lockless read in sk_clone_lock() */ WRITE_ONCE(sk->sk_prot, ops); }
net/core/skmsg.c +28 -6
@@ -494,14 +494,34 @@ static void sk_psock_backlog(struct work_struct *work)struct sk_psock *sk_psock_init (struct sock *sk,int node) { -struct sk_psock *psock =kzalloc_node (sizeof (*psock), -GFP_ATOMIC | __GFP_NOWARN, - node); -if (!psock) -return NULL ; +struct sk_psock *psock; +struct proto *prot; +write_lock_bh (&sk->sk_callback_lock); + +if (inet_csk_has_ulp (sk)) { + psock =ERR_PTR (-EINVAL ); + goto out; + } + +if (sk->sk_user_data) { + psock =ERR_PTR (-EBUSY ); + goto out; + } + + psock =kzalloc_node (sizeof (*psock),GFP_ATOMIC | __GFP_NOWARN, node); +if (!psock) { + psock =ERR_PTR (-ENOMEM ); + goto out; + } + + prot =READ_ONCE (sk->sk_prot); psock->sk = sk; - psock->eval = __SK_NONE; + psock->eval = __SK_NONE; + psock->sk_proto = prot; + psock->saved_unhash = prot->unhash; + psock->saved_close = prot->close; + psock->saved_write_space = sk->sk_write_space;INIT_LIST_HEAD (&psock->link);spin_lock_init (&psock->link_lock);
@@ -516,6 +536,8 @@ struct sk_psock *sk_psock_init(struct sock *sk, int node)rcu_assign_sk_user_data_nocopy (sk, psock);sock_hold (sk); +out: +write_unlock_bh (&sk->sk_callback_lock);return psock; }EXPORT_SYMBOL_GPL (sk_psock_init);
net/core/sock_map.c +4 -10
@@ -184,8 +184,6 @@ static int sock_map_init_proto(struct sock *sk, struct sk_psock *psock) {struct proto *prot; -sock_owned_by_me (sk); -switch (sk->sk_type) {case SOCK_STREAM : prot =tcp_bpf_get_proto (sk, psock);
@@ -272,8 +270,8 @@ static int sock_map_link(struct bpf_map *map, struct sk_psock_progs *progs, } }else { psock =sk_psock_init (sk, map->numa_node); -if (!psock) { - ret = -ENOMEM ; +if (IS_ERR (psock)) { + ret =PTR_ERR (psock); goto out_progs; } }
@@ -322,8 +320,8 @@ static int sock_map_link_no_progs(struct bpf_map *map, struct sock *sk)if (!psock) { psock =sk_psock_init (sk, map->numa_node); -if (!psock) -return -ENOMEM ; +if (IS_ERR (psock)) +return PTR_ERR (psock); } ret =sock_map_init_proto (sk, psock);
@@ -478,8 +476,6 @@ static int sock_map_update_common(struct bpf_map *map, u32 idx,return -EINVAL ;if (unlikely (idx >= map->max_entries))return -E2BIG ; -if (inet_csk_has_ulp (sk)) -return -EINVAL ; link =sk_psock_init_link ();if (!link)
@@ -855,8 +851,6 @@ static int sock_hash_update_common(struct bpf_map *map, void *key,WARN_ON_ONCE (!rcu_read_lock_held ());if (unlikely (flags >BPF_EXIST ))return -EINVAL ; -if (inet_csk_has_ulp (sk)) -return -EINVAL ; link =sk_psock_init_link ();if (!link)
net/ipv4/tcp_bpf.c +5 -8
@@ -567,10 +567,9 @@ static void tcp_bpf_rebuild_protos(struct proto prot[TCP_BPF_NUM_CFGS], prot[TCP_BPF_TX ].sendpage = tcp_bpf_sendpage; } -static void tcp_bpf_check_v6_needs_rebuild (struct sock *sk,struct proto *ops) +static void tcp_bpf_check_v6_needs_rebuild (struct proto *ops) { -if (sk->sk_family ==AF_INET6 && -unlikely (ops !=smp_load_acquire (&tcpv6_prot_saved))) { +if (unlikely (ops !=smp_load_acquire (&tcpv6_prot_saved))) {spin_lock_bh (&tcpv6_prot_lock);if (likely (ops != tcpv6_prot_saved)) {tcp_bpf_rebuild_protos (tcp_bpf_prots[TCP_BPF_IPV6 ], ops);
@@ -603,13 +602,11 @@ struct proto *tcp_bpf_get_proto(struct sock *sk, struct sk_psock *psock)int family = sk->sk_family ==AF_INET6 ?TCP_BPF_IPV6 :TCP_BPF_IPV4 ;int config = psock->progs.msg_parser ?TCP_BPF_TX :TCP_BPF_BASE ; -if (!psock->sk_proto) { -struct proto *ops =READ_ONCE (sk->sk_prot); - -if (tcp_bpf_assert_proto_ops (ops)) +if (sk->sk_family ==AF_INET6 ) { +if (tcp_bpf_assert_proto_ops (psock->sk_proto))return ERR_PTR (-EINVAL ); -tcp_bpf_check_v6_needs_rebuild (sk, ops); +tcp_bpf_check_v6_needs_rebuild (psock->sk_proto); }return &tcp_bpf_prots[family][config];
net/ipv4/udp_bpf.c +4 -5
@@ -22,10 +22,9 @@ static void udp_bpf_rebuild_protos(struct proto *prot, const struct proto *base) prot->close = sock_map_close; } -static void udp_bpf_check_v6_needs_rebuild (struct sock *sk,struct proto *ops) +static void udp_bpf_check_v6_needs_rebuild (struct proto *ops) { -if (sk->sk_family ==AF_INET6 && -unlikely (ops !=smp_load_acquire (&udpv6_prot_saved))) { +if (unlikely (ops !=smp_load_acquire (&udpv6_prot_saved))) {spin_lock_bh (&udpv6_prot_lock);if (likely (ops != udpv6_prot_saved)) {udp_bpf_rebuild_protos (&udp_bpf_prots[UDP_BPF_IPV6 ], ops);
@@ -46,8 +45,8 @@ struct proto *udp_bpf_get_proto(struct sock *sk, struct sk_psock *psock) {int family = sk->sk_family ==AF_INET ?UDP_BPF_IPV4 :UDP_BPF_IPV6 ; -if (!psock->sk_proto) -udp_bpf_check_v6_needs_rebuild (sk,READ_ONCE (sk->sk_prot)); +if (sk->sk_family ==AF_INET6 ) +udp_bpf_check_v6_needs_rebuild (psock->sk_proto);return &udp_bpf_prots[family]; }
Signature · 1 line
2.25.1
PATCH v2 2/6 bpf: sockmap: merge sockmap and sockhash update functions
In reply to Lorenz BauerMerge the two very similar functions sock_map_update_elem and sock_hash_update_elem into one.
--- net/core/sock_map.c | 49 +++++++-------------------------------------- 1 file changed, 7 insertions(+), 42 deletions(-)
net/core/sock_map.c +7 -42
@@ -559,10 +559,12 @@ static bool sock_map_sk_state_allowed(const struct sock *sk)return false; } +static int sock_hash_update_common (struct bpf_map *map,void *key, +struct sock *sk,u64 flags); +static int sock_map_update_elem (struct bpf_map *map,void *key,void *value,u64 flags) { -u32 idx = *(u32 *)key;struct socket *sock;struct sock *sk;int ret;
@@ -591,8 +593,10 @@ static int sock_map_update_elem(struct bpf_map *map, void *key,sock_map_sk_acquire (sk);if (!sock_map_sk_state_allowed (sk)) ret = -EOPNOTSUPP ; +else if (map->map_type ==BPF_MAP_TYPE_SOCKMAP ) + ret =sock_map_update_common (map, *(u32 *)key, sk, flags);else - ret =sock_map_update_common (map, idx, sk, flags); + ret =sock_hash_update_common (map, key, sk, flags);sock_map_sk_release (sk); out:fput (sock->file);
@@ -909,45 +913,6 @@ static int sock_hash_update_common(struct bpf_map *map, void *key,return ret; } -static int sock_hash_update_elem (struct bpf_map *map,void *key, -void *value,u64 flags) -{ -struct socket *sock; -struct sock *sk; -int ret; -u64 ufd; - -if (map->value_size ==sizeof (u64)) - ufd = *(u64 *)value; -else - ufd = *(u32 *)value; -if (ufd >S32_MAX ) -return -EINVAL ; - - sock =sockfd_lookup (ufd, &ret); -if (!sock) -return ret; - sk = sock->sk; -if (!sk) { - ret = -EINVAL ; - goto out; - } -if (!sock_map_sk_is_suitable (sk)) { - ret = -EOPNOTSUPP ; - goto out; - } - -sock_map_sk_acquire (sk); -if (!sock_map_sk_state_allowed (sk)) - ret = -EOPNOTSUPP ; -else - ret =sock_hash_update_common (map, key, sk, flags); -sock_map_sk_release (sk); -out: -fput (sock->file); -return ret; -} -static int sock_hash_get_next_key (struct bpf_map *map,void *key,void *key_next) {
@@ -1216,7 +1181,7 @@ const struct bpf_map_ops sock_hash_ops = { .map_alloc = sock_hash_alloc, .map_free = sock_hash_free, .map_get_next_key = sock_hash_get_next_key, - .map_update_elem = sock_hash_update_elem, + .map_update_elem = sock_map_update_elem, .map_delete_elem = sock_hash_delete_elem, .map_lookup_elem = sock_hash_lookup, .map_lookup_elem_sys_only = sock_hash_lookup_sys,
Signature · 1 line
2.25.1
PATCH v2 3/6 bpf: sockmap: call sock_map_update_elem directly
In reply to Lorenz BauerDon't go via map->ops to call sock_map_update_elem, since we know what function to call in bpf_map_update_value. Since we currently don't allow calling map_update_elem from BPF context, we can remove ops->map_update_elem and rename the function to sock_map_update_elem_sys.
--- include/linux/bpf.h | 7 +++++++ kernel/bpf/syscall.c | 5 +++-- net/core/sock_map.c | 6 ++---- 3 files changed, 12 insertions(+), 6 deletions(-)
include/linux/bpf.h +7 -0
@@ -1635,6 +1635,7 @@ int sock_map_prog_update(struct bpf_map *map, struct bpf_prog *prog, struct bpf_prog *old, u32 which);int sock_map_get_from_fd (const union bpf_attr *attr,struct bpf_prog *prog);int sock_map_prog_detach (const union bpf_attr *attr,enum bpf_prog_type ptype); +int sock_map_update_elem_sys (struct bpf_map *map,void *key,void *value,u64 flags);void sock_map_unhash (struct sock *sk);void sock_map_close (struct sock *sk,long timeout);#else
@@ -1656,6 +1657,12 @@ static inline int sock_map_prog_detach(const union bpf_attr *attr, {return -EOPNOTSUPP ; } + +static inline int sock_map_update_elem_sys (struct bpf_map *map,void *key,void *value, +u64 flags) +{ +return -EOPNOTSUPP ; +}#endif /* CONFIG_BPF_STREAM_PARSER */ #if defined(CONFIG_INET ) && defined(CONFIG_BPF_SYSCALL )
kernel/bpf/syscall.c +3 -2
@@ -157,10 +157,11 @@ static int bpf_map_update_value(struct bpf_map *map, struct fd f, void *key,if (bpf_map_is_dev_bound (map)) {return bpf_map_offload_update_elem (map, key, value, flags); }else if (map->map_type ==BPF_MAP_TYPE_CPUMAP || - map->map_type ==BPF_MAP_TYPE_SOCKHASH || - map->map_type ==BPF_MAP_TYPE_SOCKMAP || map->map_type ==BPF_MAP_TYPE_STRUCT_OPS ) {return map->ops->map_update_elem (map, key, value, flags); + }else if (map->map_type ==BPF_MAP_TYPE_SOCKHASH || + map->map_type ==BPF_MAP_TYPE_SOCKMAP ) { +return sock_map_update_elem_sys (map, key, value, flags); }else if (IS_FD_PROG_ARRAY (map)) {return bpf_fd_array_map_update_elem (map, f.file, key, value, flags);
net/core/sock_map.c +2 -4
@@ -562,8 +562,8 @@ static bool sock_map_sk_state_allowed(const struct sock *sk)static int sock_hash_update_common (struct bpf_map *map,void *key,struct sock *sk,u64 flags); -static int sock_map_update_elem (struct bpf_map *map,void *key, -void *value,u64 flags) +int sock_map_update_elem_sys (struct bpf_map *map,void *key,void *value, +u64 flags) {struct socket *sock;struct sock *sk;
@@ -687,7 +687,6 @@ const struct bpf_map_ops sock_map_ops = { .map_free = sock_map_free, .map_get_next_key = sock_map_get_next_key, .map_lookup_elem_sys_only = sock_map_lookup_sys, - .map_update_elem = sock_map_update_elem, .map_delete_elem = sock_map_delete_elem, .map_lookup_elem = sock_map_lookup, .map_release_uref = sock_map_release_progs,
@@ -1181,7 +1180,6 @@ const struct bpf_map_ops sock_hash_ops = { .map_alloc = sock_hash_alloc, .map_free = sock_hash_free, .map_get_next_key = sock_hash_get_next_key, - .map_update_elem = sock_map_update_elem, .map_delete_elem = sock_hash_delete_elem, .map_lookup_elem = sock_hash_lookup, .map_lookup_elem_sys_only = sock_hash_lookup_sys,
Signature · 1 line
2.25.1
PATCH v2 4/6 bpf: override the meaning of ARG_PTR_TO_MAP_VALUE for sockmap and sockhash
In reply to Lorenz BauerThe verifier assumes that map values are simple blobs of memory, and therefore treats ARG_PTR_TO_MAP_VALUE, etc. as such. However, there are map types where this isn't true. For example, sockmap and sockhash store sockets. In general this isn't a big problem: we can just write helpers that explicitly requests PTR_TO_SOCKET instead of ARG_PTR_TO_MAP_VALUE.
The one exception are the standard map helpers like map_update_elem, map_lookup_elem, etc. Here it would be nice we could overload the function prototype for different kinds of maps. Unfortunately, this isn't entirely straight forward:
We only know the type of the map once we have resolved meta->map_ptr in check_func_arg. This means we can't swap out the prototype in check_helper_call until we're half way through the function.
Instead, modify check_func_arg to treat ARG_PTR_TO_MAP_VALUE* to mean "the native type for the map" instead of "pointer to memory" for sockmap and sockhash. This means we don't have to modify the function prototype at all
--- kernel/bpf/verifier.c | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+)
kernel/bpf/verifier.c +37 -0
@@ -3872,6 +3872,35 @@ static int int_ptr_type_to_size(enum bpf_arg_type type)return -EINVAL ; } +static int resolve_map_arg_type (struct bpf_verifier_env *env, +const struct bpf_call_arg_meta *meta, +enum bpf_arg_type *arg_type) +{ +if (!meta->map_ptr) { +/* kernel subsystem misconfigured verifier */ +verbose (env,"invalid map_ptr to access map->type\n" ); +return -EACCES ; + } + +switch (meta->map_ptr->map_type) { +case BPF_MAP_TYPE_SOCKMAP : +case BPF_MAP_TYPE_SOCKHASH : +if (*arg_type ==ARG_PTR_TO_MAP_VALUE ) { + *arg_type =ARG_PTR_TO_SOCKET ; + }else if (*arg_type ==ARG_PTR_TO_MAP_VALUE_OR_NULL ) { + *arg_type =ARG_PTR_TO_SOCKET_OR_NULL ; + }else { +verbose (env,"invalid arg_type for sockmap/sockhash\n" ); +return -EINVAL ; + } +break ; + +default : +break ; + } +return 0 ; +} +static int check_func_arg (struct bpf_verifier_env *env,u32 arg,struct bpf_call_arg_meta *meta,const struct bpf_func_proto *fn)
@@ -3904,6 +3933,14 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,return -EACCES ; } +if (arg_type ==ARG_PTR_TO_MAP_VALUE || + arg_type ==ARG_PTR_TO_UNINIT_MAP_VALUE || + arg_type ==ARG_PTR_TO_MAP_VALUE_OR_NULL ) { + err =resolve_map_arg_type (env, meta, &arg_type); +if (err) +return err; + } + if (arg_type == ARG_PTR_TO_MAP_KEY || arg_type == ARG_PTR_TO_MAP_VALUE || arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE ||
Signature · 1 line
2.25.1
Re: [PATCH bpf-next v2 4/6] bpf: override the meaning of ARG_PTR_TO_MAP_VALUE for sockmap and sockhash
Quoted 40 lines · Lorenz Bauer
The verifier assumes that map values are simple blobs of memory, and therefore treats
ARG_PTR_TO_MAP_VALUE, etc. as such. However, there are map types where this isn't true. For example, sockmap and sockhash store sockets. In general this isn't a big problem: we can just write helpers that explicitly requestsPTR_TO_SOCKETinstead ofARG_PTR_TO_MAP_VALUE.The one exception are the standard map helpers like
map_update_elem,map_lookup_elem, etc. Here it would be nice we could overload the function prototype for different kinds of maps. Unfortunately, this isn't entirely straight forward:
We only know the type of the map once we have resolvedmeta->map_ptrincheck_func_arg. This means we can't swap out the prototype incheck_helper_calluntil we're half way through the function.Instead, modify
check_func_argto treat ARG_PTR_TO_MAP_VALUE* to mean "the native type for the map" instead of "pointer to memory" for sockmap and sockhash. This means we don't have to modify the function prototype at allSigned-off-by Lorenz Bauer <[email protected]>--- kernel/bpf/verifier.c | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+)kernel/bpf/verifier.c +7
-0@@ -3872,6 +3872,35 @@ static int int_ptr_type_to_size(enum bpf_arg_type type)return -EINVAL ; } +static int resolve_map_arg_type (struct bpf_verifier_env *env, +const struct bpf_call_arg_meta *meta, +enum bpf_arg_type *arg_type) +{ +if (!meta->map_ptr) { +/* kernel subsystem misconfigured verifier */ +verbose (env,"invalid map_ptr to access map->type\n" );
kernel/bpf/verifier.c static int int_ptr_type_to_size(enum bpf_arg_type type) · line 3891
Quoted 10 lines
kernel/bpf/verifier.c +10
-0+return -EACCES ; + } + +switch (meta->map_ptr->map_type) { +case BPF_MAP_TYPE_SOCKMAP : +case BPF_MAP_TYPE_SOCKHASH : +if (*arg_type ==ARG_PTR_TO_MAP_VALUE ) { + *arg_type =ARG_PTR_TO_SOCKET ; + }else if (*arg_type ==ARG_PTR_TO_MAP_VALUE_OR_NULL ) { + *arg_type =ARG_PTR_TO_SOCKET_OR_NULL ;
Is this *arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL possible with current implementation?
If not, we can remove this "else if" and return -EINVAL, right?
Quoted 16 lines
kernel/bpf/verifier.c +12
-0+ }else { +verbose (env,"invalid arg_type for sockmap/sockhash\n" ); +return -EINVAL ; + } +break ; + +default : +break ; + } +return 0 ; +} +static int check_func_arg (struct bpf_verifier_env *env,u32 arg,struct bpf_call_arg_meta *meta,const struct bpf_func_proto *fn)
kernel/bpf/verifier.c static int check_func_arg(struct bpf_verifier_env *env, u32 arg, · line 3939
Quoted 7 lines
kernel/bpf/verifier.c +4
-0@@ -3904,6 +3933,14 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,return -EACCES ; } +if (arg_type ==ARG_PTR_TO_MAP_VALUE || + arg_type ==ARG_PTR_TO_UNINIT_MAP_VALUE || + arg_type ==ARG_PTR_TO_MAP_VALUE_OR_NULL ) { + err =resolve_map_arg_type (env, meta, &arg_type);
I am okay with this to cover all MAP_VALUE types with func name resolve_map_arg_type as a generic helper.
Quoted 8 lines
kernel/bpf/verifier.c +4
-0+if (err) +return err; + } + if (arg_type == ARG_PTR_TO_MAP_KEY || arg_type == ARG_PTR_TO_MAP_VALUE || arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE ||
Re: [PATCH bpf-next v2 4/6] bpf: override the meaning of ARG_PTR_TO_MAP_VALUE for sockmap and sockhash
Quoted 49 lines · Yonghong Song
Quoted 45 lines · Lorenz Bauer
The verifier assumes that map values are simple blobs of memory, and therefore treats
ARG_PTR_TO_MAP_VALUE, etc. as such. However, there are map types where this isn't true. For example, sockmap and sockhash store sockets. In general this isn't a big problem: we can just write helpers that explicitly requestsPTR_TO_SOCKETinstead ofARG_PTR_TO_MAP_VALUE.The one exception are the standard map helpers like
map_update_elem,map_lookup_elem, etc. Here it would be nice we could overload the function prototype for different kinds of maps. Unfortunately, this isn't entirely straight forward:
We only know the type of the map once we have resolvedmeta->map_ptrincheck_func_arg. This means we can't swap out the prototype incheck_helper_calluntil we're half way through the function.Instead, modify
check_func_argto treat ARG_PTR_TO_MAP_VALUE* to mean "the native type for the map" instead of "pointer to memory" for sockmap and sockhash. This means we don't have to modify the function prototype at allSigned-off-by Lorenz Bauer <[email protected]>--- kernel/bpf/verifier.c | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+)kernel/bpf/verifier.c +12
-0@@ -3872,6 +3872,35 @@ static int int_ptr_type_to_size(enum bpf_arg_type type)return -EINVAL ; } +static int resolve_map_arg_type (struct bpf_verifier_env *env, +const struct bpf_call_arg_meta *meta, +enum bpf_arg_type *arg_type) +{ +if (!meta->map_ptr) { +/* kernel subsystem misconfigured verifier */ +verbose (env,"invalid map_ptr to access map->type\n" ); +return -EACCES ; + } + +switch (meta->map_ptr->map_type) { +case BPF_MAP_TYPE_SOCKMAP :
Quoted 8 lines
kernel/bpf/verifier.c static int int_ptr_type_to_size(enum bpf_arg_type type) · line 3891
Quoted 5 lines
kernel/bpf/verifier.c +5
-0+case BPF_MAP_TYPE_SOCKHASH : +if (*arg_type ==ARG_PTR_TO_MAP_VALUE ) { + *arg_type =ARG_PTR_TO_SOCKET ; + }else if (*arg_type ==ARG_PTR_TO_MAP_VALUE_OR_NULL ) { + *arg_type =ARG_PTR_TO_SOCKET_OR_NULL ;Is this
*arg_type==ARG_PTR_TO_MAP_VALUE_OR_NULLpossible with current implementation?
No, the only user is bpf_sk_storage_get and friends which requires BPF_MAP_TYPE_SK_STORAGE.
I seemed to make sense to map ARG_PTR_TO_MAP_VALUE_OR_NULL, but I can remove it as
well if you prefer. Do you think this is dangerous?
Quoted 38 lines
If not, we can remove this "else if" and return
-EINVAL, right?Quoted 16 lines
kernel/bpf/verifier.c +12
-0+ }else { +verbose (env,"invalid arg_type for sockmap/sockhash\n" ); +return -EINVAL ; + } +break ; + +default : +break ; + } +return 0 ; +} +static int check_func_arg (struct bpf_verifier_env *env,u32 arg,struct bpf_call_arg_meta *meta,const struct bpf_func_proto *fn)kernel/bpf/verifier.c static int check_func_arg(struct bpf_verifier_env *env, u32 arg, · line 3939
Quoted 7 lines
kernel/bpf/verifier.c +4
-0@@ -3904,6 +3933,14 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,return -EACCES ; } +if (arg_type ==ARG_PTR_TO_MAP_VALUE || + arg_type ==ARG_PTR_TO_UNINIT_MAP_VALUE || + arg_type ==ARG_PTR_TO_MAP_VALUE_OR_NULL ) { + err =resolve_map_arg_type (env, meta, &arg_type);I am okay with this to cover all
MAP_VALUEtypes with func nameresolve_map_arg_typeas a generic helper.Quoted 8 lines
kernel/bpf/verifier.c +4
-0+if (err) +return err; + } + if (arg_type == ARG_PTR_TO_MAP_KEY || arg_type == ARG_PTR_TO_MAP_VALUE || arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE ||
Signature · 4 lines
Lorenz Bauer | Systems Engineer 6th Floor, County Hall/The Riverside Building, SE1 7PB, UK www.cloudflare.com
Re: [PATCH bpf-next v2 4/6] bpf: override the meaning of ARG_PTR_TO_MAP_VALUE for sockmap and sockhash
Quoted 59 lines · Lorenz Bauer
Quoted 49 lines · Yonghong Song
Quoted 45 lines · Lorenz Bauer
The verifier assumes that map values are simple blobs of memory, and therefore treats
ARG_PTR_TO_MAP_VALUE, etc. as such. However, there are map types where this isn't true. For example, sockmap and sockhash store sockets. In general this isn't a big problem: we can just write helpers that explicitly requestsPTR_TO_SOCKETinstead ofARG_PTR_TO_MAP_VALUE.The one exception are the standard map helpers like
map_update_elem,map_lookup_elem, etc. Here it would be nice we could overload the function prototype for different kinds of maps. Unfortunately, this isn't entirely straight forward:
We only know the type of the map once we have resolvedmeta->map_ptrincheck_func_arg. This means we can't swap out the prototype incheck_helper_calluntil we're half way through the function.Instead, modify
check_func_argto treat ARG_PTR_TO_MAP_VALUE* to mean "the native type for the map" instead of "pointer to memory" for sockmap and sockhash. This means we don't have to modify the function prototype at allSigned-off-by Lorenz Bauer <[email protected]>--- kernel/bpf/verifier.c | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+)kernel/bpf/verifier.c +12
-0@@ -3872,6 +3872,35 @@ static int int_ptr_type_to_size(enum bpf_arg_type type)return -EINVAL ; } +static int resolve_map_arg_type (struct bpf_verifier_env *env, +const struct bpf_call_arg_meta *meta, +enum bpf_arg_type *arg_type) +{ +if (!meta->map_ptr) { +/* kernel subsystem misconfigured verifier */ +verbose (env,"invalid map_ptr to access map->type\n" ); +return -EACCES ; + } + +switch (meta->map_ptr->map_type) { +case BPF_MAP_TYPE_SOCKMAP :Quoted 8 lines
kernel/bpf/verifier.c static int int_ptr_type_to_size(enum bpf_arg_type type) · line 3891
Quoted 5 lines
kernel/bpf/verifier.c +5
-0+case BPF_MAP_TYPE_SOCKHASH : +if (*arg_type ==ARG_PTR_TO_MAP_VALUE ) { + *arg_type =ARG_PTR_TO_SOCKET ; + }else if (*arg_type ==ARG_PTR_TO_MAP_VALUE_OR_NULL ) { + *arg_type =ARG_PTR_TO_SOCKET_OR_NULL ;Is this
*arg_type==ARG_PTR_TO_MAP_VALUE_OR_NULLpossible with current implementation?
Quoted 5 lines
No, the only user is
bpf_sk_storage_getand friends which requiresBPF_MAP_TYPE_SK_STORAGE.
I seemed to make sense to mapARG_PTR_TO_MAP_VALUE_OR_NULL, but I can remove it as
well if you prefer. Do you think this is dangerous?
It is not dangerous, but is misleading. People looking at code may think it is possible but actually it is not. So I prefer you remove it.
Quoted 42 lines
Quoted 39 lines
If not, we can remove this "else if" and return
-EINVAL, right?Quoted 16 lines
kernel/bpf/verifier.c +12
-0+ }else { +verbose (env,"invalid arg_type for sockmap/sockhash\n" ); +return -EINVAL ; + } +break ; + +default : +break ; + } +return 0 ; +} +static int check_func_arg (struct bpf_verifier_env *env,u32 arg,struct bpf_call_arg_meta *meta,const struct bpf_func_proto *fn)kernel/bpf/verifier.c static int check_func_arg(struct bpf_verifier_env *env, u32 arg, · line 3939
Quoted 7 lines
kernel/bpf/verifier.c +4
-0@@ -3904,6 +3933,14 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,return -EACCES ; } +if (arg_type ==ARG_PTR_TO_MAP_VALUE || + arg_type ==ARG_PTR_TO_UNINIT_MAP_VALUE || + arg_type ==ARG_PTR_TO_MAP_VALUE_OR_NULL ) { + err =resolve_map_arg_type (env, meta, &arg_type);I am okay with this to cover all
MAP_VALUEtypes with func nameresolve_map_arg_typeas a generic helper.Quoted 8 lines
kernel/bpf/verifier.c +4
-0+if (err) +return err; + } + if (arg_type == ARG_PTR_TO_MAP_KEY || arg_type == ARG_PTR_TO_MAP_VALUE || arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE ||
PATCH v2 5/6 bpf: sockmap: allow update from BPF
In reply to Lorenz BauerAllow calling bpf_map_update_elem on sockmap and sockhash from a BPF context. The synchronization required for this is a bit fiddly: we need to prevent the socket from changing its state while we add it to the sockmap, since we rely on getting a callback via sk_prot->unhash. However, we can't just lock_sock like in sock_map_sk_acquire because that might sleep. So instead we disable softirq processing and use bh_lock_sock to prevent further modification.
--- kernel/bpf/verifier.c | 41 +++++++++++++++++++++++++++++++++++++++-- net/core/sock_map.c | 24 ++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 2 deletions(-)
kernel/bpf/verifier.c +39 -2
@@ -4180,6 +4180,41 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,return -EACCES ; } +static bool may_update_sockmap (struct bpf_verifier_env *env,int func_id) +{ +enum bpf_attach_type eatype = env->prog->expected_attach_type; +enum bpf_prog_type type = env->prog->type; + +if (func_id != BPF_FUNC_map_update_elem) +return false; + +/* It's not possible to get access to a locked struct sock in these +* contexts, so updating is safe. +*/ +switch (type) { +case BPF_PROG_TYPE_TRACING : +if (eatype ==BPF_TRACE_ITER ) +return true; +break ; +case BPF_PROG_TYPE_SOCKET_FILTER : +case BPF_PROG_TYPE_SCHED_CLS : +case BPF_PROG_TYPE_SCHED_ACT : +case BPF_PROG_TYPE_XDP : +case BPF_PROG_TYPE_CGROUP_SKB : +case BPF_PROG_TYPE_SK_SKB : +case BPF_PROG_TYPE_SK_MSG : +case BPF_PROG_TYPE_SK_REUSEPORT : +case BPF_PROG_TYPE_FLOW_DISSECTOR : +case BPF_PROG_TYPE_SK_LOOKUP : +return true; +default : +break ; + } + +verbose (env,"cannot update sockmap in this context\n" ); +return false; +} +static int check_map_func_compatibility (struct bpf_verifier_env *env,struct bpf_map *map,int func_id) {
@@ -4251,7 +4286,8 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env, func_id != BPF_FUNC_map_delete_elem && func_id != BPF_FUNC_msg_redirect_map && func_id != BPF_FUNC_sk_select_reuseport && - func_id != BPF_FUNC_map_lookup_elem) + func_id != BPF_FUNC_map_lookup_elem && + !may_update_sockmap(env, func_id)) goto error;break ;case BPF_MAP_TYPE_SOCKHASH :
@@ -4260,7 +4296,8 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env, func_id != BPF_FUNC_map_delete_elem && func_id != BPF_FUNC_msg_redirect_hash && func_id != BPF_FUNC_sk_select_reuseport && - func_id != BPF_FUNC_map_lookup_elem) + func_id != BPF_FUNC_map_lookup_elem && + !may_update_sockmap(env, func_id)) goto error;break ;case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY :
net/core/sock_map.c +24 -0
@@ -603,6 +603,28 @@ int sock_map_update_elem_sys(struct bpf_map *map, void *key, void *value,return ret; } +static int sock_map_update_elem (struct bpf_map *map,void *key, +void *value,u64 flags) +{ +struct sock *sk = (struct sock *)value; +int ret; + +if (!sock_map_sk_is_suitable (sk)) +return -EOPNOTSUPP ; + +local_bh_disable (); +bh_lock_sock (sk); +if (!sock_map_sk_state_allowed (sk)) + ret = -EOPNOTSUPP ; +else if (map->map_type ==BPF_MAP_TYPE_SOCKMAP ) + ret =sock_map_update_common (map, *(u32 *)key, sk, flags); +else + ret =sock_hash_update_common (map, key, sk, flags); +bh_unlock_sock (sk); +local_bh_enable (); +return ret; +} + BPF_CALL_4(bpf_sock_map_update, struct bpf_sock_ops_kern *, sops, struct bpf_map *, map, void *, key, u64, flags) {
@@ -687,6 +709,7 @@ const struct bpf_map_ops sock_map_ops = { .map_free = sock_map_free, .map_get_next_key = sock_map_get_next_key, .map_lookup_elem_sys_only = sock_map_lookup_sys, + .map_update_elem = sock_map_update_elem, .map_delete_elem = sock_map_delete_elem, .map_lookup_elem = sock_map_lookup, .map_release_uref = sock_map_release_progs,
@@ -1180,6 +1203,7 @@ const struct bpf_map_ops sock_hash_ops = { .map_alloc = sock_hash_alloc, .map_free = sock_hash_free, .map_get_next_key = sock_hash_get_next_key, + .map_update_elem = sock_map_update_elem, .map_delete_elem = sock_hash_delete_elem, .map_lookup_elem = sock_hash_lookup, .map_lookup_elem_sys_only = sock_hash_lookup_sys,
Signature · 1 line
2.25.1
PATCH v2 6/6 selftests: bpf: test sockmap update from BPF
In reply to Lorenz BauerAdd a test which copies a socket from a sockmap into another sockmap or sockhash. This excercises bpf_map_update_elem support from BPF context. Compare the socket cookies from source and destination to ensure that the copy succeeded.
--- .../selftests/bpf/prog_tests/sockmap_basic.c | 71 +++++++++++++++++++ .../selftests/bpf/progs/test_sockmap_copy.c | 48 +++++++++++++ 2 files changed, 119 insertions(+) create mode 100644 tools/testing/selftests/bpf/progs/test_sockmap_copy.c
tools/testing/selftests/bpf/prog_tests/sockmap_basic.c +71 -0
@@ -4,6 +4,7 @@#include "test_progs.h" #include "test_skmsg_load_helpers.skel.h" +#include "test_sockmap_copy.skel.h" #define TCP_REPAIR 19/* TCP sock is under repair right now */
@@ -101,6 +102,72 @@ static void test_skmsg_helpers(enum bpf_map_type map_type)test_skmsg_load_helpers__destroy (skel); } +static void test_sockmap_copy (enum bpf_map_type map_type) +{ +struct bpf_prog_test_run_attr tattr; +struct test_sockmap_copy *skel; +__u64 src_cookie, dst_cookie; +int err, prog, src, dst; +const __u32 zero =0 ; +char dummy[14 ] = {0 }; +__s64 sk; + + sk =connected_socket_v4 (); +if (CHECK_FAIL (sk ==-1 )) +return ; + + skel =test_sockmap_copy__open_and_load (); +if (CHECK_FAIL (!skel)) { +close (sk); +perror ("test_sockmap_copy__open_and_load" ); +return ; + } + + prog =bpf_program__fd (skel->progs.copy_sock_map); + src =bpf_map__fd (skel->maps.src); +if (map_type ==BPF_MAP_TYPE_SOCKMAP ) + dst =bpf_map__fd (skel->maps.dst_sock_map); +else + dst =bpf_map__fd (skel->maps.dst_sock_hash); + + err =bpf_map_update_elem (src, &zero, &sk,BPF_NOEXIST ); +if (CHECK_FAIL (err)) { +perror ("bpf_map_update" ); + goto out; + } + + err =bpf_map_lookup_elem (src, &zero, &src_cookie); +if (CHECK_FAIL (err)) { +perror ("bpf_map_lookup_elem(src)" ); + goto out; + } + + tattr = (struct bpf_prog_test_run_attr ){ + .prog_fd = prog, + .repeat =1 , + .data_in = dummy, + .data_size_in =sizeof (dummy), + }; + + err =bpf_prog_test_run_xattr (&tattr); +if (CHECK_ATTR (err || !tattr.retval,"bpf_prog_test_run" , +"errno=%u retval=%u\n" , errno, tattr.retval)) + goto out; + + err =bpf_map_lookup_elem (dst, &zero, &dst_cookie); +if (CHECK_FAIL (err)) { +perror ("bpf_map_lookup_elem(dst)" ); + goto out; + } + +if (dst_cookie != src_cookie) +PRINT_FAIL ("cookie %llu != %llu\n" , dst_cookie, src_cookie); + +out: +close (sk); +test_sockmap_copy__destroy (skel); +} +void test_sockmap_basic (void ) {if (test__start_subtest ("sockmap create_update_free" ))
@@ -111,4 +178,8 @@ void test_sockmap_basic(void)test_skmsg_helpers (BPF_MAP_TYPE_SOCKMAP );if (test__start_subtest ("sockhash sk_msg load helpers" ))test_skmsg_helpers (BPF_MAP_TYPE_SOCKHASH ); +if (test__start_subtest ("sockmap copy" )) +test_sockmap_copy (BPF_MAP_TYPE_SOCKMAP ); +if (test__start_subtest ("sockhash copy" )) +test_sockmap_copy (BPF_MAP_TYPE_SOCKHASH ); }
tools/testing/selftests/bpf/progs/test_sockmap_copy.c new file +48 -0
48 lines not on this page · raw message
Signature · 1 line
2.25.1