bpf, sockmap: Fix some issues with using apply_bytes
Subsystems (8)
v3 cover bpf, sockmap: Fix some issues with using apply_bytes
Patch 1~3 fixes three issues with using apply_bytes when redirecting. Patch 4 adds ingress tests for txmsg with apply_bytes in selftests.
Thanks to John Fastabend and Jakub Sitnicki for correct solution.
---
Changes in v3:
*Patch 2: Rename 'flags', modify based on Jakub Sitnicki's patch
Changes in v2:
*Patch 2: Clear psock->flags explicitly before releasing the sock lock
Pengcheng Yang (4):
bpf, sockmap: Fix repeated calls to sock_put() when msg has more_data bpf, sockmap: Fix missing BPF_F_INGRESS flag when using apply_bytes bpf, sockmap: Fix data loss caused by using apply_bytes on ingress
redirect
selftests/bpf: Add ingress tests for txmsg with apply_bytes
include/linux/skmsg.h | 1 + include/net/tcp.h | 4 ++-- net/core/skmsg.c | 9 ++++++--- net/ipv4/tcp_bpf.c | 19 ++++++++++++------- net/tls/tls_sw.c | 6 ++++-- tools/testing/selftests/bpf/test_sockmap.c | 18 ++++++++++++++++++ 6 files changed, 43 insertions(+), 14 deletions(-)
Signature · 1 line
1.8.3.1
PATCH v3 1/4 bpf, sockmap: Fix repeated calls to sock_put() when msg has more_data
In tcp_bpf_send_verdict() redirection, the eval variable is assigned to __SK_REDIRECT after the apply_bytes data is sent, if msg has more_data, sock_put() will be called multiple times.
We should reset the eval variable to __SK_NONE every time more_data starts.
This causes:
IPv4: Attempt to release TCP socket in state 1 00000000b4c925d7
------------[ cut here ]------------ refcount_t: addition on 0; use-after-free. WARNING: CPU: 5 PID: 4482 at lib/refcount.c:25 refcount_warn_saturate+0x7d/0x110 Modules linked in: CPU: 5 PID: 4482 Comm: sockhash_bypass Kdump: loaded Not tainted 6.0.0 #1 Hardware name: Red Hat KVM, BIOS 1.11.0-2.el7 04/01/2014 Call Trace: <TASK> __tcp_transmit_skb+0xa1b/0xb90 ? __alloc_skb+0x8c/0x1a0 ? __kmalloc_node_track_caller+0x184/0x320 tcp_write_xmit+0x22a/0x1110 __tcp_push_pending_frames+0x32/0xf0 do_tcp_sendpages+0x62d/0x640 tcp_bpf_push+0xae/0x2c0 tcp_bpf_sendmsg_redir+0x260/0x410 ? preempt_count_add+0x70/0xa0 tcp_bpf_send_verdict+0x386/0x4b0 tcp_bpf_sendmsg+0x21b/0x3b0 sock_sendmsg+0x58/0x70 __sys_sendto+0xfa/0x170 ? xfd_validate_state+0x1d/0x80 ? switch_fpu_return+0x59/0xe0 __x64_sys_sendto+0x24/0x30 do_syscall_64+0x37/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd
--- net/ipv4/tcp_bpf.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
net/ipv4/tcp_bpf.c +2 -1
@@ -279,7 +279,7 @@ static int tcp_bpf_send_verdict(struct sock *sk, struct sk_psock *psock,bool cork = false, enospc =sk_msg_full (msg);struct sock *sk_redir;u32 tosend, origsize, sent, delta =0 ; -u32 eval = __SK_NONE; +u32 eval;int ret; more_data:
@@ -310,6 +310,7 @@ static int tcp_bpf_send_verdict(struct sock *sk, struct sk_psock *psock, tosend = msg->sg.size;if (psock->apply_bytes && psock->apply_bytes < tosend) tosend = psock->apply_bytes; + eval = __SK_NONE;switch (psock->eval) {case __SK_PASS:
Signature · 1 line
1.8.3.1
PATCH v3 2/4 bpf, sockmap: Fix missing BPF_F_INGRESS flag when using apply_bytes
In reply to Pengcheng YangWhen redirecting, we use sk_msg_to_ingress() to get the BPF_F_INGRESS flag from the msg->flags. If apply_bytes is used and it is larger than the current data being processed, sk_psock_msg_verdict() will not be called when sendmsg() is called again. At this time, the msg->flags is 0, and we lost the BPF_F_INGRESS flag.
So we need to save the BPF_F_INGRESS flag in sk_psock and use it when redirection.
--- include/linux/skmsg.h | 1 + include/net/tcp.h | 4 ++-- net/core/skmsg.c | 9 ++++++--- net/ipv4/tcp_bpf.c | 11 ++++++----- net/tls/tls_sw.c | 6 ++++-- 5 files changed, 19 insertions(+), 12 deletions(-)
include/linux/skmsg.h +1 -0
@@ -82,6 +82,7 @@ struct sk_psock {u32 apply_bytes;u32 cork_bytes;u32 eval; +bool redir_ingress;/* undefined if sk_redir is null */ struct sk_msg *cork;struct sk_psock_progs progs;#if IS_ENABLED (CONFIG_BPF_STREAM_PARSER )
include/net/tcp.h +2 -2
@@ -2281,8 +2281,8 @@ void tcp_update_ulp(struct sock *sk, struct proto *p,void tcp_bpf_clone (const struct sock *sk,struct sock *newsk);#endif /* CONFIG_BPF_SYSCALL */ -int tcp_bpf_sendmsg_redir (struct sock *sk,struct sk_msg *msg,u32 bytes, -int flags); +int tcp_bpf_sendmsg_redir (struct sock *sk,bool ingress, +struct sk_msg *msg,u32 bytes,int flags);#endif /* CONFIG_NET_SOCK_MSG */ #if !defined(CONFIG_BPF_SYSCALL ) || !defined(CONFIG_NET_SOCK_MSG )
net/core/skmsg.c +6 -3
@@ -885,13 +885,16 @@ int sk_psock_msg_verdict(struct sock *sk, struct sk_psock *psock, ret =sk_psock_map_verd (ret, msg->sk_redir); psock->apply_bytes = msg->apply_bytes;if (ret == __SK_REDIRECT) { -if (psock->sk_redir) +if (psock->sk_redir) {sock_put (psock->sk_redir); - psock->sk_redir = msg->sk_redir; -if (!psock->sk_redir) { + psock->sk_redir =NULL ; + } +if (!msg->sk_redir) { ret = __SK_DROP; goto out; } + psock->redir_ingress =sk_msg_to_ingress (msg); + psock->sk_redir = msg->sk_redir;sock_hold (psock->sk_redir); } out:
net/ipv4/tcp_bpf.c +6 -5
@@ -131,10 +131,9 @@ static int tcp_bpf_push_locked(struct sock *sk, struct sk_msg *msg,return ret; } -int tcp_bpf_sendmsg_redir (struct sock *sk,struct sk_msg *msg, -u32 bytes,int flags) +int tcp_bpf_sendmsg_redir (struct sock *sk,bool ingress, +struct sk_msg *msg,u32 bytes,int flags) { -bool ingress =sk_msg_to_ingress (msg);struct sk_psock *psock =sk_psock_get (sk);int ret;
@@ -276,7 +275,7 @@ static int tcp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,static int tcp_bpf_send_verdict (struct sock *sk,struct sk_psock *psock,struct sk_msg *msg,int *copied,int flags) { -bool cork = false, enospc =sk_msg_full (msg); +bool cork = false, enospc =sk_msg_full (msg), redir_ingress;struct sock *sk_redir;u32 tosend, origsize, sent, delta =0 ;u32 eval;
@@ -322,6 +321,7 @@ static int tcp_bpf_send_verdict(struct sock *sk, struct sk_psock *psock,sk_msg_apply_bytes (psock, tosend);break ;case __SK_REDIRECT: + redir_ingress = psock->redir_ingress; sk_redir = psock->sk_redir;sk_msg_apply_bytes (psock, tosend);if (!psock->apply_bytes) {
@@ -338,7 +338,8 @@ static int tcp_bpf_send_verdict(struct sock *sk, struct sk_psock *psock,release_sock (sk); origsize = msg->sg.size; - ret =tcp_bpf_sendmsg_redir (sk_redir, msg, tosend, flags); + ret =tcp_bpf_sendmsg_redir (sk_redir, redir_ingress, + msg, tosend, flags); sent = origsize - msg->sg.size;if (eval == __SK_REDIRECT)
net/tls/tls_sw.c +4 -2
@@ -792,7 +792,7 @@ static int bpf_exec_tx_verdict(struct sk_msg *msg, struct sock *sk,struct sk_psock *psock;struct sock *sk_redir;struct tls_rec *rec; -bool enospc, policy; +bool enospc, policy, redir_ingress;int err =0 , send;u32 delta =0 ;
@@ -837,6 +837,7 @@ static int bpf_exec_tx_verdict(struct sk_msg *msg, struct sock *sk, }break ;case __SK_REDIRECT: + redir_ingress = psock->redir_ingress; sk_redir = psock->sk_redir;memcpy (&msg_redir, msg,sizeof (*msg));if (msg->apply_bytes < send)
@@ -846,7 +847,8 @@ static int bpf_exec_tx_verdict(struct sk_msg *msg, struct sock *sk,sk_msg_return_zero (sk, msg, send); msg->sg.size -= send;release_sock (sk); - err =tcp_bpf_sendmsg_redir (sk_redir, &msg_redir, send, flags); + err =tcp_bpf_sendmsg_redir (sk_redir, redir_ingress, + &msg_redir, send, flags);lock_sock (sk);if (err <0 ) { *copied -=sk_msg_free_nocharge (sk, &msg_redir);
Signature · 1 line
1.8.3.1
PATCH v3 3/4 bpf, sockmap: Fix data loss caused by using apply_bytes on ingress redirect
In reply to Pengcheng YangUse apply_bytes on ingress redirect, when apply_bytes is less than the length of msg data, some data may be skipped and lost in bpf_tcp_ingress().
If there is still data in the scatterlist that has not been consumed, we cannot move the msg iter.
--- net/ipv4/tcp_bpf.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)
net/ipv4/tcp_bpf.c +4 -1
@@ -45,8 +45,11 @@ static int bpf_tcp_ingress(struct sock *sk, struct sk_psock *psock, tmp->sg.end = i;if (apply) { apply_bytes -= size; -if (!apply_bytes) +if (!apply_bytes) { +if (sge->length) +sk_msg_iter_var_prev (i);break ; + } } }while (i != msg->sg.end);
Signature · 1 line
1.8.3.1
PATCH v3 4/4 selftests/bpf: Add ingress tests for txmsg with apply_bytes
In reply to Pengcheng YangCurrently, the ingress redirect is not covered in "txmsg test apply".
--- tools/testing/selftests/bpf/test_sockmap.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)
tools/testing/selftests/bpf/test_sockmap.c +18 -0
@@ -1649,24 +1649,42 @@ static void test_txmsg_apply(int cgrp, struct sockmap_options *opt) { txmsg_pass =1 ; txmsg_redir =0 ; + txmsg_ingress =0 ; txmsg_apply =1 ; txmsg_cork =0 ;test_send_one (opt, cgrp); txmsg_pass =0 ; txmsg_redir =1 ; + txmsg_ingress =0 ; + txmsg_apply =1 ; + txmsg_cork =0 ; +test_send_one (opt, cgrp); + + txmsg_pass =0 ; + txmsg_redir =1 ; + txmsg_ingress =1 ; txmsg_apply =1 ; txmsg_cork =0 ;test_send_one (opt, cgrp); txmsg_pass =1 ; txmsg_redir =0 ; + txmsg_ingress =0 ; + txmsg_apply =1024 ; + txmsg_cork =0 ; +test_send_large (opt, cgrp); + + txmsg_pass =0 ; + txmsg_redir =1 ; + txmsg_ingress =0 ; txmsg_apply =1024 ; txmsg_cork =0 ;test_send_large (opt, cgrp); txmsg_pass =0 ; txmsg_redir =1 ; + txmsg_ingress =1 ; txmsg_apply =1024 ; txmsg_cork =0 ;test_send_large (opt, cgrp);
Signature · 1 line
1.8.3.1
Quoted 19 lines · Pengcheng Yang
Patch 1~3 fixes three issues with using
apply_byteswhen redirecting. Patch 4 adds ingress tests for txmsg withapply_bytesin selftests.Thanks to John Fastabend and Jakub Sitnicki for correct solution.
---
Changes in v3:
*Patch 2: Rename 'flags', modify based on Jakub Sitnicki's patchChanges in v2:
*Patch 2: Clearpsock->flagsexplicitly before releasing the sock lockPengcheng Yang (4):
bpf, sockmap: Fix repeated calls tosock_put()when msg hasmore_databpf, sockmap: Fix missingBPF_F_INGRESSflag when usingapply_bytesbpf, sockmap: Fix data loss caused by usingapply_byteson ingress
redirect
selftests/bpf: Add ingress tests for txmsg withapply_bytes
Quoted 7 lines
include/linux/skmsg.h | 1 + include/net/tcp.h | 4 ++-- net/core/skmsg.c | 9 ++++++--- net/ipv4/tcp_bpf.c | 19 ++++++++++++------- net/tls/tls_sw.c | 6 ++++-- tools/testing/selftests/bpf/test_sockmap.c | 18 ++++++++++++++++++ 6 files changed, 43 insertions(+), 14 deletions(-)
Thanks for the fixes, Pengcheng.
For the series:
This series was applied to bpf/bpf-next.git (master) by Daniel Borkmann <[email protected]>:
Hello:
This series was applied to bpf/bpf-next.git (master) by Daniel Borkmann <[email protected]>:
Quoted 10 lines · On Tue, 29 Nov 2022 18:40:37 +0800 you wrote:
Patch 1~3 fixes three issues with using
apply_byteswhen redirecting. Patch 4 adds ingress tests for txmsg withapply_bytesin selftests.Thanks to John Fastabend and Jakub Sitnicki for correct solution.
---
Changes in v3:
*Patch 2: Rename 'flags', modify based on Jakub Sitnicki's patch[...]
Here is the summary with links:
- [bpf,v3,1/4] bpf, sockmap: Fix repeated calls to sock_put() when msg has more_data
https://git.kernel.org/bpf/bpf-next/c/7a9841ca0252 - [bpf,v3,2/4] bpf, sockmap: Fix missing BPF_F_INGRESS flag when using apply_bytes https://git.kernel.org/bpf/bpf-next/c/a351d6087bf7 - [bpf,v3,3/4] bpf, sockmap: Fix data loss caused by using apply_bytes on ingress redirect https://git.kernel.org/bpf/bpf-next/c/9072931f020b - [bpf,v3,4/4] selftests/bpf: Add ingress tests for txmsg with apply_bytes https://git.kernel.org/bpf/bpf-next/c/89903dcb3c2e
You are awesome, thank you!
Signature · 2 lines
Deet-doot-dot, I am a bot. https://korg.docs.kernel.org/patchwork/pwbot.html