bpf: misc performance improvements for cgroup hooks
Subsystems (9)
v6 cover bpf: misc performance improvements for cgroup hooks
First patch adds custom getsockopt for TCP_ZEROCOPY_RECEIVE to remove kmalloc and lock_sock overhead from the dat path.
Second patch removes kzalloc/kfree from getsockopt for the common cases.
Third patch switches cgroup_bpf_enabled to be per-attach to to add only overhead for the cgroup attach types used on the system.
No visible user-side changes.
v6:
- avoid indirect cost for new bpf_bypass_getsockopt (Eric Dumazet)
v5:
- reorder patches to reduce the churn (Martin KaFai Lau)
v4:
- update performance numbers
- bypass_bpf_getsockopt (Martin KaFai Lau)
v3:
- remove extra newline, add comment about sizeof tcp_zerocopy_receive (Martin KaFai Lau)
- add another patch to remove lock_sock overhead from TCP_ZEROCOPY_RECEIVE; technically, this makes patch #1 obsolete, but I'd still prefer to keep it to help with other socket options
v2:
- perf numbers for getsockopt kmalloc reduction (Song Liu)
- (sk) in BPF_CGROUP_PRE_CONNECT_ENABLED (Song Liu)
- 128 -> 64 buffer size, BUILD_BUG_ON (Martin KaFai Lau)
Stanislav Fomichev (3):
bpf: remove extra lock_sock for TCP_ZEROCOPY_RECEIVE
bpf: try to avoid kzalloc in cgroup/{s,g}etsockopt
bpf: split cgroup_bpf_enabled per attach type
include/linux/bpf-cgroup.h | 63 +++++++---- include/linux/filter.h | 5 + include/linux/indirect_call_wrapper.h | 6 + include/net/sock.h | 2 + include/net/tcp.h | 1 + kernel/bpf/cgroup.c | 104 +++++++++++++++--- net/ipv4/af_inet.c | 9 +- net/ipv4/tcp.c | 14 +++ net/ipv4/tcp_ipv4.c | 1 + net/ipv4/udp.c | 7 +- net/ipv6/af_inet6.c | 9 +- net/ipv6/tcp_ipv6.c | 1 + net/ipv6/udp.c | 7 +- net/socket.c | 3 + .../selftests/bpf/prog_tests/sockopt_sk.c | 22 ++++ .../testing/selftests/bpf/progs/sockopt_sk.c | 15 +++ 16 files changed, 217 insertions(+), 52 deletions(-)
Signature · 1 line
2.30.0.284.gd98b1dd5eaa7-goog
PATCH v6 1/3 bpf: remove extra lock_sock for TCP_ZEROCOPY_RECEIVE
Add custom implementation of getsockopt hook for TCP_ZEROCOPY_RECEIVE. We skip generic hooks for TCP_ZEROCOPY_RECEIVE and have a custom call in do_tcp_getsockopt using the on-stack data. This removes 3% overhead for locking/unlocking the socket.
Without this patch:
3.38% 0.07% tcp_mmap [kernel.kallsyms] [k] __cgroup_bpf_run_filter_getsockopt
|
--3.30%--__cgroup_bpf_run_filter_getsockopt
|
--0.81%--__kmallocWith the patch applied:
0.52% 0.12% tcp_mmap [kernel.kallsyms] [k] __cgroup_bpf_run_filter_getsockopt_kern
--- include/linux/bpf-cgroup.h | 27 +++++++++++-- include/linux/indirect_call_wrapper.h | 6 +++ include/net/sock.h | 2 + include/net/tcp.h | 1 + kernel/bpf/cgroup.c | 38 +++++++++++++++++++ net/ipv4/tcp.c | 14 +++++++ net/ipv4/tcp_ipv4.c | 1 + net/ipv6/tcp_ipv6.c | 1 + net/socket.c | 3 ++ .../selftests/bpf/prog_tests/sockopt_sk.c | 22 +++++++++++ .../testing/selftests/bpf/progs/sockopt_sk.c | 15 ++++++++ 11 files changed, 126 insertions(+), 4 deletions(-)
include/linux/bpf-cgroup.h +23 -4
@@ -147,6 +147,10 @@ int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level, int __user *optlen, int max_optlen, int retval); +int __cgroup_bpf_run_filter_getsockopt_kern (struct sock *sk,int level, +int optname,void *optval, +int *optlen,int retval); +static inline enum bpf_cgroup_storage_type cgroup_storage_type (struct bpf_map *map) {
@@ -364,10 +368,23 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, ({ \int __ret = retval; \if (cgroup_bpf_enabled) \ - __ret =__cgroup_bpf_run_filter_getsockopt (sock, level, \ - optname, optval, \ - optlen, max_optlen, \ - retval); \ +if (!(sock)->sk_prot->bpf_bypass_getsockopt || \ + !INDIRECT_CALL_INET_1 ((sock)->sk_prot->bpf_bypass_getsockopt, \ + tcp_bpf_bypass_getsockopt, \ + level, optname)) \ + __ret =__cgroup_bpf_run_filter_getsockopt ( \ + sock, level, optname, optval, optlen, \ + max_optlen, retval); \ + __ret; \ +}) + +#define BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN (sock, level, optname, optval, \ + optlen, retval) \ +({ \ + int __ret = retval; \ + if (cgroup_bpf_enabled) \ + __ret = __cgroup_bpf_run_filter_getsockopt_kern( \ + sock, level, optname, optval, optlen, retval); \ __ret; \ })
@@ -452,6 +469,8 @@ static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map,#define BPF_CGROUP_GETSOCKOPT_MAX_OPTLEN (optlen) ({ 0; })#define BPF_CGROUP_RUN_PROG_GETSOCKOPT (sock, level, optname, optval, \ optlen, max_optlen, retval) ({ retval; }) +#define BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN (sock, level, optname, optval, \ + optlen, retval) ({ retval; })#define BPF_CGROUP_RUN_PROG_SETSOCKOPT (sock, level, optname, optval, optlen, \ kernel_optval) ({ 0; })
include/linux/indirect_call_wrapper.h +6 -0
@@ -60,4 +60,10 @@#define INDIRECT_CALL_INET (f, f2, f1, ...) f(__VA_ARGS__)#endif +#if IS_ENABLED (CONFIG_INET ) +#define INDIRECT_CALL_INET_1 (f, f1, ...) INDIRECT_CALL_1(f, f1, __VA_ARGS__) +#else +#define INDIRECT_CALL_INET_1 (f, f1, ...) f(__VA_ARGS__) +#endif +#endif
include/net/sock.h +2 -0
@@ -1174,6 +1174,8 @@ struct proto {int (*backlog_rcv) (struct sock *sk,struct sk_buff *skb); +bool (*bpf_bypass_getsockopt)(int level, +int optname);void (*release_cb)(struct sock *sk);
include/net/tcp.h +1 -0
@@ -403,6 +403,7 @@ __poll_t tcp_poll(struct file *file, struct socket *sock, struct poll_table_struct *wait);int tcp_getsockopt (struct sock *sk,int level,int optname,char __user *optval,int __user *optlen); +bool tcp_bpf_bypass_getsockopt (int level,int optname);int tcp_setsockopt (struct sock *sk,int level,int optname,sockptr_t optval,unsigned int optlen);void tcp_set_keepalive (struct sock *sk,int val);
kernel/bpf/cgroup.c +38 -0
@@ -1485,6 +1485,44 @@ int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level,sockopt_free_buf (&ctx);return ret; } + +int __cgroup_bpf_run_filter_getsockopt_kern (struct sock *sk,int level, +int optname,void *optval, +int *optlen,int retval) +{ +struct cgroup *cgrp =sock_cgroup_ptr (&sk->sk_cgrp_data); +struct bpf_sockopt_kern ctx = { + .sk = sk, + .level = level, + .optname = optname, + .retval = retval, + .optlen = *optlen, + .optval = optval, + .optval_end = optval + *optlen, + }; +int ret; + + ret =BPF_PROG_RUN_ARRAY (cgrp->bpf.effective[BPF_CGROUP_GETSOCKOPT ], + &ctx,BPF_PROG_RUN ); +if (!ret) +return -EPERM ; + +if (ctx.optlen > *optlen) +return -EFAULT ; + +/* BPF programs only allowed to set retval to 0, not some +* arbitrary value. +*/ +if (ctx.retval !=0 && ctx.retval != retval) +return -EFAULT ; + +/* BPF programs can shrink the buffer, export the modifications. +*/ +if (ctx.optlen !=0 ) + *optlen = ctx.optlen; + +return ctx.retval; +}#endif static ssize_t sysctl_cpy_dir(const struct ctl_dir *dir, char **bufp,
net/ipv4/tcp.c +14 -0
@@ -4098,6 +4098,8 @@ static int do_tcp_getsockopt(struct sock *sk, int level,return -EFAULT ;lock_sock (sk); err =tcp_zerocopy_receive (sk, &zc); + err =BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN (sk, level, optname, + &zc, &len, err);release_sock (sk); if (len >= offsetofend(struct tcp_zerocopy_receive, err)) goto zerocopy_rcv_sk_err;
@@ -4132,6 +4134,18 @@ static int do_tcp_getsockopt(struct sock *sk, int level,return 0 ; } +bool tcp_bpf_bypass_getsockopt (int level,int optname) +{ +/* TCP do_tcp_getsockopt has optimized getsockopt implementation +* to avoid extra socket lock for TCP_ZEROCOPY_RECEIVE. +*/ +if (level ==SOL_TCP && optname ==TCP_ZEROCOPY_RECEIVE ) +return true; + +return false; +} +EXPORT_SYMBOL (tcp_bpf_bypass_getsockopt); +int tcp_getsockopt (struct sock *sk,int level,int optname,char __user *optval,int __user *optlen) {
net/ipv4/tcp_ipv4.c +1 -0
@@ -2792,6 +2792,7 @@ struct proto tcp_prot = { .shutdown = tcp_shutdown, .setsockopt = tcp_setsockopt, .getsockopt = tcp_getsockopt, + .bpf_bypass_getsockopt = tcp_bpf_bypass_getsockopt, .keepalive = tcp_set_keepalive, .recvmsg = tcp_recvmsg, .sendmsg = tcp_sendmsg,
net/ipv6/tcp_ipv6.c +1 -0
@@ -2121,6 +2121,7 @@ struct proto tcpv6_prot = { .shutdown = tcp_shutdown, .setsockopt = tcp_setsockopt, .getsockopt = tcp_getsockopt, + .bpf_bypass_getsockopt = tcp_bpf_bypass_getsockopt, .keepalive = tcp_set_keepalive, .recvmsg = tcp_recvmsg, .sendmsg = tcp_sendmsg,
net/socket.c +3 -0
@@ -2126,6 +2126,9 @@ SYSCALL_DEFINE5(setsockopt, int, fd, int, level, int, optname,return __sys_setsockopt (fd, level, optname, optval, optlen); } +INDIRECT_CALLABLE_DECLARE(bool tcp_bpf_bypass_getsockopt(int level, +int optname)); +/* * Get a socket option. Because we don't know the option lengths we have * to pass a user mode parameter for the protocols to sort out.
tools/testing/selftests/bpf/prog_tests/sockopt_sk.c +22 -0
@@ -11,6 +11,7 @@ static int getsetsockopt(void)char u8[4 ];__u32 u32;char cc[16 ];/* TCP_CA_NAME_MAX */ +struct tcp_zerocopy_receive zc; } buf = {};socklen_t optlen;char *big_buf =NULL ;
@@ -154,6 +155,27 @@ static int getsetsockopt(void) goto err; } +/* TCP_ZEROCOPY_RECEIVE triggers */ +memset (&buf,0 ,sizeof (buf)); + optlen =sizeof (buf.zc); + err =getsockopt (fd,SOL_TCP ,TCP_ZEROCOPY_RECEIVE , &buf, &optlen); +if (err) { +log_err ("Unexpected getsockopt(TCP_ZEROCOPY_RECEIVE) err=%d errno=%d" , + err, errno); + goto err; + } + +memset (&buf,0 ,sizeof (buf)); + buf.zc.address =12345 ;/* rejected by BPF */ + optlen =sizeof (buf.zc); + errno =0 ; + err =getsockopt (fd,SOL_TCP ,TCP_ZEROCOPY_RECEIVE , &buf, &optlen); +if (errno !=EPERM ) { +log_err ("Unexpected getsockopt(TCP_ZEROCOPY_RECEIVE) err=%d errno=%d" , + err, errno); + goto err; + } +free (big_buf);close (fd);return 0 ;
tools/testing/selftests/bpf/progs/sockopt_sk.c +15 -0
@@ -57,6 +57,21 @@ int _getsockopt(struct bpf_sockopt *ctx)return 1 ; } +if (ctx->level ==SOL_TCP && ctx->optname ==TCP_ZEROCOPY_RECEIVE ) { +/* Verify that TCP_ZEROCOPY_RECEIVE triggers. +* It has a custom implementation for performance +* reasons. +*/ + +if (optval +sizeof (struct tcp_zerocopy_receive ) > optval_end) +return 0 ;/* EPERM, bounds check */ + +if (((struct tcp_zerocopy_receive *)optval)->address !=0 ) +return 0 ;/* EPERM, unexpected data */ + +return 1 ; + } +if (ctx->level ==SOL_IP && ctx->optname ==IP_FREEBIND ) {if (optval +1 > optval_end)return 0 ;/* EPERM, bounds check */
Signature · 1 line
2.30.0.284.gd98b1dd5eaa7-goog
Re: [PATCH bpf-next v6 1/3] bpf: remove extra lock_sock for TCP_ZEROCOPY_RECEIVE
Quoted 33 lines · Stanislav Fomichev
Add custom implementation of getsockopt hook for
TCP_ZEROCOPY_RECEIVE. We skip generic hooks forTCP_ZEROCOPY_RECEIVEand have a custom call indo_tcp_getsockoptusing the on-stack data. This removes 3% overhead for locking/unlocking the socket.Without this patch:
3.38% 0.07% tcp_mmap [kernel.kallsyms] [k] __cgroup_bpf_run_filter_getsockopt | --3.30%--__cgroup_bpf_run_filter_getsockopt | --0.81%--__kmallocWith the patch applied:
0.52% 0.12% tcp_mmap [kernel.kallsyms] [k] __cgroup_bpf_run_filter_getsockopt_kernSigned-off-by Stanislav Fomichev <[email protected]>Cc Martin KaFai Lau <[email protected]>Cc Song Liu <[email protected]>Cc Eric Dumazet <[email protected]>--- include/linux/bpf-cgroup.h | 27 +++++++++++-- include/linux/indirect_call_wrapper.h | 6 +++ include/net/sock.h | 2 + include/net/tcp.h | 1 + kernel/bpf/cgroup.c | 38 +++++++++++++++++++ net/ipv4/tcp.c | 14 +++++++ net/ipv4/tcp_ipv4.c | 1 + net/ipv6/tcp_ipv6.c | 1 + net/socket.c | 3 ++ .../selftests/bpf/prog_tests/sockopt_sk.c | 22 +++++++++++ .../testing/selftests/bpf/progs/sockopt_sk.c | 15 ++++++++ 11 files changed, 126 insertions(+), 4 deletions(-)
[ ... ]
Quoted 15 lines
kernel/bpf/cgroup.c +7
-0@@ -1485,6 +1485,44 @@ int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level,sockopt_free_buf (&ctx);return ret; } + +int __cgroup_bpf_run_filter_getsockopt_kern (struct sock *sk,int level, +int optname,void *optval, +int *optlen,int retval) +{ +struct cgroup *cgrp =sock_cgroup_ptr (&sk->sk_cgrp_data); +struct bpf_sockopt_kern ctx = {
kernel/bpf/cgroup.c int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level, · line 1504
Quoted 10 lines
kernel/bpf/cgroup.c +10
-0+ .sk = sk, + .level = level, + .optname = optname, + .retval = retval, + .optlen = *optlen, + .optval = optval, + .optval_end = optval + *optlen, + }; +int ret; +
The current behavior only passes kernel optval to bpf prog when retval == 0. Can you explain a few words here about the difference and why it is fine?
Just in case some other options may want to reuse the __cgroup_bpf_run_filter_getsockopt_kern() in the future.
Quoted 24 lines
kernel/bpf/cgroup.c +21
-0+ ret =BPF_PROG_RUN_ARRAY (cgrp->bpf.effective[BPF_CGROUP_GETSOCKOPT ], + &ctx,BPF_PROG_RUN ); +if (!ret) +return -EPERM ; + +if (ctx.optlen > *optlen) +return -EFAULT ; + +/* BPF programs only allowed to set retval to 0, not some +* arbitrary value. +*/ +if (ctx.retval !=0 && ctx.retval != retval) +return -EFAULT ; + +/* BPF programs can shrink the buffer, export the modifications. +*/ +if (ctx.optlen !=0 ) + *optlen = ctx.optlen; + +return ctx.retval; +}#endif static ssize_t sysctl_cpy_dir(const struct ctl_dir *dir, char **bufp,
[ ... ]
tools/testing/selftests/bpf/prog_tests/sockopt_sk.c static int getsetsockopt(void) · line 14
Quoted 9 lines
tools/testing/selftests/bpf/prog_tests/sockopt_sk.c +1
-0@@ -11,6 +11,7 @@ static int getsetsockopt(void)char u8[4 ];__u32 u32;char cc[16 ];/* TCP_CA_NAME_MAX */ +struct tcp_zerocopy_receive zc;
I suspect it won't compile at least in my setup.
However, I compile tools/testing/selftests/net/tcp_mmap.c fine though. I _guess_ it is because the net's test has included kernel/usr/include.
AFAIK, bpf's tests use tools/include/uapi/.
Others LGTM.
Re: [PATCH bpf-next v6 1/3] bpf: remove extra lock_sock for TCP_ZEROCOPY_RECEIVE
Quoted 59 lines · Martin KaFai Lau
Quoted 33 lines · Stanislav Fomichev
Add custom implementation of getsockopt hook for
TCP_ZEROCOPY_RECEIVE. We skip generic hooks forTCP_ZEROCOPY_RECEIVEand have a custom call indo_tcp_getsockoptusing the on-stack data. This removes 3% overhead for locking/unlocking the socket.Without this patch:
3.38% 0.07% tcp_mmap [kernel.kallsyms] [k] __cgroup_bpf_run_filter_getsockopt | --3.30%--__cgroup_bpf_run_filter_getsockopt | --0.81%--__kmallocWith the patch applied:
0.52% 0.12% tcp_mmap [kernel.kallsyms] [k] __cgroup_bpf_run_filter_getsockopt_kernSigned-off-by Stanislav Fomichev <[email protected]>Cc Martin KaFai Lau <[email protected]>Cc Song Liu <[email protected]>Cc Eric Dumazet <[email protected]>--- include/linux/bpf-cgroup.h | 27 +++++++++++-- include/linux/indirect_call_wrapper.h | 6 +++ include/net/sock.h | 2 + include/net/tcp.h | 1 + kernel/bpf/cgroup.c | 38 +++++++++++++++++++ net/ipv4/tcp.c | 14 +++++++ net/ipv4/tcp_ipv4.c | 1 + net/ipv6/tcp_ipv6.c | 1 + net/socket.c | 3 ++ .../selftests/bpf/prog_tests/sockopt_sk.c | 22 +++++++++++ .../testing/selftests/bpf/progs/sockopt_sk.c | 15 ++++++++ 11 files changed, 126 insertions(+), 4 deletions(-)[ ... ]
Quoted 22 lines
kernel/bpf/cgroup.c +14
-0@@ -1485,6 +1485,44 @@ int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level,sockopt_free_buf (&ctx);return ret; } + +int __cgroup_bpf_run_filter_getsockopt_kern (struct sock *sk,int level, +int optname,void *optval, +int *optlen,int retval) +{ +struct cgroup *cgrp =sock_cgroup_ptr (&sk->sk_cgrp_data); +struct bpf_sockopt_kern ctx = { + .sk = sk, + .level = level, + .optname = optname, + .retval = retval, + .optlen = *optlen, + .optval = optval, + .optval_end = optval + *optlen,
Quoted 8 lines
kernel/bpf/cgroup.c int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level, · line 1504
Quoted 3 lines
kernel/bpf/cgroup.c +3
-0+ }; +int ret; +The current behavior only passes kernel optval to bpf prog when retval == 0. Can you explain a few words here about the difference and why it is fine?
Just in case some other options may want to reuse the__cgroup_bpf_run_filter_getsockopt_kern()in the future.
IIRC, whatever we do in __cgroup_bpf_run_filter_getsockopt with skipping the copy for retval != 0 is just an optimization. I was assuming that on the error, kernel wouldn't copy anything back to the users (not sure how true in real life it is). I'll add a comment here to signify the difference.
Quoted 38 lines
Quoted 24 lines
kernel/bpf/cgroup.c +21
-0+ ret =BPF_PROG_RUN_ARRAY (cgrp->bpf.effective[BPF_CGROUP_GETSOCKOPT ], + &ctx,BPF_PROG_RUN ); +if (!ret) +return -EPERM ; + +if (ctx.optlen > *optlen) +return -EFAULT ; + +/* BPF programs only allowed to set retval to 0, not some +* arbitrary value. +*/ +if (ctx.retval !=0 && ctx.retval != retval) +return -EFAULT ; + +/* BPF programs can shrink the buffer, export the modifications. +*/ +if (ctx.optlen !=0 ) + *optlen = ctx.optlen; + +return ctx.retval; +}#endif static ssize_t sysctl_cpy_dir(const struct ctl_dir *dir, char **bufp,[ ... ]
tools/testing/selftests/bpf/prog_tests/sockopt_sk.c static int getsetsockopt(void) · line 14
Quoted 9 lines
tools/testing/selftests/bpf/prog_tests/sockopt_sk.c +1
-0@@ -11,6 +11,7 @@ static int getsetsockopt(void)char u8[4 ];__u32 u32;char cc[16 ];/* TCP_CA_NAME_MAX */ +struct tcp_zerocopy_receive zc;I suspect it won't compile at least in my setup.
Quoted 6 lines
However, I compile
tools/testing/selftests/net/tcp_mmap.cfine though. I _guess_ it is because the net's test has includedkernel/usr/include.AFAIK, bpf's tests use
tools/include/uapi/.Others LGTM.
Sure, let me add export it to tools/include/uapi. I didn't do it because it also compiled for me and I assumed that tcp_zerocopy_receive was exported too long ago to care (we are using the first field anyway so don't really need the latest layout).
PATCH v6 2/3 bpf: try to avoid kzalloc in cgroup/{s,g}etsockopt
In reply to Stanislav FomichevWhen we attach a bpf program to cgroup/getsockopt any other getsockopt() syscall starts incurring kzalloc/kfree cost.
Let add a small buffer on the stack and use it for small (majority) {s,g}etsockopt values. The buffer is small enough to fit into the cache line and cover the majority of simple options (most of them are 4 byte ints).
It seems natural to do the same for setsockopt, but it's a bit more involved when the BPF program modifies the data (where we have to kmalloc). The assumption is that for the majority of setsockopt calls (which are doing pure BPF options or apply policy) this will bring some benefit as well.
Without this patch (we remove about 1% __kmalloc):
3.38% 0.07% tcp_mmap [kernel.kallsyms] [k] __cgroup_bpf_run_filter_getsockopt
|
--3.30%--__cgroup_bpf_run_filter_getsockopt
|
--0.81%--__kmalloc--- include/linux/filter.h | 5 ++++ kernel/bpf/cgroup.c | 52 ++++++++++++++++++++++++++++++++++++------ 2 files changed, 50 insertions(+), 7 deletions(-)
include/linux/filter.h +5 -0
@@ -1281,6 +1281,11 @@ struct bpf_sysctl_kern {u64 tmp_reg; }; +#define BPF_SOCKOPT_KERN_BUF_SIZE 32 +struct bpf_sockopt_buf { +u8 data[BPF_SOCKOPT_KERN_BUF_SIZE ]; +}; +struct bpf_sockopt_kern {struct sock *sk;u8 *optval;
kernel/bpf/cgroup.c +45 -7
114 lines not on this page · raw message
Signature · 1 line
2.30.0.284.gd98b1dd5eaa7-goog
Re: [PATCH bpf-next v6 2/3] bpf: try to avoid kzalloc in cgroup/{s,g}etsockopt
Quoted 113 lines · Stanislav Fomichev
When we attach a bpf program to cgroup/getsockopt any other
getsockopt()syscall starts incurring kzalloc/kfree cost.Let add a small buffer on the stack and use it for small (majority) {s,g}etsockopt values. The buffer is small enough to fit into the cache line and cover the majority of simple options (most of them are 4 byte ints).
It seems natural to do the same for setsockopt, but it's a bit more involved when the BPF program modifies the data (where we have to kmalloc). The assumption is that for the majority of setsockopt calls (which are doing pure BPF options or apply policy) this will bring some benefit as well.
Without this patch (we remove about 1%
__kmalloc):3.38% 0.07% tcp_mmap [kernel.kallsyms] [k] __cgroup_bpf_run_filter_getsockopt | --3.30%--__cgroup_bpf_run_filter_getsockopt | --0.81%--__kmallocSigned-off-by Stanislav Fomichev <[email protected]>Cc Martin KaFai Lau <[email protected]>Cc Song Liu <[email protected]>--- include/linux/filter.h | 5 ++++ kernel/bpf/cgroup.c | 52 ++++++++++++++++++++++++++++++++++++------ 2 files changed, 50 insertions(+), 7 deletions(-)include/linux/filter.h +5
-0@@ -1281,6 +1281,11 @@ struct bpf_sysctl_kern {u64 tmp_reg; }; +#define BPF_SOCKOPT_KERN_BUF_SIZE 32 +struct bpf_sockopt_buf { +u8 data[BPF_SOCKOPT_KERN_BUF_SIZE ]; +}; +struct bpf_sockopt_kern {struct sock *sk;u8 *optval;kernel/bpf/cgroup.c +23
-3@@ -1298,7 +1298,8 @@ static bool __cgroup_bpf_prog_array_is_empty(struct cgroup *cgrp,return empty; } -static int sockopt_alloc_buf (struct bpf_sockopt_kern *ctx,int max_optlen) +static int sockopt_alloc_buf (struct bpf_sockopt_kern *ctx,int max_optlen, +struct bpf_sockopt_buf *buf) {if (unlikely (max_optlen <0 ))return -EINVAL ;@@ -1310,6 +1311,15 @@ static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen) max_optlen =PAGE_SIZE ; } +if (max_optlen <=sizeof (buf->data)) { +/* When the optval fits into BPF_SOCKOPT_KERN_BUF_SIZE +* bytes avoid the cost of kzalloc. +*/ + ctx->optval = buf->data; + ctx->optval_end = ctx->optval + max_optlen; +return max_optlen; + } + ctx->optval =kzalloc (max_optlen,GFP_USER );if (!ctx->optval)return -ENOMEM ;@@ -1319,16 +1329,26 @@ static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen)return max_optlen; } -static void sockopt_free_buf (struct bpf_sockopt_kern *ctx) +static void sockopt_free_buf (struct bpf_sockopt_kern *ctx, +struct bpf_sockopt_buf *buf) { +if (ctx->optval == buf->data) +return ;kfree (ctx->optval); } +static bool sockopt_buf_allocated (struct bpf_sockopt_kern *ctx, +struct bpf_sockopt_buf *buf) +{ +return ctx->optval != buf->data; +} +int __cgroup_bpf_run_filter_setsockopt (struct sock *sk,int *level,int *optname,char __user *optval,int *optlen,char **kernel_optval) {struct cgroup *cgrp =sock_cgroup_ptr (&sk->sk_cgrp_data); +struct bpf_sockopt_buf buf = {};struct bpf_sockopt_kern ctx = { .sk = sk, .level = *level,@@ -1350,7 +1370,7 @@ int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level,*/ max_optlen =max_t (int,16 , *optlen); - max_optlen =sockopt_alloc_buf (&ctx, max_optlen); + max_optlen =sockopt_alloc_buf (&ctx, max_optlen, &buf);if (max_optlen <0 )return max_optlen;
kernel/bpf/cgroup.c int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level, · line 1411
Quoted 2 lines
kernel/bpf/cgroup.c +0
-0@@ -1390,13 +1410,30 @@ int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level,*/ if (ctx.optlen !=0 ) {
When ctx.optlen == 0, is sockopt_free_buf() called? Did I miss something?
Quoted 30 lines
kernel/bpf/cgroup.c +19
-2*optlen = ctx.optlen; - *kernel_optval = ctx.optval; +/* We've used bpf_sockopt_kern->buf as an intermediary +* storage, but the BPF program indicates that we need +* to pass this data to the kernel setsockopt handler. +* No way to export on-stack buf, have to allocate a +* new buffer. +*/ +if (!sockopt_buf_allocated (&ctx, &buf)) { +void *p =kzalloc (ctx.optlen,GFP_USER ); + +if (!p) { + ret = -ENOMEM ; + goto out; + } +memcpy (p, ctx.optval, ctx.optlen); + *kernel_optval = p; + }else { + *kernel_optval = ctx.optval; + } } } out:if (ret) -sockopt_free_buf (&ctx); +sockopt_free_buf (&ctx, &buf);return ret; }
Re: [PATCH bpf-next v6 2/3] bpf: try to avoid kzalloc in cgroup/{s,g}etsockopt
Quoted 114 lines · Martin KaFai Lau
Quoted 112 lines · Stanislav Fomichev
When we attach a bpf program to cgroup/getsockopt any other
getsockopt()syscall starts incurring kzalloc/kfree cost.Let add a small buffer on the stack and use it for small (majority) {s,g}etsockopt values. The buffer is small enough to fit into the cache line and cover the majority of simple options (most of them are 4 byte ints).
It seems natural to do the same for setsockopt, but it's a bit more involved when the BPF program modifies the data (where we have to kmalloc). The assumption is that for the majority of setsockopt calls (which are doing pure BPF options or apply policy) this will bring some benefit as well.
Without this patch (we remove about 1%
__kmalloc):3.38% 0.07% tcp_mmap [kernel.kallsyms] [k] __cgroup_bpf_run_filter_getsockopt | --3.30%--__cgroup_bpf_run_filter_getsockopt | --0.81%--__kmallocSigned-off-by Stanislav Fomichev <[email protected]>Cc Martin KaFai Lau <[email protected]>Cc Song Liu <[email protected]>--- include/linux/filter.h | 5 ++++ kernel/bpf/cgroup.c | 52 ++++++++++++++++++++++++++++++++++++------ 2 files changed, 50 insertions(+), 7 deletions(-)include/linux/filter.h +5
-0@@ -1281,6 +1281,11 @@ struct bpf_sysctl_kern {u64 tmp_reg; }; +#define BPF_SOCKOPT_KERN_BUF_SIZE 32 +struct bpf_sockopt_buf { +u8 data[BPF_SOCKOPT_KERN_BUF_SIZE ]; +}; +struct bpf_sockopt_kern {struct sock *sk;u8 *optval;kernel/bpf/cgroup.c +23
-3@@ -1298,7 +1298,8 @@ static bool __cgroup_bpf_prog_array_is_empty(struct cgroup *cgrp,return empty; } -static int sockopt_alloc_buf (struct bpf_sockopt_kern *ctx,int max_optlen) +static int sockopt_alloc_buf (struct bpf_sockopt_kern *ctx,int max_optlen, +struct bpf_sockopt_buf *buf) {if (unlikely (max_optlen <0 ))return -EINVAL ;@@ -1310,6 +1311,15 @@ static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen) max_optlen =PAGE_SIZE ; } +if (max_optlen <=sizeof (buf->data)) { +/* When the optval fits into BPF_SOCKOPT_KERN_BUF_SIZE +* bytes avoid the cost of kzalloc. +*/ + ctx->optval = buf->data; + ctx->optval_end = ctx->optval + max_optlen; +return max_optlen; + } + ctx->optval =kzalloc (max_optlen,GFP_USER );if (!ctx->optval)return -ENOMEM ;@@ -1319,16 +1329,26 @@ static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen)return max_optlen; } -static void sockopt_free_buf (struct bpf_sockopt_kern *ctx) +static void sockopt_free_buf (struct bpf_sockopt_kern *ctx, +struct bpf_sockopt_buf *buf) { +if (ctx->optval == buf->data) +return ;kfree (ctx->optval); } +static bool sockopt_buf_allocated (struct bpf_sockopt_kern *ctx, +struct bpf_sockopt_buf *buf) +{ +return ctx->optval != buf->data; +} +int __cgroup_bpf_run_filter_setsockopt (struct sock *sk,int *level,int *optname,char __user *optval,int *optlen,char **kernel_optval) {struct cgroup *cgrp =sock_cgroup_ptr (&sk->sk_cgrp_data); +struct bpf_sockopt_buf buf = {};struct bpf_sockopt_kern ctx = { .sk = sk, .level = *level,@@ -1350,7 +1370,7 @@ int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level,*/ max_optlen =max_t (int,16 , *optlen); - max_optlen =sockopt_alloc_buf (&ctx, max_optlen); + max_optlen =sockopt_alloc_buf (&ctx, max_optlen, &buf);if (max_optlen <0 )return max_optlen;
Quoted 5 lines
kernel/bpf/cgroup.c int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level, · line 1411
Quoted 3 lines
kernel/bpf/cgroup.c +0
-0@@ -1390,13 +1410,30 @@ int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level,*/ if (ctx.optlen !=0 ) {When ctx.optlen == 0, is
sockopt_free_buf()called? Did I miss something?
Ouch, good catch, it looks like we do leak the buf here with optlen == 0.
We should probably change the following below to:
out:
if (!*kernel_optval)
sockopt_free_buf(...);I'll send a bpf patch with a Fixes tag, thanks!
PATCH v6 3/3 bpf: split cgroup_bpf_enabled per attach type
In reply to Stanislav FomichevWhen we attach any cgroup hook, the rest (even if unused/unattached) start to contribute small overhead. In particular, the one we want to avoid is __cgroup_bpf_run_filter_skb which does two redirections to get to the cgroup and pushes/pulls skb.
Let's split cgroup_bpf_enabled to be per-attach to make sure only used attach types trigger.
I've dropped some existing high-level cgroup_bpf_enabled in some places because BPF_PROG_CGROUP_XXX_RUN macros usually have another cgroup_bpf_enabled check.
I also had to copy-paste BPF_CGROUP_RUN_SA_PROG_LOCK for GETPEERNAME/GETSOCKNAME because type for cgroup_bpf_enabled[type] has to be constant and known at compile time.
--- include/linux/bpf-cgroup.h | 38 ++++++++++++++++++++------------------ kernel/bpf/cgroup.c | 14 ++++++-------- net/ipv4/af_inet.c | 9 +++++---- net/ipv4/udp.c | 7 +++---- net/ipv6/af_inet6.c | 9 +++++---- net/ipv6/udp.c | 7 +++---- 6 files changed, 42 insertions(+), 42 deletions(-)
include/linux/bpf-cgroup.h +20 -18
@@ -23,8 +23,8 @@ struct ctl_table_header;#ifdef CONFIG_CGROUP_BPF -extern struct static_key_false cgroup_bpf_enabled_key; -#define cgroup_bpf_enabled static_branch_unlikely(&cgroup_bpf_enabled_key) +extern struct static_key_false cgroup_bpf_enabled_key[MAX_BPF_ATTACH_TYPE ]; +#define cgroup_bpf_enabled (type) static_branch_unlikely(&cgroup_bpf_enabled_key[type]) DECLARE_PER_CPU(struct bpf_cgroup_storage*, bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE ]);
@@ -189,7 +189,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key,#define BPF_CGROUP_RUN_PROG_INET_INGRESS (sk, skb) \ ({ \ int __ret = 0; \ - if (cgroup_bpf_enabled) \ + if (cgroup_bpf_enabled(BPF_CGROUP_INET_INGRESS)) \ __ret = __cgroup_bpf_run_filter_skb(sk, skb, \ BPF_CGROUP_INET_INGRESS); \ \
@@ -199,7 +199,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key,#define BPF_CGROUP_RUN_PROG_INET_EGRESS (sk, skb) \ ({ \ int __ret = 0; \ - if (cgroup_bpf_enabled && sk && sk == skb->sk) { \ + if (cgroup_bpf_enabled(BPF_CGROUP_INET_EGRESS) && sk && sk == skb->sk) { \ typeof(sk) __sk = sk_to_full_sk(sk); \ if (sk_fullsock(__sk)) \ __ret = __cgroup_bpf_run_filter_skb(__sk, skb, \
@@ -211,7 +211,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key,#define BPF_CGROUP_RUN_SK_PROG (sk, type) \ ({ \ int __ret = 0; \ - if (cgroup_bpf_enabled) { \ + if (cgroup_bpf_enabled(type)) { \ __ret = __cgroup_bpf_run_filter_sk(sk, type); \ } \ __ret; \
@@ -232,7 +232,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key,#define BPF_CGROUP_RUN_SA_PROG (sk, uaddr, type) \ ({ \ int __ret = 0; \ - if (cgroup_bpf_enabled) \ + if (cgroup_bpf_enabled(type)) \ __ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, type, \ NULL); \ __ret; \
@@ -241,7 +241,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key,#define BPF_CGROUP_RUN_SA_PROG_LOCK (sk, uaddr, type, t_ctx) \ ({ \ int __ret = 0; \ - if (cgroup_bpf_enabled) { \ + if (cgroup_bpf_enabled(type)) { \ lock_sock(sk); \ __ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, type, \ t_ctx); \
@@ -256,8 +256,10 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key,#define BPF_CGROUP_RUN_PROG_INET6_BIND_LOCK (sk, uaddr) \ BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_INET6_BIND, NULL) -#define BPF_CGROUP_PRE_CONNECT_ENABLED (sk) (cgroup_bpf_enabled && \ - sk->sk_prot->pre_connect) +#define BPF_CGROUP_PRE_CONNECT_ENABLED (sk) \ + ((cgroup_bpf_enabled(BPF_CGROUP_INET4_CONNECT) || \ + cgroup_bpf_enabled(BPF_CGROUP_INET6_CONNECT)) && \ + (sk)->sk_prot->pre_connect)#define BPF_CGROUP_RUN_PROG_INET4_CONNECT (sk, uaddr) \ BPF_CGROUP_RUN_SA_PROG(sk, uaddr, BPF_CGROUP_INET4_CONNECT)
@@ -301,7 +303,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key,#define BPF_CGROUP_RUN_PROG_SOCK_OPS_SK (sock_ops, sk) \ ({ \ int __ret = 0; \ - if (cgroup_bpf_enabled) \ + if (cgroup_bpf_enabled(BPF_CGROUP_SOCK_OPS)) \ __ret = __cgroup_bpf_run_filter_sock_ops(sk, \ sock_ops, \ BPF_CGROUP_SOCK_OPS); \
@@ -311,7 +313,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key,#define BPF_CGROUP_RUN_PROG_SOCK_OPS (sock_ops) \ ({ \ int __ret = 0; \ - if (cgroup_bpf_enabled && (sock_ops)->sk) { \ + if (cgroup_bpf_enabled(BPF_CGROUP_SOCK_OPS) && (sock_ops)->sk) { \ typeof(sk) __sk = sk_to_full_sk((sock_ops)->sk); \ if (__sk && sk_fullsock(__sk)) \ __ret = __cgroup_bpf_run_filter_sock_ops(__sk, \
@@ -324,7 +326,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key,#define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP (type, major, minor, access) \ ({ \ int __ret = 0; \ - if (cgroup_bpf_enabled) \ + if (cgroup_bpf_enabled(BPF_CGROUP_DEVICE)) \ __ret = __cgroup_bpf_check_dev_permission(type, major, minor, \ access, \ BPF_CGROUP_DEVICE); \
@@ -336,7 +338,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key,#define BPF_CGROUP_RUN_PROG_SYSCTL (head, table, write, buf, count, pos) \ ({ \ int __ret = 0; \ - if (cgroup_bpf_enabled) \ + if (cgroup_bpf_enabled(BPF_CGROUP_SYSCTL)) \ __ret = __cgroup_bpf_run_filter_sysctl(head, table, write, \ buf, count, pos, \ BPF_CGROUP_SYSCTL); \
@@ -347,7 +349,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, kernel_optval) \ ({ \ int __ret = 0; \ - if (cgroup_bpf_enabled) \ + if (cgroup_bpf_enabled(BPF_CGROUP_SETSOCKOPT)) \ __ret = __cgroup_bpf_run_filter_setsockopt(sock, level, \ optname, optval, \ optlen, \
@@ -358,7 +360,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key,#define BPF_CGROUP_GETSOCKOPT_MAX_OPTLEN (optlen) \ ({ \ int __ret = 0; \ - if (cgroup_bpf_enabled) \ + if (cgroup_bpf_enabled(BPF_CGROUP_GETSOCKOPT)) \ get_user(__ret, optlen); \ __ret; \ })
@@ -367,7 +369,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, max_optlen, retval) \ ({ \ int __ret = retval; \ - if (cgroup_bpf_enabled) \ + if (cgroup_bpf_enabled(BPF_CGROUP_GETSOCKOPT)) \ if (!(sock)->sk_prot->bpf_bypass_getsockopt || \ !INDIRECT_CALL_INET_1((sock)->sk_prot->bpf_bypass_getsockopt, \ tcp_bpf_bypass_getsockopt, \
@@ -382,7 +384,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, optlen, retval) \ ({ \ int __ret = retval; \ - if (cgroup_bpf_enabled) \ + if (cgroup_bpf_enabled(BPF_CGROUP_GETSOCKOPT)) \ __ret = __cgroup_bpf_run_filter_getsockopt_kern( \ sock, level, optname, optval, optlen, retval); \ __ret; \
@@ -444,7 +446,7 @@ static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map,return 0 ; } -#define cgroup_bpf_enabled (0) +#define cgroup_bpf_enabled (type) (0)#define BPF_CGROUP_RUN_SA_PROG_LOCK (sk, uaddr, type, t_ctx) ({ 0; })#define BPF_CGROUP_PRE_CONNECT_ENABLED (sk) (0)#define BPF_CGROUP_RUN_PROG_INET_INGRESS (sk,skb) ({ 0; })
kernel/bpf/cgroup.c +6 -8
@@ -19,7 +19,7 @@#include "../cgroup/cgroup-internal.h" -DEFINE_STATIC_KEY_FALSE (cgroup_bpf_enabled_key); +DEFINE_STATIC_KEY_ARRAY_FALSE (cgroup_bpf_enabled_key,MAX_BPF_ATTACH_TYPE );EXPORT_SYMBOL (cgroup_bpf_enabled_key);void cgroup_bpf_offline (struct cgroup *cgrp)
@@ -128,7 +128,7 @@ static void cgroup_bpf_release(struct work_struct *work)if (pl->link)bpf_cgroup_link_auto_detach (pl->link);kfree (pl); -static_branch_dec (&cgroup_bpf_enabled_key); +static_branch_dec (&cgroup_bpf_enabled_key[type]); } old_array = rcu_dereference_protected( cgrp->bpf.effective[type],
@@ -499,7 +499,7 @@ int __cgroup_bpf_attach(struct cgroup *cgrp,if (old_prog)bpf_prog_put (old_prog);else -static_branch_inc (&cgroup_bpf_enabled_key); +static_branch_inc (&cgroup_bpf_enabled_key[type]);bpf_cgroup_storages_link (new_storage, cgrp, type);return 0 ;
@@ -698,7 +698,7 @@ int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, cgrp->bpf.flags[type] =0 ;if (old_prog)bpf_prog_put (old_prog); -static_branch_dec (&cgroup_bpf_enabled_key); +static_branch_dec (&cgroup_bpf_enabled_key[type]);return 0 ; cleanup:
@@ -1360,8 +1360,7 @@ int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level,* attached to the hook so we don't waste time allocating * memory and locking the socket. */ -if (!cgroup_bpf_enabled || -__cgroup_bpf_prog_array_is_empty (cgrp,BPF_CGROUP_SETSOCKOPT )) +if (__cgroup_bpf_prog_array_is_empty (cgrp,BPF_CGROUP_SETSOCKOPT ))return 0 ;/* Allocate a bit more than the initial user buffer for
@@ -1456,8 +1455,7 @@ int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level,* attached to the hook so we don't waste time allocating * memory and locking the socket. */ -if (!cgroup_bpf_enabled || -__cgroup_bpf_prog_array_is_empty (cgrp,BPF_CGROUP_GETSOCKOPT )) +if (__cgroup_bpf_prog_array_is_empty (cgrp,BPF_CGROUP_GETSOCKOPT ))return retval; ctx.optlen = max_optlen;
net/ipv4/af_inet.c +5 -4
@@ -777,18 +777,19 @@ int inet_getname(struct socket *sock, struct sockaddr *uaddr,return -ENOTCONN ; sin->sin_port = inet->inet_dport; sin->sin_addr.s_addr = inet->inet_daddr; +BPF_CGROUP_RUN_SA_PROG_LOCK (sk, (struct sockaddr *)sin, +BPF_CGROUP_INET4_GETPEERNAME , +NULL ); }else {__be32 addr = inet->inet_rcv_saddr;if (!addr) addr = inet->inet_saddr; sin->sin_port = inet->inet_sport; sin->sin_addr.s_addr = addr; - } -if (cgroup_bpf_enabled)BPF_CGROUP_RUN_SA_PROG_LOCK (sk, (struct sockaddr *)sin, - peer ?BPF_CGROUP_INET4_GETPEERNAME : -BPF_CGROUP_INET4_GETSOCKNAME , +BPF_CGROUP_INET4_GETSOCKNAME ,NULL ); + }memset (sin->sin_zero,0 ,sizeof (sin->sin_zero));return sizeof (*sin); }
net/ipv4/udp.c +3 -4
@@ -1124,7 +1124,7 @@ int udp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)rcu_read_unlock (); } -if (cgroup_bpf_enabled && !connected) { +if (cgroup_bpf_enabled (BPF_CGROUP_UDP4_SENDMSG ) && !connected) { err =BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK (sk, (struct sockaddr *)usin, &ipc.addr);if (err)
@@ -1858,9 +1858,8 @@ int udp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int noblock,memset (sin->sin_zero,0 ,sizeof (sin->sin_zero)); *addr_len =sizeof (*sin); -if (cgroup_bpf_enabled) -BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK (sk, - (struct sockaddr *)sin); +BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK (sk, + (struct sockaddr *)sin); }if (udp_sk (sk)->gro_enabled)
net/ipv6/af_inet6.c +5 -4
@@ -527,18 +527,19 @@ int inet6_getname(struct socket *sock, struct sockaddr *uaddr, sin->sin6_addr = sk->sk_v6_daddr;if (np->sndflow) sin->sin6_flowinfo = np->flow_label; +BPF_CGROUP_RUN_SA_PROG_LOCK (sk, (struct sockaddr *)sin, +BPF_CGROUP_INET6_GETPEERNAME , +NULL ); }else {if (ipv6_addr_any (&sk->sk_v6_rcv_saddr)) sin->sin6_addr = np->saddr;else sin->sin6_addr = sk->sk_v6_rcv_saddr; sin->sin6_port = inet->inet_sport; - } -if (cgroup_bpf_enabled)BPF_CGROUP_RUN_SA_PROG_LOCK (sk, (struct sockaddr *)sin, - peer ?BPF_CGROUP_INET6_GETPEERNAME : -BPF_CGROUP_INET6_GETSOCKNAME , +BPF_CGROUP_INET6_GETSOCKNAME ,NULL ); + } sin->sin6_scope_id =ipv6_iface_scope_id (&sin->sin6_addr, sk->sk_bound_dev_if);return sizeof (*sin);
net/ipv6/udp.c +3 -4
@@ -409,9 +409,8 @@ int udpv6_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, } *addr_len =sizeof (*sin6); -if (cgroup_bpf_enabled) -BPF_CGROUP_RUN_PROG_UDP6_RECVMSG_LOCK (sk, - (struct sockaddr *)sin6); +BPF_CGROUP_RUN_PROG_UDP6_RECVMSG_LOCK (sk, + (struct sockaddr *)sin6); }if (udp_sk (sk)->gro_enabled)
@@ -1462,7 +1461,7 @@ int udpv6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) fl6.saddr = np->saddr; fl6.fl6_sport = inet->inet_sport; -if (cgroup_bpf_enabled && !connected) { +if (cgroup_bpf_enabled (BPF_CGROUP_UDP6_SENDMSG ) && !connected) { err =BPF_CGROUP_RUN_PROG_UDP6_SENDMSG_LOCK (sk, (struct sockaddr *)sin6, &fl6.saddr);if (err)
Signature · 1 line
2.30.0.284.gd98b1dd5eaa7-goog