bpf: misc performance improvements for cgroup hooks
Subsystems (5)
v2 cover bpf: misc performance improvements for cgroup hooks
First patch tries to remove kzalloc/kfree from getsockopt for the common cases.
Second patch switches cgroup_bpf_enabled to be per-attach to to add only overhead for the cgroup attach types used on the system.
No visible user-side changes.
v2:
- perf numbers for getsockopt kmalloc reduction (Song Liu)
- (sk) in BPF_CGROUP_PRE_CONNECT_ENABLED (Song Liu)
- 128 -> 64 buffer size, BUILD_BUG_ON (Martin KaFai Lau)
Stanislav Fomichev (2):
bpf: try to avoid kzalloc in cgroup/{s,g}etsockopt
bpf: split cgroup_bpf_enabled per attach type
include/linux/bpf-cgroup.h | 36 +++++++++++++------------ include/linux/filter.h | 3 +++ kernel/bpf/cgroup.c | 54 +++++++++++++++++++++++++++++++------- net/ipv4/af_inet.c | 9 ++++--- net/ipv4/udp.c | 7 +++-- net/ipv6/af_inet6.c | 9 ++++--- net/ipv6/udp.c | 7 +++-- 7 files changed, 83 insertions(+), 42 deletions(-)
Signature · 1 line
2.29.2.729.g45daf8777d-goog
PATCH v2 1/2 bpf: try to avoid kzalloc in cgroup/{s,g}etsockopt
When we attach a bpf program to cgroup/getsockopt any other getsockopt() syscall starts incurring kzalloc/kfree cost. While, in general, it's not an issue, sometimes it is, like in the case of TCP_ZEROCOPY_RECEIVE. TCP_ZEROCOPY_RECEIVE (ab)uses getsockopt system call to implement fastpath for incoming TCP, we don't want to have extra allocations in there.
Let add a small buffer on the stack and use it for small (majority) {s,g}etsockopt values. I've started with 128 bytes to cover the options we care about (TCP_ZEROCOPY_RECEIVE which is 32 bytes currently, with some planned extension to 64).
It seems natural to do the same for setsockopt, but it's a bit more involved when the BPF program modifies the data (where we have to kmalloc). The assumption is that for the majority of setsockopt calls (which are doing pure BPF options or apply policy) this will bring some benefit as well.
Collected some performance numbers using (on a 65k MTU localhost in a VM):
$ perf record -g -- ./tcp_mmap -s -z $ ./tcp_mmap -H ::1 -z $ ... $ perf report --symbol-filter=__cgroup_bpf_run_filter_getsockopt
Without this patch:
4.81% 0.07% tcp_mmap [kernel.kallsyms] [k] __cgroup_bpf_run_filter_>
|
--4.74%--__cgroup_bpf_run_filter_getsockopt
|
|--1.06%--__kmalloc
|
|--0.71%--lock_sock_nested
|
|--0.62%--__might_fault
|
--0.52%--release_sockWith the patch applied:
3.29% 0.07% tcp_mmap [kernel.kallsyms] [k] __cgroup_bpf_run_filter_getsockopt
|
--3.22%--__cgroup_bpf_run_filter_getsockopt
|
|--0.66%--lock_sock_nested
|
|--0.57%--__might_fault
|
--0.56%--release_sockSo it saves about 1% of the system call. Unfortunately, we still get 2-3% of overhead due to another socket lock/unlock :-(
--- include/linux/filter.h | 3 +++ kernel/bpf/cgroup.c | 40 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 1 deletion(-)
include/linux/filter.h +3 -0
@@ -1281,6 +1281,8 @@ struct bpf_sysctl_kern {u64 tmp_reg; }; +#define BPF_SOCKOPT_KERN_BUF_SIZE 64 +struct bpf_sockopt_kern {struct sock *sk;u8 *optval;
@@ -1289,6 +1291,7 @@ struct bpf_sockopt_kern {s32 optname;s32 optlen;s32 retval; +u8 buf[BPF_SOCKOPT_KERN_BUF_SIZE ]; };int copy_bpf_fprog_from_user (struct sock_fprog *dst,sockptr_t src,int len);
kernel/bpf/cgroup.c +39 -1
@@ -16,6 +16,7 @@#include <linux/bpf-cgroup.h> #include <net/sock.h> #include <net/bpf_sk_storage.h> +#include <net/tcp.h> /* sizeof(struct tcp_zerocopy_receive) */ #include "../cgroup/cgroup-internal.h"
@@ -1298,6 +1299,7 @@ static bool __cgroup_bpf_prog_array_is_empty(struct cgroup *cgrp,return empty; } +static int sockopt_alloc_buf (struct bpf_sockopt_kern *ctx,int max_optlen) {if (unlikely (max_optlen <0 ))
@@ -1310,6 +1312,18 @@ static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen) max_optlen =PAGE_SIZE ; } +if (max_optlen <=sizeof (ctx->buf)) { +/* When the optval fits into BPF_SOCKOPT_KERN_BUF_SIZE +* bytes avoid the cost of kzalloc. +*/ +BUILD_BUG_ON (sizeof (struct tcp_zerocopy_receive ) > +BPF_SOCKOPT_KERN_BUF_SIZE ); + + ctx->optval = ctx->buf; + ctx->optval_end = ctx->optval + max_optlen; +return max_optlen; + } + ctx->optval =kzalloc (max_optlen,GFP_USER );if (!ctx->optval)return -ENOMEM ;
@@ -1321,9 +1335,16 @@ static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen)static void sockopt_free_buf (struct bpf_sockopt_kern *ctx) { +if (ctx->optval == ctx->buf) +return ;kfree (ctx->optval); } +static bool sockopt_buf_allocated (struct bpf_sockopt_kern *ctx) +{ +return ctx->optval != ctx->buf; +} +int __cgroup_bpf_run_filter_setsockopt (struct sock *sk,int *level,int *optname,char __user *optval,int *optlen,char **kernel_optval)
@@ -1390,7 +1411,24 @@ int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level,*/ if (ctx.optlen !=0 ) { *optlen = ctx.optlen; - *kernel_optval = ctx.optval; +/* We've used bpf_sockopt_kern->buf as an intermediary +* storage, but the BPF program indicates that we need +* to pass this data to the kernel setsockopt handler. +* No way to export on-stack buf, have to allocate a +* new buffer. +*/ +if (!sockopt_buf_allocated (&ctx)) { +void *p =kzalloc (ctx.optlen,GFP_USER ); + +if (!p) { + ret = -ENOMEM ; + goto out; + } +memcpy (p, ctx.optval, ctx.optlen); + *kernel_optval = p; + }else { + *kernel_optval = ctx.optval; + } } }
Signature · 1 line
2.29.2.729.g45daf8777d-goog
Re: [PATCH bpf-next v2 1/2] bpf: try to avoid kzalloc in cgroup/{s,g}etsockopt
Quoted 48 lines · Stanislav Fomichev
When we attach a bpf program to cgroup/getsockopt any other
getsockopt()syscall starts incurring kzalloc/kfree cost. While, in general, it's not an issue, sometimes it is, like in the case ofTCP_ZEROCOPY_RECEIVE.TCP_ZEROCOPY_RECEIVE(ab)uses getsockopt system call to implement fastpath for incoming TCP, we don't want to have extra allocations in there.Let add a small buffer on the stack and use it for small (majority) {s,g}etsockopt values. I've started with 128 bytes to cover the options we care about (
TCP_ZEROCOPY_RECEIVEwhich is 32 bytes currently, with some planned extension to 64).It seems natural to do the same for setsockopt, but it's a bit more involved when the BPF program modifies the data (where we have to kmalloc). The assumption is that for the majority of setsockopt calls (which are doing pure BPF options or apply policy) this will bring some benefit as well.
Collected some performance numbers using (on a 65k MTU localhost in a VM):
$ perf record -g -- ./tcp_mmap -s -z $ ./tcp_mmap -H ::1 -z $ ... $ perf report --symbol-filter=__cgroup_bpf_run_filter_getsockoptWithout this patch:
4.81% 0.07% tcp_mmap [kernel.kallsyms] [k] __cgroup_bpf_run_filter_> | --4.74%--__cgroup_bpf_run_filter_getsockopt | |--1.06%--__kmalloc | |--0.71%--lock_sock_nested | |--0.62%--__might_fault | --0.52%--release_sockWith the patch applied:
3.29% 0.07% tcp_mmap [kernel.kallsyms] [k] __cgroup_bpf_run_filter_getsockopt | --3.22%--__cgroup_bpf_run_filter_getsockopt | |--0.66%--lock_sock_nested | |--0.57%--__might_fault | --0.56%--release_sock
Quoted 2 lines
So it saves about 1% of the system call. Unfortunately, we still get 2-3% of overhead due to another socket lock/unlock :-(
That could be a future exercise to optimize the fast path sockopts. ;)
[ ... ]
kernel/bpf/cgroup.c line 19
Quoted 7 lines
kernel/bpf/cgroup.c +1
-0@@ -16,6 +16,7 @@#include <linux/bpf-cgroup.h> #include <net/sock.h> #include <net/bpf_sk_storage.h> +#include <net/tcp.h> /* sizeof(struct tcp_zerocopy_receive) */
To be more specific, it should be <uapi/linux/tcp.h>.
kernel/bpf/cgroup.c static bool __cgroup_bpf_prog_array_is_empty(struct cgroup *cgrp, · line 1302
Quoted 8 lines
kernel/bpf/cgroup.c +1
-0#include "../cgroup/cgroup-internal.h" @@ -1298,6 +1299,7 @@ static bool __cgroup_bpf_prog_array_is_empty(struct cgroup *cgrp,return empty; } +
Extra newline.
kernel/bpf/cgroup.c static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen) · line 1318
Quoted 11 lines
kernel/bpf/cgroup.c +4
-0static int sockopt_alloc_buf (struct bpf_sockopt_kern *ctx,int max_optlen) {if (unlikely (max_optlen <0 ))@@ -1310,6 +1312,18 @@ static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen) max_optlen =PAGE_SIZE ; } +if (max_optlen <=sizeof (ctx->buf)) { +/* When the optval fits into BPF_SOCKOPT_KERN_BUF_SIZE +* bytes avoid the cost of kzalloc. +*/
If it needs to respin, it will be good to have a few words here on why it only BUILD_BUG checks for "struct tcp_zerocopy_receive".
Quoted 8 lines
kernel/bpf/cgroup.c +8
-0+BUILD_BUG_ON (sizeof (struct tcp_zerocopy_receive ) > +BPF_SOCKOPT_KERN_BUF_SIZE ); + + ctx->optval = ctx->buf; + ctx->optval_end = ctx->optval + max_optlen; +return max_optlen; + } +
Re: [PATCH bpf-next v2 1/2] bpf: try to avoid kzalloc in cgroup/{s,g}etsockopt
Quoted 50 lines · Martin KaFai Lau
Quoted 48 lines · Stanislav Fomichev
When we attach a bpf program to cgroup/getsockopt any other
getsockopt()syscall starts incurring kzalloc/kfree cost. While, in general, it's not an issue, sometimes it is, like in the case ofTCP_ZEROCOPY_RECEIVE.TCP_ZEROCOPY_RECEIVE(ab)uses getsockopt system call to implement fastpath for incoming TCP, we don't want to have extra allocations in there.Let add a small buffer on the stack and use it for small (majority) {s,g}etsockopt values. I've started with 128 bytes to cover the options we care about (
TCP_ZEROCOPY_RECEIVEwhich is 32 bytes currently, with some planned extension to 64).It seems natural to do the same for setsockopt, but it's a bit more involved when the BPF program modifies the data (where we have to kmalloc). The assumption is that for the majority of setsockopt calls (which are doing pure BPF options or apply policy) this will bring some benefit as well.
Collected some performance numbers using (on a 65k MTU localhost in a VM):
$ perf record -g -- ./tcp_mmap -s -z $ ./tcp_mmap -H ::1 -z $ ... $ perf report --symbol-filter=__cgroup_bpf_run_filter_getsockoptWithout this patch:
4.81% 0.07% tcp_mmap [kernel.kallsyms] [k] __cgroup_bpf_run_filter_> | --4.74%--__cgroup_bpf_run_filter_getsockopt | |--1.06%--__kmalloc | |--0.71%--lock_sock_nested | |--0.62%--__might_fault | --0.52%--release_sockWith the patch applied:
3.29% 0.07% tcp_mmap [kernel.kallsyms] [k] __cgroup_bpf_run_filter_getsockopt | --3.22%--__cgroup_bpf_run_filter_getsockopt | |--0.66%--lock_sock_nested | |--0.57%--__might_fault | --0.56%--release_sock
Quoted 3 lines
Quoted 2 lines
So it saves about 1% of the system call. Unfortunately, we still get 2-3% of overhead due to another socket lock/unlock :-(
That could be a future exercise to optimize the fast path sockopts. ;)
Yeah, I couldn't think about anything simple so far. The only idea I have is to allow custom implementation for tcp/udp (where we do lock_sock) and then have existing BPF_CGROUP_RUN_PROG_{S,G}ETSOCKOPT
in net/socket.c as a fallback. Need to experiment more with it.
Quoted 8 lines
kernel/bpf/cgroup.c line 19
Quoted 7 lines
kernel/bpf/cgroup.c +1
-0@@ -16,6 +16,7 @@#include <linux/bpf-cgroup.h> #include <net/sock.h> #include <net/bpf_sk_storage.h> +#include <net/tcp.h> /* sizeof(struct tcp_zerocopy_receive) */ To be more specific, it should be <
uapi/linux/tcp.h>.
Sure, let's do that. I went with net/tcp.h because most of the code under net/* doesn't include uapi directly.
Quoted 9 lines
kernel/bpf/cgroup.c static bool __cgroup_bpf_prog_array_is_empty(struct cgroup *cgrp, · line 1302
Quoted 8 lines
kernel/bpf/cgroup.c +1
-0#include "../cgroup/cgroup-internal.h" @@ -1298,6 +1299,7 @@ static bool __cgroup_bpf_prog_array_is_empty(struct cgroup *cgrp,return empty; } +Extra newline.
Oops, thanks, will fix.
Quoted 7 lines
Quoted 7 lines
kernel/bpf/cgroup.c +0
-0static int sockopt_alloc_buf (struct bpf_sockopt_kern *ctx,int max_optlen) {if (unlikely (max_optlen <0 ))@@ -1310,6 +1312,18 @@ static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen) max_optlen =PAGE_SIZE ; }
Quoted 6 lines
kernel/bpf/cgroup.c static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen) · line 1318
Quoted 4 lines
kernel/bpf/cgroup.c +4
-0+if (max_optlen <=sizeof (ctx->buf)) { +/* When the optval fits into BPF_SOCKOPT_KERN_BUF_SIZE +* bytes avoid the cost of kzalloc. +*/ If it needs to respin, it will be good to have a few words here on why it only
BUILD_BUGchecks for "structtcp_zerocopy_receive".
Sounds good, will add. I'll wait a day to let others comment and will respin.
Quoted 8 lines
Quoted 8 lines
kernel/bpf/cgroup.c +8
-0+BUILD_BUG_ON (sizeof (struct tcp_zerocopy_receive ) > +BPF_SOCKOPT_KERN_BUF_SIZE ); + + ctx->optval = ctx->buf; + ctx->optval_end = ctx->optval + max_optlen; +return max_optlen; + } +
PATCH v2 2/2 bpf: split cgroup_bpf_enabled per attach type
In reply to Stanislav FomichevWhen we attach any cgroup hook, the rest (even if unused/unattached) start to contribute small overhead. In particular, the one we want to avoid is __cgroup_bpf_run_filter_skb which does two redirections to get to the cgroup and pushes/pulls skb.
Let's split cgroup_bpf_enabled to be per-attach to make sure only used attach types trigger.
I've dropped some existing high-level cgroup_bpf_enabled in some places because BPF_PROG_CGROUP_XXX_RUN macros usually have another cgroup_bpf_enabled check.
I also had to copy-paste BPF_CGROUP_RUN_SA_PROG_LOCK for GETPEERNAME/GETSOCKNAME because type for cgroup_bpf_enabled[type] has to be constant and known at compile time.
--- include/linux/bpf-cgroup.h | 36 +++++++++++++++++++----------------- kernel/bpf/cgroup.c | 14 ++++++-------- net/ipv4/af_inet.c | 9 +++++---- net/ipv4/udp.c | 7 +++---- net/ipv6/af_inet6.c | 9 +++++---- net/ipv6/udp.c | 7 +++---- 6 files changed, 41 insertions(+), 41 deletions(-)
include/linux/bpf-cgroup.h +19 -17
@@ -23,8 +23,8 @@ struct ctl_table_header;#ifdef CONFIG_CGROUP_BPF -extern struct static_key_false cgroup_bpf_enabled_key; -#define cgroup_bpf_enabled static_branch_unlikely(&cgroup_bpf_enabled_key) +extern struct static_key_false cgroup_bpf_enabled_key[MAX_BPF_ATTACH_TYPE ]; +#define cgroup_bpf_enabled (type) static_branch_unlikely(&cgroup_bpf_enabled_key[type]) DECLARE_PER_CPU(struct bpf_cgroup_storage*, bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE ]);
@@ -185,7 +185,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key,#define BPF_CGROUP_RUN_PROG_INET_INGRESS (sk, skb) \ ({ \ int __ret = 0; \ - if (cgroup_bpf_enabled) \ + if (cgroup_bpf_enabled(BPF_CGROUP_INET_INGRESS)) \ __ret = __cgroup_bpf_run_filter_skb(sk, skb, \ BPF_CGROUP_INET_INGRESS); \ \
@@ -195,7 +195,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key,#define BPF_CGROUP_RUN_PROG_INET_EGRESS (sk, skb) \ ({ \ int __ret = 0; \ - if (cgroup_bpf_enabled && sk && sk == skb->sk) { \ + if (cgroup_bpf_enabled(BPF_CGROUP_INET_EGRESS) && sk && sk == skb->sk) { \ typeof(sk) __sk = sk_to_full_sk(sk); \ if (sk_fullsock(__sk)) \ __ret = __cgroup_bpf_run_filter_skb(__sk, skb, \
@@ -207,7 +207,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key,#define BPF_CGROUP_RUN_SK_PROG (sk, type) \ ({ \ int __ret = 0; \ - if (cgroup_bpf_enabled) { \ + if (cgroup_bpf_enabled(type)) { \ __ret = __cgroup_bpf_run_filter_sk(sk, type); \ } \ __ret; \
@@ -228,7 +228,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key,#define BPF_CGROUP_RUN_SA_PROG (sk, uaddr, type) \ ({ \ int __ret = 0; \ - if (cgroup_bpf_enabled) \ + if (cgroup_bpf_enabled(type)) \ __ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, type, \ NULL); \ __ret; \
@@ -237,7 +237,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key,#define BPF_CGROUP_RUN_SA_PROG_LOCK (sk, uaddr, type, t_ctx) \ ({ \ int __ret = 0; \ - if (cgroup_bpf_enabled) { \ + if (cgroup_bpf_enabled(type)) { \ lock_sock(sk); \ __ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, type, \ t_ctx); \
@@ -252,8 +252,10 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key,#define BPF_CGROUP_RUN_PROG_INET6_BIND_LOCK (sk, uaddr) \ BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_INET6_BIND, NULL) -#define BPF_CGROUP_PRE_CONNECT_ENABLED (sk) (cgroup_bpf_enabled && \ - sk->sk_prot->pre_connect) +#define BPF_CGROUP_PRE_CONNECT_ENABLED (sk) \ + ((cgroup_bpf_enabled(BPF_CGROUP_INET4_CONNECT) || \ + cgroup_bpf_enabled(BPF_CGROUP_INET6_CONNECT)) && \ + (sk)->sk_prot->pre_connect)#define BPF_CGROUP_RUN_PROG_INET4_CONNECT (sk, uaddr) \ BPF_CGROUP_RUN_SA_PROG(sk, uaddr, BPF_CGROUP_INET4_CONNECT)
@@ -297,7 +299,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key,#define BPF_CGROUP_RUN_PROG_SOCK_OPS_SK (sock_ops, sk) \ ({ \ int __ret = 0; \ - if (cgroup_bpf_enabled) \ + if (cgroup_bpf_enabled(BPF_CGROUP_SOCK_OPS)) \ __ret = __cgroup_bpf_run_filter_sock_ops(sk, \ sock_ops, \ BPF_CGROUP_SOCK_OPS); \
@@ -307,7 +309,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key,#define BPF_CGROUP_RUN_PROG_SOCK_OPS (sock_ops) \ ({ \ int __ret = 0; \ - if (cgroup_bpf_enabled && (sock_ops)->sk) { \ + if (cgroup_bpf_enabled(BPF_CGROUP_SOCK_OPS) && (sock_ops)->sk) { \ typeof(sk) __sk = sk_to_full_sk((sock_ops)->sk); \ if (__sk && sk_fullsock(__sk)) \ __ret = __cgroup_bpf_run_filter_sock_ops(__sk, \
@@ -320,7 +322,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key,#define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP (type, major, minor, access) \ ({ \ int __ret = 0; \ - if (cgroup_bpf_enabled) \ + if (cgroup_bpf_enabled(BPF_CGROUP_DEVICE)) \ __ret = __cgroup_bpf_check_dev_permission(type, major, minor, \ access, \ BPF_CGROUP_DEVICE); \
@@ -332,7 +334,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key,#define BPF_CGROUP_RUN_PROG_SYSCTL (head, table, write, buf, count, pos) \ ({ \ int __ret = 0; \ - if (cgroup_bpf_enabled) \ + if (cgroup_bpf_enabled(BPF_CGROUP_SYSCTL)) \ __ret = __cgroup_bpf_run_filter_sysctl(head, table, write, \ buf, count, pos, \ BPF_CGROUP_SYSCTL); \
@@ -343,7 +345,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, kernel_optval) \ ({ \ int __ret = 0; \ - if (cgroup_bpf_enabled) \ + if (cgroup_bpf_enabled(BPF_CGROUP_SETSOCKOPT)) \ __ret = __cgroup_bpf_run_filter_setsockopt(sock, level, \ optname, optval, \ optlen, \
@@ -354,7 +356,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key,#define BPF_CGROUP_GETSOCKOPT_MAX_OPTLEN (optlen) \ ({ \ int __ret = 0; \ - if (cgroup_bpf_enabled) \ + if (cgroup_bpf_enabled(BPF_CGROUP_GETSOCKOPT)) \ get_user(__ret, optlen); \ __ret; \ })
@@ -363,7 +365,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, max_optlen, retval) \ ({ \ int __ret = retval; \ - if (cgroup_bpf_enabled) \ + if (cgroup_bpf_enabled(BPF_CGROUP_GETSOCKOPT)) \ __ret = __cgroup_bpf_run_filter_getsockopt(sock, level, \ optname, optval, \ optlen, max_optlen, \
@@ -427,7 +429,7 @@ static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map,return 0 ; } -#define cgroup_bpf_enabled (0) +#define cgroup_bpf_enabled (type) (0)#define BPF_CGROUP_RUN_SA_PROG_LOCK (sk, uaddr, type, t_ctx) ({ 0; })#define BPF_CGROUP_PRE_CONNECT_ENABLED (sk) (0)#define BPF_CGROUP_RUN_PROG_INET_INGRESS (sk,skb) ({ 0; })
kernel/bpf/cgroup.c +6 -8
@@ -20,7 +20,7 @@#include "../cgroup/cgroup-internal.h" -DEFINE_STATIC_KEY_FALSE (cgroup_bpf_enabled_key); +DEFINE_STATIC_KEY_ARRAY_FALSE (cgroup_bpf_enabled_key,MAX_BPF_ATTACH_TYPE );EXPORT_SYMBOL (cgroup_bpf_enabled_key);void cgroup_bpf_offline (struct cgroup *cgrp)
@@ -129,7 +129,7 @@ static void cgroup_bpf_release(struct work_struct *work)if (pl->link)bpf_cgroup_link_auto_detach (pl->link);kfree (pl); -static_branch_dec (&cgroup_bpf_enabled_key); +static_branch_dec (&cgroup_bpf_enabled_key[type]); } old_array = rcu_dereference_protected( cgrp->bpf.effective[type],
@@ -500,7 +500,7 @@ int __cgroup_bpf_attach(struct cgroup *cgrp,if (old_prog)bpf_prog_put (old_prog);else -static_branch_inc (&cgroup_bpf_enabled_key); +static_branch_inc (&cgroup_bpf_enabled_key[type]);bpf_cgroup_storages_link (new_storage, cgrp, type);return 0 ;
@@ -699,7 +699,7 @@ int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, cgrp->bpf.flags[type] =0 ;if (old_prog)bpf_prog_put (old_prog); -static_branch_dec (&cgroup_bpf_enabled_key); +static_branch_dec (&cgroup_bpf_enabled_key[type]);return 0 ; cleanup:
@@ -1361,8 +1361,7 @@ int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level,* attached to the hook so we don't waste time allocating * memory and locking the socket. */ -if (!cgroup_bpf_enabled || -__cgroup_bpf_prog_array_is_empty (cgrp,BPF_CGROUP_SETSOCKOPT )) +if (__cgroup_bpf_prog_array_is_empty (cgrp,BPF_CGROUP_SETSOCKOPT ))return 0 ;/* Allocate a bit more than the initial user buffer for
@@ -1456,8 +1455,7 @@ int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level,* attached to the hook so we don't waste time allocating * memory and locking the socket. */ -if (!cgroup_bpf_enabled || -__cgroup_bpf_prog_array_is_empty (cgrp,BPF_CGROUP_GETSOCKOPT )) +if (__cgroup_bpf_prog_array_is_empty (cgrp,BPF_CGROUP_GETSOCKOPT ))return retval; ctx.optlen = max_optlen;
net/ipv4/af_inet.c +5 -4
@@ -777,18 +777,19 @@ int inet_getname(struct socket *sock, struct sockaddr *uaddr,return -ENOTCONN ; sin->sin_port = inet->inet_dport; sin->sin_addr.s_addr = inet->inet_daddr; +BPF_CGROUP_RUN_SA_PROG_LOCK (sk, (struct sockaddr *)sin, +BPF_CGROUP_INET4_GETPEERNAME , +NULL ); }else {__be32 addr = inet->inet_rcv_saddr;if (!addr) addr = inet->inet_saddr; sin->sin_port = inet->inet_sport; sin->sin_addr.s_addr = addr; - } -if (cgroup_bpf_enabled)BPF_CGROUP_RUN_SA_PROG_LOCK (sk, (struct sockaddr *)sin, - peer ?BPF_CGROUP_INET4_GETPEERNAME : -BPF_CGROUP_INET4_GETSOCKNAME , +BPF_CGROUP_INET4_GETSOCKNAME ,NULL ); + }memset (sin->sin_zero,0 ,sizeof (sin->sin_zero));return sizeof (*sin); }
net/ipv4/udp.c +3 -4
@@ -1124,7 +1124,7 @@ int udp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)rcu_read_unlock (); } -if (cgroup_bpf_enabled && !connected) { +if (cgroup_bpf_enabled (BPF_CGROUP_UDP4_SENDMSG ) && !connected) { err =BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK (sk, (struct sockaddr *)usin, &ipc.addr);if (err)
@@ -1858,9 +1858,8 @@ int udp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int noblock,memset (sin->sin_zero,0 ,sizeof (sin->sin_zero)); *addr_len =sizeof (*sin); -if (cgroup_bpf_enabled) -BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK (sk, - (struct sockaddr *)sin); +BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK (sk, + (struct sockaddr *)sin); }if (udp_sk (sk)->gro_enabled)
net/ipv6/af_inet6.c +5 -4
@@ -527,18 +527,19 @@ int inet6_getname(struct socket *sock, struct sockaddr *uaddr, sin->sin6_addr = sk->sk_v6_daddr;if (np->sndflow) sin->sin6_flowinfo = np->flow_label; +BPF_CGROUP_RUN_SA_PROG_LOCK (sk, (struct sockaddr *)sin, +BPF_CGROUP_INET6_GETPEERNAME , +NULL ); }else {if (ipv6_addr_any (&sk->sk_v6_rcv_saddr)) sin->sin6_addr = np->saddr;else sin->sin6_addr = sk->sk_v6_rcv_saddr; sin->sin6_port = inet->inet_sport; - } -if (cgroup_bpf_enabled)BPF_CGROUP_RUN_SA_PROG_LOCK (sk, (struct sockaddr *)sin, - peer ?BPF_CGROUP_INET6_GETPEERNAME : -BPF_CGROUP_INET6_GETSOCKNAME , +BPF_CGROUP_INET6_GETSOCKNAME ,NULL ); + } sin->sin6_scope_id =ipv6_iface_scope_id (&sin->sin6_addr, sk->sk_bound_dev_if);return sizeof (*sin);
net/ipv6/udp.c +3 -4
@@ -409,9 +409,8 @@ int udpv6_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, } *addr_len =sizeof (*sin6); -if (cgroup_bpf_enabled) -BPF_CGROUP_RUN_PROG_UDP6_RECVMSG_LOCK (sk, - (struct sockaddr *)sin6); +BPF_CGROUP_RUN_PROG_UDP6_RECVMSG_LOCK (sk, + (struct sockaddr *)sin6); }if (udp_sk (sk)->gro_enabled)
@@ -1462,7 +1461,7 @@ int udpv6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) fl6.saddr = np->saddr; fl6.fl6_sport = inet->inet_sport; -if (cgroup_bpf_enabled && !connected) { +if (cgroup_bpf_enabled (BPF_CGROUP_UDP6_SENDMSG ) && !connected) { err =BPF_CGROUP_RUN_PROG_UDP6_SENDMSG_LOCK (sk, (struct sockaddr *)sin6, &fl6.saddr);if (err)
Signature · 1 line
2.29.2.729.g45daf8777d-goog