bpf: performance improvements
Subsystems (5)
v2 cover bpf: performance improvements
v1->v2: dropped redundant iff_up check in patch 2
At plumbers we discussed different options on how to get rid of skb_clone from bpf_clone_redirect(), the patch 2 implements the best option. Patch 1 adds 'integrated exts' to cls_bpf to improve performance by combining simple actions into bpf classifier.
Alexei Starovoitov (1):
bpf: add bpf_redirect() helper
Daniel Borkmann (1):
cls_bpf: introduce integrated actions
include/net/sch_generic.h | 3 ++- include/uapi/linux/bpf.h | 9 +++++++ include/uapi/linux/pkt_cls.h | 4 +++ net/core/dev.c | 8 ++++++ net/core/filter.c | 58 +++++++++++++++++++++++++++++++++++++++ net/sched/act_bpf.c | 1 + net/sched/cls_bpf.c | 61 ++++++++++++++++++++++++++++++++++-------- samples/bpf/bpf_helpers.h | 4 +++ samples/bpf/tcbpf1_kern.c | 24 ++++++++++++++++- 9 files changed, 159 insertions(+), 13 deletions(-)
Signature · 1 line
1.7.9.5
PATCH v2 1/2 cls_bpf: introduce integrated actions
From: Daniel Borkmann <[email protected]>
Often cls_bpf classifier is used with single action drop attached. Optimize this use case and let cls_bpf return both classid and action. For backwards compatibility reasons enable this feature under TCA_BPF_FLAG_ACT_DIRECT flag.
Then more interesting programs like the following are easier to write:
int cls_bpf_prog(struct __sk_buff *skb)
{
/* classify arp, ip, ipv6 into different traffic classes
* and drop all other packets
*/
switch (skb->protocol) {
case htons(ETH_P_ARP):
skb->tc_classid = 1;
break;
case htons(ETH_P_IP):
skb->tc_classid = 2;
break;
case htons(ETH_P_IPV6):
skb->tc_classid = 3;
break;
default:
return TC_ACT_SHOT;}
return TC_ACT_OK;
}
Joint work with Daniel Borkmann.
---v1->v2: no changes
include/net/sch_generic.h | 2 +- include/uapi/linux/bpf.h | 1 + include/uapi/linux/pkt_cls.h | 3 +++ net/core/filter.c | 14 ++++++++++ net/sched/cls_bpf.c | 60 ++++++++++++++++++++++++++++++++++-------- 5 files changed, 68 insertions(+), 12 deletions(-)
include/net/sch_generic.h +1 -1
@@ -251,7 +251,7 @@ struct tcf_proto {struct qdisc_skb_cb {unsigned int pkt_len;u16 slave_dev_queue_mapping; -u16 _pad; +u16 tc_classid;#define QDISC_CB_PRIV_LEN 20unsigned char data[QDISC_CB_PRIV_LEN ]; };
include/uapi/linux/bpf.h +1 -0
@@ -293,6 +293,7 @@ struct __sk_buff {__u32 tc_index;__u32 cb[5 ];__u32 hash; +__u32 tc_classid; };struct bpf_tunnel_key {
include/uapi/linux/pkt_cls.h +3 -0
@@ -373,6 +373,8 @@ enum {/* BPF classifier */ +#define TCA_BPF_FLAG_ACT_DIRECT (1 << 0) +enum {TCA_BPF_UNSPEC ,TCA_BPF_ACT ,
@@ -382,6 +384,7 @@ enum {TCA_BPF_OPS ,TCA_BPF_FD ,TCA_BPF_NAME , +TCA_BPF_FLAGS , __TCA_BPF_MAX, };
net/core/filter.c +14 -0
@@ -1632,6 +1632,9 @@ static bool __is_valid_access(int off, int size, enum bpf_access_type type)static bool sk_filter_is_valid_access (int off,int size,enum bpf_access_type type) { +if (off == offsetof(struct __sk_buff , tc_classid)) +return false; +if (type ==BPF_WRITE ) {switch (off) { case offsetof(struct __sk_buff, cb[0]) ...
@@ -1648,6 +1651,9 @@ static bool sk_filter_is_valid_access(int off, int size,static bool tc_cls_act_is_valid_access (int off,int size,enum bpf_access_type type) { +if (off == offsetof(struct __sk_buff , tc_classid)) +return type ==BPF_WRITE ? true : false; +if (type ==BPF_WRITE ) {switch (off) {case offsetof(struct __sk_buff , mark):
@@ -1760,6 +1766,14 @@ static u32 bpf_net_convert_ctx_access(enum bpf_access_type type, int dst_reg, *insn++ =BPF_LDX_MEM (BPF_W , dst_reg, src_reg, ctx_off);break ; +case offsetof(struct __sk_buff , tc_classid): + ctx_off -= offsetof(struct __sk_buff , tc_classid); + ctx_off += offsetof(struct sk_buff , cb); + ctx_off += offsetof(struct qdisc_skb_cb , tc_classid); +WARN_ON (type !=BPF_WRITE ); + *insn++ =BPF_STX_MEM (BPF_H , dst_reg, src_reg, ctx_off); +break ; +case offsetof(struct __sk_buff , tc_index):#ifdef CONFIG_NET_SCHED BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, tc_index) != 2);
net/sched/cls_bpf.c +49 -11
128 lines not on this page · raw message
Signature · 1 line
1.7.9.5
Jamal Hadi Salim
maintainerRe: [PATCH v2 net-next 1/2] cls_bpf: introduce integrated actions
Quoted 7 lines · Alexei Starovoitov
From: Daniel Borkmann <[email protected]>
Often
cls_bpfclassifier is used with single action drop attached. Optimize this use case and letcls_bpfreturn both classid and action. For backwards compatibility reasons enable this feature underTCA_BPF_FLAG_ACT_DIRECTflag.
This is going off in a different direction really. You are replicating the infrastructure inside bpf.
cheers,
jamal
Daniel Borkmann
maintainerRe: [PATCH v2 net-next 1/2] cls_bpf: introduce integrated actions
Hi Jamal,
Quoted 11 lines · Jamal Hadi Salim
Quoted 7 lines · Alexei Starovoitov
From: Daniel Borkmann <[email protected]>
Often
cls_bpfclassifier is used with single action drop attached. Optimize this use case and letcls_bpfreturn both classid and action. For backwards compatibility reasons enable this feature underTCA_BPF_FLAG_ACT_DIRECTflag.This is going off in a different direction really. You are replicating the infrastructure inside bpf.
Hmm, I don't really agree. With cls_bpf you have non-linear classifications as opposed to walking a chain of classifiers:
worst case, I have to walk through N classifiers just to find out that the last one matches that I need to drop - this doesn't scale at all. Given that we can make this decision right here, we can use this fact and have simple return codes provided as well. It only supplements non-linear classification that was from the very beginning of cls_bpf a core part of it.
Thanks,
Daniel
Re: [PATCH v2 net-next 1/2] cls_bpf: introduce integrated actions
Quoted 15 lines · Daniel Borkmann
Hi Jamal,
Quoted 11 lines · Jamal Hadi Salim
Quoted 7 lines · Alexei Starovoitov
From: Daniel Borkmann <[email protected]>
Often
cls_bpfclassifier is used with single action drop attached. Optimize this use case and letcls_bpfreturn both classid and action. For backwards compatibility reasons enable this feature underTCA_BPF_FLAG_ACT_DIRECTflag.This is going off in a different direction really. You are replicating the infrastructure inside bpf.
Quoted 8 lines
Hmm, I don't really agree. With
cls_bpfyou have non-linear classifications as opposed to walking a chain of classifiers:
worst case, I have to walk through N classifiers just to find out that the last one matches that I need to drop - this doesn't scale at all. Given that we can make this decision right here, we can use this fact and have simple return codes provided as well. It only supplements non-linear classification that was from the very beginning ofcls_bpfa core part of it.
I don't see the replication either. May be the commit log was misread as bpf program now executes the actions and bypasses tcf_exts_exec() ? Well, that may be interesting idea for the future, but that's not what the patch is doing. With this patch cls_bpf can return single integer like TC_ACT_SHOT/TC_ACT_OK that gact/act_bpf can already do as an _optimization_ to avoid extra hops. To do full-fledged action chaining the tcf_exts_exec() is used.
Jamal Hadi Salim
maintainerRe: [PATCH v2 net-next 1/2] cls_bpf: introduce integrated actions
Quoted 3 lines · Alexei Starovoitov
misread as bpf program now executes the actions and bypasses
tcf_exts_exec()? Well, that may be interesting idea for the future,
And above is precisely why i raised the concern. You are already bypassing tcf_exts_exec with that patch. It is a big jump. It is kind of hard to continue the discussion because i notice Dave just took in the patches.
Please dont go the above path of fully fledged bypass. The architecture is about small tools that come together to provide complex processing. ebpf may be the best classifier today - but by no means the only one or guaranteed that nothing better will exist for speficic use cases.
If there is something in the core that needs improvement for the benefit of all, then lets do that. Example i find the classid metadata interesting but flinching at ACT_REDIRECT.
cheers,
jamal
Daniel Borkmann
maintainerRe: [PATCH v2 net-next 1/2] cls_bpf: introduce integrated actions
Hi Jamal,
On 09/18/2015 02:13 PM, Jamal Hadi Salim wrote:
...
Quoted 4 lines
The architecture is about small tools that come together to provide complex processing. ebpf may be the best classifier today - but by no means the only one or guaranteed that nothing better will exist for speficic use cases.
Yes, I agree, I think nobody claimed that, and it also doesn't limit anyone from their choices of configuration possibilities one can do nowadays.
Quoted 2 lines
If there is something in the core that needs improvement for the benefit of all, then lets do that.
Yeah, I think i.e. since the last couple of months that is already happening from various people.
Thanks,
Daniel
Jamal Hadi Salim
maintainerRe: [PATCH v2 net-next 1/2] cls_bpf: introduce integrated actions
In reply to Daniel BorkmannHi Daniel,
Quoted 2 lines · Daniel Borkmann
Hmm, I don't really agree. With
cls_bpfyou have non-linear classifications as opposed to walking a chain of classifiers:
A chain of classifiers is a better description today (non-linear would be an appropriate description before cls_bpf ;->).
Quoted 3 lines
worst case, I have to walk through N classifiers just to find out that the last one matches that I need to drop - this doesn't scale at all.
The scaling reason with that posted example is not
a strong one. You can get good performance with any classifier for that policy description.
F.E with Alexei's second best classifier:->:
tc filter add dev eth0 parent ffff: protocol arp prio 1 u32\
match all ..
tc filter add dev eth0 parent ffff: protocol ip prio 1 u32\
...
But I do get the gist of your arguement otherwise and some short circuits are ok as you had earlier.
Quoted 3 lines
Given that we can make this decision right here,
we can use this fact and have simple return codes provided as well.
I think it makes sense for the simple case.
But you have every other opcode in there, not just basic accept/drop. I am worried this is leading towards an enclave of bpf do-everything.
cheers,
jamal
PATCH v2 2/2 bpf: add bpf_redirect() helper
In reply to Alexei StarovoitovExisting bpf_clone_redirect() helper clones skb before redirecting it to RX or TX of destination netdev.
Introduce bpf_redirect() helper that does that without cloning.
Benchmarked with two hosts using 10G ixgbe NICs.
One host is doing line rate pktgen.
Another host is configured as:
$ tc qdisc add dev $dev ingress $ tc filter add dev $dev root pref 10 u32 match u32 0 0 flowid 1:2 \ action bpf run object-file tcbpf1_kern.o section clone_redirect_xmit drop so it receives the packet on $dev and immediately xmits it on $dev + 1 The section 'clone_redirect_xmit' in tcbpf1_kern.o file has the program that does bpf_clone_redirect() and performance is 2.0 Mpps
$ tc filter add dev $dev root pref 10 u32 match u32 0 0 flowid 1:2 \ action bpf run object-file tcbpf1_kern.o section redirect_xmit drop which is using bpf_redirect() - 2.4 Mpps
and using cls_bpf with integrated actions as:
$ tc filter add dev $dev root pref 10 \ bpf run object-file tcbpf1_kern.o section redirect_xmit integ_act classid 1 performance is 2.5 Mpps
To summarize:
u32+act_bpf using clone_redirect - 2.0 Mpps
u32+act_bpf using redirect - 2.4 Mppscls_bpf using redirect - 2.5 Mpps
For comparison linux bridge in this setup is doing 2.1 Mpps and ixgbe rx + drop in ip_rcv - 7.8 Mpps
---
This approach is using per_cpu scratch area to store ifindex and flags. The other alternatives discussed at plumbers are slower and more intrusive. v1->v2: dropped redundant iff_up check
include/net/sch_generic.h | 1 + include/uapi/linux/bpf.h | 8 ++++++++ include/uapi/linux/pkt_cls.h | 1 + net/core/dev.c | 8 ++++++++ net/core/filter.c | 44 ++++++++++++++++++++++++++++++++++++++++++ net/sched/act_bpf.c | 1 + net/sched/cls_bpf.c | 1 + samples/bpf/bpf_helpers.h | 4 ++++ samples/bpf/tcbpf1_kern.c | 24 ++++++++++++++++++++++- 9 files changed, 91 insertions(+), 1 deletion(-)
include/net/sch_generic.h +1 -0
@@ -402,6 +402,7 @@ void __qdisc_calculate_pkt_len(struct sk_buff *skb, const struct qdisc_size_table *stab);bool tcf_destroy (struct tcf_proto *tp,bool force);void tcf_destroy_chain (struct tcf_proto __rcu **fl); +int skb_do_redirect (struct sk_buff *);/* Reset all TX qdiscs greater then index of a device. */ static inline void qdisc_reset_all_tx_gt (struct net_device *dev,unsigned int i)
include/uapi/linux/bpf.h +8 -0
@@ -272,6 +272,14 @@ enum bpf_func_id { BPF_FUNC_skb_get_tunnel_key, BPF_FUNC_skb_set_tunnel_key, BPF_FUNC_perf_event_read,/* u64 bpf_perf_event_read(&map, index) */ +/** +* bpf_redirect(ifindex, flags) - redirect to another netdev +* @ifindex: ifindex of the net device +* @flags: bit 0 - if set, redirect to ingress instead of egress +* other bits - reserved +* Return: TC_ACT_REDIRECT +*/ + BPF_FUNC_redirect, __BPF_FUNC_MAX_ID, };
include/uapi/linux/pkt_cls.h +1 -0
@@ -87,6 +87,7 @@ enum {#define TC_ACT_STOLEN 4#define TC_ACT_QUEUED 5#define TC_ACT_REPEAT 6 +#define TC_ACT_REDIRECT 7#define TC_ACT_JUMP 0x10000000/* Action type identifiers*/
net/core/dev.c +8 -0
@@ -3668,6 +3668,14 @@ static inline struct sk_buff *handle_ing(struct sk_buff *skb,case TC_ACT_QUEUED :kfree_skb (skb);return NULL ; +case TC_ACT_REDIRECT : +/* skb_mac_header check was done by cls/act_bpf, so +* we can safely push the L2 header back before +* redirecting to another netdev +*/ +__skb_push (skb, skb->mac_len); +skb_do_redirect (skb); +return NULL ;default :break ; }
net/core/filter.c +44 -0
56 lines not on this page · raw message
net/sched/act_bpf.c +1 -0
@@ -72,6 +72,7 @@ static int tcf_bpf(struct sk_buff *skb, const struct tc_action *act,case TC_ACT_PIPE :case TC_ACT_RECLASSIFY :case TC_ACT_OK : +case TC_ACT_REDIRECT : action = filter_res;break ;case TC_ACT_SHOT :
net/sched/cls_bpf.c +1 -0
@@ -70,6 +70,7 @@ static int cls_bpf_exec_opcode(int code)case TC_ACT_PIPE :case TC_ACT_STOLEN :case TC_ACT_QUEUED : +case TC_ACT_REDIRECT :case TC_ACT_UNSPEC :return code;default :
samples/bpf/bpf_helpers.h +4 -0
@@ -33,6 +33,10 @@ static int (*bpf_get_current_comm)(void *buf, int buf_size) = (void *) BPF_FUNC_get_current_comm;static int (*bpf_perf_event_read)(void *map,int index) = (void *) BPF_FUNC_perf_event_read; +static int (*bpf_clone_redirect)(void *ctx,int ifindex,int flags) = + (void *) BPF_FUNC_clone_redirect; +static int (*bpf_redirect)(int ifindex,int flags) = + (void *) BPF_FUNC_redirect;/* llvm builtin functions that eBPF C program may use to * emit BPF_LD_ABS and BPF_LD_IND instructions
samples/bpf/tcbpf1_kern.c +23 -1
@@ -5,7 +5,7 @@#include <uapi/linux/in.h> #include <uapi/linux/tcp.h> #include <uapi/linux/filter.h> - +#include <uapi/linux/pkt_cls.h> #include "bpf_helpers.h" /* compiler workaround */
@@ -64,4 +64,26 @@ int bpf_prog1(struct __sk_buff *skb)return 0 ; } +SEC("redirect_xmit") +int _redirect_xmit (struct __sk_buff *skb) +{ +return bpf_redirect (skb->ifindex +1 ,0 ); +} +SEC("redirect_recv") +int _redirect_recv (struct __sk_buff *skb) +{ +return bpf_redirect (skb->ifindex +1 ,1 ); +} +SEC("clone_redirect_xmit") +int _clone_redirect_xmit (struct __sk_buff *skb) +{ +bpf_clone_redirect (skb, skb->ifindex +1 ,0 ); +return TC_ACT_SHOT ; +} +SEC("clone_redirect_recv") +int _clone_redirect_recv (struct __sk_buff *skb) +{ +bpf_clone_redirect (skb, skb->ifindex +1 ,1 ); +return TC_ACT_SHOT ; +} char _license[] SEC("license") = "GPL";
Signature · 1 line
1.7.9.5
Re: [PATCH v2 net-next 2/2] bpf: add bpf_redirect() helper
Quoted 38 lines · Alexei Starovoitov
Existing
bpf_clone_redirect()helper clones skb before redirecting it to RX or TX of destination netdev.
Introducebpf_redirect()helper that does that without cloning.Benchmarked with two hosts using 10G ixgbe NICs.
One host is doing line rate pktgen.
Another host is configured as:$ tc qdisc add dev $dev ingress $ tc filter add dev $dev root pref 10 u32 match u32 0 0 flowid 1:2 \ action bpf run object-file tcbpf1_kern.o section clone_redirect_xmit drop so it receives the packet on $dev and immediately xmits it on $dev + 1 The section 'clone_redirect_xmit' in tcbpf1_kern.o file has the program that does bpf_clone_redirect() and performance is 2.0 Mpps$ tc filter add dev $dev root pref 10 u32 match u32 0 0 flowid 1:2 \ action bpf run object-file tcbpf1_kern.o section redirect_xmit drop which is using bpf_redirect() - 2.4 Mppsand using
cls_bpfwith integrated actions as:$ tc filter add dev $dev root pref 10 \ bpf run object-file tcbpf1_kern.o section redirect_xmit integ_act classid 1 performance is 2.5 MppsTo summarize:
u32+act_bpf usingclone_redirect- 2.0 Mpps
u32+act_bpf using redirect - 2.4 Mppscls_bpfusing redirect - 2.5 MppsFor comparison linux bridge in this setup is doing 2.1 Mpps and ixgbe rx + drop in
ip_rcv- 7.8 MppsSigned-off-by Alexei Starovoitov <[email protected]>Acked-by Daniel Borkmann <[email protected]>---
This approach is usingper_cpuscratch area to store ifindex and flags. The other alternatives discussed at plumbers are slower and more intrusive.v1->v2: dropped redundantiff_upcheck
Quoted 11 lines
include/net/sch_generic.h | 1 + include/uapi/linux/bpf.h | 8 ++++++++ include/uapi/linux/pkt_cls.h | 1 + net/core/dev.c | 8 ++++++++ net/core/filter.c | 44 ++++++++++++++++++++++++++++++++++++++++++ net/sched/act_bpf.c | 1 + net/sched/cls_bpf.c | 1 + samples/bpf/bpf_helpers.h | 4 ++++ samples/bpf/tcbpf1_kern.c | 24 ++++++++++++++++++++++- 9 files changed, 91 insertions(+), 1 deletion(-)
David Miller
maintainerFrom: Alexei Starovoitov <[email protected]>
Date: Tue, 15 Sep 2015 23:05:41 -0700
Quoted 6 lines
v1->v2: dropped redundantiff_upcheck in patch 2At plumbers we discussed different options on how to get rid of
skb_clonefrombpf_clone_redirect(), the patch 2 implements the best option. Patch 1 adds 'integrated exts' tocls_bpfto improve performance by combining simple actions into bpf classifier.
Series applied, thanks.