AF_XDP clean up/perf improvements
Subsystems (5)
cover AF_XDP clean up/perf improvements
This series has some clean up/performance improvements for XDP sockets.
The first two patches are cleanups for the AF_XDP core, and the restructure actually give a little performance boost.
The last patch adds support for selecting AF_XDP BPF program, based on what the running kernel supports.
The patches were earlier part of the bigger "bpf_redirect_xsk()" series [1]. I pulled out the non-controversial parts into this series.
Thanks to Maciej and Magnus for the internal review/comments!
Thanks to Toke, Alexei, and Andrii for the "auto-detection" help; Instead of basing it on kernel version, a run-time test is performed. Note that I did not add the probing support to libbpf.c, where the other probes reside. Instead it's in xsk.c. The reason for that is that AF_XDP will be moved out from libbpf post-1.0, to libxdp.
Thanks,
Björn
[1] https://lore.kernel.org/bpf/[email protected]/
Björn Töpel (3):
xsk: remove explicit_free parameter from __xsk_rcv()
xsk: fold xp_assign_dev and __xp_assign_dev
libbpf, xsk: select AF_XDP BPF program based on kernel version
net/xdp/xsk.c | 47 +++++++++++++++-------- net/xdp/xsk_buff_pool.c | 12 ++---- tools/lib/bpf/xsk.c | 82 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 113 insertions(+), 28 deletions(-)
base-commit: 443edcefb8213155c0da22c4a999f4a49858fa39
Signature · 1 line
2.27.0
PATCH 1/3 xsk: remove explicit_free parameter from __xsk_rcv()
From: Björn Töpel <[email protected]>
The explicit_free parameter of the __xsk_rcv() function was used to mark whether the call was via the generic XDP or the native XDP path. Instead of clutter the code with if-statements and "true/false" parameters which are hard to understand, simply move the explicit free to the __xsk_map_redirect() which is always called from the native XDP path.
--- net/xdp/xsk.c | 47 +++++++++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 16 deletions(-)
net/xdp/xsk.c +31 -16
@@ -184,12 +184,13 @@ static void xsk_copy_xdp(struct xdp_buff *to, struct xdp_buff *from, u32 len)memcpy (to_buf, from_buf, len + metalen); } -static int __xsk_rcv (struct xdp_sock *xs,struct xdp_buff *xdp,u32 len, -bool explicit_free) +static int __xsk_rcv (struct xdp_sock *xs,struct xdp_buff *xdp) {struct xdp_buff *xsk_xdp;int err; +u32 len; + len = xdp->data_end - xdp->data;if (len >xsk_pool_get_rx_frame_size (xs->pool)) { xs->rx_dropped++;return -ENOSPC ;
@@ -207,8 +208,6 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len,xsk_buff_free (xsk_xdp);return err; } -if (explicit_free) -xdp_return_buff (xdp);return 0 ; }
@@ -230,11 +229,8 @@ static bool xsk_is_bound(struct xdp_sock *xs)return false; } -static int xsk_rcv (struct xdp_sock *xs,struct xdp_buff *xdp, -bool explicit_free) +static int xsk_rcv_check (struct xdp_sock *xs,struct xdp_buff *xdp) { -u32 len; -if (!xsk_is_bound (xs))return -EINVAL ;
@@ -242,11 +238,7 @@ static int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp,return -EINVAL ;sk_mark_napi_id_once_xdp (&xs->sk, xdp); - len = xdp->data_end - xdp->data; - -return xdp->rxq->mem.type ==MEM_TYPE_XSK_BUFF_POOL ? -__xsk_rcv_zc (xs, xdp, len) : -__xsk_rcv (xs, xdp, len, explicit_free); +return 0 ; }static void xsk_flush (struct xdp_sock *xs)
@@ -261,18 +253,41 @@ int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)int err;spin_lock_bh (&xs->rx_lock); - err =xsk_rcv (xs, xdp, false); -xsk_flush (xs); + err =xsk_rcv_check (xs, xdp); +if (!err) { + err =__xsk_rcv (xs, xdp); +xsk_flush (xs); + }spin_unlock_bh (&xs->rx_lock);return err; } +static int xsk_rcv (struct xdp_sock *xs,struct xdp_buff *xdp) +{ +int err; +u32 len; + + err =xsk_rcv_check (xs, xdp); +if (err) +return err; + +if (xdp->rxq->mem.type ==MEM_TYPE_XSK_BUFF_POOL ) { + len = xdp->data_end - xdp->data; +return __xsk_rcv_zc (xs, xdp, len); + } + + err =__xsk_rcv (xs, xdp); +if (!err) +xdp_return_buff (xdp); +return err; +} +int __xsk_map_redirect (struct xdp_sock *xs,struct xdp_buff *xdp) {struct list_head *flush_list =this_cpu_ptr (&xskmap_flush_list);int err; - err =xsk_rcv (xs, xdp, true); + err =xsk_rcv (xs, xdp);if (err)return err;
Signature · 1 line
2.27.0
PATCH 2/3 xsk: fold xp_assign_dev and __xp_assign_dev
In reply to Björn TöpelFrom: Björn Töpel <[email protected]>
Fold xp_assign_dev and __xp_assign_dev. The former directly calls the latter.
--- net/xdp/xsk_buff_pool.c | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-)
net/xdp/xsk_buff_pool.c +3 -9
@@ -119,8 +119,8 @@ static void xp_disable_drv_zc(struct xsk_buff_pool *pool) } } -static int __xp_assign_dev (struct xsk_buff_pool *pool, -struct net_device *netdev,u16 queue_id,u16 flags) +int xp_assign_dev (struct xsk_buff_pool *pool, +struct net_device *netdev,u16 queue_id,u16 flags) {bool force_zc, force_copy;struct netdev_bpf bpf;
@@ -191,12 +191,6 @@ static int __xp_assign_dev(struct xsk_buff_pool *pool,return err; } -int xp_assign_dev (struct xsk_buff_pool *pool,struct net_device *dev, -u16 queue_id,u16 flags) -{ -return __xp_assign_dev (pool, dev, queue_id, flags); -} -int xp_assign_dev_shared (struct xsk_buff_pool *pool,struct xdp_umem *umem,struct net_device *dev,u16 queue_id) {
@@ -210,7 +204,7 @@ int xp_assign_dev_shared(struct xsk_buff_pool *pool, struct xdp_umem *umem,if (pool->uses_need_wakeup) flags |=XDP_USE_NEED_WAKEUP ; -return __xp_assign_dev (pool, dev, queue_id, flags); +return xp_assign_dev (pool, dev, queue_id, flags); }void xp_clear_dev (struct xsk_buff_pool *pool)
Signature · 1 line
2.27.0
PATCH 3/3 libbpf, xsk: select AF_XDP BPF program based on kernel version
In reply to Björn TöpelFrom: Björn Töpel <[email protected]>
Add detection for kernel version, and adapt the BPF program based on kernel support. This way, users will get the best possible performance from the BPF program.
--- tools/lib/bpf/xsk.c | 82 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 79 insertions(+), 3 deletions(-)
tools/lib/bpf/xsk.c +79 -3
@@ -46,6 +46,11 @@#define PF_XDP AF_XDP#endif +enum xsk_prog { +XSK_PROG_FALLBACK , +XSK_PROG_REDIRECT_FLAGS , +}; +struct xsk_umem {struct xsk_ring_prod *fill_save;struct xsk_ring_cons *comp_save;
@@ -351,6 +356,55 @@ int xsk_umem__create_v0_0_2(struct xsk_umem **umem_ptr, void *umem_area, COMPAT_VERSION(xsk_umem__create_v0_0_2, xsk_umem__create, LIBBPF_0.0.2) DEFAULT_VERSION(xsk_umem__create_v0_0_4, xsk_umem__create, LIBBPF_0.0.4) + +static enum xsk_prog get_xsk_prog(void) +{ +enum xsk_prog detected =XSK_PROG_FALLBACK ; +struct bpf_load_program_attr prog_attr; +struct bpf_create_map_attr map_attr; +__u32 size_out, retval, duration; +char data_in =0 , data_out; +struct bpf_insn insns[] = { +BPF_LD_MAP_FD (BPF_REG_1 ,0 ), +BPF_MOV64_IMM (BPF_REG_2 ,0 ), +BPF_MOV64_IMM (BPF_REG_3 ,XDP_PASS ), +BPF_EMIT_CALL (BPF_FUNC_redirect_map), +BPF_EXIT_INSN (), + }; +int prog_fd, map_fd, ret; + +memset (&map_attr,0 ,sizeof (map_attr)); + map_attr.map_type =BPF_MAP_TYPE_XSKMAP ; + map_attr.key_size =sizeof (int ); + map_attr.value_size =sizeof (int ); + map_attr.max_entries =1 ; + + map_fd =bpf_create_map_xattr (&map_attr); +if (map_fd <0 ) +return detected; + + insns[0 ].imm = map_fd; + +memset (&prog_attr,0 ,sizeof (prog_attr)); + prog_attr.prog_type =BPF_PROG_TYPE_XDP ; + prog_attr.insns = insns; + prog_attr.insns_cnt =ARRAY_SIZE (insns); + prog_attr.license ="GPL" ; + + prog_fd =bpf_load_program_xattr (&prog_attr,NULL ,0 ); +if (prog_fd <0 ) { +close (map_fd); +return detected; + } + + ret =bpf_prog_test_run (prog_fd,0 , &data_in,1 , &data_out, &size_out, &retval, &duration); +if (!ret && retval ==XDP_PASS ) + detected =XSK_PROG_REDIRECT_FLAGS ; +close (prog_fd); +close (map_fd); +return detected; +} + static int xsk_load_xdp_prog(struct xsk_socket *xsk) { static const int log_buf_size = 16 * 1024;
@@ -358,7 +412,7 @@ static int xsk_load_xdp_prog(struct xsk_socket *xsk)char log_buf[log_buf_size];int err, prog_fd; -/* This is the C-program: +/* This is the fallback C-program: * SEC("xdp_sock") int xdp_sock_prog(struct xdp_md *ctx) * { * int ret, index = ctx->rx_queue_index;
@@ -414,9 +468,31 @@ static int xsk_load_xdp_prog(struct xsk_socket *xsk)/* The jumps are to this instruction */ BPF_EXIT_INSN(), }; -size_t insns_cnt =sizeof (prog) /sizeof (struct bpf_insn ); - prog_fd =bpf_load_program (BPF_PROG_TYPE_XDP , prog, insns_cnt, +/* This is the post-5.3 kernel C-program: +* SEC("xdp_sock") int xdp_sock_prog(struct xdp_md *ctx) +* { +* return bpf_redirect_map(&xsks_map, ctx->rx_queue_index, XDP_PASS); +* } +*/ +struct bpf_insn prog_redirect_flags[] = { +/* r2 = *(u32 *)(r1 + 16) */ +BPF_LDX_MEM (BPF_W ,BPF_REG_2 ,BPF_REG_1 ,16 ), +/* r1 = xskmap[] */ +BPF_LD_MAP_FD (BPF_REG_1 , ctx->xsks_map_fd), +/* r3 = XDP_PASS */ +BPF_MOV64_IMM (BPF_REG_3 ,2 ), +/* call bpf_redirect_map */ +BPF_EMIT_CALL (BPF_FUNC_redirect_map), +BPF_EXIT_INSN (), + }; +size_t insns_cnt[] = {sizeof (prog) /sizeof (struct bpf_insn ), +sizeof (prog_redirect_flags) /sizeof (struct bpf_insn ), + }; +struct bpf_insn *progs[] = {prog, prog_redirect_flags}; +enum xsk_prog option =get_xsk_prog (); + + prog_fd =bpf_load_program (BPF_PROG_TYPE_XDP , progs[option], insns_cnt[option],"LGPL-2.1 or BSD-2-Clause" ,0 , log_buf, log_buf_size);if (prog_fd <0 ) {
Signature · 1 line
2.27.0
Daniel Borkmann
maintainerRe: [PATCH bpf-next 3/3] libbpf, xsk: select AF_XDP BPF program based on kernel version
Quoted 31 lines · Björn Töpel
From: Björn Töpel <[email protected]>
Add detection for kernel version, and adapt the BPF program based on kernel support. This way, users will get the best possible performance from the BPF program.
Reviewed-by Maciej Fijalkowski <[email protected]>Signed-off-by Björn Töpel <[email protected]>Signed-off-by Marek Majtyka <[email protected]>--- tools/lib/bpf/xsk.c | 82 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 79 insertions(+), 3 deletions(-)tools/lib/bpf/xsk.c +5
-0@@ -46,6 +46,11 @@#define PF_XDP AF_XDP#endif +enum xsk_prog { +XSK_PROG_FALLBACK , +XSK_PROG_REDIRECT_FLAGS , +}; +struct xsk_umem {struct xsk_ring_prod *fill_save;struct xsk_ring_cons *comp_save;
tools/lib/bpf/xsk.c int xsk_umem__create_v0_0_2(struct xsk_umem **umem_ptr, void *umem_area, · line 359
Quoted 4 lines
tools/lib/bpf/xsk.c +1
-0@@ -351,6 +356,55 @@ int xsk_umem__create_v0_0_2(struct xsk_umem **umem_ptr, void *umem_area, COMPAT_VERSION(xsk_umem__create_v0_0_2, xsk_umem__create, LIBBPF_0.0.2) DEFAULT_VERSION(xsk_umem__create_v0_0_4, xsk_umem__create, LIBBPF_0.0.4) +
Fyi, removed this extra newline when I applied the series, thanks!
Quoted 13 lines
tools/lib/bpf/xsk.c +13
-0+static enum xsk_prog get_xsk_prog (void ) +{ +enum xsk_prog detected =XSK_PROG_FALLBACK ; +struct bpf_load_program_attr prog_attr; +struct bpf_create_map_attr map_attr; +__u32 size_out, retval, duration; +char data_in =0 , data_out; +struct bpf_insn insns[] = { +BPF_LD_MAP_FD (BPF_REG_1 ,0 ), +BPF_MOV64_IMM (BPF_REG_2 ,0 ), +BPF_MOV64_IMM (BPF_REG_3 ,XDP_PASS ), +BPF_EMIT_CALL (BPF_FUNC_redirect_map), +BPF_EXIT_INSN (),
[...]
Re: [PATCH bpf-next 3/3] libbpf, xsk: select AF_XDP BPF program based on kernel version
In reply to Björn TöpelQuoted 5 lines · On Fri, 22 Jan 2021 11:53:51 +0100 Björn Töpel <[email protected]> wrote:
From: Björn Töpel <[email protected]>
Add detection for kernel version, and adapt the BPF program based on kernel support. This way, users will get the best possible performance from the BPF program.
You say "detection for kernel version", but doesn't the code detect the feature rather than kernel version ?
If so, please update the description.
Quoted 123 lines
Reviewed-by Maciej Fijalkowski <[email protected]>Signed-off-by Björn Töpel <[email protected]>Signed-off-by Marek Majtyka <[email protected]>--- tools/lib/bpf/xsk.c | 82 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 79 insertions(+), 3 deletions(-)tools/lib/bpf/xsk.c +79
-3@@ -46,6 +46,11 @@#define PF_XDP AF_XDP#endif +enum xsk_prog { +XSK_PROG_FALLBACK , +XSK_PROG_REDIRECT_FLAGS , +}; +struct xsk_umem {struct xsk_ring_prod *fill_save;struct xsk_ring_cons *comp_save;@@ -351,6 +356,55 @@ int xsk_umem__create_v0_0_2(struct xsk_umem **umem_ptr, void *umem_area, COMPAT_VERSION(xsk_umem__create_v0_0_2, xsk_umem__create, LIBBPF_0.0.2) DEFAULT_VERSION(xsk_umem__create_v0_0_4, xsk_umem__create, LIBBPF_0.0.4) + +static enum xsk_prog get_xsk_prog(void) +{ +enum xsk_prog detected =XSK_PROG_FALLBACK ; +struct bpf_load_program_attr prog_attr; +struct bpf_create_map_attr map_attr; +__u32 size_out, retval, duration; +char data_in =0 , data_out; +struct bpf_insn insns[] = { +BPF_LD_MAP_FD (BPF_REG_1 ,0 ), +BPF_MOV64_IMM (BPF_REG_2 ,0 ), +BPF_MOV64_IMM (BPF_REG_3 ,XDP_PASS ), +BPF_EMIT_CALL (BPF_FUNC_redirect_map), +BPF_EXIT_INSN (), + }; +int prog_fd, map_fd, ret; + +memset (&map_attr,0 ,sizeof (map_attr)); + map_attr.map_type =BPF_MAP_TYPE_XSKMAP ; + map_attr.key_size =sizeof (int ); + map_attr.value_size =sizeof (int ); + map_attr.max_entries =1 ; + + map_fd =bpf_create_map_xattr (&map_attr); +if (map_fd <0 ) +return detected; + + insns[0 ].imm = map_fd; + +memset (&prog_attr,0 ,sizeof (prog_attr)); + prog_attr.prog_type =BPF_PROG_TYPE_XDP ; + prog_attr.insns = insns; + prog_attr.insns_cnt =ARRAY_SIZE (insns); + prog_attr.license ="GPL" ; + + prog_fd =bpf_load_program_xattr (&prog_attr,NULL ,0 ); +if (prog_fd <0 ) { +close (map_fd); +return detected; + } + + ret =bpf_prog_test_run (prog_fd,0 , &data_in,1 , &data_out, &size_out, &retval, &duration); +if (!ret && retval ==XDP_PASS ) + detected =XSK_PROG_REDIRECT_FLAGS ; +close (prog_fd); +close (map_fd); +return detected; +} + static int xsk_load_xdp_prog(struct xsk_socket *xsk) { static const int log_buf_size = 16 * 1024;@@ -358,7 +412,7 @@ static int xsk_load_xdp_prog(struct xsk_socket *xsk)char log_buf[log_buf_size];int err, prog_fd; -/* This is the C-program: +/* This is the fallback C-program: * SEC("xdp_sock") int xdp_sock_prog(struct xdp_md *ctx) * { * int ret, index = ctx->rx_queue_index; @@ -414,9 +468,31 @@ static int xsk_load_xdp_prog(struct xsk_socket *xsk)/* The jumps are to this instruction */ BPF_EXIT_INSN(), }; -size_t insns_cnt =sizeof (prog) /sizeof (struct bpf_insn ); - prog_fd =bpf_load_program (BPF_PROG_TYPE_XDP , prog, insns_cnt, +/* This is the post-5.3 kernel C-program: +* SEC("xdp_sock") int xdp_sock_prog(struct xdp_md *ctx) +* { +* return bpf_redirect_map(&xsks_map, ctx->rx_queue_index, XDP_PASS); +* } +*/ +struct bpf_insn prog_redirect_flags[] = { +/* r2 = *(u32 *)(r1 + 16) */ +BPF_LDX_MEM (BPF_W ,BPF_REG_2 ,BPF_REG_1 ,16 ), +/* r1 = xskmap[] */ +BPF_LD_MAP_FD (BPF_REG_1 , ctx->xsks_map_fd), +/* r3 = XDP_PASS */ +BPF_MOV64_IMM (BPF_REG_3 ,2 ), +/* call bpf_redirect_map */ +BPF_EMIT_CALL (BPF_FUNC_redirect_map), +BPF_EXIT_INSN (), + }; +size_t insns_cnt[] = {sizeof (prog) /sizeof (struct bpf_insn ), +sizeof (prog_redirect_flags) /sizeof (struct bpf_insn ), + }; +struct bpf_insn *progs[] = {prog, prog_redirect_flags}; +enum xsk_prog option =get_xsk_prog (); + + prog_fd =bpf_load_program (BPF_PROG_TYPE_XDP , progs[option], insns_cnt[option],"LGPL-2.1 or BSD-2-Clause" ,0 , log_buf, log_buf_size);if (prog_fd <0 ) {
Signature · 4 lines
Best regards, Jesper Dangaard Brouer MSc.CS, Principal Kernel Engineer at Red Hat LinkedIn: http://www.linkedin.com/in/brouer
Quoted 11 lines · Björn Töpel
This series has some clean up/performance improvements for XDP sockets.
The first two patches are cleanups for the
AF_XDPcore, and the restructure actually give a little performance boost.The last patch adds support for selecting
AF_XDPBPF program, based on what the running kernel supports.The patches were earlier part of the bigger "
bpf_redirect_xsk()" series [1]. I pulled out the non-controversial parts into this series.
What about the first patch from that series, refactoring the existing bpf_redirect_map() handling? I think that would be eligible for sending on its own as well :)
-Toke
Quoted 11 lines · Toke Høiland-Jørgensen
Quoted 9 lines · Björn Töpel
This series has some clean up/performance improvements for XDP sockets.
The first two patches are cleanups for the
AF_XDPcore, and the restructure actually give a little performance boost.The last patch adds support for selecting
AF_XDPBPF program, based on what the running kernel supports.
Quoted 7 lines
Quoted 2 lines
The patches were earlier part of the bigger "
bpf_redirect_xsk()" series [1]. I pulled out the non-controversial parts into this series.What about the first patch from that series, refactoring the existing
bpf_redirect_map()handling? I think that would be eligible for sending on its own as well :)
Yeah, I'm planning on doing that, but I figured I'd wait for Hangbin's work to go first.
Björn
Quoted 2 lines
-Toke
Quoted 15 lines · Björn Töpel
Quoted 14 lines · Toke Høiland-Jørgensen
Quoted 11 lines · Björn Töpel
This series has some clean up/performance improvements for XDP sockets.
The first two patches are cleanups for the
AF_XDPcore, and the restructure actually give a little performance boost.The last patch adds support for selecting
AF_XDPBPF program, based on what the running kernel supports.The patches were earlier part of the bigger "
bpf_redirect_xsk()" series [1]. I pulled out the non-controversial parts into this series.
Quoted 7 lines
Quoted 4 lines
What about the first patch from that series, refactoring the existing
bpf_redirect_map()handling? I think that would be eligible for sending on its own as well :)Yeah, I'm planning on doing that, but I figured I'd wait for Hangbin's work to go first.
Ah, right, good point; cool! :)
-Toke