bpf: remove bpf_load loader completely
Subsystems (5)
v3 cover bpf: remove bpf_load loader completely
Numerous refactoring that rewrites BPF programs written with bpf_load to use the libbpf loader was finally completed, resulting in BPF programs using bpf_load within the kernel being completely no longer present.
This patchset refactors remaining bpf programs with libbpf and completely removes bpf_load, an outdated bpf loader that is difficult to keep up with the latest kernel BPF and causes confusion.
Changes in v2:
- drop 'move tracing helpers to trace_helper' patch
- add link pinning to prevent cleaning up on process exit
- add static at global variable and remove unused variable
- change to destroy link even after link__pin()
- fix return error code on exit
- merge commit with changing Makefile
Changes in v3:
- cleanup bpf_link, bpf_object and cgroup fd both on success and error
Daniel T. Lee (7):
samples: bpf: refactor hbm program with libbpf
samples: bpf: refactor test_cgrp2_sock2 program with libbpf
samples: bpf: refactor task_fd_query program with libbpf
samples: bpf: refactor ibumad program with libbpf
samples: bpf: refactor test_overhead program with libbpf
samples: bpf: fix lwt_len_hist reusing previous BPF map
samples: bpf: remove bpf_load loader completely
samples/bpf/.gitignore | 3 + samples/bpf/Makefile | 20 +- samples/bpf/bpf_load.c | 667 ------------------------------- samples/bpf/bpf_load.h | 57 --- samples/bpf/do_hbm_test.sh | 32 +- samples/bpf/hbm.c | 111 ++--- samples/bpf/hbm_kern.h | 2 +- samples/bpf/ibumad_kern.c | 26 +- samples/bpf/ibumad_user.c | 71 +++- samples/bpf/lwt_len_hist.sh | 2 + samples/bpf/task_fd_query_user.c | 101 +++-- samples/bpf/test_cgrp2_sock2.c | 61 ++- samples/bpf/test_cgrp2_sock2.sh | 21 +- samples/bpf/test_lwt_bpf.sh | 0 samples/bpf/test_overhead_user.c | 82 ++-- samples/bpf/xdp2skb_meta_kern.c | 2 +- 16 files changed, 350 insertions(+), 908 deletions(-) delete mode 100644 samples/bpf/bpf_load.c delete mode 100644 samples/bpf/bpf_load.h mode change 100644 => 100755 samples/bpf/lwt_len_hist.sh mode change 100644 => 100755 samples/bpf/test_lwt_bpf.sh
Signature · 1 line
2.25.1
PATCH v3 1/7 samples: bpf: refactor hbm program with libbpf
This commit refactors the existing cgroup programs with libbpf bpf loader. Since bpf_program__attach doesn't support cgroup program attachment, this explicitly attaches cgroup bpf program with bpf_program__attach_cgroup(bpf_prog, cg1).
Also, to change attach_type of bpf program, this uses libbpf's bpf_program__set_expected_attach_type helper to switch EGRESS to INGRESS. To keep bpf program attached to the cgroup hierarchy even after the exit, this commit uses the BPF_LINK_PINNING to pin the link attachment even after it is closed.
Besides, this program was broken due to the typo of BPF MAP definition. But this commit solves the problem by fixing this from 'queue_stats' map struct hvm_queue_stats -> hbm_queue_stats.
---
Changes in v2:
- restore read_trace_pipe2
- remove unnecessary return code and cgroup fd compare
- add static at global variable and remove unused variable
- change cgroup path with unified controller (/unified/)
- add link pinning to prevent cleaning up on process exit
Changes in v3:
- cleanup bpf_link, bpf_object and cgroup fd both on success and error
- remove link NULL cleanup since __destroy() can handle
- fix cgroup test on cgroup fd cleanup
samples/bpf/.gitignore | 3 + samples/bpf/Makefile | 2 +- samples/bpf/do_hbm_test.sh | 32 +++++------ samples/bpf/hbm.c | 111 ++++++++++++++++++++----------------- samples/bpf/hbm_kern.h | 2 +- 5 files changed, 78 insertions(+), 72 deletions(-)
samples/bpf/.gitignore +3 -0
@@ -52,3 +52,6 @@ xdp_tx_iptunnel xdpsock xsk_fwd testfile.img +hbm_out.log +iperf.* +*.out
samples/bpf/Makefile +1 -1
@@ -110,7 +110,7 @@ xdp_fwd-objs := xdp_fwd_user.o task_fd_query-objs := bpf_load.o task_fd_query_user.o $(TRACE_HELPERS) xdp_sample_pkts-objs := xdp_sample_pkts_user.o $(TRACE_HELPERS) ibumad-objs := bpf_load.o ibumad_user.o $(TRACE_HELPERS) -hbm-objs := bpf_load.o hbm.o $(CGROUP_HELPERS) +hbm-objs := hbm.o $(CGROUP_HELPERS) # Tell kbuild to always build the programs always-y := $(tprogs-y)
samples/bpf/do_hbm_test.sh +14 -18
@@ -91,6 +91,16 @@ qdisc="" flags="" do_stats=0 +BPFFS=/sys/fs/bpf +function config_bpffs () { + if mount | grep $BPFFS > /dev/null; then + echo "bpffs already mounted" + else + echo "bpffs not mounted. Mounting..." + mount -t bpf none $BPFFS + fi +} + function start_hbm () { rm -f hbm.out echo "./hbm $dir -n $id -r $rate -t $dur $flags $dbg $prog" > hbm.out
@@ -192,6 +202,7 @@ processArgs () { } processArgs +config_bpffs if [ $debug_flag -eq 1 ] ; then rm -f hbm_out.log
@@ -201,7 +212,7 @@ hbm_pid=$(start_hbm) usleep 100000 host=`hostname` -cg_base_dir=/sys/fs/cgroup +cg_base_dir=/sys/fs/cgroup/unified cg_dir="$cg_base_dir/cgroup-test-work-dir/hbm$id" echo $$ >> $cg_dir/cgroup.procs
@@ -411,23 +422,8 @@ fi sleep 1 -# Detach any BPF programs that may have lingered -ttx=`bpftool cgroup tree | grep hbm` -v=2 -for x in $ttx ; do - if [ "${x:0:36}" == "/sys/fs/cgroup/cgroup-test-work-dir/" ] ; then - cg=$x ; v=0 - else - if [ $v -eq 0 ] ; then - id=$x ; v=1 - else - if [ $v -eq 1 ] ; then - type=$x ; bpftool cgroup detach $cg $type id $id - v=0 - fi - fi - fi -done +# Detach any pinned BPF programs that may have lingered +rm -rf $BPFFS/hbm* if [ $use_netperf -ne 0 ] ; then if [ "$server" == "" ] ; then
samples/bpf/hbm.c +59 -52
189 lines not on this page · raw message
samples/bpf/hbm_kern.h +1 -1
@@ -69,7 +69,7 @@ struct {__uint (type,BPF_MAP_TYPE_ARRAY );__uint (max_entries,1 );__type (key, u32); - __type(value, struct hvm_queue_stats); + __type(value, struct hbm_queue_stats); } queue_stats SEC(".maps");struct hbm_pkt_info {
Signature · 1 line
2.25.1
Re: [PATCH bpf-next v3 1/7] samples: bpf: refactor hbm program with libbpf
Quoted 31 lines · Daniel T. Lee
This commit refactors the existing cgroup programs with libbpf bpf loader. Since
bpf_program__attachdoesn't support cgroup program attachment, this explicitly attaches cgroup bpf program withbpf_program__attach_cgroup(bpf_prog, cg1).Also, to change
attach_typeof bpf program, this uses libbpf'sbpf_program__set_expected_attach_typehelper to switch EGRESS to INGRESS. To keep bpf program attached to the cgroup hierarchy even after the exit, this commit uses theBPF_LINK_PINNINGto pin the link attachment even after it is closed.Besides, this program was broken due to the typo of BPF MAP definition. But this commit solves the problem by fixing this from '
queue_stats' map structhvm_queue_stats->hbm_queue_stats.Fixes 36b5d471135c ("selftests/bpf: samples/bpf: Split off legacy stuff from bpf_helpers.h")Signed-off-by Daniel T. Lee <[email protected]>---
Changes in v2:
- restoreread_trace_pipe2
- remove unnecessary return code and cgroup fd compare
- add static at global variable and remove unused variable
- change cgroup path with unified controller (/unified/)
- add link pinning to prevent cleaning up on process exitChanges in v3:
- cleanupbpf_link,bpf_objectand cgroup fd both on success and error
- remove link NULL cleanup since__destroy()can handle
- fix cgroup test on cgroup fd cleanup
Quoted 7 lines
samples/bpf/.gitignore | 3 + samples/bpf/Makefile | 2 +- samples/bpf/do_hbm_test.sh | 32 +++++------ samples/bpf/hbm.c | 111 ++++++++++++++++++++----------------- samples/bpf/hbm_kern.h | 2 +- 5 files changed, 78 insertions(+), 72 deletions(-)
Thanks for the nice clean up! I've applied the series to bpf-next. If Martin finds any other problems, those can be fixed in a follow up patch(es). But see also below about find_program_by_title().
samples/bpf/hbm.c static void do_error(char *msg, bool errno_flag) · line 135
Quoted 7 lines
samples/bpf/hbm.c +1
-6-if (ret) { -printf ("ERROR: bpf_prog_load_xattr failed for: %s\n" , prog); -printf (" Output from verifier:\n%s\n------\n" , bpf_log_buf); - ret =-1 ; - }else { - ret = map_fd; + bpf_prog =bpf_object__find_program_by_title (obj,"cgroup_skb/egress" );
It would be good to avoid using find_program_by_title(), as it's going to get deprecated and eventually removed. Looking up by section name ("title") is ambiguous now that libbpf supports many BPF programs per same section. There is find_program_by_name() which looks program by its C function name. I suggest using it.
Quoted 5 lines
samples/bpf/hbm.c +5
-0+if (!bpf_prog) { +printf ("ERROR: finding a prog in obj file failed\n" ); + goto err; + } +
PATCH v3 2/7 samples: bpf: refactor test_cgrp2_sock2 program with libbpf
In reply to Daniel T. LeeThis commit refactors the existing cgroup program with libbpf bpf loader. The original test_cgrp2_sock2 has keeped the bpf program attached to the cgroup hierarchy even after the exit of user program. To implement the same functionality with libbpf, this commit uses the BPF_LINK_PINNING to pin the link attachment even after it is closed.
Since this uses LINK instead of ATTACH, detach of bpf program from cgroup with 'test_cgrp2_sock' is not used anymore.
The code to mount the bpf was added to the .sh file in case the bpff was not mounted on /sys/fs/bpf. Additionally, to fix the problem that shell script cannot find the binary object from the current path, relative path './' has been added in front of binary.
---
Changes in v2:
- change to destroy link even after link__pin()
- enhance error message
samples/bpf/Makefile | 2 +- samples/bpf/test_cgrp2_sock2.c | 61 ++++++++++++++++++++++++--------- samples/bpf/test_cgrp2_sock2.sh | 21 +++++++++--- 3 files changed, 62 insertions(+), 22 deletions(-)
samples/bpf/Makefile +1 -1
@@ -82,7 +82,7 @@ test_overhead-objs := bpf_load.o test_overhead_user.o test_cgrp2_array_pin-objs := test_cgrp2_array_pin.o test_cgrp2_attach-objs := test_cgrp2_attach.o test_cgrp2_sock-objs := test_cgrp2_sock.o -test_cgrp2_sock2-objs := bpf_load.o test_cgrp2_sock2.o +test_cgrp2_sock2-objs := test_cgrp2_sock2.o xdp1-objs := xdp1_user.o # reuse xdp1 source intentionally xdp2-objs := xdp1_user.o
samples/bpf/test_cgrp2_sock2.c +44 -17
90 lines not on this page · raw message
samples/bpf/test_cgrp2_sock2.sh +17 -4
@@ -1,6 +1,9 @@ #!/bin/bash # SPDX-License-Identifier: GPL-2.0 +BPFFS=/sys/fs/bpf +LINK_PIN=$BPFFS/test_cgrp2_sock2 + function config_device { ip netns add at_ns0 ip link add veth0 type veth peer name veth0b
@@ -21,16 +24,22 @@ function config_cgroup { echo $$ >> /tmp/cgroupv2/foo/cgroup.procs } +function config_bpffs { + if mount | grep $BPFFS > /dev/null; then + echo "bpffs already mounted" + else + echo "bpffs not mounted. Mounting..." + mount -t bpf none $BPFFS + fi +} function attach_bpf { - test_cgrp2_sock2 /tmp/cgroupv2/foo sock_flags_kern.o $1 + ./test_cgrp2_sock2 /tmp/cgroupv2/foo sock_flags_kern.o $1 [ $? -ne 0 ] && exit 1 } function cleanup { - if [ -d /tmp/cgroupv2/foo ]; then - test_cgrp2_sock -d /tmp/cgroupv2/foo - fi + rm -rf $LINK_PIN ip link del veth0b ip netns delete at_ns0 umount /tmp/cgroupv2
@@ -42,6 +51,7 @@ cleanup 2>/dev/null set -e config_device config_cgroup +config_bpffs set +e #
@@ -62,6 +72,9 @@ if [ $? -eq 0 ]; then exit 1 fi +rm -rf $LINK_PIN +sleep 1 # Wait for link detach + # # Test 2 - fail ping #
Signature · 1 line
2.25.1
PATCH v3 3/7 samples: bpf: refactor task_fd_query program with libbpf
In reply to Daniel T. LeeThis commit refactors the existing kprobe program with libbpf bpf loader. To attach bpf program, this uses generic bpf_program__attach() approach rather than using bpf_load's load_bpf_file().
To attach bpf to perf_event, instead of using previous ioctl method, this commit uses bpf_program__attach_perf_event since it manages the enable of perf_event and attach of BPF programs to it, which is much more intuitive way to achieve.
Also, explicit close(fd) has been removed since event will be closed inside bpf_link__destroy() automatically.
Furthermore, to prevent conflict of same named uprobe events, O_TRUNC flag has been used to clear 'uprobe_events' interface.
---
Changes in v2:
- add static at global variable and drop {}
- fix return error code on exit
- restore DEBUGFS macro to absolute string path
samples/bpf/Makefile | 2 +- samples/bpf/task_fd_query_user.c | 101 ++++++++++++++++++++++--------- 2 files changed, 75 insertions(+), 28 deletions(-)
samples/bpf/Makefile +1 -1
@@ -107,7 +107,7 @@ xdp_adjust_tail-objs := xdp_adjust_tail_user.o xdpsock-objs := xdpsock_user.o xsk_fwd-objs := xsk_fwd.o xdp_fwd-objs := xdp_fwd_user.o -task_fd_query-objs := bpf_load.o task_fd_query_user.o $(TRACE_HELPERS) +task_fd_query-objs := task_fd_query_user.o $(TRACE_HELPERS) xdp_sample_pkts-objs := xdp_sample_pkts_user.o $(TRACE_HELPERS) ibumad-objs := bpf_load.o ibumad_user.o $(TRACE_HELPERS) hbm-objs := hbm.o $(CGROUP_HELPERS)
samples/bpf/task_fd_query_user.c +74 -27
203 lines not on this page · raw message
Signature · 1 line
2.25.1
PATCH v3 4/7 samples: bpf: refactor ibumad program with libbpf
In reply to Daniel T. LeeThis commit refactors the existing ibumad program with libbpf bpf loader. Attach/detach of Tracepoint bpf programs has been managed with the generic bpf_program__attach() and bpf_link__destroy() from the libbpf.
Also, instead of using the previous BPF MAP definition, this commit refactors ibumad MAP definition with the new BTF-defined MAP format.
To verify that this bpf program works without an infiniband device, try loading ib_umad kernel module and test the program as follows:
# modprobe ib_umad
# ./ibumadMoreover, TRACE_HELPERS has been removed from the Makefile since it is not used on this program.
---
Changes in v2:
- add static at global variable and drop {}
- fix return error code on exit
samples/bpf/Makefile | 2 +- samples/bpf/ibumad_kern.c | 26 +++++++------- samples/bpf/ibumad_user.c | 71 +++++++++++++++++++++++++++++---------- 3 files changed, 68 insertions(+), 31 deletions(-)
samples/bpf/Makefile +1 -1
@@ -109,7 +109,7 @@ xsk_fwd-objs := xsk_fwd.o xdp_fwd-objs := xdp_fwd_user.o task_fd_query-objs := task_fd_query_user.o $(TRACE_HELPERS) xdp_sample_pkts-objs := xdp_sample_pkts_user.o $(TRACE_HELPERS) -ibumad-objs := bpf_load.o ibumad_user.o $(TRACE_HELPERS) +ibumad-objs := ibumad_user.o hbm-objs := hbm.o $(CGROUP_HELPERS) # Tell kbuild to always build the programs
samples/bpf/ibumad_kern.c +13 -13
@@ -16,19 +16,19 @@#include <bpf/bpf_helpers.h> -struct bpf_map_def SEC("maps") read_count = { - .type = BPF_MAP_TYPE_ARRAY, - .key_size = sizeof(u32), /* class; u32 required */ - .value_size = sizeof(u64), /* count of mads read */ - .max_entries = 256, /* Room for all Classes */ -}; - -struct bpf_map_def SEC("maps") write_count = { - .type = BPF_MAP_TYPE_ARRAY, - .key_size = sizeof(u32), /* class; u32 required */ - .value_size = sizeof(u64), /* count of mads written */ - .max_entries = 256, /* Room for all Classes */ -}; +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __type(key, u32); /* class; u32 required */ + __type(value, u64); /* count of mads read */ + __uint(max_entries, 256); /* Room for all Classes */ +} read_count SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __type(key, u32); /* class; u32 required */ + __type(value, u64); /* count of mads written */ + __uint(max_entries, 256); /* Room for all Classes */ +} write_count SEC(".maps");#undef DEBUG#ifndef DEBUG
samples/bpf/ibumad_user.c +54 -17
105 lines not on this page · raw message
Signature · 1 line
2.25.1
PATCH v3 5/7 samples: bpf: refactor test_overhead program with libbpf
In reply to Daniel T. LeeThis commit refactors the existing program with libbpf bpf loader. Since the kprobe, tracepoint and raw_tracepoint bpf program can be attached with single bpf_program__attach() interface, so the corresponding function of libbpf is used here.
Rather than specifying the number of cpus inside the code, this commit uses the number of available cpus with _SC_NPROCESSORS_ONLN.
---
Changes in v2:
- add static at global variable and drop {}
samples/bpf/Makefile | 2 +- samples/bpf/test_overhead_user.c | 82 +++++++++++++++++++++++--------- 2 files changed, 60 insertions(+), 24 deletions(-)
samples/bpf/Makefile +1 -1
@@ -78,7 +78,7 @@ lathist-objs := lathist_user.o offwaketime-objs := offwaketime_user.o $(TRACE_HELPERS) spintest-objs := spintest_user.o $(TRACE_HELPERS) map_perf_test-objs := map_perf_test_user.o -test_overhead-objs := bpf_load.o test_overhead_user.o +test_overhead-objs := test_overhead_user.o test_cgrp2_array_pin-objs := test_cgrp2_array_pin.o test_cgrp2_attach-objs := test_cgrp2_attach.o test_cgrp2_sock-objs := test_cgrp2_sock.o
samples/bpf/test_overhead_user.c +59 -23
127 lines not on this page · raw message
Signature · 1 line
2.25.1
PATCH v3 6/7 samples: bpf: fix lwt_len_hist reusing previous BPF map
In reply to Daniel T. LeeCurrently, lwt_len_hist's map lwt_len_hist_map is uses pinning, and the map isn't cleared on test end. This leds to reuse of that map for each test, which prevents the results of the test from being accurate.
This commit fixes the problem by removing of pinned map from bpffs. Also, this commit add the executable permission to shell script files.
--- samples/bpf/lwt_len_hist.sh | 2 ++ samples/bpf/test_lwt_bpf.sh | 0 2 files changed, 2 insertions(+) mode change 100644 => 100755 samples/bpf/lwt_len_hist.sh mode change 100644 => 100755 samples/bpf/test_lwt_bpf.sh
samples/bpf/lwt_len_hist.sh +2 -0
@@ -8,6 +8,8 @@ VETH1=tst_lwt1b TRACE_ROOT=/sys/kernel/debug/tracing function cleanup { + # To reset saved histogram, remove pinned map + rm /sys/fs/bpf/tc/globals/lwt_len_hist_map ip route del 192.168.253.2/32 dev $VETH0 2> /dev/null ip link del $VETH0 2> /dev/null ip link del $VETH1 2> /dev/null
samples/bpf/test_lwt_bpf.sh +0 -0
Signature · 1 line
2.25.1
PATCH v3 7/7 samples: bpf: remove bpf_load loader completely
In reply to Daniel T. LeeNumerous refactoring that rewrites BPF programs written with bpf_load to use the libbpf loader was finally completed, resulting in BPF programs using bpf_load within the kernel being completely no longer present.
This commit removes bpf_load, an outdated bpf loader that is difficult to keep up with the latest kernel BPF and causes confusion.
Also, this commit removes the unused trace_helper and bpf_load from samples/bpf target objects from Makefile.
---
Changes in v2:
- merge commit with changing Makefile
samples/bpf/Makefile | 10 +- samples/bpf/bpf_load.c | 667 -------------------------------- samples/bpf/bpf_load.h | 57 --- samples/bpf/xdp2skb_meta_kern.c | 2 +- 4 files changed, 5 insertions(+), 731 deletions(-) delete mode 100644 samples/bpf/bpf_load.c delete mode 100644 samples/bpf/bpf_load.h
samples/bpf/Makefile +4 -6
@@ -73,7 +73,7 @@ tracex5-objs := tracex5_user.o $(TRACE_HELPERS) tracex6-objs := tracex6_user.o tracex7-objs := tracex7_user.o test_probe_write_user-objs := test_probe_write_user_user.o -trace_output-objs := trace_output_user.o $(TRACE_HELPERS) +trace_output-objs := trace_output_user.o lathist-objs := lathist_user.o offwaketime-objs := offwaketime_user.o $(TRACE_HELPERS) spintest-objs := spintest_user.o $(TRACE_HELPERS)
@@ -91,8 +91,8 @@ test_current_task_under_cgroup-objs := $(CGROUP_HELPERS) \ test_current_task_under_cgroup_user.o trace_event-objs := trace_event_user.o $(TRACE_HELPERS) sampleip-objs := sampleip_user.o $(TRACE_HELPERS) -tc_l2_redirect-objs := bpf_load.o tc_l2_redirect_user.o -lwt_len_hist-objs := bpf_load.o lwt_len_hist_user.o +tc_l2_redirect-objs := tc_l2_redirect_user.o +lwt_len_hist-objs := lwt_len_hist_user.o xdp_tx_iptunnel-objs := xdp_tx_iptunnel_user.o test_map_in_map-objs := test_map_in_map_user.o per_socket_stats_example-objs := cookie_uid_helper_example.o
@@ -108,7 +108,7 @@ xdpsock-objs := xdpsock_user.o xsk_fwd-objs := xsk_fwd.o xdp_fwd-objs := xdp_fwd_user.o task_fd_query-objs := task_fd_query_user.o $(TRACE_HELPERS) -xdp_sample_pkts-objs := xdp_sample_pkts_user.o $(TRACE_HELPERS) +xdp_sample_pkts-objs := xdp_sample_pkts_user.o ibumad-objs := ibumad_user.o hbm-objs := hbm.o $(CGROUP_HELPERS)
@@ -197,8 +197,6 @@ TPROGS_CFLAGS += --sysroot=$(SYSROOT) TPROGS_LDFLAGS := -L$(SYSROOT)/usr/lib endif -TPROGCFLAGS_bpf_load.o += -Wno-unused-variable - TPROGS_LDLIBS += $(LIBBPF) -lelf -lz TPROGLDLIBS_tracex4 += -lrt TPROGLDLIBS_trace_output += -lrt
samples/bpf/bpf_load.c deleted +0 -667
667 lines not on this page · raw message
samples/bpf/bpf_load.h deleted +0 -57
57 lines not on this page · raw message
samples/bpf/xdp2skb_meta_kern.c +1 -1
@@ -6,7 +6,7 @@* This uses the XDP data_meta infrastructure, and is a cooperation * between two bpf-programs (1) XDP and (2) clsact at TC-ingress hook. * -* Notice: This example does not use the BPF C-loader (bpf_load.c), +* Notice: This example does not use the BPF C-loader, * but instead rely on the iproute2 TC tool for loading BPF-objects. */ #include <uapi/linux/bpf.h>
Signature · 1 line
2.25.1