Misc BPF improvements
Subsystems (17)
Daniel Borkmann
maintainercover Misc BPF improvements
This last series for this window adds various misc improvements to BPF, one is to mark registered map and prog types as __ro_after_init, another one for removing cBPF stubs in eBPF JITs and moving the stub to the core and last also improving JITs is to make generated images visible to the kernel and kallsyms, so they can be seen in traces. For details, please have a look at the individual patches.
Thanks a lot!
Daniel Borkmann (3):
bpf: mark all registered map/prog types as __ro_after_init
bpf: remove stubs for cBPF from arch code
bpf: make jited programs visible in traces
Documentation/sysctl/net.txt | 12 ++ arch/arm64/net/bpf_jit_comp.c | 20 ---- arch/powerpc/net/bpf_jit_comp64.c | 3 +- arch/s390/net/bpf_jit_comp.c | 26 ----- arch/x86/net/bpf_jit_comp.c | 23 +--- include/linux/bpf.h | 4 + include/linux/filter.h | 112 +++++++++++++++++- kernel/bpf/arraymap.c | 10 +- kernel/bpf/core.c | 235 +++++++++++++++++++++++++++++++++++++- kernel/bpf/hashtab.c | 8 +- kernel/bpf/lpm_trie.c | 2 +- kernel/bpf/stackmap.c | 2 +- kernel/bpf/syscall.c | 2 + kernel/extable.c | 9 +- kernel/kallsyms.c | 61 ++++++++-- kernel/trace/bpf_trace.c | 6 +- net/Kconfig | 3 +- net/core/filter.c | 18 +-- net/core/sysctl_net_core.c | 7 ++ 19 files changed, 453 insertions(+), 110 deletions(-)
Signature · 1 line
1.9.3
Daniel Borkmann
maintainerPATCH 1/3 bpf: mark all registered map/prog types as __ro_after_init
All map types and prog types are registered to the BPF core through bpf_register_map_type() and bpf_register_prog_type() during init and remain unchanged thereafter. As by design we don't (and never will) have any pluggable code that can register to that at any later point in time, lets mark all the existing bpf_{map,prog}_type_list objects in the tree as __ro_after_init, so they can be moved to read-only section from then onwards.
--- kernel/bpf/arraymap.c | 10 +++++----- kernel/bpf/hashtab.c | 8 ++++---- kernel/bpf/lpm_trie.c | 2 +- kernel/bpf/stackmap.c | 2 +- kernel/trace/bpf_trace.c | 6 +++--- net/core/filter.c | 18 +++++++++--------- 6 files changed, 23 insertions(+), 23 deletions(-)
kernel/bpf/arraymap.c +5 -5
@@ -269,7 +269,7 @@ static void array_map_free(struct bpf_map *map) .map_delete_elem = array_map_delete_elem, }; -static struct bpf_map_type_list array_type__read_mostly = { +static struct bpf_map_type_list array_type__ro_after_init = { .ops = &array_ops, .type =BPF_MAP_TYPE_ARRAY , };
@@ -283,7 +283,7 @@ static void array_map_free(struct bpf_map *map) .map_delete_elem = array_map_delete_elem, }; -static struct bpf_map_type_list percpu_array_type__read_mostly = { +static struct bpf_map_type_list percpu_array_type__ro_after_init = { .ops = &percpu_array_ops, .type =BPF_MAP_TYPE_PERCPU_ARRAY , };
@@ -409,7 +409,7 @@ void bpf_fd_array_map_clear(struct bpf_map *map) .map_fd_put_ptr = prog_fd_array_put_ptr, }; -static struct bpf_map_type_list prog_array_type__read_mostly = { +static struct bpf_map_type_list prog_array_type__ro_after_init = { .ops = &prog_array_ops, .type =BPF_MAP_TYPE_PROG_ARRAY , };
@@ -522,7 +522,7 @@ static void perf_event_fd_array_release(struct bpf_map *map, .map_release = perf_event_fd_array_release, }; -static struct bpf_map_type_list perf_event_array_type__read_mostly = { +static struct bpf_map_type_list perf_event_array_type__ro_after_init = { .ops = &perf_event_array_ops, .type =BPF_MAP_TYPE_PERF_EVENT_ARRAY , };
@@ -564,7 +564,7 @@ static void cgroup_fd_array_free(struct bpf_map *map) .map_fd_put_ptr = cgroup_fd_array_put_ptr, }; -static struct bpf_map_type_list cgroup_array_type__read_mostly = { +static struct bpf_map_type_list cgroup_array_type__ro_after_init = { .ops = &cgroup_array_ops, .type =BPF_MAP_TYPE_CGROUP_ARRAY , };
kernel/bpf/hashtab.c +4 -4
@@ -1023,7 +1023,7 @@ static void htab_map_free(struct bpf_map *map) .map_delete_elem = htab_map_delete_elem, }; -static struct bpf_map_type_list htab_type__read_mostly = { +static struct bpf_map_type_list htab_type__ro_after_init = { .ops = &htab_ops, .type =BPF_MAP_TYPE_HASH , };
@@ -1037,7 +1037,7 @@ static void htab_map_free(struct bpf_map *map) .map_delete_elem = htab_lru_map_delete_elem, }; -static struct bpf_map_type_list htab_lru_type__read_mostly = { +static struct bpf_map_type_list htab_lru_type__ro_after_init = { .ops = &htab_lru_ops, .type =BPF_MAP_TYPE_LRU_HASH , };
@@ -1124,7 +1124,7 @@ int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value, .map_delete_elem = htab_map_delete_elem, }; -static struct bpf_map_type_list htab_percpu_type__read_mostly = { +static struct bpf_map_type_list htab_percpu_type__ro_after_init = { .ops = &htab_percpu_ops, .type =BPF_MAP_TYPE_PERCPU_HASH , };
@@ -1138,7 +1138,7 @@ int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value, .map_delete_elem = htab_lru_map_delete_elem, }; -static struct bpf_map_type_list htab_lru_percpu_type__read_mostly = { +static struct bpf_map_type_list htab_lru_percpu_type__ro_after_init = { .ops = &htab_lru_percpu_ops, .type =BPF_MAP_TYPE_LRU_PERCPU_HASH , };
kernel/bpf/lpm_trie.c +1 -1
@@ -508,7 +508,7 @@ static void trie_free(struct bpf_map *map) .map_delete_elem = trie_delete_elem, }; -static struct bpf_map_type_list trie_type__read_mostly = { +static struct bpf_map_type_list trie_type__ro_after_init = { .ops = &trie_ops, .type =BPF_MAP_TYPE_LPM_TRIE , };
kernel/bpf/stackmap.c +1 -1
@@ -273,7 +273,7 @@ static void stack_map_free(struct bpf_map *map) .map_delete_elem = stack_map_delete_elem, }; -static struct bpf_map_type_list stack_map_type__read_mostly = { +static struct bpf_map_type_list stack_map_type__ro_after_init = { .ops = &stack_map_ops, .type =BPF_MAP_TYPE_STACK_TRACE , };
kernel/trace/bpf_trace.c +3 -3
@@ -506,7 +506,7 @@ static bool kprobe_prog_is_valid_access(int off, int size, enum bpf_access_type .is_valid_access = kprobe_prog_is_valid_access, }; -static struct bpf_prog_type_list kprobe_tl = { +static struct bpf_prog_type_list kprobe_tl__ro_after_init = { .ops = &kprobe_prog_ops, .type =BPF_PROG_TYPE_KPROBE , };
@@ -589,7 +589,7 @@ static bool tp_prog_is_valid_access(int off, int size, enum bpf_access_type type .is_valid_access = tp_prog_is_valid_access, }; -static struct bpf_prog_type_list tracepoint_tl = { +static struct bpf_prog_type_list tracepoint_tl__ro_after_init = { .ops = &tracepoint_prog_ops, .type =BPF_PROG_TYPE_TRACEPOINT , };
@@ -648,7 +648,7 @@ static u32 pe_prog_convert_ctx_access(enum bpf_access_type type, .convert_ctx_access = pe_prog_convert_ctx_access, }; -static struct bpf_prog_type_list perf_event_tl = { +static struct bpf_prog_type_list perf_event_tl__ro_after_init = { .ops = &perf_event_prog_ops, .type =BPF_PROG_TYPE_PERF_EVENT , };
net/core/filter.c +9 -9
@@ -3296,47 +3296,47 @@ static u32 xdp_convert_ctx_access(enum bpf_access_type type, .convert_ctx_access = sock_filter_convert_ctx_access, }; -static struct bpf_prog_type_list sk_filter_type__read_mostly = { +static struct bpf_prog_type_list sk_filter_type__ro_after_init = { .ops = &sk_filter_ops, .type =BPF_PROG_TYPE_SOCKET_FILTER , }; -static struct bpf_prog_type_list sched_cls_type__read_mostly = { +static struct bpf_prog_type_list sched_cls_type__ro_after_init = { .ops = &tc_cls_act_ops, .type =BPF_PROG_TYPE_SCHED_CLS , }; -static struct bpf_prog_type_list sched_act_type__read_mostly = { +static struct bpf_prog_type_list sched_act_type__ro_after_init = { .ops = &tc_cls_act_ops, .type =BPF_PROG_TYPE_SCHED_ACT , }; -static struct bpf_prog_type_list xdp_type__read_mostly = { +static struct bpf_prog_type_list xdp_type__ro_after_init = { .ops = &xdp_ops, .type =BPF_PROG_TYPE_XDP , }; -static struct bpf_prog_type_list cg_skb_type__read_mostly = { +static struct bpf_prog_type_list cg_skb_type__ro_after_init = { .ops = &cg_skb_ops, .type =BPF_PROG_TYPE_CGROUP_SKB , }; -static struct bpf_prog_type_list lwt_in_type__read_mostly = { +static struct bpf_prog_type_list lwt_in_type__ro_after_init = { .ops = &lwt_inout_ops, .type =BPF_PROG_TYPE_LWT_IN , }; -static struct bpf_prog_type_list lwt_out_type__read_mostly = { +static struct bpf_prog_type_list lwt_out_type__ro_after_init = { .ops = &lwt_inout_ops, .type =BPF_PROG_TYPE_LWT_OUT , }; -static struct bpf_prog_type_list lwt_xmit_type__read_mostly = { +static struct bpf_prog_type_list lwt_xmit_type__ro_after_init = { .ops = &lwt_xmit_ops, .type =BPF_PROG_TYPE_LWT_XMIT , }; -static struct bpf_prog_type_list cg_sock_type__read_mostly = { +static struct bpf_prog_type_list cg_sock_type__ro_after_init = { .ops = &cg_sock_ops, .type =BPF_PROG_TYPE_CGROUP_SOCK };
Signature · 1 line
1.9.3
Daniel Borkmann
maintainerPATCH 3/3 bpf: make jited programs visible in traces
In reply to Daniel BorkmannLong standing issue with JITed programs is that stack traces from function tracing check whether a given address is kernel code through {__,}kernel_text_address(), which checks for code in core kernel, modules and dynamically allocated ftrace trampolines. But what is still missing is BPF JITed programs (interpreted programs are not an issue as __bpf_prog_run() will be attributed to them), thus when a stack trace is triggered, the code walking the stack won't see any of the JITed ones. The same for address correlation done from user space via reading /proc/kallsyms. This is read by tools like perf, but the latter is also useful for permanent live tracing with eBPF itself in combination with stack maps when other eBPF types are part of the callchain. See offwaketime example on dumping stack from a map.
This work tries to tackle that issue by making the addresses and symbols known to the kernel. The lookup from *kernel_text_address() is implemented through a latched RB tree that can be read under RCU in fast-path that is also shared for symbol/size/offset lookup for a specific given address in kallsyms. The slow-path iteration through all symbols in the seq file done via RCU list, which holds a tiny fraction of all exported ksyms, usually below 0.1 percent. Function symbols are exported as bpf_prog_<tag>, in order to aide debugging and attribution. This facility is currently enabled for root-only when bpf_jit_kallsyms is set to 1, and disabled if hardening is active in any mode. The rationale behind this is that still a lot of systems ship with world read permissions on kallsyms thus addresses should not get suddenly exposed for them. If that situation gets much better in future, we always have the option to change the default on this. Likewise, unprivileged programs are not allowed to add entries there either, but that is less of a concern as most such programs types relevant in this context are for root-only anyway. If enabled, call graphs and stack traces will then show a correct attribution; one example is illustrated below, where the trace is now visible in tooling such as perf script --kallsyms=/proc/kallsyms and friends.
Before:
7fff8166889d bpf_clone_redirect+0x80007f0020ed (/lib/modules/4.9.0-rc8+/build/vmlinux)
f5d80 __sendmsg_nocancel+0xffff006451f1a007 (/usr/lib64/libc-2.18.so)
After:
7fff816688b7 bpf_clone_redirect+0x80007f002107 (/lib/modules/4.9.0-rc8+/build/vmlinux) 7fffa0575728 bpf_prog_33c45a467c9e061a+0x8000600020fb (/lib/modules/4.9.0-rc8+/build/vmlinux) 7fffa07ef1fc cls_bpf_classify+0x8000600020dc (/lib/modules/4.9.0-rc8+/build/vmlinux) 7fff81678b68 tc_classify+0x80007f002078 (/lib/modules/4.9.0-rc8+/build/vmlinux) 7fff8164d40b __netif_receive_skb_core+0x80007f0025fb (/lib/modules/4.9.0-rc8+/build/vmlinux) 7fff8164d718 __netif_receive_skb+0x80007f002018 (/lib/modules/4.9.0-rc8+/build/vmlinux) 7fff8164e565 process_backlog+0x80007f002095 (/lib/modules/4.9.0-rc8+/build/vmlinux) 7fff8164dc71 net_rx_action+0x80007f002231 (/lib/modules/4.9.0-rc8+/build/vmlinux) 7fff81767461 __softirqentry_text_start+0x80007f0020d1 (/lib/modules/4.9.0-rc8+/build/vmlinux) 7fff817658ac do_softirq_own_stack+0x80007f00201c (/lib/modules/4.9.0-rc8+/build/vmlinux) 7fff810a2c20 do_softirq+0x80007f002050 (/lib/modules/4.9.0-rc8+/build/vmlinux) 7fff810a2cb5 __local_bh_enable_ip+0x80007f002085 (/lib/modules/4.9.0-rc8+/build/vmlinux) 7fff8168d452 ip_finish_output2+0x80007f002152 (/lib/modules/4.9.0-rc8+/build/vmlinux) 7fff8168ea3d ip_finish_output+0x80007f00217d (/lib/modules/4.9.0-rc8+/build/vmlinux) 7fff8168f2af ip_output+0x80007f00203f (/lib/modules/4.9.0-rc8+/build/vmlinux) [...]
7fff81005854 do_syscall_64+0x80007f002054 (/lib/modules/4.9.0-rc8+/build/vmlinux) 7fff817649eb return_from_SYSCALL_64+0x80007f002000 (/lib/modules/4.9.0-rc8+/build/vmlinux)
f5d80 __sendmsg_nocancel+0xffff01c484812007 (/usr/lib64/libc-2.18.so)
--- Documentation/sysctl/net.txt | 12 ++ arch/arm64/net/bpf_jit_comp.c | 15 --- arch/powerpc/net/bpf_jit_comp64.c | 1 + arch/s390/net/bpf_jit_comp.c | 18 --- arch/x86/net/bpf_jit_comp.c | 15 --- include/linux/bpf.h | 4 + include/linux/filter.h | 112 ++++++++++++++++++- kernel/bpf/core.c | 223 ++++++++++++++++++++++++++++++++++++++ kernel/bpf/syscall.c | 2 + kernel/extable.c | 9 +- kernel/kallsyms.c | 61 +++++++++-- net/Kconfig | 3 +- net/core/sysctl_net_core.c | 7 ++ 13 files changed, 419 insertions(+), 63 deletions(-)
Documentation/sysctl/net.txt +12 -0
@@ -54,6 +54,18 @@ Values : 1 - enable JIT hardening for unprivileged users only 2 - enable JIT hardening for all users +bpf_jit_kallsyms +---------------- + +When Berkeley Packet Filter Just in Time compiler is enabled, then compiled +images are unknown addresses to the kernel, meaning they neither show up in +traces nor in /proc/kallsyms. This enables export of these addresses, which +can be used for debugging/tracing. If bpf_jit_harden is enabled, this feature +is disabled. +Values : + 0 - disable JIT kallsyms export (default value) + 1 - enable JIT kallsyms export for privileged users only + dev_weight --------------
arch/arm64/net/bpf_jit_comp.c +0 -15
@@ -910,18 +910,3 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) tmp : orig_prog);return prog; } - -void bpf_jit_free (struct bpf_prog *prog) -{ -unsigned long addr = (unsigned long )prog->bpf_func &PAGE_MASK ; -struct bpf_binary_header *header = (void *)addr; - -if (!prog->jited) - goto free_filter; - -set_memory_rw (addr, header->pages); -bpf_jit_binary_free (header); - -free_filter: -bpf_prog_unlock_free (prog); -}
arch/powerpc/net/bpf_jit_comp64.c +1 -0
@@ -1064,6 +1064,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)return fp; } +/* Overriding bpf_jit_free() as we don't set images read-only. */ void bpf_jit_free (struct bpf_prog *fp) {unsigned long addr = (unsigned long )fp->bpf_func &PAGE_MASK ;
arch/s390/net/bpf_jit_comp.c +0 -18
@@ -1339,21 +1339,3 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) tmp : orig_fp);return fp; } - -/* -* Free eBPF program -*/ -void bpf_jit_free (struct bpf_prog *fp) -{ -unsigned long addr = (unsigned long )fp->bpf_func &PAGE_MASK ; -struct bpf_binary_header *header = (void *)addr; - -if (!fp->jited) - goto free_filter; - -set_memory_rw (addr, header->pages); -bpf_jit_binary_free (header); - -free_filter: -bpf_prog_unlock_free (fp); -}
arch/x86/net/bpf_jit_comp.c +0 -15
@@ -1180,18 +1180,3 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) tmp : orig_prog);return prog; } - -void bpf_jit_free (struct bpf_prog *fp) -{ -unsigned long addr = (unsigned long )fp->bpf_func &PAGE_MASK ; -struct bpf_binary_header *header = (void *)addr; - -if (!fp->jited) - goto free_filter; - -set_memory_rw (addr, header->pages); -bpf_jit_binary_free (header); - -free_filter: -bpf_prog_unlock_free (fp); -}
include/linux/bpf.h +4 -0
@@ -8,10 +8,12 @@#define _LINUX_BPF_H 1#include <uapi/linux/bpf.h> +#include <linux/workqueue.h> #include <linux/file.h> #include <linux/percpu.h> #include <linux/err.h> +#include <linux/rbtree_latch.h> struct perf_event ;struct bpf_map ;
@@ -177,6 +179,8 @@ struct bpf_prog_aux {atomic_t refcnt;u32 used_map_cnt;u32 max_ctx_offset; +struct latch_tree_node ksym_tnode; +struct list_head ksym_lnode;const struct bpf_verifier_ops *ops;struct bpf_map **used_maps;struct bpf_prog *prog;
include/linux/filter.h +111 -1
154 lines not on this page · raw message
kernel/bpf/core.c +223 -0
247 lines not on this page · raw message
kernel/bpf/syscall.c +2 -0
@@ -707,6 +707,7 @@ void bpf_prog_put(struct bpf_prog *prog) {if (atomic_dec_and_test (&prog->aux->refcnt)) {trace_bpf_prog_put_rcu (prog); +bpf_prog_kallsyms_del (prog);call_rcu (&prog->aux->rcu, __bpf_prog_put_rcu); } }
@@ -903,6 +904,7 @@ static int bpf_prog_load(union bpf_attr *attr)/* failed to allocate fd */ goto free_used_maps; +bpf_prog_kallsyms_add (prog);trace_bpf_prog_load (prog, err);return err;
kernel/extable.c +8 -1
@@ -20,6 +20,7 @@#include <linux/module.h> #include <linux/mutex.h> #include <linux/init.h> +#include <linux/filter.h> #include <asm/sections.h> #include <linux/uaccess.h>
@@ -104,6 +105,8 @@ int __kernel_text_address(unsigned long addr)return 1 ;if (is_ftrace_trampoline (addr))return 1 ; +if (is_bpf_text_address (addr)) +return 1 ;/* * There might be init symbols in saved stacktraces. * Give those symbols a chance to be printed in
@@ -123,7 +126,11 @@ int kernel_text_address(unsigned long addr)return 1 ;if (is_module_text_address (addr))return 1 ; -return is_ftrace_trampoline (addr); +if (is_ftrace_trampoline (addr)) +return 1 ; +if (is_bpf_text_address (addr)) +return 1 ; +return 0 ; }/*
kernel/kallsyms.c +49 -12
115 lines not on this page · raw message
net/Kconfig +2 -1
@@ -297,7 +297,8 @@ config BPF_JIT Note, admin should enable this feature changing: /proc/sys/net/core/bpf_jit_enable - /proc/sys/net/core/bpf_jit_harden (optional) + /proc/sys/net/core/bpf_jit_harden (optional) + /proc/sys/net/core/bpf_jit_kallsyms (optional) config NET_FLOW_LIMIT bool
net/core/sysctl_net_core.c +7 -0
@@ -334,6 +334,13 @@ static int proc_do_rss_key(struct ctl_table *table, int write, .mode = 0600, .proc_handler = proc_dointvec, }, + { + .procname = "bpf_jit_kallsyms", + .data = &bpf_jit_kallsyms, + .maxlen = sizeof(int), + .mode = 0600, + .proc_handler = proc_dointvec, + },# endif #endif {
Signature · 1 line
1.9.3
Daniel Borkmann
maintainerPATCH 2/3 bpf: remove stubs for cBPF from arch code
In reply to Daniel BorkmannRemove the dummy bpf_jit_compile() stubs for eBPF JITs and make that a single __weak function in the core that can be overridden similarly to the eBPF one. Also remove stale pr_err() mentions of bpf_jit_compile.
--- arch/arm64/net/bpf_jit_comp.c | 5 ----- arch/powerpc/net/bpf_jit_comp64.c | 2 -- arch/s390/net/bpf_jit_comp.c | 8 -------- arch/x86/net/bpf_jit_comp.c | 8 ++------ include/linux/filter.h | 6 +----- kernel/bpf/core.c | 12 +++++++++++- 6 files changed, 14 insertions(+), 27 deletions(-)
arch/arm64/net/bpf_jit_comp.c +0 -5
@@ -813,11 +813,6 @@ static inline void bpf_flush_icache(void *start, void *end)flush_icache_range ((unsigned long )start, (unsigned long )end); } -void bpf_jit_compile (struct bpf_prog *prog) -{ -/* Nothing to do here. We support Internal BPF. */ -} -struct bpf_prog *bpf_int_jit_compile (struct bpf_prog *prog) {struct bpf_prog *tmp, *orig_prog = prog;
arch/powerpc/net/bpf_jit_comp64.c +0 -2
@@ -961,8 +961,6 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,return 0 ; } -void bpf_jit_compile (struct bpf_prog *fp) { } -struct bpf_prog *bpf_int_jit_compile (struct bpf_prog *fp) {u32 proglen;
arch/s390/net/bpf_jit_comp.c +0 -8
@@ -1263,14 +1263,6 @@ static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp) }/* -* Classic BPF function stub. BPF programs will be converted into -* eBPF and then bpf_int_jit_compile() will be called. -*/ -void bpf_jit_compile (struct bpf_prog *fp) -{ -} - -/* * Compile eBPF program "fp" */ struct bpf_prog *bpf_int_jit_compile (struct bpf_prog *fp)
arch/x86/net/bpf_jit_comp.c +2 -6
@@ -1067,13 +1067,13 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, ilen = prog - temp;if (ilen >BPF_MAX_INSN_SIZE ) { -pr_err ("bpf_jit_compile fatal insn size error\n" ); +pr_err ("bpf_jit: fatal insn size error\n" );return -EFAULT ; }if (image) {if (unlikely (proglen + ilen > oldproglen)) { -pr_err ("bpf_jit_compile fatal error\n" ); +pr_err ("bpf_jit: fatal error\n" );return -EFAULT ; }memcpy (image + proglen, temp, ilen);
@@ -1085,10 +1085,6 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,return proglen; } -void bpf_jit_compile (struct bpf_prog *prog) -{ -} -struct bpf_prog *bpf_int_jit_compile (struct bpf_prog *prog) {struct bpf_binary_header *header =NULL ;
include/linux/filter.h +1 -5
@@ -607,6 +607,7 @@ int sk_get_filter(struct sock *sk, struct sock_filter __user *filter,u64 __bpf_call_base (u64 r1,u64 r2,u64 r3,u64 r4,u64 r5);struct bpf_prog *bpf_int_jit_compile (struct bpf_prog *prog); +void bpf_jit_compile (struct bpf_prog *prog);bool bpf_helper_changes_pkt_data (void *func); struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,
@@ -625,7 +626,6 @@ struct bpf_binary_header * bpf_jit_fill_hole_t bpf_fill_ill_insns);void bpf_jit_binary_free (struct bpf_binary_header *hdr); -void bpf_jit_compile (struct bpf_prog *fp);void bpf_jit_free (struct bpf_prog *fp);struct bpf_prog *bpf_jit_blind_constants (struct bpf_prog *fp);
@@ -669,10 +669,6 @@ static inline bool bpf_jit_blinding_enabled(void) return true; }#else -static inline void bpf_jit_compile (struct bpf_prog *fp) -{ -} -static inline void bpf_jit_free (struct bpf_prog *fp) {bpf_prog_unlock_free (fp);
kernel/bpf/core.c +11 -1
@@ -1154,12 +1154,22 @@ const struct bpf_func_proto * __weak bpf_get_trace_printk_proto(void) .arg3_type = ARG_ANYTHING, }; -/* For classic BPF JITs that don't implement bpf_int_jit_compile(). */ +/* Stub for JITs that only support cBPF. eBPF programs are interpreted. +* It is encouraged to implement bpf_int_jit_compile() instead, so that +* eBPF and implicitly also cBPF can get JITed! +*/ struct bpf_prog *__weak bpf_int_jit_compile (struct bpf_prog *prog) {return prog; } +/* Stub for JITs that support eBPF. All cBPF code gets transformed into +* eBPF by the kernel and is later compiled by bpf_int_jit_compile(). +*/ +void __weak bpf_jit_compile (struct bpf_prog *prog) +{ +} +bool __weak bpf_helper_changes_pkt_data (void *func) {return false;
Signature · 1 line
1.9.3
David Miller
maintainerFrom: Daniel Borkmann <[email protected]>
Date: Thu, 16 Feb 2017 22:24:47 +0100
Quoted 10 lines
This last series for this window adds various misc improvements to BPF, one is to mark registered map and prog types as
__ro_after_init, another one for removing cBPF stubs in eBPF JITs and moving the stub to the core and last also improving JITs is to make generated images visible to the kernel and kallsyms, so they can be seen in traces. For details, please have a look at the individual patches.Thanks a lot!
Series applied, thanks Daniel.
Re: [PATCH net-next 3/3] bpf: make jited programs visible in traces
In reply to Daniel BorkmannQuoted 61 lines · Daniel Borkmann
Long standing issue with JITed programs is that stack traces from function tracing check whether a given address is kernel code through {__,}kernel_text_address(), which checks for code in core kernel, modules and dynamically allocated ftrace trampolines. But what is still missing is BPF JITed programs (interpreted programs are not an issue as
__bpf_prog_run()will be attributed to them), thus when a stack trace is triggered, the code walking the stack won't see any of the JITed ones. The same for address correlation done from user space via reading/proc/kallsyms. This is read by tools like perf, but the latter is also useful for permanent live tracing with eBPF itself in combination with stack maps when other eBPF types are part of the callchain. See offwaketime example on dumping stack from a map.This work tries to tackle that issue by making the addresses and symbols known to the kernel. The lookup from
*kernel_text_address()is implemented through a latched RB tree that can be read under RCU in fast-path that is also shared for symbol/size/offset lookup for a specific given address in kallsyms. The slow-path iteration through all symbols in the seq file done via RCU list, which holds a tiny fraction of all exported ksyms, usually below 0.1 percent. Function symbols are exported as bpf_prog_<tag>, in order to aide debugging and attribution. This facility is currently enabled for root-only whenbpf_jit_kallsymsis set to 1, and disabled if hardening is active in any mode. The rationale behind this is that still a lot of systems ship with world read permissions on kallsyms thus addresses should not get suddenly exposed for them. If that situation gets much better in future, we always have the option to change the default on this. Likewise, unprivileged programs are not allowed to add entries there either, but that is less of a concern as most such programs types relevant in this context are for root-only anyway. If enabled, call graphs and stack traces will then show a correct attribution; one example is illustrated below, where the trace is now visible in tooling such as perf script --kallsyms=/proc/kallsyms and friends.Before:
7fff8166889dbpf_clone_redirect+0x80007f0020ed(/lib/modules/4.9.0-rc8+/build/vmlinux)f5d80 __sendmsg_nocancel+0xffff006451f1a007 (/usr/lib64/libc-2.18.so)After:
7fff816688b7bpf_clone_redirect+0x80007f002107(/lib/modules/4.9.0-rc8+/build/vmlinux)7fffa0575728bpf_prog_33c45a467c9e061a+0x8000600020fb(/lib/modules/4.9.0-rc8+/build/vmlinux)7fffa07ef1fccls_bpf_classify+0x8000600020dc(/lib/modules/4.9.0-rc8+/build/vmlinux)7fff81678b68tc_classify+0x80007f002078(/lib/modules/4.9.0-rc8+/build/vmlinux)7fff8164d40b__netif_receive_skb_core+0x80007f0025fb(/lib/modules/4.9.0-rc8+/build/vmlinux)7fff8164d718__netif_receive_skb+0x80007f002018(/lib/modules/4.9.0-rc8+/build/vmlinux)7fff8164e565process_backlog+0x80007f002095(/lib/modules/4.9.0-rc8+/build/vmlinux)7fff8164dc71net_rx_action+0x80007f002231(/lib/modules/4.9.0-rc8+/build/vmlinux)7fff81767461__softirqentry_text_start+0x80007f0020d1(/lib/modules/4.9.0-rc8+/build/vmlinux)7fff817658acdo_softirq_own_stack+0x80007f00201c(/lib/modules/4.9.0-rc8+/build/vmlinux)7fff810a2c20do_softirq+0x80007f002050(/lib/modules/4.9.0-rc8+/build/vmlinux)7fff810a2cb5__local_bh_enable_ip+0x80007f002085(/lib/modules/4.9.0-rc8+/build/vmlinux)7fff8168d452ip_finish_output2+0x80007f002152(/lib/modules/4.9.0-rc8+/build/vmlinux)7fff8168ea3dip_finish_output+0x80007f00217d(/lib/modules/4.9.0-rc8+/build/vmlinux)7fff8168f2afip_output+0x80007f00203f(/lib/modules/4.9.0-rc8+/build/vmlinux) [...]
7fff81005854do_syscall_64+0x80007f002054(/lib/modules/4.9.0-rc8+/build/vmlinux)7fff817649ebreturn_from_SYSCALL_64+0x80007f002000(/lib/modules/4.9.0-rc8+/build/vmlinux)
Quoted 6 lines
f5d80 __sendmsg_nocancel+0xffff01c484812007 (/usr/lib64/libc-2.18.so)Signed-off-by Daniel Borkmann <[email protected]>---
Latest net-next tree dies on my hosts, and my bisection came to this commit.
[ 90.045546] BUG: unable to handle kernel paging request at ffff881fef01a000^M [ 90.052535] IP: __tlb_remove_page_size+0x57/0x90^M [ 90.057152] PGD 2247067 ^M [ 90.057153] PUD 1fdaadc063 ^M [ 90.059691] PMD 1fefb0b063 ^M [ 90.062491] PTE 8000001fef01a161^M [ 90.065287] ^M [ 90.070011] Oops: 0003 [#1] SMP^M [ 90.073478] gsmi: Log Shutdown Reason 0x03^M [ 90.077584] Modules linked in: w1_therm wire cdc_acm ehci_pci ehci_hcd mlx4_en ib_uverbs mlx4_ib ib_core mlx4_core^M [ 90.087972] CPU: 34 PID: 9747 Comm: sshd Not tainted 4.10.0-smp-DEV #14^M [ 90.101580] task: ffff881fda56a300 task.stack: ffffc900337d4000^M [ 90.107515] RIP: 0010:__tlb_remove_page_size+0x57/0x90^M [ 90.112651] RSP: 0018:ffffc900337d7c98 EFLAGS: 00010202^M [ 90.117896] RAX: ffff881fef01a000 RBX: ffffc900337d7df8 RCX: 0000000000000001^M [ 90.125086] RDX: ffff880000000000 RSI: 0000000000000011 RDI: ffff88207fffe4c0^M [ 90.132234] RBP: ffffc900337d7ca0 R08: 0000000000000010 R09: ffffc900337d7bd8^M [ 90.139371] R10: 0000000000000020 R11: 0000000000000001 R12: ffff881fda064520^M [ 90.146544] R13: ffffea00ffb28f40 R14: 00007f84584a5000 R15: ffffc900337d7df8^M [ 90.153703] FS: 0000000000000000(0000) GS:ffff881fffd80000(0000) knlGS:0000000000000000^M [ 90.161802] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033^M [ 90.167548] CR2: ffff881fef01a000 CR3: 0000000001c09000 CR4: 00000000001406e0^M [ 90.174680] Call Trace:^M [ 90.177144] unmap_page_range+0x679/0x840^M [ 90.181154] unmap_single_vma+0x7f/0xf0^M [ 90.184984] unmap_vmas+0x4a/0xa0^M [ 90.188292] exit_mmap+0xa2/0x160^M [ 90.191605] mmput+0x3d/0x100^M[ 90.194584] do_exit+0x325/0xbc0^M [ 90.197810] ? vfs_read+0x95/0x140^M [ 90.201230] do_group_exit+0x49/0xc0^M [ 90.204818] SyS_exit_group+0x14/0x20^M [ 90.208492] entry_SYSCALL_64_fastpath+0x13/0x94^M [ 90.213127] RIP: 0033:0x7f8457b10279^M [ 90.216723] RSP: 002b:00007ffef283a8f0 EFLAGS: 00000246 ORIG_RAX: 00000000000000e7^M [ 90.224286] RAX: ffffffffffffffda RBX: 000055692c599640 RCX: 00007f8457b10279^M [ 90.231432] RDX: 0000000000000000 RSI: 00000000000000ff RDI: 00000000000000ff^M [ 90.238575] RBP: 00007ffef283a9f0 R08: 000000000000003c R09: 00000000000000e7^M [ 90.245728] R10: ffffffffffffff90 R11: 0000000000000246 R12: 000055692c599640^M [ 90.252875] R13: 0000000000002614 R14: 000000000000ac60 R15: 00007ffef283aa90^M [ 90.260018] Code: 89 47 20 31 c0 c3 83 7f 78 13 74 45 55 53 31 f6 48 89 fb bf 00 02 00 01 48 8d 6c 24 08 e8 c2 05 fd ff 48 85 c0 74 30 83 43 78 01 <48> c7 00 00 00 00 00 c7 40 08 00 00 00 00 c7 40 0c fe 01 00 00 ^M [ 90.278939] RIP: __tlb_remove_page_size+0x57/0x90 RSP: ffffc900337d7c98^M [ 90.285550] CR2: ffff881fef01a000^M
Daniel Borkmann
maintainerRe: [PATCH net-next 3/3] bpf: make jited programs visible in traces
On 02/20/2017 08:05 PM, Eric Dumazet wrote:
[...]
Quoted 2 lines
Latest net-next tree dies on my hosts, and my bisection came to this commit.
Thanks for the report, I'll take a look!
Daniel Borkmann
maintainerRe: [PATCH net-next 3/3] bpf: make jited programs visible in traces
Quoted 6 lines · Daniel Borkmann
On 02/20/2017 08:05 PM, Eric Dumazet wrote:
[...]Quoted 2 lines
Latest net-next tree dies on my hosts, and my bisection came to this commit.
Thanks for the report, I'll take a look!
Fyi, fix will come tomorrow.