perf test BPF failing on 4.15.0-rc6
Arnaldo Carvalho de Melo
maintainerHi Wang,
I just updated my machine to Fedora 27 and 4.15.0-rc6 and the
only test failing for me is:
[root@jouet linux]# perf test bpf
39: BPF filter : 39.1: Basic BPF filtering : FAILED! 39.2: BPF pinning : Skip 39.3: BPF prologue generation : Skip 39.4: BPF relocation checker : Skip
[root@jouet linux]#
I haven't checked but perhaps the problem is that SyS_epoll_wait seems to now be inlined in three places and perhaps the eBPF proggie is being added to just one of them?
Seemingly relevant excerpt:
Open Debuginfo file: /lib/modules/4.15.0-rc6/build/vmlinux Try to find probe point from debuginfo.
Matched function: SyS_epoll_wait [2f40eb7]
found inline addr: 0xffffffff812b6ff1
Probe point found: compat_SyS_epoll_pwait+129 found inline addr: 0xffffffff812b6de7
Probe point found: SyS_epoll_pwait+135
found inline addr: 0xffffffff812b6c80
Probe point found: SyS_epoll_wait+0
Found 3 probe_trace_events.
- Arnaldo
P.S.: Full -v output:
[root@jouet linux]# uname -a
Linux jouet 4.15.0-rc6 #4 SMP Tue Jan 2 14:30:53 -03 2018 x86_64 x86_64 x86_64 GNU/Linux
[root@jouet linux]# gcc -v
Using built-in specs.
COLLECT_GCC=/usr/bin/gcc
COLLECT_LTO_WRAPPER=/usr/libexec/gcc/x86_64-redhat-linux/7/lto-wrapper
OFFLOAD_TARGET_NAMES=nvptx-none
OFFLOAD_TARGET_DEFAULT=1
Target: x86_64-redhat-linux
Configured with: ../configure --enable-bootstrap --enable-languages=c,c++,objc,obj-c++,fortran,ada,go,lto --prefix=/usr --mandir=/usr/share/man --infodir=/usr/share/info --with-bugurl=http://bugzilla.redhat.com/bugzilla --enable-shared --enable-threads=posix --enable-checking=release --enable-multilib --with-system-zlib --enable-__cxa_atexit --disable-libunwind-exceptions --enable-gnu-unique-object --enable-linker-build-id --with-gcc-major-version-only --with-linker-hash-style=gnu --enable-plugin --enable-initfini-array --with-isl --enable-libmpx --enable-offload-targets=nvptx-none --without-cuda-driver --enable-gnu-indirect-function --with-tune=generic --with-arch_32=i686 --build=x86_64-redhat-linux Thread model: posix
gcc version 7.2.1 20170915 (Red Hat 7.2.1-2) (GCC)
[root@jouet linux]# clang -v
clang version 6.0.0 (http://llvm.org/git/clang.git 56cc8f8880db2ebc433eeb6b6a707c101467a186) (http://llvm.org/git/llvm.git 3656d83960a4f3fedf6d8f19043abf52379f78c3)
Target: x86_64-unknown-linux-gnu
Thread model: posix
InstalledDir: /usr/local/bin
Found candidate GCC installation: /usr/lib/gcc/x86_64-redhat-linux/7
Selected GCC installation: /usr/lib/gcc/x86_64-redhat-linux/7
Candidate multilib: .;@m64
Candidate multilib: 32;@m32
Selected multilib: .;@m64
[root@jouet linux]# perf test -v BPF
39: BPF filter :
39.1: Basic BPF filtering :
--- start ---
test child forked, pid 24304
Kernel build dir is set to /lib/modules/4.15.0-rc6/build
set env: KBUILD_DIR=/lib/modules/4.15.0-rc6/build
unset env: KBUILD_OPTS
include option is set to -nostdinc -isystem /usr/lib/gcc/x86_64-redhat-linux/7/include -I/home/acme/git/linux/arch/x86/include -I./arch/x86/include/generated -I/home/acme/git/linux/include -I./include -I/home/acme/git/linux/arch/x86/include/uapi -I./arch/x86/include/generated/uapi -I/home/acme/git/linux/include/uapi -I./include/generated/uapi -include /home/acme/git/linux/include/linux/kconfig.h
set env: NR_CPUS=4
set env: LINUX_VERSION_CODE=0x40f00
set env: CLANG_EXEC=/usr/local/bin/clang
set env: CLANG_OPTIONS=-xc
set env: KERNEL_INC_OPTIONS= -nostdinc -isystem /usr/lib/gcc/x86_64-redhat-linux/7/include -I/home/acme/git/linux/arch/x86/include -I./arch/x86/include/generated -I/home/acme/git/linux/include -I./include -I/home/acme/git/linux/arch/x86/include/uapi -I./arch/x86/include/generated/uapi -I/home/acme/git/linux/include/uapi -I./include/generated/uapi -include /home/acme/git/linux/include/linux/kconfig.h
set env: WORKING_DIR=/lib/modules/4.15.0-rc6/build
set env: CLANG_SOURCE=-
llvm compiling command template: echo '/*
* bpf-script-example.c
* Test basic LLVM building
*/
#ifndef LINUX_VERSION_CODE
# error Need LINUX_VERSION_CODE # error Example: for 4.2 kernel, put 'clang-opt="-DLINUX_VERSION_CODE=0x40200" into llvm section of ~/.perfconfig' #endif #define BPF_ANY 0 #define BPF_MAP_TYPE_ARRAY 2 #define BPF_FUNC_map_lookup_elem 1 #define BPF_FUNC_map_update_elem 2
static void *(*bpf_map_lookup_elem)(void *map, void *key) =
(void *) BPF_FUNC_map_lookup_elem; static void *(*bpf_map_update_elem)(void *map, void *key, void *value, int flags) = (void *) BPF_FUNC_map_update_elem;
struct bpf_map_def {
unsigned int type;
unsigned int key_size;
unsigned int value_size;
unsigned int max_entries;
};#define SEC(NAME) __attribute__((section(NAME), used)) struct bpf_map_def SEC("maps") flip_table = {
.type = BPF_MAP_TYPE_ARRAY,
.key_size = sizeof(int),
.value_size = sizeof(int), .max_entries = 1,
};
SEC("func=SyS_epoll_wait")
int bpf_func__SyS_epoll_wait(void *ctx)
{
int ind =0;
int *flag = bpf_map_lookup_elem(&flip_table, &ind);
int new_flag;
if (!flag)
return 0;
/* flip flag and store back */
new_flag = !*flag;
bpf_map_update_elem(&flip_table, &ind, &new_flag, BPF_ANY);
return new_flag;
}char _license[] SEC("license") = "GPL";
int _version SEC("version") = LINUX_VERSION_CODE;
' | $CLANG_EXEC -D__KERNEL__ -D__NR_CPUS__=$NR_CPUS -DLINUX_VERSION_CODE=$LINUX_VERSION_CODE $CLANG_OPTIONS $KERNEL_INC_OPTIONS -Wno-unused-value -Wno-pointer-sign -working-directory $WORKING_DIR -c "$CLANG_SOURCE" -target bpf -O2 -o -
libbpf: loading object '[basic_bpf_test]' from buffer
libbpf: section .strtab, size 120, link 0, flags 0, type=3
libbpf: section .text, size 0, link 0, flags 6, type=1
libbpf: section func=SyS_epoll_wait, size 192, link 0, flags 6, type=1
libbpf: found program func=SyS_epoll_wait
libbpf: section .relfunc=SyS_epoll_wait, size 32, link 8, flags 0, type=9
libbpf: section maps, size 16, link 0, flags 3, type=1
libbpf: section license, size 4, link 0, flags 3, type=1
libbpf: license of [basic_bpf_test] is GPL
libbpf: section version, size 4, link 0, flags 3, type=1
libbpf: kernel version of [basic_bpf_test] is 40f00
libbpf: section .symtab, size 168, link 1, flags 0, type=2
libbpf: maps in [basic_bpf_test]: 1 maps in 16 bytes
libbpf: map 0 is "flip_table"
libbpf: collecting relocating info for: 'func=SyS_epoll_wait'
libbpf: relocation: insn_idx=4
libbpf: relocation: find map 0 (flip_table) for insn 4
libbpf: relocation: insn_idx=17
libbpf: relocation: find map 0 (flip_table) for insn 17
bpf: config program 'func=SyS_epoll_wait'
symbol:SyS_epoll_wait file:(null) line:0 offset:0 return:0 lazy:(null)
bpf: config 'func=SyS_epoll_wait' is ok
Looking at the vmlinux_path (8 entries long)
Using /lib/modules/4.15.0-rc6/build/vmlinux for symbols
Open Debuginfo file: /lib/modules/4.15.0-rc6/build/vmlinux
Try to find probe point from debuginfo.
Matched function: SyS_epoll_wait [2f40eb7]
found inline addr: 0xffffffff812b6ff1
Probe point found: compat_SyS_epoll_pwait+129
found inline addr: 0xffffffff812b6de7
Probe point found: SyS_epoll_pwait+135
found inline addr: 0xffffffff812b6c80
Probe point found: SyS_epoll_wait+0
Found 3 probe_trace_events.
Opening /sys/kernel/debug/tracing//kprobe_events write=1
Writing event: p:perf_bpf_probe/func _text+2846705
Writing event: p:perf_bpf_probe/func_1 _text+2846183
Writing event: p:perf_bpf_probe/func_2 _text+2845824
libbpf: create map flip_table: fd=3
add bpf event perf_bpf_probe:func and attach bpf program 4
registering plugin: /root/.traceevent/plugins/plugin_kvm.so
registering plugin: /root/.traceevent/plugins/plugin_hrtimer.so
registering plugin: /root/.traceevent/plugins/plugin_jbd2.so
registering plugin: /root/.traceevent/plugins/plugin_kmem.so
registering plugin: /root/.traceevent/plugins/plugin_scsi.so
registering plugin: /root/.traceevent/plugins/plugin_sched_switch.so
registering plugin: /root/.traceevent/plugins/plugin_function.so
registering plugin: /root/.traceevent/plugins/plugin_xen.so
registering plugin: /root/.traceevent/plugins/plugin_mac80211.so
registering plugin: /root/.traceevent/plugins/plugin_cfg80211.so
adding perf_bpf_probe:func
adding perf_bpf_probe:func to 0x24e3850
add bpf event perf_bpf_probe:func_1 and attach bpf program 4
adding perf_bpf_probe:func_1
adding perf_bpf_probe:func_1 to 0x1965020
add bpf event perf_bpf_probe:func_2 and attach bpf program 4
adding perf_bpf_probe:func_2
adding perf_bpf_probe:func_2 to 0x19656a0
mmap size 1052672B
BPF filter result incorrect, expected 56, got 0 samples
Opening /sys/kernel/debug/tracing//kprobe_events write=1
Opening /sys/kernel/debug/tracing//uprobe_events write=1
Parsing probe_events: p:perf_bpf_probe/func _text+2846705
Group:perf_bpf_probe Event:func probe:p
Parsing probe_events: p:perf_bpf_probe/func_1 _text+2846183
Group:perf_bpf_probe Event:func_1 probe:p
Parsing probe_events: p:perf_bpf_probe/func_2 _text+2845824
Group:perf_bpf_probe Event:func_2 probe:p
Writing event: -:perf_bpf_probe/func
Opening /sys/kernel/debug/tracing//kprobe_events write=1
Opening /sys/kernel/debug/tracing//uprobe_events write=1
Parsing probe_events: p:perf_bpf_probe/func_1 _text+2846183
Group:perf_bpf_probe Event:func_1 probe:p
Parsing probe_events: p:perf_bpf_probe/func_2 _text+2845824
Group:perf_bpf_probe Event:func_2 probe:p
Writing event: -:perf_bpf_probe/func_1
Opening /sys/kernel/debug/tracing//kprobe_events write=1
Opening /sys/kernel/debug/tracing//uprobe_events write=1
Parsing probe_events: p:perf_bpf_probe/func_2 _text+2845824
Group:perf_bpf_probe Event:func_2 probe:p
Writing event: -:perf_bpf_probe/func_2
test child finished with -1
---- end ----
BPF filter subtest 0: FAILED!
39.2: BPF pinning :
--- force skipped ---
BPF filter subtest 1: Skip
39.3: BPF prologue generation :
--- force skipped ---
BPF filter subtest 2: Skip
39.4: BPF relocation checker :
--- force skipped ---
BPF filter subtest 3: Skip
[root@jouet linux]#
[root@jouet linux]#
Unable to reproduce. In my kernel configuration, SyS_epoll_wait is not inlined at all.
From the log you sent, it seems that all 3 instances are attached. This testcase should work if the last one (SyS_epoll_wait) get probed correctly.
Could you please have a look if the 3rd kprobe event (p:perf_bpf_probe/func_2 _text+2845824) works?
Thank you.
Quoted 216 lines · Arnaldo Carvalho de Melo
Hi Wang,
I just updated my machine to Fedora 27 and 4.15.0-rc6 and the
only test failing for me is:[root@jouet linux]# perf test bpf
39: BPF filter : 39.1: Basic BPF filtering : FAILED! 39.2: BPF pinning : Skip 39.3: BPF prologue generation : Skip 39.4: BPF relocation checker : Skip[root@jouet linux]#
I haven't checked but perhaps the problem is that
SyS_epoll_waitseems to now be inlined in three places and perhaps the eBPF proggie is being added to just one of them?Seemingly relevant excerpt:
Open Debuginfo file:
/lib/modules/4.15.0-rc6/build/vmlinuxTry to find probe point from debuginfo.
Matched function:SyS_epoll_wait[2f40eb7]
found inline addr:0xffffffff812b6ff1
Probe point found: compat_SyS_epoll_pwait+129 found inline addr:0xffffffff812b6de7
Probe point found: SyS_epoll_pwait+135
found inline addr:0xffffffff812b6c80
Probe point found: SyS_epoll_wait+0
Found 3probe_trace_events.- Arnaldo
P.S.: Full -v output:[root@jouet linux]# uname -a
Linux jouet 4.15.0-rc6 #4 SMP Tue Jan 2 14:30:53 -03 2018x86_64x86_64x86_64GNU/Linux
[root@jouet linux]# gcc -v
Using built-in specs.
COLLECT_GCC=/usr/bin/gcc
COLLECT_LTO_WRAPPER=/usr/libexec/gcc/x86_64-redhat-linux/7/lto-wrapper
OFFLOAD_TARGET_NAMES=nvptx-none
OFFLOAD_TARGET_DEFAULT=1
Target: x86_64-redhat-linux
Configured with: ../configure --enable-bootstrap --enable-languages=c,c++,objc,obj-c++,fortran,ada,go,lto --prefix=/usr --mandir=/usr/share/man --infodir=/usr/share/info --with-bugurl=http://bugzilla.redhat.com/bugzilla --enable-shared --enable-threads=posix --enable-checking=release --enable-multilib --with-system-zlib --enable-__cxa_atexit --disable-libunwind-exceptions --enable-gnu-unique-object --enable-linker-build-id --with-gcc-major-version-only --with-linker-hash-style=gnu --enable-plugin --enable-initfini-array --with-isl --enable-libmpx --enable-offload-targets=nvptx-none --without-cuda-driver --enable-gnu-indirect-function --with-tune=generic --with-arch_32=i686 --build=x86_64-redhat-linux Thread model: posix
gcc version 7.2.1 20170915 (Red Hat 7.2.1-2) (GCC)
[root@jouet linux]# clang -v
clang version 6.0.0 (http://llvm.org/git/clang.git56cc8f8880db2ebc433eeb6b6a707c101467a186) (http://llvm.org/git/llvm.git3656d83960a4f3fedf6d8f19043abf52379f78c3)
Target: x86_64-unknown-linux-gnu
Thread model: posix
InstalledDir:/usr/local/bin
Found candidate GCC installation:/usr/lib/gcc/x86_64-redhat-linux/7
Selected GCC installation:/usr/lib/gcc/x86_64-redhat-linux/7
Candidate multilib: .;@m64
Candidate multilib: 32;@m32
Selected multilib: .;@m64
[root@jouet linux]# perf test -v BPF
39: BPF filter :
39.1: Basic BPF filtering :
--- start ---
test child forked, pid 24304
Kernel build dir is set to/lib/modules/4.15.0-rc6/build
set env: KBUILD_DIR=/lib/modules/4.15.0-rc6/build
unset env:KBUILD_OPTS
include option is set to -nostdinc -isystem/usr/lib/gcc/x86_64-redhat-linux/7/include-I/home/acme/git/linux/arch/x86/include -I./arch/x86/include/generated -I/home/acme/git/linux/include -I./include -I/home/acme/git/linux/arch/x86/include/uapi -I./arch/x86/include/generated/uapi -I/home/acme/git/linux/include/uapi -I./include/generated/uapi -include/home/acme/git/linux/include/linux/kconfig.h
set env: NR_CPUS=4
set env: LINUX_VERSION_CODE=0x40f00
set env: CLANG_EXEC=/usr/local/bin/clang
set env: CLANG_OPTIONS=-xc
set env: KERNEL_INC_OPTIONS= -nostdinc -isystem/usr/lib/gcc/x86_64-redhat-linux/7/include-I/home/acme/git/linux/arch/x86/include -I./arch/x86/include/generated -I/home/acme/git/linux/include -I./include -I/home/acme/git/linux/arch/x86/include/uapi -I./arch/x86/include/generated/uapi -I/home/acme/git/linux/include/uapi -I./include/generated/uapi -include/home/acme/git/linux/include/linux/kconfig.h
set env: WORKING_DIR=/lib/modules/4.15.0-rc6/build
set env: CLANG_SOURCE=-
llvm compiling command template: echo '/*
*bpf-script-example.c
* Test basic LLVM building
*/
#ifndefLINUX_VERSION_CODE# error Need LINUX_VERSION_CODE # error Example: for 4.2 kernel, put 'clang-opt="-DLINUX_VERSION_CODE=0x40200" into llvm section of ~/.perfconfig' #endif #define BPF_ANY 0 #define BPF_MAP_TYPE_ARRAY 2 #define BPF_FUNC_map_lookup_elem 1 #define BPF_FUNC_map_update_elem 2static void *(*bpf_map_lookup_elem)(void *map, void *key) =
(void *) BPF_FUNC_map_lookup_elem; static void *(*bpf_map_update_elem)(void *map, void *key, void *value, int flags) = (void *) BPF_FUNC_map_update_elem;struct bpf_map_def { unsigned int type; unsigned int key_size; unsigned int value_size; unsigned int max_entries; };#define SEC(NAME)
__attribute__((section(NAME), used))structbpf_map_defSEC("maps")flip_table= {
.type =BPF_MAP_TYPE_ARRAY,
.key_size = sizeof(int),
.value_size = sizeof(int), .max_entries = 1,
};
SEC("func=SyS_epoll_wait")
intbpf_func__SyS_epoll_wait(void *ctx){ int ind =0; int *flag = bpf_map_lookup_elem(&flip_table, &ind); int new_flag; if (!flag) return 0; /* flip flag and store back */ new_flag = !*flag; bpf_map_update_elem(&flip_table, &ind, &new_flag, BPF_ANY); return new_flag; }char _license[]
SEC("license")= "GPL";
int_versionSEC("version")=LINUX_VERSION_CODE;
' | $CLANG_EXEC -D__KERNEL__ -D__NR_CPUS__=$NR_CPUS -DLINUX_VERSION_CODE=$LINUX_VERSION_CODE $CLANG_OPTIONS $KERNEL_INC_OPTIONS -Wno-unused-value -Wno-pointer-sign -working-directory $WORKING_DIR -c "$CLANG_SOURCE" -target bpf -O2 -o -
libbpf: loading object '[basic_bpf_test]' from buffer
libbpf: section .strtab, size 120, link 0, flags 0, type=3
libbpf: section .text, size 0, link 0, flags 6, type=1
libbpf: section func=SyS_epoll_wait, size 192, link 0, flags 6, type=1
libbpf: found program func=SyS_epoll_wait
libbpf: section .relfunc=SyS_epoll_wait, size 32, link 8, flags 0, type=9
libbpf: section maps, size 16, link 0, flags 3, type=1
libbpf: section license, size 4, link 0, flags 3, type=1
libbpf: license of [basic_bpf_test] is GPL
libbpf: section version, size 4, link 0, flags 3, type=1
libbpf: kernel version of [basic_bpf_test] is 40f00
libbpf: section .symtab, size 168, link 1, flags 0, type=2
libbpf: maps in [basic_bpf_test]: 1 maps in 16 bytes
libbpf: map 0 is "flip_table"
libbpf: collecting relocating info for: 'func=SyS_epoll_wait'
libbpf: relocation: insn_idx=4
libbpf: relocation: find map 0 (flip_table) for insn 4
libbpf: relocation: insn_idx=17
libbpf: relocation: find map 0 (flip_table) for insn 17
bpf: config program 'func=SyS_epoll_wait'
symbol:SyS_epoll_wait file:(null) line:0 offset:0 return:0 lazy:(null)
bpf: config 'func=SyS_epoll_wait' is ok
Looking at thevmlinux_path(8 entries long)
Using/lib/modules/4.15.0-rc6/build/vmlinuxfor symbols
Open Debuginfo file:/lib/modules/4.15.0-rc6/build/vmlinux
Try to find probe point from debuginfo.
Matched function:SyS_epoll_wait[2f40eb7]
found inline addr:0xffffffff812b6ff1
Probe point found: compat_SyS_epoll_pwait+129
found inline addr:0xffffffff812b6de7
Probe point found: SyS_epoll_pwait+135
found inline addr:0xffffffff812b6c80
Probe point found: SyS_epoll_wait+0
Found 3probe_trace_events.
Opening/sys/kernel/debug/tracing//kprobe_eventswrite=1
Writing event: p:perf_bpf_probe/func _text+2846705
Writing event: p:perf_bpf_probe/func_1 _text+2846183
Writing event: p:perf_bpf_probe/func_2 _text+2845824
libbpf: create mapflip_table: fd=3
add bpf event perf_bpf_probe:func and attach bpf program 4
registering plugin:/root/.traceevent/plugins/plugin_kvm.so
registering plugin:/root/.traceevent/plugins/plugin_hrtimer.so
registering plugin:/root/.traceevent/plugins/plugin_jbd2.so
registering plugin:/root/.traceevent/plugins/plugin_kmem.so
registering plugin:/root/.traceevent/plugins/plugin_scsi.so
registering plugin:/root/.traceevent/plugins/plugin_sched_switch.so
registering plugin:/root/.traceevent/plugins/plugin_function.so
registering plugin:/root/.traceevent/plugins/plugin_xen.so
registering plugin:/root/.traceevent/plugins/plugin_mac80211.so
registering plugin:/root/.traceevent/plugins/plugin_cfg80211.so
adding perf_bpf_probe:func
adding perf_bpf_probe:func to0x24e3850
add bpf event perf_bpf_probe:func_1 and attach bpf program 4
adding perf_bpf_probe:func_1
adding perf_bpf_probe:func_1 to0x1965020
add bpf event perf_bpf_probe:func_2 and attach bpf program 4
adding perf_bpf_probe:func_2
adding perf_bpf_probe:func_2 to0x19656a0
mmap size 1052672B
BPF filter result incorrect, expected 56, got 0 samples
Opening/sys/kernel/debug/tracing//kprobe_eventswrite=1
Opening/sys/kernel/debug/tracing//uprobe_eventswrite=1
Parsingprobe_events: p:perf_bpf_probe/func _text+2846705
Group:perf_bpf_probe Event:func probe:p
Parsingprobe_events: p:perf_bpf_probe/func_1 _text+2846183
Group:perf_bpf_probe Event:func_1 probe:p
Parsingprobe_events: p:perf_bpf_probe/func_2 _text+2845824
Group:perf_bpf_probe Event:func_2 probe:p
Writing event: -:perf_bpf_probe/func
Opening/sys/kernel/debug/tracing//kprobe_eventswrite=1
Opening/sys/kernel/debug/tracing//uprobe_eventswrite=1
Parsingprobe_events: p:perf_bpf_probe/func_1 _text+2846183
Group:perf_bpf_probe Event:func_1 probe:p
Parsingprobe_events: p:perf_bpf_probe/func_2 _text+2845824
Group:perf_bpf_probe Event:func_2 probe:p
Writing event: -:perf_bpf_probe/func_1
Opening/sys/kernel/debug/tracing//kprobe_eventswrite=1
Opening/sys/kernel/debug/tracing//uprobe_eventswrite=1
Parsingprobe_events: p:perf_bpf_probe/func_2 _text+2845824
Group:perf_bpf_probe Event:func_2 probe:p
Writing event: -:perf_bpf_probe/func_2
test child finished with -1
---- end ----
BPF filter subtest 0: FAILED!
39.2: BPF pinning :
--- force skipped ---
BPF filter subtest 1: Skip
39.3: BPF prologue generation :
--- force skipped ---
BPF filter subtest 2: Skip
39.4: BPF relocation checker :
--- force skipped ---
BPF filter subtest 3: Skip
[root@jouet linux]#
[root@jouet linux]#
And please check if the kprobe created by
$ perf probe -v SyS_epoll_wait
works for the test program used by this testcase:
#include <sys/epoll.h>
#include <stdlib.h>
#define NR_ITERS 100
static int epoll_wait_loop(void)
{
int i;
/* Should fail NR_ITERS times */
for (i = 0; i < NR_ITERS; i++)
epoll_wait(-(i + 1), NULL, 0, 0);
return 0;
}int main()
{
epoll_wait_loop();
return 0;
}In my configuration it gives 100 samples:
$ sudo ./perf record -e probe:SyS_epoll_wait /tmp/a.out [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.016 MB perf.data (100 samples) ]
I guess in your configuration you would get 3 tracepoints, and can get 0 sample from them.
Thank you.
Quoted 252 lines · Wangnan (F)
Unable to reproduce. In my kernel configuration,
SyS_epoll_waitis not inlined at all.From the log you sent, it seems that all 3 instances are attached. This testcase should work if the last one (
SyS_epoll_wait) get probed correctly.Could you please have a look if the 3rd kprobe event (p:perf_bpf_probe/func_2 _text+2845824) works?
Thank you.
Quoted 242 lines · Arnaldo Carvalho de Melo
Hi Wang,
I just updated my machine to Fedora 27 and 4.15.0-rc6 and the
only test failing for me is:[root@jouet linux]# perf test bpf
39: BPF filter : 39.1: Basic BPF filtering : FAILED! 39.2: BPF pinning : Skip 39.3: BPF prologue generation : Skip 39.4: BPF relocation checker : Skip[root@jouet linux]#
I haven't checked but perhaps the problem is that
SyS_epoll_waitseems to now be inlined in three places and perhaps the eBPF proggie is being added to just one of them?Seemingly relevant excerpt:
Open Debuginfo file:
/lib/modules/4.15.0-rc6/build/vmlinuxTry to find probe point from debuginfo.
Matched function:SyS_epoll_wait[2f40eb7]
found inline addr:0xffffffff812b6ff1
Probe point found: compat_SyS_epoll_pwait+129 found inline addr:0xffffffff812b6de7
Probe point found: SyS_epoll_pwait+135
found inline addr:0xffffffff812b6c80
Probe point found: SyS_epoll_wait+0
Found 3probe_trace_events.- Arnaldo
P.S.: Full -v output:[root@jouet linux]# uname -a
Linux jouet 4.15.0-rc6 #4 SMP Tue Jan 2 14:30:53 -03 2018x86_64x86_64x86_64GNU/Linux
[root@jouet linux]# gcc -v
Using built-in specs.
COLLECT_GCC=/usr/bin/gcc
COLLECT_LTO_WRAPPER=/usr/libexec/gcc/x86_64-redhat-linux/7/lto-wrapper
OFFLOAD_TARGET_NAMES=nvptx-none
OFFLOAD_TARGET_DEFAULT=1
Target: x86_64-redhat-linux
Configured with: ../configure --enable-bootstrap
--enable-languages=c,c++,objc,obj-c++,fortran,ada,go,lto
--prefix=/usr --mandir=/usr/share/man --infodir=/usr/share/info
--with-bugurl=http://bugzilla.redhat.com/bugzilla --enable-shared
--enable-threads=posix --enable-checking=release --enable-multilib
--with-system-zlib --enable-__cxa_atexit
--disable-libunwind-exceptions --enable-gnu-unique-object
--enable-linker-build-id --with-gcc-major-version-only
--with-linker-hash-style=gnu --enable-plugin --enable-initfini-array
--with-isl --enable-libmpx --enable-offload-targets=nvptx-none
--without-cuda-driver --enable-gnu-indirect-function
--with-tune=generic --with-arch_32=i686 --build=x86_64-redhat-linux
Thread model: posix
gcc version 7.2.1 20170915 (Red Hat 7.2.1-2) (GCC)
[root@jouet linux]# clang -v
clang version 6.0.0 (http://llvm.org/git/clang.git56cc8f8880db2ebc433eeb6b6a707c101467a186) (http://llvm.org/git/llvm.git3656d83960a4f3fedf6d8f19043abf52379f78c3)
Target: x86_64-unknown-linux-gnu
Thread model: posix
InstalledDir:/usr/local/bin
Found candidate GCC installation:/usr/lib/gcc/x86_64-redhat-linux/7
Selected GCC installation:/usr/lib/gcc/x86_64-redhat-linux/7
Candidate multilib: .;@m64
Candidate multilib: 32;@m32
Selected multilib: .;@m64
[root@jouet linux]# perf test -v BPF
39: BPF filter :
39.1: Basic BPF filtering :
--- start ---
test child forked, pid 24304
Kernel build dir is set to/lib/modules/4.15.0-rc6/build
set env: KBUILD_DIR=/lib/modules/4.15.0-rc6/build
unset env:KBUILD_OPTS
include option is set to -nostdinc -isystem/usr/lib/gcc/x86_64-redhat-linux/7/include(quoted diff) +0
-6-I/home/acme/git/linux/arch/x86/include -I./arch/x86/include/generated -I/home/acme/git/linux/include -I./include -I/home/acme/git/linux/arch/x86/include/uapi -I./arch/x86/include/generated/uapi -I/home/acme/git/linux/include/uapi -I./include/generated/uapi -include /home/acme/git/linux/include/linux/kconfig.hset env: NR_CPUS=4
set env: LINUX_VERSION_CODE=0x40f00
set env: CLANG_EXEC=/usr/local/bin/clang
set env: CLANG_OPTIONS=-xc
set env: KERNEL_INC_OPTIONS= -nostdinc -isystem/usr/lib/gcc/x86_64-redhat-linux/7/include(quoted diff) +0
-6-I/home/acme/git/linux/arch/x86/include -I./arch/x86/include/generated -I/home/acme/git/linux/include -I./include -I/home/acme/git/linux/arch/x86/include/uapi -I./arch/x86/include/generated/uapi -I/home/acme/git/linux/include/uapi -I./include/generated/uapi -include /home/acme/git/linux/include/linux/kconfig.hset env: WORKING_DIR=/lib/modules/4.15.0-rc6/build set env: CLANG_SOURCE=-
llvm compiling command template: echo '/*
*bpf-script-example.c
* Test basic LLVM building
*/
#ifndefLINUX_VERSION_CODE# error Need LINUX_VERSION_CODE # error Example: for 4.2 kernel, put 'clang-opt="-DLINUX_VERSION_CODE=0x40200" into llvm section of ~/.perfconfig' #endif #define BPF_ANY 0 #define BPF_MAP_TYPE_ARRAY 2 #define BPF_FUNC_map_lookup_elem 1 #define BPF_FUNC_map_update_elem 2static void *(*bpf_map_lookup_elem)(void *map, void *key) =
(void *) BPF_FUNC_map_lookup_elem; static void *(*bpf_map_update_elem)(void *map, void *key, void *value, int flags) = (void *) BPF_FUNC_map_update_elem;struct bpf_map_def { unsigned int type; unsigned int key_size; unsigned int value_size; unsigned int max_entries; };#define SEC(NAME)
__attribute__((section(NAME), used))structbpf_map_defSEC("maps")flip_table= {
.type =BPF_MAP_TYPE_ARRAY,
.key_size = sizeof(int),
.value_size = sizeof(int),
.max_entries = 1,
};
SEC("func=SyS_epoll_wait")
intbpf_func__SyS_epoll_wait(void *ctx){ int ind =0; int *flag = bpf_map_lookup_elem(&flip_table, &ind); int new_flag; if (!flag) return 0; /* flip flag and store back */ new_flag = !*flag; bpf_map_update_elem(&flip_table, &ind, &new_flag, BPF_ANY); return new_flag; }char _license[]
SEC("license")= "GPL";
int_versionSEC("version")=LINUX_VERSION_CODE;
' | $CLANG_EXEC -D__KERNEL__ -D__NR_CPUS__=$NR_CPUS -DLINUX_VERSION_CODE=$LINUX_VERSION_CODE $CLANG_OPTIONS $KERNEL_INC_OPTIONS -Wno-unused-value -Wno-pointer-sign -working-directory $WORKING_DIR -c "$CLANG_SOURCE" -target bpf -O2 -o -
libbpf: loading object '[basic_bpf_test]' from buffer
libbpf: section .strtab, size 120, link 0, flags 0, type=3
libbpf: section .text, size 0, link 0, flags 6, type=1
libbpf: section func=SyS_epoll_wait, size 192, link 0, flags 6, type=1
libbpf: found program func=SyS_epoll_wait
libbpf: section .relfunc=SyS_epoll_wait, size 32, link 8, flags 0, type=9
libbpf: section maps, size 16, link 0, flags 3, type=1
libbpf: section license, size 4, link 0, flags 3, type=1
libbpf: license of [basic_bpf_test] is GPL
libbpf: section version, size 4, link 0, flags 3, type=1
libbpf: kernel version of [basic_bpf_test] is 40f00
libbpf: section .symtab, size 168, link 1, flags 0, type=2
libbpf: maps in [basic_bpf_test]: 1 maps in 16 bytes
libbpf: map 0 is "flip_table"
libbpf: collecting relocating info for: 'func=SyS_epoll_wait'
libbpf: relocation: insn_idx=4
libbpf: relocation: find map 0 (flip_table) for insn 4
libbpf: relocation: insn_idx=17
libbpf: relocation: find map 0 (flip_table) for insn 17
bpf: config program 'func=SyS_epoll_wait' symbol:SyS_epoll_wait file:(null) line:0 offset:0 return:0 lazy:(null)
bpf: config 'func=SyS_epoll_wait' is ok
Looking at thevmlinux_path(8 entries long)
Using/lib/modules/4.15.0-rc6/build/vmlinuxfor symbols
Open Debuginfo file:/lib/modules/4.15.0-rc6/build/vmlinuxTry to find probe point from debuginfo.
Matched function:SyS_epoll_wait[2f40eb7]
found inline addr:0xffffffff812b6ff1
Probe point found: compat_SyS_epoll_pwait+129
found inline addr:0xffffffff812b6de7
Probe point found: SyS_epoll_pwait+135
found inline addr:0xffffffff812b6c80
Probe point found: SyS_epoll_wait+0
Found 3probe_trace_events.
Opening/sys/kernel/debug/tracing//kprobe_eventswrite=1 Writing event: p:perf_bpf_probe/func _text+2846705
Writing event: p:perf_bpf_probe/func_1 _text+2846183
Writing event: p:perf_bpf_probe/func_2 _text+2845824
libbpf: create mapflip_table: fd=3
add bpf event perf_bpf_probe:func and attach bpf program 4 registering plugin:/root/.traceevent/plugins/plugin_kvm.soregistering plugin:/root/.traceevent/plugins/plugin_hrtimer.soregistering plugin:/root/.traceevent/plugins/plugin_jbd2.soregistering plugin:/root/.traceevent/plugins/plugin_kmem.soregistering plugin:/root/.traceevent/plugins/plugin_scsi.soregistering plugin:/root/.traceevent/plugins/plugin_sched_switch.soregistering plugin:/root/.traceevent/plugins/plugin_function.soregistering plugin:/root/.traceevent/plugins/plugin_xen.soregistering plugin:/root/.traceevent/plugins/plugin_mac80211.soregistering plugin:/root/.traceevent/plugins/plugin_cfg80211.soadding perf_bpf_probe:func
adding perf_bpf_probe:func to0x24e3850
add bpf event perf_bpf_probe:func_1 and attach bpf program 4 adding perf_bpf_probe:func_1
adding perf_bpf_probe:func_1 to0x1965020
add bpf event perf_bpf_probe:func_2 and attach bpf program 4 adding perf_bpf_probe:func_2
adding perf_bpf_probe:func_2 to0x19656a0
mmap size 1052672B
BPF filter result incorrect, expected 56, got 0 samples Opening/sys/kernel/debug/tracing//kprobe_eventswrite=1 Opening/sys/kernel/debug/tracing//uprobe_eventswrite=1 Parsingprobe_events: p:perf_bpf_probe/func _text+2846705 Group:perf_bpf_probe Event:func probe:p
Parsingprobe_events: p:perf_bpf_probe/func_1 _text+2846183 Group:perf_bpf_probe Event:func_1 probe:p
Parsingprobe_events: p:perf_bpf_probe/func_2 _text+2845824 Group:perf_bpf_probe Event:func_2 probe:p
Writing event: -:perf_bpf_probe/func
Opening/sys/kernel/debug/tracing//kprobe_eventswrite=1 Opening/sys/kernel/debug/tracing//uprobe_eventswrite=1 Parsingprobe_events: p:perf_bpf_probe/func_1 _text+2846183 Group:perf_bpf_probe Event:func_1 probe:p
Parsingprobe_events: p:perf_bpf_probe/func_2 _text+2845824 Group:perf_bpf_probe Event:func_2 probe:p
Writing event: -:perf_bpf_probe/func_1
Opening/sys/kernel/debug/tracing//kprobe_eventswrite=1 Opening/sys/kernel/debug/tracing//uprobe_eventswrite=1 Parsingprobe_events: p:perf_bpf_probe/func_2 _text+2845824 Group:perf_bpf_probe Event:func_2 probe:p
Writing event: -:perf_bpf_probe/func_2
test child finished with -1
---- end ----
BPF filter subtest 0: FAILED!
39.2: BPF pinning :
--- force skipped ---
BPF filter subtest 1: Skip
39.3: BPF prologue generation :
--- force skipped ---
BPF filter subtest 2: Skip
39.4: BPF relocation checker :
--- force skipped ---
BPF filter subtest 3: Skip
[root@jouet linux]#
[root@jouet linux]#
Arnaldo Carvalho de Melo
maintainerEm Wed, Jan 03, 2018 at 12:58:26PM +0800, Wangnan (F) escreveu:
Quoted 3 lines
And please check if the kprobe created by
$ perf probe -v SyS_epoll_wait
[root@seventh ~]# perf probe -v SyS_epoll_wait probe-definition(0): SyS_epoll_wait
symbol:SyS_epoll_wait file:(null) line:0 offset:0 return:0 lazy:(null) 0 arguments
Looking at the vmlinux_path (8 entries long)
Using /usr/lib/debug/lib/modules/4.14.8-300.fc27.x86_64/vmlinux for symbols Open Debuginfo file: /usr/lib/debug/lib/modules/4.14.8-300.fc27.x86_64/vmlinux Try to find probe point from debuginfo.
Matched function: SyS_epoll_wait [314b563]
found inline addr: 0xffffffff812d8087
Probe point found: compat_SyS_epoll_pwait+151
found inline addr: 0xffffffff812d7e5a
Probe point found: SyS_epoll_pwait+138
found inline addr: 0xffffffff812d7cf0
Probe point found: SyS_epoll_wait+0
Found 3 probe_trace_events.
Opening /sys/kernel/debug/tracing//kprobe_events write=1
Writing event: p:probe/SyS_epoll_wait _text+2982023
Writing event: p:probe/SyS_epoll_wait_1 _text+2981466
Writing event: p:probe/SyS_epoll_wait_2 _text+2981104
Added new events:
probe:SyS_epoll_wait (on SyS_epoll_wait)
probe:SyS_epoll_wait_1 (on SyS_epoll_wait) probe:SyS_epoll_wait_2 (on SyS_epoll_wait)
You can now use it in all perf tools, such as:
perf record -e probe:SyS_epoll_wait_2 -aR sleep 1
[root@seventh ~]#
Then, with your proggie, it seems its glibc that now maps epoll_wait() to a different syscall than in older systems:
[root@seventh ~]# trace -e probe:SyS_epoll_wait* ~acme/epoll
0.018 ( 0.001 ms): epoll/20444 brk( ) = 0x2600000
0.029 ( 0.004 ms): epoll/20444 access(filename: 0xe874ec0, mode: R ) = -1 ENOENT No such file or directory
0.035 ( 0.003 ms): epoll/20444 openat(dfd: CWD, filename: 0xe872726, flags: CLOEXEC ) = 3
0.040 ( 0.002 ms): epoll/20444 fstat(fd: 3, statbuf: 0x7fffcc37b830 ) = 0
0.042 ( 0.002 ms): epoll/20444 mmap(len: 109538, prot: READ, flags: PRIVATE, fd: 3 ) = 0x7fc50ea5d000
0.045 ( 0.001 ms): epoll/20444 close(fd: 3 ) = 0
0.052 ( 0.003 ms): epoll/20444 openat(dfd: CWD, filename: 0xea7ace0, flags: CLOEXEC ) = 3
0.056 ( 0.001 ms): epoll/20444 read(fd: 3, buf: 0x7fffcc37b9f8, count: 832 ) = 832
0.059 ( 0.001 ms): epoll/20444 fstat(fd: 3, statbuf: 0x7fffcc37b890 ) = 0
0.060 ( 0.002 ms): epoll/20444 mmap(len: 8192, prot: READ|WRITE, flags: PRIVATE|ANONYMOUS, fd: -1 ) = 0x7fc50ea5b000
0.065 ( 0.003 ms): epoll/20444 mmap(len: 4074112, prot: EXEC|READ, flags: PRIVATE|DENYWRITE, fd: 3 ) = 0x7fc50e46e000
0.068 ( 0.004 ms): epoll/20444 mprotect(start: 0x7fc50e648000, len: 2093056 ) = 0
0.073 ( 0.004 ms): epoll/20444 mmap(addr: 0x7fc50e847000, len: 24576, prot: READ|WRITE, flags: PRIVATE|DENYWRITE|FIXED, fd: 3, off: 1937408) = 0x7fc50e847000
0.079 ( 0.002 ms): epoll/20444 mmap(addr: 0x7fc50e84d000, len: 14976, prot: READ|WRITE, flags: PRIVATE|ANONYMOUS|FIXED, fd: -1) = 0x7fc50e84d000
0.085 ( 0.001 ms): epoll/20444 close(fd: 3 ) = 0
0.093 ( 0.001 ms): epoll/20444 arch_prctl(option: 4098, arg2: 140484331029696 ) = 0
0.128 ( 0.003 ms): epoll/20444 mprotect(start: 0x7fc50e847000, len: 16384, prot: READ ) = 0
0.133 ( 0.002 ms): epoll/20444 mprotect(start: 0x600000, len: 4096, prot: READ ) = 0
0.137 ( 0.002 ms): epoll/20444 mprotect(start: 0x7fc50ea78000, len: 4096, prot: READ ) = 0
0.140 ( 0.005 ms): epoll/20444 munmap(addr: 0x7fc50ea5d000, len: 109538 ) = 0
0.159 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -1, sigsetsize: 8 ) = -22
0.160 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -2, sigsetsize: 8 ) = -22
0.162 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -3, sigsetsize: 8 ) = -22
0.163 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -4, sigsetsize: 8 ) = -22
0.165 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -5, sigsetsize: 8 ) = -22
<SNIP>
0.279 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -85, sigsetsize: 8 ) = -22
0.280 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -86, sigsetsize: 8 ) = -22
0.282 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -87, sigsetsize: 8 ) = -22
0.283 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -88, sigsetsize: 8 ) = -22
0.284 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -89, sigsetsize: 8 ) = -22
0.286 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -90, sigsetsize: 8 ) = -22
0.287 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -91, sigsetsize: 8 ) = -22
0.289 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -92, sigsetsize: 8 ) = -22
0.290 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -93, sigsetsize: 8 ) = -22
0.292 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -94, sigsetsize: 8 ) = -22
0.293 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -95, sigsetsize: 8 ) = -22
0.294 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -96, sigsetsize: 8 ) = -22
0.296 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -97, sigsetsize: 8 ) = -22
0.297 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -98, sigsetsize: 8 ) = -22
0.299 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -99, sigsetsize: 8 ) = -22
0.300 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -100, sigsetsize: 8 ) = -22
0.305 ( ? ): epoll/20444 exit_group( )[root@seventh ~]#
This is on a Fedora 27 system, that has glibc 2.26, while on a RHEL7 machine, that has glibc 2.17, it still uses epoll_wait() and we get:
[root@felicio ~]# trace ./epoll
0.018 ( 0.001 ms): epoll/2818 brk( ) = 0x8e5000
0.033 ( 0.002 ms): epoll/2818 mmap(len: 4096, prot: READ|WRITE, flags: PRIVATE|ANONYMOUS ) = 0x7f3f738e3000
0.041 ( 0.004 ms): epoll/2818 access(filename: 0x736e2160, mode: R ) = -1 ENOENT No such file or directory
0.051 ( 0.003 ms): epoll/2818 open(filename: 0x736e0a58, flags: CLOEXEC ) = 3
0.056 ( 0.001 ms): epoll/2818 fstat(fd: 3, statbuf: 0x7fff8b35f5b0 ) = 0
0.057 ( 0.002 ms): epoll/2818 mmap(len: 78559, prot: READ, flags: PRIVATE, fd: 3 ) = 0x7f3f738cf000
0.060 ( 0.001 ms): epoll/2818 close(fd: 3 ) = 0
0.070 ( 0.005 ms): epoll/2818 open(filename: 0x738e3640, flags: CLOEXEC ) = 3
0.076 ( 0.001 ms): epoll/2818 read(fd: 3, buf: 0x7fff8b35f770, count: 832 ) = 832
0.079 ( 0.001 ms): epoll/2818 fstat(fd: 3, statbuf: 0x7fff8b35f610 ) = 0
0.082 ( 0.003 ms): epoll/2818 mmap(len: 3932672, prot: EXEC|READ, flags: PRIVATE|DENYWRITE, fd: 3 ) = 0x7f3f73304000
0.086 ( 0.005 ms): epoll/2818 mprotect(start: 0x7f3f734bb000, len: 2093056 ) = 0
0.091 ( 0.005 ms): epoll/2818 mmap(addr: 0x7f3f736ba000, len: 24576, prot: READ|WRITE, flags: PRIVATE|DENYWRITE|FIXED, fd: 3, off: 1794048) = 0x7f3f736ba000
0.100 ( 0.003 ms): epoll/2818 mmap(addr: 0x7f3f736c0000, len: 16896, prot: READ|WRITE, flags: PRIVATE|ANONYMOUS|FIXED) = 0x7f3f736c0000
0.106 ( 0.001 ms): epoll/2818 close(fd: 3 ) = 0
0.115 ( 0.002 ms): epoll/2818 mmap(len: 4096, prot: READ|WRITE, flags: PRIVATE|ANONYMOUS ) = 0x7f3f738ce000
0.119 ( 0.002 ms): epoll/2818 mmap(len: 8192, prot: READ|WRITE, flags: PRIVATE|ANONYMOUS ) = 0x7f3f738cc000 0.123 ( 0.001 ms): epoll/2818 arch_prctl(option: 4098, arg2: 139910498273088, arg3: 139910498275408, arg4: 34, arg5: -1) = 0
0.181 ( 0.003 ms): epoll/2818 mprotect(start: 0x7f3f736ba000, len: 16384, prot: READ ) = 0
0.186 ( 0.002 ms): epoll/2818 mprotect(start: 0x600000, len: 4096, prot: READ ) = 0
0.190 ( 0.002 ms): epoll/2818 mprotect(start: 0x7f3f738e4000, len: 4096, prot: READ ) = 0
0.193 ( 0.004 ms): epoll/2818 munmap(addr: 0x7f3f738cf000, len: 78559 ) = 0
0.218 ( 0.001 ms): epoll/2818 epoll_wait(epfd: -1 ) = -1 EINVAL Invalid argument
0.220 ( 0.000 ms): epoll/2818 epoll_wait(epfd: -2 ) = -1 EINVAL Invalid argument
0.221 ( 0.000 ms): epoll/2818 epoll_wait(epfd: -3 ) = -1 EINVAL Invalid argument
0.221 ( 0.000 ms): epoll/2818 epoll_wait(epfd: -4 ) = -1 EINVAL Invalid argument
0.222 ( 0.000 ms): epoll/2818 epoll_wait(epfd: -5 ) = -1 EINVAL Invalid argument
<SNIP>
It thought that this part would cover it:
Probe point found: SyS_epoll_pwait+138
found inline addr: 0xffffffff812d7cf0
Continuing investigation...
- Arnaldo
Quoted 303 lines
works for the test program used by this testcase:
#include <
sys/epoll.h>
#include <stdlib.h>#define
NR_ITERS100static int
epoll_wait_loop(void){ int i; /* Should fail NR_ITERS times */ for (i = 0; i < NR_ITERS; i++) epoll_wait(-(i + 1), NULL, 0, 0); return 0; }int
main(){ epoll_wait_loop(); return 0; }In my configuration it gives 100 samples:
$ sudo ./perf record -e probe:SyS_epoll_wait /tmp/a.out [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.016 MB perf.data (100 samples) ]I guess in your configuration you would get 3 tracepoints, and can get 0 sample from them.
Thank you.
Quoted 267 lines · Wangnan (F)
Unable to reproduce. In my kernel configuration,
SyS_epoll_waitis not inlined at all.From the log you sent, it seems that all 3 instances are attached. This testcase should work if the last one (
SyS_epoll_wait) get probed correctly.Could you please have a look if the 3rd kprobe event (p:perf_bpf_probe/func_2 _text+2845824) works?
Thank you.
Quoted 253 lines · Arnaldo Carvalho de Melo
Hi Wang,
I just updated my machine to Fedora 27 and 4.15.0-rc6 and the
only test failing for me is:[root@jouet linux]# perf test bpf
39: BPF filter : 39.1: Basic BPF filtering : FAILED! 39.2: BPF pinning : Skip 39.3: BPF prologue generation : Skip 39.4: BPF relocation checker : Skip[root@jouet linux]#
I haven't checked but perhaps the problem is that
SyS_epoll_waitseems to now be inlined in three places and perhaps the eBPF proggie is being added to just one of them?Seemingly relevant excerpt:
Open Debuginfo file:
/lib/modules/4.15.0-rc6/build/vmlinuxTry to find probe point from debuginfo.
Matched function:SyS_epoll_wait[2f40eb7]
found inline addr:0xffffffff812b6ff1
Probe point found: compat_SyS_epoll_pwait+129 found inline addr:0xffffffff812b6de7
Probe point found: SyS_epoll_pwait+135
found inline addr:0xffffffff812b6c80
Probe point found: SyS_epoll_wait+0
Found 3probe_trace_events.- Arnaldo
P.S.: Full -v output:[root@jouet linux]# uname -a
Linux jouet 4.15.0-rc6 #4 SMP Tue Jan 2 14:30:53 -03 2018x86_64x86_64x86_64GNU/Linux
[root@jouet linux]# gcc -v
Using built-in specs.
COLLECT_GCC=/usr/bin/gcc COLLECT_LTO_WRAPPER=/usr/libexec/gcc/x86_64-redhat-linux/7/lto-wrapper OFFLOAD_TARGET_NAMES=nvptx-none
OFFLOAD_TARGET_DEFAULT=1
Target: x86_64-redhat-linux
Configured with: ../configure --enable-bootstrap --enable-languages=c,c++,objc,obj-c++,fortran,ada,go,lto --prefix=/usr --mandir=/usr/share/man --infodir=/usr/share/info --with-bugurl=http://bugzilla.redhat.com/bugzilla --enable-shared --enable-threads=posix --enable-checking=release --enable-multilib --with-system-zlib --enable-__cxa_atexit --disable-libunwind-exceptions --enable-gnu-unique-object --enable-linker-build-id --with-gcc-major-version-only --with-linker-hash-style=gnu --enable-plugin --enable-initfini-array --with-isl --enable-libmpx --enable-offload-targets=nvptx-none --without-cuda-driver --enable-gnu-indirect-function --with-tune=generic --with-arch_32=i686 --build=x86_64-redhat-linux Thread model: posix
gcc version 7.2.1 20170915 (Red Hat 7.2.1-2) (GCC) [root@jouet linux]# clang -v
clang version 6.0.0 (http://llvm.org/git/clang.git56cc8f8880db2ebc433eeb6b6a707c101467a186)
(http://llvm.org/git/llvm.git3656d83960a4f3fedf6d8f19043abf52379f78c3)
Target: x86_64-unknown-linux-gnu
Thread model: posix
InstalledDir:/usr/local/bin
Found candidate GCC installation:/usr/lib/gcc/x86_64-redhat-linux/7Selected GCC installation:/usr/lib/gcc/x86_64-redhat-linux/7Candidate multilib: .;@m64
Candidate multilib: 32;@m32
Selected multilib: .;@m64
[root@jouet linux]# perf test -v BPF
39: BPF filter :
39.1: Basic BPF filtering :
--- start ---
test child forked, pid 24304
Kernel build dir is set to/lib/modules/4.15.0-rc6/buildset env: KBUILD_DIR=/lib/modules/4.15.0-rc6/build
unset env:KBUILD_OPTS
include option is set to -nostdinc -isystem/usr/lib/gcc/x86_64-redhat-linux/7/include(quoted diff) +0
-6-I/home/acme/git/linux/arch/x86/include -I./arch/x86/include/generated -I/home/acme/git/linux/include -I./include -I/home/acme/git/linux/arch/x86/include/uapi -I./arch/x86/include/generated/uapi -I/home/acme/git/linux/include/uapi -I./include/generated/uapi -include /home/acme/git/linux/include/linux/kconfig.hset env: NR_CPUS=4
set env: LINUX_VERSION_CODE=0x40f00
set env: CLANG_EXEC=/usr/local/bin/clang
set env: CLANG_OPTIONS=-xc
set env: KERNEL_INC_OPTIONS= -nostdinc -isystem/usr/lib/gcc/x86_64-redhat-linux/7/include(quoted diff) +0
-6-I/home/acme/git/linux/arch/x86/include -I./arch/x86/include/generated -I/home/acme/git/linux/include -I./include -I/home/acme/git/linux/arch/x86/include/uapi -I./arch/x86/include/generated/uapi -I/home/acme/git/linux/include/uapi -I./include/generated/uapi -include /home/acme/git/linux/include/linux/kconfig.hset env: WORKING_DIR=/lib/modules/4.15.0-rc6/build set env: CLANG_SOURCE=-
llvm compiling command template: echo '/*
*bpf-script-example.c
* Test basic LLVM building
*/
#ifndefLINUX_VERSION_CODE# error Need LINUX_VERSION_CODE # error Example: for 4.2 kernel, put 'clang-opt="-DLINUX_VERSION_CODE=0x40200" into llvm section of ~/.perfconfig' #endif #define BPF_ANY 0 #define BPF_MAP_TYPE_ARRAY 2 #define BPF_FUNC_map_lookup_elem 1 #define BPF_FUNC_map_update_elem 2static void *(*bpf_map_lookup_elem)(void *map, void *key) =
(void *) BPF_FUNC_map_lookup_elem; static void *(*bpf_map_update_elem)(void *map, void *key, void *value, int flags) = (void *) BPF_FUNC_map_update_elem;struct bpf_map_def { unsigned int type; unsigned int key_size; unsigned int value_size; unsigned int max_entries; };#define SEC(NAME)
__attribute__((section(NAME), used))structbpf_map_defSEC("maps")flip_table= {
.type =BPF_MAP_TYPE_ARRAY,
.key_size = sizeof(int),
.value_size = sizeof(int),
.max_entries = 1,
};
SEC("func=SyS_epoll_wait")
intbpf_func__SyS_epoll_wait(void *ctx){ int ind =0; int *flag = bpf_map_lookup_elem(&flip_table, &ind); int new_flag; if (!flag) return 0; /* flip flag and store back */ new_flag = !*flag; bpf_map_update_elem(&flip_table, &ind, &new_flag, BPF_ANY); return new_flag; }char _license[]
SEC("license")= "GPL";
int_versionSEC("version")=LINUX_VERSION_CODE; ' | $CLANG_EXEC -D__KERNEL__ -D__NR_CPUS__=$NR_CPUS -DLINUX_VERSION_CODE=$LINUX_VERSION_CODE $CLANG_OPTIONS $KERNEL_INC_OPTIONS -Wno-unused-value -Wno-pointer-sign(quoted diff) +0
-2-working-directory $WORKING_DIR -c "$CLANG_SOURCE" -target bpf -O2 -o -libbpf: loading object '[
basic_bpf_test]' from buffer
libbpf: section .strtab, size 120, link 0, flags 0, type=3
libbpf: section .text, size 0, link 0, flags 6, type=1
libbpf: section func=SyS_epoll_wait, size 192, link 0, flags 6, type=1
libbpf: found program func=SyS_epoll_wait
libbpf: section .relfunc=SyS_epoll_wait, size 32, link 8, flags 0, type=9
libbpf: section maps, size 16, link 0, flags 3, type=1
libbpf: section license, size 4, link 0, flags 3, type=1
libbpf: license of [basic_bpf_test] is GPL
libbpf: section version, size 4, link 0, flags 3, type=1
libbpf: kernel version of [basic_bpf_test] is 40f00
libbpf: section .symtab, size 168, link 1, flags 0, type=2
libbpf: maps in [basic_bpf_test]: 1 maps in 16 bytes
libbpf: map 0 is "flip_table"
libbpf: collecting relocating info for: 'func=SyS_epoll_wait'
libbpf: relocation: insn_idx=4
libbpf: relocation: find map 0 (flip_table) for insn 4
libbpf: relocation: insn_idx=17
libbpf: relocation: find map 0 (flip_table) for insn 17
bpf: config program 'func=SyS_epoll_wait' symbol:SyS_epoll_wait file:(null) line:0 offset:0 return:0 lazy:(null)
bpf: config 'func=SyS_epoll_wait' is ok
Looking at thevmlinux_path(8 entries long)
Using/lib/modules/4.15.0-rc6/build/vmlinuxfor symbols Open Debuginfo file:/lib/modules/4.15.0-rc6/build/vmlinuxTry to find probe point from debuginfo.
Matched function:SyS_epoll_wait[2f40eb7]
found inline addr:0xffffffff812b6ff1
Probe point found: compat_SyS_epoll_pwait+129
found inline addr:0xffffffff812b6de7
Probe point found: SyS_epoll_pwait+135
found inline addr:0xffffffff812b6c80
Probe point found: SyS_epoll_wait+0
Found 3probe_trace_events.
Opening/sys/kernel/debug/tracing//kprobe_eventswrite=1 Writing event: p:perf_bpf_probe/func _text+2846705
Writing event: p:perf_bpf_probe/func_1 _text+2846183 Writing event: p:perf_bpf_probe/func_2 _text+2845824
libbpf: create mapflip_table: fd=3
add bpf event perf_bpf_probe:func and attach bpf program 4 registering plugin:/root/.traceevent/plugins/plugin_kvm.soregistering plugin:/root/.traceevent/plugins/plugin_hrtimer.soregistering plugin:/root/.traceevent/plugins/plugin_jbd2.soregistering plugin:/root/.traceevent/plugins/plugin_kmem.soregistering plugin:/root/.traceevent/plugins/plugin_scsi.soregistering plugin:/root/.traceevent/plugins/plugin_sched_switch.soregistering plugin:/root/.traceevent/plugins/plugin_function.soregistering plugin:/root/.traceevent/plugins/plugin_xen.soregistering plugin:/root/.traceevent/plugins/plugin_mac80211.soregistering plugin:/root/.traceevent/plugins/plugin_cfg80211.soadding perf_bpf_probe:func
adding perf_bpf_probe:func to0x24e3850
add bpf event perf_bpf_probe:func_1 and attach bpf program 4 adding perf_bpf_probe:func_1
adding perf_bpf_probe:func_1 to0x1965020
add bpf event perf_bpf_probe:func_2 and attach bpf program 4 adding perf_bpf_probe:func_2
adding perf_bpf_probe:func_2 to0x19656a0
mmap size 1052672B
BPF filter result incorrect, expected 56, got 0 samples Opening/sys/kernel/debug/tracing//kprobe_eventswrite=1 Opening/sys/kernel/debug/tracing//uprobe_eventswrite=1 Parsingprobe_events: p:perf_bpf_probe/func _text+2846705 Group:perf_bpf_probe Event:func probe:p
Parsingprobe_events: p:perf_bpf_probe/func_1 _text+2846183 Group:perf_bpf_probe Event:func_1 probe:p
Parsingprobe_events: p:perf_bpf_probe/func_2 _text+2845824 Group:perf_bpf_probe Event:func_2 probe:p
Writing event: -:perf_bpf_probe/func
Opening/sys/kernel/debug/tracing//kprobe_eventswrite=1 Opening/sys/kernel/debug/tracing//uprobe_eventswrite=1 Parsingprobe_events: p:perf_bpf_probe/func_1 _text+2846183 Group:perf_bpf_probe Event:func_1 probe:p
Parsingprobe_events: p:perf_bpf_probe/func_2 _text+2845824 Group:perf_bpf_probe Event:func_2 probe:p
Writing event: -:perf_bpf_probe/func_1
Opening/sys/kernel/debug/tracing//kprobe_eventswrite=1 Opening/sys/kernel/debug/tracing//uprobe_eventswrite=1 Parsingprobe_events: p:perf_bpf_probe/func_2 _text+2845824 Group:perf_bpf_probe Event:func_2 probe:p
Writing event: -:perf_bpf_probe/func_2
test child finished with -1
---- end ----
BPF filter subtest 0: FAILED!
39.2: BPF pinning :
--- force skipped ---
BPF filter subtest 1: Skip
39.3: BPF prologue generation :
--- force skipped ---
BPF filter subtest 2: Skip
39.4: BPF relocation checker :
--- force skipped ---
BPF filter subtest 3: Skip
[root@jouet linux]#
[root@jouet linux]#
Arnaldo Carvalho de Melo
maintainerEm Wed, Jan 03, 2018 at 01:58:44PM -0300, Arnaldo Carvalho de Melo escreveu:
Quoted 118 lines
Em Wed, Jan 03, 2018 at 12:58:26PM +0800, Wangnan (F) escreveu:
Quoted 3 lines
And please check if the kprobe created by
$ perf probe -v SyS_epoll_wait[root@seventh ~]# perf probe -v
SyS_epoll_waitprobe-definition(0):SyS_epoll_wait
symbol:SyS_epoll_wait file:(null) line:0 offset:0 return:0 lazy:(null) 0 arguments
Looking at thevmlinux_path(8 entries long)
Using/usr/lib/debug/lib/modules/4.14.8-300.fc27.x86_64/vmlinuxfor symbols Open Debuginfo file:/usr/lib/debug/lib/modules/4.14.8-300.fc27.x86_64/vmlinuxTry to find probe point from debuginfo.
Matched function:SyS_epoll_wait[314b563]
found inline addr:0xffffffff812d8087
Probe point found: compat_SyS_epoll_pwait+151
found inline addr:0xffffffff812d7e5a
Probe point found: SyS_epoll_pwait+138
found inline addr:0xffffffff812d7cf0
Probe point found: SyS_epoll_wait+0
Found 3probe_trace_events.
Opening/sys/kernel/debug/tracing//kprobe_eventswrite=1
Writing event: p:probe/SyS_epoll_wait _text+2982023
Writing event: p:probe/SyS_epoll_wait_1 _text+2981466
Writing event: p:probe/SyS_epoll_wait_2 _text+2981104
Added new events:
probe:SyS_epoll_wait (onSyS_epoll_wait)
probe:SyS_epoll_wait_1 (onSyS_epoll_wait) probe:SyS_epoll_wait_2 (onSyS_epoll_wait)You can now use it in all perf tools, such as:
perf record -e probe:SyS_epoll_wait_2 -aR sleep 1
[root@seventh ~]#
Then, with your proggie, it seems its glibc that now maps
epoll_wait()to a different syscall than in older systems:[root@seventh ~]# trace -e probe:SyS_epoll_wait* ~acme/epoll
0.018 ( 0.001 ms): epoll/20444 brk( ) = 0x2600000 0.029 ( 0.004 ms): epoll/20444 access(filename: 0xe874ec0, mode: R ) = -1 ENOENT No such file or directory 0.035 ( 0.003 ms): epoll/20444 openat(dfd: CWD, filename: 0xe872726, flags: CLOEXEC ) = 3 0.040 ( 0.002 ms): epoll/20444 fstat(fd: 3, statbuf: 0x7fffcc37b830 ) = 0 0.042 ( 0.002 ms): epoll/20444 mmap(len: 109538, prot: READ, flags: PRIVATE, fd: 3 ) = 0x7fc50ea5d000 0.045 ( 0.001 ms): epoll/20444 close(fd: 3 ) = 0 0.052 ( 0.003 ms): epoll/20444 openat(dfd: CWD, filename: 0xea7ace0, flags: CLOEXEC ) = 3 0.056 ( 0.001 ms): epoll/20444 read(fd: 3, buf: 0x7fffcc37b9f8, count: 832 ) = 832 0.059 ( 0.001 ms): epoll/20444 fstat(fd: 3, statbuf: 0x7fffcc37b890 ) = 0 0.060 ( 0.002 ms): epoll/20444 mmap(len: 8192, prot: READ|WRITE, flags: PRIVATE|ANONYMOUS, fd: -1 ) = 0x7fc50ea5b000 0.065 ( 0.003 ms): epoll/20444 mmap(len: 4074112, prot: EXEC|READ, flags: PRIVATE|DENYWRITE, fd: 3 ) = 0x7fc50e46e000 0.068 ( 0.004 ms): epoll/20444 mprotect(start: 0x7fc50e648000, len: 2093056 ) = 0 0.073 ( 0.004 ms): epoll/20444 mmap(addr: 0x7fc50e847000, len: 24576, prot: READ|WRITE, flags: PRIVATE|DENYWRITE|FIXED, fd: 3, off: 1937408) = 0x7fc50e847000 0.079 ( 0.002 ms): epoll/20444 mmap(addr: 0x7fc50e84d000, len: 14976, prot: READ|WRITE, flags: PRIVATE|ANONYMOUS|FIXED, fd: -1) = 0x7fc50e84d000 0.085 ( 0.001 ms): epoll/20444 close(fd: 3 ) = 0 0.093 ( 0.001 ms): epoll/20444 arch_prctl(option: 4098, arg2: 140484331029696 ) = 0 0.128 ( 0.003 ms): epoll/20444 mprotect(start: 0x7fc50e847000, len: 16384, prot: READ ) = 0 0.133 ( 0.002 ms): epoll/20444 mprotect(start: 0x600000, len: 4096, prot: READ ) = 0 0.137 ( 0.002 ms): epoll/20444 mprotect(start: 0x7fc50ea78000, len: 4096, prot: READ ) = 0 0.140 ( 0.005 ms): epoll/20444 munmap(addr: 0x7fc50ea5d000, len: 109538 ) = 0 0.159 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -1, sigsetsize: 8 ) = -22 0.160 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -2, sigsetsize: 8 ) = -22 0.162 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -3, sigsetsize: 8 ) = -22 0.163 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -4, sigsetsize: 8 ) = -22 0.165 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -5, sigsetsize: 8 ) = -22 <SNIP> 0.279 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -85, sigsetsize: 8 ) = -22 0.280 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -86, sigsetsize: 8 ) = -22 0.282 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -87, sigsetsize: 8 ) = -22 0.283 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -88, sigsetsize: 8 ) = -22 0.284 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -89, sigsetsize: 8 ) = -22 0.286 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -90, sigsetsize: 8 ) = -22 0.287 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -91, sigsetsize: 8 ) = -22 0.289 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -92, sigsetsize: 8 ) = -22 0.290 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -93, sigsetsize: 8 ) = -22 0.292 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -94, sigsetsize: 8 ) = -22 0.293 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -95, sigsetsize: 8 ) = -22 0.294 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -96, sigsetsize: 8 ) = -22 0.296 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -97, sigsetsize: 8 ) = -22 0.297 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -98, sigsetsize: 8 ) = -22 0.299 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -99, sigsetsize: 8 ) = -22 0.300 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -100, sigsetsize: 8 ) = -22 0.305 ( ? ): epoll/20444 exit_group( )[root@seventh ~]#
This is on a Fedora 27 system, that has glibc 2.26, while on a RHEL7 machine, that has glibc 2.17, it still uses
epoll_wait()and we get:[root@felicio ~]# trace ./epoll
0.018 ( 0.001 ms): epoll/2818 brk( ) = 0x8e5000 0.033 ( 0.002 ms): epoll/2818 mmap(len: 4096, prot: READ|WRITE, flags: PRIVATE|ANONYMOUS ) = 0x7f3f738e3000 0.041 ( 0.004 ms): epoll/2818 access(filename: 0x736e2160, mode: R ) = -1 ENOENT No such file or directory 0.051 ( 0.003 ms): epoll/2818 open(filename: 0x736e0a58, flags: CLOEXEC ) = 3 0.056 ( 0.001 ms): epoll/2818 fstat(fd: 3, statbuf: 0x7fff8b35f5b0 ) = 0 0.057 ( 0.002 ms): epoll/2818 mmap(len: 78559, prot: READ, flags: PRIVATE, fd: 3 ) = 0x7f3f738cf000 0.060 ( 0.001 ms): epoll/2818 close(fd: 3 ) = 0 0.070 ( 0.005 ms): epoll/2818 open(filename: 0x738e3640, flags: CLOEXEC ) = 3 0.076 ( 0.001 ms): epoll/2818 read(fd: 3, buf: 0x7fff8b35f770, count: 832 ) = 832 0.079 ( 0.001 ms): epoll/2818 fstat(fd: 3, statbuf: 0x7fff8b35f610 ) = 0 0.082 ( 0.003 ms): epoll/2818 mmap(len: 3932672, prot: EXEC|READ, flags: PRIVATE|DENYWRITE, fd: 3 ) = 0x7f3f73304000 0.086 ( 0.005 ms): epoll/2818 mprotect(start: 0x7f3f734bb000, len: 2093056 ) = 0 0.091 ( 0.005 ms): epoll/2818 mmap(addr: 0x7f3f736ba000, len: 24576, prot: READ|WRITE, flags: PRIVATE|DENYWRITE|FIXED, fd: 3, off: 1794048) = 0x7f3f736ba000 0.100 ( 0.003 ms): epoll/2818 mmap(addr: 0x7f3f736c0000, len: 16896, prot: READ|WRITE, flags: PRIVATE|ANONYMOUS|FIXED) = 0x7f3f736c0000 0.106 ( 0.001 ms): epoll/2818 close(fd: 3 ) = 0 0.115 ( 0.002 ms): epoll/2818 mmap(len: 4096, prot: READ|WRITE, flags: PRIVATE|ANONYMOUS ) = 0x7f3f738ce000 0.119 ( 0.002 ms): epoll/2818 mmap(len: 8192, prot: READ|WRITE, flags: PRIVATE|ANONYMOUS ) = 0x7f3f738cc0000.123 ( 0.001 ms): epoll/2818
arch_prctl(option: 4098, arg2: 139910498273088, arg3: 139910498275408, arg4: 34, arg5: -1)= 0
0.181 ( 0.003 ms): epoll/2818mprotect(start: 0x7f3f736ba000, len: 16384, prot: READ )= 0
0.186 ( 0.002 ms): epoll/2818mprotect(start: 0x600000, len: 4096, prot: READ )= 0
0.190 ( 0.002 ms): epoll/2818mprotect(start: 0x7f3f738e4000, len: 4096, prot: READ )= 0
0.193 ( 0.004 ms): epoll/2818munmap(addr: 0x7f3f738cf000, len: 78559 )= 0
0.218 ( 0.001 ms): epoll/2818epoll_wait(epfd: -1 )= -1 EINVAL Invalid argument
0.220 ( 0.000 ms): epoll/2818epoll_wait(epfd: -2 )= -1 EINVAL Invalid argument
0.221 ( 0.000 ms): epoll/2818epoll_wait(epfd: -3 )= -1 EINVAL Invalid argument
0.221 ( 0.000 ms): epoll/2818epoll_wait(epfd: -4 )= -1 EINVAL Invalid argument
0.222 ( 0.000 ms): epoll/2818epoll_wait(epfd: -5 )= -1 EINVAL Invalid argument
<SNIP>
Quoted 6 lines
It thought that this part would cover it:
Probe point found: SyS_epoll_pwait+138
found inline addr:0xffffffff812d7cf0Continuing investigation...
After applying the fallback patch to allow new tools to work with older kernels:
[root@felicio ~]# perf test bpf
39: BPF filter : 39.1: Basic BPF filtering : Ok 39.2: BPF pinning : Ok 39.3: BPF prologue generation : Ok 39.4: BPF relocation checker : Ok
[root@felicio ~]# uname -a
Linux felicio.ghostprotocols.net 4.13.0-rc7+ #1 SMP Mon Sep 11 13:56:18 -03 2017 x86_64 x86_64 x86_64 GNU/Linux [root@felicio ~]# rpm -q glibc
glibc-2.17-157.el7_3.2.x86_64
[root@felicio ~]#
After applying the patch below I get to, which is what I am trying to fix now:
[root@jouet ~]# perf test bpf
39: BPF filter : 39.1: Basic BPF filtering : Ok 39.2: BPF pinning : Ok 39.3: BPF prologue generation : FAILED! 39.4: BPF relocation checker : Skip
[root@jouet ~]#
Its related to:
<stdin>:11:10: fatal error: 'uapi/linux/fs.h' file not found
#include <uapi/linux/fs.h>
^~~~~~~~~~~~~~~~~1 error generated.
ERROR: unable to compile -
Hint: Check error message shown above.
Hint: You can also pre-compile it into .o using:
clang -target bpf -O2 -c -
with proper -I and -D options.
Failed to compile test case: 'Compile source for BPF prologue generation' Unable to get BPF object, fix kbuild first
test child finished with -1
---- end ----
BPF filter subtest 2: FAILED!
- Arnaldo
tools/perf/tests/bpf-script-example.c +1 -1
@@ -31,7 +31,7 @@ struct bpf_map_def SEC("maps") flip_table = { .max_entries = 1, }; -SEC("func=SyS_epoll_wait") +SEC("func=SyS_epoll_pwait")int bpf_func__SyS_epoll_wait (void *ctx) {int ind =0 ;
Arnaldo Carvalho de Melo
maintainerEm Wed, Jan 03, 2018 at 03:27:01PM -0300, Arnaldo Carvalho de Melo escreveu:
Quoted 20 lines
Quoted 1 line
Continuing investigation...
After applying the fallback patch to allow new tools to work with older kernels:
[root@felicio ~]# perf test bpf
39: BPF filter : 39.1: Basic BPF filtering : Ok 39.2: BPF pinning : Ok 39.3: BPF prologue generation : Ok 39.4: BPF relocation checker : Ok[root@felicio ~]# uname -a
Linux felicio.ghostprotocols.net 4.13.0-rc7+ #1 SMP Mon Sep 11 13:56:18 -03 2017x86_64x86_64x86_64GNU/Linux [root@felicio ~]# rpm -q glibc
glibc-2.17-157.el7_3.2.x86_64
[root@felicio ~]#After applying the patch below I get to, which is what I am trying to fix now:
Quoted 7 lines
[root@jouet ~]# perf test bpf
39: BPF filter : 39.1: Basic BPF filtering : Ok 39.2: BPF pinning : Ok 39.3: BPF prologue generation : FAILED! 39.4: BPF relocation checker : Skip[root@jouet ~]#
Update the patch to the one at the end of this message to make it work with older glibcs, so that we ask for epoll_pwait() and hook into that as well().
Now checking why 39.3 fails...
- Arnaldo
tools/perf/tests/bpf-script-example.c +1 -1
@@ -31,7 +31,7 @@ struct bpf_map_def SEC("maps") flip_table = { .max_entries = 1, }; -SEC("func=SyS_epoll_wait") +SEC("func=SyS_epoll_pwait")int bpf_func__SyS_epoll_wait (void *ctx) {int ind =0 ;
tools/perf/tests/bpf.c +1 -1
@@ -25,7 +25,7 @@ static int epoll_wait_loop(void)/* Should fail NR_ITERS times */ for (i =0 ; i <NR_ITERS ; i++) -epoll_wait (-(i +1 ),NULL ,0 ,0 ); +epoll_pwait (-(i +1 ),NULL ,0 ,0 ,NULL );return 0 ; }
Arnaldo Carvalho de Melo
maintainerEm Wed, Jan 03, 2018 at 03:33:07PM -0300, Arnaldo Carvalho de Melo escreveu:
Quoted 29 lines
Em Wed, Jan 03, 2018 at 03:27:01PM -0300, Arnaldo Carvalho de Melo escreveu:
Quoted 20 lines
Quoted 1 line
Continuing investigation...
After applying the fallback patch to allow new tools to work with older kernels:
[root@felicio ~]# perf test bpf
39: BPF filter : 39.1: Basic BPF filtering : Ok 39.2: BPF pinning : Ok 39.3: BPF prologue generation : Ok 39.4: BPF relocation checker : Ok[root@felicio ~]# uname -a
Linux felicio.ghostprotocols.net 4.13.0-rc7+ #1 SMP Mon Sep 11 13:56:18 -03 2017x86_64x86_64x86_64GNU/Linux [root@felicio ~]# rpm -q glibc
glibc-2.17-157.el7_3.2.x86_64
[root@felicio ~]#After applying the patch below I get to, which is what I am trying to fix now:
Quoted 7 lines
[root@jouet ~]# perf test bpf
39: BPF filter : 39.1: Basic BPF filtering : Ok 39.2: BPF pinning : Ok 39.3: BPF prologue generation : FAILED! 39.4: BPF relocation checker : Skip[root@jouet ~]#
Quoted 5 lines
Update the patch to the one at the end of this message to make it work with older glibcs, so that we ask for
epoll_pwait()and hook into that aswell().Now checking why 39.3 fails...
Couldn't reproduce after fixing up some kernel build problems, the patch below is all I need to have this working with both Fedora 27 and RHEL7, please take a look and see if it continues to work on your systems,
- Arnaldo
Quoted 28 lines
- Arnaldo
tools/perf/tests/bpf-script-example.c +1
-1@@ -31,7 +31,7 @@ struct bpf_map_def SEC("maps") flip_table = { .max_entries = 1, }; -SEC("func=SyS_epoll_wait") +SEC("func=SyS_epoll_pwait")int bpf_func__SyS_epoll_wait (void *ctx) {int ind =0 ;tools/perf/tests/bpf.c +1
-1@@ -25,7 +25,7 @@ static int epoll_wait_loop(void)/* Should fail NR_ITERS times */ for (i =0 ; i <NR_ITERS ; i++) -epoll_wait (-(i +1 ),NULL ,0 ,0 ); +epoll_pwait (-(i +1 ),NULL ,0 ,0 ,NULL );return 0 ; }
Quoted 28 lines · Arnaldo Carvalho de Melo
Em Wed, Jan 03, 2018 at 03:33:07PM -0300, Arnaldo Carvalho de Melo escreveu:
Quoted 27 lines
Em Wed, Jan 03, 2018 at 03:27:01PM -0300, Arnaldo Carvalho de Melo escreveu:
Quoted 26 lines
Quoted 1 line
Continuing investigation...
After applying the fallback patch to allow new tools to work with older kernels:
[root@felicio ~]# perf test bpf
39: BPF filter : 39.1: Basic BPF filtering : Ok 39.2: BPF pinning : Ok 39.3: BPF prologue generation : Ok 39.4: BPF relocation checker : Ok[root@felicio ~]# uname -a
Linux felicio.ghostprotocols.net 4.13.0-rc7+ #1 SMP Mon Sep 11 13:56:18 -03 2017x86_64x86_64x86_64GNU/Linux [root@felicio ~]# rpm -q glibc
glibc-2.17-157.el7_3.2.x86_64
[root@felicio ~]#After applying the patch below I get to, which is what I am trying to fix now:
[root@jouet ~]# perf test bpf
39: BPF filter : 39.1: Basic BPF filtering : Ok 39.2: BPF pinning : Ok 39.3: BPF prologue generation : FAILED! 39.4: BPF relocation checker : Skip[root@jouet ~]#
Quoted 8 lines
Quoted 5 lines
Update the patch to the one at the end of this message to make it work with older glibcs, so that we ask for
epoll_pwait()and hook into that aswell().Now checking why 39.3 fails...
Couldn't reproduce after fixing up some kernel build problems, the patch below is all I need to have this working with both Fedora 27 and RHEL7, please take a look and see if it continues to work on your systems,
It works for me. Thank you.
Since we test epoll_pwait, we'd better correct function names:
tools/perf/tests/bpf-script-example.c +2 -2
@@ -31,8 +31,8 @@ struct bpf_map_def SEC("maps") flip_table = { .max_entries = 1, }; -SEC("func=SyS_epoll_wait") -int bpf_func__SyS_epoll_wait (void *ctx) +SEC("func=SyS_epoll_pwait") +int bpf_func__SyS_epoll_pwait (void *ctx) {int ind =0 ;int *flag =bpf_map_lookup_elem (&flip_table, &ind);
tools/perf/tests/bpf.c +4 -4
@@ -19,13 +19,13 @@#ifdef HAVE_LIBBPF_SUPPORT -static int epoll_wait_loop (void ) +static int epoll_pwait_loop (void ) {int i;/* Should fail NR_ITERS times */ for (i =0 ; i <NR_ITERS ; i++) -epoll_wait (-(i +1 ),NULL ,0 ,0 ); +epoll_pwait (-(i +1 ),NULL ,0 ,0 ,NULL );return 0 ; }
@@ -68,7 +68,7 @@ static struct {"[basic_bpf_test]" ,"fix 'perf test LLVM' first" ,"load bpf object failed" , - &epoll_wait_loop, + &epoll_pwait_loop, (NR_ITERS +1 ) /2 , false, },
@@ -78,7 +78,7 @@ static struct {"[bpf_pinning]" ,"fix kbuild first" ,"check your vmlinux setting?" , - &epoll_wait_loop, + &epoll_pwait_loop, (NR_ITERS +1 ) /2 , true, },
Arnaldo Carvalho de Melo
maintainerEm Thu, Jan 04, 2018 at 09:37:35AM +0800, Wangnan (F) escreveu:
Quoted 35 lines · Arnaldo Carvalho de Melo
Quoted 32 lines
Em Wed, Jan 03, 2018 at 03:33:07PM -0300, Arnaldo Carvalho de Melo escreveu:
Quoted 31 lines
Em Wed, Jan 03, 2018 at 03:27:01PM -0300, Arnaldo Carvalho de Melo escreveu:
Quoted 19 lines
Quoted 1 line
Continuing investigation...
After applying the fallback patch to allow new tools to work with older kernels:
[root@felicio ~]# perf test bpf
39: BPF filter : 39.1: Basic BPF filtering : Ok 39.2: BPF pinning : Ok 39.3: BPF prologue generation : Ok 39.4: BPF relocation checker : Ok[root@felicio ~]# uname -a
Linux felicio.ghostprotocols.net 4.13.0-rc7+ #1 SMP Mon Sep 11 13:56:18 -03 2017x86_64x86_64x86_64GNU/Linux [root@felicio ~]# rpm -q glibc
glibc-2.17-157.el7_3.2.x86_64
[root@felicio ~]#After applying the patch below I get to, which is what I am trying to fix now:
Quoted 7 lines
[root@jouet ~]# perf test bpf
39: BPF filter : 39.1: Basic BPF filtering : Ok 39.2: BPF pinning : Ok 39.3: BPF prologue generation : FAILED! 39.4: BPF relocation checker : Skip[root@jouet ~]#
Update the patch to the one at the end of this message to make it work with older glibcs, so that we ask for
epoll_pwait()and hook into that aswell().
Quoted 8 lines
Quoted 4 lines
Quoted 1 line
Now checking why 39.3 fails...
Couldn't reproduce after fixing up some kernel build problems, the patch below is all I need to have this working with both Fedora 27 and RHEL7, please take a look and see if it continues to work on your systems,
It works for me. Thank you.
Since we test
epoll_pwait, we'd better correct function names:
Right, that wasn't strictly needed, so I tried to restrict my changes to focus on the fix.
Now that we agree on it, I'm doing what you suggest, to make it less confusing.
I'm adding your Tested-by, thanks!
- Arnaldo
Quoted 57 lines
tools/perf/tests/bpf-script-example.c +2
-2@@ -31,8 +31,8 @@ struct bpf_map_def SEC("maps") flip_table = { .max_entries = 1, }; -SEC("func=SyS_epoll_wait") -int bpf_func__SyS_epoll_wait (void *ctx) +SEC("func=SyS_epoll_pwait") +int bpf_func__SyS_epoll_pwait (void *ctx) {int ind =0 ;int *flag =bpf_map_lookup_elem (&flip_table, &ind);tools/perf/tests/bpf.c +4
-4@@ -19,13 +19,13 @@#ifdef HAVE_LIBBPF_SUPPORT -static int epoll_wait_loop (void ) +static int epoll_pwait_loop (void ) {int i;/* Should fail NR_ITERS times */ for (i =0 ; i <NR_ITERS ; i++) -epoll_wait (-(i +1 ),NULL ,0 ,0 ); +epoll_pwait (-(i +1 ),NULL ,0 ,0 ,NULL );return 0 ; }@@ -68,7 +68,7 @@ static struct {"[basic_bpf_test]" ,"fix 'perf test LLVM' first" ,"load bpf object failed" , - &epoll_wait_loop, + &epoll_pwait_loop, (NR_ITERS +1 ) /2 , false, },@@ -78,7 +78,7 @@ static struct {"[bpf_pinning]" ,"fix kbuild first" ,"check your vmlinux setting?" , - &epoll_wait_loop, + &epoll_pwait_loop, (NR_ITERS +1 ) /2 , true, },