Skip to contentSkip to conversation

perf test BPF failing on 4.15.0-rc6

DISCUSSION Arnaldo Carvalho de Melo 2 participants 9 messages Jan 2, 2018 – Jan 4, 2018 perf events

lore
Share & tools
lore

Arnaldo Carvalho de Melo

author maintainer
#0 lore raw

Hi Wang,

I just updated my machine to Fedora 27 and 4.15.0-rc6 and the
only test failing for me is:

[root@jouet linux]# perf test bpf

39: BPF filter                                            :
39.1: Basic BPF filtering                                 : FAILED!
39.2: BPF pinning                                         : Skip
39.3: BPF prologue generation                             : Skip
39.4: BPF relocation checker                              : Skip

[root@jouet linux]#

I haven't checked but perhaps the problem is that SyS_epoll_wait seems to now be inlined in three places and perhaps the eBPF proggie is being added to just one of them?

Seemingly relevant excerpt:

Open Debuginfo file: /lib/modules/4.15.0-rc6/build/vmlinux Try to find probe point from debuginfo.
Matched function: SyS_epoll_wait [2f40eb7]
found inline addr: 0xffffffff812b6ff1
Probe point found: compat_SyS_epoll_pwait+129 found inline addr: 0xffffffff812b6de7
Probe point found: SyS_epoll_pwait+135
found inline addr: 0xffffffff812b6c80
Probe point found: SyS_epoll_wait+0
Found 3 probe_trace_events.

- Arnaldo

P.S.: Full -v output:

[root@jouet linux]# uname -a
Linux jouet 4.15.0-rc6 #4 SMP Tue Jan 2 14:30:53 -03 2018 x86_64 x86_64 x86_64 GNU/Linux
[root@jouet linux]# gcc -v
Using built-in specs.
COLLECT_GCC=/usr/bin/gcc
COLLECT_LTO_WRAPPER=/usr/libexec/gcc/x86_64-redhat-linux/7/lto-wrapper
OFFLOAD_TARGET_NAMES=nvptx-none
OFFLOAD_TARGET_DEFAULT=1
Target: x86_64-redhat-linux
Configured with: ../configure --enable-bootstrap --enable-languages=c,c++,objc,obj-c++,fortran,ada,go,lto --prefix=/usr --mandir=/usr/share/man --infodir=/usr/share/info --with-bugurl=http://bugzilla.redhat.com/bugzilla --enable-shared --enable-threads=posix --enable-checking=release --enable-multilib --with-system-zlib --enable-__cxa_atexit --disable-libunwind-exceptions --enable-gnu-unique-object --enable-linker-build-id --with-gcc-major-version-only --with-linker-hash-style=gnu --enable-plugin --enable-initfini-array --with-isl --enable-libmpx --enable-offload-targets=nvptx-none --without-cuda-driver --enable-gnu-indirect-function --with-tune=generic --with-arch_32=i686 --build=x86_64-redhat-linux Thread model: posix
gcc version 7.2.1 20170915 (Red Hat 7.2.1-2) (GCC)
[root@jouet linux]# clang -v
clang version 6.0.0 (http://llvm.org/git/clang.git 56cc8f8880db2ebc433eeb6b6a707c101467a186) (http://llvm.org/git/llvm.git 3656d83960a4f3fedf6d8f19043abf52379f78c3)
Target: x86_64-unknown-linux-gnu
Thread model: posix
InstalledDir: /usr/local/bin
Found candidate GCC installation: /usr/lib/gcc/x86_64-redhat-linux/7
Selected GCC installation: /usr/lib/gcc/x86_64-redhat-linux/7
Candidate multilib: .;@m64
Candidate multilib: 32;@m32
Selected multilib: .;@m64
[root@jouet linux]# perf test -v BPF
39: BPF filter :
39.1: Basic BPF filtering :
--- start ---
test child forked, pid 24304
Kernel build dir is set to /lib/modules/4.15.0-rc6/build
set env: KBUILD_DIR=/lib/modules/4.15.0-rc6/build
unset env: KBUILD_OPTS
include option is set to -nostdinc -isystem /usr/lib/gcc/x86_64-redhat-linux/7/include -I/home/acme/git/linux/arch/x86/include -I./arch/x86/include/generated -I/home/acme/git/linux/include -I./include -I/home/acme/git/linux/arch/x86/include/uapi -I./arch/x86/include/generated/uapi -I/home/acme/git/linux/include/uapi -I./include/generated/uapi -include /home/acme/git/linux/include/linux/kconfig.h
set env: NR_CPUS=4
set env: LINUX_VERSION_CODE=0x40f00
set env: CLANG_EXEC=/usr/local/bin/clang
set env: CLANG_OPTIONS=-xc
set env: KERNEL_INC_OPTIONS= -nostdinc -isystem /usr/lib/gcc/x86_64-redhat-linux/7/include -I/home/acme/git/linux/arch/x86/include -I./arch/x86/include/generated -I/home/acme/git/linux/include -I./include -I/home/acme/git/linux/arch/x86/include/uapi -I./arch/x86/include/generated/uapi -I/home/acme/git/linux/include/uapi -I./include/generated/uapi -include /home/acme/git/linux/include/linux/kconfig.h
set env: WORKING_DIR=/lib/modules/4.15.0-rc6/build
set env: CLANG_SOURCE=-
llvm compiling command template: echo '/*
* bpf-script-example.c
* Test basic LLVM building
*/
#ifndef LINUX_VERSION_CODE

# error Need LINUX_VERSION_CODE
# error Example: for 4.2 kernel, put 'clang-opt="-DLINUX_VERSION_CODE=0x40200" into llvm section of ~/.perfconfig'
#endif
#define BPF_ANY 0
#define BPF_MAP_TYPE_ARRAY 2
#define BPF_FUNC_map_lookup_elem 1
#define BPF_FUNC_map_update_elem 2

static void *(*bpf_map_lookup_elem)(void *map, void *key) =

	(void *) BPF_FUNC_map_lookup_elem;
static void *(*bpf_map_update_elem)(void *map, void *key, void *value, int flags) =
	(void *) BPF_FUNC_map_update_elem;
struct bpf_map_def {
	unsigned int type;
	unsigned int key_size;
	unsigned int value_size;
	unsigned int max_entries;
};

#define SEC(NAME) __attribute__((section(NAME), used)) struct bpf_map_def SEC("maps") flip_table = {
.type = BPF_MAP_TYPE_ARRAY,
.key_size = sizeof(int),
.value_size = sizeof(int), .max_entries = 1,
};

SEC("func=SyS_epoll_wait")
int bpf_func__SyS_epoll_wait(void *ctx)

{
	int ind =0;
	int *flag = bpf_map_lookup_elem(&flip_table, &ind);
	int new_flag;
	if (!flag)
		return 0;
	/* flip flag and store back */
	new_flag = !*flag;
	bpf_map_update_elem(&flip_table, &ind, &new_flag, BPF_ANY);
	return new_flag;
}

char _license[] SEC("license") = "GPL";
int _version SEC("version") = LINUX_VERSION_CODE;
' | $CLANG_EXEC -D__KERNEL__ -D__NR_CPUS__=$NR_CPUS -DLINUX_VERSION_CODE=$LINUX_VERSION_CODE $CLANG_OPTIONS $KERNEL_INC_OPTIONS -Wno-unused-value -Wno-pointer-sign -working-directory $WORKING_DIR -c "$CLANG_SOURCE" -target bpf -O2 -o -
libbpf: loading object '[basic_bpf_test]' from buffer
libbpf: section .strtab, size 120, link 0, flags 0, type=3
libbpf: section .text, size 0, link 0, flags 6, type=1
libbpf: section func=SyS_epoll_wait, size 192, link 0, flags 6, type=1
libbpf: found program func=SyS_epoll_wait
libbpf: section .relfunc=SyS_epoll_wait, size 32, link 8, flags 0, type=9
libbpf: section maps, size 16, link 0, flags 3, type=1
libbpf: section license, size 4, link 0, flags 3, type=1
libbpf: license of [basic_bpf_test] is GPL
libbpf: section version, size 4, link 0, flags 3, type=1
libbpf: kernel version of [basic_bpf_test] is 40f00
libbpf: section .symtab, size 168, link 1, flags 0, type=2
libbpf: maps in [basic_bpf_test]: 1 maps in 16 bytes
libbpf: map 0 is "flip_table"
libbpf: collecting relocating info for: 'func=SyS_epoll_wait'
libbpf: relocation: insn_idx=4
libbpf: relocation: find map 0 (flip_table) for insn 4
libbpf: relocation: insn_idx=17
libbpf: relocation: find map 0 (flip_table) for insn 17
bpf: config program 'func=SyS_epoll_wait'
symbol:SyS_epoll_wait file:(null) line:0 offset:0 return:0 lazy:(null)
bpf: config 'func=SyS_epoll_wait' is ok
Looking at the vmlinux_path (8 entries long)
Using /lib/modules/4.15.0-rc6/build/vmlinux for symbols
Open Debuginfo file: /lib/modules/4.15.0-rc6/build/vmlinux
Try to find probe point from debuginfo.
Matched function: SyS_epoll_wait [2f40eb7]
found inline addr: 0xffffffff812b6ff1
Probe point found: compat_SyS_epoll_pwait+129
found inline addr: 0xffffffff812b6de7
Probe point found: SyS_epoll_pwait+135
found inline addr: 0xffffffff812b6c80
Probe point found: SyS_epoll_wait+0
Found 3 probe_trace_events.
Opening /sys/kernel/debug/tracing//kprobe_events write=1
Writing event: p:perf_bpf_probe/func _text+2846705
Writing event: p:perf_bpf_probe/func_1 _text+2846183
Writing event: p:perf_bpf_probe/func_2 _text+2845824
libbpf: create map flip_table: fd=3
add bpf event perf_bpf_probe:func and attach bpf program 4
registering plugin: /root/.traceevent/plugins/plugin_kvm.so
registering plugin: /root/.traceevent/plugins/plugin_hrtimer.so
registering plugin: /root/.traceevent/plugins/plugin_jbd2.so
registering plugin: /root/.traceevent/plugins/plugin_kmem.so
registering plugin: /root/.traceevent/plugins/plugin_scsi.so
registering plugin: /root/.traceevent/plugins/plugin_sched_switch.so
registering plugin: /root/.traceevent/plugins/plugin_function.so
registering plugin: /root/.traceevent/plugins/plugin_xen.so
registering plugin: /root/.traceevent/plugins/plugin_mac80211.so
registering plugin: /root/.traceevent/plugins/plugin_cfg80211.so
adding perf_bpf_probe:func
adding perf_bpf_probe:func to 0x24e3850
add bpf event perf_bpf_probe:func_1 and attach bpf program 4
adding perf_bpf_probe:func_1
adding perf_bpf_probe:func_1 to 0x1965020
add bpf event perf_bpf_probe:func_2 and attach bpf program 4
adding perf_bpf_probe:func_2
adding perf_bpf_probe:func_2 to 0x19656a0
mmap size 1052672B
BPF filter result incorrect, expected 56, got 0 samples
Opening /sys/kernel/debug/tracing//kprobe_events write=1
Opening /sys/kernel/debug/tracing//uprobe_events write=1
Parsing probe_events: p:perf_bpf_probe/func _text+2846705
Group:perf_bpf_probe Event:func probe:p
Parsing probe_events: p:perf_bpf_probe/func_1 _text+2846183
Group:perf_bpf_probe Event:func_1 probe:p
Parsing probe_events: p:perf_bpf_probe/func_2 _text+2845824
Group:perf_bpf_probe Event:func_2 probe:p
Writing event: -:perf_bpf_probe/func
Opening /sys/kernel/debug/tracing//kprobe_events write=1
Opening /sys/kernel/debug/tracing//uprobe_events write=1
Parsing probe_events: p:perf_bpf_probe/func_1 _text+2846183
Group:perf_bpf_probe Event:func_1 probe:p
Parsing probe_events: p:perf_bpf_probe/func_2 _text+2845824
Group:perf_bpf_probe Event:func_2 probe:p
Writing event: -:perf_bpf_probe/func_1
Opening /sys/kernel/debug/tracing//kprobe_events write=1
Opening /sys/kernel/debug/tracing//uprobe_events write=1
Parsing probe_events: p:perf_bpf_probe/func_2 _text+2845824
Group:perf_bpf_probe Event:func_2 probe:p
Writing event: -:perf_bpf_probe/func_2
test child finished with -1
---- end ----
BPF filter subtest 0: FAILED!
39.2: BPF pinning :
--- force skipped ---
BPF filter subtest 1: Skip
39.3: BPF prologue generation :
--- force skipped ---
BPF filter subtest 2: Skip
39.4: BPF relocation checker :
--- force skipped ---
BPF filter subtest 3: Skip
[root@jouet linux]#
[root@jouet linux]#

#1 lore raw

Unable to reproduce. In my kernel configuration, SyS_epoll_wait is not inlined at all.

From the log you sent, it seems that all 3 instances are attached. This testcase should work if the last one (SyS_epoll_wait) get probed correctly.

Could you please have a look if the 3rd kprobe event (p:perf_bpf_probe/func_2 _text+2845824) works?

Thank you.

Quoted 216 lines · Arnaldo Carvalho de Melo

Hi Wang,

I just updated my machine to Fedora 27 and 4.15.0-rc6 and the
only test failing for me is:

[root@jouet linux]# perf test bpf

39: BPF filter                                            :
39.1: Basic BPF filtering                                 : FAILED!
39.2: BPF pinning                                         : Skip
39.3: BPF prologue generation                             : Skip
39.4: BPF relocation checker                              : Skip

[root@jouet linux]#

I haven't checked but perhaps the problem is that SyS_epoll_wait seems to now be inlined in three places and perhaps the eBPF proggie is being added to just one of them?

Seemingly relevant excerpt:

Open Debuginfo file: /lib/modules/4.15.0-rc6/build/vmlinux Try to find probe point from debuginfo.
Matched function: SyS_epoll_wait [2f40eb7]
found inline addr: 0xffffffff812b6ff1
Probe point found: compat_SyS_epoll_pwait+129 found inline addr: 0xffffffff812b6de7
Probe point found: SyS_epoll_pwait+135
found inline addr: 0xffffffff812b6c80
Probe point found: SyS_epoll_wait+0
Found 3 probe_trace_events.

- Arnaldo

P.S.: Full -v output:

[root@jouet linux]# uname -a
Linux jouet 4.15.0-rc6 #4 SMP Tue Jan 2 14:30:53 -03 2018 x86_64 x86_64 x86_64 GNU/Linux
[root@jouet linux]# gcc -v
Using built-in specs.
COLLECT_GCC=/usr/bin/gcc
COLLECT_LTO_WRAPPER=/usr/libexec/gcc/x86_64-redhat-linux/7/lto-wrapper
OFFLOAD_TARGET_NAMES=nvptx-none
OFFLOAD_TARGET_DEFAULT=1
Target: x86_64-redhat-linux
Configured with: ../configure --enable-bootstrap --enable-languages=c,c++,objc,obj-c++,fortran,ada,go,lto --prefix=/usr --mandir=/usr/share/man --infodir=/usr/share/info --with-bugurl=http://bugzilla.redhat.com/bugzilla --enable-shared --enable-threads=posix --enable-checking=release --enable-multilib --with-system-zlib --enable-__cxa_atexit --disable-libunwind-exceptions --enable-gnu-unique-object --enable-linker-build-id --with-gcc-major-version-only --with-linker-hash-style=gnu --enable-plugin --enable-initfini-array --with-isl --enable-libmpx --enable-offload-targets=nvptx-none --without-cuda-driver --enable-gnu-indirect-function --with-tune=generic --with-arch_32=i686 --build=x86_64-redhat-linux Thread model: posix
gcc version 7.2.1 20170915 (Red Hat 7.2.1-2) (GCC)
[root@jouet linux]# clang -v
clang version 6.0.0 (http://llvm.org/git/clang.git 56cc8f8880db2ebc433eeb6b6a707c101467a186) (http://llvm.org/git/llvm.git 3656d83960a4f3fedf6d8f19043abf52379f78c3)
Target: x86_64-unknown-linux-gnu
Thread model: posix
InstalledDir: /usr/local/bin
Found candidate GCC installation: /usr/lib/gcc/x86_64-redhat-linux/7
Selected GCC installation: /usr/lib/gcc/x86_64-redhat-linux/7
Candidate multilib: .;@m64
Candidate multilib: 32;@m32
Selected multilib: .;@m64
[root@jouet linux]# perf test -v BPF
39: BPF filter :
39.1: Basic BPF filtering :
--- start ---
test child forked, pid 24304
Kernel build dir is set to /lib/modules/4.15.0-rc6/build
set env: KBUILD_DIR=/lib/modules/4.15.0-rc6/build
unset env: KBUILD_OPTS
include option is set to -nostdinc -isystem /usr/lib/gcc/x86_64-redhat-linux/7/include -I/home/acme/git/linux/arch/x86/include -I./arch/x86/include/generated -I/home/acme/git/linux/include -I./include -I/home/acme/git/linux/arch/x86/include/uapi -I./arch/x86/include/generated/uapi -I/home/acme/git/linux/include/uapi -I./include/generated/uapi -include /home/acme/git/linux/include/linux/kconfig.h
set env: NR_CPUS=4
set env: LINUX_VERSION_CODE=0x40f00
set env: CLANG_EXEC=/usr/local/bin/clang
set env: CLANG_OPTIONS=-xc
set env: KERNEL_INC_OPTIONS= -nostdinc -isystem /usr/lib/gcc/x86_64-redhat-linux/7/include -I/home/acme/git/linux/arch/x86/include -I./arch/x86/include/generated -I/home/acme/git/linux/include -I./include -I/home/acme/git/linux/arch/x86/include/uapi -I./arch/x86/include/generated/uapi -I/home/acme/git/linux/include/uapi -I./include/generated/uapi -include /home/acme/git/linux/include/linux/kconfig.h
set env: WORKING_DIR=/lib/modules/4.15.0-rc6/build
set env: CLANG_SOURCE=-
llvm compiling command template: echo '/*
* bpf-script-example.c
* Test basic LLVM building
*/
#ifndef LINUX_VERSION_CODE

# error Need LINUX_VERSION_CODE
# error Example: for 4.2 kernel, put 'clang-opt="-DLINUX_VERSION_CODE=0x40200" into llvm section of ~/.perfconfig'
#endif
#define BPF_ANY 0
#define BPF_MAP_TYPE_ARRAY 2
#define BPF_FUNC_map_lookup_elem 1
#define BPF_FUNC_map_update_elem 2

static void *(*bpf_map_lookup_elem)(void *map, void *key) =

	(void *) BPF_FUNC_map_lookup_elem;
static void *(*bpf_map_update_elem)(void *map, void *key, void *value, int flags) =
	(void *) BPF_FUNC_map_update_elem;
struct bpf_map_def {
	unsigned int type;
	unsigned int key_size;
	unsigned int value_size;
	unsigned int max_entries;
};

#define SEC(NAME) __attribute__((section(NAME), used)) struct bpf_map_def SEC("maps") flip_table = {
.type = BPF_MAP_TYPE_ARRAY,
.key_size = sizeof(int),
.value_size = sizeof(int), .max_entries = 1,
};

SEC("func=SyS_epoll_wait")
int bpf_func__SyS_epoll_wait(void *ctx)

{
	int ind =0;
	int *flag = bpf_map_lookup_elem(&flip_table, &ind);
	int new_flag;
	if (!flag)
		return 0;
	/* flip flag and store back */
	new_flag = !*flag;
	bpf_map_update_elem(&flip_table, &ind, &new_flag, BPF_ANY);
	return new_flag;
}

char _license[] SEC("license") = "GPL";
int _version SEC("version") = LINUX_VERSION_CODE;
' | $CLANG_EXEC -D__KERNEL__ -D__NR_CPUS__=$NR_CPUS -DLINUX_VERSION_CODE=$LINUX_VERSION_CODE $CLANG_OPTIONS $KERNEL_INC_OPTIONS -Wno-unused-value -Wno-pointer-sign -working-directory $WORKING_DIR -c "$CLANG_SOURCE" -target bpf -O2 -o -
libbpf: loading object '[basic_bpf_test]' from buffer
libbpf: section .strtab, size 120, link 0, flags 0, type=3
libbpf: section .text, size 0, link 0, flags 6, type=1
libbpf: section func=SyS_epoll_wait, size 192, link 0, flags 6, type=1
libbpf: found program func=SyS_epoll_wait
libbpf: section .relfunc=SyS_epoll_wait, size 32, link 8, flags 0, type=9
libbpf: section maps, size 16, link 0, flags 3, type=1
libbpf: section license, size 4, link 0, flags 3, type=1
libbpf: license of [basic_bpf_test] is GPL
libbpf: section version, size 4, link 0, flags 3, type=1
libbpf: kernel version of [basic_bpf_test] is 40f00
libbpf: section .symtab, size 168, link 1, flags 0, type=2
libbpf: maps in [basic_bpf_test]: 1 maps in 16 bytes
libbpf: map 0 is "flip_table"
libbpf: collecting relocating info for: 'func=SyS_epoll_wait'
libbpf: relocation: insn_idx=4
libbpf: relocation: find map 0 (flip_table) for insn 4
libbpf: relocation: insn_idx=17
libbpf: relocation: find map 0 (flip_table) for insn 17
bpf: config program 'func=SyS_epoll_wait'
symbol:SyS_epoll_wait file:(null) line:0 offset:0 return:0 lazy:(null)
bpf: config 'func=SyS_epoll_wait' is ok
Looking at the vmlinux_path (8 entries long)
Using /lib/modules/4.15.0-rc6/build/vmlinux for symbols
Open Debuginfo file: /lib/modules/4.15.0-rc6/build/vmlinux
Try to find probe point from debuginfo.
Matched function: SyS_epoll_wait [2f40eb7]
found inline addr: 0xffffffff812b6ff1
Probe point found: compat_SyS_epoll_pwait+129
found inline addr: 0xffffffff812b6de7
Probe point found: SyS_epoll_pwait+135
found inline addr: 0xffffffff812b6c80
Probe point found: SyS_epoll_wait+0
Found 3 probe_trace_events.
Opening /sys/kernel/debug/tracing//kprobe_events write=1
Writing event: p:perf_bpf_probe/func _text+2846705
Writing event: p:perf_bpf_probe/func_1 _text+2846183
Writing event: p:perf_bpf_probe/func_2 _text+2845824
libbpf: create map flip_table: fd=3
add bpf event perf_bpf_probe:func and attach bpf program 4
registering plugin: /root/.traceevent/plugins/plugin_kvm.so
registering plugin: /root/.traceevent/plugins/plugin_hrtimer.so
registering plugin: /root/.traceevent/plugins/plugin_jbd2.so
registering plugin: /root/.traceevent/plugins/plugin_kmem.so
registering plugin: /root/.traceevent/plugins/plugin_scsi.so
registering plugin: /root/.traceevent/plugins/plugin_sched_switch.so
registering plugin: /root/.traceevent/plugins/plugin_function.so
registering plugin: /root/.traceevent/plugins/plugin_xen.so
registering plugin: /root/.traceevent/plugins/plugin_mac80211.so
registering plugin: /root/.traceevent/plugins/plugin_cfg80211.so
adding perf_bpf_probe:func
adding perf_bpf_probe:func to 0x24e3850
add bpf event perf_bpf_probe:func_1 and attach bpf program 4
adding perf_bpf_probe:func_1
adding perf_bpf_probe:func_1 to 0x1965020
add bpf event perf_bpf_probe:func_2 and attach bpf program 4
adding perf_bpf_probe:func_2
adding perf_bpf_probe:func_2 to 0x19656a0
mmap size 1052672B
BPF filter result incorrect, expected 56, got 0 samples
Opening /sys/kernel/debug/tracing//kprobe_events write=1
Opening /sys/kernel/debug/tracing//uprobe_events write=1
Parsing probe_events: p:perf_bpf_probe/func _text+2846705
Group:perf_bpf_probe Event:func probe:p
Parsing probe_events: p:perf_bpf_probe/func_1 _text+2846183
Group:perf_bpf_probe Event:func_1 probe:p
Parsing probe_events: p:perf_bpf_probe/func_2 _text+2845824
Group:perf_bpf_probe Event:func_2 probe:p
Writing event: -:perf_bpf_probe/func
Opening /sys/kernel/debug/tracing//kprobe_events write=1
Opening /sys/kernel/debug/tracing//uprobe_events write=1
Parsing probe_events: p:perf_bpf_probe/func_1 _text+2846183
Group:perf_bpf_probe Event:func_1 probe:p
Parsing probe_events: p:perf_bpf_probe/func_2 _text+2845824
Group:perf_bpf_probe Event:func_2 probe:p
Writing event: -:perf_bpf_probe/func_1
Opening /sys/kernel/debug/tracing//kprobe_events write=1
Opening /sys/kernel/debug/tracing//uprobe_events write=1
Parsing probe_events: p:perf_bpf_probe/func_2 _text+2845824
Group:perf_bpf_probe Event:func_2 probe:p
Writing event: -:perf_bpf_probe/func_2
test child finished with -1
---- end ----
BPF filter subtest 0: FAILED!
39.2: BPF pinning :
--- force skipped ---
BPF filter subtest 1: Skip
39.3: BPF prologue generation :
--- force skipped ---
BPF filter subtest 2: Skip
39.4: BPF relocation checker :
--- force skipped ---
BPF filter subtest 3: Skip
[root@jouet linux]#
[root@jouet linux]#

#2 lore raw

And please check if the kprobe created by

 $ perf probe -v SyS_epoll_wait

works for the test program used by this testcase:

#include <sys/epoll.h>
#include <stdlib.h>

#define NR_ITERS 100

static int epoll_wait_loop(void)

{
        int i;

        /* Should fail NR_ITERS times */
        for (i = 0; i < NR_ITERS; i++)
                epoll_wait(-(i + 1), NULL, 0, 0);
        return 0;
}

int main()

{
        epoll_wait_loop();
        return 0;
}

In my configuration it gives 100 samples:

$ sudo ./perf record -e probe:SyS_epoll_wait /tmp/a.out
[ perf record: Woken up 1 times to write data ]
[ perf record: Captured and wrote 0.016 MB perf.data (100 samples) ]

I guess in your configuration you would get 3 tracepoints, and can get 0 sample from them.

Thank you.

Quoted 252 lines · Wangnan (F)

Unable to reproduce. In my kernel configuration, SyS_epoll_wait is not inlined at all.

From the log you sent, it seems that all 3 instances are attached. This testcase should work if the last one (SyS_epoll_wait) get probed correctly.

Could you please have a look if the 3rd kprobe event (p:perf_bpf_probe/func_2 _text+2845824) works?

Thank you.

Quoted 242 lines · Arnaldo Carvalho de Melo

Hi Wang,

I just updated my machine to Fedora 27 and 4.15.0-rc6 and the
only test failing for me is:

[root@jouet linux]# perf test bpf

39: BPF filter                                            :
39.1: Basic BPF filtering                                 : FAILED!
39.2: BPF pinning                                         : Skip
39.3: BPF prologue generation                             : Skip
39.4: BPF relocation checker                              : Skip

[root@jouet linux]#

I haven't checked but perhaps the problem is that SyS_epoll_wait seems to now be inlined in three places and perhaps the eBPF proggie is being added to just one of them?

Seemingly relevant excerpt:

Open Debuginfo file: /lib/modules/4.15.0-rc6/build/vmlinux Try to find probe point from debuginfo.
Matched function: SyS_epoll_wait [2f40eb7]
found inline addr: 0xffffffff812b6ff1
Probe point found: compat_SyS_epoll_pwait+129 found inline addr: 0xffffffff812b6de7
Probe point found: SyS_epoll_pwait+135
found inline addr: 0xffffffff812b6c80
Probe point found: SyS_epoll_wait+0
Found 3 probe_trace_events.

- Arnaldo

P.S.: Full -v output:

[root@jouet linux]# uname -a
Linux jouet 4.15.0-rc6 #4 SMP Tue Jan 2 14:30:53 -03 2018 x86_64 x86_64 x86_64 GNU/Linux
[root@jouet linux]# gcc -v
Using built-in specs.
COLLECT_GCC=/usr/bin/gcc
COLLECT_LTO_WRAPPER=/usr/libexec/gcc/x86_64-redhat-linux/7/lto-wrapper
OFFLOAD_TARGET_NAMES=nvptx-none
OFFLOAD_TARGET_DEFAULT=1
Target: x86_64-redhat-linux
Configured with: ../configure --enable-bootstrap
--enable-languages=c,c++,objc,obj-c++,fortran,ada,go,lto
--prefix=/usr --mandir=/usr/share/man --infodir=/usr/share/info
--with-bugurl=http://bugzilla.redhat.com/bugzilla --enable-shared
--enable-threads=posix --enable-checking=release --enable-multilib
--with-system-zlib --enable-__cxa_atexit
--disable-libunwind-exceptions --enable-gnu-unique-object
--enable-linker-build-id --with-gcc-major-version-only
--with-linker-hash-style=gnu --enable-plugin --enable-initfini-array
--with-isl --enable-libmpx --enable-offload-targets=nvptx-none
--without-cuda-driver --enable-gnu-indirect-function
--with-tune=generic --with-arch_32=i686 --build=x86_64-redhat-linux
Thread model: posix
gcc version 7.2.1 20170915 (Red Hat 7.2.1-2) (GCC)
[root@jouet linux]# clang -v
clang version 6.0.0 (http://llvm.org/git/clang.git 56cc8f8880db2ebc433eeb6b6a707c101467a186) (http://llvm.org/git/llvm.git 3656d83960a4f3fedf6d8f19043abf52379f78c3)
Target: x86_64-unknown-linux-gnu
Thread model: posix
InstalledDir: /usr/local/bin
Found candidate GCC installation: /usr/lib/gcc/x86_64-redhat-linux/7
Selected GCC installation: /usr/lib/gcc/x86_64-redhat-linux/7
Candidate multilib: .;@m64
Candidate multilib: 32;@m32
Selected multilib: .;@m64
[root@jouet linux]# perf test -v BPF
39: BPF filter :
39.1: Basic BPF filtering :
--- start ---
test child forked, pid 24304
Kernel build dir is set to /lib/modules/4.15.0-rc6/build
set env: KBUILD_DIR=/lib/modules/4.15.0-rc6/build
unset env: KBUILD_OPTS
include option is set to -nostdinc -isystem /usr/lib/gcc/x86_64-redhat-linux/7/include

(quoted diff) +0 -6
-I/home/acme/git/linux/arch/x86/include 
-I./arch/x86/include/generated  -I/home/acme/git/linux/include 
-I./include -I/home/acme/git/linux/arch/x86/include/uapi 
-I./arch/x86/include/generated/uapi 
-I/home/acme/git/linux/include/uapi -I./include/generated/uapi 
-include /home/acme/git/linux/include/linux/kconfig.h

set env: NR_CPUS=4
set env: LINUX_VERSION_CODE=0x40f00
set env: CLANG_EXEC=/usr/local/bin/clang
set env: CLANG_OPTIONS=-xc
set env: KERNEL_INC_OPTIONS= -nostdinc -isystem /usr/lib/gcc/x86_64-redhat-linux/7/include

(quoted diff) +0 -6
-I/home/acme/git/linux/arch/x86/include 
-I./arch/x86/include/generated  -I/home/acme/git/linux/include 
-I./include -I/home/acme/git/linux/arch/x86/include/uapi 
-I./arch/x86/include/generated/uapi 
-I/home/acme/git/linux/include/uapi -I./include/generated/uapi 
-include /home/acme/git/linux/include/linux/kconfig.h

set env: WORKING_DIR=/lib/modules/4.15.0-rc6/build set env: CLANG_SOURCE=-
llvm compiling command template: echo '/*
* bpf-script-example.c
* Test basic LLVM building
*/
#ifndef LINUX_VERSION_CODE

# error Need LINUX_VERSION_CODE
# error Example: for 4.2 kernel, put 'clang-opt="-DLINUX_VERSION_CODE=0x40200" into llvm section of ~/.perfconfig'
#endif
#define BPF_ANY 0
#define BPF_MAP_TYPE_ARRAY 2
#define BPF_FUNC_map_lookup_elem 1
#define BPF_FUNC_map_update_elem 2

static void *(*bpf_map_lookup_elem)(void *map, void *key) =

    (void *) BPF_FUNC_map_lookup_elem;
static void *(*bpf_map_update_elem)(void *map, void *key, void *value, int flags) =
    (void *) BPF_FUNC_map_update_elem;
struct bpf_map_def {
    unsigned int type;
    unsigned int key_size;
    unsigned int value_size;
    unsigned int max_entries;
};

#define SEC(NAME) __attribute__((section(NAME), used)) struct bpf_map_def SEC("maps") flip_table = {
.type = BPF_MAP_TYPE_ARRAY,
.key_size = sizeof(int),
.value_size = sizeof(int),
.max_entries = 1,
};

SEC("func=SyS_epoll_wait")
int bpf_func__SyS_epoll_wait(void *ctx)

{
    int ind =0;
    int *flag = bpf_map_lookup_elem(&flip_table, &ind);
    int new_flag;
    if (!flag)
        return 0;
    /* flip flag and store back */
    new_flag = !*flag;
    bpf_map_update_elem(&flip_table, &ind, &new_flag, BPF_ANY);
    return new_flag;
}

char _license[] SEC("license") = "GPL";
int _version SEC("version") = LINUX_VERSION_CODE;
' | $CLANG_EXEC -D__KERNEL__ -D__NR_CPUS__=$NR_CPUS -DLINUX_VERSION_CODE=$LINUX_VERSION_CODE $CLANG_OPTIONS $KERNEL_INC_OPTIONS -Wno-unused-value -Wno-pointer-sign -working-directory $WORKING_DIR -c "$CLANG_SOURCE" -target bpf -O2 -o -
libbpf: loading object '[basic_bpf_test]' from buffer
libbpf: section .strtab, size 120, link 0, flags 0, type=3
libbpf: section .text, size 0, link 0, flags 6, type=1
libbpf: section func=SyS_epoll_wait, size 192, link 0, flags 6, type=1
libbpf: found program func=SyS_epoll_wait
libbpf: section .relfunc=SyS_epoll_wait, size 32, link 8, flags 0, type=9
libbpf: section maps, size 16, link 0, flags 3, type=1
libbpf: section license, size 4, link 0, flags 3, type=1
libbpf: license of [basic_bpf_test] is GPL
libbpf: section version, size 4, link 0, flags 3, type=1
libbpf: kernel version of [basic_bpf_test] is 40f00
libbpf: section .symtab, size 168, link 1, flags 0, type=2
libbpf: maps in [basic_bpf_test]: 1 maps in 16 bytes
libbpf: map 0 is "flip_table"
libbpf: collecting relocating info for: 'func=SyS_epoll_wait'
libbpf: relocation: insn_idx=4
libbpf: relocation: find map 0 (flip_table) for insn 4
libbpf: relocation: insn_idx=17
libbpf: relocation: find map 0 (flip_table) for insn 17
bpf: config program 'func=SyS_epoll_wait' symbol:SyS_epoll_wait file:(null) line:0 offset:0 return:0 lazy:(null)
bpf: config 'func=SyS_epoll_wait' is ok
Looking at the vmlinux_path (8 entries long)
Using /lib/modules/4.15.0-rc6/build/vmlinux for symbols
Open Debuginfo file: /lib/modules/4.15.0-rc6/build/vmlinux Try to find probe point from debuginfo.
Matched function: SyS_epoll_wait [2f40eb7]
found inline addr: 0xffffffff812b6ff1
Probe point found: compat_SyS_epoll_pwait+129
found inline addr: 0xffffffff812b6de7
Probe point found: SyS_epoll_pwait+135
found inline addr: 0xffffffff812b6c80
Probe point found: SyS_epoll_wait+0
Found 3 probe_trace_events.
Opening /sys/kernel/debug/tracing//kprobe_events write=1 Writing event: p:perf_bpf_probe/func _text+2846705
Writing event: p:perf_bpf_probe/func_1 _text+2846183
Writing event: p:perf_bpf_probe/func_2 _text+2845824
libbpf: create map flip_table: fd=3
add bpf event perf_bpf_probe:func and attach bpf program 4 registering plugin: /root/.traceevent/plugins/plugin_kvm.so registering plugin: /root/.traceevent/plugins/plugin_hrtimer.so registering plugin: /root/.traceevent/plugins/plugin_jbd2.so registering plugin: /root/.traceevent/plugins/plugin_kmem.so registering plugin: /root/.traceevent/plugins/plugin_scsi.so registering plugin: /root/.traceevent/plugins/plugin_sched_switch.so registering plugin: /root/.traceevent/plugins/plugin_function.so registering plugin: /root/.traceevent/plugins/plugin_xen.so registering plugin: /root/.traceevent/plugins/plugin_mac80211.so registering plugin: /root/.traceevent/plugins/plugin_cfg80211.so adding perf_bpf_probe:func
adding perf_bpf_probe:func to 0x24e3850
add bpf event perf_bpf_probe:func_1 and attach bpf program 4 adding perf_bpf_probe:func_1
adding perf_bpf_probe:func_1 to 0x1965020
add bpf event perf_bpf_probe:func_2 and attach bpf program 4 adding perf_bpf_probe:func_2
adding perf_bpf_probe:func_2 to 0x19656a0
mmap size 1052672B
BPF filter result incorrect, expected 56, got 0 samples Opening /sys/kernel/debug/tracing//kprobe_events write=1 Opening /sys/kernel/debug/tracing//uprobe_events write=1 Parsing probe_events: p:perf_bpf_probe/func _text+2846705 Group:perf_bpf_probe Event:func probe:p
Parsing probe_events: p:perf_bpf_probe/func_1 _text+2846183 Group:perf_bpf_probe Event:func_1 probe:p
Parsing probe_events: p:perf_bpf_probe/func_2 _text+2845824 Group:perf_bpf_probe Event:func_2 probe:p
Writing event: -:perf_bpf_probe/func
Opening /sys/kernel/debug/tracing//kprobe_events write=1 Opening /sys/kernel/debug/tracing//uprobe_events write=1 Parsing probe_events: p:perf_bpf_probe/func_1 _text+2846183 Group:perf_bpf_probe Event:func_1 probe:p
Parsing probe_events: p:perf_bpf_probe/func_2 _text+2845824 Group:perf_bpf_probe Event:func_2 probe:p
Writing event: -:perf_bpf_probe/func_1
Opening /sys/kernel/debug/tracing//kprobe_events write=1 Opening /sys/kernel/debug/tracing//uprobe_events write=1 Parsing probe_events: p:perf_bpf_probe/func_2 _text+2845824 Group:perf_bpf_probe Event:func_2 probe:p
Writing event: -:perf_bpf_probe/func_2
test child finished with -1
---- end ----
BPF filter subtest 0: FAILED!
39.2: BPF pinning :
--- force skipped ---
BPF filter subtest 1: Skip
39.3: BPF prologue generation :
--- force skipped ---
BPF filter subtest 2: Skip
39.4: BPF relocation checker :
--- force skipped ---
BPF filter subtest 3: Skip
[root@jouet linux]#
[root@jouet linux]#

author maintainer
#3 lore raw

Em Wed, Jan 03, 2018 at 12:58:26PM +0800, Wangnan (F) escreveu:

Quoted 3 lines

And please check if the kprobe created by

 $ perf probe -v SyS_epoll_wait

[root@seventh ~]# perf probe -v SyS_epoll_wait probe-definition(0): SyS_epoll_wait
symbol:SyS_epoll_wait file:(null) line:0 offset:0 return:0 lazy:(null) 0 arguments
Looking at the vmlinux_path (8 entries long)
Using /usr/lib/debug/lib/modules/4.14.8-300.fc27.x86_64/vmlinux for symbols Open Debuginfo file: /usr/lib/debug/lib/modules/4.14.8-300.fc27.x86_64/vmlinux Try to find probe point from debuginfo.
Matched function: SyS_epoll_wait [314b563]
found inline addr: 0xffffffff812d8087
Probe point found: compat_SyS_epoll_pwait+151
found inline addr: 0xffffffff812d7e5a
Probe point found: SyS_epoll_pwait+138
found inline addr: 0xffffffff812d7cf0
Probe point found: SyS_epoll_wait+0
Found 3 probe_trace_events.
Opening /sys/kernel/debug/tracing//kprobe_events write=1
Writing event: p:probe/SyS_epoll_wait _text+2982023
Writing event: p:probe/SyS_epoll_wait_1 _text+2981466
Writing event: p:probe/SyS_epoll_wait_2 _text+2981104
Added new events:
probe:SyS_epoll_wait (on SyS_epoll_wait)
probe:SyS_epoll_wait_1 (on SyS_epoll_wait) probe:SyS_epoll_wait_2 (on SyS_epoll_wait)

You can now use it in all perf tools, such as:

perf record -e probe:SyS_epoll_wait_2 -aR sleep 1

[root@seventh ~]#

Then, with your proggie, it seems its glibc that now maps epoll_wait() to a different syscall than in older systems:

[root@seventh ~]# trace -e probe:SyS_epoll_wait* ~acme/epoll

     0.018 ( 0.001 ms): epoll/20444 brk(                                                                  ) = 0x2600000
     0.029 ( 0.004 ms): epoll/20444 access(filename: 0xe874ec0, mode: R                                   ) = -1 ENOENT No such file or directory
     0.035 ( 0.003 ms): epoll/20444 openat(dfd: CWD, filename: 0xe872726, flags: CLOEXEC                  ) = 3
     0.040 ( 0.002 ms): epoll/20444 fstat(fd: 3, statbuf: 0x7fffcc37b830                                  ) = 0
     0.042 ( 0.002 ms): epoll/20444 mmap(len: 109538, prot: READ, flags: PRIVATE, fd: 3                   ) = 0x7fc50ea5d000
     0.045 ( 0.001 ms): epoll/20444 close(fd: 3                                                           ) = 0
     0.052 ( 0.003 ms): epoll/20444 openat(dfd: CWD, filename: 0xea7ace0, flags: CLOEXEC                  ) = 3
     0.056 ( 0.001 ms): epoll/20444 read(fd: 3, buf: 0x7fffcc37b9f8, count: 832                           ) = 832
     0.059 ( 0.001 ms): epoll/20444 fstat(fd: 3, statbuf: 0x7fffcc37b890                                  ) = 0
     0.060 ( 0.002 ms): epoll/20444 mmap(len: 8192, prot: READ|WRITE, flags: PRIVATE|ANONYMOUS, fd: -1    ) = 0x7fc50ea5b000
     0.065 ( 0.003 ms): epoll/20444 mmap(len: 4074112, prot: EXEC|READ, flags: PRIVATE|DENYWRITE, fd: 3   ) = 0x7fc50e46e000
     0.068 ( 0.004 ms): epoll/20444 mprotect(start: 0x7fc50e648000, len: 2093056                          ) = 0
     0.073 ( 0.004 ms): epoll/20444 mmap(addr: 0x7fc50e847000, len: 24576, prot: READ|WRITE, flags: PRIVATE|DENYWRITE|FIXED, fd: 3, off: 1937408) = 0x7fc50e847000
     0.079 ( 0.002 ms): epoll/20444 mmap(addr: 0x7fc50e84d000, len: 14976, prot: READ|WRITE, flags: PRIVATE|ANONYMOUS|FIXED, fd: -1) = 0x7fc50e84d000
     0.085 ( 0.001 ms): epoll/20444 close(fd: 3                                                           ) = 0
     0.093 ( 0.001 ms): epoll/20444 arch_prctl(option: 4098, arg2: 140484331029696                        ) = 0
     0.128 ( 0.003 ms): epoll/20444 mprotect(start: 0x7fc50e847000, len: 16384, prot: READ                ) = 0
     0.133 ( 0.002 ms): epoll/20444 mprotect(start: 0x600000, len: 4096, prot: READ                       ) = 0
     0.137 ( 0.002 ms): epoll/20444 mprotect(start: 0x7fc50ea78000, len: 4096, prot: READ                 ) = 0
     0.140 ( 0.005 ms): epoll/20444 munmap(addr: 0x7fc50ea5d000, len: 109538                              ) = 0
     0.159 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -1, sigsetsize: 8                                   ) = -22
     0.160 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -2, sigsetsize: 8                                   ) = -22
     0.162 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -3, sigsetsize: 8                                   ) = -22
     0.163 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -4, sigsetsize: 8                                   ) = -22
     0.165 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -5, sigsetsize: 8                                   ) = -22
<SNIP>
     0.279 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -85, sigsetsize: 8                                  ) = -22
     0.280 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -86, sigsetsize: 8                                  ) = -22
     0.282 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -87, sigsetsize: 8                                  ) = -22
     0.283 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -88, sigsetsize: 8                                  ) = -22
     0.284 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -89, sigsetsize: 8                                  ) = -22
     0.286 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -90, sigsetsize: 8                                  ) = -22
     0.287 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -91, sigsetsize: 8                                  ) = -22
     0.289 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -92, sigsetsize: 8                                  ) = -22
     0.290 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -93, sigsetsize: 8                                  ) = -22
     0.292 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -94, sigsetsize: 8                                  ) = -22
     0.293 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -95, sigsetsize: 8                                  ) = -22
     0.294 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -96, sigsetsize: 8                                  ) = -22
     0.296 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -97, sigsetsize: 8                                  ) = -22
     0.297 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -98, sigsetsize: 8                                  ) = -22
     0.299 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -99, sigsetsize: 8                                  ) = -22
     0.300 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -100, sigsetsize: 8                                 ) = -22
     0.305 (     ?   ): epoll/20444 exit_group(                                                           )

[root@seventh ~]#

This is on a Fedora 27 system, that has glibc 2.26, while on a RHEL7 machine, that has glibc 2.17, it still uses epoll_wait() and we get:

[root@felicio ~]# trace ./epoll

     0.018 ( 0.001 ms): epoll/2818 brk(                                                                  ) = 0x8e5000
     0.033 ( 0.002 ms): epoll/2818 mmap(len: 4096, prot: READ|WRITE, flags: PRIVATE|ANONYMOUS            ) = 0x7f3f738e3000
     0.041 ( 0.004 ms): epoll/2818 access(filename: 0x736e2160, mode: R                                  ) = -1 ENOENT No such file or directory
     0.051 ( 0.003 ms): epoll/2818 open(filename: 0x736e0a58, flags: CLOEXEC                             ) = 3
     0.056 ( 0.001 ms): epoll/2818 fstat(fd: 3, statbuf: 0x7fff8b35f5b0                                  ) = 0
     0.057 ( 0.002 ms): epoll/2818 mmap(len: 78559, prot: READ, flags: PRIVATE, fd: 3                    ) = 0x7f3f738cf000
     0.060 ( 0.001 ms): epoll/2818 close(fd: 3                                                           ) = 0
     0.070 ( 0.005 ms): epoll/2818 open(filename: 0x738e3640, flags: CLOEXEC                             ) = 3
     0.076 ( 0.001 ms): epoll/2818 read(fd: 3, buf: 0x7fff8b35f770, count: 832                           ) = 832
     0.079 ( 0.001 ms): epoll/2818 fstat(fd: 3, statbuf: 0x7fff8b35f610                                  ) = 0
     0.082 ( 0.003 ms): epoll/2818 mmap(len: 3932672, prot: EXEC|READ, flags: PRIVATE|DENYWRITE, fd: 3   ) = 0x7f3f73304000
     0.086 ( 0.005 ms): epoll/2818 mprotect(start: 0x7f3f734bb000, len: 2093056                          ) = 0
     0.091 ( 0.005 ms): epoll/2818 mmap(addr: 0x7f3f736ba000, len: 24576, prot: READ|WRITE, flags: PRIVATE|DENYWRITE|FIXED, fd: 3, off: 1794048) = 0x7f3f736ba000
     0.100 ( 0.003 ms): epoll/2818 mmap(addr: 0x7f3f736c0000, len: 16896, prot: READ|WRITE, flags: PRIVATE|ANONYMOUS|FIXED) = 0x7f3f736c0000
     0.106 ( 0.001 ms): epoll/2818 close(fd: 3                                                           ) = 0
     0.115 ( 0.002 ms): epoll/2818 mmap(len: 4096, prot: READ|WRITE, flags: PRIVATE|ANONYMOUS            ) = 0x7f3f738ce000
     0.119 ( 0.002 ms): epoll/2818 mmap(len: 8192, prot: READ|WRITE, flags: PRIVATE|ANONYMOUS            ) = 0x7f3f738cc000

0.123 ( 0.001 ms): epoll/2818 arch_prctl(option: 4098, arg2: 139910498273088, arg3: 139910498275408, arg4: 34, arg5: -1) = 0
0.181 ( 0.003 ms): epoll/2818 mprotect(start: 0x7f3f736ba000, len: 16384, prot: READ ) = 0
0.186 ( 0.002 ms): epoll/2818 mprotect(start: 0x600000, len: 4096, prot: READ ) = 0
0.190 ( 0.002 ms): epoll/2818 mprotect(start: 0x7f3f738e4000, len: 4096, prot: READ ) = 0
0.193 ( 0.004 ms): epoll/2818 munmap(addr: 0x7f3f738cf000, len: 78559 ) = 0
0.218 ( 0.001 ms): epoll/2818 epoll_wait(epfd: -1 ) = -1 EINVAL Invalid argument
0.220 ( 0.000 ms): epoll/2818 epoll_wait(epfd: -2 ) = -1 EINVAL Invalid argument
0.221 ( 0.000 ms): epoll/2818 epoll_wait(epfd: -3 ) = -1 EINVAL Invalid argument
0.221 ( 0.000 ms): epoll/2818 epoll_wait(epfd: -4 ) = -1 EINVAL Invalid argument
0.222 ( 0.000 ms): epoll/2818 epoll_wait(epfd: -5 ) = -1 EINVAL Invalid argument
<SNIP>

It thought that this part would cover it:

Probe point found: SyS_epoll_pwait+138
found inline addr: 0xffffffff812d7cf0

Continuing investigation...

- Arnaldo

Quoted 303 lines

works for the test program used by this testcase:

#include <sys/epoll.h>
#include <stdlib.h>

#define NR_ITERS 100

static int epoll_wait_loop(void)

{
        int i;

        /* Should fail NR_ITERS times */
        for (i = 0; i < NR_ITERS; i++)
                epoll_wait(-(i + 1), NULL, 0, 0);
        return 0;
}

int main()

{
        epoll_wait_loop();
        return 0;
}

In my configuration it gives 100 samples:

$ sudo ./perf record -e probe:SyS_epoll_wait /tmp/a.out
[ perf record: Woken up 1 times to write data ]
[ perf record: Captured and wrote 0.016 MB perf.data (100 samples) ]

I guess in your configuration you would get 3 tracepoints, and can get 0 sample from them.

Thank you.

Quoted 267 lines · Wangnan (F)

Unable to reproduce. In my kernel configuration, SyS_epoll_wait is not inlined at all.

From the log you sent, it seems that all 3 instances are attached. This testcase should work if the last one (SyS_epoll_wait) get probed correctly.

Could you please have a look if the 3rd kprobe event (p:perf_bpf_probe/func_2 _text+2845824) works?

Thank you.

Quoted 253 lines · Arnaldo Carvalho de Melo

Hi Wang,

I just updated my machine to Fedora 27 and 4.15.0-rc6 and the
only test failing for me is:

[root@jouet linux]# perf test bpf

39: BPF filter                                            :
39.1: Basic BPF filtering                                 : FAILED!
39.2: BPF pinning                                         : Skip
39.3: BPF prologue generation                             : Skip
39.4: BPF relocation checker                              : Skip

[root@jouet linux]#

I haven't checked but perhaps the problem is that SyS_epoll_wait seems to now be inlined in three places and perhaps the eBPF proggie is being added to just one of them?

Seemingly relevant excerpt:

Open Debuginfo file: /lib/modules/4.15.0-rc6/build/vmlinux Try to find probe point from debuginfo.
Matched function: SyS_epoll_wait [2f40eb7]
found inline addr: 0xffffffff812b6ff1
Probe point found: compat_SyS_epoll_pwait+129 found inline addr: 0xffffffff812b6de7
Probe point found: SyS_epoll_pwait+135
found inline addr: 0xffffffff812b6c80
Probe point found: SyS_epoll_wait+0
Found 3 probe_trace_events.

- Arnaldo

P.S.: Full -v output:

[root@jouet linux]# uname -a
Linux jouet 4.15.0-rc6 #4 SMP Tue Jan 2 14:30:53 -03 2018 x86_64 x86_64 x86_64 GNU/Linux
[root@jouet linux]# gcc -v
Using built-in specs.
COLLECT_GCC=/usr/bin/gcc COLLECT_LTO_WRAPPER=/usr/libexec/gcc/x86_64-redhat-linux/7/lto-wrapper OFFLOAD_TARGET_NAMES=nvptx-none
OFFLOAD_TARGET_DEFAULT=1
Target: x86_64-redhat-linux
Configured with: ../configure --enable-bootstrap --enable-languages=c,c++,objc,obj-c++,fortran,ada,go,lto --prefix=/usr --mandir=/usr/share/man --infodir=/usr/share/info --with-bugurl=http://bugzilla.redhat.com/bugzilla --enable-shared --enable-threads=posix --enable-checking=release --enable-multilib --with-system-zlib --enable-__cxa_atexit --disable-libunwind-exceptions --enable-gnu-unique-object --enable-linker-build-id --with-gcc-major-version-only --with-linker-hash-style=gnu --enable-plugin --enable-initfini-array --with-isl --enable-libmpx --enable-offload-targets=nvptx-none --without-cuda-driver --enable-gnu-indirect-function --with-tune=generic --with-arch_32=i686 --build=x86_64-redhat-linux Thread model: posix
gcc version 7.2.1 20170915 (Red Hat 7.2.1-2) (GCC) [root@jouet linux]# clang -v
clang version 6.0.0 (http://llvm.org/git/clang.git
56cc8f8880db2ebc433eeb6b6a707c101467a186)
(http://llvm.org/git/llvm.git
3656d83960a4f3fedf6d8f19043abf52379f78c3)
Target: x86_64-unknown-linux-gnu
Thread model: posix
InstalledDir: /usr/local/bin
Found candidate GCC installation: /usr/lib/gcc/x86_64-redhat-linux/7 Selected GCC installation: /usr/lib/gcc/x86_64-redhat-linux/7 Candidate multilib: .;@m64
Candidate multilib: 32;@m32
Selected multilib: .;@m64
[root@jouet linux]# perf test -v BPF
39: BPF filter :
39.1: Basic BPF filtering :
--- start ---
test child forked, pid 24304
Kernel build dir is set to /lib/modules/4.15.0-rc6/build set env: KBUILD_DIR=/lib/modules/4.15.0-rc6/build
unset env: KBUILD_OPTS
include option is set to -nostdinc -isystem /usr/lib/gcc/x86_64-redhat-linux/7/include

(quoted diff) +0 -6
-I/home/acme/git/linux/arch/x86/include
-I./arch/x86/include/generated  -I/home/acme/git/linux/include
-I./include -I/home/acme/git/linux/arch/x86/include/uapi
-I./arch/x86/include/generated/uapi
-I/home/acme/git/linux/include/uapi -I./include/generated/uapi
-include /home/acme/git/linux/include/linux/kconfig.h

set env: NR_CPUS=4
set env: LINUX_VERSION_CODE=0x40f00
set env: CLANG_EXEC=/usr/local/bin/clang
set env: CLANG_OPTIONS=-xc
set env: KERNEL_INC_OPTIONS= -nostdinc -isystem
/usr/lib/gcc/x86_64-redhat-linux/7/include

(quoted diff) +0 -6
-I/home/acme/git/linux/arch/x86/include
-I./arch/x86/include/generated  -I/home/acme/git/linux/include
-I./include -I/home/acme/git/linux/arch/x86/include/uapi
-I./arch/x86/include/generated/uapi
-I/home/acme/git/linux/include/uapi -I./include/generated/uapi
-include /home/acme/git/linux/include/linux/kconfig.h

set env: WORKING_DIR=/lib/modules/4.15.0-rc6/build set env: CLANG_SOURCE=-
llvm compiling command template: echo '/*
* bpf-script-example.c
* Test basic LLVM building
*/
#ifndef LINUX_VERSION_CODE

# error Need LINUX_VERSION_CODE
# error Example: for 4.2 kernel, put
'clang-opt="-DLINUX_VERSION_CODE=0x40200" into llvm section of
~/.perfconfig'
#endif
#define BPF_ANY 0
#define BPF_MAP_TYPE_ARRAY 2
#define BPF_FUNC_map_lookup_elem 1
#define BPF_FUNC_map_update_elem 2

static void *(*bpf_map_lookup_elem)(void *map, void *key) =

    (void *) BPF_FUNC_map_lookup_elem;
static void *(*bpf_map_update_elem)(void *map, void *key, void
*value, int flags) =
    (void *) BPF_FUNC_map_update_elem;
struct bpf_map_def {
    unsigned int type;
    unsigned int key_size;
    unsigned int value_size;
    unsigned int max_entries;
};

#define SEC(NAME) __attribute__((section(NAME), used)) struct bpf_map_def SEC("maps") flip_table = {
.type = BPF_MAP_TYPE_ARRAY,
.key_size = sizeof(int),
.value_size = sizeof(int),
.max_entries = 1,
};

SEC("func=SyS_epoll_wait")
int bpf_func__SyS_epoll_wait(void *ctx)

{
    int ind =0;
    int *flag = bpf_map_lookup_elem(&flip_table, &ind);
    int new_flag;
    if (!flag)
        return 0;
    /* flip flag and store back */
    new_flag = !*flag;
    bpf_map_update_elem(&flip_table, &ind, &new_flag, BPF_ANY);
    return new_flag;
}

char _license[] SEC("license") = "GPL";
int _version SEC("version") = LINUX_VERSION_CODE; ' | $CLANG_EXEC -D__KERNEL__ -D__NR_CPUS__=$NR_CPUS -DLINUX_VERSION_CODE=$LINUX_VERSION_CODE $CLANG_OPTIONS $KERNEL_INC_OPTIONS -Wno-unused-value -Wno-pointer-sign

(quoted diff) +0 -2
-working-directory $WORKING_DIR -c "$CLANG_SOURCE" -target bpf -O2
-o -

libbpf: loading object '[basic_bpf_test]' from buffer
libbpf: section .strtab, size 120, link 0, flags 0, type=3
libbpf: section .text, size 0, link 0, flags 6, type=1
libbpf: section func=SyS_epoll_wait, size 192, link 0, flags 6, type=1
libbpf: found program func=SyS_epoll_wait
libbpf: section .relfunc=SyS_epoll_wait, size 32, link 8, flags 0, type=9
libbpf: section maps, size 16, link 0, flags 3, type=1
libbpf: section license, size 4, link 0, flags 3, type=1
libbpf: license of [basic_bpf_test] is GPL
libbpf: section version, size 4, link 0, flags 3, type=1
libbpf: kernel version of [basic_bpf_test] is 40f00
libbpf: section .symtab, size 168, link 1, flags 0, type=2
libbpf: maps in [basic_bpf_test]: 1 maps in 16 bytes
libbpf: map 0 is "flip_table"
libbpf: collecting relocating info for: 'func=SyS_epoll_wait'
libbpf: relocation: insn_idx=4
libbpf: relocation: find map 0 (flip_table) for insn 4
libbpf: relocation: insn_idx=17
libbpf: relocation: find map 0 (flip_table) for insn 17
bpf: config program 'func=SyS_epoll_wait' symbol:SyS_epoll_wait file:(null) line:0 offset:0 return:0 lazy:(null)
bpf: config 'func=SyS_epoll_wait' is ok
Looking at the vmlinux_path (8 entries long)
Using /lib/modules/4.15.0-rc6/build/vmlinux for symbols Open Debuginfo file: /lib/modules/4.15.0-rc6/build/vmlinux Try to find probe point from debuginfo.
Matched function: SyS_epoll_wait [2f40eb7]
found inline addr: 0xffffffff812b6ff1
Probe point found: compat_SyS_epoll_pwait+129
found inline addr: 0xffffffff812b6de7
Probe point found: SyS_epoll_pwait+135
found inline addr: 0xffffffff812b6c80
Probe point found: SyS_epoll_wait+0
Found 3 probe_trace_events.
Opening /sys/kernel/debug/tracing//kprobe_events write=1 Writing event: p:perf_bpf_probe/func _text+2846705
Writing event: p:perf_bpf_probe/func_1 _text+2846183 Writing event: p:perf_bpf_probe/func_2 _text+2845824
libbpf: create map flip_table: fd=3
add bpf event perf_bpf_probe:func and attach bpf program 4 registering plugin: /root/.traceevent/plugins/plugin_kvm.so registering plugin: /root/.traceevent/plugins/plugin_hrtimer.so registering plugin: /root/.traceevent/plugins/plugin_jbd2.so registering plugin: /root/.traceevent/plugins/plugin_kmem.so registering plugin: /root/.traceevent/plugins/plugin_scsi.so registering plugin: /root/.traceevent/plugins/plugin_sched_switch.so registering plugin: /root/.traceevent/plugins/plugin_function.so registering plugin: /root/.traceevent/plugins/plugin_xen.so registering plugin: /root/.traceevent/plugins/plugin_mac80211.so registering plugin: /root/.traceevent/plugins/plugin_cfg80211.so adding perf_bpf_probe:func
adding perf_bpf_probe:func to 0x24e3850
add bpf event perf_bpf_probe:func_1 and attach bpf program 4 adding perf_bpf_probe:func_1
adding perf_bpf_probe:func_1 to 0x1965020
add bpf event perf_bpf_probe:func_2 and attach bpf program 4 adding perf_bpf_probe:func_2
adding perf_bpf_probe:func_2 to 0x19656a0
mmap size 1052672B
BPF filter result incorrect, expected 56, got 0 samples Opening /sys/kernel/debug/tracing//kprobe_events write=1 Opening /sys/kernel/debug/tracing//uprobe_events write=1 Parsing probe_events: p:perf_bpf_probe/func _text+2846705 Group:perf_bpf_probe Event:func probe:p
Parsing probe_events: p:perf_bpf_probe/func_1 _text+2846183 Group:perf_bpf_probe Event:func_1 probe:p
Parsing probe_events: p:perf_bpf_probe/func_2 _text+2845824 Group:perf_bpf_probe Event:func_2 probe:p
Writing event: -:perf_bpf_probe/func
Opening /sys/kernel/debug/tracing//kprobe_events write=1 Opening /sys/kernel/debug/tracing//uprobe_events write=1 Parsing probe_events: p:perf_bpf_probe/func_1 _text+2846183 Group:perf_bpf_probe Event:func_1 probe:p
Parsing probe_events: p:perf_bpf_probe/func_2 _text+2845824 Group:perf_bpf_probe Event:func_2 probe:p
Writing event: -:perf_bpf_probe/func_1
Opening /sys/kernel/debug/tracing//kprobe_events write=1 Opening /sys/kernel/debug/tracing//uprobe_events write=1 Parsing probe_events: p:perf_bpf_probe/func_2 _text+2845824 Group:perf_bpf_probe Event:func_2 probe:p
Writing event: -:perf_bpf_probe/func_2
test child finished with -1
---- end ----
BPF filter subtest 0: FAILED!
39.2: BPF pinning :
--- force skipped ---
BPF filter subtest 1: Skip
39.3: BPF prologue generation :
--- force skipped ---
BPF filter subtest 2: Skip
39.4: BPF relocation checker :
--- force skipped ---
BPF filter subtest 3: Skip
[root@jouet linux]#
[root@jouet linux]#

author maintainer
#4 lore raw

Em Wed, Jan 03, 2018 at 01:58:44PM -0300, Arnaldo Carvalho de Melo escreveu:

Quoted 118 lines

Em Wed, Jan 03, 2018 at 12:58:26PM +0800, Wangnan (F) escreveu:

Quoted 3 lines

And please check if the kprobe created by

 $ perf probe -v SyS_epoll_wait

[root@seventh ~]# perf probe -v SyS_epoll_wait probe-definition(0): SyS_epoll_wait
symbol:SyS_epoll_wait file:(null) line:0 offset:0 return:0 lazy:(null) 0 arguments
Looking at the vmlinux_path (8 entries long)
Using /usr/lib/debug/lib/modules/4.14.8-300.fc27.x86_64/vmlinux for symbols Open Debuginfo file: /usr/lib/debug/lib/modules/4.14.8-300.fc27.x86_64/vmlinux Try to find probe point from debuginfo.
Matched function: SyS_epoll_wait [314b563]
found inline addr: 0xffffffff812d8087
Probe point found: compat_SyS_epoll_pwait+151
found inline addr: 0xffffffff812d7e5a
Probe point found: SyS_epoll_pwait+138
found inline addr: 0xffffffff812d7cf0
Probe point found: SyS_epoll_wait+0
Found 3 probe_trace_events.
Opening /sys/kernel/debug/tracing//kprobe_events write=1
Writing event: p:probe/SyS_epoll_wait _text+2982023
Writing event: p:probe/SyS_epoll_wait_1 _text+2981466
Writing event: p:probe/SyS_epoll_wait_2 _text+2981104
Added new events:
probe:SyS_epoll_wait (on SyS_epoll_wait)
probe:SyS_epoll_wait_1 (on SyS_epoll_wait) probe:SyS_epoll_wait_2 (on SyS_epoll_wait)

You can now use it in all perf tools, such as:

perf record -e probe:SyS_epoll_wait_2 -aR sleep 1

[root@seventh ~]#

Then, with your proggie, it seems its glibc that now maps epoll_wait() to a different syscall than in older systems:

[root@seventh ~]# trace -e probe:SyS_epoll_wait* ~acme/epoll

     0.018 ( 0.001 ms): epoll/20444 brk(                                                                  ) = 0x2600000
     0.029 ( 0.004 ms): epoll/20444 access(filename: 0xe874ec0, mode: R                                   ) = -1 ENOENT No such file or directory
     0.035 ( 0.003 ms): epoll/20444 openat(dfd: CWD, filename: 0xe872726, flags: CLOEXEC                  ) = 3
     0.040 ( 0.002 ms): epoll/20444 fstat(fd: 3, statbuf: 0x7fffcc37b830                                  ) = 0
     0.042 ( 0.002 ms): epoll/20444 mmap(len: 109538, prot: READ, flags: PRIVATE, fd: 3                   ) = 0x7fc50ea5d000
     0.045 ( 0.001 ms): epoll/20444 close(fd: 3                                                           ) = 0
     0.052 ( 0.003 ms): epoll/20444 openat(dfd: CWD, filename: 0xea7ace0, flags: CLOEXEC                  ) = 3
     0.056 ( 0.001 ms): epoll/20444 read(fd: 3, buf: 0x7fffcc37b9f8, count: 832                           ) = 832
     0.059 ( 0.001 ms): epoll/20444 fstat(fd: 3, statbuf: 0x7fffcc37b890                                  ) = 0
     0.060 ( 0.002 ms): epoll/20444 mmap(len: 8192, prot: READ|WRITE, flags: PRIVATE|ANONYMOUS, fd: -1    ) = 0x7fc50ea5b000
     0.065 ( 0.003 ms): epoll/20444 mmap(len: 4074112, prot: EXEC|READ, flags: PRIVATE|DENYWRITE, fd: 3   ) = 0x7fc50e46e000
     0.068 ( 0.004 ms): epoll/20444 mprotect(start: 0x7fc50e648000, len: 2093056                          ) = 0
     0.073 ( 0.004 ms): epoll/20444 mmap(addr: 0x7fc50e847000, len: 24576, prot: READ|WRITE, flags: PRIVATE|DENYWRITE|FIXED, fd: 3, off: 1937408) = 0x7fc50e847000
     0.079 ( 0.002 ms): epoll/20444 mmap(addr: 0x7fc50e84d000, len: 14976, prot: READ|WRITE, flags: PRIVATE|ANONYMOUS|FIXED, fd: -1) = 0x7fc50e84d000
     0.085 ( 0.001 ms): epoll/20444 close(fd: 3                                                           ) = 0
     0.093 ( 0.001 ms): epoll/20444 arch_prctl(option: 4098, arg2: 140484331029696                        ) = 0
     0.128 ( 0.003 ms): epoll/20444 mprotect(start: 0x7fc50e847000, len: 16384, prot: READ                ) = 0
     0.133 ( 0.002 ms): epoll/20444 mprotect(start: 0x600000, len: 4096, prot: READ                       ) = 0
     0.137 ( 0.002 ms): epoll/20444 mprotect(start: 0x7fc50ea78000, len: 4096, prot: READ                 ) = 0
     0.140 ( 0.005 ms): epoll/20444 munmap(addr: 0x7fc50ea5d000, len: 109538                              ) = 0
     0.159 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -1, sigsetsize: 8                                   ) = -22
     0.160 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -2, sigsetsize: 8                                   ) = -22
     0.162 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -3, sigsetsize: 8                                   ) = -22
     0.163 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -4, sigsetsize: 8                                   ) = -22
     0.165 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -5, sigsetsize: 8                                   ) = -22
<SNIP>
     0.279 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -85, sigsetsize: 8                                  ) = -22
     0.280 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -86, sigsetsize: 8                                  ) = -22
     0.282 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -87, sigsetsize: 8                                  ) = -22
     0.283 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -88, sigsetsize: 8                                  ) = -22
     0.284 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -89, sigsetsize: 8                                  ) = -22
     0.286 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -90, sigsetsize: 8                                  ) = -22
     0.287 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -91, sigsetsize: 8                                  ) = -22
     0.289 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -92, sigsetsize: 8                                  ) = -22
     0.290 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -93, sigsetsize: 8                                  ) = -22
     0.292 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -94, sigsetsize: 8                                  ) = -22
     0.293 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -95, sigsetsize: 8                                  ) = -22
     0.294 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -96, sigsetsize: 8                                  ) = -22
     0.296 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -97, sigsetsize: 8                                  ) = -22
     0.297 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -98, sigsetsize: 8                                  ) = -22
     0.299 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -99, sigsetsize: 8                                  ) = -22
     0.300 ( 0.001 ms): epoll/20444 epoll_pwait(epfd: -100, sigsetsize: 8                                 ) = -22
     0.305 (     ?   ): epoll/20444 exit_group(                                                           )

[root@seventh ~]#

This is on a Fedora 27 system, that has glibc 2.26, while on a RHEL7 machine, that has glibc 2.17, it still uses epoll_wait() and we get:

[root@felicio ~]# trace ./epoll

     0.018 ( 0.001 ms): epoll/2818 brk(                                                                  ) = 0x8e5000
     0.033 ( 0.002 ms): epoll/2818 mmap(len: 4096, prot: READ|WRITE, flags: PRIVATE|ANONYMOUS            ) = 0x7f3f738e3000
     0.041 ( 0.004 ms): epoll/2818 access(filename: 0x736e2160, mode: R                                  ) = -1 ENOENT No such file or directory
     0.051 ( 0.003 ms): epoll/2818 open(filename: 0x736e0a58, flags: CLOEXEC                             ) = 3
     0.056 ( 0.001 ms): epoll/2818 fstat(fd: 3, statbuf: 0x7fff8b35f5b0                                  ) = 0
     0.057 ( 0.002 ms): epoll/2818 mmap(len: 78559, prot: READ, flags: PRIVATE, fd: 3                    ) = 0x7f3f738cf000
     0.060 ( 0.001 ms): epoll/2818 close(fd: 3                                                           ) = 0
     0.070 ( 0.005 ms): epoll/2818 open(filename: 0x738e3640, flags: CLOEXEC                             ) = 3
     0.076 ( 0.001 ms): epoll/2818 read(fd: 3, buf: 0x7fff8b35f770, count: 832                           ) = 832
     0.079 ( 0.001 ms): epoll/2818 fstat(fd: 3, statbuf: 0x7fff8b35f610                                  ) = 0
     0.082 ( 0.003 ms): epoll/2818 mmap(len: 3932672, prot: EXEC|READ, flags: PRIVATE|DENYWRITE, fd: 3   ) = 0x7f3f73304000
     0.086 ( 0.005 ms): epoll/2818 mprotect(start: 0x7f3f734bb000, len: 2093056                          ) = 0
     0.091 ( 0.005 ms): epoll/2818 mmap(addr: 0x7f3f736ba000, len: 24576, prot: READ|WRITE, flags: PRIVATE|DENYWRITE|FIXED, fd: 3, off: 1794048) = 0x7f3f736ba000
     0.100 ( 0.003 ms): epoll/2818 mmap(addr: 0x7f3f736c0000, len: 16896, prot: READ|WRITE, flags: PRIVATE|ANONYMOUS|FIXED) = 0x7f3f736c0000
     0.106 ( 0.001 ms): epoll/2818 close(fd: 3                                                           ) = 0
     0.115 ( 0.002 ms): epoll/2818 mmap(len: 4096, prot: READ|WRITE, flags: PRIVATE|ANONYMOUS            ) = 0x7f3f738ce000
     0.119 ( 0.002 ms): epoll/2818 mmap(len: 8192, prot: READ|WRITE, flags: PRIVATE|ANONYMOUS            ) = 0x7f3f738cc000

0.123 ( 0.001 ms): epoll/2818 arch_prctl(option: 4098, arg2: 139910498273088, arg3: 139910498275408, arg4: 34, arg5: -1) = 0
0.181 ( 0.003 ms): epoll/2818 mprotect(start: 0x7f3f736ba000, len: 16384, prot: READ ) = 0
0.186 ( 0.002 ms): epoll/2818 mprotect(start: 0x600000, len: 4096, prot: READ ) = 0
0.190 ( 0.002 ms): epoll/2818 mprotect(start: 0x7f3f738e4000, len: 4096, prot: READ ) = 0
0.193 ( 0.004 ms): epoll/2818 munmap(addr: 0x7f3f738cf000, len: 78559 ) = 0
0.218 ( 0.001 ms): epoll/2818 epoll_wait(epfd: -1 ) = -1 EINVAL Invalid argument
0.220 ( 0.000 ms): epoll/2818 epoll_wait(epfd: -2 ) = -1 EINVAL Invalid argument
0.221 ( 0.000 ms): epoll/2818 epoll_wait(epfd: -3 ) = -1 EINVAL Invalid argument
0.221 ( 0.000 ms): epoll/2818 epoll_wait(epfd: -4 ) = -1 EINVAL Invalid argument
0.222 ( 0.000 ms): epoll/2818 epoll_wait(epfd: -5 ) = -1 EINVAL Invalid argument
<SNIP>

Quoted 6 lines

It thought that this part would cover it:

Probe point found: SyS_epoll_pwait+138
found inline addr: 0xffffffff812d7cf0

Continuing investigation...

After applying the fallback patch to allow new tools to work with older kernels:

[root@felicio ~]# perf test bpf

39: BPF filter                                            :
39.1: Basic BPF filtering                                 : Ok
39.2: BPF pinning                                         : Ok
39.3: BPF prologue generation                             : Ok
39.4: BPF relocation checker                              : Ok

[root@felicio ~]# uname -a
Linux felicio.ghostprotocols.net 4.13.0-rc7+ #1 SMP Mon Sep 11 13:56:18 -03 2017 x86_64 x86_64 x86_64 GNU/Linux [root@felicio ~]# rpm -q glibc
glibc-2.17-157.el7_3.2.x86_64
[root@felicio ~]#

After applying the patch below I get to, which is what I am trying to fix now:

[root@jouet ~]# perf test bpf

39: BPF filter                                            :
39.1: Basic BPF filtering                                 : Ok
39.2: BPF pinning                                         : Ok
39.3: BPF prologue generation                             : FAILED!
39.4: BPF relocation checker                              : Skip

[root@jouet ~]#

Its related to:

<stdin>:11:10: fatal error: 'uapi/linux/fs.h' file not found

#include <uapi/linux/fs.h>
         ^~~~~~~~~~~~~~~~~

1 error generated.
ERROR: unable to compile -
Hint: Check error message shown above.
Hint: You can also pre-compile it into .o using:
clang -target bpf -O2 -c -
with proper -I and -D options.
Failed to compile test case: 'Compile source for BPF prologue generation' Unable to get BPF object, fix kbuild first
test child finished with -1
---- end ----
BPF filter subtest 2: FAILED!

- Arnaldo

tools/perf/tests/bpf-script-example.c +1 -1
index 268e5f8e4aa2..d1e67e271675 100644
@@ -31,7 +31,7 @@ struct bpf_map_def SEC("maps") flip_table = {
 	.max_entries = 1,
 };
 
-SEC("func=SyS_epoll_wait")
+SEC("func=SyS_epoll_pwait")
 int bpf_func__SyS_epoll_wait(void *ctx)
 {
 	int ind =0;
author maintainer
#5 lore raw

Em Wed, Jan 03, 2018 at 03:27:01PM -0300, Arnaldo Carvalho de Melo escreveu:

Quoted 20 lines
Quoted 1 line

Continuing investigation...

After applying the fallback patch to allow new tools to work with older kernels:

[root@felicio ~]# perf test bpf

39: BPF filter                                            :
39.1: Basic BPF filtering                                 : Ok
39.2: BPF pinning                                         : Ok
39.3: BPF prologue generation                             : Ok
39.4: BPF relocation checker                              : Ok

[root@felicio ~]# uname -a
Linux felicio.ghostprotocols.net 4.13.0-rc7+ #1 SMP Mon Sep 11 13:56:18 -03 2017 x86_64 x86_64 x86_64 GNU/Linux [root@felicio ~]# rpm -q glibc
glibc-2.17-157.el7_3.2.x86_64
[root@felicio ~]#

After applying the patch below I get to, which is what I am trying to fix now:

Quoted 7 lines

[root@jouet ~]# perf test bpf

39: BPF filter                                            :
39.1: Basic BPF filtering                                 : Ok
39.2: BPF pinning                                         : Ok
39.3: BPF prologue generation                             : FAILED!
39.4: BPF relocation checker                              : Skip

[root@jouet ~]#

Update the patch to the one at the end of this message to make it work with older glibcs, so that we ask for epoll_pwait() and hook into that as well().

Now checking why 39.3 fails...

- Arnaldo

tools/perf/tests/bpf-script-example.c +1 -1
index 268e5f8e4aa2..d1e67e271675 100644
@@ -31,7 +31,7 @@ struct bpf_map_def SEC("maps") flip_table = {
 	.max_entries = 1,
 };
 
-SEC("func=SyS_epoll_wait")
+SEC("func=SyS_epoll_pwait")
 int bpf_func__SyS_epoll_wait(void *ctx)
 {
 	int ind =0;
tools/perf/tests/bpf.c +1 -1
index 0512f1b5bfdb..7c04e2d5b60b 100644
@@ -25,7 +25,7 @@ static int epoll_wait_loop(void)
 
 	/* Should fail NR_ITERS times */
 	for (i = 0; i < NR_ITERS; i++)
-		epoll_wait(-(i + 1), NULL, 0, 0);
+		epoll_pwait(-(i + 1), NULL, 0, 0, NULL);
 	return 0;
 }
 
author maintainer
#6 lore raw

Em Wed, Jan 03, 2018 at 03:33:07PM -0300, Arnaldo Carvalho de Melo escreveu:

Quoted 29 lines

Em Wed, Jan 03, 2018 at 03:27:01PM -0300, Arnaldo Carvalho de Melo escreveu:

Quoted 20 lines
Quoted 1 line

Continuing investigation...

After applying the fallback patch to allow new tools to work with older kernels:

[root@felicio ~]# perf test bpf

39: BPF filter                                            :
39.1: Basic BPF filtering                                 : Ok
39.2: BPF pinning                                         : Ok
39.3: BPF prologue generation                             : Ok
39.4: BPF relocation checker                              : Ok

[root@felicio ~]# uname -a
Linux felicio.ghostprotocols.net 4.13.0-rc7+ #1 SMP Mon Sep 11 13:56:18 -03 2017 x86_64 x86_64 x86_64 GNU/Linux [root@felicio ~]# rpm -q glibc
glibc-2.17-157.el7_3.2.x86_64
[root@felicio ~]#

After applying the patch below I get to, which is what I am trying to fix now:

Quoted 7 lines

[root@jouet ~]# perf test bpf

39: BPF filter                                            :
39.1: Basic BPF filtering                                 : Ok
39.2: BPF pinning                                         : Ok
39.3: BPF prologue generation                             : FAILED!
39.4: BPF relocation checker                              : Skip

[root@jouet ~]#

Quoted 5 lines

Update the patch to the one at the end of this message to make it work with older glibcs, so that we ask for epoll_pwait() and hook into that as well().

Now checking why 39.3 fails...

Couldn't reproduce after fixing up some kernel build problems, the patch below is all I need to have this working with both Fedora 27 and RHEL7, please take a look and see if it continues to work on your systems,

- Arnaldo

Quoted 28 lines

- Arnaldo

tools/perf/tests/bpf-script-example.c +1 -1
index 268e5f8e4aa2..d1e67e271675 100644
@@ -31,7 +31,7 @@ struct bpf_map_def SEC("maps") flip_table = {
 	.max_entries = 1,
 };
 
-SEC("func=SyS_epoll_wait")
+SEC("func=SyS_epoll_pwait")
 int bpf_func__SyS_epoll_wait(void *ctx)
 {
 	int ind =0;
tools/perf/tests/bpf.c +1 -1
index 0512f1b5bfdb..7c04e2d5b60b 100644
@@ -25,7 +25,7 @@ static int epoll_wait_loop(void)
 
 	/* Should fail NR_ITERS times */
 	for (i = 0; i < NR_ITERS; i++)
-		epoll_wait(-(i + 1), NULL, 0, 0);
+		epoll_pwait(-(i + 1), NULL, 0, 0, NULL);
 	return 0;
 }
 
#7 lore raw
Quoted 28 lines · Arnaldo Carvalho de Melo

Em Wed, Jan 03, 2018 at 03:33:07PM -0300, Arnaldo Carvalho de Melo escreveu:

Quoted 27 lines

Em Wed, Jan 03, 2018 at 03:27:01PM -0300, Arnaldo Carvalho de Melo escreveu:

Quoted 26 lines
Quoted 1 line

Continuing investigation...

After applying the fallback patch to allow new tools to work with older kernels:

[root@felicio ~]# perf test bpf

39: BPF filter                                            :
39.1: Basic BPF filtering                                 : Ok
39.2: BPF pinning                                         : Ok
39.3: BPF prologue generation                             : Ok
39.4: BPF relocation checker                              : Ok

[root@felicio ~]# uname -a
Linux felicio.ghostprotocols.net 4.13.0-rc7+ #1 SMP Mon Sep 11 13:56:18 -03 2017 x86_64 x86_64 x86_64 GNU/Linux [root@felicio ~]# rpm -q glibc
glibc-2.17-157.el7_3.2.x86_64
[root@felicio ~]#

After applying the patch below I get to, which is what I am trying to fix now:

[root@jouet ~]# perf test bpf

39: BPF filter                                            :
39.1: Basic BPF filtering                                 : Ok
39.2: BPF pinning                                         : Ok
39.3: BPF prologue generation                             : FAILED!
39.4: BPF relocation checker                              : Skip

[root@jouet ~]#

Quoted 8 lines
Quoted 5 lines

Update the patch to the one at the end of this message to make it work with older glibcs, so that we ask for epoll_pwait() and hook into that as well().

Now checking why 39.3 fails...

Couldn't reproduce after fixing up some kernel build problems, the patch below is all I need to have this working with both Fedora 27 and RHEL7, please take a look and see if it continues to work on your systems,

It works for me. Thank you.

Since we test epoll_pwait, we'd better correct function names:

tools/perf/tests/bpf-script-example.c +2 -2
index 268e5f8..e4123c1 100644
@@ -31,8 +31,8 @@ struct bpf_map_def SEC("maps") flip_table = {
     .max_entries = 1,
 };
 
-SEC("func=SyS_epoll_wait")
-int bpf_func__SyS_epoll_wait(void *ctx)
+SEC("func=SyS_epoll_pwait")
+int bpf_func__SyS_epoll_pwait(void *ctx)
 {
     int ind =0;
     int *flag = bpf_map_lookup_elem(&flip_table, &ind);
tools/perf/tests/bpf.c +4 -4
index 34c22cd..a8f9095 100644
@@ -19,13 +19,13 @@
 
 #ifdef HAVE_LIBBPF_SUPPORT
 
-static int epoll_wait_loop(void)
+static int epoll_pwait_loop(void)
 {
     int i;
 
     /* Should fail NR_ITERS times */
     for (i = 0; i < NR_ITERS; i++)
-        epoll_wait(-(i + 1), NULL, 0, 0);
+        epoll_pwait(-(i + 1), NULL, 0, 0, NULL);
     return 0;
 }
 
@@ -68,7 +68,7 @@ static struct {
         "[basic_bpf_test]",
         "fix 'perf test LLVM' first",
         "load bpf object failed",
-        &epoll_wait_loop,
+        &epoll_pwait_loop,
         (NR_ITERS + 1) / 2,
         false,
     },
@@ -78,7 +78,7 @@ static struct {
         "[bpf_pinning]",
         "fix kbuild first",
         "check your vmlinux setting?",
-        &epoll_wait_loop,
+        &epoll_pwait_loop,
         (NR_ITERS + 1) / 2,
         true,
     },
author maintainer
#8 lore raw

Em Thu, Jan 04, 2018 at 09:37:35AM +0800, Wangnan (F) escreveu:

Quoted 35 lines · Arnaldo Carvalho de Melo
Quoted 32 lines

Em Wed, Jan 03, 2018 at 03:33:07PM -0300, Arnaldo Carvalho de Melo escreveu:

Quoted 31 lines

Em Wed, Jan 03, 2018 at 03:27:01PM -0300, Arnaldo Carvalho de Melo escreveu:

Quoted 19 lines
Quoted 1 line

Continuing investigation...

After applying the fallback patch to allow new tools to work with older kernels:

[root@felicio ~]# perf test bpf

39: BPF filter                                            :
39.1: Basic BPF filtering                                 : Ok
39.2: BPF pinning                                         : Ok
39.3: BPF prologue generation                             : Ok
39.4: BPF relocation checker                              : Ok

[root@felicio ~]# uname -a
Linux felicio.ghostprotocols.net 4.13.0-rc7+ #1 SMP Mon Sep 11 13:56:18 -03 2017 x86_64 x86_64 x86_64 GNU/Linux [root@felicio ~]# rpm -q glibc
glibc-2.17-157.el7_3.2.x86_64
[root@felicio ~]#

After applying the patch below I get to, which is what I am trying to fix now:

Quoted 7 lines

[root@jouet ~]# perf test bpf

39: BPF filter                                            :
39.1: Basic BPF filtering                                 : Ok
39.2: BPF pinning                                         : Ok
39.3: BPF prologue generation                             : FAILED!
39.4: BPF relocation checker                              : Skip

[root@jouet ~]#

Update the patch to the one at the end of this message to make it work with older glibcs, so that we ask for epoll_pwait() and hook into that as well().

Quoted 8 lines
Quoted 4 lines
Quoted 1 line

Now checking why 39.3 fails...

Couldn't reproduce after fixing up some kernel build problems, the patch below is all I need to have this working with both Fedora 27 and RHEL7, please take a look and see if it continues to work on your systems,

It works for me. Thank you.

Since we test epoll_pwait, we'd better correct function names:

Right, that wasn't strictly needed, so I tried to restrict my changes to focus on the fix.

Now that we agree on it, I'm doing what you suggest, to make it less confusing.

I'm adding your Tested-by, thanks!

- Arnaldo

Quoted 57 lines
tools/perf/tests/bpf-script-example.c +2 -2
index 268e5f8..e4123c1 100644
@@ -31,8 +31,8 @@ struct bpf_map_def SEC("maps") flip_table = {
     .max_entries = 1,
 };
 
-SEC("func=SyS_epoll_wait")
-int bpf_func__SyS_epoll_wait(void *ctx)
+SEC("func=SyS_epoll_pwait")
+int bpf_func__SyS_epoll_pwait(void *ctx)
 {
     int ind =0;
     int *flag = bpf_map_lookup_elem(&flip_table, &ind);
tools/perf/tests/bpf.c +4 -4
index 34c22cd..a8f9095 100644
@@ -19,13 +19,13 @@
 
 #ifdef HAVE_LIBBPF_SUPPORT
 
-static int epoll_wait_loop(void)
+static int epoll_pwait_loop(void)
 {
     int i;
 
     /* Should fail NR_ITERS times */
     for (i = 0; i < NR_ITERS; i++)
-        epoll_wait(-(i + 1), NULL, 0, 0);
+        epoll_pwait(-(i + 1), NULL, 0, 0, NULL);
     return 0;
 }
 
@@ -68,7 +68,7 @@ static struct {
         "[basic_bpf_test]",
         "fix 'perf test LLVM' first",
         "load bpf object failed",
-        &epoll_wait_loop,
+        &epoll_pwait_loop,
         (NR_ITERS + 1) / 2,
         false,
     },
@@ -78,7 +78,7 @@ static struct {
         "[bpf_pinning]",
         "fix kbuild first",
         "check your vmlinux setting?",
-        &epoll_wait_loop,
+        &epoll_pwait_loop,
         (NR_ITERS + 1) / 2,
         true,
     },