bpf: Mark bpf_spin_{lock,unlock}() helpers with notrace correctly
Subsystems (4)
Yonghong Song
reviewerPATCH 1/2 bpf: Mark bpf_spin_{lock,unlock}() helpers with notrace correctly
Currently tracing is supposed not to allow for bpf_spin_{lock,unlock}() helper calls. This is to prevent deadlock for the following cases:
- there is a prog (prog-A) calling bpf_spin_{lock,unlock}().
- there is a tracing program (prog-B), e.g., fentry, attached to bpf_spin_lock() and/or bpf_spin_unlock().
- prog-B calls bpf_spin_{lock,unlock}().
For such a case, when prog-A calls bpf_spin_{lock,unlock}(), a deadlock will happen.
The related source codes are below in kernel/bpf/helpers.c:
notrace BPF_CALL_1(bpf_spin_lock, struct bpf_spin_lock *, lock) notrace BPF_CALL_1(bpf_spin_unlock, struct bpf_spin_lock *, lock)
notrace is supposed to prevent fentry prog from attaching to bpf_spin_{lock,unlock}().
But actually this is not the case and fentry prog can successfully attached to bpf_spin_lock(). Siddharth Chintamaneni reported the issue in [1]. The following is the macro definition for above BPF_CALL_1:
#define BPF_CALL_x(x, name, ...) \
static __always_inline \
u64 ____##name(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__)); \
typedef u64 (*btf_##name)(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__)); \
u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__)); \
u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__)) \
{ \
return ((btf_##name)____##name)(__BPF_MAP(x,__BPF_CAST,__BPF_N,__VA_ARGS__));\
} \
static __always_inline \
u64 ____##name(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__))#define BPF_CALL_1(name, ...) BPF_CALL_x(1, name, __VA_ARGS__)
The notrace attribute is actually applied to the static always_inline function ____bpf_spin_{lock,unlock}(). The actual callback function bpf_spin_{lock,unlock}() is not marked with notrace, hence allowing fentry prog to attach to two helpers, and this
may cause the above mentioned deadlock. Siddharth Chintamaneni actually has a reproducer in [2].
To fix the issue, a new macro NOTRACE_BPF_CALL_1 is introduced which will add notrace attribute to the original function instead of the hidden always_inline function and this fixed the problem.
[1] https://lore.kernel.org/bpf/CAE5sdEigPnoGrzN8WU7Tx-h-iFuMZgW06qp0KHWtpvoXxf1OAQ@mail.gmail.com/
[2] https://lore.kernel.org/bpf/CAE5sdEg6yUc_Jz50AnUXEEUh6O73yQ1Z6NV2srJnef0ZrQkZew@mail.gmail.com/
--- include/linux/filter.h | 21 ++++++++++++--------- kernel/bpf/helpers.c | 4 ++-- 2 files changed, 14 insertions(+), 11 deletions(-)
include/linux/filter.h +12 -9
@@ -547,24 +547,27 @@ static inline bool insn_is_zext(const struct bpf_insn *insn)__BPF_MAP (n, __BPF_DECL_ARGS, __BPF_N, u64, __ur_1, u64, __ur_2, \ u64, __ur_3, u64, __ur_4, u64, __ur_5) -#define BPF_CALL_x (x, name, ...) \ +#define BPF_CALL_x (x, attr, name, ...) \ static __always_inline \ u64 ____##name(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__)); \ typedef u64 (*btf_##name)(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__)); \ - u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__)); \ - u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__)) \ + attr u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__)); \ + attr u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__)) \ { \ return ((btf_##name)____##name)(__BPF_MAP(x,__BPF_CAST,__BPF_N,__VA_ARGS__));\ } \ static __always_inline \ u64 ____##name(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__)) -#define BPF_CALL_0 (name, ...) BPF_CALL_x(0, name, __VA_ARGS__) -#define BPF_CALL_1 (name, ...) BPF_CALL_x(1, name, __VA_ARGS__) -#define BPF_CALL_2 (name, ...) BPF_CALL_x(2, name, __VA_ARGS__) -#define BPF_CALL_3 (name, ...) BPF_CALL_x(3, name, __VA_ARGS__) -#define BPF_CALL_4 (name, ...) BPF_CALL_x(4, name, __VA_ARGS__) -#define BPF_CALL_5 (name, ...) BPF_CALL_x(5, name, __VA_ARGS__) +#define __NOATTR +#define BPF_CALL_0 (name, ...) BPF_CALL_x(0, __NOATTR, name, __VA_ARGS__) +#define BPF_CALL_1 (name, ...) BPF_CALL_x(1, __NOATTR, name, __VA_ARGS__) +#define BPF_CALL_2 (name, ...) BPF_CALL_x(2, __NOATTR, name, __VA_ARGS__) +#define BPF_CALL_3 (name, ...) BPF_CALL_x(3, __NOATTR, name, __VA_ARGS__) +#define BPF_CALL_4 (name, ...) BPF_CALL_x(4, __NOATTR, name, __VA_ARGS__) +#define BPF_CALL_5 (name, ...) BPF_CALL_x(5, __NOATTR, name, __VA_ARGS__) + +#define NOTRACE_BPF_CALL_1 (name, ...) BPF_CALL_x(1, notrace, name, __VA_ARGS__)#define bpf_ctx_range (TYPE ,MEMBER ) \ offsetof(TYPE, MEMBER) ... offsetofend(TYPE, MEMBER) - 1
kernel/bpf/helpers.c +2 -2
@@ -334,7 +334,7 @@ static inline void __bpf_spin_lock_irqsave(struct bpf_spin_lock *lock)__this_cpu_write (irqsave_flags, flags); } -notrace BPF_CALL_1 (bpf_spin_lock ,struct bpf_spin_lock *,lock ) +NOTRACE_BPF_CALL_1(bpf_spin_lock, struct bpf_spin_lock *, lock) {__bpf_spin_lock_irqsave (lock);return 0 ;
@@ -357,7 +357,7 @@ static inline void __bpf_spin_unlock_irqrestore(struct bpf_spin_lock *lock)local_irq_restore (flags); } -notrace BPF_CALL_1 (bpf_spin_unlock ,struct bpf_spin_lock *,lock ) +NOTRACE_BPF_CALL_1(bpf_spin_unlock, struct bpf_spin_lock *, lock) {__bpf_spin_unlock_irqrestore (lock);return 0 ;
Signature · 1 line
2.34.1
Yonghong Song
reviewerPATCH 2/2 selftests/bpf: Ensure fentry prog cannot attach to bpf_spin_{lock,unlcok}()
Add two tests to ensure fentry programs cannot attach to bpf_spin_{lock,unlock}() helpers. The tracing_failure.c files can be used in the future for other tracing failure cases.
--- .../bpf/prog_tests/tracing_failure.c | 37 +++++++++++++++++++ .../selftests/bpf/progs/tracing_failure.c | 20 ++++++++++ 2 files changed, 57 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/tracing_failure.c create mode 100644 tools/testing/selftests/bpf/progs/tracing_failure.c
tools/testing/selftests/bpf/prog_tests/tracing_failure.c new file +37 -0
@@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */ +#include <test_progs.h> +#include "tracing_failure.skel.h" + +static void test_bpf_spin_lock (bool is_spin_lock) +{ +struct tracing_failure *skel; +int err; + + skel =tracing_failure__open (); +if (!ASSERT_OK_PTR (skel,"tracing_failure__open" )) +return ; + +if (is_spin_lock) +bpf_program__set_autoload (skel->progs.test_spin_lock, true); +else +bpf_program__set_autoload (skel->progs.test_spin_unlock, true); + + err =tracing_failure__load (skel); +if (!ASSERT_OK (err,"tracing_failure__load" )) + goto out; + + err =tracing_failure__attach (skel); +ASSERT_ERR (err,"tracing_failure__attach" ); + +out: +tracing_failure__destroy (skel); +} + +void test_tracing_failure (void ) +{ +if (test__start_subtest ("bpf_spin_lock" )) +test_bpf_spin_lock (true); +if (test__start_subtest ("bpf_spin_unlock" )) +test_bpf_spin_lock (false); +}
tools/testing/selftests/bpf/progs/tracing_failure.c new file +20 -0
@@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */ + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +char _license[] SEC("license") = "GPL"; + +SEC("?fentry/bpf_spin_lock") +int BPF_PROG (test_spin_lock ,struct bpf_spin_lock *lock) +{ +return 0 ; +} + +SEC("?fentry/bpf_spin_unlock") +int BPF_PROG (test_spin_unlock ,struct bpf_spin_lock *lock) +{ +return 0 ; +}
Signature · 1 line
2.34.1
Quoted 44 lines · Yonghong Song
Currently tracing is supposed not to allow for bpf_spin_{lock,unlock}() helper calls. This is to prevent deadlock for the following cases:
- there is a prog (prog-A) calling bpf_spin_{lock,unlock}().
- there is a tracing program (prog-B), e.g., fentry, attached tobpf_spin_lock()and/orbpf_spin_unlock().
- prog-B calls bpf_spin_{lock,unlock}().
For such a case, when prog-A calls bpf_spin_{lock,unlock}(), a deadlock will happen.The related source codes are below in
kernel/bpf/helpers.c:
notraceBPF_CALL_1(bpf_spin_lock, struct bpf_spin_lock *, lock)notraceBPF_CALL_1(bpf_spin_unlock, struct bpf_spin_lock *, lock)
notrace is supposed to prevent fentry prog from attaching to bpf_spin_{lock,unlock}().But actually this is not the case and fentry prog can successfully attached to
bpf_spin_lock(). Siddharth Chintamaneni reported the issue in [1]. The following is the macro definition for aboveBPF_CALL_1:#define BPF_CALL_x(x, name, ...) \ static __always_inline \ u64 ____##name(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__)); \ typedef u64 (*btf_##name)(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__)); \ u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__)); \ u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__)) \ { \ return ((btf_##name)____##name)(__BPF_MAP(x,__BPF_CAST,__BPF_N,__VA_ARGS__));\ } \ static __always_inline \ u64 ____##name(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__))#define BPF_CALL_1(name, ...) BPF_CALL_x(1, name, __VA_ARGS__)The notrace attribute is actually applied to the static
always_inlinefunction ____bpf_spin_{lock,unlock}(). The actual callback function bpf_spin_{lock,unlock}() is not marked with notrace, hence allowing fentry prog to attach to two helpers, and this
may cause the above mentioned deadlock. Siddharth Chintamaneni actually has a reproducer in [2].To fix the issue, a new macro
NOTRACE_BPF_CALL_1is introduced which will add notrace attribute to the original function instead of the hiddenalways_inlinefunction and this fixed the problem.
Quoted 6 lines
[1] https://lore.kernel.org/bpf/CAE5sdEigPnoGrzN8WU7Tx-h-iFuMZgW06qp0KHWtpvoXxf1OAQ@mail.gmail.com/
[2] https://lore.kernel.org/bpf/CAE5sdEg6yUc_Jz50AnUXEEUh6O73yQ1Z6NV2srJnef0ZrQkZew@mail.gmail.com/Fixes d83525ca62cf ("bpf: introduce bpf_spin_lock")Cc Siddharth Chintamaneni <[email protected]>Signed-off-by Yonghong Song <[email protected]>
jirka
Quoted 72 lines
--- include/linux/filter.h | 21 ++++++++++++--------- kernel/bpf/helpers.c | 4 ++-- 2 files changed, 14 insertions(+), 11 deletions(-)include/linux/filter.h +12
-9@@ -547,24 +547,27 @@ static inline bool insn_is_zext(const struct bpf_insn *insn)__BPF_MAP (n, __BPF_DECL_ARGS, __BPF_N, u64, __ur_1, u64, __ur_2, \ u64, __ur_3, u64, __ur_4, u64, __ur_5) -#define BPF_CALL_x (x, name, ...) \ +#define BPF_CALL_x (x, attr, name, ...) \ static __always_inline \ u64 ____##name(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__)); \ typedef u64 (*btf_##name)(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__)); \ - u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__)); \ - u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__)) \ + attr u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__)); \ + attr u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__)) \ { \ return ((btf_##name)____##name)(__BPF_MAP(x,__BPF_CAST,__BPF_N,__VA_ARGS__));\ } \ static __always_inline \ u64 ____##name(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__)) -#define BPF_CALL_0 (name, ...) BPF_CALL_x(0, name, __VA_ARGS__) -#define BPF_CALL_1 (name, ...) BPF_CALL_x(1, name, __VA_ARGS__) -#define BPF_CALL_2 (name, ...) BPF_CALL_x(2, name, __VA_ARGS__) -#define BPF_CALL_3 (name, ...) BPF_CALL_x(3, name, __VA_ARGS__) -#define BPF_CALL_4 (name, ...) BPF_CALL_x(4, name, __VA_ARGS__) -#define BPF_CALL_5 (name, ...) BPF_CALL_x(5, name, __VA_ARGS__) +#define __NOATTR +#define BPF_CALL_0 (name, ...) BPF_CALL_x(0, __NOATTR, name, __VA_ARGS__) +#define BPF_CALL_1 (name, ...) BPF_CALL_x(1, __NOATTR, name, __VA_ARGS__) +#define BPF_CALL_2 (name, ...) BPF_CALL_x(2, __NOATTR, name, __VA_ARGS__) +#define BPF_CALL_3 (name, ...) BPF_CALL_x(3, __NOATTR, name, __VA_ARGS__) +#define BPF_CALL_4 (name, ...) BPF_CALL_x(4, __NOATTR, name, __VA_ARGS__) +#define BPF_CALL_5 (name, ...) BPF_CALL_x(5, __NOATTR, name, __VA_ARGS__) + +#define NOTRACE_BPF_CALL_1 (name, ...) BPF_CALL_x(1, notrace, name, __VA_ARGS__)#define bpf_ctx_range (TYPE ,MEMBER ) \ offsetof(TYPE, MEMBER) ... offsetofend(TYPE, MEMBER) - 1kernel/bpf/helpers.c +2
-2@@ -334,7 +334,7 @@ static inline void __bpf_spin_lock_irqsave(struct bpf_spin_lock *lock)__this_cpu_write (irqsave_flags, flags); } -notrace BPF_CALL_1 (bpf_spin_lock ,struct bpf_spin_lock *,lock ) +NOTRACE_BPF_CALL_1(bpf_spin_lock, struct bpf_spin_lock *, lock) {__bpf_spin_lock_irqsave (lock);return 0 ;@@ -357,7 +357,7 @@ static inline void __bpf_spin_unlock_irqrestore(struct bpf_spin_lock *lock)local_irq_restore (flags); } -notrace BPF_CALL_1 (bpf_spin_unlock ,struct bpf_spin_lock *,lock ) +NOTRACE_BPF_CALL_1(bpf_spin_unlock, struct bpf_spin_lock *, lock) {__bpf_spin_unlock_irqrestore (lock);return 0 ;Signature · 1 line
2.34.1
This series was applied to bpf/bpf-next.git (master) by Andrii Nakryiko <[email protected]>:
Hello:
This series was applied to bpf/bpf-next.git (master) by Andrii Nakryiko <[email protected]>:
Quoted 10 lines · On Tue, 6 Feb 2024 23:01:02 -0800 you wrote:
Currently tracing is supposed not to allow for bpf_spin_{lock,unlock}() helper calls. This is to prevent deadlock for the following cases:
- there is a prog (prog-A) calling bpf_spin_{lock,unlock}().
- there is a tracing program (prog-B), e.g., fentry, attached tobpf_spin_lock()and/orbpf_spin_unlock().
- prog-B calls bpf_spin_{lock,unlock}().
For such a case, when prog-A calls bpf_spin_{lock,unlock}(), a deadlock will happen.[...]
Here is the summary with links:
- [bpf-next,1/2] bpf: Mark bpf_spin_{lock,unlock}() helpers with notrace correctly
https://git.kernel.org/bpf/bpf-next/c/178c54666f9c - [bpf-next,2/2] selftests/bpf: Ensure fentry prog cannot attach to bpf_spin_{lock,unlcok}() https://git.kernel.org/bpf/bpf-next/c/fc1c9e40da37
You are awesome, thank you!
Signature · 2 lines
Deet-doot-dot, I am a bot. https://korg.docs.kernel.org/patchwork/pwbot.html