bpf: check that BPF programs run with preemption disabled
Subsystems (5)
Alexei Starovoitov
maintainerPATCH 1/1 bpf: check that BPF programs run with preemption disabled
From: Peter Zijlstra <[email protected]>
Introduce cant_sleep() macro for annotation of functions that cannot sleep.
Use it in BPF_PROG_RUN to catch execution of BPF programs in preemptable context.
--- include/linux/filter.h | 2 +- include/linux/kernel.h | 14 ++++++++++++-- kernel/sched/core.c | 28 ++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 3 deletions(-)
include/linux/filter.h +1 -1
@@ -533,7 +533,7 @@ struct sk_filter {struct bpf_prog *prog; }; -#define BPF_PROG_RUN (filter, ctx) (*(filter)->bpf_func)(ctx, (filter)->insnsi) +#define BPF_PROG_RUN (filter, ctx) ({ cant_sleep(); (*(filter)->bpf_func)(ctx, (filter)->insnsi); })#define BPF_SKB_CB_LEN QDISC_CB_PRIV_LEN
include/linux/kernel.h +12 -2
@@ -245,8 +245,10 @@ extern int _cond_resched(void);#endif #ifdef CONFIG_DEBUG_ATOMIC_SLEEP -void ___might_sleep (const char *file,int line,int preempt_offset); -void __might_sleep (const char *file,int line,int preempt_offset); +extern void ___might_sleep (const char *file,int line,int preempt_offset); +extern void __might_sleep (const char *file,int line,int preempt_offset); +extern void __cant_sleep (const char *file,int line,int preempt_offset); +/** * might_sleep - annotation for functions that can sleep *
@@ -259,6 +261,13 @@ extern int _cond_resched(void);*/ # define might_sleep () \ do { __might_sleep(__FILE__, __LINE__, 0); might_resched(); } while (0) +/** +* cant_sleep - annotation for functions that cannot sleep +* +* this macro will print a stack trace if it is executed with preemption enabled +*/ +# define cant_sleep () \ + do { __cant_sleep(__FILE__, __LINE__, 0); } while (0)# define sched_annotate_sleep () (current->task_state_change = 0)#else static inline void ___might_sleep(const char *file, int line,
@@ -266,6 +275,7 @@ extern int _cond_resched(void);static inline void __might_sleep (const char *file,int line,int preempt_offset) { }# define might_sleep () do { might_resched(); } while (0) +# define cant_sleep () do { } while (0)# define sched_annotate_sleep () do { } while (0)#endif
kernel/sched/core.c +28 -0
@@ -6149,6 +6149,34 @@ void ___might_sleep(const char *file, int line, int preempt_offset)add_taint (TAINT_WARN ,LOCKDEP_STILL_OK ); }EXPORT_SYMBOL (___might_sleep); + +void __cant_sleep (const char *file,int line,int preempt_offset) +{ +static unsigned long prev_jiffy; + +if (irqs_disabled ()) +return ; + +if (!IS_ENABLED (CONFIG_PREEMPT_COUNT )) +return ; + +if (preempt_count () > preempt_offset) +return ; + +if (time_before (jiffies, prev_jiffy +HZ ) && prev_jiffy) +return ; + prev_jiffy = jiffies; + +printk (KERN_ERR "BUG: assuming atomic context at %s:%d\n" , file, line); +printk (KERN_ERR "in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n" , +in_atomic (),irqs_disabled (), + current->pid, current->comm); + +debug_show_held_locks (current); +dump_stack (); +add_taint (TAINT_WARN ,LOCKDEP_STILL_OK ); +} +EXPORT_SYMBOL_GPL (__cant_sleep);#endif #ifdef CONFIG_MAGIC_SYSRQ
Signature · 1 line
2.20.0
Quoted 14 lines · Alexei Starovoitov
From: Peter Zijlstra <[email protected]>
Introduce
cant_sleep()macro for annotation of functions that cannot sleep.Use it in
BPF_PROG_RUNto catch execution of BPF programs in preemptable context.Suggested-by Jann Horn <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>Signed-off-by Alexei Starovoitov <[email protected]>
Quoted 100 lines
---
include/linux/filter.h | 2 +- include/linux/kernel.h | 14 ++++++++++++-- kernel/sched/core.c | 28 ++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 3 deletions(-)include/linux/filter.h +1
-1@@ -533,7 +533,7 @@ struct sk_filter {struct bpf_prog *prog; }; -#define BPF_PROG_RUN (filter, ctx) (*(filter)->bpf_func)(ctx, (filter)->insnsi) +#define BPF_PROG_RUN (filter, ctx) ({ cant_sleep(); (*(filter)->bpf_func)(ctx, (filter)->insnsi); })#define BPF_SKB_CB_LEN QDISC_CB_PRIV_LENinclude/linux/kernel.h +12
-2@@ -245,8 +245,10 @@ extern int _cond_resched(void);#endif #ifdef CONFIG_DEBUG_ATOMIC_SLEEP -void ___might_sleep (const char *file,int line,int preempt_offset); -void __might_sleep (const char *file,int line,int preempt_offset); +extern void ___might_sleep (const char *file,int line,int preempt_offset); +extern void __might_sleep (const char *file,int line,int preempt_offset); +extern void __cant_sleep (const char *file,int line,int preempt_offset); +/** * might_sleep - annotation for functions that can sleep * @@ -259,6 +261,13 @@ extern int _cond_resched(void);*/ # define might_sleep () \ do { __might_sleep(__FILE__, __LINE__, 0); might_resched(); } while (0) +/** +* cant_sleep - annotation for functions that cannot sleep +* +* this macro will print a stack trace if it is executed with preemption enabled +*/ +# define cant_sleep () \ + do { __cant_sleep(__FILE__, __LINE__, 0); } while (0)# define sched_annotate_sleep () (current->task_state_change = 0)#else static inline void ___might_sleep(const char *file, int line,@@ -266,6 +275,7 @@ extern int _cond_resched(void);static inline void __might_sleep (const char *file,int line,int preempt_offset) { }# define might_sleep () do { might_resched(); } while (0) +# define cant_sleep () do { } while (0)# define sched_annotate_sleep () do { } while (0)#endif kernel/sched/core.c +28
-0@@ -6149,6 +6149,34 @@ void ___might_sleep(const char *file, int line, int preempt_offset)add_taint (TAINT_WARN ,LOCKDEP_STILL_OK ); }EXPORT_SYMBOL (___might_sleep); + +void __cant_sleep (const char *file,int line,int preempt_offset) +{ +static unsigned long prev_jiffy; + +if (irqs_disabled ()) +return ; + +if (!IS_ENABLED (CONFIG_PREEMPT_COUNT )) +return ; + +if (preempt_count () > preempt_offset) +return ; + +if (time_before (jiffies, prev_jiffy +HZ ) && prev_jiffy) +return ; + prev_jiffy = jiffies; + +printk (KERN_ERR "BUG: assuming atomic context at %s:%d\n" , file, line); +printk (KERN_ERR "in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n" , +in_atomic (),irqs_disabled (), + current->pid, current->comm); + +debug_show_held_locks (current); +dump_stack (); +add_taint (TAINT_WARN ,LOCKDEP_STILL_OK ); +} +EXPORT_SYMBOL_GPL (__cant_sleep);#endif #ifdef CONFIG_MAGIC_SYSRQ Signature · 1 line
2.20.0
Daniel Borkmann
maintainerQuoted 10 lines · Alexei Starovoitov
From: Peter Zijlstra <[email protected]>
Introduce
cant_sleep()macro for annotation of functions that cannot sleep.Use it in
BPF_PROG_RUNto catch execution of BPF programs in preemptable context.Suggested-by Jann Horn <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>Signed-off-by Alexei Starovoitov <[email protected]>
(Just fyi, will let this sit in patchwork till the preemption fixes have been merged back from bpf into bpf-next as otherwise people will yell that they hit this frequently.)
Daniel Borkmann
maintainerQuoted 8 lines · Daniel Borkmann
Quoted 7 lines · Alexei Starovoitov
From: Peter Zijlstra <[email protected]>
Introduce
cant_sleep()macro for annotation of functions that cannot sleep.Use it in
BPF_PROG_RUNto catch execution of BPF programs in preemptable context.
Quoted 7 lines
Quoted 3 lines
Suggested-by Jann Horn <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>Signed-off-by Alexei Starovoitov <[email protected]>(Just fyi, will let this sit in patchwork till the preemption fixes have been merged back from bpf into bpf-next as otherwise people will yell that they hit this frequently.)
Pushed out now, thanks!