bpf: Fix nested bpf_bprintf_prepare with more per-cpu buffers
PATCH 1/1 bpf: Fix nested bpf_bprintf_prepare with more per-cpu buffers
The bpf_seq_printf, bpf_trace_printk and bpf_snprintf helpers share one per-cpu buffer that they use to store temporary data (arguments to bprintf). They "get" that buffer with try_get_fmt_tmp_buf and "put" it by the end of their scope with bpf_bprintf_cleanup.
If one of these helpers gets called within the scope of one of these helpers, for example: a first bpf program gets called, uses bpf_trace_printk which calls raw_spin_lock_irqsave which is traced by another bpf program that calls bpf_snprintf, then the second "get" fails. Essentially, these helpers are not re-entrant. They would return -EBUSY and print a warning message once.
This patch triples the number of bprintf buffers to allow three levels of nesting. This is very similar to what was done for tracepoints in "9594dc3c7e7 bpf: fix nested bpf tracepoints with per-cpu data"
--- kernel/bpf/helpers.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-)
kernel/bpf/helpers.c +14 -13
@@ -696,34 +696,35 @@ static int bpf_trace_copy_string(char *buf, void *unsafe_ptr, char fmt_ptype,*/ #define MAX_PRINTF_BUF_LEN 512 -struct bpf_printf_buf { -char tmp_buf[MAX_PRINTF_BUF_LEN ]; +/* Support executing three nested bprintf helper calls on a given CPU */ +struct bpf_bprintf_buffers { +char tmp_bufs[3 ][MAX_PRINTF_BUF_LEN ]; }; -static DEFINE_PER_CPU(struct bpf_printf_buf, bpf_printf_buf); -static DEFINE_PER_CPU(int, bpf_printf_buf_used); +static DEFINE_PER_CPU(struct bpf_bprintf_bufs, bpf_bprintf_bufs); +static DEFINE_PER_CPU(int, bpf_bprintf_nest_level);static int try_get_fmt_tmp_buf (char **tmp_buf) { -struct bpf_printf_buf *bufs; -int used; +struct bpf_bprintf_buffers *bufs; +int nest_level;preempt_disable (); - used =this_cpu_inc_return (bpf_printf_buf_used); -if (WARN_ON_ONCE (used >1 )) { -this_cpu_dec (bpf_printf_buf_used); + nest_level =this_cpu_inc_return (bpf_bprintf_nest_level); +if (WARN_ON_ONCE (nest_level >ARRAY_SIZE (bufs->tmp_bufs))) { +this_cpu_dec (bpf_bprintf_nest_level);preempt_enable ();return -EBUSY ; } - bufs =this_cpu_ptr (&bpf_printf_buf); - *tmp_buf = bufs->tmp_buf; + bufs =this_cpu_ptr (&bpf_bprintf_buf); + *tmp_buf = bufs->tmp_bufs[nest_level -1 ];return 0 ; }void bpf_bprintf_cleanup (void ) { -if (this_cpu_read (bpf_printf_buf_used)) { -this_cpu_dec (bpf_printf_buf_used); +if (this_cpu_read (bpf_bprintf_nest_level)) { +this_cpu_dec (bpf_bprintf_nest_level);preempt_enable (); } }
Signature · 1 line
2.31.1.607.g51e8a6a459-goog
kernel test robot
build botI love your patch! Yet something to improve:
Hi Florent,
I love your patch! Yet something to improve:
[auto build test ERROR on bpf/master]
url: https://github.com/0day-ci/linux/commits/Florent-Revest/bpf-Fix-nested-bpf_bprintf_prepare-with-more-per-cpu-buffers/20210511-053835 base: https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf.git master config: powerpc-randconfig-r021-20210510 (attached as .config) compiler: clang version 13.0.0 (https://github.com/llvm/llvm-project a0fed635fe1701470062495a6ffee1c608f3f1bc)
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# install powerpc cross compiling tool for clang build
# apt-get install binutils-powerpc-linux-gnu
# https://github.com/0day-ci/linux/commit/1f5d45d9243d8ca8ece81e778579fc46a1946887
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Florent-Revest/bpf-Fix-nested-bpf_bprintf_prepare-with-more-per-cpu-buffers/20210511-053835 git checkout 1f5d45d9243d8ca8ece81e778579fc46a1946887
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross W=1 ARCH=powerpc
If you fix the issue, kindly add following tag as appropriate
All errors (new ones prefixed by >>):
>> kernel/bpf/helpers.c:718:23: error: use of undeclared identifier 'bpf_bprintf_buf'; did you mean 'bpf_bprintf_bufs'?
bufs = this_cpu_ptr(&bpf_bprintf_buf);
^~~~~~~~~~~~~~~
bpf_bprintf_bufs
include/linux/percpu-defs.h:265:39: note: expanded from macro 'this_cpu_ptr'
#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
^
include/linux/percpu-defs.h:264:38: note: expanded from macro 'raw_cpu_ptr'
#define raw_cpu_ptr(ptr) per_cpu_ptr(ptr, 0)
^
include/linux/percpu-defs.h:263:65: note: expanded from macro 'per_cpu_ptr'
#define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); VERIFY_PERCPU_PTR(ptr); })
^
include/linux/percpu-defs.h:259:20: note: expanded from macro 'VERIFY_PERCPU_PTR'
__verify_pcpu_ptr(__p); \
^
include/linux/percpu-defs.h:219:47: note: expanded from macro '__verify_pcpu_ptr'
const void __percpu *__vpp_verify = (typeof((ptr) + 0))NULL; \
^
kernel/bpf/helpers.c:703:48: note: 'bpf_bprintf_bufs' declared here
static DEFINE_PER_CPU(struct bpf_bprintf_bufs, bpf_bprintf_bufs);
^
>> kernel/bpf/helpers.c:718:9: error: arithmetic on a pointer to an incomplete type 'typeof(struct bpf_bprintf_bufs)' (aka 'struct bpf_bprintf_bufs')
bufs = this_cpu_ptr(&bpf_bprintf_buf);
^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
include/linux/percpu-defs.h:265:27: note: expanded from macro 'this_cpu_ptr'
#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
^~~~~~~~~~~~~~~~
include/linux/percpu-defs.h:264:26: note: expanded from macro 'raw_cpu_ptr'
#define raw_cpu_ptr(ptr) per_cpu_ptr(ptr, 0)
^~~~~~~~~~~~~~~~~~~
include/linux/percpu-defs.h:263:47: note: expanded from macro 'per_cpu_ptr'
#define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); VERIFY_PERCPU_PTR(ptr); })
^~~~~~~~~~~~~~~~~~~~~~
include/linux/percpu-defs.h:259:2: note: expanded from macro 'VERIFY_PERCPU_PTR'
__verify_pcpu_ptr(__p); \
^~~~~~~~~~~~~~~~~~~~~~
include/linux/percpu-defs.h:219:52: note: expanded from macro '__verify_pcpu_ptr'
const void __percpu *__vpp_verify = (typeof((ptr) + 0))NULL; \
~~~~~ ^
kernel/bpf/helpers.c:703:30: note: forward declaration of 'struct bpf_bprintf_bufs'
static DEFINE_PER_CPU(struct bpf_bprintf_bufs, bpf_bprintf_bufs);
^
>> kernel/bpf/helpers.c:718:23: error: use of undeclared identifier 'bpf_bprintf_buf'; did you mean 'bpf_bprintf_bufs'?
bufs = this_cpu_ptr(&bpf_bprintf_buf);
^~~~~~~~~~~~~~~
bpf_bprintf_bufs
include/linux/percpu-defs.h:265:39: note: expanded from macro 'this_cpu_ptr'
#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
^
include/linux/percpu-defs.h:264:38: note: expanded from macro 'raw_cpu_ptr'
#define raw_cpu_ptr(ptr) per_cpu_ptr(ptr, 0)
^
include/linux/percpu-defs.h:263:65: note: expanded from macro 'per_cpu_ptr'
#define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); VERIFY_PERCPU_PTR(ptr); })
^
include/linux/percpu-defs.h:260:12: note: expanded from macro 'VERIFY_PERCPU_PTR'
(typeof(*(__p)) __kernel __force *)(__p); \
^
kernel/bpf/helpers.c:703:48: note: 'bpf_bprintf_bufs' declared here
static DEFINE_PER_CPU(struct bpf_bprintf_bufs, bpf_bprintf_bufs);
^
>> kernel/bpf/helpers.c:718:23: error: use of undeclared identifier 'bpf_bprintf_buf'; did you mean 'bpf_bprintf_bufs'?
bufs = this_cpu_ptr(&bpf_bprintf_buf);
^~~~~~~~~~~~~~~
bpf_bprintf_bufs
include/linux/percpu-defs.h:265:39: note: expanded from macro 'this_cpu_ptr'
#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
^
include/linux/percpu-defs.h:264:38: note: expanded from macro 'raw_cpu_ptr'
#define raw_cpu_ptr(ptr) per_cpu_ptr(ptr, 0)
^
include/linux/percpu-defs.h:263:65: note: expanded from macro 'per_cpu_ptr'
#define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); VERIFY_PERCPU_PTR(ptr); })
^
include/linux/percpu-defs.h:260:38: note: expanded from macro 'VERIFY_PERCPU_PTR'
(typeof(*(__p)) __kernel __force *)(__p); \
^
kernel/bpf/helpers.c:703:48: note: 'bpf_bprintf_bufs' declared here
static DEFINE_PER_CPU(struct bpf_bprintf_bufs, bpf_bprintf_bufs);
^
>> kernel/bpf/helpers.c:718:7: error: incompatible pointer types assigning to 'struct bpf_bprintf_buffers *' from 'typeof (*(&bpf_bprintf_bufs)) *' (aka 'struct bpf_bprintf_bufs *') [-Werror,-Wincompatible-pointer-types]
bufs = this_cpu_ptr(&bpf_bprintf_buf);
^ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
>> kernel/bpf/helpers.c:703:48: error: tentative definition has type 'typeof(struct bpf_bprintf_bufs)' (aka 'struct bpf_bprintf_bufs') that is never completed
static DEFINE_PER_CPU(struct bpf_bprintf_bufs, bpf_bprintf_bufs);
^
kernel/bpf/helpers.c:703:30: note: forward declaration of 'struct bpf_bprintf_bufs'
static DEFINE_PER_CPU(struct bpf_bprintf_bufs, bpf_bprintf_bufs);
^6 errors generated.
vim +718 kernel/bpf/helpers.c
698
699 /* Support executing three nested bprintf helper calls on a given CPU */
700 struct bpf_bprintf_buffers {
701 char tmp_bufs[3][MAX_PRINTF_BUF_LEN];
702 };
> 703 static DEFINE_PER_CPU(struct bpf_bprintf_bufs, bpf_bprintf_bufs);
704 static DEFINE_PER_CPU(int, bpf_bprintf_nest_level);
705
706 static int try_get_fmt_tmp_buf(char **tmp_buf)
707 {
708 struct bpf_bprintf_buffers *bufs;
709 int nest_level;
710
711 preempt_disable();
712 nest_level = this_cpu_inc_return(bpf_bprintf_nest_level);
713 if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(bufs->tmp_bufs))) {
714 this_cpu_dec(bpf_bprintf_nest_level);
715 preempt_enable();
716 return -EBUSY;
717 }
> 718 bufs = this_cpu_ptr(&bpf_bprintf_buf); 719 *tmp_buf = bufs->tmp_bufs[nest_level - 1];
720
721 return 0;
722 }
723
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org
kernel test robot
build botI love your patch! Yet something to improve:
Hi Florent,
I love your patch! Yet something to improve:
[auto build test ERROR on bpf/master]
url: https://github.com/0day-ci/linux/commits/Florent-Revest/bpf-Fix-nested-bpf_bprintf_prepare-with-more-per-cpu-buffers/20210511-053835 base: https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf.git master config: s390-randconfig-r013-20210510 (attached as .config) compiler: clang version 13.0.0 (https://github.com/llvm/llvm-project a0fed635fe1701470062495a6ffee1c608f3f1bc)
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# install s390 cross compiling tool for clang build
# apt-get install binutils-s390x-linux-gnu
# https://github.com/0day-ci/linux/commit/1f5d45d9243d8ca8ece81e778579fc46a1946887
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Florent-Revest/bpf-Fix-nested-bpf_bprintf_prepare-with-more-per-cpu-buffers/20210511-053835 git checkout 1f5d45d9243d8ca8ece81e778579fc46a1946887
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross W=1 ARCH=s390
If you fix the issue, kindly add following tag as appropriate
All errors (new ones prefixed by >>):
include/linux/percpu-defs.h:252:27: note: expanded from macro 'this_cpu_ptr'
#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
^~~~~~~~~~~~~~~~
include/linux/percpu-defs.h:241:2: note: expanded from macro 'raw_cpu_ptr'
__verify_pcpu_ptr(ptr); \
^~~~~~~~~~~~~~~~~~~~~~
include/linux/percpu-defs.h:219:52: note: expanded from macro '__verify_pcpu_ptr'
const void __percpu *__vpp_verify = (typeof((ptr) + 0))NULL; \
~~~~~ ^
kernel/bpf/helpers.c:703:30: note: forward declaration of 'struct bpf_bprintf_bufs'
static DEFINE_PER_CPU(struct bpf_bprintf_bufs, bpf_bprintf_bufs);
^
kernel/bpf/helpers.c:718:23: error: use of undeclared identifier 'bpf_bprintf_buf'; did you mean 'bpf_bprintf_bufs'?
bufs = this_cpu_ptr(&bpf_bprintf_buf);
^~~~~~~~~~~~~~~
bpf_bprintf_bufs
include/linux/percpu-defs.h:252:39: note: expanded from macro 'this_cpu_ptr'
#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
^
include/linux/percpu-defs.h:242:19: note: expanded from macro 'raw_cpu_ptr'
arch_raw_cpu_ptr(ptr); \
^
include/asm-generic/percpu.h:44:48: note: expanded from macro 'arch_raw_cpu_ptr'
#define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
^
include/linux/percpu-defs.h:231:23: note: expanded from macro 'SHIFT_PERCPU_PTR'
RELOC_HIDE((typeof(*(__p)) __kernel __force *)(__p), (__offset))
^
include/linux/compiler.h:181:31: note: expanded from macro 'RELOC_HIDE'
__ptr = (unsigned long) (ptr); \
^
kernel/bpf/helpers.c:703:48: note: 'bpf_bprintf_bufs' declared here
static DEFINE_PER_CPU(struct bpf_bprintf_bufs, bpf_bprintf_bufs);
^
kernel/bpf/helpers.c:718:23: error: use of undeclared identifier 'bpf_bprintf_buf'; did you mean 'bpf_bprintf_bufs'?
bufs = this_cpu_ptr(&bpf_bprintf_buf);
^~~~~~~~~~~~~~~
bpf_bprintf_bufs
include/linux/percpu-defs.h:252:39: note: expanded from macro 'this_cpu_ptr'
#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
^
include/linux/percpu-defs.h:242:19: note: expanded from macro 'raw_cpu_ptr'
arch_raw_cpu_ptr(ptr); \
^
include/asm-generic/percpu.h:44:48: note: expanded from macro 'arch_raw_cpu_ptr'
#define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
^
include/linux/percpu-defs.h:231:49: note: expanded from macro 'SHIFT_PERCPU_PTR'
RELOC_HIDE((typeof(*(__p)) __kernel __force *)(__p), (__offset))
^
include/linux/compiler.h:181:31: note: expanded from macro 'RELOC_HIDE'
__ptr = (unsigned long) (ptr); \
^
kernel/bpf/helpers.c:703:48: note: 'bpf_bprintf_bufs' declared here
static DEFINE_PER_CPU(struct bpf_bprintf_bufs, bpf_bprintf_bufs);
^
kernel/bpf/helpers.c:718:23: error: use of undeclared identifier 'bpf_bprintf_buf'; did you mean 'bpf_bprintf_bufs'?
bufs = this_cpu_ptr(&bpf_bprintf_buf);
^~~~~~~~~~~~~~~
bpf_bprintf_bufs
include/linux/percpu-defs.h:252:39: note: expanded from macro 'this_cpu_ptr'
#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
^
include/linux/percpu-defs.h:242:19: note: expanded from macro 'raw_cpu_ptr'
arch_raw_cpu_ptr(ptr); \
^
include/asm-generic/percpu.h:44:48: note: expanded from macro 'arch_raw_cpu_ptr'
#define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
^
include/linux/percpu-defs.h:231:23: note: expanded from macro 'SHIFT_PERCPU_PTR'
RELOC_HIDE((typeof(*(__p)) __kernel __force *)(__p), (__offset))
^
include/linux/compiler.h:182:13: note: expanded from macro 'RELOC_HIDE'
(typeof(ptr)) (__ptr + (off)); })
^
kernel/bpf/helpers.c:703:48: note: 'bpf_bprintf_bufs' declared here
static DEFINE_PER_CPU(struct bpf_bprintf_bufs, bpf_bprintf_bufs);
^
kernel/bpf/helpers.c:718:23: error: use of undeclared identifier 'bpf_bprintf_buf'; did you mean 'bpf_bprintf_bufs'?
bufs = this_cpu_ptr(&bpf_bprintf_buf);
^~~~~~~~~~~~~~~
bpf_bprintf_bufs
include/linux/percpu-defs.h:252:39: note: expanded from macro 'this_cpu_ptr'
#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
^
include/linux/percpu-defs.h:242:19: note: expanded from macro 'raw_cpu_ptr'
arch_raw_cpu_ptr(ptr); \
^
include/asm-generic/percpu.h:44:48: note: expanded from macro 'arch_raw_cpu_ptr'
#define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
^
include/linux/percpu-defs.h:231:49: note: expanded from macro 'SHIFT_PERCPU_PTR'
RELOC_HIDE((typeof(*(__p)) __kernel __force *)(__p), (__offset))
^
include/linux/compiler.h:182:13: note: expanded from macro 'RELOC_HIDE'
(typeof(ptr)) (__ptr + (off)); })
^
kernel/bpf/helpers.c:703:48: note: 'bpf_bprintf_bufs' declared here
static DEFINE_PER_CPU(struct bpf_bprintf_bufs, bpf_bprintf_bufs);
^
>> kernel/bpf/helpers.c:718:7: error: incompatible pointer types assigning to 'struct bpf_bprintf_buffers *' from 'typeof ((typeof (*(&bpf_bprintf_bufs)) *)(&bpf_bprintf_bufs))' (aka 'struct bpf_bprintf_bufs *') [-Werror,-Wincompatible-pointer-types]
bufs = this_cpu_ptr(&bpf_bprintf_buf);
^ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
kernel/bpf/helpers.c:703:48: error: tentative definition has type 'typeof(struct bpf_bprintf_bufs)' (aka 'struct bpf_bprintf_bufs') that is never completed
static DEFINE_PER_CPU(struct bpf_bprintf_bufs, bpf_bprintf_bufs);
^
kernel/bpf/helpers.c:703:30: note: forward declaration of 'struct bpf_bprintf_bufs'
static DEFINE_PER_CPU(struct bpf_bprintf_bufs, bpf_bprintf_bufs);
^12 warnings and 8 errors generated.
vim +718 kernel/bpf/helpers.c
705
706 static int try_get_fmt_tmp_buf(char **tmp_buf)
707 {
708 struct bpf_bprintf_buffers *bufs;
709 int nest_level;
710
711 preempt_disable();
712 nest_level = this_cpu_inc_return(bpf_bprintf_nest_level);
713 if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(bufs->tmp_bufs))) {
714 this_cpu_dec(bpf_bprintf_nest_level);
715 preempt_enable();
716 return -EBUSY;
717 }
> 718 bufs = this_cpu_ptr(&bpf_bprintf_buf);
719 *tmp_buf = bufs->tmp_bufs[nest_level - 1];
720
721 return 0;
722 }
723
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org
Quoted 1 line · kernel test robot
kernel/bpf/helpers.c:718:23: error: use of undeclared identifier 'bpf_bprintf_buf'; did you mean 'bpf_bprintf_bufs'?
Ugh, reminder to self: don't zealously rename variables just before sending a patch out.
I'm sending a v2, sorry for the noise.