bpf: Account for BPF_FETCH in insn_has_def32()
PATCH 1/1 bpf: Account for BPF_FETCH in insn_has_def32()
insn_has_def32() returns false for 32-bit BPF_FETCH insns. This makes adjust_insn_aux_data() incorrectly set zext_dst, as can be seen in [1]. This happens because insn_no_def() does not know about BPF_FETCH variants of BPF_STX.
Fix in two steps.
First, replace insn_no_def() with insn_def_regno(), which returns which the register an insn defines. Normally insn_no_def() calls are followed by insn->dst_reg uses; replace those with insn_def_regno() return value.
Second, adjust the BPF_STX special case in is_reg64() to deal with queries made from opt_subreg_zext_lo32_rnd_hi32(), where the state information is no longer available. Add a comment, since the purpose of this special case is not clear at the first glance.
[1] https://lore.kernel.org/bpf/[email protected]/
--- kernel/bpf/verifier.c | 65 ++++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 32 deletions(-)
kernel/bpf/verifier.c +33 -32
@@ -1703,7 +1703,10 @@ static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn, }if (class ==BPF_STX ) { -if (reg->type !=SCALAR_VALUE ) +/* BPF_STX (including atomic variants) has multiple source +* operands, one of which is a ptr. Check whether the caller is +* asking about it. */ +if (t ==SRC_OP && reg->type !=SCALAR_VALUE )return true;return BPF_SIZE (code) ==BPF_DW ; }
@@ -1735,22 +1738,33 @@ static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn,return true; } -/* Return TRUE if INSN doesn't have explicit value define. */ -static bool insn_no_def (struct bpf_insn *insn) +/* Return the regno defined by the insn, or -1. */ +static int insn_def_regno (const struct bpf_insn *insn) { -u8 class =BPF_CLASS (insn->code); - -return (class ==BPF_JMP || class ==BPF_JMP32 || - class ==BPF_STX || class ==BPF_ST ); +switch (BPF_CLASS (insn->code)) { +case BPF_JMP : +case BPF_JMP32 : +case BPF_ST : +return -1 ; +case BPF_STX : +return (BPF_MODE (insn->code) ==BPF_ATOMIC && + (insn->imm &BPF_FETCH )) ? + (insn->imm ==BPF_CMPXCHG ?BPF_REG_0 : insn->src_reg) : +-1 ; +default : +return insn->dst_reg; + } }/* Return TRUE if INSN has defined any 32-bit value explicitly. */ static bool insn_has_def32 (struct bpf_verifier_env *env,struct bpf_insn *insn) { -if (insn_no_def (insn)) +int dst_reg =insn_def_regno (insn); + +if (dst_reg ==-1 )return false; -return !is_reg64 (env, insn, insn->dst_reg,NULL ,DST_OP ); +return !is_reg64 (env, insn, dst_reg,NULL ,DST_OP ); } static void mark_insn_zext(struct bpf_verifier_env *env,
@@ -11006,9 +11020,10 @@ static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env,for (i =0 ; i < len; i++) {int adj_idx = i + delta;struct bpf_insn insn; -u8 load_reg; +int load_reg; insn = insns[adj_idx]; + load_reg =insn_def_regno (&insn);if (!aux[adj_idx].zext_dst) {u8 code, class;u32 imm_rnd;
@@ -11018,14 +11033,14 @@ static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, code = insn.code; class =BPF_CLASS (code); -if (insn_no_def (&insn)) +if (load_reg ==-1 )continue ;/* NOTE: arg "reg" (the fourth one) is only used for -* BPF_STX which has been ruled out in above -* check, it is safe to pass NULL here. +* BPF_STX + SRC_OP, so it is safe to pass NULL +* here. */ -if (is_reg64 (env, &insn, insn.dst_reg,NULL ,DST_OP )) { +if (is_reg64 (env, &insn, load_reg,NULL ,DST_OP )) {if (class ==BPF_LD &&BPF_MODE (code) ==BPF_IMM ) i++;
@@ -11040,7 +11055,7 @@ static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, imm_rnd =get_random_int (); rnd_hi32_patch[0 ] = insn; rnd_hi32_patch[1 ].imm = imm_rnd; - rnd_hi32_patch[3 ].dst_reg = insn.dst_reg; + rnd_hi32_patch[3 ].dst_reg = load_reg; patch = rnd_hi32_patch; patch_len =4 ; goto apply_patch_buffer;
@@ -11049,23 +11064,9 @@ static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env,if (!bpf_jit_needs_zext ())continue ; -/* zext_dst means that we want to zero-extend whatever register -* the insn defines, which is dst_reg most of the time, with -* the notable exception of BPF_STX + BPF_ATOMIC + BPF_FETCH. -*/ -if (BPF_CLASS (insn.code) ==BPF_STX && -BPF_MODE (insn.code) ==BPF_ATOMIC ) { -/* BPF_STX + BPF_ATOMIC insns without BPF_FETCH do not -* define any registers, therefore zext_dst cannot be -* set. -*/ -if (WARN_ON (!(insn.imm &BPF_FETCH ))) -return -EINVAL ; - load_reg = insn.imm ==BPF_CMPXCHG ?BPF_REG_0 - : insn.src_reg; - }else { - load_reg = insn.dst_reg; - } +/* If no register is defined, zext_dst cannot be set. */ +if (WARN_ON (load_reg ==-1 )) +return -EINVAL ; zext_patch[0 ] = insn; zext_patch[1 ].dst_reg = load_reg;
Signature · 1 line
2.29.2
Quoted 12 lines · Ilya Leoshkevich
insn_has_def32()returns false for 32-bitBPF_FETCHinsns. This makesadjust_insn_aux_data()incorrectly setzext_dst, as can be seen in [1]. This happens becauseinsn_no_def()does not know aboutBPF_FETCHvariants ofBPF_STX.Fix in two steps.
First, replace
insn_no_def()withinsn_def_regno(), which returns which the register an insn defines. Normallyinsn_no_def()calls are followed byinsn->dst_reguses; replace those withinsn_def_regno()return value.
Quoted 4 lines
Second, adjust the
BPF_STXspecial case inis_reg64()to deal with queries made fromopt_subreg_zext_lo32_rnd_hi32(), where the state information is no longer available. Add a comment, since the purpose of this special case is not clear at the first glance.
Thanks for the fix. A few nits.
Quoted 4 lines
[1] https://lore.kernel.org/bpf/[email protected]/
Fixes 37086bfdc737 ("bpf: Propagate stack bounds to registers in atomics w/ BPF_FETCH")
Is it fixing this instead?5ffa25502b5a ("bpf: Add instructions for atomic_[cmp]xchg")
and bpf instead of bpf-next?
Quoted 10 lines
Signed-off-by Ilya Leoshkevich <[email protected]>--- kernel/bpf/verifier.c | 65 ++++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 32 deletions(-)kernel/bpf/verifier.c +0
-0
kernel/bpf/verifier.c static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn, · line 1708
Quoted 7 lines
kernel/bpf/verifier.c +3
-1@@ -1703,7 +1703,10 @@ static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn, }if (class ==BPF_STX ) { -if (reg->type !=SCALAR_VALUE ) +/* BPF_STX (including atomic variants) has multiple source +* operands, one of which is a ptr. Check whether the caller is +* asking about it. */
nit. A newline for "*/".
Quoted 17 lines
kernel/bpf/verifier.c +3
-6+if (t ==SRC_OP && reg->type !=SCALAR_VALUE )return true;return BPF_SIZE (code) ==BPF_DW ; }@@ -1735,22 +1738,33 @@ static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn,return true; } -/* Return TRUE if INSN doesn't have explicit value define. */ -static bool insn_no_def (struct bpf_insn *insn) +/* Return the regno defined by the insn, or -1. */ +static int insn_def_regno (const struct bpf_insn *insn) { -u8 class =BPF_CLASS (insn->code); - -return (class ==BPF_JMP || class ==BPF_JMP32 || - class ==BPF_STX || class ==BPF_ST );
kernel/bpf/verifier.c static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn, · line 1753
Quoted 10 lines
kernel/bpf/verifier.c +10
-0+switch (BPF_CLASS (insn->code)) { +case BPF_JMP : +case BPF_JMP32 : +case BPF_ST : +return -1 ; +case BPF_STX : +return (BPF_MODE (insn->code) ==BPF_ATOMIC && + (insn->imm &BPF_FETCH )) ? + (insn->imm ==BPF_CMPXCHG ?BPF_REG_0 : insn->src_reg) : +-1 ;
A bit hard to read with multiple "?" stacking on each other. How about using a more verbose "if else" here?
Quoted 72 lines
kernel/bpf/verifier.c +14
-17+default : +return insn->dst_reg; + } }/* Return TRUE if INSN has defined any 32-bit value explicitly. */ static bool insn_has_def32 (struct bpf_verifier_env *env,struct bpf_insn *insn) { -if (insn_no_def (insn)) +int dst_reg =insn_def_regno (insn); + +if (dst_reg ==-1 )return false; -return !is_reg64 (env, insn, insn->dst_reg,NULL ,DST_OP ); +return !is_reg64 (env, insn, dst_reg,NULL ,DST_OP ); } static void mark_insn_zext(struct bpf_verifier_env *env,@@ -11006,9 +11020,10 @@ static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env,for (i =0 ; i < len; i++) {int adj_idx = i + delta;struct bpf_insn insn; -u8 load_reg; +int load_reg; insn = insns[adj_idx]; + load_reg =insn_def_regno (&insn);if (!aux[adj_idx].zext_dst) {u8 code, class;u32 imm_rnd;@@ -11018,14 +11033,14 @@ static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, code = insn.code; class =BPF_CLASS (code); -if (insn_no_def (&insn)) +if (load_reg ==-1 )continue ;/* NOTE: arg "reg" (the fourth one) is only used for -* BPF_STX which has been ruled out in above -* check, it is safe to pass NULL here. +* BPF_STX + SRC_OP, so it is safe to pass NULL +* here. */ -if (is_reg64 (env, &insn, insn.dst_reg,NULL ,DST_OP )) { +if (is_reg64 (env, &insn, load_reg,NULL ,DST_OP )) {if (class ==BPF_LD &&BPF_MODE (code) ==BPF_IMM ) i++;@@ -11040,7 +11055,7 @@ static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, imm_rnd =get_random_int (); rnd_hi32_patch[0 ] = insn; rnd_hi32_patch[1 ].imm = imm_rnd; - rnd_hi32_patch[3 ].dst_reg = insn.dst_reg; + rnd_hi32_patch[3 ].dst_reg = load_reg; patch = rnd_hi32_patch; patch_len =4 ; goto apply_patch_buffer;@@ -11049,23 +11064,9 @@ static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env,if (!bpf_jit_needs_zext ())continue ; -/* zext_dst means that we want to zero-extend whatever register -* the insn defines, which is dst_reg most of the time, with -* the notable exception of BPF_STX + BPF_ATOMIC + BPF_FETCH. -*/ -if (BPF_CLASS (insn.code) ==BPF_STX && -BPF_MODE (insn.code) ==BPF_ATOMIC ) { -/* BPF_STX + BPF_ATOMIC insns without BPF_FETCH do not -* define any registers, therefore zext_dst cannot be -* set.
kernel/bpf/verifier.c static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, · line 11068
Quoted 10 lines
kernel/bpf/verifier.c +2
-8-*/ -if (WARN_ON (!(insn.imm &BPF_FETCH ))) -return -EINVAL ; - load_reg = insn.imm ==BPF_CMPXCHG ?BPF_REG_0 - : insn.src_reg; - }else { - load_reg = insn.dst_reg; - } +/* If no register is defined, zext_dst cannot be set. */ +if (WARN_ON (load_reg ==-1 ))
May be a WARN_ON_ONCE and then followed by verbose(env, ...).
kernel/bpf/verifier.c static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, · line 11069
Quoted 1 line
kernel/bpf/verifier.c +1
-0+return -EINVAL ;
-EFAULT instead. It is what most other places return also during verifier internal error. There are a few places return -EINVAL and probably we should change them in the future.
Quoted 6 lines
kernel/bpf/verifier.c +0
-0zext_patch[0 ] = insn; zext_patch[1 ].dst_reg = load_reg;Signature · 1 line
2.29.2