bpf: add support for BPF_ABS/BPF_IND instructions
cover bpf: add support for BPF_ABS/BPF_IND instructions
To fill the gap with linux kernel eBPF implementation,
add support for two non-generic instructions:
(BPF_ABS | <size> | BPF_LD) and (BPF_IND | <size> | BPF_LD) which are used to access packet data.
Make necessary changes in BPF verifier, interpreter and x86 JIT code.
Konstantin Ananyev (5):
test/bpf: fix few small issues
bpf: fix add/sub min/max estimations
bpf: add support for packet data load instructions test/bpf: add new test cases for mbuf load instructions
bpf: x86 JIT support for packet data load instructions
app/test/test_bpf.c | 504 ++++++++++++++++++++++++++++-- doc/guides/prog_guide/bpf_lib.rst | 30 +- lib/librte_bpf/bpf_exec.c | 57 ++++ lib/librte_bpf/bpf_jit_x86.c | 181 +++++++++++ lib/librte_bpf/bpf_validate.c | 104 +++++- 5 files changed, 843 insertions(+), 33 deletions(-)
Signature · 1 line
2.17.1
PATCH 1/5 test/bpf: fix few small issues
Address for few small issues:
- unreachable return statement
- failed test-case can finish with 'success' status
Also use unified cmp_res() function to check return value.
--- app/test/test_bpf.c | 41 +++++++++++++++-------------------------- 1 file changed, 15 insertions(+), 26 deletions(-)
app/test/test_bpf.c +15 -26
@@ -1797,13 +1797,6 @@ test_call1_check(uint64_t rc, const void *arg)dummy_func1 (arg, &v32, &v64); v64 += v32; -if (v64 != rc) { -printf ("%s@%d: invalid return value " -"expected=0x%" PRIx64", actual=0x%" PRIx64"\n" , - __func__, __LINE__, v64, rc); -return -1 ; - } -return 0 ;return cmp_res (__func__, v64, rc, dv, dv,sizeof (*dv)); }
@@ -1934,13 +1927,7 @@ test_call2_check(uint64_t rc, const void *arg)dummy_func2 (&a, &b); v = a.u64 + a.u32 + b.u16 + b.u8; -if (v != rc) { -printf ("%s@%d: invalid return value " -"expected=0x%" PRIx64", actual=0x%" PRIx64"\n" , - __func__, __LINE__, v, rc); -return -1 ; - } -return 0 ; +return cmp_res (__func__, v, rc, arg, arg,0 ); }static const struct rte_bpf_xsym test_call2_xsym[] = {
@@ -2429,7 +2416,6 @@ test_call5_check(uint64_t rc, const void *arg) v =0 ; fail: -return cmp_res (__func__, v, rc, &v, &rc,sizeof (v)); }
@@ -2458,6 +2444,7 @@ static const struct rte_bpf_xsym test_call5_xsym[] = { }, }; +/* all bpf test cases */ static const struct bpf_test tests[] = { { .name ="test_store1" ,
@@ -2738,7 +2725,6 @@ run_test(const struct bpf_test *tst) } tst->prepare (tbuf); - rc =rte_bpf_exec (bpf, tbuf); ret = tst->check_result (rc, tbuf);if (ret !=0 ) {
@@ -2746,17 +2732,20 @@ run_test(const struct bpf_test *tst) __func__, __LINE__, tst->name, ret, strerror(ret)); } +/* repeat the same test with jit, when possible */ rte_bpf_get_jit (bpf, &jit); -if (jit.func ==NULL ) -return 0 ; - - tst->prepare (tbuf); - rc = jit.func (tbuf); - rv = tst->check_result (rc, tbuf); - ret |= rv; -if (rv !=0 ) { -printf ("%s@%d: check_result(%s) failed, error: %d(%s);\n" , - __func__, __LINE__, tst->name, rv,strerror (ret)); +if (jit.func !=NULL ) { + + tst->prepare (tbuf); + rc = jit.func (tbuf); + rv = tst->check_result (rc, tbuf); + ret |= rv; +if (rv !=0 ) { +printf ("%s@%d: check_result(%s) failed, " +"error: %d(%s);\n" , + __func__, __LINE__, tst->name, + rv,strerror (ret)); + } }rte_bpf_destroy (bpf);
Signature · 1 line
2.17.1
Re: [dpdk-dev] [dpdk-stable] [PATCH 1/5] test/bpf: fix few small issues
18/05/2020 17:52, Konstantin Ananyev:
Quoted 7 lines
Address for few small issues:
- unreachable return statement
- failed test-case can finish with 'success' statusAlso use unified
cmp_res()function to check return value.Fixes a9de470cc7c0 ("test: move to app directory")
Looks like you don't want to track the original commits. In this case, better to not add any fake reference.
Quoted 3 lines
Signed-off-by Konstantin Ananyev <[email protected]>
PATCH 2/5 bpf: fix add/sub min/max estimations
In reply to Konstantin Ananyeveval_add()/eval_sub() not always correctly estimate minimum and maximum possible values of add/sub operations.
--- lib/librte_bpf/bpf_validate.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)
lib/librte_bpf/bpf_validate.c +5 -5
@@ -226,7 +226,7 @@ eval_add(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, uint64_t msk)struct bpf_reg_val rv; rv.u.min = (rd->u.min + rs->u.min) & msk; - rv.u.max = (rd->u.min + rs->u.max) & msk; + rv.u.max = (rd->u.max + rs->u.max) & msk; rv.s.min = (rd->s.min + rs->s.min) & msk; rv.s.max = (rd->s.max + rs->s.max) & msk;
@@ -254,10 +254,10 @@ eval_sub(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, uint64_t msk) {struct bpf_reg_val rv; - rv.u.min = (rd->u.min - rs->u.min) & msk; - rv.u.max = (rd->u.min - rs->u.max) & msk; - rv.s.min = (rd->s.min - rs->s.min) & msk; - rv.s.max = (rd->s.max - rs->s.max) & msk; + rv.u.min = (rd->u.min - rs->u.max) & msk; + rv.u.max = (rd->u.max - rs->u.min) & msk; + rv.s.min = (rd->s.min - rs->s.max) & msk; + rv.s.max = (rd->s.max - rs->s.min) & msk;/* * if at least one of the operands is not constant,
Signature · 1 line
2.17.1
PATCH 3/5 bpf: add support for packet data load instructions
In reply to Konstantin AnanyevTo fill the gap with linux kernel eBPF implementation, add support for two non-generic instructions:
(BPF_ABS | <size> | BPF_LD) and (BPF_IND | <size> | BPF_LD) which are used to access packet data.
These instructions can only be used when BPF context is a pointer to 'struct rte_mbuf' (i.e: RTE_BPF_ARG_PTR_MBUF type).
--- doc/guides/prog_guide/bpf_lib.rst | 30 +++++++++++- lib/librte_bpf/bpf_exec.c | 57 +++++++++++++++++++++++ lib/librte_bpf/bpf_validate.c | 77 +++++++++++++++++++++++++++++++ 3 files changed, 163 insertions(+), 1 deletion(-)
doc/guides/prog_guide/bpf_lib.rst +29 -1
@@ -27,6 +27,35 @@ The library API provides the following basic operations: * Load BPF program from the ELF file and install callback to execute it on given ethdev port/queue. +Packet data load instructions +----------------------------- + +DPDK supports two non-generic instructions: ``(BPF_ABS | size | BPF_LD)`` +and ``(BPF_IND | size | BPF_LD)`` which are used to access packet data. +These instructions can only be used when execution context is a pointer to +``struct rte_mbuf`` and have seven implicit operands. +Register ``R6`` is an implicit input that must contain pointer to ``rte_mbuf``. +Register ``R0`` is an implicit output which contains the data fetched from the +packet. Registers ``R1-R5`` are scratch registers +and must not be used to store the data across these instructions. +These instructions have implicit program exit condition as well. When +eBPF program is trying to access the data beyond the packet boundary, +the interpreter will abort the execution of the program. JIT compilers +therefore must preserve this property. ``src_reg`` and ``imm32`` fields are +explicit inputs to these instructions. +For example, ``(BPF_IND | BPF_W | BPF_LD)`` means: + +.. code-block:: c + + uint32_t tmp; + R0 = rte_pktmbuf_read((const struct rte_mbuf *)R6, src_reg + imm32, + sizeof(tmp), &tmp); + if (R0 == NULL) return FAILED; + R0 = ntohl(*(uint32_t *)R0); + +and ``R1-R5`` were scratched. + + Not currently supported eBPF features -------------------------------------
@@ -34,5 +63,4 @@ Not currently supported eBPF features - cBPF - tail-pointer call - eBPF MAP - - skb - external function calls for 32-bit platforms
lib/librte_bpf/bpf_exec.c +57 -0
75 lines not on this page · raw message
lib/librte_bpf/bpf_validate.c +77 -0
95 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH 4/5 test/bpf: add new test cases for mbuf load instructions
In reply to Konstantin AnanyevAdd new test-cases for BPF_ABS/BPF_IND load instructions.
--- app/test/test_bpf.c | 463 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 463 insertions(+)
app/test/test_bpf.c +463 -0
@@ -43,6 +43,14 @@ struct dummy_net {struct rte_ipv4_hdr ip_hdr; }; +#define DUMMY_MBUF_NUM 2 + +/* first mbuf in the packet, should always be at offset 0 */ +struct dummy_mbuf { +struct rte_mbuf mb[DUMMY_MBUF_NUM ]; +uint8_t buf[DUMMY_MBUF_NUM ][RTE_MBUF_DEFAULT_BUF_SIZE ]; +}; +#define TEST_FILL_1 0xDEADBEEF#define TEST_MUL_1 21
@@ -2444,6 +2452,413 @@ static const struct rte_bpf_xsym test_call5_xsym[] = { }, }; +/* load mbuf (BPF_ABS/BPF_IND) test-cases */ +static const struct ebpf_insn test_ld_mbuf1_prog[] = { + +/* BPF_ABS/BPF_IND implicitly expect mbuf ptr in R6 */ + { + .code = (EBPF_ALU64 |EBPF_MOV |BPF_X ), + .dst_reg =EBPF_REG_6 , + .src_reg =EBPF_REG_1 , + }, +/* load IPv4 version and IHL */ + { + .code = (BPF_LD |BPF_ABS |BPF_B ), + .imm = offsetof(struct rte_ipv4_hdr , version_ihl), + }, +/* check IP version */ + { + .code = (EBPF_ALU64 |EBPF_MOV |BPF_X ), + .dst_reg =EBPF_REG_2 , + .src_reg =EBPF_REG_0 , + }, + { + .code = (BPF_ALU |BPF_AND |BPF_K ), + .dst_reg =EBPF_REG_2 , + .imm =0xf0 , + }, + { + .code = (BPF_JMP |BPF_JEQ |BPF_K ), + .dst_reg =EBPF_REG_2 , + .imm =IPVERSION <<4 , + .off =2 , + }, +/* invalid IP version, return 0 */ + { + .code = (EBPF_ALU64 |BPF_XOR |BPF_X ), + .dst_reg =EBPF_REG_0 , + .src_reg =EBPF_REG_0 , + }, + { + .code = (BPF_JMP |EBPF_EXIT ), + }, +/* load 3-rd byte of IP data */ + { + .code = (BPF_ALU |BPF_AND |BPF_K ), + .dst_reg =EBPF_REG_0 , + .imm =RTE_IPV4_HDR_IHL_MASK , + }, + { + .code = (BPF_ALU |BPF_LSH |BPF_K ), + .dst_reg =EBPF_REG_0 , + .imm =2 , + }, + { + .code = (BPF_LD |BPF_IND |BPF_B ), + .src_reg =EBPF_REG_0 , + .imm =3 , + }, + { + .code = (EBPF_ALU64 |EBPF_MOV |BPF_X ), + .dst_reg =EBPF_REG_7 , + .src_reg =EBPF_REG_0 , + }, +/* load IPv4 src addr */ + { + .code = (BPF_LD |BPF_ABS |BPF_W ), + .imm = offsetof(struct rte_ipv4_hdr , src_addr), + }, + { + .code = (EBPF_ALU64 |BPF_ADD |BPF_X ), + .dst_reg =EBPF_REG_7 , + .src_reg =EBPF_REG_0 , + }, +/* load IPv4 total length */ + { + .code = (BPF_LD |BPF_ABS |BPF_H ), + .imm = offsetof(struct rte_ipv4_hdr , total_length), + }, + { + .code = (EBPF_ALU64 |EBPF_MOV |BPF_X ), + .dst_reg =EBPF_REG_8 , + .src_reg =EBPF_REG_0 , + }, +/* load last 4 bytes of IP data */ + { + .code = (BPF_LD |BPF_IND |BPF_W ), + .src_reg =EBPF_REG_8 , + .imm = -(int32_t )sizeof (uint32_t ), + }, + { + .code = (EBPF_ALU64 |BPF_ADD |BPF_X ), + .dst_reg =EBPF_REG_7 , + .src_reg =EBPF_REG_0 , + }, +/* load 2 bytes from the middle of IP data */ + { + .code = (EBPF_ALU64 |BPF_RSH |BPF_K ), + .dst_reg =EBPF_REG_8 , + .imm =1 , + }, + { + .code = (BPF_LD |BPF_IND |BPF_H ), + .src_reg =EBPF_REG_8 , + }, + { + .code = (EBPF_ALU64 |BPF_ADD |BPF_X ), + .dst_reg =EBPF_REG_0 , + .src_reg =EBPF_REG_7 , + }, + { + .code = (BPF_JMP |EBPF_EXIT ), + }, +}; + +static void +dummy_mbuf_prep (struct rte_mbuf *mb,uint8_t buf[],uint32_t buf_len, +uint32_t data_len) +{ +uint32_t i; +uint8_t *db; + + mb->buf_addr = buf; + mb->buf_iova = (uintptr_t )buf; + mb->buf_len = buf_len; +rte_mbuf_refcnt_set (mb,1 ); + +/* set pool pointer to dummy value, test doesn't use it */ + mb->pool = (void *)buf; + +rte_pktmbuf_reset (mb); + db = (uint8_t *)rte_pktmbuf_append (mb, data_len); + +for (i =0 ; i != data_len; i++) + db[i] = i; +} + +static void +test_ld_mbuf1_prepare (void *arg) +{ +struct dummy_mbuf *dm; +struct rte_ipv4_hdr *ph; + +const uint32_t plen =400 ; +const struct rte_ipv4_hdr iph = { + .version_ihl =RTE_IPV4_VHL_DEF , + .total_length =rte_cpu_to_be_16 (plen), + .time_to_live =IPDEFTTL , + .next_proto_id =IPPROTO_RAW , + .src_addr =rte_cpu_to_be_32 (RTE_IPV4_LOOPBACK ), + .dst_addr =rte_cpu_to_be_32 (RTE_IPV4_BROADCAST ), + }; + + dm = arg; +memset (dm,0 ,sizeof (*dm)); + +dummy_mbuf_prep (&dm->mb[0 ], dm->buf[0 ],sizeof (dm->buf[0 ]), + plen /2 +1 ); +dummy_mbuf_prep (&dm->mb[1 ], dm->buf[1 ],sizeof (dm->buf[0 ]), + plen /2 -1 ); + +rte_pktmbuf_chain (&dm->mb[0 ], &dm->mb[1 ]); + + ph =rte_pktmbuf_mtod (dm->mb,typeof (ph)); +memcpy (ph, &iph,sizeof (iph)); +} + +static uint64_t +test_ld_mbuf1 (const struct rte_mbuf *pkt) +{ +uint64_t n, v; +const uint8_t *p8; +const uint16_t *p16; +const uint32_t *p32; +struct dummy_offset dof; + +/* load IPv4 version and IHL */ + p8 =rte_pktmbuf_read (pkt, + offsetof(struct rte_ipv4_hdr , version_ihl),sizeof (*p8), + &dof); +if (p8 ==NULL ) +return 0 ; + +/* check IP version */ +if ((p8[0 ] &0xf0 ) !=IPVERSION <<4 ) +return 0 ; + + n = (p8[0 ] &RTE_IPV4_HDR_IHL_MASK ) *RTE_IPV4_IHL_MULTIPLIER ; + +/* load 3-rd byte of IP data */ + p8 =rte_pktmbuf_read (pkt, n +3 ,sizeof (*p8), &dof); +if (p8 ==NULL ) +return 0 ; + + v = p8[0 ]; + +/* load IPv4 src addr */ + p32 =rte_pktmbuf_read (pkt, + offsetof(struct rte_ipv4_hdr , src_addr),sizeof (*p32), + &dof); +if (p32 ==NULL ) +return 0 ; + + v +=rte_be_to_cpu_32 (p32[0 ]); + +/* load IPv4 total length */ + p16 =rte_pktmbuf_read (pkt, + offsetof(struct rte_ipv4_hdr , total_length),sizeof (*p16), + &dof); +if (p16 ==NULL ) +return 0 ; + + n =rte_be_to_cpu_16 (p16[0 ]); + +/* load last 4 bytes of IP data */ + p32 =rte_pktmbuf_read (pkt, n -sizeof (*p32),sizeof (*p32), &dof); +if (p32 ==NULL ) +return 0 ; + + v +=rte_be_to_cpu_32 (p32[0 ]); + +/* load 2 bytes from the middle of IP data */ + p16 =rte_pktmbuf_read (pkt, n /2 ,sizeof (*p16), &dof); +if (p16 ==NULL ) +return 0 ; + + v +=rte_be_to_cpu_16 (p16[0 ]); +return v; +} + +static int +test_ld_mbuf1_check (uint64_t rc,const void *arg) +{ +const struct dummy_mbuf *dm; +uint64_t v; + + dm = arg; + v =test_ld_mbuf1 (dm->mb); +return cmp_res (__func__, v, rc, arg, arg,0 ); +} + +/* +* same as ld_mbuf1, but then trancate the mbuf by 1B, +* so load of last 4B fail. +*/ +static void +test_ld_mbuf2_prepare (void *arg) +{ +struct dummy_mbuf *dm; + +test_ld_mbuf1_prepare (arg); + dm = arg; +rte_pktmbuf_trim (dm->mb,1 ); +} + +static int +test_ld_mbuf2_check (uint64_t rc,const void *arg) +{ +return cmp_res (__func__,0 , rc, arg, arg,0 ); +} + +/* same as test_ld_mbuf1, but now store intermediate results on the stack */ +static const struct ebpf_insn test_ld_mbuf3_prog[] = { + +/* BPF_ABS/BPF_IND implicitly expect mbuf ptr in R6 */ + { + .code = (EBPF_ALU64 |EBPF_MOV |BPF_X ), + .dst_reg =EBPF_REG_6 , + .src_reg =EBPF_REG_1 , + }, +/* load IPv4 version and IHL */ + { + .code = (BPF_LD |BPF_ABS |BPF_B ), + .imm = offsetof(struct rte_ipv4_hdr , version_ihl), + }, +/* check IP version */ + { + .code = (EBPF_ALU64 |EBPF_MOV |BPF_X ), + .dst_reg =EBPF_REG_2 , + .src_reg =EBPF_REG_0 , + }, + { + .code = (BPF_ALU |BPF_AND |BPF_K ), + .dst_reg =EBPF_REG_2 , + .imm =0xf0 , + }, + { + .code = (BPF_JMP |BPF_JEQ |BPF_K ), + .dst_reg =EBPF_REG_2 , + .imm =IPVERSION <<4 , + .off =2 , + }, +/* invalid IP version, return 0 */ + { + .code = (EBPF_ALU64 |BPF_XOR |BPF_X ), + .dst_reg =EBPF_REG_0 , + .src_reg =EBPF_REG_0 , + }, + { + .code = (BPF_JMP |EBPF_EXIT ), + }, +/* load 3-rd byte of IP data */ + { + .code = (BPF_ALU |BPF_AND |BPF_K ), + .dst_reg =EBPF_REG_0 , + .imm =RTE_IPV4_HDR_IHL_MASK , + }, + { + .code = (BPF_ALU |BPF_LSH |BPF_K ), + .dst_reg =EBPF_REG_0 , + .imm =2 , + }, + { + .code = (BPF_LD |BPF_IND |BPF_B ), + .src_reg =EBPF_REG_0 , + .imm =3 , + }, + { + .code = (BPF_STX |BPF_MEM |BPF_B ), + .dst_reg =EBPF_REG_10 , + .src_reg =EBPF_REG_0 , + .off = (int16_t )(offsetof(struct dummy_offset , u8) - +sizeof (struct dummy_offset )), + }, +/* load IPv4 src addr */ + { + .code = (BPF_LD |BPF_ABS |BPF_W ), + .imm = offsetof(struct rte_ipv4_hdr , src_addr), + }, + { + .code = (BPF_STX |BPF_MEM |BPF_W ), + .dst_reg =EBPF_REG_10 , + .src_reg =EBPF_REG_0 , + .off = (int16_t )(offsetof(struct dummy_offset , u32) - +sizeof (struct dummy_offset )), + }, +/* load IPv4 total length */ + { + .code = (BPF_LD |BPF_ABS |BPF_H ), + .imm = offsetof(struct rte_ipv4_hdr , total_length), + }, + { + .code = (EBPF_ALU64 |EBPF_MOV |BPF_X ), + .dst_reg =EBPF_REG_8 , + .src_reg =EBPF_REG_0 , + }, +/* load last 4 bytes of IP data */ + { + .code = (BPF_LD |BPF_IND |BPF_W ), + .src_reg =EBPF_REG_8 , + .imm = -(int32_t )sizeof (uint32_t ), + }, + { + .code = (BPF_STX |BPF_MEM |EBPF_DW ), + .dst_reg =EBPF_REG_10 , + .src_reg =EBPF_REG_0 , + .off = (int16_t )(offsetof(struct dummy_offset , u64) - +sizeof (struct dummy_offset )), + }, +/* load 2 bytes from the middle of IP data */ + { + .code = (EBPF_ALU64 |BPF_RSH |BPF_K ), + .dst_reg =EBPF_REG_8 , + .imm =1 , + }, + { + .code = (BPF_LD |BPF_IND |BPF_H ), + .src_reg =EBPF_REG_8 , + }, + { + .code = (BPF_LDX |BPF_MEM |EBPF_DW ), + .dst_reg =EBPF_REG_1 , + .src_reg =EBPF_REG_10 , + .off = (int16_t )(offsetof(struct dummy_offset , u64) - +sizeof (struct dummy_offset )), + }, + { + .code = (EBPF_ALU64 |BPF_ADD |BPF_X ), + .dst_reg =EBPF_REG_0 , + .src_reg =EBPF_REG_1 , + }, + { + .code = (BPF_LDX |BPF_MEM |BPF_W ), + .dst_reg =EBPF_REG_1 , + .src_reg =EBPF_REG_10 , + .off = (int16_t )(offsetof(struct dummy_offset , u32) - +sizeof (struct dummy_offset )), + }, + { + .code = (EBPF_ALU64 |BPF_ADD |BPF_X ), + .dst_reg =EBPF_REG_0 , + .src_reg =EBPF_REG_1 , + }, + { + .code = (BPF_LDX |BPF_MEM |BPF_B ), + .dst_reg =EBPF_REG_1 , + .src_reg =EBPF_REG_10 , + .off = (int16_t )(offsetof(struct dummy_offset , u8) - +sizeof (struct dummy_offset )), + }, + { + .code = (EBPF_ALU64 |BPF_ADD |BPF_X ), + .dst_reg =EBPF_REG_0 , + .src_reg =EBPF_REG_1 , + }, + { + .code = (BPF_JMP |EBPF_EXIT ), + }, +}; +/* all bpf test cases */ static const struct bpf_test tests[] = { {
@@ -2704,6 +3119,54 @@ static const struct bpf_test tests[] = {/* for now don't support function calls on 32 bit platform */ .allow_fail = (sizeof(uint64_t) != sizeof(uintptr_t)), }, + { + .name = "test_ld_mbuf1", + .arg_sz = sizeof(struct dummy_mbuf), + .prm = { + .ins = test_ld_mbuf1_prog, + .nb_ins = RTE_DIM(test_ld_mbuf1_prog), + .prog_arg = { + .type = RTE_BPF_ARG_PTR_MBUF, + .buf_size =sizeof (struct dummy_mbuf ), + }, + }, + .prepare = test_ld_mbuf1_prepare, + .check_result = test_ld_mbuf1_check, +/* mbuf as input argument is not supported on 32 bit platform */ + .allow_fail = (sizeof(uint64_t) != sizeof(uintptr_t)), + }, + { + .name ="test_ld_mbuf2" , + .arg_sz = sizeof(struct dummy_mbuf), + .prm = { + .ins = test_ld_mbuf1_prog, + .nb_ins = RTE_DIM(test_ld_mbuf1_prog), + .prog_arg = { + .type =RTE_BPF_ARG_PTR_MBUF , + .buf_size = sizeof(struct dummy_mbuf), + }, + }, + .prepare = test_ld_mbuf2_prepare, + .check_result = test_ld_mbuf2_check, +/* mbuf as input argument is not supported on 32 bit platform */ + .allow_fail = (sizeof(uint64_t) != sizeof(uintptr_t)), + }, + { + .name ="test_ld_mbuf3" , + .arg_sz = sizeof(struct dummy_mbuf), + .prm = { + .ins = test_ld_mbuf3_prog, + .nb_ins = RTE_DIM(test_ld_mbuf3_prog), + .prog_arg = { + .type =RTE_BPF_ARG_PTR_MBUF , + .buf_size = sizeof(struct dummy_mbuf), + }, + }, + .prepare = test_ld_mbuf1_prepare, + .check_result = test_ld_mbuf1_check, +/* mbuf as input argument is not supported on 32 bit platform */ + .allow_fail = (sizeof(uint64_t) != sizeof(uintptr_t)), + }, }; static int
Signature · 1 line
2.17.1
PATCH 5/5 bpf: x86 JIT support for packet data load instructions
In reply to Konstantin AnanyevMake x86 JIT to generate native code for
(BPF_ABS | <size> | BPF_LD) and (BPF_IND | <size> | BPF_LD) instructions.
--- lib/librte_bpf/bpf_jit_x86.c | 181 ++++++++++++++++++++++++++++++++++ lib/librte_bpf/bpf_validate.c | 17 +++- 2 files changed, 197 insertions(+), 1 deletion(-)
lib/librte_bpf/bpf_jit_x86.c +181 -0
@@ -87,6 +87,14 @@ enum { REG_TMP1 = R10, }; +/* LD_ABS/LD_IMM offsets */ +enum { +LDMB_FSP_OFS ,/* fast-path */ +LDMB_SLP_OFS ,/* slow-path */ +LDMB_FIN_OFS ,/* final part */ +LDMB_OFS_NUM +}; +/* * callee saved registers list. * keep RBP as the last one.
@@ -100,6 +108,9 @@ struct bpf_jit_state {uint32_t num;int32_t off; } exit; +struct { +uint32_t stack_ofs; + } ldmb;uint32_t reguse;int32_t *off;uint8_t *ins;
@@ -1024,6 +1035,166 @@ emit_div(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,emit_mov_reg (st,EBPF_ALU64 |EBPF_MOV |BPF_X ,REG_TMP1 ,RDX ); } +/* +* helper function, used by emit_ld_mbuf(). +* generates code for 'fast_path': +* calculate load offset and check is it inside first packet segment. +*/ +static void +emit_ldmb_fast_path (struct bpf_jit_state *st,const uint32_t rg[EBPF_REG_7 ], +uint32_t sreg,uint32_t mode,uint32_t sz,uint32_t imm, +const int32_t ofs[LDMB_OFS_NUM ]) +{ +/* make R2 contain *off* value */ + +if (sreg != rg[EBPF_REG_2 ]) { +emit_mov_imm (st,EBPF_ALU64 |EBPF_MOV |BPF_K , + rg[EBPF_REG_2 ], imm); +if (mode ==BPF_IND ) +emit_alu_reg (st,EBPF_ALU64 |BPF_ADD |BPF_X , + sreg, rg[EBPF_REG_2 ]); + }else +/* BPF_IND with sreg == R2 */ +emit_alu_imm (st,EBPF_ALU64 |BPF_ADD |BPF_K , + rg[EBPF_REG_2 ], imm); + +/* R3 = mbuf->data_len */ +emit_ld_reg (st,BPF_LDX |BPF_MEM |BPF_H , + rg[EBPF_REG_6 ], rg[EBPF_REG_3 ], + offsetof(struct rte_mbuf , data_len)); + +/* R3 = R3 - R2 */ +emit_alu_reg (st,EBPF_ALU64 |BPF_SUB |BPF_X , + rg[EBPF_REG_2 ], rg[EBPF_REG_3 ]); + +/* JSLE R3, <sz> <slow_path> */ +emit_cmp_imm (st,EBPF_ALU64 , rg[EBPF_REG_3 ], sz); +emit_abs_jcc (st,BPF_JMP |EBPF_JSLE |BPF_K , ofs[LDMB_SLP_OFS ]); + +/* R3 = mbuf->data_off */ +emit_ld_reg (st,BPF_LDX |BPF_MEM |BPF_H , + rg[EBPF_REG_6 ], rg[EBPF_REG_3 ], + offsetof(struct rte_mbuf , data_off)); + +/* R0 = mbuf->buf_addr */ +emit_ld_reg (st,BPF_LDX |BPF_MEM |EBPF_DW , + rg[EBPF_REG_6 ], rg[EBPF_REG_0 ], + offsetof(struct rte_mbuf , buf_addr)); + +/* R0 = R0 + R3 */ +emit_alu_reg (st,EBPF_ALU64 |BPF_ADD |BPF_X , + rg[EBPF_REG_3 ], rg[EBPF_REG_0 ]); + +/* R0 = R0 + R2 */ +emit_alu_reg (st,EBPF_ALU64 |BPF_ADD |BPF_X , + rg[EBPF_REG_2 ], rg[EBPF_REG_0 ]); + +/* JMP <fin_part> */ +emit_abs_jmp (st, ofs[LDMB_FIN_OFS ]); +} + +/* +* helper function, used by emit_ld_mbuf(). +* generates code for 'slow_path': +* call __rte_pktmbuf_read() and check return value. +*/ +static void +emit_ldmb_slow_path (struct bpf_jit_state *st,const uint32_t rg[EBPF_REG_7 ], +uint32_t sz) +{ +/* make R3 contain *len* value (1/2/4) */ + +emit_mov_imm (st,EBPF_ALU64 |EBPF_MOV |BPF_K , rg[EBPF_REG_3 ], sz); + +/* make R4 contain (RBP - ldmb.stack_ofs) */ + +emit_mov_reg (st,EBPF_ALU64 |EBPF_MOV |BPF_X ,RBP , rg[EBPF_REG_4 ]); +emit_alu_imm (st,EBPF_ALU64 |BPF_SUB |BPF_K , rg[EBPF_REG_4 ], + st->ldmb.stack_ofs); + +/* make R1 contain mbuf ptr */ + +emit_mov_reg (st,EBPF_ALU64 |EBPF_MOV |BPF_X , + rg[EBPF_REG_6 ], rg[EBPF_REG_1 ]); + +/* call rte_pktmbuf_read */ +emit_call (st, (uintptr_t )__rte_pktmbuf_read); + +/* check that return value (R0) is not zero */ +emit_tst_reg (st,EBPF_ALU64 , rg[EBPF_REG_0 ], rg[EBPF_REG_0 ]); +emit_abs_jcc (st,BPF_JMP |BPF_JEQ |BPF_K , st->exit.off); +} + +/* +* helper function, used by emit_ld_mbuf(). +* generates final part of code for BPF_ABS/BPF_IND load: +* perform data load and endianness conversion. +* expects dreg to contain valid data pointer. +*/ +static void +emit_ldmb_fin (struct bpf_jit_state *st,uint32_t dreg,uint32_t opsz, +uint32_t sz) +{ +emit_ld_reg (st,BPF_LDX |BPF_MEM | opsz, dreg, dreg,0 ); +if (sz !=sizeof (uint8_t )) +emit_be2le (st, dreg, sz *CHAR_BIT ); +} + +/* +* emit code for BPF_ABS/BPF_IND load. +* generates the following construction: +* fast_path: +* off = ins->sreg + ins->imm +* if (off + ins->opsz < mbuf->data_len) +* goto slow_path; +* ptr = mbuf->buf_addr + bmf->data_off + off; +* goto fin_part; +* slow_path: +* typeof(ins->opsz) buf; //allocate space on the stack +* ptr = __rte_pktmbuf_read(mbuf, off, ins->opsz, &buf); +* if (ptr == NULL) +* goto exit_label; +* fin_part: +* res = *(typeof(ins->opsz))ptr; +* res = bswap(res); +*/ +static void +emit_ld_mbuf (struct bpf_jit_state *st,uint32_t op,uint32_t sreg,uint32_t imm) +{ +uint32_t i, mode, opsz, sz; +uint32_t rg[EBPF_REG_7 ]; +int32_t ofs[LDMB_OFS_NUM ]; + + mode =BPF_MODE (op); + opsz =BPF_SIZE (op); + sz =bpf_size (opsz); + +for (i =0 ; i !=RTE_DIM (rg); i++) + rg[i] = ebpf2x86[i]; + +/* fill with fake offsets */ +for (i =0 ; i !=RTE_DIM (ofs); i++) + ofs[i] = st->sz +INT8_MAX ; + +/* dry run first to calculate jump offsets */ + + ofs[LDMB_FSP_OFS ] = st->sz; +emit_ldmb_fast_path (st, rg, sreg, mode, sz, imm, ofs); + ofs[LDMB_SLP_OFS ] = st->sz; +emit_ldmb_slow_path (st, rg, sz); + ofs[LDMB_FIN_OFS ] = st->sz; +emit_ldmb_fin (st, rg[EBPF_REG_0 ], opsz, sz); + +RTE_VERIFY (ofs[LDMB_FIN_OFS ] - ofs[LDMB_FSP_OFS ] <=INT8_MAX ); + +/* reset dry-run code and do a proper run */ + + st->sz = ofs[LDMB_FSP_OFS ]; +emit_ldmb_fast_path (st, rg, sreg, mode, sz, imm, ofs); +emit_ldmb_slow_path (st, rg, sz); +emit_ldmb_fin (st, rg[EBPF_REG_0 ], opsz, sz); +} +static void emit_prolog (struct bpf_jit_state *st,int32_t stack_size) {
@@ -1121,6 +1292,7 @@ emit(struct bpf_jit_state *st, const struct rte_bpf *bpf)/* reset state fields */ st->sz =0 ; st->exit.num =0 ; + st->ldmb.stack_ofs = bpf->stack_sz;emit_prolog (st, bpf->stack_sz);
@@ -1240,6 +1412,15 @@ emit(struct bpf_jit_state *st, const struct rte_bpf *bpf)emit_ld_imm64 (st, dr, ins[0 ].imm, ins[1 ].imm); i++;break ; +/* load absolute/indirect instructions */ +case (BPF_LD |BPF_ABS |BPF_B ): +case (BPF_LD |BPF_ABS |BPF_H ): +case (BPF_LD |BPF_ABS |BPF_W ): +case (BPF_LD |BPF_IND |BPF_B ): +case (BPF_LD |BPF_IND |BPF_H ): +case (BPF_LD |BPF_IND |BPF_W ): +emit_ld_mbuf (st, op, sr, ins->imm); +break ;/* store instructions */ case (BPF_STX |BPF_MEM |BPF_B ):case (BPF_STX |BPF_MEM |BPF_H ):
lib/librte_bpf/bpf_validate.c +16 -1
@@ -70,6 +70,7 @@ struct bpf_verifier {uint64_t stack_sz;uint32_t nb_nodes;uint32_t nb_jcc_nodes; +uint32_t nb_ldmb_nodes;uint32_t node_colour[MAX_NODE_COLOUR ];uint32_t edge_type[MAX_EDGE_TYPE ];struct bpf_eval_state *evst;
@@ -2020,6 +2021,14 @@ validate(struct bpf_verifier *bvf) rc |=add_edge (bvf, node, i +2 ); i++;break ; +case (BPF_LD |BPF_ABS |BPF_B ): +case (BPF_LD |BPF_ABS |BPF_H ): +case (BPF_LD |BPF_ABS |BPF_W ): +case (BPF_LD |BPF_IND |BPF_B ): +case (BPF_LD |BPF_IND |BPF_H ): +case (BPF_LD |BPF_IND |BPF_W ): + bvf->nb_ldmb_nodes++; +/* fallthrough */ default : rc |=add_edge (bvf, node, i +1 );break ;
@@ -2320,8 +2329,14 @@ bpf_validate(struct rte_bpf *bpf)free (bvf.in);/* copy collected info */ -if (rc ==0 ) +if (rc ==0 ) { bpf->stack_sz = bvf.stack_sz; +/* for LD_ABS/LD_IND, we'll need extra space on the stack */ +if (bvf.nb_ldmb_nodes !=0 ) + bpf->stack_sz =RTE_ALIGN_CEIL (bpf->stack_sz + +sizeof (uint64_t ),sizeof (uint64_t )); + } +return rc; }
Signature · 1 line
2.17.1
Re: [dpdk-dev] [PATCH 5/5] bpf: x86 JIT support for packet data loadinstructions
Quoted 151 lines
From: dev [mailto:[email protected]] On Behalf Of Konstantin Ananyev
Sent: Monday, May 18, 2020 5:53 PMMake x86 JIT to generate native code for
(BPF_ABS| <size> |BPF_LD) and (BPF_IND| <size> |BPF_LD) instructions.Signed-off-by Konstantin Ananyev <[email protected]>--- lib/librte_bpf/bpf_jit_x86.c | 181 ++++++++++++++++++++++++++++++++++ lib/librte_bpf/bpf_validate.c | 17 +++- 2 files changed, 197 insertions(+), 1 deletion(-)lib/librte_bpf/bpf_jit_x86.c +113
-0@@ -87,6 +87,14 @@ enum { REG_TMP1 = R10, }; +/* LD_ABS/LD_IMM offsets */ +enum { +LDMB_FSP_OFS ,/* fast-path */ +LDMB_SLP_OFS ,/* slow-path */ +LDMB_FIN_OFS ,/* final part */ +LDMB_OFS_NUM +}; +/* * callee saved registers list. * keep RBP as the last one. @@ -100,6 +108,9 @@ struct bpf_jit_state {uint32_t num;int32_t off; } exit; +struct { +uint32_t stack_ofs; + } ldmb;uint32_t reguse;int32_t *off;uint8_t *ins;@@ -1024,6 +1035,166 @@ emit_div(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg, emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP1, RDX); } +/* +* helper function, used by emit_ld_mbuf(). +* generates code for 'fast_path': +* calculate load offset and check is it inside first packet segment. +*/ +static void +emit_ldmb_fast_path (struct bpf_jit_state *st,const uint32_t rg[EBPF_REG_7 ], +uint32_t sreg,uint32_t mode,uint32_t sz,uint32_t imm, +const int32_t ofs[LDMB_OFS_NUM ]) +{ +/* make R2 contain *off* value */ + +if (sreg != rg[EBPF_REG_2 ]) { +emit_mov_imm (st,EBPF_ALU64 |EBPF_MOV |BPF_K , + rg[EBPF_REG_2 ], imm); +if (mode ==BPF_IND ) +emit_alu_reg (st,EBPF_ALU64 |BPF_ADD |BPF_X , + sreg, rg[EBPF_REG_2 ]); + }else +/* BPF_IND with sreg == R2 */ +emit_alu_imm (st,EBPF_ALU64 |BPF_ADD |BPF_K , + rg[EBPF_REG_2 ], imm); + +/* R3 = mbuf->data_len */ +emit_ld_reg (st,BPF_LDX |BPF_MEM |BPF_H , + rg[EBPF_REG_6 ], rg[EBPF_REG_3 ], + offsetof(struct rte_mbuf , data_len)); + +/* R3 = R3 - R2 */ +emit_alu_reg (st,EBPF_ALU64 |BPF_SUB |BPF_X , + rg[EBPF_REG_2 ], rg[EBPF_REG_3 ]); + +/* JSLE R3, <sz> <slow_path> */ +emit_cmp_imm (st,EBPF_ALU64 , rg[EBPF_REG_3 ], sz); +emit_abs_jcc (st,BPF_JMP |EBPF_JSLE |BPF_K , ofs[LDMB_SLP_OFS ]); + +/* R3 = mbuf->data_off */ +emit_ld_reg (st,BPF_LDX |BPF_MEM |BPF_H , + rg[EBPF_REG_6 ], rg[EBPF_REG_3 ], + offsetof(struct rte_mbuf , data_off)); + +/* R0 = mbuf->buf_addr */ +emit_ld_reg (st,BPF_LDX |BPF_MEM |EBPF_DW , + rg[EBPF_REG_6 ], rg[EBPF_REG_0 ], + offsetof(struct rte_mbuf , buf_addr)); + +/* R0 = R0 + R3 */ +emit_alu_reg (st,EBPF_ALU64 |BPF_ADD |BPF_X , + rg[EBPF_REG_3 ], rg[EBPF_REG_0 ]); + +/* R0 = R0 + R2 */ +emit_alu_reg (st,EBPF_ALU64 |BPF_ADD |BPF_X , + rg[EBPF_REG_2 ], rg[EBPF_REG_0 ]); + +/* JMP <fin_part> */ +emit_abs_jmp (st, ofs[LDMB_FIN_OFS ]); +} + +/* +* helper function, used by emit_ld_mbuf(). +* generates code for 'slow_path': +* call __rte_pktmbuf_read() and check return value. +*/ +static void +emit_ldmb_slow_path (struct bpf_jit_state *st,const uint32_t rg[EBPF_REG_7 ], +uint32_t sz) +{ +/* make R3 contain *len* value (1/2/4) */ + +emit_mov_imm (st,EBPF_ALU64 |EBPF_MOV |BPF_K , rg[EBPF_REG_3 ], sz); + +/* make R4 contain (RBP - ldmb.stack_ofs) */ + +emit_mov_reg (st,EBPF_ALU64 |EBPF_MOV |BPF_X ,RBP , rg[EBPF_REG_4 ]); +emit_alu_imm (st,EBPF_ALU64 |BPF_SUB |BPF_K , rg[EBPF_REG_4 ], + st->ldmb.stack_ofs); + +/* make R1 contain mbuf ptr */ + +emit_mov_reg (st,EBPF_ALU64 |EBPF_MOV |BPF_X , + rg[EBPF_REG_6 ], rg[EBPF_REG_1 ]); + +/* call rte_pktmbuf_read */ +emit_call (st, (uintptr_t )__rte_pktmbuf_read); + +/* check that return value (R0) is not zero */ +emit_tst_reg (st,EBPF_ALU64 , rg[EBPF_REG_0 ], rg[EBPF_REG_0 ]); +emit_abs_jcc (st,BPF_JMP |BPF_JEQ |BPF_K , st->exit.off); +} + +/* +* helper function, used by emit_ld_mbuf(). +* generates final part of code for BPF_ABS/BPF_IND load: +* perform data load and endianness conversion. +* expects dreg to contain valid data pointer. +*/ +static void +emit_ldmb_fin (struct bpf_jit_state *st,uint32_t dreg,uint32_t opsz, +uint32_t sz) +{ +emit_ld_reg (st,BPF_LDX |BPF_MEM | opsz, dreg, dreg,0 ); +if (sz !=sizeof (uint8_t ))
lib/librte_bpf/bpf_jit_x86.c emit_div(struct bpf_jit_state *st, uint32_t op, · line 1152
Quoted 10 lines
lib/librte_bpf/bpf_jit_x86.c +10
-0+emit_be2le (st, dreg, sz *CHAR_BIT ); +} + +/* +* emit code for BPF_ABS/BPF_IND load. +* generates the following construction: +* fast_path: +* off = ins->sreg + ins->imm +* if (off + ins->opsz < mbuf->data_len) +* goto slow_path;
I am not an eBPF expert, but I am not sure this is correct.
I think it should be > instead of <. Also, it looks like you actually emit:
if (mbuf->data_len - off <= ins->opsz)
goto slow_path;
Could you please double check that both the comment and the emitted code has the comparison turning the correct way.
Quoted 124 lines
lib/librte_bpf/bpf_jit_x86.c +13
-0+* ptr = mbuf->buf_addr + bmf->data_off + off; +* goto fin_part; +* slow_path: +* typeof(ins->opsz) buf; //allocate space on the stack +* ptr = __rte_pktmbuf_read(mbuf, off, ins->opsz, &buf); +* if (ptr == NULL) +* goto exit_label; +* fin_part: +* res = *(typeof(ins->opsz))ptr; +* res = bswap(res); +*/ +static void +emit_ld_mbuf(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
uint32_timm)lib/librte_bpf/bpf_jit_x86.c +45
-0+{ +uint32_t i, mode, opsz, sz; +uint32_t rg[EBPF_REG_7 ]; +int32_t ofs[LDMB_OFS_NUM ]; + + mode =BPF_MODE (op); + opsz =BPF_SIZE (op); + sz =bpf_size (opsz); + +for (i =0 ; i !=RTE_DIM (rg); i++) + rg[i] = ebpf2x86[i]; + +/* fill with fake offsets */ +for (i =0 ; i !=RTE_DIM (ofs); i++) + ofs[i] = st->sz +INT8_MAX ; + +/* dry run first to calculate jump offsets */ + + ofs[LDMB_FSP_OFS ] = st->sz; +emit_ldmb_fast_path (st, rg, sreg, mode, sz, imm, ofs); + ofs[LDMB_SLP_OFS ] = st->sz; +emit_ldmb_slow_path (st, rg, sz); + ofs[LDMB_FIN_OFS ] = st->sz; +emit_ldmb_fin (st, rg[EBPF_REG_0 ], opsz, sz); + +RTE_VERIFY (ofs[LDMB_FIN_OFS ] - ofs[LDMB_FSP_OFS ] <=INT8_MAX ); + +/* reset dry-run code and do a proper run */ + + st->sz = ofs[LDMB_FSP_OFS ]; +emit_ldmb_fast_path (st, rg, sreg, mode, sz, imm, ofs); +emit_ldmb_slow_path (st, rg, sz); +emit_ldmb_fin (st, rg[EBPF_REG_0 ], opsz, sz); +} +static void emit_prolog (struct bpf_jit_state *st,int32_t stack_size) {@@ -1121,6 +1292,7 @@ emit(struct bpf_jit_state *st, const struct rte_bpf *bpf)/* reset state fields */ st->sz =0 ; st->exit.num =0 ; + st->ldmb.stack_ofs = bpf->stack_sz;emit_prolog (st, bpf->stack_sz);@@ -1240,6 +1412,15 @@ emit(struct bpf_jit_state *st, const struct rte_bpf *bpf)emit_ld_imm64 (st, dr, ins[0 ].imm, ins[1 ].imm); i++;break ; +/* load absolute/indirect instructions */ +case (BPF_LD |BPF_ABS |BPF_B ): +case (BPF_LD |BPF_ABS |BPF_H ): +case (BPF_LD |BPF_ABS |BPF_W ): +case (BPF_LD |BPF_IND |BPF_B ): +case (BPF_LD |BPF_IND |BPF_H ): +case (BPF_LD |BPF_IND |BPF_W ): +emit_ld_mbuf (st, op, sr, ins->imm); +break ;/* store instructions */ case (BPF_STX |BPF_MEM |BPF_B ):lib/librte_bpf/bpf_jit_x86.c +0
-0case (BPF_STX |BPF_MEM |BPF_H ):lib/librte_bpf/bpf_validate.c +16
-2@@ -70,6 +70,7 @@ struct bpf_verifier {uint64_t stack_sz;uint32_t nb_nodes;uint32_t nb_jcc_nodes; +uint32_t nb_ldmb_nodes;uint32_t node_colour[MAX_NODE_COLOUR ];uint32_t edge_type[MAX_EDGE_TYPE ];struct bpf_eval_state *evst;@@ -2020,6 +2021,14 @@ validate(struct bpf_verifier *bvf) rc |=add_edge (bvf, node, i +2 ); i++;break ; +case (BPF_LD |BPF_ABS |BPF_B ): +case (BPF_LD |BPF_ABS |BPF_H ): +case (BPF_LD |BPF_ABS |BPF_W ): +case (BPF_LD |BPF_IND |BPF_B ): +case (BPF_LD |BPF_IND |BPF_H ): +case (BPF_LD |BPF_IND |BPF_W ): + bvf->nb_ldmb_nodes++; +/* fallthrough */ default : rc |=add_edge (bvf, node, i +1 );break ;@@ -2320,8 +2329,14 @@ bpf_validate(struct rte_bpf *bpf)free (bvf.in);/* copy collected info */ -if (rc ==0 ) +if (rc ==0 ) { bpf->stack_sz = bvf.stack_sz; +/* for LD_ABS/LD_IND, we'll need extra space on the stack */ +if (bvf.nb_ldmb_nodes !=0 ) + bpf->stack_sz =RTE_ALIGN_CEIL (bpf->stack_sz + +sizeof (uint64_t ),sizeof (uint64_t )); + } +return rc; } --2.17.1
Re: [dpdk-dev] [PATCH 5/5] bpf: x86 JIT support for packet data loadinstructions
Quoted 118 lines
Quoted 105 lines
lib/librte_bpf/bpf_jit_x86.c +7
-0+/* +* helper function, used by emit_ld_mbuf(). +* generates code for 'fast_path': +* calculate load offset and check is it inside first packet segment. +*/ +static void +emit_ldmb_fast_path(struct bpf_jit_state *st, const uint32_trg[EBPF_REG_7],
lib/librte_bpf/bpf_jit_x86.c +58
-0+ uint32_t sreg, uint32_t mode, uint32_t sz, uint32_t imm, + const int32_t ofs[LDMB_OFS_NUM]) +{ + /* make R2 contain *off* value */ + + if (sreg != rg[EBPF_REG_2]) { + emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, +rg [EBPF_REG_2 ],imm ); +if (mode ==BPF_IND ) +emit_alu_reg (st,EBPF_ALU64 |BPF_ADD |BPF_X , + sreg, rg[EBPF_REG_2 ]); + } else +/* BPF_IND with sreg == R2 */ +emit_alu_imm (st,EBPF_ALU64 |BPF_ADD |BPF_K , + rg[EBPF_REG_2 ], imm); + +/* R3 = mbuf->data_len */ +emit_ld_reg (st,BPF_LDX |BPF_MEM |BPF_H , + rg[EBPF_REG_6 ], rg[EBPF_REG_3 ], + offsetof(struct rte_mbuf , data_len)); + +/* R3 = R3 - R2 */ +emit_alu_reg (st,EBPF_ALU64 |BPF_SUB |BPF_X , + rg[EBPF_REG_2 ], rg[EBPF_REG_3 ]); + +/* JSLE R3, <sz> <slow_path> */ +emit_cmp_imm (st,EBPF_ALU64 , rg[EBPF_REG_3 ], sz); +emit_abs_jcc (st,BPF_JMP |EBPF_JSLE |BPF_K , ofs[LDMB_SLP_OFS ]); + +/* R3 = mbuf->data_off */ +emit_ld_reg (st,BPF_LDX |BPF_MEM |BPF_H , + rg[EBPF_REG_6 ], rg[EBPF_REG_3 ], + offsetof(struct rte_mbuf , data_off)); + +/* R0 = mbuf->buf_addr */ +emit_ld_reg (st,BPF_LDX |BPF_MEM |EBPF_DW , + rg[EBPF_REG_6 ], rg[EBPF_REG_0 ], + offsetof(struct rte_mbuf , buf_addr)); + +/* R0 = R0 + R3 */ +emit_alu_reg (st,EBPF_ALU64 |BPF_ADD |BPF_X , + rg[EBPF_REG_3 ], rg[EBPF_REG_0 ]); + +/* R0 = R0 + R2 */ +emit_alu_reg (st,EBPF_ALU64 |BPF_ADD |BPF_X , + rg[EBPF_REG_2 ], rg[EBPF_REG_0 ]); + +/* JMP <fin_part> */ +emit_abs_jmp (st, ofs[LDMB_FIN_OFS ]); +} + +/* +* helper function, used by emit_ld_mbuf(). +* generates code for 'slow_path': +* call __rte_pktmbuf_read() and check return value. +*/ +static void +emit_ldmb_slow_path(struct bpf_jit_state *st, const uint32_trg[EBPF_REG_7],
lib/librte_bpf/bpf_jit_x86.c +37
-0+ uint32_t sz) +{ +/* make R3 contain *len* value (1/2/4) */ + +emit_mov_imm (st,EBPF_ALU64 |EBPF_MOV |BPF_K , rg[EBPF_REG_3 ], sz); + +/* make R4 contain (RBP - ldmb.stack_ofs) */ + +emit_mov_reg (st,EBPF_ALU64 |EBPF_MOV |BPF_X ,RBP , rg[EBPF_REG_4 ]); +emit_alu_imm (st,EBPF_ALU64 |BPF_SUB |BPF_K , rg[EBPF_REG_4 ], + st->ldmb.stack_ofs); + +/* make R1 contain mbuf ptr */ + +emit_mov_reg (st,EBPF_ALU64 |EBPF_MOV |BPF_X , + rg[EBPF_REG_6 ], rg[EBPF_REG_1 ]); + +/* call rte_pktmbuf_read */ +emit_call (st, (uintptr_t )__rte_pktmbuf_read); + +/* check that return value (R0) is not zero */ +emit_tst_reg (st,EBPF_ALU64 , rg[EBPF_REG_0 ], rg[EBPF_REG_0 ]); +emit_abs_jcc (st,BPF_JMP |BPF_JEQ |BPF_K , st->exit.off); +} + +/* +* helper function, used by emit_ld_mbuf(). +* generates final part of code for BPF_ABS/BPF_IND load: +* perform data load and endianness conversion. +* expects dreg to contain valid data pointer. +*/ +static void +emit_ldmb_fin (struct bpf_jit_state *st,uint32_t dreg,uint32_t opsz, +uint32_t sz) +{ +emit_ld_reg (st,BPF_LDX |BPF_MEM | opsz, dreg, dreg,0 ); +if (sz !=sizeof (uint8_t ))lib/librte_bpf/bpf_jit_x86.c emit_div(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg, · line 1149
Quoted 10 lines
lib/librte_bpf/bpf_jit_x86.c +10
-0+emit_be2le (st, dreg, sz *CHAR_BIT ); +} + +/* +* emit code for BPF_ABS/BPF_IND load. +* generates the following construction: +* fast_path: +* off = ins->sreg + ins->imm +* if (off + ins->opsz < mbuf->data_len) +* goto slow_path; I am not an eBPF expert, but I am not sure this is correct.
Quoted 5 lines
I think it should be > instead of <. Also, it looks like you actually emit:
if (mbuf->data_len- off <=ins->opsz)
gotoslow_path;Could you please double check that both the comment and the emitted code has the comparison turning the correct way.
Ack, should be:
if (mbuf->data_len - off < ins->opsz)
Will send v2.
v2 cover bpf: add support for BPF_ABS/BPF_IND instructions
In reply to Konstantin AnanyevTo fill the gap with linux kernel eBPF implementation,
add support for two non-generic instructions:
(BPF_ABS | <size> | BPF_LD) and (BPF_IND | <size> | BPF_LD) which are used to access packet data in a safe manner. Make necessary changes in BPF verifier, interpreter and x86 JIT code.
Konstantin Ananyev (5):
test/bpf: fix few small issues
bpf: fix add/sub min/max estimations
bpf: add support for packet data load instructions test/bpf: add new test cases for mbuf load instructions
bpf: x86 JIT support for packet data load instructions
app/test/test_bpf.c | 504 +++++++++++++++++++++++-- doc/guides/prog_guide/bpf_lib.rst | 30 +- doc/guides/rel_notes/release_20_08.rst | 7 + lib/librte_bpf/bpf_exec.c | 57 +++ lib/librte_bpf/bpf_jit_x86.c | 181 +++++++++ lib/librte_bpf/bpf_validate.c | 104 ++++- 6 files changed, 850 insertions(+), 33 deletions(-)
Signature · 1 line
2.17.1
PATCH v2 1/5 test/bpf: fix few small issues
Address for few small issues:
- unreachable return statement
- failed test-case can finish with 'success' status
Also use unified cmp_res() function to check return value.
--- app/test/test_bpf.c | 41 +++++++++++++++-------------------------- 1 file changed, 15 insertions(+), 26 deletions(-)
app/test/test_bpf.c +15 -26
@@ -1797,13 +1797,6 @@ test_call1_check(uint64_t rc, const void *arg)dummy_func1 (arg, &v32, &v64); v64 += v32; -if (v64 != rc) { -printf ("%s@%d: invalid return value " -"expected=0x%" PRIx64", actual=0x%" PRIx64"\n" , - __func__, __LINE__, v64, rc); -return -1 ; - } -return 0 ;return cmp_res (__func__, v64, rc, dv, dv,sizeof (*dv)); }
@@ -1934,13 +1927,7 @@ test_call2_check(uint64_t rc, const void *arg)dummy_func2 (&a, &b); v = a.u64 + a.u32 + b.u16 + b.u8; -if (v != rc) { -printf ("%s@%d: invalid return value " -"expected=0x%" PRIx64", actual=0x%" PRIx64"\n" , - __func__, __LINE__, v, rc); -return -1 ; - } -return 0 ; +return cmp_res (__func__, v, rc, arg, arg,0 ); }static const struct rte_bpf_xsym test_call2_xsym[] = {
@@ -2429,7 +2416,6 @@ test_call5_check(uint64_t rc, const void *arg) v =0 ; fail: -return cmp_res (__func__, v, rc, &v, &rc,sizeof (v)); }
@@ -2458,6 +2444,7 @@ static const struct rte_bpf_xsym test_call5_xsym[] = { }, }; +/* all bpf test cases */ static const struct bpf_test tests[] = { { .name ="test_store1" ,
@@ -2738,7 +2725,6 @@ run_test(const struct bpf_test *tst) } tst->prepare (tbuf); - rc =rte_bpf_exec (bpf, tbuf); ret = tst->check_result (rc, tbuf);if (ret !=0 ) {
@@ -2746,17 +2732,20 @@ run_test(const struct bpf_test *tst) __func__, __LINE__, tst->name, ret, strerror(ret)); } +/* repeat the same test with jit, when possible */ rte_bpf_get_jit (bpf, &jit); -if (jit.func ==NULL ) -return 0 ; - - tst->prepare (tbuf); - rc = jit.func (tbuf); - rv = tst->check_result (rc, tbuf); - ret |= rv; -if (rv !=0 ) { -printf ("%s@%d: check_result(%s) failed, error: %d(%s);\n" , - __func__, __LINE__, tst->name, rv,strerror (ret)); +if (jit.func !=NULL ) { + + tst->prepare (tbuf); + rc = jit.func (tbuf); + rv = tst->check_result (rc, tbuf); + ret |= rv; +if (rv !=0 ) { +printf ("%s@%d: check_result(%s) failed, " +"error: %d(%s);\n" , + __func__, __LINE__, tst->name, + rv,strerror (ret)); + } }rte_bpf_destroy (bpf);
Signature · 1 line
2.17.1
PATCH v2 2/5 bpf: fix add/sub min/max estimations
In reply to Konstantin Ananyeveval_add()/eval_sub() not always correctly estimate minimum and maximum possible values of add/sub operations.
--- lib/librte_bpf/bpf_validate.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)
lib/librte_bpf/bpf_validate.c +5 -5
@@ -226,7 +226,7 @@ eval_add(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, uint64_t msk)struct bpf_reg_val rv; rv.u.min = (rd->u.min + rs->u.min) & msk; - rv.u.max = (rd->u.min + rs->u.max) & msk; + rv.u.max = (rd->u.max + rs->u.max) & msk; rv.s.min = (rd->s.min + rs->s.min) & msk; rv.s.max = (rd->s.max + rs->s.max) & msk;
@@ -254,10 +254,10 @@ eval_sub(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, uint64_t msk) {struct bpf_reg_val rv; - rv.u.min = (rd->u.min - rs->u.min) & msk; - rv.u.max = (rd->u.min - rs->u.max) & msk; - rv.s.min = (rd->s.min - rs->s.min) & msk; - rv.s.max = (rd->s.max - rs->s.max) & msk; + rv.u.min = (rd->u.min - rs->u.max) & msk; + rv.u.max = (rd->u.max - rs->u.min) & msk; + rv.s.min = (rd->s.min - rs->s.max) & msk; + rv.s.max = (rd->s.max - rs->s.min) & msk;/* * if at least one of the operands is not constant,
Signature · 1 line
2.17.1
PATCH v2 3/5 bpf: add support for packet data load instructions
In reply to Konstantin AnanyevTo fill the gap with linux kernel eBPF implementation, add support for two non-generic instructions:
(BPF_ABS | <size> | BPF_LD) and (BPF_IND | <size> | BPF_LD) which are used to access packet data.
These instructions can only be used when BPF context is a pointer to 'struct rte_mbuf' (i.e: RTE_BPF_ARG_PTR_MBUF type).
--- doc/guides/prog_guide/bpf_lib.rst | 30 +++++++++- doc/guides/rel_notes/release_20_08.rst | 7 +++ lib/librte_bpf/bpf_exec.c | 57 +++++++++++++++++++ lib/librte_bpf/bpf_validate.c | 77 ++++++++++++++++++++++++++ 4 files changed, 170 insertions(+), 1 deletion(-)
doc/guides/prog_guide/bpf_lib.rst +29 -1
@@ -27,6 +27,35 @@ The library API provides the following basic operations: * Load BPF program from the ELF file and install callback to execute it on given ethdev port/queue. +Packet data load instructions +----------------------------- + +DPDK supports two non-generic instructions: ``(BPF_ABS | size | BPF_LD)`` +and ``(BPF_IND | size | BPF_LD)`` which are used to access packet data. +These instructions can only be used when execution context is a pointer to +``struct rte_mbuf`` and have seven implicit operands. +Register ``R6`` is an implicit input that must contain pointer to ``rte_mbuf``. +Register ``R0`` is an implicit output which contains the data fetched from the +packet. Registers ``R1-R5`` are scratch registers +and must not be used to store the data across these instructions. +These instructions have implicit program exit condition as well. When +eBPF program is trying to access the data beyond the packet boundary, +the interpreter will abort the execution of the program. JIT compilers +therefore must preserve this property. ``src_reg`` and ``imm32`` fields are +explicit inputs to these instructions. +For example, ``(BPF_IND | BPF_W | BPF_LD)`` means: + +.. code-block:: c + + uint32_t tmp; + R0 = rte_pktmbuf_read((const struct rte_mbuf *)R6, src_reg + imm32, + sizeof(tmp), &tmp); + if (R0 == NULL) return FAILED; + R0 = ntohl(*(uint32_t *)R0); + +and ``R1-R5`` were scratched. + + Not currently supported eBPF features -------------------------------------
@@ -34,5 +63,4 @@ Not currently supported eBPF features - cBPF - tail-pointer call - eBPF MAP - - skb - external function calls for 32-bit platforms
doc/guides/rel_notes/release_20_08.rst +7 -0
@@ -56,6 +56,13 @@ New Features Also, make sure to start the actual text at the margin. ========================================================= +* **Added support for BPF_ABS/BPF_IND load instructions.** + + Added support for two BPF non-generic instructions: + ``(BPF_ABS | <size> | BPF_LD)`` and ``(BPF_IND | <size> | BPF_LD)`` + which are used to access packet data in a safe manner. Currently JIT support + for these instructions is implemented for x86 only. + Removed Items -------------
lib/librte_bpf/bpf_exec.c +57 -0
75 lines not on this page · raw message
lib/librte_bpf/bpf_validate.c +77 -0
95 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH v2 4/5 test/bpf: add new test cases for mbuf load instructions
In reply to Konstantin AnanyevAdd new test-cases for BPF_ABS/BPF_IND load instructions.
--- app/test/test_bpf.c | 463 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 463 insertions(+)
app/test/test_bpf.c +463 -0
@@ -43,6 +43,14 @@ struct dummy_net {struct rte_ipv4_hdr ip_hdr; }; +#define DUMMY_MBUF_NUM 2 + +/* first mbuf in the packet, should always be at offset 0 */ +struct dummy_mbuf { +struct rte_mbuf mb[DUMMY_MBUF_NUM ]; +uint8_t buf[DUMMY_MBUF_NUM ][RTE_MBUF_DEFAULT_BUF_SIZE ]; +}; +#define TEST_FILL_1 0xDEADBEEF#define TEST_MUL_1 21
@@ -2444,6 +2452,413 @@ static const struct rte_bpf_xsym test_call5_xsym[] = { }, }; +/* load mbuf (BPF_ABS/BPF_IND) test-cases */ +static const struct ebpf_insn test_ld_mbuf1_prog[] = { + +/* BPF_ABS/BPF_IND implicitly expect mbuf ptr in R6 */ + { + .code = (EBPF_ALU64 |EBPF_MOV |BPF_X ), + .dst_reg =EBPF_REG_6 , + .src_reg =EBPF_REG_1 , + }, +/* load IPv4 version and IHL */ + { + .code = (BPF_LD |BPF_ABS |BPF_B ), + .imm = offsetof(struct rte_ipv4_hdr , version_ihl), + }, +/* check IP version */ + { + .code = (EBPF_ALU64 |EBPF_MOV |BPF_X ), + .dst_reg =EBPF_REG_2 , + .src_reg =EBPF_REG_0 , + }, + { + .code = (BPF_ALU |BPF_AND |BPF_K ), + .dst_reg =EBPF_REG_2 , + .imm =0xf0 , + }, + { + .code = (BPF_JMP |BPF_JEQ |BPF_K ), + .dst_reg =EBPF_REG_2 , + .imm =IPVERSION <<4 , + .off =2 , + }, +/* invalid IP version, return 0 */ + { + .code = (EBPF_ALU64 |BPF_XOR |BPF_X ), + .dst_reg =EBPF_REG_0 , + .src_reg =EBPF_REG_0 , + }, + { + .code = (BPF_JMP |EBPF_EXIT ), + }, +/* load 3-rd byte of IP data */ + { + .code = (BPF_ALU |BPF_AND |BPF_K ), + .dst_reg =EBPF_REG_0 , + .imm =RTE_IPV4_HDR_IHL_MASK , + }, + { + .code = (BPF_ALU |BPF_LSH |BPF_K ), + .dst_reg =EBPF_REG_0 , + .imm =2 , + }, + { + .code = (BPF_LD |BPF_IND |BPF_B ), + .src_reg =EBPF_REG_0 , + .imm =3 , + }, + { + .code = (EBPF_ALU64 |EBPF_MOV |BPF_X ), + .dst_reg =EBPF_REG_7 , + .src_reg =EBPF_REG_0 , + }, +/* load IPv4 src addr */ + { + .code = (BPF_LD |BPF_ABS |BPF_W ), + .imm = offsetof(struct rte_ipv4_hdr , src_addr), + }, + { + .code = (EBPF_ALU64 |BPF_ADD |BPF_X ), + .dst_reg =EBPF_REG_7 , + .src_reg =EBPF_REG_0 , + }, +/* load IPv4 total length */ + { + .code = (BPF_LD |BPF_ABS |BPF_H ), + .imm = offsetof(struct rte_ipv4_hdr , total_length), + }, + { + .code = (EBPF_ALU64 |EBPF_MOV |BPF_X ), + .dst_reg =EBPF_REG_8 , + .src_reg =EBPF_REG_0 , + }, +/* load last 4 bytes of IP data */ + { + .code = (BPF_LD |BPF_IND |BPF_W ), + .src_reg =EBPF_REG_8 , + .imm = -(int32_t )sizeof (uint32_t ), + }, + { + .code = (EBPF_ALU64 |BPF_ADD |BPF_X ), + .dst_reg =EBPF_REG_7 , + .src_reg =EBPF_REG_0 , + }, +/* load 2 bytes from the middle of IP data */ + { + .code = (EBPF_ALU64 |BPF_RSH |BPF_K ), + .dst_reg =EBPF_REG_8 , + .imm =1 , + }, + { + .code = (BPF_LD |BPF_IND |BPF_H ), + .src_reg =EBPF_REG_8 , + }, + { + .code = (EBPF_ALU64 |BPF_ADD |BPF_X ), + .dst_reg =EBPF_REG_0 , + .src_reg =EBPF_REG_7 , + }, + { + .code = (BPF_JMP |EBPF_EXIT ), + }, +}; + +static void +dummy_mbuf_prep (struct rte_mbuf *mb,uint8_t buf[],uint32_t buf_len, +uint32_t data_len) +{ +uint32_t i; +uint8_t *db; + + mb->buf_addr = buf; + mb->buf_iova = (uintptr_t )buf; + mb->buf_len = buf_len; +rte_mbuf_refcnt_set (mb,1 ); + +/* set pool pointer to dummy value, test doesn't use it */ + mb->pool = (void *)buf; + +rte_pktmbuf_reset (mb); + db = (uint8_t *)rte_pktmbuf_append (mb, data_len); + +for (i =0 ; i != data_len; i++) + db[i] = i; +} + +static void +test_ld_mbuf1_prepare (void *arg) +{ +struct dummy_mbuf *dm; +struct rte_ipv4_hdr *ph; + +const uint32_t plen =400 ; +const struct rte_ipv4_hdr iph = { + .version_ihl =RTE_IPV4_VHL_DEF , + .total_length =rte_cpu_to_be_16 (plen), + .time_to_live =IPDEFTTL , + .next_proto_id =IPPROTO_RAW , + .src_addr =rte_cpu_to_be_32 (RTE_IPV4_LOOPBACK ), + .dst_addr =rte_cpu_to_be_32 (RTE_IPV4_BROADCAST ), + }; + + dm = arg; +memset (dm,0 ,sizeof (*dm)); + +dummy_mbuf_prep (&dm->mb[0 ], dm->buf[0 ],sizeof (dm->buf[0 ]), + plen /2 +1 ); +dummy_mbuf_prep (&dm->mb[1 ], dm->buf[1 ],sizeof (dm->buf[0 ]), + plen /2 -1 ); + +rte_pktmbuf_chain (&dm->mb[0 ], &dm->mb[1 ]); + + ph =rte_pktmbuf_mtod (dm->mb,typeof (ph)); +memcpy (ph, &iph,sizeof (iph)); +} + +static uint64_t +test_ld_mbuf1 (const struct rte_mbuf *pkt) +{ +uint64_t n, v; +const uint8_t *p8; +const uint16_t *p16; +const uint32_t *p32; +struct dummy_offset dof; + +/* load IPv4 version and IHL */ + p8 =rte_pktmbuf_read (pkt, + offsetof(struct rte_ipv4_hdr , version_ihl),sizeof (*p8), + &dof); +if (p8 ==NULL ) +return 0 ; + +/* check IP version */ +if ((p8[0 ] &0xf0 ) !=IPVERSION <<4 ) +return 0 ; + + n = (p8[0 ] &RTE_IPV4_HDR_IHL_MASK ) *RTE_IPV4_IHL_MULTIPLIER ; + +/* load 3-rd byte of IP data */ + p8 =rte_pktmbuf_read (pkt, n +3 ,sizeof (*p8), &dof); +if (p8 ==NULL ) +return 0 ; + + v = p8[0 ]; + +/* load IPv4 src addr */ + p32 =rte_pktmbuf_read (pkt, + offsetof(struct rte_ipv4_hdr , src_addr),sizeof (*p32), + &dof); +if (p32 ==NULL ) +return 0 ; + + v +=rte_be_to_cpu_32 (p32[0 ]); + +/* load IPv4 total length */ + p16 =rte_pktmbuf_read (pkt, + offsetof(struct rte_ipv4_hdr , total_length),sizeof (*p16), + &dof); +if (p16 ==NULL ) +return 0 ; + + n =rte_be_to_cpu_16 (p16[0 ]); + +/* load last 4 bytes of IP data */ + p32 =rte_pktmbuf_read (pkt, n -sizeof (*p32),sizeof (*p32), &dof); +if (p32 ==NULL ) +return 0 ; + + v +=rte_be_to_cpu_32 (p32[0 ]); + +/* load 2 bytes from the middle of IP data */ + p16 =rte_pktmbuf_read (pkt, n /2 ,sizeof (*p16), &dof); +if (p16 ==NULL ) +return 0 ; + + v +=rte_be_to_cpu_16 (p16[0 ]); +return v; +} + +static int +test_ld_mbuf1_check (uint64_t rc,const void *arg) +{ +const struct dummy_mbuf *dm; +uint64_t v; + + dm = arg; + v =test_ld_mbuf1 (dm->mb); +return cmp_res (__func__, v, rc, arg, arg,0 ); +} + +/* +* same as ld_mbuf1, but then trancate the mbuf by 1B, +* so load of last 4B fail. +*/ +static void +test_ld_mbuf2_prepare (void *arg) +{ +struct dummy_mbuf *dm; + +test_ld_mbuf1_prepare (arg); + dm = arg; +rte_pktmbuf_trim (dm->mb,1 ); +} + +static int +test_ld_mbuf2_check (uint64_t rc,const void *arg) +{ +return cmp_res (__func__,0 , rc, arg, arg,0 ); +} + +/* same as test_ld_mbuf1, but now store intermediate results on the stack */ +static const struct ebpf_insn test_ld_mbuf3_prog[] = { + +/* BPF_ABS/BPF_IND implicitly expect mbuf ptr in R6 */ + { + .code = (EBPF_ALU64 |EBPF_MOV |BPF_X ), + .dst_reg =EBPF_REG_6 , + .src_reg =EBPF_REG_1 , + }, +/* load IPv4 version and IHL */ + { + .code = (BPF_LD |BPF_ABS |BPF_B ), + .imm = offsetof(struct rte_ipv4_hdr , version_ihl), + }, +/* check IP version */ + { + .code = (EBPF_ALU64 |EBPF_MOV |BPF_X ), + .dst_reg =EBPF_REG_2 , + .src_reg =EBPF_REG_0 , + }, + { + .code = (BPF_ALU |BPF_AND |BPF_K ), + .dst_reg =EBPF_REG_2 , + .imm =0xf0 , + }, + { + .code = (BPF_JMP |BPF_JEQ |BPF_K ), + .dst_reg =EBPF_REG_2 , + .imm =IPVERSION <<4 , + .off =2 , + }, +/* invalid IP version, return 0 */ + { + .code = (EBPF_ALU64 |BPF_XOR |BPF_X ), + .dst_reg =EBPF_REG_0 , + .src_reg =EBPF_REG_0 , + }, + { + .code = (BPF_JMP |EBPF_EXIT ), + }, +/* load 3-rd byte of IP data */ + { + .code = (BPF_ALU |BPF_AND |BPF_K ), + .dst_reg =EBPF_REG_0 , + .imm =RTE_IPV4_HDR_IHL_MASK , + }, + { + .code = (BPF_ALU |BPF_LSH |BPF_K ), + .dst_reg =EBPF_REG_0 , + .imm =2 , + }, + { + .code = (BPF_LD |BPF_IND |BPF_B ), + .src_reg =EBPF_REG_0 , + .imm =3 , + }, + { + .code = (BPF_STX |BPF_MEM |BPF_B ), + .dst_reg =EBPF_REG_10 , + .src_reg =EBPF_REG_0 , + .off = (int16_t )(offsetof(struct dummy_offset , u8) - +sizeof (struct dummy_offset )), + }, +/* load IPv4 src addr */ + { + .code = (BPF_LD |BPF_ABS |BPF_W ), + .imm = offsetof(struct rte_ipv4_hdr , src_addr), + }, + { + .code = (BPF_STX |BPF_MEM |BPF_W ), + .dst_reg =EBPF_REG_10 , + .src_reg =EBPF_REG_0 , + .off = (int16_t )(offsetof(struct dummy_offset , u32) - +sizeof (struct dummy_offset )), + }, +/* load IPv4 total length */ + { + .code = (BPF_LD |BPF_ABS |BPF_H ), + .imm = offsetof(struct rte_ipv4_hdr , total_length), + }, + { + .code = (EBPF_ALU64 |EBPF_MOV |BPF_X ), + .dst_reg =EBPF_REG_8 , + .src_reg =EBPF_REG_0 , + }, +/* load last 4 bytes of IP data */ + { + .code = (BPF_LD |BPF_IND |BPF_W ), + .src_reg =EBPF_REG_8 , + .imm = -(int32_t )sizeof (uint32_t ), + }, + { + .code = (BPF_STX |BPF_MEM |EBPF_DW ), + .dst_reg =EBPF_REG_10 , + .src_reg =EBPF_REG_0 , + .off = (int16_t )(offsetof(struct dummy_offset , u64) - +sizeof (struct dummy_offset )), + }, +/* load 2 bytes from the middle of IP data */ + { + .code = (EBPF_ALU64 |BPF_RSH |BPF_K ), + .dst_reg =EBPF_REG_8 , + .imm =1 , + }, + { + .code = (BPF_LD |BPF_IND |BPF_H ), + .src_reg =EBPF_REG_8 , + }, + { + .code = (BPF_LDX |BPF_MEM |EBPF_DW ), + .dst_reg =EBPF_REG_1 , + .src_reg =EBPF_REG_10 , + .off = (int16_t )(offsetof(struct dummy_offset , u64) - +sizeof (struct dummy_offset )), + }, + { + .code = (EBPF_ALU64 |BPF_ADD |BPF_X ), + .dst_reg =EBPF_REG_0 , + .src_reg =EBPF_REG_1 , + }, + { + .code = (BPF_LDX |BPF_MEM |BPF_W ), + .dst_reg =EBPF_REG_1 , + .src_reg =EBPF_REG_10 , + .off = (int16_t )(offsetof(struct dummy_offset , u32) - +sizeof (struct dummy_offset )), + }, + { + .code = (EBPF_ALU64 |BPF_ADD |BPF_X ), + .dst_reg =EBPF_REG_0 , + .src_reg =EBPF_REG_1 , + }, + { + .code = (BPF_LDX |BPF_MEM |BPF_B ), + .dst_reg =EBPF_REG_1 , + .src_reg =EBPF_REG_10 , + .off = (int16_t )(offsetof(struct dummy_offset , u8) - +sizeof (struct dummy_offset )), + }, + { + .code = (EBPF_ALU64 |BPF_ADD |BPF_X ), + .dst_reg =EBPF_REG_0 , + .src_reg =EBPF_REG_1 , + }, + { + .code = (BPF_JMP |EBPF_EXIT ), + }, +}; +/* all bpf test cases */ static const struct bpf_test tests[] = { {
@@ -2704,6 +3119,54 @@ static const struct bpf_test tests[] = {/* for now don't support function calls on 32 bit platform */ .allow_fail = (sizeof(uint64_t) != sizeof(uintptr_t)), }, + { + .name = "test_ld_mbuf1", + .arg_sz = sizeof(struct dummy_mbuf), + .prm = { + .ins = test_ld_mbuf1_prog, + .nb_ins = RTE_DIM(test_ld_mbuf1_prog), + .prog_arg = { + .type = RTE_BPF_ARG_PTR_MBUF, + .buf_size =sizeof (struct dummy_mbuf ), + }, + }, + .prepare = test_ld_mbuf1_prepare, + .check_result = test_ld_mbuf1_check, +/* mbuf as input argument is not supported on 32 bit platform */ + .allow_fail = (sizeof(uint64_t) != sizeof(uintptr_t)), + }, + { + .name ="test_ld_mbuf2" , + .arg_sz = sizeof(struct dummy_mbuf), + .prm = { + .ins = test_ld_mbuf1_prog, + .nb_ins = RTE_DIM(test_ld_mbuf1_prog), + .prog_arg = { + .type =RTE_BPF_ARG_PTR_MBUF , + .buf_size = sizeof(struct dummy_mbuf), + }, + }, + .prepare = test_ld_mbuf2_prepare, + .check_result = test_ld_mbuf2_check, +/* mbuf as input argument is not supported on 32 bit platform */ + .allow_fail = (sizeof(uint64_t) != sizeof(uintptr_t)), + }, + { + .name ="test_ld_mbuf3" , + .arg_sz = sizeof(struct dummy_mbuf), + .prm = { + .ins = test_ld_mbuf3_prog, + .nb_ins = RTE_DIM(test_ld_mbuf3_prog), + .prog_arg = { + .type =RTE_BPF_ARG_PTR_MBUF , + .buf_size = sizeof(struct dummy_mbuf), + }, + }, + .prepare = test_ld_mbuf1_prepare, + .check_result = test_ld_mbuf1_check, +/* mbuf as input argument is not supported on 32 bit platform */ + .allow_fail = (sizeof(uint64_t) != sizeof(uintptr_t)), + }, }; static int
Signature · 1 line
2.17.1
PATCH v2 5/5 bpf: x86 JIT support for packet data load instructions
In reply to Konstantin AnanyevMake x86 JIT to generate native code for
(BPF_ABS | <size> | BPF_LD) and (BPF_IND | <size> | BPF_LD) instructions.
--- lib/librte_bpf/bpf_jit_x86.c | 181 ++++++++++++++++++++++++++++++++++ lib/librte_bpf/bpf_validate.c | 17 +++- 2 files changed, 197 insertions(+), 1 deletion(-)
lib/librte_bpf/bpf_jit_x86.c +181 -0
@@ -87,6 +87,14 @@ enum { REG_TMP1 = R10, }; +/* LD_ABS/LD_IMM offsets */ +enum { +LDMB_FSP_OFS ,/* fast-path */ +LDMB_SLP_OFS ,/* slow-path */ +LDMB_FIN_OFS ,/* final part */ +LDMB_OFS_NUM +}; +/* * callee saved registers list. * keep RBP as the last one.
@@ -100,6 +108,9 @@ struct bpf_jit_state {uint32_t num;int32_t off; } exit; +struct { +uint32_t stack_ofs; + } ldmb;uint32_t reguse;int32_t *off;uint8_t *ins;
@@ -1024,6 +1035,166 @@ emit_div(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,emit_mov_reg (st,EBPF_ALU64 |EBPF_MOV |BPF_X ,REG_TMP1 ,RDX ); } +/* +* helper function, used by emit_ld_mbuf(). +* generates code for 'fast_path': +* calculate load offset and check is it inside first packet segment. +*/ +static void +emit_ldmb_fast_path (struct bpf_jit_state *st,const uint32_t rg[EBPF_REG_7 ], +uint32_t sreg,uint32_t mode,uint32_t sz,uint32_t imm, +const int32_t ofs[LDMB_OFS_NUM ]) +{ +/* make R2 contain *off* value */ + +if (sreg != rg[EBPF_REG_2 ]) { +emit_mov_imm (st,EBPF_ALU64 |EBPF_MOV |BPF_K , + rg[EBPF_REG_2 ], imm); +if (mode ==BPF_IND ) +emit_alu_reg (st,EBPF_ALU64 |BPF_ADD |BPF_X , + sreg, rg[EBPF_REG_2 ]); + }else +/* BPF_IND with sreg == R2 */ +emit_alu_imm (st,EBPF_ALU64 |BPF_ADD |BPF_K , + rg[EBPF_REG_2 ], imm); + +/* R3 = mbuf->data_len */ +emit_ld_reg (st,BPF_LDX |BPF_MEM |BPF_H , + rg[EBPF_REG_6 ], rg[EBPF_REG_3 ], + offsetof(struct rte_mbuf , data_len)); + +/* R3 = R3 - R2 */ +emit_alu_reg (st,EBPF_ALU64 |BPF_SUB |BPF_X , + rg[EBPF_REG_2 ], rg[EBPF_REG_3 ]); + +/* JSLT R3, <sz> <slow_path> */ +emit_cmp_imm (st,EBPF_ALU64 , rg[EBPF_REG_3 ], sz); +emit_abs_jcc (st,BPF_JMP |EBPF_JSLT |BPF_K , ofs[LDMB_SLP_OFS ]); + +/* R3 = mbuf->data_off */ +emit_ld_reg (st,BPF_LDX |BPF_MEM |BPF_H , + rg[EBPF_REG_6 ], rg[EBPF_REG_3 ], + offsetof(struct rte_mbuf , data_off)); + +/* R0 = mbuf->buf_addr */ +emit_ld_reg (st,BPF_LDX |BPF_MEM |EBPF_DW , + rg[EBPF_REG_6 ], rg[EBPF_REG_0 ], + offsetof(struct rte_mbuf , buf_addr)); + +/* R0 = R0 + R3 */ +emit_alu_reg (st,EBPF_ALU64 |BPF_ADD |BPF_X , + rg[EBPF_REG_3 ], rg[EBPF_REG_0 ]); + +/* R0 = R0 + R2 */ +emit_alu_reg (st,EBPF_ALU64 |BPF_ADD |BPF_X , + rg[EBPF_REG_2 ], rg[EBPF_REG_0 ]); + +/* JMP <fin_part> */ +emit_abs_jmp (st, ofs[LDMB_FIN_OFS ]); +} + +/* +* helper function, used by emit_ld_mbuf(). +* generates code for 'slow_path': +* call __rte_pktmbuf_read() and check return value. +*/ +static void +emit_ldmb_slow_path (struct bpf_jit_state *st,const uint32_t rg[EBPF_REG_7 ], +uint32_t sz) +{ +/* make R3 contain *len* value (1/2/4) */ + +emit_mov_imm (st,EBPF_ALU64 |EBPF_MOV |BPF_K , rg[EBPF_REG_3 ], sz); + +/* make R4 contain (RBP - ldmb.stack_ofs) */ + +emit_mov_reg (st,EBPF_ALU64 |EBPF_MOV |BPF_X ,RBP , rg[EBPF_REG_4 ]); +emit_alu_imm (st,EBPF_ALU64 |BPF_SUB |BPF_K , rg[EBPF_REG_4 ], + st->ldmb.stack_ofs); + +/* make R1 contain mbuf ptr */ + +emit_mov_reg (st,EBPF_ALU64 |EBPF_MOV |BPF_X , + rg[EBPF_REG_6 ], rg[EBPF_REG_1 ]); + +/* call rte_pktmbuf_read */ +emit_call (st, (uintptr_t )__rte_pktmbuf_read); + +/* check that return value (R0) is not zero */ +emit_tst_reg (st,EBPF_ALU64 , rg[EBPF_REG_0 ], rg[EBPF_REG_0 ]); +emit_abs_jcc (st,BPF_JMP |BPF_JEQ |BPF_K , st->exit.off); +} + +/* +* helper function, used by emit_ld_mbuf(). +* generates final part of code for BPF_ABS/BPF_IND load: +* perform data load and endianness conversion. +* expects dreg to contain valid data pointer. +*/ +static void +emit_ldmb_fin (struct bpf_jit_state *st,uint32_t dreg,uint32_t opsz, +uint32_t sz) +{ +emit_ld_reg (st,BPF_LDX |BPF_MEM | opsz, dreg, dreg,0 ); +if (sz !=sizeof (uint8_t )) +emit_be2le (st, dreg, sz *CHAR_BIT ); +} + +/* +* emit code for BPF_ABS/BPF_IND load. +* generates the following construction: +* fast_path: +* off = ins->sreg + ins->imm +* if (mbuf->data_len - off < ins->opsz) +* goto slow_path; +* ptr = mbuf->buf_addr + mbuf->data_off + off; +* goto fin_part; +* slow_path: +* typeof(ins->opsz) buf; //allocate space on the stack +* ptr = __rte_pktmbuf_read(mbuf, off, ins->opsz, &buf); +* if (ptr == NULL) +* goto exit_label; +* fin_part: +* res = *(typeof(ins->opsz))ptr; +* res = bswap(res); +*/ +static void +emit_ld_mbuf (struct bpf_jit_state *st,uint32_t op,uint32_t sreg,uint32_t imm) +{ +uint32_t i, mode, opsz, sz; +uint32_t rg[EBPF_REG_7 ]; +int32_t ofs[LDMB_OFS_NUM ]; + + mode =BPF_MODE (op); + opsz =BPF_SIZE (op); + sz =bpf_size (opsz); + +for (i =0 ; i !=RTE_DIM (rg); i++) + rg[i] = ebpf2x86[i]; + +/* fill with fake offsets */ +for (i =0 ; i !=RTE_DIM (ofs); i++) + ofs[i] = st->sz +INT8_MAX ; + +/* dry run first to calculate jump offsets */ + + ofs[LDMB_FSP_OFS ] = st->sz; +emit_ldmb_fast_path (st, rg, sreg, mode, sz, imm, ofs); + ofs[LDMB_SLP_OFS ] = st->sz; +emit_ldmb_slow_path (st, rg, sz); + ofs[LDMB_FIN_OFS ] = st->sz; +emit_ldmb_fin (st, rg[EBPF_REG_0 ], opsz, sz); + +RTE_VERIFY (ofs[LDMB_FIN_OFS ] - ofs[LDMB_FSP_OFS ] <=INT8_MAX ); + +/* reset dry-run code and do a proper run */ + + st->sz = ofs[LDMB_FSP_OFS ]; +emit_ldmb_fast_path (st, rg, sreg, mode, sz, imm, ofs); +emit_ldmb_slow_path (st, rg, sz); +emit_ldmb_fin (st, rg[EBPF_REG_0 ], opsz, sz); +} +static void emit_prolog (struct bpf_jit_state *st,int32_t stack_size) {
@@ -1121,6 +1292,7 @@ emit(struct bpf_jit_state *st, const struct rte_bpf *bpf)/* reset state fields */ st->sz =0 ; st->exit.num =0 ; + st->ldmb.stack_ofs = bpf->stack_sz;emit_prolog (st, bpf->stack_sz);
@@ -1240,6 +1412,15 @@ emit(struct bpf_jit_state *st, const struct rte_bpf *bpf)emit_ld_imm64 (st, dr, ins[0 ].imm, ins[1 ].imm); i++;break ; +/* load absolute/indirect instructions */ +case (BPF_LD |BPF_ABS |BPF_B ): +case (BPF_LD |BPF_ABS |BPF_H ): +case (BPF_LD |BPF_ABS |BPF_W ): +case (BPF_LD |BPF_IND |BPF_B ): +case (BPF_LD |BPF_IND |BPF_H ): +case (BPF_LD |BPF_IND |BPF_W ): +emit_ld_mbuf (st, op, sr, ins->imm); +break ;/* store instructions */ case (BPF_STX |BPF_MEM |BPF_B ):case (BPF_STX |BPF_MEM |BPF_H ):
lib/librte_bpf/bpf_validate.c +16 -1
@@ -70,6 +70,7 @@ struct bpf_verifier {uint64_t stack_sz;uint32_t nb_nodes;uint32_t nb_jcc_nodes; +uint32_t nb_ldmb_nodes;uint32_t node_colour[MAX_NODE_COLOUR ];uint32_t edge_type[MAX_EDGE_TYPE ];struct bpf_eval_state *evst;
@@ -2020,6 +2021,14 @@ validate(struct bpf_verifier *bvf) rc |=add_edge (bvf, node, i +2 ); i++;break ; +case (BPF_LD |BPF_ABS |BPF_B ): +case (BPF_LD |BPF_ABS |BPF_H ): +case (BPF_LD |BPF_ABS |BPF_W ): +case (BPF_LD |BPF_IND |BPF_B ): +case (BPF_LD |BPF_IND |BPF_H ): +case (BPF_LD |BPF_IND |BPF_W ): + bvf->nb_ldmb_nodes++; +/* fallthrough */ default : rc |=add_edge (bvf, node, i +1 );break ;
@@ -2320,8 +2329,14 @@ bpf_validate(struct rte_bpf *bpf)free (bvf.in);/* copy collected info */ -if (rc ==0 ) +if (rc ==0 ) { bpf->stack_sz = bvf.stack_sz; +/* for LD_ABS/LD_IND, we'll need extra space on the stack */ +if (bvf.nb_ldmb_nodes !=0 ) + bpf->stack_sz =RTE_ALIGN_CEIL (bpf->stack_sz + +sizeof (uint64_t ),sizeof (uint64_t )); + } +return rc; }
Signature · 1 line
2.17.1
Re: [dpdk-dev] [PATCH v2 5/5] bpf: x86 JIT support for packet data load instructions
Quoted 9 lines
From: Konstantin Ananyev [mailto:[email protected]]
Sent: Wednesday, May 27, 2020 4:17 PMMake x86 JIT to generate native code for
(BPF_ABS| <size> |BPF_LD) and (BPF_IND| <size> |BPF_LD) instructions.Signed-off-by Konstantin Ananyev <[email protected]>---
[...]
Quoted 37 lines
lib/librte_bpf/bpf_jit_x86.c +7
-0+/* +* helper function, used by emit_ld_mbuf(). +* generates code for 'fast_path': +* calculate load offset and check is it inside first packet segment. +*/ +static void +emit_ldmb_fast_path(struct bpf_jit_state *st, const uint32_trg[EBPF_REG_7],
lib/librte_bpf/bpf_jit_x86.c +29
-0+ uint32_t sreg, uint32_t mode, uint32_t sz, uint32_t imm, + const int32_t ofs[LDMB_OFS_NUM]) +{ + /* make R2 contain *off* value */ + + if (sreg != rg[EBPF_REG_2]) { + emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, +rg [EBPF_REG_2 ],imm ); +if (mode ==BPF_IND ) +emit_alu_reg (st,EBPF_ALU64 |BPF_ADD |BPF_X , + sreg, rg[EBPF_REG_2 ]); + } else +/* BPF_IND with sreg == R2 */ + emit_alu_imm(st, EBPF_ALU64 | BPF_ADD | BPF_K, +rg [EBPF_REG_2 ],imm ); + +/* R3 = mbuf->data_len */ +emit_ld_reg (st,BPF_LDX |BPF_MEM |BPF_H , + rg[EBPF_REG_6 ], rg[EBPF_REG_3 ], + offsetof(struct rte_mbuf , data_len)); + +/* R3 = R3 - R2 */ +emit_alu_reg (st,EBPF_ALU64 |BPF_SUB |BPF_X , + rg[EBPF_REG_2 ], rg[EBPF_REG_3 ]); + +/* JSLT R3, <sz> <slow_path> */ +emit_cmp_imm (st,EBPF_ALU64 , rg[EBPF_REG_3 ], sz); +emit_abs_jcc (st,BPF_JMP |EBPF_JSLT |BPF_K , ofs[LDMB_SLP_OFS ]); +
[...]
Quoted 9 lines
lib/librte_bpf/bpf_jit_x86.c +9
-0+ +/* +* emit code for BPF_ABS/BPF_IND load. +* generates the following construction: +* fast_path: +* off = ins->sreg + ins->imm +* if (mbuf->data_len - off < ins->opsz) +* goto slow_path; +* ptr = mbuf->buf_addr + mbuf->data_off + off;
lib/librte_bpf/bpf_jit_x86.c emit_div(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg, · line 1160
Quoted 10 lines
lib/librte_bpf/bpf_jit_x86.c +10
-0+* goto fin_part; +* slow_path: +* typeof(ins->opsz) buf; //allocate space on the stack +* ptr = __rte_pktmbuf_read(mbuf, off, ins->opsz, &buf); +* if (ptr == NULL) +* goto exit_label; +* fin_part: +* res = *(typeof(ins->opsz))ptr; +* res = bswap(res); +*/
[...]
The comparison for jumping to the slow path looks correct now.
I haven't reviewed it all in depth, but it certainly deserves an:
Thomas Monjalon
Applied27/05/2020 16:16, Konstantin Ananyev:
Quoted 6 lines
To fill the gap with linux kernel eBPF implementation,
add support for two non-generic instructions:
(BPF_ABS| <size> |BPF_LD) and (BPF_IND| <size> |BPF_LD) which are used to access packet data in a safe manner. Make necessary changes in BPF verifier, interpreter and x86 JIT code.
Quoted 6 lines
Konstantin Ananyev (5):
test/bpf: fix few small issues
bpf: fix add/sub min/max estimations
bpf: add support for packet data load instructions test/bpf: add new test cases for mbuf load instructions
bpf: x86 JIT support for packet data load instructions
Applied, thanks