Skip to contentSkip to conversation

bpf: add support for BPF_ABS/BPF_IND instructions

PATCH Superseded Konstantin Ananyev 3 participants 17 messages May 18, 2020 – Jun 24, 2020

lore
Share & tools
lore
#0 lore raw

cover bpf: add support for BPF_ABS/BPF_IND instructions

To fill the gap with linux kernel eBPF implementation,
add support for two non-generic instructions:
(BPF_ABS | <size> | BPF_LD) and (BPF_IND | <size> | BPF_LD) which are used to access packet data.
Make necessary changes in BPF verifier, interpreter and x86 JIT code.

Konstantin Ananyev (5):
test/bpf: fix few small issues
bpf: fix add/sub min/max estimations
bpf: add support for packet data load instructions test/bpf: add new test cases for mbuf load instructions
bpf: x86 JIT support for packet data load instructions

 app/test/test_bpf.c               | 504 ++++++++++++++++++++++++++++--
 doc/guides/prog_guide/bpf_lib.rst |  30 +-
 lib/librte_bpf/bpf_exec.c         |  57 ++++
 lib/librte_bpf/bpf_jit_x86.c      | 181 +++++++++++
 lib/librte_bpf/bpf_validate.c     | 104 +++++-
 5 files changed, 843 insertions(+), 33 deletions(-)
Signature · 1 line
2.17.1
#1 lore raw

PATCH 1/5 test/bpf: fix few small issues

Address for few small issues:
- unreachable return statement
- failed test-case can finish with 'success' status

Also use unified cmp_res() function to check return value.

Fixes a9de470cc7c0 ("test: move to app directory")
---
 app/test/test_bpf.c | 41 +++++++++++++++--------------------------
 1 file changed, 15 insertions(+), 26 deletions(-)
app/test/test_bpf.c +15 -26
index ee534687a..4a61a7d7c 100644
@@ -1797,13 +1797,6 @@ test_call1_check(uint64_t rc, const void *arg)
 	dummy_func1(arg, &v32, &v64);
 	v64 += v32;
 
-	if (v64 != rc) {
-		printf("%s@%d: invalid return value "
-			"expected=0x%" PRIx64 ", actual=0x%" PRIx64 "\n",
-			__func__, __LINE__, v64, rc);
-		return -1;
-	}
-	return 0;
 	return cmp_res(__func__, v64, rc, dv, dv, sizeof(*dv));
 }
 
@@ -1934,13 +1927,7 @@ test_call2_check(uint64_t rc, const void *arg)
 	dummy_func2(&a, &b);
 	v = a.u64 + a.u32 + b.u16 + b.u8;
 
-	if (v != rc) {
-		printf("%s@%d: invalid return value "
-			"expected=0x%" PRIx64 ", actual=0x%" PRIx64 "\n",
-			__func__, __LINE__, v, rc);
-		return -1;
-	}
-	return 0;
+	return cmp_res(__func__, v, rc, arg, arg, 0);
 }
 
 static const struct rte_bpf_xsym test_call2_xsym[] = {
@@ -2429,7 +2416,6 @@ test_call5_check(uint64_t rc, const void *arg)
 	v = 0;
 
 fail:
-
 	return cmp_res(__func__, v, rc, &v, &rc, sizeof(v));
 }
 
@@ -2458,6 +2444,7 @@ static const struct rte_bpf_xsym test_call5_xsym[] = {
 	},
 };
 
+/* all bpf test cases */
 static const struct bpf_test tests[] = {
 	{
 		.name = "test_store1",
@@ -2738,7 +2725,6 @@ run_test(const struct bpf_test *tst)
 	}
 
 	tst->prepare(tbuf);
-
 	rc = rte_bpf_exec(bpf, tbuf);
 	ret = tst->check_result(rc, tbuf);
 	if (ret != 0) {
@@ -2746,17 +2732,20 @@ run_test(const struct bpf_test *tst)
 			__func__, __LINE__, tst->name, ret, strerror(ret));
 	}
 
+	/* repeat the same test with jit, when possible */
 	rte_bpf_get_jit(bpf, &jit);
-	if (jit.func == NULL)
-		return 0;
-
-	tst->prepare(tbuf);
-	rc = jit.func(tbuf);
-	rv = tst->check_result(rc, tbuf);
-	ret |= rv;
-	if (rv != 0) {
-		printf("%s@%d: check_result(%s) failed, error: %d(%s);\n",
-			__func__, __LINE__, tst->name, rv, strerror(ret));
+	if (jit.func != NULL) {
+
+		tst->prepare(tbuf);
+		rc = jit.func(tbuf);
+		rv = tst->check_result(rc, tbuf);
+		ret |= rv;
+		if (rv != 0) {
+			printf("%s@%d: check_result(%s) failed, "
+				"error: %d(%s);\n",
+				__func__, __LINE__, tst->name,
+				rv, strerror(ret));
+		}
 	}
 
 	rte_bpf_destroy(bpf);
Signature · 1 line
2.17.1
#2 lore raw

Re: [dpdk-dev] [dpdk-stable] [PATCH 1/5] test/bpf: fix few small issues

18/05/2020 17:52, Konstantin Ananyev:

Quoted 7 lines

Address for few small issues:
- unreachable return statement
- failed test-case can finish with 'success' status

Also use unified cmp_res() function to check return value.

Fixes a9de470cc7c0 ("test: move to app directory")

Looks like you don't want to track the original commits. In this case, better to not add any fake reference.

Quoted 3 lines
#3 lore raw

PATCH 2/5 bpf: fix add/sub min/max estimations

In reply to Konstantin Ananyev

eval_add()/eval_sub() not always correctly estimate minimum and maximum possible values of add/sub operations.

Fixes 8021917293d0 ("bpf: add extra validation for input BPF program")
---
 lib/librte_bpf/bpf_validate.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)
lib/librte_bpf/bpf_validate.c +5 -5
index 6bd6f78e9..80d21fabb 100644
@@ -226,7 +226,7 @@ eval_add(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, uint64_t msk)
 	struct bpf_reg_val rv;
 
 	rv.u.min = (rd->u.min + rs->u.min) & msk;
-	rv.u.max = (rd->u.min + rs->u.max) & msk;
+	rv.u.max = (rd->u.max + rs->u.max) & msk;
 	rv.s.min = (rd->s.min + rs->s.min) & msk;
 	rv.s.max = (rd->s.max + rs->s.max) & msk;
 
@@ -254,10 +254,10 @@ eval_sub(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, uint64_t msk)
 {
 	struct bpf_reg_val rv;
 
-	rv.u.min = (rd->u.min - rs->u.min) & msk;
-	rv.u.max = (rd->u.min - rs->u.max) & msk;
-	rv.s.min = (rd->s.min - rs->s.min) & msk;
-	rv.s.max = (rd->s.max - rs->s.max) & msk;
+	rv.u.min = (rd->u.min - rs->u.max) & msk;
+	rv.u.max = (rd->u.max - rs->u.min) & msk;
+	rv.s.min = (rd->s.min - rs->s.max) & msk;
+	rv.s.max = (rd->s.max - rs->s.min) & msk;
 
 	/*
 	 * if at least one of the operands is not constant,
Signature · 1 line
2.17.1
#4 lore raw

PATCH 3/5 bpf: add support for packet data load instructions

In reply to Konstantin Ananyev

To fill the gap with linux kernel eBPF implementation, add support for two non-generic instructions:
(BPF_ABS | <size> | BPF_LD) and (BPF_IND | <size> | BPF_LD) which are used to access packet data.
These instructions can only be used when BPF context is a pointer to 'struct rte_mbuf' (i.e: RTE_BPF_ARG_PTR_MBUF type).

---
 doc/guides/prog_guide/bpf_lib.rst | 30 +++++++++++-
 lib/librte_bpf/bpf_exec.c         | 57 +++++++++++++++++++++++
 lib/librte_bpf/bpf_validate.c     | 77 +++++++++++++++++++++++++++++++
 3 files changed, 163 insertions(+), 1 deletion(-)
doc/guides/prog_guide/bpf_lib.rst +29 -1
index 9c728da7b..1feb7734a 100644
@@ -27,6 +27,35 @@ The library API provides the following basic operations:
 
 *   Load BPF program from the ELF file and install callback to execute it on given ethdev port/queue.
 
+Packet data load instructions
+-----------------------------
+
+DPDK supports two non-generic instructions: ``(BPF_ABS | size | BPF_LD)``
+and ``(BPF_IND | size | BPF_LD)`` which are used to access packet data.
+These instructions can only be used when execution context is a pointer to
+``struct rte_mbuf`` and have seven implicit operands.
+Register ``R6`` is an implicit input that must contain pointer to ``rte_mbuf``.
+Register ``R0`` is an implicit output which contains the data fetched from the
+packet. Registers ``R1-R5`` are scratch registers
+and must not be used to store the data across these instructions.
+These instructions have implicit program exit condition as well. When
+eBPF program is trying to access the data beyond the packet boundary,
+the interpreter will abort the execution of the program. JIT compilers
+therefore must preserve this property. ``src_reg`` and ``imm32`` fields are
+explicit inputs to these instructions.
+For example, ``(BPF_IND | BPF_W | BPF_LD)`` means:
+
+.. code-block:: c
+
+    uint32_t tmp;
+    R0 = rte_pktmbuf_read((const struct rte_mbuf *)R6,  src_reg + imm32,
+	sizeof(tmp), &tmp);
+    if (R0 == NULL) return FAILED;
+    R0 = ntohl(*(uint32_t *)R0);
+
+and ``R1-R5`` were scratched.
+
+
 Not currently supported eBPF features
 -------------------------------------
 
@@ -34,5 +63,4 @@ Not currently supported eBPF features
  - cBPF
  - tail-pointer call
  - eBPF MAP
- - skb
  - external function calls for 32-bit platforms
lib/librte_bpf/bpf_exec.c +57 -0

75 lines not on this page · raw message

lib/librte_bpf/bpf_validate.c +77 -0

95 lines not on this page · raw message

Signature · 1 line
2.17.1
#5 lore raw

PATCH 4/5 test/bpf: add new test cases for mbuf load instructions

In reply to Konstantin Ananyev

Add new test-cases for BPF_ABS/BPF_IND load instructions.

---
 app/test/test_bpf.c | 463 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 463 insertions(+)
app/test/test_bpf.c +463 -0
index 4a61a7d7c..7c3de96c6 100644
@@ -43,6 +43,14 @@ struct dummy_net {
 	struct rte_ipv4_hdr ip_hdr;
 };
 
+#define	DUMMY_MBUF_NUM	2
+
+/* first mbuf in the packet, should always be at offset 0 */
+struct dummy_mbuf {
+	struct rte_mbuf mb[DUMMY_MBUF_NUM];
+	uint8_t buf[DUMMY_MBUF_NUM][RTE_MBUF_DEFAULT_BUF_SIZE];
+};
+
 #define	TEST_FILL_1	0xDEADBEEF
 
 #define	TEST_MUL_1	21
@@ -2444,6 +2452,413 @@ static const struct rte_bpf_xsym test_call5_xsym[] = {
 	},
 };
 
+/* load mbuf (BPF_ABS/BPF_IND) test-cases */
+static const struct ebpf_insn test_ld_mbuf1_prog[] = {
+
+	/* BPF_ABS/BPF_IND implicitly expect mbuf ptr in R6 */
+	{
+		.code = (EBPF_ALU64 | EBPF_MOV | BPF_X),
+		.dst_reg = EBPF_REG_6,
+		.src_reg = EBPF_REG_1,
+	},
+	/* load IPv4 version and IHL */
+	{
+		.code = (BPF_LD | BPF_ABS | BPF_B),
+		.imm = offsetof(struct rte_ipv4_hdr, version_ihl),
+	},
+	/* check IP version */
+	{
+		.code = (EBPF_ALU64 | EBPF_MOV | BPF_X),
+		.dst_reg = EBPF_REG_2,
+		.src_reg = EBPF_REG_0,
+	},
+	{
+		.code = (BPF_ALU | BPF_AND | BPF_K),
+		.dst_reg = EBPF_REG_2,
+		.imm = 0xf0,
+	},
+	{
+		.code = (BPF_JMP | BPF_JEQ | BPF_K),
+		.dst_reg = EBPF_REG_2,
+		.imm = IPVERSION << 4,
+		.off = 2,
+	},
+	/* invalid IP version, return 0 */
+	{
+		.code = (EBPF_ALU64 | BPF_XOR | BPF_X),
+		.dst_reg = EBPF_REG_0,
+		.src_reg = EBPF_REG_0,
+	},
+	{
+		.code = (BPF_JMP | EBPF_EXIT),
+	},
+	/* load 3-rd byte of IP data */
+	{
+		.code = (BPF_ALU | BPF_AND | BPF_K),
+		.dst_reg = EBPF_REG_0,
+		.imm = RTE_IPV4_HDR_IHL_MASK,
+	},
+	{
+		.code = (BPF_ALU | BPF_LSH | BPF_K),
+		.dst_reg = EBPF_REG_0,
+		.imm = 2,
+	},
+	{
+		.code = (BPF_LD | BPF_IND | BPF_B),
+		.src_reg = EBPF_REG_0,
+		.imm = 3,
+	},
+	{
+		.code = (EBPF_ALU64 | EBPF_MOV | BPF_X),
+		.dst_reg = EBPF_REG_7,
+		.src_reg = EBPF_REG_0,
+	},
+	/* load IPv4 src addr */
+	{
+		.code = (BPF_LD | BPF_ABS | BPF_W),
+		.imm = offsetof(struct rte_ipv4_hdr, src_addr),
+	},
+	{
+		.code = (EBPF_ALU64 | BPF_ADD | BPF_X),
+		.dst_reg = EBPF_REG_7,
+		.src_reg = EBPF_REG_0,
+	},
+	/* load IPv4 total length */
+	{
+		.code = (BPF_LD | BPF_ABS | BPF_H),
+		.imm = offsetof(struct rte_ipv4_hdr, total_length),
+	},
+	{
+		.code = (EBPF_ALU64 | EBPF_MOV | BPF_X),
+		.dst_reg = EBPF_REG_8,
+		.src_reg = EBPF_REG_0,
+	},
+	/* load last 4 bytes of IP data */
+	{
+		.code = (BPF_LD | BPF_IND | BPF_W),
+		.src_reg = EBPF_REG_8,
+		.imm = -(int32_t)sizeof(uint32_t),
+	},
+	{
+		.code = (EBPF_ALU64 | BPF_ADD | BPF_X),
+		.dst_reg = EBPF_REG_7,
+		.src_reg = EBPF_REG_0,
+	},
+	/* load 2 bytes from the middle of IP data */
+	{
+		.code = (EBPF_ALU64 | BPF_RSH | BPF_K),
+		.dst_reg = EBPF_REG_8,
+		.imm = 1,
+	},
+	{
+		.code = (BPF_LD | BPF_IND | BPF_H),
+		.src_reg = EBPF_REG_8,
+	},
+	{
+		.code = (EBPF_ALU64 | BPF_ADD | BPF_X),
+		.dst_reg = EBPF_REG_0,
+		.src_reg = EBPF_REG_7,
+	},
+	{
+		.code = (BPF_JMP | EBPF_EXIT),
+	},
+};
+
+static void
+dummy_mbuf_prep(struct rte_mbuf *mb, uint8_t buf[], uint32_t buf_len,
+	uint32_t data_len)
+{
+	uint32_t i;
+	uint8_t *db;
+
+	mb->buf_addr = buf;
+	mb->buf_iova = (uintptr_t)buf;
+	mb->buf_len = buf_len;
+	rte_mbuf_refcnt_set(mb, 1);
+
+	/* set pool pointer to dummy value, test doesn't use it */
+	mb->pool = (void *)buf;
+
+	rte_pktmbuf_reset(mb);
+	db = (uint8_t *)rte_pktmbuf_append(mb, data_len);
+
+	for (i = 0; i != data_len; i++)
+		db[i] = i;
+}
+
+static void
+test_ld_mbuf1_prepare(void *arg)
+{
+	struct dummy_mbuf *dm;
+	struct rte_ipv4_hdr *ph;
+
+	const uint32_t plen = 400;
+	const struct rte_ipv4_hdr iph = {
+		.version_ihl = RTE_IPV4_VHL_DEF,
+		.total_length = rte_cpu_to_be_16(plen),
+		.time_to_live = IPDEFTTL,
+		.next_proto_id = IPPROTO_RAW,
+		.src_addr = rte_cpu_to_be_32(RTE_IPV4_LOOPBACK),
+		.dst_addr = rte_cpu_to_be_32(RTE_IPV4_BROADCAST),
+	};
+
+	dm = arg;
+	memset(dm, 0, sizeof(*dm));
+
+	dummy_mbuf_prep(&dm->mb[0], dm->buf[0], sizeof(dm->buf[0]),
+		plen / 2 + 1);
+	dummy_mbuf_prep(&dm->mb[1], dm->buf[1], sizeof(dm->buf[0]),
+		plen / 2 - 1);
+
+	rte_pktmbuf_chain(&dm->mb[0], &dm->mb[1]);
+
+	ph = rte_pktmbuf_mtod(dm->mb, typeof(ph));
+	memcpy(ph, &iph, sizeof(iph));
+}
+
+static uint64_t
+test_ld_mbuf1(const struct rte_mbuf *pkt)
+{
+	uint64_t n, v;
+	const uint8_t *p8;
+	const uint16_t *p16;
+	const uint32_t *p32;
+	struct dummy_offset dof;
+
+	/* load IPv4 version and IHL */
+	p8 = rte_pktmbuf_read(pkt,
+		offsetof(struct rte_ipv4_hdr, version_ihl), sizeof(*p8),
+		&dof);
+	if (p8 == NULL)
+		return 0;
+
+	/* check IP version */
+	if ((p8[0] & 0xf0) != IPVERSION << 4)
+		return 0;
+
+	n = (p8[0] & RTE_IPV4_HDR_IHL_MASK) * RTE_IPV4_IHL_MULTIPLIER;
+
+	/* load 3-rd byte of IP data */
+	p8 = rte_pktmbuf_read(pkt, n + 3, sizeof(*p8), &dof);
+	if (p8 == NULL)
+		return 0;
+
+	v = p8[0];
+
+	/* load IPv4 src addr */
+	p32 = rte_pktmbuf_read(pkt,
+		offsetof(struct rte_ipv4_hdr, src_addr), sizeof(*p32),
+		&dof);
+	if (p32 == NULL)
+		return 0;
+
+	v += rte_be_to_cpu_32(p32[0]);
+
+	/* load IPv4 total length */
+	p16 = rte_pktmbuf_read(pkt,
+		offsetof(struct rte_ipv4_hdr, total_length), sizeof(*p16),
+		&dof);
+	if (p16 == NULL)
+		return 0;
+
+	n = rte_be_to_cpu_16(p16[0]);
+
+	/* load last 4 bytes of IP data */
+	p32 = rte_pktmbuf_read(pkt, n - sizeof(*p32), sizeof(*p32), &dof);
+	if (p32 == NULL)
+		return 0;
+
+	v += rte_be_to_cpu_32(p32[0]);
+
+	/* load 2 bytes from the middle of IP data */
+	p16 = rte_pktmbuf_read(pkt, n / 2, sizeof(*p16), &dof);
+	if (p16 == NULL)
+		return 0;
+
+	v += rte_be_to_cpu_16(p16[0]);
+	return v;
+}
+
+static int
+test_ld_mbuf1_check(uint64_t rc, const void *arg)
+{
+	const struct dummy_mbuf *dm;
+	uint64_t v;
+
+	dm = arg;
+	v = test_ld_mbuf1(dm->mb);
+	return cmp_res(__func__, v, rc, arg, arg, 0);
+}
+
+/*
+ * same as ld_mbuf1, but then trancate the mbuf by 1B,
+ * so load of last 4B fail.
+ */
+static void
+test_ld_mbuf2_prepare(void *arg)
+{
+	struct dummy_mbuf *dm;
+
+	test_ld_mbuf1_prepare(arg);
+	dm = arg;
+	rte_pktmbuf_trim(dm->mb, 1);
+}
+
+static int
+test_ld_mbuf2_check(uint64_t rc, const void *arg)
+{
+	return cmp_res(__func__, 0, rc, arg, arg, 0);
+}
+
+/* same as test_ld_mbuf1, but now store intermediate results on the stack */
+static const struct ebpf_insn test_ld_mbuf3_prog[] = {
+
+	/* BPF_ABS/BPF_IND implicitly expect mbuf ptr in R6 */
+	{
+		.code = (EBPF_ALU64 | EBPF_MOV | BPF_X),
+		.dst_reg = EBPF_REG_6,
+		.src_reg = EBPF_REG_1,
+	},
+	/* load IPv4 version and IHL */
+	{
+		.code = (BPF_LD | BPF_ABS | BPF_B),
+		.imm = offsetof(struct rte_ipv4_hdr, version_ihl),
+	},
+	/* check IP version */
+	{
+		.code = (EBPF_ALU64 | EBPF_MOV | BPF_X),
+		.dst_reg = EBPF_REG_2,
+		.src_reg = EBPF_REG_0,
+	},
+	{
+		.code = (BPF_ALU | BPF_AND | BPF_K),
+		.dst_reg = EBPF_REG_2,
+		.imm = 0xf0,
+	},
+	{
+		.code = (BPF_JMP | BPF_JEQ | BPF_K),
+		.dst_reg = EBPF_REG_2,
+		.imm = IPVERSION << 4,
+		.off = 2,
+	},
+	/* invalid IP version, return 0 */
+	{
+		.code = (EBPF_ALU64 | BPF_XOR | BPF_X),
+		.dst_reg = EBPF_REG_0,
+		.src_reg = EBPF_REG_0,
+	},
+	{
+		.code = (BPF_JMP | EBPF_EXIT),
+	},
+	/* load 3-rd byte of IP data */
+	{
+		.code = (BPF_ALU | BPF_AND | BPF_K),
+		.dst_reg = EBPF_REG_0,
+		.imm = RTE_IPV4_HDR_IHL_MASK,
+	},
+	{
+		.code = (BPF_ALU | BPF_LSH | BPF_K),
+		.dst_reg = EBPF_REG_0,
+		.imm = 2,
+	},
+	{
+		.code = (BPF_LD | BPF_IND | BPF_B),
+		.src_reg = EBPF_REG_0,
+		.imm = 3,
+	},
+	{
+		.code = (BPF_STX | BPF_MEM | BPF_B),
+		.dst_reg = EBPF_REG_10,
+		.src_reg = EBPF_REG_0,
+		.off = (int16_t)(offsetof(struct dummy_offset, u8) -
+			sizeof(struct dummy_offset)),
+	},
+	/* load IPv4 src addr */
+	{
+		.code = (BPF_LD | BPF_ABS | BPF_W),
+		.imm = offsetof(struct rte_ipv4_hdr, src_addr),
+	},
+	{
+		.code = (BPF_STX | BPF_MEM | BPF_W),
+		.dst_reg = EBPF_REG_10,
+		.src_reg = EBPF_REG_0,
+		.off = (int16_t)(offsetof(struct dummy_offset, u32) -
+			sizeof(struct dummy_offset)),
+	},
+	/* load IPv4 total length */
+	{
+		.code = (BPF_LD | BPF_ABS | BPF_H),
+		.imm = offsetof(struct rte_ipv4_hdr, total_length),
+	},
+	{
+		.code = (EBPF_ALU64 | EBPF_MOV | BPF_X),
+		.dst_reg = EBPF_REG_8,
+		.src_reg = EBPF_REG_0,
+	},
+	/* load last 4 bytes of IP data */
+	{
+		.code = (BPF_LD | BPF_IND | BPF_W),
+		.src_reg = EBPF_REG_8,
+		.imm = -(int32_t)sizeof(uint32_t),
+	},
+	{
+		.code = (BPF_STX | BPF_MEM | EBPF_DW),
+		.dst_reg = EBPF_REG_10,
+		.src_reg = EBPF_REG_0,
+		.off = (int16_t)(offsetof(struct dummy_offset, u64) -
+			sizeof(struct dummy_offset)),
+	},
+	/* load 2 bytes from the middle of IP data */
+	{
+		.code = (EBPF_ALU64 | BPF_RSH | BPF_K),
+		.dst_reg = EBPF_REG_8,
+		.imm = 1,
+	},
+	{
+		.code = (BPF_LD | BPF_IND | BPF_H),
+		.src_reg = EBPF_REG_8,
+	},
+	{
+		.code = (BPF_LDX | BPF_MEM | EBPF_DW),
+		.dst_reg = EBPF_REG_1,
+		.src_reg = EBPF_REG_10,
+		.off = (int16_t)(offsetof(struct dummy_offset, u64) -
+			sizeof(struct dummy_offset)),
+	},
+	{
+		.code = (EBPF_ALU64 | BPF_ADD | BPF_X),
+		.dst_reg = EBPF_REG_0,
+		.src_reg = EBPF_REG_1,
+	},
+	{
+		.code = (BPF_LDX | BPF_MEM | BPF_W),
+		.dst_reg = EBPF_REG_1,
+		.src_reg = EBPF_REG_10,
+		.off = (int16_t)(offsetof(struct dummy_offset, u32) -
+			sizeof(struct dummy_offset)),
+	},
+	{
+		.code = (EBPF_ALU64 | BPF_ADD | BPF_X),
+		.dst_reg = EBPF_REG_0,
+		.src_reg = EBPF_REG_1,
+	},
+	{
+		.code = (BPF_LDX | BPF_MEM | BPF_B),
+		.dst_reg = EBPF_REG_1,
+		.src_reg = EBPF_REG_10,
+		.off = (int16_t)(offsetof(struct dummy_offset, u8) -
+			sizeof(struct dummy_offset)),
+	},
+	{
+		.code = (EBPF_ALU64 | BPF_ADD | BPF_X),
+		.dst_reg = EBPF_REG_0,
+		.src_reg = EBPF_REG_1,
+	},
+	{
+		.code = (BPF_JMP | EBPF_EXIT),
+	},
+};
+
 /* all bpf test cases */
 static const struct bpf_test tests[] = {
 	{
@@ -2704,6 +3119,54 @@ static const struct bpf_test tests[] = {
 		/* for now don't support function calls on 32 bit platform */
 		.allow_fail = (sizeof(uint64_t) != sizeof(uintptr_t)),
 	},
+	{
+		.name = "test_ld_mbuf1",
+		.arg_sz = sizeof(struct dummy_mbuf),
+		.prm = {
+			.ins = test_ld_mbuf1_prog,
+			.nb_ins = RTE_DIM(test_ld_mbuf1_prog),
+			.prog_arg = {
+				.type = RTE_BPF_ARG_PTR_MBUF,
+				.buf_size = sizeof(struct dummy_mbuf),
+			},
+		},
+		.prepare = test_ld_mbuf1_prepare,
+		.check_result = test_ld_mbuf1_check,
+		/* mbuf as input argument is not supported on 32 bit platform */
+		.allow_fail = (sizeof(uint64_t) != sizeof(uintptr_t)),
+	},
+	{
+		.name = "test_ld_mbuf2",
+		.arg_sz = sizeof(struct dummy_mbuf),
+		.prm = {
+			.ins = test_ld_mbuf1_prog,
+			.nb_ins = RTE_DIM(test_ld_mbuf1_prog),
+			.prog_arg = {
+				.type = RTE_BPF_ARG_PTR_MBUF,
+				.buf_size = sizeof(struct dummy_mbuf),
+			},
+		},
+		.prepare = test_ld_mbuf2_prepare,
+		.check_result = test_ld_mbuf2_check,
+		/* mbuf as input argument is not supported on 32 bit platform */
+		.allow_fail = (sizeof(uint64_t) != sizeof(uintptr_t)),
+	},
+	{
+		.name = "test_ld_mbuf3",
+		.arg_sz = sizeof(struct dummy_mbuf),
+		.prm = {
+			.ins = test_ld_mbuf3_prog,
+			.nb_ins = RTE_DIM(test_ld_mbuf3_prog),
+			.prog_arg = {
+				.type = RTE_BPF_ARG_PTR_MBUF,
+				.buf_size = sizeof(struct dummy_mbuf),
+			},
+		},
+		.prepare = test_ld_mbuf1_prepare,
+		.check_result = test_ld_mbuf1_check,
+		/* mbuf as input argument is not supported on 32 bit platform */
+		.allow_fail = (sizeof(uint64_t) != sizeof(uintptr_t)),
+	},
 };
 
 static int
Signature · 1 line
2.17.1
#6 lore raw

PATCH 5/5 bpf: x86 JIT support for packet data load instructions

In reply to Konstantin Ananyev

Make x86 JIT to generate native code for
(BPF_ABS | <size> | BPF_LD) and (BPF_IND | <size> | BPF_LD) instructions.

---
 lib/librte_bpf/bpf_jit_x86.c  | 181 ++++++++++++++++++++++++++++++++++
 lib/librte_bpf/bpf_validate.c |  17 +++-
 2 files changed, 197 insertions(+), 1 deletion(-)
lib/librte_bpf/bpf_jit_x86.c +181 -0
index f70cd6be5..8a7ad45b3 100644
@@ -87,6 +87,14 @@ enum {
 	REG_TMP1 = R10,
 };
 
+/* LD_ABS/LD_IMM offsets */
+enum {
+	LDMB_FSP_OFS, /* fast-path */
+	LDMB_SLP_OFS, /* slow-path */
+	LDMB_FIN_OFS, /* final part */
+	LDMB_OFS_NUM
+};
+
 /*
  * callee saved registers list.
  * keep RBP as the last one.
@@ -100,6 +108,9 @@ struct bpf_jit_state {
 		uint32_t num;
 		int32_t off;
 	} exit;
+	struct {
+		uint32_t stack_ofs;
+	} ldmb;
 	uint32_t reguse;
 	int32_t *off;
 	uint8_t *ins;
@@ -1024,6 +1035,166 @@ emit_div(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,
 		emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP1, RDX);
 }
 
+/*
+ * helper function, used by emit_ld_mbuf().
+ * generates code for 'fast_path':
+ * calculate load offset and check is it inside first packet segment.
+ */
+static void
+emit_ldmb_fast_path(struct bpf_jit_state *st, const uint32_t rg[EBPF_REG_7],
+	uint32_t sreg, uint32_t mode, uint32_t sz, uint32_t imm,
+	const int32_t ofs[LDMB_OFS_NUM])
+{
+	/* make R2 contain *off* value */
+
+	if (sreg != rg[EBPF_REG_2]) {
+		emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K,
+			rg[EBPF_REG_2], imm);
+		if (mode == BPF_IND)
+			emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X,
+				sreg, rg[EBPF_REG_2]);
+	} else
+		/* BPF_IND with sreg == R2 */
+		emit_alu_imm(st, EBPF_ALU64 | BPF_ADD | BPF_K,
+			rg[EBPF_REG_2], imm);
+
+	/* R3 = mbuf->data_len */
+	emit_ld_reg(st, BPF_LDX | BPF_MEM | BPF_H,
+		rg[EBPF_REG_6], rg[EBPF_REG_3],
+		offsetof(struct rte_mbuf, data_len));
+
+	/* R3 = R3 - R2 */
+	emit_alu_reg(st, EBPF_ALU64 | BPF_SUB | BPF_X,
+		rg[EBPF_REG_2], rg[EBPF_REG_3]);
+
+	/* JSLE R3, <sz> <slow_path> */
+	emit_cmp_imm(st, EBPF_ALU64, rg[EBPF_REG_3], sz);
+	emit_abs_jcc(st, BPF_JMP | EBPF_JSLE | BPF_K, ofs[LDMB_SLP_OFS]);
+
+	/* R3 = mbuf->data_off */
+	emit_ld_reg(st, BPF_LDX | BPF_MEM | BPF_H,
+		rg[EBPF_REG_6], rg[EBPF_REG_3],
+		offsetof(struct rte_mbuf, data_off));
+
+	/* R0 = mbuf->buf_addr */
+	emit_ld_reg(st, BPF_LDX | BPF_MEM | EBPF_DW,
+		rg[EBPF_REG_6], rg[EBPF_REG_0],
+		offsetof(struct rte_mbuf, buf_addr));
+
+	/* R0 = R0 + R3 */
+	emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X,
+		rg[EBPF_REG_3], rg[EBPF_REG_0]);
+
+	/* R0 = R0 + R2 */
+	emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X,
+		rg[EBPF_REG_2], rg[EBPF_REG_0]);
+
+	/* JMP <fin_part> */
+	emit_abs_jmp(st, ofs[LDMB_FIN_OFS]);
+}
+
+/*
+ * helper function, used by emit_ld_mbuf().
+ * generates code for 'slow_path':
+ * call __rte_pktmbuf_read() and check return value.
+ */
+static void
+emit_ldmb_slow_path(struct bpf_jit_state *st, const uint32_t rg[EBPF_REG_7],
+	uint32_t sz)
+{
+	/* make R3 contain *len* value (1/2/4) */
+
+	emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, rg[EBPF_REG_3], sz);
+
+	/* make R4 contain (RBP - ldmb.stack_ofs) */
+
+	emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RBP, rg[EBPF_REG_4]);
+	emit_alu_imm(st, EBPF_ALU64 | BPF_SUB | BPF_K, rg[EBPF_REG_4],
+		st->ldmb.stack_ofs);
+
+	/* make R1 contain mbuf ptr */
+
+	emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X,
+		rg[EBPF_REG_6], rg[EBPF_REG_1]);
+
+	/* call rte_pktmbuf_read */
+	emit_call(st, (uintptr_t)__rte_pktmbuf_read);
+
+	/* check that return value (R0) is not zero */
+	emit_tst_reg(st, EBPF_ALU64, rg[EBPF_REG_0], rg[EBPF_REG_0]);
+	emit_abs_jcc(st, BPF_JMP | BPF_JEQ | BPF_K, st->exit.off);
+}
+
+/*
+ * helper function, used by emit_ld_mbuf().
+ * generates final part of code for BPF_ABS/BPF_IND load:
+ * perform data load and endianness conversion.
+ * expects dreg to contain valid data pointer.
+ */
+static void
+emit_ldmb_fin(struct bpf_jit_state *st, uint32_t dreg, uint32_t opsz,
+	uint32_t sz)
+{
+	emit_ld_reg(st, BPF_LDX | BPF_MEM | opsz, dreg, dreg, 0);
+	if (sz != sizeof(uint8_t))
+		emit_be2le(st, dreg, sz * CHAR_BIT);
+}
+
+/*
+ * emit code for BPF_ABS/BPF_IND load.
+ * generates the following construction:
+ * fast_path:
+ *   off = ins->sreg + ins->imm
+ *   if (off + ins->opsz < mbuf->data_len)
+ *      goto slow_path;
+ *   ptr = mbuf->buf_addr + bmf->data_off + off;
+ *   goto fin_part;
+ * slow_path:
+ *   typeof(ins->opsz) buf; //allocate space on the stack
+ *   ptr = __rte_pktmbuf_read(mbuf, off, ins->opsz, &buf);
+ *   if (ptr == NULL)
+ *      goto exit_label;
+ * fin_part:
+ *   res = *(typeof(ins->opsz))ptr;
+ *   res = bswap(res);
+ */
+static void
+emit_ld_mbuf(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t imm)
+{
+	uint32_t i, mode, opsz, sz;
+	uint32_t rg[EBPF_REG_7];
+	int32_t ofs[LDMB_OFS_NUM];
+
+	mode = BPF_MODE(op);
+	opsz = BPF_SIZE(op);
+	sz = bpf_size(opsz);
+
+	for (i = 0; i != RTE_DIM(rg); i++)
+		rg[i] = ebpf2x86[i];
+
+	/* fill with fake offsets */
+	for (i = 0; i != RTE_DIM(ofs); i++)
+		ofs[i] = st->sz + INT8_MAX;
+
+	/* dry run first to calculate jump offsets */
+
+	ofs[LDMB_FSP_OFS] = st->sz;
+	emit_ldmb_fast_path(st, rg, sreg, mode, sz, imm, ofs);
+	ofs[LDMB_SLP_OFS] = st->sz;
+	emit_ldmb_slow_path(st, rg, sz);
+	ofs[LDMB_FIN_OFS] = st->sz;
+	emit_ldmb_fin(st, rg[EBPF_REG_0], opsz, sz);
+
+	RTE_VERIFY(ofs[LDMB_FIN_OFS] - ofs[LDMB_FSP_OFS] <= INT8_MAX);
+
+	/* reset dry-run code and do a proper run */
+
+	st->sz = ofs[LDMB_FSP_OFS];
+	emit_ldmb_fast_path(st, rg, sreg, mode, sz, imm, ofs);
+	emit_ldmb_slow_path(st, rg, sz);
+	emit_ldmb_fin(st, rg[EBPF_REG_0], opsz, sz);
+}
+
 static void
 emit_prolog(struct bpf_jit_state *st, int32_t stack_size)
 {
@@ -1121,6 +1292,7 @@ emit(struct bpf_jit_state *st, const struct rte_bpf *bpf)
 	/* reset state fields */
 	st->sz = 0;
 	st->exit.num = 0;
+	st->ldmb.stack_ofs = bpf->stack_sz;
 
 	emit_prolog(st, bpf->stack_sz);
 
@@ -1240,6 +1412,15 @@ emit(struct bpf_jit_state *st, const struct rte_bpf *bpf)
 			emit_ld_imm64(st, dr, ins[0].imm, ins[1].imm);
 			i++;
 			break;
+		/* load absolute/indirect instructions */
+		case (BPF_LD | BPF_ABS | BPF_B):
+		case (BPF_LD | BPF_ABS | BPF_H):
+		case (BPF_LD | BPF_ABS | BPF_W):
+		case (BPF_LD | BPF_IND | BPF_B):
+		case (BPF_LD | BPF_IND | BPF_H):
+		case (BPF_LD | BPF_IND | BPF_W):
+			emit_ld_mbuf(st, op, sr, ins->imm);
+			break;
 		/* store instructions */
 		case (BPF_STX | BPF_MEM | BPF_B):
 		case (BPF_STX | BPF_MEM | BPF_H):
lib/librte_bpf/bpf_validate.c +16 -1
index fecdda0e1..9214f1503 100644
@@ -70,6 +70,7 @@ struct bpf_verifier {
 	uint64_t stack_sz;
 	uint32_t nb_nodes;
 	uint32_t nb_jcc_nodes;
+	uint32_t nb_ldmb_nodes;
 	uint32_t node_colour[MAX_NODE_COLOUR];
 	uint32_t edge_type[MAX_EDGE_TYPE];
 	struct bpf_eval_state *evst;
@@ -2020,6 +2021,14 @@ validate(struct bpf_verifier *bvf)
 			rc |= add_edge(bvf, node, i + 2);
 			i++;
 			break;
+		case (BPF_LD | BPF_ABS | BPF_B):
+		case (BPF_LD | BPF_ABS | BPF_H):
+		case (BPF_LD | BPF_ABS | BPF_W):
+		case (BPF_LD | BPF_IND | BPF_B):
+		case (BPF_LD | BPF_IND | BPF_H):
+		case (BPF_LD | BPF_IND | BPF_W):
+			bvf->nb_ldmb_nodes++;
+			/* fallthrough */
 		default:
 			rc |= add_edge(bvf, node, i + 1);
 			break;
@@ -2320,8 +2329,14 @@ bpf_validate(struct rte_bpf *bpf)
 	free(bvf.in);
 
 	/* copy collected info */
-	if (rc == 0)
+	if (rc == 0) {
 		bpf->stack_sz = bvf.stack_sz;
 
+		/* for LD_ABS/LD_IND, we'll need extra space on the stack */
+		if (bvf.nb_ldmb_nodes != 0)
+			bpf->stack_sz = RTE_ALIGN_CEIL(bpf->stack_sz +
+				sizeof(uint64_t), sizeof(uint64_t));
+	}
+
 	return rc;
 }
Signature · 1 line
2.17.1
#7 lore raw

Re: [dpdk-dev] [PATCH 5/5] bpf: x86 JIT support for packet data loadinstructions

Quoted 151 lines

From: dev [mailto:[email protected]] On Behalf Of Konstantin Ananyev
Sent: Monday, May 18, 2020 5:53 PM

Make x86 JIT to generate native code for
(BPF_ABS | <size> | BPF_LD) and (BPF_IND | <size> | BPF_LD) instructions.

---
 lib/librte_bpf/bpf_jit_x86.c  | 181 ++++++++++++++++++++++++++++++++++
 lib/librte_bpf/bpf_validate.c |  17 +++-
 2 files changed, 197 insertions(+), 1 deletion(-)
lib/librte_bpf/bpf_jit_x86.c +113 -0
index f70cd6be5..8a7ad45b3 100644
@@ -87,6 +87,14 @@ enum {
 	REG_TMP1 = R10,
 };
 
+/* LD_ABS/LD_IMM offsets */
+enum {
+	LDMB_FSP_OFS, /* fast-path */
+	LDMB_SLP_OFS, /* slow-path */
+	LDMB_FIN_OFS, /* final part */
+	LDMB_OFS_NUM
+};
+
 /*
  * callee saved registers list.
  * keep RBP as the last one.
@@ -100,6 +108,9 @@ struct bpf_jit_state {
 		uint32_t num;
 		int32_t off;
 	} exit;
+	struct {
+		uint32_t stack_ofs;
+	} ldmb;
 	uint32_t reguse;
 	int32_t *off;
 	uint8_t *ins;
@@ -1024,6 +1035,166 @@ emit_div(struct bpf_jit_state *st, uint32_t op,
 uint32_t sreg, uint32_t dreg,
 		emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP1, RDX);
 }
 
+/*
+ * helper function, used by emit_ld_mbuf().
+ * generates code for 'fast_path':
+ * calculate load offset and check is it inside first packet segment.
+ */
+static void
+emit_ldmb_fast_path(struct bpf_jit_state *st, const uint32_t
 rg[EBPF_REG_7],
+	uint32_t sreg, uint32_t mode, uint32_t sz, uint32_t imm,
+	const int32_t ofs[LDMB_OFS_NUM])
+{
+	/* make R2 contain *off* value */
+
+	if (sreg != rg[EBPF_REG_2]) {
+		emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K,
+			rg[EBPF_REG_2], imm);
+		if (mode == BPF_IND)
+			emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X,
+				sreg, rg[EBPF_REG_2]);
+	} else
+		/* BPF_IND with sreg == R2 */
+		emit_alu_imm(st, EBPF_ALU64 | BPF_ADD | BPF_K,
+			rg[EBPF_REG_2], imm);
+
+	/* R3 = mbuf->data_len */
+	emit_ld_reg(st, BPF_LDX | BPF_MEM | BPF_H,
+		rg[EBPF_REG_6], rg[EBPF_REG_3],
+		offsetof(struct rte_mbuf, data_len));
+
+	/* R3 = R3 - R2 */
+	emit_alu_reg(st, EBPF_ALU64 | BPF_SUB | BPF_X,
+		rg[EBPF_REG_2], rg[EBPF_REG_3]);
+
+	/* JSLE R3, <sz> <slow_path> */
+	emit_cmp_imm(st, EBPF_ALU64, rg[EBPF_REG_3], sz);
+	emit_abs_jcc(st, BPF_JMP | EBPF_JSLE | BPF_K, ofs[LDMB_SLP_OFS]);
+
+	/* R3 = mbuf->data_off */
+	emit_ld_reg(st, BPF_LDX | BPF_MEM | BPF_H,
+		rg[EBPF_REG_6], rg[EBPF_REG_3],
+		offsetof(struct rte_mbuf, data_off));
+
+	/* R0 = mbuf->buf_addr */
+	emit_ld_reg(st, BPF_LDX | BPF_MEM | EBPF_DW,
+		rg[EBPF_REG_6], rg[EBPF_REG_0],
+		offsetof(struct rte_mbuf, buf_addr));
+
+	/* R0 = R0 + R3 */
+	emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X,
+		rg[EBPF_REG_3], rg[EBPF_REG_0]);
+
+	/* R0 = R0 + R2 */
+	emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X,
+		rg[EBPF_REG_2], rg[EBPF_REG_0]);
+
+	/* JMP <fin_part> */
+	emit_abs_jmp(st, ofs[LDMB_FIN_OFS]);
+}
+
+/*
+ * helper function, used by emit_ld_mbuf().
+ * generates code for 'slow_path':
+ * call __rte_pktmbuf_read() and check return value.
+ */
+static void
+emit_ldmb_slow_path(struct bpf_jit_state *st, const uint32_t
 rg[EBPF_REG_7],
+	uint32_t sz)
+{
+	/* make R3 contain *len* value (1/2/4) */
+
+	emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, rg[EBPF_REG_3], sz);
+
+	/* make R4 contain (RBP - ldmb.stack_ofs) */
+
+	emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RBP, rg[EBPF_REG_4]);
+	emit_alu_imm(st, EBPF_ALU64 | BPF_SUB | BPF_K, rg[EBPF_REG_4],
+		st->ldmb.stack_ofs);
+
+	/* make R1 contain mbuf ptr */
+
+	emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X,
+		rg[EBPF_REG_6], rg[EBPF_REG_1]);
+
+	/* call rte_pktmbuf_read */
+	emit_call(st, (uintptr_t)__rte_pktmbuf_read);
+
+	/* check that return value (R0) is not zero */
+	emit_tst_reg(st, EBPF_ALU64, rg[EBPF_REG_0], rg[EBPF_REG_0]);
+	emit_abs_jcc(st, BPF_JMP | BPF_JEQ | BPF_K, st->exit.off);
+}
+
+/*
+ * helper function, used by emit_ld_mbuf().
+ * generates final part of code for BPF_ABS/BPF_IND load:
+ * perform data load and endianness conversion.
+ * expects dreg to contain valid data pointer.
+ */
+static void
+emit_ldmb_fin(struct bpf_jit_state *st, uint32_t dreg, uint32_t opsz,
+	uint32_t sz)
+{
+	emit_ld_reg(st, BPF_LDX | BPF_MEM | opsz, dreg, dreg, 0);
+	if (sz != sizeof(uint8_t))

lib/librte_bpf/bpf_jit_x86.c emit_div(struct bpf_jit_state *st, uint32_t op, · line 1152

Quoted 10 lines
lib/librte_bpf/bpf_jit_x86.c +10 -0
+		emit_be2le(st, dreg, sz * CHAR_BIT);
+}
+
+/*
+ * emit code for BPF_ABS/BPF_IND load.
+ * generates the following construction:
+ * fast_path:
+ *   off = ins->sreg + ins->imm
+ *   if (off + ins->opsz < mbuf->data_len)
+ *      goto slow_path;

I am not an eBPF expert, but I am not sure this is correct.

I think it should be > instead of <. Also, it looks like you actually emit:
if (mbuf->data_len - off <= ins->opsz)
goto slow_path;

Could you please double check that both the comment and the emitted code has the comparison turning the correct way.

Quoted 124 lines
lib/librte_bpf/bpf_jit_x86.c +13 -0
+ *   ptr = mbuf->buf_addr + bmf->data_off + off;
+ *   goto fin_part;
+ * slow_path:
+ *   typeof(ins->opsz) buf; //allocate space on the stack
+ *   ptr = __rte_pktmbuf_read(mbuf, off, ins->opsz, &buf);
+ *   if (ptr == NULL)
+ *      goto exit_label;
+ * fin_part:
+ *   res = *(typeof(ins->opsz))ptr;
+ *   res = bswap(res);
+ */
+static void
+emit_ld_mbuf(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,

uint32_t imm)

lib/librte_bpf/bpf_jit_x86.c +45 -0
+{
+	uint32_t i, mode, opsz, sz;
+	uint32_t rg[EBPF_REG_7];
+	int32_t ofs[LDMB_OFS_NUM];
+
+	mode = BPF_MODE(op);
+	opsz = BPF_SIZE(op);
+	sz = bpf_size(opsz);
+
+	for (i = 0; i != RTE_DIM(rg); i++)
+		rg[i] = ebpf2x86[i];
+
+	/* fill with fake offsets */
+	for (i = 0; i != RTE_DIM(ofs); i++)
+		ofs[i] = st->sz + INT8_MAX;
+
+	/* dry run first to calculate jump offsets */
+
+	ofs[LDMB_FSP_OFS] = st->sz;
+	emit_ldmb_fast_path(st, rg, sreg, mode, sz, imm, ofs);
+	ofs[LDMB_SLP_OFS] = st->sz;
+	emit_ldmb_slow_path(st, rg, sz);
+	ofs[LDMB_FIN_OFS] = st->sz;
+	emit_ldmb_fin(st, rg[EBPF_REG_0], opsz, sz);
+
+	RTE_VERIFY(ofs[LDMB_FIN_OFS] - ofs[LDMB_FSP_OFS] <= INT8_MAX);
+
+	/* reset dry-run code and do a proper run */
+
+	st->sz = ofs[LDMB_FSP_OFS];
+	emit_ldmb_fast_path(st, rg, sreg, mode, sz, imm, ofs);
+	emit_ldmb_slow_path(st, rg, sz);
+	emit_ldmb_fin(st, rg[EBPF_REG_0], opsz, sz);
+}
+
 static void
 emit_prolog(struct bpf_jit_state *st, int32_t stack_size)
 {
@@ -1121,6 +1292,7 @@ emit(struct bpf_jit_state *st, const struct rte_bpf
 *bpf)
 	/* reset state fields */
 	st->sz = 0;
 	st->exit.num = 0;
+	st->ldmb.stack_ofs = bpf->stack_sz;
 
 	emit_prolog(st, bpf->stack_sz);
@@ -1240,6 +1412,15 @@ emit(struct bpf_jit_state *st, const struct rte_bpf
 *bpf)
 			emit_ld_imm64(st, dr, ins[0].imm, ins[1].imm);
 			i++;
 			break;
+		/* load absolute/indirect instructions */
+		case (BPF_LD | BPF_ABS | BPF_B):
+		case (BPF_LD | BPF_ABS | BPF_H):
+		case (BPF_LD | BPF_ABS | BPF_W):
+		case (BPF_LD | BPF_IND | BPF_B):
+		case (BPF_LD | BPF_IND | BPF_H):
+		case (BPF_LD | BPF_IND | BPF_W):
+			emit_ld_mbuf(st, op, sr, ins->imm);
+			break;
 		/* store instructions */
 		case (BPF_STX | BPF_MEM | BPF_B):
lib/librte_bpf/bpf_jit_x86.c +0 -0
 		case (BPF_STX | BPF_MEM | BPF_H):
lib/librte_bpf/bpf_validate.c +16 -2
index fecdda0e1..9214f1503 100644
@@ -70,6 +70,7 @@ struct bpf_verifier {
 	uint64_t stack_sz;
 	uint32_t nb_nodes;
 	uint32_t nb_jcc_nodes;
+	uint32_t nb_ldmb_nodes;
 	uint32_t node_colour[MAX_NODE_COLOUR];
 	uint32_t edge_type[MAX_EDGE_TYPE];
 	struct bpf_eval_state *evst;
@@ -2020,6 +2021,14 @@ validate(struct bpf_verifier *bvf)
 			rc |= add_edge(bvf, node, i + 2);
 			i++;
 			break;
+		case (BPF_LD | BPF_ABS | BPF_B):
+		case (BPF_LD | BPF_ABS | BPF_H):
+		case (BPF_LD | BPF_ABS | BPF_W):
+		case (BPF_LD | BPF_IND | BPF_B):
+		case (BPF_LD | BPF_IND | BPF_H):
+		case (BPF_LD | BPF_IND | BPF_W):
+			bvf->nb_ldmb_nodes++;
+			/* fallthrough */
 		default:
 			rc |= add_edge(bvf, node, i + 1);
 			break;
@@ -2320,8 +2329,14 @@ bpf_validate(struct rte_bpf *bpf)
 	free(bvf.in);
 
 	/* copy collected info */
-	if (rc == 0)
+	if (rc == 0) {
 		bpf->stack_sz = bvf.stack_sz;
 
+		/* for LD_ABS/LD_IND, we'll need extra space on the stack */
+		if (bvf.nb_ldmb_nodes != 0)
+			bpf->stack_sz = RTE_ALIGN_CEIL(bpf->stack_sz +
+				sizeof(uint64_t), sizeof(uint64_t));
+	}
+
 	return rc;
 }
--

2.17.1

#8 lore raw

Re: [dpdk-dev] [PATCH 5/5] bpf: x86 JIT support for packet data loadinstructions

Quoted 118 lines
Quoted 105 lines
lib/librte_bpf/bpf_jit_x86.c +7 -0
+/*
+ * helper function, used by emit_ld_mbuf().
+ * generates code for 'fast_path':
+ * calculate load offset and check is it inside first packet segment.
+ */
+static void
+emit_ldmb_fast_path(struct bpf_jit_state *st, const uint32_t

rg[EBPF_REG_7],

lib/librte_bpf/bpf_jit_x86.c +58 -0
+	uint32_t sreg, uint32_t mode, uint32_t sz, uint32_t imm,
+	const int32_t ofs[LDMB_OFS_NUM])
+{
+	/* make R2 contain *off* value */
+
+	if (sreg != rg[EBPF_REG_2]) {
+		emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K,
+			rg[EBPF_REG_2], imm);
+		if (mode == BPF_IND)
+			emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X,
+				sreg, rg[EBPF_REG_2]);
+	} else
+		/* BPF_IND with sreg == R2 */
+		emit_alu_imm(st, EBPF_ALU64 | BPF_ADD | BPF_K,
+			rg[EBPF_REG_2], imm);
+
+	/* R3 = mbuf->data_len */
+	emit_ld_reg(st, BPF_LDX | BPF_MEM | BPF_H,
+		rg[EBPF_REG_6], rg[EBPF_REG_3],
+		offsetof(struct rte_mbuf, data_len));
+
+	/* R3 = R3 - R2 */
+	emit_alu_reg(st, EBPF_ALU64 | BPF_SUB | BPF_X,
+		rg[EBPF_REG_2], rg[EBPF_REG_3]);
+
+	/* JSLE R3, <sz> <slow_path> */
+	emit_cmp_imm(st, EBPF_ALU64, rg[EBPF_REG_3], sz);
+	emit_abs_jcc(st, BPF_JMP | EBPF_JSLE | BPF_K, ofs[LDMB_SLP_OFS]);
+
+	/* R3 = mbuf->data_off */
+	emit_ld_reg(st, BPF_LDX | BPF_MEM | BPF_H,
+		rg[EBPF_REG_6], rg[EBPF_REG_3],
+		offsetof(struct rte_mbuf, data_off));
+
+	/* R0 = mbuf->buf_addr */
+	emit_ld_reg(st, BPF_LDX | BPF_MEM | EBPF_DW,
+		rg[EBPF_REG_6], rg[EBPF_REG_0],
+		offsetof(struct rte_mbuf, buf_addr));
+
+	/* R0 = R0 + R3 */
+	emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X,
+		rg[EBPF_REG_3], rg[EBPF_REG_0]);
+
+	/* R0 = R0 + R2 */
+	emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X,
+		rg[EBPF_REG_2], rg[EBPF_REG_0]);
+
+	/* JMP <fin_part> */
+	emit_abs_jmp(st, ofs[LDMB_FIN_OFS]);
+}
+
+/*
+ * helper function, used by emit_ld_mbuf().
+ * generates code for 'slow_path':
+ * call __rte_pktmbuf_read() and check return value.
+ */
+static void
+emit_ldmb_slow_path(struct bpf_jit_state *st, const uint32_t

rg[EBPF_REG_7],

lib/librte_bpf/bpf_jit_x86.c +37 -0
+	uint32_t sz)
+{
+	/* make R3 contain *len* value (1/2/4) */
+
+	emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, rg[EBPF_REG_3], sz);
+
+	/* make R4 contain (RBP - ldmb.stack_ofs) */
+
+	emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RBP, rg[EBPF_REG_4]);
+	emit_alu_imm(st, EBPF_ALU64 | BPF_SUB | BPF_K, rg[EBPF_REG_4],
+		st->ldmb.stack_ofs);
+
+	/* make R1 contain mbuf ptr */
+
+	emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X,
+		rg[EBPF_REG_6], rg[EBPF_REG_1]);
+
+	/* call rte_pktmbuf_read */
+	emit_call(st, (uintptr_t)__rte_pktmbuf_read);
+
+	/* check that return value (R0) is not zero */
+	emit_tst_reg(st, EBPF_ALU64, rg[EBPF_REG_0], rg[EBPF_REG_0]);
+	emit_abs_jcc(st, BPF_JMP | BPF_JEQ | BPF_K, st->exit.off);
+}
+
+/*
+ * helper function, used by emit_ld_mbuf().
+ * generates final part of code for BPF_ABS/BPF_IND load:
+ * perform data load and endianness conversion.
+ * expects dreg to contain valid data pointer.
+ */
+static void
+emit_ldmb_fin(struct bpf_jit_state *st, uint32_t dreg, uint32_t opsz,
+	uint32_t sz)
+{
+	emit_ld_reg(st, BPF_LDX | BPF_MEM | opsz, dreg, dreg, 0);
+	if (sz != sizeof(uint8_t))

lib/librte_bpf/bpf_jit_x86.c emit_div(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg, · line 1149

Quoted 10 lines
lib/librte_bpf/bpf_jit_x86.c +10 -0
+		emit_be2le(st, dreg, sz * CHAR_BIT);
+}
+
+/*
+ * emit code for BPF_ABS/BPF_IND load.
+ * generates the following construction:
+ * fast_path:
+ *   off = ins->sreg + ins->imm
+ *   if (off + ins->opsz < mbuf->data_len)
+ *      goto slow_path;

I am not an eBPF expert, but I am not sure this is correct.

Quoted 5 lines

I think it should be > instead of <. Also, it looks like you actually emit:
if (mbuf->data_len - off <= ins->opsz)
goto slow_path;

Could you please double check that both the comment and the emitted code has the comparison turning the correct way.

Ack, should be:
if (mbuf->data_len - off < ins->opsz)
Will send v2.

#9 lore raw

v2 cover bpf: add support for BPF_ABS/BPF_IND instructions

In reply to Konstantin Ananyev

To fill the gap with linux kernel eBPF implementation,
add support for two non-generic instructions:
(BPF_ABS | <size> | BPF_LD) and (BPF_IND | <size> | BPF_LD) which are used to access packet data in a safe manner. Make necessary changes in BPF verifier, interpreter and x86 JIT code.

Konstantin Ananyev (5):
test/bpf: fix few small issues
bpf: fix add/sub min/max estimations
bpf: add support for packet data load instructions test/bpf: add new test cases for mbuf load instructions
bpf: x86 JIT support for packet data load instructions

 app/test/test_bpf.c                    | 504 +++++++++++++++++++++++--
 doc/guides/prog_guide/bpf_lib.rst      |  30 +-
 doc/guides/rel_notes/release_20_08.rst |   7 +
 lib/librte_bpf/bpf_exec.c              |  57 +++
 lib/librte_bpf/bpf_jit_x86.c           | 181 +++++++++
 lib/librte_bpf/bpf_validate.c          | 104 ++++-
 6 files changed, 850 insertions(+), 33 deletions(-)
Signature · 1 line
2.17.1
#10 lore raw

PATCH v2 1/5 test/bpf: fix few small issues

Address for few small issues:
- unreachable return statement
- failed test-case can finish with 'success' status

Also use unified cmp_res() function to check return value.

Fixes a9de470cc7c0 ("test: move to app directory")
---
 app/test/test_bpf.c | 41 +++++++++++++++--------------------------
 1 file changed, 15 insertions(+), 26 deletions(-)
app/test/test_bpf.c +15 -26
index ee534687a..4a61a7d7c 100644
@@ -1797,13 +1797,6 @@ test_call1_check(uint64_t rc, const void *arg)
 	dummy_func1(arg, &v32, &v64);
 	v64 += v32;
 
-	if (v64 != rc) {
-		printf("%s@%d: invalid return value "
-			"expected=0x%" PRIx64 ", actual=0x%" PRIx64 "\n",
-			__func__, __LINE__, v64, rc);
-		return -1;
-	}
-	return 0;
 	return cmp_res(__func__, v64, rc, dv, dv, sizeof(*dv));
 }
 
@@ -1934,13 +1927,7 @@ test_call2_check(uint64_t rc, const void *arg)
 	dummy_func2(&a, &b);
 	v = a.u64 + a.u32 + b.u16 + b.u8;
 
-	if (v != rc) {
-		printf("%s@%d: invalid return value "
-			"expected=0x%" PRIx64 ", actual=0x%" PRIx64 "\n",
-			__func__, __LINE__, v, rc);
-		return -1;
-	}
-	return 0;
+	return cmp_res(__func__, v, rc, arg, arg, 0);
 }
 
 static const struct rte_bpf_xsym test_call2_xsym[] = {
@@ -2429,7 +2416,6 @@ test_call5_check(uint64_t rc, const void *arg)
 	v = 0;
 
 fail:
-
 	return cmp_res(__func__, v, rc, &v, &rc, sizeof(v));
 }
 
@@ -2458,6 +2444,7 @@ static const struct rte_bpf_xsym test_call5_xsym[] = {
 	},
 };
 
+/* all bpf test cases */
 static const struct bpf_test tests[] = {
 	{
 		.name = "test_store1",
@@ -2738,7 +2725,6 @@ run_test(const struct bpf_test *tst)
 	}
 
 	tst->prepare(tbuf);
-
 	rc = rte_bpf_exec(bpf, tbuf);
 	ret = tst->check_result(rc, tbuf);
 	if (ret != 0) {
@@ -2746,17 +2732,20 @@ run_test(const struct bpf_test *tst)
 			__func__, __LINE__, tst->name, ret, strerror(ret));
 	}
 
+	/* repeat the same test with jit, when possible */
 	rte_bpf_get_jit(bpf, &jit);
-	if (jit.func == NULL)
-		return 0;
-
-	tst->prepare(tbuf);
-	rc = jit.func(tbuf);
-	rv = tst->check_result(rc, tbuf);
-	ret |= rv;
-	if (rv != 0) {
-		printf("%s@%d: check_result(%s) failed, error: %d(%s);\n",
-			__func__, __LINE__, tst->name, rv, strerror(ret));
+	if (jit.func != NULL) {
+
+		tst->prepare(tbuf);
+		rc = jit.func(tbuf);
+		rv = tst->check_result(rc, tbuf);
+		ret |= rv;
+		if (rv != 0) {
+			printf("%s@%d: check_result(%s) failed, "
+				"error: %d(%s);\n",
+				__func__, __LINE__, tst->name,
+				rv, strerror(ret));
+		}
 	}
 
 	rte_bpf_destroy(bpf);
Signature · 1 line
2.17.1
#11 lore raw

PATCH v2 2/5 bpf: fix add/sub min/max estimations

In reply to Konstantin Ananyev

eval_add()/eval_sub() not always correctly estimate minimum and maximum possible values of add/sub operations.

Fixes 8021917293d0 ("bpf: add extra validation for input BPF program")
---
 lib/librte_bpf/bpf_validate.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)
lib/librte_bpf/bpf_validate.c +5 -5
index 6bd6f78e9..80d21fabb 100644
@@ -226,7 +226,7 @@ eval_add(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, uint64_t msk)
 	struct bpf_reg_val rv;
 
 	rv.u.min = (rd->u.min + rs->u.min) & msk;
-	rv.u.max = (rd->u.min + rs->u.max) & msk;
+	rv.u.max = (rd->u.max + rs->u.max) & msk;
 	rv.s.min = (rd->s.min + rs->s.min) & msk;
 	rv.s.max = (rd->s.max + rs->s.max) & msk;
 
@@ -254,10 +254,10 @@ eval_sub(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, uint64_t msk)
 {
 	struct bpf_reg_val rv;
 
-	rv.u.min = (rd->u.min - rs->u.min) & msk;
-	rv.u.max = (rd->u.min - rs->u.max) & msk;
-	rv.s.min = (rd->s.min - rs->s.min) & msk;
-	rv.s.max = (rd->s.max - rs->s.max) & msk;
+	rv.u.min = (rd->u.min - rs->u.max) & msk;
+	rv.u.max = (rd->u.max - rs->u.min) & msk;
+	rv.s.min = (rd->s.min - rs->s.max) & msk;
+	rv.s.max = (rd->s.max - rs->s.min) & msk;
 
 	/*
 	 * if at least one of the operands is not constant,
Signature · 1 line
2.17.1
#12 lore raw

PATCH v2 3/5 bpf: add support for packet data load instructions

In reply to Konstantin Ananyev

To fill the gap with linux kernel eBPF implementation, add support for two non-generic instructions:
(BPF_ABS | <size> | BPF_LD) and (BPF_IND | <size> | BPF_LD) which are used to access packet data.
These instructions can only be used when BPF context is a pointer to 'struct rte_mbuf' (i.e: RTE_BPF_ARG_PTR_MBUF type).

---
 doc/guides/prog_guide/bpf_lib.rst      | 30 +++++++++-
 doc/guides/rel_notes/release_20_08.rst |  7 +++
 lib/librte_bpf/bpf_exec.c              | 57 +++++++++++++++++++
 lib/librte_bpf/bpf_validate.c          | 77 ++++++++++++++++++++++++++
 4 files changed, 170 insertions(+), 1 deletion(-)
doc/guides/prog_guide/bpf_lib.rst +29 -1
index 9c728da7b..1feb7734a 100644
@@ -27,6 +27,35 @@ The library API provides the following basic operations:
 
 *   Load BPF program from the ELF file and install callback to execute it on given ethdev port/queue.
 
+Packet data load instructions
+-----------------------------
+
+DPDK supports two non-generic instructions: ``(BPF_ABS | size | BPF_LD)``
+and ``(BPF_IND | size | BPF_LD)`` which are used to access packet data.
+These instructions can only be used when execution context is a pointer to
+``struct rte_mbuf`` and have seven implicit operands.
+Register ``R6`` is an implicit input that must contain pointer to ``rte_mbuf``.
+Register ``R0`` is an implicit output which contains the data fetched from the
+packet. Registers ``R1-R5`` are scratch registers
+and must not be used to store the data across these instructions.
+These instructions have implicit program exit condition as well. When
+eBPF program is trying to access the data beyond the packet boundary,
+the interpreter will abort the execution of the program. JIT compilers
+therefore must preserve this property. ``src_reg`` and ``imm32`` fields are
+explicit inputs to these instructions.
+For example, ``(BPF_IND | BPF_W | BPF_LD)`` means:
+
+.. code-block:: c
+
+    uint32_t tmp;
+    R0 = rte_pktmbuf_read((const struct rte_mbuf *)R6,  src_reg + imm32,
+	sizeof(tmp), &tmp);
+    if (R0 == NULL) return FAILED;
+    R0 = ntohl(*(uint32_t *)R0);
+
+and ``R1-R5`` were scratched.
+
+
 Not currently supported eBPF features
 -------------------------------------
 
@@ -34,5 +63,4 @@ Not currently supported eBPF features
  - cBPF
  - tail-pointer call
  - eBPF MAP
- - skb
  - external function calls for 32-bit platforms
doc/guides/rel_notes/release_20_08.rst +7 -0
index 39064afbe..3c0824616 100644
@@ -56,6 +56,13 @@ New Features
      Also, make sure to start the actual text at the margin.
      =========================================================
 
+* **Added support for BPF_ABS/BPF_IND load instructions.**
+
+  Added support for two BPF non-generic instructions:
+  ``(BPF_ABS | <size> | BPF_LD)`` and ``(BPF_IND | <size> | BPF_LD)``
+  which are used to access packet data in a safe manner. Currently JIT support
+  for these instructions is implemented for x86 only.
+
 
 Removed Items
 -------------
lib/librte_bpf/bpf_exec.c +57 -0

75 lines not on this page · raw message

lib/librte_bpf/bpf_validate.c +77 -0

95 lines not on this page · raw message

Signature · 1 line
2.17.1
#13 lore raw

PATCH v2 4/5 test/bpf: add new test cases for mbuf load instructions

In reply to Konstantin Ananyev

Add new test-cases for BPF_ABS/BPF_IND load instructions.

---
 app/test/test_bpf.c | 463 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 463 insertions(+)
app/test/test_bpf.c +463 -0
index 4a61a7d7c..7c3de96c6 100644
@@ -43,6 +43,14 @@ struct dummy_net {
 	struct rte_ipv4_hdr ip_hdr;
 };
 
+#define	DUMMY_MBUF_NUM	2
+
+/* first mbuf in the packet, should always be at offset 0 */
+struct dummy_mbuf {
+	struct rte_mbuf mb[DUMMY_MBUF_NUM];
+	uint8_t buf[DUMMY_MBUF_NUM][RTE_MBUF_DEFAULT_BUF_SIZE];
+};
+
 #define	TEST_FILL_1	0xDEADBEEF
 
 #define	TEST_MUL_1	21
@@ -2444,6 +2452,413 @@ static const struct rte_bpf_xsym test_call5_xsym[] = {
 	},
 };
 
+/* load mbuf (BPF_ABS/BPF_IND) test-cases */
+static const struct ebpf_insn test_ld_mbuf1_prog[] = {
+
+	/* BPF_ABS/BPF_IND implicitly expect mbuf ptr in R6 */
+	{
+		.code = (EBPF_ALU64 | EBPF_MOV | BPF_X),
+		.dst_reg = EBPF_REG_6,
+		.src_reg = EBPF_REG_1,
+	},
+	/* load IPv4 version and IHL */
+	{
+		.code = (BPF_LD | BPF_ABS | BPF_B),
+		.imm = offsetof(struct rte_ipv4_hdr, version_ihl),
+	},
+	/* check IP version */
+	{
+		.code = (EBPF_ALU64 | EBPF_MOV | BPF_X),
+		.dst_reg = EBPF_REG_2,
+		.src_reg = EBPF_REG_0,
+	},
+	{
+		.code = (BPF_ALU | BPF_AND | BPF_K),
+		.dst_reg = EBPF_REG_2,
+		.imm = 0xf0,
+	},
+	{
+		.code = (BPF_JMP | BPF_JEQ | BPF_K),
+		.dst_reg = EBPF_REG_2,
+		.imm = IPVERSION << 4,
+		.off = 2,
+	},
+	/* invalid IP version, return 0 */
+	{
+		.code = (EBPF_ALU64 | BPF_XOR | BPF_X),
+		.dst_reg = EBPF_REG_0,
+		.src_reg = EBPF_REG_0,
+	},
+	{
+		.code = (BPF_JMP | EBPF_EXIT),
+	},
+	/* load 3-rd byte of IP data */
+	{
+		.code = (BPF_ALU | BPF_AND | BPF_K),
+		.dst_reg = EBPF_REG_0,
+		.imm = RTE_IPV4_HDR_IHL_MASK,
+	},
+	{
+		.code = (BPF_ALU | BPF_LSH | BPF_K),
+		.dst_reg = EBPF_REG_0,
+		.imm = 2,
+	},
+	{
+		.code = (BPF_LD | BPF_IND | BPF_B),
+		.src_reg = EBPF_REG_0,
+		.imm = 3,
+	},
+	{
+		.code = (EBPF_ALU64 | EBPF_MOV | BPF_X),
+		.dst_reg = EBPF_REG_7,
+		.src_reg = EBPF_REG_0,
+	},
+	/* load IPv4 src addr */
+	{
+		.code = (BPF_LD | BPF_ABS | BPF_W),
+		.imm = offsetof(struct rte_ipv4_hdr, src_addr),
+	},
+	{
+		.code = (EBPF_ALU64 | BPF_ADD | BPF_X),
+		.dst_reg = EBPF_REG_7,
+		.src_reg = EBPF_REG_0,
+	},
+	/* load IPv4 total length */
+	{
+		.code = (BPF_LD | BPF_ABS | BPF_H),
+		.imm = offsetof(struct rte_ipv4_hdr, total_length),
+	},
+	{
+		.code = (EBPF_ALU64 | EBPF_MOV | BPF_X),
+		.dst_reg = EBPF_REG_8,
+		.src_reg = EBPF_REG_0,
+	},
+	/* load last 4 bytes of IP data */
+	{
+		.code = (BPF_LD | BPF_IND | BPF_W),
+		.src_reg = EBPF_REG_8,
+		.imm = -(int32_t)sizeof(uint32_t),
+	},
+	{
+		.code = (EBPF_ALU64 | BPF_ADD | BPF_X),
+		.dst_reg = EBPF_REG_7,
+		.src_reg = EBPF_REG_0,
+	},
+	/* load 2 bytes from the middle of IP data */
+	{
+		.code = (EBPF_ALU64 | BPF_RSH | BPF_K),
+		.dst_reg = EBPF_REG_8,
+		.imm = 1,
+	},
+	{
+		.code = (BPF_LD | BPF_IND | BPF_H),
+		.src_reg = EBPF_REG_8,
+	},
+	{
+		.code = (EBPF_ALU64 | BPF_ADD | BPF_X),
+		.dst_reg = EBPF_REG_0,
+		.src_reg = EBPF_REG_7,
+	},
+	{
+		.code = (BPF_JMP | EBPF_EXIT),
+	},
+};
+
+static void
+dummy_mbuf_prep(struct rte_mbuf *mb, uint8_t buf[], uint32_t buf_len,
+	uint32_t data_len)
+{
+	uint32_t i;
+	uint8_t *db;
+
+	mb->buf_addr = buf;
+	mb->buf_iova = (uintptr_t)buf;
+	mb->buf_len = buf_len;
+	rte_mbuf_refcnt_set(mb, 1);
+
+	/* set pool pointer to dummy value, test doesn't use it */
+	mb->pool = (void *)buf;
+
+	rte_pktmbuf_reset(mb);
+	db = (uint8_t *)rte_pktmbuf_append(mb, data_len);
+
+	for (i = 0; i != data_len; i++)
+		db[i] = i;
+}
+
+static void
+test_ld_mbuf1_prepare(void *arg)
+{
+	struct dummy_mbuf *dm;
+	struct rte_ipv4_hdr *ph;
+
+	const uint32_t plen = 400;
+	const struct rte_ipv4_hdr iph = {
+		.version_ihl = RTE_IPV4_VHL_DEF,
+		.total_length = rte_cpu_to_be_16(plen),
+		.time_to_live = IPDEFTTL,
+		.next_proto_id = IPPROTO_RAW,
+		.src_addr = rte_cpu_to_be_32(RTE_IPV4_LOOPBACK),
+		.dst_addr = rte_cpu_to_be_32(RTE_IPV4_BROADCAST),
+	};
+
+	dm = arg;
+	memset(dm, 0, sizeof(*dm));
+
+	dummy_mbuf_prep(&dm->mb[0], dm->buf[0], sizeof(dm->buf[0]),
+		plen / 2 + 1);
+	dummy_mbuf_prep(&dm->mb[1], dm->buf[1], sizeof(dm->buf[0]),
+		plen / 2 - 1);
+
+	rte_pktmbuf_chain(&dm->mb[0], &dm->mb[1]);
+
+	ph = rte_pktmbuf_mtod(dm->mb, typeof(ph));
+	memcpy(ph, &iph, sizeof(iph));
+}
+
+static uint64_t
+test_ld_mbuf1(const struct rte_mbuf *pkt)
+{
+	uint64_t n, v;
+	const uint8_t *p8;
+	const uint16_t *p16;
+	const uint32_t *p32;
+	struct dummy_offset dof;
+
+	/* load IPv4 version and IHL */
+	p8 = rte_pktmbuf_read(pkt,
+		offsetof(struct rte_ipv4_hdr, version_ihl), sizeof(*p8),
+		&dof);
+	if (p8 == NULL)
+		return 0;
+
+	/* check IP version */
+	if ((p8[0] & 0xf0) != IPVERSION << 4)
+		return 0;
+
+	n = (p8[0] & RTE_IPV4_HDR_IHL_MASK) * RTE_IPV4_IHL_MULTIPLIER;
+
+	/* load 3-rd byte of IP data */
+	p8 = rte_pktmbuf_read(pkt, n + 3, sizeof(*p8), &dof);
+	if (p8 == NULL)
+		return 0;
+
+	v = p8[0];
+
+	/* load IPv4 src addr */
+	p32 = rte_pktmbuf_read(pkt,
+		offsetof(struct rte_ipv4_hdr, src_addr), sizeof(*p32),
+		&dof);
+	if (p32 == NULL)
+		return 0;
+
+	v += rte_be_to_cpu_32(p32[0]);
+
+	/* load IPv4 total length */
+	p16 = rte_pktmbuf_read(pkt,
+		offsetof(struct rte_ipv4_hdr, total_length), sizeof(*p16),
+		&dof);
+	if (p16 == NULL)
+		return 0;
+
+	n = rte_be_to_cpu_16(p16[0]);
+
+	/* load last 4 bytes of IP data */
+	p32 = rte_pktmbuf_read(pkt, n - sizeof(*p32), sizeof(*p32), &dof);
+	if (p32 == NULL)
+		return 0;
+
+	v += rte_be_to_cpu_32(p32[0]);
+
+	/* load 2 bytes from the middle of IP data */
+	p16 = rte_pktmbuf_read(pkt, n / 2, sizeof(*p16), &dof);
+	if (p16 == NULL)
+		return 0;
+
+	v += rte_be_to_cpu_16(p16[0]);
+	return v;
+}
+
+static int
+test_ld_mbuf1_check(uint64_t rc, const void *arg)
+{
+	const struct dummy_mbuf *dm;
+	uint64_t v;
+
+	dm = arg;
+	v = test_ld_mbuf1(dm->mb);
+	return cmp_res(__func__, v, rc, arg, arg, 0);
+}
+
+/*
+ * same as ld_mbuf1, but then trancate the mbuf by 1B,
+ * so load of last 4B fail.
+ */
+static void
+test_ld_mbuf2_prepare(void *arg)
+{
+	struct dummy_mbuf *dm;
+
+	test_ld_mbuf1_prepare(arg);
+	dm = arg;
+	rte_pktmbuf_trim(dm->mb, 1);
+}
+
+static int
+test_ld_mbuf2_check(uint64_t rc, const void *arg)
+{
+	return cmp_res(__func__, 0, rc, arg, arg, 0);
+}
+
+/* same as test_ld_mbuf1, but now store intermediate results on the stack */
+static const struct ebpf_insn test_ld_mbuf3_prog[] = {
+
+	/* BPF_ABS/BPF_IND implicitly expect mbuf ptr in R6 */
+	{
+		.code = (EBPF_ALU64 | EBPF_MOV | BPF_X),
+		.dst_reg = EBPF_REG_6,
+		.src_reg = EBPF_REG_1,
+	},
+	/* load IPv4 version and IHL */
+	{
+		.code = (BPF_LD | BPF_ABS | BPF_B),
+		.imm = offsetof(struct rte_ipv4_hdr, version_ihl),
+	},
+	/* check IP version */
+	{
+		.code = (EBPF_ALU64 | EBPF_MOV | BPF_X),
+		.dst_reg = EBPF_REG_2,
+		.src_reg = EBPF_REG_0,
+	},
+	{
+		.code = (BPF_ALU | BPF_AND | BPF_K),
+		.dst_reg = EBPF_REG_2,
+		.imm = 0xf0,
+	},
+	{
+		.code = (BPF_JMP | BPF_JEQ | BPF_K),
+		.dst_reg = EBPF_REG_2,
+		.imm = IPVERSION << 4,
+		.off = 2,
+	},
+	/* invalid IP version, return 0 */
+	{
+		.code = (EBPF_ALU64 | BPF_XOR | BPF_X),
+		.dst_reg = EBPF_REG_0,
+		.src_reg = EBPF_REG_0,
+	},
+	{
+		.code = (BPF_JMP | EBPF_EXIT),
+	},
+	/* load 3-rd byte of IP data */
+	{
+		.code = (BPF_ALU | BPF_AND | BPF_K),
+		.dst_reg = EBPF_REG_0,
+		.imm = RTE_IPV4_HDR_IHL_MASK,
+	},
+	{
+		.code = (BPF_ALU | BPF_LSH | BPF_K),
+		.dst_reg = EBPF_REG_0,
+		.imm = 2,
+	},
+	{
+		.code = (BPF_LD | BPF_IND | BPF_B),
+		.src_reg = EBPF_REG_0,
+		.imm = 3,
+	},
+	{
+		.code = (BPF_STX | BPF_MEM | BPF_B),
+		.dst_reg = EBPF_REG_10,
+		.src_reg = EBPF_REG_0,
+		.off = (int16_t)(offsetof(struct dummy_offset, u8) -
+			sizeof(struct dummy_offset)),
+	},
+	/* load IPv4 src addr */
+	{
+		.code = (BPF_LD | BPF_ABS | BPF_W),
+		.imm = offsetof(struct rte_ipv4_hdr, src_addr),
+	},
+	{
+		.code = (BPF_STX | BPF_MEM | BPF_W),
+		.dst_reg = EBPF_REG_10,
+		.src_reg = EBPF_REG_0,
+		.off = (int16_t)(offsetof(struct dummy_offset, u32) -
+			sizeof(struct dummy_offset)),
+	},
+	/* load IPv4 total length */
+	{
+		.code = (BPF_LD | BPF_ABS | BPF_H),
+		.imm = offsetof(struct rte_ipv4_hdr, total_length),
+	},
+	{
+		.code = (EBPF_ALU64 | EBPF_MOV | BPF_X),
+		.dst_reg = EBPF_REG_8,
+		.src_reg = EBPF_REG_0,
+	},
+	/* load last 4 bytes of IP data */
+	{
+		.code = (BPF_LD | BPF_IND | BPF_W),
+		.src_reg = EBPF_REG_8,
+		.imm = -(int32_t)sizeof(uint32_t),
+	},
+	{
+		.code = (BPF_STX | BPF_MEM | EBPF_DW),
+		.dst_reg = EBPF_REG_10,
+		.src_reg = EBPF_REG_0,
+		.off = (int16_t)(offsetof(struct dummy_offset, u64) -
+			sizeof(struct dummy_offset)),
+	},
+	/* load 2 bytes from the middle of IP data */
+	{
+		.code = (EBPF_ALU64 | BPF_RSH | BPF_K),
+		.dst_reg = EBPF_REG_8,
+		.imm = 1,
+	},
+	{
+		.code = (BPF_LD | BPF_IND | BPF_H),
+		.src_reg = EBPF_REG_8,
+	},
+	{
+		.code = (BPF_LDX | BPF_MEM | EBPF_DW),
+		.dst_reg = EBPF_REG_1,
+		.src_reg = EBPF_REG_10,
+		.off = (int16_t)(offsetof(struct dummy_offset, u64) -
+			sizeof(struct dummy_offset)),
+	},
+	{
+		.code = (EBPF_ALU64 | BPF_ADD | BPF_X),
+		.dst_reg = EBPF_REG_0,
+		.src_reg = EBPF_REG_1,
+	},
+	{
+		.code = (BPF_LDX | BPF_MEM | BPF_W),
+		.dst_reg = EBPF_REG_1,
+		.src_reg = EBPF_REG_10,
+		.off = (int16_t)(offsetof(struct dummy_offset, u32) -
+			sizeof(struct dummy_offset)),
+	},
+	{
+		.code = (EBPF_ALU64 | BPF_ADD | BPF_X),
+		.dst_reg = EBPF_REG_0,
+		.src_reg = EBPF_REG_1,
+	},
+	{
+		.code = (BPF_LDX | BPF_MEM | BPF_B),
+		.dst_reg = EBPF_REG_1,
+		.src_reg = EBPF_REG_10,
+		.off = (int16_t)(offsetof(struct dummy_offset, u8) -
+			sizeof(struct dummy_offset)),
+	},
+	{
+		.code = (EBPF_ALU64 | BPF_ADD | BPF_X),
+		.dst_reg = EBPF_REG_0,
+		.src_reg = EBPF_REG_1,
+	},
+	{
+		.code = (BPF_JMP | EBPF_EXIT),
+	},
+};
+
 /* all bpf test cases */
 static const struct bpf_test tests[] = {
 	{
@@ -2704,6 +3119,54 @@ static const struct bpf_test tests[] = {
 		/* for now don't support function calls on 32 bit platform */
 		.allow_fail = (sizeof(uint64_t) != sizeof(uintptr_t)),
 	},
+	{
+		.name = "test_ld_mbuf1",
+		.arg_sz = sizeof(struct dummy_mbuf),
+		.prm = {
+			.ins = test_ld_mbuf1_prog,
+			.nb_ins = RTE_DIM(test_ld_mbuf1_prog),
+			.prog_arg = {
+				.type = RTE_BPF_ARG_PTR_MBUF,
+				.buf_size = sizeof(struct dummy_mbuf),
+			},
+		},
+		.prepare = test_ld_mbuf1_prepare,
+		.check_result = test_ld_mbuf1_check,
+		/* mbuf as input argument is not supported on 32 bit platform */
+		.allow_fail = (sizeof(uint64_t) != sizeof(uintptr_t)),
+	},
+	{
+		.name = "test_ld_mbuf2",
+		.arg_sz = sizeof(struct dummy_mbuf),
+		.prm = {
+			.ins = test_ld_mbuf1_prog,
+			.nb_ins = RTE_DIM(test_ld_mbuf1_prog),
+			.prog_arg = {
+				.type = RTE_BPF_ARG_PTR_MBUF,
+				.buf_size = sizeof(struct dummy_mbuf),
+			},
+		},
+		.prepare = test_ld_mbuf2_prepare,
+		.check_result = test_ld_mbuf2_check,
+		/* mbuf as input argument is not supported on 32 bit platform */
+		.allow_fail = (sizeof(uint64_t) != sizeof(uintptr_t)),
+	},
+	{
+		.name = "test_ld_mbuf3",
+		.arg_sz = sizeof(struct dummy_mbuf),
+		.prm = {
+			.ins = test_ld_mbuf3_prog,
+			.nb_ins = RTE_DIM(test_ld_mbuf3_prog),
+			.prog_arg = {
+				.type = RTE_BPF_ARG_PTR_MBUF,
+				.buf_size = sizeof(struct dummy_mbuf),
+			},
+		},
+		.prepare = test_ld_mbuf1_prepare,
+		.check_result = test_ld_mbuf1_check,
+		/* mbuf as input argument is not supported on 32 bit platform */
+		.allow_fail = (sizeof(uint64_t) != sizeof(uintptr_t)),
+	},
 };
 
 static int
Signature · 1 line
2.17.1
#14 lore raw

PATCH v2 5/5 bpf: x86 JIT support for packet data load instructions

In reply to Konstantin Ananyev

Make x86 JIT to generate native code for
(BPF_ABS | <size> | BPF_LD) and (BPF_IND | <size> | BPF_LD) instructions.

---
 lib/librte_bpf/bpf_jit_x86.c  | 181 ++++++++++++++++++++++++++++++++++
 lib/librte_bpf/bpf_validate.c |  17 +++-
 2 files changed, 197 insertions(+), 1 deletion(-)
lib/librte_bpf/bpf_jit_x86.c +181 -0
index f70cd6be5..aa22ea78a 100644
@@ -87,6 +87,14 @@ enum {
 	REG_TMP1 = R10,
 };
 
+/* LD_ABS/LD_IMM offsets */
+enum {
+	LDMB_FSP_OFS, /* fast-path */
+	LDMB_SLP_OFS, /* slow-path */
+	LDMB_FIN_OFS, /* final part */
+	LDMB_OFS_NUM
+};
+
 /*
  * callee saved registers list.
  * keep RBP as the last one.
@@ -100,6 +108,9 @@ struct bpf_jit_state {
 		uint32_t num;
 		int32_t off;
 	} exit;
+	struct {
+		uint32_t stack_ofs;
+	} ldmb;
 	uint32_t reguse;
 	int32_t *off;
 	uint8_t *ins;
@@ -1024,6 +1035,166 @@ emit_div(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,
 		emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP1, RDX);
 }
 
+/*
+ * helper function, used by emit_ld_mbuf().
+ * generates code for 'fast_path':
+ * calculate load offset and check is it inside first packet segment.
+ */
+static void
+emit_ldmb_fast_path(struct bpf_jit_state *st, const uint32_t rg[EBPF_REG_7],
+	uint32_t sreg, uint32_t mode, uint32_t sz, uint32_t imm,
+	const int32_t ofs[LDMB_OFS_NUM])
+{
+	/* make R2 contain *off* value */
+
+	if (sreg != rg[EBPF_REG_2]) {
+		emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K,
+			rg[EBPF_REG_2], imm);
+		if (mode == BPF_IND)
+			emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X,
+				sreg, rg[EBPF_REG_2]);
+	} else
+		/* BPF_IND with sreg == R2 */
+		emit_alu_imm(st, EBPF_ALU64 | BPF_ADD | BPF_K,
+			rg[EBPF_REG_2], imm);
+
+	/* R3 = mbuf->data_len */
+	emit_ld_reg(st, BPF_LDX | BPF_MEM | BPF_H,
+		rg[EBPF_REG_6], rg[EBPF_REG_3],
+		offsetof(struct rte_mbuf, data_len));
+
+	/* R3 = R3 - R2 */
+	emit_alu_reg(st, EBPF_ALU64 | BPF_SUB | BPF_X,
+		rg[EBPF_REG_2], rg[EBPF_REG_3]);
+
+	/* JSLT R3, <sz> <slow_path> */
+	emit_cmp_imm(st, EBPF_ALU64, rg[EBPF_REG_3], sz);
+	emit_abs_jcc(st, BPF_JMP | EBPF_JSLT | BPF_K, ofs[LDMB_SLP_OFS]);
+
+	/* R3 = mbuf->data_off */
+	emit_ld_reg(st, BPF_LDX | BPF_MEM | BPF_H,
+		rg[EBPF_REG_6], rg[EBPF_REG_3],
+		offsetof(struct rte_mbuf, data_off));
+
+	/* R0 = mbuf->buf_addr */
+	emit_ld_reg(st, BPF_LDX | BPF_MEM | EBPF_DW,
+		rg[EBPF_REG_6], rg[EBPF_REG_0],
+		offsetof(struct rte_mbuf, buf_addr));
+
+	/* R0 = R0 + R3 */
+	emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X,
+		rg[EBPF_REG_3], rg[EBPF_REG_0]);
+
+	/* R0 = R0 + R2 */
+	emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X,
+		rg[EBPF_REG_2], rg[EBPF_REG_0]);
+
+	/* JMP <fin_part> */
+	emit_abs_jmp(st, ofs[LDMB_FIN_OFS]);
+}
+
+/*
+ * helper function, used by emit_ld_mbuf().
+ * generates code for 'slow_path':
+ * call __rte_pktmbuf_read() and check return value.
+ */
+static void
+emit_ldmb_slow_path(struct bpf_jit_state *st, const uint32_t rg[EBPF_REG_7],
+	uint32_t sz)
+{
+	/* make R3 contain *len* value (1/2/4) */
+
+	emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, rg[EBPF_REG_3], sz);
+
+	/* make R4 contain (RBP - ldmb.stack_ofs) */
+
+	emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RBP, rg[EBPF_REG_4]);
+	emit_alu_imm(st, EBPF_ALU64 | BPF_SUB | BPF_K, rg[EBPF_REG_4],
+		st->ldmb.stack_ofs);
+
+	/* make R1 contain mbuf ptr */
+
+	emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X,
+		rg[EBPF_REG_6], rg[EBPF_REG_1]);
+
+	/* call rte_pktmbuf_read */
+	emit_call(st, (uintptr_t)__rte_pktmbuf_read);
+
+	/* check that return value (R0) is not zero */
+	emit_tst_reg(st, EBPF_ALU64, rg[EBPF_REG_0], rg[EBPF_REG_0]);
+	emit_abs_jcc(st, BPF_JMP | BPF_JEQ | BPF_K, st->exit.off);
+}
+
+/*
+ * helper function, used by emit_ld_mbuf().
+ * generates final part of code for BPF_ABS/BPF_IND load:
+ * perform data load and endianness conversion.
+ * expects dreg to contain valid data pointer.
+ */
+static void
+emit_ldmb_fin(struct bpf_jit_state *st, uint32_t dreg, uint32_t opsz,
+	uint32_t sz)
+{
+	emit_ld_reg(st, BPF_LDX | BPF_MEM | opsz, dreg, dreg, 0);
+	if (sz != sizeof(uint8_t))
+		emit_be2le(st, dreg, sz * CHAR_BIT);
+}
+
+/*
+ * emit code for BPF_ABS/BPF_IND load.
+ * generates the following construction:
+ * fast_path:
+ *   off = ins->sreg + ins->imm
+ *   if (mbuf->data_len - off < ins->opsz)
+ *      goto slow_path;
+ *   ptr = mbuf->buf_addr + mbuf->data_off + off;
+ *   goto fin_part;
+ * slow_path:
+ *   typeof(ins->opsz) buf; //allocate space on the stack
+ *   ptr = __rte_pktmbuf_read(mbuf, off, ins->opsz, &buf);
+ *   if (ptr == NULL)
+ *      goto exit_label;
+ * fin_part:
+ *   res = *(typeof(ins->opsz))ptr;
+ *   res = bswap(res);
+ */
+static void
+emit_ld_mbuf(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t imm)
+{
+	uint32_t i, mode, opsz, sz;
+	uint32_t rg[EBPF_REG_7];
+	int32_t ofs[LDMB_OFS_NUM];
+
+	mode = BPF_MODE(op);
+	opsz = BPF_SIZE(op);
+	sz = bpf_size(opsz);
+
+	for (i = 0; i != RTE_DIM(rg); i++)
+		rg[i] = ebpf2x86[i];
+
+	/* fill with fake offsets */
+	for (i = 0; i != RTE_DIM(ofs); i++)
+		ofs[i] = st->sz + INT8_MAX;
+
+	/* dry run first to calculate jump offsets */
+
+	ofs[LDMB_FSP_OFS] = st->sz;
+	emit_ldmb_fast_path(st, rg, sreg, mode, sz, imm, ofs);
+	ofs[LDMB_SLP_OFS] = st->sz;
+	emit_ldmb_slow_path(st, rg, sz);
+	ofs[LDMB_FIN_OFS] = st->sz;
+	emit_ldmb_fin(st, rg[EBPF_REG_0], opsz, sz);
+
+	RTE_VERIFY(ofs[LDMB_FIN_OFS] - ofs[LDMB_FSP_OFS] <= INT8_MAX);
+
+	/* reset dry-run code and do a proper run */
+
+	st->sz = ofs[LDMB_FSP_OFS];
+	emit_ldmb_fast_path(st, rg, sreg, mode, sz, imm, ofs);
+	emit_ldmb_slow_path(st, rg, sz);
+	emit_ldmb_fin(st, rg[EBPF_REG_0], opsz, sz);
+}
+
 static void
 emit_prolog(struct bpf_jit_state *st, int32_t stack_size)
 {
@@ -1121,6 +1292,7 @@ emit(struct bpf_jit_state *st, const struct rte_bpf *bpf)
 	/* reset state fields */
 	st->sz = 0;
 	st->exit.num = 0;
+	st->ldmb.stack_ofs = bpf->stack_sz;
 
 	emit_prolog(st, bpf->stack_sz);
 
@@ -1240,6 +1412,15 @@ emit(struct bpf_jit_state *st, const struct rte_bpf *bpf)
 			emit_ld_imm64(st, dr, ins[0].imm, ins[1].imm);
 			i++;
 			break;
+		/* load absolute/indirect instructions */
+		case (BPF_LD | BPF_ABS | BPF_B):
+		case (BPF_LD | BPF_ABS | BPF_H):
+		case (BPF_LD | BPF_ABS | BPF_W):
+		case (BPF_LD | BPF_IND | BPF_B):
+		case (BPF_LD | BPF_IND | BPF_H):
+		case (BPF_LD | BPF_IND | BPF_W):
+			emit_ld_mbuf(st, op, sr, ins->imm);
+			break;
 		/* store instructions */
 		case (BPF_STX | BPF_MEM | BPF_B):
 		case (BPF_STX | BPF_MEM | BPF_H):
lib/librte_bpf/bpf_validate.c +16 -1
index fecdda0e1..9214f1503 100644
@@ -70,6 +70,7 @@ struct bpf_verifier {
 	uint64_t stack_sz;
 	uint32_t nb_nodes;
 	uint32_t nb_jcc_nodes;
+	uint32_t nb_ldmb_nodes;
 	uint32_t node_colour[MAX_NODE_COLOUR];
 	uint32_t edge_type[MAX_EDGE_TYPE];
 	struct bpf_eval_state *evst;
@@ -2020,6 +2021,14 @@ validate(struct bpf_verifier *bvf)
 			rc |= add_edge(bvf, node, i + 2);
 			i++;
 			break;
+		case (BPF_LD | BPF_ABS | BPF_B):
+		case (BPF_LD | BPF_ABS | BPF_H):
+		case (BPF_LD | BPF_ABS | BPF_W):
+		case (BPF_LD | BPF_IND | BPF_B):
+		case (BPF_LD | BPF_IND | BPF_H):
+		case (BPF_LD | BPF_IND | BPF_W):
+			bvf->nb_ldmb_nodes++;
+			/* fallthrough */
 		default:
 			rc |= add_edge(bvf, node, i + 1);
 			break;
@@ -2320,8 +2329,14 @@ bpf_validate(struct rte_bpf *bpf)
 	free(bvf.in);
 
 	/* copy collected info */
-	if (rc == 0)
+	if (rc == 0) {
 		bpf->stack_sz = bvf.stack_sz;
 
+		/* for LD_ABS/LD_IND, we'll need extra space on the stack */
+		if (bvf.nb_ldmb_nodes != 0)
+			bpf->stack_sz = RTE_ALIGN_CEIL(bpf->stack_sz +
+				sizeof(uint64_t), sizeof(uint64_t));
+	}
+
 	return rc;
 }
Signature · 1 line
2.17.1
Acked-by on v2 5/5
#15 lore raw

Re: [dpdk-dev] [PATCH v2 5/5] bpf: x86 JIT support for packet data load instructions

Quoted 9 lines

From: Konstantin Ananyev [mailto:[email protected]]
Sent: Wednesday, May 27, 2020 4:17 PM

Make x86 JIT to generate native code for
(BPF_ABS | <size> | BPF_LD) and (BPF_IND | <size> | BPF_LD) instructions.

---

[...]

Quoted 37 lines
lib/librte_bpf/bpf_jit_x86.c +7 -0
+/*
+ * helper function, used by emit_ld_mbuf().
+ * generates code for 'fast_path':
+ * calculate load offset and check is it inside first packet segment.
+ */
+static void
+emit_ldmb_fast_path(struct bpf_jit_state *st, const uint32_t

rg[EBPF_REG_7],

lib/librte_bpf/bpf_jit_x86.c +29 -0
+	uint32_t sreg, uint32_t mode, uint32_t sz, uint32_t imm,
+	const int32_t ofs[LDMB_OFS_NUM])
+{
+	/* make R2 contain *off* value */
+
+	if (sreg != rg[EBPF_REG_2]) {
+		emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K,
+			rg[EBPF_REG_2], imm);
+		if (mode == BPF_IND)
+			emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X,
+				sreg, rg[EBPF_REG_2]);
+	} else
+		/* BPF_IND with sreg == R2 */
+		emit_alu_imm(st, EBPF_ALU64 | BPF_ADD | BPF_K,
+			rg[EBPF_REG_2], imm);
+
+	/* R3 = mbuf->data_len */
+	emit_ld_reg(st, BPF_LDX | BPF_MEM | BPF_H,
+		rg[EBPF_REG_6], rg[EBPF_REG_3],
+		offsetof(struct rte_mbuf, data_len));
+
+	/* R3 = R3 - R2 */
+	emit_alu_reg(st, EBPF_ALU64 | BPF_SUB | BPF_X,
+		rg[EBPF_REG_2], rg[EBPF_REG_3]);
+
+	/* JSLT R3, <sz> <slow_path> */
+	emit_cmp_imm(st, EBPF_ALU64, rg[EBPF_REG_3], sz);
+	emit_abs_jcc(st, BPF_JMP | EBPF_JSLT | BPF_K, ofs[LDMB_SLP_OFS]);
+

[...]

Quoted 9 lines
lib/librte_bpf/bpf_jit_x86.c +9 -0
+
+/*
+ * emit code for BPF_ABS/BPF_IND load.
+ * generates the following construction:
+ * fast_path:
+ *   off = ins->sreg + ins->imm
+ *   if (mbuf->data_len - off < ins->opsz)
+ *      goto slow_path;
+ *   ptr = mbuf->buf_addr + mbuf->data_off + off;

lib/librte_bpf/bpf_jit_x86.c emit_div(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg, · line 1160

Quoted 10 lines
lib/librte_bpf/bpf_jit_x86.c +10 -0
+ *   goto fin_part;
+ * slow_path:
+ *   typeof(ins->opsz) buf; //allocate space on the stack
+ *   ptr = __rte_pktmbuf_read(mbuf, off, ins->opsz, &buf);
+ *   if (ptr == NULL)
+ *      goto exit_label;
+ * fin_part:
+ *   res = *(typeof(ins->opsz))ptr;
+ *   res = bswap(res);
+ */

[...]

The comparison for jumping to the slow path looks correct now.

I haven't reviewed it all in depth, but it certainly deserves an:

#16 lore raw
In reply to Konstantin Ananyev

27/05/2020 16:16, Konstantin Ananyev:

Quoted 6 lines

To fill the gap with linux kernel eBPF implementation,
add support for two non-generic instructions:
(BPF_ABS | <size> | BPF_LD) and (BPF_IND | <size> | BPF_LD) which are used to access packet data in a safe manner. Make necessary changes in BPF verifier, interpreter and x86 JIT code.

Quoted 6 lines

Konstantin Ananyev (5):
test/bpf: fix few small issues
bpf: fix add/sub min/max estimations
bpf: add support for packet data load instructions test/bpf: add new test cases for mbuf load instructions
bpf: x86 JIT support for packet data load instructions

Applied, thanks