Skip to contentSkip to conversation

libbpf: Fix ring buffers for 32-bit userspace on 64-bit kernels

PATCH New Samuel Wu 1 participant 1 message Nov 20

Subsystems (2)
lore
Share & tools
lore

Samuel Wu

author
#0 lore raw

PATCH 1/1 libbpf: Fix ring buffers for 32-bit userspace on 64-bit kernels

The kernel keeps ring buffer positions in unsigned long, so a 32-bit little-endian process on a 64-bit kernel accesses only the low 32 bits. Reading them is fine, as positions are never more than one ring size apart, but the upper 32 bits are always zero when writing. After 4GB:
- RINGBUF: the kernel sees consumer_pos 4GB behind, so bpf_ringbuf_reserve() always fails and epoll consumers spin.
- USER_RINGBUF: the kernel sees producer_pos behind consumer_pos, so bpf_user_ringbuf_drain() stops returning samples.

In a 32-bit little-endian process, write the position as a 64-bit value that carries into the upper half. On a 32-bit kernel, the upper 4 bytes land in user-writable memory that the kernel does not use for that map
type: consumer-page padding (RINGBUF), or padding/pending_pos (USER_RINGBUF). Big-endian 32-bit compat is not fixed because offset 0 maps the upper 32 bits, so even reads do not work without knowing the kernel word size.

Tested transferring >4GB over BPF_MAP_TYPE_RINGBUF with 32-bit userspace (armeabi-v7a) on a 64-bit kernel (aarch64), and verified no regression on 32-bit i386.

This needs commit fdd4fad0bbbd ("libbpf: Fix ring buffer consumer loop on 32-bit position wrap"): without it, the consumer stops when the low 32 bits wrap.

Fixes bf99c936f947 ("libbpf: Add BPF ring buffer support")
Fixes b66ccae01f1d ("bpf: Add libbpf logic for user-space ring buffer")
---
 tools/lib/bpf/ringbuf.c | 38 +++++++++++++++++++++++++++++++++-----
 1 file changed, 33 insertions(+), 5 deletions(-)
tools/lib/bpf/ringbuf.c +33 -5
index 78fcb3df243f..254e19ec51f2 100644
@@ -58,6 +58,35 @@ struct ringbuf_hdr {
 	__u32 pad;
 };
 
+/* 8-byte alignment lets compiler inline 64-bit __atomic_store_n() on i386. */
+typedef __u64 __attribute__((aligned(8), may_alias)) ringbuf_pos64_t;
+
+/*
+ * Carry 32-bit little-endian position updates into the upper 32 bits for
+ * 64-bit kernels. On 32-bit kernels the upper half lands in unused padding
+ * (RINGBUF) or unused pending_pos (USER_RINGBUF).
+ */
+static void ringbuf_store_pos(unsigned long *pos, unsigned long val)
+{
+	ringbuf_pos64_t *p = (void *)pos;
+	__u64 pos64;
+
+	if (sizeof(long) == 8 || __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) {
+		/* Pairs with smp_load_acquire() in the kernel. */
+		smp_store_release(pos, val);
+		return;
+	}
+
+	pos64 = READ_ONCE(*p);
+	pos64 += (__u32)(val - pos64);
+#if __GCC_ATOMIC_LLONG_LOCK_FREE == 2
+	__atomic_store_n(p, pos64, __ATOMIC_RELEASE);
+#else
+	/* Non-atomic store, but it only transiently fails reserve/peek. */
+	smp_store_release(p, pos64);
+#endif
+}
+
 static void ringbuf_free_ring(struct ring_buffer *rb, struct ring *r)
 {
 	if (r->consumer_pos) {
@@ -290,14 +319,13 @@ static int64_t ringbuf_process_ring(struct ring *r, size_t n)
 				err = r->sample_cb(r->ctx, sample, len);
 				if (err < 0) {
 					/* update consumer pos and bail out */
-					smp_store_release(r->consumer_pos,
-							  cons_pos);
+					ringbuf_store_pos(r->consumer_pos, cons_pos);
 					return err;
 				}
 				cnt++;
 			}
 
-			smp_store_release(r->consumer_pos, cons_pos);
+			ringbuf_store_pos(r->consumer_pos, cons_pos);
 
 			if (cnt >= n)
 				goto done;
@@ -412,7 +440,7 @@ struct ring *ring_buffer__ring(struct ring_buffer *rb, unsigned int idx)
 
 unsigned long ring__consumer_pos(const struct ring *r)
 {
-	/* Synchronizes with smp_store_release() in ringbuf_process_ring(). */
+	/* Synchronizes with ringbuf_store_pos() in ringbuf_process_ring(). */
 	return smp_load_acquire(r->consumer_pos);
 }
 
@@ -652,7 +680,7 @@ void *user_ring_buffer__reserve(struct user_ring_buffer *rb, __u32 size)
 	/* Synchronizes with smp_load_acquire() in __bpf_user_ringbuf_peek() in
 	 * the kernel.
 	 */
-	smp_store_release(rb->producer_pos, prod_pos + total_size);
+	ringbuf_store_pos(rb->producer_pos, prod_pos + total_size);
 
 	return (void *)rb->data + ((prod_pos + BPF_RINGBUF_HDR_SZ) & rb->mask);
 }
Signature · 1 line
2.56.1.809.ga0b61035b5-goog