Skip to contentSkip to conversation

napi tracking strategy

PATCH Superseded Olivier Langlois 2 participants 6 messages Sep 16, 2024 – Sep 17, 2024 io_uring

v2 of 4 Superseded Latest is v4 →

lore
Share & tools
lore
#0 lore raw

v2 cover napi tracking strategy

the actual napi tracking strategy is inducing a non-negligeable overhead. Everytime a multishot poll is triggered or any poll armed, if the napi is enabled on the ring a lookup is performed to either add a new napi id into the napi_list or its timeout value is updated.

For many scenarios, this is overkill as the napi id list will be pretty much static most of the time. To address this common scenario, a new abstraction has been created following the common Linux kernel idiom of creating an abstract interface with a struct filled with function pointers.

the patch serie consist of very minor fixes followed by the core of the changes to implement the new feature.

v2 changes:
- extract small changes from the core changes to ease minor fixes backport
- totally remove the io_napi_tracking_ops interface

Olivier Langlois (3):
io_uring/napi: protect concurrent io_napi_entry timeout accesses io_uring/napi: fix io_napi_entry RCU accesses io_uring/napi: add static napi tracking strategy

 include/linux/io_uring_types.h |   2 +-
 include/uapi/linux/io_uring.h  |  32 +++++++-
 io_uring/fdinfo.c              |  36 +++++++++
 io_uring/napi.c                | 135 ++++++++++++++++++++++++++-------
 io_uring/napi.h                |  15 +++-
 5 files changed, 186 insertions(+), 34 deletions(-)
Signature · 1 line
2.46.0
#1 lore raw

PATCH v2 1/3 io_uring/napi: protect concurrent io_napi_entry timeout accesses

io_napi_entry timeout value can be updated while accessed from the poll functions.

Its concurrent accesses are wrapped with READ_ONCE()/WRITE_ONCE() macros to avoid incorrect compiler optimizations.

---
 io_uring/napi.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
io_uring/napi.c +3 -3
index 1de1d4d62925..1dd2df9da468 100644
@@ -60,7 +60,7 @@ void __io_napi_add(struct io_ring_ctx *ctx, struct socket *sock)
 	rcu_read_lock();
 	e = io_napi_hash_find(hash_list, napi_id);
 	if (e) {
-		e->timeout = jiffies + NAPI_TIMEOUT;
+		WRITE_ONCE(e->timeout, jiffies + NAPI_TIMEOUT);
 		rcu_read_unlock();
 		return;
 	}
@@ -92,7 +92,7 @@ static void __io_napi_remove_stale(struct io_ring_ctx *ctx)
 
 	spin_lock(&ctx->napi_lock);
 	hash_for_each(ctx->napi_ht, i, e, node) {
-		if (time_after(jiffies, e->timeout)) {
+		if (time_after(jiffies, READ_ONCE(e->timeout))) {
 			list_del(&e->list);
 			hash_del_rcu(&e->node);
 			kfree_rcu(e, rcu);
@@ -150,7 +150,7 @@ static bool __io_napi_do_busy_loop(struct io_ring_ctx *ctx,
 		napi_busy_loop_rcu(e->napi_id, loop_end, loop_end_arg,
 				   ctx->napi_prefer_busy_poll, BUSY_POLL_BUDGET);
 
-		if (time_after(jiffies, e->timeout))
+		if (time_after(jiffies, READ_ONCE(e->timeout)))
 			is_stale = true;
 	}
 
Signature · 1 line
2.46.0
#2 lore raw

PATCH v2 2/3 io_uring/napi: fix io_napi_entry RCU accesses

In reply to Olivier Langlois

correct 3 RCU structures modifications that were not using the RCU functions to make their update.

---
 io_uring/napi.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)
io_uring/napi.c +9 -2
index 1dd2df9da468..738d88bc050a 100644
@@ -81,7 +81,7 @@ void __io_napi_add(struct io_ring_ctx *ctx, struct socket *sock)
 	}
 
 	hlist_add_tail_rcu(&e->node, hash_list);
-	list_add_tail(&e->list, &ctx->napi_list);
+	list_add_tail_rcu(&e->list, &ctx->napi_list);
 	spin_unlock(&ctx->napi_lock);
 }
 
@@ -91,9 +91,15 @@ static void __io_napi_remove_stale(struct io_ring_ctx *ctx)
 	unsigned int i;
 
 	spin_lock(&ctx->napi_lock);
+	/*
+	 * hash_for_each_safe() is not required as long as:
+	 * 1. hash_del_rcu() does not reset the deleted node next pointer
+	 * 2. kfree_rcu() delays the memory freeing until the next quiescent
+	 *    state
+	 */
 	hash_for_each(ctx->napi_ht, i, e, node) {
 		if (time_after(jiffies, READ_ONCE(e->timeout))) {
-			list_del(&e->list);
+			list_del_rcu(&e->list);
 			hash_del_rcu(&e->node);
 			kfree_rcu(e, rcu);
 		}
@@ -207,6 +213,7 @@ void io_napi_free(struct io_ring_ctx *ctx)
 	unsigned int i;
 
 	spin_lock(&ctx->napi_lock);
+	INIT_LIST_HEAD_RCU(&ctx->napi_list);
 	hash_for_each(ctx->napi_ht, i, e, node) {
 		hash_del_rcu(&e->node);
 		kfree_rcu(e, rcu);
Signature · 1 line
2.46.0
#3 lore raw

PATCH v2 3/3 io_uring/napi: add static napi tracking strategy

In reply to Olivier Langlois

add the static napi tracking strategy that allows the user to manually manage the napi ids list to busy poll and offload the ring from dynamically update the list.

---
 include/linux/io_uring_types.h |   2 +-
 include/uapi/linux/io_uring.h  |  32 ++++++++-
 io_uring/fdinfo.c              |  36 ++++++++++
 io_uring/napi.c                | 118 +++++++++++++++++++++++++++------
 io_uring/napi.h                |  15 +++--
 5 files changed, 174 insertions(+), 29 deletions(-)
include/linux/io_uring_types.h +1 -1
index 3315005df117..f0a0cb2518bb 100644
@@ -406,7 +406,7 @@ struct io_ring_ctx {
 	/* napi busy poll default timeout */
 	ktime_t			napi_busy_poll_dt;
 	bool			napi_prefer_busy_poll;
-	bool			napi_enabled;
+	u8			napi_track_mode;
 
 	DECLARE_HASHTABLE(napi_ht, 4);
 #endif
include/uapi/linux/io_uring.h +30 -2
index adc2524fd8e3..10d9030c4242 100644
@@ -728,12 +728,40 @@ struct io_uring_buf_status {
 	__u32	resv[8];
 };
 
+enum io_uring_napi_op {
+	/* register/ungister backward compatible opcode */
+	IO_URING_NAPI_REGISTER_OP = 0,
+
+	/* opcodes to update napi_list when static tracking is used */
+	IO_URING_NAPI_STATIC_ADD_ID = 1,
+	IO_URING_NAPI_STATIC_DEL_ID = 2
+};
+
+enum io_uring_napi_tracking_strategy {
+	/* value must be 0 for backward compatibility */
+	IO_URING_NAPI_TRACKING_DYNAMIC = 0,
+	IO_URING_NAPI_TRACKING_STATIC = 1,
+	IO_URING_NAPI_TRACKING_INACTIVE = 255
+};
+
 /* argument for IORING_(UN)REGISTER_NAPI */
 struct io_uring_napi {
 	__u32	busy_poll_to;
 	__u8	prefer_busy_poll;
-	__u8	pad[3];
-	__u64	resv;
+
+	/* a io_uring_napi_op value */
+	__u8	opcode;
+	__u8	pad[2];
+
+	/*
+	 * for IO_URING_NAPI_REGISTER_OP, it is a
+	 * io_uring_napi_tracking_strategy value.
+	 *
+	 * for IO_URING_NAPI_STATIC_ADD_ID/IO_URING_NAPI_STATIC_DEL_ID
+	 * it is the napi id to add/del from napi_list.
+	 */
+	__u32	op_param;
+	__u32	resv;
 };
 
 /*
io_uring/fdinfo.c +36 -0
index b1e0e0d85349..595a066ba250 100644
@@ -46,6 +46,41 @@ static __cold int io_uring_show_cred(struct seq_file *m, unsigned int id,
 	return 0;
 }
 
+#ifdef CONFIG_NET_RX_BUSY_POLL
+static void common_tracking_show_fdinfo(struct io_ring_ctx *ctx,
+					struct seq_file *m,
+					const char *tracking_strategy)
+{
+	seq_puts(m, "NAPI:\tenabled\n");
+	seq_printf(m, "napi tracking:\t%s\n", tracking_strategy);
+	seq_printf(m, "napi_busy_poll_dt:\t%llu\n", ctx->napi_busy_poll_dt);
+	if (ctx->napi_prefer_busy_poll)
+		seq_puts(m, "napi_prefer_busy_poll:\ttrue\n");
+	else
+		seq_puts(m, "napi_prefer_busy_poll:\tfalse\n");
+}
+
+static void napi_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)
+{
+	switch (READ_ONCE(ctx->napi_track_mode)) {
+	case IO_URING_NAPI_TRACKING_INACTIVE:
+		seq_puts(m, "NAPI:\tdisabled\n");
+		break;
+	case IO_URING_NAPI_TRACKING_DYNAMIC:
+		common_tracking_show_fdinfo(ctx, m, "dynamic");
+		break;
+	case IO_URING_NAPI_TRACKING_STATIC:
+		common_tracking_show_fdinfo(ctx, m, "static");
+		break;
+	}
+}
+#else
+static inline void napi_show_fdinfo(struct io_ring_ctx *ctx,
+				    struct seq_file *m)
+{
+}
+#endif
+
 /*
  * Caller holds a reference to the file already, we don't need to do
  * anything else to get an extra reference.
@@ -223,5 +258,6 @@ __cold void io_uring_show_fdinfo(struct seq_file *m, struct file *file)
 	}
 
 	spin_unlock(&ctx->completion_lock);
+	napi_show_fdinfo(ctx, m);
 }
 #endif
io_uring/napi.c +96 -22

199 lines not on this page · raw message

io_uring/napi.h +11 -4
index 27b88c3eb428..220574522484 100644
@@ -15,7 +15,7 @@ void io_napi_free(struct io_ring_ctx *ctx);
 int io_register_napi(struct io_ring_ctx *ctx, void __user *arg);
 int io_unregister_napi(struct io_ring_ctx *ctx, void __user *arg);
 
-void __io_napi_add(struct io_ring_ctx *ctx, struct socket *sock);
+int __io_napi_add_id(struct io_ring_ctx *ctx, unsigned int napi_id);
 
 void __io_napi_adjust_timeout(struct io_ring_ctx *ctx,
 		struct io_wait_queue *iowq, ktime_t to_wait);
@@ -54,13 +54,20 @@ static inline void io_napi_add(struct io_kiocb *req)
 {
 	struct io_ring_ctx *ctx = req->ctx;
 	struct socket *sock;
+	struct sock *sk;
 
-	if (!READ_ONCE(ctx->napi_enabled))
+	if (READ_ONCE(ctx->napi_track_mode) != IO_URING_NAPI_TRACKING_DYNAMIC)
 		return;
 
 	sock = sock_from_file(req->file);
-	if (sock)
-		__io_napi_add(ctx, sock);
+	if (!sock)
+		return;
+
+	sk = sock->sk;
+	if (!sk)
+		return;
+
+	__io_napi_add_id(ctx, READ_ONCE(sk->sk_napi_id));
 }
 
 #else
Signature · 1 line
2.46.0
maintainer
#4 lore raw

Re: [PATCH v2 3/3] io_uring/napi: add static napi tracking strategy

Quoted 3 lines · Olivier Langlois

add the static napi tracking strategy that allows the user to manually manage the napi ids list to busy poll and offload the ring from dynamically update the list.

I like this, I suspect for many cases this is all you will need rather than try and dynamically track multiple instances.

Away for the next week or so, so won't have time to do a proper review until I'm back. Timing wise this doesn't matter too much as the 6.12 merge window is currently open, hence we cannot target this to anything sooner than 6.13. So we have time to get this reviewed and queued up.

A few minor comments below.

Quoted 13 lines
include/uapi/linux/io_uring.h +5 -0
index adc2524fd8e3..10d9030c4242 100644
@@ -728,12 +728,40 @@ struct io_uring_buf_status {
 	__u32	resv[8];
 };
 
+enum io_uring_napi_op {
+	/* register/ungister backward compatible opcode */
+	IO_URING_NAPI_REGISTER_OP = 0,
+
+	/* opcodes to update napi_list when static tracking is used */

include/uapi/linux/io_uring.h struct io_uring_buf_status { · line 745

Quoted 10 lines
include/uapi/linux/io_uring.h +10 -0
+	IO_URING_NAPI_STATIC_ADD_ID = 1,
+	IO_URING_NAPI_STATIC_DEL_ID = 2
+};
+
+enum io_uring_napi_tracking_strategy {
+	/* value must be 0 for backward compatibility */
+	IO_URING_NAPI_TRACKING_DYNAMIC = 0,
+	IO_URING_NAPI_TRACKING_STATIC = 1,
+	IO_URING_NAPI_TRACKING_INACTIVE = 255
+};

I think this is a fine way to do it, retaining compatability with what we have now on the registration side.

Quoted 11 lines
include/uapi/linux/io_uring.h +5 -2
 /* argument for IORING_(UN)REGISTER_NAPI */
 struct io_uring_napi {
 	__u32	busy_poll_to;
 	__u8	prefer_busy_poll;
-	__u8	pad[3];
-	__u64	resv;
+
+	/* a io_uring_napi_op value */
+	__u8	opcode;
+	__u8	pad[2];
+

include/uapi/linux/io_uring.h struct io_uring_buf_status { · line 765

Quoted 10 lines
include/uapi/linux/io_uring.h +9 -0
+	/*
+	 * for IO_URING_NAPI_REGISTER_OP, it is a
+	 * io_uring_napi_tracking_strategy value.
+	 *
+	 * for IO_URING_NAPI_STATIC_ADD_ID/IO_URING_NAPI_STATIC_DEL_ID
+	 * it is the napi id to add/del from napi_list.
+	 */
+	__u32	op_param;
+	__u32	resv;
 };

Looks good too.

Quoted 17 lines
include/uapi/linux/io_uring.h +17 -0
+static void common_tracking_show_fdinfo(struct io_ring_ctx *ctx,
+					struct seq_file *m,
+					const char *tracking_strategy)
+{
+	seq_puts(m, "NAPI:\tenabled\n");
+	seq_printf(m, "napi tracking:\t%s\n", tracking_strategy);
+	seq_printf(m, "napi_busy_poll_dt:\t%llu\n", ctx->napi_busy_poll_dt);
+	if (ctx->napi_prefer_busy_poll)
+		seq_puts(m, "napi_prefer_busy_poll:\ttrue\n");
+	else
+		seq_puts(m, "napi_prefer_busy_poll:\tfalse\n");
+}
+
+static void napi_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)
+{
+	switch (READ_ONCE(ctx->napi_track_mode)) {
+	case IO_URING_NAPI_TRACKING_INACTIVE:

include/uapi/linux/io_uring.h struct io_uring_buf_status {

Quoted 10 lines
include/uapi/linux/io_uring.h +10 -0
+		seq_puts(m, "NAPI:\tdisabled\n");
+		break;
+	case IO_URING_NAPI_TRACKING_DYNAMIC:
+		common_tracking_show_fdinfo(ctx, m, "dynamic");
+		break;
+	case IO_URING_NAPI_TRACKING_STATIC:
+		common_tracking_show_fdinfo(ctx, m, "static");
+		break;
+	}
+}

Maybe add an "unknown" default entry here, just in case it ever changes and someone forgets to update the fdinfo code.

include/uapi/linux/io_uring.h

Quoted 9 lines
include/uapi/linux/io_uring.h +9 -0
+static inline bool __io_napi_do_busy_loop(struct io_ring_ctx *ctx,
+					  void *loop_end_arg)
+{
+	if (READ_ONCE(ctx->napi_track_mode) == IO_URING_NAPI_TRACKING_STATIC)
+		return static_tracking_do_busy_loop(ctx, loop_end_arg);
+	else
+		return dynamic_tracking_do_busy_loop(ctx, loop_end_arg);
+}
+

Minor style nit:

	if (READ_ONCE(ctx->napi_track_mode) == IO_URING_NAPI_TRACKING_STATIC)
		return static_tracking_do_busy_loop(ctx, loop_end_arg);
	return dynamic_tracking_do_busy_loop(ctx, loop_end_arg);

would do.

Signature · 1 line
Jens Axboe
#5 lore raw

Re: [PATCH v2 3/3] io_uring/napi: add static napi tracking strategy

Quoted 13 lines · Jens Axboe
Quoted 12 lines · Olivier Langlois

+static void napi_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)

io_uring/fdinfo.c +9 -0
+{
+	switch (READ_ONCE(ctx->napi_track_mode)) {
+	case IO_URING_NAPI_TRACKING_INACTIVE:
+		seq_puts(m, "NAPI:\tdisabled\n");
+		break;
+	case IO_URING_NAPI_TRACKING_DYNAMIC:
+		common_tracking_show_fdinfo(ctx, m, "dynamic");
+		break;
+	case IO_URING_NAPI_TRACKING_STATIC:
Quoted 8 lines

io_uring/fdinfo.c static __cold int io_uring_show_cred(struct seq_file *m, unsigned int id, · line 76

Quoted 4 lines
io_uring/fdinfo.c +4 -0
+		common_tracking_show_fdinfo(ctx, m, "static");
+		break;
+	}
+}

Maybe add an "unknown" default entry here, just in case it ever changes
and someone forgets to update the fdinfo code.

ok

Quoted 16 lines

Review

Quoted 13 lines
io_uring/napi.c +4 -0
+static inline bool __io_napi_do_busy_loop(struct io_ring_ctx *ctx,
+					  void *loop_end_arg)
+{
+	if (READ_ONCE(ctx->napi_track_mode) ==

IO_URING_NAPI_TRACKING_STATIC)

(quoted diff) +1 -0
+		return static_tracking_do_busy_loop(ctx,

loop_end_arg);

(quoted diff) +2 -0
+	else
+		return dynamic_tracking_do_busy_loop(ctx,

loop_end_arg);

(quoted diff) +2 -0
+}
+

Minor style nit:

Quoted 8 lines
	if (READ_ONCE(ctx->napi_track_mode) ==
IO_URING_NAPI_TRACKING_STATIC)
		return static_tracking_do_busy_loop(ctx,
loop_end_arg);
	return dynamic_tracking_do_busy_loop(ctx, loop_end_arg);

would do.

will do.

Also, after looking at Pavel work at

https://github.com/isilence/linux/commits/manual-napi/

I have found a way to improve patch #2 of this serie...

v3 is going to be sweet!