Allow io-wq user configurable CPU masks
Jens Axboe
maintainercover Allow io-wq user configurable CPU masks
Hi,
This adds IORING_REGISTER_IOWQ_AFF that allows an application to specificy CPU affinities for any IO threads that may get created to service requests. Explanation in patch 2, patch 1 is just a prep patch that makes this easier to accomplish.
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 1/2 io-wq: use private CPU mask
In preparation for allowing user specific CPU masks for IO thread creation, switch to using a mask embedded in the per-node wqe structure.
--- fs/io-wq.c | 50 +++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 7 deletions(-)
fs/io-wq.c +43 -7
@@ -94,6 +94,8 @@ struct io_wqe {struct io_wq *wq;struct io_wq_work *hash_tail[IO_WQ_NR_HASH_BUCKETS ]; + +cpumask_var_t cpu_mask; };/*
@@ -638,7 +640,7 @@ static void create_io_worker(struct io_wq *wq, struct io_wqe *wqe, int index) tsk->pf_io_worker = worker; worker->task = tsk; -set_cpus_allowed_ptr (tsk,cpumask_of_node (wqe->node)); +set_cpus_allowed_ptr (tsk, wqe->cpu_mask); tsk->flags |=PF_NO_SETAFFINITY ;raw_spin_lock_irq (&wqe->lock);
@@ -922,6 +924,9 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) wqe =kzalloc_node (sizeof (struct io_wqe ),GFP_KERNEL , alloc_node);if (!wqe) goto err; +if (!alloc_cpumask_var (&wqe->cpu_mask,GFP_KERNEL )) + goto err; +cpumask_copy (wqe->cpu_mask,cpumask_of_node (node)); wq->wqes[node] = wqe; wqe->node = alloc_node; wqe->acct[IO_WQ_ACCT_BOUND ].index =IO_WQ_ACCT_BOUND ;
@@ -947,8 +952,12 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) err:io_wq_put_hash (data->hash);cpuhp_state_remove_instance_nocalls (io_wq_online, &wq->cpuhp_node); -for_each_node (node) +for_each_node (node) { +if (!wq->wqes[node]) +continue ; +free_cpumask_var (wq->wqes[node]->cpu_mask);kfree (wq->wqes[node]); + } err_wq:kfree (wq);return ERR_PTR (ret);
@@ -1018,6 +1027,7 @@ static void io_wq_destroy(struct io_wq *wq) .cancel_all = true, };io_wqe_cancel_pending_work (wqe, &match); +free_cpumask_var (wqe->cpu_mask);kfree (wqe); }io_wq_put_hash (wq->hash);
@@ -1032,31 +1042,57 @@ void io_wq_put_and_exit(struct io_wq *wq)io_wq_destroy (wq); } +struct online_data { +unsigned int cpu; +bool online; +}; +static bool io_wq_worker_affinity (struct io_worker *worker,void *data) { -set_cpus_allowed_ptr (worker->task,cpumask_of_node (worker->wqe->node)); +struct online_data *od = data; +if (od->online) +cpumask_set_cpu (od->cpu, worker->wqe->cpu_mask); +else +cpumask_clear_cpu (od->cpu, worker->wqe->cpu_mask);return false; } -static int io_wq_cpu_online (unsigned int cpu,struct hlist_node *node) +static int __io_wq_cpu_online (struct io_wq *wq,unsigned int cpu,bool online) { - struct io_wq *wq = hlist_entry_safe(node, struct io_wq, cpuhp_node); +struct online_data od = { + .cpu = cpu, + .online = online + };int i;rcu_read_lock ();for_each_node (i) -io_wq_for_each_worker (wq->wqes[i], io_wq_worker_affinity,NULL ); +io_wq_for_each_worker (wq->wqes[i], io_wq_worker_affinity, &od);rcu_read_unlock ();return 0 ; } +static int io_wq_cpu_online (unsigned int cpu,struct hlist_node *node) +{ + struct io_wq *wq = hlist_entry_safe(node, struct io_wq, cpuhp_node); + +return __io_wq_cpu_online (wq, cpu, true); +} + +static int io_wq_cpu_offline (unsigned int cpu,struct hlist_node *node) +{ + struct io_wq *wq = hlist_entry_safe(node, struct io_wq, cpuhp_node); + +return __io_wq_cpu_online (wq, cpu, false); +} +static __init int io_wq_init (void ) {int ret; ret =cpuhp_setup_state_multi (CPUHP_AP_ONLINE_DYN ,"io-wq/online" , - io_wq_cpu_online,NULL ); + io_wq_cpu_online, io_wq_cpu_offline);if (ret <0 )return ret; io_wq_online = ret;
Signature · 1 line
2.32.0
Jens Axboe
maintainerPATCH 2/2 io_uring: allow user configurable IO thread CPU affinity
In reply to Jens Axboeio-wq defaults to per-node masks for IO workers. This works fine by default, but isn't particularly handy for workloads that prefer more specific affinities, for either performance or isolation reasons.
This adds IORING_REGISTER_IOWQ_AFF that allows the user to pass in a CPU mask that is then applied to IO thread workers, and an IORING_UNREGISTER_IOWQ_AFF that simply resets the masks back to the default of per-node.
Note that no care is given to existing IO threads, they will need to go through a reschedule before the affinity is correct if they are already running or sleeping.
--- fs/io-wq.c | 17 ++++++++++++ fs/io-wq.h | 2 ++ fs/io_uring.c | 51 +++++++++++++++++++++++++++++++++++ include/uapi/linux/io_uring.h | 4 +++ 4 files changed, 74 insertions(+)
fs/io-wq.c +17 -0
@@ -1087,6 +1087,23 @@ static int io_wq_cpu_offline(unsigned int cpu, struct hlist_node *node)return __io_wq_cpu_online (wq, cpu, false); } +int io_wq_cpu_affinity (struct io_wq *wq,cpumask_var_t mask) +{ +int i; + +rcu_read_lock (); +for_each_node (i) { +struct io_wqe *wqe = wq->wqes[i]; + +if (mask) +cpumask_copy (wqe->cpu_mask, mask); +else +cpumask_copy (wqe->cpu_mask,cpumask_of_node (i)); + } +rcu_read_unlock (); +return 0 ; +} +static __init int io_wq_init (void ) {int ret;
fs/io-wq.h +2 -0
@@ -128,6 +128,8 @@ void io_wq_put_and_exit(struct io_wq *wq);void io_wq_enqueue (struct io_wq *wq,struct io_wq_work *work);void io_wq_hash_work (struct io_wq_work *work,void *val); +int io_wq_cpu_affinity (struct io_wq *wq,cpumask_var_t mask); +static inline bool io_wq_is_hashed (struct io_wq_work *work) {return work->flags &IO_WQ_WORK_HASHED ;
fs/io_uring.c +51 -0
69 lines not on this page · raw message
include/uapi/linux/io_uring.h +4 -0
@@ -306,6 +306,10 @@ enum {IORING_REGISTER_BUFFERS2 =15 ,IORING_REGISTER_BUFFERS_UPDATE =16 , +/* set/clear io-wq thread affinities */ +IORING_REGISTER_IOWQ_AFF =17 , +IORING_UNREGISTER_IOWQ_AFF =18 , +/* this goes last */ IORING_REGISTER_LAST };
Signature · 1 line
2.32.0