Fixes for 5.10 v2
Jens Axboe
maintainerHi,
Resending, as git send-email aborted on me...
- fsize unification with the flags based setup
- NUMA affinity fix for hotplug/unplug
- loop_rw_iter() fix for the set_fs changes
- splice fix for the set_fs changes
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 1/4 io_uring: unify fsize with def->work_flags
This one was missed in the earlier conversion, should be included like any of the other IO identity flags. Make sure we restore to RLIM_INIFITY when dropping the personality again.
--- fs/io-wq.c | 8 ++++++-- fs/io-wq.h | 1 + fs/io_uring.c | 23 +++++++++++------------ 3 files changed, 18 insertions(+), 14 deletions(-)
fs/io-wq.c +6 -2
@@ -187,7 +187,8 @@ static bool __io_worker_unuse(struct io_wqe *wqe, struct io_worker *worker) worker->blkcg_css = NULL; }#endif - +if (current->signal->rlim[RLIMIT_FSIZE ].rlim_cur !=RLIM_INFINITY ) + current->signal->rlim[RLIMIT_FSIZE ].rlim_cur =RLIM_INFINITY ;return dropped_lock; }
@@ -483,7 +484,10 @@ static void io_impersonate_work(struct io_worker *worker,if ((work->flags &IO_WQ_WORK_CREDS ) && worker->cur_creds != work->identity->creds)io_wq_switch_creds (worker, work); - current->signal->rlim[RLIMIT_FSIZE ].rlim_cur = work->identity->fsize; +if (work->flags &IO_WQ_WORK_FSIZE ) + current->signal->rlim[RLIMIT_FSIZE ].rlim_cur = work->identity->fsize; +else if (current->signal->rlim[RLIMIT_FSIZE ].rlim_cur !=RLIM_INFINITY ) + current->signal->rlim[RLIMIT_FSIZE ].rlim_cur =RLIM_INFINITY ;io_wq_switch_blkcg (worker, work);#ifdef CONFIG_AUDIT current->loginuid = work->identity->loginuid;
fs/io-wq.h +1 -0
@@ -17,6 +17,7 @@ enum {IO_WQ_WORK_MM =128 ,IO_WQ_WORK_CREDS =256 ,IO_WQ_WORK_BLKCG =512 , +IO_WQ_WORK_FSIZE =1024 , IO_WQ_HASH_SHIFT = 24, /* upper 8 bits are used for hash key */ };
fs/io_uring.c +11 -12
@@ -753,8 +753,6 @@ struct io_op_def { unsigned pollout : 1;/* op supports buffer selection */ unsigned buffer_select : 1; -/* needs rlimit(RLIMIT_FSIZE) assigned */ - unsigned needs_fsize : 1;/* must always have async data allocated */ unsigned needs_async_data : 1;/* size of async data needed, if any */
@@ -778,10 +776,10 @@ static const struct io_op_def io_op_defs[] = { .hash_reg_file = 1, .unbound_nonreg_file = 1, .pollout = 1, - .needs_fsize = 1, .needs_async_data = 1, .async_size = sizeof(struct io_async_rw), - .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG, + .work_flags =IO_WQ_WORK_MM |IO_WQ_WORK_BLKCG | + IO_WQ_WORK_FSIZE, }, [IORING_OP_FSYNC] = { .needs_file = 1,
@@ -799,9 +797,8 @@ static const struct io_op_def io_op_defs[] = { .hash_reg_file = 1, .unbound_nonreg_file = 1, .pollout = 1, - .needs_fsize = 1, .async_size = sizeof(struct io_async_rw), - .work_flags = IO_WQ_WORK_BLKCG, + .work_flags = IO_WQ_WORK_BLKCG | IO_WQ_WORK_FSIZE, }, [IORING_OP_POLL_ADD] = { .needs_file = 1,
@@ -859,8 +856,7 @@ static const struct io_op_def io_op_defs[] = { }, [IORING_OP_FALLOCATE ] = { .needs_file =1 , - .needs_fsize =1 , - .work_flags =IO_WQ_WORK_BLKCG , + .work_flags =IO_WQ_WORK_BLKCG |IO_WQ_WORK_FSIZE , }, [IORING_OP_OPENAT ] = { .work_flags = IO_WQ_WORK_FILES | IO_WQ_WORK_BLKCG |
@@ -890,9 +886,9 @@ static const struct io_op_def io_op_defs[] = { .needs_file = 1, .unbound_nonreg_file = 1, .pollout = 1, - .needs_fsize = 1, .async_size = sizeof(struct io_async_rw), - .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG, + .work_flags =IO_WQ_WORK_MM |IO_WQ_WORK_BLKCG | + IO_WQ_WORK_FSIZE, }, [IORING_OP_FADVISE] = { .needs_file = 1,
@@ -1293,8 +1289,11 @@ static bool io_grab_identity(struct io_kiocb *req)struct io_identity *id = req->work.identity;struct io_ring_ctx *ctx = req->ctx; -if (def->needs_fsize && id->fsize !=rlimit (RLIMIT_FSIZE )) -return false; +if (def->work_flags &IO_WQ_WORK_FSIZE ) { +if (id->fsize !=rlimit (RLIMIT_FSIZE )) +return false; + req->work.flags |=IO_WQ_WORK_FSIZE ; + } if (!(req->work.flags & IO_WQ_WORK_FILES) && (def->work_flags & IO_WQ_WORK_FILES) &&
Signature · 1 line
2.29.0
Jens Axboe
maintainerPATCH 2/4 io-wq: re-set NUMA node affinities if CPUs come online
In reply to Jens AxboeWe correctly set io-wq NUMA node affinities when the io-wq context is setup, but if an entire node CPU set is offlined and then brought back online, the per node affinities are broken. Ensure that we set them again whenever a CPU comes online. This ensures that we always track the right node affinity. The usual cpuhp notifiers are used to drive it.
--- fs/io-wq.c | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 54 insertions(+), 4 deletions(-)
fs/io-wq.c +54 -4
@@ -19,6 +19,7 @@#include <linux/task_work.h> #include <linux/blk-cgroup.h> #include <linux/audit.h> +#include <linux/cpu.h> #include "io-wq.h"
@@ -123,9 +124,13 @@ struct io_wq {refcount_t refs;struct completion done; +struct hlist_node cpuhp_node; +refcount_t use_refs; }; +static enum cpuhp_state io_wq_online; +static bool io_worker_get (struct io_worker *worker) {return refcount_inc_not_zero (&worker->ref);
@@ -1091,10 +1096,12 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data)return ERR_PTR (-ENOMEM ); wq->wqes =kcalloc (nr_node_ids,sizeof (struct io_wqe *),GFP_KERNEL ); -if (!wq->wqes) { -kfree (wq); -return ERR_PTR (-ENOMEM ); - } +if (!wq->wqes) + goto err_wq; + + ret =cpuhp_state_add_instance_nocalls (io_wq_online, &wq->cpuhp_node); +if (ret) + goto err_wqes; wq->free_work = data->free_work; wq->do_work = data->do_work;
@@ -1102,6 +1109,7 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data)/* caller must already hold a reference to this */ wq->user = data->user; + ret = -ENOMEM ;for_each_node (node) {struct io_wqe *wqe;int alloc_node = node;
@@ -1145,9 +1153,12 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) ret =PTR_ERR (wq->manager);complete (&wq->done); err: +cpuhp_state_remove_instance_nocalls (io_wq_online, &wq->cpuhp_node);for_each_node (node)kfree (wq->wqes[node]); +err_wqes:kfree (wq->wqes); +err_wq:kfree (wq);return ERR_PTR (ret); }
@@ -1164,6 +1175,8 @@ static void __io_wq_destroy(struct io_wq *wq) {int node; +cpuhp_state_remove_instance_nocalls (io_wq_online, &wq->cpuhp_node); +set_bit (IO_WQ_BIT_EXIT , &wq->state);if (wq->manager)kthread_stop (wq->manager);
@@ -1191,3 +1204,40 @@ struct task_struct *io_wq_get_task(struct io_wq *wq) {return wq->manager; } + +static bool io_wq_worker_affinity (struct io_worker *worker,void *data) +{ +struct task_struct *task = worker->task; +unsigned long flags; + +raw_spin_lock_irqsave (&task->pi_lock, flags); +do_set_cpus_allowed (task,cpumask_of_node (worker->wqe->node)); + task->flags |=PF_NO_SETAFFINITY ; +raw_spin_unlock_irqrestore (&task->pi_lock, flags); +return false; +} + +static int io_wq_cpu_online (unsigned int cpu,struct hlist_node *node) +{ + struct io_wq *wq = hlist_entry_safe(node, struct io_wq, cpuhp_node); +int i; + +rcu_read_lock (); +for_each_node (i) +io_wq_for_each_worker (wq->wqes[i], io_wq_worker_affinity,NULL ); +rcu_read_unlock (); +return 0 ; +} + +static __init int io_wq_init (void ) +{ +int ret; + + ret =cpuhp_setup_state_multi (CPUHP_AP_ONLINE_DYN ,"io-wq/online" , + io_wq_cpu_online,NULL ); +if (ret <0 ) +return ret; + io_wq_online = ret; +return 0 ; +} +subsys_initcall (io_wq_init);
Signature · 1 line
2.29.0
Jens Axboe
maintainerPATCH 3/4 io_uring: make loop_rw_iter() use original user supplied pointers
In reply to Jens AxboeWe jump through a hoop for fixed buffers, where we first map these to a bvec(), then kmap() the bvec to obtain the pointer we copy to/from. This was always a bit ugly, and with the set_fs changes, it ends up being practically problematic as well.
There's no need to jump through these hoops, just use the original user pointers and length for the non iter based read/write.
--- fs/io_uring.c | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-)
fs/io_uring.c +9 -12
@@ -3115,9 +3115,10 @@ static inline loff_t *io_kiocb_ppos(struct kiocb *kiocb)* For files that don't have ->read_iter() and ->write_iter(), handle them * by looping over ->read() or ->write() manually. */ -static ssize_t loop_rw_iter (int rw,struct file *file,struct kiocb *kiocb, -struct iov_iter *iter) +static ssize_t loop_rw_iter (int rw,struct io_kiocb *req,struct iov_iter *iter) { +struct kiocb *kiocb = &req->rw.kiocb; +struct file *file = req->file;ssize_t ret =0 ;/*
@@ -3137,11 +3138,8 @@ static ssize_t loop_rw_iter(int rw, struct file *file, struct kiocb *kiocb,if (!iov_iter_is_bvec (iter)) { iovec =iov_iter_iovec (iter); }else { -/* fixed buffers import bvec */ - iovec.iov_base =kmap (iter->bvec->bv_page) - + iter->iov_offset; - iovec.iov_len =min (iter->count, - iter->bvec->bv_len - iter->iov_offset); + iovec.iov_base = (void __user *) req->rw.addr; + iovec.iov_len = req->rw.len; }if (rw ==READ ) {
@@ -3152,9 +3150,6 @@ static ssize_t loop_rw_iter(int rw, struct file *file, struct kiocb *kiocb, iovec.iov_len, io_kiocb_ppos(kiocb)); } -if (iov_iter_is_bvec (iter)) -kunmap (iter->bvec->bv_page); -if (nr <0 ) {if (!ret) ret = nr;
@@ -3163,6 +3158,8 @@ static ssize_t loop_rw_iter(int rw, struct file *file, struct kiocb *kiocb, ret += nr;if (nr != iovec.iov_len)break ; + req->rw.len -= nr; + req->rw.addr += nr;iov_iter_advance (iter, nr); }
@@ -3352,7 +3349,7 @@ static int io_iter_do_read(struct io_kiocb *req, struct iov_iter *iter)if (req->file->f_op->read_iter)return call_read_iter (req->file, &req->rw.kiocb, iter);else if (req->file->f_op->read) -return loop_rw_iter (READ , req->file, &req->rw.kiocb, iter); +return loop_rw_iter (READ , req, iter);else return -EINVAL ; }
@@ -3543,7 +3540,7 @@ static int io_write(struct io_kiocb *req, bool force_nonblock,if (req->file->f_op->write_iter) ret2 =call_write_iter (req->file, kiocb, iter);else if (req->file->f_op->write) - ret2 =loop_rw_iter (WRITE , req->file, kiocb, iter); + ret2 =loop_rw_iter (WRITE , req, iter);else ret2 = -EINVAL ;
Signature · 1 line
2.29.0
Jens Axboe
maintainerPATCH 4/4 splice: change exported internal do_splice() helper to take kernel offset
In reply to Jens AxboeWith the set_fs change, we can no longer rely on copy_{to,from}_user() accepting a kernel pointer. Clean this up and change the internal helper that io_uring uses to deal with kernel pointers instead. This puts the offset copy in/out in __do_splice() instead, which just calls the same helper.
--- fs/splice.c | 63 +++++++++++++++++++++++++++++++++--------- include/linux/splice.h | 4 +-- 2 files changed, 52 insertions(+), 15 deletions(-)
fs/splice.c +50 -13
@@ -1107,9 +1107,8 @@ static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,/* * Determine where to splice to/from. */ -long do_splice (struct file *in,loff_t __user *off_in, -struct file *out,loff_t __user *off_out, -size_t len,unsigned int flags) +long do_splice (struct file *in,loff_t *off_in,struct file *out, +loff_t *off_out,size_t len,unsigned int flags) {struct pipe_inode_info *ipipe;struct pipe_inode_info *opipe;
@@ -1143,8 +1142,7 @@ long do_splice(struct file *in, loff_t __user *off_in,if (off_out) {if (!(out->f_mode &FMODE_PWRITE ))return -EINVAL ; -if (copy_from_user (&offset, off_out,sizeof (loff_t))) -return -EFAULT ; + offset = *off_out; }else { offset = out->f_pos; }
@@ -1165,8 +1163,8 @@ long do_splice(struct file *in, loff_t __user *off_in,if (!off_out) out->f_pos = offset; -else if (copy_to_user (off_out, &offset,sizeof (loff_t))) - ret = -EFAULT ; +else + *off_out = offset;return ret; }
@@ -1177,8 +1175,7 @@ long do_splice(struct file *in, loff_t __user *off_in,if (off_in) {if (!(in->f_mode &FMODE_PREAD ))return -EINVAL ; -if (copy_from_user (&offset, off_in,sizeof (loff_t))) -return -EFAULT ; + offset = *off_in; }else { offset = in->f_pos; }
@@ -1202,8 +1199,8 @@ long do_splice(struct file *in, loff_t __user *off_in,wakeup_pipe_readers (opipe);if (!off_in) in->f_pos = offset; -else if (copy_to_user (off_in, &offset,sizeof (loff_t))) - ret = -EFAULT ; +else + *off_in = offset;return ret; }
@@ -1211,6 +1208,46 @@ long do_splice(struct file *in, loff_t __user *off_in,return -EINVAL ; } +static long __do_splice (struct file *in,loff_t __user *off_in, +struct file *out,loff_t __user *off_out, +size_t len,unsigned int flags) +{ +struct pipe_inode_info *ipipe; +struct pipe_inode_info *opipe; +loff_t offset, *__off_in =NULL , *__off_out =NULL ; +long ret; + + ipipe =get_pipe_info (in, true); + opipe =get_pipe_info (out, true); + +if (ipipe && off_in) +return -ESPIPE ; +if (opipe && off_out) +return -ESPIPE ; + +if (off_out) { +if (copy_from_user (&offset, off_out,sizeof (loff_t))) +return -EFAULT ; + __off_out = &offset; + } +if (off_in) { +if (copy_from_user (&offset, off_in,sizeof (loff_t))) +return -EFAULT ; + __off_in = &offset; + } + + ret =do_splice (in, __off_in, out, __off_out, len, flags); +if (ret <0 ) +return ret; + +if (__off_out &©_to_user (off_out, __off_out,sizeof (loff_t))) +return -EFAULT ; +if (__off_in &©_to_user (off_in, __off_in,sizeof (loff_t))) +return -EFAULT ; + +return ret; +} +static int iter_to_pipe (struct iov_iter *from,struct pipe_inode_info *pipe,unsigned flags)
@@ -1405,8 +1442,8 @@ SYSCALL_DEFINE6(splice, int, fd_in, loff_t __user *, off_in,if (in.file) { out =fdget (fd_out);if (out.file) { - error =do_splice (in.file, off_in, out.file, off_out, - len, flags); + error =__do_splice (in.file, off_in, out.file, off_out, + len, flags);fdput (out); }fdput (in);
include/linux/splice.h +2 -2
@@ -78,8 +78,8 @@ extern ssize_t add_to_pipe(struct pipe_inode_info *, struct pipe_buffer *);extern ssize_t splice_direct_to_actor (struct file *,struct splice_desc *,splice_direct_actor *); -extern long do_splice (struct file *in,loff_t __user *off_in, -struct file *out,loff_t __user *off_out, +extern long do_splice (struct file *in,loff_t *off_in, +struct file *out,loff_t *off_out,size_t len,unsigned int flags); extern long do_tee(struct file *in, struct file *out, size_t len,
Signature · 1 line
2.29.0