ublk: io_desc optimizations
Subsystems (3)
v2 cover ublk: io_desc optimizations
This series is based on the "ublk: harden user buffer handling" series.
ublk passes the parameters of incoming I/O in memory shared between the kernel ublk driver and userspace ublk server in struct ublksrv_io_desc.
The thread submitting the ublk I/O writes to the io_desc, while the ublk server thread handling the I/O reads the io_desc. This basically guarantees a cache miss on both threads for each ublk I/O. Avoid the cache misses by writing the io_desc on the server thread in the kernel before dispatching the I/O to userspace.
The size of each io_desc is currently fixed to 24 bytes, which has been an obstacle to extending it with additional fields [1]. Additionally, with multiple ublk server threads handling I/Os from the same ublk queue (possible with UBLK_F_PER_IO_DAEMON or UBLK_F_BATCH_IO), false sharing results from adjacent io_descs sharing the same cache line. Add a UBLK_F_IO_DESC_SIZE feature allowing the ublk server to increase the size of the io_descs for its ublk devices.
[1]: https://lore.kernel.org/linux-block/aV8QfvaNO5P6vOs6@fedora/
v2:
- Cap io_desc_size arbitrarily at 256 (Ming)
- Add kublk list support, selftest (Ming)
- Add Reviewed-by tags, Suggested-by tag (Ming)
v1: https://lore.kernel.org/linux-block/[email protected]/
Caleb Sander Mateos (8):
ublk: consistently use u16 for queue and tag numbers
ublk: remove struct ublk_zoned_report_desc's operation field
ublk: split request validation from io_desc init
ublk: initialize io_desc on daemon task
ublk: add UBLK_F_IO_DESC_SIZE
selftests: ublk: add support for --io_desc_size
selftests: ublk: add UBLK_F_IO_DESC_SIZE test
ublk: lift checks out of ublk_{,un}map_io()
drivers/block/ublk_drv.c | 261 ++++++++++--------- include/uapi/linux/ublk_cmd.h | 5 +- tools/testing/selftests/ublk/Makefile | 1 + tools/testing/selftests/ublk/kublk.c | 30 ++- tools/testing/selftests/ublk/kublk.h | 6 +- tools/testing/selftests/ublk/test_loop_08.sh | 25 ++ 6 files changed, 188 insertions(+), 140 deletions(-) create mode 100755 tools/testing/selftests/ublk/test_loop_08.sh
Signature · 1 line
2.54.0
PATCH v2 1/8 ublk: consistently use u16 for queue and tag numbers
The u16 nr_hw_queues and queue_depth fields of the ublk UAPI struct ublksrv_ctrl_dev_info constrain the number of queues and queue depth of each ublk device. However, the ublk driver is a bit inconsistent with the type it uses to represent these values, mixing u16 with int and unsigned int. Change all queue number, queue depth, q_id, and tag variables/fields to u16 to save some space.
--- drivers/block/ublk_drv.c | 80 ++++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 39 deletions(-)
drivers/block/ublk_drv.c +41 -39
@@ -235,22 +235,22 @@ struct ublk_io {void *buf_ctx_handle;spinlock_t lock; }____cacheline_aligned_in_smp ;struct ublk_queue { -int q_id; -int q_depth; +u16 q_id; +u16 q_depth;unsigned long flags;struct ublksrv_io_desc *io_cmd_buf;bool force_abort;bool canceling;bool fail_io;/* copy of dev->state == UBLK_S_DEV_FAIL_IO */ spinlock_t cancel_lock;struct ublk_device *dev; -u32 nr_io_ready; +u16 nr_io_ready;/* * For supporting UBLK_F_BATCH_IO only. * * Inflight ublk request tag is saved in this fifo
@@ -325,11 +325,11 @@ struct ublk_device {spinlock_t lock;struct mm_struct *mm;struct ublk_params params; -u32 nr_queue_ready; +u16 nr_queue_ready;bool unprivileged_daemons;struct mutex cancel_mutex;bool canceling;pid_t ublksrv_tgid;struct delayed_work exit_work;
@@ -401,11 +401,11 @@ static inline void ublk_io_evts_deinit(struct ublk_queue *q)WARN_ON_ONCE (!kfifo_is_empty (&q->evts_fifo));kfifo_free (&q->evts_fifo); }static inline struct ublksrv_io_desc * -ublk_get_iod (const struct ublk_queue *ubq,unsigned tag) +ublk_get_iod (const struct ublk_queue *ubq,u16 tag) {return &ubq->io_cmd_buf[tag]; }static inline bool ublk_support_zero_copy (const struct ublk_queue *ubq)
@@ -421,12 +421,11 @@ static inline bool ublk_dev_support_zero_copy(const struct ublk_device *ub)static inline bool ublk_support_shmem_zc (const struct ublk_queue *ubq) {return ubq->flags &UBLK_F_SHMEM_ZC ; } -static inline bool ublk_iod_is_shmem_zc (const struct ublk_queue *ubq, -unsigned int tag) +static inline bool ublk_iod_is_shmem_zc (const struct ublk_queue *ubq,u16 tag) {return ublk_get_iod (ubq, tag)->op_flags &UBLK_IO_F_SHMEM_ZC ; }static inline bool ublk_dev_support_shmem_zc (const struct ublk_device *ub)
@@ -862,22 +861,22 @@ static ssize_t ublk_batch_copy_io_tags(struct ublk_batch_fetch_cmd *fcmd,static unsigned int unprivileged_ublks_max =64 ;static unsigned int unprivileged_ublks_added;/* protected by ublk_ctl_mutex */ static struct miscdevice ublk_misc; -static inline unsigned ublk_pos_to_hwq (loff_t pos) +static inline u16 ublk_pos_to_hwq (loff_t pos) {return ((pos -UBLKSRV_IO_BUF_OFFSET ) >>UBLK_QID_OFF ) &UBLK_QID_BITS_MASK ; }static inline unsigned ublk_pos_to_buf_off (loff_t pos) {return (pos -UBLKSRV_IO_BUF_OFFSET ) &UBLK_IO_BUF_BITS_MASK ; } -static inline unsigned ublk_pos_to_tag (loff_t pos) +static inline u16 ublk_pos_to_tag (loff_t pos) {return ((pos -UBLKSRV_IO_BUF_OFFSET ) >>UBLK_TAG_OFF ) &UBLK_TAG_BITS_MASK ; }
@@ -1229,22 +1228,22 @@ static noinline void ublk_put_device(struct ublk_device *ub) {put_device (&ub->cdev_dev); }static inline struct ublk_queue *ublk_get_queue (struct ublk_device *dev, -int qid) +u16 qid) {return dev->queues[qid]; }static inline struct ublksrv_io_desc * -ublk_queue_cmd_buf (struct ublk_device *ub,int q_id) +ublk_queue_cmd_buf (struct ublk_device *ub,u16 q_id) {return ublk_get_queue (ub, q_id)->io_cmd_buf; } -static inline int __ublk_queue_cmd_buf_size (int depth) +static inline int __ublk_queue_cmd_buf_size (u16 depth) {return round_up (depth *sizeof (struct ublksrv_io_desc ),PAGE_SIZE ); }static inline int ublk_queue_cmd_buf_size (struct ublk_device *ub)
@@ -1665,11 +1664,11 @@ static inline void __ublk_abort_rq(struct ublk_queue *ubq,else ublk_end_request (rq,BLK_STS_IOERR ); }static void -ublk_auto_buf_reg_fallback (const struct ublk_queue *ubq,unsigned tag) +ublk_auto_buf_reg_fallback (const struct ublk_queue *ubq,u16 tag) {struct ublksrv_io_desc *iod =ublk_get_iod (ubq, tag); iod->op_flags |=UBLK_IO_F_NEED_REG_BUF ; }
@@ -1776,11 +1775,11 @@ static bool ublk_start_io(const struct ublk_queue *ubq, struct request *req, }static void ublk_dispatch_req (struct ublk_queue *ubq,struct request *req) {unsigned int issue_flags =IO_URING_CMD_TASK_WORK_ISSUE_FLAGS ; -int tag = req->tag; +u16 tag = req->tag;struct ublk_io *io = &ubq->ios[tag];pr_devel ("%s: complete: qid %d tag %d io_flags %x addr %llx\n" , __func__, ubq->q_id, req->tag, io->flags,ublk_get_iod (ubq, req->tag)->addr);
@@ -2366,11 +2365,11 @@ static const struct blk_mq_ops ublk_batch_mq_ops = { .timeout = ublk_timeout, };static void ublk_queue_reinit (struct ublk_device *ub,struct ublk_queue *ubq) { -int i; +u16 i; ubq->nr_io_ready =0 ;for (i =0 ; i < ubq->q_depth; i++) {struct ublk_io *io = &ubq->ios[i];
@@ -2411,11 +2410,11 @@ static int ublk_ch_open(struct inode *inode, struct file *filp)return 0 ; }static void ublk_reset_ch_dev (struct ublk_device *ub) { -int i; +u16 i;for (i =0 ; i < ub->dev_info.nr_hw_queues; i++) {struct ublk_queue *ubq =ublk_get_queue (ub, i);/* Sync with ublk_cancel_cmd() */
@@ -2483,20 +2482,20 @@ static void ublk_partition_scan_work(struct work_struct *work)* means. */ static void ublk_set_canceling (struct ublk_device *ub,bool canceling) __must_hold(&ub->cancel_mutex) { -int i; +u16 i; ub->canceling = canceling;for (i =0 ; i < ub->dev_info.nr_hw_queues; i++)ublk_get_queue (ub, i)->canceling = canceling; }static bool ublk_check_and_reset_active_ref (struct ublk_device *ub) { -int i, j; +u16 i, j;if (!ublk_dev_need_req_ref (ub))return false;for (i =0 ; i < ub->dev_info.nr_hw_queues; i++) {
@@ -2525,11 +2524,11 @@ static bool ublk_check_and_reset_active_ref(struct ublk_device *ub)static void ublk_ch_release_work_fn (struct work_struct *work) {struct ublk_device *ub = container_of(work, struct ublk_device, exit_work.work);struct gendisk *disk; -int i; +u16 i;/* * For zero-copy and auto buffer register modes, I/O references * might not be dropped naturally when the daemon is killed, but * io_uring guarantees that registered bvec kernel buffers are
@@ -2644,11 +2643,12 @@ static int ublk_ch_mmap(struct file *filp, struct vm_area_struct *vma) {struct ublk_device *ub = filp->private_data;size_t sz = vma->vm_end - vma->vm_start;unsigned max_sz =ublk_max_cmd_buf_size ();unsigned long pfn, end, phys_off = vma->vm_pgoff <<PAGE_SHIFT ; -int q_id, ret =0 ; +int ret =0 ; +u16 q_id;spin_lock (&ub->lock);if (!ub->mm) ub->mm = current->mm;if (current->mm != ub->mm)
@@ -2717,11 +2717,11 @@ static void ublk_abort_batch_queue(struct ublk_device *ub,* So no one can hold our request IO reference any more, simply ignore the * reference, and complete the request immediately */ static void ublk_abort_queue (struct ublk_device *ub,struct ublk_queue *ubq) { -int i; +u16 i;for (i =0 ; i < ubq->q_depth; i++) {struct ublk_io *io = &ubq->ios[i];if (io->flags &UBLK_IO_FLAG_OWNED_BY_SRV )
@@ -2760,11 +2760,11 @@ static void ublk_start_cancel(struct ublk_device *ub) out:mutex_unlock (&ub->cancel_mutex);ublk_put_disk (disk); } -static void ublk_cancel_cmd (struct ublk_queue *ubq,unsigned tag, +static void ublk_cancel_cmd (struct ublk_queue *ubq,u16 tag,unsigned int issue_flags) {struct ublk_io *io = &ubq->ios[tag];struct ublk_device *ub = ubq->dev;struct io_uring_cmd *cmd =NULL ;
@@ -2911,11 +2911,11 @@ static inline bool ublk_dev_ready(const struct ublk_device *ub)return ub->nr_queue_ready == ub->dev_info.nr_hw_queues; }static void ublk_cancel_queue (struct ublk_queue *ubq) { -int i; +u16 i;if (ublk_support_batch_io (ubq)) {ublk_batch_cancel_queue (ubq);return ; }
@@ -2925,11 +2925,11 @@ static void ublk_cancel_queue(struct ublk_queue *ubq) }/* Cancel all pending commands, must be called after del_gendisk() returns */ static void ublk_cancel_dev (struct ublk_device *ub) { -int i; +u16 i;for (i =0 ; i < ub->dev_info.nr_hw_queues; i++)ublk_cancel_queue (ublk_get_queue (ub, i)); }
@@ -2959,11 +2959,11 @@ static void ublk_wait_tagset_rqs_idle(struct ublk_device *ub) } }static void ublk_force_abort_dev (struct ublk_device *ub) { -int i; +u16 i;pr_devel ("%s: force abort ub: dev_id %d state %s\n" , __func__, ub->dev_info.dev_id, ub->dev_info.state ==UBLK_S_DEV_LIVE ?"LIVE" :"QUIESCED" );
@@ -3156,11 +3156,11 @@ ublk_config_io_buf(const struct ublk_device *ub, struct ublk_io *io,return 0 ; }static inline void ublk_prep_cancel (struct io_uring_cmd *cmd,unsigned int issue_flags, -struct ublk_queue *ubq,unsigned int tag) +struct ublk_queue *ubq,u16 tag) {struct ublk_uring_cmd_pdu *pdu =ublk_get_uring_cmd_pdu (cmd);/* * Safe to refer to @ubq since ublk_queue won't be died until its
@@ -3960,12 +3960,12 @@ static int ublk_handle_non_batch_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) {const struct ublksrv_io_cmd *ub_cmd =io_uring_sqe_cmd (cmd->sqe, struct ublksrv_io_cmd);struct ublk_device *ub = cmd->file->private_data; -unsigned tag =READ_ONCE (ub_cmd->tag); -unsigned q_id =READ_ONCE (ub_cmd->q_id); +u16 tag =READ_ONCE (ub_cmd->tag); +u16 q_id =READ_ONCE (ub_cmd->q_id);unsigned index =READ_ONCE (ub_cmd->addr);struct ublk_queue *ubq;struct ublk_io *io;if (cmd->cmd_op ==UBLK_U_IO_UNREGISTER_IO_BUF )
@@ -4167,11 +4167,12 @@ static const struct file_operations ublk_ch_batch_io_fops = { .mmap = ublk_ch_mmap, };static void __ublk_deinit_queue (struct ublk_device *ub,struct ublk_queue *ubq) { -int size, i; +int size; +u16 i; size =ublk_queue_cmd_buf_size (ub);for (i =0 ; i < ubq->q_depth; i++) {struct ublk_io *io = &ubq->ios[i];
@@ -4188,22 +4189,22 @@ static void __ublk_deinit_queue(struct ublk_device *ub, struct ublk_queue *ubq)ublk_io_evts_deinit (ubq);kvfree (ubq); } -static void ublk_deinit_queue (struct ublk_device *ub,int q_id) +static void ublk_deinit_queue (struct ublk_device *ub,u16 q_id) {struct ublk_queue *ubq = ub->queues[q_id];if (!ubq)return ;__ublk_deinit_queue (ub, ubq); ub->queues[q_id] =NULL ; } -static int ublk_get_queue_numa_node (struct ublk_device *ub,int q_id) +static int ublk_get_queue_numa_node (struct ublk_device *ub,u16 q_id) {unsigned int cpu;/* Find first CPU mapped to this queue */ for_each_possible_cpu (cpu) {
@@ -4212,18 +4213,19 @@ static int ublk_get_queue_numa_node(struct ublk_device *ub, int q_id) }return NUMA_NO_NODE ; } -static int ublk_init_queue (struct ublk_device *ub,int q_id) +static int ublk_init_queue (struct ublk_device *ub,u16 q_id) { -int depth = ub->dev_info.queue_depth; +u16 depth = ub->dev_info.queue_depth;gfp_t gfp_flags =GFP_KERNEL | __GFP_ZERO;struct ublk_queue *ubq;struct page *page;int numa_node; -int size, i, ret; +int size, ret; +u16 i;/* Determine NUMA node based on queue's CPU affinity */ numa_node =ublk_get_queue_numa_node (ub, q_id);/* Allocate queue structure on local NUMA node */
@@ -4264,19 +4266,20 @@ static int ublk_init_queue(struct ublk_device *ub, int q_id)return ret; }static void ublk_deinit_queues (struct ublk_device *ub) { -int i; +u16 i;for (i =0 ; i < ub->dev_info.nr_hw_queues; i++)ublk_deinit_queue (ub, i); }static int ublk_init_queues (struct ublk_device *ub) { -int i, ret; +int ret; +u16 i;for (i =0 ; i < ub->dev_info.nr_hw_queues; i++) { ret =ublk_init_queue (ub, i);if (ret) goto fail;
@@ -5179,11 +5182,11 @@ static int ublk_ctrl_set_size(struct ublk_device *ub, const struct ublksrv_ctrl_return ret; }struct count_busy {const struct ublk_queue *ubq; -unsigned int nr_busy; +u16 nr_busy; };static bool ublk_count_busy_req (struct request *rq,void *data) {struct count_busy *idle = data;
@@ -5217,12 +5220,11 @@ static int ublk_wait_for_idle_io(struct ublk_device *ub,*/ if (ublk_dev_support_batch_io (ub))return 0 ;while (elapsed < timeout_ms && !signal_pending (current)) { -unsigned int queues_cancelable =0 ; -int i; +u16 i, queues_cancelable =0 ;for (i =0 ; i < ub->dev_info.nr_hw_queues; i++) {struct ublk_queue *ubq =ublk_get_queue (ub, i); queues_cancelable += !!ubq_has_idle_io (ubq);
Signature · 1 line
2.54.0
PATCH v2 2/8 ublk: remove struct ublk_zoned_report_desc's operation field
In reply to Caleb Sander Mateosstruct ublk_zoned_report_desc's operation field is only ever set to UBLK_IO_OP_REPORT_ZONES, so remove it. Replace its one load with the constant.
--- drivers/block/ublk_drv.c | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-)
drivers/block/ublk_drv.c +3 -11
@@ -526,11 +526,10 @@ static void ublk_init_iod(struct ublk_queue *ubq, struct request *req,#ifdef CONFIG_BLK_DEV_ZONED struct ublk_zoned_report_desc {__u64 sector; -__u32 operation;__u32 nr_zones; };static DEFINE_XARRAY (ublk_zoned_report_descs);
@@ -656,11 +655,10 @@ static int ublk_report_zones(struct gendisk *disk, sector_t sector,if (IS_ERR (req)) { ret =PTR_ERR (req); goto out; } - desc.operation =UBLK_IO_OP_REPORT_ZONES ; desc.sector = sector; desc.nr_zones = zones_in_request; ret =ublk_zoned_insert_report_desc (req, &desc);if (ret) goto free_req;
@@ -729,19 +727,13 @@ static blk_status_t ublk_setup_iod_zoned(struct ublk_queue *ubq,break ;case REQ_OP_DRV_IN : desc =ublk_zoned_get_report_desc (req);if (!desc)return BLK_STS_IOERR ; - ublk_op = desc->operation; -switch (ublk_op) { -case UBLK_IO_OP_REPORT_ZONES : -ublk_init_iod (ubq, req, ublk_op, desc->nr_zones, - desc->sector); -return BLK_STS_OK ; -default : -return BLK_STS_IOERR ; - } +ublk_init_iod (ubq, req,UBLK_IO_OP_REPORT_ZONES , desc->nr_zones, + desc->sector); +return BLK_STS_OK ;case REQ_OP_DRV_OUT :/* We do not support drv_out */ return BLK_STS_NOTSUPP ;default :return BLK_STS_IOERR ;
Signature · 1 line
2.54.0
PATCH v2 3/8 ublk: split request validation from io_desc init
In reply to Caleb Sander MateosIn preparation for moving the struct ublksrv_io_desc initialization from the thread submitting ublk requests to the daemon thread receiving them, split the fallible part of ublk_setup_iod{,_zoned}() into new helper ublk_validate_req{,_zoned}(). Only ublk_setup_iod{,_zoned}() accesses the io_desc and cannot error out.
Return a bool value from ublk_validate_req{,_zoned}() as the existing error code ublk_setup_iod{,_zoned}() returns is only checked against BLK_STS_OK.
--- drivers/block/ublk_drv.c | 70 +++++++++++++++++++++++++++------------- 1 file changed, 48 insertions(+), 22 deletions(-)
drivers/block/ublk_drv.c +48 -22
@@ -698,12 +698,28 @@ static int ublk_report_zones(struct gendisk *disk, sector_t sector, out:kvfree (buffer);return ret; } -static blk_status_t ublk_setup_iod_zoned (struct ublk_queue *ubq, -struct request *req) +static bool ublk_validate_req_zoned (const struct request *req) +{ +switch (req_op (req)) { +case REQ_OP_ZONE_OPEN : +case REQ_OP_ZONE_CLOSE : +case REQ_OP_ZONE_FINISH : +case REQ_OP_ZONE_RESET : +case REQ_OP_ZONE_APPEND : +case REQ_OP_ZONE_RESET_ALL : +return true; +case REQ_OP_DRV_IN : +return !!ublk_zoned_get_report_desc (req); +default : +return false; + } +} + +static void ublk_setup_iod_zoned (struct ublk_queue *ubq,struct request *req) {struct ublk_zoned_report_desc *desc;u32 ublk_op;switch (req_op (req)) {
@@ -725,24 +741,19 @@ static blk_status_t ublk_setup_iod_zoned(struct ublk_queue *ubq,case REQ_OP_ZONE_RESET_ALL : ublk_op =UBLK_IO_OP_ZONE_RESET_ALL ;break ;case REQ_OP_DRV_IN : desc =ublk_zoned_get_report_desc (req); -if (!desc) -return BLK_STS_IOERR ;ublk_init_iod (ubq, req,UBLK_IO_OP_REPORT_ZONES , desc->nr_zones, desc->sector); -return BLK_STS_OK ; -case REQ_OP_DRV_OUT : -/* We do not support drv_out */ -return BLK_STS_NOTSUPP ; +return ;default : - return BLK_STS_IOERR; +WARN_ON_ONCE (1 ); + return; }ublk_init_iod (ubq, req, ublk_op,blk_rq_sectors (req),blk_rq_pos (req)); -return BLK_STS_OK ; }#else #define ublk_report_zones (NULL)
@@ -759,14 +770,18 @@ static void ublk_dev_param_zoned_apply(struct ublk_device *ub)static int ublk_revalidate_disk_zones (struct ublk_device *ub) {return 0 ; } -static blk_status_t ublk_setup_iod_zoned (struct ublk_queue *ubq, -struct request *req) +static bool ublk_validate_req_zoned (const struct request *req) { -return BLK_STS_NOTSUPP ; +return false; +} + +static void ublk_setup_iod_zoned (struct ublk_queue *ubq,struct request *req) +{ +WARN_ON_ONCE (1 ); }#endif static inline void __ublk_complete_rq(struct request *req, struct ublk_io *io,
@@ -1495,11 +1510,26 @@ static unsigned int ublk_unmap_io(bool need_map,return ublk_copy_user_pages (req,0 , &iter, dir); }return rq_bytes; } -static blk_status_t ublk_setup_iod (struct ublk_queue *ubq,struct request *req) +static bool ublk_validate_req (const struct ublk_queue *ubq, +const struct request *req) +{ +switch (req_op (req)) { +case REQ_OP_READ : +case REQ_OP_WRITE : +case REQ_OP_FLUSH : +case REQ_OP_DISCARD : +case REQ_OP_WRITE_ZEROES : +return true; +default : +return ublk_queue_is_zoned (ubq) &&ublk_validate_req_zoned (req); + } +} + +static void ublk_setup_iod (struct ublk_queue *ubq,struct request *req) {u32 ublk_op;switch (req_op (req)) {case REQ_OP_READ :
@@ -1516,17 +1546,15 @@ static blk_status_t ublk_setup_iod(struct ublk_queue *ubq, struct request *req)break ;case REQ_OP_WRITE_ZEROES : ublk_op =UBLK_IO_OP_WRITE_ZEROES ;break ;default : -if (ublk_queue_is_zoned (ubq)) -return ublk_setup_iod_zoned (ubq, req); -return BLK_STS_IOERR ; +ublk_setup_iod_zoned (ubq, req); +return ; }ublk_init_iod (ubq, req, ublk_op,blk_rq_sectors (req),blk_rq_pos (req)); -return BLK_STS_OK ; }static inline struct ublk_uring_cmd_pdu *ublk_get_uring_cmd_pdu (struct io_uring_cmd *ioucmd) {
@@ -2136,12 +2164,10 @@ static enum blk_eh_timer_return ublk_timeout(struct request *rq) }static blk_status_t ublk_prep_req (struct ublk_queue *ubq,struct request *rq,bool check_cancel) { -blk_status_t res; -if (unlikely (READ_ONCE (ubq->fail_io)))return BLK_STS_TARGET ;/* With recovery feature enabled, force_abort is set in * ublk_stop_dev() before calling del_gendisk(). We have to
@@ -2158,14 +2184,14 @@ static blk_status_t ublk_prep_req(struct ublk_queue *ubq, struct request *rq,if (check_cancel &&unlikely (ubq->canceling))return BLK_STS_IOERR ;/* fill iod to slot in io cmd buffer */ - res =ublk_setup_iod (ubq, rq); -if (unlikely (res !=BLK_STS_OK )) +if (unlikely (!ublk_validate_req (ubq, rq)))return BLK_STS_IOERR ; +ublk_setup_iod (ubq, rq);blk_mq_start_request (rq);return BLK_STS_OK ; }/*
Signature · 1 line
2.54.0
PATCH v2 4/8 ublk: initialize io_desc on daemon task
In reply to Caleb Sander Mateosublk_setup_iod() is currently called to populate struct ublksrv_io_desc on the thread submitting I/O to a ublk device. However, only the ublk server threads read the io_descs. This basically guarantees a cache miss on both threads for each ublk I/O. There's really no need to initialize the io_descs on the submitting thread. Move the ublk_setup_iod() call to ublk_dispatch_req() (for non-UBLK_F_BATCH_IO) and __ublk_batch_prep_dispatch() (for UBLK_F_BATCH_IO), which runs on the ublk server daemon thread before dispatching the I/O to userspace.
--- drivers/block/ublk_drv.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
drivers/block/ublk_drv.c +2 -1
@@ -1798,10 +1798,11 @@ static void ublk_dispatch_req(struct ublk_queue *ubq, struct request *req) {unsigned int issue_flags =IO_URING_CMD_TASK_WORK_ISSUE_FLAGS ;u16 tag = req->tag;struct ublk_io *io = &ubq->ios[tag]; +ublk_setup_iod (ubq, req);pr_devel ("%s: complete: qid %d tag %d io_flags %x addr %llx\n" , __func__, ubq->q_id, req->tag, io->flags,ublk_get_iod (ubq, req->tag)->addr);/*
@@ -1851,10 +1852,11 @@ static bool __ublk_batch_prep_dispatch(struct ublk_queue *ubq,struct ublk_io *io = &ubq->ios[tag];struct request *req =blk_mq_tag_to_rq (ub->tag_set.tags[ubq->q_id], tag);enum auto_buf_reg_res res =AUTO_BUF_REG_FALLBACK ;struct io_uring_cmd *cmd = data->cmd; +ublk_setup_iod (ubq, req);if (!ublk_start_io (ubq, req, io))return false;if (ublk_support_auto_buf_reg (ubq) &&blk_rq_has_data (req)) { res = ublk_auto_buf_register(ubq, req, io, cmd,
@@ -2187,11 +2189,10 @@ static blk_status_t ublk_prep_req(struct ublk_queue *ubq, struct request *rq,/* fill iod to slot in io cmd buffer */ if (unlikely (!ublk_validate_req (ubq, rq)))return BLK_STS_IOERR ; -ublk_setup_iod (ubq, rq);blk_mq_start_request (rq);return BLK_STS_OK ; }/*
Signature · 1 line
2.54.0
PATCH v2 5/8 ublk: add UBLK_F_IO_DESC_SIZE
In reply to Caleb Sander Mateosublk passes the parameters of incoming I/O in memory shared between the kernel ublk driver and userspace ublk server in struct ublksrv_io_desc. The size of this struct is currently fixed to 24 bytes, which has been an obstacle to extending it with additional fields [1]. Additionally, with multiple ublk server threads handling I/Os from the same ublk queue (possible with UBLK_F_PER_IO_DAEMON or UBLK_F_BATCH_IO), false sharing results from adjacent io_descs sharing the same cache line.
Add a ublk feature UBLK_F_IO_DESC_SIZE to allow a ublk server to override the size of each io_desc. The size must be at least 24 and a multiple of 8 to store a properly-aligned struct ublksrv_io_desc. It's also limited to a maximum of 256, though this bound could be lifted in the future.
The struct ublksrv_io_desc is located at the beginning of each io_desc and the remainder is padding. The mmap() performed for each queue must have a length of queue_depth * io_desc_size rounded up to the page size. The mmap() offset must be q_id * UBLK_MAX_QUEUE_DEPTH * io_desc_size, also rounded up to the page size.
[1]: https://lore.kernel.org/linux-block/aV8QfvaNO5P6vOs6@fedora/
--- drivers/block/ublk_drv.c | 38 ++++++++++++++++++++-------- include/uapi/linux/ublk_cmd.h | 5 +++- tools/testing/selftests/ublk/kublk.c | 1 + 3 files changed, 32 insertions(+), 12 deletions(-)
drivers/block/ublk_drv.c +27 -11
@@ -87,11 +87,12 @@ | UBLK_F_BUF_REG_OFF_DAEMON \ | (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) ? UBLK_F_INTEGRITY : 0) \ | UBLK_F_SAFE_STOP_DEV \ | UBLK_F_BATCH_IO \ | UBLK_F_NO_AUTO_PART_SCAN \ - | UBLK_F_SHMEM_ZC) + | UBLK_F_SHMEM_ZC \ + | UBLK_F_IO_DESC_SIZE)#define UBLK_F_ALL_RECOVERY_FLAGS (UBLK_F_USER_RECOVERY \ | UBLK_F_USER_RECOVERY_REISSUE \ | UBLK_F_USER_RECOVERY_FAIL_IO)
@@ -105,10 +106,12 @@#define UBLK_BATCH_F_ALL \ (UBLK_BATCH_F_HAS_ZONE_LBA | \ UBLK_BATCH_F_HAS_BUF_ADDR | \ UBLK_BATCH_F_AUTO_BUF_REG_FALLBACK) +#define UBLK_MAX_IO_DESC_SIZE 256 +/* ublk batch fetch uring_cmd */ struct ublk_batch_fetch_cmd {struct list_head node;struct io_uring_cmd *cmd;unsigned short buf_group;
@@ -237,10 +240,11 @@ struct ublk_io { }____cacheline_aligned_in_smp ;struct ublk_queue {u16 q_id;u16 q_depth; +u16 io_desc_size;unsigned long flags;struct ublksrv_io_desc *io_cmd_buf;bool force_abort;
@@ -403,11 +407,11 @@ static inline void ublk_io_evts_deinit(struct ublk_queue *q) }static inline struct ublksrv_io_desc *ublk_get_iod (const struct ublk_queue *ubq,u16 tag) { -return &ubq->io_cmd_buf[tag]; +return (void *)ubq->io_cmd_buf + tag * (size_t )ubq->io_desc_size; }static inline bool ublk_support_zero_copy (const struct ublk_queue *ubq) {return ubq->flags &UBLK_F_SUPPORT_ZERO_COPY ;
@@ -1246,23 +1250,24 @@ static inline struct ublksrv_io_desc * ublk_queue_cmd_buf(struct ublk_device *ub, u16 q_id) {return ublk_get_queue (ub, q_id)->io_cmd_buf; } -static inline int __ublk_queue_cmd_buf_size (u16 depth) +static inline size_t __ublk_queue_cmd_buf_size (const struct ublk_device *ub, +u16 depth) { -return round_up (depth *sizeof (struct ublksrv_io_desc ),PAGE_SIZE ); +return round_up (depth * (size_t )ub->dev_info.io_desc_size,PAGE_SIZE ); } -static inline int ublk_queue_cmd_buf_size (struct ublk_device *ub) +static inline size_t ublk_queue_cmd_buf_size (const struct ublk_device *ub) { -return __ublk_queue_cmd_buf_size (ub->dev_info.queue_depth); +return __ublk_queue_cmd_buf_size (ub, ub->dev_info.queue_depth); } -static int ublk_max_cmd_buf_size (void ) +static size_t ublk_max_cmd_buf_size (const struct ublk_device *ub) { -return __ublk_queue_cmd_buf_size (UBLK_MAX_QUEUE_DEPTH ); +return __ublk_queue_cmd_buf_size (ub,UBLK_MAX_QUEUE_DEPTH ); }/* * Should I/O outstanding to the ublk server when it exits be reissued? * If not, outstanding I/O will get errors.
@@ -2660,11 +2665,11 @@ static int ublk_ch_release(struct inode *inode, struct file *filp)/* map pre-allocated per-queue cmd buffer to ublksrv daemon */ static int ublk_ch_mmap (struct file *filp,struct vm_area_struct *vma) {struct ublk_device *ub = filp->private_data;size_t sz = vma->vm_end - vma->vm_start; -unsigned max_sz =ublk_max_cmd_buf_size (); +size_t max_sz =ublk_max_cmd_buf_size (ub);unsigned long pfn, end, phys_off = vma->vm_pgoff <<PAGE_SHIFT ;int ret =0 ;u16 q_id;spin_lock (&ub->lock);
@@ -4186,11 +4191,11 @@ static const struct file_operations ublk_ch_batch_io_fops = { .mmap = ublk_ch_mmap, };static void __ublk_deinit_queue (struct ublk_device *ub,struct ublk_queue *ubq) { -int size; +size_t size;u16 i; size =ublk_queue_cmd_buf_size (ub);for (i =0 ; i < ubq->q_depth; i++) {
@@ -4239,11 +4244,12 @@ static int ublk_init_queue(struct ublk_device *ub, u16 q_id)u16 depth = ub->dev_info.queue_depth;gfp_t gfp_flags =GFP_KERNEL | __GFP_ZERO;struct ublk_queue *ubq;struct page *page;int numa_node; -int size, ret; +size_t size; +int ret;u16 i;/* Determine NUMA node based on queue's CPU affinity */ numa_node =ublk_get_queue_numa_node (ub, q_id);
@@ -4264,10 +4270,11 @@ static int ublk_init_queue(struct ublk_device *ub, u16 q_id)if (!page) {kvfree (ubq);return -ENOMEM ; } ubq->io_cmd_buf =page_address (page); + ubq->io_desc_size = ub->dev_info.io_desc_size;for (i =0 ; i < ubq->q_depth; i++)spin_lock_init (&ubq->ios[i].lock);if (ublk_dev_support_batch_io (ub)) {
@@ -4748,10 +4755,19 @@ static int ublk_ctrl_add_dev(const struct ublksrv_ctrl_cmd *header)/* User copy is required to access integrity buffer */ if (info.flags &UBLK_F_INTEGRITY && !(info.flags &UBLK_F_USER_COPY ))return -EINVAL ; +if (info.flags &UBLK_F_IO_DESC_SIZE ) { +if (info.io_desc_size <sizeof (struct ublksrv_io_desc ) || + info.io_desc_size % _Alignof(struct ublksrv_io_desc ) || + info.io_desc_size >UBLK_MAX_IO_DESC_SIZE ) +return -EINVAL ; + }else { + info.io_desc_size =sizeof (struct ublksrv_io_desc ); + } +/* the created device is always owned by current user */ ublk_store_owner_uid_gid (&info.owner_uid, &info.owner_gid);if (header->dev_id != info.dev_id) { pr_warn("%s: dev id not match %u %u\n",
include/uapi/linux/ublk_cmd.h +4 -1
@@ -415,10 +415,13 @@ struct ublk_shmem_buf_reg {* pages match a registered buffer, UBLK_IO_F_SHMEM_ZC is set and addr * encodes the buffer index + offset instead of a userspace buffer address. */ #define UBLK_F_SHMEM_ZC (1ULL << 19) +/* ublksrv_io_desc size is specified by ublksrv_ctrl_dev_info's io_desc_size */ +#define UBLK_F_IO_DESC_SIZE (1ULL << 20) +/* device state */ #define UBLK_S_DEV_DEAD 0#define UBLK_S_DEV_LIVE 1#define UBLK_S_DEV_QUIESCED 2#define UBLK_S_DEV_FAIL_IO 3
@@ -450,11 +453,11 @@ struct ublksrv_ctrl_cmd {struct ublksrv_ctrl_dev_info {__u16 nr_hw_queues;__u16 queue_depth;__u16 state; -__u16 pad0; +__u16 io_desc_size;__u32 max_io_buf_bytes;__u32 dev_id;__s32 ublksrv_pid;
tools/testing/selftests/ublk/kublk.c +1 -0
@@ -1968,10 +1968,11 @@ static int cmd_dev_get_features(void)FEAT_NAME (UBLK_F_INTEGRITY ),FEAT_NAME (UBLK_F_SAFE_STOP_DEV ),FEAT_NAME (UBLK_F_BATCH_IO ),FEAT_NAME (UBLK_F_NO_AUTO_PART_SCAN ),FEAT_NAME (UBLK_F_SHMEM_ZC ), + FEAT_NAME(UBLK_F_IO_DESC_SIZE), };struct ublk_dev *dev;__u64 features =0 ;int ret;
Signature · 1 line
2.54.0
PATCH v2 6/8 selftests: ublk: add support for --io_desc_size
In reply to Caleb Sander MateosAdd an optional --io_desc_size argument to the kublk add/recover commands to enable UBLK_F_IO_DESC on the ublk device. The mmap() arguments and ublk_get_iod() computation are adjusted accordingly.
Display the configured io_desc_size in the kublk list output for ublk devices with UBLK_F_IO_DESC.
--- tools/testing/selftests/ublk/kublk.c | 29 +++++++++++++++++++--------- tools/testing/selftests/ublk/kublk.h | 6 ++++-- 2 files changed, 24 insertions(+), 11 deletions(-)
tools/testing/selftests/ublk/kublk.c +20 -9
@@ -350,10 +350,12 @@ static void ublk_ctrl_dump(struct ublk_dev *dev) info->dev_id, info->nr_hw_queues, info->queue_depth, 1 << p.basic.logical_bs_shift, p.basic.dev_sectors);ublk_log ("\tmax rq size %d daemon pid %d flags 0x%llx state %s\n" , info->max_io_buf_bytes, info->ublksrv_pid, info->flags,ublk_dev_state_desc (dev)); +if (info->flags &UBLK_F_IO_DESC_SIZE ) +ublk_log ("\tio_desc_size %u\n" , info->io_desc_size);if (affinity) {char buf[512 ];int i;
@@ -398,26 +400,26 @@ static struct ublk_dev *ublk_ctrl_init(void) dev->nr_fds =1 ;return dev; } -static int __ublk_queue_cmd_buf_sz (unsigned depth) +static size_t __ublk_queue_cmd_buf_sz (const struct ublk_queue *q,__u16 depth) { -int size = depth *sizeof (struct ublksrv_io_desc ); -unsigned int page_sz =getpagesize (); +size_t size = depth * (size_t )q->io_desc_size; +size_t page_sz =getpagesize ();return round_up (size, page_sz); } -static int ublk_queue_max_cmd_buf_sz (void ) +static size_t ublk_queue_max_cmd_buf_sz (const struct ublk_queue *q) { -return __ublk_queue_cmd_buf_sz (UBLK_MAX_QUEUE_DEPTH ); +return __ublk_queue_cmd_buf_sz (q,UBLK_MAX_QUEUE_DEPTH ); } -static int ublk_queue_cmd_buf_sz (struct ublk_queue *q) +static size_t ublk_queue_cmd_buf_sz (const struct ublk_queue *q) { -return __ublk_queue_cmd_buf_sz (q->q_depth); +return __ublk_queue_cmd_buf_sz (q, q->q_depth); }static void ublk_queue_deinit (struct ublk_queue *q) {int i;
@@ -451,26 +453,27 @@ static int ublk_queue_init(struct ublk_queue *q, unsigned long long extra_flags, __u8 metadata_size) {struct ublk_dev *dev = q->dev;int depth = dev->dev_info.queue_depth;int i; -int cmd_buf_size, io_buf_size, integrity_size; +size_t cmd_buf_size, io_buf_size, integrity_size;unsigned long off;pthread_spin_init (&q->lock,PTHREAD_PROCESS_PRIVATE ); q->tgt_ops = dev->tgt.ops; q->flags =0 ; q->q_depth = depth; q->flags = dev->dev_info.flags; q->flags |= extra_flags; q->metadata_size = metadata_size; + q->io_desc_size = dev->dev_info.io_desc_size;/* Cache fd in queue for fast path access */ q->ublk_fd = dev->fds[0 ]; cmd_buf_size =ublk_queue_cmd_buf_sz (q); - off =UBLKSRV_CMD_BUF_OFFSET + q->q_id *ublk_queue_max_cmd_buf_sz (); + off =UBLKSRV_CMD_BUF_OFFSET + q->q_id *ublk_queue_max_cmd_buf_sz (q); q->io_cmd_buf =mmap (0 , cmd_buf_size,PROT_READ ,MAP_SHARED |MAP_POPULATE , dev->fds[0 ], off);if (q->io_cmd_buf ==MAP_FAILED ) {ublk_err ("ublk dev %d queue %d map io_cmd_buf failed %m\n" , q->dev->dev_info.dev_id, q->q_id);
@@ -1706,10 +1709,11 @@ static int __cmd_dev_add(const struct dev_ctx *ctx) info = &dev->dev_info; info->dev_id = ctx->dev_id; info->nr_hw_queues = nr_queues; info->queue_depth = depth; + info->io_desc_size = ctx->io_desc_size; info->flags = ctx->flags;if ((features &UBLK_F_QUIESCE ) && (info->flags &UBLK_F_USER_RECOVERY )) info->flags |=UBLK_F_QUIESCE ; dev->nthreads = nthreads;
@@ -2067,10 +2071,11 @@ static void __cmd_create_help(char *exe, bool recovery)printf ("\t[-e 0|1 ] [-i 0|1] [--no_ublk_fixed_fd]\n" );printf ("\t[--nthreads threads] [--per_io_tasks]\n" );printf ("\t[--integrity_capable] [--integrity_reftag] [--metadata_size SIZE] " "[--pi_offset OFFSET] [--csum_type ip|t10dif|nvme] [--tag_size SIZE]\n" );printf ("\t[--batch|-b] [--no_auto_part_scan]\n" ); +printf ("\t[--io_desc_size SIZE]\n" );printf ("\t[target options] [backfile1] [backfile2] ...\n" );printf ("\tdefault: nr_queues=2(max 32), depth=128(max 1024), dev_id=-1(auto allocation)\n" );printf ("\tdefault: nthreads=nr_queues" );for (i =0 ; i <ARRAY_SIZE (tgt_ops_list); i++) {
@@ -2144,10 +2149,11 @@ int main(int argc, char *argv[]) { "batch", 0, NULL, 'b'}, { "no_auto_part_scan", 0, NULL, 0 }, { "shmem_zc", 0, NULL, 0 }, { "htlb", 1, NULL, 0 }, { "rdonly_shmem_buf", 0, NULL, 0 }, + { "io_desc_size", 1, NULL, 0 }, { 0, 0, 0, 0 } };const struct ublk_tgt_ops *ops =NULL ;int option_idx, opt;const char *cmd = argv[1 ];
@@ -2156,10 +2162,11 @@ int main(int argc, char *argv[]) .queue_depth = 128, .nr_hw_queues = 2, .dev_id = -1, .tgt_type = "unknown", .csum_type = LBMD_PI_CSUM_NONE, + .io_desc_size = sizeof(struct ublksrv_io_desc), };int ret = -EINVAL , i;int tgt_argc =1 ;char *tgt_argv[MAX_NR_TGT_ARG ] = {NULL };int value;
@@ -2265,10 +2272,14 @@ int main(int argc, char *argv[]) ctx.flags |=UBLK_F_SHMEM_ZC ;if (!strcmp (longopts[option_idx].name,"htlb" )) ctx.htlb_path =strdup (optarg);if (!strcmp (longopts[option_idx].name,"rdonly_shmem_buf" )) ctx.rdonly_shmem_buf =1 ; +if (!strcmp (longopts[option_idx].name,"io_desc_size" )) { + ctx.flags |=UBLK_F_IO_DESC_SIZE ; + ctx.io_desc_size =strtoul (optarg,NULL ,0 ); + }break ;case '?' :/* * target requires every option must have argument */
tools/testing/selftests/ublk/kublk.h +4 -2
@@ -85,10 +85,11 @@ struct dev_ctx {__u32 integrity_flags;__u8 metadata_size;__u8 pi_offset;__u8 csum_type;__u8 tag_size; +__u16 io_desc_size;int _evtfd;int _shmid;/* built from shmem, only for ublk_dump_dev() */
@@ -185,10 +186,11 @@ struct ublk_queue {#define UBLKS_Q_NO_UBLK_FIXED_FD (1ULL << 62)#define UBLKS_Q_PREPARED (1ULL << 61)__u64 flags;int ublk_fd;/* cached ublk char device fd */ __u8 metadata_size; +__u16 io_desc_size;struct ublk_io ios[UBLK_QUEUE_DEPTH ];/* used for prep io commands */ pthread_spinlock_t lock; };
@@ -459,13 +461,13 @@ static inline void ublk_mark_io_done(struct ublk_io *io, int res) { io->flags |= (UBLKS_IO_NEED_COMMIT_RQ_COMP |UBLKS_IO_FREE ); io->result = res; } -static inline const struct ublksrv_io_desc *ublk_get_iod (const struct ublk_queue *q,int tag) +static inline const struct ublksrv_io_desc *ublk_get_iod (const struct ublk_queue *q,__u16 tag) { -return &q->io_cmd_buf[tag]; +return (void *)q->io_cmd_buf + tag * (size_t )q->io_desc_size; }static inline void ublk_set_sqe_cmd_op (struct io_uring_sqe *sqe,__u32 cmd_op) {__u32 *addr = (__u32 *)&sqe->off;
Signature · 1 line
2.54.0
PATCH v2 7/8 selftests: ublk: add UBLK_F_IO_DESC_SIZE test
In reply to Caleb Sander MateosAdd test loop_08, which creates a ublk device with UBLK_F_IO_DESC_SIZE enabled and io_desc_size set to 64. The test issues verified I/O to the device using fio.
--- tools/testing/selftests/ublk/Makefile | 1 + tools/testing/selftests/ublk/test_loop_08.sh | 25 ++++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100755 tools/testing/selftests/ublk/test_loop_08.sh
tools/testing/selftests/ublk/Makefile +1 -0
@@ -32,10 +32,11 @@ TEST_PROGS += test_loop_02.sh TEST_PROGS += test_loop_03.sh TEST_PROGS += test_loop_04.sh TEST_PROGS += test_loop_05.sh TEST_PROGS += test_loop_06.sh TEST_PROGS += test_loop_07.sh +TEST_PROGS += test_loop_08.sh TEST_PROGS += test_integrity_01.sh TEST_PROGS += test_integrity_02.sh TEST_PROGS += test_integrity_03.sh
tools/testing/selftests/ublk/test_loop_08.sh new file +25 -0
@@ -0,0 +1,25 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +. "$(cd "$(dirname "$0")" && pwd)"/test_common.sh + +ERR_CODE=0 + +if ! _have_program fio; then + exit "$UBLK_SKIP_CODE" +fi + +_prep_test "loop" "write and verify with io_desc_size" + +_create_backfile 0 256M + +dev_id=$(_add_ublk_dev -t loop --io_desc_size 64 "${UBLK_BACKFILES[0]}") +_check_add_dev $TID $? + +# run fio over the ublk disk +_run_fio_verify_io --filename=/dev/ublkb"${dev_id}" --size=256M +ERR_CODE=$? + +_cleanup_test + +_show_result $TID $ERR_CODE
Signature · 1 line
2.54.0
PATCH v2 8/8 ublk: lift checks out of ublk_{,un}map_io()
In reply to Caleb Sander Mateosublk_map_io() and ublk_unmap_io() are no-ops for ublk devices that enable user copy or zero copy, as well as for requests without data to copy in the given direction. However, the implementation is a bit convoluted, returning the full request data length and relying on the caller to check the return value against the request length. UBLK_F_SHMEM_ZC recently added branches to skip the ublk_{,un}map_io() call for I/Os using a shared-memory buffer. This is a more logical place for the device need_map and the ublk_need_{,un}map_req() checks, so move them there from ublk_{,un}map_io().
Checking these conditions early also skips the expensive pointer-chasing for the ublk_iod_is_shmem_zc() check in __ublk_complete_rq() for the common case of a ublk device using user copy or zero copy.
Drop the req_op() filter in __ublk_complete_rq(), as it's redundant with the ublk_need_unmap_req() check.
--- drivers/block/ublk_drv.c | 70 +++++++++++----------------------------- 1 file changed, 19 insertions(+), 51 deletions(-)
drivers/block/ublk_drv.c +19 -51
@@ -1466,57 +1466,33 @@ static inline bool ublk_need_unmap_req(const struct request *req) {return blk_rq_has_data (req) && (req_op (req) ==REQ_OP_READ ||req_op (req) ==REQ_OP_DRV_IN ); } -static unsigned int ublk_map_io (const struct ublk_queue *ubq, -const struct request *req, +static unsigned int ublk_map_io (const struct request *req,const struct ublk_io *io) { -const unsigned int rq_bytes =blk_rq_bytes (req); +struct iov_iter iter; +const int dir =ITER_DEST ; -if (!ublk_need_map_io (ubq)) -return rq_bytes; - -/* -* no zero copy, we delay copy WRITE request data into ublksrv -* context and the big benefit is that pinning pages in current -* context is pretty fast, see ublk_pin_user_pages -*/ -if (ublk_need_map_req (req)) { -struct iov_iter iter; -const int dir =ITER_DEST ; - -if (import_ubuf (dir,u64_to_user_ptr (io->buf.addr), rq_bytes, - &iter) <0 ) -return 0 ; +if (import_ubuf (dir,u64_to_user_ptr (io->buf.addr),blk_rq_bytes (req), + &iter) <0 ) +return 0 ; -return ublk_copy_user_pages (req,0 , &iter, dir); - } -return rq_bytes; +return ublk_copy_user_pages (req,0 , &iter, dir); } -static unsigned int ublk_unmap_io (bool need_map, -const struct request *req, +static unsigned int ublk_unmap_io (const struct request *req,const struct ublk_io *io) { -const unsigned int rq_bytes =blk_rq_bytes (req); - -if (!need_map) -return rq_bytes; - -if (ublk_need_unmap_req (req)) { -struct iov_iter iter; -const int dir =ITER_SOURCE ; +struct iov_iter iter; +const int dir =ITER_SOURCE ; -if (import_ubuf (dir,u64_to_user_ptr (io->buf.addr), io->res, - &iter) <0 ) -return 0 ; +if (import_ubuf (dir,u64_to_user_ptr (io->buf.addr), io->res, &iter) <0 ) +return 0 ; -return ublk_copy_user_pages (req,0 , &iter, dir); - } -return rq_bytes; +return ublk_copy_user_pages (req,0 , &iter, dir); }static bool ublk_validate_req (const struct ublk_queue *ubq,const struct request *req) {
@@ -1588,26 +1564,17 @@ static inline void __ublk_complete_rq(struct request *req, struct ublk_io *io,if (io->res <0 ) { res =errno_to_blk_status (io->res); goto exit; } -/* -* FLUSH, DISCARD or WRITE_ZEROES usually won't return bytes returned, so end them -* directly. -* -* Both the two needn't unmap. -*/ -if (req_op (req) !=REQ_OP_READ &&req_op (req) !=REQ_OP_WRITE && -req_op (req) !=REQ_OP_DRV_IN ) - goto exit; -/* shmem zero copy: no data to unmap, pages already shared */ -if (ublk_iod_is_shmem_zc (req->mq_hctx->driver_data, req->tag)) +if (!need_map || !ublk_need_unmap_req (req) || +ublk_iod_is_shmem_zc (req->mq_hctx->driver_data, req->tag)) goto exit;/* for READ request, writing data in iod->addr to rq buffers */ - unmapped_bytes =ublk_unmap_io (need_map, req, io); + unmapped_bytes =ublk_unmap_io (req, io);/* * Extremely impossible since we got data filled in just before * * Re-read simply for this unlikely case.
@@ -1769,14 +1736,15 @@ static bool ublk_start_io(const struct ublk_queue *ubq, struct request *req, struct ublk_io *io) {unsigned mapped_bytes;/* shmem zero copy: skip data copy, pages already shared */ -if (ublk_iod_is_shmem_zc (ubq, req->tag)) +if (!ublk_need_map_io (ubq) || !ublk_need_map_req (req) || +ublk_iod_is_shmem_zc (ubq, req->tag))return true; - mapped_bytes =ublk_map_io (ubq, req, io); + mapped_bytes =ublk_map_io (req, io);/* partially mapped, update io descriptor */ if (unlikely (mapped_bytes !=blk_rq_bytes (req))) {/* * Nothing mapped, retry until we succeed.
Signature · 1 line
2.54.0
Quoted 5 lines · Caleb Sander Mateos
This series is based on the "ublk: harden user buffer handling" series.
ublk passes the parameters of incoming I/O in memory shared between the kernel ublk driver and userspace ublk server in struct
ublksrv_io_desc.
Quoted 7 lines
The thread submitting the ublk I/O writes to the
io_desc, while the ublk server thread handling the I/O reads theio_desc. This basically guarantees a cache miss on both threads for each ublk I/O. Avoid the cache misses by writing theio_descon the server thread in the kernel before dispatching the I/O to userspace.[...]
Applied, thanks!
[1/8] ublk: consistently use u16 for queue and tag numbers
commit: f510198855b6ddac0ffe62684dc407c56d3e0f24
[2/8] ublk: remove struct ublk_zoned_report_desc's operation field
commit: 3a00b782a7b63b1876261ab883e1d8e1aa0e09fb
[3/8] ublk: split request validation from io_desc init
commit: 8c76625ff9360cbe9ce0c47a624877aea14b3499
[4/8] ublk: initialize io_desc on daemon task
commit: 735409f58b3da45094f2dfd7c18fb9e1937431f1
[5/8] ublk: add UBLK_F_IO_DESC_SIZE
commit: 5c0958d80190822c4614ca00478d9acd9671bba6
[6/8] selftests: ublk: add support for --io_desc_size
commit: fc01b96d74b3a9eec2b558d49ec9f5176473766a
[7/8] selftests: ublk: add UBLK_F_IO_DESC_SIZE test
commit: d61d0f95e686be015cfaf193c0ae15156d1a0cc4
[8/8] ublk: lift checks out of ublk_{,un}map_io()
commit: a8a79eba22dc4c11f2877bcf9e8557f6d95541acBest regards,
Signature · 1 line
Jens Axboe