virtio-blk: support polling I/O and mq_ops->queue_rqs()
Subsystems (3)
v2 cover virtio-blk: support polling I/O and mq_ops->queue_rqs()
This patch serise adds support for polling I/O and mq_ops->queue_rqs() to virtio-blk driver.
Changes v1 -> v2
- To receive the number of poll queues from user, use module parameter instead of QEMU uapi change.
- Add the comment about virtblk_map_queues().
- Add support for mq_ops->queue_rqs() to implement submit side batch.
Suwan Kim (2):
virtio-blk: support polling I/O
virtio-blk: support mq_ops->queue_rqs()
drivers/block/virtio_blk.c | 194 ++++++++++++++++++++++++++++++++++--- 1 file changed, 181 insertions(+), 13 deletions(-)
Signature · 1 line
2.26.3
PATCH v2 1/2 virtio-blk: support polling I/O
This patch supports polling I/O via virtio-blk driver. Polling feature is enabled by module parameter "num_poll_queues" and it sets dedicated polling queues for virtio-blk. This patch improves the polling I/O throughput and latency.
The virtio-blk driver doesn't not have a poll function and a poll queue and it has been operating in interrupt driven method even if the polling function is called in the upper layer.
virtio-blk polling is implemented upon 'batched completion' of block layer. virtblk_poll() queues completed request to io_comp_batch->req_list and later, virtblk_complete_batch() calls unmap function and ends the requests in batch.
virtio-blk reads the number of poll queues from module parameter "num_poll_queues". If VM sets queue parameter as below, ("num-queues=N" [QEMU property], "num_poll_queues=M" [module parameter]) It allocates N virtqueues to virtio_blk->vqs[N] and it uses [0..(N-M-1)] as default queues and [(N-M)..(N-1)] as poll queues. Unlike the default queues, the poll queues have no callback function.
Regarding HW-SW queue mapping, the default queue mapping uses the existing method that condsiders MSI irq vector. But the poll queue doesn't have an irq, so it uses the regular blk-mq cpu mapping.
For verifying the improvement, I did Fio polling I/O performance test with io_uring engine with the options below.
(io_uring, hipri, randread, direct=1, bs=512, iodepth=64 numjobs=N) I set 4 vcpu and 4 virtio-blk queues - 2 default queues and 2 poll queues for VM.
As a result, IOPS and average latency improved about 10%.
Test result:
- Fio io_uring poll without virtio-blk poll support
-- numjobs=1 : IOPS = 339K, avg latency = 188.33us -- numjobs=2 : IOPS = 367K, avg latency = 347.33us -- numjobs=4 : IOPS = 383K, avg latency = 682.06us
- Fio io_uring poll with virtio-blk poll support
-- numjobs=1 : IOPS = 380K, avg latency = 167.87us -- numjobs=2 : IOPS = 409K, avg latency = 312.6us -- numjobs=4 : IOPS = 413K, avg latency = 619.72us
--- drivers/block/virtio_blk.c | 101 +++++++++++++++++++++++++++++++++++-- 1 file changed, 97 insertions(+), 4 deletions(-)
drivers/block/virtio_blk.c +97 -4
@@ -37,6 +37,10 @@ MODULE_PARM_DESC(num_request_queues,"0 for no limit. " "Values > nr_cpu_ids truncated to nr_cpu_ids."); +static unsigned int num_poll_queues; +module_param (num_poll_queues, uint,0644 ); +MODULE_PARM_DESC (num_poll_queues,"The number of dedicated virtqueues for polling I/O" ); +static int major;static DEFINE_IDA (vd_index_ida);
@@ -81,6 +85,7 @@ struct virtio_blk {/* num of vqs */ int num_vqs; +int io_queues[HCTX_MAX_TYPES ];struct virtio_blk_vq *vqs; };
@@ -548,6 +553,7 @@ static int init_vq(struct virtio_blk *vblk)const char **names;struct virtqueue **vqs;unsigned short num_vqs; +unsigned int num_poll_vqs;struct virtio_device *vdev = vblk->vdev;struct irq_affinity desc = {0 , };
@@ -556,6 +562,7 @@ static int init_vq(struct virtio_blk *vblk) &num_vqs);if (err) num_vqs =1 ; +if (!err && !num_vqs) {dev_err (&vdev->dev,"MQ advertised but zero queues reported\n" );return -EINVAL ;
@@ -565,6 +572,13 @@ static int init_vq(struct virtio_blk *vblk)min_not_zero (num_request_queues, nr_cpu_ids), num_vqs); + num_poll_vqs = min_t(unsigned int, num_poll_queues, num_vqs - 1); + +memset (vblk->io_queues,0 ,sizeof (int ) *HCTX_MAX_TYPES ); + vblk->io_queues[HCTX_TYPE_DEFAULT ] = num_vqs - num_poll_vqs; + vblk->io_queues[HCTX_TYPE_READ ] =0 ; + vblk->io_queues[HCTX_TYPE_POLL ] = num_poll_vqs; + vblk->vqs =kmalloc_array (num_vqs,sizeof (*vblk->vqs),GFP_KERNEL );if (!vblk->vqs)return -ENOMEM ;
@@ -578,8 +592,13 @@ static int init_vq(struct virtio_blk *vblk) }for (i =0 ; i < num_vqs; i++) { - callbacks[i] = virtblk_done; -snprintf (vblk->vqs[i].name,VQ_NAME_LEN ,"req.%d" , i); +if (i < num_vqs - num_poll_vqs) { + callbacks[i] = virtblk_done; +snprintf (vblk->vqs[i].name,VQ_NAME_LEN ,"req.%d" , i); + }else { + callbacks[i] =NULL ; +snprintf (vblk->vqs[i].name,VQ_NAME_LEN ,"req_poll.%d" , i); + } names[i] = vblk->vqs[i].name; }
@@ -728,16 +747,87 @@ static const struct attribute_group *virtblk_attr_groups[] = {static int virtblk_map_queues (struct blk_mq_tag_set *set) {struct virtio_blk *vblk = set->driver_data; +int i, qoff; + +for (i =0 , qoff =0 ; i < set->nr_maps; i++) { +struct blk_mq_queue_map *map = &set->map[i]; + + map->nr_queues = vblk->io_queues[i]; + map->queue_offset = qoff; + qoff += map->nr_queues; + +if (map->nr_queues ==0 ) +continue ; + +/* +* Regular queues have interrupts and hence CPU affinity is +* defined by the core virtio code, but polling queues have +* no interrupts so we let the block layer assign CPU affinity. +*/ +if (i ==HCTX_TYPE_DEFAULT ) +blk_mq_virtio_map_queues (&set->map[i], vblk->vdev,0 ); +else +blk_mq_map_queues (&set->map[i]); + } + +return 0 ; +} + +static void virtblk_complete_batch (struct io_comp_batch *iob) +{ +struct request *req; +struct virtblk_req *vbr; -return blk_mq_virtio_map_queues (&set->map[HCTX_TYPE_DEFAULT ], - vblk->vdev,0 ); +rq_list_for_each (&iob->req_list, req) { + vbr =blk_mq_rq_to_pdu (req); +virtblk_unmap_data (req, vbr); +virtblk_cleanup_cmd (req); + } +blk_mq_end_request_batch (iob); +} + +static int virtblk_poll (struct blk_mq_hw_ctx *hctx,struct io_comp_batch *iob) +{ +struct virtio_blk_vq *vq = hctx->driver_data; +struct virtblk_req *vbr; +unsigned long flags; +unsigned int len; +int found =0 ; + +spin_lock_irqsave (&vq->lock, flags); + +while ((vbr =virtqueue_get_buf (vq->vq, &len)) !=NULL ) { +struct request *req =blk_mq_rq_from_pdu (vbr); + + found++; +if (!blk_mq_add_to_batch (req, iob,virtblk_result (vbr), + virtblk_complete_batch)) +blk_mq_complete_request (req); + } + +spin_unlock_irqrestore (&vq->lock, flags); + +return found; +} + +static int virtblk_init_hctx (struct blk_mq_hw_ctx *hctx,void *data, +unsigned int hctx_idx) +{ +struct virtio_blk *vblk = data; +struct virtio_blk_vq *vq = &vblk->vqs[hctx_idx]; + +WARN_ON (vblk->tag_set.tags[hctx_idx] != hctx->tags); + hctx->driver_data = vq; +return 0 ; }static const struct blk_mq_ops virtio_mq_ops = { .queue_rq = virtio_queue_rq, .commit_rqs = virtio_commit_rqs, + .init_hctx = virtblk_init_hctx, .complete = virtblk_request_done, .map_queues = virtblk_map_queues, + .poll = virtblk_poll, };static unsigned int virtblk_queue_depth;
@@ -816,6 +906,9 @@ static int virtblk_probe(struct virtio_device *vdev)sizeof (struct scatterlist ) *VIRTIO_BLK_INLINE_SG_CNT ; vblk->tag_set.driver_data = vblk; vblk->tag_set.nr_hw_queues = vblk->num_vqs; + vblk->tag_set.nr_maps =1 ; +if (vblk->io_queues[HCTX_TYPE_POLL ]) + vblk->tag_set.nr_maps =3 ; err =blk_mq_alloc_tag_set (&vblk->tag_set);if (err)
Signature · 1 line
2.26.3
kernel test robot
build botRe: [PATCH v2 1/2] virtio-blk: support polling I/O
Thank you for the patch! Perhaps something to improve:
Hi Suwan,
Thank you for the patch! Perhaps something to improve:
[auto build test WARNING on v5.17]
[also build test WARNING on next-20220321]
[cannot apply to axboe-block/for-next]
[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]
url: https://github.com/0day-ci/linux/commits/Suwan-Kim/virtio-blk-support-polling-I-O-and-mq_ops-queue_rqs/20220321-223027 base: f443e374ae131c168a065ea1748feac6b2e76613 config: x86_64-rhel-8.3-kselftests (https://download.01.org/0day-ci/archive/20220322/[email protected]/config) compiler: gcc-9 (Ubuntu 9.4.0-1ubuntu1~20.04) 9.4.0 reproduce: # apt-get install sparse # sparse version: v0.6.4-dirty # https://github.com/0day-ci/linux/commit/5388c0729248236712cf865643a2190989dabbb5 git remote add linux-review https://github.com/0day-ci/linux git fetch --no-tags linux-review Suwan-Kim/virtio-blk-support-polling-I-O-and-mq_ops-queue_rqs/20220321-223027 git checkout 5388c0729248236712cf865643a2190989dabbb5 # save the config file to linux build tree mkdir build_dir make W=1 C=1 CF='-fdiagnostic-prefix -D__CHECK_ENDIAN__' O=build_dir ARCH=x86_64 SHELL=/bin/bash drivers/block/
If you fix the issue, kindly add following tag as appropriate
sparse warnings: (new ones prefixed by >>)
>> drivers/block/virtio_blk.c:803:66: sparse: sparse: incorrect type in argument 3 (different base types) @@ expected int ioerror @@ got restricted blk_status_t @@
drivers/block/virtio_blk.c:803:66: sparse: expected int ioerror
drivers/block/virtio_blk.c:803:66: sparse: got restricted blk_status_t
vim +803 drivers/block/virtio_blk.c
788
789 static int virtblk_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
790 {
791 struct virtio_blk_vq *vq = hctx->driver_data;
792 struct virtblk_req *vbr;
793 unsigned long flags;
794 unsigned int len;
795 int found = 0;
796
797 spin_lock_irqsave(&vq->lock, flags);
798
799 while ((vbr = virtqueue_get_buf(vq->vq, &len)) != NULL) {
800 struct request *req = blk_mq_rq_from_pdu(vbr);
801
802 found++;
> 803 if (!blk_mq_add_to_batch(req, iob, virtblk_result(vbr),
804 virtblk_complete_batch))
805 blk_mq_complete_request(req);
806 }
807
808 spin_unlock_irqrestore(&vq->lock, flags);
809
810 return found;
811 }
812
Signature · 2 lines
0-DAY CI Kernel Test Service https://01.org/lkp
PATCH v2 2/2 virtio-blk: support mq_ops->queue_rqs()
In reply to Suwan KimThis patch supports mq_ops->queue_rqs() hook. It has an advantage of batch submission to virtio-blk driver. It also helps polling I/O because polling uses batched completion of block layer. Batch submission in queue_rqs() can boost polling performance.
In queue_rqs(), it iterates plug->mq_list, collects requests that belong to same HW queue and adds requests into virtqueue until it encounters a request from other HW queue or sees the end of the list. Then, virtio-blk kicks virtqueue to submit requests.
If there is an error, it inserts error request to requeue_list and passes it to ordinary block layer path.
For verification, I did fio test.
(io_uring, randread, direct=1, bs=4K, iodepth=64 numjobs=N) I set 4 vcpu and 2 virtio-blk queues for VM and run fio test 5 times. It shows about 1-4% improvement.
| numjobs=2 | numjobs=4
-----------------------------------------------------------
fio without queue_rqs() | 282K IOPS | 245K IOPS
-----------------------------------------------------------
fio with queue_rqs() | 294K IOPS | 249K IOPS
For polling I/O performance, I also did fio test as below. (io_uring, hipri, randread, direct=1, bs=512, iodepth=64 numjobs=4) I set 4 vcpu and 2 poll queues for VM.
It shows upto 7% improvement in polling I/O.
| IOPS | avg latency
-----------------------------------------------------------
fio poll without queue_rqs() | 413K | 619.72us
-----------------------------------------------------------
fio poll with queue_rqs() | 445K | 581.2 usec--- drivers/block/virtio_blk.c | 93 ++++++++++++++++++++++++++++++++++---- 1 file changed, 84 insertions(+), 9 deletions(-)
drivers/block/virtio_blk.c +84 -9
@@ -311,6 +311,28 @@ static void virtio_commit_rqs(struct blk_mq_hw_ctx *hctx)virtqueue_notify (vq->vq); } +static blk_status_t virtblk_prep_rq (struct blk_mq_hw_ctx *hctx, +struct virtio_blk *vblk, +struct request *req, +struct virtblk_req *vbr,int *num) +{ +blk_status_t status; + + status =virtblk_setup_cmd (vblk->vdev, req, vbr); +if (unlikely (status)) +return status; + +blk_mq_start_request (req); + + *num =virtblk_map_data (hctx, req, vbr); +if (unlikely (*num <0 )) { +virtblk_cleanup_cmd (req); +return BLK_STS_RESOURCE ; + } + +return BLK_STS_OK ; +} +static blk_status_t virtio_queue_rq (struct blk_mq_hw_ctx *hctx,const struct blk_mq_queue_data *bd) {
@@ -324,18 +346,10 @@ static blk_status_t virtio_queue_rq(struct blk_mq_hw_ctx *hctx,blk_status_t status;int err; - status =virtblk_setup_cmd (vblk->vdev, req, vbr); + status =virtblk_prep_rq (hctx, vblk, req, vbr, &num);if (unlikely (status))return status; -blk_mq_start_request (req); - - num =virtblk_map_data (hctx, req, vbr); -if (unlikely (num <0 )) { -virtblk_cleanup_cmd (req); -return BLK_STS_RESOURCE ; - } -spin_lock_irqsave (&vblk->vqs[qid].lock, flags); err =virtblk_add_req (vblk->vqs[qid].vq, vbr, vbr->sg_table.sgl, num);if (err) {
@@ -367,6 +381,66 @@ static blk_status_t virtio_queue_rq(struct blk_mq_hw_ctx *hctx,return BLK_STS_OK ; } +static bool virtblk_prep_rq_batch (struct virtio_blk_vq *vq,struct request *req) +{ +struct virtio_blk *vblk = req->mq_hctx->queue->queuedata; +struct virtblk_req *vbr =blk_mq_rq_to_pdu (req); +unsigned long flags; +int num, err; + + req->mq_hctx->tags->rqs[req->tag] = req; + +if (virtblk_prep_rq (req->mq_hctx, vblk, req, vbr, &num) !=BLK_STS_OK ) +return false; + +spin_lock_irqsave (&vq->lock, flags); + err =virtblk_add_req (vq->vq, vbr, vbr->sg_table.sgl, num); +if (err) { +spin_unlock_irqrestore (&vq->lock, flags); +virtblk_unmap_data (req, vbr); +virtblk_cleanup_cmd (req); +return false; + } +spin_unlock_irqrestore (&vq->lock, flags); + +return true; +} + +static void virtio_queue_rqs (struct request **rqlist) +{ +struct request *req, *next, *prev =NULL ; +struct request *requeue_list =NULL ; + +rq_list_for_each_safe (rqlist, req, next) { +struct virtio_blk_vq *vq = req->mq_hctx->driver_data; +unsigned long flags; +bool kick; + +if (!virtblk_prep_rq_batch (vq, req)) { +rq_list_move (rqlist, &requeue_list, req, prev); + req = prev; + +if (!req) +continue ; + } + +if (!next || req->mq_hctx != next->mq_hctx) { +spin_lock_irqsave (&vq->lock, flags); + kick =virtqueue_kick_prepare (vq->vq); +spin_unlock_irqrestore (&vq->lock, flags); +if (kick) +virtqueue_notify (vq->vq); + + req->rq_next =NULL ; + *rqlist = next; + prev =NULL ; + }else + prev = req; + } + + *rqlist = requeue_list; +} +/* return id (s/n) string for *disk to *id_str */ static int virtblk_get_id (struct gendisk *disk,char *id_str)
@@ -823,6 +897,7 @@ static int virtblk_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,static const struct blk_mq_ops virtio_mq_ops = { .queue_rq = virtio_queue_rq, + .queue_rqs = virtio_queue_rqs, .commit_rqs = virtio_commit_rqs, .init_hctx = virtblk_init_hctx, .complete = virtblk_request_done,
Signature · 1 line
2.26.3