nvmet: support polling queue task for bio request
Subsystems (2)
cover nvmet: support polling queue task for bio request
Since nvme target currently does not support to submit bio to a polling queue, the bio's completion relies on system interrupt. But when there is high workload in system and the competition is very high, so it makes sense to add polling queue task to submit bio to disk's polling queue and poll the completion queue of disk.
Ping Gan (4):
nvmet: Add nvme target polling queue task parameters
nvmet: Add polling queue task for nvme target
nvmet: support bio polling queue request
nvme-core: Get lowlevel disk for target polling queue task
drivers/nvme/host/multipath.c | 20 + drivers/nvme/target/Makefile | 2 +- drivers/nvme/target/core.c | 55 +- drivers/nvme/target/io-cmd-bdev.c | 243 ++++++++- drivers/nvme/target/nvmet.h | 13 + drivers/nvme/target/polling-queue-thread.c | 594 +++++++++++++++++++++ 6 files changed, 895 insertions(+), 32 deletions(-) create mode 100644 drivers/nvme/target/polling-queue-thread.c
Signature · 1 line
2.26.2
PATCH 1/4 nvmet: Add nvme target polling queue task parameters
To define a polling task's running parameters when nvme target submits bio to a nvme polling queue.
--- drivers/nvme/target/core.c | 55 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 2 deletions(-)
drivers/nvme/target/core.c +53 -2
@@ -17,6 +17,29 @@#include "nvmet.h" +/* Define the polling queue thread's affinity cpu core. +* */ +static int pqt_affinity_core =-1 ; +module_param (pqt_affinity_core, int,0644 ); +MODULE_PARM_DESC (pqt_affinity_core, +"nvme polling queue thread's affinity core, -1 for all online cpus" ); + +/* Define a time (in usecs) that polling queue thread shall sample the +* * io request ring before determining it to be idle. +* */ +static int pqt_idle_usecs; +module_param (pqt_idle_usecs, int,0644 ); +MODULE_PARM_DESC (pqt_idle_usecs, +"polling queue task will poll io request till idle time in usecs" ); + +/* Define the polling queue thread ring's size. +* * The ring will be consumed by polling queue thread. +* */ +static int pqt_ring_size; +module_param (pqt_ring_size, int,0644 ); +MODULE_PARM_DESC (pqt_ring_size, +"nvme target polling queue thread ring size" ); +struct kmem_cache *nvmet_bvec_cache;struct workqueue_struct *buffered_io_wq;struct workqueue_struct *zbd_wq;
@@ -1648,13 +1671,34 @@ static int __init nvmet_init(void) {int error = -ENOMEM ; +if ((pqt_affinity_core >=-1 && + pqt_affinity_core < nr_cpu_ids) || + pqt_idle_usecs >0 || pqt_ring_size >0 ) { +if (pqt_idle_usecs ==0 ) + pqt_idle_usecs =1000 ;//default 1ms +if (pqt_affinity_core <-1 || + pqt_affinity_core >= nr_cpu_ids) { +printk (KERN_ERR "bad parameter for affinity core \n" ); + error = -EINVAL ; +return error; + } +if (pqt_ring_size ==0 ) + pqt_ring_size =4096 ;//default 4k + error =nvmet_init_pq_thread (pqt_idle_usecs, + pqt_affinity_core, pqt_ring_size); +if (error) +return error; + } + nvmet_ana_group_enabled[NVMET_DEFAULT_ANA_GRPID ] =1 ; nvmet_bvec_cache =kmem_cache_create ("nvmet-bvec" ,NVMET_MAX_MPOOL_BVEC *sizeof (struct bio_vec ),0 ,SLAB_HWCACHE_ALIGN ,NULL ); -if (!nvmet_bvec_cache) -return -ENOMEM ; +if (!nvmet_bvec_cache) { + error = -ENOMEM ; + goto out_free_pqt; + } zbd_wq =alloc_workqueue ("nvmet-zbd-wq" ,WQ_MEM_RECLAIM ,0 );if (!zbd_wq)
@@ -1688,6 +1732,8 @@ static int __init nvmet_init(void)destroy_workqueue (zbd_wq); out_destroy_bvec_cache:kmem_cache_destroy (nvmet_bvec_cache); +out_free_pqt: +nvmet_exit_pq_thread ();return error; }
@@ -1701,6 +1747,11 @@ static void __exit nvmet_exit(void)destroy_workqueue (zbd_wq);kmem_cache_destroy (nvmet_bvec_cache); +if ((pqt_affinity_core >=-1 && + pqt_affinity_core < nr_cpu_ids) || + pqt_idle_usecs >0 || pqt_ring_size >0 ) +nvmet_exit_pq_thread (); +BUILD_BUG_ON (sizeof (struct nvmf_disc_rsp_page_entry ) !=1024 );BUILD_BUG_ON (sizeof (struct nvmf_disc_rsp_page_hdr ) !=1024 ); }
Signature · 1 line
2.26.2
PATCH 4/4 nvme-core: Get lowlevel disk for target polling queue task
In reply to Ping GanWhen enabling multipath, if the block device of nvmet is a nvme_ns_head disk, then we should get the lowlevel block device to do bio split.
--- drivers/nvme/host/multipath.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+)
drivers/nvme/host/multipath.c +20 -0
@@ -371,6 +371,25 @@ static bool nvme_available_path(struct nvme_ns_head *head)return false; } +//for polling queue task to get lowlevel block device +struct block_device *nvme_mpath_get_bdev (struct block_device *bdev) +{ +struct nvme_ns_head *head = bdev->bd_disk->private_data; +int srcu_idx; +struct nvme_ns *ns; +struct block_device *ret =NULL ; + +if (!multipath) +return NULL ; + srcu_idx =srcu_read_lock (&head->srcu); + ns =nvme_find_path (head); +if (likely (ns)) + ret = ns->disk->part0; +srcu_read_unlock (&head->srcu, srcu_idx); +return ret; +} +EXPORT_SYMBOL_GPL (nvme_mpath_get_bdev); +static void nvme_ns_head_submit_bio (struct bio *bio) {struct nvme_ns_head *head = bio->bi_bdev->bd_disk->private_data;
@@ -452,6 +471,7 @@ const struct block_device_operations nvme_ns_head_ops = { .report_zones = nvme_ns_head_report_zones, .pr_ops = &nvme_pr_ops, }; +EXPORT_SYMBOL_GPL (nvme_ns_head_ops);static inline struct nvme_ns_head *cdev_to_ns_head (struct cdev *cdev) {
Signature · 1 line
2.26.2
PATCH 2/4 nvmet: Add polling queue task for nvme target
In reply to Ping GanThe polling queue task of nvme target will fetch bio requests from the lossless ring which is filled by io-cmd-bdev's rw , then submit the requests to nvme's polling queue, at last do polling to check the coplemented status of the requests and complete the request.
--- drivers/nvme/target/Makefile | 2 +- drivers/nvme/target/nvmet.h | 13 + drivers/nvme/target/polling-queue-thread.c | 594 +++++++++++++++++++++ 3 files changed, 608 insertions(+), 1 deletion(-) create mode 100644 drivers/nvme/target/polling-queue-thread.c
drivers/nvme/target/Makefile +1 -1
@@ -10,7 +10,7 @@ obj-$(CONFIG_NVME_TARGET_FCLOOP) += nvme-fcloop.o obj-$(CONFIG_NVME_TARGET_TCP) += nvmet-tcp.o nvmet-y += core.o configfs.o admin-cmd.o fabrics-cmd.o \ - discovery.o io-cmd-file.o io-cmd-bdev.o + discovery.o io-cmd-file.o io-cmd-bdev.o polling-queue-thread.o nvmet-$(CONFIG_NVME_TARGET_PASSTHRU) += passthru.o nvmet-$(CONFIG_BLK_DEV_ZONED) += zns.o nvmet-$(CONFIG_NVME_TARGET_AUTH) += fabrics-cmd-auth.o auth.o
drivers/nvme/target/nvmet.h +13 -0
@@ -395,6 +395,12 @@ struct nvmet_req {u64 error_slba; }; +struct nvmet_pqt_bio_req { +struct nvmet_req *req; +struct bio_list blist; +unsigned short io_completed; +}; +#define NVMET_MAX_MPOOL_BVEC 16extern struct kmem_cache *nvmet_bvec_cache;extern struct workqueue_struct *buffered_io_wq;
@@ -455,6 +461,13 @@ u16 nvmet_parse_discovery_cmd(struct nvmet_req *req);u16 nvmet_parse_fabrics_admin_cmd (struct nvmet_req *req);u16 nvmet_parse_fabrics_io_cmd (struct nvmet_req *req); +//below is for enabling nvmet polling queue task +int nvmet_init_pq_thread (u32 thread_idle,int affinity_cpu,u32 ring_size); +void nvmet_exit_pq_thread (void ); +bool nvmet_pqt_enabled (void ); +int nvmet_pqt_ring_enqueue (void *pt); +void nvmet_wakeup_pq_thread (void ); +bool nvmet_req_init (struct nvmet_req *req,struct nvmet_cq *cq,struct nvmet_sq *sq,const struct nvmet_fabrics_ops *ops);void nvmet_req_uninit (struct nvmet_req *req);
drivers/nvme/target/polling-queue-thread.c new file +594 -0
594 lines not on this page · raw message
Signature · 1 line
2.26.2
PATCH 3/4 nvmet: support bio polling queue request
In reply to Ping GanIf enabling bio polling queue task, we will split and chain the bios if needed, then fill the request to the lossless ring of polling queue task.
--- drivers/nvme/target/io-cmd-bdev.c | 243 ++++++++++++++++++++++++++---- 1 file changed, 214 insertions(+), 29 deletions(-)
drivers/nvme/target/io-cmd-bdev.c +214 -29
@@ -184,6 +184,16 @@ static void nvmet_bio_done(struct bio *bio)nvmet_req_bio_put (req, bio); } +static void nvmet_pqt_bio_done (struct bio *bio) +{ +struct nvmet_pqt_bio_req *req_done = bio->bi_private; + +nvmet_req_complete (req_done->req,blk_to_nvme_status (req_done->req, + bio->bi_status)); +nvmet_req_bio_put (req_done->req, bio); + req_done->io_completed =1 ; +} +#ifdef CONFIG_BLK_DEV_INTEGRITY static int nvmet_bdev_alloc_bip (struct nvmet_req *req,struct bio *bio,struct sg_mapping_iter *miter)
@@ -237,6 +247,38 @@ static int nvmet_bdev_alloc_bip(struct nvmet_req *req, struct bio *bio, }#endif /* CONFIG_BLK_DEV_INTEGRITY */ +#ifdef CONFIG_NVME_MULTIPATH +extern struct block_device *nvme_mpath_get_bdev (struct block_device *bdev); +extern const struct block_device_operations nvme_ns_head_ops; +#endif + +static inline int nvmet_chain_par_bio (struct nvmet_req *req,struct bio **bio, +struct sg_mapping_iter *prot_miter,struct block_device *bdev, +sector_t sector,struct bio_list *blist) +{ +struct bio *parent, *child; +unsigned int vec_cnt; +int rc; + + parent = *bio; + vec_cnt =queue_max_segments (bdev->bd_disk->queue); +if (req->metadata_len) { + rc =nvmet_bdev_alloc_bip (req, parent, + prot_miter); +if (unlikely (rc)) +return rc; + } + child =bio_alloc (bdev, vec_cnt, parent->bi_opf,GFP_KERNEL ); + child->bi_iter.bi_sector = sector; + *bio = child; +bio_chain (*bio, parent); + parent->bi_opf |=REQ_POLLED ; + parent->bi_opf |=REQ_NOWAIT ; + parent->bi_opf |=REQ_NOMERGE ; +bio_list_add (blist, parent); +return 0 ; +} +static void nvmet_bdev_execute_rw (struct nvmet_req *req) {unsigned int sg_cnt = req->sg_cnt;
@@ -247,8 +289,13 @@ static void nvmet_bdev_execute_rw(struct nvmet_req *req)blk_opf_t opf;int i, rc;struct sg_mapping_iter prot_miter; -unsigned int iter_flags; +unsigned int iter_flags, max_sectors; +unsigned short vec_cnt, max_segments;unsigned int total_len =nvmet_rw_data_len (req) + req->metadata_len; +bool pqt_enabled =nvmet_pqt_enabled (); +unsigned int sg_len; +struct nvmet_pqt_bio_req *req_done =NULL ; +struct block_device *bdev = req->ns->bdev;if (!nvmet_check_transfer_len (req, total_len))return ;
@@ -268,6 +315,24 @@ static void nvmet_bdev_execute_rw(struct nvmet_req *req) iter_flags =SG_MITER_FROM_SG ; } +#ifdef CONFIG_NVME_MULTIPATH +if (pqt_enabled && bdev->bd_disk->fops == &nvme_ns_head_ops) { + bdev =nvme_mpath_get_bdev (bdev); +if (!bdev) { +nvmet_req_complete (req,0 ); +return ; + } + opf |=REQ_DRV ; + } +#endif +if (pqt_enabled) { + req_done =kmalloc (sizeof (struct nvmet_pqt_bio_req ),GFP_KERNEL ); +if (!req_done) { +nvmet_req_complete (req,0 ); +return ; + } + } +if (is_pci_p2pdma_page (sg_page (req->sg))) opf |=REQ_NOMERGE ;
@@ -278,54 +343,174 @@ static void nvmet_bdev_execute_rw(struct nvmet_req *req)bio_init (bio, req->ns->bdev, req->inline_bvec,ARRAY_SIZE (req->inline_bvec), opf); }else { - bio =bio_alloc (req->ns->bdev,bio_max_segs (sg_cnt), opf, + vec_cnt =bio_max_segs (sg_cnt); +if (pqt_enabled) + vec_cnt =queue_max_segments (bdev->bd_disk->queue); + bio =bio_alloc (bdev, vec_cnt, opf,GFP_KERNEL ); } bio->bi_iter.bi_sector = sector; - bio->bi_private = req; - bio->bi_end_io = nvmet_bio_done; +if (!pqt_enabled) { + bio->bi_private = req; + bio->bi_end_io = nvmet_bio_done; + }else { + req_done->req = req; + bio->bi_private = req_done; + bio->bi_end_io = nvmet_pqt_bio_done; + } -blk_start_plug (&plug); +if (!pqt_enabled) +blk_start_plug (&plug);if (req->metadata_len)sg_miter_start (&prot_miter, req->metadata_sg, req->metadata_sg_cnt, iter_flags); -for_each_sg (req->sg, sg, req->sg_cnt, i) { -while (bio_add_page (bio,sg_page (sg), sg->length, sg->offset) - != sg->length) { -struct bio *prev = bio; - -if (req->metadata_len) { - rc =nvmet_bdev_alloc_bip (req, bio, - &prot_miter); -if (unlikely (rc)) { -bio_io_error (bio); -return ; +if (!pqt_enabled) { +for_each_sg (req->sg, sg, req->sg_cnt, i) { +while (bio_add_page (bio,sg_page (sg), sg->length, sg->offset) + != sg->length) { +struct bio *prev = bio; + +if (req->metadata_len) { + rc =nvmet_bdev_alloc_bip (req, bio, + &prot_miter); +if (unlikely (rc)) { +bio_io_error (bio); +return ; + } } - } - bio =bio_alloc (req->ns->bdev,bio_max_segs (sg_cnt), - opf,GFP_KERNEL ); - bio->bi_iter.bi_sector = sector; + bio =bio_alloc (req->ns->bdev,bio_max_segs (sg_cnt), + opf,GFP_KERNEL ); + bio->bi_iter.bi_sector = sector; -bio_chain (bio, prev); -submit_bio (prev); - } +bio_chain (bio, prev); +submit_bio (prev); + } - sector += sg->length >>9 ; - sg_cnt--; + sector += sg->length >>9 ; + sg_cnt--; + } + }else { +bio_list_init (&req_done->blist); +if (!test_bit (QUEUE_FLAG_POLL , &bdev->bd_disk->queue->queue_flags)) + goto err_bio; + max_sectors = bdev->bd_disk->queue->limits.max_sectors; + max_sectors <<=9 ; + max_segments =queue_max_segments (bdev->bd_disk->queue); + sg_len =0 ; +unsigned int offset, len, vec_len, i; +bool sg_start_pg = true, need_chain_bio = false; +struct page *sglist_page, *max_sector_align; +sector_t temp_sector; + +/* +* for bio's polling mode we will split bio to +* avoid low level's bio splitting when submit. +*/ +for_each_sg (req->sg, sg, req->sg_cnt, i) { + temp_sector = sector; + offset = (sg->offset %PAGE_SIZE ); +if (offset + sg->length >PAGE_SIZE ) {// need to split + len = sg->length; + i =0 ; + sglist_page =virt_to_page (page_to_virt (sg_page (sg)) + offset); +if (offset !=0 ) + sg_start_pg = false; +while (len >PAGE_SIZE ) { + max_sector_align =virt_to_page (page_to_virt (sglist_page) + + (PAGE_SIZE *i)); + vec_len = sg_start_pg?PAGE_SIZE :(PAGE_SIZE - offset); +if (bio->bi_vcnt == max_segments -1 || + sg_len + vec_len > max_sectors) + need_chain_bio = true; +else { +__bio_add_page (bio, max_sector_align, + vec_len, sg_start_pg?0 :offset); + temp_sector += vec_len >>9 ; + sg_len += vec_len; + } +if (need_chain_bio) { + rc =nvmet_chain_par_bio (req, &bio, &prot_miter, + bdev, temp_sector, &req_done->blist); +if (unlikely (rc)) + goto err_bio; +__bio_add_page (bio, max_sector_align, vec_len, + sg_start_pg?0 :(PAGE_SIZE - offset)); + temp_sector += vec_len >>9 ; + sg_len = vec_len; + need_chain_bio = false; + } +if (!sg_start_pg) { + len -= (PAGE_SIZE - offset); + sg_start_pg = true; + }else { + len -=PAGE_SIZE ; + } + i++; + } +if (len >0 ) { + max_sector_align =virt_to_page (page_to_virt (sglist_page) + + (i *PAGE_SIZE )); +if (bio->bi_vcnt == max_segments -1 || + sg_len + len > max_sectors) { + rc =nvmet_chain_par_bio (req, &bio, &prot_miter, + bdev, temp_sector, &req_done->blist); +if (unlikely (rc)) + goto err_bio; + sg_len = len; + }else { + sg_len += len; + } +__bio_add_page (bio, max_sector_align, len,0 ); + temp_sector += len >>9 ; + } + }else { +if (bio->bi_vcnt == max_segments -1 || + sg_len + sg->length > max_sectors) { + rc =nvmet_chain_par_bio (req, &bio, &prot_miter, + bdev, temp_sector, &req_done->blist); +if (unlikely (rc)) + goto err_bio; + sg_len = sg->length; + }else { + sg_len += sg->length; + } +__bio_add_page (bio,sg_page (sg), sg->length, sg->offset); + } + sector += sg->length >>9 ; + sg_cnt--; + } }if (req->metadata_len) { rc =nvmet_bdev_alloc_bip (req, bio, &prot_miter);if (unlikely (rc)) { -bio_io_error (bio); -return ; + goto err_bio; } } -submit_bio (bio); -blk_finish_plug (&plug); +if (pqt_enabled) { + bio->bi_opf |=REQ_POLLED ; + bio->bi_opf |=REQ_NOWAIT ; + bio->bi_opf |=REQ_NOMERGE ; +bio_list_add (&req_done->blist, bio); + req_done->io_completed =0 ; + rc =nvmet_pqt_ring_enqueue (req_done); +if (rc <0 ) + goto err_bio; +nvmet_wakeup_pq_thread (); + }else { +submit_bio (bio); + } +if (!pqt_enabled) +blk_finish_plug (&plug); +return ; +err_bio: +bio_io_error (bio); +if (pqt_enabled) +kfree (req_done); +return ; }static void nvmet_bdev_execute_flush (struct nvmet_req *req)
Signature · 1 line
2.26.2
kernel test robot
build botRe: [PATCH 4/4] nvme-core: Get lowlevel disk for target polling queue task
In reply to Ping Gankernel test robot noticed the following build warnings:
Hi Ping,
kernel test robot noticed the following build warnings:
[auto build test WARNING on v6.6-rc1]
[also build test WARNING on linus/master next-20230913] [cannot apply to hch-configfs/for-next]
[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Ping-Gan/nvmet-Add-nvme-target-polling-queue-task-parameters/20230913-164112 base: v6.6-rc1 patch link: https://lore.kernel.org/r/006b6aefe94d73ee64931c769af4a908616439ad.1694592708.git.jacky_gam_2001%40163.com patch subject: [PATCH 4/4] nvme-core: Get lowlevel disk for target polling queue task config: m68k-allyesconfig (https://download.01.org/0day-ci/archive/20230913/[email protected]/config) compiler: m68k-linux-gcc (GCC) 13.2.0 reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20230913/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All warnings (new ones prefixed by >>):
>> drivers/nvme/host/multipath.c:375:22: warning: no previous prototype for 'nvme_mpath_get_bdev' [-Wmissing-prototypes]
375 | struct block_device *nvme_mpath_get_bdev(struct block_device *bdev)
| ^~~~~~~~~~~~~~~~~~~
vim +/nvme_mpath_get_bdev +375 drivers/nvme/host/multipath.c
373
374 //for polling queue task to get lowlevel block device
> 375 struct block_device *nvme_mpath_get_bdev(struct block_device *bdev)
376 {
377 struct nvme_ns_head *head = bdev->bd_disk->private_data;
378 int srcu_idx;
379 struct nvme_ns *ns;
380 struct block_device *ret = NULL;
381
382 if (!multipath)
383 return NULL;
384 srcu_idx = srcu_read_lock(&head->srcu);
385 ns = nvme_find_path(head);
386 if (likely(ns))
387 ret = ns->disk->part0;
388 srcu_read_unlock(&head->srcu, srcu_idx);
389 return ret;
390 }
391 EXPORT_SYMBOL_GPL(nvme_mpath_get_bdev);
392
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
kernel test robot
build botRe: [PATCH 2/4] nvmet: Add polling queue task for nvme target
In reply to Ping Gankernel test robot noticed the following build warnings:
Hi Ping,
kernel test robot noticed the following build warnings:
[auto build test WARNING on v6.6-rc1]
[also build test WARNING on linus/master next-20230913] [cannot apply to hch-configfs/for-next]
[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Ping-Gan/nvmet-Add-nvme-target-polling-queue-task-parameters/20230913-164112 base: v6.6-rc1 patch link: https://lore.kernel.org/r/84d7f188e892b5b0ba251a4601455d7a137075f3.1694592708.git.jacky_gam_2001%40163.com patch subject: [PATCH 2/4] nvmet: Add polling queue task for nvme target config: arc-randconfig-r004-20230913 (https://download.01.org/0day-ci/archive/20230913/[email protected]/config) compiler: arc-elf-gcc (GCC) 13.2.0 reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20230913/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All warnings (new ones prefixed by >>):
drivers/nvme/target/polling-queue-thread.c: In function '__nvmet_pq_ring_do_enqueue':
>> drivers/nvme/target/polling-queue-thread.c:150:37: warning: this statement may fall through [-Wimplicit-fallthrough=]
150 | ring[idx++] = obj_table[i++]; /* fallthrough */ \
| ~~~~~~~~~~~~^~~~~~~~~~~~~~~~
drivers/nvme/target/polling-queue-thread.c:278:9: note: in expansion of macro 'ENQUEUE_PTRS'
278 | ENQUEUE_PTRS(r, &r[1], prod_head, obj_table, n, void *);
| ^~~~~~~~~~~~
drivers/nvme/target/polling-queue-thread.c:151:17: note: here
151 | case 2: \
| ^~~~
drivers/nvme/target/polling-queue-thread.c:278:9: note: in expansion of macro 'ENQUEUE_PTRS'
278 | ENQUEUE_PTRS(r, &r[1], prod_head, obj_table, n, void *);
| ^~~~~~~~~~~~
drivers/nvme/target/polling-queue-thread.c:152:37: warning: this statement may fall through [-Wimplicit-fallthrough=]
152 | ring[idx++] = obj_table[i++]; /* fallthrough */ \
| ~~~~~~~~~~~~^~~~~~~~~~~~~~~~
drivers/nvme/target/polling-queue-thread.c:278:9: note: in expansion of macro 'ENQUEUE_PTRS'
278 | ENQUEUE_PTRS(r, &r[1], prod_head, obj_table, n, void *);
| ^~~~~~~~~~~~
drivers/nvme/target/polling-queue-thread.c:153:17: note: here
153 | case 1: \
| ^~~~
drivers/nvme/target/polling-queue-thread.c:278:9: note: in expansion of macro 'ENQUEUE_PTRS'
278 | ENQUEUE_PTRS(r, &r[1], prod_head, obj_table, n, void *);
| ^~~~~~~~~~~~
drivers/nvme/target/polling-queue-thread.c: In function '__nvmet_pq_ring_do_dequeue':
drivers/nvme/target/polling-queue-thread.c:181:40: warning: this statement may fall through [-Wimplicit-fallthrough=]
181 | obj_table[i++] = ring[idx++]; /* fallthrough */ \
| ^
drivers/nvme/target/polling-queue-thread.c:298:9: note: in expansion of macro 'DEQUEUE_PTRS'
298 | DEQUEUE_PTRS(r, &r[1], cons_head, obj_table, n, void *);
| ^~~~~~~~~~~~
drivers/nvme/target/polling-queue-thread.c:182:17: note: here
182 | case 2: \
| ^~~~
drivers/nvme/target/polling-queue-thread.c:298:9: note: in expansion of macro 'DEQUEUE_PTRS'
298 | DEQUEUE_PTRS(r, &r[1], cons_head, obj_table, n, void *);
| ^~~~~~~~~~~~
drivers/nvme/target/polling-queue-thread.c:183:40: warning: this statement may fall through [-Wimplicit-fallthrough=]
183 | obj_table[i++] = ring[idx++]; /* fallthrough */ \
| ^
drivers/nvme/target/polling-queue-thread.c:298:9: note: in expansion of macro 'DEQUEUE_PTRS'
298 | DEQUEUE_PTRS(r, &r[1], cons_head, obj_table, n, void *);
| ^~~~~~~~~~~~
drivers/nvme/target/polling-queue-thread.c:184:17: note: here
184 | case 1: \
| ^~~~
drivers/nvme/target/polling-queue-thread.c:298:9: note: in expansion of macro 'DEQUEUE_PTRS'
298 | DEQUEUE_PTRS(r, &r[1], cons_head, obj_table, n, void *);
| ^~~~~~~~~~~~vim +150 drivers/nvme/target/polling-queue-thread.c
131
132 //below is derived from FreeBSD's bufring.h
133 /* the actual enqueue of pointers on the ring.
134 * Placed here since identical code needed in both
135 * single and multi producer enqueue functions */
136 #define ENQUEUE_PTRS(r, ring_start, prod_head, obj_table, n, obj_type) do { \
137 unsigned int i; \
138 const u32 size = (r)->size; \
139 u32 idx = prod_head & (r)->mask; \
140 obj_type *ring = (obj_type *)ring_start; \
141 if (likely(idx + n < size)) { \
142 for (i = 0; i < (n & ((~(unsigned)0x3))); i += 4, idx += 4) { \
143 ring[idx] = obj_table[i]; \
144 ring[idx+1] = obj_table[i+1]; \
145 ring[idx+2] = obj_table[i+2]; \
146 ring[idx+3] = obj_table[i+3]; \
147 } \
148 switch (n & 0x3) { \
149 case 3: \
> 150 ring[idx++] = obj_table[i++]; /* fallthrough */ \
151 case 2: \
152 ring[idx++] = obj_table[i++]; /* fallthrough */ \
153 case 1: \
154 ring[idx++] = obj_table[i++]; \
155 } \
156 } else { \
157 for (i = 0; idx < size; i++, idx++)\
158 ring[idx] = obj_table[i]; \
159 for (idx = 0; i < n; i++, idx++) \
160 ring[idx] = obj_table[i]; \
161 } \
162 } while (0)
163
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
kernel test robot
build botRe: [PATCH 2/4] nvmet: Add polling queue task for nvme target
In reply to Ping Gankernel test robot noticed the following build warnings:
Hi Ping,
kernel test robot noticed the following build warnings:
[auto build test WARNING on v6.6-rc1]
[also build test WARNING on linus/master next-20230913] [cannot apply to hch-configfs/for-next]
[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Ping-Gan/nvmet-Add-nvme-target-polling-queue-task-parameters/20230913-164112 base: v6.6-rc1 patch link: https://lore.kernel.org/r/84d7f188e892b5b0ba251a4601455d7a137075f3.1694592708.git.jacky_gam_2001%40163.com patch subject: [PATCH 2/4] nvmet: Add polling queue task for nvme target config: i386-randconfig-013-20230913 (https://download.01.org/0day-ci/archive/20230913/[email protected]/config) compiler: clang version 16.0.4 (https://github.com/llvm/llvm-project.git ae42196bc493ffe877a7e3dff8be32035dea4d07) reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20230913/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All warnings (new ones prefixed by >>):
>> drivers/nvme/target/polling-queue-thread.c:278:2: warning: unannotated fall-through between switch labels [-Wimplicit-fallthrough]
ENQUEUE_PTRS(r, &r[1], prod_head, obj_table, n, void *);
^
drivers/nvme/target/polling-queue-thread.c:151:3: note: expanded from macro 'ENQUEUE_PTRS'
case 2: \
^
>> drivers/nvme/target/polling-queue-thread.c:278:2: warning: unannotated fall-through between switch labels [-Wimplicit-fallthrough]
drivers/nvme/target/polling-queue-thread.c:153:3: note: expanded from macro 'ENQUEUE_PTRS'
case 1: \
^
drivers/nvme/target/polling-queue-thread.c:298:2: warning: unannotated fall-through between switch labels [-Wimplicit-fallthrough]
DEQUEUE_PTRS(r, &r[1], cons_head, obj_table, n, void *);
^
drivers/nvme/target/polling-queue-thread.c:182:3: note: expanded from macro 'DEQUEUE_PTRS'
case 2: \
^
drivers/nvme/target/polling-queue-thread.c:298:2: warning: unannotated fall-through between switch labels [-Wimplicit-fallthrough]
drivers/nvme/target/polling-queue-thread.c:184:3: note: expanded from macro 'DEQUEUE_PTRS'
case 1: \
^ drivers/nvme/target/polling-queue-thread.c:357:1: warning: unused function '__nvmet_pq_ring_full' [-Wunused-function] __nvmet_pq_ring_full(const struct nvmet_pq_ring *r)
^
drivers/nvme/target/polling-queue-thread.c:366:1: warning: unused function '__nvmet_pq_ring_empty' [-Wunused-function] __nvmet_pq_ring_empty(const struct nvmet_pq_ring *r)
^
drivers/nvme/target/polling-queue-thread.c:375:1: warning: unused function '__nvmet_pq_ring_get_size' [-Wunused-function] __nvmet_pq_ring_get_size(const struct nvmet_pq_ring *r)
^
drivers/nvme/target/polling-queue-thread.c:384:1: warning: unused function '__nvmet_pq_ring_get_capacity' [-Wunused-function] __nvmet_pq_ring_get_capacity(const struct nvmet_pq_ring *r)
^
8 warnings generated.
--
>> drivers/nvme/target/polling-queue-thread.c:345: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* Return the number of free entries in a ring.
drivers/nvme/target/polling-queue-thread.c:354: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* Test if a ring is full.
drivers/nvme/target/polling-queue-thread.c:363: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* Test if a ring is empty.
drivers/nvme/target/polling-queue-thread.c:372: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* Return the size of the ring.
drivers/nvme/target/polling-queue-thread.c:381: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* Return the number of elements which can be stored in the ring.
vim +278 drivers/nvme/target/polling-queue-thread.c
265
266 static inline u32
267 __nvmet_pq_ring_do_enqueue(struct nvmet_pq_ring *r,
268 void **obj_table, u32 n, u32 *free_space)
269 {
270 uint32_t prod_head, prod_next;
271 uint32_t free_entries;
272
273 n = __nvmet_pq_ring_move_prod_head(r, n, &prod_head,
274 &prod_next, &free_entries);
275 if (n == 0)
276 goto end;
277
> 278 ENQUEUE_PTRS(r, &r[1], prod_head, obj_table, n, void *);
279
280 __nvmet_pq_ring_update_tail(&r->prod, prod_head, prod_next, 1);
281 end:
282 if (free_space != NULL)
283 *free_space = free_entries - n;
284 return n;
285 }
286
287 static inline u32
288 __nvmet_pq_ring_do_dequeue(struct nvmet_pq_ring *r,
289 void **obj_table, u32 n, u32 *available)
290 {
291 uint32_t cons_head, cons_next;
292 uint32_t entries;
293
294 n = __nvmet_pq_ring_move_cons_head(r, n, &cons_head, &cons_next, &entries);
295 if (n == 0)
296 goto end;
297
298 DEQUEUE_PTRS(r, &r[1], cons_head, obj_table, n, void *);
299
300 __nvmet_pq_ring_update_tail(&r->cons, cons_head, cons_next, 0);
301
302 end:
303 if (available != NULL)
304 *available = entries - n;
305 return n;
306 }
307
308 static inline u32
309 nvmet_pq_ring_enqueue_bulk(struct nvmet_pq_ring *r,
310 void **obj_table, u32 n, u32 *free_space)
311 {
312 return __nvmet_pq_ring_do_enqueue(r, obj_table, n, free_space);
313 }
314
315 static inline int
316 nvmet_pq_ring_enqueue(struct nvmet_pq_ring *r, void **obj)
317 {
318 return nvmet_pq_ring_enqueue_bulk(r, obj, 1, NULL) ? 0 : -ENOBUFS;
319 }
320
321 static inline u32
322 nvmet_pq_ring_dequeue_bulk(struct nvmet_pq_ring *r,
323 void **obj_table, u32 n, u32 *available)
324 {
325 return __nvmet_pq_ring_do_dequeue(r, obj_table, n, available);
326 }
327
328 static inline int
329 nvmet_pq_ring_dequeue(struct nvmet_pq_ring *r, void **obj_p)
330 {
331 return nvmet_pq_ring_dequeue_bulk(r, obj_p, 1, NULL) ? 0 : -ENOENT;
332 }
333
334 static inline u32
335 __nvmet_pq_ring_count(const struct nvmet_pq_ring *r)
336 {
337 u32 prod_tail = smp_load_acquire(&r->prod.tail);
338 u32 cons_tail = smp_load_acquire(&r->cons.tail);
339 u32 count = (prod_tail - cons_tail) & r->mask;
340 u32 capacity = smp_load_acquire(&r->capacity);
341 return (count > capacity) ? capacity : count;
342 }
343
344 /**
> 345 * Return the number of free entries in a ring. 346 */
347 static inline u32
348 __nvmet_pq_ring_free_count(const struct nvmet_pq_ring *r)
349 {
350 return smp_load_acquire(&r->capacity) - __nvmet_pq_ring_count(r);
351 }
352
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
kernel test robot
build botRe: [PATCH 2/4] nvmet: Add polling queue task for nvme target
In reply to Ping Gankernel test robot noticed the following build warnings:
Hi Ping,
kernel test robot noticed the following build warnings:
[auto build test WARNING on v6.6-rc1]
[also build test WARNING on linus/master next-20230913] [cannot apply to hch-configfs/for-next]
[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Ping-Gan/nvmet-Add-nvme-target-polling-queue-task-parameters/20230913-164112 base: v6.6-rc1 patch link: https://lore.kernel.org/r/84d7f188e892b5b0ba251a4601455d7a137075f3.1694592708.git.jacky_gam_2001%40163.com patch subject: [PATCH 2/4] nvmet: Add polling queue task for nvme target config: x86_64-randconfig-121-20230913 (https://download.01.org/0day-ci/archive/20230913/[email protected]/config) compiler: gcc-12 (Debian 12.2.0-14) 12.2.0 reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20230913/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
sparse warnings: (new ones prefixed by >>)
>> drivers/nvme/target/polling-queue-thread.c:72:29: sparse: sparse: symbol 'nvmet_pqt_data' was not declared. Should it be static?
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
Quoted 7 lines · Ping Gan
Since nvme target currently does not support to submit bio to a polling queue, the bio's completion relies on system interrupt. But when there is high workload in system and the competition is very high, so it makes sense to add polling queue task to submit bio to disk's polling queue and poll the completion queue of disk.
I did some work in the past for nvmet polling and saw good performance improvement.
Can you please share performance numbers for this series ?
-ck
Quoted 11 lines · Ping Gan
Quoted 9 lines
Since nvme target currently does not support to submit bio to a polling
queue, the bio's completion relies on system interrupt. But when there is high workload in system and the competition is very high, so it makes
sense to add polling queue task to submit bio to disk's polling queue and poll the completion queue of disk.
Quoted 6 lines
I did some work in the past for nvmet polling and saw good performance improvement.
Can you please share performance numbers for this series ?
-ck
hi,
I have verified this patch on two testbeds one for host and the other for target. I used tcp as transport protocol, spdk perf as initiator. I do two group tests. The IO size of first is 4K, and the other is 2M. Both includ randrw, randwrite and randrw. Both have same prerequisites. At the initiator side I used 1 qp, 32 queue depth,and 1 spdk perf application, and for target side I bound tcp queue to 1 target core. And I get below results.iosize_4k polling queue interrupt
randrw NIC_rx:338M/s NIC_tx:335M/s NIC_rx:260M/s
NIC_tx:258M/s
randwrite NIC_rx:587M/s NIC_rx:431M/s randread NIC_tx:873M/s NIC_tx:654M/s
iosize_2M polling queue interrupt
randrw NIC_rx:738M/s NIC_tx:741M/s NIC_rx:674M/s
NIC_tx:674M/s
randwrite NIC_rx:1199M/s NIC_rx:1146M/s randread NIC_tx:2226M/s NIC_tx:2119M/s
For iosize 4k the NIC's bandwidth of poling queue is more than 30% than bandwidth of interrupt. But for iosize 2M the improvement is not obvious, the randrw of polling queue is about 9% more than interrupt; randwrite and randread of polling queue is about 5% more than interrupt.
Thanks,
Ping
Re: [PATCH 0/4] nvmet: support polling queue task for bio
In reply to Chaitanya KulkarniQuoted 13 lines · Ping Gan
Quoted 11 lines
Since nvme target currently does not support to submit bio to a polling
queue, the bio's completion relies on system interrupt. But when there
is high workload in system and the competition is very high, so it makes
sense to add polling queue task to submit bio to disk's polling queue
and poll the completion queue of disk.
Quoted 6 lines
I did some work in the past for nvmet polling and saw good performance improvement.
Can you please share performance numbers for this series ?
-ck
hi,
I have verified this patch on two testbeds one for host and the other for target. I used tcp as transport protocol, spdk perf as initiator. I did two group tests. The IO size of first is 4K, and the other is 2M. Both include randrw, randwrite and randrw. Both also have same prerequisites. At the initiator side I used 1 qp, 32 queue depth,and 1 spdk perf application, and for target side I bound tcp queue to 1 target core. And I get below results.iosize_4k polling queue interrupt
randrw NIC_rx:338M/s NIC_tx:335M/s NIC_rx:260M/s
NIC_tx:258M/s
randwrite NIC_rx:587M/s NIC_rx:431M/s randread NIC_tx:873M/s NIC_tx:654M/s
iosize_2M polling queue interrupt
randrw NIC_rx:738M/s NIC_tx:741M/s NIC_rx:674M/s
NIC_tx:674M/s
randwrite NIC_rx:1199M/s NIC_rx:1146M/s randread NIC_tx:2226M/s NIC_tx:2119M/s
For iosize 4k the NIC's bandwidth of poling queue is more than 30% than bandwidth of interrupt. But for iosize 2M the improvement is not obvious,
the randrw of polling queue is about 9% more than interrupt; randwrite and randread of polling queue is about 5% more than interrupt.
Thanks,
Ping