virtio-pmem: Asynchronous flush
Subsystems (3)
Pankaj Gupta
maintainerv2 cover virtio-pmem: Asynchronous flush
From: Pankaj Gupta <[email protected]>
Jeff reported preflush order issue with the existing implementation of virtio pmem preflush. Dan suggested[1] to implement asynchronous flush for virtio pmem using work queue as done in md/RAID. This patch series intends to solve the preflush ordering issue and also makes the flush asynchronous for the submitting thread.
Submitting this patch series for review. Sorry, It took me long time to come back to this due to some personal reasons.
RFC v1 -> RFC v2
- More testing and bug fix.
[1] https://marc.info/?l=linux-kernel&m=157446316409937&w=2
Pankaj Gupta (2):
virtio-pmem: Async virtio-pmem flush
pmem: enable pmem_submit_bio for asynchronous flush
drivers/nvdimm/nd_virtio.c | 72 ++++++++++++++++++++++++++++-------- drivers/nvdimm/pmem.c | 17 ++++++--- drivers/nvdimm/virtio_pmem.c | 10 ++++- drivers/nvdimm/virtio_pmem.h | 14 +++++++ 4 files changed, 91 insertions(+), 22 deletions(-)
Signature · 1 line
2.25.1
Pankaj Gupta
maintainerRFC v2 1/2 virtio-pmem: Async virtio-pmem flush
From: Pankaj Gupta <[email protected]>
Implement asynchronous flush for virtio pmem using work queue to solve the preflush ordering issue. Also, coalesce the flush requests when a flush is already in process.
--- drivers/nvdimm/nd_virtio.c | 72 ++++++++++++++++++++++++++++-------- drivers/nvdimm/virtio_pmem.c | 10 ++++- drivers/nvdimm/virtio_pmem.h | 14 +++++++ 3 files changed, 79 insertions(+), 17 deletions(-)
drivers/nvdimm/nd_virtio.c +56 -16
@@ -97,29 +97,69 @@ static int virtio_pmem_flush(struct nd_region *nd_region)return err; }; +static void submit_async_flush (struct work_struct *ws); +/* The asynchronous flush callback function */ int async_pmem_flush (struct nd_region *nd_region,struct bio *bio) { -/* -* Create child bio for asynchronous flush and chain with -* parent bio. Otherwise directly call nd_region flush. +/* queue asynchronous flush and coalesce the flush requests */ +struct virtio_device *vdev = nd_region->provider_data; +struct virtio_pmem *vpmem = vdev->priv; +ktime_t req_start =ktime_get_boottime (); + +spin_lock_irq (&vpmem->lock); +/* flush requests wait until ongoing flush completes, +* hence coalescing all the pending requests. */ -if (bio && bio->bi_iter.bi_sector !=-1 ) { -struct bio *child =bio_alloc (GFP_ATOMIC ,0 ); - -if (!child) -return -ENOMEM ; -bio_copy_dev (child, bio); - child->bi_opf =REQ_PREFLUSH ; - child->bi_iter.bi_sector =-1 ; -bio_chain (child, bio); -submit_bio (child); -return 0 ; +wait_event_lock_irq (vpmem->sb_wait, + !vpmem->flush_bio || +ktime_before (req_start, vpmem->prev_flush_start), + vpmem->lock); +/* new request after previous flush is completed */ +if (ktime_after (req_start, vpmem->prev_flush_start)) { +WARN_ON (vpmem->flush_bio); + vpmem->flush_bio = bio; + bio =NULL ; + } +spin_unlock_irq (&vpmem->lock); + +if (!bio) { +INIT_WORK (&vpmem->flush_work, submit_async_flush); +queue_work (vpmem->pmem_wq, &vpmem->flush_work); +return 1 ; + } + +/* flush completed in other context while we waited */ +if (bio && (bio->bi_opf &REQ_PREFLUSH )) { + bio->bi_opf &= ~REQ_PREFLUSH ; +submit_bio (bio); + }else if (bio && (bio->bi_opf &REQ_FUA )) { + bio->bi_opf &= ~REQ_FUA ; +bio_endio (bio); } -if (virtio_pmem_flush (nd_region)) -return -EIO ;return 0 ; };EXPORT_SYMBOL_GPL (async_pmem_flush); + +static void submit_async_flush (struct work_struct *ws) +{ + struct virtio_pmem *vpmem = container_of(ws, struct virtio_pmem, flush_work); +struct bio *bio = vpmem->flush_bio; + + vpmem->start_flush =ktime_get_boottime (); + bio->bi_status =errno_to_blk_status (virtio_pmem_flush (vpmem->nd_region)); + vpmem->prev_flush_start = vpmem->start_flush; + vpmem->flush_bio =NULL ; +wake_up (&vpmem->sb_wait); + +/* Submit parent bio only for PREFLUSH */ +if (bio && (bio->bi_opf &REQ_PREFLUSH )) { + bio->bi_opf &= ~REQ_PREFLUSH ; +submit_bio (bio); + }else if (bio && (bio->bi_opf &REQ_FUA )) { + bio->bi_opf &= ~REQ_FUA ; +bio_endio (bio); + } +}MODULE_LICENSE ("GPL" );
drivers/nvdimm/virtio_pmem.c +9 -1
@@ -24,6 +24,7 @@ static int init_vq(struct virtio_pmem *vpmem)return PTR_ERR (vpmem->req_vq);spin_lock_init (&vpmem->pmem_lock); +spin_lock_init (&vpmem->lock);INIT_LIST_HEAD (&vpmem->req_list);return 0 ;
@@ -57,7 +58,12 @@ static int virtio_pmem_probe(struct virtio_device *vdev)dev_err (&vdev->dev,"failed to initialize virtio pmem vq's\n" ); goto out_err; } - + vpmem->pmem_wq =alloc_workqueue ("vpmem_wq" ,WQ_MEM_RECLAIM ,0 ); +if (!vpmem->pmem_wq) { + err = -ENOMEM ; + goto out_err; + } +init_waitqueue_head (&vpmem->sb_wait); virtio_cread_le(vpmem->vdev, struct virtio_pmem_config, start, &vpmem->start); virtio_cread_le(vpmem->vdev, struct virtio_pmem_config,
@@ -90,10 +96,12 @@ static int virtio_pmem_probe(struct virtio_device *vdev) goto out_nd; } nd_region->provider_data =dev_to_virtio (nd_region->dev.parent->parent); + vpmem->nd_region = nd_region;return 0 ; out_nd:nvdimm_bus_unregister (vpmem->nvdimm_bus); out_vq: +destroy_workqueue (vpmem->pmem_wq); vdev->config->del_vqs (vdev); out_err:return err;
drivers/nvdimm/virtio_pmem.h +14 -0
@@ -35,9 +35,23 @@ struct virtio_pmem {/* Virtio pmem request queue */ struct virtqueue *req_vq; +struct bio *flush_bio; +/* last_flush is when the last completed flush was started */ +ktime_t prev_flush_start, start_flush; + +/* work queue for deferred flush */ +struct work_struct flush_work; +struct workqueue_struct *pmem_wq; + +/* Synchronize flush wait queue data */ +spinlock_t lock; +/* for waiting for previous flush to complete */ +wait_queue_head_t sb_wait; +/* nvdimm bus registers virtio pmem device */ struct nvdimm_bus *nvdimm_bus;struct nvdimm_bus_descriptor nd_desc; +struct nd_region *nd_region;/* List to store deferred work if virtqueue is full */ struct list_head req_list;
Signature · 1 line
2.25.1
Re: [RFC v2 1/2] virtio-pmem: Async virtio-pmem flush
Quoted 50 lines · Pankaj Gupta
From: Pankaj Gupta <[email protected]>
Implement asynchronous flush for virtio pmem using work queue to solve the preflush ordering issue. Also, coalesce the flush requests when a flush is already in process.
Signed-off-by Pankaj Gupta <[email protected]>--- drivers/nvdimm/nd_virtio.c | 72 ++++++++++++++++++++++++++++-------- drivers/nvdimm/virtio_pmem.c | 10 ++++- drivers/nvdimm/virtio_pmem.h | 14 +++++++ 3 files changed, 79 insertions(+), 17 deletions(-)drivers/nvdimm/nd_virtio.c +10
-14@@ -97,29 +97,69 @@ static int virtio_pmem_flush(struct nd_region *nd_region)return err; }; +static void submit_async_flush (struct work_struct *ws); +/* The asynchronous flush callback function */ int async_pmem_flush (struct nd_region *nd_region,struct bio *bio) { -/* -* Create child bio for asynchronous flush and chain with -* parent bio. Otherwise directly call nd_region flush. +/* queue asynchronous flush and coalesce the flush requests */ +struct virtio_device *vdev = nd_region->provider_data; +struct virtio_pmem *vpmem = vdev->priv; +ktime_t req_start =ktime_get_boottime (); + +spin_lock_irq (&vpmem->lock); +/* flush requests wait until ongoing flush completes, +* hence coalescing all the pending requests. */ -if (bio && bio->bi_iter.bi_sector !=-1 ) { -struct bio *child =bio_alloc (GFP_ATOMIC ,0 ); - -if (!child) -return -ENOMEM ; -bio_copy_dev (child, bio); - child->bi_opf =REQ_PREFLUSH ; - child->bi_iter.bi_sector =-1 ; -bio_chain (child, bio); -submit_bio (child); -return 0 ;
drivers/nvdimm/nd_virtio.c static int virtio_pmem_flush(struct nd_region *nd_region) · line 123
Quoted 10 lines
drivers/nvdimm/nd_virtio.c +10
-0+wait_event_lock_irq (vpmem->sb_wait, + !vpmem->flush_bio || +ktime_before (req_start, vpmem->prev_flush_start), + vpmem->lock); +/* new request after previous flush is completed */ +if (ktime_after (req_start, vpmem->prev_flush_start)) { +WARN_ON (vpmem->flush_bio); + vpmem->flush_bio = bio; + bio =NULL ; + }
Why the dance with ->prev_flush_start vs just calling queue_work() again. queue_work() is naturally coalescing in that if the last work request has not started execution another queue attempt will be dropped.
drivers/nvdimm/nd_virtio.c static int virtio_pmem_flush(struct nd_region *nd_region) · line 127
Quoted 4 lines
drivers/nvdimm/nd_virtio.c +4
-0+spin_unlock_irq (&vpmem->lock); + +if (!bio) { +INIT_WORK (&vpmem->flush_work, submit_async_flush);
I expect this only needs to be initialized once at driver init time.
drivers/nvdimm/nd_virtio.c static int virtio_pmem_flush(struct nd_region *nd_region) · line 138
Quoted 11 lines
drivers/nvdimm/nd_virtio.c +11
-0+queue_work (vpmem->pmem_wq, &vpmem->flush_work); +return 1 ; + } + +/* flush completed in other context while we waited */ +if (bio && (bio->bi_opf &REQ_PREFLUSH )) { + bio->bi_opf &= ~REQ_PREFLUSH ; +submit_bio (bio); + }else if (bio && (bio->bi_opf &REQ_FUA )) { + bio->bi_opf &= ~REQ_FUA ; +bio_endio (bio);
It's not clear to me how this happens, shouldn't all flush completions be driven from the work completion?
Quoted 17 lines
drivers/nvdimm/nd_virtio.c +10
-2} -if (virtio_pmem_flush (nd_region)) -return -EIO ;return 0 ; };EXPORT_SYMBOL_GPL (async_pmem_flush); + +static void submit_async_flush (struct work_struct *ws) +{ + struct virtio_pmem *vpmem = container_of(ws, struct virtio_pmem, flush_work); +struct bio *bio = vpmem->flush_bio; + + vpmem->start_flush =ktime_get_boottime (); + bio->bi_status =errno_to_blk_status (virtio_pmem_flush (vpmem->nd_region)); + vpmem->prev_flush_start = vpmem->start_flush; + vpmem->flush_bio =NULL ;
drivers/nvdimm/nd_virtio.c static int virtio_pmem_flush(struct nd_region *nd_region) · line 163
Quoted 10 lines
drivers/nvdimm/nd_virtio.c +10
-0+wake_up (&vpmem->sb_wait); + +/* Submit parent bio only for PREFLUSH */ +if (bio && (bio->bi_opf &REQ_PREFLUSH )) { + bio->bi_opf &= ~REQ_PREFLUSH ; +submit_bio (bio); + }else if (bio && (bio->bi_opf &REQ_FUA )) { + bio->bi_opf &= ~REQ_FUA ; +bio_endio (bio); + }
Shouldn't the wait_event_lock_irq() be here rather than in async_pmem_flush()? That will cause the workqueue to back up and flush requests to coalesce.
drivers/nvdimm/virtio_pmem.c static int init_vq(struct virtio_pmem *vpmem) · line 27
Quoted 11 lines
drivers/nvdimm/nd_virtio.c +1
-0+}MODULE_LICENSE ("GPL" );drivers/nvdimm/virtio_pmem.c +1
-0@@ -24,6 +24,7 @@ static int init_vq(struct virtio_pmem *vpmem)return PTR_ERR (vpmem->req_vq);spin_lock_init (&vpmem->pmem_lock); +spin_lock_init (&vpmem->lock);
Why 2 locks?
Pankaj Gupta
maintainerRe: [RFC v2 1/2] virtio-pmem: Async virtio-pmem flush
Hi Dan,
Thank you for the review. Please see my reply inline.
Quoted 54 lines
Quoted 54 lines
Implement asynchronous flush for virtio pmem using work queue to solve the preflush ordering issue. Also, coalesce the flush requests when a flush is already in process.
Signed-off-by Pankaj Gupta <[email protected]>--- drivers/nvdimm/nd_virtio.c | 72 ++++++++++++++++++++++++++++-------- drivers/nvdimm/virtio_pmem.c | 10 ++++- drivers/nvdimm/virtio_pmem.h | 14 +++++++ 3 files changed, 79 insertions(+), 17 deletions(-)drivers/nvdimm/nd_virtio.c +17
-14@@ -97,29 +97,69 @@ static int virtio_pmem_flush(struct nd_region *nd_region)return err; }; +static void submit_async_flush (struct work_struct *ws); +/* The asynchronous flush callback function */ int async_pmem_flush (struct nd_region *nd_region,struct bio *bio) { -/* -* Create child bio for asynchronous flush and chain with -* parent bio. Otherwise directly call nd_region flush. +/* queue asynchronous flush and coalesce the flush requests */ +struct virtio_device *vdev = nd_region->provider_data; +struct virtio_pmem *vpmem = vdev->priv; +ktime_t req_start =ktime_get_boottime (); + +spin_lock_irq (&vpmem->lock); +/* flush requests wait until ongoing flush completes, +* hence coalescing all the pending requests. */ -if (bio && bio->bi_iter.bi_sector !=-1 ) { -struct bio *child =bio_alloc (GFP_ATOMIC ,0 ); - -if (!child) -return -ENOMEM ; -bio_copy_dev (child, bio); - child->bi_opf =REQ_PREFLUSH ; - child->bi_iter.bi_sector =-1 ; -bio_chain (child, bio); -submit_bio (child); -return 0 ; +wait_event_lock_irq (vpmem->sb_wait, + !vpmem->flush_bio || +ktime_before (req_start, vpmem->prev_flush_start), + vpmem->lock); +/* new request after previous flush is completed */ +if (ktime_after (req_start, vpmem->prev_flush_start)) { +WARN_ON (vpmem->flush_bio);
Quoted 8 lines
drivers/nvdimm/nd_virtio.c static int virtio_pmem_flush(struct nd_region *nd_region) · line 123
Quoted 3 lines
drivers/nvdimm/nd_virtio.c +3
-0+ vpmem->flush_bio = bio; + bio =NULL ; + }Why the dance with ->prev_flush_start vs just calling
queue_work()again.queue_work()is naturally coalescing in that if the last work request has not started execution another queue attempt will be dropped.
How parent flush request will know when corresponding flush is completed?
Quoted 7 lines
drivers/nvdimm/nd_virtio.c static int virtio_pmem_flush(struct nd_region *nd_region) · line 127
Quoted 5 lines
drivers/nvdimm/nd_virtio.c +4
-0+spin_unlock_irq (&vpmem->lock); + +if (!bio) { +INIT_WORK (&vpmem->flush_work, submit_async_flush);I expect this only needs to be initialized once at driver init time.
yes, will fix this.
Quoted 13 lines
drivers/nvdimm/nd_virtio.c static int virtio_pmem_flush(struct nd_region *nd_region) · line 138
Quoted 12 lines
drivers/nvdimm/nd_virtio.c +11
-0+queue_work (vpmem->pmem_wq, &vpmem->flush_work); +return 1 ; + } + +/* flush completed in other context while we waited */ +if (bio && (bio->bi_opf &REQ_PREFLUSH )) { + bio->bi_opf &= ~REQ_PREFLUSH ; +submit_bio (bio); + }else if (bio && (bio->bi_opf &REQ_FUA )) { + bio->bi_opf &= ~REQ_FUA ; +bio_endio (bio);
Quoted 2 lines
It's not clear to me how this happens, shouldn't all flush completions be driven from the work completion?
Requests should progress after notified by ongoing flush completion event.
Quoted 24 lines
Quoted 24 lines
drivers/nvdimm/nd_virtio.c +16
-2} -if (virtio_pmem_flush (nd_region)) -return -EIO ;return 0 ; };EXPORT_SYMBOL_GPL (async_pmem_flush); + +static void submit_async_flush (struct work_struct *ws) +{ + struct virtio_pmem *vpmem = container_of(ws, struct virtio_pmem, flush_work); +struct bio *bio = vpmem->flush_bio; + + vpmem->start_flush =ktime_get_boottime (); + bio->bi_status =errno_to_blk_status (virtio_pmem_flush (vpmem->nd_region)); + vpmem->prev_flush_start = vpmem->start_flush; + vpmem->flush_bio =NULL ; +wake_up (&vpmem->sb_wait); + +/* Submit parent bio only for PREFLUSH */ +if (bio && (bio->bi_opf &REQ_PREFLUSH )) { + bio->bi_opf &= ~REQ_PREFLUSH ; +submit_bio (bio);
Quoted 8 lines
drivers/nvdimm/nd_virtio.c static int virtio_pmem_flush(struct nd_region *nd_region) · line 163
Quoted 4 lines
drivers/nvdimm/nd_virtio.c +4
-0+ }else if (bio && (bio->bi_opf &REQ_FUA )) { + bio->bi_opf &= ~REQ_FUA ; +bio_endio (bio); + }Shouldn't the
wait_event_lock_irq()be here rather than inasync_pmem_flush()? That will cause the workqueue to back up and flush requests to coalesce.
but this is coalesced flush request?
Quoted 9 lines
Quoted 9 lines
drivers/nvdimm/nd_virtio.c +1
-0+}MODULE_LICENSE ("GPL" );drivers/nvdimm/virtio_pmem.c +0
-0@@ -24,6 +24,7 @@ static int init_vq(struct virtio_pmem *vpmem)return PTR_ERR (vpmem->req_vq);
Quoted 4 lines
drivers/nvdimm/virtio_pmem.c static int init_vq(struct virtio_pmem *vpmem) · line 27
Quoted 2 lines
drivers/nvdimm/virtio_pmem.c +1
-0spin_lock_init (&vpmem->pmem_lock); +spin_lock_init (&vpmem->lock);Why 2 locks?
One lock is for work queue and other for virtio flush completion.
Thanks,
Pankaj
Re: [RFC v2 1/2] virtio-pmem: Async virtio-pmem flush
Quoted 68 lines · Pankaj Gupta
Hi Dan,
Thank you for the review. Please see my reply inline.
Quoted 54 lines
Quoted 54 lines
Implement asynchronous flush for virtio pmem using work queue to solve the preflush ordering issue. Also, coalesce the flush requests when a flush is already in process.
Signed-off-by Pankaj Gupta <[email protected]>--- drivers/nvdimm/nd_virtio.c | 72 ++++++++++++++++++++++++++++-------- drivers/nvdimm/virtio_pmem.c | 10 ++++- drivers/nvdimm/virtio_pmem.h | 14 +++++++ 3 files changed, 79 insertions(+), 17 deletions(-)drivers/nvdimm/nd_virtio.c +17
-14@@ -97,29 +97,69 @@ static int virtio_pmem_flush(struct nd_region *nd_region)return err; }; +static void submit_async_flush (struct work_struct *ws); +/* The asynchronous flush callback function */ int async_pmem_flush (struct nd_region *nd_region,struct bio *bio) { -/* -* Create child bio for asynchronous flush and chain with -* parent bio. Otherwise directly call nd_region flush. +/* queue asynchronous flush and coalesce the flush requests */ +struct virtio_device *vdev = nd_region->provider_data; +struct virtio_pmem *vpmem = vdev->priv; +ktime_t req_start =ktime_get_boottime (); + +spin_lock_irq (&vpmem->lock); +/* flush requests wait until ongoing flush completes, +* hence coalescing all the pending requests. */ -if (bio && bio->bi_iter.bi_sector !=-1 ) { -struct bio *child =bio_alloc (GFP_ATOMIC ,0 ); - -if (!child) -return -ENOMEM ; -bio_copy_dev (child, bio); - child->bi_opf =REQ_PREFLUSH ; - child->bi_iter.bi_sector =-1 ; -bio_chain (child, bio); -submit_bio (child); -return 0 ; +wait_event_lock_irq (vpmem->sb_wait, + !vpmem->flush_bio || +ktime_before (req_start, vpmem->prev_flush_start), + vpmem->lock); +/* new request after previous flush is completed */ +if (ktime_after (req_start, vpmem->prev_flush_start)) { +WARN_ON (vpmem->flush_bio);Quoted 8 lines
drivers/nvdimm/nd_virtio.c static int virtio_pmem_flush(struct nd_region *nd_region) · line 123
Quoted 3 lines
drivers/nvdimm/nd_virtio.c +3
-0+ vpmem->flush_bio = bio; + bio =NULL ; + }Why the dance with ->prev_flush_start vs just calling
queue_work()again.queue_work()is naturally coalescing in that if the last work request has not started execution another queue attempt will be dropped.
Quoted 1 line
How parent flush request will know when corresponding flush is completed?
The eventual bio_endio() is what signals upper layers that the flush completed...
Hold on... it's been so long that I forgot that you are copying md_flush_request() here. It would help immensely if that was mentioned in the changelog and at a minimum have a comment in the code that this was copied from md. In fact it would be extra helpful if you refactored a common helper that bio based block drivers could share for implementing flush handling, but that can come later.
Let me go re-review this with respect to whether the md case is fully applicable here.
Re: [RFC v2 1/2] virtio-pmem: Async virtio-pmem flush
Quoted 69 lines
Quoted 67 lines
Hi Dan,
Thank you for the review. Please see my reply inline.
Quoted 54 lines
Quoted 54 lines
Implement asynchronous flush for virtio pmem using work queue to solve the preflush ordering issue. Also, coalesce the flush requests when a flush is already in process.
Signed-off-by Pankaj Gupta <[email protected]>--- drivers/nvdimm/nd_virtio.c | 72 ++++++++++++++++++++++++++++-------- drivers/nvdimm/virtio_pmem.c | 10 ++++- drivers/nvdimm/virtio_pmem.h | 14 +++++++ 3 files changed, 79 insertions(+), 17 deletions(-)drivers/nvdimm/nd_virtio.c +17
-14@@ -97,29 +97,69 @@ static int virtio_pmem_flush(struct nd_region *nd_region)return err; }; +static void submit_async_flush (struct work_struct *ws); +/* The asynchronous flush callback function */ int async_pmem_flush (struct nd_region *nd_region,struct bio *bio) { -/* -* Create child bio for asynchronous flush and chain with -* parent bio. Otherwise directly call nd_region flush. +/* queue asynchronous flush and coalesce the flush requests */ +struct virtio_device *vdev = nd_region->provider_data; +struct virtio_pmem *vpmem = vdev->priv; +ktime_t req_start =ktime_get_boottime (); + +spin_lock_irq (&vpmem->lock); +/* flush requests wait until ongoing flush completes, +* hence coalescing all the pending requests. */ -if (bio && bio->bi_iter.bi_sector !=-1 ) { -struct bio *child =bio_alloc (GFP_ATOMIC ,0 ); - -if (!child) -return -ENOMEM ; -bio_copy_dev (child, bio); - child->bi_opf =REQ_PREFLUSH ; - child->bi_iter.bi_sector =-1 ; -bio_chain (child, bio); -submit_bio (child); -return 0 ; +wait_event_lock_irq (vpmem->sb_wait, + !vpmem->flush_bio || +ktime_before (req_start, vpmem->prev_flush_start), + vpmem->lock); +/* new request after previous flush is completed */ +if (ktime_after (req_start, vpmem->prev_flush_start)) { +WARN_ON (vpmem->flush_bio);Quoted 8 lines
drivers/nvdimm/nd_virtio.c static int virtio_pmem_flush(struct nd_region *nd_region) · line 123
Quoted 3 lines
drivers/nvdimm/nd_virtio.c +3
-0+ vpmem->flush_bio = bio; + bio =NULL ; + }Why the dance with ->prev_flush_start vs just calling
queue_work()again.queue_work()is naturally coalescing in that if the last work request has not started execution another queue attempt will be dropped.Quoted 1 line
How parent flush request will know when corresponding flush is completed?
Quoted 8 lines
The eventual
bio_endio()is what signals upper layers that the flush completed...Hold on... it's been so long that I forgot that you are copying
md_flush_request()here. It would help immensely if that was mentioned in the changelog and at a minimum have a comment in the code that this was copied from md. In fact it would be extra helpful if you
My bad. I only mentioned this in the cover letter.
Quoted 2 lines
refactored a common helper that bio based block drivers could share for implementing flush handling, but that can come later.
Sure.
Quoted 3 lines
Let me go re-review this with respect to whether the md case is fully applicable here.
o.k.
Best regards,
Pankaj
Re: [RFC v2 1/2] virtio-pmem: Async virtio-pmem flush
Quoted 74 lines · Pankaj Gupta
Quoted 74 lines
Quoted 68 lines
Hi Dan,
Thank you for the review. Please see my reply inline.
Quoted 54 lines
Quoted 54 lines
Implement asynchronous flush for virtio pmem using work queue to solve the preflush ordering issue. Also, coalesce the flush requests when a flush is already in process.
Signed-off-by Pankaj Gupta <[email protected]>--- drivers/nvdimm/nd_virtio.c | 72 ++++++++++++++++++++++++++++-------- drivers/nvdimm/virtio_pmem.c | 10 ++++- drivers/nvdimm/virtio_pmem.h | 14 +++++++ 3 files changed, 79 insertions(+), 17 deletions(-)drivers/nvdimm/nd_virtio.c +17
-14@@ -97,29 +97,69 @@ static int virtio_pmem_flush(struct nd_region *nd_region)return err; }; +static void submit_async_flush (struct work_struct *ws); +/* The asynchronous flush callback function */ int async_pmem_flush (struct nd_region *nd_region,struct bio *bio) { -/* -* Create child bio for asynchronous flush and chain with -* parent bio. Otherwise directly call nd_region flush. +/* queue asynchronous flush and coalesce the flush requests */ +struct virtio_device *vdev = nd_region->provider_data; +struct virtio_pmem *vpmem = vdev->priv; +ktime_t req_start =ktime_get_boottime (); + +spin_lock_irq (&vpmem->lock); +/* flush requests wait until ongoing flush completes, +* hence coalescing all the pending requests. */ -if (bio && bio->bi_iter.bi_sector !=-1 ) { -struct bio *child =bio_alloc (GFP_ATOMIC ,0 ); - -if (!child) -return -ENOMEM ; -bio_copy_dev (child, bio); - child->bi_opf =REQ_PREFLUSH ; - child->bi_iter.bi_sector =-1 ; -bio_chain (child, bio); -submit_bio (child); -return 0 ; +wait_event_lock_irq (vpmem->sb_wait, + !vpmem->flush_bio || +ktime_before (req_start, vpmem->prev_flush_start), + vpmem->lock); +/* new request after previous flush is completed */ +if (ktime_after (req_start, vpmem->prev_flush_start)) { +WARN_ON (vpmem->flush_bio);Quoted 8 lines
drivers/nvdimm/nd_virtio.c static int virtio_pmem_flush(struct nd_region *nd_region) · line 123
Quoted 3 lines
drivers/nvdimm/nd_virtio.c +3
-0+ vpmem->flush_bio = bio; + bio =NULL ; + }Why the dance with ->prev_flush_start vs just calling
queue_work()again.queue_work()is naturally coalescing in that if the last work request has not started execution another queue attempt will be dropped.Quoted 1 line
How parent flush request will know when corresponding flush is completed?
The eventual
bio_endio()is what signals upper layers that the flush completed...
Quoted 6 lines
Quoted 4 lines
Hold on... it's been so long that I forgot that you are copying
md_flush_request()here. It would help immensely if that was mentioned in the changelog and at a minimum have a comment in the code that this was copied from md. In fact it would be extra helpful if youMy bad. I only mentioned this in the cover letter.
Yeah, sorry about that. Having come back to this after so long I just decided to jump straight into the patches, but even if I had read that cover I still would have given the feedback that md_flush_request() heritage should also be noted with a comment in the code.
Re: [RFC v2 1/2] virtio-pmem: Async virtio-pmem flush
Quoted 74 lines
Quoted 74 lines
Quoted 73 lines
Quoted 63 lines
Quoted 54 lines
Quoted 54 lines
Implement asynchronous flush for virtio pmem using work queue to solve the preflush ordering issue. Also, coalesce the flush requests when a flush is already in process.
Signed-off-by Pankaj Gupta <[email protected]>--- drivers/nvdimm/nd_virtio.c | 72 ++++++++++++++++++++++++++++-------- drivers/nvdimm/virtio_pmem.c | 10 ++++- drivers/nvdimm/virtio_pmem.h | 14 +++++++ 3 files changed, 79 insertions(+), 17 deletions(-)drivers/nvdimm/nd_virtio.c +17
-14@@ -97,29 +97,69 @@ static int virtio_pmem_flush(struct nd_region *nd_region)return err; }; +static void submit_async_flush (struct work_struct *ws); +/* The asynchronous flush callback function */ int async_pmem_flush (struct nd_region *nd_region,struct bio *bio) { -/* -* Create child bio for asynchronous flush and chain with -* parent bio. Otherwise directly call nd_region flush. +/* queue asynchronous flush and coalesce the flush requests */ +struct virtio_device *vdev = nd_region->provider_data; +struct virtio_pmem *vpmem = vdev->priv; +ktime_t req_start =ktime_get_boottime (); + +spin_lock_irq (&vpmem->lock); +/* flush requests wait until ongoing flush completes, +* hence coalescing all the pending requests. */ -if (bio && bio->bi_iter.bi_sector !=-1 ) { -struct bio *child =bio_alloc (GFP_ATOMIC ,0 ); - -if (!child) -return -ENOMEM ; -bio_copy_dev (child, bio); - child->bi_opf =REQ_PREFLUSH ; - child->bi_iter.bi_sector =-1 ; -bio_chain (child, bio); -submit_bio (child); -return 0 ; +wait_event_lock_irq (vpmem->sb_wait, + !vpmem->flush_bio || +ktime_before (req_start, vpmem->prev_flush_start), + vpmem->lock); +/* new request after previous flush is completed */ +if (ktime_after (req_start, vpmem->prev_flush_start)) { +WARN_ON (vpmem->flush_bio);Quoted 8 lines
drivers/nvdimm/nd_virtio.c static int virtio_pmem_flush(struct nd_region *nd_region) · line 123
Quoted 3 lines
drivers/nvdimm/nd_virtio.c +3
-0+ vpmem->flush_bio = bio; + bio =NULL ; + }Why the dance with ->prev_flush_start vs just calling
queue_work()again.queue_work()is naturally coalescing in that if the last work request has not started execution another queue attempt will be dropped.Quoted 1 line
How parent flush request will know when corresponding flush is completed?
The eventual
bio_endio()is what signals upper layers that the flush completed...Hold on... it's been so long that I forgot that you are copying
md_flush_request()here. It would help immensely if that was mentioned in the changelog and at a minimum have a comment in the code that this was copied from md. In fact it would be extra helpful if you
Quoted 6 lines
Quoted 1 line
My bad. I only mentioned this in the cover letter.
Yeah, sorry about that. Having come back to this after so long I just decided to jump straight into the patches, but even if I had read that cover I still would have given the feedback that
md_flush_request()heritage should also be noted with a comment in the code.
Sure.
Thanks,
Pankaj
Pankaj Gupta
maintainerRFC v2 2/2 pmem: Enable pmem_submit_bio for asynchronous flush
In reply to Pankaj GuptaFrom: Pankaj Gupta <[email protected]>
Return from "pmem_submit_bio" when asynchronous flush is in process in other context.
--- drivers/nvdimm/pmem.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-)
drivers/nvdimm/pmem.c +12 -5
@@ -201,8 +201,13 @@ static blk_qc_t pmem_submit_bio(struct bio *bio)struct pmem_device *pmem = bio->bi_bdev->bd_disk->private_data;struct nd_region *nd_region =to_region (pmem); -if (bio->bi_opf &REQ_PREFLUSH ) - ret =nvdimm_flush (nd_region, bio); +if ((bio->bi_opf &REQ_PREFLUSH ) && +nvdimm_flush (nd_region, bio)) { + +/* asynchronous flush completes in other context */ +if (nd_region->flush) +return BLK_QC_T_NONE ; + } do_acct =blk_queue_io_stat (bio->bi_bdev->bd_disk->queue);if (do_acct)
@@ -222,11 +227,13 @@ static blk_qc_t pmem_submit_bio(struct bio *bio)if (do_acct)bio_end_io_acct (bio, start); -if (bio->bi_opf &REQ_FUA ) +if (bio->bi_opf &REQ_FUA ) { ret =nvdimm_flush (nd_region, bio); -if (ret) - bio->bi_status =errno_to_blk_status (ret); +/* asynchronous flush completes in other context */ +if (nd_region->flush) +return BLK_QC_T_NONE ; + }bio_endio (bio);return BLK_QC_T_NONE ;
Signature · 1 line
2.25.1
kernel test robot
build botRe: [RFC v2 2/2] pmem: Enable pmem_submit_bio for asynchronous flush
[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in https://git-scm.com/docs/git-format-patch]
Hi Pankaj,
[FYI, it's a private test report for your RFC patch.]
[auto build test WARNING on linux/master]
[also build test WARNING on linus/master linux-nvdimm/libnvdimm-for-next v5.14-rc3 next-20210723] [If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]
url: https://github.com/0day-ci/linux/commits/Pankaj-Gupta/virtio-pmem-Async-virtio-pmem-flush/20210726-150913 base: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git 349a2d52ffe59b7a0c5876fa7ee9f3eaf188b830 config: arc-allyesconfig (attached as .config) compiler: arceb-elf-gcc (GCC) 10.3.0
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# https://github.com/0day-ci/linux/commit/e6040aad1930816dc183a36f6ff5eae1412a9713
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Pankaj-Gupta/virtio-pmem-Async-virtio-pmem-flush/20210726-150913 git checkout e6040aad1930816dc183a36f6ff5eae1412a9713
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-10.3.0 make.cross ARCH=arc
If you fix the issue, kindly add following tag as appropriate
All warnings (new ones prefixed by >>):
drivers/nvdimm/pmem.c: In function 'pmem_submit_bio':
>> drivers/nvdimm/pmem.c:195:6: warning: variable 'ret' set but not used [-Wunused-but-set-variable]
195 | int ret = 0;
| ^~~vim +/ret +195 drivers/nvdimm/pmem.c
9e853f2313e5eb drivers/block/pmem.c Ross Zwisler 2015-04-01 192
c62b37d96b6eb3 drivers/nvdimm/pmem.c Christoph Hellwig 2020-07-01 193 static blk_qc_t pmem_submit_bio(struct bio *bio)
9e853f2313e5eb drivers/block/pmem.c Ross Zwisler 2015-04-01 194 {
c5d4355d10d414 drivers/nvdimm/pmem.c Pankaj Gupta 2019-07-05 @195 int ret = 0;
4e4cbee93d5613 drivers/nvdimm/pmem.c Christoph Hellwig 2017-06-03 196 blk_status_t rc = 0;
f0dc089ce217e7 drivers/nvdimm/pmem.c Dan Williams 2015-05-16 197 bool do_acct;
f0dc089ce217e7 drivers/nvdimm/pmem.c Dan Williams 2015-05-16 198 unsigned long start;
9e853f2313e5eb drivers/block/pmem.c Ross Zwisler 2015-04-01 199 struct bio_vec bvec;
9e853f2313e5eb drivers/block/pmem.c Ross Zwisler 2015-04-01 200 struct bvec_iter iter;
309dca309fc39a drivers/nvdimm/pmem.c Christoph Hellwig 2021-01-24 201 struct pmem_device *pmem = bio->bi_bdev->bd_disk->private_data;
7e267a8c790edf drivers/nvdimm/pmem.c Dan Williams 2016-06-01 202 struct nd_region *nd_region = to_region(pmem);
7e267a8c790edf drivers/nvdimm/pmem.c Dan Williams 2016-06-01 203
e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 204 if ((bio->bi_opf & REQ_PREFLUSH) &&
e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 205 nvdimm_flush(nd_region, bio)) {
e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 206
e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 207 /* asynchronous flush completes in other context */
e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 208 if (nd_region->flush)
e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 209 return BLK_QC_T_NONE;
e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 210 }
9e853f2313e5eb drivers/block/pmem.c Ross Zwisler 2015-04-01 211
309dca309fc39a drivers/nvdimm/pmem.c Christoph Hellwig 2021-01-24 212 do_acct = blk_queue_io_stat(bio->bi_bdev->bd_disk->queue);
0fd92f89a44d3b drivers/nvdimm/pmem.c Christoph Hellwig 2020-05-27 213 if (do_acct)
0fd92f89a44d3b drivers/nvdimm/pmem.c Christoph Hellwig 2020-05-27 214 start = bio_start_io_acct(bio);
e10624f8c09710 drivers/nvdimm/pmem.c Dan Williams 2016-01-06 215 bio_for_each_segment(bvec, bio, iter) {
5d64efe79703c4 drivers/nvdimm/pmem.c Vivek Goyal 2020-02-28 216 if (op_is_write(bio_op(bio)))
5d64efe79703c4 drivers/nvdimm/pmem.c Vivek Goyal 2020-02-28 217 rc = pmem_do_write(pmem, bvec.bv_page, bvec.bv_offset,
5d64efe79703c4 drivers/nvdimm/pmem.c Vivek Goyal 2020-02-28 218 iter.bi_sector, bvec.bv_len);
5d64efe79703c4 drivers/nvdimm/pmem.c Vivek Goyal 2020-02-28 219 else
5d64efe79703c4 drivers/nvdimm/pmem.c Vivek Goyal 2020-02-28 220 rc = pmem_do_read(pmem, bvec.bv_page, bvec.bv_offset,
5d64efe79703c4 drivers/nvdimm/pmem.c Vivek Goyal 2020-02-28 221 iter.bi_sector, bvec.bv_len);
e10624f8c09710 drivers/nvdimm/pmem.c Dan Williams 2016-01-06 222 if (rc) {
4e4cbee93d5613 drivers/nvdimm/pmem.c Christoph Hellwig 2017-06-03 223 bio->bi_status = rc;
e10624f8c09710 drivers/nvdimm/pmem.c Dan Williams 2016-01-06 224 break;
e10624f8c09710 drivers/nvdimm/pmem.c Dan Williams 2016-01-06 225 }
e10624f8c09710 drivers/nvdimm/pmem.c Dan Williams 2016-01-06 226 }
f0dc089ce217e7 drivers/nvdimm/pmem.c Dan Williams 2015-05-16 227 if (do_acct)
0fd92f89a44d3b drivers/nvdimm/pmem.c Christoph Hellwig 2020-05-27 228 bio_end_io_acct(bio, start);
61031952f4c89d drivers/nvdimm/pmem.c Ross Zwisler 2015-06-25 229
e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 230 if (bio->bi_opf & REQ_FUA) {
c5d4355d10d414 drivers/nvdimm/pmem.c Pankaj Gupta 2019-07-05 231 ret = nvdimm_flush(nd_region, bio);
c5d4355d10d414 drivers/nvdimm/pmem.c Pankaj Gupta 2019-07-05 232
e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 233 /* asynchronous flush completes in other context */
e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 234 if (nd_region->flush)
e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 235 return BLK_QC_T_NONE;
e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 236 }
61031952f4c89d drivers/nvdimm/pmem.c Ross Zwisler 2015-06-25 237
4246a0b63bd8f5 drivers/nvdimm/pmem.c Christoph Hellwig 2015-07-20 238 bio_endio(bio);
dece16353ef47d drivers/nvdimm/pmem.c Jens Axboe 2015-11-05 239 return BLK_QC_T_NONE;
9e853f2313e5eb drivers/block/pmem.c Ross Zwisler 2015-04-01 240 }
9e853f2313e5eb drivers/block/pmem.c Ross Zwisler 2015-04-01 241 ---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org
Pankaj Gupta
maintainerRe: [RFC v2 2/2] pmem: Enable pmem_submit_bio for asynchronous flush
Quoted 24 lines
Hi Pankaj,
[FYI, it's a private test report for your RFC patch.]
[auto build test WARNING on linux/master]
[also build test WARNING on linus/master linux-nvdimm/libnvdimm-for-next v5.14-rc3 next-20210723] [If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]url: https://github.com/0day-ci/linux/commits/Pankaj-Gupta/virtio-pmem-Async-virtio-pmem-flush/20210726-150913 base: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git 349a2d52ffe59b7a0c5876fa7ee9f3eaf188b830 config: arc-allyesconfig (attached as .config) compiler: arceb-elf-gcc (GCC) 10.3.0reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# https://github.com/0day-ci/linux/commit/e6040aad1930816dc183a36f6ff5eae1412a9713
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Pankaj-Gupta/virtio-pmem-Async-virtio-pmem-flush/20210726-150913 git checkoute6040aad1930816dc183a36f6ff5eae1412a9713
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-10.3.0 make.cross ARCH=arcIf you fix the issue, kindly add following tag as appropriate
Quoted 8 lines
Reported-by kernel test robot <[email protected]>All warnings (new ones prefixed by >>):
drivers/nvdimm/pmem.c: In function 'pmem_submit_bio':Quoted 1 line
Quoted 1 line
drivers/nvdimm/pmem.c:195:6: warning: variable 'ret' set but not used [-Wunused-but-set-variable]195 | int ret = 0; | ^~~
Thank you, will fix this.
Quoted 58 lines
vim +/ret +195
drivers/nvdimm/pmem.c9e853f2313e5eb drivers/block/pmem.c Ross Zwisler 2015-04-01 192 c62b37d96b6eb3 drivers/nvdimm/pmem.c Christoph Hellwig 2020-07-01 193 static blk_qc_t pmem_submit_bio(struct bio *bio) 9e853f2313e5eb drivers/block/pmem.c Ross Zwisler 2015-04-01 194 { c5d4355d10d414 drivers/nvdimm/pmem.c Pankaj Gupta 2019-07-05 @195 int ret = 0; 4e4cbee93d5613 drivers/nvdimm/pmem.c Christoph Hellwig 2017-06-03 196 blk_status_t rc = 0; f0dc089ce217e7 drivers/nvdimm/pmem.c Dan Williams 2015-05-16 197 bool do_acct; f0dc089ce217e7 drivers/nvdimm/pmem.c Dan Williams 2015-05-16 198 unsigned long start; 9e853f2313e5eb drivers/block/pmem.c Ross Zwisler 2015-04-01 199 struct bio_vec bvec; 9e853f2313e5eb drivers/block/pmem.c Ross Zwisler 2015-04-01 200 struct bvec_iter iter; 309dca309fc39a drivers/nvdimm/pmem.c Christoph Hellwig 2021-01-24 201 struct pmem_device *pmem = bio->bi_bdev->bd_disk->private_data; 7e267a8c790edf drivers/nvdimm/pmem.c Dan Williams 2016-06-01 202 struct nd_region *nd_region = to_region(pmem); 7e267a8c790edf drivers/nvdimm/pmem.c Dan Williams 2016-06-01 203 e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 204 if ((bio->bi_opf & REQ_PREFLUSH) && e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 205 nvdimm_flush(nd_region, bio)) { e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 206 e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 207 /* asynchronous flush completes in other context */ e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 208 if (nd_region->flush) e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 209 return BLK_QC_T_NONE; e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 210 } 9e853f2313e5eb drivers/block/pmem.c Ross Zwisler 2015-04-01 211 309dca309fc39a drivers/nvdimm/pmem.c Christoph Hellwig 2021-01-24 212 do_acct = blk_queue_io_stat(bio->bi_bdev->bd_disk->queue); 0fd92f89a44d3b drivers/nvdimm/pmem.c Christoph Hellwig 2020-05-27 213 if (do_acct) 0fd92f89a44d3b drivers/nvdimm/pmem.c Christoph Hellwig 2020-05-27 214 start = bio_start_io_acct(bio); e10624f8c09710 drivers/nvdimm/pmem.c Dan Williams 2016-01-06 215 bio_for_each_segment(bvec, bio, iter) { 5d64efe79703c4 drivers/nvdimm/pmem.c Vivek Goyal 2020-02-28 216 if (op_is_write(bio_op(bio))) 5d64efe79703c4 drivers/nvdimm/pmem.c Vivek Goyal 2020-02-28 217 rc = pmem_do_write(pmem, bvec.bv_page, bvec.bv_offset, 5d64efe79703c4 drivers/nvdimm/pmem.c Vivek Goyal 2020-02-28 218 iter.bi_sector, bvec.bv_len); 5d64efe79703c4 drivers/nvdimm/pmem.c Vivek Goyal 2020-02-28 219 else 5d64efe79703c4 drivers/nvdimm/pmem.c Vivek Goyal 2020-02-28 220 rc = pmem_do_read(pmem, bvec.bv_page, bvec.bv_offset, 5d64efe79703c4 drivers/nvdimm/pmem.c Vivek Goyal 2020-02-28 221 iter.bi_sector, bvec.bv_len); e10624f8c09710 drivers/nvdimm/pmem.c Dan Williams 2016-01-06 222 if (rc) { 4e4cbee93d5613 drivers/nvdimm/pmem.c Christoph Hellwig 2017-06-03 223 bio->bi_status = rc; e10624f8c09710 drivers/nvdimm/pmem.c Dan Williams 2016-01-06 224 break; e10624f8c09710 drivers/nvdimm/pmem.c Dan Williams 2016-01-06 225 } e10624f8c09710 drivers/nvdimm/pmem.c Dan Williams 2016-01-06 226 } f0dc089ce217e7 drivers/nvdimm/pmem.c Dan Williams 2015-05-16 227 if (do_acct) 0fd92f89a44d3b drivers/nvdimm/pmem.c Christoph Hellwig 2020-05-27 228 bio_end_io_acct(bio, start); 61031952f4c89d drivers/nvdimm/pmem.c Ross Zwisler 2015-06-25 229 e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 230 if (bio->bi_opf & REQ_FUA) { c5d4355d10d414 drivers/nvdimm/pmem.c Pankaj Gupta 2019-07-05 231 ret = nvdimm_flush(nd_region, bio); c5d4355d10d414 drivers/nvdimm/pmem.c Pankaj Gupta 2019-07-05 232 e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 233 /* asynchronous flush completes in other context */ e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 234 if (nd_region->flush) e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 235 return BLK_QC_T_NONE; e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 236 } 61031952f4c89d drivers/nvdimm/pmem.c Ross Zwisler 2015-06-25 237 4246a0b63bd8f5 drivers/nvdimm/pmem.c Christoph Hellwig 2015-07-20 238 bio_endio(bio); dece16353ef47d drivers/nvdimm/pmem.c Jens Axboe 2015-11-05 239 return BLK_QC_T_NONE; 9e853f2313e5eb drivers/block/pmem.c Ross Zwisler 2015-04-01 240 } 9e853f2313e5eb drivers/block/pmem.c Ross Zwisler 2015-04-01 241---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org
kernel test robot
build botRe: [RFC v2 2/2] pmem: Enable pmem_submit_bio for asynchronous flush
In reply to Pankaj Gupta[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in https://git-scm.com/docs/git-format-patch]
Hi Pankaj,
[FYI, it's a private test report for your RFC patch.]
[auto build test WARNING on linux/master]
[also build test WARNING on linus/master linux-nvdimm/libnvdimm-for-next v5.14-rc3 next-20210723] [If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]
url: https://github.com/0day-ci/linux/commits/Pankaj-Gupta/virtio-pmem-Async-virtio-pmem-flush/20210726-150913 base: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git 349a2d52ffe59b7a0c5876fa7ee9f3eaf188b830 config: x86_64-randconfig-a013-20210726 (attached as .config) compiler: clang version 13.0.0 (https://github.com/llvm/llvm-project c63dbd850182797bc4b76124d08e1c320ab2365d)
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# install x86_64 cross compiling tool for clang build
# apt-get install binutils-x86-64-linux-gnu
# https://github.com/0day-ci/linux/commit/e6040aad1930816dc183a36f6ff5eae1412a9713
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Pankaj-Gupta/virtio-pmem-Async-virtio-pmem-flush/20210726-150913 git checkout e6040aad1930816dc183a36f6ff5eae1412a9713
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross ARCH=x86_64
If you fix the issue, kindly add following tag as appropriate
All warnings (new ones prefixed by >>):
>> drivers/nvdimm/pmem.c:195:6: warning: variable 'ret' set but not used [-Wunused-but-set-variable]
int ret = 0;
^1 warning generated.
vim +/ret +195 drivers/nvdimm/pmem.c
9e853f2313e5eb drivers/block/pmem.c Ross Zwisler 2015-04-01 192
c62b37d96b6eb3 drivers/nvdimm/pmem.c Christoph Hellwig 2020-07-01 193 static blk_qc_t pmem_submit_bio(struct bio *bio)
9e853f2313e5eb drivers/block/pmem.c Ross Zwisler 2015-04-01 194 {
c5d4355d10d414 drivers/nvdimm/pmem.c Pankaj Gupta 2019-07-05 @195 int ret = 0;
4e4cbee93d5613 drivers/nvdimm/pmem.c Christoph Hellwig 2017-06-03 196 blk_status_t rc = 0;
f0dc089ce217e7 drivers/nvdimm/pmem.c Dan Williams 2015-05-16 197 bool do_acct;
f0dc089ce217e7 drivers/nvdimm/pmem.c Dan Williams 2015-05-16 198 unsigned long start;
9e853f2313e5eb drivers/block/pmem.c Ross Zwisler 2015-04-01 199 struct bio_vec bvec;
9e853f2313e5eb drivers/block/pmem.c Ross Zwisler 2015-04-01 200 struct bvec_iter iter;
309dca309fc39a drivers/nvdimm/pmem.c Christoph Hellwig 2021-01-24 201 struct pmem_device *pmem = bio->bi_bdev->bd_disk->private_data;
7e267a8c790edf drivers/nvdimm/pmem.c Dan Williams 2016-06-01 202 struct nd_region *nd_region = to_region(pmem);
7e267a8c790edf drivers/nvdimm/pmem.c Dan Williams 2016-06-01 203
e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 204 if ((bio->bi_opf & REQ_PREFLUSH) &&
e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 205 nvdimm_flush(nd_region, bio)) {
e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 206
e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 207 /* asynchronous flush completes in other context */
e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 208 if (nd_region->flush)
e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 209 return BLK_QC_T_NONE;
e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 210 }
9e853f2313e5eb drivers/block/pmem.c Ross Zwisler 2015-04-01 211
309dca309fc39a drivers/nvdimm/pmem.c Christoph Hellwig 2021-01-24 212 do_acct = blk_queue_io_stat(bio->bi_bdev->bd_disk->queue);
0fd92f89a44d3b drivers/nvdimm/pmem.c Christoph Hellwig 2020-05-27 213 if (do_acct)
0fd92f89a44d3b drivers/nvdimm/pmem.c Christoph Hellwig 2020-05-27 214 start = bio_start_io_acct(bio);
e10624f8c09710 drivers/nvdimm/pmem.c Dan Williams 2016-01-06 215 bio_for_each_segment(bvec, bio, iter) {
5d64efe79703c4 drivers/nvdimm/pmem.c Vivek Goyal 2020-02-28 216 if (op_is_write(bio_op(bio)))
5d64efe79703c4 drivers/nvdimm/pmem.c Vivek Goyal 2020-02-28 217 rc = pmem_do_write(pmem, bvec.bv_page, bvec.bv_offset,
5d64efe79703c4 drivers/nvdimm/pmem.c Vivek Goyal 2020-02-28 218 iter.bi_sector, bvec.bv_len);
5d64efe79703c4 drivers/nvdimm/pmem.c Vivek Goyal 2020-02-28 219 else
5d64efe79703c4 drivers/nvdimm/pmem.c Vivek Goyal 2020-02-28 220 rc = pmem_do_read(pmem, bvec.bv_page, bvec.bv_offset,
5d64efe79703c4 drivers/nvdimm/pmem.c Vivek Goyal 2020-02-28 221 iter.bi_sector, bvec.bv_len);
e10624f8c09710 drivers/nvdimm/pmem.c Dan Williams 2016-01-06 222 if (rc) {
4e4cbee93d5613 drivers/nvdimm/pmem.c Christoph Hellwig 2017-06-03 223 bio->bi_status = rc;
e10624f8c09710 drivers/nvdimm/pmem.c Dan Williams 2016-01-06 224 break;
e10624f8c09710 drivers/nvdimm/pmem.c Dan Williams 2016-01-06 225 }
e10624f8c09710 drivers/nvdimm/pmem.c Dan Williams 2016-01-06 226 }
f0dc089ce217e7 drivers/nvdimm/pmem.c Dan Williams 2015-05-16 227 if (do_acct)
0fd92f89a44d3b drivers/nvdimm/pmem.c Christoph Hellwig 2020-05-27 228 bio_end_io_acct(bio, start);
61031952f4c89d drivers/nvdimm/pmem.c Ross Zwisler 2015-06-25 229
e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 230 if (bio->bi_opf & REQ_FUA) {
c5d4355d10d414 drivers/nvdimm/pmem.c Pankaj Gupta 2019-07-05 231 ret = nvdimm_flush(nd_region, bio);
c5d4355d10d414 drivers/nvdimm/pmem.c Pankaj Gupta 2019-07-05 232
e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 233 /* asynchronous flush completes in other context */
e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 234 if (nd_region->flush)
e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 235 return BLK_QC_T_NONE;
e6040aad193081 drivers/nvdimm/pmem.c Pankaj Gupta 2021-07-26 236 }
61031952f4c89d drivers/nvdimm/pmem.c Ross Zwisler 2015-06-25 237
4246a0b63bd8f5 drivers/nvdimm/pmem.c Christoph Hellwig 2015-07-20 238 bio_endio(bio);
dece16353ef47d drivers/nvdimm/pmem.c Jens Axboe 2015-11-05 239 return BLK_QC_T_NONE;
9e853f2313e5eb drivers/block/pmem.c Ross Zwisler 2015-04-01 240 }
9e853f2313e5eb drivers/block/pmem.c Ross Zwisler 2015-04-01 241 ---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org
Pankaj Gupta
maintainerGentle ping.
Quoted 28 lines
Jeff reported preflush order issue with the existing implementation of virtio pmem preflush. Dan suggested[1] to implement asynchronous flush for virtio pmem using work queue as done in md/RAID. This patch series intends to solve the preflush ordering issue and also makes the flush asynchronous for the submitting thread.
Submitting this patch series for review. Sorry, It took me long time to come back to this due to some personal reasons.
RFC v1 -> RFC v2
- More testing and bug fix.[1] https://marc.info/?l=linux-kernel&m=157446316409937&w=2
Pankaj Gupta (2):
virtio-pmem: Async virtio-pmem flush
pmem: enablepmem_submit_biofor asynchronous flushdrivers/nvdimm/nd_virtio.c | 72 ++++++++++++++++++++++++++++-------- drivers/nvdimm/pmem.c | 17 ++++++--- drivers/nvdimm/virtio_pmem.c | 10 ++++- drivers/nvdimm/virtio_pmem.h | 14 +++++++ 4 files changed, 91 insertions(+), 22 deletions(-)--
2.25.1
Pankaj Gupta
maintainerFriendly ping!
Thanks,
Pankaj
Quoted 31 lines · Pankaj Gupta
Gentle ping.
Quoted 28 lines
Jeff reported preflush order issue with the existing implementation of virtio pmem preflush. Dan suggested[1] to implement asynchronous flush for virtio pmem using work queue as done in md/RAID. This patch series intends to solve the preflush ordering issue and also makes the flush asynchronous for the submitting thread.
Submitting this patch series for review. Sorry, It took me long time to come back to this due to some personal reasons.
RFC v1 -> RFC v2
- More testing and bug fix.[1] https://marc.info/?l=linux-kernel&m=157446316409937&w=2
Pankaj Gupta (2):
virtio-pmem: Async virtio-pmem flush
pmem: enablepmem_submit_biofor asynchronous flushdrivers/nvdimm/nd_virtio.c | 72 ++++++++++++++++++++++++++++-------- drivers/nvdimm/pmem.c | 17 ++++++--- drivers/nvdimm/virtio_pmem.c | 10 ++++- drivers/nvdimm/virtio_pmem.h | 14 +++++++ 4 files changed, 91 insertions(+), 22 deletions(-)--
2.25.1