virtiofs: Fix couple of deadlocks
Subsystems (4)
Vivek Goyal
maintainercover virtiofs: Fix couple of deadlocks
Hi,
We have couple of places which can result in deadlock. This patch series fixes these.
We can be called with fc->bg_lock (for background requests) while submitting a request. This leads to two constraints.
- We can't end requests in submitter's context and call fuse_end_request() as it tries to take fc->bg_lock as well. So queue these requests on a list and use a worker to end these requests.
- If virtqueue is full, we can wait with fc->bg_lock held for queue to have space. Worker which is completing the request gets blocked on fc->bg_lock as well. And that means requests are not completing, that means descriptors are not being freed and that means submitter can't make progress. Deadlock.
Fix this by punting the requests to a list and retry submission later with the help of a worker.
Thanks
Vivek
Vivek Goyal (5):
virtiofs: Do not end request in submission context
virtiofs: No need to check fpq->connected state
virtiofs: Set FR_SENT flag only after request has been sent
virtiofs: Count pending forgets as in_flight forgets
virtiofs: Retry request submission from worker context
fs/fuse/virtio_fs.c | 165 +++++++++++++++++++++++++++++--------------- 1 file changed, 111 insertions(+), 54 deletions(-)
Signature · 1 line
2.20.1
Vivek Goyal
maintainerPATCH 1/5 virtiofs: Do not end request in submission context
Submission context can hold some locks which end request code tries to hold again and deadlock can occur. For example, fc->bg_lock. If a background request is being submitted, it might hold fc->bg_lock and if we could not submit request (because device went away) and tried to end request, then deadlock happens. During testing, I also got a warning from deadlock detection code.
So put requests on a list and end requests from a worker thread.
I got following warning from deadlock detector.
[ 603.137138] WARNING: possible recursive locking detected
[ 603.137142] --------------------------------------------
[ 603.137144] blogbench/2036 is trying to acquire lock:
[ 603.137149] 00000000f0f51107 (&(&fc->bg_lock)->rlock){+.+.}, at: fuse_request_end+0xdf/0x1c0 [fuse]
[ 603.140701]
[ 603.140701] but task is already holding lock:
[ 603.140703] 00000000f0f51107 (&(&fc->bg_lock)->rlock){+.+.}, at: fuse_simple_background+0x92/0x1d0 [fuse]
[ 603.140713]
[ 603.140713] other info that might help us debug this:
[ 603.140714] Possible unsafe locking scenario:
[ 603.140714]
[ 603.140715] CPU0
[ 603.140716] ----
[ 603.140716] lock(&(&fc->bg_lock)->rlock);
[ 603.140718] lock(&(&fc->bg_lock)->rlock);
[ 603.140719]
[ 603.140719] *** DEADLOCK ***--- fs/fuse/virtio_fs.c | 38 ++++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-)
fs/fuse/virtio_fs.c +34 -4
@@ -30,6 +30,7 @@ struct virtio_fs_vq {struct virtqueue *vq;/* protected by ->lock */ struct work_struct done_work;struct list_head queued_reqs; +struct list_head end_reqs;/* End these requests */ struct delayed_work dispatch_work;struct fuse_dev *fud;bool connected;
@@ -259,8 +260,27 @@ static void virtio_fs_hiprio_done_work(struct work_struct *work)spin_unlock (&fsvq->lock); } -static void virtio_fs_dummy_dispatch_work (struct work_struct *work) +static void virtio_fs_request_dispatch_work (struct work_struct *work) { +struct fuse_req *req; + struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, + dispatch_work.work); +struct fuse_conn *fc = fsvq->fud->fc; + +pr_debug ("virtio-fs: worker %s called.\n" , __func__); +while (1 ) { +spin_lock (&fsvq->lock); + req = list_first_entry_or_null(&fsvq->end_reqs, struct fuse_req, + list); +if (!req) { +spin_unlock (&fsvq->lock); +return ; + } + +list_del_init (&req->list); +spin_unlock (&fsvq->lock); +fuse_request_end (fc, req); + } }static void virtio_fs_hiprio_dispatch_work (struct work_struct *work)
@@ -502,6 +522,7 @@ static int virtio_fs_setup_vqs(struct virtio_device *vdev, names[VQ_HIPRIO ] = fs->vqs[VQ_HIPRIO ].name;INIT_WORK (&fs->vqs[VQ_HIPRIO ].done_work, virtio_fs_hiprio_done_work);INIT_LIST_HEAD (&fs->vqs[VQ_HIPRIO ].queued_reqs); +INIT_LIST_HEAD (&fs->vqs[VQ_HIPRIO ].end_reqs);INIT_DELAYED_WORK (&fs->vqs[VQ_HIPRIO ].dispatch_work, virtio_fs_hiprio_dispatch_work);spin_lock_init (&fs->vqs[VQ_HIPRIO ].lock);
@@ -511,8 +532,9 @@ static int virtio_fs_setup_vqs(struct virtio_device *vdev,spin_lock_init (&fs->vqs[i].lock);INIT_WORK (&fs->vqs[i].done_work, virtio_fs_requests_done_work);INIT_DELAYED_WORK (&fs->vqs[i].dispatch_work, - virtio_fs_dummy_dispatch_work); + virtio_fs_request_dispatch_work);INIT_LIST_HEAD (&fs->vqs[i].queued_reqs); +INIT_LIST_HEAD (&fs->vqs[i].end_reqs);snprintf (fs->vqs[i].name,sizeof (fs->vqs[i].name),"requests.%u" , i -VQ_REQUEST ); callbacks[i] = virtio_fs_vq_done;
@@ -918,6 +940,7 @@ __releases(fiq->lock)struct fuse_conn *fc;struct fuse_req *req;struct fuse_pqueue *fpq; +struct virtio_fs_vq *fsvq;int ret;WARN_ON (list_empty (&fiq->pending));
@@ -951,7 +974,8 @@ __releases(fiq->lock)smp_mb__after_atomic (); retry: - ret =virtio_fs_enqueue_req (&fs->vqs[queue_id], req); + fsvq = &fs->vqs[queue_id]; + ret =virtio_fs_enqueue_req (fsvq, req);if (ret <0 ) {if (ret == -ENOMEM || ret == -ENOSPC ) {/* Virtqueue full. Retry submission */
@@ -965,7 +989,13 @@ __releases(fiq->lock)clear_bit (FR_SENT , &req->flags);list_del_init (&req->list);spin_unlock (&fpq->lock); -fuse_request_end (fc, req); + +/* Can't end request in submission context. Use a worker */ +spin_lock (&fsvq->lock); +list_add_tail (&req->list, &fsvq->end_reqs); +schedule_delayed_work (&fsvq->dispatch_work, +msecs_to_jiffies (1 )); +spin_unlock (&fsvq->lock);return ; } }
Signature · 1 line
2.20.1
Miklos Szeredi
maintainerRe: [PATCH 1/5] virtiofs: Do not end request in submission context
Quoted 114 lines · Vivek Goyal
Submission context can hold some locks which end request code tries to hold again and deadlock can occur. For example,
fc->bg_lock. If a background request is being submitted, it might holdfc->bg_lockand if we could not submit request (because device went away) and tried to end request, then deadlock happens. During testing, I also got a warning from deadlock detection code.So put requests on a list and end requests from a worker thread.
I got following warning from deadlock detector.
[ 603.137138] WARNING: possible recursive locking detected [ 603.137142] -------------------------------------------- [ 603.137144] blogbench/2036 is trying to acquire lock: [ 603.137149] 00000000f0f51107 (&(&fc->bg_lock)->rlock){+.+.}, at: fuse_request_end+0xdf/0x1c0 [fuse] [ 603.140701] [ 603.140701] but task is already holding lock: [ 603.140703] 00000000f0f51107 (&(&fc->bg_lock)->rlock){+.+.}, at: fuse_simple_background+0x92/0x1d0 [fuse] [ 603.140713] [ 603.140713] other info that might help us debug this: [ 603.140714] Possible unsafe locking scenario: [ 603.140714] [ 603.140715] CPU0 [ 603.140716] ---- [ 603.140716] lock(&(&fc->bg_lock)->rlock); [ 603.140718] lock(&(&fc->bg_lock)->rlock); [ 603.140719] [ 603.140719] *** DEADLOCK ***Signed-off-by Vivek Goyal <[email protected]>--- fs/fuse/virtio_fs.c | 38 ++++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-)fs/fuse/virtio_fs.c +27
-3@@ -30,6 +30,7 @@ struct virtio_fs_vq {struct virtqueue *vq;/* protected by ->lock */ struct work_struct done_work;struct list_head queued_reqs; +struct list_head end_reqs;/* End these requests */ struct delayed_work dispatch_work;struct fuse_dev *fud;bool connected;@@ -259,8 +260,27 @@ static void virtio_fs_hiprio_done_work(struct work_struct *work)spin_unlock (&fsvq->lock); } -static void virtio_fs_dummy_dispatch_work (struct work_struct *work) +static void virtio_fs_request_dispatch_work (struct work_struct *work) { +struct fuse_req *req; + struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, + dispatch_work.work); +struct fuse_conn *fc = fsvq->fud->fc; + +pr_debug ("virtio-fs: worker %s called.\n" , __func__); +while (1 ) { +spin_lock (&fsvq->lock); + req = list_first_entry_or_null(&fsvq->end_reqs, struct fuse_req, + list); +if (!req) { +spin_unlock (&fsvq->lock); +return ; + } + +list_del_init (&req->list); +spin_unlock (&fsvq->lock); +fuse_request_end (fc, req); + } }static void virtio_fs_hiprio_dispatch_work (struct work_struct *work)@@ -502,6 +522,7 @@ static int virtio_fs_setup_vqs(struct virtio_device *vdev, names[VQ_HIPRIO ] = fs->vqs[VQ_HIPRIO ].name;INIT_WORK (&fs->vqs[VQ_HIPRIO ].done_work, virtio_fs_hiprio_done_work);INIT_LIST_HEAD (&fs->vqs[VQ_HIPRIO ].queued_reqs); +INIT_LIST_HEAD (&fs->vqs[VQ_HIPRIO ].end_reqs);INIT_DELAYED_WORK (&fs->vqs[VQ_HIPRIO ].dispatch_work, virtio_fs_hiprio_dispatch_work);spin_lock_init (&fs->vqs[VQ_HIPRIO ].lock);@@ -511,8 +532,9 @@ static int virtio_fs_setup_vqs(struct virtio_device *vdev,spin_lock_init (&fs->vqs[i].lock);INIT_WORK (&fs->vqs[i].done_work, virtio_fs_requests_done_work);INIT_DELAYED_WORK (&fs->vqs[i].dispatch_work, - virtio_fs_dummy_dispatch_work); + virtio_fs_request_dispatch_work);INIT_LIST_HEAD (&fs->vqs[i].queued_reqs); +INIT_LIST_HEAD (&fs->vqs[i].end_reqs);snprintf (fs->vqs[i].name,sizeof (fs->vqs[i].name),"requests.%u" , i -VQ_REQUEST ); callbacks[i] = virtio_fs_vq_done;@@ -918,6 +940,7 @@ __releases(fiq->lock)struct fuse_conn *fc;struct fuse_req *req;struct fuse_pqueue *fpq; +struct virtio_fs_vq *fsvq;int ret;WARN_ON (list_empty (&fiq->pending));@@ -951,7 +974,8 @@ __releases(fiq->lock)smp_mb__after_atomic (); retry: - ret =virtio_fs_enqueue_req (&fs->vqs[queue_id], req); + fsvq = &fs->vqs[queue_id]; + ret =virtio_fs_enqueue_req (fsvq, req);if (ret <0 ) {if (ret == -ENOMEM || ret == -ENOSPC ) {/* Virtqueue full. Retry submission */
fs/fuse/virtio_fs.c __releases(fiq->lock) · line 997
Quoted 10 lines
fs/fuse/virtio_fs.c +6
-1@@ -965,7 +989,13 @@ __releases(fiq->lock)clear_bit (FR_SENT , &req->flags);list_del_init (&req->list);spin_unlock (&fpq->lock); -fuse_request_end (fc, req); + +/* Can't end request in submission context. Use a worker */ +spin_lock (&fsvq->lock); +list_add_tail (&req->list, &fsvq->end_reqs); +schedule_delayed_work (&fsvq->dispatch_work, +msecs_to_jiffies (1 ));
What's the reason to delay by one msec? If this is purely for deadlock avoidance, then a zero delay would work better, no?
Thanks,
Miklos
Vivek Goyal
maintainerRe: [PATCH 1/5] virtiofs: Do not end request in submission context
On Mon, Oct 21, 2019 at 10:03:39AM +0200, Miklos Szeredi wrote:
[..]
Quoted 44 lines
Quoted 44 lines
static void
virtio_fs_hiprio_dispatch_work(struct work_struct *work)fs/fuse/virtio_fs.c +7
-3@@ -502,6 +522,7 @@ static int virtio_fs_setup_vqs(struct virtio_device *vdev, names[VQ_HIPRIO ] = fs->vqs[VQ_HIPRIO ].name;INIT_WORK (&fs->vqs[VQ_HIPRIO ].done_work, virtio_fs_hiprio_done_work);INIT_LIST_HEAD (&fs->vqs[VQ_HIPRIO ].queued_reqs); +INIT_LIST_HEAD (&fs->vqs[VQ_HIPRIO ].end_reqs);INIT_DELAYED_WORK (&fs->vqs[VQ_HIPRIO ].dispatch_work, virtio_fs_hiprio_dispatch_work);spin_lock_init (&fs->vqs[VQ_HIPRIO ].lock);@@ -511,8 +532,9 @@ static int virtio_fs_setup_vqs(struct virtio_device *vdev,spin_lock_init (&fs->vqs[i].lock);INIT_WORK (&fs->vqs[i].done_work, virtio_fs_requests_done_work);INIT_DELAYED_WORK (&fs->vqs[i].dispatch_work, - virtio_fs_dummy_dispatch_work); + virtio_fs_request_dispatch_work);INIT_LIST_HEAD (&fs->vqs[i].queued_reqs); +INIT_LIST_HEAD (&fs->vqs[i].end_reqs);snprintf (fs->vqs[i].name,sizeof (fs->vqs[i].name),"requests.%u" , i -VQ_REQUEST ); callbacks[i] = virtio_fs_vq_done;@@ -918,6 +940,7 @@ __releases(fiq->lock)struct fuse_conn *fc;struct fuse_req *req;struct fuse_pqueue *fpq; +struct virtio_fs_vq *fsvq;int ret;WARN_ON (list_empty (&fiq->pending));@@ -951,7 +974,8 @@ __releases(fiq->lock)smp_mb__after_atomic (); retry: - ret =virtio_fs_enqueue_req (&fs->vqs[queue_id], req); + fsvq = &fs->vqs[queue_id]; + ret =virtio_fs_enqueue_req (fsvq, req);if (ret <0 ) {if (ret == -ENOMEM || ret == -ENOSPC ) {/* Virtqueue full. Retry submission */ @@ -965,7 +989,13 @@ __releases(fiq->lock)clear_bit (FR_SENT , &req->flags);list_del_init (&req->list);spin_unlock (&fpq->lock); -fuse_request_end (fc, req); +
Quoted 8 lines
fs/fuse/virtio_fs.c __releases(fiq->lock) · line 997
Quoted 5 lines
fs/fuse/virtio_fs.c +5
-0+/* Can't end request in submission context. Use a worker */ +spin_lock (&fsvq->lock); +list_add_tail (&req->list, &fsvq->end_reqs); +schedule_delayed_work (&fsvq->dispatch_work, +msecs_to_jiffies (1 ));What's the reason to delay by one msec? If this is purely for deadlock avoidance, then a zero delay would work better, no?
Hi Miklos,
I have no good reason to do that. Will change it to zero delay.
Thanks
Vivek
Miklos Szeredi
maintainerRe: [PATCH 1/5] virtiofs: Do not end request in submission context
Quoted 57 lines · Vivek Goyal
On Mon, Oct 21, 2019 at 10:03:39AM +0200, Miklos Szeredi wrote:
[..]
Quoted 44 lines
Quoted 44 lines
static void
virtio_fs_hiprio_dispatch_work(struct work_struct *work)fs/fuse/virtio_fs.c +7
-3@@ -502,6 +522,7 @@ static int virtio_fs_setup_vqs(struct virtio_device *vdev, names[VQ_HIPRIO ] = fs->vqs[VQ_HIPRIO ].name;INIT_WORK (&fs->vqs[VQ_HIPRIO ].done_work, virtio_fs_hiprio_done_work);INIT_LIST_HEAD (&fs->vqs[VQ_HIPRIO ].queued_reqs); +INIT_LIST_HEAD (&fs->vqs[VQ_HIPRIO ].end_reqs);INIT_DELAYED_WORK (&fs->vqs[VQ_HIPRIO ].dispatch_work, virtio_fs_hiprio_dispatch_work);spin_lock_init (&fs->vqs[VQ_HIPRIO ].lock);@@ -511,8 +532,9 @@ static int virtio_fs_setup_vqs(struct virtio_device *vdev,spin_lock_init (&fs->vqs[i].lock);INIT_WORK (&fs->vqs[i].done_work, virtio_fs_requests_done_work);INIT_DELAYED_WORK (&fs->vqs[i].dispatch_work, - virtio_fs_dummy_dispatch_work); + virtio_fs_request_dispatch_work);INIT_LIST_HEAD (&fs->vqs[i].queued_reqs); +INIT_LIST_HEAD (&fs->vqs[i].end_reqs);snprintf (fs->vqs[i].name,sizeof (fs->vqs[i].name),"requests.%u" , i -VQ_REQUEST ); callbacks[i] = virtio_fs_vq_done;@@ -918,6 +940,7 @@ __releases(fiq->lock)struct fuse_conn *fc;struct fuse_req *req;struct fuse_pqueue *fpq; +struct virtio_fs_vq *fsvq;int ret;WARN_ON (list_empty (&fiq->pending));@@ -951,7 +974,8 @@ __releases(fiq->lock)smp_mb__after_atomic (); retry: - ret =virtio_fs_enqueue_req (&fs->vqs[queue_id], req); + fsvq = &fs->vqs[queue_id]; + ret =virtio_fs_enqueue_req (fsvq, req);if (ret <0 ) {if (ret == -ENOMEM || ret == -ENOSPC ) {/* Virtqueue full. Retry submission */ @@ -965,7 +989,13 @@ __releases(fiq->lock)clear_bit (FR_SENT , &req->flags);list_del_init (&req->list);spin_unlock (&fpq->lock); -fuse_request_end (fc, req); +Quoted 8 lines
fs/fuse/virtio_fs.c __releases(fiq->lock) · line 997
Quoted 5 lines
fs/fuse/virtio_fs.c +5
-0+/* Can't end request in submission context. Use a worker */ +spin_lock (&fsvq->lock); +list_add_tail (&req->list, &fsvq->end_reqs); +schedule_delayed_work (&fsvq->dispatch_work, +msecs_to_jiffies (1 ));What's the reason to delay by one msec? If this is purely for deadlock avoidance, then a zero delay would work better, no?
Quoted 3 lines
Hi Miklos,
I have no good reason to do that. Will change it to zero delay.
Okay, fixed and pushed out to fuse.git#for-next.
Thanks,
Miklos
Vivek Goyal
maintainerPATCH 2/5 virtiofs: No need to check fpq->connected state
In reply to Vivek GoyalIn virtiofs we keep per queue connected state in virtio_fs_vq->connected and use that to end request if queue is not connected. And virtiofs does not even touch fpq->connected state.
We probably need to merge these two at some point of time. For now, simplify the code a bit and do not worry about checking state of fpq->connected.
--- fs/fuse/virtio_fs.c | 7 ------- 1 file changed, 7 deletions(-)
fs/fuse/virtio_fs.c +0 -7
@@ -960,13 +960,6 @@ __releases(fiq->lock) fpq = &fs->vqs[queue_id].fud->pq;spin_lock (&fpq->lock); -if (!fpq->connected) { -spin_unlock (&fpq->lock); - req->out.h.error = -ENODEV ; -pr_err ("virtio-fs: %s disconnected\n" , __func__); -fuse_request_end (fc, req); -return ; - }list_add_tail (&req->list, fpq->processing);spin_unlock (&fpq->lock);set_bit (FR_SENT , &req->flags);
Signature · 1 line
2.20.1
Vivek Goyal
maintainerPATCH 3/5 virtiofs: Set FR_SENT flag only after request has been sent
In reply to Vivek GoyalFR_SENT flag should be set when request has been sent successfuly sent over virtqueue. This is used by interrupt logic to figure out if interrupt request should be sent or not.
Also add it to fqp->processing list after sending it successfully.
--- fs/fuse/virtio_fs.c | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-)
fs/fuse/virtio_fs.c +10 -13
@@ -857,6 +857,7 @@ static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq,unsigned int i;int ret;bool notify; +struct fuse_pqueue *fpq;/* Does the sglist fit on the stack? */ total_sgs =sg_count_fuse_req (req);
@@ -911,6 +912,15 @@ static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq, goto out; } +/* Request successfuly sent. */ + fpq = &fsvq->fud->pq; +spin_lock (&fpq->lock); +list_add_tail (&req->list, fpq->processing); +spin_unlock (&fpq->lock); +set_bit (FR_SENT , &req->flags); +/* matches barrier in request_wait_answer() */ +smp_mb__after_atomic (); + fsvq->in_flight++; notify =virtqueue_kick_prepare (vq);
@@ -939,7 +949,6 @@ __releases(fiq->lock)struct virtio_fs *fs;struct fuse_conn *fc;struct fuse_req *req; -struct fuse_pqueue *fpq;struct virtio_fs_vq *fsvq;int ret;
@@ -958,14 +967,6 @@ __releases(fiq->lock) req->in.h.nodeid, req->in.h.len, fuse_len_args(req->args->out_numargs, req->args->out_args)); - fpq = &fs->vqs[queue_id].fud->pq; -spin_lock (&fpq->lock); -list_add_tail (&req->list, fpq->processing); -spin_unlock (&fpq->lock); -set_bit (FR_SENT , &req->flags); -/* matches barrier in request_wait_answer() */ -smp_mb__after_atomic (); - retry: fsvq = &fs->vqs[queue_id]; ret =virtio_fs_enqueue_req (fsvq, req);
@@ -978,10 +979,6 @@ __releases(fiq->lock) } req->out.h.error = ret;pr_err ("virtio-fs: virtio_fs_enqueue_req() failed %d\n" , ret); -spin_lock (&fpq->lock); -clear_bit (FR_SENT , &req->flags); -list_del_init (&req->list); -spin_unlock (&fpq->lock);/* Can't end request in submission context. Use a worker */ spin_lock (&fsvq->lock);
Signature · 1 line
2.20.1
Vivek Goyal
maintainerPATCH 4/5 virtiofs: Count pending forgets as in_flight forgets
In reply to Vivek GoyalIf virtqueue is full, we put forget requests on a list and these forgets are dispatched later using a worker. As of now we don't count these forgets in fsvq->in_flight variable. This means when queue is being drained, we have to have special logic to first drain these pending requests and then wait for fsvq->in_flight to go to zero.
By counting pending forgets in fsvq->in_flight, we can get rid of special logic and just wait for in_flight to go to zero. Worker thread will kick and drain all the forgets anyway, leading in_flight to zero.
I also need similar logic for normal request queue in next patch where I am about to defer request submission in the worker context if queue is full.
This simplifies the code a bit.
Also add two helper functions to inc/dec in_flight. Decrement in_flight helper will later used to call completion when in_flight reaches zero.
--- fs/fuse/virtio_fs.c | 44 ++++++++++++++++++++------------------------ 1 file changed, 20 insertions(+), 24 deletions(-)
fs/fuse/virtio_fs.c +20 -24
@@ -67,6 +67,19 @@ static inline struct fuse_pqueue *vq_to_fpq(struct virtqueue *vq)return &vq_to_fsvq (vq)->fud->pq; } +/* Should be called with fsvq->lock held. */ +static inline void inc_in_flight_req (struct virtio_fs_vq *fsvq) +{ + fsvq->in_flight++; +} + +/* Should be called with fsvq->lock held. */ +static inline void dec_in_flight_req (struct virtio_fs_vq *fsvq) +{ +WARN_ON (fsvq->in_flight <=0 ); + fsvq->in_flight--; +} +static void release_virtio_fs_obj (struct kref *ref) { struct virtio_fs *vfs = container_of(ref, struct virtio_fs, refcount);
@@ -110,22 +123,6 @@ static void virtio_fs_drain_queue(struct virtio_fs_vq *fsvq)flush_delayed_work (&fsvq->dispatch_work); } -static inline void drain_hiprio_queued_reqs (struct virtio_fs_vq *fsvq) -{ -struct virtio_fs_forget *forget; - -spin_lock (&fsvq->lock); -while (1 ) { - forget =list_first_entry_or_null (&fsvq->queued_reqs, - struct virtio_fs_forget, list); -if (!forget) -break ; -list_del (&forget->list); -kfree (forget); - } -spin_unlock (&fsvq->lock); -} -static void virtio_fs_drain_all_queues (struct virtio_fs *fs) {struct virtio_fs_vq *fsvq;
@@ -133,9 +130,6 @@ static void virtio_fs_drain_all_queues(struct virtio_fs *fs)for (i =0 ; i < fs->nvqs; i++) { fsvq = &fs->vqs[i]; -if (i ==VQ_HIPRIO ) -drain_hiprio_queued_reqs (fsvq); -virtio_fs_drain_queue (fsvq); } }
@@ -254,7 +248,7 @@ static void virtio_fs_hiprio_done_work(struct work_struct *work)while ((req =virtqueue_get_buf (vq, &len)) !=NULL ) {kfree (req); - fsvq->in_flight--; +dec_in_flight_req (fsvq); } }while (!virtqueue_enable_cb (vq) &&likely (!virtqueue_is_broken (vq)));spin_unlock (&fsvq->lock);
@@ -306,6 +300,7 @@ static void virtio_fs_hiprio_dispatch_work(struct work_struct *work)list_del (&forget->list);if (!fsvq->connected) { +dec_in_flight_req (fsvq);spin_unlock (&fsvq->lock);kfree (forget);continue ;
@@ -327,13 +322,13 @@ static void virtio_fs_hiprio_dispatch_work(struct work_struct *work) }else {pr_debug ("virtio-fs: Could not queue FORGET: err=%d. Dropping it.\n" , ret); +dec_in_flight_req (fsvq);kfree (forget); }spin_unlock (&fsvq->lock);return ; } - fsvq->in_flight++; notify =virtqueue_kick_prepare (vq);spin_unlock (&fsvq->lock);
@@ -472,7 +467,7 @@ static void virtio_fs_requests_done_work(struct work_struct *work)fuse_request_end (fc, req);spin_lock (&fsvq->lock); - fsvq->in_flight--; +dec_in_flight_req (fsvq);spin_unlock (&fsvq->lock); } }
@@ -730,6 +725,7 @@ __releases(fiq->lock)list_add_tail (&forget->list, &fsvq->queued_reqs);schedule_delayed_work (&fsvq->dispatch_work,msecs_to_jiffies (1 )); +inc_in_flight_req (fsvq); }else {pr_debug ("virtio-fs: Could not queue FORGET: err=%d. Dropping it.\n" , ret);
@@ -739,7 +735,7 @@ __releases(fiq->lock) goto out; } - fsvq->in_flight++; +inc_in_flight_req (fsvq); notify =virtqueue_kick_prepare (vq);spin_unlock (&fsvq->lock);
@@ -921,7 +917,7 @@ static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq,/* matches barrier in request_wait_answer() */ smp_mb__after_atomic (); - fsvq->in_flight++; +inc_in_flight_req (fsvq); notify =virtqueue_kick_prepare (vq);spin_unlock (&fsvq->lock);
Signature · 1 line
2.20.1
Vivek Goyal
maintainerPATCH 5/5 virtiofs: Retry request submission from worker context
In reply to Vivek GoyalIf regular request queue gets full, currently we sleep for a bit and retrying submission in submitter's context. This assumes submitter is not holding any spin lock. But this assumption is not true for background requests. For background requests, we are called with fc->bg_lock held.
This can lead to deadlock where one thread is trying submission with fc->bg_lock held while request completion thread has called fuse_request_end() which tries to acquire fc->bg_lock and gets blocked. As request completion thread gets blocked, it does not make further progress and that means queue does not get empty and submitter can't submit more requests.
To solve this issue, retry submission with the help of a worker, instead of retrying in submitter's context. We already do this for hiprio/forget requests.
--- fs/fuse/virtio_fs.c | 59 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 50 insertions(+), 9 deletions(-)
fs/fuse/virtio_fs.c +50 -9
@@ -55,6 +55,9 @@ struct virtio_fs_forget {struct list_head list; }; +static int virtio_fs_enqueue_req (struct virtio_fs_vq *fsvq, +struct fuse_req *req,bool in_flight); +static inline struct virtio_fs_vq *vq_to_fsvq (struct virtqueue *vq) {struct virtio_fs *fs = vq->vdev->priv;
@@ -260,6 +263,7 @@ static void virtio_fs_request_dispatch_work(struct work_struct *work) struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, dispatch_work.work);struct fuse_conn *fc = fsvq->fud->fc; +int ret;pr_debug ("virtio-fs: worker %s called.\n" , __func__);while (1 ) {
@@ -268,13 +272,43 @@ static void virtio_fs_request_dispatch_work(struct work_struct *work) list);if (!req) {spin_unlock (&fsvq->lock); -return ; +break ; }list_del_init (&req->list);spin_unlock (&fsvq->lock);fuse_request_end (fc, req); } + +/* Dispatch pending requests */ +while (1 ) { +spin_lock (&fsvq->lock); + req =list_first_entry_or_null (&fsvq->queued_reqs, + struct fuse_req, list); +if (!req) { +spin_unlock (&fsvq->lock); +return ; + } +list_del_init (&req->list); +spin_unlock (&fsvq->lock); + + ret =virtio_fs_enqueue_req (fsvq, req, true); +if (ret <0 ) { +if (ret == -ENOMEM || ret == -ENOSPC ) { +spin_lock (&fsvq->lock); +list_add_tail (&req->list, &fsvq->queued_reqs); +schedule_delayed_work (&fsvq->dispatch_work, +msecs_to_jiffies (1 )); +spin_unlock (&fsvq->lock); +return ; + } + req->out.h.error = ret; +dec_in_flight_req (fsvq); +pr_err ("virtio-fs: virtio_fs_enqueue_req() failed %d\n" , + ret); +fuse_request_end (fc, req); + } + } }static void virtio_fs_hiprio_dispatch_work (struct work_struct *work)
@@ -837,7 +871,7 @@ static unsigned int sg_init_fuse_args(struct scatterlist *sg,/* Add a request to a virtqueue and kick the device */ static int virtio_fs_enqueue_req (struct virtio_fs_vq *fsvq, -struct fuse_req *req) +struct fuse_req *req,bool in_flight) {/* requests need at least 4 elements */ struct scatterlist *stack_sgs[6 ];
@@ -917,7 +951,8 @@ static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq,/* matches barrier in request_wait_answer() */ smp_mb__after_atomic (); -inc_in_flight_req (fsvq); +if (!in_flight) +inc_in_flight_req (fsvq); notify =virtqueue_kick_prepare (vq);spin_unlock (&fsvq->lock);
@@ -963,15 +998,21 @@ __releases(fiq->lock) req->in.h.nodeid, req->in.h.len, fuse_len_args(req->args->out_numargs, req->args->out_args)); -retry: fsvq = &fs->vqs[queue_id]; - ret =virtio_fs_enqueue_req (fsvq, req); + ret =virtio_fs_enqueue_req (fsvq, req, false);if (ret <0 ) {if (ret == -ENOMEM || ret == -ENOSPC ) { -/* Virtqueue full. Retry submission */ -/* TODO use completion instead of timeout */ -usleep_range (20 ,30 ); - goto retry; +/* +* Virtqueue full. Retry submission from worker +* context as we might be holding fc->bg_lock. +*/ +spin_lock (&fsvq->lock); +list_add_tail (&req->list, &fsvq->queued_reqs); +inc_in_flight_req (fsvq); +schedule_delayed_work (&fsvq->dispatch_work, +msecs_to_jiffies (1 )); +spin_unlock (&fsvq->lock); +return ; } req->out.h.error = ret;pr_err ("virtio-fs: virtio_fs_enqueue_req() failed %d\n" , ret);
Signature · 1 line
2.20.1
Miklos Szeredi
maintainerRe: [PATCH 5/5] virtiofs: Retry request submission from worker context
Quoted 71 lines · Vivek Goyal
If regular request queue gets full, currently we sleep for a bit and retrying submission in submitter's context. This assumes submitter is not holding any spin lock. But this assumption is not true for background requests. For background requests, we are called with
fc->bg_lockheld.This can lead to deadlock where one thread is trying submission with
fc->bg_lockheld while request completion thread has calledfuse_request_end()which tries to acquirefc->bg_lockand gets blocked. As request completion thread gets blocked, it does not make further progress and that means queue does not get empty and submitter can't submit more requests.To solve this issue, retry submission with the help of a worker, instead of retrying in submitter's context. We already do this for hiprio/forget requests.
Reported-by Chirantan Ekbote <[email protected]>Signed-off-by Vivek Goyal <[email protected]>--- fs/fuse/virtio_fs.c | 59 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 50 insertions(+), 9 deletions(-)fs/fuse/virtio_fs.c +20
-1@@ -55,6 +55,9 @@ struct virtio_fs_forget {struct list_head list; }; +static int virtio_fs_enqueue_req (struct virtio_fs_vq *fsvq, +struct fuse_req *req,bool in_flight); +static inline struct virtio_fs_vq *vq_to_fsvq (struct virtqueue *vq) {struct virtio_fs *fs = vq->vdev->priv;@@ -260,6 +263,7 @@ static void virtio_fs_request_dispatch_work(struct work_struct *work) struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, dispatch_work.work);struct fuse_conn *fc = fsvq->fud->fc; +int ret;pr_debug ("virtio-fs: worker %s called.\n" , __func__);while (1 ) {@@ -268,13 +272,43 @@ static void virtio_fs_request_dispatch_work(struct work_struct *work) list);if (!req) {spin_unlock (&fsvq->lock); -return ; +break ; }list_del_init (&req->list);spin_unlock (&fsvq->lock);fuse_request_end (fc, req); } + +/* Dispatch pending requests */ +while (1 ) { +spin_lock (&fsvq->lock); + req =list_first_entry_or_null (&fsvq->queued_reqs, + struct fuse_req, list); +if (!req) { +spin_unlock (&fsvq->lock); +return ; + } +list_del_init (&req->list); +spin_unlock (&fsvq->lock); + + ret =virtio_fs_enqueue_req (fsvq, req, true); +if (ret <0 ) {
fs/fuse/virtio_fs.c static void virtio_fs_request_dispatch_work(struct work_struct *work) · line 306
Quoted 10 lines
fs/fuse/virtio_fs.c +10
-0+if (ret == -ENOMEM || ret == -ENOSPC ) { +spin_lock (&fsvq->lock); +list_add_tail (&req->list, &fsvq->queued_reqs); +schedule_delayed_work (&fsvq->dispatch_work, +msecs_to_jiffies (1 )); +spin_unlock (&fsvq->lock); +return ; + } + req->out.h.error = ret; +dec_in_flight_req (fsvq);
Missing locking. Fixed.
Thanks,
Miklos
Vivek Goyal
maintainerRe: [PATCH 5/5] virtiofs: Retry request submission from worker context
On Mon, Oct 21, 2019 at 10:15:18AM +0200, Miklos Szeredi wrote:
[..]
Quoted 31 lines
Quoted 31 lines
fs/fuse/virtio_fs.c +20
-1@@ -268,13 +272,43 @@ static void virtio_fs_request_dispatch_work(struct work_struct *work) list);if (!req) {spin_unlock (&fsvq->lock); -return ; +break ; }list_del_init (&req->list);spin_unlock (&fsvq->lock);fuse_request_end (fc, req); } + +/* Dispatch pending requests */ +while (1 ) { +spin_lock (&fsvq->lock); + req =list_first_entry_or_null (&fsvq->queued_reqs, + struct fuse_req, list); +if (!req) { +spin_unlock (&fsvq->lock); +return ; + } +list_del_init (&req->list); +spin_unlock (&fsvq->lock); + + ret =virtio_fs_enqueue_req (fsvq, req, true); +if (ret <0 ) { +if (ret == -ENOMEM || ret == -ENOSPC ) { +spin_lock (&fsvq->lock); +list_add_tail (&req->list, &fsvq->queued_reqs); + schedule_delayed_work(&fsvq->dispatch_work,
Quoted 8 lines
fs/fuse/virtio_fs.c static void virtio_fs_request_dispatch_work(struct work_struct *work) · line 306
Quoted 6 lines
fs/fuse/virtio_fs.c +6
-0+ msecs_to_jiffies(1)); +spin_unlock (&fsvq->lock); +return ; + } + req->out.h.error = ret; +dec_in_flight_req (fsvq);Missing locking. Fixed.
Good catch. Thanks.
Vivek