Skip to contentSkip to conversation

virtio-fs: Flush forget requests before sending destroy

PATCH In review Vivek Goyal 3 participants 13 messages May 31, 2019 – Jun 6, 2019

Subsystems (2)
lore
Share & tools
lore

Vivek Goyal

author maintainer
#0 lore raw

cover virtio-fs: Flush forget requests before sending destroy

Hi Peng Tao,

Can you plase try following patches and see if it helps with the issue you are facing.

One of the patch is cleanup. Rest of the two patches make sure all forget requests requests are flushed and no forget reuqest is sent after destroy has been sent.

Thanks
Vivek

Vivek Goyal (3):
Use helper function to get forget requests from fuse Do not dispatch forget requests once queue is disconnected
virtio-fs: Waiting for pending forget requests to finish

 fs/fuse/dev.c       |  9 +++---
 fs/fuse/fuse_i.h    |  3 ++
 fs/fuse/virtio_fs.c | 70 ++++++++++++++++++++++++++++++++++++---------
 3 files changed, 65 insertions(+), 17 deletions(-)
Signature · 1 line
2.20.1
author maintainer
#1 lore raw

PATCH 1/3 Use helper function to get forget requests from fuse

stacked file systems like virtio-fs do not have to play directly with forget list data structures. There is a helper function use that instead.

Rename dequeue_forget() to fuse_dequeue_forget() and export it so that stacked filesystems can use it.

---
 fs/fuse/dev.c       | 9 +++++----
 fs/fuse/fuse_i.h    | 3 +++
 fs/fuse/virtio_fs.c | 7 +------
 3 files changed, 9 insertions(+), 10 deletions(-)
fs/fuse/dev.c +5 -4
index ee9dd38bc0f0..39da7ec19fa2 100644
@@ -1203,7 +1203,7 @@ __releases(fiq->waitq.lock)
 	return err ? err : reqsize;
 }
 
-static struct fuse_forget_link *dequeue_forget(struct fuse_iqueue *fiq,
+struct fuse_forget_link *fuse_dequeue_forget(struct fuse_iqueue *fiq,
 					       unsigned max,
 					       unsigned *countp)
 {
@@ -1224,6 +1224,7 @@ static struct fuse_forget_link *dequeue_forget(struct fuse_iqueue *fiq,
 
 	return head;
 }
+EXPORT_SYMBOL(fuse_dequeue_forget);
 
 static int fuse_read_single_forget(struct fuse_iqueue *fiq,
 				   struct fuse_copy_state *cs,
@@ -1231,7 +1232,7 @@ static int fuse_read_single_forget(struct fuse_iqueue *fiq,
 __releases(fiq->waitq.lock)
 {
 	int err;
-	struct fuse_forget_link *forget = dequeue_forget(fiq, 1, NULL);
+	struct fuse_forget_link *forget = fuse_dequeue_forget(fiq, 1, NULL);
 	struct fuse_forget_in arg = {
 		.nlookup = forget->forget_one.nlookup,
 	};
@@ -1279,7 +1280,7 @@ __releases(fiq->waitq.lock)
 	}
 
 	max_forgets = (nbytes - ih.len) / sizeof(struct fuse_forget_one);
-	head = dequeue_forget(fiq, max_forgets, &count);
+	head = fuse_dequeue_forget(fiq, max_forgets, &count);
 	spin_unlock(&fiq->waitq.lock);
 
 	arg.count = count;
@@ -2249,7 +2250,7 @@ void fuse_abort_conn(struct fuse_conn *fc)
 			clear_bit(FR_PENDING, &req->flags);
 		list_splice_tail_init(&fiq->pending, &to_end);
 		while (forget_pending(fiq))
-			kfree(dequeue_forget(fiq, 1, NULL));
+			kfree(fuse_dequeue_forget(fiq, 1, NULL));
 		wake_up_all_locked(&fiq->waitq);
 		spin_unlock(&fiq->waitq.lock);
 		kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
fs/fuse/fuse_i.h +3 -0
index 7ac7f9a0b81b..ad8272c98c82 100644
@@ -972,6 +972,9 @@ void fuse_queue_forget(struct fuse_conn *fc, struct fuse_forget_link *forget,
 
 struct fuse_forget_link *fuse_alloc_forget(void);
 
+struct fuse_forget_link *fuse_dequeue_forget(struct fuse_iqueue *fiq,
+					     unsigned max, unsigned *countp);
+
 /* Used by READDIRPLUS */
 void fuse_force_forget(struct file *file, u64 nodeid);
 
fs/fuse/virtio_fs.c +1 -6
index 18fc0dca0abc..cbda7ebd5bf9 100644
@@ -697,12 +697,7 @@ __releases(fiq->waitq.lock)
 	u64 unique;
 	int ret;
 
-	BUG_ON(!fiq->forget_list_head.next);
-	link = fiq->forget_list_head.next;
-	BUG_ON(link->next);
-	fiq->forget_list_head.next = NULL;
-	fiq->forget_list_tail = &fiq->forget_list_head;
-
+	link = fuse_dequeue_forget(fiq, 1, NULL);
 	unique = fuse_get_unique(fiq);
 
 	fs = fiq->priv;
Signature · 1 line
2.20.1
Reviewed-by on v1 1/3
#2 lore raw

Re: [Virtio-fs] [PATCH 1/3] Use helper function to get forget requests from fuse

Quoted 5 lines · Vivek Goyal

stacked file systems like virtio-fs do not have to play directly with forget list data structures. There is a helper function use that instead.

Rename dequeue_forget() to fuse_dequeue_forget() and export it so that stacked filesystems can use it.

thanks,
-liubo

Quoted 95 lines
---
 fs/fuse/dev.c       | 9 +++++----
 fs/fuse/fuse_i.h    | 3 +++
 fs/fuse/virtio_fs.c | 7 +------
 3 files changed, 9 insertions(+), 10 deletions(-)
fs/fuse/dev.c +5 -4
index ee9dd38bc0f0..39da7ec19fa2 100644
@@ -1203,7 +1203,7 @@ __releases(fiq->waitq.lock)
 	return err ? err : reqsize;
 }
 
-static struct fuse_forget_link *dequeue_forget(struct fuse_iqueue *fiq,
+struct fuse_forget_link *fuse_dequeue_forget(struct fuse_iqueue *fiq,
 					       unsigned max,
 					       unsigned *countp)
 {
@@ -1224,6 +1224,7 @@ static struct fuse_forget_link *dequeue_forget(struct fuse_iqueue *fiq,
 
 	return head;
 }
+EXPORT_SYMBOL(fuse_dequeue_forget);
 
 static int fuse_read_single_forget(struct fuse_iqueue *fiq,
 				   struct fuse_copy_state *cs,
@@ -1231,7 +1232,7 @@ static int fuse_read_single_forget(struct fuse_iqueue *fiq,
 __releases(fiq->waitq.lock)
 {
 	int err;
-	struct fuse_forget_link *forget = dequeue_forget(fiq, 1, NULL);
+	struct fuse_forget_link *forget = fuse_dequeue_forget(fiq, 1, NULL);
 	struct fuse_forget_in arg = {
 		.nlookup = forget->forget_one.nlookup,
 	};
@@ -1279,7 +1280,7 @@ __releases(fiq->waitq.lock)
 	}
 
 	max_forgets = (nbytes - ih.len) / sizeof(struct fuse_forget_one);
-	head = dequeue_forget(fiq, max_forgets, &count);
+	head = fuse_dequeue_forget(fiq, max_forgets, &count);
 	spin_unlock(&fiq->waitq.lock);
 
 	arg.count = count;
@@ -2249,7 +2250,7 @@ void fuse_abort_conn(struct fuse_conn *fc)
 			clear_bit(FR_PENDING, &req->flags);
 		list_splice_tail_init(&fiq->pending, &to_end);
 		while (forget_pending(fiq))
-			kfree(dequeue_forget(fiq, 1, NULL));
+			kfree(fuse_dequeue_forget(fiq, 1, NULL));
 		wake_up_all_locked(&fiq->waitq);
 		spin_unlock(&fiq->waitq.lock);
 		kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
fs/fuse/fuse_i.h +3 -0
index 7ac7f9a0b81b..ad8272c98c82 100644
@@ -972,6 +972,9 @@ void fuse_queue_forget(struct fuse_conn *fc, struct fuse_forget_link *forget,
 
 struct fuse_forget_link *fuse_alloc_forget(void);
 
+struct fuse_forget_link *fuse_dequeue_forget(struct fuse_iqueue *fiq,
+					     unsigned max, unsigned *countp);
+
 /* Used by READDIRPLUS */
 void fuse_force_forget(struct file *file, u64 nodeid);
 
fs/fuse/virtio_fs.c +1 -6
index 18fc0dca0abc..cbda7ebd5bf9 100644
@@ -697,12 +697,7 @@ __releases(fiq->waitq.lock)
 	u64 unique;
 	int ret;
 
-	BUG_ON(!fiq->forget_list_head.next);
-	link = fiq->forget_list_head.next;
-	BUG_ON(link->next);
-	fiq->forget_list_head.next = NULL;
-	fiq->forget_list_tail = &fiq->forget_list_head;
-
+	link = fuse_dequeue_forget(fiq, 1, NULL);
 	unique = fuse_get_unique(fiq);
 
 	fs = fiq->priv;
Signature · 6 lines
2.20.1

_______________________________________________
Virtio-fs mailing list
[email protected]
https://www.redhat.com/mailman/listinfo/virtio-fs
author maintainer
#3 lore raw

PATCH 2/3 Do not dispatch forget requests once queue is disconnected

In reply to Vivek Goyal

Keep track of virtiofs queue status and once queue is disconnected do not dispatch any more forget reuqests. Queue is disconnected once unmount request comes in.

---
 fs/fuse/virtio_fs.c | 25 +++++++++++++++++++------
 1 file changed, 19 insertions(+), 6 deletions(-)
fs/fuse/virtio_fs.c +19 -6
index cbda7ebd5bf9..737e92cdc5ed 100644
@@ -31,6 +31,7 @@ struct virtio_fs_vq {
 	struct list_head queued_reqs;
 	struct delayed_work dispatch_work;
 	struct fuse_dev *fud;
+	bool connected;
 	char name[24];
 } ____cacheline_aligned_in_smp;
 
@@ -215,6 +216,12 @@ static void virtio_fs_hiprio_dispatch_work(struct work_struct *work)
 		}
 
 		list_del(&forget->list);
+		if (!fsvq->connected) {
+			spin_unlock(&fsvq->lock);
+			kfree(forget);
+			continue;
+		}
+
 		sg_init_one(&sg, forget, sizeof(*forget));
 
 		/* Enqueue the request */
@@ -425,9 +432,10 @@ static int virtio_fs_setup_vqs(struct virtio_device *vdev,
 	if (ret < 0)
 		goto out;
 
-	for (i = 0; i < fs->nvqs; i++)
+	for (i = 0; i < fs->nvqs; i++) {
 		fs->vqs[i].vq = vqs[i];
-
+		fs->vqs[i].connected = true;
+	}
 out:
 	kfree(names);
 	kfree(callbacks);
@@ -1071,11 +1079,16 @@ static int virtio_fs_fill_super(struct super_block *sb, void *data,
 static void virtio_kill_sb(struct super_block *sb)
 {
 	struct fuse_conn *fc = get_fuse_conn_super(sb);
+	struct virtio_fs *vfs = fc->iq.priv;
+	struct virtio_fs_vq *fsvq = &vfs->vqs[VQ_HIPRIO];
+
+	/* Stop forget queue. Soon destroy will be sent */
+	spin_lock(&fsvq->lock);
+	fsvq->connected = false;
+	spin_unlock(&fsvq->lock);
+
 	fuse_kill_sb_anon(sb);
-	if (fc) {
-		struct virtio_fs *vfs = fc->iq.priv;
-		virtio_fs_free_devs(vfs);
-	}
+	virtio_fs_free_devs(vfs);
 }
 
 static struct dentry *virtio_fs_mount(struct file_system_type *fs_type,
Signature · 1 line
2.20.1
#4 lore raw

Re: [Virtio-fs] [PATCH 2/3] Do not dispatch forget requests once queue is disconnected

Quoted 4 lines · Vivek Goyal

Keep track of virtiofs queue status and once queue is disconnected do not dispatch any more forget reuqests. Queue is disconnected once unmount request comes in.

Reviewed-and-tested-by Peng Tao <[email protected]>
Quoted 65 lines
---
  fs/fuse/virtio_fs.c | 25 +++++++++++++++++++------
  1 file changed, 19 insertions(+), 6 deletions(-)
fs/fuse/virtio_fs.c +19 -6
index cbda7ebd5bf9..737e92cdc5ed 100644
@@ -31,6 +31,7 @@ struct virtio_fs_vq {
  	struct list_head queued_reqs;
  	struct delayed_work dispatch_work;
  	struct fuse_dev *fud;
+	bool connected;
  	char name[24];
  } ____cacheline_aligned_in_smp;
 
@@ -215,6 +216,12 @@ static void virtio_fs_hiprio_dispatch_work(struct work_struct *work)
  		}
 
  		list_del(&forget->list);
+		if (!fsvq->connected) {
+			spin_unlock(&fsvq->lock);
+			kfree(forget);
+			continue;
+		}
+
  		sg_init_one(&sg, forget, sizeof(*forget));
 
  		/* Enqueue the request */
@@ -425,9 +432,10 @@ static int virtio_fs_setup_vqs(struct virtio_device *vdev,
  	if (ret < 0)
  		goto out;
 
-	for (i = 0; i < fs->nvqs; i++)
+	for (i = 0; i < fs->nvqs; i++) {
  		fs->vqs[i].vq = vqs[i];
-
+		fs->vqs[i].connected = true;
+	}
  out:
  	kfree(names);
  	kfree(callbacks);
@@ -1071,11 +1079,16 @@ static int virtio_fs_fill_super(struct super_block *sb, void *data,
  static void virtio_kill_sb(struct super_block *sb)
  {
  	struct fuse_conn *fc = get_fuse_conn_super(sb);
+	struct virtio_fs *vfs = fc->iq.priv;
+	struct virtio_fs_vq *fsvq = &vfs->vqs[VQ_HIPRIO];
+
+	/* Stop forget queue. Soon destroy will be sent */
+	spin_lock(&fsvq->lock);
+	fsvq->connected = false;
+	spin_unlock(&fsvq->lock);
+
  	fuse_kill_sb_anon(sb);
-	if (fc) {
-		struct virtio_fs *vfs = fc->iq.priv;
-		virtio_fs_free_devs(vfs);
-	}
+	virtio_fs_free_devs(vfs);
  }
 
  static struct dentry *virtio_fs_mount(struct file_system_type *fs_type,
Reviewed-by on v1 2/3
#5 lore raw

Re: [Virtio-fs] [PATCH 2/3] Do not dispatch forget requests once queue is disconnected

In reply to Vivek Goyal
Quoted 3 lines · Vivek Goyal

Keep track of virtiofs queue status and once queue is disconnected do not dispatch any more forget reuqests. Queue is disconnected once unmount request comes in.

thanks,
-liubo

Quoted 72 lines
---
 fs/fuse/virtio_fs.c | 25 +++++++++++++++++++------
 1 file changed, 19 insertions(+), 6 deletions(-)
fs/fuse/virtio_fs.c +19 -6
index cbda7ebd5bf9..737e92cdc5ed 100644
@@ -31,6 +31,7 @@ struct virtio_fs_vq {
 	struct list_head queued_reqs;
 	struct delayed_work dispatch_work;
 	struct fuse_dev *fud;
+	bool connected;
 	char name[24];
 } ____cacheline_aligned_in_smp;
 
@@ -215,6 +216,12 @@ static void virtio_fs_hiprio_dispatch_work(struct work_struct *work)
 		}
 
 		list_del(&forget->list);
+		if (!fsvq->connected) {
+			spin_unlock(&fsvq->lock);
+			kfree(forget);
+			continue;
+		}
+
 		sg_init_one(&sg, forget, sizeof(*forget));
 
 		/* Enqueue the request */
@@ -425,9 +432,10 @@ static int virtio_fs_setup_vqs(struct virtio_device *vdev,
 	if (ret < 0)
 		goto out;
 
-	for (i = 0; i < fs->nvqs; i++)
+	for (i = 0; i < fs->nvqs; i++) {
 		fs->vqs[i].vq = vqs[i];
-
+		fs->vqs[i].connected = true;
+	}
 out:
 	kfree(names);
 	kfree(callbacks);
@@ -1071,11 +1079,16 @@ static int virtio_fs_fill_super(struct super_block *sb, void *data,
 static void virtio_kill_sb(struct super_block *sb)
 {
 	struct fuse_conn *fc = get_fuse_conn_super(sb);
+	struct virtio_fs *vfs = fc->iq.priv;
+	struct virtio_fs_vq *fsvq = &vfs->vqs[VQ_HIPRIO];
+
+	/* Stop forget queue. Soon destroy will be sent */
+	spin_lock(&fsvq->lock);
+	fsvq->connected = false;
+	spin_unlock(&fsvq->lock);
+
 	fuse_kill_sb_anon(sb);
-	if (fc) {
-		struct virtio_fs *vfs = fc->iq.priv;
-		virtio_fs_free_devs(vfs);
-	}
+	virtio_fs_free_devs(vfs);
 }
 
 static struct dentry *virtio_fs_mount(struct file_system_type *fs_type,
Signature · 6 lines
2.20.1

_______________________________________________
Virtio-fs mailing list
[email protected]
https://www.redhat.com/mailman/listinfo/virtio-fs
author maintainer
#6 lore raw

PATCH 3/3 virtio-fs: Waiting for pending forget requests to finish

In reply to Vivek Goyal

Keep a track of pending forget requests and wait for these to finish during ->kill_sb()

---
 fs/fuse/virtio_fs.c | 38 +++++++++++++++++++++++++++++++++++++-
 1 file changed, 37 insertions(+), 1 deletion(-)
fs/fuse/virtio_fs.c +37 -1
index 737e92cdc5ed..abdb58285805 100644
@@ -32,6 +32,7 @@ struct virtio_fs_vq {
 	struct delayed_work dispatch_work;
 	struct fuse_dev *fud;
 	bool connected;
+	long in_flight;
 	char name[24];
 } ____cacheline_aligned_in_smp;
 
@@ -183,8 +184,10 @@ static void virtio_fs_hiprio_done_work(struct work_struct *work)
 
 		virtqueue_disable_cb(vq);
 
-		while ((req = virtqueue_get_buf(vq, &len)) != NULL)
+		while ((req = virtqueue_get_buf(vq, &len)) != NULL) {
 			kfree(req);
+			fsvq->in_flight--;
+		}
 	} while (!virtqueue_enable_cb(vq) && likely(!virtqueue_is_broken(vq)));
 	spin_unlock(&fsvq->lock);
 }
@@ -244,6 +247,7 @@ static void virtio_fs_hiprio_dispatch_work(struct work_struct *work)
 			return;
 		}
 
+		fsvq->in_flight++;
 		notify = virtqueue_kick_prepare(vq);
 		spin_unlock(&fsvq->lock);
 
@@ -753,6 +757,7 @@ __releases(fiq->waitq.lock)
 		goto out;
 	}
 
+	fsvq->in_flight++;
 	notify = virtqueue_kick_prepare(vq);
 
 	spin_unlock(&fsvq->lock);
@@ -982,6 +987,36 @@ __releases(fiq->waitq.lock)
 	}
 }
 
+static void virtio_fs_flush_hiprio_queue(struct virtio_fs_vq *fsvq)
+{
+	struct virtio_fs_forget *forget;
+
+	WARN_ON(fsvq->in_flight < 0);
+
+	/* Go through pending forget reuests and free them */
+	spin_lock(&fsvq->lock);
+	while(1) {
+		forget = list_first_entry_or_null(&fsvq->queued_reqs,
+					struct virtio_fs_forget, list);
+		if (!forget)
+			break;
+		kfree(forget);
+	}
+
+	spin_unlock(&fsvq->lock);
+
+	/* Wait for in flight requests to finish.*/
+	while (1) {
+		spin_lock(&fsvq->lock);
+		if (!fsvq->in_flight) {
+			spin_unlock(&fsvq->lock);
+			break;
+		}
+		spin_unlock(&fsvq->lock);
+		usleep_range(1000, 2000);
+	}
+}
+
 const static struct fuse_iqueue_ops virtio_fs_fiq_ops = {
 	.wake_forget_and_unlock		= virtio_fs_wake_forget_and_unlock,
 	.wake_interrupt_and_unlock	= virtio_fs_wake_interrupt_and_unlock,
@@ -1086,6 +1121,7 @@ static void virtio_kill_sb(struct super_block *sb)
 	spin_lock(&fsvq->lock);
 	fsvq->connected = false;
 	spin_unlock(&fsvq->lock);
+	virtio_fs_flush_hiprio_queue(fsvq);
 
 	fuse_kill_sb_anon(sb);
 	virtio_fs_free_devs(vfs);
Signature · 1 line
2.20.1
#7 lore raw

Re: [Virtio-fs] [PATCH 3/3] virtio-fs: Waiting for pending forget requests to finish

Quoted 55 lines · Vivek Goyal

Keep a track of pending forget requests and wait for these to finish during ->kill_sb()

---
  fs/fuse/virtio_fs.c | 38 +++++++++++++++++++++++++++++++++++++-
  1 file changed, 37 insertions(+), 1 deletion(-)
fs/fuse/virtio_fs.c +9 -1
index 737e92cdc5ed..abdb58285805 100644
@@ -32,6 +32,7 @@ struct virtio_fs_vq {
  	struct delayed_work dispatch_work;
  	struct fuse_dev *fud;
  	bool connected;
+	long in_flight;
  	char name[24];
  } ____cacheline_aligned_in_smp;
 
@@ -183,8 +184,10 @@ static void virtio_fs_hiprio_done_work(struct work_struct *work)
 
  		virtqueue_disable_cb(vq);
 
-		while ((req = virtqueue_get_buf(vq, &len)) != NULL)
+		while ((req = virtqueue_get_buf(vq, &len)) != NULL) {
  			kfree(req);
+			fsvq->in_flight--;
+		}
  	} while (!virtqueue_enable_cb(vq) && likely(!virtqueue_is_broken(vq)));
  	spin_unlock(&fsvq->lock);
  }
@@ -244,6 +247,7 @@ static void virtio_fs_hiprio_dispatch_work(struct work_struct *work)
  			return;
  		}
 
+		fsvq->in_flight++;
  		notify = virtqueue_kick_prepare(vq);
  		spin_unlock(&fsvq->lock);
 
@@ -753,6 +757,7 @@ __releases(fiq->waitq.lock)
  		goto out;
  	}
 
+	fsvq->in_flight++;
  	notify = virtqueue_kick_prepare(vq);
 
  	spin_unlock(&fsvq->lock);
@@ -982,6 +987,36 @@ __releases(fiq->waitq.lock)
  	}
  }
 
+static void virtio_fs_flush_hiprio_queue(struct virtio_fs_vq *fsvq)
+{
+	struct virtio_fs_forget *forget;

fs/fuse/virtio_fs.c __releases(fiq->waitq.lock) · line 1002

Quoted 10 lines
fs/fuse/virtio_fs.c +10 -0
+
+	WARN_ON(fsvq->in_flight < 0);
+
+	/* Go through pending forget reuests and free them */
+	spin_lock(&fsvq->lock);
+	while(1) {
+		forget = list_first_entry_or_null(&fsvq->queued_reqs,
+					struct virtio_fs_forget, list);
+		if (!forget)
+			break;

missing list_del()?

Cheers,
Tao

author maintainer
#8 lore raw

Re: [Virtio-fs] [PATCH 3/3] virtio-fs: Waiting for pending forget requests to finish

On Tue, Jun 04, 2019 at 12:06:40PM +0800, Peng Tao wrote:
[..]

Quoted 6 lines
Quoted 6 lines
fs/fuse/virtio_fs.c +6 -0
+static void virtio_fs_flush_hiprio_queue(struct virtio_fs_vq *fsvq)
+{
+	struct virtio_fs_forget *forget;
+
+	WARN_ON(fsvq->in_flight < 0);
+
Quoted 8 lines

fs/fuse/virtio_fs.c __releases(fiq->waitq.lock) · line 1002

Quoted 7 lines
fs/fuse/virtio_fs.c +7 -0
+	/* Go through pending forget reuests and free them */
+	spin_lock(&fsvq->lock);
+	while(1) {
+		forget = list_first_entry_or_null(&fsvq->queued_reqs,
+					struct virtio_fs_forget, list);
+		if (!forget)
+			break;

missing list_del()?

Good catch. Will fix it.

Vivek

Reviewed-by on v1 3/3
#9 lore raw

Re: [Virtio-fs] [PATCH 3/3] virtio-fs: Waiting for pending forget requests to finish

In reply to Vivek Goyal
Quoted 70 lines · Vivek Goyal

Keep a track of pending forget requests and wait for these to finish during ->kill_sb()

---
 fs/fuse/virtio_fs.c | 38 +++++++++++++++++++++++++++++++++++++-
 1 file changed, 37 insertions(+), 1 deletion(-)
fs/fuse/virtio_fs.c +24 -1
index 737e92cdc5ed..abdb58285805 100644
@@ -32,6 +32,7 @@ struct virtio_fs_vq {
 	struct delayed_work dispatch_work;
 	struct fuse_dev *fud;
 	bool connected;
+	long in_flight;
 	char name[24];
 } ____cacheline_aligned_in_smp;
 
@@ -183,8 +184,10 @@ static void virtio_fs_hiprio_done_work(struct work_struct *work)
 
 		virtqueue_disable_cb(vq);
 
-		while ((req = virtqueue_get_buf(vq, &len)) != NULL)
+		while ((req = virtqueue_get_buf(vq, &len)) != NULL) {
 			kfree(req);
+			fsvq->in_flight--;
+		}
 	} while (!virtqueue_enable_cb(vq) && likely(!virtqueue_is_broken(vq)));
 	spin_unlock(&fsvq->lock);
 }
@@ -244,6 +247,7 @@ static void virtio_fs_hiprio_dispatch_work(struct work_struct *work)
 			return;
 		}
 
+		fsvq->in_flight++;
 		notify = virtqueue_kick_prepare(vq);
 		spin_unlock(&fsvq->lock);
 
@@ -753,6 +757,7 @@ __releases(fiq->waitq.lock)
 		goto out;
 	}
 
+	fsvq->in_flight++;
 	notify = virtqueue_kick_prepare(vq);
 
 	spin_unlock(&fsvq->lock);
@@ -982,6 +987,36 @@ __releases(fiq->waitq.lock)
 	}
 }
 
+static void virtio_fs_flush_hiprio_queue(struct virtio_fs_vq *fsvq)
+{
+	struct virtio_fs_forget *forget;
+
+	WARN_ON(fsvq->in_flight < 0);
+
+	/* Go through pending forget reuests and free them */
+	spin_lock(&fsvq->lock);
+	while(1) {
+		forget = list_first_entry_or_null(&fsvq->queued_reqs,
+					struct virtio_fs_forget, list);
+		if (!forget)
+			break;
+		kfree(forget);
+	}
+
+	spin_unlock(&fsvq->lock);
+

fs/fuse/virtio_fs.c __releases(fiq->waitq.lock) · line 1017

Quoted 10 lines
fs/fuse/virtio_fs.c +10 -0
+	/* Wait for in flight requests to finish.*/
+	while (1) {
+		spin_lock(&fsvq->lock);
+		if (!fsvq->in_flight) {
+			spin_unlock(&fsvq->lock);
+			break;
+		}
+		spin_unlock(&fsvq->lock);
+		usleep_range(1000, 2000);
+	}

why not a wait_queue for fsvq->in_flight == 0?

Otherwise, it looks good.

thanks,
-liubo

Quoted 20 lines
fs/fuse/virtio_fs.c +3 -0
+}
+
 const static struct fuse_iqueue_ops virtio_fs_fiq_ops = {
 	.wake_forget_and_unlock		= virtio_fs_wake_forget_and_unlock,
 	.wake_interrupt_and_unlock	= virtio_fs_wake_interrupt_and_unlock,
@@ -1086,6 +1121,7 @@ static void virtio_kill_sb(struct super_block *sb)
 	spin_lock(&fsvq->lock);
 	fsvq->connected = false;
 	spin_unlock(&fsvq->lock);
+	virtio_fs_flush_hiprio_queue(fsvq);
 
 	fuse_kill_sb_anon(sb);
 	virtio_fs_free_devs(vfs);
Signature · 6 lines
2.20.1

_______________________________________________
Virtio-fs mailing list
[email protected]
https://www.redhat.com/mailman/listinfo/virtio-fs
author maintainer
#10 lore raw

Re: [Virtio-fs] [PATCH 3/3] virtio-fs: Waiting for pending forget requests to finish

On Wed, Jun 05, 2019 at 11:26:07AM -0700, Liu Bo wrote:

[..]

Quoted 22 lines
Quoted 22 lines
fs/fuse/virtio_fs.c +22 -0
+static void virtio_fs_flush_hiprio_queue(struct virtio_fs_vq *fsvq)
+{
+	struct virtio_fs_forget *forget;
+
+	WARN_ON(fsvq->in_flight < 0);
+
+	/* Go through pending forget reuests and free them */
+	spin_lock(&fsvq->lock);
+	while(1) {
+		forget = list_first_entry_or_null(&fsvq->queued_reqs,
+					struct virtio_fs_forget, list);
+		if (!forget)
+			break;
+		kfree(forget);
+	}
+
+	spin_unlock(&fsvq->lock);
+
+	/* Wait for in flight requests to finish.*/
+	while (1) {
+		spin_lock(&fsvq->lock);
+		if (!fsvq->in_flight) {
Quoted 8 lines

fs/fuse/virtio_fs.c __releases(fiq->waitq.lock) · line 1017

Quoted 6 lines
fs/fuse/virtio_fs.c +6 -0
+			spin_unlock(&fsvq->lock);
+			break;
+		}
+		spin_unlock(&fsvq->lock);
+		usleep_range(1000, 2000);
+	}

why not a wait_queue for fsvq->in_flight == 0?

It was much simpler to do busy wait as I am not expecting this wait to be long. But if this turns out to be issue later, we can switch to wait queue.

Vivek

#11 lore raw

Re: [Virtio-fs] [PATCH 3/3] virtio-fs: Waiting for pending forget requests to finish

In reply to Vivek Goyal
Quoted 69 lines · Vivek Goyal

Keep a track of pending forget requests and wait for these to finish during ->kill_sb()

---
 fs/fuse/virtio_fs.c | 38 +++++++++++++++++++++++++++++++++++++-
 1 file changed, 37 insertions(+), 1 deletion(-)
fs/fuse/virtio_fs.c +23 -1
index 737e92cdc5ed..abdb58285805 100644
@@ -32,6 +32,7 @@ struct virtio_fs_vq {
 	struct delayed_work dispatch_work;
 	struct fuse_dev *fud;
 	bool connected;
+	long in_flight;
 	char name[24];
 } ____cacheline_aligned_in_smp;
 
@@ -183,8 +184,10 @@ static void virtio_fs_hiprio_done_work(struct work_struct *work)
 
 		virtqueue_disable_cb(vq);
 
-		while ((req = virtqueue_get_buf(vq, &len)) != NULL)
+		while ((req = virtqueue_get_buf(vq, &len)) != NULL) {
 			kfree(req);
+			fsvq->in_flight--;
+		}
 	} while (!virtqueue_enable_cb(vq) && likely(!virtqueue_is_broken(vq)));
 	spin_unlock(&fsvq->lock);
 }
@@ -244,6 +247,7 @@ static void virtio_fs_hiprio_dispatch_work(struct work_struct *work)
 			return;
 		}
 
+		fsvq->in_flight++;
 		notify = virtqueue_kick_prepare(vq);
 		spin_unlock(&fsvq->lock);
 
@@ -753,6 +757,7 @@ __releases(fiq->waitq.lock)
 		goto out;
 	}
 
+	fsvq->in_flight++;
 	notify = virtqueue_kick_prepare(vq);
 
 	spin_unlock(&fsvq->lock);
@@ -982,6 +987,36 @@ __releases(fiq->waitq.lock)
 	}
 }
 
+static void virtio_fs_flush_hiprio_queue(struct virtio_fs_vq *fsvq)
+{
+	struct virtio_fs_forget *forget;
+
+	WARN_ON(fsvq->in_flight < 0);
+
+	/* Go through pending forget reuests and free them */
+	spin_lock(&fsvq->lock);
+	while(1) {
+		forget = list_first_entry_or_null(&fsvq->queued_reqs,
+					struct virtio_fs_forget, list);
+		if (!forget)
+			break;
+		kfree(forget);
+	}
+
+	spin_unlock(&fsvq->lock);

fs/fuse/virtio_fs.c __releases(fiq->waitq.lock) · line 1016

Quoted 10 lines
fs/fuse/virtio_fs.c +10 -0
+
+	/* Wait for in flight requests to finish.*/
+	while (1) {
+		spin_lock(&fsvq->lock);
+		if (!fsvq->in_flight) {
+			spin_unlock(&fsvq->lock);
+			break;
+		}
+		spin_unlock(&fsvq->lock);
+		usleep_range(1000, 2000);

Looks like #include <linux/delay.h> is also needed in order to pass building.

thanks,
-liubo

Quoted 21 lines
fs/fuse/virtio_fs.c +4 -0
+	}
+}
+
 const static struct fuse_iqueue_ops virtio_fs_fiq_ops = {
 	.wake_forget_and_unlock		= virtio_fs_wake_forget_and_unlock,
 	.wake_interrupt_and_unlock	= virtio_fs_wake_interrupt_and_unlock,
@@ -1086,6 +1121,7 @@ static void virtio_kill_sb(struct super_block *sb)
 	spin_lock(&fsvq->lock);
 	fsvq->connected = false;
 	spin_unlock(&fsvq->lock);
+	virtio_fs_flush_hiprio_queue(fsvq);
 
 	fuse_kill_sb_anon(sb);
 	virtio_fs_free_devs(vfs);
Signature · 6 lines
2.20.1

_______________________________________________
Virtio-fs mailing list
[email protected]
https://www.redhat.com/mailman/listinfo/virtio-fs
author maintainer
#12 lore raw

Re: [Virtio-fs] [PATCH 3/3] virtio-fs: Waiting for pending forget requests to finish

On Thu, Jun 06, 2019 at 02:06:46PM -0700, Liu Bo wrote:

[..]

Quoted 22 lines
Quoted 22 lines
fs/fuse/virtio_fs.c +22 -0
+static void virtio_fs_flush_hiprio_queue(struct virtio_fs_vq *fsvq)
+{
+	struct virtio_fs_forget *forget;
+
+	WARN_ON(fsvq->in_flight < 0);
+
+	/* Go through pending forget reuests and free them */
+	spin_lock(&fsvq->lock);
+	while(1) {
+		forget = list_first_entry_or_null(&fsvq->queued_reqs,
+					struct virtio_fs_forget, list);
+		if (!forget)
+			break;
+		kfree(forget);
+	}
+
+	spin_unlock(&fsvq->lock);
+
+	/* Wait for in flight requests to finish.*/
+	while (1) {
+		spin_lock(&fsvq->lock);
+		if (!fsvq->in_flight) {
Quoted 8 lines

fs/fuse/virtio_fs.c __releases(fiq->waitq.lock) · line 1016

Quoted 5 lines
fs/fuse/virtio_fs.c +5 -0
+			spin_unlock(&fsvq->lock);
+			break;
+		}
+		spin_unlock(&fsvq->lock);
+		usleep_range(1000, 2000);

Looks like #include <linux/delay.h> is also needed in order to pass building.

Its already there.

https://gitlab.com/virtio-fs/linux/blob/virtio-fs-dev/fs/fuse/virtio_fs.c#L14

Vivek