Skip to contentSkip to conversation

move layout get/return synchronization to rpc prepare

RFC In review Benny Halevy 2 participants 9 messages Sep 14, 2010

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

cover move layout get/return synchronization to rpc prepare

The following patchset introduces a per-inode rpc wait queue to be used to synchronize layout gets and returns during their respective rpc prepare phase.

Essentially, layout gets wait on layout segments marked as invalid, currently meaning there is a layout return in progress. It should be ok to send the layout get anyway but waiting avoids the race rather than having to possibly resolve it on the done path.

In the future layout gets could insert the layout segment they ask for as "invalid" so that parallel layout gets would wait too. However, since the server may return a different segment than requested (bigger or smaller), there's no guarantee that the parallel layoutget will overlap the one in progress.

Layout returns wait on layout segments while they are being used (e.g. for I/O operations).

[PATCH 1/4] SQUASHME: pnfs-post-submit: remove take_ref and only_valid params from pnfs_has_layout [PATCH 2/4] pnfs: allow nfs4_proc_layoutget to sleep on invalid lsegs
[PATCH 3/4] pnfs: allow nfs4_proc_layoutreturn to sleep on barrier
[PATCH 4/4] SQUASHME: pnfs: get rid of lo_waitq

#1 lore raw

PATCH 1/4 SQUASHME: pnfs-post-submit: remove take_ref and only_valid params from pnfs_has_layout

They are always evaluated to treu and false, respectively as the lseg ptr is always needed.

---
 fs/nfs/pnfs.c |   23 ++++++++---------------
 1 files changed, 8 insertions(+), 15 deletions(-)
fs/nfs/pnfs.c +8 -15
index b576470..bc4f0c1 100644
@@ -1001,9 +1001,7 @@ has_matching_lseg(struct pnfs_layout_segment *lseg,
  */
 static struct pnfs_layout_segment *
 pnfs_has_layout(struct pnfs_layout_hdr *lo,
-		struct pnfs_layout_range *range,
-		bool take_ref,
-		bool only_valid)
+		struct pnfs_layout_range *range)
 {
 	struct pnfs_layout_segment *lseg, *ret = NULL;
 
@@ -1011,19 +1009,17 @@ pnfs_has_layout(struct pnfs_layout_hdr *lo,
 
 	BUG_ON_UNLOCKED_LO(lo);
 	list_for_each_entry (lseg, &lo->segs, fi_list) {
-		if (has_matching_lseg(lseg, range) &&
-		    (lseg->valid || !only_valid)) {
+		if (has_matching_lseg(lseg, range)) {
 			ret = lseg;
-			if (take_ref)
-				get_lseg(ret);
+			get_lseg(ret);
 			break;
 		}
 		if (cmp_layout(range, &lseg->range) > 0)
 			break;
 	}
 
-	dprintk("%s:Return lseg %p take_ref %d ref %d valid %d\n",
-		__func__, ret, take_ref,
+	dprintk("%s:Return lseg %p ref %d valid %d\n",
+		__func__, ret,
 		ret ? atomic_read(&ret->kref.refcount) : 0,
 		ret ? ret->valid : 0);
 	return ret;
@@ -1050,10 +1046,8 @@ _pnfs_update_layout(struct inode *ino,
 	struct nfs_inode *nfsi = NFS_I(ino);
 	struct pnfs_layout_hdr *lo;
 	struct pnfs_layout_segment *lseg = NULL;
-	bool take_ref = (lsegpp != NULL);
 
-	if (take_ref)
-		*lsegpp = NULL;
+	*lsegpp = NULL;
 	spin_lock(&ino->i_lock);
 	lo = pnfs_alloc_layout(ino);
 	if (lo == NULL) {
@@ -1062,10 +1056,9 @@ _pnfs_update_layout(struct inode *ino,
 	}
 
 	/* Check to see if the layout for the given range already exists */
-	lseg = pnfs_has_layout(lo, &arg, take_ref, !take_ref);
+	lseg = pnfs_has_layout(lo, &arg);
 	if (lseg && !lseg->valid) {
-		if (take_ref)
-			put_lseg_locked(lseg);
+		put_lseg_locked(lseg);
 		/* someone is cleaning the layout */
 		lseg = NULL;
 		goto out_unlock;
Signature · 1 line
1.7.2.2
#2 lore raw

PATCH 2/4 pnfs: allow nfs4_proc_layoutget to sleep on invalid lsegs

In reply to Benny Halevy

Introduce lo_rpcwaitq and sleep on it while there's an lseg marked invalid, i.e. it is being returned (in the future, it could also be layoutget in progress).

---
 fs/nfs/inode.c         |    1 +
 fs/nfs/nfs4proc.c      |   25 ++++++++++++++++++++++---
 fs/nfs/pnfs.c          |   37 ++++++++++++++++++-------------------
 fs/nfs/pnfs.h          |    3 +++
 include/linux/nfs_fs.h |    1 +
 5 files changed, 45 insertions(+), 22 deletions(-)
fs/nfs/inode.c +1 -0
index 7621cfc..059bdf7 100644
@@ -1462,6 +1462,7 @@ static inline void nfs4_init_once(struct nfs_inode *nfsi)
 	init_rwsem(&nfsi->rwsem);
 #ifdef CONFIG_NFS_V4_1
 	init_waitqueue_head(&nfsi->lo_waitq);
+	rpc_init_wait_queue(&nfsi->lo_rpcwaitq, "pNFS Layout");
 	nfsi->pnfs_layout_suspend = 0;
 	nfsi->layout = NULL;
 #endif /* CONFIG_NFS_V4_1 */
fs/nfs/nfs4proc.c +22 -3
index 48a763e..f12568d 100644
@@ -5442,13 +5442,32 @@ nfs4_layoutget_prepare(struct rpc_task *task, void *calldata)
 {
 	struct nfs4_layoutget *lgp = calldata;
 	struct inode *ino = lgp->args.inode;
+	struct nfs_inode *nfsi = NFS_I(ino);
 	struct nfs_server *server = NFS_SERVER(ino);
+	struct pnfs_layout_segment *lseg;
 
 	dprintk("--> %s\n", __func__);
-	if (nfs4_setup_sequence(server, NULL, &lgp->args.seq_args,
-				&lgp->res.seq_res, 0, task))
+	spin_lock(&ino->i_lock);
+	lseg = pnfs_has_layout(nfsi->layout, &lgp->args.range);
+	if (likely(!lseg)) {
+		spin_unlock(&ino->i_lock);
+		dprintk("%s: no lseg found, proceeding\n", __func__);
+		if (!nfs4_setup_sequence(server, NULL, &lgp->args.seq_args,
+					 &lgp->res.seq_res, 0, task))
+			rpc_call_start(task);
 		return;
-	rpc_call_start(task);
+	}
+	if (!lseg->valid) {
+		put_lseg_locked(lseg);
+		spin_unlock(&ino->i_lock);
+		dprintk("%s: invalid lseg found, waiting\n", __func__);
+		rpc_sleep_on(&nfsi->lo_rpcwaitq, task, NULL);
+		return;
+	}
+	*lgp->lsegpp = lseg;
+	spin_unlock(&ino->i_lock);
+	dprintk("%s: valid lseg found, no rpc required\n", __func__);
+	rpc_exit(task, NFS4_OK);
 }
 
 static void nfs4_layoutget_done(struct rpc_task *task, void *calldata)
fs/nfs/pnfs.c +18 -19
index bc4f0c1..2450383 100644
@@ -430,7 +430,7 @@ destroy_lseg(struct kref *kref)
 	PNFS_LD_IO_OPS(lseg->layout)->free_lseg(lseg);
 }
 
-static void
+void
 put_lseg_locked(struct pnfs_layout_segment *lseg)
 {
 	bool do_wake_up;
@@ -444,8 +444,10 @@ put_lseg_locked(struct pnfs_layout_segment *lseg)
 	do_wake_up = !lseg->valid;
 	nfsi = PNFS_NFS_INODE(lseg->layout);
 	kref_put(&lseg->kref, destroy_lseg);
-	if (do_wake_up)
+	if (do_wake_up) {
 		wake_up(&nfsi->lo_waitq);
+		rpc_wake_up(&nfsi->lo_rpcwaitq);
+	}
 }
 
 void
@@ -999,7 +1001,7 @@ has_matching_lseg(struct pnfs_layout_segment *lseg,
 /*
  * lookup range in layout
  */
-static struct pnfs_layout_segment *
+struct pnfs_layout_segment *
 pnfs_has_layout(struct pnfs_layout_hdr *lo,
 		struct pnfs_layout_range *range)
 {
@@ -1057,22 +1059,20 @@ _pnfs_update_layout(struct inode *ino,
 
 	/* Check to see if the layout for the given range already exists */
 	lseg = pnfs_has_layout(lo, &arg);
-	if (lseg && !lseg->valid) {
-		put_lseg_locked(lseg);
-		/* someone is cleaning the layout */
-		lseg = NULL;
-		goto out_unlock;
-	}
-
 	if (lseg) {
-		dprintk("%s: Using cached lseg %p for %llu@%llu iomode %d)\n",
-			__func__,
-			lseg,
-			arg.length,
-			arg.offset,
-			arg.iomode);
+		if (lseg->valid) {
+			dprintk("%s: Using cached lseg %p for %llu@%llu "
+				"iomode %d)\n",
+				__func__,
+				lseg,
+				arg.length,
+				arg.offset,
+				arg.iomode);
 
-		goto out_unlock;
+			goto out_unlock;
+		}
+		/* someone is cleaning the layout */
+		put_lseg_locked(lseg);
 	}
 
 	/* if get layout already failed once goto out */
@@ -1097,8 +1097,7 @@ out:
 		nfsi->layout->state, lseg);
 	return;
 out_unlock:
-	if (lsegpp)
-		*lsegpp = lseg;
+	*lsegpp = lseg;
 	spin_unlock(&ino->i_lock);
 	goto out;
 }
fs/nfs/pnfs.h +3 -0
index ea70385..f7f567e 100644
@@ -34,6 +34,9 @@ extern int nfs4_proc_layoutreturn(struct nfs4_layoutreturn *lrp, bool wait);
 /* pnfs.c */
 extern const nfs4_stateid zero_stateid;
 
+void put_lseg_locked(struct pnfs_layout_segment *);
+struct pnfs_layout_segment *pnfs_has_layout(struct pnfs_layout_hdr *,
+					    struct pnfs_layout_range *);
 void _pnfs_update_layout(struct inode *ino, struct nfs_open_context *ctx,
 	loff_t pos, u64 count, enum pnfs_iomode access_type,
 	struct pnfs_layout_segment **lsegpp);
include/linux/nfs_fs.h +1 -0
index 5bafa95..196c0c6 100644
@@ -213,6 +213,7 @@ struct nfs_inode {
 	/* pNFS layout information */
 #if defined(CONFIG_NFS_V4_1)
 	wait_queue_head_t lo_waitq;
+	struct rpc_wait_queue lo_rpcwaitq;
 	struct pnfs_layout_hdr *layout;
 	time_t pnfs_layout_suspend;
 #endif /* CONFIG_NFS_V4_1 */
Signature · 1 line
1.7.2.2
#7 lore raw

PATCH 3/4 pnfs: allow nfs4_proc_layoutreturn to sleep on barrier

In reply to Benny Halevy

use lo_rpcwaitq and sleep on it in the rpc prepare phase while there's need to wait on the layoutreturn barrier.

---
 fs/nfs/nfs4proc.c |    8 ++++++++
 fs/nfs/pnfs.c     |   12 +-----------
 fs/nfs/pnfs.h     |    1 +
 3 files changed, 10 insertions(+), 11 deletions(-)
fs/nfs/nfs4proc.c +8 -0
index f12568d..b21a711 100644
@@ -5683,9 +5683,17 @@ nfs4_layoutreturn_prepare(struct rpc_task *task, void *calldata)
 {
 	struct nfs4_layoutreturn *lrp = calldata;
 	struct inode *ino = lrp->args.inode;
+	struct nfs_inode *nfsi = NFS_I(ino);
 	struct nfs_server *server = NFS_SERVER(ino);
 
 	dprintk("--> %s\n", __func__);
+	if ((lrp->args.return_type == RETURN_FILE) &&
+	    pnfs_return_layout_barrier(nfsi, &lrp->args.range)) {
+		dprintk("%s: waiting on barrier\n", __func__);
+		rpc_sleep_on(&nfsi->lo_rpcwaitq, task, NULL);
+		return;
+	}
+
 	if (nfs4_setup_sequence(server, NULL, &lrp->args.seq_args,
 				&lrp->res.seq_res, 0, task))
 		return;
fs/nfs/pnfs.c +1 -11
index 2450383..0e8bb6f 100644
@@ -714,7 +714,7 @@ pnfs_free_layout(struct pnfs_layout_hdr *lo,
 	dprintk("%s:Return\n", __func__);
 }
 
-static bool
+bool
 pnfs_return_layout_barrier(struct nfs_inode *nfsi,
 			   struct pnfs_layout_range *range)
 {
@@ -804,16 +804,6 @@ _pnfs_return_layout(struct inode *ino, struct pnfs_layout_range *range,
 
 		spin_unlock(&ino->i_lock);
 
-		if (pnfs_return_layout_barrier(nfsi, &arg)) {
-			if (stateid) { /* callback */
-				status = -EAGAIN;
-				goto out_put;
-			}
-			dprintk("%s: waiting\n", __func__);
-			wait_event(nfsi->lo_waitq,
-				   !pnfs_return_layout_barrier(nfsi, &arg));
-		}
-
 		if (layoutcommit_needed(nfsi)) {
 			if (stateid && !wait) { /* callback */
 				dprintk("%s: layoutcommit pending\n", __func__);
fs/nfs/pnfs.h +1 -0
index f7f567e..81534aa 100644
@@ -41,6 +41,7 @@ void _pnfs_update_layout(struct inode *ino, struct nfs_open_context *ctx,
 	loff_t pos, u64 count, enum pnfs_iomode access_type,
 	struct pnfs_layout_segment **lsegpp);
 
+bool pnfs_return_layout_barrier(struct nfs_inode *, struct pnfs_layout_range *);
 int _pnfs_return_layout(struct inode *, struct pnfs_layout_range *,
 			const nfs4_stateid *stateid, /* optional */
 			enum pnfs_layoutreturn_type, bool wait);
Signature · 1 line
1.7.2.2
#8 lore raw

PATCH 4/4 SQUASHME: pnfs: get rid of lo_waitq

In reply to Benny Halevy

it has no more users at this point.

SPLITME: refactor put_lseg{,_locked}

---
 fs/nfs/inode.c         |    1 -
 fs/nfs/pnfs.c          |   32 ++++++++++++++------------------
 include/linux/nfs_fs.h |    1 -
 3 files changed, 14 insertions(+), 20 deletions(-)
fs/nfs/inode.c +0 -1
index 059bdf7..8e5fbb5 100644
@@ -1461,7 +1461,6 @@ static inline void nfs4_init_once(struct nfs_inode *nfsi)
 	nfsi->delegation_state = 0;
 	init_rwsem(&nfsi->rwsem);
 #ifdef CONFIG_NFS_V4_1
-	init_waitqueue_head(&nfsi->lo_waitq);
 	rpc_init_wait_queue(&nfsi->lo_rpcwaitq, "pNFS Layout");
 	nfsi->pnfs_layout_suspend = 0;
 	nfsi->layout = NULL;
fs/nfs/pnfs.c +14 -18
index 0e8bb6f..e6261a3 100644
@@ -360,7 +360,6 @@ pnfs_layout_release(struct pnfs_layout_hdr *lo,
 	 */
 	put_layout_locked(lo);
 	spin_unlock(&nfsi->vfs_inode.i_lock);
-	wake_up_all(&nfsi->lo_waitq);
 }
 
 void
@@ -430,44 +429,41 @@ destroy_lseg(struct kref *kref)
 	PNFS_LD_IO_OPS(lseg->layout)->free_lseg(lseg);
 }
 
-void
-put_lseg_locked(struct pnfs_layout_segment *lseg)
+static void
+put_lseg_common(struct nfs_inode *nfsi, struct pnfs_layout_segment *lseg)
 {
 	bool do_wake_up;
-	struct nfs_inode *nfsi;
-
-	if (!lseg)
-		return;
 
 	dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg,
 		atomic_read(&lseg->kref.refcount), lseg->valid);
 	do_wake_up = !lseg->valid;
-	nfsi = PNFS_NFS_INODE(lseg->layout);
 	kref_put(&lseg->kref, destroy_lseg);
-	if (do_wake_up) {
-		wake_up(&nfsi->lo_waitq);
+	if (do_wake_up)
 		rpc_wake_up(&nfsi->lo_rpcwaitq);
-	}
+}
+
+void
+put_lseg_locked(struct pnfs_layout_segment *lseg)
+{
+	if (!lseg)
+		return;
+
+	BUG_ON_UNLOCKED_LO(lseg->layout);
+	put_lseg_common(PNFS_NFS_INODE(lseg->layout), lseg);
 }
 
 void
 put_lseg(struct pnfs_layout_segment *lseg)
 {
-	bool do_wake_up;
 	struct nfs_inode *nfsi;
 
 	if (!lseg)
 		return;
 
-	dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg,
-		atomic_read(&lseg->kref.refcount), lseg->valid);
-	do_wake_up = !lseg->valid;
 	nfsi = PNFS_NFS_INODE(lseg->layout);
 	spin_lock(&nfsi->vfs_inode.i_lock);
-	kref_put(&lseg->kref, destroy_lseg);
+	put_lseg_common(nfsi, lseg);
 	spin_unlock(&nfsi->vfs_inode.i_lock);
-	if (do_wake_up)
-		wake_up(&nfsi->lo_waitq);
 }
 EXPORT_SYMBOL(put_lseg);
 
include/linux/nfs_fs.h +0 -1
index 196c0c6..745d14e 100644
@@ -212,7 +212,6 @@ struct nfs_inode {
 
 	/* pNFS layout information */
 #if defined(CONFIG_NFS_V4_1)
-	wait_queue_head_t lo_waitq;
 	struct rpc_wait_queue lo_rpcwaitq;
 	struct pnfs_layout_hdr *layout;
 	time_t pnfs_layout_suspend;
Signature · 1 line
1.7.2.2
#3 lore raw

Re: [PATCH 2/4] pnfs: allow nfs4_proc_layoutget to sleep on invalid lsegs

In reply to Benny Halevy

Note that this waits on any range that overlaps both an outstanding LAYOUTRETURN and a current layout. Whereas what you really want is to wait on any range that overlaps an outstanding LAYOUTRETURN. One way to get the desired behavior would be to have LAYOUTRETURN insert an "invalid" lseg covering its range.

Fred

Quoted 177 lines · Benny Halevy

Introduce lo_rpcwaitq and sleep on it while there's an lseg marked invalid, i.e. it is being returned (in the future, it could also be layoutget in progress).

---
 fs/nfs/inode.c         |    1 +
 fs/nfs/nfs4proc.c      |   25 ++++++++++++++++++++++--- fs/nfs/pnfs.c          |   37 ++++++++++++++++++------------------- fs/nfs/pnfs.h          |    3 +++
 include/linux/nfs_fs.h |    1 +

 5 files changed, 45 insertions(+), 22 deletions(-)
fs/nfs/inode.c +1 -0
index 7621cfc..059bdf7 100644
@@ -1462,6 +1462,7 @@ static inline void nfs4_init_once(struct nfs_inode *nfsi)
        init_rwsem(&nfsi->rwsem);
  #ifdef CONFIG_NFS_V4_1
        init_waitqueue_head(&nfsi->lo_waitq);
+       rpc_init_wait_queue(&nfsi->lo_rpcwaitq, "pNFS Layout");
        nfsi->pnfs_layout_suspend = 0;
        nfsi->layout = NULL;
  #endif /* CONFIG_NFS_V4_1 */
fs/nfs/nfs4proc.c +22 -3
index 48a763e..f12568d 100644
@@ -5442,13 +5442,32 @@ nfs4_layoutget_prepare(struct rpc_task *task, void *calldata)
  {
        struct nfs4_layoutget *lgp = calldata;
        struct inode *ino = lgp->args.inode;
+       struct nfs_inode *nfsi = NFS_I(ino);
        struct nfs_server *server = NFS_SERVER(ino);
+       struct pnfs_layout_segment *lseg;
 
        dprintk("--> %s\n", __func__);
-       if (nfs4_setup_sequence(server, NULL, &lgp->args.seq_args,
-                               &lgp->res.seq_res, 0, task))
+       spin_lock(&ino->i_lock);
+       lseg = pnfs_has_layout(nfsi->layout, &lgp->args.range);
+       if (likely(!lseg)) {
+               spin_unlock(&ino->i_lock);
+               dprintk("%s: no lseg found, proceeding\n", __func__);
+               if (!nfs4_setup_sequence(server, NULL, &lgp->args.seq_args,
+                                        &lgp->res.seq_res, 0, task))
+                       rpc_call_start(task);
                return;
-       rpc_call_start(task);
+       }
+       if (!lseg->valid) {
+               put_lseg_locked(lseg);
+               spin_unlock(&ino->i_lock);
+               dprintk("%s: invalid lseg found, waiting\n", __func__);
+               rpc_sleep_on(&nfsi->lo_rpcwaitq, task, NULL);
+               return;
+       }
+       *lgp->lsegpp = lseg;
+       spin_unlock(&ino->i_lock);
+       dprintk("%s: valid lseg found, no rpc required\n", __func__);
+       rpc_exit(task, NFS4_OK);
  }
 
  static void nfs4_layoutget_done(struct rpc_task *task, void *calldata)
fs/nfs/pnfs.c +4 -2
index bc4f0c1..2450383 100644
@@ -430,7 +430,7 @@ destroy_lseg(struct kref *kref)
        PNFS_LD_IO_OPS(lseg->layout)->free_lseg(lseg);
  }
 
-static void
+void
  put_lseg_locked(struct pnfs_layout_segment *lseg)
  {
        bool do_wake_up;
@@ -444,8 +444,10 @@ put_lseg_locked(struct pnfs_layout_segment *lseg)
        do_wake_up = !lseg->valid;
        nfsi = PNFS_NFS_INODE(lseg->layout);
        kref_put(&lseg->kref, destroy_lseg);
-       if (do_wake_up)
+       if (do_wake_up) {
                wake_up(&nfsi->lo_waitq);
+               rpc_wake_up(&nfsi->lo_rpcwaitq);
+       }
  }
 

 void

fs/nfs/pnfs.c +13 -15
@@ -999,7 +1001,7 @@ has_matching_lseg(struct pnfs_layout_segment *lseg,
  /*
  * lookup range in layout
  */
-static struct pnfs_layout_segment *
+struct pnfs_layout_segment *
  pnfs_has_layout(struct pnfs_layout_hdr *lo,
                struct pnfs_layout_range *range)
  {
@@ -1057,22 +1059,20 @@ _pnfs_update_layout(struct inode *ino,
 
        /* Check to see if the layout for the given range already exists */
        lseg = pnfs_has_layout(lo, &arg);
-       if (lseg && !lseg->valid) {
-               put_lseg_locked(lseg);
-               /* someone is cleaning the layout */
-               lseg = NULL;
-               goto out_unlock;
-       }
-
        if (lseg) {
-               dprintk("%s: Using cached lseg %p for %llu@%llu iomode %d)\n",
-                       __func__,
-                       lseg,
-                       arg.length,
-                       arg.offset,
-                       arg.iomode);
+               if (lseg->valid) {
+                       dprintk("%s: Using cached lseg %p for %llu@%llu "
+                               "iomode %d)\n",
+                               __func__,
+                               lseg,
+                               arg.length,
+                               arg.offset,
+                               arg.iomode);
 
-               goto out_unlock;
+                       goto out_unlock;
+               }
+               /* someone is cleaning the layout */
+               put_lseg_locked(lseg);
        }
 
        /* if get layout already failed once goto out */
@@ -1097,8 +1097,7 @@ out:
                nfsi->layout->state, lseg);
        return;

 out_unlock:
-       if (lsegpp)
-               *lsegpp = lseg;
+       *lsegpp = lseg;
       spin_unlock(&ino->i_lock);
       goto out;
 }

fs/nfs/pnfs.h +3 -0
index ea70385..f7f567e 100644
@@ -34,6 +34,9 @@ extern int nfs4_proc_layoutreturn(struct nfs4_layoutreturn *lrp, bool wait);
  /* pnfs.c */
  extern const nfs4_stateid zero_stateid;
 
+void put_lseg_locked(struct pnfs_layout_segment *);
+struct pnfs_layout_segment *pnfs_has_layout(struct pnfs_layout_hdr *,
+                                           struct pnfs_layout_range *);
  void _pnfs_update_layout(struct inode *ino, struct nfs_open_context *ctx,
        loff_t pos, u64 count, enum pnfs_iomode access_type,
        struct pnfs_layout_segment **lsegpp);
include/linux/nfs_fs.h +1 -1
index 5bafa95..196c0c6 100644
@@ -213,6 +213,7 @@ struct nfs_inode {
        /* pNFS layout information */
  #if defined(CONFIG_NFS_V4_1)
        wait_queue_head_t lo_waitq;
+       struct rpc_wait_queue lo_rpcwaitq;
        struct pnfs_layout_hdr *layout;
        time_t pnfs_layout_suspend;
  #endif /* CONFIG_NFS_V4_1 */
--

1.7.2.2

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

#4 lore raw

Re: [PATCH 2/4] pnfs: allow nfs4_proc_layoutget to sleep on invalid lsegs

Quoted 5 lines · Fred Isaman

Note that this waits on any range that overlaps both an outstanding LAYOUTRETURN and a current layout. Whereas what you really want is to wait on any range that overlaps an outstanding LAYOUTRETURN. One way to get the desired behavior would be to have LAYOUTRETURN insert an "invalid" lseg covering its range.

What do you mean by "current layout"?
The way it works now is that layout return does mark the layout segments it's about to return as invalid (in pnfs_return_layout_barrier). layout get inserts a valid layout segment only upon completion (in pnfs_layout_process) and it does not yet insert an invalid one before sending the RPC, something we may wan to consider in the future to prevent superfluous parallel gets.

Benny

Quoted 182 lines

Fred

Quoted 177 lines · Benny Halevy

Introduce lo_rpcwaitq and sleep on it while there's an lseg marked invalid, i.e. it is being returned (in the future, it could also be layoutget in progress).

---
 fs/nfs/inode.c         |    1 +
 fs/nfs/nfs4proc.c      |   25 ++++++++++++++++++++++---
 fs/nfs/pnfs.c          |   37 ++++++++++++++++++-------------------
 fs/nfs/pnfs.h          |    3 +++
 include/linux/nfs_fs.h |    1 +
 5 files changed, 45 insertions(+), 22 deletions(-)
fs/nfs/inode.c +1 -0
index 7621cfc..059bdf7 100644
@@ -1462,6 +1462,7 @@ static inline void nfs4_init_once(struct nfs_inode *nfsi)
       init_rwsem(&nfsi->rwsem);
 #ifdef CONFIG_NFS_V4_1
       init_waitqueue_head(&nfsi->lo_waitq);
+       rpc_init_wait_queue(&nfsi->lo_rpcwaitq, "pNFS Layout");
       nfsi->pnfs_layout_suspend = 0;
       nfsi->layout = NULL;
 #endif /* CONFIG_NFS_V4_1 */
fs/nfs/nfs4proc.c +22 -3
index 48a763e..f12568d 100644
@@ -5442,13 +5442,32 @@ nfs4_layoutget_prepare(struct rpc_task *task, void *calldata)
 {
       struct nfs4_layoutget *lgp = calldata;
       struct inode *ino = lgp->args.inode;
+       struct nfs_inode *nfsi = NFS_I(ino);
       struct nfs_server *server = NFS_SERVER(ino);
+       struct pnfs_layout_segment *lseg;
 
       dprintk("--> %s\n", __func__);
-       if (nfs4_setup_sequence(server, NULL, &lgp->args.seq_args,
-                               &lgp->res.seq_res, 0, task))
+       spin_lock(&ino->i_lock);
+       lseg = pnfs_has_layout(nfsi->layout, &lgp->args.range);
+       if (likely(!lseg)) {
+               spin_unlock(&ino->i_lock);
+               dprintk("%s: no lseg found, proceeding\n", __func__);
+               if (!nfs4_setup_sequence(server, NULL, &lgp->args.seq_args,
+                                        &lgp->res.seq_res, 0, task))
+                       rpc_call_start(task);
               return;
-       rpc_call_start(task);
+       }
+       if (!lseg->valid) {
+               put_lseg_locked(lseg);
+               spin_unlock(&ino->i_lock);
+               dprintk("%s: invalid lseg found, waiting\n", __func__);
+               rpc_sleep_on(&nfsi->lo_rpcwaitq, task, NULL);
+               return;
+       }
+       *lgp->lsegpp = lseg;
+       spin_unlock(&ino->i_lock);
+       dprintk("%s: valid lseg found, no rpc required\n", __func__);
+       rpc_exit(task, NFS4_OK);
 }
 
 static void nfs4_layoutget_done(struct rpc_task *task, void *calldata)
fs/nfs/pnfs.c +18 -19
index bc4f0c1..2450383 100644
@@ -430,7 +430,7 @@ destroy_lseg(struct kref *kref)
       PNFS_LD_IO_OPS(lseg->layout)->free_lseg(lseg);
 }
 
-static void
+void
 put_lseg_locked(struct pnfs_layout_segment *lseg)
 {
       bool do_wake_up;
@@ -444,8 +444,10 @@ put_lseg_locked(struct pnfs_layout_segment *lseg)
       do_wake_up = !lseg->valid;
       nfsi = PNFS_NFS_INODE(lseg->layout);
       kref_put(&lseg->kref, destroy_lseg);
-       if (do_wake_up)
+       if (do_wake_up) {
               wake_up(&nfsi->lo_waitq);
+               rpc_wake_up(&nfsi->lo_rpcwaitq);
+       }
 }
 
 void
@@ -999,7 +1001,7 @@ has_matching_lseg(struct pnfs_layout_segment *lseg,
 /*
 * lookup range in layout
 */
-static struct pnfs_layout_segment *
+struct pnfs_layout_segment *
 pnfs_has_layout(struct pnfs_layout_hdr *lo,
               struct pnfs_layout_range *range)
 {
@@ -1057,22 +1059,20 @@ _pnfs_update_layout(struct inode *ino,
 
       /* Check to see if the layout for the given range already exists */
       lseg = pnfs_has_layout(lo, &arg);
-       if (lseg && !lseg->valid) {
-               put_lseg_locked(lseg);
-               /* someone is cleaning the layout */
-               lseg = NULL;
-               goto out_unlock;
-       }
-
       if (lseg) {
-               dprintk("%s: Using cached lseg %p for %llu@%llu iomode %d)\n",
-                       __func__,
-                       lseg,
-                       arg.length,
-                       arg.offset,
-                       arg.iomode);
+               if (lseg->valid) {
+                       dprintk("%s: Using cached lseg %p for %llu@%llu "
+                               "iomode %d)\n",
+                               __func__,
+                               lseg,
+                               arg.length,
+                               arg.offset,
+                               arg.iomode);
 
-               goto out_unlock;
+                       goto out_unlock;
+               }
+               /* someone is cleaning the layout */
+               put_lseg_locked(lseg);
       }
 
       /* if get layout already failed once goto out */
@@ -1097,8 +1097,7 @@ out:
               nfsi->layout->state, lseg);
       return;
 out_unlock:
-       if (lsegpp)
-               *lsegpp = lseg;
+       *lsegpp = lseg;
       spin_unlock(&ino->i_lock);
       goto out;
 }
fs/nfs/pnfs.h +3 -0
index ea70385..f7f567e 100644
@@ -34,6 +34,9 @@ extern int nfs4_proc_layoutreturn(struct nfs4_layoutreturn *lrp, bool wait);
 /* pnfs.c */
 extern const nfs4_stateid zero_stateid;
 
+void put_lseg_locked(struct pnfs_layout_segment *);
+struct pnfs_layout_segment *pnfs_has_layout(struct pnfs_layout_hdr *,
+                                           struct pnfs_layout_range *);
 void _pnfs_update_layout(struct inode *ino, struct nfs_open_context *ctx,
       loff_t pos, u64 count, enum pnfs_iomode access_type,
       struct pnfs_layout_segment **lsegpp);
include/linux/nfs_fs.h +1 -1
index 5bafa95..196c0c6 100644
@@ -213,6 +213,7 @@ struct nfs_inode {
       /* pNFS layout information */
 #if defined(CONFIG_NFS_V4_1)
       wait_queue_head_t lo_waitq;
+       struct rpc_wait_queue lo_rpcwaitq;
       struct pnfs_layout_hdr *layout;
       time_t pnfs_layout_suspend;
 #endif /* CONFIG_NFS_V4_1 */
--

1.7.2.2

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

#5 lore raw

Re: [PATCH 2/4] pnfs: allow nfs4_proc_layoutget to sleep on invalid lsegs

Quoted 8 lines · Benny Halevy
Quoted 5 lines · Fred Isaman

Note that this waits on any range that overlaps both an outstanding LAYOUTRETURN and a current layout.  Whereas what you really want is to wait on any range that overlaps an outstanding LAYOUTRETURN.  One way to get the desired behavior would be to have LAYOUTRETURN insert an "invalid" lseg covering its range.

What do you mean by "current layout"?

Consider the case where the server asks for a LAYOUTRETURN of the range 0-1000, but the client currently only has lsegs covering 0-500. The client should wait for the LAYOUTRETURN to complete before sending any LAYOUTGET for the range 500-1000, but there is currently no mechanism to do this.

Fred

Quoted 192 lines

The way it works now is that layout return does mark the layout segments it's about to return as invalid (in pnfs_return_layout_barrier). layout get inserts a valid layout segment only upon completion (in pnfs_layout_process) and it does not yet insert an invalid one before sending the RPC, something we may wan to consider in the future to prevent superfluous parallel gets.

Benny

Quoted 182 lines

Fred

Quoted 177 lines · Benny Halevy

Introduce lo_rpcwaitq and sleep on it while there's an lseg marked invalid, i.e. it is being returned (in the future, it could also be layoutget in progress).

---
 fs/nfs/inode.c         |    1 +
 fs/nfs/nfs4proc.c      |   25 ++++++++++++++++++++++--- fs/nfs/pnfs.c          |   37 ++++++++++++++++++------------------- fs/nfs/pnfs.h          |    3 +++
 include/linux/nfs_fs.h |    1 +

 5 files changed, 45 insertions(+), 22 deletions(-)
fs/nfs/inode.c +1 -0
index 7621cfc..059bdf7 100644
@@ -1462,6 +1462,7 @@ static inline void nfs4_init_once(struct nfs_inode *nfsi)
        init_rwsem(&nfsi->rwsem);
  #ifdef CONFIG_NFS_V4_1
        init_waitqueue_head(&nfsi->lo_waitq);
+       rpc_init_wait_queue(&nfsi->lo_rpcwaitq, "pNFS Layout");
        nfsi->pnfs_layout_suspend = 0;
        nfsi->layout = NULL;
  #endif /* CONFIG_NFS_V4_1 */
fs/nfs/nfs4proc.c +22 -3
index 48a763e..f12568d 100644
@@ -5442,13 +5442,32 @@ nfs4_layoutget_prepare(struct rpc_task *task, void *calldata)
  {
        struct nfs4_layoutget *lgp = calldata;
        struct inode *ino = lgp->args.inode;
+       struct nfs_inode *nfsi = NFS_I(ino);
        struct nfs_server *server = NFS_SERVER(ino);
+       struct pnfs_layout_segment *lseg;
 
        dprintk("--> %s\n", __func__);
-       if (nfs4_setup_sequence(server, NULL, &lgp->args.seq_args,
-                               &lgp->res.seq_res, 0, task))
+       spin_lock(&ino->i_lock);
+       lseg = pnfs_has_layout(nfsi->layout, &lgp->args.range);
+       if (likely(!lseg)) {
+               spin_unlock(&ino->i_lock);
+               dprintk("%s: no lseg found, proceeding\n", __func__);
+               if (!nfs4_setup_sequence(server, NULL, &lgp->args.seq_args,
+                                        &lgp->res.seq_res, 0, task))
+                       rpc_call_start(task);
                return;
-       rpc_call_start(task);
+       }
+       if (!lseg->valid) {
+               put_lseg_locked(lseg);
+               spin_unlock(&ino->i_lock);
+               dprintk("%s: invalid lseg found, waiting\n", __func__);
+               rpc_sleep_on(&nfsi->lo_rpcwaitq, task, NULL);
+               return;
+       }
+       *lgp->lsegpp = lseg;
+       spin_unlock(&ino->i_lock);
+       dprintk("%s: valid lseg found, no rpc required\n", __func__);
+       rpc_exit(task, NFS4_OK);
  }
 
  static void nfs4_layoutget_done(struct rpc_task *task, void *calldata)
fs/nfs/pnfs.c +4 -2
index bc4f0c1..2450383 100644
@@ -430,7 +430,7 @@ destroy_lseg(struct kref *kref)
        PNFS_LD_IO_OPS(lseg->layout)->free_lseg(lseg);
  }
 
-static void
+void
  put_lseg_locked(struct pnfs_layout_segment *lseg)
  {
        bool do_wake_up;
@@ -444,8 +444,10 @@ put_lseg_locked(struct pnfs_layout_segment *lseg)
        do_wake_up = !lseg->valid;
        nfsi = PNFS_NFS_INODE(lseg->layout);
        kref_put(&lseg->kref, destroy_lseg);
-       if (do_wake_up)
+       if (do_wake_up) {
                wake_up(&nfsi->lo_waitq);
+               rpc_wake_up(&nfsi->lo_rpcwaitq);
+       }
  }
 

 void

fs/nfs/pnfs.c +13 -15
@@ -999,7 +1001,7 @@ has_matching_lseg(struct pnfs_layout_segment *lseg,
  /*
  * lookup range in layout
  */
-static struct pnfs_layout_segment *
+struct pnfs_layout_segment *
  pnfs_has_layout(struct pnfs_layout_hdr *lo,
                struct pnfs_layout_range *range)
  {
@@ -1057,22 +1059,20 @@ _pnfs_update_layout(struct inode *ino,
 
        /* Check to see if the layout for the given range already exists */
        lseg = pnfs_has_layout(lo, &arg);
-       if (lseg && !lseg->valid) {
-               put_lseg_locked(lseg);
-               /* someone is cleaning the layout */
-               lseg = NULL;
-               goto out_unlock;
-       }
-
        if (lseg) {
-               dprintk("%s: Using cached lseg %p for %llu@%llu iomode %d)\n",
-                       __func__,
-                       lseg,
-                       arg.length,
-                       arg.offset,
-                       arg.iomode);
+               if (lseg->valid) {
+                       dprintk("%s: Using cached lseg %p for %llu@%llu "
+                               "iomode %d)\n",
+                               __func__,
+                               lseg,
+                               arg.length,
+                               arg.offset,
+                               arg.iomode);
 
-               goto out_unlock;
+                       goto out_unlock;
+               }
+               /* someone is cleaning the layout */
+               put_lseg_locked(lseg);
        }
 
        /* if get layout already failed once goto out */
@@ -1097,8 +1097,7 @@ out:
                nfsi->layout->state, lseg);
        return;

 out_unlock:
-       if (lsegpp)
-               *lsegpp = lseg;
+       *lsegpp = lseg;
       spin_unlock(&ino->i_lock);
       goto out;
 }

fs/nfs/pnfs.h +3 -0
index ea70385..f7f567e 100644
@@ -34,6 +34,9 @@ extern int nfs4_proc_layoutreturn(struct nfs4_layoutreturn *lrp, bool wait);
  /* pnfs.c */
  extern const nfs4_stateid zero_stateid;
 
+void put_lseg_locked(struct pnfs_layout_segment *);
+struct pnfs_layout_segment *pnfs_has_layout(struct pnfs_layout_hdr *,
+                                           struct pnfs_layout_range *);
  void _pnfs_update_layout(struct inode *ino, struct nfs_open_context *ctx,
        loff_t pos, u64 count, enum pnfs_iomode access_type,
        struct pnfs_layout_segment **lsegpp);
include/linux/nfs_fs.h +1 -1
index 5bafa95..196c0c6 100644
@@ -213,6 +213,7 @@ struct nfs_inode {
        /* pNFS layout information */
  #if defined(CONFIG_NFS_V4_1)
        wait_queue_head_t lo_waitq;
+       struct rpc_wait_queue lo_rpcwaitq;
        struct pnfs_layout_hdr *layout;
        time_t pnfs_layout_suspend;
  #endif /* CONFIG_NFS_V4_1 */
--

1.7.2.2

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

#6 lore raw

Re: [PATCH 2/4] pnfs: allow nfs4_proc_layoutget to sleep on invalid lsegs

Quoted 8 lines · Fred Isaman
Quoted 7 lines · Benny Halevy
Quoted 5 lines · Fred Isaman

Note that this waits on any range that overlaps both an outstanding LAYOUTRETURN and a current layout. Whereas what you really want is to wait on any range that overlaps an outstanding LAYOUTRETURN. One way to get the desired behavior would be to have LAYOUTRETURN insert an "invalid" lseg covering its range.

Quoted 8 lines
Quoted 1 line

What do you mean by "current layout"?

Consider the case where the server asks for a LAYOUTRETURN of the range 0-1000, but the client currently only has lsegs covering 0-500. The client should wait for the LAYOUTRETURN to complete before sending any LAYOUTGET for the range 500-1000, but there is currently no mechanism to do this.

Yes, this could be yet another optimization to avoid getting NFS4ERR_RECALLCONFLICT.

Benny

Quoted 194 lines

Fred

Quoted 192 lines

The way it works now is that layout return does mark the layout segments it's about to return as invalid (in pnfs_return_layout_barrier). layout get inserts a valid layout segment only upon completion (in pnfs_layout_process) and it does not yet insert an invalid one before sending the RPC, something we may wan to consider in the future to prevent superfluous parallel gets.

Benny

Quoted 182 lines

Fred

Quoted 177 lines · Benny Halevy

Introduce lo_rpcwaitq and sleep on it while there's an lseg marked invalid, i.e. it is being returned (in the future, it could also be layoutget in progress).

---
 fs/nfs/inode.c         |    1 +
 fs/nfs/nfs4proc.c      |   25 ++++++++++++++++++++++---
 fs/nfs/pnfs.c          |   37 ++++++++++++++++++-------------------
 fs/nfs/pnfs.h          |    3 +++
 include/linux/nfs_fs.h |    1 +
 5 files changed, 45 insertions(+), 22 deletions(-)
fs/nfs/inode.c +1 -0
index 7621cfc..059bdf7 100644
@@ -1462,6 +1462,7 @@ static inline void nfs4_init_once(struct nfs_inode *nfsi)
       init_rwsem(&nfsi->rwsem);
 #ifdef CONFIG_NFS_V4_1
       init_waitqueue_head(&nfsi->lo_waitq);
+       rpc_init_wait_queue(&nfsi->lo_rpcwaitq, "pNFS Layout");
       nfsi->pnfs_layout_suspend = 0;
       nfsi->layout = NULL;
 #endif /* CONFIG_NFS_V4_1 */
fs/nfs/nfs4proc.c +22 -3
index 48a763e..f12568d 100644
@@ -5442,13 +5442,32 @@ nfs4_layoutget_prepare(struct rpc_task *task, void *calldata)
 {
       struct nfs4_layoutget *lgp = calldata;
       struct inode *ino = lgp->args.inode;
+       struct nfs_inode *nfsi = NFS_I(ino);
       struct nfs_server *server = NFS_SERVER(ino);
+       struct pnfs_layout_segment *lseg;
 
       dprintk("--> %s\n", __func__);
-       if (nfs4_setup_sequence(server, NULL, &lgp->args.seq_args,
-                               &lgp->res.seq_res, 0, task))
+       spin_lock(&ino->i_lock);
+       lseg = pnfs_has_layout(nfsi->layout, &lgp->args.range);
+       if (likely(!lseg)) {
+               spin_unlock(&ino->i_lock);
+               dprintk("%s: no lseg found, proceeding\n", __func__);
+               if (!nfs4_setup_sequence(server, NULL, &lgp->args.seq_args,
+                                        &lgp->res.seq_res, 0, task))
+                       rpc_call_start(task);
               return;
-       rpc_call_start(task);
+       }
+       if (!lseg->valid) {
+               put_lseg_locked(lseg);
+               spin_unlock(&ino->i_lock);
+               dprintk("%s: invalid lseg found, waiting\n", __func__);
+               rpc_sleep_on(&nfsi->lo_rpcwaitq, task, NULL);
+               return;
+       }
+       *lgp->lsegpp = lseg;
+       spin_unlock(&ino->i_lock);
+       dprintk("%s: valid lseg found, no rpc required\n", __func__);
+       rpc_exit(task, NFS4_OK);
 }
 
 static void nfs4_layoutget_done(struct rpc_task *task, void *calldata)
fs/nfs/pnfs.c +18 -19
index bc4f0c1..2450383 100644
@@ -430,7 +430,7 @@ destroy_lseg(struct kref *kref)
       PNFS_LD_IO_OPS(lseg->layout)->free_lseg(lseg);
 }
 
-static void
+void
 put_lseg_locked(struct pnfs_layout_segment *lseg)
 {
       bool do_wake_up;
@@ -444,8 +444,10 @@ put_lseg_locked(struct pnfs_layout_segment *lseg)
       do_wake_up = !lseg->valid;
       nfsi = PNFS_NFS_INODE(lseg->layout);
       kref_put(&lseg->kref, destroy_lseg);
-       if (do_wake_up)
+       if (do_wake_up) {
               wake_up(&nfsi->lo_waitq);
+               rpc_wake_up(&nfsi->lo_rpcwaitq);
+       }
 }
 
 void
@@ -999,7 +1001,7 @@ has_matching_lseg(struct pnfs_layout_segment *lseg,
 /*
 * lookup range in layout
 */
-static struct pnfs_layout_segment *
+struct pnfs_layout_segment *
 pnfs_has_layout(struct pnfs_layout_hdr *lo,
               struct pnfs_layout_range *range)
 {
@@ -1057,22 +1059,20 @@ _pnfs_update_layout(struct inode *ino,
 
       /* Check to see if the layout for the given range already exists */
       lseg = pnfs_has_layout(lo, &arg);
-       if (lseg && !lseg->valid) {
-               put_lseg_locked(lseg);
-               /* someone is cleaning the layout */
-               lseg = NULL;
-               goto out_unlock;
-       }
-
       if (lseg) {
-               dprintk("%s: Using cached lseg %p for %llu@%llu iomode %d)\n",
-                       __func__,
-                       lseg,
-                       arg.length,
-                       arg.offset,
-                       arg.iomode);
+               if (lseg->valid) {
+                       dprintk("%s: Using cached lseg %p for %llu@%llu "
+                               "iomode %d)\n",
+                               __func__,
+                               lseg,
+                               arg.length,
+                               arg.offset,
+                               arg.iomode);
 
-               goto out_unlock;
+                       goto out_unlock;
+               }
+               /* someone is cleaning the layout */
+               put_lseg_locked(lseg);
       }
 
       /* if get layout already failed once goto out */
@@ -1097,8 +1097,7 @@ out:
               nfsi->layout->state, lseg);
       return;
 out_unlock:
-       if (lsegpp)
-               *lsegpp = lseg;
+       *lsegpp = lseg;
       spin_unlock(&ino->i_lock);
       goto out;
 }
fs/nfs/pnfs.h +3 -0
index ea70385..f7f567e 100644
@@ -34,6 +34,9 @@ extern int nfs4_proc_layoutreturn(struct nfs4_layoutreturn *lrp, bool wait);
 /* pnfs.c */
 extern const nfs4_stateid zero_stateid;
 
+void put_lseg_locked(struct pnfs_layout_segment *);
+struct pnfs_layout_segment *pnfs_has_layout(struct pnfs_layout_hdr *,
+                                           struct pnfs_layout_range *);
 void _pnfs_update_layout(struct inode *ino, struct nfs_open_context *ctx,
       loff_t pos, u64 count, enum pnfs_iomode access_type,
       struct pnfs_layout_segment **lsegpp);
include/linux/nfs_fs.h +1 -1
index 5bafa95..196c0c6 100644
@@ -213,6 +213,7 @@ struct nfs_inode {
       /* pNFS layout information */
 #if defined(CONFIG_NFS_V4_1)
       wait_queue_head_t lo_waitq;
+       struct rpc_wait_queue lo_rpcwaitq;
       struct pnfs_layout_hdr *layout;
       time_t pnfs_layout_suspend;
 #endif /* CONFIG_NFS_V4_1 */
--

1.7.2.2

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html