Skip to contentSkip to conversation

pnfs-submit: embed pnfs_layout_type in per layout structure

PATCH In review [email protected] 3 participants 18 messages Jun 29, 2010 – Jun 30, 2010

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

cover pnfs-submit: embed pnfs_layout_type in per layout structure

A cleanup from Fred. 0001-SQUASHME-pnfs-submit-add-state-flag-for-layoutcommit.patch

These patches introduce the allocation of the pnfs_layout_type changing the static declaration in struct nfs_inode to a pointer.

0002-SQUASHME-pnfs_submit-move-pnfs_layout_state-back-to-.patch 0003-SQUASHME-pnfs-submit-move-pnfs_layout_suspend-back-t.patch 0004-SQUASHME-pnfs-submit-embed-pnfs_layout_type.patch 0005-SQUASHME-pnfs-submit-filelayout-use-new-alloc_layout.patch

Testing

CONFIG_NFS_V4_1 set:
Connectathon passes on a VM SMP client against GFS2/pNFS with return_on_close and write layouts turned on, and against the pyNFS file layout server without return on close set.

CONFIG_NFS_V4_1 not set
NFSv4.0 Cthon tests pass

-->Andy

#1 lore raw

PATCH 1/5 SQUASHME pnfs-submit: add state flag for layoutcommit_needed

From: Fred Isaman <[email protected]>

This avoids locking and existance of layout issues

---
 fs/nfs/pnfs.c             |    2 ++
 include/linux/nfs4_pnfs.h |    2 +-
 include/linux/nfs_fs.h    |    1 +
 3 files changed, 4 insertions(+), 1 deletions(-)
fs/nfs/pnfs.c +2 -0
index 9a295cf..bf15b5c 100644
@@ -154,6 +154,7 @@ pnfs_need_layoutcommit(struct nfs_inode *nfsi, struct nfs_open_context *ctx)
 	spin_lock(&nfsi->vfs_inode.i_lock);
 	if (has_layout(nfsi) && !layoutcommit_needed(nfsi)) {
 		nfsi->layout.lo_cred = get_rpccred(ctx->state->owner->so_cred);
+		__set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->pnfs_layout_state);
 		nfsi->change_attr++;
 		spin_unlock(&nfsi->vfs_inode.i_lock);
 		dprintk("%s: Set layoutcommit\n", __func__);
@@ -1635,6 +1636,7 @@ pnfs_layoutcommit_inode(struct inode *inode, int sync)
 	nfsi->layout.pnfs_write_begin_pos = 0;
 	nfsi->layout.pnfs_write_end_pos = 0;
 	nfsi->layout.lo_cred = NULL;
+	__clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->pnfs_layout_state);
 	pnfs_get_layout_stateid(&data->args.stateid, &nfsi->layout);
 
 	spin_unlock(&inode->i_lock);
include/linux/nfs4_pnfs.h +1 -1
index d20b5de..4d9b15c 100644
@@ -83,7 +83,7 @@ has_layout(struct nfs_inode *nfsi)
 static inline bool
 layoutcommit_needed(struct nfs_inode *nfsi)
 {
-	return nfsi->layout.lo_cred != NULL;
+	return test_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->pnfs_layout_state);
 }
 
 #else /* CONFIG_NFS_V4_1 */
include/linux/nfs_fs.h +1 -0
index a60d880..a33e86e 100644
@@ -109,6 +109,7 @@ struct pnfs_layout_type {
 	#define NFS_INO_RO_LAYOUT_FAILED 0      /* get ro layout failed stop trying */
 	#define NFS_INO_RW_LAYOUT_FAILED 1      /* get rw layout failed stop trying */
 	#define NFS_INO_LAYOUT_ALLOC     2      /* bit lock for layout allocation */
+	#define NFS_INO_LAYOUTCOMMIT     3	/* LAYOUTCOMMIT needed */
 	time_t pnfs_layout_suspend;
 	struct rpc_cred         *lo_cred; /* layoutcommit credential */
 	/* DH: These vars keep track of the maximum write range
Signature · 1 line
1.6.6
#2 lore raw

PATCH 2/5 SQUASHME: pnfs_submit: move pnfs_layout_state back to nfs_inode

From: Andy Adamson <[email protected]>

Prepare to embed struct pnfs_layout_tupe into layout driver private structs and to make layout a pointer: state flags need to be accessible before allocation.

---
 fs/nfs/inode.c         |    2 +-
 fs/nfs/pnfs.c          |   16 ++++++++--------
 include/linux/nfs_fs.h |   10 +++++-----
 3 files changed, 14 insertions(+), 14 deletions(-)
fs/nfs/inode.c +1 -1
index 7989dea..66d7be2 100644
@@ -1364,7 +1364,6 @@ void nfs4_clear_inode(struct inode *inode)
 static void pnfs_alloc_init_inode(struct nfs_inode *nfsi)
 {
 #ifdef CONFIG_NFS_V4_1
-	nfsi->layout.pnfs_layout_state = 0;
 	memset(&nfsi->layout.stateid, 0, NFS4_STATEID_SIZE);
 	nfsi->layout.roc_iomode = 0;
 	nfsi->layout.lo_cred = NULL;
@@ -1420,6 +1419,7 @@ static void pnfs_init_once(struct nfs_inode *nfsi)
 {
 #ifdef CONFIG_NFS_V4_1
 	init_waitqueue_head(&nfsi->lo_waitq);
+	nfsi->pnfs_layout_state = 0;
 	seqlock_init(&nfsi->layout.seqlock);
 	INIT_LIST_HEAD(&nfsi->layout.lo_layouts);
 	INIT_LIST_HEAD(&nfsi->layout.segs);
fs/nfs/pnfs.c +8 -8
index bf15b5c..d05cb03 100644
@@ -951,7 +951,7 @@ get_lock_alloc_layout(struct inode *ino)
 		 * wait until bit is cleared if we lost this race.
 		 */
 		res = wait_on_bit_lock(
-			&nfsi->layout.pnfs_layout_state,
+			&nfsi->pnfs_layout_state,
 			NFS_INO_LAYOUT_ALLOC,
 			pnfs_wait_schedule, TASK_KILLABLE);
 		if (res) {
@@ -975,8 +975,8 @@ get_lock_alloc_layout(struct inode *ino)
 
 		/* release the NFS_INO_LAYOUT_ALLOC bit and wake up waiters */
 		clear_bit_unlock(NFS_INO_LAYOUT_ALLOC,
-				 &nfsi->layout.pnfs_layout_state);
-		wake_up_bit(&nfsi->layout.pnfs_layout_state,
+				 &nfsi->pnfs_layout_state);
+		wake_up_bit(&nfsi->pnfs_layout_state,
 			    NFS_INO_LAYOUT_ALLOC);
 		break;
 	}
@@ -1104,12 +1104,12 @@ _pnfs_update_layout(struct inode *ino,
 	}
 
 	/* if get layout already failed once goto out */
-	if (test_bit(lo_fail_bit(iomode), &nfsi->layout.pnfs_layout_state)) {
+	if (test_bit(lo_fail_bit(iomode), &nfsi->pnfs_layout_state)) {
 		if (unlikely(nfsi->layout.pnfs_layout_suspend &&
 		    get_seconds() >= nfsi->layout.pnfs_layout_suspend)) {
 			dprintk("%s: layout_get resumed\n", __func__);
 			clear_bit(lo_fail_bit(iomode),
-				  &nfsi->layout.pnfs_layout_state);
+				  &nfsi->pnfs_layout_state);
 			nfsi->layout.pnfs_layout_suspend = 0;
 		} else
 			goto out_put;
@@ -1121,7 +1121,7 @@ _pnfs_update_layout(struct inode *ino,
 	send_layoutget(ino, ctx, &arg, lsegpp, lo);
 out:
 	dprintk("%s end, state 0x%lx lseg %p\n", __func__,
-		nfsi->layout.pnfs_layout_state, lseg);
+		nfsi->pnfs_layout_state, lseg);
 	return;
 out_put:
 	if (lsegpp)
@@ -1226,12 +1226,12 @@ pnfs_get_layout_done(struct nfs4_pnfs_layoutget *lgp, int rpc_status)
 	/* remember that get layout failed and suspend trying */
 	nfsi->layout.pnfs_layout_suspend = suspend;
 	set_bit(lo_fail_bit(lgp->args.lseg.iomode),
-		&nfsi->layout.pnfs_layout_state);
+		&nfsi->pnfs_layout_state);
 	dprintk("%s: layout_get suspended until %ld\n",
 		__func__, suspend);
 out:
 	dprintk("%s end (err:%d) state 0x%lx lseg %p\n",
-		__func__, lgp->status, nfsi->layout.pnfs_layout_state, lseg);
+		__func__, lgp->status, nfsi->pnfs_layout_state, lseg);
 	return;
 }
 
include/linux/nfs_fs.h +5 -5
index a33e86e..e216e24 100644
@@ -105,11 +105,6 @@ struct pnfs_layout_type {
 	seqlock_t seqlock;		/* Protects the stateid */
 	nfs4_stateid stateid;
 	void *ld_data;			/* layout driver private data */
-	unsigned long pnfs_layout_state;
-	#define NFS_INO_RO_LAYOUT_FAILED 0      /* get ro layout failed stop trying */
-	#define NFS_INO_RW_LAYOUT_FAILED 1      /* get rw layout failed stop trying */
-	#define NFS_INO_LAYOUT_ALLOC     2      /* bit lock for layout allocation */
-	#define NFS_INO_LAYOUTCOMMIT     3	/* LAYOUTCOMMIT needed */
 	time_t pnfs_layout_suspend;
 	struct rpc_cred         *lo_cred; /* layoutcommit credential */
 	/* DH: These vars keep track of the maximum write range
@@ -208,6 +203,11 @@ struct nfs_inode {
 #if defined(CONFIG_NFS_V4_1)
 	wait_queue_head_t lo_waitq;
 	struct pnfs_layout_type layout;
+	unsigned long pnfs_layout_state;
+	#define NFS_INO_RO_LAYOUT_FAILED 0 /* ro LAYOUTGET failed stop trying */
+	#define NFS_INO_RW_LAYOUT_FAILED 1 /* rw LAYOUTGET failed stop trying */
+	#define NFS_INO_LAYOUT_ALLOC     2 /* bit lock for layout allocation */
+	#define NFS_INO_LAYOUTCOMMIT     3 /* LAYOUTCOMMIT needed */
 #endif /* CONFIG_NFS_V4_1 */
 #endif /* CONFIG_NFS_V4*/
 #ifdef CONFIG_NFS_FSCACHE
Signature · 1 line
1.6.6
#3 lore raw

PATCH 3/5 SQUASHME: pnfs-submit: move pnfs_layout_suspend back to nfs_inode

From: Fred Isaman <[email protected]>

Prepare to embed struct pnfs_layout_tupe into layout driver private structs and to make layout a pointer. Since a temp error on first LAYOUTGET erases the layout, the suspend timer needs to be moved.

---
 fs/nfs/inode.c         |    1 +
 fs/nfs/pnfs.c          |    8 ++++----
 include/linux/nfs_fs.h |    2 +-
 3 files changed, 6 insertions(+), 5 deletions(-)
fs/nfs/inode.c +1 -0
index 66d7be2..cb12d33 100644
@@ -1420,6 +1420,7 @@ static void pnfs_init_once(struct nfs_inode *nfsi)
 #ifdef CONFIG_NFS_V4_1
 	init_waitqueue_head(&nfsi->lo_waitq);
 	nfsi->pnfs_layout_state = 0;
+	nfsi->pnfs_layout_suspend = 0;
 	seqlock_init(&nfsi->layout.seqlock);
 	INIT_LIST_HEAD(&nfsi->layout.lo_layouts);
 	INIT_LIST_HEAD(&nfsi->layout.segs);
fs/nfs/pnfs.c +4 -4
index d05cb03..7f6ace2 100644
@@ -1105,12 +1105,12 @@ _pnfs_update_layout(struct inode *ino,
 
 	/* if get layout already failed once goto out */
 	if (test_bit(lo_fail_bit(iomode), &nfsi->pnfs_layout_state)) {
-		if (unlikely(nfsi->layout.pnfs_layout_suspend &&
-		    get_seconds() >= nfsi->layout.pnfs_layout_suspend)) {
+		if (unlikely(nfsi->pnfs_layout_suspend &&
+		    get_seconds() >= nfsi->pnfs_layout_suspend)) {
 			dprintk("%s: layout_get resumed\n", __func__);
 			clear_bit(lo_fail_bit(iomode),
 				  &nfsi->pnfs_layout_state);
-			nfsi->layout.pnfs_layout_suspend = 0;
+			nfsi->pnfs_layout_suspend = 0;
 		} else
 			goto out_put;
 	}
@@ -1224,7 +1224,7 @@ pnfs_get_layout_done(struct nfs4_pnfs_layoutget *lgp, int rpc_status)
 	}
 
 	/* remember that get layout failed and suspend trying */
-	nfsi->layout.pnfs_layout_suspend = suspend;
+	nfsi->pnfs_layout_suspend = suspend;
 	set_bit(lo_fail_bit(lgp->args.lseg.iomode),
 		&nfsi->pnfs_layout_state);
 	dprintk("%s: layout_get suspended until %ld\n",
include/linux/nfs_fs.h +1 -1
index e216e24..cebc958 100644
@@ -105,7 +105,6 @@ struct pnfs_layout_type {
 	seqlock_t seqlock;		/* Protects the stateid */
 	nfs4_stateid stateid;
 	void *ld_data;			/* layout driver private data */
-	time_t pnfs_layout_suspend;
 	struct rpc_cred         *lo_cred; /* layoutcommit credential */
 	/* DH: These vars keep track of the maximum write range
 	 * so the values can be used for layoutcommit.
@@ -208,6 +207,7 @@ struct nfs_inode {
 	#define NFS_INO_RW_LAYOUT_FAILED 1 /* rw LAYOUTGET failed stop trying */
 	#define NFS_INO_LAYOUT_ALLOC     2 /* bit lock for layout allocation */
 	#define NFS_INO_LAYOUTCOMMIT     3 /* LAYOUTCOMMIT needed */
+	time_t pnfs_layout_suspend;
 #endif /* CONFIG_NFS_V4_1 */
 #endif /* CONFIG_NFS_V4*/
 #ifdef CONFIG_NFS_FSCACHE
Signature · 1 line
1.6.6
#4 lore raw

PATCH 4/5 SQUASHME pnfs-submit embed pnfs_layout_type

From: Andy Adamson <[email protected]>

Each layoutdriver embeds the pnfs_layout_type in it's private layout cache head structure, and allocates them both with the alloc_layout layoutdriver_io_operation.

Move all nfs inode pnfs field initialization into nfs4_init_once.

---
 fs/nfs/callback_proc.c    |    2 +-
 fs/nfs/inode.c            |   43 ++++---------------------
 fs/nfs/nfs4proc.c         |    2 +-
 fs/nfs/nfs4state.c        |    4 +-
 fs/nfs/nfs4xdr.c          |    2 +-
 fs/nfs/pnfs.c             |   78 +++++++++++++++++++++++++--------------------
 include/linux/nfs4_pnfs.h |   14 ++------
 include/linux/nfs_fs.h    |    4 +-
 8 files changed, 61 insertions(+), 88 deletions(-)
fs/nfs/callback_proc.c +1 -1
index 4766695..d999ea8 100644
@@ -233,7 +233,7 @@ static int pnfs_recall_layout(void *data)
 	rl.cbl_seg.length = NFS4_MAX_UINT64;
 
 	if (rl.cbl_recall_type == RETURN_FILE) {
-		if (pnfs_is_next_layout_stateid(&NFS_I(inode)->layout,
+		if (pnfs_is_next_layout_stateid(NFS_I(inode)->layout,
 						rl.cbl_stateid))
 			status = pnfs_return_layout(inode, &rl.cbl_seg,
 						    &rl.cbl_stateid, RETURN_FILE,
fs/nfs/inode.c +7 -36

80 lines not on this page · raw message

fs/nfs/nfs4proc.c +1 -1
index 6283996..61f4aa9 100644
@@ -1086,7 +1086,7 @@ static void pnfs4_layout_reclaim(struct nfs4_state *state)
 	 * flag during grace period
 	 */
 	pnfs_destroy_layout(NFS_I(state->inode));
-	pnfs_set_layout_stateid(&NFS_I(state->inode)->layout, &zero_stateid);
+	pnfs_set_layout_stateid(NFS_I(state->inode)->layout, &zero_stateid);
 #endif /* CONFIG_NFS_V4_1 */
 }
 
fs/nfs/nfs4state.c +2 -2
index bfe679b..6f44cb0 100644
@@ -597,10 +597,10 @@ static void __nfs4_close(struct path *path, struct nfs4_state *state,
 #ifdef CONFIG_NFS_V4_1
 		struct nfs_inode *nfsi = NFS_I(state->inode);
 
-		if (has_layout(nfsi) && nfsi->layout.roc_iomode) {
+		if (has_layout(nfsi) && nfsi->layout->roc_iomode) {
 			struct nfs4_pnfs_layout_segment range;
 
-			range.iomode = nfsi->layout.roc_iomode;
+			range.iomode = nfsi->layout->roc_iomode;
 			range.offset = 0;
 			range.length = NFS4_MAX_UINT64;
 			pnfs_return_layout(state->inode, &range, NULL,
fs/nfs/nfs4xdr.c +1 -1
index eeee855..0fd02b1 100644
@@ -1886,7 +1886,7 @@ encode_layoutreturn(struct xdr_stream *xdr,
 		ld_io_ops->encode_layoutreturn);
 		if (ld_io_ops->encode_layoutreturn) {
 			ld_io_ops->encode_layoutreturn(
-				&NFS_I(args->inode)->layout, xdr, args);
+				NFS_I(args->inode)->layout, xdr, args);
 		} else {
 			p = reserve_space(xdr, 4);
 			*p = cpu_to_be32(0);
fs/nfs/pnfs.c +43 -35

174 lines not on this page · raw message

include/linux/nfs4_pnfs.h +4 -10
index 4d9b15c..6e46589 100644
@@ -35,13 +35,13 @@ struct pnfs_layoutdriver_type {
 static inline struct nfs_inode *
 PNFS_NFS_INODE(struct pnfs_layout_type *lo)
 {
-	return container_of(lo, struct nfs_inode, layout);
+	return NFS_I(lo->lo_inode);
 }
 
 static inline struct inode *
 PNFS_INODE(struct pnfs_layout_type *lo)
 {
-	return &PNFS_NFS_INODE(lo)->vfs_inode;
+	return lo->lo_inode;
 }
 
 static inline struct nfs_server *
@@ -50,12 +50,6 @@ PNFS_NFS_SERVER(struct pnfs_layout_type *lo)
 	return NFS_SERVER(PNFS_INODE(lo));
 }
 
-static inline void *
-PNFS_LD_DATA(struct pnfs_layout_type *lo)
-{
-	return lo->ld_data;
-}
-
 static inline struct pnfs_layoutdriver_type *
 PNFS_LD(struct pnfs_layout_type *lo)
 {
@@ -77,7 +71,7 @@ PNFS_LD_POLICY_OPS(struct pnfs_layout_type *lo)
 static inline bool
 has_layout(struct nfs_inode *nfsi)
 {
-	return nfsi->layout.ld_data != NULL;
+	return nfsi->layout != NULL;
 }
 
 static inline bool
@@ -149,7 +143,7 @@ struct layoutdriver_io_operations {
 	/* Layout information. For each inode, alloc_layout is executed once to retrieve an
 	 * inode specific layout structure.  Each subsequent layoutget operation results in
 	 * a set_layout call to set the opaque layout in the layout driver.*/
-	void * (*alloc_layout) (struct inode *inode);
+	struct pnfs_layout_type * (*alloc_layout) (struct inode *inode);
 	void (*free_layout) (void *layoutid);
 	struct pnfs_layout_segment * (*alloc_lseg) (struct pnfs_layout_type *layoutid, struct nfs4_pnfs_layoutget_res *lgr);
 	void (*free_lseg) (struct pnfs_layout_segment *lseg);
include/linux/nfs_fs.h +2 -2
index cebc958..43d516e 100644
@@ -104,13 +104,13 @@ struct pnfs_layout_type {
 	int roc_iomode;			/* iomode to return on close, 0=none */
 	seqlock_t seqlock;		/* Protects the stateid */
 	nfs4_stateid stateid;
-	void *ld_data;			/* layout driver private data */
 	struct rpc_cred         *lo_cred; /* layoutcommit credential */
 	/* DH: These vars keep track of the maximum write range
 	 * so the values can be used for layoutcommit.
 	 */
 	loff_t                  pnfs_write_begin_pos;
 	loff_t                  pnfs_write_end_pos;
+	struct inode		*lo_inode;
 };
 
 /*
@@ -201,7 +201,7 @@ struct nfs_inode {
 	/* pNFS layout information */
 #if defined(CONFIG_NFS_V4_1)
 	wait_queue_head_t lo_waitq;
-	struct pnfs_layout_type layout;
+	struct pnfs_layout_type *layout;
 	unsigned long pnfs_layout_state;
 	#define NFS_INO_RO_LAYOUT_FAILED 0 /* ro LAYOUTGET failed stop trying */
 	#define NFS_INO_RW_LAYOUT_FAILED 1 /* rw LAYOUTGET failed stop trying */
Signature · 1 line
1.6.6
#5 lore raw

PATCH 5/5 SQUASHME pnfs-submit: filelayout: use new alloc_layout API

From: Andy Adamson <[email protected]>

---
 fs/nfs/nfs4filelayout.c |   11 +++++++----
 fs/nfs/nfs4filelayout.h |    7 +++++++
 2 files changed, 14 insertions(+), 4 deletions(-)
fs/nfs/nfs4filelayout.c +7 -4
index 57a0010..07d5bf6 100644
@@ -301,11 +301,14 @@ filelayout_write_pagelist(struct pnfs_layout_type *layoutid,
  * will use the pnfs_layout_type type to refer to the layout for this
  * inode from now on.
  */
-static void *
+static struct pnfs_layout_type *
 filelayout_alloc_layout(struct inode *inode)
 {
+	struct nfs4_filelayout *flp;
+
 	dprintk("NFS_FILELAYOUT: allocating layout\n");
-	return kzalloc(sizeof(struct nfs4_filelayout), GFP_KERNEL);
+	flp =  kzalloc(sizeof(struct nfs4_filelayout), GFP_KERNEL);
+	return flp ? &flp->fl_cache : NULL;
 }
 
 /* Free a filelayout layout structure
@@ -471,7 +474,7 @@ static struct pnfs_layout_segment *
 filelayout_alloc_lseg(struct pnfs_layout_type *layoutid,
 		      struct nfs4_pnfs_layoutget_res *lgr)
 {
-	struct nfs4_filelayout *flo = PNFS_LD_DATA(layoutid);
+	struct nfs4_filelayout *flo = FILE_LO_CACHE(layoutid);
 	struct pnfs_layout_segment *lseg;
 	int rc;
 
@@ -693,7 +696,7 @@ filelayout_commit(struct pnfs_layout_type *layoutid, int sync,
 ssize_t
 filelayout_get_stripesize(struct pnfs_layout_type *layoutid)
 {
-	struct nfs4_filelayout *flo = PNFS_LD_DATA(layoutid);
+	struct nfs4_filelayout *flo = FILE_LO_CACHE(layoutid);
 
 	return flo->stripe_unit;
 }
fs/nfs/nfs4filelayout.h +7 -0
index de8391f..eca459f 100644
@@ -68,6 +68,7 @@ struct nfs4_filelayout_segment {
 };
 
 struct nfs4_filelayout {
+	struct pnfs_layout_type fl_cache;
 	int uncommitted_write;
 	loff_t last_commit_size;
 	u64 layout_id;
@@ -77,6 +78,12 @@ struct nfs4_filelayout {
 extern struct nfs_fh *
 nfs4_fl_select_ds_fh(struct pnfs_layout_segment *lseg, loff_t offset);
 
+static inline struct nfs4_filelayout *
+FILE_LO_CACHE(struct pnfs_layout_type *lo_cache)
+{
+	return container_of(lo_cache, struct nfs4_filelayout, fl_cache);
+}
+
 extern struct pnfs_client_operations *pnfs_callback_ops;
 
 extern void nfs4_fl_free_deviceid_callback(struct kref *);
Signature · 1 line
1.6.6
#6 lore raw

Re: [PATCH 5/5] SQUASHME pnfs-submit: filelayout: use new alloc_layout API

Quoted 8 lines · On 06/29/2010 07:42 PM, [email protected] wrote:

From: Andy Adamson <[email protected]>

---
 fs/nfs/nfs4filelayout.c |   11 +++++++----
 fs/nfs/nfs4filelayout.h |    7 +++++++
 2 files changed, 14 insertions(+), 4 deletions(-)

Won't you go head and submit an API change to the other two layout drivers?

Quoted 52 lines
fs/nfs/nfs4filelayout.c +7 -4
index 57a0010..07d5bf6 100644
@@ -301,11 +301,14 @@ filelayout_write_pagelist(struct pnfs_layout_type *layoutid,
  * will use the pnfs_layout_type type to refer to the layout for this
  * inode from now on.
  */
-static void *
+static struct pnfs_layout_type *
 filelayout_alloc_layout(struct inode *inode)
 {
+	struct nfs4_filelayout *flp;
+
 	dprintk("NFS_FILELAYOUT: allocating layout\n");
-	return kzalloc(sizeof(struct nfs4_filelayout), GFP_KERNEL);
+	flp =  kzalloc(sizeof(struct nfs4_filelayout), GFP_KERNEL);
+	return flp ? &flp->fl_cache : NULL;
 }
 
 /* Free a filelayout layout structure
@@ -471,7 +474,7 @@ static struct pnfs_layout_segment *
 filelayout_alloc_lseg(struct pnfs_layout_type *layoutid,
 		      struct nfs4_pnfs_layoutget_res *lgr)
 {
-	struct nfs4_filelayout *flo = PNFS_LD_DATA(layoutid);
+	struct nfs4_filelayout *flo = FILE_LO_CACHE(layoutid);
 	struct pnfs_layout_segment *lseg;
 	int rc;
 
@@ -693,7 +696,7 @@ filelayout_commit(struct pnfs_layout_type *layoutid, int sync,
 ssize_t
 filelayout_get_stripesize(struct pnfs_layout_type *layoutid)
 {
-	struct nfs4_filelayout *flo = PNFS_LD_DATA(layoutid);
+	struct nfs4_filelayout *flo = FILE_LO_CACHE(layoutid);
 
 	return flo->stripe_unit;
 }
fs/nfs/nfs4filelayout.h +1 -0
index de8391f..eca459f 100644
@@ -68,6 +68,7 @@ struct nfs4_filelayout_segment {
 };
 
 struct nfs4_filelayout {
+	struct pnfs_layout_type fl_cache;
 	int uncommitted_write;
 	loff_t last_commit_size;
 	u64 layout_id;

fs/nfs/nfs4filelayout.h struct nfs4_filelayout { · line 86

Quoted 9 lines
fs/nfs/nfs4filelayout.h +6 -0
@@ -77,6 +78,12 @@ struct nfs4_filelayout {
 extern struct nfs_fh *
 nfs4_fl_select_ds_fh(struct pnfs_layout_segment *lseg, loff_t offset);
 
+static inline struct nfs4_filelayout *
+FILE_LO_CACHE(struct pnfs_layout_type *lo_cache)
+{
+	return container_of(lo_cache, struct nfs4_filelayout, fl_cache);
+}
+

The name? why the __CACHE? why not just FILE_LO()? the struct it returns name is nfs4_filelayout, simple.

Boaz

Quoted 3 lines
fs/nfs/nfs4filelayout.h +0 -0
 extern struct pnfs_client_operations *pnfs_callback_ops;
 
 extern void nfs4_fl_free_deviceid_callback(struct kref *);
#7 lore raw

Re: [PATCH 5/5] SQUASHME pnfs-submit: filelayout: use new alloc_layout API

Quoted 11 lines · Boaz Harrosh
Quoted 8 lines · On 06/29/2010 07:42 PM, [email protected] wrote:

From: Andy Adamson <[email protected]>

---
fs/nfs/nfs4filelayout.c | 11 +++++++----
fs/nfs/nfs4filelayout.h | 7 +++++++

2 files changed, 14 insertions(+), 4 deletions(-)

Won't you go head and submit an API change to the other two layout drivers?

OK. Just wanted comments first.

-->Andy

Quoted 74 lines
Quoted 58 lines
fs/nfs/nfs4filelayout.c +5 -2
index 57a0010..07d5bf6 100644
@@ -301,11 +301,14 @@ filelayout_write_pagelist(struct  
 pnfs_layout_type *layoutid,
 * will use the pnfs_layout_type type to refer to the layout for this
 * inode from now on.
 */
-static void *
+static struct pnfs_layout_type *
 filelayout_alloc_layout(struct inode *inode)
 {
+	struct nfs4_filelayout *flp;
+
 	dprintk("NFS_FILELAYOUT: allocating layout\n");
-	return kzalloc(sizeof(struct nfs4_filelayout), GFP_KERNEL);
+	flp =  kzalloc(sizeof(struct nfs4_filelayout), GFP_KERNEL);
+	return flp ? &flp->fl_cache : NULL;
 }
 

/* Free a filelayout layout structure

fs/nfs/nfs4filelayout.c +1 -1
@@ -471,7 +474,7 @@ static struct pnfs_layout_segment *
 filelayout_alloc_lseg(struct pnfs_layout_type *layoutid,
 		      struct nfs4_pnfs_layoutget_res *lgr)
 {
-	struct nfs4_filelayout *flo = PNFS_LD_DATA(layoutid);
+	struct nfs4_filelayout *flo = FILE_LO_CACHE(layoutid);
 	struct pnfs_layout_segment *lseg;
 	int rc;
 
@@ -693,7 +696,7 @@ filelayout_commit(struct pnfs_layout_type  
 *layoutid, int sync,

ssize_t
filelayout_get_stripesize(struct pnfs_layout_type *layoutid) {

fs/nfs/nfs4filelayout.c +1 -1
-	struct nfs4_filelayout *flo = PNFS_LD_DATA(layoutid);
+	struct nfs4_filelayout *flo = FILE_LO_CACHE(layoutid);
	return flo->stripe_unit;
}
fs/nfs/nfs4filelayout.h +1 -0
index de8391f..eca459f 100644
@@ -68,6 +68,7 @@ struct nfs4_filelayout_segment {
 };
 
 struct nfs4_filelayout {
+	struct pnfs_layout_type fl_cache;
 	int uncommitted_write;
 	loff_t last_commit_size;
 	u64 layout_id;
@@ -77,6 +78,12 @@ struct nfs4_filelayout {

extern struct nfs_fh *
nfs4_fl_select_ds_fh(struct pnfs_layout_segment *lseg, loff_t offset);

fs/nfs/nfs4filelayout.h struct nfs4_filelayout { · line 86

Quoted 6 lines
fs/nfs/nfs4filelayout.h +6 -0
+static inline struct nfs4_filelayout *
+FILE_LO_CACHE(struct pnfs_layout_type *lo_cache)
+{
+	return container_of(lo_cache, struct nfs4_filelayout, fl_cache);
+}
+

The name? why the __CACHE? why not just FILE_LO()? the struct it returns name is nfs4_filelayout, simple.

Boaz

Quoted 3 lines

extern struct pnfs_client_operations *pnfs_callback_ops;

extern void nfs4_fl_free_deviceid_callback(struct kref *);

#8 lore raw

Re: [PATCH 4/5] SQUASHME pnfs-submit embed pnfs_layout_type

In reply to [email protected]
Quoted 66 lines · On 06/29/2010 07:42 PM, [email protected] wrote:

From: Andy Adamson <[email protected]>

Each layoutdriver embeds the pnfs_layout_type in it's private layout cache head structure, and allocates them both with the alloc_layout layoutdriver_io_operation.

Move all nfs inode pnfs field initialization into nfs4_init_once.

---
 fs/nfs/callback_proc.c    |    2 +-
 fs/nfs/inode.c            |   43 ++++---------------------
 fs/nfs/nfs4proc.c         |    2 +-
 fs/nfs/nfs4state.c        |    4 +-
 fs/nfs/nfs4xdr.c          |    2 +-
 fs/nfs/pnfs.c             |   78 +++++++++++++++++++++++++--------------------
 include/linux/nfs4_pnfs.h |   14 ++------
 include/linux/nfs_fs.h    |    4 +-
 8 files changed, 61 insertions(+), 88 deletions(-)
fs/nfs/callback_proc.c +1 -1
index 4766695..d999ea8 100644
@@ -233,7 +233,7 @@ static int pnfs_recall_layout(void *data)
 	rl.cbl_seg.length = NFS4_MAX_UINT64;
 
 	if (rl.cbl_recall_type == RETURN_FILE) {
-		if (pnfs_is_next_layout_stateid(&NFS_I(inode)->layout,
+		if (pnfs_is_next_layout_stateid(NFS_I(inode)->layout,
 						rl.cbl_stateid))
 			status = pnfs_return_layout(inode, &rl.cbl_seg,
 						    &rl.cbl_stateid, RETURN_FILE,
fs/nfs/inode.c +0 -12
index cb12d33..c290806 100644
@@ -1361,17 +1361,6 @@ void nfs4_clear_inode(struct inode *inode)
 }
 #endif
 
-static void pnfs_alloc_init_inode(struct nfs_inode *nfsi)
-{
-#ifdef CONFIG_NFS_V4_1
-	memset(&nfsi->layout.stateid, 0, NFS4_STATEID_SIZE);
-	nfsi->layout.roc_iomode = 0;
-	nfsi->layout.lo_cred = NULL;
-	nfsi->layout.pnfs_write_begin_pos = 0;
-	nfsi->layout.pnfs_write_end_pos = 0;
-#endif /* CONFIG_NFS_V4_1 */
-}
-
 struct inode *nfs_alloc_inode(struct super_block *sb)
 {
 	struct nfs_inode *nfsi;
@@ -1387,23 +1376,14 @@ struct inode *nfs_alloc_inode(struct super_block *sb)
 #ifdef CONFIG_NFS_V4
 	nfsi->nfs4_acl = NULL;
 #endif /* CONFIG_NFS_V4 */
-	pnfs_alloc_init_inode(nfsi);
 	return &nfsi->vfs_inode;
 }
 
 static void pnfs_destroy_inode(struct nfs_inode *nfsi)
 {
 #ifdef CONFIG_NFS_V4_1

fs/nfs/inode.c struct inode *nfs_alloc_inode(struct super_block *sb) · line 1405

Quoted 10 lines
fs/nfs/inode.c +1 -8
-	if (!list_empty(&nfsi->layout.segs))
+	if (has_layout(nfsi))
 		pnfs_destroy_layout(nfsi);
-
-	WARN_ON(!list_empty(&nfsi->layout.segs));
-	if (nfsi->layout.refcount)
-		printk("%s: WARNING: layout.refcount %d\n", __func__,
-			nfsi->layout.refcount);
-	WARN_ON(nfsi->layout.refcount);
-	WARN_ON(!list_empty(&nfsi->layout.lo_layouts));

Andy, these problems did not go away with this patch. Please re-instate them for the new code. All 3 WARN_ONs still apply. Perhaps in pnfs_destroy_layout()

Quoted 127 lines
fs/nfs/inode.c +6 -16
-	WARN_ON(nfsi->layout.ld_data);
 #endif /* CONFIG_NFS_V4_1 */
 }
 
@@ -1415,20 +1395,6 @@ void nfs_destroy_inode(struct inode *inode)
 	kmem_cache_free(nfs_inode_cachep, nfsi);
 }
 
-static void pnfs_init_once(struct nfs_inode *nfsi)
-{
-#ifdef CONFIG_NFS_V4_1
-	init_waitqueue_head(&nfsi->lo_waitq);
-	nfsi->pnfs_layout_state = 0;
-	nfsi->pnfs_layout_suspend = 0;
-	seqlock_init(&nfsi->layout.seqlock);
-	INIT_LIST_HEAD(&nfsi->layout.lo_layouts);
-	INIT_LIST_HEAD(&nfsi->layout.segs);
-	nfsi->layout.refcount = 0;
-	nfsi->layout.ld_data = NULL;
-#endif /* CONFIG_NFS_V4_1 */
-}
-
 static inline void nfs4_init_once(struct nfs_inode *nfsi)
 {
 #ifdef CONFIG_NFS_V4
@@ -1436,6 +1402,12 @@ static inline void nfs4_init_once(struct nfs_inode *nfsi)
 	nfsi->delegation = NULL;
 	nfsi->delegation_state = 0;
 	init_rwsem(&nfsi->rwsem);
+#ifdef CONFIG_NFS_V4_1
+	init_waitqueue_head(&nfsi->lo_waitq);
+	nfsi->layout = NULL;
+	nfsi->pnfs_layout_state = 0;
+	nfsi->pnfs_layout_suspend = 0;
+#endif /* CONFIG_NFS_V4_1 */
 #endif
 }
 
@@ -1454,7 +1426,6 @@ static void init_once(void *foo)
 	INIT_HLIST_HEAD(&nfsi->silly_list);
 	init_waitqueue_head(&nfsi->waitqueue);
 	nfs4_init_once(nfsi);
-	pnfs_init_once(nfsi);
 }
 
 static int __init nfs_init_inodecache(void)
fs/nfs/nfs4proc.c +1 -1
index 6283996..61f4aa9 100644
@@ -1086,7 +1086,7 @@ static void pnfs4_layout_reclaim(struct nfs4_state *state)
 	 * flag during grace period
 	 */
 	pnfs_destroy_layout(NFS_I(state->inode));
-	pnfs_set_layout_stateid(&NFS_I(state->inode)->layout, &zero_stateid);
+	pnfs_set_layout_stateid(NFS_I(state->inode)->layout, &zero_stateid);
 #endif /* CONFIG_NFS_V4_1 */
 }
 
fs/nfs/nfs4state.c +2 -2
index bfe679b..6f44cb0 100644
@@ -597,10 +597,10 @@ static void __nfs4_close(struct path *path, struct nfs4_state *state,
 #ifdef CONFIG_NFS_V4_1
 		struct nfs_inode *nfsi = NFS_I(state->inode);
 
-		if (has_layout(nfsi) && nfsi->layout.roc_iomode) {
+		if (has_layout(nfsi) && nfsi->layout->roc_iomode) {
 			struct nfs4_pnfs_layout_segment range;
 
-			range.iomode = nfsi->layout.roc_iomode;
+			range.iomode = nfsi->layout->roc_iomode;
 			range.offset = 0;
 			range.length = NFS4_MAX_UINT64;
 			pnfs_return_layout(state->inode, &range, NULL,
fs/nfs/nfs4xdr.c +1 -1
index eeee855..0fd02b1 100644
@@ -1886,7 +1886,7 @@ encode_layoutreturn(struct xdr_stream *xdr,
 		ld_io_ops->encode_layoutreturn);
 		if (ld_io_ops->encode_layoutreturn) {
 			ld_io_ops->encode_layoutreturn(
-				&NFS_I(args->inode)->layout, xdr, args);
+				NFS_I(args->inode)->layout, xdr, args);
 		} else {
 			p = reserve_space(xdr, 4);
 			*p = cpu_to_be32(0);
fs/nfs/pnfs.c +7 -7
index 7f6ace2..9275140 100644
@@ -153,7 +153,7 @@ pnfs_need_layoutcommit(struct nfs_inode *nfsi, struct nfs_open_context *ctx)
 	dprintk("%s: has_layout=%d ctx=%p\n", __func__, has_layout(nfsi), ctx);
 	spin_lock(&nfsi->vfs_inode.i_lock);
 	if (has_layout(nfsi) && !layoutcommit_needed(nfsi)) {
-		nfsi->layout.lo_cred = get_rpccred(ctx->state->owner->so_cred);
+		nfsi->layout->lo_cred = get_rpccred(ctx->state->owner->so_cred);
 		__set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->pnfs_layout_state);
 		nfsi->change_attr++;
 		spin_unlock(&nfsi->vfs_inode.i_lock);
@@ -174,17 +174,17 @@ pnfs_update_last_write(struct nfs_inode *nfsi, loff_t offset, size_t extent)
 	loff_t end_pos;
 
 	spin_lock(&nfsi->vfs_inode.i_lock);
-	if (offset < nfsi->layout.pnfs_write_begin_pos)
-		nfsi->layout.pnfs_write_begin_pos = offset;
+	if (offset < nfsi->layout->pnfs_write_begin_pos)
+		nfsi->layout->pnfs_write_begin_pos = offset;
 	end_pos = offset + extent - 1; /* I'm being inclusive */
-	if (end_pos > nfsi->layout.pnfs_write_end_pos)
-		nfsi->layout.pnfs_write_end_pos = end_pos;
+	if (end_pos > nfsi->layout->pnfs_write_end_pos)
+		nfsi->layout->pnfs_write_end_pos = end_pos;
 	dprintk("%s: Wrote %lu@%lu bpos %lu, epos: %lu\n",
 		__func__,
 		(unsigned long) extent,
 		(unsigned long) offset ,
-		(unsigned long) nfsi->layout.pnfs_write_begin_pos,
-		(unsigned long) nfsi->layout.pnfs_write_end_pos);
+		(unsigned long) nfsi->layout->pnfs_write_begin_pos,
+		(unsigned long) nfsi->layout->pnfs_write_end_pos);
 	spin_unlock(&nfsi->vfs_inode.i_lock);
 }
 

fs/nfs/pnfs.c get_layout(struct pnfs_layout_type *lo) · line 330

Quoted 7 lines
fs/nfs/pnfs.c +1 -2
@@ -325,10 +325,9 @@ get_layout(struct pnfs_layout_type *lo)
 static inline struct pnfs_layout_type *
 grab_current_layout(struct nfs_inode *nfsi)
 {
-	struct pnfs_layout_type *lo = &nfsi->layout;
+	struct pnfs_layout_type *lo = nfsi->layout;
 
-	BUG_ON_UNLOCKED_LO(lo);

Why did you drop this one?

Quoted 25 lines
fs/nfs/pnfs.c +6 -6
-	if (!lo->ld_data)
+	if (!lo)
 		return NULL;
 	get_layout(lo);
 	return lo;
@@ -342,13 +341,13 @@ put_layout(struct pnfs_layout_type *lo)
 
 	lo->refcount--;
 	if (!lo->refcount) {
-		struct layoutdriver_io_operations *io_ops =
-			PNFS_LD_IO_OPS(lo);
+		struct layoutdriver_io_operations *io_ops = PNFS_LD_IO_OPS(lo);
+		struct nfs_inode *nfsi = PNFS_NFS_INODE(lo);
 
-		dprintk("%s: freeing layout %p\n", __func__, lo->ld_data);
+		dprintk("%s: freeing layout cache %p\n", __func__, lo);
 		WARN_ON(!list_empty(&lo->lo_layouts));
-		io_ops->free_layout(lo->ld_data);
-		lo->ld_data = NULL;
+		io_ops->free_layout(lo);
+		nfsi->layout = NULL;
 	}
 }
 

fs/nfs/pnfs.c pnfs_return_layout_barrier(struct nfs_inode *nfsi, · line 690

Quoted 5 lines
fs/nfs/pnfs.c +1 -1
@@ -688,7 +687,7 @@ pnfs_return_layout_barrier(struct nfs_inode *nfsi,
 	bool ret = false;
 
 	spin_lock(&nfsi->vfs_inode.i_lock);
-	list_for_each_entry (lseg, &nfsi->layout.segs, fi_list) {
+	list_for_each_entry(lseg, &nfsi->layout->segs, fi_list) {

White space cleanup. actually I like the space. I thought checkpatch was fixed to accept these spaces?

Quoted 14 lines
fs/nfs/pnfs.c +4 -0
 		if (!should_free_lseg(lseg, range))
 			continue;
 		lseg->valid = false;
@@ -894,17 +893,20 @@ pnfs_insert_layout(struct pnfs_layout_type *lo,
 	dprintk("%s:Return\n", __func__);
 }
 
+/*
+ * Each layoutdriver embeds pnfs_layout_type in it's per-layout type layout
+ * cache structure. Initialize the common pnfs_layout_type fields.
+ */
 static struct pnfs_layout_type *
 alloc_init_layout(struct inode *ino)
 {

fs/nfs/pnfs.c pnfs_insert_layout(struct pnfs_layout_type *lo, · line 908

Quoted 10 lines
fs/nfs/pnfs.c +2 -4
-	void *ld_data;
 	struct pnfs_layout_type *lo;
 	struct layoutdriver_io_operations *io_ops;
 
 	io_ops = NFS_SERVER(ino)->pnfs_curr_ld->ld_io_ops;
-	lo = &NFS_I(ino)->layout;
-	ld_data = io_ops->alloc_layout(ino);
-	if (!ld_data) {
+	lo = NFS_I(ino)->layout;
+	lo = io_ops->alloc_layout(ino);

Oops, ;-)

fs/nfs/pnfs.c alloc_init_layout(struct inode *ino) · line 915

Quoted 9 lines
fs/nfs/pnfs.c +1 -1
+	if (!lo) {
 		printk(KERN_ERR
 			"%s: out of memory: io_ops->alloc_layout failed\n",
 			__func__);
@@ -912,11 +914,17 @@ alloc_init_layout(struct inode *ino)
 	}
 
 	spin_lock(&ino->i_lock);
-	BUG_ON(lo->ld_data != NULL);

You could still do a BUG_ON(nfsi->layout) above before the io_ops->alloc_layout(ino) call.

fs/nfs/pnfs.c alloc_init_layout(struct inode *ino) · line 921

Quoted 7 lines
fs/nfs/pnfs.c +3 -2
-	lo->ld_data = ld_data;
-	memset(&lo->stateid, 0, NFS4_STATEID_SIZE);
 	lo->refcount = 1;
+	INIT_LIST_HEAD(&lo->lo_layouts);
+	INIT_LIST_HEAD(&lo->segs);
 	lo->roc_iomode = 0;
+	seqlock_init(&lo->seqlock);

All these below, I'd drop. The comment above io_ops->alloc_layout should say that it must return a ZEROed out structure, which it is in your files-layout patch.

fs/nfs/pnfs.c alloc_init_layout(struct inode *ino) · line 928

Quoted 8 lines
fs/nfs/pnfs.c +6 -0
+	memset(&lo->stateid, 0, NFS4_STATEID_SIZE);
+	lo->lo_cred = NULL;
+	lo->pnfs_write_begin_pos = 0;
+	lo->pnfs_write_end_pos = 0;
 
+	lo->lo_inode = ino;
+	NFS_I(ino)->layout = lo;
 	spin_unlock(&ino->i_lock);

Just as a note:
A pointer/word atomic inspection theory is based on the fact that the set is done with a memory barrier. Same as the test/set_bit operations. So here the spin_unlock provides that for us, naturally. (*Not* that the inode state will really need it, because no one will actually care before the allocation is done)

Quoted 140 lines
fs/nfs/pnfs.c +0 -0
 	return lo;

}

fs/nfs/pnfs.c +12 -12
@@ -962,7 +970,7 @@ get_lock_alloc_layout(struct inode *ino)
 		/* Was current_layout already allocated while we slept?
 		 * If so, retry get_lock'ing it. Otherwise, allocate it.
 		 */
-		if (nfsi->layout.ld_data) {
+		if (nfsi->layout) {
 			spin_lock(&ino->i_lock);
 			continue;
 		}
@@ -1312,7 +1320,7 @@ pnfs_set_pg_test(struct inode *inode, struct nfs_pageio_descriptor *pgio)
 
 	pgio->pg_test = NULL;
 
-	laytype = &NFS_I(inode)->layout;
+	laytype = NFS_I(inode)->layout;
 	ld = NFS_SERVER(inode)->pnfs_curr_ld;
 	if (!pnfs_enabled_sb(NFS_SERVER(inode)) || !laytype)
 		return;
@@ -1445,7 +1453,7 @@ pnfs_writepages(struct nfs_write_data *wdata, int how)
 		numpages);
 	wdata->pdata.lseg = lseg;
 	trypnfs = nfss->pnfs_curr_ld->ld_io_ops->write_pagelist(
-							&nfsi->layout,
+							nfsi->layout,
 							args->pages,
 							args->pgbase,
 							numpages,
@@ -1498,7 +1506,7 @@ pnfs_readpages(struct nfs_read_data *rdata)
 		__func__, numpages);
 	rdata->pdata.lseg = lseg;
 	trypnfs = nfss->pnfs_curr_ld->ld_io_ops->read_pagelist(
-							&nfsi->layout,
+							nfsi->layout,
 							args->pages,
 							args->pgbase,
 							numpages,
@@ -1559,7 +1567,7 @@ pnfs_commit(struct nfs_write_data *data, int sync)
 	 * This will be done by passing the buck to the layout driver.
 	 */
 	data->pdata.lseg = NULL;
-	trypnfs = nfss->pnfs_curr_ld->ld_io_ops->commit(&nfsi->layout,
+	trypnfs = nfss->pnfs_curr_ld->ld_io_ops->commit(nfsi->layout,
 							sync, data);
 	dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs);
 	return trypnfs;
@@ -1630,14 +1638,14 @@ pnfs_layoutcommit_inode(struct inode *inode, int sync)
 	/* Clear layoutcommit properties in the inode so
 	 * new lc info can be generated
 	 */
-	write_begin_pos = nfsi->layout.pnfs_write_begin_pos;
-	write_end_pos = nfsi->layout.pnfs_write_end_pos;
-	data->cred = nfsi->layout.lo_cred;
-	nfsi->layout.pnfs_write_begin_pos = 0;
-	nfsi->layout.pnfs_write_end_pos = 0;
-	nfsi->layout.lo_cred = NULL;
+	write_begin_pos = nfsi->layout->pnfs_write_begin_pos;
+	write_end_pos = nfsi->layout->pnfs_write_end_pos;
+	data->cred = nfsi->layout->lo_cred;
+	nfsi->layout->pnfs_write_begin_pos = 0;
+	nfsi->layout->pnfs_write_end_pos = 0;
+	nfsi->layout->lo_cred = NULL;
 	__clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->pnfs_layout_state);
-	pnfs_get_layout_stateid(&data->args.stateid, &nfsi->layout);
+	pnfs_get_layout_stateid(&data->args.stateid, nfsi->layout);
 
 	spin_unlock(&inode->i_lock);
 
include/linux/nfs4_pnfs.h +4 -10
index 4d9b15c..6e46589 100644
@@ -35,13 +35,13 @@ struct pnfs_layoutdriver_type {
 static inline struct nfs_inode *
 PNFS_NFS_INODE(struct pnfs_layout_type *lo)
 {
-	return container_of(lo, struct nfs_inode, layout);
+	return NFS_I(lo->lo_inode);
 }
 
 static inline struct inode *
 PNFS_INODE(struct pnfs_layout_type *lo)
 {
-	return &PNFS_NFS_INODE(lo)->vfs_inode;
+	return lo->lo_inode;
 }
 
 static inline struct nfs_server *
@@ -50,12 +50,6 @@ PNFS_NFS_SERVER(struct pnfs_layout_type *lo)
 	return NFS_SERVER(PNFS_INODE(lo));
 }
 
-static inline void *
-PNFS_LD_DATA(struct pnfs_layout_type *lo)
-{
-	return lo->ld_data;
-}
-
 static inline struct pnfs_layoutdriver_type *
 PNFS_LD(struct pnfs_layout_type *lo)
 {
@@ -77,7 +71,7 @@ PNFS_LD_POLICY_OPS(struct pnfs_layout_type *lo)
 static inline bool
 has_layout(struct nfs_inode *nfsi)
 {
-	return nfsi->layout.ld_data != NULL;
+	return nfsi->layout != NULL;
 }
 
 static inline bool
@@ -149,7 +143,7 @@ struct layoutdriver_io_operations {
 	/* Layout information. For each inode, alloc_layout is executed once to retrieve an
 	 * inode specific layout structure.  Each subsequent layoutget operation results in
 	 * a set_layout call to set the opaque layout in the layout driver.*/
-	void * (*alloc_layout) (struct inode *inode);
+	struct pnfs_layout_type * (*alloc_layout) (struct inode *inode);
 	void (*free_layout) (void *layoutid);
 	struct pnfs_layout_segment * (*alloc_lseg) (struct pnfs_layout_type *layoutid, struct nfs4_pnfs_layoutget_res *lgr);
 	void (*free_lseg) (struct pnfs_layout_segment *lseg);
include/linux/nfs_fs.h +1 -1
index cebc958..43d516e 100644
@@ -104,13 +104,13 @@ struct pnfs_layout_type {
 	int roc_iomode;			/* iomode to return on close, 0=none */
 	seqlock_t seqlock;		/* Protects the stateid */
 	nfs4_stateid stateid;
-	void *ld_data;			/* layout driver private data */
 	struct rpc_cred         *lo_cred; /* layoutcommit credential */
 	/* DH: These vars keep track of the maximum write range
 	 * so the values can be used for layoutcommit.
 	 */
 	loff_t                  pnfs_write_begin_pos;
 	loff_t                  pnfs_write_end_pos;
+	struct inode		*lo_inode;
 };
 
 /*

include/linux/nfs_fs.h struct nfs_inode { · line 207

Quoted 8 lines
include/linux/nfs_fs.h +1 -1
@@ -201,7 +201,7 @@ struct nfs_inode {
 	/* pNFS layout information */
 #if defined(CONFIG_NFS_V4_1)
 	wait_queue_head_t lo_waitq;
-	struct pnfs_layout_type layout;
+	struct pnfs_layout_type *layout;
 	unsigned long pnfs_layout_state;
 	#define NFS_INO_RO_LAYOUT_FAILED 0 /* ro LAYOUTGET failed stop trying */
 	#define NFS_INO_RW_LAYOUT_FAILED 1 /* rw LAYOUTGET failed stop trying */

Boaz

#9 lore raw

Re: [PATCH 4/5] SQUASHME pnfs-submit embed pnfs_layout_type

Quoted 66 lines · Boaz Harrosh
Quoted 65 lines · On 06/29/2010 07:42 PM, [email protected] wrote:

From: Andy Adamson <[email protected]>

Each layoutdriver embeds the pnfs_layout_type in it's private layout cache head structure, and allocates them both with the alloc_layout layoutdriver_io_operation.

Move all nfs inode pnfs field initialization into nfs4_init_once.

---

fs/nfs/callback_proc.c    |    2 +-
fs/nfs/inode.c            |   43 ++++---------------------
fs/nfs/nfs4proc.c         |    2 +-
fs/nfs/nfs4state.c        |    4 +-
fs/nfs/nfs4xdr.c          |    2 +-
fs/nfs/pnfs.c             |   78 ++++++++++++++++++++++++ 
+--------------------
include/linux/nfs4_pnfs.h |   14 ++------
include/linux/nfs_fs.h    |    4 +-
8 files changed, 61 insertions(+), 88 deletions(-)
fs/nfs/callback_proc.c +1 -1
index 4766695..d999ea8 100644
@@ -233,7 +233,7 @@ static int pnfs_recall_layout(void *data)
 	rl.cbl_seg.length = NFS4_MAX_UINT64;
 
 	if (rl.cbl_recall_type == RETURN_FILE) {
-		if (pnfs_is_next_layout_stateid(&NFS_I(inode)->layout,
+		if (pnfs_is_next_layout_stateid(NFS_I(inode)->layout,
 						rl.cbl_stateid))
 			status = pnfs_return_layout(inode, &rl.cbl_seg,
 						    &rl.cbl_stateid, RETURN_FILE,
fs/nfs/inode.c +0 -12
index cb12d33..c290806 100644
@@ -1361,17 +1361,6 @@ void nfs4_clear_inode(struct inode *inode)
 }
 #endif
 
-static void pnfs_alloc_init_inode(struct nfs_inode *nfsi)
-{
-#ifdef CONFIG_NFS_V4_1
-	memset(&nfsi->layout.stateid, 0, NFS4_STATEID_SIZE);
-	nfsi->layout.roc_iomode = 0;
-	nfsi->layout.lo_cred = NULL;
-	nfsi->layout.pnfs_write_begin_pos = 0;
-	nfsi->layout.pnfs_write_end_pos = 0;
-#endif /* CONFIG_NFS_V4_1 */
-}
-
 struct inode *nfs_alloc_inode(struct super_block *sb)
 {
 	struct nfs_inode *nfsi;
@@ -1387,23 +1376,14 @@ struct inode *nfs_alloc_inode(struct  
 super_block *sb)
 #ifdef CONFIG_NFS_V4
 	nfsi->nfs4_acl = NULL;
 #endif /* CONFIG_NFS_V4 */
-	pnfs_alloc_init_inode(nfsi);
 	return &nfsi->vfs_inode;
 }
 
Quoted 16 lines

fs/nfs/inode.c struct inode *nfs_alloc_inode(struct super_block *sb) · line 1405

Quoted 13 lines
fs/nfs/inode.c +1 -8
 static void pnfs_destroy_inode(struct nfs_inode *nfsi)
 {
 #ifdef CONFIG_NFS_V4_1
-	if (!list_empty(&nfsi->layout.segs))
+	if (has_layout(nfsi))
 		pnfs_destroy_layout(nfsi);
-
-	WARN_ON(!list_empty(&nfsi->layout.segs));
-	if (nfsi->layout.refcount)
-		printk("%s: WARNING: layout.refcount %d\n", __func__,
-			nfsi->layout.refcount);
-	WARN_ON(nfsi->layout.refcount);
-	WARN_ON(!list_empty(&nfsi->layout.lo_layouts));

Andy, these problems did not go away with this patch. Please re-instate them for the new code. All 3 WARN_ONs still apply. Perhaps in pnfs_destroy_layout()

OK - good idea.

Quoted 142 lines
Quoted 142 lines
fs/nfs/inode.c +6 -16
-	WARN_ON(nfsi->layout.ld_data);
 #endif /* CONFIG_NFS_V4_1 */
 }
@@ -1415,20 +1395,6 @@ void nfs_destroy_inode(struct inode *inode)
 	kmem_cache_free(nfs_inode_cachep, nfsi);
 }
 
-static void pnfs_init_once(struct nfs_inode *nfsi)
-{
-#ifdef CONFIG_NFS_V4_1
-	init_waitqueue_head(&nfsi->lo_waitq);
-	nfsi->pnfs_layout_state = 0;
-	nfsi->pnfs_layout_suspend = 0;
-	seqlock_init(&nfsi->layout.seqlock);
-	INIT_LIST_HEAD(&nfsi->layout.lo_layouts);
-	INIT_LIST_HEAD(&nfsi->layout.segs);
-	nfsi->layout.refcount = 0;
-	nfsi->layout.ld_data = NULL;
-#endif /* CONFIG_NFS_V4_1 */
-}
-
 static inline void nfs4_init_once(struct nfs_inode *nfsi)
 {
 #ifdef CONFIG_NFS_V4
@@ -1436,6 +1402,12 @@ static inline void nfs4_init_once(struct  
 nfs_inode *nfsi)
 	nfsi->delegation = NULL;
 	nfsi->delegation_state = 0;
 	init_rwsem(&nfsi->rwsem);
+#ifdef CONFIG_NFS_V4_1
+	init_waitqueue_head(&nfsi->lo_waitq);
+	nfsi->layout = NULL;
+	nfsi->pnfs_layout_state = 0;
+	nfsi->pnfs_layout_suspend = 0;
+#endif /* CONFIG_NFS_V4_1 */
 #endif
 }
@@ -1454,7 +1426,6 @@ static void init_once(void *foo)
 	INIT_HLIST_HEAD(&nfsi->silly_list);
 	init_waitqueue_head(&nfsi->waitqueue);
 	nfs4_init_once(nfsi);
-	pnfs_init_once(nfsi);
 }
 
 static int __init nfs_init_inodecache(void)
fs/nfs/nfs4proc.c +1 -1
index 6283996..61f4aa9 100644
@@ -1086,7 +1086,7 @@ static void pnfs4_layout_reclaim(struct  
 nfs4_state *state)
 	 * flag during grace period
 	 */
 	pnfs_destroy_layout(NFS_I(state->inode));
-	pnfs_set_layout_stateid(&NFS_I(state->inode)->layout,  
 &zero_stateid);
+	pnfs_set_layout_stateid(NFS_I(state->inode)->layout,  
 &zero_stateid);

#endif /* CONFIG_NFS_V4_1 */
}

fs/nfs/nfs4state.c +2 -2
index bfe679b..6f44cb0 100644
@@ -597,10 +597,10 @@ static void __nfs4_close(struct path *path,  
 struct nfs4_state *state,
 #ifdef CONFIG_NFS_V4_1
 		struct nfs_inode *nfsi = NFS_I(state->inode);
 
-		if (has_layout(nfsi) && nfsi->layout.roc_iomode) {
+		if (has_layout(nfsi) && nfsi->layout->roc_iomode) {
 			struct nfs4_pnfs_layout_segment range;
 
-			range.iomode = nfsi->layout.roc_iomode;
+			range.iomode = nfsi->layout->roc_iomode;
 			range.offset = 0;
 			range.length = NFS4_MAX_UINT64;
			pnfs_return_layout(state->inode, &range, NULL,
fs/nfs/nfs4xdr.c +1 -1
index eeee855..0fd02b1 100644
@@ -1886,7 +1886,7 @@ encode_layoutreturn(struct xdr_stream *xdr,
 		ld_io_ops->encode_layoutreturn);
 		if (ld_io_ops->encode_layoutreturn) {
 			ld_io_ops->encode_layoutreturn(
-				&NFS_I(args->inode)->layout, xdr, args);
+				NFS_I(args->inode)->layout, xdr, args);
 		} else {
 			p = reserve_space(xdr, 4);
 			*p = cpu_to_be32(0);
fs/nfs/pnfs.c +1 -1
index 7f6ace2..9275140 100644
@@ -153,7 +153,7 @@ pnfs_need_layoutcommit(struct nfs_inode *nfsi,  
 struct nfs_open_context *ctx)
 	dprintk("%s: has_layout=%d ctx=%p\n", __func__, has_layout(nfsi),  
 ctx);
 	spin_lock(&nfsi->vfs_inode.i_lock);
 	if (has_layout(nfsi) && !layoutcommit_needed(nfsi)) {
-		nfsi->layout.lo_cred = get_rpccred(ctx->state->owner->so_cred);
+		nfsi->layout->lo_cred = get_rpccred(ctx->state->owner->so_cred);
 		__set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->pnfs_layout_state);
		nfsi->change_attr++;
		spin_unlock(&nfsi->vfs_inode.i_lock);
fs/nfs/pnfs.c +6 -6
@@ -174,17 +174,17 @@ pnfs_update_last_write(struct nfs_inode  
 *nfsi, loff_t offset, size_t extent)
 	loff_t end_pos;
 
 	spin_lock(&nfsi->vfs_inode.i_lock);
-	if (offset < nfsi->layout.pnfs_write_begin_pos)
-		nfsi->layout.pnfs_write_begin_pos = offset;
+	if (offset < nfsi->layout->pnfs_write_begin_pos)
+		nfsi->layout->pnfs_write_begin_pos = offset;
 	end_pos = offset + extent - 1; /* I'm being inclusive */
-	if (end_pos > nfsi->layout.pnfs_write_end_pos)
-		nfsi->layout.pnfs_write_end_pos = end_pos;
+	if (end_pos > nfsi->layout->pnfs_write_end_pos)
+		nfsi->layout->pnfs_write_end_pos = end_pos;
 	dprintk("%s: Wrote %lu@%lu bpos %lu, epos: %lu\n",
 		__func__,
 		(unsigned long) extent,
 		(unsigned long) offset ,
-		(unsigned long) nfsi->layout.pnfs_write_begin_pos,
-		(unsigned long) nfsi->layout.pnfs_write_end_pos);
+		(unsigned long) nfsi->layout->pnfs_write_begin_pos,
+		(unsigned long) nfsi->layout->pnfs_write_end_pos);
 	spin_unlock(&nfsi->vfs_inode.i_lock);
 }
@@ -325,10 +325,9 @@ get_layout(struct pnfs_layout_type *lo)

static inline struct pnfs_layout_type *
grab_current_layout(struct nfs_inode *nfsi)
{

fs/nfs/pnfs.c +1 -1
-	struct pnfs_layout_type *lo = &nfsi->layout;
+	struct pnfs_layout_type *lo = nfsi->layout;
Quoted 3 lines

fs/nfs/pnfs.c get_layout(struct pnfs_layout_type *lo) · line 330

Quoted 1 line
fs/nfs/pnfs.c +0 -1
-	BUG_ON_UNLOCKED_LO(lo);

Why did you drop this one?

It is called in get_layout.

Quoted 29 lines
Quoted 29 lines
fs/nfs/pnfs.c +1 -1
-	if (!lo->ld_data)
+	if (!lo)
		return NULL;
	get_layout(lo);
	return lo;
fs/nfs/pnfs.c +5 -5
@@ -342,13 +341,13 @@ put_layout(struct pnfs_layout_type *lo)
 
 	lo->refcount--;
 	if (!lo->refcount) {
-		struct layoutdriver_io_operations *io_ops =
-			PNFS_LD_IO_OPS(lo);
+		struct layoutdriver_io_operations *io_ops = PNFS_LD_IO_OPS(lo);
+		struct nfs_inode *nfsi = PNFS_NFS_INODE(lo);
 
-		dprintk("%s: freeing layout %p\n", __func__, lo->ld_data);
+		dprintk("%s: freeing layout cache %p\n", __func__, lo);
 		WARN_ON(!list_empty(&lo->lo_layouts));
-		io_ops->free_layout(lo->ld_data);
-		lo->ld_data = NULL;
+		io_ops->free_layout(lo);
+		nfsi->layout = NULL;
 	}
 }
 
@@ -688,7 +687,7 @@ pnfs_return_layout_barrier(struct nfs_inode  
 *nfsi,
 	bool ret = false;
 
Quoted 6 lines

fs/nfs/pnfs.c pnfs_return_layout_barrier(struct nfs_inode *nfsi, · line 690

Quoted 3 lines
fs/nfs/pnfs.c +1 -1
 	spin_lock(&nfsi->vfs_inode.i_lock);
-	list_for_each_entry (lseg, &nfsi->layout.segs, fi_list) {
+	list_for_each_entry(lseg, &nfsi->layout->segs, fi_list) {

White space cleanup. actually I like the space. I thought checkpatch was fixed to accept these spaces?

Nope - still there in Fedora 12.

Quoted 20 lines
Quoted 20 lines
fs/nfs/pnfs.c +0 -0
 		if (!should_free_lseg(lseg, range))
 			continue;
		lseg->valid = false;
fs/nfs/pnfs.c +2 -0
@@ -894,17 +893,20 @@ pnfs_insert_layout(struct pnfs_layout_type *lo,
 	dprintk("%s:Return\n", __func__);
 }
 
+/*
+ * Each layoutdriver embeds pnfs_layout_type in it's per-layout  

type layout
+ * cache structure. Initialize the common pnfs_layout_type fields.
+ */
static struct pnfs_layout_type * alloc_init_layout(struct inode *ino)
{

fs/nfs/pnfs.c +0 -1
-	void *ld_data;
	struct pnfs_layout_type *lo;
	struct layoutdriver_io_operations *io_ops;
Quoted 8 lines

fs/nfs/pnfs.c pnfs_insert_layout(struct pnfs_layout_type *lo, · line 908

Quoted 6 lines
	io_ops = NFS_SERVER(ino)->pnfs_curr_ld->ld_io_ops;
fs/nfs/pnfs.c +2 -3
-	lo = &NFS_I(ino)->layout;
-	ld_data = io_ops->alloc_layout(ino);
-	if (!ld_data) {
+	lo = NFS_I(ino)->layout;
+	lo = io_ops->alloc_layout(ino);

Oops, ;-)

Yeah... :)

Quoted 8 lines
Quoted 8 lines
fs/nfs/pnfs.c +1 -0
+	if (!lo) {
		printk(KERN_ERR
			"%s: out of memory: io_ops->alloc_layout failed\n",
			__func__);
fs/nfs/pnfs.c +0 -0
@@ -912,11 +914,17 @@ alloc_init_layout(struct inode *ino)
 	}
 
Quoted 6 lines

fs/nfs/pnfs.c alloc_init_layout(struct inode *ino) · line 915

Quoted 2 lines
fs/nfs/pnfs.c +0 -1
 	spin_lock(&ino->i_lock);
-	BUG_ON(lo->ld_data != NULL);

You could still do a BUG_ON(nfsi->layout) above before the io_ops-

fs/nfs/pnfs.c alloc_init_layout(struct inode *ino) · line 917

Quoted 1 line
fs/nfs/pnfs.c +0 -0
 alloc_layout(ino)

call.

OK.

Quoted 12 lines

fs/nfs/pnfs.c alloc_init_layout(struct inode *ino) · line 921

Quoted 8 lines
fs/nfs/pnfs.c +3 -2
-	lo->ld_data = ld_data;
-	memset(&lo->stateid, 0, NFS4_STATEID_SIZE);
 	lo->refcount = 1;
+	INIT_LIST_HEAD(&lo->lo_layouts);
+	INIT_LIST_HEAD(&lo->segs);
 	lo->roc_iomode = 0;
+	seqlock_init(&lo->seqlock);

All these below, I'd drop. The comment above io_ops->alloc_layout should say that it must return a ZEROed out structure, which it is in your files-layout patch.

Alright.

-->Andy

Quoted 180 lines

fs/nfs/pnfs.c alloc_init_layout(struct inode *ino) · line 928

Quoted 9 lines
fs/nfs/pnfs.c +6 -0
+	memset(&lo->stateid, 0, NFS4_STATEID_SIZE);
+	lo->lo_cred = NULL;
+	lo->pnfs_write_begin_pos = 0;
+	lo->pnfs_write_end_pos = 0;
 
+	lo->lo_inode = ino;
+	NFS_I(ino)->layout = lo;
 	spin_unlock(&ino->i_lock);

Just as a note:
A pointer/word atomic inspection theory is based on the fact that
the set is
done with a memory barrier. Same as the test/set_bit operations. So
here
the spin_unlock provides that for us, naturally. (*Not* that the
inode state will
really need it, because no one will actually care before the
allocation is done)

Quoted 156 lines
	return lo;
}
fs/nfs/pnfs.c +2 -2
@@ -962,7 +970,7 @@ get_lock_alloc_layout(struct inode *ino)
 		/* Was current_layout already allocated while we slept?
 		 * If so, retry get_lock'ing it. Otherwise, allocate it.
 		 */
-		if (nfsi->layout.ld_data) {
+		if (nfsi->layout) {
 			spin_lock(&ino->i_lock);
 			continue;
 		}
@@ -1312,7 +1320,7 @@ pnfs_set_pg_test(struct inode *inode, struct  
 nfs_pageio_descriptor *pgio)
 
 	pgio->pg_test = NULL;
 
-	laytype = &NFS_I(inode)->layout;
+	laytype = NFS_I(inode)->layout;
 	ld = NFS_SERVER(inode)->pnfs_curr_ld;
 	if (!pnfs_enabled_sb(NFS_SERVER(inode)) || !laytype)
		return;
fs/nfs/pnfs.c +1 -1
@@ -1445,7 +1453,7 @@ pnfs_writepages(struct nfs_write_data *wdata,  
 int how)
 		numpages);
 	wdata->pdata.lseg = lseg;
 	trypnfs = nfss->pnfs_curr_ld->ld_io_ops->write_pagelist(
-							&nfsi->layout,
+							nfsi->layout,
 							args->pages,
 							args->pgbase,

numpages,

fs/nfs/pnfs.c +2 -2
@@ -1498,7 +1506,7 @@ pnfs_readpages(struct nfs_read_data *rdata)
 		__func__, numpages);
 	rdata->pdata.lseg = lseg;
 	trypnfs = nfss->pnfs_curr_ld->ld_io_ops->read_pagelist(
-							&nfsi->layout,
+							nfsi->layout,
 							args->pages,
 							args->pgbase,
 							numpages,
@@ -1559,7 +1567,7 @@ pnfs_commit(struct nfs_write_data *data, int  
 sync)
 	 * This will be done by passing the buck to the layout driver.
 	 */
 	data->pdata.lseg = NULL;
-	trypnfs = nfss->pnfs_curr_ld->ld_io_ops->commit(&nfsi->layout,
+	trypnfs = nfss->pnfs_curr_ld->ld_io_ops->commit(nfsi->layout,
 							sync, data);
 	dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs);
	return trypnfs;
fs/nfs/pnfs.c +7 -7
@@ -1630,14 +1638,14 @@ pnfs_layoutcommit_inode(struct inode  
 *inode, int sync)
 	/* Clear layoutcommit properties in the inode so
 	 * new lc info can be generated
 	 */
-	write_begin_pos = nfsi->layout.pnfs_write_begin_pos;
-	write_end_pos = nfsi->layout.pnfs_write_end_pos;
-	data->cred = nfsi->layout.lo_cred;
-	nfsi->layout.pnfs_write_begin_pos = 0;
-	nfsi->layout.pnfs_write_end_pos = 0;
-	nfsi->layout.lo_cred = NULL;
+	write_begin_pos = nfsi->layout->pnfs_write_begin_pos;
+	write_end_pos = nfsi->layout->pnfs_write_end_pos;
+	data->cred = nfsi->layout->lo_cred;
+	nfsi->layout->pnfs_write_begin_pos = 0;
+	nfsi->layout->pnfs_write_end_pos = 0;
+	nfsi->layout->lo_cred = NULL;
 	__clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->pnfs_layout_state);
-	pnfs_get_layout_stateid(&data->args.stateid, &nfsi->layout);
+	pnfs_get_layout_stateid(&data->args.stateid, nfsi->layout);
 
 	spin_unlock(&inode->i_lock);
include/linux/nfs4_pnfs.h +0 -0
index 4d9b15c..6e46589 100644
@@ -35,13 +35,13 @@ struct pnfs_layoutdriver_type {

static inline struct nfs_inode *
PNFS_NFS_INODE(struct pnfs_layout_type *lo)
{

include/linux/nfs4_pnfs.h +1 -1
-	return container_of(lo, struct nfs_inode, layout);
+	return NFS_I(lo->lo_inode);

}

static inline struct inode *
PNFS_INODE(struct pnfs_layout_type *lo)
{

include/linux/nfs4_pnfs.h +1 -1
-	return &PNFS_NFS_INODE(lo)->vfs_inode;
+	return lo->lo_inode;

}

static inline struct nfs_server *

include/linux/nfs4_pnfs.h +0 -6
@@ -50,12 +50,6 @@ PNFS_NFS_SERVER(struct pnfs_layout_type *lo)
 	return NFS_SERVER(PNFS_INODE(lo));
 }
 
-static inline void *
-PNFS_LD_DATA(struct pnfs_layout_type *lo)
-{
-	return lo->ld_data;
-}
-

static inline struct pnfs_layoutdriver_type *
PNFS_LD(struct pnfs_layout_type *lo)
{

include/linux/nfs4_pnfs.h +0 -0
@@ -77,7 +71,7 @@ PNFS_LD_POLICY_OPS(struct pnfs_layout_type *lo)

static inline bool
has_layout(struct nfs_inode *nfsi)
{

include/linux/nfs4_pnfs.h +1 -1
-	return nfsi->layout.ld_data != NULL;
+	return nfsi->layout != NULL;

}

static inline bool

include/linux/nfs4_pnfs.h +0 -0
@@ -149,7 +143,7 @@ struct layoutdriver_io_operations {
 	/* Layout information. For each inode, alloc_layout is executed  

once to retrieve an
* inode specific layout structure. Each subsequent layoutget
operation results in
* a set_layout call to set the opaque layout in the layout
driver.*/

include/linux/nfs4_pnfs.h +1 -1
-	void * (*alloc_layout) (struct inode *inode);
+	struct pnfs_layout_type * (*alloc_layout) (struct inode *inode);
	void (*free_layout) (void *layoutid);
	struct pnfs_layout_segment * (*alloc_lseg) (struct  
pnfs_layout_type *layoutid, struct nfs4_pnfs_layoutget_res *lgr);
	void (*free_lseg) (struct pnfs_layout_segment *lseg);
include/linux/nfs_fs.h +2 -2
index cebc958..43d516e 100644
@@ -104,13 +104,13 @@ struct pnfs_layout_type {
 	int roc_iomode;			/* iomode to return on close, 0=none */
 	seqlock_t seqlock;		/* Protects the stateid */
 	nfs4_stateid stateid;
-	void *ld_data;			/* layout driver private data */
 	struct rpc_cred         *lo_cred; /* layoutcommit credential */
 	/* DH: These vars keep track of the maximum write range
 	 * so the values can be used for layoutcommit.
 	 */
 	loff_t                  pnfs_write_begin_pos;
 	loff_t                  pnfs_write_end_pos;
+	struct inode		*lo_inode;
 };
 
 /*
@@ -201,7 +201,7 @@ struct nfs_inode {
 	/* pNFS layout information */
 #if defined(CONFIG_NFS_V4_1)
 	wait_queue_head_t lo_waitq;
-	struct pnfs_layout_type layout;
+	struct pnfs_layout_type *layout;
 	unsigned long pnfs_layout_state;
 	#define NFS_INO_RO_LAYOUT_FAILED 0 /* ro LAYOUTGET failed stop  

trying */

Quoted 2 lines
	#define NFS_INO_RW_LAYOUT_FAILED 1 /* rw LAYOUTGET failed stop  

trying */

Boaz

#10 lore raw

Re: [PATCH 3/5] SQUASHME: pnfs-submit: move pnfs_layout_suspend back to nfs_inode

In reply to [email protected]
Quoted 6 lines · On 06/29/2010 07:42 PM, [email protected] wrote:

From: Fred Isaman <[email protected]>

Prepare to embed struct pnfs_layout_tupe into layout driver private structs and to make layout a pointer. Since a temp error on first LAYOUTGET erases the layout, the suspend timer needs to be moved.

OK Fred, Andy. (I'm also referring to the 1/5 and 2/5 issues here)

There is a total mess up. LAYOUTGET has nothing to do with layout (.eg struct pnfs_layout_type) and the layout must not be deallocated if there is any kind of error and/or if IO is actually to be done with MDS or not. LAYOUTGET corresponds to layout_seg held on a list in layout. The list might be empty and so it's flags and state.

Half of the problem was before and this here made it worse. The life time of nfsi->layout should be the same as nfsi itself just as it was before.

Once the life_time of nfsi && nfsi->layout are unified then all your problems go away because you are not checking for existence of nfsi anywhere right? that's because there is no such problem.

Look at it this way. If a layout_driver is in the game it gets to allocate it's own area in NFS_I. part of that area is for generic pnfs.c use, i.e struct pnfs_layout_type, the rest is for private use. once existing it is part of the NFS_I life up till death.

If it's not so it should be fixed, not some members leaking out to generic structures.

Boaz

Quoted 68 lines
---
 fs/nfs/inode.c         |    1 +
 fs/nfs/pnfs.c          |    8 ++++----
 include/linux/nfs_fs.h |    2 +-
 3 files changed, 6 insertions(+), 5 deletions(-)
fs/nfs/inode.c +1 -0
index 66d7be2..cb12d33 100644
@@ -1420,6 +1420,7 @@ static void pnfs_init_once(struct nfs_inode *nfsi)
 #ifdef CONFIG_NFS_V4_1
 	init_waitqueue_head(&nfsi->lo_waitq);
 	nfsi->pnfs_layout_state = 0;
+	nfsi->pnfs_layout_suspend = 0;
 	seqlock_init(&nfsi->layout.seqlock);
 	INIT_LIST_HEAD(&nfsi->layout.lo_layouts);
 	INIT_LIST_HEAD(&nfsi->layout.segs);
fs/nfs/pnfs.c +4 -4
index d05cb03..7f6ace2 100644
@@ -1105,12 +1105,12 @@ _pnfs_update_layout(struct inode *ino,
 
 	/* if get layout already failed once goto out */
 	if (test_bit(lo_fail_bit(iomode), &nfsi->pnfs_layout_state)) {
-		if (unlikely(nfsi->layout.pnfs_layout_suspend &&
-		    get_seconds() >= nfsi->layout.pnfs_layout_suspend)) {
+		if (unlikely(nfsi->pnfs_layout_suspend &&
+		    get_seconds() >= nfsi->pnfs_layout_suspend)) {
 			dprintk("%s: layout_get resumed\n", __func__);
 			clear_bit(lo_fail_bit(iomode),
 				  &nfsi->pnfs_layout_state);
-			nfsi->layout.pnfs_layout_suspend = 0;
+			nfsi->pnfs_layout_suspend = 0;
 		} else
 			goto out_put;
 	}
@@ -1224,7 +1224,7 @@ pnfs_get_layout_done(struct nfs4_pnfs_layoutget *lgp, int rpc_status)
 	}
 
 	/* remember that get layout failed and suspend trying */
-	nfsi->layout.pnfs_layout_suspend = suspend;
+	nfsi->pnfs_layout_suspend = suspend;
 	set_bit(lo_fail_bit(lgp->args.lseg.iomode),
 		&nfsi->pnfs_layout_state);
 	dprintk("%s: layout_get suspended until %ld\n",
include/linux/nfs_fs.h +1 -1
index e216e24..cebc958 100644
@@ -105,7 +105,6 @@ struct pnfs_layout_type {
 	seqlock_t seqlock;		/* Protects the stateid */
 	nfs4_stateid stateid;
 	void *ld_data;			/* layout driver private data */
-	time_t pnfs_layout_suspend;
 	struct rpc_cred         *lo_cred; /* layoutcommit credential */
 	/* DH: These vars keep track of the maximum write range
 	 * so the values can be used for layoutcommit.
@@ -208,6 +207,7 @@ struct nfs_inode {
 	#define NFS_INO_RW_LAYOUT_FAILED 1 /* rw LAYOUTGET failed stop trying */
 	#define NFS_INO_LAYOUT_ALLOC     2 /* bit lock for layout allocation */
 	#define NFS_INO_LAYOUTCOMMIT     3 /* LAYOUTCOMMIT needed */
+	time_t pnfs_layout_suspend;
 #endif /* CONFIG_NFS_V4_1 */
 #endif /* CONFIG_NFS_V4*/
 #ifdef CONFIG_NFS_FSCACHE
#11 lore raw

Re: [PATCH 3/5] SQUASHME: pnfs-submit: move pnfs_layout_suspend back to nfs_inode

Quoted 8 lines · Boaz Harrosh
Quoted 6 lines · On 06/29/2010 07:42 PM, [email protected] wrote:

From: Fred Isaman <[email protected]>

Prepare to embed struct pnfs_layout_tupe into layout driver private structs and to make layout a pointer. Since a temp error on first LAYOUTGET erases the layout, the suspend timer needs to be moved.

Quoted 7 lines

OK Fred, Andy. (I'm also referring to the 1/5 and 2/5 issues here)

There is a total mess up. LAYOUTGET has nothing to do with layout (.eg struct pnfs_layout_type) and the layout must not be deallocated if there is any kind of error and/or if IO is actually to be done with MDS or not. LAYOUTGET corresponds to layout_seg held on a list in layout. The list might be empty and so it's flags and state.

pnfs_layout_type is the layout cache head structure which has no business in nfsi unless there are layout segments populating it.

Quoted 4 lines

Half of the problem was before and this here made it worse. The life time of nfsi->layout should be the same as nfsi itself just as it was before.

No, the nfs_inode exists with no regard to pnfs_layout_type (horrible name).

Quoted 4 lines

Once the life_time of nfsi && nfsi->layout are unified then all your problems go away because you are not checking for existence of nfsi anywhere right? that's because there is no such problem.

No, the problems don't go away simply because you statically allocate a cache head struct. We have problems with races and reference counting that exist independently of how a structure is allocated.

We don't need to check for the existence of nfsi, The inode is guaranteed to exist until nfs_destroy_inode is called by the VFS.

Quoted 5 lines

Look at it this way. If a layout_driver is in the game it gets to allocate it's own area in NFS_I. part of that area is for generic pnfs.c use, i.e struct pnfs_layout_type, the rest is for private use. once existing it is part of the NFS_I life up till death.

No. It should only exists when needed - just like nfsi->nfs_acl and nfsi->delegation etc.

Quoted 79 lines

If it's not so it should be fixed, not some members leaking out to generic structures.

Boaz

Quoted 72 lines

---

fs/nfs/inode.c         |    1 +
fs/nfs/pnfs.c          |    8 ++++----
include/linux/nfs_fs.h |    2 +-
3 files changed, 6 insertions(+), 5 deletions(-)
fs/nfs/inode.c +1 -0
index 66d7be2..cb12d33 100644
@@ -1420,6 +1420,7 @@ static void pnfs_init_once(struct nfs_inode  
 *nfsi)
 #ifdef CONFIG_NFS_V4_1
 	init_waitqueue_head(&nfsi->lo_waitq);
 	nfsi->pnfs_layout_state = 0;
+	nfsi->pnfs_layout_suspend = 0;
 	seqlock_init(&nfsi->layout.seqlock);
 	INIT_LIST_HEAD(&nfsi->layout.lo_layouts);
	INIT_LIST_HEAD(&nfsi->layout.segs);
fs/nfs/pnfs.c +4 -4
index d05cb03..7f6ace2 100644
@@ -1105,12 +1105,12 @@ _pnfs_update_layout(struct inode *ino,
 
 	/* if get layout already failed once goto out */
 	if (test_bit(lo_fail_bit(iomode), &nfsi->pnfs_layout_state)) {
-		if (unlikely(nfsi->layout.pnfs_layout_suspend &&
-		    get_seconds() >= nfsi->layout.pnfs_layout_suspend)) {
+		if (unlikely(nfsi->pnfs_layout_suspend &&
+		    get_seconds() >= nfsi->pnfs_layout_suspend)) {
 			dprintk("%s: layout_get resumed\n", __func__);
 			clear_bit(lo_fail_bit(iomode),
 				  &nfsi->pnfs_layout_state);
-			nfsi->layout.pnfs_layout_suspend = 0;
+			nfsi->pnfs_layout_suspend = 0;
 		} else
 			goto out_put;
 	}
@@ -1224,7 +1224,7 @@ pnfs_get_layout_done(struct  
 nfs4_pnfs_layoutget *lgp, int rpc_status)
 	}
 
 	/* remember that get layout failed and suspend trying */
-	nfsi->layout.pnfs_layout_suspend = suspend;
+	nfsi->pnfs_layout_suspend = suspend;
 	set_bit(lo_fail_bit(lgp->args.lseg.iomode),
 		&nfsi->pnfs_layout_state);

dprintk("%s: layout_get suspended until %ld\n",

include/linux/nfs_fs.h +0 -1
index e216e24..cebc958 100644
@@ -105,7 +105,6 @@ struct pnfs_layout_type {
 	seqlock_t seqlock;		/* Protects the stateid */
 	nfs4_stateid stateid;
 	void *ld_data;			/* layout driver private data */
-	time_t pnfs_layout_suspend;
 	struct rpc_cred         *lo_cred; /* layoutcommit credential */
 	/* DH: These vars keep track of the maximum write range
 	 * so the values can be used for layoutcommit.
@@ -208,6 +207,7 @@ struct nfs_inode {
 	#define NFS_INO_RW_LAYOUT_FAILED 1 /* rw LAYOUTGET failed stop  

trying */

	#define NFS_INO_LAYOUT_ALLOC     2 /* bit lock for layout  
allocation */
	#define NFS_INO_LAYOUTCOMMIT     3 /* LAYOUTCOMMIT needed */
include/linux/nfs_fs.h +1 -0
+	time_t pnfs_layout_suspend;

#endif /* CONFIG_NFS_V4_1 */
#endif /* CONFIG_NFS_V4*/
#ifdef CONFIG_NFS_FSCACHE

#12 lore raw

Re: [PATCH 3/5] SQUASHME: pnfs-submit: move pnfs_layout_suspend back to nfs_inode

Quoted 14 lines · Andy Adamson
Quoted 11 lines · Boaz Harrosh
Quoted 7 lines · On 06/29/2010 07:42 PM, [email protected] wrote:

From: Fred Isaman <[email protected]>

Prepare to embed struct pnfs_layout_tupe into layout driver private structs
and to make layout a pointer. Since a temp error on first LAYOUTGET erases the layout, the suspend timer needs to be moved.

OK Fred, Andy. (I'm also referring to the 1/5 and 2/5 issues here)

Quoted 8 lines
Quoted 6 lines

There is a total mess up. LAYOUTGET has nothing to do with layout (.eg struct pnfs_layout_type) and the layout must not be deallocated if there is any kind of error and/or if IO is actually to be done with MDS or not. LAYOUTGET corresponds to layout_seg held on a list in layout.
The list might be empty and so it's flags and state.

pnfs_layout_type is the layout cache head structure which has no

Why are you calling this "cache" head? Are you referring to the segments list .ie layout->lo_layouts. For me it's a list head why is it a cache ?

Quoted 2 lines

business in nfsi unless there are layout segments populating it.

This is where we disagree. You say layout is lo_layouts. I say it is more, it is that plus the state and additional information. An empty list is not a none-existent list. There are two states here.

Quoted 8 lines
Quoted 4 lines

Half of the problem was before and this here made it worse. The life time of nfsi->layout should be the same as nfsi itself just as it was before.

No, the nfs_inode exists with no regard to pnfs_layout_type (horrible name).

Yes

Quoted 5 lines
Quoted 4 lines

Once the life_time of nfsi && nfsi->layout are unified then all your problems go away because you are not checking for existence of nfsi anywhere right? that's because there is no such problem.

Quoted 7 lines

No, the problems don't go away simply because you statically allocate a cache head struct. We have problems with races and reference counting that exist independently of how a structure is allocated.

We don't need to check for the existence of nfsi, The inode is guaranteed to exist until nfs_destroy_inode is called by the VFS.

And so if you restrict the life time of layout with nfsi your problem will go away as well, no?

Quoted 9 lines
Quoted 5 lines

Look at it this way. If a layout_driver is in the game it gets to allocate it's own area in NFS_I. part of that area is for generic pnfs.c use, i.e struct pnfs_layout_type, the rest is for private use. once existing it is part of the NFS_I life up till death.

No. It should only exists when needed - just like nfsi->nfs_acl and nfsi->delegation etc.

It is the same problem with what we had with commit. We had it all over the place and had races refcounting and shit for ever, until we simplified it and moved it to the proper NFS checkpoint.

Here too we can just do much better and drop lots of accounting by saying:

If this nfsi is eligible for pnfs_io and layouts. We allocate a layout governing structure for it. That will be freed at the end together with nfsi. These that never need it like directories links and so on will not have one.

Now if that first layout_get() never gets through, well nu, we only used it to say "Don't have any".

Don't you think that is a much simpler model. Why should one shoot one's foot?

Boaz

Quoted 80 lines
Quoted 79 lines

If it's not so it should be fixed, not some members leaking out to generic structures.

Boaz

Quoted 72 lines

---

fs/nfs/inode.c         |    1 +
fs/nfs/pnfs.c          |    8 ++++----
include/linux/nfs_fs.h |    2 +-
3 files changed, 6 insertions(+), 5 deletions(-)
fs/nfs/inode.c +1 -0
index 66d7be2..cb12d33 100644
@@ -1420,6 +1420,7 @@ static void pnfs_init_once(struct nfs_inode  
 *nfsi)
 #ifdef CONFIG_NFS_V4_1
 	init_waitqueue_head(&nfsi->lo_waitq);
 	nfsi->pnfs_layout_state = 0;
+	nfsi->pnfs_layout_suspend = 0;
 	seqlock_init(&nfsi->layout.seqlock);
 	INIT_LIST_HEAD(&nfsi->layout.lo_layouts);
	INIT_LIST_HEAD(&nfsi->layout.segs);
fs/nfs/pnfs.c +4 -4
index d05cb03..7f6ace2 100644
@@ -1105,12 +1105,12 @@ _pnfs_update_layout(struct inode *ino,
 
 	/* if get layout already failed once goto out */
 	if (test_bit(lo_fail_bit(iomode), &nfsi->pnfs_layout_state)) {
-		if (unlikely(nfsi->layout.pnfs_layout_suspend &&
-		    get_seconds() >= nfsi->layout.pnfs_layout_suspend)) {
+		if (unlikely(nfsi->pnfs_layout_suspend &&
+		    get_seconds() >= nfsi->pnfs_layout_suspend)) {
 			dprintk("%s: layout_get resumed\n", __func__);
 			clear_bit(lo_fail_bit(iomode),
 				  &nfsi->pnfs_layout_state);
-			nfsi->layout.pnfs_layout_suspend = 0;
+			nfsi->pnfs_layout_suspend = 0;
 		} else
 			goto out_put;
 	}
@@ -1224,7 +1224,7 @@ pnfs_get_layout_done(struct  
 nfs4_pnfs_layoutget *lgp, int rpc_status)
 	}
 
 	/* remember that get layout failed and suspend trying */
-	nfsi->layout.pnfs_layout_suspend = suspend;
+	nfsi->pnfs_layout_suspend = suspend;
 	set_bit(lo_fail_bit(lgp->args.lseg.iomode),
 		&nfsi->pnfs_layout_state);

dprintk("%s: layout_get suspended until %ld\n",

include/linux/nfs_fs.h +0 -1
index e216e24..cebc958 100644
@@ -105,7 +105,6 @@ struct pnfs_layout_type {
 	seqlock_t seqlock;		/* Protects the stateid */
 	nfs4_stateid stateid;
 	void *ld_data;			/* layout driver private data */
-	time_t pnfs_layout_suspend;
 	struct rpc_cred         *lo_cred; /* layoutcommit credential */
 	/* DH: These vars keep track of the maximum write range
 	 * so the values can be used for layoutcommit.
@@ -208,6 +207,7 @@ struct nfs_inode {
 	#define NFS_INO_RW_LAYOUT_FAILED 1 /* rw LAYOUTGET failed stop  

trying */

	#define NFS_INO_LAYOUT_ALLOC     2 /* bit lock for layout  
allocation */
	#define NFS_INO_LAYOUTCOMMIT     3 /* LAYOUTCOMMIT needed */
include/linux/nfs_fs.h +1 -0
+	time_t pnfs_layout_suspend;

#endif /* CONFIG_NFS_V4_1 */
#endif /* CONFIG_NFS_V4*/
#ifdef CONFIG_NFS_FSCACHE

#13 lore raw

Re: [PATCH 3/5] SQUASHME: pnfs-submit: move pnfs_layout_suspend back to nfs_inode

Quoted 15 lines · Boaz Harrosh
Quoted 14 lines · Andy Adamson
Quoted 11 lines · Boaz Harrosh
Quoted 7 lines · On 06/29/2010 07:42 PM, [email protected] wrote:

From: Fred Isaman <[email protected]>

Prepare to embed struct pnfs_layout_tupe into layout driver private structs
and to make layout a pointer. Since a temp error on first LAYOUTGET erases the layout, the suspend timer needs to be moved.

OK Fred, Andy. (I'm also referring to the 1/5 and 2/5 issues here)

Quoted 8 lines
Quoted 8 lines
Quoted 6 lines

There is a total mess up. LAYOUTGET has nothing to do with layout (.eg struct pnfs_layout_type) and the layout must not be deallocated if there is any kind of error and/or if IO is actually to be done with MDS or not. LAYOUTGET corresponds to layout_seg held on a list in layout.
The list might be empty and so it's flags and state.

pnfs_layout_type is the layout cache head structure which has no

Hi Boaz

Quoted 4 lines

Why are you calling this "cache" head? Are you referring to the segments list .ie layout->lo_layouts. For me it's a list head why is it a cache ?

It's a cache of layout segments implemented as a list. We may at some point use a b-tree - whatever.

Quoted 8 lines
Quoted 3 lines

business in nfsi unless there are layout segments populating it.

This is where we disagree. You say layout is lo_layouts. I say it is more, it is that plus the state and additional information. An empty list is not a none-existent list. There are two states here.

The pnfs_layout_state is in the nfsi. For file layout, there is no need for pnfs_layout_type when there are no layout segments.

What does the object layout driver need in pnfs_layout_type (and/or in private data) when there are no layout segments?

Quoted 45 lines
Quoted 9 lines
Quoted 5 lines

Half of the problem was before and this here made it worse. The life time of nfsi->layout should be the same as nfsi itself just as it was before.

No, the nfs_inode exists with no regard to pnfs_layout_type (horrible name).

Yes

Quoted 5 lines
Quoted 4 lines

Once the life_time of nfsi && nfsi->layout are unified then all your problems go away because you are not checking for existence of nfsi anywhere right? that's because there is no such problem.

Quoted 7 lines

No, the problems don't go away simply because you statically allocate a cache head struct. We have problems with races and reference counting that exist independently of how a structure is allocated.

We don't need to check for the existence of nfsi, The inode is guaranteed to exist until nfs_destroy_inode is called by the VFS.

And so if you restrict the life time of layout with nfsi your problem will go away as well, no?

Quoted 9 lines
Quoted 5 lines

Look at it this way. If a layout_driver is in the game it gets to allocate it's own area in NFS_I. part of that area is for generic pnfs.c use, i.e struct pnfs_layout_type, the rest is for private use. once existing it is part of the NFS_I life up till death.

No. It should only exists when needed - just like nfsi->nfs_acl and nfsi->delegation etc.

It is the same problem with what we had with commit. We had it all over the place and had races refcounting and shit for ever, until we simplified it and moved it to the proper NFS checkpoint.

Here too we can just do much better and drop lots of accounting by saying:

Quoted 7 lines

If this nfsi is eligible for pnfs_io and layouts. We allocate a
layout
governing structure for it. That will be freed at the end together
with
nfsi. These that never need it like directories links and so on
will not
have one.

What accounting are you removing with your scheme? If at nfs_destroy_inode you have LAYOUTRETURNS (or any LAYOUTXXX) on the wire, you have to wait until they are resolved. Exactly the same accounting as with my patch - which by the way, did not change any refcounting so I'm confused as to your complaint. Note that prior to my patches, pnfs_layout_type->lo_data was freed at the last reference! All I did was include the pnfs_layout_type because just like the lo_data, it is no longer needed.

Quoted 4 lines

Now if that first layout_get() never gets through, well nu, we only
used
it to say "Don't have any".

A NULL nfsi->layout and/or a state in nfsi->pnfs_layout_state can indicate "don't have any layout segments in the cache".

If pNFS is never used, why have the pnfs_layout_type? Should NFSv3/4.0/4.1 mounts have all these extra fields in the inode (which is a precious resource)?

Quoted 2 lines

Don't you think that is a much simpler model. Why should one shoot one's foot?

No, I don't think it's any simpler. Allocating the structure when it's needed and freeing it when it's not is the normal way to do this - just like delegations.

I do agree that we need to review the refcounting.

-->Andy

Quoted 84 lines

Boaz

Quoted 80 lines
Quoted 79 lines

If it's not so it should be fixed, not some members leaking out to generic structures.

Boaz

Quoted 72 lines

---

fs/nfs/inode.c         |    1 +
fs/nfs/pnfs.c          |    8 ++++----
include/linux/nfs_fs.h |    2 +-
3 files changed, 6 insertions(+), 5 deletions(-)
fs/nfs/inode.c +1 -0
index 66d7be2..cb12d33 100644
@@ -1420,6 +1420,7 @@ static void pnfs_init_once(struct nfs_inode
 *nfsi)
 #ifdef CONFIG_NFS_V4_1
 	init_waitqueue_head(&nfsi->lo_waitq);
 	nfsi->pnfs_layout_state = 0;
+	nfsi->pnfs_layout_suspend = 0;
 	seqlock_init(&nfsi->layout.seqlock);
 	INIT_LIST_HEAD(&nfsi->layout.lo_layouts);
	INIT_LIST_HEAD(&nfsi->layout.segs);
fs/nfs/pnfs.c +4 -4
index d05cb03..7f6ace2 100644
@@ -1105,12 +1105,12 @@ _pnfs_update_layout(struct inode *ino,
 
 	/* if get layout already failed once goto out */
 	if (test_bit(lo_fail_bit(iomode), &nfsi->pnfs_layout_state)) {
-		if (unlikely(nfsi->layout.pnfs_layout_suspend &&
-		    get_seconds() >= nfsi->layout.pnfs_layout_suspend)) {
+		if (unlikely(nfsi->pnfs_layout_suspend &&
+		    get_seconds() >= nfsi->pnfs_layout_suspend)) {
 			dprintk("%s: layout_get resumed\n", __func__);
 			clear_bit(lo_fail_bit(iomode),
 				  &nfsi->pnfs_layout_state);
-			nfsi->layout.pnfs_layout_suspend = 0;
+			nfsi->pnfs_layout_suspend = 0;
 		} else
 			goto out_put;
 	}
@@ -1224,7 +1224,7 @@ pnfs_get_layout_done(struct
 nfs4_pnfs_layoutget *lgp, int rpc_status)
 	}
 
 	/* remember that get layout failed and suspend trying */
-	nfsi->layout.pnfs_layout_suspend = suspend;
+	nfsi->pnfs_layout_suspend = suspend;
 	set_bit(lo_fail_bit(lgp->args.lseg.iomode),
 		&nfsi->pnfs_layout_state);

dprintk("%s: layout_get suspended until %ld\n",

include/linux/nfs_fs.h +0 -1
index e216e24..cebc958 100644
@@ -105,7 +105,6 @@ struct pnfs_layout_type {
 	seqlock_t seqlock;		/* Protects the stateid */
 	nfs4_stateid stateid;
 	void *ld_data;			/* layout driver private data */
-	time_t pnfs_layout_suspend;
 	struct rpc_cred         *lo_cred; /* layoutcommit credential */
 	/* DH: These vars keep track of the maximum write range
 	 * so the values can be used for layoutcommit.
@@ -208,6 +207,7 @@ struct nfs_inode {
 	#define NFS_INO_RW_LAYOUT_FAILED 1 /* rw LAYOUTGET failed stop

trying */

	#define NFS_INO_LAYOUT_ALLOC     2 /* bit lock for layout
allocation */
	#define NFS_INO_LAYOUTCOMMIT     3 /* LAYOUTCOMMIT needed */
include/linux/nfs_fs.h +1 -0
+	time_t pnfs_layout_suspend;

#endif /* CONFIG_NFS_V4_1 */
#endif /* CONFIG_NFS_V4*/
#ifdef CONFIG_NFS_FSCACHE

#14 lore raw

Re: [PATCH 3/5] SQUASHME: pnfs-submit: move pnfs_layout_suspend back to nfs_inode

Quoted 31 lines · Andy Adamson
Quoted 23 lines · Boaz Harrosh
Quoted 22 lines · Andy Adamson
Quoted 18 lines · Boaz Harrosh
Quoted 7 lines · On 06/29/2010 07:42 PM, [email protected] wrote:

From: Fred Isaman <[email protected]>

Prepare to embed struct pnfs_layout_tupe into layout driver private structs
and to make layout a pointer. Since a temp error on first LAYOUTGET erases the layout, the suspend timer needs to be moved.

OK Fred, Andy. (I'm also referring to the 1/5 and 2/5 issues here)

There is a total mess up. LAYOUTGET has nothing to do with layout (.eg struct pnfs_layout_type) and the layout must not be deallocated if there is any kind of error and/or if IO is actually to be done with
MDS or not. LAYOUTGET corresponds to layout_seg held on a list in layout.
The list might be empty and so it's flags and state.

Quoted 1 line
Quoted 1 line

pnfs_layout_type is the layout cache head structure which has no

Hi Boaz

Quoted 1 line
Quoted 7 lines
Quoted 3 lines

Why are you calling this "cache" head? Are you referring to the segments list .ie layout->lo_layouts. For me it's a list head why is it a cache ?

It's a cache of layout segments implemented as a list. We may at some point use a b-tree - whatever.

First of all It's not my day. layout->lo_layouts is that client list head which keeps the global list of nfsi's that have layouts.

So we are talking about pnfs_layout_type->segs (Why don't you fix me ;-))

cache for me is a bag-of-objects that is kept around for later use even though I was done with it. Well it might be that. We could theoretically call layout_get for every IO and immediately return it after commit, so we keep it until close or destroy, so it's a cache. (I never looked at it this way. I always look at it as a file's state. now I'm open, now I'm active IO...)

OK got you

Quoted 4 lines
Quoted 4 lines
Quoted 3 lines

business in nfsi unless there are layout segments populating it.

Quoted 8 lines
Quoted 4 lines

This is where we disagree. You say layout is lo_layouts. I say it is more, it is that plus the state and additional information. An empty list is not a none-existent list. There are two states here.

The pnfs_layout_state is in the nfsi. For file layout, there is no need for pnfs_layout_type when there are no layout segments.

There is in the generic layer. The before-layout-get state and the after-layout-return state. The layout-segments "potential" is part of the game. see your own patches. why did you have to leak some of these layout members to outside?

Quoted 3 lines

What does the object layout driver need in pnfs_layout_type (and/or in private data) when there are no layout segments?

Nothing

Quoted 57 lines
Quoted 49 lines
Quoted 9 lines
Quoted 5 lines

Half of the problem was before and this here made it worse. The life time of nfsi->layout should be the same as nfsi itself just as it was before.

No, the nfs_inode exists with no regard to pnfs_layout_type (horrible name).

Yes

Quoted 5 lines
Quoted 4 lines

Once the life_time of nfsi && nfsi->layout are unified then all your problems go away because you are not checking for existence of nfsi anywhere right? that's because there is no such problem.

Quoted 7 lines

No, the problems don't go away simply because you statically allocate a cache head struct. We have problems with races and reference counting that exist independently of how a structure is allocated.

We don't need to check for the existence of nfsi, The inode is guaranteed to exist until nfs_destroy_inode is called by the VFS.

And so if you restrict the life time of layout with nfsi your problem will go away as well, no?

Quoted 10 lines
Quoted 6 lines

Look at it this way. If a layout_driver is in the game it gets to allocate it's own area in NFS_I. part of that area is for generic pnfs.c use, i.e struct pnfs_layout_type, the rest is for private use.
once existing it is part of the NFS_I life up till death.

No. It should only exists when needed - just like nfsi->nfs_acl and nfsi->delegation etc.

It is the same problem with what we had with commit. We had it all over
the place and had races refcounting and shit for ever, until we simplified
it and moved it to the proper NFS checkpoint.

Here too we can just do much better and drop lots of accounting by saying:

Quoted 7 lines

If this nfsi is eligible for pnfs_io and layouts. We allocate a
layout
governing structure for it. That will be freed at the end together
with
nfsi. These that never need it like directories links and so on
will not
have one.

Quoted 9 lines

What accounting are you removing with your scheme? If at nfs_destroy_inode you have LAYOUTRETURNS (or any LAYOUTXXX) on the wire, you have to wait until they are resolved. Exactly the same accounting as with my patch - which by the way, did not change any refcounting so I'm confused as to your complaint. Note that prior to my patches, pnfs_layout_type->lo_data was freed at the last reference! All I did was include the pnfs_layout_type because just like the lo_data, it is no longer needed.

Yes the pnfs_layout_type->lo_data was half used as a state variable. We don't have to do that. We can just keep it around up until nfsi destructor. This one is surly properly ref-counted, right. So we can just drop our private ref-counting.

Quoted 5 lines
Quoted 4 lines

Now if that first layout_get() never gets through, well nu, we only
used
it to say "Don't have any".

Quoted 7 lines

A NULL nfsi->layout and/or a state in nfsi->pnfs_layout_state can indicate "don't have any layout segments in the cache".

If pNFS is never used, why have the pnfs_layout_type? Should NFSv3/4.0/4.1 mounts have all these extra fields in the inode (which is a precious resource)?

What? I thought it is understood that these patches was to remove that, sure, and I agree.
For all modes other then pnfs+regular-rw-files this structure pointer is NULL. For pnfs-IO-potential files this structure is allocated, together with all it's members and glory.

so:
* nfsi->layout == NULL - This file will not need pnfs (v234 other)
* nfsi->layout->segs == EMPTY - Before layout_get or after layout_return
* nfsi->layout->XXX - other layout state global to any segments.

Quoted 8 lines
Quoted 3 lines

Don't you think that is a much simpler model. Why should one shoot one's foot?

No, I don't think it's any simpler. Allocating the structure when it's needed and freeing it when it's not is the normal way to do this - just like delegations.

I don't know about delegations. Just here. What I'm saying is lets collapse some of the extra stuff, by using existing life-time rules.

You know what, it looks like this is a big jump for you right now. Leave it like that. Lets run and stabilize this stage. I will put it on my todo list to try and sanitize it farther before we submit.

Quoted 5 lines

I do agree that we need to review the refcounting.

-->Andy

Quoted 1 line

Thanks, this all is moving in the right directions, eventually we will get there.

Boaz

#15 lore raw

Re: [PATCH 2/5] SQUASHME: pnfs_submit: move pnfs_layout_state back to nfs_inode

In reply to [email protected]
Quoted 6 lines · On 06/29/2010 07:42 PM, [email protected] wrote:

From: Andy Adamson <[email protected]>

Prepare to embed struct pnfs_layout_tupe into layout driver private structs and to make layout a pointer: state flags need to be accessible before allocation.

I don't get that. A pointer inspection operation is atomic. (gone are the days of 16-bit large/huge models where a pointer is two registers.)

So an: if (lo->pointer) is just as atomic as test_bit()

And I don't see, not here, and not in the next patch, where are these used before the allocation. if you throw away Fred's first patch and make:
layoutcommit_needed(struct nfs_inode *nfsi)
{
- return nfsi->layout.lo_cred != NULL;
+ return nfsi->layout != NULL;
}

Then all three patches (first one thrown away) can just collapse to a single embedded-to-pointer conversion.

Boaz

Quoted 118 lines
---
 fs/nfs/inode.c         |    2 +-
 fs/nfs/pnfs.c          |   16 ++++++++--------
 include/linux/nfs_fs.h |   10 +++++-----
 3 files changed, 14 insertions(+), 14 deletions(-)
fs/nfs/inode.c +1 -1
index 7989dea..66d7be2 100644
@@ -1364,7 +1364,6 @@ void nfs4_clear_inode(struct inode *inode)
 static void pnfs_alloc_init_inode(struct nfs_inode *nfsi)
 {
 #ifdef CONFIG_NFS_V4_1
-	nfsi->layout.pnfs_layout_state = 0;
 	memset(&nfsi->layout.stateid, 0, NFS4_STATEID_SIZE);
 	nfsi->layout.roc_iomode = 0;
 	nfsi->layout.lo_cred = NULL;
@@ -1420,6 +1419,7 @@ static void pnfs_init_once(struct nfs_inode *nfsi)
 {
 #ifdef CONFIG_NFS_V4_1
 	init_waitqueue_head(&nfsi->lo_waitq);
+	nfsi->pnfs_layout_state = 0;
 	seqlock_init(&nfsi->layout.seqlock);
 	INIT_LIST_HEAD(&nfsi->layout.lo_layouts);
 	INIT_LIST_HEAD(&nfsi->layout.segs);
fs/nfs/pnfs.c +8 -8
index bf15b5c..d05cb03 100644
@@ -951,7 +951,7 @@ get_lock_alloc_layout(struct inode *ino)
 		 * wait until bit is cleared if we lost this race.
 		 */
 		res = wait_on_bit_lock(
-			&nfsi->layout.pnfs_layout_state,
+			&nfsi->pnfs_layout_state,
 			NFS_INO_LAYOUT_ALLOC,
 			pnfs_wait_schedule, TASK_KILLABLE);
 		if (res) {
@@ -975,8 +975,8 @@ get_lock_alloc_layout(struct inode *ino)
 
 		/* release the NFS_INO_LAYOUT_ALLOC bit and wake up waiters */
 		clear_bit_unlock(NFS_INO_LAYOUT_ALLOC,
-				 &nfsi->layout.pnfs_layout_state);
-		wake_up_bit(&nfsi->layout.pnfs_layout_state,
+				 &nfsi->pnfs_layout_state);
+		wake_up_bit(&nfsi->pnfs_layout_state,
 			    NFS_INO_LAYOUT_ALLOC);
 		break;
 	}
@@ -1104,12 +1104,12 @@ _pnfs_update_layout(struct inode *ino,
 	}
 
 	/* if get layout already failed once goto out */
-	if (test_bit(lo_fail_bit(iomode), &nfsi->layout.pnfs_layout_state)) {
+	if (test_bit(lo_fail_bit(iomode), &nfsi->pnfs_layout_state)) {
 		if (unlikely(nfsi->layout.pnfs_layout_suspend &&
 		    get_seconds() >= nfsi->layout.pnfs_layout_suspend)) {
 			dprintk("%s: layout_get resumed\n", __func__);
 			clear_bit(lo_fail_bit(iomode),
-				  &nfsi->layout.pnfs_layout_state);
+				  &nfsi->pnfs_layout_state);
 			nfsi->layout.pnfs_layout_suspend = 0;
 		} else
 			goto out_put;
@@ -1121,7 +1121,7 @@ _pnfs_update_layout(struct inode *ino,
 	send_layoutget(ino, ctx, &arg, lsegpp, lo);
 out:
 	dprintk("%s end, state 0x%lx lseg %p\n", __func__,
-		nfsi->layout.pnfs_layout_state, lseg);
+		nfsi->pnfs_layout_state, lseg);
 	return;
 out_put:
 	if (lsegpp)
@@ -1226,12 +1226,12 @@ pnfs_get_layout_done(struct nfs4_pnfs_layoutget *lgp, int rpc_status)
 	/* remember that get layout failed and suspend trying */
 	nfsi->layout.pnfs_layout_suspend = suspend;
 	set_bit(lo_fail_bit(lgp->args.lseg.iomode),
-		&nfsi->layout.pnfs_layout_state);
+		&nfsi->pnfs_layout_state);
 	dprintk("%s: layout_get suspended until %ld\n",
 		__func__, suspend);
 out:
 	dprintk("%s end (err:%d) state 0x%lx lseg %p\n",
-		__func__, lgp->status, nfsi->layout.pnfs_layout_state, lseg);
+		__func__, lgp->status, nfsi->pnfs_layout_state, lseg);
 	return;
 }
 
include/linux/nfs_fs.h +5 -5
index a33e86e..e216e24 100644
@@ -105,11 +105,6 @@ struct pnfs_layout_type {
 	seqlock_t seqlock;		/* Protects the stateid */
 	nfs4_stateid stateid;
 	void *ld_data;			/* layout driver private data */
-	unsigned long pnfs_layout_state;
-	#define NFS_INO_RO_LAYOUT_FAILED 0      /* get ro layout failed stop trying */
-	#define NFS_INO_RW_LAYOUT_FAILED 1      /* get rw layout failed stop trying */
-	#define NFS_INO_LAYOUT_ALLOC     2      /* bit lock for layout allocation */
-	#define NFS_INO_LAYOUTCOMMIT     3	/* LAYOUTCOMMIT needed */
 	time_t pnfs_layout_suspend;
 	struct rpc_cred         *lo_cred; /* layoutcommit credential */
 	/* DH: These vars keep track of the maximum write range
@@ -208,6 +203,11 @@ struct nfs_inode {
 #if defined(CONFIG_NFS_V4_1)
 	wait_queue_head_t lo_waitq;
 	struct pnfs_layout_type layout;
+	unsigned long pnfs_layout_state;
+	#define NFS_INO_RO_LAYOUT_FAILED 0 /* ro LAYOUTGET failed stop trying */
+	#define NFS_INO_RW_LAYOUT_FAILED 1 /* rw LAYOUTGET failed stop trying */
+	#define NFS_INO_LAYOUT_ALLOC     2 /* bit lock for layout allocation */
+	#define NFS_INO_LAYOUTCOMMIT     3 /* LAYOUTCOMMIT needed */
 #endif /* CONFIG_NFS_V4_1 */
 #endif /* CONFIG_NFS_V4*/
 #ifdef CONFIG_NFS_FSCACHE
#16 lore raw

Re: [PATCH 2/5] SQUASHME: pnfs_submit: move pnfs_layout_state back to nfs_inode

Quoted 18 lines · Boaz Harrosh
Quoted 6 lines · On 06/29/2010 07:42 PM, [email protected] wrote:

From: Andy Adamson <[email protected]>

Prepare to embed struct pnfs_layout_tupe into layout driver private structs and to make layout a pointer: state flags need to be accessible before allocation.

I don't get that. A pointer inspection operation is atomic. (gone are the days of 16-bit large/huge models where a pointer is two registers.)

So an: if (lo->pointer) is just as atomic as test_bit()

And I don't see, not here, and not in the next patch, where are these used before the allocation. if you throw away Fred's first patch and make:
layoutcommit_needed(struct nfs_inode *nfsi)
{

Review

Quoted 3 lines
(quoted diff) +1 -1
-	return nfsi->layout.lo_cred != NULL;
+	return nfsi->layout != NULL;
 }

Rrrr need my morning coffee, sorry.

But still, Fred's patch can go both sites of the pointer assignment have an MB and a pointer inspection is there-for atomic. Above can then be:

layoutcommit_needed(struct nfs_inode *nfsi)
{
- return nfsi->layout.lo_cred != NULL;
+ return nfsi->layout && (nfsi->layout->lo_cred != NULL);
}

Quoted 3 lines

Then all three patches (first one thrown away) can just collapse to a single embedded-to-pointer conversion.

This still applies

Boaz

Quoted 120 lines

Boaz

Quoted 118 lines
---
 fs/nfs/inode.c         |    2 +-
 fs/nfs/pnfs.c          |   16 ++++++++--------
 include/linux/nfs_fs.h |   10 +++++-----
 3 files changed, 14 insertions(+), 14 deletions(-)
fs/nfs/inode.c +1 -1
index 7989dea..66d7be2 100644
@@ -1364,7 +1364,6 @@ void nfs4_clear_inode(struct inode *inode)
 static void pnfs_alloc_init_inode(struct nfs_inode *nfsi)
 {
 #ifdef CONFIG_NFS_V4_1
-	nfsi->layout.pnfs_layout_state = 0;
 	memset(&nfsi->layout.stateid, 0, NFS4_STATEID_SIZE);
 	nfsi->layout.roc_iomode = 0;
 	nfsi->layout.lo_cred = NULL;
@@ -1420,6 +1419,7 @@ static void pnfs_init_once(struct nfs_inode *nfsi)
 {
 #ifdef CONFIG_NFS_V4_1
 	init_waitqueue_head(&nfsi->lo_waitq);
+	nfsi->pnfs_layout_state = 0;
 	seqlock_init(&nfsi->layout.seqlock);
 	INIT_LIST_HEAD(&nfsi->layout.lo_layouts);
 	INIT_LIST_HEAD(&nfsi->layout.segs);
fs/nfs/pnfs.c +8 -8
index bf15b5c..d05cb03 100644
@@ -951,7 +951,7 @@ get_lock_alloc_layout(struct inode *ino)
 		 * wait until bit is cleared if we lost this race.
 		 */
 		res = wait_on_bit_lock(
-			&nfsi->layout.pnfs_layout_state,
+			&nfsi->pnfs_layout_state,
 			NFS_INO_LAYOUT_ALLOC,
 			pnfs_wait_schedule, TASK_KILLABLE);
 		if (res) {
@@ -975,8 +975,8 @@ get_lock_alloc_layout(struct inode *ino)
 
 		/* release the NFS_INO_LAYOUT_ALLOC bit and wake up waiters */
 		clear_bit_unlock(NFS_INO_LAYOUT_ALLOC,
-				 &nfsi->layout.pnfs_layout_state);
-		wake_up_bit(&nfsi->layout.pnfs_layout_state,
+				 &nfsi->pnfs_layout_state);
+		wake_up_bit(&nfsi->pnfs_layout_state,
 			    NFS_INO_LAYOUT_ALLOC);
 		break;
 	}
@@ -1104,12 +1104,12 @@ _pnfs_update_layout(struct inode *ino,
 	}
 
 	/* if get layout already failed once goto out */
-	if (test_bit(lo_fail_bit(iomode), &nfsi->layout.pnfs_layout_state)) {
+	if (test_bit(lo_fail_bit(iomode), &nfsi->pnfs_layout_state)) {
 		if (unlikely(nfsi->layout.pnfs_layout_suspend &&
 		    get_seconds() >= nfsi->layout.pnfs_layout_suspend)) {
 			dprintk("%s: layout_get resumed\n", __func__);
 			clear_bit(lo_fail_bit(iomode),
-				  &nfsi->layout.pnfs_layout_state);
+				  &nfsi->pnfs_layout_state);
 			nfsi->layout.pnfs_layout_suspend = 0;
 		} else
 			goto out_put;
@@ -1121,7 +1121,7 @@ _pnfs_update_layout(struct inode *ino,
 	send_layoutget(ino, ctx, &arg, lsegpp, lo);
 out:
 	dprintk("%s end, state 0x%lx lseg %p\n", __func__,
-		nfsi->layout.pnfs_layout_state, lseg);
+		nfsi->pnfs_layout_state, lseg);
 	return;
 out_put:
 	if (lsegpp)
@@ -1226,12 +1226,12 @@ pnfs_get_layout_done(struct nfs4_pnfs_layoutget *lgp, int rpc_status)
 	/* remember that get layout failed and suspend trying */
 	nfsi->layout.pnfs_layout_suspend = suspend;
 	set_bit(lo_fail_bit(lgp->args.lseg.iomode),
-		&nfsi->layout.pnfs_layout_state);
+		&nfsi->pnfs_layout_state);
 	dprintk("%s: layout_get suspended until %ld\n",
 		__func__, suspend);
 out:
 	dprintk("%s end (err:%d) state 0x%lx lseg %p\n",
-		__func__, lgp->status, nfsi->layout.pnfs_layout_state, lseg);
+		__func__, lgp->status, nfsi->pnfs_layout_state, lseg);
 	return;
 }
 
include/linux/nfs_fs.h +5 -5
index a33e86e..e216e24 100644
@@ -105,11 +105,6 @@ struct pnfs_layout_type {
 	seqlock_t seqlock;		/* Protects the stateid */
 	nfs4_stateid stateid;
 	void *ld_data;			/* layout driver private data */
-	unsigned long pnfs_layout_state;
-	#define NFS_INO_RO_LAYOUT_FAILED 0      /* get ro layout failed stop trying */
-	#define NFS_INO_RW_LAYOUT_FAILED 1      /* get rw layout failed stop trying */
-	#define NFS_INO_LAYOUT_ALLOC     2      /* bit lock for layout allocation */
-	#define NFS_INO_LAYOUTCOMMIT     3	/* LAYOUTCOMMIT needed */
 	time_t pnfs_layout_suspend;
 	struct rpc_cred         *lo_cred; /* layoutcommit credential */
 	/* DH: These vars keep track of the maximum write range
@@ -208,6 +203,11 @@ struct nfs_inode {
 #if defined(CONFIG_NFS_V4_1)
 	wait_queue_head_t lo_waitq;
 	struct pnfs_layout_type layout;
+	unsigned long pnfs_layout_state;
+	#define NFS_INO_RO_LAYOUT_FAILED 0 /* ro LAYOUTGET failed stop trying */
+	#define NFS_INO_RW_LAYOUT_FAILED 1 /* rw LAYOUTGET failed stop trying */
+	#define NFS_INO_LAYOUT_ALLOC     2 /* bit lock for layout allocation */
+	#define NFS_INO_LAYOUTCOMMIT     3 /* LAYOUTCOMMIT needed */
 #endif /* CONFIG_NFS_V4_1 */
 #endif /* CONFIG_NFS_V4*/
 #ifdef CONFIG_NFS_FSCACHE
#17 lore raw

Re: [PATCH 2/5] SQUASHME: pnfs_submit: move pnfs_layout_state back to nfs_inode

On Wed, Jun 30, 2010 at 5:31 AM, Boaz Harrosh <[email protected]> wr= ote:

Quoted 5 lines · Boaz Harrosh
Quoted 4 lines · On 06/29/2010 07:42 PM, [email protected] wrote:
Quoted 3 lines

From: Andy Adamson <[email protected]>

Prepare to embed struct pnfs_layout_tupe into layout driver private=

structs

Quoted 1 line
Quoted 1 line
Quoted 1 line

and to make layout a pointer: state flags need to be accessible bef=

ore

Quoted 4 lines
Quoted 4 lines
Quoted 2 lines

allocation.

I don't get that. A pointer inspection operation is atomic. (gone ar=

e the

Quoted 5 lines
Quoted 5 lines

days of 16-bit large/huge models where a pointer is two registers.)

So an: if (lo->pointer) is just as atomic as test_bit()

And I don't see, not here, and not in the next patch, where are thes=

e used before

Quoted 11 lines
Quoted 7 lines

the allocation. if you throw away Fred's first patch and make:
=A0layoutcommit_needed(struct nfs_inode *nfsi)
=A0{
- =A0 =A0 return nfsi->layout.lo_cred !=3D NULL;
+ =A0 =A0 return nfsi->layout !=3D NULL;
=A0}

Rrrr need my morning coffee, sorry.

But still, Fred's patch can go both sites of the pointer assignment h=

ave an

Quoted 6 lines

MB and a pointer inspection is there-for atomic. Above can then be:

=A0layoutcommit_needed(struct nfs_inode *nfsi)
=A0{
- =A0 =A0 =A0 return nfsi->layout.lo_cred !=3D NULL;
+ =A0 =A0 =A0 return nfsi->layout && (nfsi->layout->lo_cred !=3D NULL=

);

Quoted 2 lines

=A0}

This is called outside of pnfs.c, without lock. Thus there is no guarantee that nfsi->layout won't go to to NULL during the '&&'.

=46red

Quoted 2 lines
Quoted 2 lines

Then all three patches (first one thrown away) can just collapse to =

a single

Quoted 14 lines
Quoted 2 lines

embedded-to-pointer conversion.

This still applies

Boaz

Quoted 8 lines

Boaz

Quoted 7 lines

---
=A0fs/nfs/inode.c =A0 =A0 =A0 =A0 | =A0 =A02 +- =A0fs/nfs/pnfs.c =A0 =A0 =A0 =A0 =A0| =A0 16 ++++++++-------- =A0include/linux/nfs_fs.h | =A0 10 +++++-----
=A03 files changed, 14 insertions(+), 14 deletions(-)

Quoted 13 lines
Quoted 5 lines
Quoted 5 lines
fs/nfs/inode.c +0 -0
index 7989dea..66d7be2 100644
@@ -1364,7 +1364,6 @@ void nfs4_clear_inode(struct inode *inode)
Quoted 8 lines

fs/nfs/inode.c static void pnfs_init_once(struct nfs_inode *=

Quoted 8 lines
fs/nfs/inode.c +0 -0
 =A0static void pnfs_alloc_init_inode(struct nfs_inode *nfsi)
 =A0{

=A0#ifdef CONFIG_NFS_V4_1
- =A0 =A0nfsi->layout.pnfs_layout_state =3D 0;
=A0 =A0 =A0memset(&nfsi->layout.stateid, 0, NFS4_STATEID_SIZE); =A0 =A0 =A0nfsi->layout.roc_iomode =3D 0;
=A0 =A0 =A0nfsi->layout.lo_cred =3D NULL;

fs/nfs/inode.c +0 -0
@@ -1420,6 +1419,7 @@ static void pnfs_init_once(struct nfs_inode *=

nfsi)

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
fs/nfs/inode.c +0 -0
 =A0{

=A0#ifdef CONFIG_NFS_V4_1
=A0 =A0 =A0init_waitqueue_head(&nfsi->lo_waitq);
+ =A0 =A0nfsi->pnfs_layout_state =3D 0;
=A0 =A0 =A0seqlock_init(&nfsi->layout.seqlock);

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

=A0 =A0 =A0INIT_LIST_HEAD(&nfsi->layout.lo_layouts); =A0 =A0 =A0INIT_LIST_HEAD(&nfsi->layout.segs);

fs/nfs/pnfs.c +0 -0
index bf15b5c..d05cb03 100644
@@ -951,7 +951,7 @@ get_lock_alloc_layout(struct inode *ino)

=A0 =A0 =A0 =A0 =A0 =A0 =A0 * wait until bit is cleared if we lost =

this race.

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

=A0 =A0 =A0 =A0 =A0 =A0 =A0 */
=A0 =A0 =A0 =A0 =A0 =A0 =A0res =3D wait_on_bit_lock(
- =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0&nfsi->layout.pnfs_layout_=

state,

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0&nfsi->pnfs_layout_state, =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0NFS_INO_LAYOUT_ALLOC, =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0pnfs_wait_schedule, TASK=

_KILLABLE);

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

=A0 =A0 =A0 =A0 =A0 =A0 =A0if (res) {

fs/nfs/pnfs.c +0 -0
@@ -975,8 +975,8 @@ get_lock_alloc_layout(struct inode *ino)
 

=A0 =A0 =A0 =A0 =A0 =A0 =A0/* release the NFS_INO_LAYOUT_ALLOC bit =

and wake up waiters */

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

=A0 =A0 =A0 =A0 =A0 =A0 =A0clear_bit_unlock(NFS_INO_LAYOUT_ALLOC,
- =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 &nfsi->la=

yout.pnfs_layout_state);

Quoted 1 line
Quoted 1 line
Quoted 1 line

- =A0 =A0 =A0 =A0 =A0 =A0wake_up_bit(&nfsi->layout.pnfs_layout_stat=

e,

Quoted 1 line
Quoted 1 line
Quoted 1 line

+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 &nfsi->pn=

fs_layout_state);

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

+ =A0 =A0 =A0 =A0 =A0 =A0wake_up_bit(&nfsi->pnfs_layout_state, =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0NFS_INO_LAYOUT_A=

LLOC);

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

=A0 =A0 =A0 =A0 =A0 =A0 =A0break;
=A0 =A0 =A0}

fs/nfs/pnfs.c +0 -0
@@ -1104,12 +1104,12 @@ _pnfs_update_layout(struct inode *ino,
 =A0 =A0 =A0}
 

=A0 =A0 =A0/* if get layout already failed once goto out */
- =A0 =A0if (test_bit(lo_fail_bit(iomode), &nfsi->layout.pnfs_layou=

t_state)) {

Quoted 1 line
Quoted 1 line
Quoted 1 line

+ =A0 =A0if (test_bit(lo_fail_bit(iomode), &nfsi->pnfs_layout_state=

)) {

Quoted 1 line
Quoted 1 line
Quoted 1 line

=A0 =A0 =A0 =A0 =A0 =A0 =A0if (unlikely(nfsi->layout.pnfs_layout_su=

spend &&

Quoted 1 line
Quoted 1 line
Quoted 1 line

=A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0get_seconds() >=3D nfsi->layout.=

pnfs_layout_suspend)) {

Quoted 1 line
Quoted 1 line
Quoted 1 line

=A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0dprintk("%s: layout_get =

resumed\n", __func__);

Quoted 1 line
Quoted 1 line
Quoted 1 line

=A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0clear_bit(lo_fail_bit(io=

mode),

Quoted 5 lines
Quoted 1 line
Quoted 1 line

- =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0&nfsi-=

layout.pnfs_layout_state);

Quoted 1 line
Quoted 1 line

+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0&nfsi-=

pnfs_layout_state);

Quoted 1 line
Quoted 1 line

=A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0nfsi->layout.pnfs_layout=

_suspend =3D 0;

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

=A0 =A0 =A0 =A0 =A0 =A0 =A0} else
=A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0goto out_put;

fs/nfs/pnfs.c +0 -0
@@ -1121,7 +1121,7 @@ _pnfs_update_layout(struct inode *ino,
 =A0 =A0 =A0send_layoutget(ino, ctx, &arg, lsegpp, lo);
Quoted 8 lines
Quoted 8 lines

fs/nfs/pnfs.c pnfs_get_layout_done(struct nfs4_pnfs_layou=

Quoted 8 lines

=A0out:
=A0 =A0 =A0dprintk("%s end, state 0x%lx lseg %p\n", __func__,
- =A0 =A0 =A0 =A0 =A0 =A0nfsi->layout.pnfs_layout_state, lseg);
+ =A0 =A0 =A0 =A0 =A0 =A0nfsi->pnfs_layout_state, lseg); =A0 =A0 =A0return;
=A0out_put:
=A0 =A0 =A0if (lsegpp)

fs/nfs/pnfs.c +0 -0
@@ -1226,12 +1226,12 @@ pnfs_get_layout_done(struct nfs4_pnfs_layou=

tget *lgp, int rpc_status)

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines

=A0 =A0 =A0/* remember that get layout failed and suspend trying */ =A0 =A0 =A0nfsi->layout.pnfs_layout_suspend =3D suspend; =A0 =A0 =A0set_bit(lo_fail_bit(lgp->args.lseg.iomode),
- =A0 =A0 =A0 =A0 =A0 =A0&nfsi->layout.pnfs_layout_state);
+ =A0 =A0 =A0 =A0 =A0 =A0&nfsi->pnfs_layout_state);
=A0 =A0 =A0dprintk("%s: layout_get suspended until %ld\n", =A0 =A0 =A0 =A0 =A0 =A0 =A0__func__, suspend);
=A0out:
=A0 =A0 =A0dprintk("%s end (err:%d) state 0x%lx lseg %p\n",
- =A0 =A0 =A0 =A0 =A0 =A0__func__, lgp->status, nfsi->layout.pnfs_l=

ayout_state, lseg);

Quoted 1 line
Quoted 1 line
Quoted 1 line

+ =A0 =A0 =A0 =A0 =A0 =A0__func__, lgp->status, nfsi->pnfs_layout_s=

tate, lseg);

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines

=A0 =A0 =A0return;
=A0}

include/linux/nfs_fs.h +0 -0
index a33e86e..e216e24 100644
@@ -105,11 +105,6 @@ struct pnfs_layout_type {

=A0 =A0 =A0seqlock_t seqlock; =A0 =A0 =A0 =A0 =A0 =A0 =A0/* Protect=

s the stateid */

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

=A0 =A0 =A0nfs4_stateid stateid;
=A0 =A0 =A0void *ld_data; =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0/* lay=

out driver private data */

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

- =A0 =A0unsigned long pnfs_layout_state;
- =A0 =A0#define NFS_INO_RO_LAYOUT_FAILED 0 =A0 =A0 =A0/* get ro la=

yout failed stop trying */

Quoted 1 line
Quoted 1 line
Quoted 1 line

- =A0 =A0#define NFS_INO_RW_LAYOUT_FAILED 1 =A0 =A0 =A0/* get rw la=

yout failed stop trying */

Quoted 1 line
Quoted 1 line
Quoted 1 line

- =A0 =A0#define NFS_INO_LAYOUT_ALLOC =A0 =A0 2 =A0 =A0 =A0/* bit l=

ock for layout allocation */

Quoted 1 line
Quoted 1 line
Quoted 1 line

- =A0 =A0#define NFS_INO_LAYOUTCOMMIT =A0 =A0 3 =A0 =A0 =A0/* LAYOU=

TCOMMIT needed */

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

=A0 =A0 =A0time_t pnfs_layout_suspend;
=A0 =A0 =A0struct rpc_cred =A0 =A0 =A0 =A0 *lo_cred; /* layoutcommi=

t credential */

Quoted 7 lines
Quoted 7 lines

include/linux/nfs_fs.h struct nfs_inode { · line 207

Quoted 7 lines

=A0 =A0 =A0/* DH: These vars keep track of the maximum write range

include/linux/nfs_fs.h +2 -0
@@ -208,6 +203,11 @@ struct nfs_inode {
 =A0#if defined(CONFIG_NFS_V4_1)
 =A0 =A0 =A0wait_queue_head_t lo_waitq;
 =A0 =A0 =A0struct pnfs_layout_type layout;
+ =A0 =A0unsigned long pnfs_layout_state;
+ =A0 =A0#define NFS_INO_RO_LAYOUT_FAILED 0 /* ro LAYOUTGET failed =

stop trying */

Quoted 1 line
Quoted 1 line

include/linux/nfs_fs.h struct nfs_inode { · line 208

Quoted 1 line
include/linux/nfs_fs.h +1 -0
+ =A0 =A0#define NFS_INO_RW_LAYOUT_FAILED 1 /* rw LAYOUTGET failed =

stop trying */

Quoted 1 line
Quoted 1 line

include/linux/nfs_fs.h struct nfs_inode { · line 209

Quoted 1 line
include/linux/nfs_fs.h +1 -0
+ =A0 =A0#define NFS_INO_LAYOUT_ALLOC =A0 =A0 2 /* bit lock for lay=

out allocation */

Quoted 1 line
Quoted 1 line

include/linux/nfs_fs.h struct nfs_inode { · line 210

Quoted 1 line
include/linux/nfs_fs.h +1 -0
+ =A0 =A0#define NFS_INO_LAYOUTCOMMIT =A0 =A0 3 /* LAYOUTCOMMIT nee=

ded */

Quoted 7 lines
Quoted 4 lines
Quoted 3 lines

=A0#endif /* CONFIG_NFS_V4_1 */
=A0#endif /* CONFIG_NFS_V4*/
=A0#ifdef CONFIG_NFS_FSCACHE

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" =

in

Quoted 3 lines

the body of a message to [email protected]
More majordomo info at =A0http://vger.kernel.org/majordomo-info.html