Skip to contentSkip to conversation

OCFS2: anti stale inode for nfs (V4)

PATCH New wengang wang 3 participants 10 messages Feb 20, 2009 – Feb 27, 2009 OCFS2

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 OCFS2: anti stale inode for nfs (V4)

changes from v3:
1, move codes that checks inode allocation bit to subfunction ocfs2_test_inode_bit().

2, release the suballoc lock just after we get it. we should release it asap and doing so doesn't affect functionility.

3, add inode alloc slot validation.

--

 dlmglue.c      |   45 +++++++++++++++++
 dlmglue.h      |    2
 export.c       |   75 +++++++++++++++++++++++++---
 inode.c        |   24 ++++++++-
 inode.h        |    1
 ocfs2.h        |    1
 ocfs2_lockid.h |    4 +
 suballoc.c     |  150 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 suballoc.h     |    2
 9 files changed, 295 insertions(+), 9 deletions(-)
dlmglue.h +2 -0
Index: dlmglue.h
===================================================================
@@ -115,6 +115,8 @@ void ocfs2_super_unlock(struct ocfs2_sup
 			int ex);
 int ocfs2_rename_lock(struct ocfs2_super *osb);
 void ocfs2_rename_unlock(struct ocfs2_super *osb);
+int ocfs2_nfs_sync_lock(struct ocfs2_super *osb, int ex);
+void ocfs2_nfs_sync_unlock(struct ocfs2_super *osb, int ex);
 int ocfs2_dentry_lock(struct dentry *dentry, int ex);
 void ocfs2_dentry_unlock(struct dentry *dentry, int ex);
 int ocfs2_file_lock(struct file *file, int ex, int trylock);
export.c +67 -8

102 lines not on this page · raw message

inode.c +23 -1
Index: inode.c
===================================================================
@@ -112,6 +112,17 @@ void ocfs2_get_inode_flags(struct ocfs2_
 		oi->ip_attr |= OCFS2_DIRSYNC_FL;
 }
 
+struct inode *ocfs2_ilookup(struct super_block *sb, u64 blkno)
+{
+	struct ocfs2_find_inode_args args;
+
+	args.fi_blkno = blkno;
+	args.fi_flags = 0;
+	args.fi_ino = ino_from_blkno(sb, blkno);
+	args.fi_sysfile_type = 0;
+
+	return ilookup5(sb, blkno, ocfs2_find_actor, &args);
+}
 struct inode *ocfs2_iget(struct ocfs2_super *osb, u64 blkno, unsigned flags,
 			 int sysfile_type)
 {
@@ -949,6 +960,13 @@ void ocfs2_delete_inode(struct inode *in
 		goto bail;
 	}
 
+	/* Lock down the nfs_sync lock in PR mode */
+	status = ocfs2_nfs_sync_lock(OCFS2_SB(inode->i_sb), 0);
+	if (status < 0) {
+		mlog(ML_ERROR, "getting nfs sync lock(PR) failed %d\n", status);
+		ocfs2_cleanup_delete_inode(inode, 0);
+		goto bail_unblock;
+	}
 	/* Lock down the inode. This gives us an up to date view of
 	 * it's metadata (for verification), and allows us to
 	 * serialize delete_inode on multiple nodes.
@@ -962,7 +980,7 @@ void ocfs2_delete_inode(struct inode *in
 		if (status != -ENOENT)
 			mlog_errno(status);
 		ocfs2_cleanup_delete_inode(inode, 0);
-		goto bail_unblock;
+		goto bail_unlock_nfs_sync;
 	}
 
 	/* Query the cluster. This will be the final decision made
@@ -1005,6 +1023,10 @@ void ocfs2_delete_inode(struct inode *in
 bail_unlock_inode:
 	ocfs2_inode_unlock(inode, 1);
 	brelse(di_bh);
+
+bail_unlock_nfs_sync:
+	ocfs2_nfs_sync_unlock(OCFS2_SB(inode->i_sb), 0);
+
 bail_unblock:
 	status = sigprocmask(SIG_SETMASK, &oldset, NULL);
 	if (status < 0)
inode.h +1 -0
Index: inode.h
===================================================================
@@ -124,6 +124,7 @@ void ocfs2_drop_inode(struct inode *inod
 /* Flags for ocfs2_iget() */
 #define OCFS2_FI_FLAG_SYSFILE		0x1
 #define OCFS2_FI_FLAG_ORPHAN_RECOVERY	0x2
+struct inode *ocfs2_ilookup(struct super_block *sb, u64 feoff);
 struct inode *ocfs2_iget(struct ocfs2_super *osb, u64 feoff, unsigned flags,
 			 int sysfile_type);
 int ocfs2_inode_init_private(struct inode *inode);
ocfs2_lockid.h +4 -0
Index: ocfs2_lockid.h
===================================================================
@@ -47,6 +47,7 @@ enum ocfs2_lock_type {
 	OCFS2_LOCK_TYPE_OPEN,
 	OCFS2_LOCK_TYPE_FLOCK,
 	OCFS2_LOCK_TYPE_QINFO,
+	OCFS2_LOCK_TYPE_NFS_SYNC,
 	OCFS2_NUM_LOCK_TYPES
 };
 
@@ -81,6 +82,9 @@ static inline char ocfs2_lock_type_char(
 		case OCFS2_LOCK_TYPE_QINFO:
 			c = 'Q';
 			break;
+		case OCFS2_LOCK_TYPE_NFS_SYNC:
+			c = 'Y';
+			break;
 		default:
 			c = '\0';
 	}
suballoc.c +150 -0

153 lines not on this page · raw message

suballoc.h +2 -0
Index: suballoc.h
===================================================================
@@ -186,4 +186,6 @@ int ocfs2_lock_allocators(struct inode *
 			  u32 clusters_to_add, u32 extents_to_split,
 			  struct ocfs2_alloc_context **data_ac,
 			  struct ocfs2_alloc_context **meta_ac);
+
+int ocfs2_test_inode_bit(struct ocfs2_super *osb, u64 blkno, int *res);
 #endif /* _CHAINALLOC_H_ */
ocfs2.h +1 -0
Index: ocfs2.h
===================================================================
@@ -305,6 +305,7 @@ struct ocfs2_super
 	struct ocfs2_cluster_connection *cconn;
 	struct ocfs2_lock_res osb_super_lockres;
 	struct ocfs2_lock_res osb_rename_lockres;
+	struct ocfs2_lock_res osb_nfs_sync_lockres;
 	struct ocfs2_dlm_debug *osb_dlm_debug;
 
 	struct dentry *osb_debug_root;
dlmglue.c +45 -0

81 lines not on this page · raw message

#1 lore raw

PATCH 1/1 OCFS2: anti stale inode for nfs (V4)

Quoted 8 lines · wengang wang

changes from v3:
1, move codes that checks inode allocation bit to subfunction ocfs2_test_inode_bit().

2, release the suballoc lock just after we get it. we should release it asap and doing so doesn't affect functionility.

3, add inode alloc slot validation.

Almost there!

Quoted 8 lines
export.c +8 -0
+	/* takes nfs_sync_lock in EX mode */
+	status = ocfs2_nfs_sync_lock(osb, 1);
+	if (status < 0) {
+		mlog(ML_ERROR, "getting nfs sync lock(EX) failed %d\n", status);
+		goto check_err;
+	}
+
+	status = ocfs2_test_inode_bit(osb, blkno, &set);

export.c static struct dentry *ocfs2_get_dentry(s · line 88

Quoted 10 lines
export.c +10 -0
+	if (status < 0) {
+		if (status == -EINVAL) {
+			/* meta block never be re-allocated as data block.
+			 * nfsd gives us wrong blkno */
+			status = -EEXIST;
+		} else {
+			mlog(ML_ERROR, "test inode bit failed %d\n", status);
+		}
+		goto unlock_nfs_sync;
+	}

This looks very nice, but it should return -ESTALE instead
of -EEXIST; it most likely does not exist, and nfs knows how to handle that error.

export.c static struct dentry *ocfs2_get_dentry(s · line 131

Quoted 7 lines
export.c +4 -1
 	result = d_obtain_alias(inode);
-	if (!IS_ERR(result))
+	if (!IS_ERR(result)) {
 		result->d_op = &ocfs2_dentry_ops;
+	} else {
+		mlog_errno((int)result);
+	}

Use PTR_ERR(result), not (int)result.

Quoted 21 lines

Index: suballoc.c
===================================================================

suballoc.c +13 -0
@@ -2116,3 +2116,153 @@ out:
 
 	return ret;
 }
+
+/* reads(hit disk) the inode specified by blkno to get suballoc_slot
+ * and suballoc_bit
+ * */
+static int ocfs2_get_suballoc_slot_bit(struct ocfs2_super *osb, u64 blkno,
+				u16 *suballoc_slot, u16 *suballoc_bit)
+{
+	int status;
+	struct buffer_head *inode_bh = NULL;
+	struct ocfs2_dinode *inode_fe;
+
+	mlog_entry("blkno: %llu\n", blkno);
+

suballoc.c out: · line 2141

Quoted 10 lines
suballoc.c +10 -0
+	/* dirty read disk */
+	status = ocfs2_read_blocks_sync(osb, blkno, 1, &inode_bh);
+	if (status < 0)
+		goto bail;
+
+	inode_fe = (struct ocfs2_dinode *) inode_bh->b_data;
+	if (!OCFS2_IS_VALID_DINODE(inode_fe)) {
+		status = -EINVAL;
+		goto bail;
+	}

Let's add an ML_ERROR here: "invalid inode %llu requested".

suballoc.c out: · line 2150

Quoted 8 lines
suballoc.c +8 -0
+	if (le16_to_cpu(inode_fe->i_suballoc_slot) != OCFS2_INVALID_SLOT &&
+	    (u32)le16_to_cpu(inode_fe->i_suballoc_slot) > osb->max_slots -1) {
+		mlog(ML_ERROR, "inode %llu has invalid suballoc slot %u"
+		     "this may be caused by file system crash", blkno,
+		     (u32)le16_to_cpu(inode_fe->i_suballoc_slot));
+		status = -EINVAL;
+		goto bail;
+	}

Don't print "this may be caused by file system crash", as it is
much more likely that nfs is asking for a bad block. Let's jsut say "inode %llu has an invalid suballoc slot %u".

Quoted 42 lines
suballoc.c +42 -0
+/* test if the bit, which is for the inode specified by blkno, in suballoc is
+ * set or not.
+ * on success, 0 is returned and *res is 1 for SET; 0 otherwise.
+ * when fails, errno is returned and *res is meaningless.
+ * MAKE SURE to hold nfs_sync_lock to revent ocfs2_delete_inode() on another
+ * node from accessing the same suballoc concurrently.
+ * */
+int ocfs2_test_inode_bit(struct ocfs2_super *osb, u64 blkno, int *res)
+{
+	int status;
+	u16 suballoc_bit = 0, suballoc_slot = 0;
+	struct inode *inode_alloc_inode;
+	struct buffer_head *alloc_bh = NULL;
+
+	/* MAKE SURE nfs_sync_lock is held */
+
+	mlog_entry("blkno: %llu", blkno);
+
+	status = ocfs2_get_suballoc_slot_bit(osb, blkno, &suballoc_slot,
+					     &suballoc_bit);
+	if (status < 0) {
+		mlog(ML_ERROR, "get alloc slot and bit failed %d\n", status);
+		goto bail;
+	}
+
+	inode_alloc_inode =
+		ocfs2_get_system_file_inode(osb, INODE_ALLOC_SYSTEM_INODE,
+					    suballoc_slot);
+	if (!inode_alloc_inode) {
+		/* the error code could be inaccurate, but we are not able to
+		 * get the correct one. */
+		status = -EEXIST;
+		mlog(ML_ERROR, "unable to get alloc inode in slot %u\n",
+		     (u32)suballoc_slot);
+		goto bail;
+	}
+
+	mutex_lock(&inode_alloc_inode->i_mutex);
+	/* lock down the suballoc lock it to cause other nodes flush disk and
+	 * then release it immediately to let allocation on other nodes to go
+	 * asap. the reason we can read the group without lock is that we just
+	 * want to know if the bit is clear and there can't be a concurrent

suballoc.c out: · line 2254

Quoted 10 lines
suballoc.c +10 -0
+	 * clearing action because we hold the nfs_sync_lock. sure that there
+	 * can be a concurrent setting action then. it doesn't matter, we can
+	 * check generation later. */
+	status = ocfs2_inode_lock(inode_alloc_inode, &alloc_bh, 0);
+	if (status < 0) {
+		mutex_unlock(&inode_alloc_inode->i_mutex);
+		mlog(ML_ERROR, "lock on alloc inode on slot %u failed %d\n",
+		     (u32)suballoc_slot, status);
+		goto bail;
+	}

You need to test the suballoc bit inside the lock. The current
code is unsafe because while the inode set value won't change in a way that matters, the structure of the alloc inode's groups may. So move the ocfs2_test_suballoc_bit() call inside the lock here:

+ status = ocfs2_test_suballoc_bit(osb, inode_alloc_inode, alloc_bh,
+ blkno, suballoc_bit, res);
+ if (status < 0)
+ mlog(ML_ERROR, "test suballoc bit failed %d\n", status);
+ ocfs2_inode_unlock(inode_alloc_inode, 0);
+ mutex_unlock(&inode_alloc_inode->i_mutex);

Update the comment, of course. Your points about the nfs_sync_lock are correct. The only change is that we read the group under the lock.

suballoc.c out: · line 2268

Quoted 6 lines
suballoc.c +6 -0
+	iput(inode_alloc_inode);
+	brelse(alloc_bh);
+bail:
+	mlog_exit(status);
+	return status;
+}

Otherwise this looks good.

Joel

Signature · 9 lines
#!/bin/perl -sp0777i<X+d*lMLa^*lN%0]dsXx++lMlN/dsM0<j]dsj
$/=unpack('H*',$_);$_=`echo 16dio\U$k"SK$/SM$n\EsN0p[lN*1
lK[d2%Sa2/d0$^Ixp"|dc`;s/\W//g;$_=pack('H*',/((..)*)$/)

Joel Becker
Principal Software Developer
Oracle
E-mail: [email protected]
Phone: (650) 506-8127
#2 lore raw

PATCH 1/1 OCFS2: anti stale inode for nfs (V4)

Quoted 6 lines
Quoted 6 lines
export.c +6 -0
+	status = ocfs2_test_inode_bit(osb, blkno, &set);
+	if (status < 0) {
+		if (status == -EINVAL) {
+			/* meta block never be re-allocated as data block.
+			 * nfsd gives us wrong blkno */
+			status = -EEXIST;
Quoted 8 lines

export.c static struct dentry *ocfs2_get_dentry(s · line 88

Quoted 5 lines
export.c +5 -0
+		} else {
+			mlog(ML_ERROR, "test inode bit failed %d\n", status);
+		}
+		goto unlock_nfs_sync;
+	}

This looks very nice, but it should return -ESTALE instead
of -EEXIST;

why? I think it's an EEXIST.
if you say nfs don't know how to deal with the EEXIST, I agree to change it. it most likely does not exist, and nfs knows how to handle

Quoted 7 lines

that error.

export.c static struct dentry *ocfs2_get_dentry(s · line 131

Quoted 2 lines
export.c +2 -0
+		mlog_errno((int)result);
+	}

Use PTR_ERR(result), not (int)result.

yes:)

Quoted 6 lines

suballoc.c out: · line 2141

Quoted 4 lines
suballoc.c +4 -0
+	if (!OCFS2_IS_VALID_DINODE(inode_fe)) {
+		status = -EINVAL;
+		goto bail;
+	}

Let's add an ML_ERROR here: "invalid inode %llu requested".

Ok.

Quoted 4 lines
Quoted 4 lines
suballoc.c +4 -0
+	if (le16_to_cpu(inode_fe->i_suballoc_slot) != OCFS2_INVALID_SLOT &&
+	    (u32)le16_to_cpu(inode_fe->i_suballoc_slot) > osb->max_slots -1) {
+		mlog(ML_ERROR, "inode %llu has invalid suballoc slot %u"
+		     "this may be caused by file system crash", blkno,
Quoted 8 lines

suballoc.c out: · line 2150

Quoted 4 lines
suballoc.c +4 -0
+		     (u32)le16_to_cpu(inode_fe->i_suballoc_slot));
+		status = -EINVAL;
+		goto bail;
+	}

Don't print "this may be caused by file system crash", as it is
much more likely that nfs is asking for a bad block. Let's jsut say "inode %llu has an invalid suballoc slot %u".

Ok.

Quoted 10 lines
Quoted 10 lines
suballoc.c +10 -0
+	/* lock down the suballoc lock it to cause other nodes flush disk and
+	 * then release it immediately to let allocation on other nodes to go
+	 * asap. the reason we can read the group without lock is that we just
+	 * want to know if the bit is clear and there can't be a concurrent
+	 * clearing action because we hold the nfs_sync_lock. sure that there
+	 * can be a concurrent setting action then. it doesn't matter, we can
+	 * check generation later. */
+	status = ocfs2_inode_lock(inode_alloc_inode, &alloc_bh, 0);
+	if (status < 0) {
+		mutex_unlock(&inode_alloc_inode->i_mutex);
Quoted 8 lines

suballoc.c out: · line 2254

Quoted 4 lines
suballoc.c +4 -0
+		mlog(ML_ERROR, "lock on alloc inode on slot %u failed %d\n",
+		     (u32)suballoc_slot, status);
+		goto bail;
+	}

You need to test the suballoc bit inside the lock. The current
code is unsafe because while the inode set value won't change in a way that matters, the structure of the alloc inode's groups may.

sorry, i don't understand the above very well.
i know the structure of the group may change since we released the suballoc lock. while, the change can only be manipulating a bit from "clear" to "set". according to our purpose, changing from "set" to "clear" is dangerous, the reverse changing only leads us to generation check. and the gen check can tell a stale inode.
I agree as a common function, it's not good if you meant that. but if it's used only for stale inode checking, it's workable I think.

thanks,
wengang.

#3 lore raw

PATCH 1/1 OCFS2: anti stale inode for nfs (V4)

Quoted 13 lines · Wengang Wang
Quoted 5 lines
Quoted 5 lines
export.c +5 -0
+	status = ocfs2_test_inode_bit(osb, blkno, &set);
+	if (status < 0) {
+		if (status == -EINVAL) {
+			/* meta block never be re-allocated as data block.
+			 * nfsd gives us wrong blkno */
Quoted 8 lines

export.c static struct dentry *ocfs2_get_dentry(s · line 88

Quoted 6 lines
export.c +6 -0
+			status = -EEXIST;
+		} else {
+			mlog(ML_ERROR, "test inode bit failed %d\n", status);
+		}
+		goto unlock_nfs_sync;
+	}

This looks very nice, but it should return -ESTALE instead
of -EEXIST;

Quoted 5 lines

why? I think it's an EEXIST.
if you say nfs don't know how to deal with the EEXIST, I agree to change it.
it most likely does not exist, and nfs knows how to handle

Quoted 1 line

that error.

EEXIST means "the file exists", which it clearly does not.
I think you mean ENOENT. But what is really happening here is nfs asks "I used to have file XXX, is it still there?" The correct response seems to me to be "Um, not anymore - it's stale".

Quoted 18 lines
Quoted 9 lines
Quoted 9 lines
suballoc.c +9 -0
+	/* lock down the suballoc lock it to cause other nodes flush disk and
+	 * then release it immediately to let allocation on other nodes to go
+	 * asap. the reason we can read the group without lock is that we just
+	 * want to know if the bit is clear and there can't be a concurrent
+	 * clearing action because we hold the nfs_sync_lock. sure that there
+	 * can be a concurrent setting action then. it doesn't matter, we can
+	 * check generation later. */
+	status = ocfs2_inode_lock(inode_alloc_inode, &alloc_bh, 0);
+	if (status < 0) {
Quoted 8 lines

suballoc.c out: · line 2254

Quoted 5 lines
suballoc.c +5 -0
+		mutex_unlock(&inode_alloc_inode->i_mutex);
+		mlog(ML_ERROR, "lock on alloc inode on slot %u failed %d\n",
+		     (u32)suballoc_slot, status);
+		goto bail;
+	}

You need to test the suballoc bit inside the lock. The current
code is unsafe because while the inode set value won't change in a way that matters, the structure of the alloc inode's groups may.

Quoted 8 lines

sorry, i don't understand the above very well.
i know the structure of the group may change since we released the suballoc lock. while, the change can only be manipulating a bit from "clear" to "set". according to our purpose, changing from "set" to "clear" is dangerous, the reverse changing only leads us to generation check. and the gen check can tell a stale inode.
I agree as a common function, it's not good if you meant that. but if it's used only for stale inode checking, it's workable I think.

	The issue is not the clear<->set transition.  Remember, we're

only blocking out deletes with the nfs sync lock. Other nodes can do a create (the clear->set), but they also might do that on other group descriptors. This means the layout of groups, including their pointers, might be changing beneath us.
Also, it is a correctness issue. Your way may happen to work
for the stale inode checking, but that's by sheer luck. Putting the access of the group inside the lock is definitively correct.

Joel

Signature · 10 lines
You can use a screwdriver to screw in screws or to clean your ears,
however, the latter needs real skill, determination and a lack of fear
of injuring yourself.  It is much the same with JavaScript.
	- Chris Heilmann

Joel Becker
Principal Software Developer
Oracle
E-mail: joel.becker at oracle.com
Phone: (650) 506-8127
#4 lore raw

PATCH 1/1 OCFS2: anti stale inode for nfs (V4)

Quoted 14 lines · Joel Becker
Quoted 13 lines · Wengang Wang
Quoted 13 lines

export.c static struct dentry *ocfs2_get_dentry(s · line 88

Quoted 11 lines
export.c +11 -0
+	status = ocfs2_test_inode_bit(osb, blkno, &set);
+	if (status < 0) {
+		if (status == -EINVAL) {
+			/* meta block never be re-allocated as data block.
+			 * nfsd gives us wrong blkno */
+			status = -EEXIST;
+		} else {
+			mlog(ML_ERROR, "test inode bit failed %d\n", status);
+		}
+		goto unlock_nfs_sync;
+	}

This looks very nice, but it should return -ESTALE instead
of -EEXIST;

Quoted 8 lines
Quoted 5 lines

why? I think it's an EEXIST.
if you say nfs don't know how to deal with the EEXIST, I agree to change it.
it most likely does not exist, and nfs knows how to handle

Quoted 1 line

that error.

EEXIST means "the file exists", which it clearly does not.
I think you mean ENOENT.

Sorry, yes I meant ENOENT.
also, the following EEXIST is wrong too(and also in some existing codes).

+ inode_alloc_inode =
+ ocfs2_get_system_file_inode(osb, INODE_ALLOC_SYSTEM_INODE,
+ suballoc_slot);
+ if (!inode_alloc_inode) {
+ /* the error code could be inaccurate, but we are not able to
+ * get the correct one. */
+ status = -EEXIST;

here, we don't have the accurate reason. I suggest we return PTR_ERR(xxx) instead of NULL for ocfs2_get_system_file_inode() ( _ocfs2_get_system_file_inode() actually) and change code where ocfs2_get_system_file_inode() is called.

Quoted 3 lines

But what is really happening here is nfs asks
"I used to have file XXX, is it still there?" The correct response seems to me to be "Um, not anymore - it's stale".

yes, your case is the "normal" stale case.
checking where EINVAL is returned, it's either (!OCFS2_IS_VALID_DINODE(inode_fe)) or alloc slot out of range. in the two cases, I prefer nfs passed a wrong fh(blkno) instead of a stale one, that's nfs does a buggy request.
well, returning ESTALE is also OK. I just think ENOENT is more accurate.

Quoted 26 lines
Quoted 17 lines
Quoted 9 lines
Quoted 9 lines
suballoc.c +9 -0
+	/* lock down the suballoc lock it to cause other nodes flush disk and
+	 * then release it immediately to let allocation on other nodes to go
+	 * asap. the reason we can read the group without lock is that we just
+	 * want to know if the bit is clear and there can't be a concurrent
+	 * clearing action because we hold the nfs_sync_lock. sure that there
+	 * can be a concurrent setting action then. it doesn't matter, we can
+	 * check generation later. */
+	status = ocfs2_inode_lock(inode_alloc_inode, &alloc_bh, 0);
+	if (status < 0) {
Quoted 8 lines

suballoc.c out: · line 2254

Quoted 5 lines
suballoc.c +5 -0
+		mutex_unlock(&inode_alloc_inode->i_mutex);
+		mlog(ML_ERROR, "lock on alloc inode on slot %u failed %d\n",
+		     (u32)suballoc_slot, status);
+		goto bail;
+	}

You need to test the suballoc bit inside the lock. The current
code is unsafe because while the inode set value won't change in a way that matters, the structure of the alloc inode's groups may.

Quoted 8 lines

sorry, i don't understand the above very well.
i know the structure of the group may change since we released the suballoc lock. while, the change can only be manipulating a bit from "clear" to "set". according to our purpose, changing from "set" to "clear" is dangerous, the reverse changing only leads us to generation check. and the gen check can tell a stale inode.
I agree as a common function, it's not good if you meant that. but if it's used only for stale inode checking, it's workable I think.

Quoted 5 lines
	The issue is not the clear<->set transition.  Remember, we're

only blocking out deletes with the nfs sync lock. Other nodes can do a create (the clear->set), but they also might do that on other group descriptors. This means the layout of groups, including their pointers, might be changing beneath us.

if my understand is correct:),
yes, when there could be a new group added to a chain. and the chain list changed(pointers). and there could be a chain relink(ocfs2_relink_block_group()).
but I don't understand why it affects the group in question. the group in question is gotten by reading the block which is numbered of inode blkno minus alloc bit. that doesn't involve any pointer. so, could you tell me more detail?

Quoted 2 lines

Also, it is a correctness issue. Your way may happen to work
for the stale inode checking, but that's by sheer luck.

until I got clear your idea about the groups, I persist :P.

Quoted 2 lines

Putting the
access of the group inside the lock is definitively correct.

Yes, agree.
I just want a better performance when we are correct.

thanks,
wengang.

#5 lore raw

PATCH 1/1 OCFS2: anti stale inode for nfs (V4)

Quoted 34 lines · Wengang Wang
Quoted 14 lines · Joel Becker
Quoted 13 lines · Wengang Wang
Quoted 13 lines

export.c static struct dentry *ocfs2_get_dentry(s · line 88

Quoted 11 lines
export.c +11 -0
+	status = ocfs2_test_inode_bit(osb, blkno, &set);
+	if (status < 0) {
+		if (status == -EINVAL) {
+			/* meta block never be re-allocated as data block.
+			 * nfsd gives us wrong blkno */
+			status = -EEXIST;
+		} else {
+			mlog(ML_ERROR, "test inode bit failed %d\n", status);
+		}
+		goto unlock_nfs_sync;
+	}

This looks very nice, but it should return -ESTALE instead
of -EEXIST;

Quoted 7 lines
Quoted 5 lines

why? I think it's an EEXIST.
if you say nfs don't know how to deal with the EEXIST, I agree to change it.
it most likely does not exist, and nfs knows how to handle

Quoted 1 line

that error.

EEXIST means "the file exists", which it clearly does not.
I think you mean ENOENT.

Sorry, yes I meant ENOENT.
also, the following EEXIST is wrong too(and also in some existing codes).

suballoc.c +7 -0
+	inode_alloc_inode =
+		ocfs2_get_system_file_inode(osb, INODE_ALLOC_SYSTEM_INODE,
+					    suballoc_slot);
+	if (!inode_alloc_inode) {
+		/* the error code could be inaccurate, but we are not able to
+		 * get the correct one. */
+		status = -EEXIST;
Quoted 4 lines

here, we don't have the accurate reason. I suggest we return PTR_ERR(xxx) instead of NULL for ocfs2_get_system_file_inode() ( _ocfs2_get_system_file_inode() actually) and change code where ocfs2_get_system_file_inode() is called.

Yeah, I noticed that we don't have an error from
ocfs2_get_system_file_inode(). But we don't have any need to fix that right now. That can be a later patch if we care.

Quoted 10 lines
Quoted 3 lines

But what is really happening here is nfs asks
"I used to have file XXX, is it still there?" The correct response seems to me to be "Um, not anymore - it's stale".

yes, your case is the "normal" stale case.
checking where EINVAL is returned, it's either (!OCFS2_IS_VALID_DINODE(inode_fe)) or alloc slot out of range. in the two cases, I prefer nfs passed a wrong fh(blkno) instead of a stale one, that's nfs does a buggy request.
well, returning ESTALE is also OK. I just think ENOENT is more accurate.

I agree with you that ENOENT is more accurate, but right now
we're playing nice with nfsd. Actually, return -ENOENT from the test function, but in ocfs2_get_dentry() change it to -ESTALE for nfsd.

Quoted 8 lines

if my understand is correct:),
yes, when there could be a new group added to a chain. and the chain list changed(pointers). and there could be a chain relink(ocfs2_relink_block_group()).
but I don't understand why it affects the group in question. the group in question is gotten by reading the block which is numbered of inode blkno minus alloc bit. that doesn't involve any pointer. so, could you tell me more detail?

I agree it probably doesn't affect this direct block group. At
least, unless we change the code to do some walking or whatever. But we want this function to be right no matter the behavior.

Quoted 10 lines
Quoted 2 lines

Also, it is a correctness issue. Your way may happen to work
for the stale inode checking, but that's by sheer luck.

until I got clear your idea about the groups, I persist :P.

Quoted 2 lines

Putting the
access of the group inside the lock is definitively correct.

Yes, agree.
I just want a better performance when we are correct.

We will never sacrifice correctness for performance. You don't
access data outside of locks unless you have some really, really good reason. This won't hurt any performance - it's a standard operation we do all the time: "take lock, do something with group descriptor, drop lock". If it was a performance problem it would be a problem everywhere, and we'd solve it in a correct fashion.
Secondly, if some later code decides to use this function, they
certainly won't expect it to be doing unsafe things. And they won't be inside a very specific nfs code path.

Joel

Signature · 9 lines
Life's Little Instruction Book #69

	"Whistle"

Joel Becker
Principal Software Developer
Oracle
E-mail: joel.becker at oracle.com
Phone: (650) 506-8127
#6 lore raw

PATCH 1/1 OCFS2: anti stale inode for nfs (V4)

Quoted 34 lines · Joel Becker
Quoted 33 lines · Wengang Wang
Quoted 14 lines · Joel Becker
Quoted 13 lines · Wengang Wang
Quoted 13 lines

export.c static struct dentry *ocfs2_get_dentry(s · line 88

Quoted 11 lines
export.c +11 -0
+	status = ocfs2_test_inode_bit(osb, blkno, &set);
+	if (status < 0) {
+		if (status == -EINVAL) {
+			/* meta block never be re-allocated as data block.
+			 * nfsd gives us wrong blkno */
+			status = -EEXIST;
+		} else {
+			mlog(ML_ERROR, "test inode bit failed %d\n", status);
+		}
+		goto unlock_nfs_sync;
+	}

This looks very nice, but it should return -ESTALE instead
of -EEXIST;

Quoted 7 lines
Quoted 5 lines

why? I think it's an EEXIST.
if you say nfs don't know how to deal with the EEXIST, I agree to change it.
it most likely does not exist, and nfs knows how to handle

Quoted 1 line

that error.

EEXIST means "the file exists", which it clearly does not.
I think you mean ENOENT.

Sorry, yes I meant ENOENT.
also, the following EEXIST is wrong too(and also in some existing codes).

suballoc.c +7 -0
+	inode_alloc_inode =
+		ocfs2_get_system_file_inode(osb, INODE_ALLOC_SYSTEM_INODE,
+					    suballoc_slot);
+	if (!inode_alloc_inode) {
+		/* the error code could be inaccurate, but we are not able to
+		 * get the correct one. */
+		status = -EEXIST;
Quoted 9 lines
Quoted 4 lines

here, we don't have the accurate reason. I suggest we return PTR_ERR(xxx) instead of NULL for ocfs2_get_system_file_inode() ( _ocfs2_get_system_file_inode() actually) and change code where ocfs2_get_system_file_inode() is called.

Yeah, I noticed that we don't have an error from
ocfs2_get_system_file_inode(). But we don't have any need to fix that right now. That can be a later patch if we care.

Ok.

Quoted 10 lines
Quoted 9 lines
Quoted 3 lines

But what is really happening here is nfs asks
"I used to have file XXX, is it still there?" The correct response seems to me to be "Um, not anymore - it's stale".

yes, your case is the "normal" stale case.
checking where EINVAL is returned, it's either (!OCFS2_IS_VALID_DINODE(inode_fe)) or alloc slot out of range. in the two cases, I prefer nfs passed a wrong fh(blkno) instead of a stale one, that's nfs does a buggy request.
well, returning ESTALE is also OK. I just think ENOENT is more accurate.

Quoted 4 lines

I agree with you that ENOENT is more accurate, but right now
we're playing nice with nfsd. Actually, return -ENOENT from the test function, but in ocfs2_get_dentry() change it to -ESTALE for nfsd.

Ok.

Quoted 9 lines
Quoted 8 lines

if my understand is correct:),
yes, when there could be a new group added to a chain. and the chain list changed(pointers). and there could be a chain relink(ocfs2_relink_block_group()).
but I don't understand why it affects the group in question. the group in question is gotten by reading the block which is numbered of inode blkno minus alloc bit. that doesn't involve any pointer. so, could you tell me more detail?

Quoted 4 lines

I agree it probably doesn't affect this direct block group. At
least, unless we change the code to do some walking or whatever. But we want this function to be right no matter the behavior.

Ok.

Quoted 9 lines
Quoted 8 lines
Quoted 2 lines

Also, it is a correctness issue. Your way may happen to work
for the stale inode checking, but that's by sheer luck.

until I got clear your idea about the groups, I persist :P.

Quoted 2 lines

Putting the
access of the group inside the lock is definitively correct.

Yes, agree.
I just want a better performance when we are correct.

Quoted 9 lines

We will never sacrifice correctness for performance. You don't
access data outside of locks unless you have some really, really good reason. This won't hurt any performance - it's a standard operation we do all the time: "take lock, do something with group descriptor, drop lock". If it was a performance problem it would be a problem everywhere, and we'd solve it in a correct fashion.
Secondly, if some later code decides to use this function, they
certainly won't expect it to be doing unsafe things. And they won't be inside a very specific nfs code path.

Yeah, as a common function it shouldn't be nfs specific.

thanks,
wengang.

#7 lore raw

PATCH 1/1 OCFS2: anti stale inode for nfs (V4)

In reply to wengang wang
Quoted 8 lines · wengang wang

changes from v3:
1, move codes that checks inode allocation bit to subfunction ocfs2_test_inode_bit().

2, release the suballoc lock just after we get it. we should release it asap and doing so doesn't affect functionility.

3, add inode alloc slot validation.

One last thing, sorry I missed it.

Quoted 13 lines
suballoc.c +13 -0
+static int ocfs2_test_suballoc_bit(struct ocfs2_super *osb, struct inode *suballoc,
+			    struct buffer_head *alloc_bh, u64 blkno, u16 bit,
+			    int *res)
+{
+	struct ocfs2_dinode *alloc_fe;
+	struct ocfs2_group_desc *group;
+	struct buffer_head *group_bh = NULL;
+	u64 bg_blkno;
+	int status;
+
+	mlog_entry("blkno: %llu bit: %u\n", blkno, (unsigned int)bit);
+
+	alloc_fe = (struct ocfs2_dinode *)alloc_bh->b_data;

suballoc.c out: · line 2191

Quoted 10 lines
suballoc.c +10 -0
+	BUG_ON((bit + 1) > ocfs2_bits_per_group(&alloc_fe->id2.i_chain));
+
+	bg_blkno = ocfs2_which_suballoc_group(blkno, bit);
+	status = ocfs2_read_blocks_sync(osb, bg_blkno, 1, &group_bh);
+	if (status < 0)
+		goto bail;
+
+	status = ocfs2_check_group_descriptor(osb->sb, alloc_fe, group_bh);
+	if (status < 0)
+		goto bail;

Just use ocfs2_read_group_descriptor() here. The locking code
will make sure it reads from disk if necessary.

Joel

Signature · 10 lines
 One look at the From:
 understanding has blossomed
 .procmailrc grows
	- Alexander Viro

Joel Becker
Principal Software Developer
Oracle
E-mail: joel.becker at oracle.com
Phone: (650) 506-8127
#8 lore raw

PATCH 1/1 OCFS2: anti stale inode for nfs (V4)

In reply to wengang wang

Hi wengang,
thanks for the work. Just one comment. See below.

Quoted 10 lines · wengang wang

changes from v3:
1, move codes that checks inode allocation bit to subfunction ocfs2_test_inode_bit().

2, release the suballoc lock just after we get it. we should release it asap and doing so doesn't affect functionility.

3, add inode alloc slot validation.

<snip>

Quoted 53 lines
suballoc.c +53 -0
+/* reads(hit disk) the inode specified by blkno to get suballoc_slot
+ * and suballoc_bit
+ * */
+static int ocfs2_get_suballoc_slot_bit(struct ocfs2_super *osb, u64 blkno,
+				u16 *suballoc_slot, u16 *suballoc_bit)
+{
+	int status;
+	struct buffer_head *inode_bh = NULL;
+	struct ocfs2_dinode *inode_fe;
+
+	mlog_entry("blkno: %llu\n", blkno);
+
+	/* dirty read disk */
+	status = ocfs2_read_blocks_sync(osb, blkno, 1, &inode_bh);
+	if (status < 0)
+		goto bail;
+
+	inode_fe = (struct ocfs2_dinode *) inode_bh->b_data;
+	if (!OCFS2_IS_VALID_DINODE(inode_fe)) {
+		status = -EINVAL;
+		goto bail;
+	}
+
+	if (le16_to_cpu(inode_fe->i_suballoc_slot) != OCFS2_INVALID_SLOT &&
+	    (u32)le16_to_cpu(inode_fe->i_suballoc_slot) > osb->max_slots -1) {
+		mlog(ML_ERROR, "inode %llu has invalid suballoc slot %u"
+		     "this may be caused by file system crash", blkno,
+		     (u32)le16_to_cpu(inode_fe->i_suballoc_slot));
+		status = -EINVAL;
+		goto bail;
+	}
+	if (suballoc_slot)
+		*suballoc_slot = le16_to_cpu(inode_fe->i_suballoc_slot);
+	if (suballoc_bit)
+		*suballoc_bit= le16_to_cpu(inode_fe->i_suballoc_bit);
+
+bail:
+	brelse(inode_bh);
+
+	mlog_exit(status);
+	return status;
+}
+
+/* test whether bit is SET in allocator bitmap or not.
+ * on success, 0 is returned and *res is 1 for SET; 0 otherwise.
+ * when fails, errno is returned and *res is meaningless.
+ * calls this after you have cluster locked against suballoc, or you may
+ * get a result based on non-up2date contents
+ * */
+static int ocfs2_test_suballoc_bit(struct ocfs2_super *osb, struct inode *suballoc,
+			    struct buffer_head *alloc_bh, u64 blkno, u16 bit,
+			    int *res)
+{

suballoc.c out: · line 2182

Quoted 10 lines
suballoc.c +10 -0
+	struct ocfs2_dinode *alloc_fe;
+	struct ocfs2_group_desc *group;
+	struct buffer_head *group_bh = NULL;
+	u64 bg_blkno;
+	int status;
+
+	mlog_entry("blkno: %llu bit: %u\n", blkno, (unsigned int)bit);
+
+	alloc_fe = (struct ocfs2_dinode *)alloc_bh->b_data;
+	BUG_ON((bit + 1) > ocfs2_bits_per_group(&alloc_fe->id2.i_chain));

here, we shouldn't BUG_ON. It actually isn't a kernel bug. the 'bit' is got from the disk by function ocfs2_get_suballoc_slot_bit. So maybe a corrupt inode, maybe something else, but never a bug of ocfs2. ;) So just return error please.

Regards,
Tao

#9 lore raw

PATCH 1/1 OCFS2: anti stale inode for nfs (V4)

Hi,

good point! thanks!

regards,
wengang.

Quoted 7 lines

suballoc.c out: · line 2182

Quoted 2 lines
suballoc.c +2 -0
+    alloc_fe = (struct ocfs2_dinode *)alloc_bh->b_data;
+    BUG_ON((bit + 1) > ocfs2_bits_per_group(&alloc_fe->id2.i_chain));

here, we shouldn't BUG_ON. It actually isn't a kernel bug. the 'bit' is got from the disk by function ocfs2_get_suballoc_slot_bit. So maybe a corrupt inode, maybe something else, but never a bug of ocfs2. ;) So just return error please.