Skip to contentSkip to conversation

OCFS2: anti stale inode for nfs (V5)

PATCH New wengang wang 2 participants 7 messages Feb 27, 2009 – Mar 3, 2009 OCFS2

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 OCFS2: anti stale inode for nfs (V5)

changes from v4:
1, let suballoc lock covers the checking of the group.

2, add/correct some log messages.

3, use ocfs2_read_group_descriptor() instead of diry reading the group.

--

 dlmglue.c      |   45 ++++++++++++++++
 dlmglue.h      |    2
 export.c       |   77 +++++++++++++++++++++++++--
 inode.c        |   24 ++++++++
 inode.h        |    1
 ocfs2.h        |    1
 ocfs2_lockid.h |    4 +
 suballoc.c     |  157 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 suballoc.h     |    2
 9 files changed, 304 insertions(+), 9 deletions(-)
dlmglue.h +2 -0
Index: dlmglue.h
===================================================================
@@ -115,6 +115,8 @@ void ocfs2_super_unlock(struct ocfs2_sup
 			int ex);
 int ocfs2_rename_lock(struct ocfs2_super *osb);
 void ocfs2_rename_unlock(struct ocfs2_super *osb);
+int ocfs2_nfs_sync_lock(struct ocfs2_super *osb, int ex);
+void ocfs2_nfs_sync_unlock(struct ocfs2_super *osb, int ex);
 int ocfs2_dentry_lock(struct dentry *dentry, int ex);
 void ocfs2_dentry_unlock(struct dentry *dentry, int ex);
 int ocfs2_file_lock(struct file *file, int ex, int trylock);
export.c +69 -8

104 lines not on this page · raw message

inode.c +23 -1
Index: inode.c
===================================================================
@@ -112,6 +112,17 @@ void ocfs2_get_inode_flags(struct ocfs2_
 		oi->ip_attr |= OCFS2_DIRSYNC_FL;
 }
 
+struct inode *ocfs2_ilookup(struct super_block *sb, u64 blkno)
+{
+	struct ocfs2_find_inode_args args;
+
+	args.fi_blkno = blkno;
+	args.fi_flags = 0;
+	args.fi_ino = ino_from_blkno(sb, blkno);
+	args.fi_sysfile_type = 0;
+
+	return ilookup5(sb, blkno, ocfs2_find_actor, &args);
+}
 struct inode *ocfs2_iget(struct ocfs2_super *osb, u64 blkno, unsigned flags,
 			 int sysfile_type)
 {
@@ -949,6 +960,13 @@ void ocfs2_delete_inode(struct inode *in
 		goto bail;
 	}
 
+	/* Lock down the nfs_sync lock in PR mode */
+	status = ocfs2_nfs_sync_lock(OCFS2_SB(inode->i_sb), 0);
+	if (status < 0) {
+		mlog(ML_ERROR, "getting nfs sync lock(PR) failed %d\n", status);
+		ocfs2_cleanup_delete_inode(inode, 0);
+		goto bail_unblock;
+	}
 	/* Lock down the inode. This gives us an up to date view of
 	 * it's metadata (for verification), and allows us to
 	 * serialize delete_inode on multiple nodes.
@@ -962,7 +980,7 @@ void ocfs2_delete_inode(struct inode *in
 		if (status != -ENOENT)
 			mlog_errno(status);
 		ocfs2_cleanup_delete_inode(inode, 0);
-		goto bail_unblock;
+		goto bail_unlock_nfs_sync;
 	}
 
 	/* Query the cluster. This will be the final decision made
@@ -1005,6 +1023,10 @@ void ocfs2_delete_inode(struct inode *in
 bail_unlock_inode:
 	ocfs2_inode_unlock(inode, 1);
 	brelse(di_bh);
+
+bail_unlock_nfs_sync:
+	ocfs2_nfs_sync_unlock(OCFS2_SB(inode->i_sb), 0);
+
 bail_unblock:
 	status = sigprocmask(SIG_SETMASK, &oldset, NULL);
 	if (status < 0)
inode.h +1 -0
Index: inode.h
===================================================================
@@ -124,6 +124,7 @@ void ocfs2_drop_inode(struct inode *inod
 /* Flags for ocfs2_iget() */
 #define OCFS2_FI_FLAG_SYSFILE		0x1
 #define OCFS2_FI_FLAG_ORPHAN_RECOVERY	0x2
+struct inode *ocfs2_ilookup(struct super_block *sb, u64 feoff);
 struct inode *ocfs2_iget(struct ocfs2_super *osb, u64 feoff, unsigned flags,
 			 int sysfile_type);
 int ocfs2_inode_init_private(struct inode *inode);
ocfs2_lockid.h +4 -0
Index: ocfs2_lockid.h
===================================================================
@@ -47,6 +47,7 @@ enum ocfs2_lock_type {
 	OCFS2_LOCK_TYPE_OPEN,
 	OCFS2_LOCK_TYPE_FLOCK,
 	OCFS2_LOCK_TYPE_QINFO,
+	OCFS2_LOCK_TYPE_NFS_SYNC,
 	OCFS2_NUM_LOCK_TYPES
 };
 
@@ -81,6 +82,9 @@ static inline char ocfs2_lock_type_char(
 		case OCFS2_LOCK_TYPE_QINFO:
 			c = 'Q';
 			break;
+		case OCFS2_LOCK_TYPE_NFS_SYNC:
+			c = 'Y';
+			break;
 		default:
 			c = '\0';
 	}
suballoc.c +157 -0

160 lines not on this page · raw message

suballoc.h +2 -0
Index: suballoc.h
===================================================================
@@ -186,4 +186,6 @@ int ocfs2_lock_allocators(struct inode *
 			  u32 clusters_to_add, u32 extents_to_split,
 			  struct ocfs2_alloc_context **data_ac,
 			  struct ocfs2_alloc_context **meta_ac);
+
+int ocfs2_test_inode_bit(struct ocfs2_super *osb, u64 blkno, int *res);
 #endif /* _CHAINALLOC_H_ */
ocfs2.h +1 -0
Index: ocfs2.h
===================================================================
@@ -305,6 +305,7 @@ struct ocfs2_super
 	struct ocfs2_cluster_connection *cconn;
 	struct ocfs2_lock_res osb_super_lockres;
 	struct ocfs2_lock_res osb_rename_lockres;
+	struct ocfs2_lock_res osb_nfs_sync_lockres;
 	struct ocfs2_dlm_debug *osb_dlm_debug;
 
 	struct dentry *osb_debug_root;
dlmglue.c +45 -0

81 lines not on this page · raw message

#1 lore raw

PATCH 1/1 OCFS2: anti stale inode for nfs (V5)

Quoted 8 lines · wengang wang

changes from v4:
1, let suballoc lock covers the checking of the group.

2, add/correct some log messages.

3, use ocfs2_read_group_descriptor() instead of diry reading the group.

Almost there!

suballoc.c out: · line 2204

Quoted 13 lines
suballoc.c +13 -0
+	bg_blkno = ocfs2_which_suballoc_group(blkno, bit);
+	status = ocfs2_read_group_descriptor(suballoc, alloc_fe, bg_blkno,
+					     &group_bh);
+	if (status < 0) {
+		mlog(ML_ERROR, "read group %llu failed %d\n", bg_blkno, status);
+		goto bail;
+	}
+
+	status = ocfs2_check_group_descriptor(osb->sb, alloc_fe, group_bh);
+	if (status < 0) {
+		mlog(ML_ERROR, "check group %llu faild %d\n", bg_blkno, status);
+		goto bail;
+	}

The call to ocfs2_read_group_descriptor() already does the same
checkin as ocfs2_check_group_descriptor(), so you don't need to call ocfs2_check_group_descriptor().
Other than that, I think this patch is done! Do you have a test
to test it?

Joel

Signature · 10 lines
 One look at the From:
 understanding has blossomed
 .procmailrc grows
	- Alexander Viro

Joel Becker
Principal Software Developer
Oracle
E-mail: joel.becker at oracle.com
Phone: (650) 506-8127
#2 lore raw

PATCH 1/1 OCFS2: anti stale inode for nfs (V5)

Quoted 10 lines
Quoted 10 lines
suballoc.c +9 -0
+	bg_blkno = ocfs2_which_suballoc_group(blkno, bit);
+	status = ocfs2_read_group_descriptor(suballoc, alloc_fe, bg_blkno,
+					     &group_bh);
+	if (status < 0) {
+		mlog(ML_ERROR, "read group %llu failed %d\n", bg_blkno, status);
+		goto bail;
+	}
+
+	status = ocfs2_check_group_descriptor(osb->sb, alloc_fe, group_bh);
Quoted 8 lines

suballoc.c out: · line 2204

Quoted 4 lines
suballoc.c +4 -0
+	if (status < 0) {
+		mlog(ML_ERROR, "check group %llu faild %d\n", bg_blkno, status);
+		goto bail;
+	}

The call to ocfs2_read_group_descriptor() already does the same
checkin as ocfs2_check_group_descriptor(), so you don't need to call ocfs2_check_group_descriptor().

ah, yes. I only verified that ocfs2_check_group_descriptor() is not called in ocfs2_read_group_descriptor(). but didn't notice the same contents are done in ocfs2_read_group_descriptor() as well. will repost as V6.

Quoted 2 lines

Other than that, I think this patch is done! Do you have a test
to test it?

I tested against V4 which checks group without sub allocator cluster lock taken.
the test case is:
1) 2 nodes env
2) each node exports ocfs2 fs(/wwg-ocfs2_test).
3) each node does the script:

OCFS2_TOP=/wwg-ocfs2-test
TOP_DIR=/ocfs2-export
write_thread()

{
         while [ 1 ] ; do
                 host=`/bin/hostname`
                 /bin/rm -rf $TOP_DIR/$host
                 /bin/mkdir $TOP_DIR/$host
                 /bin/cp -r /usr/share $TOP_DIR/$host/
         done
}

mkdir $OCFS2_TOP $TOP_DIR
mount -L ocfs2_anti_stale $OCFS2_TOP
mount -o nordirplus 127.0.0.1:/${OCFS2_TOP} ${TOP_DIR} write_thread&
while [ 1 ]; do

         ls -lR $TOP_DIR >/dev/null 2>&1

done

is it enough?

regards,
wengang.

#3 lore raw

PATCH 1/1 OCFS2: anti stale inode for nfs (V5)

Quoted 9 lines · Wengang Wang
Quoted 2 lines

Other than that, I think this patch is done! Do you have a test
to test it?

I tested against V4 which checks group without sub allocator cluster lock taken.
the test case is:
1) 2 nodes env
2) each node exports ocfs2 fs(/wwg-ocfs2_test).
3) each node does the script:

Does this script reproduce the problem in the un-patched case?
If so, and the patch fixes it, then we're good.

Joel

Signature · 10 lines
"And yet I find,
 And yet I find repeating in my head.
 If I can't be my own, 
 I'd feel better dead."

Joel Becker
Principal Software Developer
Oracle
E-mail: joel.becker at oracle.com
Phone: (650) 506-8127
#4 lore raw

PATCH 1/1 OCFS2: anti stale inode for nfs (V5)

Quoted 10 lines · Joel Becker
Quoted 8 lines · Wengang Wang
Quoted 2 lines

Other than that, I think this patch is done! Do you have a test
to test it?

I tested against V4 which checks group without sub allocator cluster lock taken.
the test case is:
1) 2 nodes env
2) each node exports ocfs2 fs(/wwg-ocfs2_test).
3) each node does the script:

Quoted 2 lines

Does this script reproduce the problem in the un-patched case?
If so, and the patch fixes it, then we're good.

No, I never reproduced the bug on 1.4(did it only on 1.2 with adding sleep and force disk updating). but theoretically there is the problem on 1.4.

regards,
wengang.

#5 lore raw

PATCH 1/1 OCFS2: anti stale inode for nfs (V5)

Quoted 7 lines · Wengang Wang
Quoted 2 lines · Joel Becker

Does this script reproduce the problem in the un-patched case?
If so, and the patch fixes it, then we're good.

No, I never reproduced the bug on 1.4(did it only on 1.2 with adding sleep and force disk updating). but theoretically there is the problem on 1.4.

Oh, well. I'd hope to have a reproduction case that would prove
our solution. That said, we have a good idea what was happening and the patch solves that.
Please cook up the final patch so that Mark can queue it up.
Thanks for all the hard work!

Joel

Signature · 9 lines
"There are only two ways to live your life. One is as though nothing
 is a miracle. The other is as though everything is a miracle."
        - Albert Einstein

Joel Becker
Principal Software Developer
Oracle
E-mail: joel.becker at oracle.com
Phone: (650) 506-8127
#6 lore raw

PATCH 1/1 OCFS2: anti stale inode for nfs (V5)

Quoted 10 lines · Joel Becker
Quoted 6 lines · Wengang Wang
Quoted 2 lines · Joel Becker

Does this script reproduce the problem in the un-patched case?
If so, and the patch fixes it, then we're good.

No, I never reproduced the bug on 1.4(did it only on 1.2 with adding sleep and force disk updating). but theoretically there is the problem on 1.4.

Oh, well. I'd hope to have a reproduction case that would prove
our solution.

yes. I meant I tried but no luck.

That said, we have a good idea what was happening and the

Quoted 2 lines

patch solves that.
Please cook up the final patch so that Mark can queue it up.

will very soon.

regards,
wengang