Skip to contentSkip to conversation

NLM failover - nlm_unlock

PATCH In review Wendy Cheng 2 participants 3 messages Aug 14, 2006 – Aug 18, 2006

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/5 NLM failover - nlm_unlock

By writing exported filesytem id into /proc/fs/nfsd/nlm_unlock, this patch walks thru lockd's global nlm_files list to release all the locks associated with the particular id. It is used to enable NFS lock failover with active-active clustered servers.

Relevant steps:
1) Exports filesystem with "fsid" option as:
/etc/exports entry> /mnt/ext3/exports *(fsid=1234,sync,rw)

2) Drops locks based on fsid by:
shell> echo 1234 > /proc/fs/nfsd/nlm_unlock

 fs/lockd/svcsubs.c          |   79 ++++++++++++++++++++++++++++++++++++
+++++--- 
 fs/nfsd/nfsctl.c            |   41 ++++++++++++++++++++++
 include/linux/lockd/bind.h  |    5 ++
 include/linux/lockd/lockd.h |    2 +
 include/linux/nfsd/debug.h  |    1
 5 files changed, 123 insertions(+), 5 deletions(-)
include/linux/lockd/lockd.h +2 -0
@@ -133,6 +133,7 @@ struct nlm_block {
 #define NLM_ACT_CHECK		0		/* check for locks */
 #define NLM_ACT_MARK		1		/* mark & sweep */
 #define NLM_ACT_UNLOCK		2		/* release all locks */
+#define NLM_ACT_FO_UNLOCK	3		/* failover release locks */
 
 /*
  * Global variables
@@ -196,6 +197,7 @@ void		  nlm_release_file(struct nlm_file
 void		  nlmsvc_mark_resources(void);
 void		  nlmsvc_free_host_resources(struct nlm_host *);
 void		  nlmsvc_invalidate_all(void);
+int 		  nlmsvc_fo_unlock(int *fsid);
 
 static __inline__ struct inode *
 nlmsvc_file_inode(struct nlm_file *file)
fs/lockd/svcsubs.c +74 -5

124 lines not on this page · raw message

include/linux/lockd/bind.h +5 -0
@@ -33,4 +33,9 @@ extern int	nlmclnt_proc(struct inode *, 
 extern int	lockd_up(void);
 extern void	lockd_down(void);
 
+/*
+ * NLM failover
+ */
+extern int     nlmsvc_fo_unlock(int *fsid);
+
 #endif /* LINUX_LOCKD_BIND_H */
fs/nfsd/nfsctl.c +41 -0

77 lines not on this page · raw message

include/linux/nfsd/debug.h +1 -0
@@ -32,6 +32,7 @@
 #define NFSDDBG_REPCACHE	0x0080
 #define NFSDDBG_XDR		0x0100
 #define NFSDDBG_LOCKD		0x0200
+#define NFSDDBG_CLUSTER 	0x0400
 #define NFSDDBG_ALL		0x7FFF
 #define NFSDDBG_NOCHANGE	0xFFFF
 

-------------------------------------------------------------------------
Using Tomcat but need to do more? Need to support web services, security? Get stuff done quickly with pre-integrated technology to make your job easier Download IBM WebSphere Application Server v.1.0.1 based on Apache Geronimo
http://sel.as-us.falkag.net/sel?cmd=lnk&kid=120709&bid=263057&dat=121642

_______________________________________________
NFS maillist - [email protected]
https://lists.sourceforge.net/lists/listinfo/nfs

#1 lore raw

G'day Wendy,

Your fsid-based approach does seem to make dropping locks less fiddly than using the virtual server address. Some minor nits...

Quoted 6 lines · Wendy Cheng

By writing exported filesytem id into /proc/fs/nfsd/nlm_unlock, this patch walks thru lockd's global nlm_files list to release all the locks associated with the particular id. It is used to enable NFS lock failover with active-active clustered servers.
[...]
static int

Review

Quoted 10 lines
(quoted diff) +3 -2
-nlm_traverse_files(struct nlm_host *host, int action)
+nlm_traverse_files(struct nlm_host *host, int *fsid_p, int action)
[...]
 {
 	struct nlm_file	*file, **fp;
-	int		i;
+	int		i, rc, fsid, act=action;
 
 	mutex_lock(&nlm_file_mutex);
+	if (fsid_p) fsid = *fsid_p;

I don't see any point initialising fsid like this, as you throw away that value before ever using it. Initialising to zero should be enough to stop the compiler complaining.

Review

Quoted 2 lines
(quoted diff) +2 -0
+				dprintk("lockd: drop lock file=0x%x fsid=%d\n",
+					(int) file, fsid);

`file' is a pointer, and will not be the same size as an int on 64bit machines. This is why %p exists, see examples in the existing NLM code.

Review nlm_traverse_files(struct nlm_host *host · line 311

Quoted 6 lines
(quoted diff) +2 -0
@@ -253,6 +307,8 @@ nlm_traverse_files(struct nlm_host *host
 			/* No more references to this file. Let go of it. */
 			if (!file->f_blocks && !file->f_locks
 			 && !file->f_shares && !file->f_count) {
+				dprintk("lockd: fo_unlock close file=0x%x\n", 
+					(int) file);

Ditto.

Review static ssize_t (*write_op[])(struct file · line 109

Quoted 8 lines
(quoted diff) +1 -0
@@ -90,6 +103,7 @@ static ssize_t (*write_op[])(struct file
 	[NFSD_Getfd] = write_getfd,
 	[NFSD_Getfs] = write_getfs,
 	[NFSD_Fh] = write_filehandle,
+	[NFSD_Nlm_unlock] = do_nlm_fo_unlock,
 	[NFSD_Threads] = write_threads,
 	[NFSD_Versions] = write_versions,
 #ifdef CONFIG_NFSD_V4

All the other entries in write_op[] have a consistent naming scheme, being called write_foo().

Quoted 7 lines
(quoted diff) +3 -0
@@ -334,6 +348,32 @@ static ssize_t write_filehandle(struct f
 	return mesg - buf;	
 }
 
+static ssize_t do_nlm_fo_unlock(struct file *file, char *buf, size_t size)
+{
+	char *mesg = buf;

Review static ssize_t write_filehandle(struct f · line 363

Quoted 10 lines
(quoted diff) +10 -0
+	int fsid, rc;
+
+	if (size <= 0) return -EINVAL;
+
+	/* convert string into a valid fsid */
+	rc = get_int(&mesg, &fsid);
+	if (rc) {
+		dprintk("nfsd: do_nlm_ip_unlock invalid ip(%s)\n", buf);
+		return rc;
+	}

This dprintk seems to reflect the function's previous name and semantics.

Review static ssize_t write_filehandle(struct f · line 366

Quoted 3 lines
(quoted diff) +3 -0
+
+	/* call nlm to release the locks */
+	rc = nlmsvc_fo_unlock(&fsid);

I don't understand why you pass this parameter by reference? It's not large and none of the functions called by nlmsvc_fo_unlock() need to write to it.

include/linux/nfsd/debug.h line 38

Quoted 11 lines
(quoted diff) +0 -0
 #ifdef CONFIG_NFSD_V4
include/linux/nfsd/debug.h +1 -0
@@ -32,6 +32,7 @@
 #define NFSDDBG_REPCACHE	0x0080
 #define NFSDDBG_XDR		0x0100
 #define NFSDDBG_LOCKD		0x0200
+#define NFSDDBG_CLUSTER 	0x0400
 #define NFSDDBG_ALL		0x7FFF
 #define NFSDDBG_NOCHANGE	0xFFFF
 

You don't seem to use this anywhere.

Greg.

Signature · 2 lines
Greg Banks, R&D Software Engineer, SGI Australian Software Group.
I don't speak for SGI.
#2 lore raw
Quoted 5 lines · Greg Banks

G'day Wendy,

Your fsid-based approach does seem to make dropping locks less fiddly than using the virtual server address. Some minor nits...

Thanks ! Another pair of eyes are always helpful. Will work on it - ditto for Patch 5-2.

-- Wendy

Quoted 94 lines · Wendy Cheng
Quoted 6 lines

By writing exported filesytem id into /proc/fs/nfsd/nlm_unlock, this patch walks thru lockd's global nlm_files list to release all the locks associated with the particular id. It is used to enable NFS lock failover with active-active clustered servers.
[...]
static int

Review

Quoted 10 lines
(quoted diff) +3 -2
-nlm_traverse_files(struct nlm_host *host, int action)
+nlm_traverse_files(struct nlm_host *host, int *fsid_p, int action)
[...]
 {
 	struct nlm_file	*file, **fp;
-	int		i;
+	int		i, rc, fsid, act=action;
 
 	mutex_lock(&nlm_file_mutex);
+	if (fsid_p) fsid = *fsid_p;

I don't see any point initialising fsid like this, as you throw away that value before ever using it. Initialising to zero should be enough to stop the compiler complaining.

Review

Quoted 2 lines
(quoted diff) +2 -0
+				dprintk("lockd: drop lock file=0x%x fsid=%d\n",
+					(int) file, fsid);

`file' is a pointer, and will not be the same size as an int on 64bit machines. This is why %p exists, see examples in the existing NLM code.

Review nlm_traverse_files(struct nlm_host *host · line 311

Quoted 6 lines
(quoted diff) +2 -0
@@ -253,6 +307,8 @@ nlm_traverse_files(struct nlm_host *host
 			/* No more references to this file. Let go of it. */
 			if (!file->f_blocks && !file->f_locks
 			 && !file->f_shares && !file->f_count) {
+				dprintk("lockd: fo_unlock close file=0x%x\n", 
+					(int) file);

Ditto.

Review static ssize_t (*write_op[])(struct file · line 109

Quoted 8 lines
(quoted diff) +1 -0
@@ -90,6 +103,7 @@ static ssize_t (*write_op[])(struct file
 	[NFSD_Getfd] = write_getfd,
 	[NFSD_Getfs] = write_getfs,
 	[NFSD_Fh] = write_filehandle,
+	[NFSD_Nlm_unlock] = do_nlm_fo_unlock,
 	[NFSD_Threads] = write_threads,
 	[NFSD_Versions] = write_versions,
 #ifdef CONFIG_NFSD_V4

All the other entries in write_op[] have a consistent naming scheme, being called write_foo().

Quoted 7 lines
(quoted diff) +3 -0
@@ -334,6 +348,32 @@ static ssize_t write_filehandle(struct f
 	return mesg - buf;	
 }
 
+static ssize_t do_nlm_fo_unlock(struct file *file, char *buf, size_t size)
+{
+	char *mesg = buf;

Review static ssize_t write_filehandle(struct f · line 363

Quoted 10 lines
(quoted diff) +10 -0
+	int fsid, rc;
+
+	if (size <= 0) return -EINVAL;
+
+	/* convert string into a valid fsid */
+	rc = get_int(&mesg, &fsid);
+	if (rc) {
+		dprintk("nfsd: do_nlm_ip_unlock invalid ip(%s)\n", buf);
+		return rc;
+	}

This dprintk seems to reflect the function's previous name and semantics.

Review static ssize_t write_filehandle(struct f · line 366

Quoted 3 lines
(quoted diff) +3 -0
+
+	/* call nlm to release the locks */
+	rc = nlmsvc_fo_unlock(&fsid);

I don't understand why you pass this parameter by reference? It's not large and none of the functions called by nlmsvc_fo_unlock() need to write to it.

include/linux/nfsd/debug.h line 38

Quoted 11 lines
(quoted diff) +0 -0
 #ifdef CONFIG_NFSD_V4
include/linux/nfsd/debug.h +1 -0
@@ -32,6 +32,7 @@
 #define NFSDDBG_REPCACHE	0x0080
 #define NFSDDBG_XDR		0x0100
 #define NFSDDBG_LOCKD		0x0200
+#define NFSDDBG_CLUSTER 	0x0400
 #define NFSDDBG_ALL		0x7FFF
 #define NFSDDBG_NOCHANGE	0xFFFF
 

You don't seem to use this anywhere.

Quoted 1 line

Greg.

-------------------------------------------------------------------------
Using Tomcat but need to do more? Need to support web services, security? Get stuff done quickly with pre-integrated technology to make your job easier Download IBM WebSphere Application Server v.1.0.1 based on Apache Geronimo
http://sel.as-us.falkag.net/sel?cmd=lnk&kid=120709&bid=263057&dat=121642
_______________________________________________
NFS maillist - [email protected]
https://lists.sourceforge.net/lists/listinfo/nfs