Skip to contentSkip to conversation

pNFS generic device ID cache version 3

PATCH In review [email protected] 3 participants 6 messages Apr 26, 2010 – May 3, 2010

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

cover pNFS generic device ID cache version 3

This patch set implements a shared RCU device ID cache servicing multiple mounts of a single layout type per meta data server (struct nfs_client).

Device id's are referenced by layout segments which hold a pointer to the nfs4_deviceid struct.

An hlist is used for the cache due to the large number of devices used by the object and block layout drivers.

Note that nfs4_deviceid_hash() is the same as the NFSD opaque_hash function. Perhaps they should be shared.

0001-SQUASHME-pnfs_submit-generic-device-ID-cache.patch 0002-SQUASHME-pnfs_submit-fix-multiple-mount-set_pnfs_lay.patch 0003-SQUASHME-pnfs-submit-file-layout-driver-generic-devi.patch

These patches apply to the 2.6.34-rc3 pnfs-submit branch.

Testing:
-------
CONFIG_NFS_V4_1 set:

NFSv4.1/pNFS mounts:
Connectathon tests pass against GFS2/pNFS with a single AUTH_SYS mount, a double AUTH_SYS mount, and an AUTH_SYS and AUTH_GSS/KRB5 mount (which creates two superblocks under a struct nfs_client and both share the device id cache).

NFSv4.0 mount;
Connectathon tests pass

Did not test with multiple device ID's. I will create a mulitple device ID test with the pynfs file layout server.

CONFIG_NFS_V4_1 not set:

NFSv4.0 mount: Connectathon tests pass.

-->Andy

#1 lore raw

PATCH 1/3 SQUASHME pnfs_submit: generic device ID cache

From: Andy Adamson <[email protected]>

A shared RCU device ID cache servicing multiple mounts of a single layout type per meta data server (struct nfs_client).

Device IDs of type deviceid4 are required by all layout types, long lived and read at each I/O. They are added to the deviceid cache at first reference by a layout via GETDEVICEINFO and (currently) are only removed at umount.

Reference count the device ID cache for each mounted file system in the initialize_mountpoint layoutdriver_io_operation.

Dereference the device id cache on file system in the uninitialize_mountpoint layoutdriver_io_operation called at umount

Each layoutsegment assigns a pointer and takes a reference to the nfs4_deviceid structure identified by the layout deviceid. This is so that there are no deviceid lookups for the normal I/O path.

Even thought required by all layouttypes, the deviceid is not exposed in the LAYOUTGET4res but is instead hidden in the opaque layouttype4.

Therefore, each layout type alloc_lseg calls nfs4_set_layout_deviceid, and free_lseg calls nfs4_unset_layout_deviceid.

While the file layout driver will not cache very many deviceid's, the object and block layout drivers could cache 100's for a large installation. Use an hlist.

---
 fs/nfs/pnfs.c             |  167 +++++++++++++++++++++++++++++++++++++++++++++
 include/linux/nfs4_pnfs.h |   50 +++++++++++++
 include/linux/nfs_fs_sb.h |    1 +
 3 files changed, 218 insertions(+), 0 deletions(-)
fs/nfs/pnfs.c +167 -0
index 91572aa..bf906cc 100644
@@ -45,6 +45,7 @@
 #include <linux/nfs4.h>
 #include <linux/pnfs_xdr.h>
 #include <linux/nfs4_pnfs.h>
+#include <linux/rculist.h>
 
 #include "internal.h"
 #include "nfs4_fs.h"
@@ -2296,3 +2297,169 @@ struct pnfs_client_operations pnfs_ops = {
 
 EXPORT_SYMBOL(pnfs_unregister_layoutdriver);
 EXPORT_SYMBOL(pnfs_register_layoutdriver);
+
+
+/* Device ID cache. Supports one layout type per struct nfs_client */
+int
+nfs4_alloc_init_deviceid_cache(struct nfs_client *clp,
+			 void (*free_callback)(struct kref *))
+{
+	struct nfs4_deviceid_cache *c;
+
+	c = kzalloc(sizeof(struct nfs4_deviceid_cache), GFP_KERNEL);
+	if (!c)
+		return -ENOMEM;
+	spin_lock(&clp->cl_lock);
+	if (clp->cl_devid_cache != NULL) {
+		kref_get(&clp->cl_devid_cache->dc_kref);
+		spin_unlock(&clp->cl_lock);
+		dprintk("%s [kref [%d]]\n", __func__,
+			atomic_read(&clp->cl_devid_cache->dc_kref.refcount));
+		kfree(c);
+	} else {
+		int i;
+
+		spin_lock_init(&c->dc_lock);
+		for (i = 0; i < NFS4_DEVICE_ID_HASH_SIZE ; i++)
+			INIT_HLIST_HEAD(&c->dc_deviceids[i]);
+		kref_init(&c->dc_kref);
+		c->dc_free_callback = free_callback;
+		clp->cl_devid_cache = c;
+		spin_unlock(&clp->cl_lock);
+		dprintk("%s [new]\n", __func__);
+	}
+	return 0;
+}
+EXPORT_SYMBOL(nfs4_alloc_init_deviceid_cache);
+
+void
+nfs4_init_deviceid_node(struct nfs4_deviceid *d)
+{
+	INIT_HLIST_NODE(&d->de_node);
+	kref_init(&d->de_kref);
+}
+EXPORT_SYMBOL(nfs4_init_deviceid_node);
+
+/* Called from layoutdriver_io_operations->alloc_lseg */
+void
+nfs4_set_layout_deviceid(struct pnfs_layout_segment *l, struct nfs4_deviceid *d)
+{
+	dprintk("%s [%d]\n", __func__, atomic_read(&d->de_kref.refcount));
+	l->deviceid = d;
+	kref_get(&d->de_kref);
+}
+EXPORT_SYMBOL(nfs4_set_layout_deviceid);
+
+/* Called from layoutdriver_io_operations->free_lseg */
+void
+nfs4_unset_layout_deviceid(struct pnfs_layout_segment *l,
+			   struct nfs4_deviceid *d,
+			   void (*free_callback)(struct kref *))
+{
+	dprintk("%s [%d]\n", __func__, atomic_read(&d->de_kref.refcount));
+	l->deviceid = NULL;
+	kref_put(&d->de_kref, free_callback);
+}
+EXPORT_SYMBOL(nfs4_unset_layout_deviceid);
+
+struct nfs4_deviceid *
+nfs4_find_deviceid(struct nfs4_deviceid_cache *c, struct pnfs_deviceid *id)
+{
+	struct nfs4_deviceid *d;
+	struct hlist_node *n;
+	long hash = nfs4_deviceid_hash(id);
+
+	dprintk("--> %s hash %ld\n", __func__, hash);
+	rcu_read_lock();
+	hlist_for_each_entry_rcu(d, n, &c->dc_deviceids[hash], de_node) {
+		if (!memcmp(&d->de_id, id, NFS4_PNFS_DEVICEID4_SIZE)) {
+			rcu_read_unlock();
+			return d;
+		}
+	}
+	rcu_read_unlock();
+	return NULL;
+}
+EXPORT_SYMBOL(nfs4_find_deviceid);
+
+/*
+ * Add or kref_get a deviceid.
+ * GETDEVICEINFOs for same deviceid can race. If deviceid is found, discard new
+ */
+struct nfs4_deviceid *
+nfs4_add_deviceid(struct nfs4_deviceid_cache *c, struct nfs4_deviceid *new)
+{
+	struct nfs4_deviceid *d;
+	struct hlist_node *n;
+	long hash = nfs4_deviceid_hash(&new->de_id);
+
+	dprintk("--> %s hash %ld\n", __func__, hash);
+	spin_lock(&c->dc_lock);
+	hlist_for_each_entry_rcu(d, n, &c->dc_deviceids[hash], de_node) {
+		if (!memcmp(&d->de_id, &new->de_id, NFS4_PNFS_DEVICEID4_SIZE)) {
+			spin_unlock(&c->dc_lock);
+			dprintk("%s [discard]\n", __func__);
+			c->dc_free_callback(&new->de_kref);
+			return d;
+		}
+	}
+	hlist_add_head_rcu(&new->de_node, &c->dc_deviceids[hash]);
+	spin_unlock(&c->dc_lock);
+	dprintk("%s [new]\n", __func__);
+	return new;
+}
+EXPORT_SYMBOL(nfs4_add_deviceid);
+
+static int
+nfs4_remove_deviceid(struct nfs4_deviceid_cache *c, long hash)
+{
+	struct nfs4_deviceid *d;
+	struct hlist_node *n;
+
+	dprintk("--> %s hash %ld\n", __func__, hash);
+	spin_lock(&c->dc_lock);
+	hlist_for_each_entry_rcu(d, n, &c->dc_deviceids[hash], de_node) {
+		hlist_del_rcu(&d->de_node);
+		spin_unlock(&c->dc_lock);
+		synchronize_rcu();
+		dprintk("%s [%d]\n", __func__,
+			atomic_read(&d->de_kref.refcount));
+		kref_put(&d->de_kref, c->dc_free_callback);
+		return 1;
+	}
+	spin_unlock(&c->dc_lock);
+	return 0;
+}
+
+static void
+nfs4_free_deviceid_cache(struct kref *kref)
+{
+	struct nfs4_deviceid_cache *cache =
+		container_of(kref, struct nfs4_deviceid_cache, dc_kref);
+	int more;
+	long i;
+
+	for (i = 0; i < NFS4_DEVICE_ID_HASH_SIZE; i++) {
+		more = 1;
+		while (more)
+			more = nfs4_remove_deviceid(cache, i);
+	}
+	kfree(cache);
+}
+
+void
+nfs4_put_deviceid_cache(struct nfs_client *clp)
+{
+	struct nfs4_deviceid_cache *tmp = clp->cl_devid_cache;
+	int refcount;
+
+	dprintk("--> %s cl_devid_cache %p\n", __func__, clp->cl_devid_cache);
+	spin_lock(&clp->cl_lock);
+	refcount = atomic_read(&clp->cl_devid_cache->dc_kref.refcount);
+	if (refcount == 1)
+		clp->cl_devid_cache = NULL;
+	spin_unlock(&clp->cl_lock);
+	dprintk("%s [%d]\n", __func__, refcount);
+	kref_put(&tmp->dc_kref, nfs4_free_deviceid_cache);
+}
+EXPORT_SYMBOL(nfs4_put_deviceid_cache);
include/linux/nfs4_pnfs.h +50 -0

62 lines not on this page · raw message

include/linux/nfs_fs_sb.h +1 -0
index 8522461..ef2e18e 100644
@@ -87,6 +87,7 @@ struct nfs_client {
 	u32			cl_exchange_flags;
 	struct nfs4_session	*cl_session; 	/* sharred session */
 	struct list_head	cl_lo_inodes;	/* Inodes having layouts */
+	struct nfs4_deviceid_cache *cl_devid_cache; /* pNFS deviceid cache */
 #endif /* CONFIG_NFS_V4_1 */
 
 #ifdef CONFIG_NFS_FSCACHE
Signature · 1 line
1.6.6
#2 lore raw

PATCH 2/3 SQUASHME pnfs_submit: fix multiple mount set_pnfs_layoutdriver

From: Andy Adamson <[email protected]>

The same struct nfs_server can enter set_pnfs_layoutdriver for mounts that share a super block. Don't initialize a pnfs mountpoint more than once.

Don't set the pnfs_curr_ld until the pnfs mountpoint initialization succeeds

---
 fs/nfs/pnfs.c |   15 ++++++++++-----
 1 files changed, 10 insertions(+), 5 deletions(-)
fs/nfs/pnfs.c +10 -5
index bf906cc..a3e8231 100644
@@ -215,20 +215,25 @@ set_pnfs_layoutdriver(struct super_block *sb, struct nfs_fh *fh, u32 id)
 	struct pnfs_mount_type *mt;
 	struct nfs_server *server = NFS_SB(sb);
 
+	if (server->pnfs_curr_ld)
+		return;
+
 	if (id > 0 && find_pnfs(id, &mod)) {
-		dprintk("%s: Setting pNFS module\n", __func__);
-		server->pnfs_curr_ld = mod->pnfs_ld_type;
-		mt = server->pnfs_curr_ld->ld_io_ops->initialize_mountpoint(
+		mt = mod->pnfs_ld_type->ld_io_ops->initialize_mountpoint(
 			sb, fh);
 		if (!mt) {
 			printk(KERN_ERR "%s: Error initializing mount point "
 			       "for layout driver %u. ", __func__, id);
 			goto out_err;
 		}
-		/* Layout driver succeeded in initializing mountpoint */
+		/*
+		 * Layout driver succeeded in initializing mountpoint
+		 * and has taken a reference on the nfs_client cl_devid_cache
+		 */
+		server->pnfs_curr_ld = mod->pnfs_ld_type;
 		server->pnfs_mountid = mt;
-		/* Set the rpc_ops */
 		server->nfs_client->rpc_ops = &pnfs_v4_clientops;
+		dprintk("%s: pNFS module for %u set\n", __func__, id);
 		return;
 	}
 
Signature · 1 line
1.6.6
#3 lore raw

PATCH 3/3 SQUASHME pnfs-submit: file layout driver generic device ID cache

From: Andy Adamson <[email protected]>

Replace the per superblock deviceid cache with the generic deviceid cache.

Embed struct nfs4_deviceid into struct nfs4_file_layout_dsaddr, the file layout specific deviceid structure. Provide a free_deviceid_callback.

---
 fs/nfs/client.c            |    1 +
 fs/nfs/nfs4filelayout.c    |   54 ++++++------
 fs/nfs/nfs4filelayout.h    |   12 +--
 fs/nfs/nfs4filelayoutdev.c |  199 +++++++++-----------------------------------
 4 files changed, 71 insertions(+), 195 deletions(-)
fs/nfs/client.c +1 -0
index e13ccb7..887d71e 100644
@@ -38,6 +38,7 @@
 #include <net/ipv6.h>
 #include <linux/nfs_xdr.h>
 #include <linux/sunrpc/bc_xprt.h>
+#include <linux/nfs4_pnfs.h>
 
 #include <asm/system.h>
 
fs/nfs/nfs4filelayout.c +27 -27

137 lines not on this page · raw message

fs/nfs/nfs4filelayout.h +4 -8
index 12498a2..fbf307c 100644
@@ -43,8 +43,7 @@ struct nfs4_pnfs_ds {
 };
 
 struct nfs4_file_layout_dsaddr {
-	struct hlist_node	hash_node;   /* nfs4_pnfs_dev_hlist dev_list */
-	struct pnfs_deviceid	dev_id;
+	struct nfs4_deviceid	deviceid;
 	u32 			stripe_count;
 	u8			*stripe_indices;
 	u32			ds_num;
@@ -86,15 +85,13 @@ struct nfs4_filelayout {
 
 struct filelayout_mount_type {
 	struct super_block *fl_sb;
-	struct nfs4_pnfs_dev_hlist *hlist;
 };
 
 extern struct pnfs_client_operations *pnfs_callback_ops;
 
+extern void nfs4_fl_free_deviceid_callback(struct kref *);
 extern void print_ds(struct nfs4_pnfs_ds *ds);
 char *deviceid_fmt(const struct pnfs_deviceid *dev_id);
-int  nfs4_pnfs_devlist_init(struct nfs4_pnfs_dev_hlist *hlist);
-void nfs4_pnfs_devlist_destroy(struct nfs4_pnfs_dev_hlist *hlist);
 int nfs4_pnfs_dserver_get(struct pnfs_layout_segment *lseg,
 			  loff_t offset,
 			  size_t count,
@@ -102,9 +99,8 @@ int nfs4_pnfs_dserver_get(struct pnfs_layout_segment *lseg,
 u32 filelayout_dserver_get_index(loff_t offset,
 				 struct nfs4_file_layout_dsaddr *di,
 				 struct nfs4_filelayout_segment *layout);
-struct nfs4_file_layout_dsaddr *
-nfs4_pnfs_device_item_find(struct nfs4_pnfs_dev_hlist *hlist,
-			   struct pnfs_deviceid *dev_id);
+extern struct nfs4_file_layout_dsaddr *
+nfs4_pnfs_device_item_find(struct nfs_client *, struct pnfs_deviceid *dev_id);
 struct nfs4_file_layout_dsaddr *
 get_device_info(struct inode *inode, struct pnfs_deviceid *dev_id);
 
fs/nfs/nfs4filelayoutdev.c +39 -160

297 lines not on this page · raw message

Signature · 1 line
1.6.6
#4 lore raw

Re: [pnfs] [PATCH 1/3] SQUASHME pnfs_submit: generic device ID cache

In reply to [email protected]
Quoted 188 lines · On Apr. 26, 2010, 19:18 +0300, [email protected] wrote:

From: Andy Adamson <[email protected]>

A shared RCU device ID cache servicing multiple mounts of a single layout type per meta data server (struct nfs_client).

Device IDs of type deviceid4 are required by all layout types, long lived and read at each I/O. They are added to the deviceid cache at first reference by a layout via GETDEVICEINFO and (currently) are only removed at umount.

Reference count the device ID cache for each mounted file system in the initialize_mountpoint layoutdriver_io_operation.

Dereference the device id cache on file system in the uninitialize_mountpoint layoutdriver_io_operation called at umount

Each layoutsegment assigns a pointer and takes a reference to the nfs4_deviceid structure identified by the layout deviceid. This is so that there are no deviceid lookups for the normal I/O path.

Even thought required by all layouttypes, the deviceid is not exposed in the LAYOUTGET4res but is instead hidden in the opaque layouttype4.

Therefore, each layout type alloc_lseg calls nfs4_set_layout_deviceid, and free_lseg calls nfs4_unset_layout_deviceid.

While the file layout driver will not cache very many deviceid's, the object and block layout drivers could cache 100's for a large installation. Use an hlist.

---
 fs/nfs/pnfs.c             |  167 +++++++++++++++++++++++++++++++++++++++++++++
 include/linux/nfs4_pnfs.h |   50 +++++++++++++
 include/linux/nfs_fs_sb.h |    1 +
 3 files changed, 218 insertions(+), 0 deletions(-)
fs/nfs/pnfs.c +137 -0
index 91572aa..bf906cc 100644
@@ -45,6 +45,7 @@
 #include <linux/nfs4.h>
 #include <linux/pnfs_xdr.h>
 #include <linux/nfs4_pnfs.h>
+#include <linux/rculist.h>
 
 #include "internal.h"
 #include "nfs4_fs.h"
@@ -2296,3 +2297,169 @@ struct pnfs_client_operations pnfs_ops = {
 
 EXPORT_SYMBOL(pnfs_unregister_layoutdriver);
 EXPORT_SYMBOL(pnfs_register_layoutdriver);
+
+
+/* Device ID cache. Supports one layout type per struct nfs_client */
+int
+nfs4_alloc_init_deviceid_cache(struct nfs_client *clp,
+			 void (*free_callback)(struct kref *))
+{
+	struct nfs4_deviceid_cache *c;
+
+	c = kzalloc(sizeof(struct nfs4_deviceid_cache), GFP_KERNEL);
+	if (!c)
+		return -ENOMEM;
+	spin_lock(&clp->cl_lock);
+	if (clp->cl_devid_cache != NULL) {
+		kref_get(&clp->cl_devid_cache->dc_kref);
+		spin_unlock(&clp->cl_lock);
+		dprintk("%s [kref [%d]]\n", __func__,
+			atomic_read(&clp->cl_devid_cache->dc_kref.refcount));
+		kfree(c);
+	} else {
+		int i;
+
+		spin_lock_init(&c->dc_lock);
+		for (i = 0; i < NFS4_DEVICE_ID_HASH_SIZE ; i++)
+			INIT_HLIST_HEAD(&c->dc_deviceids[i]);
+		kref_init(&c->dc_kref);
+		c->dc_free_callback = free_callback;
+		clp->cl_devid_cache = c;
+		spin_unlock(&clp->cl_lock);
+		dprintk("%s [new]\n", __func__);
+	}
+	return 0;
+}
+EXPORT_SYMBOL(nfs4_alloc_init_deviceid_cache);
+
+void
+nfs4_init_deviceid_node(struct nfs4_deviceid *d)
+{
+	INIT_HLIST_NODE(&d->de_node);
+	kref_init(&d->de_kref);
+}
+EXPORT_SYMBOL(nfs4_init_deviceid_node);
+
+/* Called from layoutdriver_io_operations->alloc_lseg */
+void
+nfs4_set_layout_deviceid(struct pnfs_layout_segment *l, struct nfs4_deviceid *d)
+{
+	dprintk("%s [%d]\n", __func__, atomic_read(&d->de_kref.refcount));
+	l->deviceid = d;
+	kref_get(&d->de_kref);
+}
+EXPORT_SYMBOL(nfs4_set_layout_deviceid);
+
+/* Called from layoutdriver_io_operations->free_lseg */
+void
+nfs4_unset_layout_deviceid(struct pnfs_layout_segment *l,
+			   struct nfs4_deviceid *d,
+			   void (*free_callback)(struct kref *))
+{
+	dprintk("%s [%d]\n", __func__, atomic_read(&d->de_kref.refcount));
+	l->deviceid = NULL;
+	kref_put(&d->de_kref, free_callback);
+}
+EXPORT_SYMBOL(nfs4_unset_layout_deviceid);
+
+struct nfs4_deviceid *
+nfs4_find_deviceid(struct nfs4_deviceid_cache *c, struct pnfs_deviceid *id)
+{
+	struct nfs4_deviceid *d;
+	struct hlist_node *n;
+	long hash = nfs4_deviceid_hash(id);
+
+	dprintk("--> %s hash %ld\n", __func__, hash);
+	rcu_read_lock();
+	hlist_for_each_entry_rcu(d, n, &c->dc_deviceids[hash], de_node) {
+		if (!memcmp(&d->de_id, id, NFS4_PNFS_DEVICEID4_SIZE)) {
+			rcu_read_unlock();
+			return d;
+		}
+	}
+	rcu_read_unlock();
+	return NULL;
+}
+EXPORT_SYMBOL(nfs4_find_deviceid);
+
+/*
+ * Add or kref_get a deviceid.
+ * GETDEVICEINFOs for same deviceid can race. If deviceid is found, discard new
+ */
+struct nfs4_deviceid *
+nfs4_add_deviceid(struct nfs4_deviceid_cache *c, struct nfs4_deviceid *new)
+{
+	struct nfs4_deviceid *d;
+	struct hlist_node *n;
+	long hash = nfs4_deviceid_hash(&new->de_id);
+
+	dprintk("--> %s hash %ld\n", __func__, hash);
+	spin_lock(&c->dc_lock);
+	hlist_for_each_entry_rcu(d, n, &c->dc_deviceids[hash], de_node) {
+		if (!memcmp(&d->de_id, &new->de_id, NFS4_PNFS_DEVICEID4_SIZE)) {
+			spin_unlock(&c->dc_lock);
+			dprintk("%s [discard]\n", __func__);
+			c->dc_free_callback(&new->de_kref);
+			return d;
+		}
+	}
+	hlist_add_head_rcu(&new->de_node, &c->dc_deviceids[hash]);
+	spin_unlock(&c->dc_lock);
+	dprintk("%s [new]\n", __func__);
+	return new;
+}
+EXPORT_SYMBOL(nfs4_add_deviceid);
+
+static int
+nfs4_remove_deviceid(struct nfs4_deviceid_cache *c, long hash)
+{
+	struct nfs4_deviceid *d;
+	struct hlist_node *n;
+
+	dprintk("--> %s hash %ld\n", __func__, hash);
+	spin_lock(&c->dc_lock);
+	hlist_for_each_entry_rcu(d, n, &c->dc_deviceids[hash], de_node) {
+		hlist_del_rcu(&d->de_node);
+		spin_unlock(&c->dc_lock);
+		synchronize_rcu();
+		dprintk("%s [%d]\n", __func__,
+			atomic_read(&d->de_kref.refcount));
+		kref_put(&d->de_kref, c->dc_free_callback);
+		return 1;
+	}
+	spin_unlock(&c->dc_lock);
+	return 0;
+}
+
+static void
+nfs4_free_deviceid_cache(struct kref *kref)

fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops = { · line 2445

Quoted 10 lines
fs/nfs/pnfs.c +10 -0
+{
+	struct nfs4_deviceid_cache *cache =
+		container_of(kref, struct nfs4_deviceid_cache, dc_kref);
+	int more;
+	long i;
+
+	for (i = 0; i < NFS4_DEVICE_ID_HASH_SIZE; i++) {
+		more = 1;
+		while (more)
+			more = nfs4_remove_deviceid(cache, i);

Andy, this can be simplified to

		while (nfs4_remove_deviceid(cache, i))
			;

If ok with you, I'll make this change upon merging.

Benny

Quoted 100 lines
fs/nfs/pnfs.c +20 -0
+	}
+	kfree(cache);
+}
+
+void
+nfs4_put_deviceid_cache(struct nfs_client *clp)
+{
+	struct nfs4_deviceid_cache *tmp = clp->cl_devid_cache;
+	int refcount;
+
+	dprintk("--> %s cl_devid_cache %p\n", __func__, clp->cl_devid_cache);
+	spin_lock(&clp->cl_lock);
+	refcount = atomic_read(&clp->cl_devid_cache->dc_kref.refcount);
+	if (refcount == 1)
+		clp->cl_devid_cache = NULL;
+	spin_unlock(&clp->cl_lock);
+	dprintk("%s [%d]\n", __func__, refcount);
+	kref_put(&tmp->dc_kref, nfs4_free_deviceid_cache);
+}
+EXPORT_SYMBOL(nfs4_put_deviceid_cache);
include/linux/nfs4_pnfs.h +50 -0
index 3caac60..3b7aeb7 100644
@@ -106,6 +106,7 @@ struct pnfs_layout_segment {
 	struct kref kref;
 	bool valid;
 	struct pnfs_layout_type *layout;
+	struct nfs4_deviceid *deviceid;
 	u8 ld_data[];			/* layout driver private data */
 };
 
@@ -275,6 +276,55 @@ struct pnfs_devicelist {
 	struct pnfs_deviceid	dev_id[NFS4_PNFS_GETDEVLIST_MAXNUM];
 };
 
+/*
+ * Device ID RCU cache. A device ID is unique per client ID and layout type.
+ */
+#define NFS4_DEVICE_ID_HASH_BITS	5
+#define NFS4_DEVICE_ID_HASH_SIZE	(1 << NFS4_DEVICE_ID_HASH_BITS)
+#define NFS4_DEVICE_ID_HASH_MASK	(NFS4_DEVICE_ID_HASH_SIZE - 1)
+
+static inline u32
+nfs4_deviceid_hash(struct pnfs_deviceid *id)
+{
+	unsigned char *cptr = (unsigned char *)id->data;
+	unsigned int nbytes = NFS4_PNFS_DEVICEID4_SIZE;
+	u32 x = 0;
+
+	while (nbytes--) {
+		x *= 37;
+		x += *cptr++;
+	}
+	return x & NFS4_DEVICE_ID_HASH_MASK;
+}
+
+struct nfs4_deviceid_cache {
+	spinlock_t		dc_lock;
+	struct kref		dc_kref;
+	void			(*dc_free_callback)(struct kref *);
+	struct hlist_head	dc_deviceids[NFS4_DEVICE_ID_HASH_SIZE];
+};
+
+/* Device ID cache node */
+struct nfs4_deviceid {
+	struct hlist_node	de_node;
+	struct pnfs_deviceid	de_id;
+	struct kref		de_kref;
+};
+
+extern int nfs4_alloc_init_deviceid_cache(struct nfs_client *,
+				void (*free_callback)(struct kref *));
+extern void nfs4_put_deviceid_cache(struct nfs_client *);
+extern void nfs4_init_deviceid_node(struct nfs4_deviceid *);
+extern struct nfs4_deviceid *nfs4_find_deviceid(struct nfs4_deviceid_cache *,
+				struct pnfs_deviceid *);
+extern struct nfs4_deviceid *nfs4_add_deviceid(struct nfs4_deviceid_cache *,
+				struct nfs4_deviceid *);
+extern void nfs4_set_layout_deviceid(struct pnfs_layout_segment *,
+				struct nfs4_deviceid *);
+extern void nfs4_unset_layout_deviceid(struct pnfs_layout_segment *,
+				struct nfs4_deviceid *,
+				void (*free_callback)(struct kref *));
+
 /* pNFS client callback functions.
  * These operations allow the layout driver to access pNFS client
  * specific information or call pNFS client->server operations.
include/linux/nfs_fs_sb.h +1 -0
index 8522461..ef2e18e 100644
@@ -87,6 +87,7 @@ struct nfs_client {
 	u32			cl_exchange_flags;
 	struct nfs4_session	*cl_session; 	/* sharred session */
 	struct list_head	cl_lo_inodes;	/* Inodes having layouts */
+	struct nfs4_deviceid_cache *cl_devid_cache; /* pNFS deviceid cache */
 #endif /* CONFIG_NFS_V4_1 */
 
 #ifdef CONFIG_NFS_FSCACHE
#5 lore raw

Re: [pnfs] [PATCH 1/3] SQUASHME pnfs_submit: generic device ID cache

On Mon, May 3, 2010 at 7:48 AM, Benny Halevy <[email protected]> wrot= e:

Quoted 4 lines · On Apr. 26, 2010, 19:18 +0300, [email protected] wrote:
Quoted 3 lines

From: Andy Adamson <[email protected]>

A shared RCU device ID cache servicing multiple mounts of a single l=

ayout type

Quoted 3 lines
Quoted 3 lines

per meta data server (struct nfs_client).

Device IDs of type deviceid4 are required by all layout types, long =

lived and

Quoted 1 line
Quoted 1 line

read at each I/O. =A0They are added to the deviceid cache at first r=

eference by

Quoted 1 line
Quoted 1 line

a layout via GETDEVICEINFO and (currently) are only removed at umoun=

t.

Quoted 5 lines
Quoted 5 lines

Reference count the device ID cache for each mounted file system in the initialize_mountpoint layoutdriver_io_operation.

Dereference the device id cache on file system in the uninitialize_m=

ountpoint

Quoted 5 lines
Quoted 5 lines

layoutdriver_io_operation called at umount

Each layoutsegment assigns a pointer and takes a reference to the nfs4_deviceid structure identified by the layout deviceid. This is so that there are no deviceid lookups for the normal I/O pat=

h.

Quoted 2 lines
Quoted 2 lines

Even thought required by all layouttypes, the deviceid is not expose=

d in the

Quoted 3 lines
Quoted 3 lines

LAYOUTGET4res but is instead hidden in the opaque layouttype4.

Therefore, each layout type alloc_lseg calls nfs4_set_layout_devicei=

d,

Quoted 3 lines
Quoted 3 lines

and free_lseg calls nfs4_unset_layout_deviceid.

While the file layout driver will not cache very many deviceid's, th=

e object

Quoted 6 lines
Quoted 6 lines

and block layout drivers could cache 100's for a large installation. Use an hlist.

---
=A0fs/nfs/pnfs.c =A0 =A0 =A0 =A0 =A0 =A0 | =A0167 ++++++++++++++++++=

+++++++++++++++++++++++++++

Quoted 18 lines
Quoted 18 lines

=A0include/linux/nfs4_pnfs.h | =A0 50 +++++++++++++ =A0include/linux/nfs_fs_sb.h | =A0 =A01 + =A03 files changed, 218 insertions(+), 0 deletions(-)

fs/nfs/pnfs.c +0 -0
index 91572aa..bf906cc 100644
@@ -45,6 +45,7 @@

=A0#include <linux/nfs4.h>
=A0#include <linux/pnfs_xdr.h>
=A0#include <linux/nfs4_pnfs.h>
+#include <linux/rculist.h>

=A0#include "internal.h"
=A0#include "nfs4_fs.h"

fs/nfs/pnfs.c +0 -0
@@ -2296,3 +2297,169 @@ struct pnfs_client_operations pnfs_ops =3D {
 
Quoted 5 lines

fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D { · line 2302

Quoted 5 lines
fs/nfs/pnfs.c +3 -0
 =A0EXPORT_SYMBOL(pnfs_unregister_layoutdriver);
 =A0EXPORT_SYMBOL(pnfs_register_layoutdriver);
+
+
+/* Device ID cache. Supports one layout type per struct nfs_client =

*/

Quoted 3 lines

fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops = {

Quoted 3 lines
fs/nfs/pnfs.c +3 -0
+int
+nfs4_alloc_init_deviceid_cache(struct nfs_client *clp,
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0void (*free_callback)(s=

truct kref *))

Quoted 4 lines

fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops = {

Quoted 4 lines
fs/nfs/pnfs.c +4 -0
+{
+ =A0 =A0 struct nfs4_deviceid_cache *c;
+
+ =A0 =A0 c =3D kzalloc(sizeof(struct nfs4_deviceid_cache), GFP_KERN=

EL);

Quoted 8 lines

fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D { · line 2317

Quoted 8 lines
fs/nfs/pnfs.c +8 -0
+ =A0 =A0 if (!c)
+ =A0 =A0 =A0 =A0 =A0 =A0 return -ENOMEM;
+ =A0 =A0 spin_lock(&clp->cl_lock);
+ =A0 =A0 if (clp->cl_devid_cache !=3D NULL) {
+ =A0 =A0 =A0 =A0 =A0 =A0 kref_get(&clp->cl_devid_cache->dc_kref);
+ =A0 =A0 =A0 =A0 =A0 =A0 spin_unlock(&clp->cl_lock);
+ =A0 =A0 =A0 =A0 =A0 =A0 dprintk("%s [kref [%d]]\n", __func__,
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 atomic_read(&clp->cl_devid=

_cache->dc_kref.refcount));

Quoted 6 lines

fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D { · line 2323

Quoted 6 lines
fs/nfs/pnfs.c +6 -0
+ =A0 =A0 =A0 =A0 =A0 =A0 kfree(c);
+ =A0 =A0 } else {
+ =A0 =A0 =A0 =A0 =A0 =A0 int i;
+
+ =A0 =A0 =A0 =A0 =A0 =A0 spin_lock_init(&c->dc_lock);
+ =A0 =A0 =A0 =A0 =A0 =A0 for (i =3D 0; i < NFS4_DEVICE_ID_HASH_SIZE=

; i++)

Quoted 1 line

fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D { · line 2324

Quoted 1 line
fs/nfs/pnfs.c +1 -0
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 INIT_HLIST_HEAD(&c->dc_dev=

iceids[i]);

Quoted 13 lines
Quoted 13 lines
fs/nfs/pnfs.c +13 -0
+ =A0 =A0 =A0 =A0 =A0 =A0 kref_init(&c->dc_kref);
+ =A0 =A0 =A0 =A0 =A0 =A0 c->dc_free_callback =3D free_callback;
+ =A0 =A0 =A0 =A0 =A0 =A0 clp->cl_devid_cache =3D c;
+ =A0 =A0 =A0 =A0 =A0 =A0 spin_unlock(&clp->cl_lock);
+ =A0 =A0 =A0 =A0 =A0 =A0 dprintk("%s [new]\n", __func__);
+ =A0 =A0 }
+ =A0 =A0 return 0;
+}
+EXPORT_SYMBOL(nfs4_alloc_init_deviceid_cache);
+
+void
+nfs4_init_deviceid_node(struct nfs4_deviceid *d)
+{
Quoted 8 lines

fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops = {

Quoted 8 lines
fs/nfs/pnfs.c +8 -0
+ =A0 =A0 INIT_HLIST_NODE(&d->de_node);
+ =A0 =A0 kref_init(&d->de_kref);
+}
+EXPORT_SYMBOL(nfs4_init_deviceid_node);
+
+/* Called from layoutdriver_io_operations->alloc_lseg */
+void
+nfs4_set_layout_deviceid(struct pnfs_layout_segment *l, struct nfs4=

_deviceid *d)

Quoted 2 lines

fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D { · line 2347

Quoted 2 lines
fs/nfs/pnfs.c +2 -0
+{
+ =A0 =A0 dprintk("%s [%d]\n", __func__, atomic_read(&d->de_kref.ref=

count));

Quoted 9 lines

fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops = {

Quoted 9 lines
fs/nfs/pnfs.c +9 -0
+ =A0 =A0 l->deviceid =3D d;
+ =A0 =A0 kref_get(&d->de_kref);
+}
+EXPORT_SYMBOL(nfs4_set_layout_deviceid);
+
+/* Called from layoutdriver_io_operations->free_lseg */
+void
+nfs4_unset_layout_deviceid(struct pnfs_layout_segment *l,
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0struct nfs4_devicei=

d *d,

Quoted 1 line

fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D { · line 2357

Quoted 1 line
fs/nfs/pnfs.c +1 -0
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0void (*free_callbac=

k)(struct kref *))

Quoted 2 lines

fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D { · line 2359

Quoted 2 lines
fs/nfs/pnfs.c +2 -0
+{
+ =A0 =A0 dprintk("%s [%d]\n", __func__, atomic_read(&d->de_kref.ref=

count));

Quoted 7 lines

fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops = {

Quoted 7 lines
fs/nfs/pnfs.c +7 -0
+ =A0 =A0 l->deviceid =3D NULL;
+ =A0 =A0 kref_put(&d->de_kref, free_callback);
+}
+EXPORT_SYMBOL(nfs4_unset_layout_deviceid);
+
+struct nfs4_deviceid *
+nfs4_find_deviceid(struct nfs4_deviceid_cache *c, struct pnfs_devic=

eid *id)

Quoted 8 lines

fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D { · line 2374

Quoted 8 lines
fs/nfs/pnfs.c +8 -0
+{
+ =A0 =A0 struct nfs4_deviceid *d;
+ =A0 =A0 struct hlist_node *n;
+ =A0 =A0 long hash =3D nfs4_deviceid_hash(id);
+
+ =A0 =A0 dprintk("--> %s hash %ld\n", __func__, hash);
+ =A0 =A0 rcu_read_lock();
+ =A0 =A0 hlist_for_each_entry_rcu(d, n, &c->dc_deviceids[hash], de_=

node) {

Quoted 1 line

fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D { · line 2375

Quoted 1 line
fs/nfs/pnfs.c +1 -0
+ =A0 =A0 =A0 =A0 =A0 =A0 if (!memcmp(&d->de_id, id, NFS4_PNFS_DEVIC=

EID4_SIZE)) {

Quoted 4 lines
Quoted 4 lines
fs/nfs/pnfs.c +4 -0
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 rcu_read_unlock();
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 return d;
+ =A0 =A0 =A0 =A0 =A0 =A0 }
+ =A0 =A0 }
Quoted 8 lines

fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops = {

Quoted 8 lines
fs/nfs/pnfs.c +8 -0
+ =A0 =A0 rcu_read_unlock();
+ =A0 =A0 return NULL;
+}
+EXPORT_SYMBOL(nfs4_find_deviceid);
+
+/*
+ * Add or kref_get a deviceid.
+ * GETDEVICEINFOs for same deviceid can race. If deviceid is found,=

discard new

Quoted 3 lines

fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops = {

Quoted 3 lines
fs/nfs/pnfs.c +3 -0
+ */
+struct nfs4_deviceid *
+nfs4_add_deviceid(struct nfs4_deviceid_cache *c, struct nfs4_device=

id *new)

Quoted 8 lines

fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D { · line 2398

Quoted 8 lines
fs/nfs/pnfs.c +8 -0
+{
+ =A0 =A0 struct nfs4_deviceid *d;
+ =A0 =A0 struct hlist_node *n;
+ =A0 =A0 long hash =3D nfs4_deviceid_hash(&new->de_id);
+
+ =A0 =A0 dprintk("--> %s hash %ld\n", __func__, hash);
+ =A0 =A0 spin_lock(&c->dc_lock);
+ =A0 =A0 hlist_for_each_entry_rcu(d, n, &c->dc_deviceids[hash], de_=

node) {

Quoted 1 line

fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D { · line 2399

Quoted 1 line
fs/nfs/pnfs.c +1 -0
+ =A0 =A0 =A0 =A0 =A0 =A0 if (!memcmp(&d->de_id, &new->de_id, NFS4_P=

NFS_DEVICEID4_SIZE)) {

Quoted 2 lines

fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D { · line 2401

Quoted 2 lines
fs/nfs/pnfs.c +2 -0
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 spin_unlock(&c->dc_lock);
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 dprintk("%s [discard]\n", =

__func__);

Quoted 1 line

fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D { · line 2402

Quoted 1 line
fs/nfs/pnfs.c +1 -0
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 c->dc_free_callback(&new->=

de_kref);

Quoted 11 lines
Quoted 11 lines
fs/nfs/pnfs.c +11 -0
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 return d;
+ =A0 =A0 =A0 =A0 =A0 =A0 }
+ =A0 =A0 }
+ =A0 =A0 hlist_add_head_rcu(&new->de_node, &c->dc_deviceids[hash]);
+ =A0 =A0 spin_unlock(&c->dc_lock);
+ =A0 =A0 dprintk("%s [new]\n", __func__);
+ =A0 =A0 return new;
+}
+EXPORT_SYMBOL(nfs4_add_deviceid);
+
+static int
Quoted 8 lines

fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D { · line 2421

Quoted 8 lines
fs/nfs/pnfs.c +8 -0
+nfs4_remove_deviceid(struct nfs4_deviceid_cache *c, long hash)
+{
+ =A0 =A0 struct nfs4_deviceid *d;
+ =A0 =A0 struct hlist_node *n;
+
+ =A0 =A0 dprintk("--> %s hash %ld\n", __func__, hash);
+ =A0 =A0 spin_lock(&c->dc_lock);
+ =A0 =A0 hlist_for_each_entry_rcu(d, n, &c->dc_deviceids[hash], de_=

node) {

Quoted 5 lines

fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D { · line 2426

Quoted 5 lines
fs/nfs/pnfs.c +5 -0
+ =A0 =A0 =A0 =A0 =A0 =A0 hlist_del_rcu(&d->de_node);
+ =A0 =A0 =A0 =A0 =A0 =A0 spin_unlock(&c->dc_lock);
+ =A0 =A0 =A0 =A0 =A0 =A0 synchronize_rcu();
+ =A0 =A0 =A0 =A0 =A0 =A0 dprintk("%s [%d]\n", __func__,
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 atomic_read(&d->de_kref.re=

fcount));

Quoted 1 line

fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D { · line 2427

Quoted 1 line
fs/nfs/pnfs.c +1 -0
+ =A0 =A0 =A0 =A0 =A0 =A0 kref_put(&d->de_kref, c->dc_free_callback)=

;

Quoted 11 lines

fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D { · line 2438

Quoted 11 lines
fs/nfs/pnfs.c +11 -0
+ =A0 =A0 =A0 =A0 =A0 =A0 return 1;
+ =A0 =A0 }
+ =A0 =A0 spin_unlock(&c->dc_lock);
+ =A0 =A0 return 0;
+}
+
+static void
+nfs4_free_deviceid_cache(struct kref *kref)
+{
+ =A0 =A0 struct nfs4_deviceid_cache *cache =3D
+ =A0 =A0 =A0 =A0 =A0 =A0 container_of(kref, struct nfs4_deviceid_ca=

che, dc_kref);

Quoted 7 lines

fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D { · line 2445

Quoted 7 lines
fs/nfs/pnfs.c +7 -0
+ =A0 =A0 int more;
+ =A0 =A0 long i;
+
+ =A0 =A0 for (i =3D 0; i < NFS4_DEVICE_ID_HASH_SIZE; i++) {
+ =A0 =A0 =A0 =A0 =A0 =A0 more =3D 1;
+ =A0 =A0 =A0 =A0 =A0 =A0 while (more)
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 more =3D nfs4_remove_devic=

eid(cache, i);

Quoted 7 lines

Andy, this can be simplified to

=A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0while (nfs4_remove_deviceid(cache, i)) =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0;

If ok with you, I'll make this change upon merging.

Yes - looks fine, thanks.

-->Andy

Quoted 14 lines

Benny

fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops = {

Quoted 11 lines
fs/nfs/pnfs.c +11 -0
+ =A0 =A0 }
+ =A0 =A0 kfree(cache);
+}
+
+void
+nfs4_put_deviceid_cache(struct nfs_client *clp)
+{
+ =A0 =A0 struct nfs4_deviceid_cache *tmp =3D clp->cl_devid_cache;
+ =A0 =A0 int refcount;
+
+ =A0 =A0 dprintk("--> %s cl_devid_cache %p\n", __func__, clp->cl_de=

vid_cache);

Quoted 2 lines

fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D { · line 2458

Quoted 2 lines
fs/nfs/pnfs.c +2 -0
+ =A0 =A0 spin_lock(&clp->cl_lock);
+ =A0 =A0 refcount =3D atomic_read(&clp->cl_devid_cache->dc_kref.ref=

count);

Quoted 9 lines
Quoted 9 lines
fs/nfs/pnfs.c +7 -0
+ =A0 =A0 if (refcount =3D=3D 1)
+ =A0 =A0 =A0 =A0 =A0 =A0 clp->cl_devid_cache =3D NULL;
+ =A0 =A0 spin_unlock(&clp->cl_lock);
+ =A0 =A0 dprintk("%s [%d]\n", __func__, refcount);
+ =A0 =A0 kref_put(&tmp->dc_kref, nfs4_free_deviceid_cache);
+}
+EXPORT_SYMBOL(nfs4_put_deviceid_cache);
include/linux/nfs4_pnfs.h +0 -0
index 3caac60..3b7aeb7 100644
Quoted 8 lines
Quoted 8 lines
include/linux/nfs4_pnfs.h +1 -0
@@ -106,6 +106,7 @@ struct pnfs_layout_segment {
 =A0 =A0 =A0 struct kref kref;
 =A0 =A0 =A0 bool valid;
 =A0 =A0 =A0 struct pnfs_layout_type *layout;
+ =A0 =A0 struct nfs4_deviceid *deviceid;

=A0 =A0 =A0 u8 ld_data[]; =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 /* lay=

out driver private data */

Quoted 4 lines
Quoted 4 lines

=A0};

include/linux/nfs4_pnfs.h +0 -0
@@ -275,6 +276,55 @@ struct pnfs_devicelist {

=A0 =A0 =A0 struct pnfs_deviceid =A0 =A0dev_id[NFS4_PNFS_GETDEVLIST_=

MAXNUM];

Quoted 4 lines
Quoted 4 lines

=A0};

+/*
+ * Device ID RCU cache. A device ID is unique per client ID and lay=

out type.

Quoted 3 lines

include/linux/nfs4_pnfs.h

Quoted 3 lines

+ */

include/linux/nfs4_pnfs.h +2 -0
+#define NFS4_DEVICE_ID_HASH_BITS =A0 =A0 5
+#define NFS4_DEVICE_ID_HASH_SIZE =A0 =A0 (1 << NFS4_DEVICE_ID_HASH_=

BITS)

Quoted 1 line
Quoted 1 line

+#define NFS4_DEVICE_ID_HASH_MASK =A0 =A0 (NFS4_DEVICE_ID_HASH_SIZE =

- 1)

Quoted 11 lines
Quoted 11 lines
include/linux/nfs4_pnfs.h +11 -0
+
+static inline u32
+nfs4_deviceid_hash(struct pnfs_deviceid *id)
+{
+ =A0 =A0 unsigned char *cptr =3D (unsigned char *)id->data;
+ =A0 =A0 unsigned int nbytes =3D NFS4_PNFS_DEVICEID4_SIZE;
+ =A0 =A0 u32 x =3D 0;
+
+ =A0 =A0 while (nbytes--) {
+ =A0 =A0 =A0 =A0 =A0 =A0 x *=3D 37;
+ =A0 =A0 =A0 =A0 =A0 =A0 x +=3D *cptr++;
Quoted 8 lines

include/linux/nfs4_pnfs.h struct pnfs_devicelist {

Quoted 8 lines

+ =A0 =A0 }
+ =A0 =A0 return x & NFS4_DEVICE_ID_HASH_MASK;

include/linux/nfs4_pnfs.h +6 -0
+}
+
+struct nfs4_deviceid_cache {
+ =A0 =A0 spinlock_t =A0 =A0 =A0 =A0 =A0 =A0 =A0dc_lock;
+ =A0 =A0 struct kref =A0 =A0 =A0 =A0 =A0 =A0 dc_kref;
+ =A0 =A0 void =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0(*dc_free_call=

back)(struct kref *);

Quoted 1 line
Quoted 1 line

+ =A0 =A0 struct hlist_head =A0 =A0 =A0 dc_deviceids[NFS4_DEVICE_ID_=

HASH_SIZE];

Quoted 11 lines

include/linux/nfs4_pnfs.h struct pnfs_devicelist {

Quoted 11 lines
include/linux/nfs4_pnfs.h +11 -0
+};
+
+/* Device ID cache node */
+struct nfs4_deviceid {
+ =A0 =A0 struct hlist_node =A0 =A0 =A0 de_node;
+ =A0 =A0 struct pnfs_deviceid =A0 =A0de_id;
+ =A0 =A0 struct kref =A0 =A0 =A0 =A0 =A0 =A0 de_kref;
+};
+
+extern int nfs4_alloc_init_deviceid_cache(struct nfs_client *,
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 void (*fre=

e_callback)(struct kref *));

Quoted 3 lines

include/linux/nfs4_pnfs.h struct pnfs_devicelist {

Quoted 3 lines
include/linux/nfs4_pnfs.h +3 -0
+extern void nfs4_put_deviceid_cache(struct nfs_client *);
+extern void nfs4_init_deviceid_node(struct nfs4_deviceid *);
+extern struct nfs4_deviceid *nfs4_find_deviceid(struct nfs4_devicei=

d_cache *,

Quoted 1 line
Quoted 1 line

+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 struct pnf=

s_deviceid *);

Quoted 1 line
Quoted 1 line

+extern struct nfs4_deviceid *nfs4_add_deviceid(struct nfs4_deviceid=

_cache *,

Quoted 1 line
Quoted 1 line

+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 struct nfs=

4_deviceid *);

Quoted 2 lines
Quoted 2 lines

+extern void nfs4_set_layout_deviceid(struct pnfs_layout_segment *,
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 struct nfs=

4_deviceid *);

Quoted 1 line
Quoted 1 line

+extern void nfs4_unset_layout_deviceid(struct pnfs_layout_segment *=

,

Quoted 1 line
Quoted 1 line

+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 struct nfs=

4_deviceid *,

Quoted 1 line
Quoted 1 line

+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 void (*fre=

e_callback)(struct kref *));

Quoted 10 lines
Quoted 10 lines

+
=A0/* pNFS client callback functions.
=A0 * These operations allow the layout driver to access pNFS client =A0 * specific information or call pNFS client->server operations.

include/linux/nfs_fs_sb.h +0 -0
index 8522461..ef2e18e 100644
@@ -87,6 +87,7 @@ struct nfs_client {

=A0 =A0 =A0 u32 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 cl_exchange_=

flags;

Quoted 1 line
Quoted 1 line

=A0 =A0 =A0 struct nfs4_session =A0 =A0 *cl_session; =A0 =A0/* sharr=

ed session */

Quoted 1 line
Quoted 1 line

=A0 =A0 =A0 struct list_head =A0 =A0 =A0 =A0cl_lo_inodes; =A0 /* Ino=

des having layouts */

Quoted 1 line
Quoted 1 line

+ =A0 =A0 struct nfs4_deviceid_cache *cl_devid_cache; /* pNFS device=

id cache */

Quoted 6 lines
Quoted 3 lines

=A0#endif /* CONFIG_NFS_V4_1 */

=A0#ifdef CONFIG_NFS_FSCACHE

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" =

in

Quoted 3 lines

the body of a message to [email protected]
More majordomo info at =A0http://vger.kernel.org/majordomo-info.html