pNFS generic device ID cache
Subsystems (2)
cover pNFS generic device ID cache
This patch set implements a shared RCU device ID cache servicing multiple mounts of a single layout type per meta data server (struct nfs_client).
0001-pnfs_submit-generic-device-ID-cache.patch 0002-pnfs_submit-fix-multiple-mount-set_pnfs_layoutdriver.patch 0003-pnfs-submit-file-layout-driver-generic-device-ID-cac.patch
These patches apply to the 2.6.34-rc3 pnfs branch below the following patch which is the head of the revert patches - otherwise known as the first patch not included in the pnfs file layout driver submission.
9a75b7356ca22b10903b507383646201cdcc9020
pnfs: filelayout: CB_NOTIFY_DEVICE support
This reverts commit 933b55c15ed2b7fd89d8e0342bd5b2825726201a "SQUASHME pnfs_submit: remove filelayout CB_NOTIFY_DEVICE support"
Testing:
CONFIG_NFS_V4_1 set:
NFSv4.1/pNFS mounts:
Connectathon tests pass against GFS2/pNFS with a single AUTH_SYS mount, a double AUTH_SYS mount, and an AUTH_SYS and AUTH_GSS/KRB5 mount (which creates two superblocks under a struct nfs_client and both share the device id cache).
NFSv4.0 mount;
Connectathon tests pass
Did not test with multiple device ID's. I will create a mulitple device ID test with the pynfs file layout server.
CONFIG_NFS_V4_1 not set:
NFSv4.0 mount: Connectathon tests pass.
-->Andy
PATCH 1/3 SQUASHME pnfs_submit: generic device ID cache
From: Andy Adamson <[email protected]>
A shared RCU device ID cache servicing multiple mounts of a single layout type per meta data server (struct nfs_client).
Device IDs of type deviceid4 are required by all layout types, long lived and read at each I/O. They are added to the deviceid cache at first reference by a layout via GETDEVICEINFO and (currently) are only removed at umount.
Reference count the device ID cache for each mounted file system in the initialize_mountpoint layoutdriver_io_operation.
Dereference the device id cache on file system in the uninitialize_mountpoint layoutdriver_io_operation called at umount
--- fs/nfs/pnfs.c | 119 +++++++++++++++++++++++++++++++++++++++++++++ include/linux/nfs4_pnfs.h | 27 ++++++++++ include/linux/nfs_fs_sb.h | 1 + 3 files changed, 147 insertions(+), 0 deletions(-)
fs/nfs/pnfs.c +119 -0
@@ -45,6 +45,7 @@#include <linux/nfs4.h> #include <linux/pnfs_xdr.h> #include <linux/nfs4_pnfs.h> +#include <linux/rculist.h> #include "internal.h" #include "nfs4_fs.h"
@@ -2296,3 +2297,121 @@ struct pnfs_client_operations pnfs_ops = {EXPORT_SYMBOL (pnfs_unregister_layoutdriver);EXPORT_SYMBOL (pnfs_register_layoutdriver); + + +/* Device ID cache. Supports one layout type per struct nfs_client */ +int +nfs4_alloc_init_deviceid_cache (struct nfs_client *clp, +void (*free_callback)(struct rcu_head *)) +{ +struct nfs4_deviceid_cache *c; + + c =kzalloc (sizeof (struct nfs4_deviceid_cache ),GFP_KERNEL ); +if (!c) +return -ENOMEM ; +spin_lock (&clp->cl_lock); +if (clp->cl_devid_cache !=NULL ) { +kref_get (&clp->cl_devid_cache->dc_kref); +spin_unlock (&clp->cl_lock); +dprintk ("%s [kref [%d]]\n" , __func__, +atomic_read (&clp->cl_devid_cache->dc_kref.refcount)); +kfree (c); + }else { +spin_lock_init (&c->dc_lock); +INIT_LIST_HEAD (&c->dc_deviceids); +kref_init (&c->dc_kref); + c->dc_free_callback = free_callback; + c->dc_clp = clp; + clp->cl_devid_cache = c; +spin_unlock (&clp->cl_lock); +dprintk ("%s [new]\n" , __func__); + } +return 0 ; +} +EXPORT_SYMBOL (nfs4_alloc_init_deviceid_cache); + +void +nfs4_init_deviceid_node (struct nfs4_deviceid *d) +{ +INIT_LIST_HEAD (&d->de_node); +INIT_RCU_HEAD (&d->de_rcu); +} +EXPORT_SYMBOL (nfs4_init_deviceid_node); + +struct nfs4_deviceid * +nfs4_find_deviceid (struct nfs4_deviceid_cache *c,struct pnfs_deviceid *id) +{ +struct nfs4_deviceid *d; + +rcu_read_lock (); +list_for_each_entry_rcu (d, &c->dc_deviceids, de_node) { +if (!memcmp (&d->de_id, id,NFS4_PNFS_DEVICEID4_SIZE )) { +rcu_read_unlock (); +return d; + } + } +rcu_read_unlock (); +return NULL ; +} +EXPORT_SYMBOL (nfs4_find_deviceid); + +/* +* Add or kref_get a deviceid. +* GETDEVICEINFOs for same deviceid can race. If deviceid is found, discard new +*/ +void +nfs4_add_deviceid (struct nfs4_deviceid_cache *c,struct nfs4_deviceid *new) +{ +struct nfs4_deviceid *d; + +spin_lock (&c->dc_lock); +list_for_each_entry_rcu (d, &c->dc_deviceids, de_node) { +if (!memcmp (&d->de_id, &new->de_id,NFS4_PNFS_DEVICEID4_SIZE )) { +spin_unlock (&c->dc_lock); +dprintk ("%s [discard]\n" , __func__); + c->dc_free_callback (&new->de_rcu); + } + } +list_add_rcu (&new->de_node, &c->dc_deviceids); +spin_unlock (&c->dc_lock); +dprintk ("%s [new]\n" , __func__); +} +EXPORT_SYMBOL (nfs4_add_deviceid); + +static int +nfs4_remove_deviceid (struct nfs4_deviceid_cache *c) +{ +struct nfs4_deviceid *d; + +spin_lock (&c->dc_lock); +list_for_each_entry_rcu (d, &c->dc_deviceids, de_node) { +list_del_rcu (&d->de_node); +spin_unlock (&c->dc_lock); +synchronize_rcu (); + c->dc_free_callback (&d->de_rcu); +return 1 ; + } +spin_unlock (&c->dc_lock); +return 0 ; +} + +static void +nfs4_free_deviceid_cache (struct kref *kref) +{ +struct nfs4_deviceid_cache *cache = + container_of(kref, struct nfs4_deviceid_cache, dc_kref); +int more =1 ; + +while (more) + more =nfs4_remove_deviceid (cache); + cache->dc_clp->cl_devid_cache =NULL ; +kfree (cache); +} + +void +nfs4_put_deviceid_cache (struct nfs4_deviceid_cache *c) +{ +dprintk ("%s [%d]\n" , __func__,atomic_read (&c->dc_kref.refcount)); +kref_put (&c->dc_kref, nfs4_free_deviceid_cache); +} +EXPORT_SYMBOL (nfs4_put_deviceid_cache);
include/linux/nfs4_pnfs.h +27 -0
@@ -281,6 +281,33 @@ struct pnfs_devicelist {struct pnfs_deviceid dev_id[NFS4_PNFS_GETDEVLIST_MAXNUM ]; }; +/* +* Device ID RCU cache. A device ID is unique per client ID and layout type. +*/ +struct nfs4_deviceid_cache { +spinlock_t dc_lock; +struct list_head dc_deviceids; +struct kref dc_kref; +void (*dc_free_callback)(struct rcu_head *); +struct nfs_client *dc_clp; +}; + +/* Device ID cache node */ +struct nfs4_deviceid { +struct list_head de_node; +struct rcu_head de_rcu; +struct pnfs_deviceid de_id; +}; + +extern int nfs4_alloc_init_deviceid_cache (struct nfs_client *, +void (*free_callback)(struct rcu_head *)); +extern void nfs4_put_deviceid_cache (struct nfs4_deviceid_cache *); +extern void nfs4_init_deviceid_node (struct nfs4_deviceid *); +extern struct nfs4_deviceid *nfs4_find_deviceid (struct nfs4_deviceid_cache *, +struct pnfs_deviceid *); +extern void nfs4_add_deviceid (struct nfs4_deviceid_cache *, +struct nfs4_deviceid *); +/* pNFS client callback functions. * These operations allow the layout driver to access pNFS client * specific information or call pNFS client->server operations.
include/linux/nfs_fs_sb.h +1 -0
@@ -87,6 +87,7 @@ struct nfs_client {u32 cl_exchange_flags;struct nfs4_session *cl_session;/* sharred session */ struct list_head cl_lo_inodes;/* Inodes having layouts */ +struct nfs4_deviceid_cache *cl_devid_cache;/* pNFS deviceid cache */ #endif /* CONFIG_NFS_V4_1 */ #ifdef CONFIG_NFS_FSCACHE
Signature · 1 line
1.6.6
PATCH 2/3 SQUASHME pnfs_submit: fix multiple mount set_pnfs_layoutdriver
From: Andy Adamson <[email protected]>
The same struct nfs_server can enter set_pnfs_layoutdriver for mounts that share a super block. Don't initialize a pnfs mountpoint more than once.
Don't set the pnfs_curr_ld until the pnfs mountpoint initialization succeeds
--- fs/nfs/pnfs.c | 15 ++++++++++----- 1 files changed, 10 insertions(+), 5 deletions(-)
fs/nfs/pnfs.c +10 -5
@@ -215,20 +215,25 @@ set_pnfs_layoutdriver(struct super_block *sb, struct nfs_fh *fh, u32 id)struct pnfs_mount_type *mt;struct nfs_server *server =NFS_SB (sb); +if (server->pnfs_curr_ld) +return ; +if (id >0 &&find_pnfs (id, &mod)) { -dprintk ("%s: Setting pNFS module\n" , __func__); - server->pnfs_curr_ld = mod->pnfs_ld_type; - mt = server->pnfs_curr_ld->ld_io_ops->initialize_mountpoint ( + mt = mod->pnfs_ld_type->ld_io_ops->initialize_mountpoint ( sb, fh);if (!mt) {printk (KERN_ERR "%s: Error initializing mount point " "for layout driver %u. " , __func__, id); goto out_err; } -/* Layout driver succeeded in initializing mountpoint */ +/* +* Layout driver succeeded in initializing mountpoint +* and has taken a reference on the nfs_client cl_devid_cache +*/ + server->pnfs_curr_ld = mod->pnfs_ld_type; server->pnfs_mountid = mt; -/* Set the rpc_ops */ server->nfs_client->rpc_ops = &pnfs_v4_clientops; +dprintk ("%s: pNFS module for %u set\n" , __func__, id);return ; }
Signature · 1 line
1.6.6
PATCH 3/3 SQUASHME pnfs-submit: file layout driver generic device ID cache
From: Andy Adamson <[email protected]>
Replace the per superblock deviceid cache with the generic deviceid cache.
Embed struct nfs4_deviceid into struct nfs4_file_layout_dsaddr, the file layout specific deviceid structure. Provide a free_deviceid_callback.
--- fs/nfs/client.c | 1 + fs/nfs/nfs4filelayout.c | 31 +++----- fs/nfs/nfs4filelayout.h | 12 +-- fs/nfs/nfs4filelayoutdev.c | 190 ++++++++------------------------------------ 4 files changed, 51 insertions(+), 183 deletions(-)
fs/nfs/client.c +1 -0
@@ -38,6 +38,7 @@#include <net/ipv6.h> #include <linux/nfs_xdr.h> #include <linux/sunrpc/bc_xprt.h> +#include <linux/nfs4_pnfs.h> #include <asm/system.h>
fs/nfs/nfs4filelayout.c +12 -19
@@ -76,17 +76,11 @@ filelayout_initialize_mountpoint(struct super_block *sb, struct nfs_fh *fh) {struct filelayout_mount_type *fl_mt;struct pnfs_mount_type *mt; -int status; fl_mt =kmalloc (sizeof (struct filelayout_mount_type ),GFP_KERNEL );if (!fl_mt) goto error_ret; -/* Initialize nfs4 file layout specific device list structure */ - fl_mt->hlist =kmalloc (sizeof (struct nfs4_pnfs_dev_hlist ),GFP_KERNEL ); -if (!fl_mt->hlist) - goto cleanup_fl_mt; - mt =kmalloc (sizeof (struct pnfs_mount_type ),GFP_KERNEL );if (!mt) goto cleanup_fl_mt;
@@ -94,11 +88,11 @@ filelayout_initialize_mountpoint(struct super_block *sb, struct nfs_fh *fh) fl_mt->fl_sb = sb; mt->mountid = (void *)fl_mt; - status =nfs4_pnfs_devlist_init (fl_mt->hlist); -if (status) +if (nfs4_alloc_init_deviceid_cache (NFS_SB (sb)->nfs_client, + nfs4_fl_free_deviceid_callback)) goto cleanup_mt; -dprintk ("%s: device list has been initialized successfully\n" , +dprintk ("%s: deviceid cache has been initialized successfully\n" , __func__);return mt;
@@ -106,11 +100,10 @@ cleanup_mt: ;kfree (mt); cleanup_fl_mt: ; -kfree (fl_mt->hlist);kfree (fl_mt); error_ret: ; -printk (KERN_WARNING "%s: device list could not be initialized\n" , +printk (KERN_WARNING "%s: deviceid cache could not be initialized\n" , __func__);return NULL ;
@@ -123,13 +116,14 @@ filelayout_uninitialize_mountpoint(struct pnfs_mount_type *mountid) {struct filelayout_mount_type *fl_mt =NULL ; +dprintk ("--> %s\n" , __func__);if (mountid) { - fl_mt = (struct filelayout_mount_type *)mountid->mountid; +struct nfs4_deviceid_cache *cache; -if (fl_mt !=NULL ) { -nfs4_pnfs_devlist_destroy (fl_mt->hlist); -kfree (fl_mt); - } + fl_mt = (struct filelayout_mount_type *)mountid->mountid; + cache =NFS_SB (fl_mt->fl_sb)->nfs_client->cl_devid_cache; +nfs4_put_deviceid_cache (cache); +kfree (fl_mt);kfree (mountid); }return 0 ;
@@ -381,8 +375,7 @@ filelayout_check_layout(struct pnfs_layout_type *lo,struct nfs_server *nfss =NFS_SERVER (PNFS_INODE (lo));dprintk ("--> %s\n" , __func__); - dsaddr =nfs4_pnfs_device_item_find (FILE_MT (PNFS_INODE (lo))->hlist, - &fl->dev_id); + dsaddr =nfs4_pnfs_device_item_find (nfss->nfs_client, &fl->dev_id);if (dsaddr ==NULL ) { dsaddr =get_device_info (PNFS_INODE (lo), &fl->dev_id);if (dsaddr ==NULL ) {
@@ -618,7 +611,7 @@ filelayout_commit(struct pnfs_layout_type *layoutid, int sync, stripesz =filelayout_get_stripesize (layoutid);dprintk ("%s stripesize %Zd\n" , __func__, stripesz); - dsaddr =nfs4_pnfs_device_item_find (FILE_MT (data->inode)->hlist, + dsaddr =nfs4_pnfs_device_item_find (NFS_SERVER (data->inode)->nfs_client, &nfslay->dev_id);if (dsaddr ==NULL ) { data->pdata.pnfs_error = -EIO ;
fs/nfs/nfs4filelayout.h +4 -8
@@ -43,8 +43,7 @@ struct nfs4_pnfs_ds { };struct nfs4_file_layout_dsaddr { -struct hlist_node hash_node;/* nfs4_pnfs_dev_hlist dev_list */ -struct pnfs_deviceid dev_id; +struct nfs4_deviceid deviceid;u32 stripe_count;u8 *stripe_indices;u32 ds_num;
@@ -86,15 +85,13 @@ struct nfs4_filelayout {struct filelayout_mount_type {struct super_block *fl_sb; -struct nfs4_pnfs_dev_hlist *hlist; };extern struct pnfs_client_operations *pnfs_callback_ops; +extern void nfs4_fl_free_deviceid_callback (struct rcu_head *);extern void print_ds (struct nfs4_pnfs_ds *ds);char *deviceid_fmt (const struct pnfs_deviceid *dev_id); -int nfs4_pnfs_devlist_init (struct nfs4_pnfs_dev_hlist *hlist); -void nfs4_pnfs_devlist_destroy (struct nfs4_pnfs_dev_hlist *hlist); int nfs4_pnfs_dserver_get(struct pnfs_layout_segment *lseg, loff_t offset, size_t count,
@@ -102,9 +99,8 @@ int nfs4_pnfs_dserver_get(struct pnfs_layout_segment *lseg,u32 filelayout_dserver_get_index (loff_t offset,struct nfs4_file_layout_dsaddr *di,struct nfs4_filelayout_segment *layout); -struct nfs4_file_layout_dsaddr * -nfs4_pnfs_device_item_find (struct nfs4_pnfs_dev_hlist *hlist, -struct pnfs_deviceid *dev_id); +extern struct nfs4_file_layout_dsaddr * +nfs4_pnfs_device_item_find (struct nfs_client *,struct pnfs_deviceid *dev_id);struct nfs4_file_layout_dsaddr *get_device_info (struct inode *inode,struct pnfs_deviceid *dev_id);
fs/nfs/nfs4filelayoutdev.c +34 -156
285 lines not on this page · raw message
Signature · 1 line
1.6.6
Re: [PATCH 1/3] SQUASHME pnfs_submit: generic device ID cache
In reply to [email protected]Quoted 3 lines · On Fri, Apr 16, 2010 at 11:52 AM, <[email protected]> wrote:
From: Andy Adamson <[email protected]>
A shared RCU device ID cache servicing multiple mounts of a single la=
yout type
Quoted 3 lines
per meta data server (struct
nfs_client).Device IDs of type deviceid4 are required by all layout types, long l=
ived and
Quoted 1 line
read at each I/O. =A0They are added to the deviceid cache at first re=
ference by
Quoted 1 line
a layout via GETDEVICEINFO and (currently) are only removed at umount=
=2E
Quoted 5 lines
Reference count the device ID cache for each mounted file system in the
initialize_mountpointlayoutdriver_io_operation.Dereference the device id cache on file system in the uninitialize_mo=
untpoint
Quoted 5 lines
layoutdriver_io_operationcalled at umountSigned-off-by Andy Adamson <[email protected]>---
=A0fs/nfs/pnfs.c =A0 =A0 =A0 =A0 =A0 =A0 | =A0119 +++++++++++++++++++=
++++++++++++++++++++++++++
Quoted 17 lines
=A0include/linux/nfs4_pnfs.h | =A0 27 ++++++++++ =A0include/linux/nfs_fs_sb.h | =A0 =A01 + =A03 files changed, 147 insertions(+), 0 deletions(-)
fs/nfs/pnfs.c +0
-0@@ -45,6 +45,7 @@=A0#include <
linux/nfs4.h>
=A0#include <linux/pnfs_xdr.h>
=A0#include <linux/nfs4_pnfs.h>
+#include <linux/rculist.h>=A0#include "
internal.h"
=A0#include "nfs4_fs.h"
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D { · line 2302
Quoted 6 lines
fs/nfs/pnfs.c +3
-0@@ -2296,3 +2297,121 @@ struct pnfs_client_operations pnfs_ops =3D { =A0EXPORT_SYMBOL (pnfs_unregister_layoutdriver); =A0EXPORT_SYMBOL(pnfs_register_layoutdriver); + + +/* Device ID cache. Supports one layout type per struct nfs_client *=
/
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops = {
Quoted 3 lines
fs/nfs/pnfs.c +3
-0+int +nfs4_alloc_init_deviceid_cache(struct nfs_client *clp, + =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0void (*free_callback=
)(struct rcu_head *))
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops = {
Quoted 4 lines
fs/nfs/pnfs.c +4
-0+{ + =A0 =A0 =A0 struct nfs4_deviceid_cache *c; + + =A0 =A0 =A0 c =3D kzalloc(sizeof(struct nfs4_deviceid_cache), GFP_K=
ERNEL);
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D { · line 2314
Quoted 5 lines
fs/nfs/pnfs.c +5
-0+ =A0 =A0 =A0 if (!c) + =A0 =A0 =A0 =A0 =A0 =A0 =A0 return -ENOMEM; + =A0 =A0 =A0 spin_lock(&clp->cl_lock); + =A0 =A0 =A0 if (clp->cl_devid_cache !=3D NULL) { + =A0 =A0 =A0 =A0 =A0 =A0 =A0 kref_get(&clp->cl_devid_cache->dc_kref)=
;
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D { · line 2317
Quoted 3 lines
fs/nfs/pnfs.c +3
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 spin_unlock(&clp->cl_lock); + =A0 =A0 =A0 =A0 =A0 =A0 =A0 dprintk("%s [kref [%d]]\n", __func__, + =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 atomic_read(&clp->cl_de=
vid_cache->dc_kref.refcount));
Quoted 15 lines
fs/nfs/pnfs.c +15
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 kfree(c); + =A0 =A0 =A0 } else { + =A0 =A0 =A0 =A0 =A0 =A0 =A0 spin_lock_init(&c->dc_lock); + =A0 =A0 =A0 =A0 =A0 =A0 =A0 INIT_LIST_HEAD(&c->dc_deviceids); + =A0 =A0 =A0 =A0 =A0 =A0 =A0 kref_init(&c->dc_kref); + =A0 =A0 =A0 =A0 =A0 =A0 =A0 c->dc_free_callback =3D free_callback; + =A0 =A0 =A0 =A0 =A0 =A0 =A0 c->dc_clp =3D clp; + =A0 =A0 =A0 =A0 =A0 =A0 =A0 clp->cl_devid_cache =3D c; + =A0 =A0 =A0 =A0 =A0 =A0 =A0 spin_unlock(&clp->cl_lock); + =A0 =A0 =A0 =A0 =A0 =A0 =A0 dprintk("%s [new]\n", __func__); + =A0 =A0 =A0 } + =A0 =A0 =A0 return 0; +} +EXPORT_SYMBOL (nfs4_alloc_init_deviceid_cache); +
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D { · line 2342
Quoted 10 lines
fs/nfs/pnfs.c +10
-0+void +nfs4_init_deviceid_node (struct nfs4_deviceid *d) +{ + =A0 =A0 =A0 INIT_LIST_HEAD(&d->de_node); + =A0 =A0 =A0 INIT_RCU_HEAD(&d->de_rcu); +} +EXPORT_SYMBOL (nfs4_init_deviceid_node); + +struct nfs4_deviceid * +nfs4_find_deviceid(struct nfs4_deviceid_cache *c, struct pnfs_device=
id *id)
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D { · line 2348
Quoted 6 lines
fs/nfs/pnfs.c +6
-0+{ + =A0 =A0 =A0 struct nfs4_deviceid *d; + + =A0 =A0 =A0 rcu_read_lock(); + =A0 =A0 =A0 list_for_each_entry_rcu(d, &c->dc_deviceids, de_node) { + =A0 =A0 =A0 =A0 =A0 =A0 =A0 if (!memcmp(&d->de_id, id, NFS4_PNFS_DE=
VICEID4_SIZE)) {
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D { · line 2360
Quoted 12 lines
fs/nfs/pnfs.c +12
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 rcu_read_unlock(); + =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 return d; + =A0 =A0 =A0 =A0 =A0 =A0 =A0 } + =A0 =A0 =A0 } + =A0 =A0 =A0 rcu_read_unlock(); + =A0 =A0 =A0 return NULL; +} +EXPORT_SYMBOL (nfs4_find_deviceid); + +/* +* Add or kref_get a deviceid. +* GETDEVICEINFOs for same deviceid can race. If deviceid is found, =
discard new
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops = {
Quoted 3 lines
fs/nfs/pnfs.c +3
-0+*/ +void +nfs4_add_deviceid(struct nfs4_deviceid_cache *c, struct nfs4_devicei=
d *new)
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D { · line 2369
Quoted 6 lines
fs/nfs/pnfs.c +6
-0+{ + =A0 =A0 =A0 struct nfs4_deviceid *d; + + =A0 =A0 =A0 spin_lock(&c->dc_lock); + =A0 =A0 =A0 list_for_each_entry_rcu(d, &c->dc_deviceids, de_node) { + =A0 =A0 =A0 =A0 =A0 =A0 =A0 if (!memcmp(&d->de_id, &new->de_id, NFS=
4_PNFS_DEVICEID4_SIZE)) {
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D { · line 2370
Quoted 1 line
fs/nfs/pnfs.c +1
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 spin_unlock(&c->dc_lock=
);
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D { · line 2371
Quoted 1 line
fs/nfs/pnfs.c +1
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 dprintk("%s [discard]\n=
", __func__);
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D { · line 2372
Quoted 1 line
fs/nfs/pnfs.c +1
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 c->dc_free_callback(&ne=
w->de_rcu);
Quoted 20 lines
fs/nfs/pnfs.c +20
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 } + =A0 =A0 =A0 } + =A0 =A0 =A0 list_add_rcu(&new->de_node, &c->dc_deviceids); + =A0 =A0 =A0 spin_unlock(&c->dc_lock); + =A0 =A0 =A0 dprintk("%s [new]\n", __func__); +} +EXPORT_SYMBOL (nfs4_add_deviceid); + +static int +nfs4_remove_deviceid (struct nfs4_deviceid_cache *c) +{ + =A0 =A0 =A0 struct nfs4_deviceid *d; + + =A0 =A0 =A0 spin_lock(&c->dc_lock); + =A0 =A0 =A0 list_for_each_entry_rcu(d, &c->dc_deviceids, de_node) { + =A0 =A0 =A0 =A0 =A0 =A0 =A0 list_del_rcu(&d->de_node); + =A0 =A0 =A0 =A0 =A0 =A0 =A0 spin_unlock(&c->dc_lock); + =A0 =A0 =A0 =A0 =A0 =A0 =A0 synchronize_rcu(); + =A0 =A0 =A0 =A0 =A0 =A0 =A0 c->dc_free_callback(&d->de_rcu); + =A0 =A0 =A0 =A0 =A0 =A0 =A0 return 1;
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D { · line 2402
Quoted 10 lines
fs/nfs/pnfs.c +10
-0+ =A0 =A0 =A0 } + =A0 =A0 =A0 spin_unlock(&c->dc_lock); + =A0 =A0 =A0 return 0; +} + +static void +nfs4_free_deviceid_cache(struct kref *kref) +{ + =A0 =A0 =A0 struct nfs4_deviceid_cache *cache =3D + =A0 =A0 =A0 =A0 =A0 =A0 =A0 container_of(kref, struct nfs4_deviceid=
_cache, dc_kref);
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D { · line 2407
Quoted 5 lines
fs/nfs/pnfs.c +5
-0+ =A0 =A0 =A0 int more =3D 1; + + =A0 =A0 =A0 while (more) + =A0 =A0 =A0 =A0 =A0 =A0 =A0 more =3D nfs4_remove_deviceid(cache); + =A0 =A0 =A0 cache->dc_clp->cl_devid_cache =3D NULL;
Need to take the cl_lock around this assignment
spin_lock(&cache->dc_clp->cl_lock);cache->dc_clp->cl_devid_cache =3D NULLspin_unlock(&cache->dc_clp->cl_lock);
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops = {
Quoted 7 lines
fs/nfs/pnfs.c +7
-0+ =A0 =A0 =A0 kfree(cache); +} + +void +nfs4_put_deviceid_cache(struct nfs4_deviceid_cache *c) +{ + =A0 =A0 =A0 dprintk("%s [%d]\n", __func__, atomic_read(&c->dc_kref.=
refcount));
Quoted 9 lines
fs/nfs/pnfs.c +3
-0+ =A0 =A0 =A0 kref_put(&c->dc_kref, nfs4_free_deviceid_cache); +} +EXPORT_SYMBOL (nfs4_put_deviceid_cache);include/linux/nfs4_pnfs.h +0
-0@@ -281,6 +281,33 @@ struct pnfs_devicelist {=A0 =A0 =A0 =A0struct
pnfs_deviceid=A0 =A0dev_id[NFS4_PNFS_GETDEVLIS=
T_MAXNUM];
Quoted 4 lines
=A0};
+/*
+ * Device ID RCU cache. A device ID is unique per client ID and layo=
ut type.
Quoted 6 lines
+ */
+structnfs4_deviceid_cache{
+ =A0 =A0 =A0spinlock_t=A0 =A0 =A0 =A0 =A0 =A0 =A0dc_lock;
+ =A0 =A0 =A0 structlist_head=A0 =A0 =A0 =A0dc_deviceids;
+ =A0 =A0 =A0 struct kref =A0 =A0 =A0 =A0 =A0 =A0dc_kref;
+ =A0 =A0 =A0 void =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0(*dc_free_c=
allback)(struct rcu_head *);
include/linux/nfs4_pnfs.h struct pnfs_devicelist {
Quoted 12 lines
+ =A0 =A0 =A0 struct
nfs_client=A0 =A0 =A0*dc_clp;include/linux/nfs4_pnfs.h +11
-0+}; + +/* Device ID cache node */ +struct nfs4_deviceid { + =A0 =A0 =A0 struct list_head =A0 =A0 =A0 =A0de_node; + =A0 =A0 =A0 struct rcu_head =A0 =A0 =A0 =A0 de_rcu; + =A0 =A0 =A0 struct pnfs_deviceid =A0 =A0de_id; +}; + +extern int nfs4_alloc_init_deviceid_cache(struct nfs_client *, + =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 void (*=
free_callback)(struct rcu_head *));
include/linux/nfs4_pnfs.h struct pnfs_devicelist {
Quoted 3 lines
include/linux/nfs4_pnfs.h +3
-0+extern void nfs4_put_deviceid_cache (struct nfs4_deviceid_cache *); +extern void nfs4_init_deviceid_node (struct nfs4_deviceid *); +extern struct nfs4_deviceid *nfs4_find_deviceid(struct nfs4_deviceid=
_cache *,
Quoted 1 line
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 struct =
pnfs_deviceid *);
Quoted 2 lines
+extern void nfs4_add_deviceid(struct
nfs4_deviceid_cache*,
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 struct =
nfs4_deviceid *);
Quoted 10 lines
+
=A0/* pNFS client callback functions.
=A0* These operations allow the layout driver to access pNFS client =A0* specific information or call pNFSclient->serveroperations.include/linux/nfs_fs_sb.h +0
-0@@ -87,6 +87,7 @@ struct nfs_client {=A0 =A0 =A0 =A0u32 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 cl_exchang=
e_flags;
Quoted 1 line
=A0 =A0 =A0 =A0struct
nfs4_session=A0 =A0*cl_session; =A0 =A0/* sha=
rred session */
Quoted 1 line
=A0 =A0 =A0 =A0struct
list_head=A0 =A0 =A0 =A0cl_lo_inodes; =A0 /* I=
nodes having layouts */
Quoted 1 line
+ =A0 =A0 =A0 struct
nfs4_deviceid_cache*cl_devid_cache; /* pNFS dev=
iceid cache */
Quoted 8 lines
=A0#endif /*
CONFIG_NFS_V4_1*/=A0#ifdef
CONFIG_NFS_FSCACHE
--
1.6.6--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" =
in
Quoted 3 lines
the body of a message to [email protected]
More majordomo info at =A0http://vger.kernel.org/majordomo-info.html
A message that is not in the archive (sent off-list or not yet received): u2n89c397151004160904m9e862360xcaf0e187640b0177-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org
Re: [pnfs] [PATCH 1/3] SQUASHME pnfs_submit: generic device ID cache
Quoted 85 lines · William A. (Andy) Adamson
Quoted 84 lines · On Fri, Apr 16, 2010 at 11:52 AM, <[email protected]> wrote:
From: Andy Adamson <[email protected]>
A shared RCU device ID cache servicing multiple mounts of a single layout type per meta data server (struct
nfs_client).Device IDs of type deviceid4 are required by all layout types, long lived and read at each I/O. They are added to the deviceid cache at first reference by a layout via GETDEVICEINFO and (currently) are only removed at umount.
Reference count the device ID cache for each mounted file system in the
initialize_mountpointlayoutdriver_io_operation.Dereference the device id cache on file system in the
uninitialize_mountpointlayoutdriver_io_operationcalled at umountSigned-off-by Andy Adamson <[email protected]>--- fs/nfs/pnfs.c | 119 +++++++++++++++++++++++++++++++++++++++++++++ include/linux/nfs4_pnfs.h | 27 ++++++++++ include/linux/nfs_fs_sb.h | 1 + 3 files changed, 147 insertions(+), 0 deletions(-)fs/nfs/pnfs.c +47
-0@@ -45,6 +45,7 @@#include <linux/nfs4.h> #include <linux/pnfs_xdr.h> #include <linux/nfs4_pnfs.h> +#include <linux/rculist.h> #include "internal.h" #include "nfs4_fs.h" @@ -2296,3 +2297,121 @@ struct pnfs_client_operations pnfs_ops = {EXPORT_SYMBOL (pnfs_unregister_layoutdriver);EXPORT_SYMBOL (pnfs_register_layoutdriver); + + +/* Device ID cache. Supports one layout type per struct nfs_client */ +int +nfs4_alloc_init_deviceid_cache (struct nfs_client *clp, +void (*free_callback)(struct rcu_head *)) +{ +struct nfs4_deviceid_cache *c; + + c =kzalloc (sizeof (struct nfs4_deviceid_cache ),GFP_KERNEL ); +if (!c) +return -ENOMEM ; +spin_lock (&clp->cl_lock); +if (clp->cl_devid_cache !=NULL ) { +kref_get (&clp->cl_devid_cache->dc_kref); +spin_unlock (&clp->cl_lock); +dprintk ("%s [kref [%d]]\n" , __func__, +atomic_read (&clp->cl_devid_cache->dc_kref.refcount)); +kfree (c); + }else { +spin_lock_init (&c->dc_lock); +INIT_LIST_HEAD (&c->dc_deviceids); +kref_init (&c->dc_kref); + c->dc_free_callback = free_callback; + c->dc_clp = clp; + clp->cl_devid_cache = c; +spin_unlock (&clp->cl_lock); +dprintk ("%s [new]\n" , __func__); + } +return 0 ; +} +EXPORT_SYMBOL (nfs4_alloc_init_deviceid_cache); + +void +nfs4_init_deviceid_node (struct nfs4_deviceid *d) +{ +INIT_LIST_HEAD (&d->de_node); +INIT_RCU_HEAD (&d->de_rcu); +} +EXPORT_SYMBOL (nfs4_init_deviceid_node); + +struct nfs4_deviceid * +nfs4_find_deviceid (struct nfs4_deviceid_cache *c,struct pnfs_deviceid *id) +{ +struct nfs4_deviceid *d; +
Quoted 8 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops = { · line 2353
Quoted 8 lines
fs/nfs/pnfs.c +8
-0+rcu_read_lock (); +list_for_each_entry_rcu (d, &c->dc_deviceids, de_node) { +if (!memcmp (&d->de_id, id,NFS4_PNFS_DEVICEID4_SIZE )) { +rcu_read_unlock (); +return d; + } + } +rcu_read_unlock ();
I hope this is worth the added complexity...
Out of curiosity, do you have a benchmark comparing the cost of rcu_read_lock (and unlock) vs. spin_lock(&c->dc_lock)?
Quoted 57 lines
Quoted 44 lines
fs/nfs/pnfs.c +44
-0+return NULL ; +} +EXPORT_SYMBOL (nfs4_find_deviceid); + +/* +* Add or kref_get a deviceid. +* GETDEVICEINFOs for same deviceid can race. If deviceid is found, discard new +*/ +void +nfs4_add_deviceid (struct nfs4_deviceid_cache *c,struct nfs4_deviceid *new) +{ +struct nfs4_deviceid *d; + +spin_lock (&c->dc_lock); +list_for_each_entry_rcu (d, &c->dc_deviceids, de_node) { +if (!memcmp (&d->de_id, &new->de_id,NFS4_PNFS_DEVICEID4_SIZE )) { +spin_unlock (&c->dc_lock); +dprintk ("%s [discard]\n" , __func__); + c->dc_free_callback (&new->de_rcu); + } + } +list_add_rcu (&new->de_node, &c->dc_deviceids); +spin_unlock (&c->dc_lock); +dprintk ("%s [new]\n" , __func__); +} +EXPORT_SYMBOL (nfs4_add_deviceid); + +static int +nfs4_remove_deviceid (struct nfs4_deviceid_cache *c) +{ +struct nfs4_deviceid *d; + +spin_lock (&c->dc_lock); +list_for_each_entry_rcu (d, &c->dc_deviceids, de_node) { +list_del_rcu (&d->de_node); +spin_unlock (&c->dc_lock); +synchronize_rcu (); + c->dc_free_callback (&d->de_rcu); +return 1 ; + } +spin_unlock (&c->dc_lock); +return 0 ; +} +fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops = { · line 2407
Quoted 10 lines
fs/nfs/pnfs.c +10
-0+static void +nfs4_free_deviceid_cache (struct kref *kref) +{ +struct nfs4_deviceid_cache *cache = + container_of(kref, struct nfs4_deviceid_cache, dc_kref); +int more =1 ; + +while (more) + more =nfs4_remove_deviceid (cache); + cache->dc_clp->cl_devid_cache =NULL ;Need to take the
cl_lockaround this assignment
Quoted 5 lines
spin_lock(&cache->dc_clp->cl_lock);cache->dc_clp->cl_devid_cache= NULLspin_unlock(&cache->dc_clp->cl_lock);
That must happen atomically before kref_put.
It's illegal to have cl_devid_cache be referenced by cache->dc_clp without a reference count backing it up.
Otherwise, if accessed concurrently to this piece of code someone might call kref_get while the refcount is zero.
Normally, you'd first clear the referencing pointer to prevent any new reference to it and only then kref_put it, e.g.:
spin_lock(&cache->dc_clp->cl_lock);
tmp = cache->dc_clp->cl_devid_cache; cache->dc_clp->cl_devid_cache = NULL spin_unlock(&cache->dc_clp->cl_lock); kref_put(&tmp->dc_kref, nfs4_free_deviceid_cache);
Benny
Quoted 72 lines
Quoted 63 lines
fs/nfs/pnfs.c +10
-0+kfree (cache); +} + +void +nfs4_put_deviceid_cache (struct nfs4_deviceid_cache *c) +{ +dprintk ("%s [%d]\n" , __func__,atomic_read (&c->dc_kref.refcount)); +kref_put (&c->dc_kref, nfs4_free_deviceid_cache); +} +EXPORT_SYMBOL (nfs4_put_deviceid_cache);include/linux/nfs4_pnfs.h +27
-0@@ -281,6 +281,33 @@ struct pnfs_devicelist {struct pnfs_deviceid dev_id[NFS4_PNFS_GETDEVLIST_MAXNUM ]; }; +/* +* Device ID RCU cache. A device ID is unique per client ID and layout type. +*/ +struct nfs4_deviceid_cache { +spinlock_t dc_lock; +struct list_head dc_deviceids; +struct kref dc_kref; +void (*dc_free_callback)(struct rcu_head *); +struct nfs_client *dc_clp; +}; + +/* Device ID cache node */ +struct nfs4_deviceid { +struct list_head de_node; +struct rcu_head de_rcu; +struct pnfs_deviceid de_id; +}; + +extern int nfs4_alloc_init_deviceid_cache (struct nfs_client *, +void (*free_callback)(struct rcu_head *)); +extern void nfs4_put_deviceid_cache (struct nfs4_deviceid_cache *); +extern void nfs4_init_deviceid_node (struct nfs4_deviceid *); +extern struct nfs4_deviceid *nfs4_find_deviceid (struct nfs4_deviceid_cache *, +struct pnfs_deviceid *); +extern void nfs4_add_deviceid (struct nfs4_deviceid_cache *, +struct nfs4_deviceid *); +/* pNFS client callback functions. * These operations allow the layout driver to access pNFS client * specific information or call pNFS client->server operations. include/linux/nfs_fs_sb.h +1
-1@@ -87,6 +87,7 @@ struct nfs_client {u32 cl_exchange_flags;struct nfs4_session *cl_session;/* sharred session */ struct list_head cl_lo_inodes;/* Inodes having layouts */ +struct nfs4_deviceid_cache *cl_devid_cache;/* pNFS deviceid cache */ #endif /* CONFIG_NFS_V4_1 */ #ifdef CONFIG_NFS_FSCACHE --1.6.6
Quoted 5 lines
--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html_______________________________________________
pNFS mailing list
[email protected]
http://linux-nfs.org/cgi-bin/mailman/listinfo/pnfs
Re: [pnfs] [PATCH 1/3] SQUASHME pnfs_submit: generic device ID cache
On Wed, Apr 21, 2010 at 1:59 AM, Benny Halevy <[email protected]> wro= te:
Quoted 1 line
On Apr. 16, 2010, 19:04 +0300, "William A. (Andy) Adamson" <androsada=
Quoted 4 lines · [email protected]> wrote:
Quoted 4 lines · On Fri, Apr 16, 2010 at 11:52 AM, =A0<[email protected]> wrote:
Quoted 3 lines
From: Andy Adamson <[email protected]>
A shared RCU device ID cache servicing multiple mounts of a single =
layout type
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
per meta data server (struct
nfs_client).Device IDs of type deviceid4 are required by all layout types, long=
lived and
Quoted 1 line
Quoted 1 line
Quoted 1 line
read at each I/O. =A0They are added to the deviceid cache at first =
reference by
Quoted 1 line
Quoted 1 line
Quoted 1 line
a layout via GETDEVICEINFO and (currently) are only removed at umou=
nt.
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Reference count the device ID cache for each mounted file system in the
initialize_mountpointlayoutdriver_io_operation.Dereference the device id cache on file system in the uninitialize_=
mountpoint
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
layoutdriver_io_operationcalled at umountSigned-off-by Andy Adamson <[email protected]>---
=A0fs/nfs/pnfs.c =A0 =A0 =A0 =A0 =A0 =A0 | =A0119 +++++++++++++++++=
++++++++++++++++++++++++++++
Quoted 14 lines
Quoted 14 lines
Quoted 14 lines
=A0include/linux/nfs4_pnfs.h | =A0 27 ++++++++++ =A0include/linux/nfs_fs_sb.h | =A0 =A01 + =A03 files changed, 147 insertions(+), 0 deletions(-)
fs/nfs/pnfs.c +0
-0@@ -45,6 +45,7 @@=A0#include <
linux/nfs4.h>
=A0#include <linux/pnfs_xdr.h>
=A0#include <linux/nfs4_pnfs.h>
+#include <linux/rculist.h>
Quoted 3 lines
Quoted 3 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D =
Quoted 3 lines
=A0#include "
internal.h"
=A0#include "nfs4_fs.h"fs/nfs/pnfs.c +0
-0@@ -2296,3 +2297,121 @@ struct pnfs_client_operations pnfs_ops =3D =
{
Quoted 6 lines
Quoted 6 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D = · line 2301
Quoted 6 lines
fs/nfs/pnfs.c +3
-0=A0EXPORT_SYMBOL (pnfs_unregister_layoutdriver); =A0EXPORT_SYMBOL(pnfs_register_layoutdriver); + + +/* Device ID cache. Supports one layout type per struct nfs_client=
*/
Quoted 3 lines
Quoted 3 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops = {
Quoted 3 lines
fs/nfs/pnfs.c +3
-0+int +nfs4_alloc_init_deviceid_cache(struct nfs_client *clp, + =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0void (*free_callba=
ck)(struct rcu_head *))
Quoted 4 lines
Quoted 4 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops = {
Quoted 4 lines
fs/nfs/pnfs.c +4
-0+{ + =A0 =A0 =A0 struct nfs4_deviceid_cache *c; + + =A0 =A0 =A0 c =3D kzalloc(sizeof(struct nfs4_deviceid_cache), GFP=
_KERNEL);
Quoted 5 lines
Quoted 5 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D = · line 2313
Quoted 5 lines
fs/nfs/pnfs.c +5
-0+ =A0 =A0 =A0 if (!c) + =A0 =A0 =A0 =A0 =A0 =A0 =A0 return -ENOMEM; + =A0 =A0 =A0 spin_lock(&clp->cl_lock); + =A0 =A0 =A0 if (clp->cl_devid_cache !=3D NULL) { + =A0 =A0 =A0 =A0 =A0 =A0 =A0 kref_get(&clp->cl_devid_cache->dc_kre=
f);
Quoted 3 lines
Quoted 3 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D = · line 2316
Quoted 3 lines
fs/nfs/pnfs.c +3
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 spin_unlock(&clp->cl_lock); + =A0 =A0 =A0 =A0 =A0 =A0 =A0 dprintk("%s [kref [%d]]\n", __func__, + =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 atomic_read(&clp->cl_=
devid_cache->dc_kref.refcount));
Quoted 6 lines
Quoted 6 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D = · line 2322
Quoted 6 lines
fs/nfs/pnfs.c +6
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 kfree(c); + =A0 =A0 =A0 } else { + =A0 =A0 =A0 =A0 =A0 =A0 =A0 spin_lock_init(&c->dc_lock); + =A0 =A0 =A0 =A0 =A0 =A0 =A0 INIT_LIST_HEAD(&c->dc_deviceids); + =A0 =A0 =A0 =A0 =A0 =A0 =A0 kref_init(&c->dc_kref); + =A0 =A0 =A0 =A0 =A0 =A0 =A0 c->dc_free_callback =3D free_callback=
;
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
fs/nfs/pnfs.c +11
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 c->dc_clp =3D clp; + =A0 =A0 =A0 =A0 =A0 =A0 =A0 clp->cl_devid_cache =3D c; + =A0 =A0 =A0 =A0 =A0 =A0 =A0 spin_unlock(&clp->cl_lock); + =A0 =A0 =A0 =A0 =A0 =A0 =A0 dprintk("%s [new]\n", __func__); + =A0 =A0 =A0 } + =A0 =A0 =A0 return 0; +} +EXPORT_SYMBOL (nfs4_alloc_init_deviceid_cache); + +void +nfs4_init_deviceid_node (struct nfs4_deviceid *d)
Quoted 8 lines
Quoted 8 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops = {
Quoted 8 lines
fs/nfs/pnfs.c +8
-0+{ + =A0 =A0 =A0 INIT_LIST_HEAD(&d->de_node); + =A0 =A0 =A0 INIT_RCU_HEAD(&d->de_rcu); +} +EXPORT_SYMBOL (nfs4_init_deviceid_node); + +struct nfs4_deviceid * +nfs4_find_deviceid(struct nfs4_deviceid_cache *c, struct pnfs_devi=
ceid *id)
Quoted 5 lines
Quoted 5 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D = · line 2346
Quoted 5 lines
fs/nfs/pnfs.c +5
-0+{ + =A0 =A0 =A0 struct nfs4_deviceid *d; + + =A0 =A0 =A0 rcu_read_lock(); + =A0 =A0 =A0 list_for_each_entry_rcu(d, &c->dc_deviceids, de_node)=
{
Quoted 1 line
Quoted 1 line
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D = · line 2347
Quoted 1 line
fs/nfs/pnfs.c +1
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 if (!memcmp(&d->de_id, id, NFS4_PNFS_=
DEVICEID4_SIZE)) {
Quoted 10 lines
Quoted 5 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D = · line 2352
Quoted 5 lines
fs/nfs/pnfs.c +5
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 rcu_read_unlock(); + =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 return d; + =A0 =A0 =A0 =A0 =A0 =A0 =A0 } + =A0 =A0 =A0 } + =A0 =A0 =A0 rcu_read_unlock();I hope this is worth the added complexity...
Out of curiosity, do you have a benchmark comparing the cost of
rcu_read_lock(and unlock) vs.spin_lock(&c->dc_lock)?
The deviceid cache is read at each I/O. If we use a spin_lock to protect the deviceid cache, this would mean that all I/0 is serialized behind the spin_lock even though the deviceid cache is changed infrequently. The RCU allows readers to "run almost naked" and does not serialize I/O behind reading the deviceid cache.
So I think it is worth it. I have not benchmarked the difference.
Quoted 8 lines
Quoted 8 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops = {
Quoted 8 lines
fs/nfs/pnfs.c +7
-0+ =A0 =A0 =A0 return NULL; +} +EXPORT_SYMBOL (nfs4_find_deviceid); + +/* +* Add or kref_get a deviceid. +* GETDEVICEINFOs for same deviceid can race. If deviceid is found=
, discard new
Quoted 3 lines
Quoted 3 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops = {
Quoted 3 lines
fs/nfs/pnfs.c +3
-0+*/ +void +nfs4_add_deviceid(struct nfs4_deviceid_cache *c, struct nfs4_devic=
eid *new)
Quoted 5 lines
Quoted 5 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D = · line 2367
Quoted 5 lines
fs/nfs/pnfs.c +5
-0+{ + =A0 =A0 =A0 struct nfs4_deviceid *d; + + =A0 =A0 =A0 spin_lock(&c->dc_lock); + =A0 =A0 =A0 list_for_each_entry_rcu(d, &c->dc_deviceids, de_node)=
{
Quoted 1 line
Quoted 1 line
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D = · line 2368
Quoted 1 line
fs/nfs/pnfs.c +1
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 if (!memcmp(&d->de_id, &new->de_id, N=
=46S4_PNFS_DEVICEID4_SIZE)) {
Quoted 1 line
Quoted 1 line
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D = · line 2369
Quoted 1 line
fs/nfs/pnfs.c +1
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 spin_unlock(&c->dc_lo=
ck);
Quoted 1 line
Quoted 1 line
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D = · line 2370
Quoted 1 line
fs/nfs/pnfs.c +1
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 dprintk("%s [discard]=
\n", __func__);
Quoted 1 line
Quoted 1 line
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D = · line 2371
Quoted 1 line
fs/nfs/pnfs.c +1
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 c->dc_free_callback(&=
new->de_rcu);
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
fs/nfs/pnfs.c +7
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 } + =A0 =A0 =A0 } + =A0 =A0 =A0 list_add_rcu(&new->de_node, &c->dc_deviceids); + =A0 =A0 =A0 spin_unlock(&c->dc_lock); + =A0 =A0 =A0 dprintk("%s [new]\n", __func__); +} +EXPORT_SYMBOL (nfs4_add_deviceid);
Quoted 8 lines
Quoted 8 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops = {
Quoted 8 lines
fs/nfs/pnfs.c +8
-0+ +static int +nfs4_remove_deviceid (struct nfs4_deviceid_cache *c) +{ + =A0 =A0 =A0 struct nfs4_deviceid *d; + + =A0 =A0 =A0 spin_lock(&c->dc_lock); + =A0 =A0 =A0 list_for_each_entry_rcu(d, &c->dc_deviceids, de_node)=
{
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
fs/nfs/pnfs.c +7
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 list_del_rcu(&d->de_node); + =A0 =A0 =A0 =A0 =A0 =A0 =A0 spin_unlock(&c->dc_lock); + =A0 =A0 =A0 =A0 =A0 =A0 =A0 synchronize_rcu(); + =A0 =A0 =A0 =A0 =A0 =A0 =A0 c->dc_free_callback(&d->de_rcu); + =A0 =A0 =A0 =A0 =A0 =A0 =A0 return 1; + =A0 =A0 =A0 } + =A0 =A0 =A0 spin_unlock(&c->dc_lock);
Quoted 8 lines
Quoted 8 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops = {
Quoted 8 lines
fs/nfs/pnfs.c +8
-0+ =A0 =A0 =A0 return 0; +} + +static void +nfs4_free_deviceid_cache(struct kref *kref) +{ + =A0 =A0 =A0 struct nfs4_deviceid_cache *cache =3D + =A0 =A0 =A0 =A0 =A0 =A0 =A0 container_of(kref, struct nfs4_device=
id_cache, dc_kref);
Quoted 20 lines
Quoted 6 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D = · line 2406
Quoted 5 lines
fs/nfs/pnfs.c +5
-0+ =A0 =A0 =A0 int more =3D 1; + + =A0 =A0 =A0 while (more) + =A0 =A0 =A0 =A0 =A0 =A0 =A0 more =3D nfs4_remove_deviceid(cache); + =A0 =A0 =A0 cache->dc_clp->cl_devid_cache =3D NULL;Quoted 7 lines
Need to take the
cl_lockaround this assignment
spin_lock(&cache->dc_clp->cl_lock);cache->dc_clp->cl_devid_cache=3D NULLspin_unlock(&cache->dc_clp->cl_lock);That must happen atomically before
kref_put.
It's illegal to havecl_devid_cachebe referenced bycache->dc_clpwithout a reference count backing it up.
Otherwise, if accessed concurrently to this piece of code someone might callkref_getwhile the refcount is zero.
Quoted 8 lines
Normally, you'd first clear the referencing pointer to prevent any new reference to it and only then
kref_putit, e.g.:
spin_lock(&cache->dc_clp->cl_lock);
tmp =3Dcache->dc_clp->cl_devid_cache;cache->dc_clp->cl_devid_cache=3D NULLspin_unlock(&cache->dc_clp->cl_lock);kref_put(&tmp->dc_kref, nfs4_free_deviceid_cache);
Good point. Thanks for the review. I'll rethink and resend
-->Andy
Quoted 10 lines
Benny
Quoted 7 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops = {
Quoted 7 lines
fs/nfs/pnfs.c +7
-0+ =A0 =A0 =A0 kfree(cache); +} + +void +nfs4_put_deviceid_cache(struct nfs4_deviceid_cache *c) +{ + =A0 =A0 =A0 dprintk("%s [%d]\n", __func__, atomic_read(&c->dc_kre=
f.refcount));
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
fs/nfs/pnfs.c +3
-0+ =A0 =A0 =A0 kref_put(&c->dc_kref, nfs4_free_deviceid_cache); +} +EXPORT_SYMBOL (nfs4_put_deviceid_cache);include/linux/nfs4_pnfs.h +0
-0@@ -281,6 +281,33 @@ struct pnfs_devicelist {=A0 =A0 =A0 =A0struct
pnfs_deviceid=A0 =A0dev_id[NFS4_PNFS_GETDEVL=
IST_MAXNUM];
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
=A0};
+/*
+ * Device ID RCU cache. A device ID is unique per client ID and la=
yout type.
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
+ */
+structnfs4_deviceid_cache{
+ =A0 =A0 =A0spinlock_t=A0 =A0 =A0 =A0 =A0 =A0 =A0dc_lock;
+ =A0 =A0 =A0 structlist_head=A0 =A0 =A0 =A0dc_deviceids;
+ =A0 =A0 =A0 struct kref =A0 =A0 =A0 =A0 =A0 =A0dc_kref;
+ =A0 =A0 =A0 void =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0(*dc_free=
_callback)(struct rcu_head *);
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
+ =A0 =A0 =A0 struct
nfs_client=A0 =A0 =A0*dc_clp;include/linux/nfs4_pnfs.h +3
-0+}; + +/* Device ID cache node */
Quoted 8 lines
Quoted 8 lines
include/linux/nfs4_pnfs.h struct pnfs_devicelist {
Quoted 8 lines
include/linux/nfs4_pnfs.h +8
-0+struct nfs4_deviceid { + =A0 =A0 =A0 struct list_head =A0 =A0 =A0 =A0de_node; + =A0 =A0 =A0 struct rcu_head =A0 =A0 =A0 =A0 de_rcu; + =A0 =A0 =A0 struct pnfs_deviceid =A0 =A0de_id; +}; + +extern int nfs4_alloc_init_deviceid_cache(struct nfs_client *, + =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 void =
(*free_callback)(struct rcu_head *));
Quoted 3 lines
Quoted 3 lines
include/linux/nfs4_pnfs.h struct pnfs_devicelist {
Quoted 3 lines
include/linux/nfs4_pnfs.h +3
-0+extern void nfs4_put_deviceid_cache (struct nfs4_deviceid_cache *); +extern void nfs4_init_deviceid_node (struct nfs4_deviceid *); +extern struct nfs4_deviceid *nfs4_find_deviceid(struct nfs4_device=
id_cache *,
Quoted 1 line
Quoted 1 line
Quoted 1 line
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 struc=
t pnfs_deviceid *);
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
+extern void nfs4_add_deviceid(struct
nfs4_deviceid_cache*,
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 struc=
t nfs4_deviceid *);
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
+
=A0/* pNFS client callback functions.
=A0* These operations allow the layout driver to access pNFS client =A0* specific information or call pNFSclient->serveroperations.include/linux/nfs_fs_sb.h +0
-0@@ -87,6 +87,7 @@ struct nfs_client {=A0 =A0 =A0 =A0u32 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 cl_excha=
nge_flags;
Quoted 1 line
Quoted 1 line
Quoted 1 line
=A0 =A0 =A0 =A0struct
nfs4_session=A0 =A0*cl_session; =A0 =A0/* s=
harred session */
Quoted 1 line
Quoted 1 line
Quoted 1 line
=A0 =A0 =A0 =A0struct
list_head=A0 =A0 =A0 =A0cl_lo_inodes; =A0 /*=
Inodes having layouts */
Quoted 1 line
Quoted 1 line
Quoted 1 line
+ =A0 =A0 =A0 struct
nfs4_deviceid_cache*cl_devid_cache; /* pNFS d=
eviceid cache */
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
=A0#endif /*
CONFIG_NFS_V4_1*/=A0#ifdef
CONFIG_NFS_FSCACHE
--
1.6.6--
To unsubscribe from this list: send the line "unsubscribe linux-nfs=
" in
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
the body of a message to [email protected]
More majordomo info at =A0http://vger.kernel.org/majordomo-info.htm=
l
Quoted 6 lines
Quoted 5 lines
Quoted 1 line
_______________________________________________
pNFS mailing list
[email protected]
http://linux-nfs.org/cgi-bin/mailman/listinfo/pnfs
A message that is not in the archive (sent off-list or not yet received): l2l89c397151004210822j8b43009o3a9e78ceed901fd9-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org
Re: [pnfs] [PATCH 1/3] SQUASHME pnfs_submit: generic device ID cache
Quoted 98 lines · William A. (Andy) Adamson
Quoted 97 lines · Benny Halevy
Quoted 85 lines · William A. (Andy) Adamson
Quoted 84 lines · On Fri, Apr 16, 2010 at 11:52 AM, <[email protected]> wrote:
From: Andy Adamson <[email protected]>
A shared RCU device ID cache servicing multiple mounts of a single layout type per meta data server (struct
nfs_client).Device IDs of type deviceid4 are required by all layout types, long lived and read at each I/O. They are added to the deviceid cache at first reference by a layout via GETDEVICEINFO and (currently) are only removed at umount.
Reference count the device ID cache for each mounted file system in the
initialize_mountpointlayoutdriver_io_operation.Dereference the device id cache on file system in the
uninitialize_mountpointlayoutdriver_io_operationcalled at umountSigned-off-by Andy Adamson <[email protected]>--- fs/nfs/pnfs.c | 119 +++++++++++++++++++++++++++++++++++++++++++++ include/linux/nfs4_pnfs.h | 27 ++++++++++ include/linux/nfs_fs_sb.h | 1 + 3 files changed, 147 insertions(+), 0 deletions(-)fs/nfs/pnfs.c +47
-0@@ -45,6 +45,7 @@#include <linux/nfs4.h> #include <linux/pnfs_xdr.h> #include <linux/nfs4_pnfs.h> +#include <linux/rculist.h> #include "internal.h" #include "nfs4_fs.h" @@ -2296,3 +2297,121 @@ struct pnfs_client_operations pnfs_ops = {EXPORT_SYMBOL (pnfs_unregister_layoutdriver);EXPORT_SYMBOL (pnfs_register_layoutdriver); + + +/* Device ID cache. Supports one layout type per struct nfs_client */ +int +nfs4_alloc_init_deviceid_cache (struct nfs_client *clp, +void (*free_callback)(struct rcu_head *)) +{ +struct nfs4_deviceid_cache *c; + + c =kzalloc (sizeof (struct nfs4_deviceid_cache ),GFP_KERNEL ); +if (!c) +return -ENOMEM ; +spin_lock (&clp->cl_lock); +if (clp->cl_devid_cache !=NULL ) { +kref_get (&clp->cl_devid_cache->dc_kref); +spin_unlock (&clp->cl_lock); +dprintk ("%s [kref [%d]]\n" , __func__, +atomic_read (&clp->cl_devid_cache->dc_kref.refcount)); +kfree (c); + }else { +spin_lock_init (&c->dc_lock); +INIT_LIST_HEAD (&c->dc_deviceids); +kref_init (&c->dc_kref); + c->dc_free_callback = free_callback; + c->dc_clp = clp; + clp->cl_devid_cache = c; +spin_unlock (&clp->cl_lock); +dprintk ("%s [new]\n" , __func__); + } +return 0 ; +} +EXPORT_SYMBOL (nfs4_alloc_init_deviceid_cache); + +void +nfs4_init_deviceid_node (struct nfs4_deviceid *d) +{ +INIT_LIST_HEAD (&d->de_node); +INIT_RCU_HEAD (&d->de_rcu); +} +EXPORT_SYMBOL (nfs4_init_deviceid_node); + +struct nfs4_deviceid * +nfs4_find_deviceid (struct nfs4_deviceid_cache *c,struct pnfs_deviceid *id) +{ +struct nfs4_deviceid *d; +Quoted 8 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops = { · line 2353
Quoted 8 lines
fs/nfs/pnfs.c +8
-0+rcu_read_lock (); +list_for_each_entry_rcu (d, &c->dc_deviceids, de_node) { +if (!memcmp (&d->de_id, id,NFS4_PNFS_DEVICEID4_SIZE )) { +rcu_read_unlock (); +return d; + } + } +rcu_read_unlock ();I hope this is worth the added complexity...
Quoted 4 lines
Quoted 2 lines
Out of curiosity, do you have a benchmark comparing the cost of
rcu_read_lock(and unlock) vs.spin_lock(&c->dc_lock)?The deviceid cache is read at each I/O. If we use a
spin_lockto
Yeah, I see where this goes...
In the objects layout driver we get a reference on the device structure at alloc_lseg time and keep a pointer to it throughout the lseg's life time. This saves the deviceid lookup on every I/O.
Benny
Quoted 167 lines
protect the deviceid cache, this would mean that all I/0 is serialized behind the
spin_lockeven though the deviceid cache is changed infrequently. The RCU allows readers to "run almost naked" and does not serialize I/O behind reading the deviceid cache.So I think it is worth it. I have not benchmarked the difference.
Quoted 70 lines
Quoted 58 lines
Quoted 45 lines
fs/nfs/pnfs.c +44
-0+return NULL ; +} +EXPORT_SYMBOL (nfs4_find_deviceid); + +/* +* Add or kref_get a deviceid. +* GETDEVICEINFOs for same deviceid can race. If deviceid is found, discard new +*/ +void +nfs4_add_deviceid (struct nfs4_deviceid_cache *c,struct nfs4_deviceid *new) +{ +struct nfs4_deviceid *d; + +spin_lock (&c->dc_lock); +list_for_each_entry_rcu (d, &c->dc_deviceids, de_node) { +if (!memcmp (&d->de_id, &new->de_id,NFS4_PNFS_DEVICEID4_SIZE )) { +spin_unlock (&c->dc_lock); +dprintk ("%s [discard]\n" , __func__); + c->dc_free_callback (&new->de_rcu); + } + } +list_add_rcu (&new->de_node, &c->dc_deviceids); +spin_unlock (&c->dc_lock); +dprintk ("%s [new]\n" , __func__); +} +EXPORT_SYMBOL (nfs4_add_deviceid); + +static int +nfs4_remove_deviceid (struct nfs4_deviceid_cache *c) +{ +struct nfs4_deviceid *d; + +spin_lock (&c->dc_lock); +list_for_each_entry_rcu (d, &c->dc_deviceids, de_node) { +list_del_rcu (&d->de_node); +spin_unlock (&c->dc_lock); +synchronize_rcu (); + c->dc_free_callback (&d->de_rcu); +return 1 ; + } +spin_unlock (&c->dc_lock); +return 0 ; +} +fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops = { · line 2407
Quoted 10 lines
fs/nfs/pnfs.c +10
-0+static void +nfs4_free_deviceid_cache (struct kref *kref) +{ +struct nfs4_deviceid_cache *cache = + container_of(kref, struct nfs4_deviceid_cache, dc_kref); +int more =1 ; + +while (more) + more =nfs4_remove_deviceid (cache); + cache->dc_clp->cl_devid_cache =NULL ;Need to take the
cl_lockaround this assignmentQuoted 5 lines
spin_lock(&cache->dc_clp->cl_lock);cache->dc_clp->cl_devid_cache= NULLspin_unlock(&cache->dc_clp->cl_lock);That must happen atomically before
kref_put.
It's illegal to havecl_devid_cachebe referenced bycache->dc_clpwithout a reference count backing it up.
Otherwise, if accessed concurrently to this piece of code someone might callkref_getwhile the refcount is zero.Quoted 8 lines
Normally, you'd first clear the referencing pointer to prevent any new reference to it and only then
kref_putit, e.g.:
spin_lock(&cache->dc_clp->cl_lock);
tmp =cache->dc_clp->cl_devid_cache;cache->dc_clp->cl_devid_cache= NULLspin_unlock(&cache->dc_clp->cl_lock);kref_put(&tmp->dc_kref, nfs4_free_deviceid_cache);Good point. Thanks for the review. I'll rethink and resend
-->Andy
Quoted 76 lines
Benny
Quoted 72 lines
Quoted 63 lines
fs/nfs/pnfs.c +10
-0+kfree (cache); +} + +void +nfs4_put_deviceid_cache (struct nfs4_deviceid_cache *c) +{ +dprintk ("%s [%d]\n" , __func__,atomic_read (&c->dc_kref.refcount)); +kref_put (&c->dc_kref, nfs4_free_deviceid_cache); +} +EXPORT_SYMBOL (nfs4_put_deviceid_cache);include/linux/nfs4_pnfs.h +27
-0@@ -281,6 +281,33 @@ struct pnfs_devicelist {struct pnfs_deviceid dev_id[NFS4_PNFS_GETDEVLIST_MAXNUM ]; }; +/* +* Device ID RCU cache. A device ID is unique per client ID and layout type. +*/ +struct nfs4_deviceid_cache { +spinlock_t dc_lock; +struct list_head dc_deviceids; +struct kref dc_kref; +void (*dc_free_callback)(struct rcu_head *); +struct nfs_client *dc_clp; +}; + +/* Device ID cache node */ +struct nfs4_deviceid { +struct list_head de_node; +struct rcu_head de_rcu; +struct pnfs_deviceid de_id; +}; + +extern int nfs4_alloc_init_deviceid_cache (struct nfs_client *, +void (*free_callback)(struct rcu_head *)); +extern void nfs4_put_deviceid_cache (struct nfs4_deviceid_cache *); +extern void nfs4_init_deviceid_node (struct nfs4_deviceid *); +extern struct nfs4_deviceid *nfs4_find_deviceid (struct nfs4_deviceid_cache *, +struct pnfs_deviceid *); +extern void nfs4_add_deviceid (struct nfs4_deviceid_cache *, +struct nfs4_deviceid *); +/* pNFS client callback functions. * These operations allow the layout driver to access pNFS client * specific information or call pNFS client->server operations. include/linux/nfs_fs_sb.h +1
-1@@ -87,6 +87,7 @@ struct nfs_client {u32 cl_exchange_flags;struct nfs4_session *cl_session;/* sharred session */ struct list_head cl_lo_inodes;/* Inodes having layouts */ +struct nfs4_deviceid_cache *cl_devid_cache;/* pNFS deviceid cache */ #endif /* CONFIG_NFS_V4_1 */ #ifdef CONFIG_NFS_FSCACHE --1.6.6
Quoted 5 lines
--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html_______________________________________________
pNFS mailing list
[email protected]
http://linux-nfs.org/cgi-bin/mailman/listinfo/pnfs
Signature · 9 lines
Benny Halevy Software Architect Panasas, Inc. [email protected] Tel/Fax: +972-3-647-8340 Mobile: +972-54-802-8340 Panasas: The Leader in Parallel Storage www.panasas.com
Re: [pnfs] [PATCH 1/3] SQUASHME pnfs_submit: generic device ID cache
On Thu, Apr 22, 2010 at 7:20 AM, Benny Halevy <[email protected]> wro= te:
Quoted 1 line
On Apr. 21, 2010, 18:22 +0300, "William A. (Andy) Adamson" <androsada=
Quoted 1 line · [email protected]> wrote:
Quoted 1 line
On Wed, Apr 21, 2010 at 1:59 AM, Benny Halevy <[email protected]> =
Quoted 1 line · wrote:
Quoted 1 line
Quoted 1 line
On Apr. 16, 2010, 19:04 +0300, "William A. (Andy) Adamson" <androsa=
Quoted 4 lines · [email protected]> wrote:
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines · On Fri, Apr 16, 2010 at 11:52 AM, =A0<[email protected]> wrote:
Quoted 3 lines
From: Andy Adamson <[email protected]>
A shared RCU device ID cache servicing multiple mounts of a singl=
e layout type
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
per meta data server (struct
nfs_client).Device IDs of type deviceid4 are required by all layout types, lo=
ng lived and
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
read at each I/O. =A0They are added to the deviceid cache at firs=
t reference by
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
a layout via GETDEVICEINFO and (currently) are only removed at um=
ount.
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Reference count the device ID cache for each mounted file system in the
initialize_mountpointlayoutdriver_io_operation.Dereference the device id cache on file system in the uninitializ=
e_mountpoint
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
layoutdriver_io_operationcalled at umountSigned-off-by Andy Adamson <[email protected]>---
=A0fs/nfs/pnfs.c =A0 =A0 =A0 =A0 =A0 =A0 | =A0119 +++++++++++++++=
++++++++++++++++++++++++++++++
Quoted 14 lines
Quoted 14 lines
Quoted 14 lines
Quoted 14 lines
Quoted 14 lines
=A0include/linux/nfs4_pnfs.h | =A0 27 ++++++++++ =A0include/linux/nfs_fs_sb.h | =A0 =A01 + =A03 files changed, 147 insertions(+), 0 deletions(-)
fs/nfs/pnfs.c +0
-0@@ -45,6 +45,7 @@=A0#include <
linux/nfs4.h>
=A0#include <linux/pnfs_xdr.h>
=A0#include <linux/nfs4_pnfs.h>
+#include <linux/rculist.h>
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D=
Quoted 3 lines
=A0#include "
internal.h"
=A0#include "nfs4_fs.h"fs/nfs/pnfs.c +0
-0@@ -2296,3 +2297,121 @@ struct pnfs_client_operations pnfs_ops =3D=
{
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D= · line 2301
Quoted 6 lines
fs/nfs/pnfs.c +3
-0=A0EXPORT_SYMBOL (pnfs_unregister_layoutdriver); =A0EXPORT_SYMBOL(pnfs_register_layoutdriver); + + +/* Device ID cache. Supports one layout type per struct nfs_clie=
nt */
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D= · line 2304
Quoted 3 lines
fs/nfs/pnfs.c +3
-0+int +nfs4_alloc_init_deviceid_cache(struct nfs_client *clp, + =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0void (*free_call=
back)(struct rcu_head *))
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D= · line 2308
Quoted 4 lines
fs/nfs/pnfs.c +4
-0+{ + =A0 =A0 =A0 struct nfs4_deviceid_cache *c; + + =A0 =A0 =A0 c =3D kzalloc(sizeof(struct nfs4_deviceid_cache), G=
=46P_KERNEL);
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D= · line 2313
Quoted 5 lines
fs/nfs/pnfs.c +5
-0+ =A0 =A0 =A0 if (!c) + =A0 =A0 =A0 =A0 =A0 =A0 =A0 return -ENOMEM; + =A0 =A0 =A0 spin_lock(&clp->cl_lock); + =A0 =A0 =A0 if (clp->cl_devid_cache !=3D NULL) { + =A0 =A0 =A0 =A0 =A0 =A0 =A0 kref_get(&clp->cl_devid_cache->dc_k=
ref);
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D= · line 2315
Quoted 2 lines
fs/nfs/pnfs.c +2
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 spin_unlock(&clp->cl_lock); + =A0 =A0 =A0 =A0 =A0 =A0 =A0 dprintk("%s [kref [%d]]\n", __func_=
_,
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D= · line 2316
Quoted 1 line
fs/nfs/pnfs.c +1
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 atomic_read(&clp->c=
l_devid_cache->dc_kref.refcount));
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D= · line 2322
Quoted 6 lines
fs/nfs/pnfs.c +6
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 kfree(c); + =A0 =A0 =A0 } else { + =A0 =A0 =A0 =A0 =A0 =A0 =A0 spin_lock_init(&c->dc_lock); + =A0 =A0 =A0 =A0 =A0 =A0 =A0 INIT_LIST_HEAD(&c->dc_deviceids); + =A0 =A0 =A0 =A0 =A0 =A0 =A0 kref_init(&c->dc_kref); + =A0 =A0 =A0 =A0 =A0 =A0 =A0 c->dc_free_callback =3D free_callba=
ck;
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
fs/nfs/pnfs.c +11
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 c->dc_clp =3D clp; + =A0 =A0 =A0 =A0 =A0 =A0 =A0 clp->cl_devid_cache =3D c; + =A0 =A0 =A0 =A0 =A0 =A0 =A0 spin_unlock(&clp->cl_lock); + =A0 =A0 =A0 =A0 =A0 =A0 =A0 dprintk("%s [new]\n", __func__); + =A0 =A0 =A0 } + =A0 =A0 =A0 return 0; +} +EXPORT_SYMBOL (nfs4_alloc_init_deviceid_cache); + +void +nfs4_init_deviceid_node (struct nfs4_deviceid *d)
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D= · line 2341
Quoted 8 lines
fs/nfs/pnfs.c +8
-0+{ + =A0 =A0 =A0 INIT_LIST_HEAD(&d->de_node); + =A0 =A0 =A0 INIT_RCU_HEAD(&d->de_rcu); +} +EXPORT_SYMBOL (nfs4_init_deviceid_node); + +struct nfs4_deviceid * +nfs4_find_deviceid(struct nfs4_deviceid_cache *c, struct pnfs_de=
viceid *id)
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D= · line 2346
Quoted 5 lines
fs/nfs/pnfs.c +5
-0+{ + =A0 =A0 =A0 struct nfs4_deviceid *d; + + =A0 =A0 =A0 rcu_read_lock(); + =A0 =A0 =A0 list_for_each_entry_rcu(d, &c->dc_deviceids, de_nod=
e) {
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D= · line 2347
Quoted 1 line
fs/nfs/pnfs.c +1
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 if (!memcmp(&d->de_id, id, NFS4_PNF=
S_DEVICEID4_SIZE)) {
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 5 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D= · line 2352
Quoted 5 lines
fs/nfs/pnfs.c +5
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 rcu_read_unlock(); + =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 return d; + =A0 =A0 =A0 =A0 =A0 =A0 =A0 } + =A0 =A0 =A0 } + =A0 =A0 =A0 rcu_read_unlock();I hope this is worth the added complexity...
Quoted 7 lines
Quoted 4 lines
Quoted 2 lines
Out of curiosity, do you have a benchmark comparing the cost of
rcu_read_lock(and unlock) vs.spin_lock(&c->dc_lock)?The deviceid cache is read at each I/O. If we use a
spin_locktoYeah, I see where this goes...
In the objects layout driver we get a reference on the device structu=
re
Quoted 1 line
at
alloc_lsegtime and keep a pointer to it throughout the lseg's lif=
e time.
Quoted 1 line
This saves the deviceid lookup on every I/O.
Perhaps that is a better way to go. How many deviceid's is 'normal' for the object driver?
-->Andy
Quoted 4 lines
Benny
Quoted 1 line
protect the deviceid cache, this would mean that all I/0 is serializ=
ed
Quoted 8 lines
Quoted 8 lines
behind the
spin_lockeven though the deviceid cache is changed infrequently. The RCU allows readers to "run almost naked" and does not serialize I/O behind reading the deviceid cache.So I think it is worth it. I have not benchmarked the difference.
Quoted 1 line
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D= · line 2359
Quoted 7 lines
fs/nfs/pnfs.c +7
-0+ =A0 =A0 =A0 return NULL; +} +EXPORT_SYMBOL (nfs4_find_deviceid); + +/* +* Add or kref_get a deviceid. +* GETDEVICEINFOs for same deviceid can race. If deviceid is fou=
nd, discard new
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D= · line 2362
Quoted 3 lines
fs/nfs/pnfs.c +3
-0+*/ +void +nfs4_add_deviceid(struct nfs4_deviceid_cache *c, struct nfs4_dev=
iceid *new)
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D= · line 2367
Quoted 5 lines
fs/nfs/pnfs.c +5
-0+{ + =A0 =A0 =A0 struct nfs4_deviceid *d; + + =A0 =A0 =A0 spin_lock(&c->dc_lock); + =A0 =A0 =A0 list_for_each_entry_rcu(d, &c->dc_deviceids, de_nod=
e) {
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D= · line 2368
Quoted 1 line
fs/nfs/pnfs.c +1
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 if (!memcmp(&d->de_id, &new->de_id,=
NFS4_PNFS_DEVICEID4_SIZE)) {
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D= · line 2369
Quoted 1 line
fs/nfs/pnfs.c +1
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 spin_unlock(&c->dc_=
lock);
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D= · line 2370
Quoted 1 line
fs/nfs/pnfs.c +1
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 dprintk("%s [discar=
d]\n", __func__);
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D= · line 2371
Quoted 1 line
fs/nfs/pnfs.c +1
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 c->dc_free_callback=
(&new->de_rcu);
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
fs/nfs/pnfs.c +7
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 } + =A0 =A0 =A0 } + =A0 =A0 =A0 list_add_rcu(&new->de_node, &c->dc_deviceids); + =A0 =A0 =A0 spin_unlock(&c->dc_lock); + =A0 =A0 =A0 dprintk("%s [new]\n", __func__); +} +EXPORT_SYMBOL (nfs4_add_deviceid);
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D= · line 2386
Quoted 8 lines
fs/nfs/pnfs.c +8
-0+ +static int +nfs4_remove_deviceid (struct nfs4_deviceid_cache *c) +{ + =A0 =A0 =A0 struct nfs4_deviceid *d; + + =A0 =A0 =A0 spin_lock(&c->dc_lock); + =A0 =A0 =A0 list_for_each_entry_rcu(d, &c->dc_deviceids, de_nod=
e) {
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
fs/nfs/pnfs.c +7
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 list_del_rcu(&d->de_node); + =A0 =A0 =A0 =A0 =A0 =A0 =A0 spin_unlock(&c->dc_lock); + =A0 =A0 =A0 =A0 =A0 =A0 =A0 synchronize_rcu(); + =A0 =A0 =A0 =A0 =A0 =A0 =A0 c->dc_free_callback(&d->de_rcu); + =A0 =A0 =A0 =A0 =A0 =A0 =A0 return 1; + =A0 =A0 =A0 } + =A0 =A0 =A0 spin_unlock(&c->dc_lock);
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D= · line 2401
Quoted 8 lines
fs/nfs/pnfs.c +8
-0+ =A0 =A0 =A0 return 0; +} + +static void +nfs4_free_deviceid_cache(struct kref *kref) +{ + =A0 =A0 =A0 struct nfs4_deviceid_cache *cache =3D + =A0 =A0 =A0 =A0 =A0 =A0 =A0 container_of(kref, struct nfs4_devi=
ceid_cache, dc_kref);
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D= · line 2405
Quoted 4 lines
fs/nfs/pnfs.c +4
-0+ =A0 =A0 =A0 int more =3D 1; + + =A0 =A0 =A0 while (more) + =A0 =A0 =A0 =A0 =A0 =A0 =A0 more =3D nfs4_remove_deviceid(cache=
);
Quoted 32 lines
Quoted 32 lines
Quoted 16 lines
Quoted 9 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D= · line 2406
Quoted 1 line
fs/nfs/pnfs.c +1
-0+ =A0 =A0 =A0 cache->dc_clp->cl_devid_cache =3D NULL;Need to take the
cl_lockaround this assignment
spin_lock(&cache->dc_clp->cl_lock);cache->dc_clp->cl_devid_cache=3D NULLspin_unlock(&cache->dc_clp->cl_lock);That must happen atomically before
kref_put.
It's illegal to havecl_devid_cachebe referenced bycache->dc_clpwithout a reference count backing it up.
Otherwise, if accessed concurrently to this piece of code someone might callkref_getwhile the refcount is zero.Quoted 8 lines
Normally, you'd first clear the referencing pointer to prevent any new reference to it and only then
kref_putit, e.g.:
spin_lock(&cache->dc_clp->cl_lock);
tmp =3Dcache->dc_clp->cl_devid_cache;cache->dc_clp->cl_devid_cache=3D NULLspin_unlock(&cache->dc_clp->cl_lock);kref_put(&tmp->dc_kref, nfs4_free_deviceid_cache);Good point. Thanks for the review. I'll rethink and resend
-->Andy
Quoted 3 lines
Benny
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops =3D= · line 2413
Quoted 7 lines
fs/nfs/pnfs.c +7
-0+ =A0 =A0 =A0 kfree(cache); +} + +void +nfs4_put_deviceid_cache(struct nfs4_deviceid_cache *c) +{ + =A0 =A0 =A0 dprintk("%s [%d]\n", __func__, atomic_read(&c->dc_k=
ref.refcount));
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
include/linux/nfs4_pnfs.=
Quoted 4 lines
fs/nfs/pnfs.c +3
-0+ =A0 =A0 =A0 kref_put(&c->dc_kref, nfs4_free_deviceid_cache); +} +EXPORT_SYMBOL (nfs4_put_deviceid_cache);include/linux/nfs4_pnfs.= renamed from include/linux/nfs4_pnfs.h +0
-0
h
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
index 1d521f4..2a88a06 100644
include/linux/nfs4_pnfs.h +0
-0@@ -281,6 +281,33 @@ struct pnfs_devicelist {=A0 =A0 =A0 =A0struct
pnfs_deviceid=A0 =A0dev_id[NFS4_PNFS_GETDE=
VLIST_MAXNUM];
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
=A0};
+/*
+ * Device ID RCU cache. A device ID is unique per client ID and =
layout type.
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
+ */
+structnfs4_deviceid_cache{
+ =A0 =A0 =A0spinlock_t=A0 =A0 =A0 =A0 =A0 =A0 =A0dc_lock;
+ =A0 =A0 =A0 structlist_head=A0 =A0 =A0 =A0dc_deviceids;
+ =A0 =A0 =A0 struct kref =A0 =A0 =A0 =A0 =A0 =A0dc_kref;
+ =A0 =A0 =A0 void =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0(*dc_fr=
ee_callback)(struct rcu_head *);
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
+ =A0 =A0 =A0 struct
nfs_client=A0 =A0 =A0*dc_clp;include/linux/nfs4_pnfs.h +3
-0+}; + +/* Device ID cache node */
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
include/linux/nfs4_pnfs.h
Quoted 8 lines
include/linux/nfs4_pnfs.h +8
-0+struct nfs4_deviceid { + =A0 =A0 =A0 struct list_head =A0 =A0 =A0 =A0de_node; + =A0 =A0 =A0 struct rcu_head =A0 =A0 =A0 =A0 de_rcu; + =A0 =A0 =A0 struct pnfs_deviceid =A0 =A0de_id; +}; + +extern int nfs4_alloc_init_deviceid_cache(struct nfs_client *, + =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 voi=
d (*free_callback)(struct rcu_head *));
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
+extern void nfs4_put_deviceid_cache(struct
nfs4_deviceid_cache*=
);
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
include/linux/nfs4_pnfs.h
Quoted 2 lines
include/linux/nfs4_pnfs.h +2
-0+extern void nfs4_init_deviceid_node (struct nfs4_deviceid *); +extern struct nfs4_deviceid *nfs4_find_deviceid(struct nfs4_devi=
ceid_cache *,
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 str=
uct pnfs_deviceid *);
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
+extern void nfs4_add_deviceid(struct
nfs4_deviceid_cache*,
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 str=
uct nfs4_deviceid *);
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
+
=A0/* pNFS client callback functions.
=A0* These operations allow the layout driver to access pNFS clie=
nt
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
include/linux/nfs_fs_sb.=
Quoted 2 lines
=A0* specific information or call pNFS
client->serveroperations.include/linux/nfs_fs_sb.= renamed from include/linux/nfs_fs_sb.h +0
-0
h
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
index 8522461..ef2e18e 100644
include/linux/nfs_fs_sb.h +0
-0@@ -87,6 +87,7 @@ struct nfs_client {=A0 =A0 =A0 =A0u32 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 cl_exc=
hange_flags;
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
=A0 =A0 =A0 =A0struct
nfs4_session=A0 =A0*cl_session; =A0 =A0/*=
sharred session */
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
=A0 =A0 =A0 =A0struct
list_head=A0 =A0 =A0 =A0cl_lo_inodes; =A0 =
/* Inodes having layouts */
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
+ =A0 =A0 =A0 struct
nfs4_deviceid_cache*cl_devid_cache; /* pNFS=
deviceid cache */
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
=A0#endif /*
CONFIG_NFS_V4_1*/=A0#ifdef
CONFIG_NFS_FSCACHE
--
1.6.6--
To unsubscribe from this list: send the line "unsubscribe linux-n=
fs" in
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
the body of a message to [email protected]
More majordomo info at =A0http://vger.kernel.org/majordomo-info.h=
tml
Quoted 19 lines
Quoted 6 lines
Quoted 6 lines
Quoted 5 lines
Quoted 1 line
_______________________________________________
pNFS mailing list
[email protected]
http://linux-nfs.org/cgi-bin/mailman/listinfo/pnfs--
Benny Halevy
Software Architect
Panasas, Inc.
[email protected]
Tel/Fax: +972-3-647-8340
Mobile: +972-54-802-8340Panasas: The Leader in Parallel Storage
www.panasas.com
A message that is not in the archive (sent off-list or not yet received): v2h89c397151004220847v3a31c493s4089d0cd53cf3e19-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org
Re: [pnfs] [PATCH 1/3] SQUASHME pnfs_submit: generic device ID cache
Quoted 105 lines · William A. (Andy) Adamson
Quoted 104 lines · Benny Halevy
Quoted 102 lines · William A. (Andy) Adamson
Quoted 97 lines · Benny Halevy
Quoted 85 lines · William A. (Andy) Adamson
Quoted 84 lines · On Fri, Apr 16, 2010 at 11:52 AM, <[email protected]> wrote:
From: Andy Adamson <[email protected]>
A shared RCU device ID cache servicing multiple mounts of a single layout type per meta data server (struct
nfs_client).Device IDs of type deviceid4 are required by all layout types, long lived and read at each I/O. They are added to the deviceid cache at first reference by a layout via GETDEVICEINFO and (currently) are only removed at umount.
Reference count the device ID cache for each mounted file system in the
initialize_mountpointlayoutdriver_io_operation.Dereference the device id cache on file system in the
uninitialize_mountpointlayoutdriver_io_operationcalled at umountSigned-off-by Andy Adamson <[email protected]>--- fs/nfs/pnfs.c | 119 +++++++++++++++++++++++++++++++++++++++++++++ include/linux/nfs4_pnfs.h | 27 ++++++++++ include/linux/nfs_fs_sb.h | 1 + 3 files changed, 147 insertions(+), 0 deletions(-)fs/nfs/pnfs.c +47
-0@@ -45,6 +45,7 @@#include <linux/nfs4.h> #include <linux/pnfs_xdr.h> #include <linux/nfs4_pnfs.h> +#include <linux/rculist.h> #include "internal.h" #include "nfs4_fs.h" @@ -2296,3 +2297,121 @@ struct pnfs_client_operations pnfs_ops = {EXPORT_SYMBOL (pnfs_unregister_layoutdriver);EXPORT_SYMBOL (pnfs_register_layoutdriver); + + +/* Device ID cache. Supports one layout type per struct nfs_client */ +int +nfs4_alloc_init_deviceid_cache (struct nfs_client *clp, +void (*free_callback)(struct rcu_head *)) +{ +struct nfs4_deviceid_cache *c; + + c =kzalloc (sizeof (struct nfs4_deviceid_cache ),GFP_KERNEL ); +if (!c) +return -ENOMEM ; +spin_lock (&clp->cl_lock); +if (clp->cl_devid_cache !=NULL ) { +kref_get (&clp->cl_devid_cache->dc_kref); +spin_unlock (&clp->cl_lock); +dprintk ("%s [kref [%d]]\n" , __func__, +atomic_read (&clp->cl_devid_cache->dc_kref.refcount)); +kfree (c); + }else { +spin_lock_init (&c->dc_lock); +INIT_LIST_HEAD (&c->dc_deviceids); +kref_init (&c->dc_kref); + c->dc_free_callback = free_callback; + c->dc_clp = clp; + clp->cl_devid_cache = c; +spin_unlock (&clp->cl_lock); +dprintk ("%s [new]\n" , __func__); + } +return 0 ; +} +EXPORT_SYMBOL (nfs4_alloc_init_deviceid_cache); + +void +nfs4_init_deviceid_node (struct nfs4_deviceid *d) +{ +INIT_LIST_HEAD (&d->de_node); +INIT_RCU_HEAD (&d->de_rcu); +} +EXPORT_SYMBOL (nfs4_init_deviceid_node); + +struct nfs4_deviceid * +nfs4_find_deviceid (struct nfs4_deviceid_cache *c,struct pnfs_deviceid *id) +{ +struct nfs4_deviceid *d; +Quoted 8 lines
fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops = { · line 2353
Quoted 8 lines
fs/nfs/pnfs.c +8
-0+rcu_read_lock (); +list_for_each_entry_rcu (d, &c->dc_deviceids, de_node) { +if (!memcmp (&d->de_id, id,NFS4_PNFS_DEVICEID4_SIZE )) { +rcu_read_unlock (); +return d; + } + } +rcu_read_unlock ();I hope this is worth the added complexity...
Quoted 2 lines
Out of curiosity, do you have a benchmark comparing the cost of
rcu_read_lock(and unlock) vs.spin_lock(&c->dc_lock)?The deviceid cache is read at each I/O. If we use a
spin_lockto
Quoted 8 lines
Quoted 4 lines
Yeah, I see where this goes...
In the objects layout driver we get a reference on the device structure atalloc_lsegtime and keep a pointer to it throughout the lseg's life time. This saves the deviceid lookup on every I/O.Perhaps that is a better way to go. How many deviceid's is 'normal' for the object driver?
I'd say that order of 10's to a few 100's is normal. 1000 and up is a big installation.
Benny
Quoted 186 lines
-->Andy
Quoted 183 lines
Benny
Quoted 87 lines
protect the deviceid cache, this would mean that all I/0 is serialized behind the
spin_lockeven though the deviceid cache is changed infrequently. The RCU allows readers to "run almost naked" and does not serialize I/O behind reading the deviceid cache.So I think it is worth it. I have not benchmarked the difference.
Quoted 70 lines
Quoted 58 lines
Quoted 45 lines
fs/nfs/pnfs.c +44
-0+return NULL ; +} +EXPORT_SYMBOL (nfs4_find_deviceid); + +/* +* Add or kref_get a deviceid. +* GETDEVICEINFOs for same deviceid can race. If deviceid is found, discard new +*/ +void +nfs4_add_deviceid (struct nfs4_deviceid_cache *c,struct nfs4_deviceid *new) +{ +struct nfs4_deviceid *d; + +spin_lock (&c->dc_lock); +list_for_each_entry_rcu (d, &c->dc_deviceids, de_node) { +if (!memcmp (&d->de_id, &new->de_id,NFS4_PNFS_DEVICEID4_SIZE )) { +spin_unlock (&c->dc_lock); +dprintk ("%s [discard]\n" , __func__); + c->dc_free_callback (&new->de_rcu); + } + } +list_add_rcu (&new->de_node, &c->dc_deviceids); +spin_unlock (&c->dc_lock); +dprintk ("%s [new]\n" , __func__); +} +EXPORT_SYMBOL (nfs4_add_deviceid); + +static int +nfs4_remove_deviceid (struct nfs4_deviceid_cache *c) +{ +struct nfs4_deviceid *d; + +spin_lock (&c->dc_lock); +list_for_each_entry_rcu (d, &c->dc_deviceids, de_node) { +list_del_rcu (&d->de_node); +spin_unlock (&c->dc_lock); +synchronize_rcu (); + c->dc_free_callback (&d->de_rcu); +return 1 ; + } +spin_unlock (&c->dc_lock); +return 0 ; +} +fs/nfs/pnfs.c struct pnfs_client_operations pnfs_ops = { · line 2407
Quoted 10 lines
fs/nfs/pnfs.c +10
-0+static void +nfs4_free_deviceid_cache (struct kref *kref) +{ +struct nfs4_deviceid_cache *cache = + container_of(kref, struct nfs4_deviceid_cache, dc_kref); +int more =1 ; + +while (more) + more =nfs4_remove_deviceid (cache); + cache->dc_clp->cl_devid_cache =NULL ;Need to take the
cl_lockaround this assignmentQuoted 5 lines
spin_lock(&cache->dc_clp->cl_lock);cache->dc_clp->cl_devid_cache= NULLspin_unlock(&cache->dc_clp->cl_lock);That must happen atomically before
kref_put.
It's illegal to havecl_devid_cachebe referenced bycache->dc_clpwithout a reference count backing it up.
Otherwise, if accessed concurrently to this piece of code someone might callkref_getwhile the refcount is zero.Quoted 8 lines
Normally, you'd first clear the referencing pointer to prevent any new reference to it and only then
kref_putit, e.g.:
spin_lock(&cache->dc_clp->cl_lock);
tmp =cache->dc_clp->cl_devid_cache;cache->dc_clp->cl_devid_cache= NULLspin_unlock(&cache->dc_clp->cl_lock);kref_put(&tmp->dc_kref, nfs4_free_deviceid_cache);Quoted 80 lines
Good point. Thanks for the review. I'll rethink and resend
-->Andy
Quoted 76 lines
Benny
Quoted 72 lines
Quoted 63 lines
fs/nfs/pnfs.c +10
-0+kfree (cache); +} + +void +nfs4_put_deviceid_cache (struct nfs4_deviceid_cache *c) +{ +dprintk ("%s [%d]\n" , __func__,atomic_read (&c->dc_kref.refcount)); +kref_put (&c->dc_kref, nfs4_free_deviceid_cache); +} +EXPORT_SYMBOL (nfs4_put_deviceid_cache);include/linux/nfs4_pnfs.h +27
-0@@ -281,6 +281,33 @@ struct pnfs_devicelist {struct pnfs_deviceid dev_id[NFS4_PNFS_GETDEVLIST_MAXNUM ]; }; +/* +* Device ID RCU cache. A device ID is unique per client ID and layout type. +*/ +struct nfs4_deviceid_cache { +spinlock_t dc_lock; +struct list_head dc_deviceids; +struct kref dc_kref; +void (*dc_free_callback)(struct rcu_head *); +struct nfs_client *dc_clp; +}; + +/* Device ID cache node */ +struct nfs4_deviceid { +struct list_head de_node; +struct rcu_head de_rcu; +struct pnfs_deviceid de_id; +}; + +extern int nfs4_alloc_init_deviceid_cache (struct nfs_client *, +void (*free_callback)(struct rcu_head *)); +extern void nfs4_put_deviceid_cache (struct nfs4_deviceid_cache *); +extern void nfs4_init_deviceid_node (struct nfs4_deviceid *); +extern struct nfs4_deviceid *nfs4_find_deviceid (struct nfs4_deviceid_cache *, +struct pnfs_deviceid *); +extern void nfs4_add_deviceid (struct nfs4_deviceid_cache *, +struct nfs4_deviceid *); +/* pNFS client callback functions. * These operations allow the layout driver to access pNFS client * specific information or call pNFS client->server operations. include/linux/nfs_fs_sb.h +1
-1@@ -87,6 +87,7 @@ struct nfs_client {u32 cl_exchange_flags;struct nfs4_session *cl_session;/* sharred session */ struct list_head cl_lo_inodes;/* Inodes having layouts */ +struct nfs4_deviceid_cache *cl_devid_cache;/* pNFS deviceid cache */ #endif /* CONFIG_NFS_V4_1 */ #ifdef CONFIG_NFS_FSCACHE --1.6.6
Quoted 5 lines
--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html_______________________________________________
pNFS mailing list
[email protected]
http://linux-nfs.org/cgi-bin/mailman/listinfo/pnfs--
Benny Halevy
Software Architect
Panasas, Inc.
[email protected]
Tel/Fax: +972-3-647-8340
Mobile: +972-54-802-8340Panasas: The Leader in Parallel Storage
www.panasas.com