pnfs: LAYOUTGET/DEVINFO submission v5
Subsystems (3)
cover pnfs: LAYOUTGET/DEVINFO submission v5
This is try 5 of the first wave of the linux pnfs client submission. The only change (besides rebase on top of Trond's most recent nfs-for-2.6.37 branch) between this series and the previous is that owner/sign-off issues have been addressed, which allows us to remove the RFC label of the previous series.
For those interested, the patches themselves can also be found atgit://linux-nfs.org/~iisaman/linux-pnfs.git,
branch devinfo-submit-rc7-trond
This is the start of code implementing pnfs, based on RFC 5661. Since sending the whole thing at once would be overwhelming, we are trying to break it into bite sized waves. This first wave implements the mount/umount infrastructure, as well as sending the LAYOUTGET and GETDEVTICEINFO calls on io (but not actually using the information for io). Note that two major simplifications to the protocol will be made throughout the initial submission process: only the file layout driver is considered, and only whole file layouts are requested.
These patches apply against Trond's nfs-for-2.6.37 branch.
patches 01-08 implement the mount/umount hooks patches 09-12 implement LAYOUTGET and GETDEVICEINFO
PATCH 1/12 NFSD: remove duplicate NFS4_STATEID_SIZE
From: Andy Adamson <[email protected]>
Already accepted by Bruce
--- fs/nfsd/nfs4callback.c | 1 - 1 files changed, 0 insertions(+), 1 deletions(-)
fs/nfsd/nfs4callback.c +0 -1
@@ -41,7 +41,6 @@#define NFSPROC4_CB_NULL 0#define NFSPROC4_CB_COMPOUND 1 -#define NFS4_STATEID_SIZE 16/* Index of predefined Linux callback client operations */
Signature · 1 line
1.7.2.1
PATCH 2/12 SUNRPC: define xdr_decode_opaque_fixed
In reply to Fred IsamanFrom: Benny Halevy <[email protected]>
A helper for decoding a fixed length opaque value. Returns a pointer to the next item in the xdr stream.
--- include/linux/sunrpc/xdr.h | 7 +++++++ 1 files changed, 7 insertions(+), 0 deletions(-)
include/linux/sunrpc/xdr.h +7 -0
@@ -132,6 +132,13 @@ xdr_decode_hyper(__be32 *p, __u64 *valp)return p +2 ; } +static inline __be32 * +xdr_decode_opaque_fixed (__be32 *p,void *ptr,unsigned int len) +{ +memcpy (ptr, p, len); +return p +XDR_QUADLEN (len); +} +/* * Adjust kvec to reflect end of xdr'ed data (RPC client XDR) */
Signature · 1 line
1.7.2.1
PATCH 3/12 NFSv4.1: pnfsd, pnfs: protocol level pnfs constants
In reply to Fred IsamanFrom: Dean Hildebrand <[email protected]>
Use only layoutreturn constant for both returns and recalls. (return_* works better for recall_type rather the other way around)
--- include/linux/nfs4.h | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 45 insertions(+), 0 deletions(-)
include/linux/nfs4.h +45 -0
@@ -471,6 +471,8 @@ enum lock_type4 {#define FATTR4_WORD1_TIME_MODIFY (1UL << 21)#define FATTR4_WORD1_TIME_MODIFY_SET (1UL << 22)#define FATTR4_WORD1_MOUNTED_ON_FILEID (1UL << 23) +#define FATTR4_WORD1_FS_LAYOUT_TYPES (1UL << 30) +#define FATTR4_WORD2_LAYOUT_BLKSIZE (1UL << 1)#define NFSPROC4_NULL 0#define NFSPROC4_COMPOUND 1
@@ -550,6 +552,49 @@ enum state_protect_how4 {SP4_SSV =2 }; +enum pnfs_layouttype { +LAYOUT_NFSV4_1_FILES =1 , +LAYOUT_OSD2_OBJECTS =2 , +LAYOUT_BLOCK_VOLUME =3 , +}; + +/* used for both layout return and recall */ +enum pnfs_layoutreturn_type { +RETURN_FILE =1 , +RETURN_FSID =2 , +RETURN_ALL =3 +}; + +enum pnfs_iomode { +IOMODE_READ =1 , +IOMODE_RW =2 , +IOMODE_ANY =3 , +}; + +enum pnfs_notify_deviceid_type4 { +NOTIFY_DEVICEID4_CHANGE =1 <<1 , +NOTIFY_DEVICEID4_DELETE =1 <<2 , +}; + +#define NFL4_UFLG_MASK 0x0000003F +#define NFL4_UFLG_DENSE 0x00000001 +#define NFL4_UFLG_COMMIT_THRU_MDS 0x00000002 +#define NFL4_UFLG_STRIPE_UNIT_SIZE_MASK 0xFFFFFFC0 + +/* Encoded in the loh_body field of type layouthint4 */ +enum filelayout_hint_care4 { +NFLH4_CARE_DENSE =NFL4_UFLG_DENSE , +NFLH4_CARE_COMMIT_THRU_MDS =NFL4_UFLG_COMMIT_THRU_MDS , +NFLH4_CARE_STRIPE_UNIT_SIZE =0x00000040 , +NFLH4_CARE_STRIPE_COUNT =0x00000080 +}; + +#define NFS4_DEVICEID4_SIZE 16 + +struct nfs4_deviceid { +char data[NFS4_DEVICEID4_SIZE ]; +}; +#endif #endif
Signature · 1 line
1.7.2.1
PATCH 4/12 NFS: change stateid to be a union
In reply to Fred IsamanFrom: Alexandros Batsakis <[email protected]>
In NFSv4.1 the stateid consists of the other and seqid fields. For layout processing we need to numerically compare the seqid value of layout stateids. To do so, introduce a union to nfs4_stateid to switch between opaque(16 bytes) and opaque(12 bytes) / __be32
--- fs/nfs/callback_proc.c | 8 ++++---- include/linux/nfs4.h | 15 +++++++++++++-- 2 files changed, 17 insertions(+), 6 deletions(-)
fs/nfs/callback_proc.c +4 -4
@@ -118,11 +118,11 @@ int nfs41_validate_delegation_stateid(struct nfs_delegation *delegation, const nif (delegation ==NULL )return 0 ; -/* seqid is 4-bytes long */ -if (((u32 *) &stateid->data)[0 ] !=0 ) +if (stateid->stateid.seqid !=0 )return 0 ; -if (memcmp (&delegation->stateid.data[4 ], &stateid->data[4 ], -sizeof (stateid->data)-4 )) +if (memcmp (&delegation->stateid.stateid.other, + &stateid->stateid.other, +NFS4_STATEID_OTHER_SIZE ))return 0 ;return 1 ;
include/linux/nfs4.h +13 -2
@@ -17,7 +17,9 @@#define NFS4_BITMAP_SIZE 2#define NFS4_VERIFIER_SIZE 8 -#define NFS4_STATEID_SIZE 16 +#define NFS4_STATEID_SEQID_SIZE 4 +#define NFS4_STATEID_OTHER_SIZE 12 +#define NFS4_STATEID_SIZE (NFS4_STATEID_SEQID_SIZE + NFS4_STATEID_OTHER_SIZE)#define NFS4_FHSIZE 128#define NFS4_MAXPATHLEN PATH_MAX#define NFS4_MAXNAMLEN NAME_MAX
@@ -167,7 +169,16 @@ struct nfs4_acl { };typedef struct {char data[NFS4_VERIFIER_SIZE ]; }nfs4_verifier ; -typedef struct {char data[NFS4_STATEID_SIZE ]; }nfs4_stateid ; + +struct nfs41_stateid { +__be32 seqid; +char other[NFS4_STATEID_OTHER_SIZE ]; +} __attribute__ ((packed)); + +typedef union { +char data[NFS4_STATEID_SIZE ]; +struct nfs41_stateid stateid; +}nfs4_stateid ;enum nfs_opnum4 {OP_ACCESS =3 ,
Signature · 1 line
1.7.2.1
PATCH 5/12 NFS: ask for layouttypes during v4 fsinfo call
In reply to Fred IsamanFrom: Andy Adamson <[email protected]>
This information will be used to determine which layout driver, if any, to use for subsequent IO on this filesystem. Each driver is assigned an integer id, with 0 reserved to indicate no driver.
The server can in theory return multiple ids. However, our current client implementation only notes the first entry and ignores the rest.
--- fs/nfs/nfs4proc.c | 2 +- fs/nfs/nfs4xdr.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++ include/linux/nfs_xdr.h | 1 + 3 files changed, 59 insertions(+), 1 deletions(-)
fs/nfs/nfs4proc.c +1 -1
@@ -129,7 +129,7 @@ const u32 nfs4_fsinfo_bitmap[2] = { FATTR4_WORD0_MAXFILESIZE | FATTR4_WORD0_MAXREAD | FATTR4_WORD0_MAXWRITE | FATTR4_WORD0_LEASE_TIME, - 0 + FATTR4_WORD1_FS_LAYOUT_TYPES };const u32 nfs4_fs_locations_bitmap[2 ] = {
fs/nfs/nfs4xdr.c +57 -0
69 lines not on this page · raw message
include/linux/nfs_xdr.h +1 -0
@@ -113,6 +113,7 @@ struct nfs_fsinfo {__u32 dtpref;/* pref. readdir transfer size */ __u64 maxfilesize;__u32 lease_time;/* in seconds */ +__u32 layouttype;/* supported pnfs layout driver */ };struct nfs_fsstat {
Signature · 1 line
1.7.2.1
PATCH 6/12 NFS: set layout driver
In reply to Fred IsamanFrom: Ricardo Labiaga <[email protected]>
Put in the infrastructure that uses information returned from the server at mount to select a layout driver module.
In this patch, a stub is used that always returns "no driver found".
--- fs/nfs/Makefile | 1 + fs/nfs/client.c | 5 +++ fs/nfs/pnfs.c | 84 +++++++++++++++++++++++++++++++++++++++++++++ fs/nfs/pnfs.h | 56 ++++++++++++++++++++++++++++++ include/linux/nfs_fs.h | 1 + include/linux/nfs_fs_sb.h | 1 + 6 files changed, 148 insertions(+), 0 deletions(-) create mode 100644 fs/nfs/pnfs.c create mode 100644 fs/nfs/pnfs.h
fs/nfs/Makefile +1 -0
@@ -15,5 +15,6 @@ nfs-$(CONFIG_NFS_V4) += nfs4proc.o nfs4xdr.o nfs4state.o nfs4renewd.o \ delegation.o idmap.o \ callback.o callback_xdr.o callback_proc.o \ nfs4namespace.o +nfs-$(CONFIG_NFS_V4_1) += pnfs.o nfs-$(CONFIG_SYSCTL) += sysctl.o nfs-$(CONFIG_NFS_FSCACHE) += fscache.o fscache-index.o
fs/nfs/client.c +5 -0
@@ -48,6 +48,7 @@#include "iostat.h" #include "internal.h" #include "fscache.h" +#include "pnfs.h" #define NFSDBG_FACILITY NFSDBG_CLIENT
@@ -900,6 +901,8 @@ static void nfs_server_set_fsinfo(struct nfs_server *server, struct nfs_fsinfo *if (server->wsize >NFS_MAX_FILE_IO_SIZE ) server->wsize =NFS_MAX_FILE_IO_SIZE ; server->wpages = (server->wsize +PAGE_CACHE_SIZE -1 ) >>PAGE_CACHE_SHIFT ; +set_pnfs_layoutdriver (server, fsinfo->layouttype); + server->wtmult =nfs_block_bits (fsinfo->wtmult,NULL ); server->dtsize =nfs_block_size (fsinfo->dtpref,NULL );
@@ -937,6 +940,7 @@ static int nfs_probe_fsinfo(struct nfs_server *server, struct nfs_fh *mntfh, str } fsinfo.fattr = fattr; + fsinfo.layouttype =0 ; error = clp->rpc_ops->fsinfo (server, mntfh, &fsinfo);if (error <0 ) goto out_error;
@@ -1019,6 +1023,7 @@ void nfs_free_server(struct nfs_server *server) {dprintk ("--> nfs_free_server()\n" ); +unset_pnfs_layoutdriver (server);spin_lock (&nfs_client_lock);list_del (&server->client_link);list_del (&server->master_link);
fs/nfs/pnfs.c new file +84 -0
84 lines not on this page · raw message
fs/nfs/pnfs.h new file +56 -0
56 lines not on this page · raw message
include/linux/nfs_fs.h +1 -0
@@ -615,6 +615,7 @@ nfs_fileid_to_ino_t(u64 fileid)#define NFSDBG_CLIENT 0x0200#define NFSDBG_MOUNT 0x0400#define NFSDBG_FSCACHE 0x0800 +#define NFSDBG_PNFS 0x1000#define NFSDBG_ALL 0xFFFF#ifdef __KERNEL__
include/linux/nfs_fs_sb.h +1 -0
@@ -144,6 +144,7 @@ struct nfs_server {u32 acl_bitmask;/* V4 bitmask representing the ACEs that are supported on this filesystem */ +struct pnfs_layoutdriver_type *pnfs_curr_ld;/* Active layout driver */ #endif void (*destroy)(struct nfs_server *);
Signature · 1 line
1.7.2.1
PATCH 7/12 NFSv4.1: pnfs: full mount/umount infrastructure
In reply to Fred IsamanAllow a module implementing a layout type to register, and have its mount/umount routines called for filesystems that the server declares support it.
--- Documentation/filesystems/nfs/00-INDEX | 2 + Documentation/filesystems/nfs/pnfs.txt | 48 +++++++++++++++++ fs/nfs/Kconfig | 8 ++- fs/nfs/pnfs.c | 88 +++++++++++++++++++++++++++++++- fs/nfs/pnfs.h | 9 +++ 5 files changed, 151 insertions(+), 4 deletions(-) create mode 100644 Documentation/filesystems/nfs/pnfs.txt
Documentation/filesystems/nfs/00-INDEX +2 -0
@@ -12,6 +12,8 @@ nfs-rdma.txt - how to install and setup the Linux NFS/RDMA client and server software nfsroot.txt - short guide on setting up a diskless box with NFS root filesystem. +pnfs.txt + - short explanation of some of the internals of the pnfs client code rpc-cache.txt - introduction to the caching mechanisms in the sunrpc layer. idmapper.txt
Documentation/filesystems/nfs/pnfs.txt new file +48 -0
48 lines not on this page · raw message
fs/nfs/Kconfig +6 -2
@@ -75,13 +75,17 @@ config NFS_V4 config NFS_V4_1 bool "NFS client support for NFSv4.1 (EXPERIMENTAL)" - depends on NFS_V4 && EXPERIMENTAL + depends on NFS_FS && NFS_V4 && EXPERIMENTAL + select PNFS_FILE_LAYOUT help This option enables support for minor version 1 of the NFSv4 protocol - (draft-ietf-nfsv4-minorversion1) in the kernel's NFS client. + (RFC 5661) in the kernel's NFS client. If unsure, say N. +config PNFS_FILE_LAYOUT + tristate + config ROOT_NFS bool "Root file system on NFS" depends on NFS_FS=y && IP_PNP
fs/nfs/pnfs.c +86 -2
112 lines not on this page · raw message
fs/nfs/pnfs.h +9 -0
@@ -36,8 +36,17 @@/* Per-layout driver specific registration structure */ struct pnfs_layoutdriver_type { +struct list_head pnfs_tblid; +const u32 id; +const char *name; +struct module *owner; +int (*initialize_mountpoint) (struct nfs_server *); +int (*uninitialize_mountpoint) (struct nfs_server *); }; +extern int pnfs_register_layoutdriver (struct pnfs_layoutdriver_type *); +extern void pnfs_unregister_layoutdriver (struct pnfs_layoutdriver_type *); +void set_pnfs_layoutdriver (struct nfs_server *,u32 id);void unset_pnfs_layoutdriver (struct nfs_server *);
Signature · 1 line
1.7.2.1
PATCH 8/12 NFSv4.1: pnfs: filelayout: introduce minimal file layout driver
In reply to Fred IsamanFrom: Dean Hildebrand <[email protected]>
This driver just registers itself and supplies trivial mount/umount functions.
--- fs/nfs/Makefile | 3 ++ fs/nfs/nfs4filelayout.c | 78 +++++++++++++++++++++++++++++++++++++++++++++++ include/linux/nfs_fs.h | 1 + 3 files changed, 82 insertions(+), 0 deletions(-) create mode 100644 fs/nfs/nfs4filelayout.c
fs/nfs/Makefile +3 -0
@@ -18,3 +18,6 @@ nfs-$(CONFIG_NFS_V4) += nfs4proc.o nfs4xdr.o nfs4state.o nfs4renewd.o \ nfs-$(CONFIG_NFS_V4_1) += pnfs.o nfs-$(CONFIG_SYSCTL) += sysctl.o nfs-$(CONFIG_NFS_FSCACHE) += fscache.o fscache-index.o + +obj-$(CONFIG_PNFS_FILE_LAYOUT) += nfs_layout_nfsv41_files.o +nfs_layout_nfsv41_files-y := nfs4filelayout.o
fs/nfs/nfs4filelayout.c new file +78 -0
78 lines not on this page · raw message
include/linux/nfs_fs.h +1 -0
@@ -616,6 +616,7 @@ nfs_fileid_to_ino_t(u64 fileid)#define NFSDBG_MOUNT 0x0400#define NFSDBG_FSCACHE 0x0800#define NFSDBG_PNFS 0x1000 +#define NFSDBG_PNFS_LD 0x2000#define NFSDBG_ALL 0xFFFF#ifdef __KERNEL__
Signature · 1 line
1.7.2.1
PATCH 9/12 NFS: create and destroy inode's layout cache
In reply to Fred IsamanFrom: Benny Halevy <[email protected]>
At the start of the io paths, try to grab the relevant layout information. This will initiate the inode's layout cache, but stubs ensure the cache stays empty.
--- fs/nfs/file.c | 5 ++ fs/nfs/inode.c | 3 + fs/nfs/pnfs.c | 140 ++++++++++++++++++++++++++++++++++++++++++++++++ fs/nfs/pnfs.h | 39 +++++++++++++ fs/nfs/read.c | 3 + include/linux/nfs_fs.h | 3 + 6 files changed, 193 insertions(+), 0 deletions(-)
fs/nfs/file.c +5 -0
@@ -36,6 +36,7 @@#include "internal.h" #include "iostat.h" #include "fscache.h" +#include "pnfs.h" #define NFSDBG_FACILITY NFSDBG_FILE
@@ -386,6 +387,10 @@ static int nfs_write_begin(struct file *file, struct address_space *mapping, file->f_path.dentry->d_name.name, mapping->host->i_ino, len, (long long) pos); +pnfs_update_layout (mapping->host, +nfs_file_open_context (file), +IOMODE_RW ); + start:/* * Prevent starvation issues if someone is doing a consistency
fs/nfs/inode.c +3 -0
@@ -48,6 +48,7 @@#include "internal.h" #include "fscache.h" #include "dns_resolve.h" +#include "pnfs.h" #define NFSDBG_FACILITY NFSDBG_VFS
@@ -1414,6 +1415,7 @@ void nfs4_evict_inode(struct inode *inode) {truncate_inode_pages (&inode->i_data,0 );end_writeback (inode); +pnfs_destroy_layout (NFS_I (inode));/* If we are holding a delegation, return it! */ nfs_inode_return_delegation_noreclaim (inode);/* First call standard NFS clear_inode() code */
@@ -1451,6 +1453,7 @@ static inline void nfs4_init_once(struct nfs_inode *nfsi) nfsi->delegation =NULL ; nfsi->delegation_state =0 ;init_rwsem (&nfsi->rwsem); + nfsi->layout =NULL ;#endif }
fs/nfs/pnfs.c +140 -0
143 lines not on this page · raw message
fs/nfs/pnfs.h +39 -0
@@ -34,6 +34,11 @@#define LAYOUT_NFSV4_1_MODULE_PREFIX "nfs-layouttype4" +enum { +NFS_LAYOUT_RO_FAILED =0 ,/* get ro layout failed stop trying */ +NFS_LAYOUT_RW_FAILED ,/* get rw layout failed stop trying */ +}; +/* Per-layout driver specific registration structure */ struct pnfs_layoutdriver_type {struct list_head pnfs_tblid;
@@ -44,14 +49,48 @@ struct pnfs_layoutdriver_type {int (*uninitialize_mountpoint) (struct nfs_server *); }; +struct pnfs_layout_hdr { +unsigned long refcount; +unsigned long state; +struct inode *inode; +}; +extern int pnfs_register_layoutdriver (struct pnfs_layoutdriver_type *);extern void pnfs_unregister_layoutdriver (struct pnfs_layoutdriver_type *); +struct pnfs_layout_segment * +pnfs_update_layout (struct inode *ino,struct nfs_open_context *ctx, +enum pnfs_iomode access_type);void set_pnfs_layoutdriver (struct nfs_server *,u32 id);void unset_pnfs_layoutdriver (struct nfs_server *); +void pnfs_destroy_layout (struct nfs_inode *); + + +static inline int lo_fail_bit (u32 iomode) +{ +return iomode ==IOMODE_RW ? +NFS_LAYOUT_RW_FAILED :NFS_LAYOUT_RO_FAILED ; +} + +/* Return true if a layout driver is being used for this mountpoint */ +static inline int pnfs_enabled_sb (struct nfs_server *nfss) +{ +return nfss->pnfs_curr_ld !=NULL ; +}#else /* CONFIG_NFS_V4_1 */ +static inline void pnfs_destroy_layout (struct nfs_inode *nfsi) +{ +} + +static inline struct pnfs_layout_segment * +pnfs_update_layout (struct inode *ino,struct nfs_open_context *ctx, +enum pnfs_iomode access_type) +{ +return NULL ; +} +static inline void set_pnfs_layoutdriver (struct nfs_server *s,u32 id) { }
fs/nfs/read.c +3 -0
@@ -25,6 +25,7 @@#include "internal.h" #include "iostat.h" #include "fscache.h" +#include "pnfs.h" #define NFSDBG_FACILITY NFSDBG_PAGECACHE
@@ -120,6 +121,7 @@ int nfs_readpage_async(struct nfs_open_context *ctx, struct inode *inode, len =nfs_page_length (page);if (len ==0 )return nfs_return_empty_page (page); +pnfs_update_layout (inode, ctx,IOMODE_READ ); new =nfs_create_request (ctx, inode, page,0 , len);if (IS_ERR (new)) {unlock_page (page);
@@ -624,6 +626,7 @@ int nfs_readpages(struct file *filp, struct address_space *mapping,if (ret ==0 ) goto read_complete;/* all pages were read */ +pnfs_update_layout (inode, desc.ctx,IOMODE_READ );if (rsize <PAGE_CACHE_SIZE )nfs_pageio_init (&pgio, inode, nfs_pagein_multi, rsize,0 );else
include/linux/nfs_fs.h +3 -0
@@ -188,6 +188,9 @@ struct nfs_inode {struct nfs_delegation *delegation;fmode_t delegation_state;struct rw_semaphore rwsem; + +/* pNFS layout information */ +struct pnfs_layout_hdr *layout;#endif /* CONFIG_NFS_V4*/ #ifdef CONFIG_NFS_FSCACHE struct fscache_cookie *fscache;
Signature · 1 line
1.7.2.1
PATCH 10/12 NFS: client needs to maintain list of inodes with active layouts
In reply to Fred IsamanFrom: Andy Adamson <[email protected]>
In particular, server reboot will invalidate all layouts.
Note that in order to have an active layout, we must get a successful response from the server. To avoid adding that machinery, this patch just includes a stub that fakes up a successful return. Since the layout is never referenced for io, this is not a problem.
--- fs/nfs/client.c | 4 +- fs/nfs/nfs4state.c | 2 + fs/nfs/pnfs.c | 160 ++++++++++++++++++++++++++++++++++++++++++++- fs/nfs/pnfs.h | 14 ++++ include/linux/nfs_fs_sb.h | 1 + 5 files changed, 177 insertions(+), 4 deletions(-)
fs/nfs/client.c +3 -1
@@ -156,7 +156,9 @@ static struct nfs_client *nfs_alloc_client(const struct nfs_client_initdata *cl_ cred =rpc_lookup_machine_cred ();if (!IS_ERR (cred)) clp->cl_machine_cred = cred; - +#if defined(CONFIG_NFS_V4_1 ) +INIT_LIST_HEAD (&clp->cl_layouts); +#endif nfs_fscache_get_client_cookie (clp);return clp;
fs/nfs/nfs4state.c +2 -0
@@ -53,6 +53,7 @@#include "callback.h" #include "delegation.h" #include "internal.h" +#include "pnfs.h" #define OPENOWNER_POOL_SIZE 8
@@ -1453,6 +1454,7 @@ static void nfs4_state_manager(struct nfs_client *clp) }clear_bit (NFS4CLNT_CHECK_LEASE , &clp->cl_state);set_bit (NFS4CLNT_RECLAIM_REBOOT , &clp->cl_state); +pnfs_destroy_all_layouts (clp); }if (test_and_clear_bit (NFS4CLNT_CHECK_LEASE , &clp->cl_state)) {
fs/nfs/pnfs.c +157 -3
207 lines not on this page · raw message
fs/nfs/pnfs.h +14 -0
@@ -30,6 +30,13 @@#ifndef FS_NFS_PNFS_H #define FS_NFS_PNFS_H +struct pnfs_layout_segment { +struct list_head fi_list; +u32 iomode; +struct kref kref; +struct pnfs_layout_hdr *layout; +}; +#ifdef CONFIG_NFS_V4_1 #define LAYOUT_NFSV4_1_MODULE_PREFIX "nfs-layouttype4"
@@ -51,6 +58,8 @@ struct pnfs_layoutdriver_type {struct pnfs_layout_hdr {unsigned long refcount; +struct list_head layouts;/* other client layouts */ +struct list_head segs;/* layout segments list */ unsigned long state;struct inode *inode; };
@@ -64,6 +73,7 @@ pnfs_update_layout(struct inode *ino, struct nfs_open_context *ctx,void set_pnfs_layoutdriver (struct nfs_server *,u32 id);void unset_pnfs_layoutdriver (struct nfs_server *);void pnfs_destroy_layout (struct nfs_inode *); +void pnfs_destroy_all_layouts (struct nfs_client *);static inline int lo_fail_bit (u32 iomode)
@@ -80,6 +90,10 @@ static inline int pnfs_enabled_sb(struct nfs_server *nfss)#else /* CONFIG_NFS_V4_1 */ +static inline void pnfs_destroy_all_layouts (struct nfs_client *clp) +{ +} +static inline void pnfs_destroy_layout (struct nfs_inode *nfsi) { }
include/linux/nfs_fs_sb.h +1 -0
@@ -82,6 +82,7 @@ struct nfs_client {/* The flags used for obtaining the clientid during EXCHANGE_ID */ u32 cl_exchange_flags;struct nfs4_session *cl_session;/* sharred session */ +struct list_head cl_layouts;#endif /* CONFIG_NFS_V4_1 */ #ifdef CONFIG_NFS_FSCACHE
Signature · 1 line
1.7.2.1
PATCH 11/12 NFSv4.1: pnfs: add LAYOUTGET and GETDEVICEINFO infrastructure
In reply to Fred IsamanFrom: Andy Adamson <[email protected]>
Add the ability to actually send LAYOUTGET and GETDEVICEINFO. This also adds in the machinery to handle layout state and the deviceid cache. Note that GETDEVICEINFO is not called directly by the generic layer. Instead it is called by the drivers while parsing the LAYOUTGET opaque data in response to an unknown device id embedded therein. RFC 5661 only encodes device ids within the driver-specific opaque data.
--- fs/nfs/nfs4proc.c | 142 +++++++++++++++++ fs/nfs/nfs4xdr.c | 302 +++++++++++++++++++++++++++++++++++ fs/nfs/pnfs.c | 385 +++++++++++++++++++++++++++++++++++++++++---- fs/nfs/pnfs.h | 73 +++++++++- include/linux/nfs4.h | 2 + include/linux/nfs_fs_sb.h | 1 + include/linux/nfs_xdr.h | 49 ++++++ 7 files changed, 921 insertions(+), 33 deletions(-)
fs/nfs/nfs4proc.c +142 -0
@@ -55,6 +55,7 @@#include "internal.h" #include "iostat.h" #include "callback.h" +#include "pnfs.h" #define NFSDBG_FACILITY NFSDBG_PROC
@@ -5245,6 +5246,147 @@ out:dprintk ("<-- %s status=%d\n" , __func__, status);return status; } + +static void +nfs4_layoutget_prepare (struct rpc_task *task,void *calldata) +{ +struct nfs4_layoutget *lgp = calldata; +struct inode *ino = lgp->args.inode; +struct nfs_server *server =NFS_SERVER (ino); + +dprintk ("--> %s\n" , __func__); +if (nfs4_setup_sequence (server, &lgp->args.seq_args, + &lgp->res.seq_res,0 , task)) +return ; +rpc_call_start (task); +} + +static void nfs4_layoutget_done (struct rpc_task *task,void *calldata) +{ +struct nfs4_layoutget *lgp = calldata; +struct nfs_server *server =NFS_SERVER (lgp->args.inode); + +dprintk ("--> %s\n" , __func__); + +if (!nfs4_sequence_done (task, &lgp->res.seq_res)) +return ; + +switch (task->tk_status) { +case 0 : +break ; +case -NFS4ERR_LAYOUTTRYLATER : +case -NFS4ERR_RECALLCONFLICT : + task->tk_status = -NFS4ERR_DELAY ; +/* Fall through */ +default : +if (nfs4_async_handle_error (task, server,NULL ) == -EAGAIN ) { +rpc_restart_call_prepare (task); +return ; + } + } + lgp->status = task->tk_status; +dprintk ("<-- %s\n" , __func__); +} + +static void nfs4_layoutget_release (void *calldata) +{ +struct nfs4_layoutget *lgp = calldata; + +dprintk ("--> %s\n" , __func__); +put_layout_hdr (lgp->args.inode); +if (lgp->res.layout.buf !=NULL ) +free_page ((unsigned long ) lgp->res.layout.buf); +put_nfs_open_context (lgp->args.ctx); +kfree (calldata); +dprintk ("<-- %s\n" , __func__); +} + +static const struct rpc_call_ops nfs4_layoutget_call_ops = { + .rpc_call_prepare = nfs4_layoutget_prepare, + .rpc_call_done = nfs4_layoutget_done, + .rpc_release = nfs4_layoutget_release, +}; + +int nfs4_proc_layoutget (struct nfs4_layoutget *lgp) +{ +struct nfs_server *server =NFS_SERVER (lgp->args.inode); +struct rpc_task *task; +struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_LAYOUTGET ], + .rpc_argp = &lgp->args, + .rpc_resp = &lgp->res, + }; +struct rpc_task_setup task_setup_data = { + .rpc_client = server->client, + .rpc_message = &msg, + .callback_ops = &nfs4_layoutget_call_ops, + .callback_data = lgp, + .flags =RPC_TASK_ASYNC , + }; +int status =0 ; + +dprintk ("--> %s\n" , __func__); + + lgp->res.layout.buf = (void *)__get_free_page (GFP_NOFS ); +if (lgp->res.layout.buf ==NULL ) { +nfs4_layoutget_release (lgp); +return -ENOMEM ; + } + + lgp->res.seq_res.sr_slot =NULL ; + task =rpc_run_task (&task_setup_data); +if (IS_ERR (task)) +return PTR_ERR (task); + status =nfs4_wait_for_completion_rpc_task (task); +if (status !=0 ) + goto out; + status = lgp->status; +if (status !=0 ) + goto out; + status =pnfs_layout_process (lgp); +out: +rpc_put_task (task); +dprintk ("<-- %s status=%d\n" , __func__, status); +return status; +} + +static int +_nfs4_proc_getdeviceinfo (struct nfs_server *server,struct pnfs_device *pdev) +{ +struct nfs4_getdeviceinfo_args args = { + .pdev = pdev, + }; +struct nfs4_getdeviceinfo_res res = { + .pdev = pdev, + }; +struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_GETDEVICEINFO ], + .rpc_argp = &args, + .rpc_resp = &res, + }; +int status; + +dprintk ("--> %s\n" , __func__); + status =nfs4_call_sync (server, &msg, &args, &res,0 ); +dprintk ("<-- %s status=%d\n" , __func__, status); + +return status; +} + +int nfs4_proc_getdeviceinfo (struct nfs_server *server,struct pnfs_device *pdev) +{ +struct nfs4_exception exception = { }; +int err; + +do { + err =nfs4_handle_exception (server, +_nfs4_proc_getdeviceinfo (server, pdev), + &exception); + }while (exception.retry); +return err; +} +EXPORT_SYMBOL_GPL (nfs4_proc_getdeviceinfo); +#endif /* CONFIG_NFS_V4_1 */ struct nfs4_state_recovery_ops nfs40_reboot_recovery_ops = {
fs/nfs/nfs4xdr.c +302 -0
350 lines not on this page · raw message
fs/nfs/pnfs.c +353 -32
471 lines not on this page · raw message
fs/nfs/pnfs.h +72 -1
111 lines not on this page · raw message
include/linux/nfs4.h +2 -0
@@ -545,6 +545,8 @@ enum {NFSPROC4_CLNT_SEQUENCE ,NFSPROC4_CLNT_GET_LEASE_TIME ,NFSPROC4_CLNT_RECLAIM_COMPLETE , +NFSPROC4_CLNT_LAYOUTGET , + NFSPROC4_CLNT_GETDEVICEINFO, };/* nfs41 types */
include/linux/nfs_fs_sb.h +1 -0
@@ -83,6 +83,7 @@ struct nfs_client {u32 cl_exchange_flags;struct nfs4_session *cl_session;/* sharred session */ struct list_head cl_layouts; +struct pnfs_deviceid_cache *cl_devid_cache;/* pNFS deviceid cache */ #endif /* CONFIG_NFS_V4_1 */ #ifdef CONFIG_NFS_FSCACHE
include/linux/nfs_xdr.h +49 -0
55 lines not on this page · raw message
Signature · 1 line
1.7.2.1
PATCH 12/12 NFSv4.1: pnfs: filelayout: add driver's LAYOUTGET and GETDEVICEINFO infrastructure
In reply to Fred IsamanFrom: Andy Adamson <[email protected]>
Implement the driver's io_ops->alloc_lseg and free_lseg functions, which integrate into the deviceid cache and calls out to nfs4_proc_getdeviceinfo when necessary.
--- fs/nfs/Makefile | 2 +- fs/nfs/client.c | 1 + fs/nfs/nfs4filelayout.c | 204 ++++++++++++++++++++- fs/nfs/nfs4filelayout.h | 94 +++++++++ fs/nfs/nfs4filelayoutdev.c | 448 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 747 insertions(+), 2 deletions(-) create mode 100644 fs/nfs/nfs4filelayout.h create mode 100644 fs/nfs/nfs4filelayoutdev.c
fs/nfs/Makefile +1 -1
@@ -20,4 +20,4 @@ nfs-$(CONFIG_SYSCTL) += sysctl.o nfs-$(CONFIG_NFS_FSCACHE) += fscache.o fscache-index.o obj-$(CONFIG_PNFS_FILE_LAYOUT) += nfs_layout_nfsv41_files.o -nfs_layout_nfsv41_files-y := nfs4filelayout.o +nfs_layout_nfsv41_files-y := nfs4filelayout.o nfs4filelayoutdev.o
fs/nfs/client.c +1 -0
@@ -255,6 +255,7 @@ void nfs_put_client(struct nfs_client *clp)nfs_free_client (clp); } } +EXPORT_SYMBOL_GPL (nfs_put_client);#if defined(CONFIG_IPV6 ) || defined(CONFIG_IPV6_MODULE )/*
fs/nfs/nfs4filelayout.c +203 -1
233 lines not on this page · raw message
fs/nfs/nfs4filelayout.h new file +94 -0
94 lines not on this page · raw message
fs/nfs/nfs4filelayoutdev.c new file +448 -0
448 lines not on this page · raw message
Signature · 1 line
1.7.2.1
Quoted 21 lines · Fred Isaman
This is try 5 of the first wave of the linux pnfs client submission. The only change (besides rebase on top of Trond's most recent nfs-for-2.6.37 branch) between this series and the previous is that owner/sign-off issues have been addressed, which allows us to remove the RFC label of the previous series.
For those interested, the patches themselves can also be found at
git://linux-nfs.org/~iisaman/linux-pnfs.git,
branch devinfo-submit-rc7-trondThis is the start of code implementing pnfs, based on RFC 5661. Since sending the whole thing at once would be overwhelming, we are trying to break it into bite sized waves. This first wave implements the mount/umount infrastructure, as well as sending the LAYOUTGET and GETDEVTICEINFO calls on io (but not actually using the information for io). Note that two major simplifications to the protocol will be made throughout the initial submission process: only the file layout driver is considered, and only whole file layouts are requested.
Quoted 4 lines
These patches apply against Trond's nfs-for-2.6.37 branch.
patches 01-08 implement the mount/umount hooks patches 09-12 implement LAYOUTGET and GETDEVICEINFO
OK. I merged these patches for testing in linux-next last night. If all goes well, I'll send them to Linus in this merge window.
Trond