Skip to contentSkip to conversation

NFSv4.1 Use MDS auth flavor for data server connection

PATCH In review [email protected] 3 participants 3 messages Sep 5, 2013

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 NFSv4.1 Use MDS auth flavor for data server connection

From: Andy Adamson <[email protected]>

Commit 4edaa308 "NFS: Use "krb5i" to establish NFSv4 state whenever possible" uses the nfs_client cl_rpcclient for all state management operations, and will use krb5i or auth_sys with no regard to the mount command authflavor choice.

The MDS, as any NFSv4.1 mount point, uses the nfs_server rpc client for all non-state management operations with a different nfs_server for each fsid encountered traversing the mount point, each with a potentially different auth flavor.

pNFS data servers are not mounted in the normal sense as there is no associated nfs_server structure. Data servers can also export multiple fsids, each with a potentially different auth flavor.

Data servers need to use the same authflavor as the MDS server rpc client for non-state management operations. Populate a list of rpc clients with the MDS server rpc client auth flavor for the DS to use.

---
 fs/nfs/internal.h           |  2 ++
 fs/nfs/nfs4client.c         | 75 +++++++++++++++++++++++++++++++++++++++++++++
 fs/nfs/nfs4filelayout.c     | 35 ++++++++++++++++-----
 include/linux/nfs_fs_sb.h   |  1 +
 include/linux/sunrpc/clnt.h |  2 ++
 net/sunrpc/clnt.c           |  1 +
 6 files changed, 108 insertions(+), 8 deletions(-)
fs/nfs/internal.h +2 -0
index 2415198..23ec6e8 100644
@@ -186,6 +186,8 @@ extern struct nfs_client *nfs4_set_ds_client(struct nfs_client* mds_clp,
 					     int ds_addrlen, int ds_proto,
 					     unsigned int ds_timeo,
 					     unsigned int ds_retrans);
+extern struct rpc_clnt *nfs4_find_or_create_ds_client(struct nfs_client *,
+						struct inode *);
 #ifdef CONFIG_PROC_FS
 extern int __init nfs_fs_proc_init(void);
 extern void nfs_fs_proc_exit(void);
fs/nfs/nfs4client.c +75 -0

97 lines not on this page · raw message

fs/nfs/nfs4filelayout.c +27 -8
index a70cb3a..b86464b 100644
@@ -528,6 +528,7 @@ filelayout_read_pagelist(struct nfs_read_data *data)
 	struct nfs_pgio_header *hdr = data->header;
 	struct pnfs_layout_segment *lseg = hdr->lseg;
 	struct nfs4_pnfs_ds *ds;
+	struct rpc_clnt *ds_clnt;
 	loff_t offset = data->args.offset;
 	u32 j, idx;
 	struct nfs_fh *fh;
@@ -542,6 +543,11 @@ filelayout_read_pagelist(struct nfs_read_data *data)
 	ds = nfs4_fl_prepare_ds(lseg, idx);
 	if (!ds)
 		return PNFS_NOT_ATTEMPTED;
+
+	ds_clnt = nfs4_find_or_create_ds_client(ds->ds_clp, hdr->inode);
+	if (IS_ERR(ds_clnt))
+		return PNFS_NOT_ATTEMPTED;
+
 	dprintk("%s USE DS: %s cl_count %d\n", __func__,
 		ds->ds_remotestr, atomic_read(&ds->ds_clp->cl_count));
 
@@ -556,7 +562,7 @@ filelayout_read_pagelist(struct nfs_read_data *data)
 	data->mds_offset = offset;
 
 	/* Perform an asynchronous read to ds */
-	nfs_initiate_read(ds->ds_clp->cl_rpcclient, data,
+	nfs_initiate_read(ds_clnt, data,
 				  &filelayout_read_call_ops, RPC_TASK_SOFTCONN);
 	return PNFS_ATTEMPTED;
 }
@@ -568,6 +574,7 @@ filelayout_write_pagelist(struct nfs_write_data *data, int sync)
 	struct nfs_pgio_header *hdr = data->header;
 	struct pnfs_layout_segment *lseg = hdr->lseg;
 	struct nfs4_pnfs_ds *ds;
+	struct rpc_clnt *ds_clnt;
 	loff_t offset = data->args.offset;
 	u32 j, idx;
 	struct nfs_fh *fh;
@@ -578,6 +585,11 @@ filelayout_write_pagelist(struct nfs_write_data *data, int sync)
 	ds = nfs4_fl_prepare_ds(lseg, idx);
 	if (!ds)
 		return PNFS_NOT_ATTEMPTED;
+
+	ds_clnt = nfs4_find_or_create_ds_client(ds->ds_clp, hdr->inode);
+	if (IS_ERR(ds_clnt))
+		return PNFS_NOT_ATTEMPTED;
+
 	dprintk("%s ino %lu sync %d req %Zu@%llu DS: %s cl_count %d\n",
 		__func__, hdr->inode->i_ino, sync, (size_t) data->args.count,
 		offset, ds->ds_remotestr, atomic_read(&ds->ds_clp->cl_count));
@@ -595,7 +607,7 @@ filelayout_write_pagelist(struct nfs_write_data *data, int sync)
 	data->args.offset = filelayout_get_dserver_offset(lseg, offset);
 
 	/* Perform an asynchronous write */
-	nfs_initiate_write(ds->ds_clp->cl_rpcclient, data,
+	nfs_initiate_write(ds_clnt, data,
 				    &filelayout_write_call_ops, sync,
 				    RPC_TASK_SOFTCONN);
 	return PNFS_ATTEMPTED;
@@ -1105,16 +1117,19 @@ static int filelayout_initiate_commit(struct nfs_commit_data *data, int how)
 {
 	struct pnfs_layout_segment *lseg = data->lseg;
 	struct nfs4_pnfs_ds *ds;
+	struct rpc_clnt *ds_clnt;
 	u32 idx;
 	struct nfs_fh *fh;
 
 	idx = calc_ds_index_from_commit(lseg, data->ds_commit_index);
 	ds = nfs4_fl_prepare_ds(lseg, idx);
-	if (!ds) {
-		prepare_to_resend_writes(data);
-		filelayout_commit_release(data);
-		return -EAGAIN;
-	}
+	if (!ds)
+		goto out_err;
+
+	ds_clnt = nfs4_find_or_create_ds_client(ds->ds_clp, data->inode);
+	if (IS_ERR(ds_clnt))
+		goto out_err;
+
 	dprintk("%s ino %lu, how %d cl_count %d\n", __func__,
 		data->inode->i_ino, how, atomic_read(&ds->ds_clp->cl_count));
 	data->commit_done_cb = filelayout_commit_done_cb;
@@ -1123,9 +1138,13 @@ static int filelayout_initiate_commit(struct nfs_commit_data *data, int how)
 	fh = select_ds_fh_from_commit(lseg, data->ds_commit_index);
 	if (fh)
 		data->args.fh = fh;
-	return nfs_initiate_commit(ds->ds_clp->cl_rpcclient, data,
+	return nfs_initiate_commit(ds_clnt, data,
 				   &filelayout_commit_call_ops, how,
 				   RPC_TASK_SOFTCONN);
+out_err:
+	prepare_to_resend_writes(data);
+	filelayout_commit_release(data);
+	return -EAGAIN;
 }
 
 static int
include/linux/nfs_fs_sb.h +1 -0
index d221243..4d476e7 100644
@@ -56,6 +56,7 @@ struct nfs_client {
 	struct rpc_cred		*cl_machine_cred;
 
 #if IS_ENABLED(CONFIG_NFS_V4)
+	struct list_head	cl_ds_clients; /* auth flavor data servers */
 	u64			cl_clientid;	/* constant */
 	nfs4_verifier		cl_confirm;	/* Clientid verifier */
 	unsigned long		cl_state;
include/linux/sunrpc/clnt.h +2 -0
index 76c0bf6..3919f2b 100644
@@ -35,6 +35,8 @@ struct rpc_clnt {
 	atomic_t		cl_count;	/* Number of references */
 	struct list_head	cl_clients;	/* Global list of clients */
 	struct list_head	cl_tasks;	/* List of tasks */
+	struct list_head	cl_ds_clnts;	/* list of per auth flavor
+						   data servers */
 	spinlock_t		cl_lock;	/* spinlock */
 	struct rpc_xprt __rcu *	cl_xprt;	/* transport */
 	struct rpc_procinfo *	cl_procinfo;	/* procedure info */
net/sunrpc/clnt.c +1 -0
index baf0df6..3c18394 100644
@@ -356,6 +356,7 @@ static struct rpc_clnt * rpc_new_client(const struct rpc_create_args *args,
 		goto out_no_stats;
 	clnt->cl_program  = program;
 	INIT_LIST_HEAD(&clnt->cl_tasks);
+	INIT_LIST_HEAD(&clnt->cl_ds_clnts);
 	spin_lock_init(&clnt->cl_lock);
 
 	if (!xprt_bound(xprt))
Signature · 1 line
1.8.3.1
#1 lore raw

T24gVGh1LCAyMDEzLTA5LTA1IGF0IDExOjE0IC0wNDAwLCBhbmRyb3NAbmV0YXBwLmNvbSB3cm90 ZToNCj4gRnJvbTogQW5keSBBZGFtc29uIDxhbmRyb3NAbmV0YXBwLmNvbT4NCj4gDQo+IENvbW1p dCA0ZWRhYTMwOCAiTkZTOiBVc2UgImtyYjVpIiB0byBlc3RhYmxpc2ggTkZTdjQgc3RhdGUgd2hl bmV2ZXIgcG9zc2libGUiDQo+IHVzZXMgdGhlIG5mc19jbGllbnQgY2xfcnBjY2xpZW50IGZvciBh bGwgc3RhdGUgbWFuYWdlbWVudCBvcGVyYXRpb25zLCBhbmQNCj4gd2lsbCB1c2Uga3JiNWkgb3Ig YXV0aF9zeXMgd2l0aCBubyByZWdhcmQgdG8gdGhlIG1vdW50IGNvbW1hbmQgYXV0aGZsYXZvcg0K PiBjaG9pY2UuDQo+IA0KPiBUaGUgTURTLCBhcyBhbnkgTkZTdjQuMSBtb3VudCBwb2ludCwgdXNl cyB0aGUgbmZzX3NlcnZlciBycGMgY2xpZW50IGZvciBhbGwNCj4gbm9uLXN0YXRlIG1hbmFnZW1l bnQgb3BlcmF0aW9ucyB3aXRoIGEgZGlmZmVyZW50IG5mc19zZXJ2ZXIgZm9yIGVhY2ggZnNpZA0K PiBlbmNvdW50ZXJlZCB0cmF2ZXJzaW5nIHRoZSBtb3VudCBwb2ludCwgZWFjaCB3aXRoIGEgcG90 ZW50aWFsbHkgZGlmZmVyZW50DQo+IGF1dGggZmxhdm9yLg0KPiANCj4gcE5GUyBkYXRhIHNlcnZl cnMgYXJlIG5vdCBtb3VudGVkIGluIHRoZSBub3JtYWwgc2Vuc2UgYXMgdGhlcmUgaXMgbm8gYXNz b2NpYXRlZA0KPiBuZnNfc2VydmVyIHN0cnVjdHVyZS4gRGF0YSBzZXJ2ZXJzIGNhbiBhbHNvIGV4 cG9ydCBtdWx0aXBsZSBmc2lkcywgZWFjaCB3aXRoDQo+IGEgcG90ZW50aWFsbHkgZGlmZmVyZW50 IGF1dGggZmxhdm9yLg0KPiANCj4gRGF0YSBzZXJ2ZXJzIG5lZWQgdG8gdXNlIHRoZSBzYW1lIGF1 dGhmbGF2b3IgYXMgdGhlIE1EUyBzZXJ2ZXIgcnBjIGNsaWVudCBmb3INCj4gbm9uLXN0YXRlIG1h bmFnZW1lbnQgb3BlcmF0aW9ucy4gUG9wdWxhdGUgYSBsaXN0IG9mIHJwYyBjbGllbnRzIHdpdGgg dGhlIE1EUw0KPiBzZXJ2ZXIgcnBjIGNsaWVudCBhdXRoIGZsYXZvciBmb3IgdGhlIERTIHRvIHVz ZS4NCj4gDQo+IFNpZ25lZC1vZmYtYnk6IEFuZHkgQWRhbXNvbiA8YW5kcm9zQG5ldGFwcC5jb20+ DQo+IC0tLQ0KPiAgZnMvbmZzL2ludGVybmFsLmggICAgICAgICAgIHwgIDIgKysNCj4gIGZzL25m cy9uZnM0Y2xpZW50LmMgICAgICAgICB8IDc1ICsrKysrKysrKysrKysrKysrKysrKysrKysrKysr KysrKysrKysrKysrKysrKw0KPiAgZnMvbmZzL25mczRmaWxlbGF5b3V0LmMgICAgIHwgMzUgKysr KysrKysrKysrKysrKy0tLS0tDQo+ICBpbmNsdWRlL2xpbnV4L25mc19mc19zYi5oICAgfCAgMSAr DQo+ICBpbmNsdWRlL2xpbnV4L3N1bnJwYy9jbG50LmggfCAgMiArKw0KPiAgbmV0L3N1bnJwYy9j bG50LmMgICAgICAgICAgIHwgIDEgKw0KPiAgNiBmaWxlcyBjaGFuZ2VkLCAxMDggaW5zZXJ0aW9u cygrKSwgOCBkZWxldGlvbnMoLSkNCj4gDQo8c25pcD4NCj4gZGlmZiAtLWdpdCBhL2luY2x1ZGUv bGludXgvc3VucnBjL2NsbnQuaCBiL2luY2x1ZGUvbGludXgvc3VucnBjL2NsbnQuaA0KPiBpbmRl eCA3NmMwYmY2Li4zOTE5ZjJiIDEwMDY0NA0KPiAtLS0gYS9pbmNsdWRlL2xpbnV4L3N1bnJwYy9j bG50LmgNCj4gKysrIGIvaW5jbHVkZS9saW51eC9zdW5ycGMvY2xudC5oDQo+IEBAIC0zNSw2ICsz NSw4IEBAIHN0cnVjdCBycGNfY2xudCB7DQo+ICAJYXRvbWljX3QJCWNsX2NvdW50OwkvKiBOdW1i ZXIgb2YgcmVmZXJlbmNlcyAqLw0KPiAgCXN0cnVjdCBsaXN0X2hlYWQJY2xfY2xpZW50czsJLyog R2xvYmFsIGxpc3Qgb2YgY2xpZW50cyAqLw0KPiAgCXN0cnVjdCBsaXN0X2hlYWQJY2xfdGFza3M7 CS8qIExpc3Qgb2YgdGFza3MgKi8NCj4gKwlzdHJ1Y3QgbGlzdF9oZWFkCWNsX2RzX2NsbnRzOwkv KiBsaXN0IG9mIHBlciBhdXRoIGZsYXZvcg0KPiArCQkJCQkJICAgZGF0YSBzZXJ2ZXJzICovDQoN ClRoaXMgbGlzdCBkb2Vzbid0IGJlbG9uZyBpbiB0aGUgUlBDIGxheWVyLiBQbGVhc2UgbWFrZSBh IHNlcGFyYXRlDQpzdHJ1Y3R1cmUgdGhhdCBsaXZlcyBpbiB0aGUgTkZTIGxheWVyLCBhbmQgdGhh dCBvd25zIHRoZSBycGNfY2xudC4NCg0KSU9XIHNvbWV0aGluZyBhbG9uZyB0aGUgbGluZXMgb2Y6 DQoNCnN0cnVjdCBuZnNfZHNfc2VydmVyIHsNCglzdHJ1Y3QgbGlzdF9oZWFkIGxpc3Q7CS8qIGRz X2NscC0+Y2xfZHNfY2xpZW50cyAqLw0KCXN0cnVjdCBycGNfY2xudCAqcnBjX2NsbnQ7DQp9Ow0K DQo+ICAJc3BpbmxvY2tfdAkJY2xfbG9jazsJLyogc3BpbmxvY2sgKi8NCj4gIAlzdHJ1Y3QgcnBj X3hwcnQgX19yY3UgKgljbF94cHJ0OwkvKiB0cmFuc3BvcnQgKi8NCj4gIAlzdHJ1Y3QgcnBjX3By b2NpbmZvICoJY2xfcHJvY2luZm87CS8qIHByb2NlZHVyZSBpbmZvICovDQo+IGRpZmYgLS1naXQg YS9uZXQvc3VucnBjL2NsbnQuYyBiL25ldC9zdW5ycGMvY2xudC5jDQo+IGluZGV4IGJhZjBkZjYu LjNjMTgzOTQgMTAwNjQ0DQo+IC0tLSBhL25ldC9zdW5ycGMvY2xudC5jDQo+ICsrKyBiL25ldC9z dW5ycGMvY2xudC5jDQo+IEBAIC0zNTYsNiArMzU2LDcgQEAgc3RhdGljIHN0cnVjdCBycGNfY2xu dCAqIHJwY19uZXdfY2xpZW50KGNvbnN0IHN0cnVjdCBycGNfY3JlYXRlX2FyZ3MgKmFyZ3MsDQo+ ICAJCWdvdG8gb3V0X25vX3N0YXRzOw0KPiAgCWNsbnQtPmNsX3Byb2dyYW0gID0gcHJvZ3JhbTsN Cj4gIAlJTklUX0xJU1RfSEVBRCgmY2xudC0+Y2xfdGFza3MpOw0KPiArCUlOSVRfTElTVF9IRUFE KCZjbG50LT5jbF9kc19jbG50cyk7DQo+ICAJc3Bpbl9sb2NrX2luaXQoJmNsbnQtPmNsX2xvY2sp Ow0KPiAgDQo+ICAJaWYgKCF4cHJ0X2JvdW5kKHhwcnQpKQ0KDQotLSANClRyb25kIE15a2xlYnVz dA0KTGludXggTkZTIGNsaWVudCBtYWludGFpbmVyDQoNCk5ldEFwcA0KVHJvbmQuTXlrbGVidXN0 QG5ldGFwcC5jb20NCnd3dy5uZXRhcHAuY29tDQo=

#2 lore raw
Quoted 46 lines · Trond
Quoted 30 lines · On Thu, 2013-09-05 at 11:14 -0400, [email protected] wrote:

From: Andy Adamson <[email protected]>

Commit 4edaa308 "NFS: Use "krb5i" to establish NFSv4 state whenever possible" uses the nfs_client cl_rpcclient for all state management operations, and will use krb5i or auth_sys with no regard to the mount command authflavor choice.

The MDS, as any NFSv4.1 mount point, uses the nfs_server rpc client for all non-state management operations with a different nfs_server for each fsid encountered traversing the mount point, each with a potentially different auth flavor.

pNFS data servers are not mounted in the normal sense as there is no associated nfs_server structure. Data servers can also export multiple fsids, each with a potentially different auth flavor.

Data servers need to use the same authflavor as the MDS server rpc client for non-state management operations. Populate a list of rpc clients with the MDS server rpc client auth flavor for the DS to use.

---
 fs/nfs/internal.h           |  2 ++
 fs/nfs/nfs4client.c         | 75 +++++++++++++++++++++++++++++++++++++++++++++
 fs/nfs/nfs4filelayout.c     | 35 ++++++++++++++++-----
 include/linux/nfs_fs_sb.h   |  1 +
 include/linux/sunrpc/clnt.h |  2 ++
 net/sunrpc/clnt.c           |  1 +
 6 files changed, 108 insertions(+), 8 deletions(-)

<snip>

include/linux/sunrpc/clnt.h struct rpc_clnt { · line 39

Quoted 10 lines
include/linux/sunrpc/clnt.h +2 -0
index 76c0bf6..3919f2b 100644
@@ -35,6 +35,8 @@ struct rpc_clnt {
      atomic_t                cl_count;       /* Number of references */
      struct list_head        cl_clients;     /* Global list of clients */
      struct list_head        cl_tasks;       /* List of tasks */
+     struct list_head        cl_ds_clnts;    /* list of per auth flavor
+                                                data servers */

This list doesn't belong in the RPC layer. Please make a separate structure that lives in the NFS layer, and that owns the rpc_clnt.

Quoted 6 lines

IOW something along the lines of:

struct nfs_ds_server {
        struct list_head list;  /* ds_clp->cl_ds_clients */
        struct rpc_clnt *rpc_clnt;
};

OK

-->Andy

Quoted 24 lines
Quoted 9 lines
include/linux/sunrpc/clnt.h +0 -0
      spinlock_t              cl_lock;        /* spinlock */
      struct rpc_xprt __rcu * cl_xprt;        /* transport */
      struct rpc_procinfo *   cl_procinfo;    /* procedure info */
net/sunrpc/clnt.c +0 -0
index baf0df6..3c18394 100644

net/sunrpc/clnt.c static struct rpc_clnt * rpc_new_client(const struct rpc_create_args *args, · line 362

Quoted 7 lines
net/sunrpc/clnt.c +1 -0
@@ -356,6 +356,7 @@ static struct rpc_clnt * rpc_new_client(const struct rpc_create_args *args,
              goto out_no_stats;
      clnt->cl_program  = program;
      INIT_LIST_HEAD(&clnt->cl_tasks);
+     INIT_LIST_HEAD(&clnt->cl_ds_clnts);
      spin_lock_init(&clnt->cl_lock);
 
      if (!xprt_bound(xprt))

--
Trond Myklebust
Linux NFS client maintainer

NetApp
[email protected]
www.netapp.com