Skip to contentSkip to conversation

SUNRPC: destroy gss_cred and context on Kerberos credential destruction

PATCH In review [email protected] 2 participants 7 messages Oct 22, 2013 – Oct 28, 2013

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

cover SUNRPC: destroy gss_cred and context on Kerberos credential destruction

From: Andy Adamson <[email protected]>

This patchset requires the nfs-utils "GSSD: Use gss-ctx keys and gsskeyd to sync Kerber os credentials and kernel gss_contexts" patchset which triggers gss-ctx key creation on kinit and destruction on kdestroy.

This is an RFC patch set.

We need to do a lot of testing to ensure that
once kdestroy and gss-ctx gss_user_destroy is called, all existing buffered writes using the 'destroyed gss credential + context' are serviced.

Differences from version1

- Replaced request_key with keyring_search
- Used flag based method to service buffered writes after kdestroy instead of calling sys_sync.
- Use of the session keyring (KEY_SPEC_USER_SESSION_KEYRING) instead of the user keyring.

Version 1 of these patches were presented along with the GSS expiry patch set.

A new key type, gss-ctx is registered by the auth_gss module. If a gss-ctx key is created at kinit (via gsskeyd), the key serial is saved in the resultant gss_cred. When kdestroy triggers the gss-ctx key destruction (via gsskeyd), all gss_contexts matching the key-serial are found and the gss_cred->base (rpc_cred) cr_flags RPCAUTH_CRED_KEY_DESTROYED bit is set.

The new RPC_TASK_BUF_WRITE task flag is set in the NFS layer in nfs_initiate_write, the buffered write path. Pass the flag to the crmatch routine via the rpcauth_bindcred lookup_flags.

In gss_match, if the RPCAUTH_CRED_KEY_DESTROYED rpc_cred bit is set and if the RPC_TASK_BUF_WRITE flag is set, return found - e.g. use the DESTROYED gss_cred for the buffered writes. Return not-found for all other calls. The RPC_CRED_KEY_EXPIRE_SOON acred flag is also set which tells the NFS layer to discontinue any new buffered writes, turning to sync writes instead. Note that these sync writes will fail as gss_match will find but not return the DESTROYED gss_cred.

We want to pay close attention to the number of upcalls this code triggers as a result of the destroyed Kerberos credentials, and destroyed gss_cred/gss_context.

Andy Adamson (5):
SUNRPC: register the gss-ctx key type
SUNRPC set gss_cred gss-ctx key serial
SUNRPC: invalidate gss_context upon gss-ctx keyring key destruction
SUNRPC: allow only existing buffered writes on creds with destroyed
keys
SUNRPC add EKEYEXPIRED case to call_bind_status

 fs/nfs/write.c                  |   2 +-
 include/linux/sunrpc/auth.h     |   1 +
 include/linux/sunrpc/auth_gss.h |   1 +
 include/linux/sunrpc/sched.h    |   1 +
 net/sunrpc/auth.c               |   2 +
 net/sunrpc/auth_gss/auth_gss.c  | 170 +++++++++++++++++++++++++++++++++++++++-
 net/sunrpc/clnt.c               |   5 ++
 7 files changed, 178 insertions(+), 4 deletions(-)
Signature · 1 line
1.8.3.1
#1 lore raw

PATCH 1/5 SUNRPC: register the gss-ctx key type

From: Andy Adamson <[email protected]>

The gss-ctx key is used for gss_credential and gss_context management to sync kernel gss_cred and gss_context lifetimes with Kerberos credential lifetimes. It will be created upon kinit and destroyed upon kdestroy.

---
 net/sunrpc/auth_gss/auth_gss.c | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)
net/sunrpc/auth_gss/auth_gss.c +30 -0
index 0846566..ae42a83 100644
@@ -52,6 +52,8 @@
 #include <linux/sunrpc/gss_api.h>
 #include <asm/uaccess.h>
 #include <linux/hashtable.h>
+#include <linux/key-type.h>
+#include <keys/user-type.h>
 
 #include "../netns.h"
 
@@ -113,6 +115,32 @@ static void gss_free_ctx(struct gss_cl_ctx *);
 static const struct rpc_pipe_ops gss_upcall_ops_v0;
 static const struct rpc_pipe_ops gss_upcall_ops_v1;
 
+static struct key_type key_type_gss_ctx = {
+	.name		= "gss-ctx",
+	.instantiate	= user_instantiate,
+	.match		= user_match,
+	.revoke		= user_revoke,
+	.destroy	= user_destroy,
+	.describe	= user_describe,
+	.read		= user_read,
+};
+
+/* Register the gss-ctx key type for use by nfslogin and nfslogout */
+static int gss_register_ctx_keytype(void)
+{
+	int ret;
+
+	ret = register_key_type(&key_type_gss_ctx);
+	pr_notice("NFS: Registering the %s key type ret %d\n",
+		key_type_gss_ctx.name, ret);
+	return ret;
+}
+
+static void gss_unregister_ctx_keytype(void)
+{
+	unregister_key_type(&key_type_gss_ctx);
+}
+
 static inline struct gss_cl_ctx *
 gss_get_ctx(struct gss_cl_ctx *ctx)
 {
@@ -1959,6 +1987,7 @@ static int __init init_rpcsec_gss(void)
 	if (err)
 		goto out_svc_exit;
 	rpc_init_wait_queue(&pipe_version_rpc_waitqueue, "gss pipe version");
+	gss_register_ctx_keytype();
 	return 0;
 out_svc_exit:
 	gss_svc_shutdown();
@@ -1972,6 +2001,7 @@ static void __exit exit_rpcsec_gss(void)
 {
 	unregister_pernet_subsys(&rpcsec_gss_net_ops);
 	gss_svc_shutdown();
+	gss_unregister_ctx_keytype();
 	rpcauth_unregister(&authgss_ops);
 	rcu_barrier(); /* Wait for completion of call_rcu()'s */
 }
Signature · 1 line
1.8.3.1
#2 lore raw

PATCH 2/5 SUNRPC set gss_cred gss-ctx key serial

In reply to [email protected]

From: Andy Adamson <[email protected]>

the gc_serial will be used to find gss_cred's associated with the gss-ctx key which in turn is associated with the user kerberos credentials.

Add the ccname to the gssd upcall. Gssd uses this name to locate the principals' Kerbeos credential cache which helps gssd to stop rooting around the file system.

---
 include/linux/sunrpc/auth_gss.h |  1 +
 net/sunrpc/auth_gss/auth_gss.c  | 91 +++++++++++++++++++++++++++++++++++++++--
 2 files changed, 89 insertions(+), 3 deletions(-)
include/linux/sunrpc/auth_gss.h +1 -0
index f1cfd4c..4b20b53 100644
@@ -79,6 +79,7 @@ struct gss_cl_ctx {
 struct gss_upcall_msg;
 struct gss_cred {
 	struct rpc_cred		gc_base;
+	key_serial_t		gc_serial;
 	enum rpc_gss_svc	gc_service;
 	struct gss_cl_ctx __rcu	*gc_ctx;
 	struct gss_upcall_msg	*gc_upcall;
net/sunrpc/auth_gss/auth_gss.c +88 -3

152 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#3 lore raw

PATCH 3/5 SUNRPC: invalidate gss_context upon gss-ctx keyring key destruction

In reply to [email protected]

From: Andy Adamson <[email protected]>

Lookup all gss_contexts matching the key-serial and set the gss_cred->base (rpc_cred) cr_flags RPCAUTH_CRED_KEY_DESTROYED bit.

In gss_match, which is called prior to any use of the gss_cred, if the RPCAUTH_CRED_KEY_DESTROYED rpc_cred bit is set, return no match.

A future patch will make an exception, returning a match for any buffered writes setup before the RPCAUTH_CRED_KEY_DESTROYED flag was set.

When crmatch fails, the rpc code will then try to create a new gss_cred + context, which will fail due to destroyed kerberos credentials.

Note: Currently we leave the RPCAUTH_CRED_KEY_DESTROYED cred in the unused lru list to be garbage collected.

---
 include/linux/sunrpc/auth.h    |  1 +
 net/sunrpc/auth_gss/auth_gss.c | 45 +++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 45 insertions(+), 1 deletion(-)
include/linux/sunrpc/auth.h +1 -0
index 790be14..f1151e3 100644
@@ -68,6 +68,7 @@ struct rpc_cred {
 #define RPCAUTH_CRED_UPTODATE	1
 #define RPCAUTH_CRED_HASHED	2
 #define RPCAUTH_CRED_NEGATIVE	3
+#define RPCAUTH_CRED_KEY_DESTROYED 4
 
 #define RPCAUTH_CRED_MAGIC	0x0f4aa4f0
 
net/sunrpc/auth_gss/auth_gss.c +44 -1

62 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#4 lore raw

Re: [PATCH Version 2 3/5] SUNRPC: invalidate gss_context upon gss-ctx keyring key destruction

Quoted 72 lines · On Tue, 2013-10-22 at 10:21 -0400, [email protected] wrote:

From: Andy Adamson <[email protected]>

Lookup all gss_contexts matching the key-serial and set the gss_cred->base (rpc_cred) cr_flags RPCAUTH_CRED_KEY_DESTROYED bit.

In gss_match, which is called prior to any use of the gss_cred, if the RPCAUTH_CRED_KEY_DESTROYED rpc_cred bit is set, return no match.

A future patch will make an exception, returning a match for any buffered writes setup before the RPCAUTH_CRED_KEY_DESTROYED flag was set.

When crmatch fails, the rpc code will then try to create a new gss_cred +- context, which will fail due to destroyed kerberos credentials.

Note: Currently we leave the RPCAUTH_CRED_KEY_DESTROYED cred in the unused lru list to be garbage collected.

---
 include/linux/sunrpc/auth.h    |  1 +-
 net/sunrpc/auth_gss/auth_gss.c | 45 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+--
 2 files changed, 45 insertions(+-), 1 deletion(-)
include/linux/sunrpc/auth.h +1 -1
index 790be14..f1151e3 100644
--- a/include/linux/sunrpc/auth.h
+-+-+- b/include/linux/sunrpc/auth.h

@@ -68,6 +-68,7 @@ struct rpc_cred {
#define RPCAUTH_CRED_UPTODATE 1
#define RPCAUTH_CRED_HASHED 2
#define RPCAUTH_CRED_NEGATIVE 3
+-#define RPCAUTH_CRED_KEY_DESTROYED 4

#define RPCAUTH_CRED_MAGIC 0x0f4aa4f0

net/sunrpc/auth_gss/auth_gss.c +1 -1
index 10d6e53..b7365b9 100644
--- a/net/sunrpc/auth_gss/auth_gss.c
+-+-+- b/net/sunrpc/auth_gss/auth_gss.c

@@ -115,12 +-115,52 @@ static void gss_free_ctx(struct gss_cl_ctx *);
static const struct rpc_pipe_ops gss_upcall_ops_v0; static const struct rpc_pipe_ops gss_upcall_ops_v1;

+-/**
+- * The UID Kerberos credential has been destroyed. Search all gss_auth
+- * credential caches and mark all UID gss_creds RPCAUTH_CRED_KEY_DESTROYED.
+- */
+-static void
+-gss_mark_cred_destroy(uid_t uid, key_serial_t serial)
+-{
+-	struct gss_auth *ga;
+-	struct rpc_cred *cr;
+-	struct gss_cred *gc;
+-	struct auth_cred ac = {
+-		.uid = uid,
+-	};
+-	int i;
+-
+-	spin_lock(&gss_auth_hash_lock);
+-	hash_for_each(gss_auth_hash_table, i, ga, hash) {
+-		/* check all supported pseudoflavors */
+-		if (ga->rpc_auth.au_flavor > RPC_AUTH_MAXFLAVOR) {
+-			cr = rpcauth_lookup_credcache(&ga->rpc_auth, &ac, 0);
+-			if (IS_ERR(cr) || cr == NULL)
+-				continue;
+-			gc = container_of(cr, struct gss_cred, gc_base);
+-			if (gc->gc_serial == serial) {
+-				set_bit(RPCAUTH_CRED_KEY_DESTROYED,
+-					&cr->cr_flags);
+-			}
+-			put_rpccred(cr); /* balance get in lookup credcache */
+-		}
Quoted 8 lines
+-	}
+-	spin_unlock(&gss_auth_hash_lock);
+-}
+-
+-static void
+-gss_user_destroy(struct key *key)
+-{
+-	gss_mark_cred_destroy(key->uid, key->serial);

This won't compile when CONFIG_USER_NS=y. key->uid is of type kuid_t, and not uid_t...

Signature · 6 lines
Trond Myklebust
Linux NFS client maintainer

NetApp
[email protected]
www.netapp.com
#5 lore raw

PATCH 4/5 SUNRPC: allow only existing buffered writes on creds with destroyed keys

In reply to [email protected]

From: Andy Adamson <[email protected]>

Set the new RPC_TASK_BUF_WRITE task flag in the NFS layer in initiate_write, the buffered write path. Pass the flag to the crmatch routine via the rpcauth_bindcred lookup_flags.

This flag is only used when RPCAUTH_CRED_KEY_DESTROYED
is set on the gss_cred, and allows for buffered writes to use a "destroyed" gss_context.

If the RPCAUTH_CRED_KEY_DESTROYED rpc_cred bit is set, set the RPC_CRED_KEY_EXPIRE_SOON acred bit which tells the NFS layer to discontinue any new buffered writes, turning to sync writes instead. The sync writes will fail as gss_match will find but not return the DESTROYED gss_cred.

---
 fs/nfs/write.c                 |  2 +-
 include/linux/sunrpc/sched.h   |  1 +
 net/sunrpc/auth.c              |  2 ++
 net/sunrpc/auth_gss/auth_gss.c | 10 ++++++++--
 4 files changed, 12 insertions(+), 3 deletions(-)
fs/nfs/write.c +1 -1
index ac1dc33..40c3e8e 100644
@@ -1006,7 +1006,7 @@ int nfs_initiate_write(struct rpc_clnt *clnt,
 		.callback_ops = call_ops,
 		.callback_data = data,
 		.workqueue = nfsiod_workqueue,
-		.flags = RPC_TASK_ASYNC | flags,
+		.flags = RPC_TASK_ASYNC | RPC_TASK_BUF_WRITE | flags,
 		.priority = priority,
 	};
 	int ret = 0;
include/linux/sunrpc/sched.h +1 -0
index 3a847de..8ea50a7 100644
@@ -123,6 +123,7 @@ struct rpc_task_setup {
 #define RPC_TASK_TIMEOUT	0x1000		/* fail with ETIMEDOUT on timeout */
 #define RPC_TASK_NOCONNECT	0x2000		/* return ENOTCONN if not connected */
 #define RPC_TASK_NO_RETRANS_TIMEOUT	0x4000		/* wait forever for a reply */
+#define RPC_TASK_BUF_WRITE	0x8000		/* allow buffered writes with destroyed credential key */
 
 #define RPC_IS_ASYNC(t)		((t)->tk_flags & RPC_TASK_ASYNC)
 #define RPC_IS_SWAPPER(t)	((t)->tk_flags & RPC_TASK_SWAPPER)
net/sunrpc/auth.c +2 -0
index 5285ead..646d38f 100644
@@ -650,6 +650,8 @@ rpcauth_bindcred(struct rpc_task *task, struct rpc_cred *cred, int flags)
 	struct rpc_cred *new;
 	int lookupflags = 0;
 
+	if (flags & RPC_TASK_BUF_WRITE)
+		lookupflags |= RPC_TASK_BUF_WRITE;
 	if (flags & RPC_TASK_ASYNC)
 		lookupflags |= RPCAUTH_LOOKUP_NEW;
 	if (cred != NULL)
net/sunrpc/auth_gss/auth_gss.c +8 -2
index b7365b9..31c72c5 100644
@@ -1511,8 +1511,14 @@ check_expire:
 	if (ret == 0)
 		return ret;
 
-	if (test_bit(RPCAUTH_CRED_KEY_DESTROYED, &rc->cr_flags))
-		return 0;
+	/* Has kdestroy -> gss-ctx key destroy occurred? */
+	if (test_bit(RPCAUTH_CRED_KEY_DESTROYED, &rc->cr_flags)) {
+		/* tell NFS layer no more buffered writes */
+		set_bit(RPC_CRED_KEY_EXPIRE_SOON, &acred->ac_flags);
+                /* if this is not a buffered write, return not found */
+		if (!(flags & RPC_TASK_BUF_WRITE))
+			return 0;
+	}
 
 	/* Notify acred users of GSS context expiration timeout */
 	if (test_bit(RPC_CRED_NOTIFY_TIMEOUT, &acred->ac_flags) &&
Signature · 1 line
1.8.3.1
#6 lore raw

PATCH 5/5 SUNRPC add EKEYEXPIRED case to call_bind_status

In reply to [email protected]

From: Andy Adamson <[email protected]>

---
 net/sunrpc/clnt.c | 5 +++++
 1 file changed, 5 insertions(+)
net/sunrpc/clnt.c +5 -0
index 7352aef..8a5c5b6 100644
@@ -1622,6 +1622,11 @@ call_bind_status(struct rpc_task *task)
 		dprintk("RPC: %5u unrecognized remote rpcbind service\n",
 				task->tk_pid);
 		break;
+	case -EKEYEXPIRED:
+		/* credential key has expired */
+		dprintk("RPC: %5u Credential key has expired\n",
+				task->tk_pid);
+		break;
 	case -EPROTONOSUPPORT:
 		dprintk("RPC: %5u remote rpcbind version unavailable, retrying\n",
 				task->tk_pid);
Signature · 1 line
1.8.3.1