nfsidmap: use multiple child keyrings

The kernel keyring has a max of ~508 entries on 64-bit systems. For installations with more distict users than this limit, create a specified number of child keyrings and fill them evenly.

---
 utils/nfsidmap/nfsidmap.c   |   76 +++++++++++++++++++++++++++++++++---------
 utils/nfsidmap/nfsidmap.man |    5 ++-
 2 files changed, 63 insertions(+), 18 deletions(-)

utils/nfsidmap/nfsidmap.c +1 -1

index 2518ed6..41abede 100644
@@ -15,7 +15,7 @@
 #include "conffile.h"
 
 int verbose = 0;
-char *usage="Usage: %s [-v] [-c || [-u|-g|-r key] || [-t timeout] key desc]";
+char *usage="Usage: %s [-v] [-c || [-u|-g|-r key] || [-t timeout] [-n count] key desc]";
 
 #define MAX_ID_LEN   11
 #define IDMAP_NAMESZ 128

utils/nfsidmap/nfsidmap.c +57 -15

 #endif
 
 #ifndef PATH_IDMAPDCONF

@@ -39,7 +39,7 @@ static int keyring_clear(char *keyring);
 /*
  * Find either a user or group id based on the name@domain string
  */
-int id_lookup(char *name_at_domain, key_serial_t key, int type)
+int id_lookup(char *name_at_domain, key_serial_t key, int type, key_serial_t dest_keyring)
 {
 	char id[MAX_ID_LEN];
 	uid_t uid = 0;

@@ -58,7 +58,7 @@ int id_lookup(char *name_at_domain, key_serial_t key, int type)
 			(type == USER ? "nfs4_owner_to_uid" : "nfs4_group_owner_to_gid"));
 
 	if (rc == 0) {
-		rc = keyctl_instantiate(key, id, strlen(id) + 1, 0);
+		rc = keyctl_instantiate(key, id, strlen(id) + 1, dest_keyring);
 		if (rc < 0) {
 			switch(rc) {
 			case -EDQUOT:

@@ -67,9 +67,9 @@ int id_lookup(char *name_at_domain, key_serial_t key, int type)
 				/*
 			 	 * The keyring is full. Clear the keyring and try again
 			 	 */
-				rc = keyring_clear(DEFAULT_KEYRING);
+				rc = keyctl_clear(dest_keyring);
 				if (rc == 0)
-					rc = keyctl_instantiate(key, id, strlen(id) + 1, 0);
+					rc = keyctl_instantiate(key, id, strlen(id) + 1, dest_keyring);
 				break;
 			default:
 				break;

@@ -85,7 +85,7 @@ int id_lookup(char *name_at_domain, key_serial_t key, int type)
 /*
  * Find the name@domain string from either a user or group id
  */
-int name_lookup(char *id, key_serial_t key, int type)
+int name_lookup(char *id, key_serial_t key, int type, key_serial_t dest_keyring)
 {
 	char name[IDMAP_NAMESZ];
 	char domain[NFS4_MAX_DOMAIN_LEN];

@@ -113,7 +113,7 @@ int name_lookup(char *id, key_serial_t key, int type)
 			(type == USER ? "nfs4_uid_to_name" : "nfs4_gid_to_name"));
 
 	if (rc == 0) {
-		rc = keyctl_instantiate(key, &name, strlen(name), 0);
+		rc = keyctl_instantiate(key, &name, strlen(name), dest_keyring);
 		if (rc < 0)
 			xlog_err("name_lookup: keyctl_instantiate failed: %m");
 	}

@@ -127,7 +127,7 @@ static int keyring_clear(char *keyring)
 {
 	FILE *fp;
 	char buf[BUFSIZ];
-	key_serial_t key;
+	key_serial_t key, child_key;
 
 	if (keyring == NULL)
 		keyring = DEFAULT_KEYRING;

@@ -151,6 +151,7 @@ static int keyring_clear(char *keyring)
 		 */
 		*(strchr(buf, ' ')) = '\0';
 		sscanf(buf, "%x", &key);
+
 		if (keyctl_clear(key) < 0) {
 			xlog_err("keyctl_clear(0x%x) failed: %m", key);
 			fclose(fp);

@@ -159,7 +160,8 @@ static int keyring_clear(char *keyring)
 		fclose(fp);
 		return 0;
 	}
-	xlog_err("'%s' keyring was not found.", keyring);
+	if (strstr(keyring, DEFAULT_KEYRING":"))
+		xlog_err("'%s' keyring was not found.", keyring);
 	fclose(fp);
 	return 1;
 }

@@ -232,8 +234,10 @@ int main(int argc, char **argv)
 	char *type;
 	int rc = 1, opt;
 	int timeout = 600;
-	key_serial_t key;
+	int childrings = 0;
+	key_serial_t key, parent_keyring, dest_keyring;
 	char *progname, *keystr = NULL;
+	char child_name[BUFSIZ];
 	int clearing = 0, keymask = 0;
 
 	/* Set the basename */

@@ -244,7 +248,7 @@ int main(int argc, char **argv)
 
 	xlog_open(progname);
 
-	while ((opt = getopt(argc, argv, "u:g:r:ct:v")) != -1) {
+	while ((opt = getopt(argc, argv, "u:g:r:ct:vn:d:")) != -1) {
 		switch (opt) {
 		case 'u':
 			keymask = UIDKEYS;

@@ -267,6 +271,9 @@ int main(int argc, char **argv)
 		case 't':
 			timeout = atoi(optarg);
 			break;
+		case 'n':
+			childrings = atoi(optarg);
+			break;
 		default:
 			xlog_warn(usage, progname);
 			break;

@@ -284,9 +291,16 @@ int main(int argc, char **argv)
 		rc = key_revoke(keystr, keymask);
 		return rc;		
 	}
+
 	if (clearing) {
 		xlog_syslog(0);
-		rc = keyring_clear(DEFAULT_KEYRING);
+		int i = 1;
+		for(rc = 0; rc == 0; i++) {
+			snprintf(child_name, sizeof(child_name), DEFAULT_KEYRING "_child_%d", i);
+			rc = keyring_clear(child_name);
+		}
+
+		rc = keyring_clear(DEFAULT_KEYRING ":");
 		return rc;		
 	}
 

@@ -315,14 +329,42 @@ int main(int argc, char **argv)
 			key, type, value, timeout);
 	}
 
+	if (childrings) {
+		int i;
+		long child_size, smallest_size = 2032;
+		parent_keyring = request_key("keyring", DEFAULT_KEYRING, NULL, KEY_SPEC_THREAD_KEYRING);
+
+		for (i = 1; i <= childrings; i++) {
+			key_serial_t child_keyring;
+
+			snprintf(child_name, sizeof(child_name), DEFAULT_KEYRING "_child_%d", i);
+
+			child_keyring = keyctl_search(parent_keyring, "keyring", child_name, 0);
+			if (child_keyring < 0) {
+				child_keyring = add_key("keyring", child_name, NULL, 0, parent_keyring);
+				xlog_warn("added new child %s: %m", child_name);
+			}
+
+			child_size = keyctl_read(child_keyring, NULL, 0);
+			if (child_size < smallest_size) {
+				dest_keyring = child_keyring;
+				smallest_size = child_size;
+			}
+		}
+	}
+
 	if (strcmp(type, "uid") == 0)
-		rc = id_lookup(value, key, USER);
+		rc = id_lookup(value, key, USER, dest_keyring);
 	else if (strcmp(type, "gid") == 0)
-		rc = id_lookup(value, key, GROUP);
+		rc = id_lookup(value, key, GROUP, dest_keyring);
 	else if (strcmp(type, "user") == 0)
-		rc = name_lookup(value, key, USER);
+		rc = name_lookup(value, key, USER, dest_keyring);
 	else if (strcmp(type, "group") == 0)
-		rc = name_lookup(value, key, GROUP);
+		rc = name_lookup(value, key, GROUP, dest_keyring);
+
+	/* if we hung this off a child, unlink from the parent */
+	if (dest_keyring)
+		keyctl_unlink(key, parent_keyring);
 
 	/* Set timeout to 10 (600 seconds) minutes */
 	if (rc == 0)

utils/nfsidmap/nfsidmap.man +4 -1

index 3a3a523..04f0014 100644
@@ -6,7 +6,7 @@
 .SH NAME
 nfsidmap \- The NFS idmapper upcall program
 .SH SYNOPSIS
-.B "nfsidmap [-v] [-t timeout] key desc"
+.B "nfsidmap [-v] [-t timeout] [-n count] key desc"
 .br
 .B "nfsidmap [-v] [-c]"
 .br

@@ -42,6 +42,9 @@ Revoke both the uid and gid key of the given user.
 Set the expiration timer, in seconds, on the key.
 The default is 600 seconds (10 mins).
 .TP
+.B -n count
+Set the the number of child keyrings to create.
+.TP
 .B -u user
 Revoke the uid key of the given user.
 .TP

Quoted 3 lines · Benjamin Coddington

The kernel keyring has a max of ~508 entries on 64-bit systems. For installations with more distict users than this limit, create a specified number of child keyrings and fill them evenly.

A couple things...

1) no Signed-off-by: line

2) Its seems you can create key rings but can't delete them. Here is what I'm doing:
in /etc/request-key.d/id_resolver.conf I have
create id_resolver * * /usr/sbin/nfsidmap -n 10 %k %d
but when I tried to delete the keys

   # nfsidmap -vc
   nfsidmap: clearing '08aa156c I--Q---     1perm 3f010000     0     0 keyring   .id_resolver_child_10: empty'
   nfsidmap: keyctl_clear(0x8aa156c) failed: Permission denied

utils/nfsidmap/nfsidmap.c char *usage="Usage: %s [-v] [-c || [-u|-g|-r key] || [-t timeout] key desc]"; · line 27

Quoted 4 lines

#define PROCKEYS "/proc/keys"
#ifndef DEFAULT_KEYRING

utils/nfsidmap/nfsidmap.c +1 -1

-#define DEFAULT_KEYRING "id_resolver"
+#define DEFAULT_KEYRING ".id_resolver"

3) Why is changing the default needed?

Finally, what -n value do plan on using? Maybe a blurb in the man page on what a good number is and why....

steved.

Quoted 189 lines

#endif

#ifndef PATH_IDMAPDCONF

utils/nfsidmap/nfsidmap.c +57 -15

@@ -39,7 +39,7 @@ static int keyring_clear(char *keyring);
 /*
 * Find either a user or group id based on the name@domain string
 */
-int id_lookup(char *name_at_domain, key_serial_t key, int type)
+int id_lookup(char *name_at_domain, key_serial_t key, int type, key_serial_t dest_keyring)
 {
 	char id[MAX_ID_LEN];
 	uid_t uid = 0;

@@ -58,7 +58,7 @@ int id_lookup(char *name_at_domain, key_serial_t key, int type)
 			(type == USER ? "nfs4_owner_to_uid" : "nfs4_group_owner_to_gid"));
 
 	if (rc == 0) {
-		rc = keyctl_instantiate(key, id, strlen(id) + 1, 0);
+		rc = keyctl_instantiate(key, id, strlen(id) + 1, dest_keyring);
 		if (rc < 0) {
 			switch(rc) {
 			case -EDQUOT:

@@ -67,9 +67,9 @@ int id_lookup(char *name_at_domain, key_serial_t key, int type)
 				/*
 			 	 * The keyring is full. Clear the keyring and try again
 			 	 */
-				rc = keyring_clear(DEFAULT_KEYRING);
+				rc = keyctl_clear(dest_keyring);
 				if (rc == 0)
-					rc = keyctl_instantiate(key, id, strlen(id) + 1, 0);
+					rc = keyctl_instantiate(key, id, strlen(id) + 1, dest_keyring);
 				break;
 			default:
 				break;

@@ -85,7 +85,7 @@ int id_lookup(char *name_at_domain, key_serial_t key, int type)
 /*
 * Find the name@domain string from either a user or group id
 */
-int name_lookup(char *id, key_serial_t key, int type)
+int name_lookup(char *id, key_serial_t key, int type, key_serial_t dest_keyring)
 {
 	char name[IDMAP_NAMESZ];
 	char domain[NFS4_MAX_DOMAIN_LEN];

@@ -113,7 +113,7 @@ int name_lookup(char *id, key_serial_t key, int type)
 			(type == USER ? "nfs4_uid_to_name" : "nfs4_gid_to_name"));
 
 	if (rc == 0) {
-		rc = keyctl_instantiate(key, &name, strlen(name), 0);
+		rc = keyctl_instantiate(key, &name, strlen(name), dest_keyring);
 		if (rc < 0)
 			xlog_err("name_lookup: keyctl_instantiate failed: %m");
 	}

@@ -127,7 +127,7 @@ static int keyring_clear(char *keyring)
 {
 	FILE *fp;
 	char buf[BUFSIZ];
-	key_serial_t key;
+	key_serial_t key, child_key;
 
 	if (keyring == NULL)
 		keyring = DEFAULT_KEYRING;

@@ -151,6 +151,7 @@ static int keyring_clear(char *keyring)
 		 */
 		*(strchr(buf, ' ')) = '\0';
 		sscanf(buf, "%x", &key);
+
 		if (keyctl_clear(key) < 0) {
 			xlog_err("keyctl_clear(0x%x) failed: %m", key);
 			fclose(fp);

@@ -159,7 +160,8 @@ static int keyring_clear(char *keyring)
 		fclose(fp);
 		return 0;
 	}
-	xlog_err("'%s' keyring was not found.", keyring);
+	if (strstr(keyring, DEFAULT_KEYRING":"))
+		xlog_err("'%s' keyring was not found.", keyring);
 	fclose(fp);
 	return 1;
 }

@@ -232,8 +234,10 @@ int main(int argc, char **argv)
 	char *type;
 	int rc = 1, opt;
 	int timeout = 600;
-	key_serial_t key;
+	int childrings = 0;
+	key_serial_t key, parent_keyring, dest_keyring;
 	char *progname, *keystr = NULL;
+	char child_name[BUFSIZ];
 	int clearing = 0, keymask = 0;
 
 	/* Set the basename */

@@ -244,7 +248,7 @@ int main(int argc, char **argv)
 
 	xlog_open(progname);
 
-	while ((opt = getopt(argc, argv, "u:g:r:ct:v")) != -1) {
+	while ((opt = getopt(argc, argv, "u:g:r:ct:vn:d:")) != -1) {
 		switch (opt) {
 		case 'u':
 			keymask = UIDKEYS;

@@ -267,6 +271,9 @@ int main(int argc, char **argv)
 		case 't':
 			timeout = atoi(optarg);
 			break;
+		case 'n':
+			childrings = atoi(optarg);
+			break;
 		default:
 			xlog_warn(usage, progname);
 			break;

@@ -284,9 +291,16 @@ int main(int argc, char **argv)
 		rc = key_revoke(keystr, keymask);
 		return rc;		
 	}
+
 	if (clearing) {
 		xlog_syslog(0);
-		rc = keyring_clear(DEFAULT_KEYRING);
+		int i = 1;
+		for(rc = 0; rc == 0; i++) {
+			snprintf(child_name, sizeof(child_name), DEFAULT_KEYRING "_child_%d", i);
+			rc = keyring_clear(child_name);
+		}
+
+		rc = keyring_clear(DEFAULT_KEYRING ":");
 		return rc;		
 	}
 

@@ -315,14 +329,42 @@ int main(int argc, char **argv)
 			key, type, value, timeout);
 	}
 
+	if (childrings) {
+		int i;
+		long child_size, smallest_size = 2032;
+		parent_keyring = request_key("keyring", DEFAULT_KEYRING, NULL, KEY_SPEC_THREAD_KEYRING);
+
+		for (i = 1; i <= childrings; i++) {
+			key_serial_t child_keyring;
+
+			snprintf(child_name, sizeof(child_name), DEFAULT_KEYRING "_child_%d", i);
+
+			child_keyring = keyctl_search(parent_keyring, "keyring", child_name, 0);
+			if (child_keyring < 0) {
+				child_keyring = add_key("keyring", child_name, NULL, 0, parent_keyring);
+				xlog_warn("added new child %s: %m", child_name);
+			}
+
+			child_size = keyctl_read(child_keyring, NULL, 0);
+			if (child_size < smallest_size) {
+				dest_keyring = child_keyring;
+				smallest_size = child_size;
+			}
+		}
+	}
+
 	if (strcmp(type, "uid") == 0)
-		rc = id_lookup(value, key, USER);
+		rc = id_lookup(value, key, USER, dest_keyring);
 	else if (strcmp(type, "gid") == 0)
-		rc = id_lookup(value, key, GROUP);
+		rc = id_lookup(value, key, GROUP, dest_keyring);
 	else if (strcmp(type, "user") == 0)
-		rc = name_lookup(value, key, USER);
+		rc = name_lookup(value, key, USER, dest_keyring);
 	else if (strcmp(type, "group") == 0)
-		rc = name_lookup(value, key, GROUP);
+		rc = name_lookup(value, key, GROUP, dest_keyring);
+
+	/* if we hung this off a child, unlink from the parent */
+	if (dest_keyring)
+		keyctl_unlink(key, parent_keyring);
 
 	/* Set timeout to 10 (600 seconds) minutes */
 	if (rc == 0)

utils/nfsidmap/nfsidmap.man +0 -0

index 3a3a523..04f0014 100644
@@ -6,7 +6,7 @@

.SH NAME
nfsidmap \- The NFS idmapper upcall program .SH SYNOPSIS
-.B "nfsidmap [-v] [-t timeout] key desc" +.B "nfsidmap [-v] [-t timeout] [-n count] key desc" .br
.B "nfsidmap [-v] [-c]"
.br

utils/nfsidmap/nfsidmap.man +0 -0

@@ -42,6 +42,9 @@ Revoke both the uid and gid key of the given user.

Set the expiration timer, in seconds, on the key.

Quoted 9 lines

The default is 600 seconds (10 mins).
.TP
+.B -n count
+Set the the number of child keyrings to create.
+.TP
.B -u user
Revoke the uid key of the given user.
.TP

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Quoted 8 lines · Steve Dickson

Quoted 3 lines · Benjamin Coddington

The kernel keyring has a max of ~508 entries on 64-bit systems. For installations with more distict users than this limit, create a specified number of child keyrings and fill them evenly.

A couple things...

1) no Signed-off-by: line

Quoted 8 lines

2) Its seems you can create key rings but can't delete them. Here is what I'm doing:
in /etc/request-key.d/id_resolver.conf I have
create id_resolver * * /usr/sbin/nfsidmap -n 10 %k %d
but when I tried to delete the keys

   # nfsidmap -vc
   nfsidmap: clearing '08aa156c I--Q---     1perm 3f010000     0     0 keyring   .id_resolver_child_10: empty'
   nfsidmap: keyctl_clear(0x8aa156c) failed: Permission denied

This mess works on my fleet of RHEL6 boxes which is where I was trying to fix this. They create the child keyrings with

perm 3b3f0000

Instead of yours which appears to be

perm 3f010000

Are you testing on a later kernel? Likely this behavior has changed.

Quoted 6 lines

Review

Quoted 5 lines

#define PROCKEYS "/proc/keys"
#ifndef DEFAULT_KEYRING

(quoted diff) +1 -1

-#define DEFAULT_KEYRING "id_resolver"
+#define DEFAULT_KEYRING ".id_resolver"

3) Why is changing the default needed?

The default is wrong. I think that's the first thing I changed when trying to fix this problem, since it looked like id_lookup() should gracefully recover in the case that the keyring was full (but it still doesn't).

This actually doesn't have to change, since
the strcmp()s that use it will work either way -- but it might catch someone by surprise later. I'll split this out.

Quoted 3 lines

Finally, what -n value do plan on using? Maybe a blurb in the man page on what a good number is and why....

I've got this running now with -n160, since
we have ~60K distinct uid/gid s. Ideally, I'd like to re-submit this to self-scale which wouldn't require any sysadmin tuning, but I haven't had the time. Really, this is just a quick fix for the brokenness that's in current RHEL and less-new Fedora.

If you want it done right it will take me a week or two to find the time.

Ben

Quoted 204 lines

steved.

Quoted 189 lines

#endif

#ifndef PATH_IDMAPDCONF

(quoted diff) +57 -15

@@ -39,7 +39,7 @@ static int keyring_clear(char *keyring);
 /*
 * Find either a user or group id based on the name@domain string
 */
-int id_lookup(char *name_at_domain, key_serial_t key, int type)
+int id_lookup(char *name_at_domain, key_serial_t key, int type, key_serial_t dest_keyring)
 {
 	char id[MAX_ID_LEN];
 	uid_t uid = 0;

@@ -58,7 +58,7 @@ int id_lookup(char *name_at_domain, key_serial_t key, int type)
 			(type == USER ? "nfs4_owner_to_uid" : "nfs4_group_owner_to_gid"));
 
 	if (rc == 0) {
-		rc = keyctl_instantiate(key, id, strlen(id) + 1, 0);
+		rc = keyctl_instantiate(key, id, strlen(id) + 1, dest_keyring);
 		if (rc < 0) {
 			switch(rc) {
 			case -EDQUOT:

@@ -67,9 +67,9 @@ int id_lookup(char *name_at_domain, key_serial_t key, int type)
 				/*
 			 	 * The keyring is full. Clear the keyring and try again
 			 	 */
-				rc = keyring_clear(DEFAULT_KEYRING);
+				rc = keyctl_clear(dest_keyring);
 				if (rc == 0)
-					rc = keyctl_instantiate(key, id, strlen(id) + 1, 0);
+					rc = keyctl_instantiate(key, id, strlen(id) + 1, dest_keyring);
 				break;
 			default:
 				break;

@@ -85,7 +85,7 @@ int id_lookup(char *name_at_domain, key_serial_t key, int type)
 /*
 * Find the name@domain string from either a user or group id
 */
-int name_lookup(char *id, key_serial_t key, int type)
+int name_lookup(char *id, key_serial_t key, int type, key_serial_t dest_keyring)
 {
 	char name[IDMAP_NAMESZ];
 	char domain[NFS4_MAX_DOMAIN_LEN];

@@ -113,7 +113,7 @@ int name_lookup(char *id, key_serial_t key, int type)
 			(type == USER ? "nfs4_uid_to_name" : "nfs4_gid_to_name"));
 
 	if (rc == 0) {
-		rc = keyctl_instantiate(key, &name, strlen(name), 0);
+		rc = keyctl_instantiate(key, &name, strlen(name), dest_keyring);
 		if (rc < 0)
 			xlog_err("name_lookup: keyctl_instantiate failed: %m");
 	}

@@ -127,7 +127,7 @@ static int keyring_clear(char *keyring)
 {
 	FILE *fp;
 	char buf[BUFSIZ];
-	key_serial_t key;
+	key_serial_t key, child_key;
 
 	if (keyring == NULL)
 		keyring = DEFAULT_KEYRING;

@@ -151,6 +151,7 @@ static int keyring_clear(char *keyring)
 		 */
 		*(strchr(buf, ' ')) = '\0';
 		sscanf(buf, "%x", &key);
+
 		if (keyctl_clear(key) < 0) {
 			xlog_err("keyctl_clear(0x%x) failed: %m", key);
 			fclose(fp);

@@ -159,7 +160,8 @@ static int keyring_clear(char *keyring)
 		fclose(fp);
 		return 0;
 	}
-	xlog_err("'%s' keyring was not found.", keyring);
+	if (strstr(keyring, DEFAULT_KEYRING":"))
+		xlog_err("'%s' keyring was not found.", keyring);
 	fclose(fp);
 	return 1;
 }

@@ -232,8 +234,10 @@ int main(int argc, char **argv)
 	char *type;
 	int rc = 1, opt;
 	int timeout = 600;
-	key_serial_t key;
+	int childrings = 0;
+	key_serial_t key, parent_keyring, dest_keyring;
 	char *progname, *keystr = NULL;
+	char child_name[BUFSIZ];
 	int clearing = 0, keymask = 0;
 
 	/* Set the basename */

@@ -244,7 +248,7 @@ int main(int argc, char **argv)
 
 	xlog_open(progname);
 
-	while ((opt = getopt(argc, argv, "u:g:r:ct:v")) != -1) {
+	while ((opt = getopt(argc, argv, "u:g:r:ct:vn:d:")) != -1) {
 		switch (opt) {
 		case 'u':
 			keymask = UIDKEYS;

@@ -267,6 +271,9 @@ int main(int argc, char **argv)
 		case 't':
 			timeout = atoi(optarg);
 			break;
+		case 'n':
+			childrings = atoi(optarg);
+			break;
 		default:
 			xlog_warn(usage, progname);
 			break;

@@ -284,9 +291,16 @@ int main(int argc, char **argv)
 		rc = key_revoke(keystr, keymask);
 		return rc;		
 	}
+
 	if (clearing) {
 		xlog_syslog(0);
-		rc = keyring_clear(DEFAULT_KEYRING);
+		int i = 1;
+		for(rc = 0; rc == 0; i++) {
+			snprintf(child_name, sizeof(child_name), DEFAULT_KEYRING "_child_%d", i);
+			rc = keyring_clear(child_name);
+		}
+
+		rc = keyring_clear(DEFAULT_KEYRING ":");
 		return rc;		
 	}
 

@@ -315,14 +329,42 @@ int main(int argc, char **argv)
 			key, type, value, timeout);
 	}
 
+	if (childrings) {
+		int i;
+		long child_size, smallest_size = 2032;
+		parent_keyring = request_key("keyring", DEFAULT_KEYRING, NULL, KEY_SPEC_THREAD_KEYRING);
+
+		for (i = 1; i <= childrings; i++) {
+			key_serial_t child_keyring;
+
+			snprintf(child_name, sizeof(child_name), DEFAULT_KEYRING "_child_%d", i);
+
+			child_keyring = keyctl_search(parent_keyring, "keyring", child_name, 0);
+			if (child_keyring < 0) {
+				child_keyring = add_key("keyring", child_name, NULL, 0, parent_keyring);
+				xlog_warn("added new child %s: %m", child_name);
+			}
+
+			child_size = keyctl_read(child_keyring, NULL, 0);
+			if (child_size < smallest_size) {
+				dest_keyring = child_keyring;
+				smallest_size = child_size;
+			}
+		}
+	}
+
 	if (strcmp(type, "uid") == 0)
-		rc = id_lookup(value, key, USER);
+		rc = id_lookup(value, key, USER, dest_keyring);
 	else if (strcmp(type, "gid") == 0)
-		rc = id_lookup(value, key, GROUP);
+		rc = id_lookup(value, key, GROUP, dest_keyring);
 	else if (strcmp(type, "user") == 0)
-		rc = name_lookup(value, key, USER);
+		rc = name_lookup(value, key, USER, dest_keyring);
 	else if (strcmp(type, "group") == 0)
-		rc = name_lookup(value, key, GROUP);
+		rc = name_lookup(value, key, GROUP, dest_keyring);
+
+	/* if we hung this off a child, unlink from the parent */
+	if (dest_keyring)
+		keyctl_unlink(key, parent_keyring);
 
 	/* Set timeout to 10 (600 seconds) minutes */
 	if (rc == 0)

utils/nfsidmap/nfsidmap.man +0 -0

index 3a3a523..04f0014 100644
@@ -6,7 +6,7 @@

.SH NAME
nfsidmap \- The NFS idmapper upcall program .SH SYNOPSIS
-.B "nfsidmap [-v] [-t timeout] key desc" +.B "nfsidmap [-v] [-t timeout] [-n count] key desc" .br
.B "nfsidmap [-v] [-c]"
.br

utils/nfsidmap/nfsidmap.man +0 -0

@@ -42,6 +42,9 @@ Revoke both the uid and gid key of the given user.

Set the expiration timer, in seconds, on the key.

Quoted 9 lines

The default is 600 seconds (10 mins).
.TP
+.B -n count
+Set the the number of child keyrings to create.
+.TP
.B -u user
Revoke the uid key of the given user.
.TP

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Quoted 8 lines

Quoted 2 lines

Finally, what -n value do plan on using? Maybe a blurb in the man page on what a good number is and why....

I've got this running now with -n160, since
we have ~60K distinct uid/gid s. Ideally, I'd like to re-submit this to self-scale which wouldn't require any sysadmin tuning, but I haven't had the time. Really, this is just a quick fix for the brokenness that's in current RHEL and less-new Fedora.

The brokenness in RHEL will be healing very soon... See
https://bugzilla.redhat.com/show_bug.cgi?id=1033708. RHEL is basically going back to using rpc.idmapd on the client and nfsidmap is going away... It as just a bad dream... It never happen! ;-)

Quoted 8 lines · Steve Dickson

Quoted 8 lines

Quoted 2 lines

Finally, what -n value do plan on using? Maybe a blurb in the man page on what a good number is and why....

I've got this running now with -n160, since
we have ~60K distinct uid/gid s. Ideally, I'd like to re-submit this to self-scale which wouldn't require any sysadmin tuning, but I haven't had the time. Really, this is just a quick fix for the brokenness that's in current RHEL and less-new Fedora.

Quoted 5 lines

The brokenness in RHEL will be healing very soon... See
https://bugzilla.redhat.com/show_bug.cgi?id=1033708. RHEL is basically going back to using rpc.idmapd on the client and nfsidmap is going away... It as just a bad dream... It never happen! ;-)

This BZ says the fix is coming in nfs-utils by removing the nfsidmap command.

But IIRC, unless the kernel side of the idmapper is changed, it will exec request-key for every lookup before falling back to the upcall, and there's no cache in front of that. Isn't that going to have some performance problems?

I'm much more interested in getting the keyrings to work, since they seem to offer significant performance gains.

If RH is going to change the kernel side of the idmapper to only upcall, then I'd be satisfied to use that.

Ben

Quoted 14 lines · Benjamin Coddington

Quoted 12 lines · Steve Dickson

Quoted 7 lines

Quoted 2 lines

Finally, what -n value do plan on using? Maybe a blurb in the man page on what a good number is and why....

I've got this running now with -n160, since
we have ~60K distinct uid/gid s. Ideally, I'd like to re-submit this to self-scale which wouldn't require any sysadmin tuning, but I haven't had the time. Really, this is just a quick fix for the brokenness that's in current RHEL and less-new Fedora.

The brokenness in RHEL will be healing very soon... See
https://bugzilla.redhat.com/show_bug.cgi?id=1033708. RHEL is basically going back to using rpc.idmapd on the client and nfsidmap is going away... It as just a bad dream... It never happen! ;-)

Quoted 1 line

This BZ says the fix is coming in nfs-utils by removing the nfsidmap command.

Yeah.. I bet the RHEL police are not going to be happy about that... so I might have to put it back...

Quoted 3 lines

But IIRC, unless the kernel side of the idmapper is changed, it will exec request-key for every lookup before falling back to the upcall, and there's no cache in front of that. Isn't that going to have some performance problems?

I don't thinks so, since things will eventually get cached but I'll definitely look into it...

Quoted 3 lines

I'm much more interested in getting the keyrings to work, since they seem to offer significant performance gains.

I too, but the back port of the current key is a bit too much for RHEL at this point.

Quoted 1 line

If RH is going to change the kernel side of the idmapper to only upcall, then I'd be satisfied to use that.

It was a bone-head decision on my part to flip this type of switch midstream... I too just want to put things back to the way were,,,

steved.

Quoted 2 lines

Ben

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Quoted 22 lines · Benjamin Coddington

Quoted 8 lines · Steve Dickson

Quoted 3 lines · Benjamin Coddington

The kernel keyring has a max of ~508 entries on 64-bit systems. For installations with more distict users than this limit, create a specified number of child keyrings and fill them evenly.

A couple things...

1) no Signed-off-by: line

Quoted 8 lines

2) Its seems you can create key rings but can't delete them. Here is what I'm doing:
in /etc/request-key.d/id_resolver.conf I have
create id_resolver * * /usr/sbin/nfsidmap -n 10 %k %d
but when I tried to delete the keys

   # nfsidmap -vc
   nfsidmap: clearing '08aa156c I--Q---     1perm 3f010000     0     0 keyring   .id_resolver_child_10: empty'
   nfsidmap: keyctl_clear(0x8aa156c) failed: Permission denied

This mess works on my fleet of RHEL6 boxes which is where I was trying to fix this. They create the child keyrings with

Quoted 7 lines

perm 3b3f0000

Instead of yours which appears to be

perm 3f010000

Are you testing on a later kernel? Likely this behavior has changed.

Yes... Much later...

Quoted 11 lines

Quoted 6 lines

Review

Quoted 5 lines

#define PROCKEYS "/proc/keys"
#ifndef DEFAULT_KEYRING

(quoted diff) +1 -1

-#define DEFAULT_KEYRING "id_resolver"
+#define DEFAULT_KEYRING ".id_resolver"

3) Why is changing the default needed?

The default is wrong. I think that's the first thing I changed when trying to fix this problem, since it looked like id_lookup() should gracefully recover in the case that the keyring was full (but it still doesn't).

I'm think the "id_resolver" default can from the face the entry /etc/request-key.d/id_resolver.conf which tells nfsidmap put the keys on the id_resolver key ring... so I'm not really sure where the .id_resolver is coming from... CC-ing David Howells maybe he knows...

Quoted 45 lines · Steve Dickson

Quoted 22 lines · Benjamin Coddington

Quoted 8 lines · Steve Dickson

Quoted 3 lines · Benjamin Coddington

The kernel keyring has a max of ~508 entries on 64-bit systems. For installations with more distict users than this limit, create a specified number of child keyrings and fill them evenly.

A couple things...

1) no Signed-off-by: line

Quoted 8 lines

2) Its seems you can create key rings but can't delete them. Here is what I'm doing:
in /etc/request-key.d/id_resolver.conf I have
create id_resolver * * /usr/sbin/nfsidmap -n 10 %k %d
but when I tried to delete the keys

  # nfsidmap -vc
  nfsidmap: clearing '08aa156c I--Q---     1perm 3f010000     0     0 keyring   .id_resolver_child_10: empty'
  nfsidmap: keyctl_clear(0x8aa156c) failed: Permission denied

This mess works on my fleet of RHEL6 boxes which is where I was trying to fix this. They create the child keyrings with

Quoted 7 lines

perm 3b3f0000

Instead of yours which appears to be

perm 3f010000

Are you testing on a later kernel? Likely this behavior has changed.

Yes... Much later...

Quoted 11 lines

Quoted 6 lines

Review

Quoted 5 lines

#define PROCKEYS "/proc/keys"
#ifndef DEFAULT_KEYRING

(quoted diff) +1 -1

-#define DEFAULT_KEYRING "id_resolver"
+#define DEFAULT_KEYRING ".id_resolver"

3) Why is changing the default needed?

The default is wrong. I think that's the first thing I changed when trying to fix this problem, since it looked like id_lookup() should gracefully recover in the case that the keyring was full (but it still doesn't).

Quoted 6 lines

I'm think the "id_resolver" default can from the face the entry /etc/request-key.d/id_resolver.conf which tells nfsidmap put the keys on the id_resolver key ring... so I'm not really sure where the .id_resolver is coming from... CC-ing David Howells maybe he knows...

To translate in king's English...

The reason the default is "id_resolver" is because the is the name of the key ring defined in id_resolver.conf is id_resolver. Now how that is translated into ".id_resolver" in /proc/keys is not clear....

What kernel are you dealing with?

David
--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

The reason the default is "id_resolver" is because the is the name of the key ring defined in id_resolver.conf is id_resolver. Now how that is translated into ".id_resolver" in /proc/keys is not clear....

Quoted 4 lines · Steve Dickson

The reason the default is "id_resolver" is because the is the name of the key ring defined in id_resolver.conf is id_resolver. Now how that is translated into ".id_resolver" in /proc/keys is not clear....

Where in /etc/request-key.d/id_resolver.conf does it mention the name of a keyring?

create id_resolver * * /usr/sbin/nfsidmap %k %d

Match it against this line from request-key.conf:

#OP TYPE DESCRIPTION CALLOUT INFO PROGRAM ARG1 ARG2 ARG3 ...

The "id_resolver" here is the name of the *key type* to be matched for that line. There is nothing here to do with keyrings.

David

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

The kernel keyring has a max of ~508 entries on 64-bit systems. For installations with more distict users than this limit, create a specified number of child keyrings and fill them evenly.

Quoted 3 lines · Steve Dickson

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

The kernel keyring has a max of ~508 entries on 64-bit systems. For installations with more distict users than this limit, create a specified number of child keyrings and fill them evenly.

This got fixed recently.

	Commit b2a4df200d570b2c33a57e1ebfa5896e4bc81b69
	Author: David Howells <[email protected]>  2013-09-24 10:35:18
	Subject: KEYS: Expand the capacity of a keyring

With that, the capacity is more or less infinite.

Yep, I'm aware of that - but unless RH backports this fix (which I was told is not going to happen) or uses this multiple keyring nfsidmap approach, things are going to stay broken in RHEL6.