Skip to contentSkip to conversation

nfsd4: drc containerization

PATCH New J. Bruce Fields 1 participant 2 messages May 17, 2019

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/2 nfsd4: drc containerization

From: "J. Bruce Fields" <[email protected]>

The nfsd duplicate reply cache should not be shared between network namespaces.

The most straightforward way to fix this is just to move every global in the code to per-net-namespace memory, so that's what we do.

Still todo: sort out which members of nfsd_stats should be global and which per-net-namespace.

---
 fs/nfsd/cache.h    |   5 +-
 fs/nfsd/netns.h    |  33 +++++++
 fs/nfsd/nfscache.c | 226 ++++++++++++++++++++++-----------------------
 fs/nfsd/nfsctl.c   |  14 +--
 4 files changed, 153 insertions(+), 125 deletions(-)
fs/nfsd/cache.h +3 -2
index 4a98537efb0f..10ec5ecdf117 100644
@@ -10,6 +10,7 @@
 #define NFSCACHE_H
 
 #include <linux/sunrpc/svc.h>
+#include "netns.h"
 
 /*
  * Representation of a reply cache entry.
@@ -77,8 +78,8 @@ enum {
 /* Checksum this amount of the request */
 #define RC_CSUMLEN		(256U)
 
-int	nfsd_reply_cache_init(void);
-void	nfsd_reply_cache_shutdown(void);
+int	nfsd_reply_cache_init(struct nfsd_net *);
+void	nfsd_reply_cache_shutdown(struct nfsd_net *);
 int	nfsd_cache_lookup(struct svc_rqst *);
 void	nfsd_cache_update(struct svc_rqst *, int, __be32 *);
 int	nfsd_reply_cache_stats_open(struct inode *, struct file *);
fs/nfsd/netns.h +33 -0
index 32cb8c027483..4146dca94c5f 100644
@@ -131,6 +131,39 @@ struct nfsd_net {
 	u32		s2s_cp_cl_id;
 	struct idr	s2s_cp_stateids;
 	spinlock_t	s2s_cp_lock;
+
+	/* Duplicate reply cache: */
+	struct nfsd_drc_bucket   *drc_hashtbl;
+	struct kmem_cache        *drc_slab;
+
+	/* max number of entries allowed in the cache */
+	unsigned int             max_drc_entries;
+
+	/* number of significant bits in the hash value */
+	unsigned int             maskbits;
+	unsigned int             drc_hashsize;
+
+	/*
+	 * Stats and other tracking of on the duplicate reply cache. All of these and
+	 * the "rc" fields in nfsdstats are protected by the cache_lock
+	 */
+
+	/* total number of entries */
+	atomic_t                 num_drc_entries;
+
+	/* cache misses due only to checksum comparison failures */
+	unsigned int             payload_misses;
+
+	/* amount of memory (in bytes) currently consumed by the DRC */
+	unsigned int             drc_mem_usage;
+
+	/* longest hash chain seen */
+	unsigned int             longest_chain;
+
+	/* size of cache when we saw the longest hash chain */
+	unsigned int             longest_chain_cachesize;
+
+	struct shrinker		nfsd_reply_cache_shrinker;
 };
 
 /* Simple check to find out if a given net was properly initialized */
fs/nfsd/nfscache.c +109 -117

467 lines not on this page · raw message

fs/nfsd/nfsctl.c +8 -6
index f2feb2d11bae..9e2d038c3426 100644
@@ -1239,6 +1239,9 @@ static __net_init int nfsd_init_net(struct net *net)
 	retval = nfsd_idmap_init(net);
 	if (retval)
 		goto out_idmap_error;
+	retval = nfsd_reply_cache_init(nn);
+	if (retval)
+		goto out_drc_error;
 	nn->nfsd4_lease = 90;	/* default lease time */
 	nn->nfsd4_grace = 90;
 	nn->somebody_reclaimed = false;
@@ -1250,6 +1253,8 @@ static __net_init int nfsd_init_net(struct net *net)
 	init_waitqueue_head(&nn->ntf_wq);
 	return 0;
 
+out_drc_error:
+	nfsd_idmap_shutdown(net);
 out_idmap_error:
 	nfsd_export_shutdown(net);
 out_export_error:
@@ -1258,6 +1263,9 @@ static __net_init int nfsd_init_net(struct net *net)
 
 static __net_exit void nfsd_exit_net(struct net *net)
 {
+	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
+
+	nfsd_reply_cache_shutdown(nn);
 	nfsd_idmap_shutdown(net);
 	nfsd_export_shutdown(net);
 }
@@ -1290,9 +1298,6 @@ static int __init init_nfsd(void)
 	if (retval)
 		goto out_exit_pnfs;
 	nfsd_stat_init();	/* Statistics */
-	retval = nfsd_reply_cache_init();
-	if (retval)
-		goto out_free_stat;
 	nfsd_lockd_init();	/* lockd->nfsd callbacks */
 	retval = create_proc_exports_entry();
 	if (retval)
@@ -1306,8 +1311,6 @@ static int __init init_nfsd(void)
 	remove_proc_entry("fs/nfs", NULL);
 out_free_lockd:
 	nfsd_lockd_shutdown();
-	nfsd_reply_cache_shutdown();
-out_free_stat:
 	nfsd_stat_shutdown();
 	nfsd_fault_inject_cleanup();
 out_exit_pnfs:
@@ -1323,7 +1326,6 @@ static int __init init_nfsd(void)
 
 static void __exit exit_nfsd(void)
 {
-	nfsd_reply_cache_shutdown();
 	remove_proc_entry("fs/nfs/exports", NULL);
 	remove_proc_entry("fs/nfs", NULL);
 	nfsd_stat_shutdown();
Signature · 1 line
2.21.0
#1 lore raw

PATCH 2/2 nfsd: note inadequate stats locking

From: "J. Bruce Fields" <[email protected]>

After 89a26b3d295d "nfsd: split DRC global spinlock into per-bucket locks", there is no longer a single global spinlock to protect these stats.

So, really we need to fix that. For now, at least fix the comment.

---
 fs/nfsd/netns.h | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)
fs/nfsd/netns.h +5 -2
index 4146dca94c5f..e01c22fd1485 100644
@@ -144,8 +144,11 @@ struct nfsd_net {
 	unsigned int             drc_hashsize;
 
 	/*
-	 * Stats and other tracking of on the duplicate reply cache. All of these and
-	 * the "rc" fields in nfsdstats are protected by the cache_lock
+	 * Stats and other tracking of on the duplicate reply cache.
+	 * These fields and the "rc" fields in nfsdstats are modified
+	 * with only the per-bucket cache lock, which isn't really safe
+	 * and should be fixed if we want the statistics to be
+	 * completely accurate.
 	 */
 
 	/* total number of entries */
Signature · 1 line
2.21.0