Convert inode cache locking to RCU

Index: 2.6.x-xfs-new/fs/xfs/xfs_iget.c
===================================================================
@@ -40,6 +40,37 @@
 #include "xfs_utils.h"
 
 /*
+ * Attempt to move the inode out of the IRECLAIMABLE state.
+ * Must be called under rcu_read_lock() and with the ip->i_flags_lock
+ * held for synchronisation with xfs_ireclaim_finish().
+ */
+STATIC int
+xfs_iget_reclaim_check(
+	xfs_inode_t	*ip,
+	int		flags)
+{
+	/*
+	 * If IRECLAIM is set this inode is on its way out of the system, we
+	 * need to pause and try again.
+	 */
+	if (__xfs_iflags_test(ip, XFS_IRECLAIM))
+		return EAGAIN;
+	ASSERT(__xfs_iflags_test(ip, XFS_IRECLAIMABLE));
+
+	/*
+	 * If lookup is racing with unlink, then we should return an error
+	 * immediately so we don't remove it from the reclaim list and
+	 * potentially leak the inode.
+	 */
+
+	if ((ip->i_d.di_mode == 0) && !(flags & XFS_IGET_CREATE))
+		return ENOENT;
+
+	__xfs_iflags_clear(ip, XFS_IRECLAIMABLE);
+	return 0;
+}
+
+/*
  * Look up an inode by number in the given file system.
  * The inode is looked up in the cache held in each AG.
  * If the inode is found in the cache, attach it to the provided

@@ -94,7 +125,7 @@ xfs_iget_core(
 	agino = XFS_INO_TO_AGINO(mp, ino);
 
 again:
-	read_lock(&pag->pag_ici_lock);
+	rcu_read_lock();
 	ip = radix_tree_lookup(&pag->pag_ici_root, agino);
 
 	if (ip != NULL) {

@@ -103,52 +134,44 @@ again:
 		 * we need to pause and try again.
 		 */
 		if (xfs_iflags_test(ip, XFS_INEW)) {
-			read_unlock(&pag->pag_ici_lock);
+			rcu_read_unlock();
 			delay(1);
 			XFS_STATS_INC(xs_ig_frecycle);
 
 			goto again;
 		}
 
+		/*
+		 * Determine if the inode is queued for reclaim or being
+		 * reclaimed.  This is trickier now we are under RCU locking.
+		 *
+		 * Basically, xfs_ireclaim_finish() uses the i_flags_lock to
+		 * atomically move the inode out of the IRECLAIMABLE state and
+		 * inode the IRECLAIM state, so we have to use the same lock to
+		 * do an equivalent set of tests and move the inode out of the
+		 * IRECLAIMABLE state.
+		 */
 		old_inode = ip->i_vnode;
 		if (old_inode == NULL) {
-			/*
-			 * If IRECLAIM is set this inode is
-			 * on its way out of the system,
-			 * we need to pause and try again.
-			 */
-			if (xfs_iflags_test(ip, XFS_IRECLAIM)) {
-				read_unlock(&pag->pag_ici_lock);
-				delay(1);
+			spin_lock(&ip->i_flags_lock);
+			error = xfs_iget_reclaim_check(ip, flags);
+			spin_unlock(&ip->i_flags_lock);
+			rcu_read_unlock();
+			if (error) {
 				XFS_STATS_INC(xs_ig_frecycle);
-
-				goto again;
-			}
-			ASSERT(xfs_iflags_test(ip, XFS_IRECLAIMABLE));
-
-			/*
-			 * If lookup is racing with unlink, then we
-			 * should return an error immediately so we
-			 * don't remove it from the reclaim list and
-			 * potentially leak the inode.
-			 */
-			if ((ip->i_d.di_mode == 0) &&
-			    !(flags & XFS_IGET_CREATE)) {
-				read_unlock(&pag->pag_ici_lock);
+				if (error == EAGAIN) {
+					delay(1);
+					goto again;
+				}
 				xfs_put_perag(mp, pag);
-				return ENOENT;
+				return error;
 			}
-
-			xfs_itrace_exit_tag(ip, "xfs_iget.alloc");
-
-			XFS_STATS_INC(xs_ig_found);
-			xfs_iflags_clear(ip, XFS_IRECLAIMABLE);
-			read_unlock(&pag->pag_ici_lock);
-
 			XFS_MOUNT_ILOCK(mp);
 			list_del_init(&ip->i_reclaim);
 			XFS_MOUNT_IUNLOCK(mp);
 
+			XFS_STATS_INC(xs_ig_found);
+			xfs_itrace_exit_tag(ip, "xfs_iget.alloc");
 			goto finish_inode;
 
 		} else if (inode != old_inode) {

@@ -156,7 +179,7 @@ again:
 			 * try again.
 			 */
 			if (old_inode->i_state & (I_FREEING | I_CLEAR)) {
-				read_unlock(&pag->pag_ici_lock);
+				rcu_read_unlock();
 				delay(1);
 				XFS_STATS_INC(xs_ig_frecycle);
 

@@ -174,7 +197,7 @@ again:
 		/*
 		 * Inode cache hit
 		 */
-		read_unlock(&pag->pag_ici_lock);
+		rcu_read_unlock();
 		XFS_STATS_INC(xs_ig_found);
 
 finish_inode:

@@ -194,7 +217,7 @@ finish_inode:
 	/*
 	 * Inode cache miss
 	 */
-	read_unlock(&pag->pag_ici_lock);
+	rcu_read_unlock();
 	XFS_STATS_INC(xs_ig_missed);
 
 	/*

@@ -237,14 +260,14 @@ finish_inode:
 	}
 	mask = ~(((XFS_INODE_CLUSTER_SIZE(mp) >> mp->m_sb.sb_inodelog)) - 1);
 	first_index = agino & mask;
-	write_lock(&pag->pag_ici_lock);
+	spin_lock(&pag->pag_ici_lock);
 	/*
 	 * insert the new inode
 	 */
 	error = radix_tree_insert(&pag->pag_ici_root, agino, ip);
 	if (unlikely(error)) {
 		BUG_ON(error != -EEXIST);
-		write_unlock(&pag->pag_ici_lock);
+		spin_unlock(&pag->pag_ici_lock);
 		radix_tree_preload_end();
 		xfs_idestroy(ip);
 		XFS_STATS_INC(xs_ig_dup);

@@ -257,7 +280,7 @@ finish_inode:
 	ip->i_udquot = ip->i_gdquot = NULL;
 	xfs_iflags_set(ip, XFS_INEW);
 
-	write_unlock(&pag->pag_ici_lock);
+	spin_unlock(&pag->pag_ici_lock);
 	radix_tree_preload_end();
 
 	/*

@@ -378,9 +401,9 @@ xfs_inode_incore(xfs_mount_t	*mp,
 	xfs_perag_t	*pag;
 
 	pag = xfs_get_perag(mp, ino);
-	read_lock(&pag->pag_ici_lock);
+	rcu_read_lock();
 	ip = radix_tree_lookup(&pag->pag_ici_root, XFS_INO_TO_AGINO(mp, ino));
-	read_unlock(&pag->pag_ici_lock);
+	rcu_read_unlock();
 	xfs_put_perag(mp, pag);
 
 	/* the returned inode must match the transaction */

@@ -491,9 +514,9 @@ xfs_iextract(
 	xfs_perag_t	*pag = xfs_get_perag(mp, ip->i_ino);
 	xfs_inode_t	*iq;
 
-	write_lock(&pag->pag_ici_lock);
+	spin_lock(&pag->pag_ici_lock);
 	radix_tree_delete(&pag->pag_ici_root, XFS_INO_TO_AGINO(mp, ip->i_ino));
-	write_unlock(&pag->pag_ici_lock);
+	spin_unlock(&pag->pag_ici_lock);
 	xfs_put_perag(mp, pag);
 
 	/*

133 lines not on this page · raw message

Index: 2.6.x-xfs-new/fs/xfs/xfs_inode.h
===================================================================
@@ -203,7 +203,12 @@ typedef struct xfs_inode {
 	struct xfs_inode	*i_mnext;	/* next inode in mount list */
 	struct xfs_inode	*i_mprev;	/* ptr to prev inode */
 	struct xfs_mount	*i_mount;	/* fs mount struct ptr */
-	struct list_head	i_reclaim;	/* reclaim list */
+	union {
+		struct list_head i_reclaim;	/* reclaim list */
+		struct rcu_head	i_rcu;		/* rcu reclaim list */
+	} i_ru;
+#define i_reclaim	i_ru.i_reclaim
+#define i_rcu		i_ru.i_rcu
 	bhv_vnode_t		*i_vnode;	/* vnode backpointer */
 	struct xfs_dquot	*i_udquot;	/* user dquot */
 	struct xfs_dquot	*i_gdquot;	/* group dquot */

@@ -285,10 +290,15 @@ xfs_iflags_set(xfs_inode_t *ip, unsigned
 }
 
 static inline void
+__xfs_iflags_clear(xfs_inode_t *ip, unsigned short flags)
+{
+	ip->i_flags &= ~flags;
+}
+static inline void
 xfs_iflags_clear(xfs_inode_t *ip, unsigned short flags)
 {
 	spin_lock(&ip->i_flags_lock);
-	ip->i_flags &= ~flags;
+	__xfs_iflags_clear(ip, flags);
 	spin_unlock(&ip->i_flags_lock);
 }
 

Index: 2.6.x-xfs-new/fs/xfs/xfs_ag.h
===================================================================
@@ -199,7 +199,7 @@ typedef struct xfs_perag
 	atomic_t        pagf_fstrms;    /* # of filestreams active in this AG */
 
 	int		pag_ici_init;	/* incore inode cache initialised */
-	rwlock_t	pag_ici_lock;	/* incore inode lock */
+	spinlock_t	pag_ici_lock;	/* incore inode lock */
 	struct radix_tree_root pag_ici_root;	/* incore inode cache root */
 } xfs_perag_t;
 

Index: 2.6.x-xfs-new/fs/xfs/xfs_mount.c
===================================================================
@@ -334,7 +334,7 @@ xfs_initialize_perag_icache(
 	xfs_perag_t	*pag)
 {
 	if (!pag->pag_ici_init) {
-		rwlock_init(&pag->pag_ici_lock);
+		spin_lock_init(&pag->pag_ici_lock);
 		INIT_RADIX_TREE(&pag->pag_ici_root, GFP_ATOMIC);
 		pag->pag_ici_init = 1;
 	}

Index: 2.6.x-xfs-new/fs/xfs/xfs_vnodeops.c
===================================================================
@@ -3671,24 +3671,22 @@ xfs_finish_reclaim(
 	int		locked,
 	int		sync_mode)
 {
-	xfs_perag_t	*pag = xfs_get_perag(ip->i_mount, ip->i_ino);
 	bhv_vnode_t	*vp = XFS_ITOV_NULL(ip);
 	int		error;
 
 	if (vp && VN_BAD(vp))
 		goto reclaim;
 
-	/* The hash lock here protects a thread in xfs_iget_core from
+	/*
+	 * The flags lock here protects a thread in xfs_iget_core from
 	 * racing with us on linking the inode back with a vnode.
 	 * Once we have the XFS_IRECLAIM flag set it will not touch
 	 * us.
 	 */
-	write_lock(&pag->pag_ici_lock);
 	spin_lock(&ip->i_flags_lock);
 	if (__xfs_iflags_test(ip, XFS_IRECLAIM) ||
 	    (!__xfs_iflags_test(ip, XFS_IRECLAIMABLE) && vp == NULL)) {
 		spin_unlock(&ip->i_flags_lock);
-		write_unlock(&pag->pag_ici_lock);
 		if (locked) {
 			xfs_ifunlock(ip);
 			xfs_iunlock(ip, XFS_ILOCK_EXCL);

@@ -3697,8 +3695,6 @@ xfs_finish_reclaim(
 	}
 	__xfs_iflags_set(ip, XFS_IRECLAIM);
 	spin_unlock(&ip->i_flags_lock);
-	write_unlock(&pag->pag_ici_lock);
-	xfs_put_perag(ip->i_mount, pag);
 
 	/*
 	 * If the inode is still dirty, then flush it out.  If the inode