Skip to contentSkip to conversation

fs: remove retry loop

RFC PATCH In review John Ogness 3 participants 5 messages Jun 17, 2020 VFS

lore
Share & tools
lore
#0 lore raw

cover fs: remove retry loop

Hello,

This patch removes the last retry loop in VFS. It is partially reverting

commit d3ef3d7351cc ("fs: mnt_want_write speedup")

by re-introducing per-cpu spinlocks for each mount. The patch includes benchmark results in the diffstat section to show that the previous optimization work is not undone.

I would have liked to use a percpu_rw_semaphore per mount instead of the many spinlocks. However, percpu_rw_semaphore can sleep, which is a problem for sb_prepare_remount_readonly() since it needs to take the spinlock in @mount_lock in order to iterate @sb->s_mounts. Perhaps using a mutex to sychronize @sb->s_mounts is an option. I am not sure. That is why this is an RFC.

I am suggesting this partial revert because it removes the retry loop and does not show any obvious negative benchmark effects.

John Ogness (1):
fs/namespace.c: use spinlock instead of busy loop

 fs/mount.h     |   7 +++
 fs/namespace.c | 118 +++++++++++++++++++++++++++++++++----------------
 2 files changed, 86 insertions(+), 39 deletions(-)
Signature · 1 line
2.20.1
#1 lore raw

RFC PATCH 1/1 fs/namespace.c: use spinlock instead of busy loop

The MNT_WRITE_HOLD flag is used to manually implement a per-cpu optimized rwsem using busy waiting. This implementation is a problem on PREEMPT_RT because write_seqlock() on @mount_lock (i.e. taking a spinlock) does not disable preemption. This allows a writer to preempt a task that has set MNT_WRITE_HOLD and thus that writer will live lock in __mnt_want_write() due to busy looping with preemption disabled.

Reimplement the same semantics using per-cpu spinlocks. This provides lockdep coverage and makes the code RT ready.

Since this reverts some of the optimization work of
commit d3ef3d7351cc ("fs: mnt_want_write speedup")
lmbench lat_mmap tests were performed to verify that there is no obvious performance degradation.

---
Here is the detailed test information...

TEST COMMAND
lat_mmap -P $pval -W 32 -N 50 64m file

OUTPUT FORMAT
pval: avg std

RESULTS (32 CPUs)
No Forced Preemption
BEFORE AFTER

  1:   275.60  1.82   274.40  0.55
  2:   296.20  3.83   286.80  1.92
  4:   310.20  4.44   304.40  2.51
  8:   359.20  2.28   357.80  2.95
 16:   417.40  2.51   412.20  3.90
 32:   625.60  2.07   622.00  3.08
 64:  1202.60 15.87  1202.20  6.14

No Forced Preemption, no PTI
BEFORE AFTER

  1:   278.00  2.12   274.40  1.52
  2:   301.00  3.67   289.80  6.06
  4:   333.40  7.73   303.80  2.39
  8:   389.80  3.56   351.80  3.42
 16:   425.00  3.46   408.20  4.87
 32:   606.00  1.22   605.60  1.82
 64:  1193.60  7.09  1184.80  4.27

Voluntary Kernel Preemption
BEFORE AFTER

  1:   277.80  1.30   278.20  1.10
  2:   291.20  0.84   286.60  2.30
  4:   310.00  1.87   304.80  1.30
  8:   360.00  2.55   354.60  1.14
 16:   414.00  1.58   414.00  2.35
 32:   619.60  5.50   607.00  3.74
 64:  1224.00  8.40  1219.60  6.19

Voluntary Kernel Preemption, no PTI
BEFORE AFTER

  1:   277.80  4.66   276.40  0.89
  2:   291.40  6.54   286.40  3.58
  4:   310.00  1.22   315.40  1.14
  8:   357.20  0.84   361.40  2.61
 16:   405.60  2.88   407.60  2.51
 32:   615.40  2.30   611.60  5.55
 64:  1219.80  9.91  1207.40 10.88

Preemptible Kernel
BEFORE AFTER

  1:   283.80  0.45   286.80  0.84
  2:   293.40  2.51   294.40  3.51
  4:   318.20  1.30   315.60  1.95
  8:   367.00  0.71   363.00  1.22
 16:   416.20  1.64   413.20  4.87
 32:   628.80  2.28   617.40  2.97
 64:  1277.20  9.88  1254.20  4.87

Preemptible Kernel, no PTI
BEFORE AFTER

  1:   283.00  1.73   288.40  1.67
  2:   305.80  2.05   297.00  3.24
  4:   321.40  4.34   318.60  2.79
  8:   370.20  2.39   366.40  2.70
 16:   413.20  3.11   412.40  2.41
 32:   616.40  2.61   620.20  2.05
 64:  1266.00  6.48  1255.80  3.90
 fs/mount.h     |   7 +++
 fs/namespace.c | 118 +++++++++++++++++++++++++++++++++----------------
 2 files changed, 86 insertions(+), 39 deletions(-)
fs/mount.h +7 -0
index c7abb7b394d8..627e635cd7d1 100644
@@ -28,6 +28,12 @@ struct mnt_namespace {
 struct mnt_pcp {
 	int mnt_count;
 	int mnt_writers;
+	/*
+	 * If holding multiple instances of this lock, they
+	 * must be ordered by cpu number.
+	 */
+	spinlock_t mnt_writers_lock;
+	struct lock_class_key mnt_writers_lock_key;
 };
 
 struct mountpoint {
@@ -51,6 +57,7 @@ struct mount {
 #else
 	int mnt_count;
 	int mnt_writers;
+	spinlock_t mnt_writers_lock;
 #endif
 	struct list_head mnt_mounts;	/* list of children, anchored here */
 	struct list_head mnt_child;	/* and going through their mnt_child */
fs/namespace.c +79 -39

196 lines not on this page · raw message

Signature · 1 line
2.20.1
#2 lore raw

Re: [RFC PATCH 1/1] fs/namespace.c: use spinlock instead of busy loop

Quoted 21 lines · John Ogness
fs/namespace.c +17 -0
@@ -459,17 +469,39 @@ void mnt_drop_write_file(struct file *file)
 }
 EXPORT_SYMBOL(mnt_drop_write_file);
 
+static void mnt_lock_writers(struct mount *mnt)
+{
+#ifdef CONFIG_SMP
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+		spin_lock(&per_cpu_ptr(mnt->mnt_pcp,
+				       cpu)->mnt_writers_lock);
+	}
+#else
+	spin_lock(&mnt->mnt_writers_lock);
+#endif
+}
+
+static void mnt_unlock_writers(struct mount *mnt)
+{
+#ifdef CONFIG_SMP

fs/namespace.c void mnt_drop_write_file(struct file *file) · line 498

Quoted 10 lines
fs/namespace.c +10 -0
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+		spin_unlock(&per_cpu_ptr(mnt->mnt_pcp,
+					 cpu)->mnt_writers_lock);
+	}
+#else
+	spin_unlock(&mnt->mnt_writers_lock);
+#endif
+}

*groan*.. this is brlock reincarnate :-/ Also broken.

#3 lore raw

Re: [RFC PATCH 1/1] fs/namespace.c: use spinlock instead of busy loop

In reply to John Ogness
Quoted 8 lines · John Ogness

The MNT_WRITE_HOLD flag is used to manually implement a per-cpu optimized rwsem using busy waiting. This implementation is a problem on PREEMPT_RT because write_seqlock() on @mount_lock (i.e. taking a spinlock) does not disable preemption. This allows a writer to preempt a task that has set MNT_WRITE_HOLD and thus that writer will live lock in __mnt_want_write() due to busy looping with preemption disabled.

Why can't you use a regular percpu-rwsem for this?

#4 lore raw

Re: [RFC PATCH 1/1] fs/namespace.c: use spinlock instead of busy loop

Quoted 1 line · Peter Zijlstra

Why can't you use a regular percpu-rwsem for this?

This is the percpu-rwsem version. The lock is per struct super_block not per mount point. This is due to the restriction in sb_prepare_remount_readonly().

fs/mount.h +2 -0
index 711a4093e475..344fae9931b5 100644
@@ -4,6 +4,7 @@
 #include <linux/poll.h>
 #include <linux/ns_common.h>
 #include <linux/fs_pin.h>
+#include <linux/percpu-rwsem.h>
 
 struct mnt_namespace {
 	atomic_t		count;
@@ -40,6 +41,7 @@ struct mount {
 		struct rcu_head mnt_rcu;
 		struct llist_node mnt_llist;
 	};
+	struct percpu_rw_semaphore mnt_writers_rws;
 #ifdef CONFIG_SMP
 	struct mnt_pcp __percpu *mnt_pcp;
 #else
fs/namespace.c +26 -62

171 lines not on this page · raw message

fs/super.c +2 -0
index a288cd60d2ae..b6036e5fbe8d 100644
@@ -210,6 +210,8 @@ static struct super_block *alloc_super(struct file_system_type *type, int flags,
 	INIT_LIST_HEAD(&s->s_mounts);
 	s->s_user_ns = get_user_ns(user_ns);
 	init_rwsem(&s->s_umount);
+	percpu_init_rwsem(&s->mnt_writers_rws);
+
 	lockdep_set_class(&s->s_umount, &type->s_umount_key);
 	/*
 	 * sget() can have s_umount recursion.
include/linux/fs.h +1 -0
index 45cc10cdf6dd..cc259b7c4ee8 100644
@@ -1458,6 +1458,7 @@ struct super_block {
 #endif
 	struct hlist_bl_head	s_roots;	/* alternate root dentries for NFS */
 	struct list_head	s_mounts;	/* list of mounts; _not_ for fs use */
+	struct percpu_rw_semaphore mnt_writers_rws;
 	struct block_device	*s_bdev;
 	struct backing_dev_info *s_bdi;
 	struct mtd_info		*s_mtd;
include/linux/mount.h +1 -2
index bf8cc4108b8f..8daf217108e8 100644
@@ -32,7 +32,6 @@ struct fs_context;
 #define MNT_READONLY	0x40	/* does the user want this to be r/o? */
 
 #define MNT_SHRINKABLE	0x100
-#define MNT_WRITE_HOLD	0x200
 
 #define MNT_SHARED	0x1000	/* if the vfsmount is a shared mount */
 #define MNT_UNBINDABLE	0x2000	/* if the vfsmount is a unbindable mount */
@@ -49,7 +48,7 @@ struct fs_context;
 				 | MNT_READONLY)
 #define MNT_ATIME_MASK (MNT_NOATIME | MNT_NODIRATIME | MNT_RELATIME )
 
-#define MNT_INTERNAL_FLAGS (MNT_SHARED | MNT_WRITE_HOLD | MNT_INTERNAL | \
+#define MNT_INTERNAL_FLAGS (MNT_SHARED | MNT_INTERNAL | \
 			    MNT_DOOMED | MNT_SYNC_UMOUNT | MNT_MARKED)
 
 #define MNT_INTERNAL	0x4000
Signature · 1 line
2.27.0