cgroup: kernel BUG at kernel/cgroup.c:1038!

Hi all,

While fuzzing with trinity on a KVM tools guest running latest -next kernel, I've stumbled on the following spew:

[   88.247018] kernel BUG at kernel/cgroup.c:1038!
[   88.250738] invalid opcode: 0000 [#1] PREEMPT SMP DEBUG_PAGEALLOC
[   88.251169] Dumping ftrace buffer:
[   88.251169]    (ftrace buffer empty)
[   88.251169] CPU: 1 PID: 7973 Comm: mount Tainted: G        W    3.10.0-rc7-next-2013
0625-sasha-00011-g1c1dc0e #1105
[   88.251169] task: ffff880fc0ae8000 ti: ffff880fc0b9a000 task.ti: ffff880fc0b9a000
[   88.251437] RIP: 0010:[<ffffffff81249b29>]  [<ffffffff81249b29>] rebind_subsystems+0
x409/0x5f0
[   88.252442] RSP: 0018:ffff880fc0b9bba8  EFLAGS: 00010202
[   88.252994] RAX: 0000000000000001 RBX: ffff880fc0b8a148 RCX: 0000000000000001
[   88.253839] RDX: 0000000000000003 RSI: 0000000000000001 RDI: ffffffff8860eec0
[   88.254695] RBP: ffff880fc0b9bc28 R08: ffff880fc0b8a170 R09: ffffffff889e64f8
[   88.255325] R10: ffffffff889e64d0 R11: ffffffff8a28b600 R12: 0000000000000001
[   88.255325] R13: 0000000000000000 R14: ffff880fc0b8a170 R15: ffffffff87a6f060
[   88.255325] FS:  00007f1289eba7e0(0000) GS:ffff880fe5800000(0000) knlGS:000000000000
0000
[   88.255325] CS:  0010 DS: 0000 ES: 0000 CR0: 000000008005003b
[   88.255325] CR2: 00007f12894382f0 CR3: 0000000fc01e0000 CR4: 00000000000006e0
[   88.255325] Stack:
[   88.255325]  ffffffff87a6e4d8 ffffffff87a6e468 2222222222222222 2222222222222222
[   88.255325]  2222222222222222 2222222200000001 0000000000000000 0000000187a6e460
[   88.255325]  ffff880fc0b8a160 0000000000000000 ffff880fcc2f8b88 ffff880fc0b8a148
[   88.255325] Call Trace:
[   88.255325]  [<ffffffff8124bd4f>] cgroup_kill_sb+0xff/0x210
[   88.255325]  [<ffffffff813d21af>] deactivate_locked_super+0x4f/0x90
[   88.255325]  [<ffffffff8124f3b3>] cgroup_mount+0x673/0x6e0
[   88.255325]  [<ffffffff81401a6d>] ? get_fs_type+0x7d/0xd0
[   88.255325]  [<ffffffff854e95ec>] ? _raw_read_unlock+0x5c/0x80
[   88.255325]  [<ffffffff81257169>] cpuset_mount+0xd9/0x110
[   88.255325]  [<ffffffff81337908>] ? pcpu_alloc+0x4e8/0x5f0
[   88.255325]  [<ffffffff813d2580>] mount_fs+0xb0/0x2d0
[   88.255325]  [<ffffffff81404afd>] vfs_kern_mount+0xbd/0x180
[   88.255325]  [<ffffffff814070b5>] do_new_mount+0x145/0x2c0
[   88.255325]  [<ffffffff81de79b8>] ? security_capable+0x18/0x20
[   88.255325]  [<ffffffff814085d6>] do_mount+0x356/0x3c0
[   88.255325]  [<ffffffff8140873d>] SyS_mount+0xfd/0x140
[   88.255325]  [<ffffffff854eb600>] tracesys+0xdd/0xe2
[   88.255325] Code: 89 c6 89 45 a8 e8 28 b7 07 00 8b 45 a8 48 63 d0 48 83 c2 02 48 8b 0c d5 48 63 9e 88 48 ff c1 85 c0 48 89 0c d5 48 63 9e 88 74 04 <0f> 0b eb fe 4a 8b 84 2b 88 00 00 00 48 c7 c7 98 ee 60 88 4c 39
[   88.255325] RIP  [<ffffffff81249b29>] rebind_subsystems+0x409/0x5f0
[   88.255325]  RSP <ffff880fc0b9bba8>

Thanks,
Sasha

cgroupfs_root used to have ->actual_subsys_mask in addition to ->subsys_mask. a8a648c4ac ("cgroup: remove cgroup->actual_subsys_mask") removed it noting that the subsys_mask is essentially temporary and doesn't belong in cgroupfs_root; however, the patch made it impossible to tell whether a cgroupfs_root actually has the subsystems bound or just have the bits set leading to the following BUG when trying to mount with subsystems which are already mounted elsewhere.

kernel BUG at kernel/cgroup.c:1038!
invalid opcode: 0000 [#1] PREEMPT SMP DEBUG_PAGEALLOC
...

 CPU: 1 PID: 7973 Comm: mount Tainted: G        W    3.10.0-rc7-next-20130625-sasha-00011-g1c1dc0e #1105
 task: ffff880fc0ae8000 ti: ffff880fc0b9a000 task.ti: ffff880fc0b9a000
 RIP: 0010:[<ffffffff81249b29>]  [<ffffffff81249b29>] rebind_subsystems+0x409/0x5f0
 ...
 Call Trace:
  [<ffffffff8124bd4f>] cgroup_kill_sb+0xff/0x210
  [<ffffffff813d21af>] deactivate_locked_super+0x4f/0x90
  [<ffffffff8124f3b3>] cgroup_mount+0x673/0x6e0
  [<ffffffff81257169>] cpuset_mount+0xd9/0x110
  [<ffffffff813d2580>] mount_fs+0xb0/0x2d0
  [<ffffffff81404afd>] vfs_kern_mount+0xbd/0x180
  [<ffffffff814070b5>] do_new_mount+0x145/0x2c0
  [<ffffffff814085d6>] do_mount+0x356/0x3c0
  [<ffffffff8140873d>] SyS_mount+0xfd/0x140
  [<ffffffff854eb600>] tracesys+0xdd/0xe2

We still want rebind_subsystems() to take added/removed masks, so let's fix it by marking whether a cgroupfs_root has finished binding or not. Also, document what's going on around ->subsys_mask initialization so that similar mistakes aren't repeated.

---
 include/linux/cgroup.h |    1 +
 kernel/cgroup.c        |   22 +++++++++++++++++++---
 2 files changed, 20 insertions(+), 3 deletions(-)

@@ -276,6 +276,7 @@ enum {
 
 	CGRP_ROOT_NOPREFIX	= (1 << 1), /* mounted subsystems have no named prefix */
 	CGRP_ROOT_XATTR		= (1 << 2), /* supports extended attributes */
+	CGRP_ROOT_SUBSYS_BOUND	= (1 << 3), /* subsystems finished binding */
 };
 
 /*

@@ -1086,6 +1086,12 @@ static int rebind_subsystems(struct cgro
 		}
 	}
 
+	/*
+	 * Mark @root has finished binding subsystems.  @root->subsys_mask
+	 * now matches the bound subsystems.
+	 */
+	root->flags |= CGRP_ROOT_SUBSYS_BOUND;
+
 	return 0;
 }
 

@@ -1485,6 +1491,14 @@ static struct cgroupfs_root *cgroup_root
 
 	init_cgroup_root(root);
 
+	/*
+	 * We need to set @root->subsys_mask now so that @root can be
+	 * matched by cgroup_test_super() before it finishes
+	 * initialization; otherwise, competing mounts with the same
+	 * options may try to bind the same subsystems instead of waiting
+	 * for the first one leading to unexpected mount errors.
+	 * SUBSYS_BOUND will be set once actual binding is complete.
+	 */
 	root->subsys_mask = opts->subsys_mask;
 	root->flags = opts->flags;
 	ida_init(&root->cgroup_ida);

@@ -1734,9 +1748,11 @@ static void cgroup_kill_sb(struct super_
 	mutex_lock(&cgroup_root_mutex);
 
 	/* Rebind all subsystems back to the default hierarchy */
-	ret = rebind_subsystems(root, 0, root->subsys_mask);
-	/* Shouldn't be able to fail ... */
-	BUG_ON(ret);
+	if (root->flags & CGRP_ROOT_SUBSYS_BOUND) {
+		ret = rebind_subsystems(root, 0, root->subsys_mask);
+		/* Shouldn't be able to fail ... */
+		BUG_ON(ret);
+	}
 
 	/*
 	 * Release all the links from cset_links to this hierarchy's

This isn't strictly necessary as all subsystems specified in @subsys_mask are guaranteed to be pinned; however, it does spuriously trigger lockdep warning. Let's grab cgroup_mutex around it.

---
 kernel/cgroup.c |   10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

@@ -1325,11 +1325,11 @@ static void drop_parsed_module_refcounts
 	struct cgroup_subsys *ss;
 	int i;
 
-	for_each_subsys(ss, i) {
-		if (!(subsys_mask & (1UL << i)))
-			continue;
-		module_put(cgroup_subsys[i]->module);
-	}
+	mutex_lock(&cgroup_mutex);
+	for_each_subsys(ss, i)
+		if (subsys_mask & (1UL << i))
+			module_put(cgroup_subsys[i]->module);
+	mutex_unlock(&cgroup_mutex);
 }
 
 static int cgroup_remount(struct super_block *sb, int *flags, char *data)

Applied both patches to cgroup/for-3.11.

Thanks.

tejun

FWIW, for both patches,

Thanks,
Sasha