cgroup: implement cgroup.populated, v2

PATCH New Tejun Heo 7 participants 20 messages Apr 14, 2014 – Apr 17, 2014 4 messages not in the archive

Subsystems (2)

cgroup kernfs

lore

Share & tools

lore

Tejun Heo

author maintainer

2014-04-14 21:44 UTC #0 lore raw

PATCH 1/1 cgroup: implement cgroup.populated, v2

Hello,

This is v2 of cgroup.populated patchset changes from v1[L] are,

* Rebased on top of v3.15-rc1

* Patch description updated for the third patch as per Serge.

(inotify folks, can you please review the second patch which hooks up
kernfs_notify() with fsnotify? It works but directly invoking fsnotify_parent() and fsnotify() feels a bit dirty. Maybe something like fsnotify_notify_modify_by_dentry() would make more sense?)

cgroup users often need a way to determine when a cgroup's subhierarchy becomes empty so that it can be cleaned up. cgroup currently provides release_agent for it; unfortunately, this mechanism is riddled with issues. This patchset implements a replacement mechanism "cgroup.subtree_populated" which can be used to monitor whether the cgroup's subhierarchy has tasks in it or not and triggers poll and [di]notify events when its content changes.

This patchset contains the following three patches.

0001-kernfs-implement-kernfs_root-supers-list.patch 0002-kernfs-make-kernfs_notify-trigger-inotify-events-too.patch 0003-cgroup-implement-cgroup.subtree_populated-for-the-de.patch

0001-0002 add [di]notify notification to kernfs_notify().

0003 implements cgroup.subtree_populated.

This patchset is on top of "cgroup: implement unified hierarchy, v2" patchset[1] and availalbe in the following git branch.

git://git.kernel.org/pub/scm/linux/kernel/git/tj/cgroup.git review-populated

diffstat follows.

 fs/kernfs/dir.c             |    1 
 fs/kernfs/file.c            |   41 +++++++++++++++++++++++----
 fs/kernfs/kernfs-internal.h |    5 +++
 fs/kernfs/mount.c           |   11 +++++++
 include/linux/cgroup.h      |   15 ++++++++++
 include/linux/kernfs.h      |    4 ++
 kernel/cgroup.c             |   65 +++++++++++++++++++++++++++++++++++++++++---
 7 files changed, 132 insertions(+), 10 deletions(-)

Thanks.

--
tejun

[L] http://lkml.kernel.org/g/1397056052-2829-1-git-send-email-tj-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org
[1] http://lkml.kernel.org/g/1397511430-2673-1-git-send-email-tj-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org

Tejun Heo

author maintainer

2014-04-14 21:44 UTC #1 lore raw

PATCH 1/3 kernfs: implement kernfs_root->supers list

Currently, there's no way to find out which super_blocks are associated with a given kernfs_root. Let's implement it - the planned inotify extension to kernfs_notify() needs it.

Make kernfs_super_info point back to the super_block and chain it at kernfs_root->supers.

Signed-off-by Tejun Heo <[email protected]>

Cc Greg Kroah-Hartman <[email protected]>

---
 fs/kernfs/dir.c             |  1 +
 fs/kernfs/kernfs-internal.h |  5 +++++
 fs/kernfs/mount.c           | 11 +++++++++++
 include/linux/kernfs.h      |  4 ++++
 4 files changed, 21 insertions(+)

fs/kernfs/dir.c +1 -0

index 78f3403..43aa979 100644
@@ -711,6 +711,7 @@ struct kernfs_root *kernfs_create_root(struct kernfs_syscall_ops *scops,
 		return ERR_PTR(-ENOMEM);
 
 	ida_init(&root->ino_ida);
+	INIT_LIST_HEAD(&root->supers);
 
 	kn = __kernfs_new_node(root, "", S_IFDIR | S_IRUGO | S_IXUGO,
 			       KERNFS_DIR);

fs/kernfs/kernfs-internal.h +5 -0

index 8be13b2..dc84a3e 100644
@@ -49,6 +49,8 @@ static inline struct kernfs_root *kernfs_root(struct kernfs_node *kn)
  * mount.c
  */
 struct kernfs_super_info {
+	struct super_block	*sb;
+
 	/*
 	 * The root associated with this super_block.  Each super_block is
 	 * identified by the root and ns it's associated with.

@@ -62,6 +64,9 @@ struct kernfs_super_info {
 	 * an array and compare kernfs_node tag against every entry.
 	 */
 	const void		*ns;
+
+	/* anchored at kernfs_root->supers, protected by kernfs_mutex */
+	struct list_head	node;
 };
 #define kernfs_info(SB) ((struct kernfs_super_info *)(SB->s_fs_info))

fs/kernfs/mount.c +11 -0

index 6a5f04a..f25a7c0 100644
@@ -68,6 +68,7 @@ static int kernfs_fill_super(struct super_block *sb)
 	struct inode *inode;
 	struct dentry *root;
 
+	info->sb = sb;
 	sb->s_blocksize = PAGE_CACHE_SIZE;
 	sb->s_blocksize_bits = PAGE_CACHE_SHIFT;
 	sb->s_magic = SYSFS_MAGIC;

@@ -166,12 +167,18 @@ struct dentry *kernfs_mount_ns(struct file_system_type *fs_type, int flags,
 		*new_sb_created = !sb->s_root;
 
 	if (!sb->s_root) {
+		struct kernfs_super_info *info = kernfs_info(sb);
+
 		error = kernfs_fill_super(sb);
 		if (error) {
 			deactivate_locked_super(sb);
 			return ERR_PTR(error);
 		}
 		sb->s_flags |= MS_ACTIVE;
+
+		mutex_lock(&kernfs_mutex);
+		list_add(&info->node, &root->supers);
+		mutex_unlock(&kernfs_mutex);
 	}
 
 	return dget(sb->s_root);

@@ -190,6 +197,10 @@ void kernfs_kill_sb(struct super_block *sb)
 	struct kernfs_super_info *info = kernfs_info(sb);
 	struct kernfs_node *root_kn = sb->s_root->d_fsdata;
 
+	mutex_lock(&kernfs_mutex);
+	list_del(&info->node);
+	mutex_unlock(&kernfs_mutex);
+
 	/*
 	 * Remove the superblock from fs_supers/s_instances
 	 * so we can't find it, before freeing kernfs_super_info.

include/linux/kernfs.h +4 -0

index b0122dc..589318b 100644
@@ -144,6 +144,10 @@ struct kernfs_root {
 	/* private fields, do not use outside kernfs proper */
 	struct ida		ino_ida;
 	struct kernfs_syscall_ops *syscall_ops;
+
+	/* list of kernfs_super_info of this root, protected by kernfs_mutex */
+	struct list_head	supers;
+
 	wait_queue_head_t	deactivate_waitq;
 };

Signature · 1 line

1.9.0

Tejun Heo

author maintainer

2014-04-14 21:44 UTC #2 lore raw

PATCH 2/3 kernfs: make kernfs_notify() trigger inotify events too

In reply to Tejun Heo

kernfs_notify() is used to indicate either new data is available or the content of a file has changed. It currently only triggers poll which may not be the most convenient to monitor especially when there are a lot to monitor. Let's hook it up to fsnotify too so that the events can be monitored via inotify too.

fsnotify_modify() requires file * but kernfs_notify() doesn't have any specific file associated; however, we can walk all super_blocks associated with a kernfs_root and as kernfs always associate one ino with inode and one dentry with an inode, it's trivial to look up the dentry associated with a given kernfs_node. As any active monitor would pin dentry, just looking up existing dentry is enough. This patch looks up the dentry associated with the specified kernfs_node and generates events equivalent to fsnotify_modify().

Note that as fsnotify doesn't provide fsnotify_modify() equivalent which can be called with dentry, kernfs_notify() directly calls fsnotify_parent() and fsnotify(). It might be better to add a wrapper in fsnotify.h instead.

Signed-off-by Tejun Heo <[email protected]>

Cc Greg Kroah-Hartman <[email protected]>

Cc John McCutchan <john-jueV0HHMeujJJrXXpGQQMAC/[email protected]>

Cc Robert Love <[email protected]>

Cc Eric Paris <[email protected]>

---
 fs/kernfs/file.c | 41 +++++++++++++++++++++++++++++++++++------
 1 file changed, 35 insertions(+), 6 deletions(-)

fs/kernfs/file.c +35 -6

index 8034706..98bacd9 100644
@@ -14,6 +14,7 @@
 #include <linux/poll.h>
 #include <linux/pagemap.h>
 #include <linux/sched.h>
+#include <linux/fsnotify.h>
 
 #include "kernfs-internal.h"

@@ -785,20 +786,48 @@ static unsigned int kernfs_fop_poll(struct file *filp, poll_table *wait)
  */
 void kernfs_notify(struct kernfs_node *kn)
 {
+	struct kernfs_root *root = kernfs_root(kn);
 	struct kernfs_open_node *on;
+	struct kernfs_super_info *info;
 	unsigned long flags;
 
+	if (WARN_ON(kernfs_type(kn) != KERNFS_FILE))
+		return;
+
+	/* kick poll */
 	spin_lock_irqsave(&kernfs_open_node_lock, flags);
 
-	if (!WARN_ON(kernfs_type(kn) != KERNFS_FILE)) {
-		on = kn->attr.open;
-		if (on) {
-			atomic_inc(&on->event);
-			wake_up_interruptible(&on->poll);
-		}
+	on = kn->attr.open;
+	if (on) {
+		atomic_inc(&on->event);
+		wake_up_interruptible(&on->poll);
 	}
 
 	spin_unlock_irqrestore(&kernfs_open_node_lock, flags);
+
+	/* kick fsnotify */
+	mutex_lock(&kernfs_mutex);
+
+	list_for_each_entry(info, &root->supers, node) {
+		struct inode *inode;
+		struct dentry *dentry;
+
+		inode = ilookup(info->sb, kn->ino);
+		if (!inode)
+			continue;
+
+		dentry = d_find_any_alias(inode);
+		if (dentry) {
+			fsnotify_parent(NULL, dentry, FS_MODIFY);
+			fsnotify(inode, FS_MODIFY, inode, FSNOTIFY_EVENT_INODE,
+				 NULL, 0);
+			dput(dentry);
+		}
+
+		iput(inode);
+	}
+
+	mutex_unlock(&kernfs_mutex);
 }
 EXPORT_SYMBOL_GPL(kernfs_notify);

Signature · 1 line

1.9.0

Tejun Heo

author maintainer

2014-04-14 21:44 UTC #3 lore raw

PATCH 3/3 cgroup: implement cgroup.subtree_populated for the default hierarchy

In reply to Tejun Heo

* It delivers events by forking and execing a userland binary specified as the release_agent. This is a long deprecated method of notification delivery. It's extremely heavy, slow and cumbersome to integrate with larger infrastructure.

* There is single monitoring point at the root. There's no way to delegate management of subtree.

* The event isn't recursive. It triggers when a cgroup doesn't have any tasks or child cgroups. Events for internal nodes trigger only after all children are removed. This again makes it impossible to delegate management of subtree.

* Events are filtered from the kernel side. "notify_on_release" file is used to subscribe to or suppress release event. This is unnecessarily complicated and probably done this way because event delivery itself was expensive.

This patch implements interface file "cgroup.subtree_populated" which can be used to monitor whether the cgroup's subhierarchy has tasks in it or not. Its value is 0 if there is no task in the cgroup and its descendants; otherwise, 1, and kernfs_notify() notificaiton is triggers when the value changes, which can be monitored through poll and [di]notify.

This is a lot ligther and simpler and trivially allows delegating management of subhierarchy - subhierarchy monitoring can block further propgation simply by putting itself or another process in the root of the subhierarchy and monitor events that it's interested in from there without interfering with monitoring higher in the tree.

v2: Patch description updated as per Serge.

Signed-off-by Tejun Heo <[email protected]>

Acked-by Serge Hallyn <serge.hallyn-GeWIH/[email protected]>

Cc Lennart Poettering <[email protected]>

---
 include/linux/cgroup.h | 15 ++++++++++++
 kernel/cgroup.c        | 65 ++++++++++++++++++++++++++++++++++++++++++++++----
 2 files changed, 76 insertions(+), 4 deletions(-)

include/linux/cgroup.h +15 -0

index ada2392..4b38e2d 100644
@@ -154,6 +154,14 @@ struct cgroup {
 	/* the number of attached css's */
 	int nr_css;
 
+	/*
+	 * If this cgroup contains any tasks, it contributes one to
+	 * populated_cnt.  All children with non-zero popuplated_cnt of
+	 * their own contribute one.  The count is zero iff there's no task
+	 * in this cgroup or its subtree.
+	 */
+	int populated_cnt;
+
 	atomic_t refcnt;
 
 	/*

@@ -166,6 +174,7 @@ struct cgroup {
 	struct cgroup *parent;		/* my parent */
 	struct kernfs_node *kn;		/* cgroup kernfs entry */
 	struct kernfs_node *control_kn;	/* kn for "cgroup.subtree_control" */
+	struct kernfs_node *populated_kn; /* kn for "cgroup.subtree_populated" */
 
 	/*
 	 * Monotonically increasing unique serial number which defines a

@@ -264,6 +273,12 @@ enum {
 	 *
 	 * - "cgroup.clone_children" is removed.
 	 *
+	 * - "cgroup.subtree_populated" is available.  Its value is 0 if
+	 *   the cgroup and its descendants contain no task; otherwise, 1.
+	 *   The file also generates kernfs notification which can be
+	 *   monitored through poll and [di]notify when the value of the
+	 *   file changes.
+	 *
 	 * - If mount is requested with sane_behavior but without any
 	 *   subsystem, the default unified hierarchy is mounted.
 	 *

kernel/cgroup.c +61 -4

102 lines not on this page · raw message

Signature · 1 line

1.9.0

Li Zefan

2014-04-15 00:57 UTC #4 lore raw

Re: [PATCH 3/3] cgroup: implement cgroup.subtree_populated for the default hierarchy

Quoted 23 lines · Tejun Heo

cgroup users often need a way to determine when a cgroup's subhierarchy becomes empty so that it can be cleaned up. cgroup currently provides release_agent for it; unfortunately, this mechanism is riddled with issues.
* It delivers events by forking and execing a userland binary specified as the release_agent. This is a long deprecated method of notification delivery. It's extremely heavy, slow and cumbersome to integrate with larger infrastructure.
* There is single monitoring point at the root. There's no way to delegate management of subtree.
* The event isn't recursive. It triggers when a cgroup doesn't have any tasks or child cgroups. Events for internal nodes trigger only after all children are removed. This again makes it impossible to delegate management of subtree.
* Events are filtered from the kernel side. "notify_on_release" file is used to subscribe to or suppress release event. This is unnecessarily complicated and probably done this way because event delivery itself was expensive.

Quoted 4 lines

This patch implements interface file "cgroup.subtree_populated" which can be used to monitor whether the cgroup's subhierarchy has tasks in it or not. Its value is 0 if there is no task in the cgroup and its descendants; otherwise, 1,

Is cgroup.tree_populated a better name?

cgroup.subtree_control controls child cgroups only, but .subtree_populated shows 1 if there're tasks in the cgroup or its children, so the two are a bit inconsistent to me.

Quoted 15 lines

and kernfs_notify() notificaiton is
triggers when the value changes, which can be monitored through poll and [di]notify.
This is a lot ligther and simpler and trivially allows delegating management of subhierarchy - subhierarchy monitoring can block further propgation simply by putting itself or another process in the root of the subhierarchy and monitor events that it's interested in from there without interfering with monitoring higher in the tree.
v2: Patch description updated as per Serge.
Signed-off-by Tejun Heo <[email protected]>
Acked-by Serge Hallyn <serge.hallyn-GeWIH/[email protected]>
Cc Lennart Poettering <[email protected]>

Tejun Heo

author maintainer

2014-04-15 14:54 UTC #5 lore raw

Re: [PATCH 3/3] cgroup: implement cgroup.subtree_populated for the default hierarchy

Hello,

Quoted 5 lines · Li Zefan

Is cgroup.tree_populated a better name?
cgroup.subtree_control controls child cgroups only, but .subtree_populated shows 1 if there're tasks in the cgroup or its children, so the two are a bit inconsistent to me.

Yes, good catch. subtree_control affects subtree proper. subtree_populated covers self too. The difference is subtle and the trade off between shared pattern in names and clarifying the subtlety didn't seem clear-cut to me. Hmmm....

Signature · 1 line

tejun

Tejun Heo

author maintainer

2014-04-15 16:52 UTC #6 lore raw

Re: [PATCH 3/3] cgroup: implement cgroup.subtree_populated for the default hierarchy

Quoted 5 lines · Tejun Heo

Hello,
Quoted 2 lines · Li Zefan
Is cgroup.tree_populated a better name?

Quoted 8 lines

Quoted 3 lines
cgroup.subtree_control controls child cgroups only, but .subtree_populated shows 1 if there're tasks in the cgroup or its children, so the two are a bit inconsistent to me.
Yes, good catch. subtree_control affects subtree proper. subtree_populated covers self too. The difference is subtle and the trade off between shared pattern in names and clarifying the subtlety didn't seem clear-cut to me. Hmmm....

How about cgroup.populated?

Signature · 1 line

tejun

Li Zefan

2014-04-16 01:30 UTC #7 lore raw

Re: [PATCH 3/3] cgroup: implement cgroup.subtree_populated for the default hierarchy

Quoted 10 lines · Tejun Heo

Quoted 9 lines · Tejun Heo
Hello,
Quoted 5 lines · Li Zefan
Is cgroup.tree_populated a better name?
cgroup.subtree_control controls child cgroups only, but .subtree_populated shows 1 if there're tasks in the cgroup or its children, so the two are a bit inconsistent to me.

Quoted 7 lines

Quoted 4 lines
Yes, good catch. subtree_control affects subtree proper. subtree_populated covers self too. The difference is subtle and the trade off between shared pattern in names and clarifying the subtlety didn't seem clear-cut to me. Hmmm....
How about cgroup.populated?

Yeah, fine for me.

A message that is not in the archive (sent off-list or not yet received): 20140415165221.GD30990-Gd/HAXX7CRxy/[email protected]

Li Zefan

2014-04-16 01:30 UTC #9 lore raw

Re: [PATCH 3/3] cgroup: implement cgroup.subtree_populated for the default hierarchy

Quoted 10 lines · Tejun Heo

Quoted 9 lines · Tejun Heo
Hello,
Quoted 5 lines · Li Zefan
Is cgroup.tree_populated a better name?
cgroup.subtree_control controls child cgroups only, but .subtree_populated shows 1 if there're tasks in the cgroup or its children, so the two are a bit inconsistent to me.

Quoted 7 lines

Quoted 4 lines
Yes, good catch. subtree_control affects subtree proper. subtree_populated covers self too. The difference is subtle and the trade off between shared pattern in names and clarifying the subtlety didn't seem clear-cut to me. Hmmm....
How about cgroup.populated?

Yeah, fine for me.

A message that is not in the archive (sent off-list or not yet received): [email protected]

Tejun Heo

2014-04-15 14:54 UTC #11 lore raw

Re: [PATCH 3/3] cgroup: implement cgroup.subtree_populated for the default hierarchy

Hello,

Quoted 5 lines · Li Zefan

Is cgroup.tree_populated a better name?
cgroup.subtree_control controls child cgroups only, but .subtree_populated shows 1 if there're tasks in the cgroup or its children, so the two are a bit inconsistent to me.

Signature · 1 line

tejun

Li Zefan

2014-04-16 02:48 UTC #12 lore raw

Re: [PATCH 3/3] cgroup: implement cgroup.subtree_populated for the default hierarchy

In reply to Tejun Heo

Hi Tejun,

Quoted 23 lines · Tejun Heo

cgroup users often need a way to determine when a cgroup's subhierarchy becomes empty so that it can be cleaned up. cgroup currently provides release_agent for it; unfortunately, this mechanism is riddled with issues.
* It delivers events by forking and execing a userland binary specified as the release_agent. This is a long deprecated method of notification delivery. It's extremely heavy, slow and cumbersome to integrate with larger infrastructure.
* There is single monitoring point at the root. There's no way to delegate management of subtree.
* The event isn't recursive. It triggers when a cgroup doesn't have any tasks or child cgroups. Events for internal nodes trigger only after all children are removed. This again makes it impossible to delegate management of subtree.
* Events are filtered from the kernel side. "notify_on_release" file is used to subscribe to or suppress release event. This is unnecessarily complicated and probably done this way because event delivery itself was expensive.

Quoted 7 lines

This patch implements interface file "cgroup.subtree_populated" which can be used to monitor whether the cgroup's subhierarchy has tasks in it or not. Its value is 0 if there is no task in the cgroup and its descendants; otherwise, 1, and kernfs_notify() notificaiton is triggers when the value changes, which can be monitored through poll and [di]notify.

For the old notification mechanism, the path of the cgroup that becomes empty will be passed to the user specified release agent. Like this:

# cat /sbin/cpuset_release_agent
#!/bin/sh
rmdir /dev/cpuset/$1

How do we achieve this using inotify?

- monitor all the cgroups, or
- monitor all the leaf cgroups, and travel cgrp->parent to delete all empty cgroups.
- monitor root cgroup only, and travel the whole hierarchy to find empy cgroups when it gets an fs event.

Seems none of them is scalible.

Quoted 8 lines

This is a lot ligther and simpler and trivially allows delegating management of subhierarchy - subhierarchy monitoring can block further propgation simply by putting itself or another process in the root of the subhierarchy and monitor events that it's interested in from there without interfering with monitoring higher in the tree.
v2: Patch description updated as per Serge.

Kay Sievers

2014-04-16 03:33 UTC #13 lore raw

Re: [PATCH 3/3] cgroup: implement cgroup.subtree_populated for the default hierarchy

Quoted 41 lines · Li Zefan

Quoted 23 lines · Tejun Heo
cgroup users often need a way to determine when a cgroup's subhierarchy becomes empty so that it can be cleaned up. cgroup currently provides release_agent for it; unfortunately, this mechanism is riddled with issues.
* It delivers events by forking and execing a userland binary specified as the release_agent. This is a long deprecated method of notification delivery. It's extremely heavy, slow and cumbersome to integrate with larger infrastructure.
* There is single monitoring point at the root. There's no way to delegate management of subtree.
* The event isn't recursive. It triggers when a cgroup doesn't have any tasks or child cgroups. Events for internal nodes trigger only after all children are removed. This again makes it impossible to delegate management of subtree.
* Events are filtered from the kernel side. "notify_on_release" file is used to subscribe to or suppress release event. This is unnecessarily complicated and probably done this way because event delivery itself was expensive.
Quoted 7 lines
This patch implements interface file "cgroup.subtree_populated" which can be used to monitor whether the cgroup's subhierarchy has tasks in it or not. Its value is 0 if there is no task in the cgroup and its descendants; otherwise, 1, and kernfs_notify() notificaiton is triggers when the value changes, which can be monitored through poll and [di]notify.
For the old notification mechanism, the path of the cgroup that becomes empty will be passed to the user specified release agent. Like this:
# cat /sbin/cpuset_release_agent
#!/bin/sh
rmdir /dev/cpuset/$1
How do we achieve this using inotify?

Quoted 7 lines

- monitor all the cgroups, or
- monitor all the leaf cgroups, and travel cgrp->parent to delete all empty cgroups.
- monitor root cgroup only, and travel the whole hierarchy to find empy cgroups when it gets an fs event.
Seems none of them is scalible.

The manager would add all cgroups as watches to one inotify file descriptor, it should not be problem to do that.

Kay

Eric W. Biederman

2014-04-16 03:50 UTC #14 lore raw

Re: [PATCH 3/3] cgroup: implement cgroup.subtree_populated for the default hierarchy

Quoted 48 lines · Kay Sievers

Quoted 47 lines · Li Zefan
Quoted 23 lines · Tejun Heo
cgroup users often need a way to determine when a cgroup's subhierarchy becomes empty so that it can be cleaned up. cgroup currently provides release_agent for it; unfortunately, this mechanism is riddled with issues.
* It delivers events by forking and execing a userland binary specified as the release_agent. This is a long deprecated method of notification delivery. It's extremely heavy, slow and cumbersome to integrate with larger infrastructure.
* There is single monitoring point at the root. There's no way to delegate management of subtree.
* The event isn't recursive. It triggers when a cgroup doesn't have any tasks or child cgroups. Events for internal nodes trigger only after all children are removed. This again makes it impossible to delegate management of subtree.
* Events are filtered from the kernel side. "notify_on_release" file is used to subscribe to or suppress release event. This is unnecessarily complicated and probably done this way because event delivery itself was expensive.
Quoted 7 lines
This patch implements interface file "cgroup.subtree_populated" which can be used to monitor whether the cgroup's subhierarchy has tasks in it or not. Its value is 0 if there is no task in the cgroup and its descendants; otherwise, 1, and kernfs_notify() notificaiton is triggers when the value changes, which can be monitored through poll and [di]notify.
For the old notification mechanism, the path of the cgroup that becomes empty will be passed to the user specified release agent. Like this:
# cat /sbin/cpuset_release_agent
#!/bin/sh
rmdir /dev/cpuset/$1
How do we achieve this using inotify?
- monitor all the cgroups, or
- monitor all the leaf cgroups, and travel cgrp->parent to delete all empty cgroups.
- monitor root cgroup only, and travel the whole hierarchy to find empy cgroups when it gets an fs event.

Quoted 4 lines

Quoted 1 line
Seems none of them is scalible.
The manager would add all cgroups as watches to one inotify file descriptor, it should not be problem to do that.

inotify won't work on cgroupfs.

Eric

Kay Sievers

2014-04-16 04:15 UTC #15 lore raw

Re: [PATCH 3/3] cgroup: implement cgroup.subtree_populated for the default hierarchy

Quoted 50 lines · Eric W. Biederman

Quoted 48 lines · Kay Sievers
Quoted 47 lines · Li Zefan
Quoted 23 lines · Tejun Heo
cgroup users often need a way to determine when a cgroup's subhierarchy becomes empty so that it can be cleaned up. cgroup currently provides release_agent for it; unfortunately, this mechanism is riddled with issues.
* It delivers events by forking and execing a userland binary specified as the release_agent. This is a long deprecated method of notification delivery. It's extremely heavy, slow and cumbersome to integrate with larger infrastructure.
* There is single monitoring point at the root. There's no way to delegate management of subtree.
* The event isn't recursive. It triggers when a cgroup doesn't have any tasks or child cgroups. Events for internal nodes trigger only after all children are removed. This again makes it impossible to delegate management of subtree.
* Events are filtered from the kernel side. "notify_on_release" file is used to subscribe to or suppress release event. This is unnecessarily complicated and probably done this way because event delivery itself was expensive.
Quoted 7 lines
This patch implements interface file "cgroup.subtree_populated" which can be used to monitor whether the cgroup's subhierarchy has tasks in it or not. Its value is 0 if there is no task in the cgroup and its descendants; otherwise, 1, and kernfs_notify() notificaiton is triggers when the value changes, which can be monitored through poll and [di]notify.
For the old notification mechanism, the path of the cgroup that becomes empty will be passed to the user specified release agent. Like this:
# cat /sbin/cpuset_release_agent
#!/bin/sh
rmdir /dev/cpuset/$1
How do we achieve this using inotify?
- monitor all the cgroups, or
- monitor all the leaf cgroups, and travel cgrp->parent to delete all empty cgroups.
- monitor root cgroup only, and travel the whole hierarchy to find empy cgroups when it gets an fs event.

Quoted 6 lines

Quoted 4 lines
Quoted 1 line
Seems none of them is scalible.
The manager would add all cgroups as watches to one inotify file descriptor, it should not be problem to do that.
inotify won't work on cgroupfs.

Inotify on kernfs will work.

Kay

Li Zefan

2014-04-16 04:20 UTC #16 lore raw

Re: [PATCH 3/3] cgroup: implement cgroup.subtree_populated for the default hierarchy

In reply to Eric W. Biederman

Quoted 50 lines · Eric W. Biederman

Quoted 48 lines · Kay Sievers
Quoted 47 lines · Li Zefan
Quoted 23 lines · Tejun Heo
cgroup users often need a way to determine when a cgroup's subhierarchy becomes empty so that it can be cleaned up. cgroup currently provides release_agent for it; unfortunately, this mechanism is riddled with issues.
* It delivers events by forking and execing a userland binary specified as the release_agent. This is a long deprecated method of notification delivery. It's extremely heavy, slow and cumbersome to integrate with larger infrastructure.
* There is single monitoring point at the root. There's no way to delegate management of subtree.
* The event isn't recursive. It triggers when a cgroup doesn't have any tasks or child cgroups. Events for internal nodes trigger only after all children are removed. This again makes it impossible to delegate management of subtree.
* Events are filtered from the kernel side. "notify_on_release" file is used to subscribe to or suppress release event. This is unnecessarily complicated and probably done this way because event delivery itself was expensive.
Quoted 7 lines
This patch implements interface file "cgroup.subtree_populated" which can be used to monitor whether the cgroup's subhierarchy has tasks in it or not. Its value is 0 if there is no task in the cgroup and its descendants; otherwise, 1, and kernfs_notify() notificaiton is triggers when the value changes, which can be monitored through poll and [di]notify.
For the old notification mechanism, the path of the cgroup that becomes empty will be passed to the user specified release agent. Like this:
# cat /sbin/cpuset_release_agent
#!/bin/sh
rmdir /dev/cpuset/$1
How do we achieve this using inotify?
- monitor all the cgroups, or
- monitor all the leaf cgroups, and travel cgrp->parent to delete all empty cgroups.
- monitor root cgroup only, and travel the whole hierarchy to find empy cgroups when it gets an fs event.

Quoted 7 lines

Quoted 4 lines
Quoted 1 line
Seems none of them is scalible.
The manager would add all cgroups as watches to one inotify file descriptor, it should not be problem to do that.
inotify won't work on cgroupfs.

Tejun's working on inotify support for cgroupfs, and I believe this patchset has been tested, so it works.

So what do you mean by saying it won't work? Could you be more specific?

A message that is not in the archive (sent off-list or not yet received): CAPXgP12kvPdX0QExwN2JphDfEW=d+7K2c_Y8DbomGd=YVy=VGQ-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org

Eric W. Biederman

2014-04-16 03:50 UTC #18 lore raw

Re: [PATCH 3/3] cgroup: implement cgroup.subtree_populated for the default hierarchy

Quoted 48 lines · Kay Sievers

Quoted 47 lines · Li Zefan
Quoted 23 lines · Tejun Heo
cgroup users often need a way to determine when a cgroup's subhierarchy becomes empty so that it can be cleaned up. cgroup currently provides release_agent for it; unfortunately, this mechanism is riddled with issues.
* It delivers events by forking and execing a userland binary specified as the release_agent. This is a long deprecated method of notification delivery. It's extremely heavy, slow and cumbersome to integrate with larger infrastructure.
* There is single monitoring point at the root. There's no way to delegate management of subtree.
* The event isn't recursive. It triggers when a cgroup doesn't have any tasks or child cgroups. Events for internal nodes trigger only after all children are removed. This again makes it impossible to delegate management of subtree.
* Events are filtered from the kernel side. "notify_on_release" file is used to subscribe to or suppress release event. This is unnecessarily complicated and probably done this way because event delivery itself was expensive.
Quoted 7 lines
This patch implements interface file "cgroup.subtree_populated" which can be used to monitor whether the cgroup's subhierarchy has tasks in it or not. Its value is 0 if there is no task in the cgroup and its descendants; otherwise, 1, and kernfs_notify() notificaiton is triggers when the value changes, which can be monitored through poll and [di]notify.
For the old notification mechanism, the path of the cgroup that becomes empty will be passed to the user specified release agent. Like this:
# cat /sbin/cpuset_release_agent
#!/bin/sh
rmdir /dev/cpuset/$1
How do we achieve this using inotify?
- monitor all the cgroups, or
- monitor all the leaf cgroups, and travel cgrp->parent to delete all empty cgroups.
- monitor root cgroup only, and travel the whole hierarchy to find empy cgroups when it gets an fs event.

Quoted 4 lines

Quoted 1 line
Seems none of them is scalible.
The manager would add all cgroups as watches to one inotify file descriptor, it should not be problem to do that.

inotify won't work on cgroupfs.

Eric

Li Zefan

2014-04-16 04:16 UTC #19 lore raw

Re: [PATCH 3/3] cgroup: implement cgroup.subtree_populated for the default hierarchy

In reply to Kay Sievers

Quoted 48 lines · Kay Sievers

Quoted 47 lines · Li Zefan
Quoted 23 lines · Tejun Heo
cgroup users often need a way to determine when a cgroup's subhierarchy becomes empty so that it can be cleaned up. cgroup currently provides release_agent for it; unfortunately, this mechanism is riddled with issues.
* It delivers events by forking and execing a userland binary specified as the release_agent. This is a long deprecated method of notification delivery. It's extremely heavy, slow and cumbersome to integrate with larger infrastructure.
* There is single monitoring point at the root. There's no way to delegate management of subtree.
* The event isn't recursive. It triggers when a cgroup doesn't have any tasks or child cgroups. Events for internal nodes trigger only after all children are removed. This again makes it impossible to delegate management of subtree.
* Events are filtered from the kernel side. "notify_on_release" file is used to subscribe to or suppress release event. This is unnecessarily complicated and probably done this way because event delivery itself was expensive.
Quoted 7 lines
This patch implements interface file "cgroup.subtree_populated" which can be used to monitor whether the cgroup's subhierarchy has tasks in it or not. Its value is 0 if there is no task in the cgroup and its descendants; otherwise, 1, and kernfs_notify() notificaiton is triggers when the value changes, which can be monitored through poll and [di]notify.
For the old notification mechanism, the path of the cgroup that becomes empty will be passed to the user specified release agent. Like this:
# cat /sbin/cpuset_release_agent
#!/bin/sh
rmdir /dev/cpuset/$1
How do we achieve this using inotify?
- monitor all the cgroups, or
- monitor all the leaf cgroups, and travel cgrp->parent to delete all empty cgroups.
- monitor root cgroup only, and travel the whole hierarchy to find empy cgroups when it gets an fs event.

Quoted 5 lines

Quoted 1 line
Seems none of them is scalible.
The manager would add all cgroups as watches to one inotify file descriptor, it should not be problem to do that.

Never use inotify. Thanks for explanation, so I think inotify can scale to thounsands of cgroups after I googled a bit.

Tejun Heo

author maintainer

2014-04-16 14:50 UTC #20 lore raw

PATCH v3 3/3 cgroup: implement cgroup.populated for the default hierarchy

In reply to Tejun Heo

Hello,

Renamed to cgroup.populated. git branch updated accordingly.

Thanks.
------- 8< -------
From bf3fd307267ce307c4c8ec41834d7bddb6441907 Mon Sep 17 00:00:00 2001
From: Tejun Heo <[email protected]>
Date: Wed, 16 Apr 2014 10:48:38 -0400

* There is single monitoring point at the root. There's no way to delegate management of a subtree.

This patch implements interface file "cgroup.populated" which can be used to monitor whether the cgroup's subhierarchy has tasks in it or not. Its value is 0 if there is no task in the cgroup and its descendants; otherwise, 1, and kernfs_notify() notificaiton is triggers when the value changes, which can be monitored through poll and [di]notify.

v2: Patch description updated as per Serge.

v3: "cgroup.subtree_populated" renamed to "cgroup.populated". The
subtree_ prefix was a bit confusing because "cgroup.subtree_control" uses it to denote the tree rooted at the cgroup sans the cgroup itself while the populated state includes the cgroup itself.

Signed-off-by Tejun Heo <[email protected]>

Acked-by Serge Hallyn <serge.hallyn-GeWIH/[email protected]>

Cc Lennart Poettering <[email protected]>

---
 include/linux/cgroup.h | 15 ++++++++++++
 kernel/cgroup.c        | 65 ++++++++++++++++++++++++++++++++++++++++++++++----
 2 files changed, 76 insertions(+), 4 deletions(-)

include/linux/cgroup.h +15 -0

index ada2392..4b38e2d 100644
@@ -154,6 +154,14 @@ struct cgroup {
 	/* the number of attached css's */
 	int nr_css;
 
+	/*
+	 * If this cgroup contains any tasks, it contributes one to
+	 * populated_cnt.  All children with non-zero popuplated_cnt of
+	 * their own contribute one.  The count is zero iff there's no task
+	 * in this cgroup or its subtree.
+	 */
+	int populated_cnt;
+
 	atomic_t refcnt;
 
 	/*

@@ -166,6 +174,7 @@ struct cgroup {
 	struct cgroup *parent;		/* my parent */
 	struct kernfs_node *kn;		/* cgroup kernfs entry */
 	struct kernfs_node *control_kn;	/* kn for "cgroup.subtree_control" */
+	struct kernfs_node *populated_kn; /* kn for "cgroup.subtree_populated" */
 
 	/*
 	 * Monotonically increasing unique serial number which defines a

@@ -264,6 +273,12 @@ enum {
 	 *
 	 * - "cgroup.clone_children" is removed.
 	 *
+	 * - "cgroup.subtree_populated" is available.  Its value is 0 if
+	 *   the cgroup and its descendants contain no task; otherwise, 1.
+	 *   The file also generates kernfs notification which can be
+	 *   monitored through poll and [di]notify when the value of the
+	 *   file changes.
+	 *
 	 * - If mount is requested with sane_behavior but without any
 	 *   subsystem, the default unified hierarchy is mounted.
 	 *

kernel/cgroup.c +61 -4

102 lines not on this page · raw message

Signature · 1 line

1.9.0

Li Zefan

Acked-by on v3 3/3

2014-04-17 01:23 UTC #21 lore raw

Re: [PATCH v3 3/3] cgroup: implement cgroup.populated for the default hierarchy

Quoted 44 lines

cgroup users often need a way to determine when a cgroup's subhierarchy becomes empty so that it can be cleaned up. cgroup currently provides release_agent for it; unfortunately, this mechanism is riddled with issues.
* It delivers events by forking and execing a userland binary specified as the release_agent. This is a long deprecated method of notification delivery. It's extremely heavy, slow and cumbersome to integrate with larger infrastructure.
* There is single monitoring point at the root. There's no way to delegate management of a subtree.
* The event isn't recursive. It triggers when a cgroup doesn't have any tasks or child cgroups. Events for internal nodes trigger only after all children are removed. This again makes it impossible to delegate management of a subtree.
* Events are filtered from the kernel side. "notify_on_release" file is used to subscribe to or suppress release event. This is unnecessarily complicated and probably done this way because event delivery itself was expensive.
This patch implements interface file "cgroup.populated" which can be used to monitor whether the cgroup's subhierarchy has tasks in it or not. Its value is 0 if there is no task in the cgroup and its descendants; otherwise, 1, and kernfs_notify() notificaiton is triggers when the value changes, which can be monitored through poll and [di]notify.
This is a lot ligther and simpler and trivially allows delegating management of subhierarchy - subhierarchy monitoring can block further propgation simply by putting itself or another process in the root of the subhierarchy and monitor events that it's interested in from there without interfering with monitoring higher in the tree.
v2: Patch description updated as per Serge.
v3: "cgroup.subtree_populated" renamed to "cgroup.populated". The
subtree_ prefix was a bit confusing because "cgroup.subtree_control" uses it to denote the tree rooted at the cgroup sans the cgroup itself while the populated state includes the cgroup itself.

Quoted 3 lines

Signed-off-by Tejun Heo <[email protected]>
Acked-by Serge Hallyn <serge.hallyn-GeWIH/[email protected]>
Cc Lennart Poettering <[email protected]>

Acked-by Li Zefan <[email protected]>

A message that is not in the archive (sent off-list or not yet received): 1397511846-2904-4-git-send-email-tj-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org

Tejun Heo

2014-04-16 14:50 UTC #23 lore raw

PATCH v3 3/3 cgroup: implement cgroup.populated for the default hierarchy

Hello,

Renamed to cgroup.populated. git branch updated accordingly.

Thanks.
------- 8< -------
From bf3fd307267ce307c4c8ec41834d7bddb6441907 Mon Sep 17 00:00:00 2001
From: Tejun Heo <[email protected]>
Date: Wed, 16 Apr 2014 10:48:38 -0400

* There is single monitoring point at the root. There's no way to delegate management of a subtree.

v2: Patch description updated as per Serge.

Signed-off-by Tejun Heo <[email protected]>

Acked-by Serge Hallyn <serge.hallyn-GeWIH/[email protected]>

Cc Lennart Poettering <[email protected]>

---
 include/linux/cgroup.h | 15 ++++++++++++
 kernel/cgroup.c        | 65 ++++++++++++++++++++++++++++++++++++++++++++++----
 2 files changed, 76 insertions(+), 4 deletions(-)

include/linux/cgroup.h +15 -0

index ada2392..4b38e2d 100644
@@ -154,6 +154,14 @@ struct cgroup {
 	/* the number of attached css's */
 	int nr_css;
 
+	/*
+	 * If this cgroup contains any tasks, it contributes one to
+	 * populated_cnt.  All children with non-zero popuplated_cnt of
+	 * their own contribute one.  The count is zero iff there's no task
+	 * in this cgroup or its subtree.
+	 */
+	int populated_cnt;
+
 	atomic_t refcnt;
 
 	/*

@@ -166,6 +174,7 @@ struct cgroup {
 	struct cgroup *parent;		/* my parent */
 	struct kernfs_node *kn;		/* cgroup kernfs entry */
 	struct kernfs_node *control_kn;	/* kn for "cgroup.subtree_control" */
+	struct kernfs_node *populated_kn; /* kn for "cgroup.subtree_populated" */
 
 	/*
 	 * Monotonically increasing unique serial number which defines a

@@ -264,6 +273,12 @@ enum {
 	 *
 	 * - "cgroup.clone_children" is removed.
 	 *
+	 * - "cgroup.subtree_populated" is available.  Its value is 0 if
+	 *   the cgroup and its descendants contain no task; otherwise, 1.
+	 *   The file also generates kernfs notification which can be
+	 *   monitored through poll and [di]notify when the value of the
+	 *   file changes.
+	 *
 	 * - If mount is requested with sane_behavior but without any
 	 *   subsystem, the default unified hierarchy is mounted.
 	 *

kernel/cgroup.c +61 -4

102 lines not on this page · raw message

Signature · 1 line

1.9.0