Generic container system
Subsystems (8)
cover Generic container system
There have recently been various proposals floating around for resource management/accounting subsystems in the kernel, including Res Groups, User BeanCounters and others. These all need the basic abstraction of being able to group together multiple processes in an aggregate, in order to track/limit the resources permitted to those processes, and all implement this grouping in different ways.
Already existing in the kernel is the cpuset subsystem; this has a process grouping mechanism that is mature, tested, and well documented (particularly with regards to synchronization rules).
This patchset extracts the process grouping code from cpusets into a generic container system, and makes the cpusets code a client of the container system.
It also provides a very simple additional container subsystem to do per-container CPU usage accounting; this is primarily to demonstrate use of the container subsystem API, but is useful in its own right.
The change is implemented in four stages:
1) extract the process grouping code from cpusets into a standalone system
2) remove the process grouping code from cpusets and hook into the container system
3) convert the container system to present a generic API, and make cpusets a client of that API
4) add a simple CPU accounting container subsystem
The intention is that the various resource management efforts can also become container clients, with the result that:
- the userspace APIs are (somewhat) normalised
- it's easier to test out e.g. the ResGroups CPU controller in conjunction with the UBC memory controller
- the additional kernel footprint of any of the competing resource management systems is substantially reduced, since it doesn't need to provide process grouping/containment, hence improving their chances of getting into the kernel
Possible TODOs include:
- define a convention for populating the per-container directories so that different subsystems don't clash with one another
- provide higher-level primitives (e.g. an easy interface to seq_file) for files registered by subsystems.
- support subsystem deregistering
---
RFC PATCH 1/4 Generic container system abstracted from cpusets code
This patch creates a generic process container system based on (and parallel top) the cpusets code. At a coarse level it was created by copying kernel/cpuset.c, doing s/cpuset/container/g, and stripping out any code that was cpuset-specific rather than applicable to any process container subsystem.
--- Documentation/containers.txt | 229 +++++++ fs/proc/base.c | 19 include/linux/container.h | 96 +++ include/linux/sched.h | 5 include/linux/utsrelease.h | 1 init/Kconfig | 9 init/main.c | 3 kernel/Makefile | 1 kernel/container.c | 1342 +++++++++++++++++++++++++++++++++++++++++++ kernel/exit.c | 2 kernel/fork.c | 3 11 files changed, 1709 insertions(+), 1 deletion(-)
fs/proc/base.c +19 -0
@@ -69,6 +69,7 @@#include <linux/ptrace.h> #include <linux/seccomp.h> #include <linux/cpuset.h> +#include <linux/container.h> #include <linux/audit.h> #include <linux/poll.h> #include "internal.h"
@@ -124,6 +125,9 @@ enum pid_directory_inos {#ifdef CONFIG_CPUSETS PROC_TGID_CPUSET,#endif +#ifdef CONFIG_CONTAINERS + PROC_TGID_CONTAINER, +#endif #ifdef CONFIG_SECURITY PROC_TGID_ATTR , PROC_TGID_ATTR_CURRENT,
@@ -167,6 +171,9 @@ enum pid_directory_inos {#ifdef CONFIG_CPUSETS PROC_TID_CPUSET,#endif +#ifdef CONFIG_CONTAINERS + PROC_TID_CONTAINER, +#endif #ifdef CONFIG_SECURITY PROC_TID_ATTR , PROC_TID_ATTR_CURRENT,
@@ -235,6 +242,9 @@ static struct pid_entry tgid_base_stuff[#ifdef CONFIG_CPUSETS E(PROC_TGID_CPUSET, "cpuset", S_IFREG|S_IRUGO),#endif +#ifdef CONFIG_CONTAINERS +E (PROC_TGID_CONTAINER ,"container" ,S_IFREG |S_IRUGO ), +#endifE (PROC_TGID_OOM_SCORE ,"oom_score" ,S_IFREG |S_IRUGO ),E (PROC_TGID_OOM_ADJUST ,"oom_adj" ,S_IFREG |S_IRUGO |S_IWUSR ), #ifdef CONFIG_AUDITSYSCALL
@@ -277,6 +287,9 @@ static struct pid_entry tid_base_stuff[]#ifdef CONFIG_CPUSETS E(PROC_TID_CPUSET, "cpuset", S_IFREG|S_IRUGO),#endif +#ifdef CONFIG_CONTAINERS +E (PROC_TID_CONTAINER ,"container" ,S_IFREG |S_IRUGO ), +#endifE (PROC_TID_OOM_SCORE ,"oom_score" ,S_IFREG |S_IRUGO ),E (PROC_TID_OOM_ADJUST ,"oom_adj" ,S_IFREG |S_IRUGO |S_IWUSR ), #ifdef CONFIG_AUDITSYSCALL
@@ -1819,6 +1832,12 @@ static struct dentry *proc_pident_lookup inode->i_fop = &proc_cpuset_operations;break ;#endif +#ifdef CONFIG_CONTAINERS +case PROC_TID_CONTAINER : +case PROC_TGID_CONTAINER : + inode->i_fop = &proc_container_operations; +break ; +#endif case PROC_TID_OOM_SCORE :case PROC_TGID_OOM_SCORE : inode->i_fop = &proc_info_file_operations;
include/linux/container.h new file +96 -0
96 lines not on this page · raw message
include/linux/sched.h +4 -1
@@ -710,8 +710,8 @@ extern unsigned int max_cache_size;struct io_context ;/* See blkdev.h */ void exit_io_context (void ); +struct container ;struct cpuset ; -#define NGROUPS_SMALL 32#define NGROUPS_PER_BLOCK ((int)(PAGE_SIZE / sizeof(gid_t)))struct group_info {
@@ -979,6 +979,9 @@ struct task_struct {int cpuset_mems_generation;int cpuset_mem_spread_rotor;#endif +#ifdef CONFIG_CONTAINERS +struct container *container; +#endif struct robust_list_head __user *robust_list;#ifdef CONFIG_COMPAT struct compat_robust_list_head __user *compat_robust_list;
include/linux/utsrelease.h new file +1 -0
@@ -0,0 +1 @@ +#define UTS_RELEASE "2.6.18"
init/Kconfig +9 -0
@@ -220,6 +220,15 @@ config IKCONFIG_PROC This option enables access to the kernel configuration file through /proc/config.gz. +config CONTAINERS + bool "Container support" + help + This option will let you create and manage process containers, + which can be used to aggregate multiple processes, e.g. for + the purposes of resource tracking. + + Say N if unsure + config CPUSETS bool "Cpuset support" depends on SMP
init/main.c +3 -0
@@ -40,6 +40,7 @@#include <linux/writeback.h> #include <linux/cpu.h> #include <linux/cpuset.h> +#include <linux/container.h> #include <linux/efi.h> #include <linux/taskstats_kern.h> #include <linux/delayacct.h>
@@ -545,6 +546,7 @@ asmlinkage void __init start_kernel(void }#endif vfs_caches_init_early (); +container_init_early ();cpuset_init_early ();mem_init ();kmem_cache_init ();
@@ -575,6 +577,7 @@ asmlinkage void __init start_kernel(void#ifdef CONFIG_PROC_FS proc_root_init ();#endif +container_init ();cpuset_init ();taskstats_init_early ();delayacct_init ();
kernel/container.c new file +1342 -0
1,342 lines not on this page · raw message
kernel/exit.c +2 -0
@@ -28,6 +28,7 @@#include <linux/taskstats_kern.h> #include <linux/delayacct.h> #include <linux/cpuset.h> +#include <linux/container.h> #include <linux/syscalls.h> #include <linux/signal.h> #include <linux/posix-timers.h>
@@ -919,6 +920,7 @@ fastcall NORET_TYPE void do_exit(long coexit_namespace (tsk);exit_thread ();cpuset_exit (tsk); +container_exit (tsk);exit_keys (tsk);if (group_dead && tsk->signal->leader)
kernel/fork.c +3 -0
@@ -30,6 +30,7 @@#include <linux/capability.h> #include <linux/cpu.h> #include <linux/cpuset.h> +#include <linux/container.h> #include <linux/security.h> #include <linux/swap.h> #include <linux/syscalls.h>
@@ -1044,6 +1045,7 @@ static struct task_struct *copy_process( p->io_context =NULL ; p->io_wait =NULL ; p->audit_context =NULL ; +container_fork (p);cpuset_fork (p);#ifdef CONFIG_NUMA p->mempolicy =mpol_copy (p->mempolicy);
@@ -1278,6 +1280,7 @@ bad_fork_cleanup_policy: bad_fork_cleanup_cpuset:#endif cpuset_exit (p); +container_exit (p); bad_fork_cleanup_delays_binfmt:delayacct_tsk_free (p);if (p->binfmt)
kernel/Makefile +1 -0
@@ -36,6 +36,7 @@ obj-$(CONFIG_PM) += power/ obj-$(CONFIG_BSD_PROCESS_ACCT) += acct.o obj-$(CONFIG_KEXEC) += kexec.o obj-$(CONFIG_COMPAT) += compat.o +obj-$(CONFIG_CONTAINERS) += container.o obj-$(CONFIG_CPUSETS) += cpuset.o obj-$(CONFIG_IKCONFIG) += configs.o obj-$(CONFIG_STOP_MACHINE) += stop_machine.o
Documentation/containers.txt new file +229 -0
229 lines not on this page · raw message
--
Re: [RFC][PATCH 1/4] Generic container system abstracted from cpusets code
Be sure to spell out, if you haven't already, that there is just a single container hierarchy, with each task attached to a single container in that hierarchy, and each subsystem (e.g. cpusets) adding its own special files to the common container hierarchy directories.
It is not the case that each subsystem using containers gets to concoct its own hierarchy, independent of any other subsystem.
init/Kconfig config IKCONFIG_PROC · line 228
Quoted 6 lines · Menage
init/Kconfig +6
-0+config CONTAINERS + bool "Container support" + help + This option will let you create and manage process containers, + which can be used to aggregate multiple processes, e.g. for + the purposes of resource tracking.
I wonder if this should be a configurable ... perhaps not.
If someone configures in one or more of the subsystems, such as cpusets, that requires CONTAINERS, then we need to enable containers. Otherwise we don't need to enable CONTAINERS. I don't see what user input we need here specific to CONTAINERS.
Quoted 2 lines
+ * Copyright (C) 2003 BULL SA.
+ * Copyright (C) 2004-2006 Silicon Graphics, Inc.
Perhaps you want to add a Google or Menage copyright here?
Quoted 2 lines
+ * 2003-10-22 Updates by Stephen Hemminger.
+ * 2004 May-July Rework by Paul Jackson.
Perhaps time for another log line here, such as:
+ * 2006 Generalized to containers by Paul Menage.
include/linux/container.h line 49
Quoted 5 lines
include/linux/container.h new file +5
-0+struct list_head sibling;/* my parents children */ +struct list_head children;/* my children */ + +struct container *parent;/* my parent */ +struct dentry *dentry;/* container fs entry */
Delete extra tab after semi-colon on parent line.
Quoted 1 line
+ 2) mount -t container none
/dev/container
(Yes - this pertains to something this you took as is from the current cpuset docmentation ...) Please don't use 'none' for this argument to mount. Use some string relevant to the type of filesystem being mounted. This argument shows up in error messages from mount, and as it notes in the mount(8) man page, when speaking of the proc file system, though it applies here too:
The proc file system is not associated with a special device, and when mounting it, an arbitrary keyword, such as proc can be used instead of a device specification. (The customary choice none is less fortunate:
the error message `none busy' from umount can be confusing.)
For example, make the above:
mount -t container container /dev/container
Signature · 3 lines
I won't rest till it's the best ...
Programmer, Linux Scalability
Paul Jackson <[email protected]> 1.925.600.0401Re: [RFC][PATCH 1/4] Generic container system abstracted from cpusets code
Quoted 9 lines · Paul Jackson
Be sure to spell out, if you haven't already, that there is just a single container hierarchy, with each task attached to a single container in that hierarchy, and each subsystem (e.g. cpusets) adding its own special files to the common container hierarchy directories.
It is not the case that each subsystem using containers gets to concoct its own hierarchy, independent of any other subsystem.
Well, it wouldn't be too hard to have the container code manage multiple container hierarchies in parallel if people wanted the ability to do this. You'd just replace the
struct container *container;
field in task_struct with
struct container *containers[MAX_CONTAINER_HIERARCHIES]
and have each instance of the container filesystem manage a different hierarchy. Perhaps to configure this from userspace you could do something like
mount -t container xxx -octype=cpuset /dev/cpuset
mount -t container yyy -octype=resgroup,ctype=cpuacct /mnt/rescontainer
i.e. mount one container hierarchy on /dev/cpuset with just the cpuset container type, and mount an independent hierarchy on /mnt/rescontainer with the resgroup and cpuacct container types attached to it. You'd probably want to enforce a rule that each container type could only be attached to a single hierarchy at a time.
Anyone have any thoughts on the usefulness/insanity of such an idea? The most useful part of it may be the ability to e.g. create a container hierarchy that explicitly doesn't cause e.g. multiple cpusets to be created, if you want to avoid the overhead that kicks in when there are multiple cpusets in use. That could be approximated without the overhead of multiple container hierarchy support by having a control file(s) in the top container dir that lets you configure which container subsystems are actually in use. So e.g. if you turned cpusets off before creating any subcontainers, all containers would actually share the same cpuset instance, and we could avoid any of the overheads associated with having multiple cpusets.
This would be somewhere in between the existing state of a single hierarchy with multiple fixed container types, and the full-blown multiple hierarchies with configurably-attachable container types that I outlined above.
Quoted 5 lines
If someone configures in one or more of the subsystems, such as cpusets, that requires CONTAINERS, then we need to enable containers. Otherwise we don't need to enable CONTAINERS. I don't see what user input we need here specific to CONTAINERS.
Fair enough - so have the various container types do "select CONTAINERS" rather than "depends on CONTAINERS"?
Quoted 11 lines
Quoted 3 lines
+ * Copyright (C) 2003 BULL SA.
+ * Copyright (C) 2004-2006 Silicon Graphics, Inc.Perhaps you want to add a Google or Menage copyright here?
Quoted 2 lines
+ * 2003-10-22 Updates by Stephen Hemminger.
+ * 2004 May-July Rework by Paul Jackson.Perhaps time for another log line here, such as:
+ * 2006 Generalized to containers by Paul Menage.
I already had something like that at the top - I've now made it clearer that these copyright statements came from the cpuset.c code from which this code was abstracted.
Paul
Re: [ckrm-tech] [RFC][PATCH 1/4] Generic container system abstracted from cpusets code
Quoted 1 line · Paul M
Anyone have any thoughts on the usefulness/insanity of such an idea?
[multiple hierarchies] I'd say let's pretend we don't know how to do them, until we have good reason otherwise. Having just a single resource hierarchy simplifies the thinking of these things. If doing so fits 90% of the needs, and can be stretched to get most of the remaining needs, then that simplification is well worth it. It might mean that a reasonable number of people can actually understand this stuff, which is always a nice property.
Signature · 3 lines
I won't rest till it's the best ...
Programmer, Linux Scalability
Paul Jackson <[email protected]> 1.925.600.0401Re: [ckrm-tech] [RFC][PATCH 1/4] Generic container system abstracted from cpusets code
In reply to Paul MenageQuoted 5 lines
Quoted 2 lines
Perhaps time for another log line here, such as:
+ * 2006 Generalized to containers by Paul Menage.I already had something like that at the top - I've now made it clearer that ...
Sure - so long as you're happy with your Copyright statements, I'm happy.
Signature · 3 lines
I won't rest till it's the best ...
Programmer, Linux Scalability
Paul Jackson <[email protected]> 1.925.600.0401RFC PATCH 2/4 Cpusets hooked into containers
In reply to [email protected]This patch removes the process grouping code from the cpusets code, instead hooking it into the generic container system. This temporarily adds cpuset-specific code in kernel/container.c, which is removed by the next patch in the series.
--- Documentation/cpusets.txt | 80 +- fs/proc/base.c | 18 include/linux/container.h | 4 include/linux/cpuset.h | 27 include/linux/sched.h | 4 init/Kconfig | 2 kernel/container.c | 50 + kernel/cpuset.c | 1317 ++++------------------------------------------ kernel/exit.c | 2 kernel/fork.c | 7 mm/oom_kill.c | 6 11 files changed, 228 insertions(+), 1289 deletions(-)
include/linux/container.h +4 -0
@@ -47,6 +47,10 @@ struct container {struct container *parent;/* my parent */ struct dentry *dentry;/* container fs entry */ + +#ifdef CONFIG_CPUSETS +struct cpuset *cpuset; +#endif };/* struct cftype:
include/linux/cpuset.h +13 -14
@@ -11,16 +11,13 @@#include <linux/sched.h> #include <linux/cpumask.h> #include <linux/nodemask.h> +#include <linux/container.h> #ifdef CONFIG_CPUSETS -extern int number_of_cpusets;/* How many cpusets are defined in system? */ -extern int cpuset_init_early (void );extern int cpuset_init (void );extern void cpuset_init_smp (void ); -extern void cpuset_fork (struct task_struct *p); -extern void cpuset_exit (struct task_struct *p);extern cpumask_t cpuset_cpus_allowed (struct task_struct *p);extern nodemask_t cpuset_mems_allowed (struct task_struct *p);void cpuset_init_current_mems_allowed (void );
@@ -32,7 +29,7 @@ int cpuset_zonelist_valid_mems_allowed(sextern int __cpuset_zone_allowed (struct zone *z,gfp_t gfp_mask);static int inline cpuset_zone_allowed (struct zone *z,gfp_t gfp_mask) { -return number_of_cpusets <=1 ||__cpuset_zone_allowed (z, gfp_mask); +return number_of_containers <=1 ||__cpuset_zone_allowed (z, gfp_mask); }extern int cpuset_excl_nodes_overlap (const struct task_struct *p);
@@ -47,10 +44,6 @@ extern void __cpuset_memory_pressure_bumextern struct file_operations proc_cpuset_operations;extern char *cpuset_task_status_allowed (struct task_struct *task,char *buffer); - -extern void cpuset_lock (void ); -extern void cpuset_unlock (void ); -extern int cpuset_mem_spread_node (void );static inline int cpuset_do_page_mem_spread (void )
@@ -63,13 +56,22 @@ static inline int cpuset_do_slab_mem_sprreturn current->flags &PF_SPREAD_SLAB ; } +extern int cpuset_can_attach_task (struct container *cont, +struct task_struct *tsk); +extern void cpuset_attach_task (struct container *cont, +struct task_struct *tsk); +extern void cpuset_post_attach_task (struct container *cont, +struct container *oldcont, +struct task_struct *tsk); +extern int cpuset_populate_dir (struct container *cont); +extern int cpuset_create (struct container *cont); +extern void cpuset_destroy (struct container *cont); +#else /* !CONFIG_CPUSETS */ static inline int cpuset_init_early (void ) {return 0 ; }static inline int cpuset_init (void ) {return 0 ; }static inline void cpuset_init_smp (void ) {} -static inline void cpuset_fork (struct task_struct *p) {} -static inline void cpuset_exit (struct task_struct *p) {}static inline cpumask_t cpuset_cpus_allowed (struct task_struct *p) {
@@ -108,9 +110,6 @@ static inline char *cpuset_task_status_areturn buffer; } -static inline void cpuset_lock (void ) {} -static inline void cpuset_unlock (void ) {} -static inline int cpuset_mem_spread_node (void ) {return 0 ;
kernel/exit.c +0 -2
@@ -27,7 +27,6 @@#include <linux/mempolicy.h> #include <linux/taskstats_kern.h> #include <linux/delayacct.h> -#include <linux/cpuset.h> #include <linux/container.h> #include <linux/syscalls.h> #include <linux/signal.h>
@@ -919,7 +918,6 @@ fastcall NORET_TYPE void do_exit(long co__exit_fs (tsk);exit_namespace (tsk);exit_thread (); -cpuset_exit (tsk);container_exit (tsk);exit_keys (tsk);
kernel/fork.c +2 -5
@@ -29,7 +29,6 @@#include <linux/fs.h> #include <linux/capability.h> #include <linux/cpu.h> -#include <linux/cpuset.h> #include <linux/container.h> #include <linux/security.h> #include <linux/swap.h>
@@ -1046,13 +1045,12 @@ static struct task_struct *copy_process( p->io_wait =NULL ; p->audit_context =NULL ;container_fork (p); -cpuset_fork (p);#ifdef CONFIG_NUMA p->mempolicy =mpol_copy (p->mempolicy);if (IS_ERR (p->mempolicy)) { retval =PTR_ERR (p->mempolicy); p->mempolicy =NULL ; - goto bad_fork_cleanup_cpuset; + goto bad_fork_cleanup_container; }mpol_fix_fork_child_flag (p);#endif
@@ -1277,9 +1275,8 @@ bad_fork_cleanup_security: bad_fork_cleanup_policy:#ifdef CONFIG_NUMA mpol_free (p->mempolicy); -bad_fork_cleanup_cpuset: +bad_fork_cleanup_container:#endif -cpuset_exit (p);container_exit (p); bad_fork_cleanup_delays_binfmt:delayacct_tsk_free (p);
kernel/container.c +47 -3
110 lines not on this page · raw message
kernel/cpuset.c +126 -1191
1,754 lines not on this page · raw message
init/Kconfig +1 -1
@@ -231,7 +231,7 @@ config CONTAINERS config CPUSETS bool "Cpuset support" - depends on SMP + depends on SMP && CONTAINERS help This option will let you create and manage CPUSETs which allow dynamically partitioning a system into sets of CPUs and
mm/oom_kill.c +3 -3
@@ -326,7 +326,7 @@ void out_of_memory(struct zonelist *zoneshow_mem (); } -cpuset_lock (); +container_lock ();read_lock (&tasklist_lock);/*
@@ -360,7 +360,7 @@ retry:/* Found nothing?!?! Either we hang forever, or we panic. */ if (!p) {read_unlock (&tasklist_lock); -cpuset_unlock (); +container_unlock ();panic ("Out of memory and no killable processes...\n" ); }
@@ -372,7 +372,7 @@ retry: out:read_unlock (&tasklist_lock); -cpuset_unlock (); +container_unlock ();/* * Give "p" a good chance of killing itself before we
fs/proc/base.c +0 -18
@@ -122,9 +122,6 @@ enum pid_directory_inos {#ifdef CONFIG_SCHEDSTATS PROC_TGID_SCHEDSTAT,#endif -#ifdef CONFIG_CPUSETS - PROC_TGID_CPUSET, -#endif #ifdef CONFIG_CONTAINERS PROC_TGID_CONTAINER,#endif
@@ -168,9 +165,6 @@ enum pid_directory_inos {#ifdef CONFIG_SCHEDSTATS PROC_TID_SCHEDSTAT,#endif -#ifdef CONFIG_CPUSETS - PROC_TID_CPUSET, -#endif #ifdef CONFIG_CONTAINERS PROC_TID_CONTAINER,#endif
@@ -239,9 +233,6 @@ static struct pid_entry tgid_base_stuff[#ifdef CONFIG_SCHEDSTATS E(PROC_TGID_SCHEDSTAT, "schedstat", S_IFREG|S_IRUGO),#endif -#ifdef CONFIG_CPUSETS - E(PROC_TGID_CPUSET, "cpuset", S_IFREG|S_IRUGO), -#endif #ifdef CONFIG_CONTAINERS E(PROC_TGID_CONTAINER, "container", S_IFREG|S_IRUGO),#endif
@@ -284,9 +275,6 @@ static struct pid_entry tid_base_stuff[]#ifdef CONFIG_SCHEDSTATS E(PROC_TID_SCHEDSTAT, "schedstat",S_IFREG|S_IRUGO),#endif -#ifdef CONFIG_CPUSETS - E(PROC_TID_CPUSET, "cpuset", S_IFREG|S_IRUGO), -#endif #ifdef CONFIG_CONTAINERS E(PROC_TID_CONTAINER, "container", S_IFREG|S_IRUGO),#endif
@@ -1826,12 +1814,6 @@ static struct dentry *proc_pident_lookup ei->op.proc_read = proc_pid_schedstat;break ;#endif -#ifdef CONFIG_CPUSETS -case PROC_TID_CPUSET : -case PROC_TGID_CPUSET : - inode->i_fop = &proc_cpuset_operations; -break ; -#endif #ifdef CONFIG_CONTAINERS case PROC_TID_CONTAINER :case PROC_TGID_CONTAINER :
include/linux/sched.h +1 -3
@@ -711,7 +711,6 @@ extern unsigned int max_cache_size;struct io_context ;/* See blkdev.h */ void exit_io_context (void );struct container ; -struct cpuset ;#define NGROUPS_SMALL 32#define NGROUPS_PER_BLOCK ((int)(PAGE_SIZE / sizeof(gid_t)))struct group_info {
@@ -974,7 +973,6 @@ struct task_struct {short il_next;#endif #ifdef CONFIG_CPUSETS -struct cpuset *cpuset;nodemask_t mems_allowed;int cpuset_mems_generation;int cpuset_mem_spread_rotor;
@@ -1352,7 +1350,7 @@ static inline int thread_group_empty(str/* * Protects ->fs, ->files, ->mm, ->group_info, ->comm, keyring * subscriptions and synchronises with wait4(). Also used in procfs. Also -* pins the final release of task.io_context. Also protects ->cpuset. +* pins the final release of task.io_context. Also protects ->container. * * Nests both inside and outside of read_lock(&tasklist_lock). * It must not be nested with write_lock_irq(&tasklist_lock),
Documentation/cpusets.txt +31 -50
166 lines not on this page · raw message
--
Re: [RFC][PATCH 2/4] Cpusets hooked into containers
kernel/container.c static long container_create(struct cont · line 1033
Quoted 4 lines · Menage
kernel/container.c +4
-0+#ifdef CONFIG_CPUSETS + err =cpuset_create (cont); +if (err) goto err_unlock_free; +#endif
I believe recommended kernel style puts the if() statement on a separate line, as in:
+#ifdef CONFIG_CPUSETS + err = cpuset_create(cont); + if (err) + goto err_unlock_free; +#endif
Signature · 3 lines
I won't rest till it's the best ...
Programmer, Linux Scalability
Paul Jackson <[email protected]> 1.925.600.0401RFC PATCH 3/4 Add generic multi-subsystem API to containers
In reply to [email protected]This patch removes all cpuset-specific knowlege from the container system, replacing it with a generic API that can be used by multiple subsystems. Cpusets is adapted to be a container subsystem.
--- Documentation/containers.txt | 121 +++++++++++++++++++++++++++++++++++++++++-- include/linux/container.h | 24 +++++++- include/linux/cpuset.h | 11 --- kernel/container.c | 95 ++++++++++++++++++++++++--------- kernel/cpuset.c | 101 ++++++++++++++++++++++------------- 5 files changed, 269 insertions(+), 83 deletions(-)
include/linux/container.h +21 -3
@@ -30,6 +30,8 @@ extern void container_unlock(void);extern void container_manage_lock (void );extern void container_manage_unlock (void ); +#define MAX_CONTAINER_SUBSYS 8 +struct container {unsigned long flags;/* "unsigned long" so bitops work */
@@ -48,9 +50,8 @@ struct container {struct container *parent;/* my parent */ struct dentry *dentry;/* container fs entry */ -#ifdef CONFIG_CPUSETS -struct cpuset *cpuset; -#endif +/* Private pointers for each registered subsystem */ +void *subsys[MAX_CONTAINER_SUBSYS ]; };/* struct cftype:
@@ -84,6 +85,23 @@ int container_add_file(struct containerint container_is_removed (const struct container *cont); +/* Container subsystem type. See Documentation/containers.txt for details */ + +struct container_subsys { +int (*create)(struct container *cont); +void (*destroy)(struct container *cont); +int (*can_attach)(struct container *cont,struct task_struct *tsk); +void (*attach)(struct container *cont,struct task_struct *tsk); +void (*post_attach)(struct container *cont, +struct container *old_cont, +struct task_struct *tsk); +int (*populate)(struct container *cont); + +int subsys_id; +}; + +int container_register_subsys (struct container_subsys *subsys); +#else /* !CONFIG_CONTAINERS */ static inline int container_init_early (void ) {return 0 ; }
include/linux/cpuset.h +0 -11
@@ -56,17 +56,6 @@ static inline int cpuset_do_slab_mem_spr return current->flags & PF_SPREAD_SLAB; } -extern int cpuset_can_attach_task (struct container *cont, -struct task_struct *tsk); -extern void cpuset_attach_task (struct container *cont, -struct task_struct *tsk); -extern void cpuset_post_attach_task (struct container *cont, -struct container *oldcont, -struct task_struct *tsk); -extern int cpuset_populate_dir (struct container *cont); -extern int cpuset_create (struct container *cont); -extern void cpuset_destroy (struct container *cont); -#else /* !CONFIG_CPUSETS */ static inline int cpuset_init_early (void ) {return 0 ; }
kernel/container.c +68 -27
175 lines not on this page · raw message
kernel/cpuset.c +62 -39
260 lines not on this page · raw message
Documentation/containers.txt +118 -3
134 lines not on this page · raw message
--
RFC PATCH 4/4 Simple CPU accounting container subsystem
In reply to [email protected]This demonstrates how to use the generic container subsystem for a simple resource tracker that counts the total CPU time used by all processes in a container, during the time that they're members of the container.
--- include/linux/cpu_acct.h | 10 ++++ init/Kconfig | 8 +++ kernel/Makefile | 1 kernel/cpu_acct.c | 104 +++++++++++++++++++++++++++++++++++++++++++++++ kernel/sched.c | 6 ++ 5 files changed, 129 insertions(+)
include/linux/cpu_acct.h new file +10 -0
@@ -0,0 +1,10 @@ + +#ifndef _LINUX_CPU_ACCT_H +#define _LINUX_CPU_ACCT_H + +#include <linux/container.h> +#include <asm/cputime.h> + +extern void cpuacct_charge (struct task_struct *,cputime_t cputime); + +#endif
init/Kconfig +8 -0
@@ -240,6 +240,14 @@ config CPUSETS Say N if unsure. +config CONTAINER_CPUACCT + bool "Simple CPU accounting container subsystem" + depends on CONTAINERS + help + Provides a simple Resource Controller for monitoring the + total CPU consumed by the tasks in a container + + config RELAY bool "Kernel->user space relay support (formerly relayfs)" help
kernel/cpu_acct.c new file +104 -0
104 lines not on this page · raw message
kernel/Makefile +1 -0
@@ -38,6 +38,7 @@ obj-$(CONFIG_KEXEC) += kexec.o obj-$(CONFIG_COMPAT) += compat.o obj-$(CONFIG_CONTAINERS) += container.o obj-$(CONFIG_CPUSETS) += cpuset.o +obj-$(CONFIG_CONTAINER_CPUACCT) += cpu_acct.o obj-$(CONFIG_IKCONFIG) += configs.o obj-$(CONFIG_STOP_MACHINE) += stop_machine.o obj-$(CONFIG_AUDIT) += audit.o auditfilter.o
kernel/sched.c +6 -0
@@ -52,6 +52,7 @@#include <linux/acct.h> #include <linux/kprobes.h> #include <linux/delayacct.h> +#include <linux/cpu_acct.h> #include <asm/tlb.h> #include <asm/unistd.h>
@@ -2918,6 +2919,8 @@ void account_user_time(struct task_struc p->utime =cputime_add (p->utime, cputime); +cpuacct_charge (p, cputime); +/* Add user time to cpustat. */ tmp =cputime_to_cputime64 (cputime);if (TASK_NICE (p) >0 )
@@ -2941,6 +2944,9 @@ void account_system_time(struct task_str p->stime =cputime_add (p->stime, cputime); +if (p != rq->idle) +cpuacct_charge (p, cputime); +/* Add system time to cpustat. */ tmp =cputime_to_cputime64 (cputime);if (hardirq_count () - hardirq_offset)
--
Quoted 2 lines · Menage
This patchset extracts the process grouping code from cpusets into a generic container system,
Interesting. I'll try to give it a careful review in the next couple of days.
I've added Jay Lan <[email protected]> to the cc list. I encourage you to include him on your cc list in the future - thanks. He has expertise in some of SGI's earlier container-like efforts.
Quoted 2 lines
cpusets ... well documented
(particularly with regards to synchronization rules).
thanks ;
Question (perhaps already answered in your code - I haven't looked yet): can loadable kernel modules register containers? I'd like to see at least GPL modules be able to register containers, but I appreciate that this could be a controversial issue. Technically, I am guessing that some EXPORT_GPL_SYMPOL declarations on the necessary container registration routines would provide GPL modules with this capability.
Guess I should read the code ...
Signature · 3 lines
I won't rest till it's the best ...
Programmer, Linux Scalability
Paul Jackson <[email protected]> 1.925.600.0401Quoted 3 lines · Paul Jackson
Question (perhaps already answered in your code - I haven't looked yet): can loadable kernel modules register containers? I'd like to
There's no reason in principle why not, although this patch doesn't export the symbols.
Paul
Aha - I must have missed the memo - I take it this patch set doesn't compile yet.
If I just apply it to 2.6.18, and try my favorite config 'make sn2_defconfig', then CONFIG_CPUSETS is disabled, because neither of the CONFIG_CONTAINER* settings are enabled, and furthermore it doesn't build, with the complaints:
===================================================================================kernel/built-in.o: In function `account_system_time':
(.text+0xc1e2): undefined reference to cpuacct_charge' kernel/built-in.o: In function account_user_time':
(.text+0xc332): undefined reference to `cpuacct_charge'
make: *** [.tmp_vmlinux1] Error 1
===================================================================================
Apparently some generic code in kernel/sched.c is using cpuacct_charge, but cpuacct_charge is only defined if CONFIG_CONTAINER_CPUACCT is enabled. Perhaps you need an empty stub for cpuacct_charge in cpu_acct.h, in the event that CONTAINER_CPUACCT is not enabled.
If I then apply the following patch to my sn2_defconfig:
===================================================================================
arch/ia64/configs/sn2_defconfig 2006-09-28 21:05:22.119968786 -0700 renamed from arch/ia64/configs/sn2_defconfig 2006-09-28 20:37:02.162797053 -0700 +2 -0
@@ -1214,3 +1214,5 @@ CONFIG_CRYPTO_CRC32C=m # # Hardware crypto devices # +CONFIG_CONTAINERS=y +CONFIG_CONTAINER_CPUACCT=y
===================================================================================
it gets further (well, less far, actually), dying in the builds of kernel/cpu_acct.c and kernel/cpuset.o, with the complaints:
===================================================================================kernel/cpu_acct.c: In function 'cpuusage_read':kernel/cpu_acct.c:57: warning: format '%llu' expects type 'long long unsigned int', but argument 3 has type 'cputime64_t' kernel/cpu_acct.c:57: warning: format '%llu' expects type 'long long unsigned int', but argument 3 has type 'cputime64_t' kernel/cpuset.c:1000: warning: initialization from incompatible pointer typekernel/cpuset.c:1007: warning: initialization from incompatible pointer typekernel/cpuset.c:1014: warning: initialization from incompatible pointer typekernel/cpuset.c:1021: warning: initialization from incompatible pointer typekernel/cpuset.c:1028: warning: initialization from incompatible pointer typekernel/cpuset.c:1035: warning: initialization from incompatible pointer typekernel/cpuset.c:1042: warning: initialization from incompatible pointer typekernel/cpuset.c:1049: warning: initialization from incompatible pointer typekernel/cpuset.c:1056: warning: initialization from incompatible pointer typekernel/cpuset.c: In function 'cpuset_create':kernel/cpuset.c:1102: error: invalid lvalue in assignmentkernel/cpuset.c:1126: error: invalid lvalue in assignmentkernel/cpuset.c: At top level:kernel/cpuset.c:1153: error: static declaration of 'cpuset_subsys' follows non-static declaration kernel/cpuset.c:58: error: previous declaration of 'cpuset_subsys' was here
make[1]: *** [kernel/cpuset.o] Error 1
===================================================================================
Looking at the relevant lines of kernel/cpuset.c, I see:
58: extern struct container_subsys cpuset_subsys;
59: #define container_cs(_cont) ((struct cpuset *)(_cont)->subsys[cpuset_subsys.subsys_id]) 1102: container_cs(cont) = &top_cpuset;
1126: container_cs(cont) = cs;
1153: static struct container_subsys cpuset_subsys = {
Looks like cpuset_subsys doesn't know if it is extern or static, and container_cs() is a cast thingie, which isn't allowed on the left side in modern gcc.
That's enough damage for one night ...
Signature · 3 lines
I won't rest till it's the best ...
Programmer, Linux Scalability
Paul Jackson <[email protected]> 1.925.600.0401Quoted 5 lines · Paul Jackson
Apparently some generic code in
kernel/sched.cis usingcpuacct_charge, butcpuacct_chargeis only defined ifCONFIG_CONTAINER_CPUACCTis enabled. Perhaps you need an empty stub forcpuacct_chargeincpu_acct.h, in the event thatCONTAINER_CPUACCTis not enabled.
You're right, the cpu_acct patch was a late example addition and I forgot to compile with CONFIG_CONTAINER_CPUACCT disabled ...
Quoted 4 lines
Looks like
cpuset_subsysdoesn't know if it is extern or static, andcontainer_cs()is a cast thingie, which isn't allowed on the left side in modern gcc.
It builds fine on gcc 3.2.2, which is still a supported compiler according to Documentation/Changes - maybe that's a bit out of date?
Paul