Going forward with Resource Management - A cpu controller

That's another 4.5kbytes/cpu of our precious per-cpu memory. Is it feasible to make this dependent upon CONFIG_CPUMETER?

This patch splits the single main runqueue into several runqueues on each CPU. One (main) runqueue is used to hold task-groups and one runqueue for each task-group in the system.

Signed-off-by Srivatsa Vaddagiri <[email protected]>

 init/Kconfig   |    8 +
 kernel/sched.c |  251 +++++++++++++++++++++++++++++++++++++++++++++++++--------
 2 files changed, 228 insertions(+), 31 deletions(-)

diff -puN kernel/sched.c~cpu_ctlr_base_changes kernel/sched.c

kernel/sched.c renamed from kernel/sched.c~cpu_ctlr_base_changes +220 -31

@@ -191,10 +191,48 @@ static inline unsigned int task_timeslic
 
 struct prio_array {
 	unsigned int nr_active;
+	int best_static_prio, best_dyn_prio;
 	DECLARE_BITMAP(bitmap, MAX_PRIO+1); /* include 1 bit for delimiter */
 	struct list_head queue[MAX_PRIO];
 };
 
+/* 
+ * Each task belong to some task-group. Task-groups and what tasks they contain 
+ * are defined by the administrator using some suitable interface.
+ * Administrator can also define the CPU bandwidth provided to each task-group.
+ *
+ * Task-groups are given a certain priority to run on every CPU. Currently
+ * task-group priority on a CPU is defined to be the same as that of 
+ * highest-priority runnable task it has on that CPU. Task-groups also
+ * get their own runqueue on every CPU. The main runqueue on each CPU is
+ * used to hold task-groups, rather than tasks.
+ *
+ * Scheduling decision on a CPU is now two-step : first pick highest priority
+ * task-group from the main runqueue and next pick highest priority task from
+ * the runqueue of that group. Both decisions are of O(1) complexity.
+ */
+
+/* runqueue used for every task-group */
+struct task_grp_rq {
+	struct prio_array arrays[2];
+	struct prio_array *active, *expired, *rq_array;
+	unsigned long expired_timestamp;
+	unsigned int ticks;
+	int prio;	/* Priority of the task-group */
+	struct list_head list;
+};
+
+static DEFINE_PER_CPU(struct task_grp_rq, init_tg_rq);
+
+/* task-group object - maintains information about each task-group */
+struct task_grp {
+	int ticks;	/* bandwidth given to the task-group */
+	struct task_grp_rq *rq[NR_CPUS];   /* runqueue pointer for every cpu */
+};
+
+/* The "default" task-group */
+struct task_grp init_task_grp;
+
 /*
  * This is the main, per-CPU runqueue data structure.
  *

@@ -224,12 +262,11 @@ struct rq {
 	 */
 	unsigned long nr_uninterruptible;
 
-	unsigned long expired_timestamp;
 	unsigned long long timestamp_last_tick;
 	struct task_struct *curr, *idle;
 	struct mm_struct *prev_mm;
+	/* these arrays hold task-groups */
 	struct prio_array *active, *expired, arrays[2];
-	int best_expired_prio;
 	atomic_t nr_iowait;
 
 #ifdef CONFIG_SMP

@@ -244,6 +281,7 @@ struct rq {
 #endif
 
 #ifdef CONFIG_SCHEDSTATS
+	/* xxx: move these to task-group runqueue */
 	/* latency stats */
 	struct sched_info rq_sched_info;
 

@@ -657,6 +695,63 @@ sched_info_switch(struct task_struct *pr
 #define sched_info_switch(t, next)	do { } while (0)
 #endif /* CONFIG_SCHEDSTATS || CONFIG_TASK_DELAY_ACCT */
 
+static unsigned int task_grp_timeslice(struct task_grp *tg)
+{
+	/* xxx: take into account sleep_avg etc of the group */
+	return tg->ticks;
+}
+
+/* Dequeue task-group object from the main runqueue */
+static void dequeue_task_grp(struct task_grp_rq *tgrq)
+{
+	struct prio_array *array = tgrq->rq_array;
+
+	array->nr_active--;
+	list_del(&tgrq->list);
+	if (list_empty(array->queue + tgrq->prio))
+		__clear_bit(tgrq->prio, array->bitmap);
+	tgrq->rq_array = NULL;
+}
+
+/* Enqueue task-group object on the main runqueue */
+static void enqueue_task_grp(struct task_grp_rq *tgrq, struct prio_array *array,
+				int head)
+{
+	if (!head)
+		list_add_tail(&tgrq->list, array->queue + tgrq->prio);
+	else
+        	list_add(&tgrq->list, array->queue + tgrq->prio);
+        __set_bit(tgrq->prio, array->bitmap);
+        array->nr_active++;
+        tgrq->rq_array = array;
+}
+
+/* return the task-group to which a task belongs */
+static inline struct task_grp *task_grp(struct task_struct *p)
+{
+	return &init_task_grp;
+}
+
+static inline void update_task_grp_prio(struct task_grp_rq *tgrq, struct rq *rq,
+								 int head)
+{
+	int new_prio;
+	struct prio_array *array = tgrq->rq_array;
+
+	new_prio = tgrq->active->best_dyn_prio;
+	if (new_prio == MAX_PRIO)
+		new_prio = tgrq->expired->best_dyn_prio;
+
+	if (array)
+		dequeue_task_grp(tgrq);
+	tgrq->prio = new_prio;
+	if (new_prio != MAX_PRIO) {
+		if (!array)
+			array = rq->active;
+		enqueue_task_grp(tgrq, array, head);
+	}
+}
+
 /*
  * Adding/removing a task to/from a priority array:
  */

@@ -664,8 +759,17 @@ static void dequeue_task(struct task_str
 {
 	array->nr_active--;
 	list_del(&p->run_list);
-	if (list_empty(array->queue + p->prio))
+	if (list_empty(array->queue + p->prio)) {
 		__clear_bit(p->prio, array->bitmap);
+		if (p->prio == array->best_dyn_prio) {
+			struct task_grp_rq *tgrq = task_grp(p)->rq[task_cpu(p)];
+
+			array->best_dyn_prio =
+					sched_find_first_bit(array->bitmap);
+			if (array == tgrq->active || !tgrq->active->nr_active)
+				update_task_grp_prio(tgrq, task_rq(p), 0);
+		}
+	}
 }
 
 static void enqueue_task(struct task_struct *p, struct prio_array *array)

@@ -675,6 +779,14 @@ static void enqueue_task(struct task_str
 	__set_bit(p->prio, array->bitmap);
 	array->nr_active++;
 	p->array = array;
+
+	if (p->prio < array->best_dyn_prio) {
+		struct task_grp_rq *tgrq = task_grp(p)->rq[task_cpu(p)];
+
+		array->best_dyn_prio = p->prio;
+		if (array == tgrq->active || !tgrq->active->nr_active)
+			update_task_grp_prio(tgrq, task_rq(p), 0);
+	}
 }
 
 /*

@@ -693,6 +805,14 @@ enqueue_task_head(struct task_struct *p,
 	__set_bit(p->prio, array->bitmap);
 	array->nr_active++;
 	p->array = array;
+
+	if (p->prio < array->best_dyn_prio) {
+		struct task_grp_rq *tgrq = task_grp(p)->rq[task_cpu(p)];
+
+		array->best_dyn_prio = p->prio;
+		if (array == tgrq->active || !tgrq->active->nr_active)
+			update_task_grp_prio(tgrq, task_rq(p), 1);
+	}
 }
 
 /*

@@ -831,10 +951,11 @@ static int effective_prio(struct task_st
  */
 static void __activate_task(struct task_struct *p, struct rq *rq)
 {
-	struct prio_array *target = rq->active;
+	struct task_grp_rq *tgrq = task_grp(p)->rq[task_cpu(p)];
+	struct prio_array *target = tgrq->active;
 
 	if (batch_task(p))
-		target = rq->expired;
+		target = tgrq->expired;
 	enqueue_task(p, target);
 	inc_nr_running(p, rq);
 }

@@ -844,7 +965,10 @@ static void __activate_task(struct task_
  */
 static inline void __activate_idle_task(struct task_struct *p, struct rq *rq)
 {
-	enqueue_task_head(p, rq->active);
+	struct task_grp_rq *tgrq = task_grp(p)->rq[task_cpu(p)];
+	struct prio_array *target = tgrq->active;
+
+	enqueue_task_head(p, target);
 	inc_nr_running(p, rq);
 }
 

@@ -2077,7 +2201,7 @@ int can_migrate_task(struct task_struct 
 	return 1;
 }
 
-#define rq_best_prio(rq) min((rq)->curr->prio, (rq)->best_expired_prio)
+#define rq_best_prio(rq) min((rq)->curr->prio, (rq)->expired->best_static_prio)
 
 /*
  * move_tasks tries to move up to max_nr_move tasks and max_load_move weighted

@@ -2884,6 +3008,8 @@ unsigned long long current_sched_time(co
 	return ns;
 }
 
+#define nr_tasks(tgrq)	(tgrq->active->nr_active + tgrq->expired->nr_active)
+
 /*
  * We place interactive tasks back into the active array, if possible.
  *

@@ -2894,13 +3020,13 @@ unsigned long long current_sched_time(co
  * increasing number of running tasks. We also ignore the interactivity
  * if a better static_prio task has expired:
  */
-static inline int expired_starving(struct rq *rq)
+static inline int expired_starving(struct task_grp_rq *rq)
 {
-	if (rq->curr->static_prio > rq->best_expired_prio)
+	if (current->static_prio > rq->expired->best_static_prio)
 		return 1;
 	if (!STARVATION_LIMIT || !rq->expired_timestamp)
 		return 0;
-	if (jiffies - rq->expired_timestamp > STARVATION_LIMIT * rq->nr_running)
+	if (jiffies - rq->expired_timestamp > STARVATION_LIMIT * nr_tasks(rq))
 		return 1;
 	return 0;
 }

@@ -2991,6 +3117,7 @@ void scheduler_tick(void)
 	struct task_struct *p = current;
 	int cpu = smp_processor_id();
 	struct rq *rq = cpu_rq(cpu);
+	struct task_grp_rq *tgrq = task_grp(p)->rq[cpu];
 
 	update_cpu_clock(p, rq, now);
 

@@ -3004,7 +3131,7 @@ void scheduler_tick(void)
 	}
 
 	/* Task might have expired already, but not scheduled off yet */
-	if (p->array != rq->active) {
+	if (p->array != tgrq->active) {
 		set_tsk_need_resched(p);
 		goto out;
 	}

@@ -3027,25 +3154,26 @@ void scheduler_tick(void)
 			set_tsk_need_resched(p);
 
 			/* put it at the end of the queue: */
-			requeue_task(p, rq->active);
+			requeue_task(p, tgrq->active);
 		}
 		goto out_unlock;
 	}
 	if (!--p->time_slice) {
-		dequeue_task(p, rq->active);
+		dequeue_task(p, tgrq->active);
 		set_tsk_need_resched(p);
 		p->prio = effective_prio(p);
 		p->time_slice = task_timeslice(p);
 		p->first_time_slice = 0;
 
-		if (!rq->expired_timestamp)
-			rq->expired_timestamp = jiffies;
-		if (!TASK_INTERACTIVE(p) || expired_starving(rq)) {
-			enqueue_task(p, rq->expired);
-			if (p->static_prio < rq->best_expired_prio)
-				rq->best_expired_prio = p->static_prio;
+		if (!tgrq->expired_timestamp)
+			tgrq->expired_timestamp = jiffies;
+		if (!TASK_INTERACTIVE(p) || expired_starving(tgrq)) {
+			enqueue_task(p, tgrq->expired);
+			if (p->static_prio < tgrq->expired->best_static_prio)
+				tgrq->expired->best_static_prio =
+							 p->static_prio;
 		} else
-			enqueue_task(p, rq->active);
+			enqueue_task(p, tgrq->active);
 	} else {
 		/*
 		 * Prevent a too long timeslice allowing a task to monopolize

@@ -3066,12 +3194,21 @@ void scheduler_tick(void)
 		if (TASK_INTERACTIVE(p) && !((task_timeslice(p) -
 			p->time_slice) % TIMESLICE_GRANULARITY(p)) &&
 			(p->time_slice >= TIMESLICE_GRANULARITY(p)) &&
-			(p->array == rq->active)) {
+			(p->array == tgrq->active)) {
 
-			requeue_task(p, rq->active);
+			requeue_task(p, tgrq->active);
 			set_tsk_need_resched(p);
 		}
 	}
+
+	if (!--tgrq->ticks) {
+		/* Move the task group to expired list */
+		dequeue_task_grp(tgrq);
+		tgrq->ticks = task_grp_timeslice(task_grp(p));
+		enqueue_task_grp(tgrq, rq->expired, 0);
+		set_tsk_need_resched(p);
+	}
+
 out_unlock:
 	spin_unlock(&rq->lock);
 out:

@@ -3264,6 +3401,7 @@ asmlinkage void __sched schedule(void)
 	int cpu, idx, new_prio;
 	long *switch_count;
 	struct rq *rq;
+	struct task_grp_rq *next_grp;
 
 	/*
 	 * Test if we are atomic.  Since do_exit() needs to call into

@@ -3332,23 +3470,41 @@ need_resched_nonpreemptible:
 		idle_balance(cpu, rq);
 		if (!rq->nr_running) {
 			next = rq->idle;
-			rq->expired_timestamp = 0;
 			wake_sleeping_dependent(cpu);
 			goto switch_tasks;
 		}
 	}
 
+	/* Pick a task group first */
+#ifdef CONFIG_CPUMETER
 	array = rq->active;
 	if (unlikely(!array->nr_active)) {
 		/*
 		 * Switch the active and expired arrays.
 		 */
-		schedstat_inc(rq, sched_switch);
 		rq->active = rq->expired;
 		rq->expired = array;
 		array = rq->active;
-		rq->expired_timestamp = 0;
-		rq->best_expired_prio = MAX_PRIO;
+	}
+	idx = sched_find_first_bit(array->bitmap);
+	queue = array->queue + idx;
+	next_grp = list_entry(queue->next, struct task_grp_rq, list);
+#else
+	next_grp = init_task_grp.rq[cpu];
+#endif
+
+	/* Pick a task within that group next */
+	array = next_grp->active;
+	if (!array->nr_active) {
+		/*
+		 * Switch the active and expired arrays.
+		 */
+		schedstat_inc(next_grp, sched_switch);
+		next_grp->active = next_grp->expired;
+		next_grp->expired = array;
+		array = next_grp->active;
+		next_grp->expired_timestamp = 0;
+		next_grp->expired->best_static_prio = MAX_PRIO;
 	}
 
 	idx = sched_find_first_bit(array->bitmap);

@@ -4413,7 +4569,8 @@ asmlinkage long sys_sched_getaffinity(pi
 asmlinkage long sys_sched_yield(void)
 {
 	struct rq *rq = this_rq_lock();
-	struct prio_array *array = current->array, *target = rq->expired;
+	struct task_grp_rq *tgrq = task_grp(current)->rq[task_cpu(current)];
+	struct prio_array *array = current->array, *target = tgrq->expired;
 
 	schedstat_inc(rq, yld_cnt);
 	/*

@@ -4424,13 +4581,13 @@ asmlinkage long sys_sched_yield(void)
 	 *  array.)
 	 */
 	if (rt_task(current))
-		target = rq->active;
+		target = tgrq->active;
 
 	if (array->nr_active == 1) {
 		schedstat_inc(rq, yld_act_empty);
-		if (!rq->expired->nr_active)
+		if (!tgrq->expired->nr_active)
 			schedstat_inc(rq, yld_both_empty);
-	} else if (!rq->expired->nr_active)
+	} else if (!tgrq->expired->nr_active)
 		schedstat_inc(rq, yld_exp_empty);
 
 	if (array != target) {

@@ -6722,21 +6879,53 @@ int in_sched_functions(unsigned long add
 		&& addr < (unsigned long)__sched_text_end);
 }
 
+static void task_grp_rq_init(struct task_grp_rq *tgrq, int ticks)
+{
+	int j, k;
+
+	tgrq->active = tgrq->arrays;
+	tgrq->expired = tgrq->arrays + 1;
+	tgrq->rq_array = NULL;
+	tgrq->expired->best_static_prio = MAX_PRIO;
+	tgrq->expired->best_dyn_prio = MAX_PRIO;
+	tgrq->active->best_static_prio = MAX_PRIO;
+	tgrq->active->best_dyn_prio = MAX_PRIO;
+	tgrq->prio = MAX_PRIO;
+	tgrq->ticks = ticks;
+	INIT_LIST_HEAD(&tgrq->list);
+
+	for (j = 0; j < 2; j++) {
+		struct prio_array *array;
+
+		array = tgrq->arrays + j;
+		for (k = 0; k < MAX_PRIO; k++) {
+			INIT_LIST_HEAD(array->queue + k);
+			__clear_bit(k, array->bitmap);
+		}
+		// delimiter for bitsearch
+		__set_bit(MAX_PRIO, array->bitmap);
+	}
+}
+
 void __init sched_init(void)
 {
 	int i, j, k;
 
+	init_task_grp.ticks = -1;     /* Unlimited bandwidth */
+
 	for_each_possible_cpu(i) {
 		struct prio_array *array;
 		struct rq *rq;
+		struct task_grp_rq *tgrq;
 
 		rq = cpu_rq(i);
+		tgrq = init_task_grp.rq[i] = &per_cpu(init_tg_rq, i);
 		spin_lock_init(&rq->lock);
+		task_grp_rq_init(tgrq, init_task_grp.ticks);
 		lockdep_set_class(&rq->lock, &rq->rq_lock_key);
 		rq->nr_running = 0;
 		rq->active = rq->arrays;
 		rq->expired = rq->arrays + 1;
-		rq->best_expired_prio = MAX_PRIO;
 
 #ifdef CONFIG_SMP
 		rq->sd = NULL;

diff -puN init/Kconfig~cpu_ctlr_base_changes init/Kconfig

init/Kconfig renamed from init/Kconfig~cpu_ctlr_base_changes +8 -0

@@ -248,6 +248,14 @@ config CPUSETS
 
 	  Say N if unsure.
 
+config CPUMETER
+	bool "CPU resource control"
+	depends on CPUSETS
+	help
+	  This options lets you create cpu resource partitions within
+	  cpusets. Each resource partition can be given a different quota
+	  of CPU usage.
+
 config RELAY
 	bool "Kernel->user space relay support (formerly relayfs)"
 	help

_

Srivatsa,

AFAICS, you wanted to go the way we used in OpenVZ - 2-level scheduling. However, you don't do any process balancing between runqueues taking into account
other groups.
In many cases you will simply endup with tasks collected on the same physical
CPU and poor performance. I'm not talking about fairness (proportinal CPU scheduling).

I don't think it is possible to make any estimations for QoS of such a scheduler.

What do you think about a full runqueue virtualization, so that first level CPU scheduler could select task-group on any basis and then arbitrary runqueue was selected for running?

P.S. BTW, this patch doesn't allow hierarchy in CPU controler.

Srivatsa,

AFAICS, you wanted to go the way we used in OpenVZ - 2-level scheduling. However, you don't do any process balancing between runqueues taking into account other groups.

What do you think about a full runqueue virtualization, so that first level CPU scheduler could select task-group on any basis and then arbitrary runqueue was selected for running?

Thanks,
Kirill
P.S. BTW, this patch doesn't allow hierarchy in CPU controler.

Define these operations for a task-group:

- create new group
- destroy existing group
- assign bandwidth (quota) for a group
- get bandwidth (quota) of a group

Signed-off-by Srivatsa Vaddagiri <[email protected]>

 include/linux/sched.h |   12 +++++++
 kernel/sched.c        |   79 ++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 91 insertions(+)

diff -puN kernel/sched.c~cpu_ctlr_grp_ops kernel/sched.c

kernel/sched.c renamed from kernel/sched.c~cpu_ctlr_grp_ops +15 -0

@@ -7063,3 +7063,82 @@ void set_curr_task(int cpu, struct task_
 }
 
 #endif
+
+#ifdef CONFIG_CPUMETER
+
+/* Allocate runqueue structures for the new task-group */
+void *sched_alloc_group(void)
+{
+	struct task_grp *tg;
+	struct task_grp_rq *tgrq;
+	int i;
+
+	tg = kzalloc(sizeof(*tg), GFP_KERNEL);
+	if (!tg)
+		return NULL;
+
+	tg->ticks = -1;		/* No limit */

kernel/sched.c renamed from kernel/sched.c~cpu_ctlr_grp_ops +13 -0

+	int i;
+
+	tg->ticks = (quota * 5 * HZ) / 100;
+
+	for_each_possible_cpu(i)
+		tg->rq[i]->ticks = tg->ticks;
+
+}
+
+/* Return assigned quota for this group */
+int sched_get_quota(void *grp)
+{
+	struct task_grp *tg = (struct task_grp *)grp;

Resource management has been talked about quite extensively in the past, more recently in the context of containers. The basic requirement here is to provide isolation between *groups* of task wrt their use of various resources like CPU, Memory, I/O bandwidth, open file-descriptors etc.

Different maintainers have however expressed different opinions over the need to complicate the kernel to meet this need, especially since it involves core kernel code like the resource schedulers.

A BoF was hence held at OLS this year to come to a consensus on the minimum requirements of a resource management solution for Linux kernel. Some notes taken at the BoF are posted here:

	http://www.uwsg.indiana.edu/hypermail/linux/kernel/0607.3/0896.html

An important consensus point of the BoF seemed to be "focus on real controllers more, preferably memory first, using some simple interface and task grouping mechanism".

In going forward, following points will need to be addressed:

- Grouping and interface
- What mechanism to use for grouping tasks and for specifying task-group resource usage limits?

- Design of individual resource controllers like memory and cpu

This patch series is an attempt to take forward the design discussion of a CPU controller.

For simplicity and convenience, cpuset has been chosen as the means to group tasks here, primarily because cpuset already exists in the kernel and also perhaps resource container definition should be unique only inside a cpuset.

I thought the most recently posted CKRM core was a fine piece of code.

The downside to such a strategy is that there is a risk that nobody ever gets around to implementing useful controllers, so it ends up dead code. I'd judge that the interest in resource management is such that the risk of this happening is low.

I think the risk is that OpenVZ has all the controls and resource managers we need, while CKRM is still more research-ish. I find the OpenVZ code much clearer, cleaner and complete at the moment, although also much more conservative in its approach to solving problems.

Quoted 4 lines

I think the risk is that OpenVZ has all the controls and resource managers we need, while CKRM is still more research-ish. I find the OpenVZ code much clearer, cleaner and complete at the moment, although also much more conservative in its approach to solving problems.

I think it would be nice to compare first the features provided by ckrm and openvz at some point and agree upon the minimum common features we need to have as we go forward. For instance I think Openvz assumes that tasks do not need to move between containers (task-groups), whereas ckrm provides this
flexibility for workload management. This may have some effect on the controller/interface design, no?

OpenVZ assumes that tasks can't move between task-groups for a single reason:
user shouldn't be able to escape from the container.
But this have no implication on the design/implementation.

BTW, do you see any practical use cases for tasks jumping between resource-containers?

Quoted 3 lines

OpenVZ assumes that tasks can't move between task-groups for a single reason:
user shouldn't be able to escape from the container.
But this have no implication on the design/implementation.

Doesnt the ability to move tasks between groups dynamically affect (atleast) memory controller design (in giving up ownership etc)?

Also if we need to support this movement, we need to have some corresponding system call/file-system interface which supports this move operation.

Quoted 1 line

BTW, do you see any practical use cases for tasks jumping between resource-containers?

The use cases I have heard of which would benefit such a feature is
(say) for database threads which want to change their "resource
affinity" status depending on which customer query they are currently handling. If they are handling a query for a "important" customer, they will want affinied to a high bandwidth resource container and later if they start handling
a less important query they will want to give up this affinity and
instead move to a low-bandwidth container.

Quoted 8 lines

The use cases I have heard of which would benefit such a feature is (say) for database threads which want to change their "resource affinity" status depending on which customer query they are currently handling. If they are handling a query for a "important" customer, they will want affinied
to a high bandwidth resource container and later if they start handling a less important query they will want to give up this affinity and instead move to a low-bandwidth container.

this works mostly for CPU only.

And OpenVZ design allows to change CPU
resource container dynamically.

But such a trick works poorly for memory, because:
1. threads share lots of resources.
2. complex databases can have more complicated handling than a thread per request.
e.g. one thread servers memory pools, another one caches, some for
stored procedures, some for requests etc.

BTW, exactly this difference shows the reason to have different groups for different resources.

Thanks,
Kirill

Quoted 2 lines

Doesnt the ability to move tasks between groups dynamically affect (atleast) memory controller design (in giving up ownership etc)?

we save object owner on the object. So if you change the container, objects are still correctly charged to the creator and are uncharged correctly on free.

Quoted 3 lines

Also if we need to support this movement, we need to have some corresponding system call/file-system interface which supports this move operation.

it can be done by the same syscall or whatever which sets your container group.
we have the same syscall for creating/setting/entering to the container. i.e. chaning the container dynamically doesn't change the interface.

Quoted 3 lines

Quoted 2 lines

BTW, do you see any practical use cases for tasks jumping between resource-containers?

Quoted 8 lines

The use cases I have heard of which would benefit such a feature is (say) for database threads which want to change their "resource affinity" status depending on which customer query they are currently handling. If they are handling a query for a "important" customer, they will want affinied to a high bandwidth resource container and later if they start handling a less important query they will want to give up this affinity and instead move to a low-bandwidth container.

this works mostly for CPU only. And OpenVZ design allows to change CPU
resource container dynamically.
But such a trick works poorly for memory, because:
1. threads share lots of resources.
2. complex databases can have more complicated handling than a thread per request.
e.g. one thread servers memory pools, another one caches, some for stored procedures, some for requests etc.

BTW, exactly this difference shows the reason to have different groups for different resources.

Quoted 6 lines

Quoted 6 lines

The use cases I have heard of which would benefit such a feature is
(say) for database threads which want to change their "resource
affinity" status depending on which customer query they are currently handling. If they are handling a query for a "important" customer, they will want affinied to a high bandwidth resource container and later if they start handling
a less important query they will want to give up this affinity and
instead move to a low-bandwidth container.

hmm, would it not be better to have a thread each in two different containers for handling different kind of requests.

Or if there is too
much of sharing between threads, then setting the individual priority should help.

Quoted 9 lines · Rohit Seth

Quoted 7 lines

Quoted 7 lines

The use cases I have heard of which would benefit such a feature is (say) for database threads which want to change their "resource affinity" status depending on which customer query they are currently handling. If they are handling a query for a "important" customer, they will want affinied to a high bandwidth resource container and later if they start handling a less important query they will want to give up this affinity and instead move to a low-bandwidth container.

Quoted 2 lines

hmm, would it not be better to have a thread each in two different containers for handling different kind of requests.

Its possible but now you're effectively requiring the thread pool to expand as many times as service levels supported.

any long running job whose prioritization changes during its lifetime also benefits from being able to be moved.

Quoted 5 lines

Or if there is too
much of sharing between threads, then setting the individual priority should help.

Quoted 7 lines

Quoted 2 lines

Doesnt the ability to move tasks between groups dynamically affect (atleast) memory controller design (in giving up ownership etc)?

we save object owner on the object. So if you change the container, objects are still correctly charged to the creator and are uncharged correctly on free.

Seems like the object owner should also change when the object moves from one container to another.

Quoted 9 lines

Quoted 7 lines

Quoted 2 lines

Doesnt the ability to move tasks between groups dynamically affect (atleast) memory controller design (in giving up ownership etc)?

we save object owner on the object. So if you change the container, objects are still correctly charged to the creator and are uncharged correctly on free.

Quoted 2 lines

Seems like the object owner should also change when the object moves from one container to another.

Consider a file which is opened in 2 processes. one of the processes wants to move to another container then. How would you decide whether to change the file owner or not?

Quoted 11 lines

Quoted 11 lines

Quoted 7 lines

Quoted 2 lines

Doesnt the ability to move tasks between groups dynamically affect (atleast) memory controller design (in giving up ownership etc)?

we save object owner on the object. So if you change the container, objects are still correctly charged to the creator and are uncharged correctly on free.

Seems like the object owner should also change when the object moves from one container to another.

Quoted 4 lines

Consider a file which is opened in 2 processes. one of the processes wants to move to another container then. How would you decide whether to change the file owner or not?

If a process has sufficient rights to move a file to a new container then it should be okay to assign the file to the new container.

Though the point is, if a resource (like file) is getting migrated to a new container then all the attributes (like owner, #pages in memory etc.) attached to that resource (file) should also migrate to this new container. Otherwise the semantics of where does the resource belong becomes very difficult.

And if you really want a resource to not be able to migrate from one container then we could define IMMUTABLE flag to indicate that behavior.

Quoted 19 lines

Quoted 17 lines

Quoted 11 lines

Quoted 7 lines

Quoted 2 lines

Doesnt the ability to move tasks between groups dynamically affect (atleast) memory controller design (in giving up ownership etc)?

we save object owner on the object. So if you change the container, objects are still correctly charged to the creator and are uncharged correctly on free.

Seems like the object owner should also change when the object moves from one container to another.

Consider a file which is opened in 2 processes. one of the processes wants to move to another container then. How would you decide whether to change the file owner or not?

Quoted 2 lines

If a process has sufficient rights to move a file to a new container then it should be okay to assign the file to the new container.

there is no such notion as "rights to move a file to a new container". The same file can be opened in processes belonging to other containers. And you have no any clue whether to have to change the owner or not.

Quoted 5 lines

Though the point is, if a resource (like file) is getting migrated to a new container then all the attributes (like owner, #pages in memory etc.) attached to that resource (file) should also migrate to this new container. Otherwise the semantics of where does the resource belong becomes very difficult.

The same for many other resources. It is a big mistake thinking that most resources belong to the processes and the owner process can be easily determined.

OpenVZ assumes that tasks can't move between task-groups for a single reason:
user shouldn't be able to escape from the container.
But this have no implication on the design/implementation.

BTW, do you see any practical use cases for tasks jumping between resource-containers?

Quoted 3 lines

OpenVZ assumes that tasks can't move between task-groups for a single reason:
user shouldn't be able to escape from the container.
But this have no implication on the design/implementation.

It does, for the memory controller at least. Things like shared anon_vma's between tasks across containers make it somewhat harder. It's much worse if you allow threads to split across containers.

Quoted 1 line

BTW, do you see any practical use cases for tasks jumping between resource-containers?

Quoted 6 lines

Quoted 3 lines

OpenVZ assumes that tasks can't move between task-groups for a single reason:
user shouldn't be able to escape from the container.
But this have no implication on the design/implementation.

Quoted 3 lines

It does, for the memory controller at least. Things like shared anon_vma's between tasks across containers make it somewhat harder. It's much worse if you allow threads to split across containers.

we already have the code to account page fractions shared between containers.
Though, it is quite useless to do so for threads... Since this numbers have no meaning (not a real usage) and only the sum of it will be a correct value.

Quoted 4 lines

we already have the code to account page fractions shared between containers.
Though, it is quite useless to do so for threads... Since this numbers have no meaning (not a real usage) and only the sum of it will be a correct value.

THat sort of accounting poses various horrible problems, which is why we steered away from it. If you share pages between containers (presumably billing them equal shares per user), what happens when you're already at your limit, and one of your sharer's exits?

Plus, are you billing by vma or address_space?

Quoted 10 lines

Quoted 6 lines

we already have the code to account page fractions shared between containers.
Though, it is quite useless to do so for threads... Since this numbers have no meaning (not a real usage)
and only the sum of it will be a correct value.

THat sort of accounting poses various horrible problems, which is why we steered away from it. If you share pages between containers (presumably billing them equal shares per user), what happens when you're already at your limit, and one of your sharer's exits?

you come across your limit and new allocations will fail.
BUT! IMPORTANT!
in real life use case with OpenVZ we allow sharing not that much data across containers:
vmas mapped as private, i.e. glibc and other libraries .data section
(and .code if it is writable). So if you use the same glibc and other executables in the containers then your are charged only a fraction of phys memory used by it. This kind of sharing is not that huge (<< memory limits usually),
so the situation you described is not a problem
in real life (at least for OpenVZ).

I think it is not a problem for OpenVZ because there is not that much of
sharing going between containers as you mentioned (btw, this least amount of sharing is a very good thing). Though I'm not sure if one has
to go to the extent of doing fractions with memory accounting. If the containers are set up in such a way that there is some sharing across containers then it is okay to be unfair and charge one of those containers for the specific resource completely.

Quoted 9 lines · Rohit Seth

I think it is not a problem for OpenVZ because there is not that much of
sharing going between containers as you mentioned (btw, this least amount of sharing is a very good thing). Though I'm not sure if one has
to go to the extent of doing fractions with memory accounting. If the containers are set up in such a way that there is some sharing across containers then it is okay to be unfair and charge one of those containers for the specific resource completely.

Right, and if you do reclaim against containers which are over their limits, the containers being unfairly charged will tend to get hit first. But, once this happens, I would hope that the ownership of those shared pages should settle out among all of the users.

If you have 100 containers sharing 100 pages, container0 might be charged for all 100 pages at first, but I'd hope that eventually containers 0->99 would each get charged for a single page.

Quoted 12 lines · Dave Hansen

Quoted 9 lines · Rohit Seth

I think it is not a problem for OpenVZ because there is not that much of
sharing going between containers as you mentioned (btw, this least amount of sharing is a very good thing). Though I'm not sure if one has
to go to the extent of doing fractions with memory accounting. If the containers are set up in such a way that there is some sharing across containers then it is okay to be unfair and charge one of those containers for the specific resource completely.

Quoted 5 lines

Right, and if you do reclaim against containers which are over their limits, the containers being unfairly charged will tend to get hit first. But, once this happens, I would hope that the ownership of those shared pages should settle out among all of the users.

I think there is lot of simplicity and value add by charging one container (even unfairly) for one resource completely. This puts the onus on system admin to set up the containers appropriately.

Rohit Seth wrote:

Quoted 3 lines

I think there is lot of simplicity and value add by charging one container (even unfairly) for one resource completely. This puts the onus on system admin to set up the containers appropriately.

It also saves you from maintaining huge lists against each page.

Worse case, you want to bill everyone who opens that address_space equally. But the semantics on exit still suck.

What was Alan's quote again? "unfair, unreliable, inefficient ... pick at least one out of the three". or something like that.

This does give you kernel (slab, pagetable, etc) allocations as well as userspace. I don't like the idea of doing controllers for inode cache and controllers for dentry cache, etc, etc, ad infinitum.

Quoted 3 lines · Nick Piggin

This does give you kernel (slab, pagetable, etc) allocations as well as userspace. I don't like the idea of doing controllers for inode cache and controllers for dentry cache, etc, etc, ad infinitum.

Those two might not be such a bad idea. Of the slab in my system, 90% is reliably from those two slabs alone. Now, a controller for the 'Acpi-Operand' slab might be going too far. ;)

Certainly something we should at least consider down the road.

But if you have a unified struct page accounting, you don't need that. You don't need struct radix_tree_node accounting, you don't need buffer_head accounting, pagetable page accounting, vm_area_struct accounting, task_struct accounting, etc etc in order to do your memory accounting if what you just want to know is "who allocated what".

And remember that if you have one container going crazy with inode/dentry cache, it will get hit by its resource limit and end up having to reclaim them or go OOM.

Now you *may* want to split the actual accounting into kernel and user parts if you're worried about obscure corner cases in kernel memory accounting. But this would basically come for free when you have the GFP_EASYRECLAIM thingy (at any rate, it is quite unintrusive).

Basically, what I have been hearing is that people want to be able to surgically isolate the memory allocation of one container from that of another. IMO this is simply infeasible (and exploit prone) to do it on a per-kernel-object basis.

Quoted 5 lines

It also saves you from maintaining huge lists against each page.

Worse case, you want to bill everyone who opens that address_space equally. But the semantics on exit still suck.

Quoted 2 lines

What was Alan's quote again? "unfair, unreliable, inefficient ... pick at least one out of the three". or something like that.

What's the sucking semantics on exit? I haven't looked much at the existing memory controllers going around, but the implementation I imagine looks something like this (I think it is conceptually similar to the basic beancounters idea):

- anyone who allocates a page for anything gets charged for that page. Except interrupt/softirq context. Could we ignore these for the moment?

This does give you kernel (slab, pagetable, etc) allocations as well as userspace. I don't like the idea of doing controllers for inode cache and controllers for dentry cache, etc, etc, ad infinitum.

- each struct page has a backpointer to its billed container. At the mm summit Linus said he didn't want back pointers, but I clarified with him and he isn't against them if they are easily configured out when not using memory controllers.

- memory accounting containers are in a hierarchy. If you want to destroy a container but it still has billed memory outstanding, that gets charged back to the parent. The data structure itself obviously still needs to stay around, to keep the backpointers from going stale... but that could be as little as a word or two in size.

The reason I like this way of accounting is that it can be done with a couple of hooks into page_alloc.c and an ifdef in mm.h, and that is the extent of the impact on core mm/ so I'd be against anything more intrusive unless this really doesn't work.

Quoted 5 lines

Quoted 5 lines

It also saves you from maintaining huge lists against each page.

Worse case, you want to bill everyone who opens that address_space equally. But the semantics on exit still suck.

Quoted 8 lines

Quoted 2 lines

What was Alan's quote again? "unfair, unreliable, inefficient ... pick at least one out of the three". or something like that.

What's the sucking semantics on exit? I haven't looked much at the existing memory controllers going around, but the implementation I imagine looks something like this (I think it is conceptually similar to the basic beancounters idea):

You have to increase the other processes allocations, putting them over their limits. If you then force them into reclaim, they're going to stall, and give bad latency.

Quoted 25 lines

- anyone who allocates a page for anything gets charged for that page. Except interrupt/softirq context. Could we ignore these for the
moment?

This does give you kernel (slab, pagetable, etc) allocations as
well as
userspace. I don't like the idea of doing controllers for inode cache and controllers for dentry cache, etc, etc, ad infinitum.

- each struct page has a backpointer to its billed container. At the mm summit Linus said he didn't want back pointers, but I clarified
with him
and he isn't against them if they are easily configured out when
not using memory controllers.

- memory accounting containers are in a hierarchy. If you want to destroy a
container but it still has billed memory outstanding, that gets
charged
back to the parent. The data structure itself obviously still needs to stay around, to keep the backpointers from going stale... but that
could
be as little as a word or two in size.

Quoted 5 lines

The reason I like this way of accounting is that it can be done with a couple of hooks into page_alloc.c and an ifdef in mm.h, and that is the extent of the impact on core mm/ so I'd be against anything more intrusive unless this really doesn't work.

See "inefficent" above (sorry ;-)) What you've chosen is more correct, but much higher overhead. The point was that there's tradeoffs either way - the conclusion we came to last time was that to make it 100% correct, you'd be better off going with a model like Xen.

1. You're adding a backpointer to struct page.

2. Each page is not accounted to one container, but shared across them, so the billing changes every time someone forks or exits. And not just for that container, but all of them. Think pte chain based rmap ... except worse.

3. When a container needs to "shrink" when somebody else exits, how do we do reclaim pages from a specific container?

What's the sucking semantics on exit? I haven't looked much at the existing memory controllers going around, but the implementation I imagine looks something like this (I think it is conceptually similar to the basic beancounters idea):

- anyone who allocates a page for anything gets charged for that page. Except interrupt/softirq context. Could we ignore these for the moment?

This does give you kernel (slab, pagetable, etc) allocations as well as userspace. I don't like the idea of doing controllers for inode cache and controllers for dentry cache, etc, etc, ad infinitum.

- each struct page has a backpointer to its billed container. At the mm summit Linus said he didn't want back pointers, but I clarified with him and he isn't against them if they are easily configured out when not using memory controllers.

- memory accounting containers are in a hierarchy. If you want to destroy a container but it still has billed memory outstanding, that gets charged back to the parent. The data structure itself obviously still needs to stay around, to keep the backpointers from going stale... but that could be as little as a word or two in size.

The reason I like this way of accounting is that it can be done with a couple of hooks into page_alloc.c and an ifdef in mm.h, and that is the extent of the impact on core mm/ so I'd be against anything more intrusive unless this really doesn't work.

- each struct page has a backpointer to its billed container. At the mm summit Linus said he didn't want back pointers, but I clarified with him and he isn't against them if they are easily configured out when not using memory controllers.

Quoted 5 lines · Nick Piggin

- each struct page has a backpointer to its billed container. At the mm summit Linus said he didn't want back pointers, but I clarified with him and he isn't against them if they are easily configured out when not using memory controllers.

This would need to be done at runtime though, otherwise it's useless for distributions and other people who want single kernel binary images.

Probably would need a second parallel table, but for that you would need to know at boot already if you plan to use them or not. Ugly.

I've been thinking a bit about replacing the mapping and index members in struct page with a single pointer that point into a cluster data type. The cluster data type is aligned to a power of two and contains a header that is shared between all pages within the cluster. The header contains a base index and mapping. The rest of the cluster is an array of pfn:s that point back to the actual page.

What's the sucking semantics on exit? I haven't looked much at the existing memory controllers going around, but the implementation I imagine looks something like this (I think it is conceptually similar to the basic beancounters idea):

- anyone who allocates a page for anything gets charged for that page. Except interrupt/softirq context. Could we ignore these for the moment?

This does give you kernel (slab, pagetable, etc) allocations as well as userspace. I don't like the idea of doing controllers for inode cache and controllers for dentry cache, etc, etc, ad infinitum.

- each struct page has a backpointer to its billed container. At the mm summit Linus said he didn't want back pointers, but I clarified with him and he isn't against them if they are easily configured out when not using memory controllers.

Quoted 10 lines

you come across your limit and new allocations will fail.
BUT! IMPORTANT!
in real life use case with OpenVZ we allow sharing not that much data across containers:
vmas mapped as private, i.e. glibc and other libraries .data section
(and .code if it is writable). So if you use the same glibc and other executables in the containers then your are charged only a fraction of phys memory used by it. This kind of sharing is not that huge (<< memory limits usually),
so the situation you described is not a problem
in real life (at least for OpenVZ).

I think it is not a problem for OpenVZ because there is not that much of sharing going between containers as you mentioned (btw, this least amount of sharing is a very good thing). Though I'm not sure if one has to go to the extent of doing fractions with memory accounting. If the containers are set up in such a way that there is some sharing across containers then it is okay to be unfair and charge one of those containers for the specific resource completely.

Quoted 4 lines

I think the risk is that OpenVZ has all the controls and resource managers we need, while CKRM is still more research-ish. I find the OpenVZ code much clearer, cleaner and complete at the moment, although also much more conservative in its approach to solving problems.

I think it would be nice to compare first the features provided by ckrm and openvz at some point and agree upon the minimum common features we need to have as we go forward. For instance I think Openvz assumes that tasks do not need to move between containers (task-groups), whereas ckrm provides this
flexibility for workload management. This may have some effect on the controller/interface design, no?

BTW, to help to compare (as you noted above) here is the list of features provided by OpenVZ:

Quoted 4 lines

BTW, to help to compare (as you noted above) here is the list of features provided by OpenVZ:

Could you point to a place where we can get a broken-down set of patches for OpenVZ or (even better), UBC ?

For purposes of the resource management discussion, it will be useful to be able to look at the UBC patches in isolation and perhaps port them over to some common interface for testing comparing with other implementations.

Quoted 10 lines · Shailabh Nagar

Quoted 2 lines

BTW, to help to compare (as you noted above) here is the list of features provided by OpenVZ:

Could you point to a place where we can get a broken-down set of patches for OpenVZ or (even better), UBC ?

For purposes of the resource management discussion, it will be useful to be able to look at the UBC patches in isolation and perhaps port them over to some common interface for testing comparing with other implementations.

Kirill,

Is this the latest set (based on your last publication) for people to look at ?

http://download.openvz.org/kernel/broken-out/2.6.16-026test005.1/

Thanks
Dipankar

Quoted 10 lines · Shailabh Nagar

Quoted 2 lines

BTW, to help to compare (as you noted above) here is the list of features provided by OpenVZ:

Could you point to a place where we can get a broken-down set of patches for OpenVZ or (even better), UBC ?

For purposes of the resource management discussion, it will be useful to be able to look at the UBC patches in isolation and perhaps port them over to some common interface for testing comparing with other implementations.

Kirill,

Is this the latest set (based on your last publication) for people to look at ?

http://download.openvz.org/kernel/broken-out/2.6.16-026test005.1/

Quoted 4 lines

The downside to such a strategy is that there is a risk that nobody ever
gets around to implementing useful controllers, so it ends up dead code. I'd judge that the interest in resource management is such that the risk of this happening is low.

I think the risk is that OpenVZ has all the controls and resource managers we need, while CKRM is still more research-ish. I find the OpenVZ code much clearer, cleaner and complete at the moment, although also much more conservative in its approach to solving problems.

And now we've dumped the good infrastructure and instead we've contentrated on the controller, wired up via some imaginative ab^H^Hreuse of the cpuset layer.

I haven't read it yet, but I will likely agree that this is an abuse of cpusets.

On Fri, 4 Aug 2006 10:37:53 +0530

Quoted 13 lines

Resource management has been talked about quite extensively in the past, more recently in the context of containers. The basic requirement here is to provide isolation between *groups* of task wrt their use of various resources like CPU, Memory, I/O bandwidth, open file-descriptors etc.

Different maintainers have however expressed different opinions over the need to complicate the kernel to meet this need, especially since it involves core kernel code like the resource schedulers.

A BoF was hence held at OLS this year to come to a consensus on the minimum requirements of a resource management solution for Linux kernel. Some notes taken at the BoF are posted here:

Quoted 5 lines

	http://www.uwsg.indiana.edu/hypermail/linux/kernel/0607.3/0896.html

An important consensus point of the BoF seemed to be "focus on real controllers more, preferably memory first, using some simple interface and task grouping mechanism".

ug, I didn't know this. Had I been there (sorry) I'd have disagreed with this whole strategy.

I thought the most recently posted CKRM core was a fine piece of code. It provides the machinery for grouping tasks together and the machinery for establishing and viewing those groupings via configfs, and other such common functionality. My 20-minute impression was that this code was an easy merge and it was just awaiting some useful controllers to come along.

And now we've dumped the good infrastructure and instead we've contentrated on the controller, wired up via some imaginative ab^H^Hreuse of the cpuset layer.

I wonder how many of the consensus-makers were familiar with the contemporary CKRM core?

f-series infrastructure

Quoted 1 line · Dipankar

f-series infrastructure

Do you have a good link to follow for more on this?

Quoted 5 lines · Paul Jackson

Quoted 1 line · Dipankar

f-series infrastructure

Do you have a good link to follow for more on this?

Here is the link to the last posting of the series by Chandra -

http://lkml.org/lkml/2006/4/27/378

Quoted 3 lines

And now we've dumped the good infrastructure and instead we've contentrated on the controller, wired up via some imaginative ab^H^Hreuse of the cpuset layer.

FWIW, this controller was originally written for f-series.

It should
be trivial to put it back there. So really, f-series isn't gone anywhere. If you want to merge it, I am sure it can be re-submitted.

Quoted 2 lines

I wonder how many of the consensus-makers were familiar with the contemporary CKRM core?

I think what would be nice is a clear strategy on whether we need to work out the infrastructure or the controllers first.

One of
the strongest points raised in the BoF was - forget the infrastructure for now, get some mergable controllers developed.

If you
want to stick to f-series infrastructure and want to see some consensus controllers evolve on top of it, that can be done too.

On Fri, 4 Aug 2006 11:50:36 +0530

Quoted 2 lines

FWIW, this controller was originally written for f-series.

What the heck is an f-series?

Quoted 3 lines

It should
be trivial to put it back there. So really, f-series isn't gone anywhere. If you want to merge it, I am sure it can be re-submitted.

Well. It shouldn't be a matter of what I want to merge - you're the resource-controller guys. But...

Quoted 3 lines

One of
the strongest points raised in the BoF was - forget the infrastructure for now, get some mergable controllers developed.

I wonder what inspired that approach. Perhaps it was a reaction to CKRM's long and difficult history? Perhaps it was felt that doing standalne controllers with ad-hoc interfaces would make things easier to merge?

Perhaps. But I think you guys know that the end result would be inferior, and that getting good infrastructure in place first will produce a better end result, but you decided to go this way because you want to get _something_ going?

Quoted 3 lines

If you
want to stick to f-series infrastructure and want to see some consensus controllers evolve on top of it, that can be done too.

Do you think that the CKRM core as last posted had any unnecessary features? I don't have the operational experience to be able to judge that, so I'd prefer to trust your experience and judgement on that. But the features which _were_ there looked quite OK from an implementation POV.

So my take was "looks good, done deal - let's go get some sane controllers working". And now this!

Quoted 2 lines

So my take was "looks good, done deal - let's go get some sane controllers working". And now this!

My recollection from the BoF is that people felt interface wasn't a major issue and it wasn't discussed much. Most of the discussion centered around grouping of tasks and the mem/cpu controllers and some additional resource control requirements from openvz folks.

One way to go forward with the interface could be to request Chandra to repost the ckrm infrastructure and see if the stake holders (primarily container folks) can review and agree on it.

Quoted 5 lines · On Fri, 4 Aug 2006 11:50:36 +0530 Dipankar Sarma <[email protected]> wrote:

Quoted 3 lines

And now we've dumped the good infrastructure and instead we've contentrated on the controller, wired up via some imaginative ab^H^Hreuse of the cpuset layer.

FWIW, this controller was originally written for f-series.

What the heck is an f-series?

<googles, looks at
http://images.automotive.com/cob/factory_automotive/images/Features/auto_shows/2005_IEAS/2005_Ford_F-series_Harley-Davidson_front.JPG,
gets worried about IBM design methodologies>

Quoted 4 lines

It should
be trivial to put it back there. So really, f-series isn't gone anywhere. If you want to merge it, I am sure it can be re-submitted.

Well. It shouldn't be a matter of what I want to merge - you're the resource-controller guys. But...

Quoted 5 lines

Quoted 2 lines

I wonder how many of the consensus-makers were familiar with the contemporary CKRM core?

I think what would be nice is a clear strategy on whether we need to work out the infrastructure or the controllers first.

We should put our thinking hats on and decide what sort of infrastructure will need to be in place by the time we have a useful number of useful controllers implemented.

Quoted 3 lines

One of
the strongest points raised in the BoF was - forget the infrastructure for now, get some mergable controllers developed.

I wonder what inspired that approach. Perhaps it was a reaction to CKRM's long and difficult history? Perhaps it was felt that doing standalne controllers with ad-hoc interfaces would make things easier to merge?

Perhaps. But I think you guys know that the end result would be inferior, and that getting good infrastructure in place first will produce a better end result, but you decided to go this way because you want to get _something_ going?

Quoted 4 lines

If you
want to stick to f-series infrastructure and want to see some consensus controllers evolve on top of it, that can be done too.

Do you think that the CKRM core as last posted had any unnecessary features? I don't have the operational experience to be able to judge

that, so I'd prefer to trust your experience and judgement on that. But the features which _were_ there looked quite OK from an implementation POV.

So my take was "looks good, done deal - let's go get some sane controllers working". And now this!

ug, I didn't know this. Had I been there (sorry) I'd have disagreed with this whole strategy.

I thought the most recently posted CKRM core was a fine piece of code. It provides the machinery for grouping tasks together and the machinery for establishing and viewing those groupings via configfs, and other such common functionality. My 20-minute impression was that this code was an easy merge and it was just awaiting some useful controllers to come along.

And now we've dumped the good infrastructure and instead we've contentrated on the controller, wired up via some imaginative ab^H^Hreuse of the cpuset layer.

Right. We won't be controlling memory, numtasks, disk, network etc controllers via cpusets, will we?

Correct me if I'm wrong, but a cpuset isn't the appropriate machinery to be using to group tasks.

And if this whole resource-control effort is to end up being successful, it should have as core infrastructure a flexible, appropriate and uniform way of grouping tasks together and of getting data into and out of those aggregates. We already have that, don't we?

Quoted 9 lines · Andrew Morton

ug, I didn't know this. Had I been there (sorry) I'd have disagreed with this whole strategy.

I thought the most recently posted CKRM core was a fine piece of code. It provides the machinery for grouping tasks together and the machinery for establishing and viewing those groupings via configfs, and other such common functionality. My 20-minute impression was that this code was an easy merge and it was just awaiting some useful controllers to come along.

Quoted 3 lines

And now we've dumped the good infrastructure and instead we've contentrated on the controller, wired up via some imaginative ab^H^Hreuse of the cpuset layer.

Andrew,
CPUset was used in this patch series primarily because it
represent a task-grouping mechanism already in the kernel and because people at the BoF wanted to start with something simple. The idea of using cpusets here was not to push this as a final solution, but use it as a means to discuss the effects of task-grouping on CPU scheduler.

We had be more than happy to work with the ckrm core which was posted last. In fact I had sent out the same cpu controller against ckrm core itself last time around to Nick/Ingo.

There was a lot of discussion last time - Mike, Ingo, others. It would be a useful starting point if we could be refreshed on what the main issues were, and whether/how this new patchset addresses them.

Quoted 3 lines · Andrew Morton

There was a lot of discussion last time - Mike, Ingo, others. It would be a useful starting point if we could be refreshed on what the main issues were, and whether/how this new patchset addresses them.

The main issues raised against the CPU controller posted last time were these:

Ingo/Nick, what are your thoughts here?

ug, I didn't know this. Had I been there (sorry) I'd have disagreed with this whole strategy.

I thought the most recently posted CKRM core was a fine piece of code. It provides the machinery for grouping tasks together and the machinery for establishing and viewing those groupings via configfs, and other such common functionality. My 20-minute impression was that this code was an easy merge and it was just awaiting some useful controllers to come along.

And now we've dumped the good infrastructure and instead we've contentrated on the controller, wired up via some imaginative ab^H^Hreuse of the cpuset layer.

I wonder how many of the consensus-makers were familiar with the contemporary CKRM core?

Quoted 1 line

- Design of individual resource controllers like memory and cpu

Right. We won't be controlling memory, numtasks, disk, network etc controllers via cpusets, will we?

For example:
1. Should task-group be changeable after set/inherited once? Are you planning to recalculate resources on group change? e.g. shared memory or used kernel memory is hard to recalculate.

2. should task-group resource container manage all the resources as a whole? e.g. in OpenVZ tasks can belong to different CPU and UBC containers. It is more flexible and e.g. we used to put some vital kernel threads to a separate CPU group to decrease delays in service.

3. I also don't understand why normal binary interface like system call is not used.
We have set_uid, sys_setrlimit and it works pretty good, does it?

4. do we want hierarchical grouping?

I was familiar. And I can arise many arguable points in the CKRM infrastructure.

For example:
1. Should task-group be changeable after set/inherited once? Are you planning to recalculate resources on group change? e.g. shared memory or used kernel memory is hard to recalculate.

2. should task-group resource container manage all the resources as a whole?
e.g. in OpenVZ tasks can belong to different CPU and UBC containers. It is more flexible and e.g. we used to put some vital kernel threads to a separate CPU group to decrease delays in service.

3. I also don't understand why normal binary interface like system call is not used.
We have set_uid, sys_setrlimit and it works pretty good, does it?

4. do we want hierarchical grouping?

Quoted 6 lines

Quoted 1 line

- Design of individual resource controllers like memory and cpu

Right. We won't be controlling memory, numtasks, disk, network etc controllers via cpusets, will we?

I hope so :)

Thanks,
Kirill

Quoted 3 lines

3. I also don't understand why normal binary interface like system call is not used.
We have set_uid, sys_setrlimit and it works pretty good, does it?

If there are no hierarchies, a syscall interface is fine since the namespace for the task-group is flat (so one can export to userspace either a number or a string as a handle to that task-group for operations like create, delete, set limit, get usage, etc)

A filesystem based interface is useful when you have hierarchies (as resource groups and cpusets do) since it naturally defines a convenient to use hierarchical namespace.

Quoted 3 lines

A filesystem based interface is useful when you have hierarchies (as resource groups and cpusets do) since it naturally defines a convenient to use hierarchical namespace.

but it is not much convinient for applications then.

Quoted 4 lines · Kirill

Quoted 3 lines

A filesystem based interface is useful when you have hierarchies (as resource groups and cpusets do) since it naturally defines a convenient to use hierarchical namespace.

but it is not much convinient for applications then.

Is this simply a language issue? File systems hierarchies are more easily manipulated with shell utilities (ls, cat, find, grep, ...) and system call API's are easier to access from C?

If so, then perhaps all that's lacking for convenient C access to a filesystem based interface is a good library, that presents an API convenient for use from C code, but underneath makes the necessary file system calls (open, read, diropen, stat, ...).

Quoted 5 lines

Quoted 3 lines

A filesystem based interface is useful when you have hierarchies (as resource groups and cpusets do) since it naturally defines a convenient to use hierarchical namespace.

but it is not much convinient for applications then.

Is this simply a language issue? File systems hierarchies are more easily manipulated with shell utilities (ls, cat, find, grep, ...) and system call API's are easier to access from C?

If so, then perhaps all that's lacking for convenient C access to a filesystem based interface is a good library, that presents an API convenient for use from C code, but underneath makes the necessary file system calls (open, read, diropen, stat, ...).

we have a /proc which is very convenient for use from shell etc. but is not good for applications, not fast enough etc.
moreover, /proc had always problems with locking, races and people tend to feel like they can change text presention of data, while applications parsing it tend to break.

Quoted 5 lines

we have a /proc which is very convenient for use from shell etc. but is not good for applications, not fast enough etc.
moreover, /proc had always problems with locking, races and people tend to feel like they can change text presention of data, while applications parsing it tend to break.

Yes - one can botch a file system API.

One can botch syscalls, too. Do you love 'ioctl'?

For some calls, the performance of a raw syscall is critical. And eventually, filesystem API's must resolve to raw file i/o syscalls.

But for these sorts of system configuration and management, the difference in performance between file system API's and raw syscall API's is not one of the decisive issues that determines success or failure.

Getting a decent API that naturally reflects the long term essential shape of the data is one of these decisive issues.

Quoted 8 lines · On Fri, 4 Aug 2006 10:37:53 +0530 Srivatsa Vaddagiri <[email protected]> wrote:

Resource management has been talked about quite extensively in the past, more recently in the context of containers. The basic requirement here is to provide isolation between *groups* of task wrt their use of various resources like CPU, Memory, I/O bandwidth, open file-descriptors etc.

Different maintainers have however expressed different opinions over the need to
complicate the kernel to meet this need, especially since it involves core kernel code like the resource schedulers.

Quoted 6 lines

A BoF was hence held at OLS this year to come to a consensus on the minimum requirements of a resource management solution for Linux kernel. Some notes taken at the BoF are posted here:

	http://www.uwsg.indiana.edu/hypermail/linux/kernel/0607.3/0896.html

An important consensus point of the BoF seemed to be "focus on real controllers more, preferably memory first, using some simple interface and task grouping mechanism".

ug, I didn't know this. Had I been there (sorry) I'd have disagreed with this whole strategy.

I thought the most recently posted CKRM core was a fine piece of code. It provides the machinery for grouping tasks together and the machinery for establishing and viewing those groupings via configfs, and other such common functionality. My 20-minute impression was that this code was an easy merge and it was just awaiting some useful controllers to come along.

And now we've dumped the good infrastructure and instead we've contentrated on the controller, wired up via some imaginative ab^H^Hreuse of the cpuset layer.

I wonder how many of the consensus-makers were familiar with the contemporary CKRM core?