CFS Hard limits - v2

kernel/sched_fair.c +22 -0

index 652e8bd..eeeddb8 100644
@@ -243,6 +243,27 @@ find_matching_se(struct sched_entity **se, struct sched_entity **pse)
 
 #endif	/* CONFIG_FAIR_GROUP_SCHED */
 
+static void add_cfs_rq_tasks_running(struct sched_entity *se,
+		unsigned long count)
+{
+	struct cfs_rq *cfs_rq;
+
+	for_each_sched_entity(se) {
+		cfs_rq = cfs_rq_of(se);
+		cfs_rq->nr_tasks_running += count;
+	}
+}
+
+static void sub_cfs_rq_tasks_running(struct sched_entity *se,
+		unsigned long count)
+{
+	struct cfs_rq *cfs_rq;
+
+	for_each_sched_entity(se) {
+		cfs_rq = cfs_rq_of(se);
+		cfs_rq->nr_tasks_running -= count;
+	}
+}
 
 /**************************************************************
  * Scheduling class tree data structure manipulation methods:

@@ -969,6 +990,7 @@ static void enqueue_task_fair(struct rq *rq, struct task_struct *p, int wakeup)
 		wakeup = 1;
 	}
 
+	add_cfs_rq_tasks_running(&p->se, 1);
 	hrtick_update(rq);
 }
 

Quoted 44 lines · Bharata B Rao

kernel/sched_fair.c +22 -0

index 652e8bd..eeeddb8 100644
@@ -243,6 +243,27 @@ find_matching_se(struct sched_entity **se, struct sched_entity **pse)
 
 #endif	/* CONFIG_FAIR_GROUP_SCHED */
 
+static void add_cfs_rq_tasks_running(struct sched_entity *se,
+		unsigned long count)
+{
+	struct cfs_rq *cfs_rq;
+
+	for_each_sched_entity(se) {
+		cfs_rq = cfs_rq_of(se);
+		cfs_rq->nr_tasks_running += count;
+	}
+}
+
+static void sub_cfs_rq_tasks_running(struct sched_entity *se,
+		unsigned long count)
+{
+	struct cfs_rq *cfs_rq;
+
+	for_each_sched_entity(se) {
+		cfs_rq = cfs_rq_of(se);
+		cfs_rq->nr_tasks_running -= count;
+	}
+}
 
 /**************************************************************
  * Scheduling class tree data structure manipulation methods:

@@ -969,6 +990,7 @@ static void enqueue_task_fair(struct rq *rq, struct task_struct *p, int wakeup)
 		wakeup = 1;
 	}
 
+	add_cfs_rq_tasks_running(&p->se, 1);
 	hrtick_update(rq);
 }
 

@@ -991,6 +1013,7 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int sleep)
 		sleep = 1;
 	}
 

kernel/sched_fair.c static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int sleep) · line 1018

Quoted 3 lines

kernel/sched_fair.c +1 -0

+	sub_cfs_rq_tasks_running(&p->se, 1);
 	hrtick_update(rq);
 }

This seems daft, why not add the incement to the for_each_sched_entity() loop already present in both functions?

kernel/sched.c +0 -0

index c283d0f..0147f6f 100644
@@ -262,6 +262,15 @@ static DEFINE_MUTEX(sched_domains_mutex);
 

kernel/sched.c +0 -0

 #endif
 
 #ifdef CONFIG_RT_GROUP_SCHED

@@ -477,6 +491,16 @@ struct cfs_rq {
 	unsigned long rq_weight;
 #endif
 #endif

kernel/sched.c +5 -0

 	/*
 	 * Number of tasks at this heirarchy.
 	 */

@@ -665,6 +689,11 @@ struct rq {
 	/* BKL stats */
 	unsigned int bkl_count;
 #endif
+	/*
+	 * Protects the cfs runtime related fields of all cfs_rqs under
+	 * this rq
+	 */
+	spinlock_t runtime_lock;
 };
 
 static DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
 

CONFIG_RT_GROUP_SCHED

kernel/sched.c +10 -0

@@ -10317,6 +10617,23 @@ static struct cftype cpu_files[] = {
 		.read_u64 = cpu_shares_read_u64,
 		.write_u64 = cpu_shares_write_u64,
 	},
+#ifdef CONFIG_CFS_HARD_LIMITS
+	{
+		.name = "cfs_runtime_us",
+		.read_s64 = cpu_cfs_runtime_read_s64,
+		.write_s64 = cpu_cfs_runtime_write_s64,
+	},
+	{
+		.name = "cfs_period_us",
+		.read_u64 = cpu_cfs_period_read_u64,
+		.write_u64 = cpu_cfs_period_write_u64,

Quoted 8 lines · Bharata B Rao

kernel/sched.c +0 -0

index c283d0f..0147f6f 100644
@@ -262,6 +262,15 @@ static DEFINE_MUTEX(sched_domains_mutex);
 
 #include <linux/cgroup.h>
 

kernel/sched.c static DEFINE_MUTEX(sched_domains_mutex); · line 272

Quoted 8 lines

kernel/sched.c +8 -0

+#if defined(CONFIG_FAIR_GROUP_SCHED) && defined(CONFIG_CFS_HARD_LIMITS)
+struct cfs_bandwidth {
+	spinlock_t		cfs_runtime_lock;
+	ktime_t			cfs_period;
+	u64			cfs_runtime;
+	struct hrtimer		cfs_period_timer;
+};
+#endif

too much cfs here..

kernel/sched.c struct task_group { · line 298

Quoted 13 lines

kernel/sched.c +5 -0

 struct cfs_rq;
 
 static LIST_HEAD(task_groups);

@@ -282,6 +291,11 @@ struct task_group {
 	/* runqueue "owned" by this group on each cpu */
 	struct cfs_rq **cfs_rq;
 	unsigned long shares;
+#ifdef CONFIG_CFS_HARD_LIMITS
+	struct cfs_bandwidth cfs_bandwidth;
+	/* If set, throttle when the group exceeds its bandwidth */
+	int hard_limit_enabled;
+#endif

What's wrong with doing something like cfs_bandwidth.cfs_runtime == RUNTIME_INF ?

Quoted 12 lines

kernel/sched.c +4 -0

 #endif
 
 #ifdef CONFIG_RT_GROUP_SCHED

@@ -477,6 +491,16 @@ struct cfs_rq {
 	unsigned long rq_weight;
 #endif
 #endif
+#ifdef CONFIG_CFS_HARD_LIMITS
+	/* set when the group is throttled  on this cpu */
+	int cfs_throttled;
+

kernel/sched.c struct cfs_rq { · line 503

Quoted 6 lines

kernel/sched.c +6 -0

+	/* runtime currently consumed by the group on this rq */
+	u64 cfs_time;
+
+	/* runtime available to the group on this rq */
+	u64 cfs_runtime;
+#endif

too much cfs_ again.

Quoted 16 lines

kernel/sched.c +5 -0

 	/*
 	 * Number of tasks at this heirarchy.
 	 */

@@ -665,6 +689,11 @@ struct rq {
 	/* BKL stats */
 	unsigned int bkl_count;
 #endif
+	/*
+	 * Protects the cfs runtime related fields of all cfs_rqs under
+	 * this rq
+	 */
+	spinlock_t runtime_lock;
 };
 
 static DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);

kernel/sched.c static inline const struct cpumask *sched_bw_period_mask(void) · line 1829

Quoted 9 lines

kernel/sched.c +9 -0

+static inline void rq_runtime_lock(struct rq *rq)
+{
+	spin_lock(&rq->runtime_lock);
+}
+
+static inline void rq_runtime_unlock(struct rq *rq)
+{
+	spin_unlock(&rq->runtime_lock);
+}

needless obfuscation.

Quoted 20 lines

CONFIG_RT_GROUP_SCHED

kernel/sched.c +14 -0

@@ -10317,6 +10617,23 @@ static struct cftype cpu_files[] = {
 		.read_u64 = cpu_shares_read_u64,
 		.write_u64 = cpu_shares_write_u64,
 	},
+#ifdef CONFIG_CFS_HARD_LIMITS
+	{
+		.name = "cfs_runtime_us",
+		.read_s64 = cpu_cfs_runtime_read_s64,
+		.write_s64 = cpu_cfs_runtime_write_s64,
+	},
+	{
+		.name = "cfs_period_us",
+		.read_u64 = cpu_cfs_period_read_u64,
+		.write_u64 = cpu_cfs_period_write_u64,
+	},
+	{
+		.name = "cfs_hard_limit",
+		.read_u64 = cpu_cfs_hard_limit_read_u64,

kernel/sched.c static struct cftype cpu_files[] = { · line 10639

Quoted 6 lines

kernel/sched.c +3 -0

+		.write_u64 = cpu_cfs_hard_limit_write_u64,
+	},
+#endif /* CONFIG_CFS_HARD_LIMITS */
 #endif
 #ifdef CONFIG_RT_GROUP_SCHED
 	{

I guess that cfs_hard_limit thing is superfluous as well.

include/linux/sched.h +5 -0

@@ -3414,6 +3443,18 @@ int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu,
 	}
 
 	/*
+	 * Don't migrate the task if it belongs to a
+	 * - throttled group on its current cpu
+	 * - throttled group on this_cpu
+	 * - group whose hierarchy is throttled on this_cpu
+	 */

include/linux/sched.h struct sched_domain; · line 1030

Quoted 13 lines · Bharata B Rao

include/linux/sched.h +1 -1

index 0f1ea4a..77ace43 100644
@@ -1024,7 +1024,7 @@ struct sched_domain;
 struct sched_class {
 	const struct sched_class *next;
 
-	void (*enqueue_task) (struct rq *rq, struct task_struct *p, int wakeup);
+	int (*enqueue_task) (struct rq *rq, struct task_struct *p, int wakeup);
 	void (*dequeue_task) (struct rq *rq, struct task_struct *p, int sleep);
 	void (*yield_task) (struct rq *rq);
 

I really hate this, it uglfies all the enqueue code in a horrid way (which is most of this patch).

Why can't we simply enqueue the task on a throttled group just like rt?

Quoted 14 lines

include/linux/sched.h +9 -0

@@ -3414,6 +3443,18 @@ int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu,
 	}
 
 	/*
+	 * Don't migrate the task if it belongs to a
+	 * - throttled group on its current cpu
+	 * - throttled group on this_cpu
+	 * - group whose hierarchy is throttled on this_cpu
+	 */
+	if (cfs_rq_throttled(cfs_rq_of(&p->se)) ||
+		task_group_throttled(task_group(p), this_cpu)) {
+		schedstat_inc(p, se.nr_failed_migrations_throttled);
+		return 0;

include/linux/sched.h int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu, · line 3460

Quoted 6 lines

include/linux/sched.h +3 -0

+	}
+
+	/*
 	 * Aggressive migration if:
 	 * 1) task is cache cold, or
 	 * 2) too many balance attempts have failed.

Simply don't iterate throttled groups?

Quoted 19 lines · Peter Zijlstra

include/linux/sched.h struct sched_domain; · line 1030

Quoted 13 lines · Bharata B Rao

include/linux/sched.h +1 -1

index 0f1ea4a..77ace43 100644
@@ -1024,7 +1024,7 @@ struct sched_domain;
 struct sched_class {
    const struct sched_class *next;
 
-   void (*enqueue_task) (struct rq *rq, struct task_struct *p, int wakeup);
+   int (*enqueue_task) (struct rq *rq, struct task_struct *p, int wakeup);
    void (*dequeue_task) (struct rq *rq, struct task_struct *p, int sleep);
    void (*yield_task) (struct rq *rq);
 

I really hate this, it uglfies all the enqueue code in a horrid way (which is most of this patch).

Quoted 1 line

Why can't we simply enqueue the task on a throttled group just like rt?

We do enqueue a task to its group even if the group is throttled. However such throttled groups are not enqueued further. In such scenarios, even though the task enqueue to its parent group succeeded, it really didn't add any task to the cpu runqueue (rq). So we need to identify this condition and don't increment rq->running. That is why this return value is needed.

Quoted 22 lines · Bharata B Rao

Quoted 20 lines · Peter Zijlstra

include/linux/sched.h struct sched_domain; · line 1030

Quoted 13 lines · Bharata B Rao

include/linux/sched.h +1 -1

index 0f1ea4a..77ace43 100644
@@ -1024,7 +1024,7 @@ struct sched_domain;
 struct sched_class {
    const struct sched_class *next;
 
-   void (*enqueue_task) (struct rq *rq, struct task_struct *p, int wakeup);
+   int (*enqueue_task) (struct rq *rq, struct task_struct *p, int wakeup);
    void (*dequeue_task) (struct rq *rq, struct task_struct *p, int sleep);
    void (*yield_task) (struct rq *rq);
 

I really hate this, it uglfies all the enqueue code in a horrid way (which is most of this patch).

Why can't we simply enqueue the task on a throttled group just like rt?

Quoted 5 lines

We do enqueue a task to its group even if the group is throttled. However such throttled groups are not enqueued further. In such scenarios, even though the task enqueue to its parent group succeeded, it really didn't add any task to the cpu runqueue (rq). So we need to identify this condition and don't increment rq->running. That is why this return value is needed.

I would still consider those tasks running, the fact that they don't get to run is a different matter.

This added return value really utterly craps up the code and I'm not going to take it.

What I'm not seeing is why all this code looks so very much different from the rt bits.

Quoted 23 lines · Peter Zijlstra

Quoted 22 lines · Bharata B Rao

Quoted 20 lines · Peter Zijlstra

include/linux/sched.h struct sched_domain; · line 1030

Quoted 13 lines · Bharata B Rao

include/linux/sched.h +1 -1

index 0f1ea4a..77ace43 100644
@@ -1024,7 +1024,7 @@ struct sched_domain;
 struct sched_class {
    const struct sched_class *next;
 
-   void (*enqueue_task) (struct rq *rq, struct task_struct *p, int wakeup);
+   int (*enqueue_task) (struct rq *rq, struct task_struct *p, int wakeup);
    void (*dequeue_task) (struct rq *rq, struct task_struct *p, int sleep);
    void (*yield_task) (struct rq *rq);
 

I really hate this, it uglfies all the enqueue code in a horrid way (which is most of this patch).

Why can't we simply enqueue the task on a throttled group just like rt?

Quoted 8 lines

Quoted 5 lines

We do enqueue a task to its group even if the group is throttled. However such throttled groups are not enqueued further. In such scenarios, even though the task enqueue to its parent group succeeded, it really didn't add any task to the cpu runqueue (rq). So we need to identify this condition and don't increment rq->running. That is why this return value is needed.

I would still consider those tasks running, the fact that they don't get to run is a different matter.

Ok, that's how rt also considers them I realize. I thought that we should update rq->running when tasks go off the runqueue due to throttling. When a task is throttled, it is no doubt present on its group's cfs_rq, but it doesn't contribute to the CPU load as the throttled group entity isn't there on any cfs_rq. rq->running is used to obtain a few load balancing metrics and they might go wrong if rq->running isn't uptodate.

Do you still think we shouldn't update rq->running ? If so, I can get rid of this return value change.

Quoted 2 lines

This added return value really utterly craps up the code and I'm not going to take it.

OK :) I will work towards making them more acceptable in future iterations.

Quoted 3 lines

What I'm not seeing is why all this code looks so very much different from the rt bits.

Throttling code here looks different than rt for the following reasons:

- As I mentioned earlier, I update rq->running during throttling which is not done in rt afaics.
- There are special conditions to prevent movement of tasks in and out of the throttled groups during load balancing and migration.
- rt dequeues the throttled entity by walking the entity hierachy from update_curr_rt(). But I found it difficult to do the same in cfs because update_curr() is called from many different places and from places where we are actually walking the entity hiearchy. A second walk (in update_curr) of the hiearchy while we are in the middle of a hierarchy walk didn't look all that good. So I resorted to just marking the entity as throttled in update_curr() and later doing the dequeing from put_prev_entity() ? Isn't this acceptable ?

Regards,
Bharata.

Quoted 33 lines · Bharata B Rao

Quoted 23 lines · Peter Zijlstra

Quoted 22 lines · Bharata B Rao

Quoted 20 lines · Peter Zijlstra

include/linux/sched.h struct sched_domain; · line 1030

Quoted 13 lines · Bharata B Rao

include/linux/sched.h +1 -1

index 0f1ea4a..77ace43 100644
@@ -1024,7 +1024,7 @@ struct sched_domain;
 struct sched_class {
    const struct sched_class *next;
 
-   void (*enqueue_task) (struct rq *rq, struct task_struct *p, int wakeup);
+   int (*enqueue_task) (struct rq *rq, struct task_struct *p, int wakeup);
    void (*dequeue_task) (struct rq *rq, struct task_struct *p, int sleep);
    void (*yield_task) (struct rq *rq);
 

I really hate this, it uglfies all the enqueue code in a horrid way (which is most of this patch).

Why can't we simply enqueue the task on a throttled group just like rt?

Quoted 8 lines

Quoted 5 lines

We do enqueue a task to its group even if the group is throttled. However such throttled groups are not enqueued further. In such scenarios, even though the task enqueue to its parent group succeeded, it really didn't add any task to the cpu runqueue (rq). So we need to identify this condition and don't increment rq->running. That is why this return value is needed.

I would still consider those tasks running, the fact that they don't get to run is a different matter.

Quoted 7 lines

Ok, that's how rt also considers them I realize. I thought that we should update rq->running when tasks go off the runqueue due to throttling. When a task is throttled, it is no doubt present on its group's cfs_rq, but it doesn't contribute to the CPU load as the throttled group entity isn't there on any cfs_rq. rq->running is used to obtain a few load balancing metrics and they might go wrong if rq->running isn't uptodate.

for all practical purposes throttled tasks _are_ running (i.e. they would like to run, but the hardware/software doesn't allow them to do more work) ...

Quoted 3 lines

Do you still think we shouldn't update rq->running ? If so, I can get rid of this return value change.

Linux-VServer marked throttled tasks as 'H' (on hold) but counted them as running, which seems to work fine and reflect the expected behaviour ...

Hi,

Here is the v2 post of hard limits feature for CFS group scheduler. This RFC post mainly adds runtime borrowing feature and has a new locking scheme to protect CFS runtime related fields.

It would be nice to have some comments on this set!

Quoted 6 lines · Bharata B Rao

Hi,

Here is the v2 post of hard limits feature for CFS group scheduler. This RFC post mainly adds runtime borrowing feature and has a new locking scheme to protect CFS runtime related fields.

Quoted 1 line

It would be nice to have some comments on this set!

I have a question I'd like to ask before diving into the code. Consider I'm a user, that has a 4CPUs box 2GHz each and I'd like to create a container with 2CPUs 1GHz each. Can I achieve this after your patches?

Quoted 7 lines · Pavel Emelyanov

Quoted 6 lines · Bharata B Rao

Hi,

Here is the v2 post of hard limits feature for CFS group scheduler. This RFC post mainly adds runtime borrowing feature and has a new locking scheme to protect CFS runtime related fields.

Quoted 6 lines

Quoted 1 line

It would be nice to have some comments on this set!

I have a question I'd like to ask before diving into the code. Consider I'm a user, that has a 4CPUs box 2GHz each and I'd like to create a container with 2CPUs 1GHz each. Can I achieve this after your patches?

I am not sure if I understand the GHz specification you mention here. Are you saying that you want run a container with 2 CPUS with each of them running at half their (frequency)capacity ?

This hard limits scheme is about time based rate limiting where you can specify a runtime(=hard limit) and a period for the container and the container will not be allowed to consume more than the specified CPU time within a given period.

Quoted 13 lines · Bharata B Rao

Quoted 7 lines · Pavel Emelyanov

Quoted 6 lines · Bharata B Rao

Hi,

Here is the v2 post of hard limits feature for CFS group scheduler. This RFC post mainly adds runtime borrowing feature and has a new locking scheme to protect CFS runtime related fields.

Quoted 5 lines

Quoted 1 line

It would be nice to have some comments on this set!

I have a question I'd like to ask before diving into the code. Consider I'm a user, that has a 4CPUs box 2GHz each and I'd like to create a container with 2CPUs 1GHz each. Can I achieve this after your patches?

Quoted 8 lines

I am not sure if I understand the GHz specification you mention here. Are you saying that you want run a container with 2 CPUS with each of them running at half their (frequency)capacity ?

This hard limits scheme is about time based rate limiting where you can specify a runtime(=hard limit) and a period for the container and the container will not be allowed to consume more than the specified CPU time within a given period.

IMO Pavel's requirement can be met with a hard limit of 25%

	2 CPU of 1GHz = (1/2 x 4) (1/2 x 2) GHz CPUs
		      = 1/4 x 4 2GHz CPU
		      = 25% of (4 2GHz CPU)

IOW by hard-limiting a container thread to run just 0.5sec every sec on a 2GHz cpu, it is effectively making progress at the rate of 1GHz?

- vatsa

IMO Pavel's requirement can be met with a hard limit of 25%

	2 CPU of 1GHz = (1/2 x 4) (1/2 x 2) GHz CPUs
		      = 1/4 x 4 2GHz CPU
		      = 25% of (4 2GHz CPU)

IOW by hard-limiting a container thread to run just 0.5sec every sec on a 2GHz cpu, it is effectively making progress at the rate of 1GHz?

- vatsa

Quoted 6 lines

IMO Pavel's requirement can be met with a hard limit of 25%

	2 CPU of 1GHz = (1/2 x 4) (1/2 x 2) GHz CPUs
		      = 1/4 x 4 2GHz CPU
		      = 25% of (4 2GHz CPU)

Quoted 3 lines

IOW by hard-limiting a container thread to run just 0.5sec every sec on a 2GHz cpu, it is effectively making progress at the rate of 1GHz?

So, any suggestions on this? I'm not against the patches themselves. I'm just trying to tell, that setting a cpulimit with 2 numbers is not a good way to go (at least a clean explanation of how to calculate them should go with the patches).

I propose to first collect what *can* be done. I see the following possibilities:

1) two times (as it is now) - I believe this is inconvenient.
2) the amount in percents (like 50%) - this is how it works in OpenVZ and customers are quite happy with it. It's better than two numbers, since you need to specify only one clean number.

3) virtual cpu power in M/GHz - I don't agree with Balbir that this is difficult for administrator. This is better than two numbers and better that the percentage, since the amount of cpu time got by a container will not change after migrating to a more powerful CPU.

Thoughts?

Quoted 2 lines

- vatsa

as I already stated, it seems perfectly fine for me

can be trivially mapped to the two values, by chosing a fixed multiplicative base (let's say '1s' to simplify :)

with 50%, you get 1s/0.5s
with 20%, you get 1s/0.2s
with 5%, you get 1s/0.05s

well, you get the idea :)

Quoted 1 line

as I already stated, it seems perfectly fine for me

You're not the only one interested in it, sorry. Besides, I got your point in "I'm find with it". Now get mine which is about "I am not".

Quoted 7 lines

can be trivially mapped to the two values, by chosing a fixed multiplicative base (let's say '1s' to simplify :)

with 50%, you get 1s/0.5s
with 20%, you get 1s/0.2s
with 5%, you get 1s/0.05s

Quoted 1 line

well, you get the idea :)

No I don't.
Is 1s/0.5s worse or better than 2s/1s?
How should I make a choice?

Quoted 8 lines · Pavel Emelyanov

Quoted 1 line

as I already stated, it seems perfectly fine for me

You're not the only one interested in it, sorry. Besides, I got your point in "I'm find with it". Now get mine which is about "I am not".

Quoted 3 lines

can be trivially mapped to the two values, by chosing a fixed multiplicative base (let's say '1s' to simplify :)

Quoted 8 lines

Quoted 5 lines

with 50%, you get 1s/0.5s
with 20%, you get 1s/0.2s
with 5%, you get 1s/0.05s

well, you get the idea :)

No I don't.
Is 1s/0.5s worse or better than 2s/1s?
How should I make a choice?

I would say it depends on your requirement. How fast do you want to respond back to the user? Wiht lower bandwidth, you would want to have shorter periods so that the user would not get the impression that he has to "wait" to get CPU time. But having a very short period is not a good thing, since there are other considerations (such as the overhead of hard limits).

thanks,

Quoted 18 lines · Dhaval Giani

Quoted 8 lines · Pavel Emelyanov

Quoted 1 line

as I already stated, it seems perfectly fine for me

You're not the only one interested in it, sorry. Besides, I got your point in "I'm find with it". Now get mine which is about "I am not".

Quoted 3 lines

can be trivially mapped to the two values, by chosing a fixed multiplicative base (let's say '1s' to simplify :)

Quoted 8 lines

Quoted 5 lines

with 50%, you get 1s/0.5s
with 20%, you get 1s/0.2s
with 5%, you get 1s/0.05s

well, you get the idea :)

No I don't.
Is 1s/0.5s worse or better than 2s/1s?
How should I make a choice?

Quoted 6 lines

I would say it depends on your requirement. How fast do you want to respond back to the user? Wiht lower bandwidth, you would want to have shorter periods so that the user would not get the impression that he has to "wait" to get CPU time. But having a very short period is not a good thing, since there are other considerations (such as the overhead of hard limits).

That's it - long period is bad for one reason, short period is bad for some other one and neither of them is clearly described unlike the limit itself.

In other words there are two numbers we're essentially playing with:
* the limit (int percents, Hz, whatever)
* and this abstract "badness"

Can't we give the user one of them for "must be configured" usage, put the other one in some "good for most users" position and let the user move it later on demand?

Yet again - weights in CFQ CPU-sched, ionoce in CFQ-iosched, bandwidth in tc (traffic shaping), etc. are all clean for end-user. Plus there are other fine tunes, that user should not configure by default, but which change the default behavior. I propose to create simple and clean interface for limits as well. If you think that virtual cpu power is not good, ok. Let's ask user for a percentage and give him yet another option to control this "badness" or "responsiveness".

Quoted 18 lines · Dhaval Giani

Quoted 8 lines · Pavel Emelyanov

Quoted 1 line

as I already stated, it seems perfectly fine for me

You're not the only one interested in it, sorry. Besides, I got your point in "I'm find with it". Now get mine which is about "I am not".

Quoted 3 lines

can be trivially mapped to the two values, by chosing a fixed multiplicative base (let's say '1s' to simplify :)

Quoted 8 lines

Quoted 5 lines

with 50%, you get 1s/0.5s
with 20%, you get 1s/0.2s
with 5%, you get 1s/0.05s

well, you get the idea :)

No I don't.
Is 1s/0.5s worse or better than 2s/1s?
How should I make a choice?

Quoted 6 lines

I would say it depends on your requirement. How fast do you want to respond back to the user? Wiht lower bandwidth, you would want to have shorter periods so that the user would not get the impression that he has to "wait" to get CPU time. But having a very short period is not a good thing, since there are other considerations (such as the overhead of hard limits).

That's it - long period is bad for one reason, short period is bad for some other one and neither of them is clearly described unlike the limit itself.

In other words there are two numbers we're essentially playing with:
* the limit (int percents, Hz, whatever)
* and this abstract "badness"

Can't we give the user one of them for "must be configured" usage, put the other one in some "good for most users" position and let the user move it later on demand?

This is what we have right now. All the groups will start with a default period (of .5s) and user has the option of changing it based on his requirement. However we need to find out if the default period is really "good for most users"

Regards,
Bharata.

Quoted 5 lines

with 50%, you get 1s/0.5s
with 20%, you get 1s/0.2s
with 5%, you get 1s/0.05s

well, you get the idea :)

No I don't.
Is 1s/0.5s worse or better than 2s/1s?
How should I make a choice?

Quoted 1 line

as I already stated, it seems perfectly fine for me

You're not the only one interested in it, sorry. Besides, I got your point in "I'm find with it". Now get mine which is about "I am not".

Quoted 6 lines

can be trivially mapped to the two values, by chosing a fixed multiplicative base (let's say '1s' to simplify :)

with 50%, you get 1s/0.5s
with 20%, you get 1s/0.2s
with 5%, you get 1s/0.05s

Quoted 1 line

well, you get the idea :)

No I don't.
Is 1s/0.5s worse or better than 2s/1s?
How should I make a choice?

3) virtual cpu power in M/GHz - I don't agree with Balbir that this is difficult for administrator. This is better than two numbers and better that the percentage, since the amount of cpu time got by a container will not change after migrating to a more powerful CPU.

Quoted 6 lines · Bharata B Rao

Hi,

Here is the v2 post of hard limits feature for CFS group scheduler. This RFC post mainly adds runtime borrowing feature and has a new locking scheme to protect CFS runtime related fields.

Quoted 1 line

It would be nice to have some comments on this set!

I have a question I'd like to ask before diving into the code. Consider I'm a user, that has a 4CPUs box 2GHz each and I'd like to create a container with 2CPUs 1GHz each. Can I achieve this after your patches?

Quoted 7 lines · Pavel Emelyanov

Quoted 6 lines · Bharata B Rao

Hi,

Here is the v2 post of hard limits feature for CFS group scheduler. This RFC post mainly adds runtime borrowing feature and has a new locking scheme to protect CFS runtime related fields.

Quoted 5 lines

Quoted 1 line

It would be nice to have some comments on this set!

I have a question I'd like to ask before diving into the code. Consider I'm a user, that has a 4CPUs box 2GHz each and I'd like to create a container with 2CPUs 1GHz each. Can I achieve this after your patches?

I don't think the GHz makes any sense, consider CPUs with frequency scaling. If I can scale from 1.6GHz to say 2.6GHz or 2GHz to 4GHz, what does it mean for hard limit control? Hard limits define control over existing bandwidth, anything else would be superficial and hard hard to get right for both developers and users.

Quoted 15 lines · Balbir Singh

Quoted 7 lines · Pavel Emelyanov

Quoted 6 lines · Bharata B Rao

Hi,

Here is the v2 post of hard limits feature for CFS group scheduler. This RFC post mainly adds runtime borrowing feature and has a new locking scheme to protect CFS runtime related fields.

Quoted 5 lines

Quoted 1 line

It would be nice to have some comments on this set!

I have a question I'd like to ask before diving into the code. Consider I'm a user, that has a 4CPUs box 2GHz each and I'd like to create a container with 2CPUs 1GHz each. Can I achieve this after your patches?

Quoted 5 lines

I don't think the GHz makes any sense, consider CPUs with frequency scaling. If I can scale from 1.6GHz to say 2.6GHz or 2GHz to 4GHz, what does it mean for hard limit control? Hard limits define control over existing bandwidth, anything else would be superficial and hard hard to get right for both developers and users.

Two numbers for configuring limits make even less sense OTOH ;) By assigning 2GHz for 4GHz CPU I obviously want half of its power ;) Please, see my reply to vatsa@ in this thread.

Quoted 21 lines · Pavel Emelyanov

Quoted 14 lines · Balbir Singh

Quoted 7 lines · Pavel Emelyanov

Quoted 6 lines · Bharata B Rao

Hi,

Here is the v2 post of hard limits feature for CFS group scheduler. This RFC post mainly adds runtime borrowing feature and has a new locking scheme to protect CFS runtime related fields.

Quoted 5 lines

Quoted 1 line

It would be nice to have some comments on this set!

I have a question I'd like to ask before diving into the code. Consider I'm a user, that has a 4CPUs box 2GHz each and I'd like to create a container with 2CPUs 1GHz each. Can I achieve this after your patches?

Quoted 5 lines

I don't think the GHz makes any sense, consider CPUs with frequency scaling. If I can scale from 1.6GHz to say 2.6GHz or 2GHz to 4GHz, what does it mean for hard limit control? Hard limits define control over existing bandwidth, anything else would be superficial and hard hard to get right for both developers and users.

Quoted 3 lines

Two numbers for configuring limits make even less sense OTOH ;) By assigning 2GHz for 4GHz CPU I obviously want half of its power ;) Please, see my reply to vatsa@ in this thread.

But it makes life more difficult for the administrator to think in terms of GHz -- no? Specifically with different heterogeneous systems. I think it would be chaotic in a data center to configure GHz for every partition. Not to say that it makes it even more confusing when running on top of KVM. Lets say I create two vCPUs and I specifiy GHz outside, do I expect to see it in /proc/cpuinfo?

I'd like to hear what others think about GHz as well.

Signature · 1 line

	Balbir

Linux-VServer uses interval and rate, which is very similar to your setup, except that we use two pairs to achive a differentiation for 'busy' and 'idle' cases, i.e. when the cpu(s) would go idle, we switch from R1/I1 to R2/I2 for all guests, allowing to distribute the excess differently than the 'active' part ... but only one set is needed for hard limits ...