Skip to contentSkip to conversation

Core scheduling (v9)

PATCH In review Joel Fernandes (Google) 12 participants 150 messages Nov 17, 2020 – Jan 15, 2021 Page 2 of 2

Subsystems (19)
lore
Share & tools
lore
#121 lore raw

Re: [PATCH -tip 23/32] sched: Add a per-thread core scheduling interface

In reply to Joel Fernandes
Quoted 1,253 lines · Joel Fernandes
Quoted 15 lines · chris hyser
Quoted 14 lines · Joel Fernandes

kernel/sched/debug.c void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, · line 1033

Quoted 9 lines
kernel/sched/debug.c +1 -0
index cffdfab..50c31f3 100644
@@ -1030,6 +1030,7 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns,
   #ifdef CONFIG_SCHED_CORE
   	__PS("core_cookie", p->core_cookie);
+	__PS("core_task_cookie", p->core_task_cookie);
   #endif

Hmm, so the final cookie of the task is always p->core_cookie. This is what the scheduler uses. All other fields are ingredients to derive the final cookie value.

Quoted 5 lines
Quoted 2 lines

I will drop this hunk from your overall diff, but let me know if you disagree!

No problem. That was there primarily for debugging.

Ok. I squashed Josh's changes into this patch and several of my fixups. So there'll be 3 patches:
1. CGroup + prctl (single patch as it is hell to split it)
2. Documentation
3. ksefltests

Below is the diff of #1. I still have to squash in the stop_machine removal and some more review changes. But other than that, please take a look and let me know anything that's odd. I will test further as well.

Also next series will only be interface as I want to see if I can get lucky enough to have Peter look at it before he leaves for PTO next week. For the other features, I will post different series as I prepare them. One series for interface, and another for kernel protection / migration changes.

---8<-----------------------

include/linux/sched.h +6 -0
index a60868165590..73baca11d743 100644
@@ -688,6 +688,8 @@ struct task_struct {
  #ifdef CONFIG_SCHED_CORE
  	struct rb_node			core_node;
  	unsigned long			core_cookie;
+	unsigned long			core_task_cookie;
+	unsigned long			core_group_cookie;
  	unsigned int			core_occupation;
  #endif
 
@@ -2081,11 +2083,15 @@ void sched_core_unsafe_enter(void);
  void sched_core_unsafe_exit(void);
  bool sched_core_wait_till_safe(unsigned long ti_check);
  bool sched_core_kernel_protected(void);
+int sched_core_share_pid(unsigned long flags, pid_t pid);
+void sched_tsk_free(struct task_struct *tsk);
  #else
  #define sched_core_unsafe_enter(ignore) do { } while (0)
  #define sched_core_unsafe_exit(ignore) do { } while (0)
  #define sched_core_wait_till_safe(ignore) do { } while (0)
  #define sched_core_kernel_protected(ignore) do { } while (0)
+#define sched_core_share_pid(flags, pid) do { } while (0)
+#define sched_tsk_free(tsk) do { } while (0)
  #endif
 
  #endif
include/uapi/linux/prctl.h +6 -0
index c334e6a02e5f..3752006842e1 100644
@@ -248,4 +248,10 @@ struct prctl_mm_map {
  #define PR_SET_IO_FLUSHER		57
  #define PR_GET_IO_FLUSHER		58
 
+/* Request the scheduler to share a core */
+#define PR_SCHED_CORE_SHARE		59
+#define PR_SCHED_CORE_CLEAR		0  /* clear core_sched cookie of pid */
+#define PR_SCHED_CORE_SHARE_FROM	1  /* get core_sched cookie from pid */
+#define PR_SCHED_CORE_SHARE_TO		2  /* push core_sched cookie to pid */
+
  #endif /* _LINUX_PRCTL_H */
kernel/fork.c +1 -0
index 7199d359690c..5468c93829c5 100644
@@ -736,6 +736,7 @@ void __put_task_struct(struct task_struct *tsk)
  	exit_creds(tsk);
  	delayacct_tsk_free(tsk);
  	put_signal_struct(tsk->signal);
+	sched_tsk_free(tsk);
 
  	if (!profile_handoff_task(tsk))
  		free_task(tsk);
kernel/sched/Makefile +1 -0
index 5fc9c9b70862..c526c20adf9d 100644
@@ -36,3 +36,4 @@ obj-$(CONFIG_CPU_FREQ_GOV_SCHEDUTIL) += cpufreq_schedutil.o
  obj-$(CONFIG_MEMBARRIER) += membarrier.o
  obj-$(CONFIG_CPU_ISOLATION) += isolation.o
  obj-$(CONFIG_PSI) += psi.o
+obj-$(CONFIG_SCHED_CORE) += coretag.o
kernel/sched/core.c +120 -19
index 7f807a84cc30..80daca9c5930 100644
@@ -157,7 +157,33 @@ static inline bool __sched_core_less(struct task_struct *a, struct task_struct *
  	return false;
  }
 
-static void sched_core_enqueue(struct rq *rq, struct task_struct *p)
+static bool sched_core_empty(struct rq *rq)
+{
+	return RB_EMPTY_ROOT(&rq->core_tree);
+}
+
+static struct task_struct *sched_core_first(struct rq *rq)
+{
+	struct task_struct *task;
+
+	task = container_of(rb_first(&rq->core_tree), struct task_struct, core_node);
+	return task;
+}
+
+static void sched_core_flush(int cpu)
+{
+	struct rq *rq = cpu_rq(cpu);
+	struct task_struct *task;
+
+	while (!sched_core_empty(rq)) {
+		task = sched_core_first(rq);
+		rb_erase(&task->core_node, &rq->core_tree);
+		RB_CLEAR_NODE(&task->core_node);
+	}
+	rq->core->core_task_seq++;
+}
+
+void sched_core_enqueue(struct rq *rq, struct task_struct *p)
  {
  	struct rb_node *parent, **node;
  	struct task_struct *node_task;
@@ -184,14 +210,15 @@ static void sched_core_enqueue(struct rq *rq, struct task_struct *p)
  	rb_insert_color(&p->core_node, &rq->core_tree);
  }
 
-static void sched_core_dequeue(struct rq *rq, struct task_struct *p)
+void sched_core_dequeue(struct rq *rq, struct task_struct *p)
  {
  	rq->core->core_task_seq++;
 
-	if (!p->core_cookie)
+	if (!sched_core_enqueued(p))
  		return;
 
  	rb_erase(&p->core_node, &rq->core_tree);
+	RB_CLEAR_NODE(&p->core_node);
  }
 
  /*
@@ -255,8 +282,24 @@ static int __sched_core_stopper(void *data)
  	bool enabled = !!(unsigned long)data;
  	int cpu;
 
-	for_each_possible_cpu(cpu)
-		cpu_rq(cpu)->core_enabled = enabled;
+	for_each_possible_cpu(cpu) {
+		struct rq *rq = cpu_rq(cpu);
+
+		WARN_ON_ONCE(enabled == rq->core_enabled);
+
+		if (!enabled || (enabled && cpumask_weight(cpu_smt_mask(cpu)) >= 2)) {
+			/*
+			 * All active and migrating tasks will have already
+			 * been removed from core queue when we clear the
+			 * cgroup tags. However, dying tasks could still be
+			 * left in core queue. Flush them here.
+			 */
+			if (!enabled)
+				sched_core_flush(cpu);
+
+			rq->core_enabled = enabled;
+		}
+	}
 
  	return 0;
  }
@@ -266,7 +309,11 @@ static int sched_core_count;
 
  static void __sched_core_enable(void)
  {
-	// XXX verify there are no cookie tasks (yet)
+	int cpu;
+
+	/* verify there are no cookie tasks (yet) */
+	for_each_online_cpu(cpu)
+		BUG_ON(!sched_core_empty(cpu_rq(cpu)));
 
  	static_branch_enable(&__sched_core_enabled);
  	stop_machine(__sched_core_stopper, (void *)true, NULL);
@@ -274,8 +321,6 @@ static void __sched_core_enable(void)
 
  static void __sched_core_disable(void)
  {
-	// XXX verify there are no cookie tasks (left)
-
  	stop_machine(__sched_core_stopper, (void *)false, NULL);
  	static_branch_disable(&__sched_core_enabled);
  }
@@ -295,12 +340,6 @@ void sched_core_put(void)
  		__sched_core_disable();
  	mutex_unlock(&sched_core_mutex);
  }
-
-#else /* !CONFIG_SCHED_CORE */
-
-static inline void sched_core_enqueue(struct rq *rq, struct task_struct *p) { }
-static inline void sched_core_dequeue(struct rq *rq, struct task_struct *p) { }
-
  #endif /* CONFIG_SCHED_CORE */
 
  /*
@@ -3779,6 +3818,9 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
  	p->capture_control = NULL;
  #endif
  	init_numa_balancing(clone_flags, p);
+#ifdef CONFIG_SCHED_CORE
+	p->core_task_cookie = 0;
+#endif
  #ifdef CONFIG_SMP
  	p->wake_entry.u_flags = CSD_TYPE_TTWU;
  	p->migration_pending = NULL;
@@ -3903,6 +3945,7 @@ static inline void init_schedstats(void) {}
  int sched_fork(unsigned long clone_flags, struct task_struct *p)
  {
  	unsigned long flags;
+	int __maybe_unused ret;
 
  	__sched_fork(clone_flags, p);
  	/*
@@ -3978,6 +4021,13 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p)
  #ifdef CONFIG_SMP
  	plist_node_init(&p->pushable_tasks, MAX_PRIO);
  	RB_CLEAR_NODE(&p->pushable_dl_tasks);
+#endif
+#ifdef CONFIG_SCHED_CORE
+	RB_CLEAR_NODE(&p->core_node);
+
+	ret = sched_core_fork(p, clone_flags);
+	if (ret)
+		return ret;
  #endif
  	return 0;
  }
@@ -7979,6 +8029,9 @@ void init_idle(struct task_struct *idle, int cpu)
  #ifdef CONFIG_SMP
  	sprintf(idle->comm, "%s/%d", INIT_TASK_COMM, cpu);
  #endif
+#ifdef CONFIG_SCHED_CORE
+	RB_CLEAR_NODE(&idle->core_node);
+#endif
  }
 
  #ifdef CONFIG_SMP
@@ -8983,6 +9036,9 @@ void sched_offline_group(struct task_group *tg)
  	spin_unlock_irqrestore(&task_group_lock, flags);
  }
 
+void cpu_core_get_group_cookie(struct task_group *tg,
+			       unsigned long *group_cookie_ptr);
+
  static void sched_change_group(struct task_struct *tsk, int type)
  {
  	struct task_group *tg;
@@ -8995,6 +9051,11 @@ static void sched_change_group(struct task_struct *tsk, int type)
  	tg = container_of(task_css_check(tsk, cpu_cgrp_id, true),
  			  struct task_group, css);
  	tg = autogroup_task_group(tsk, tg);
+
+#ifdef CONFIG_SCHED_CORE
+	sched_core_change_group(tsk, tg);
+#endif
+
  	tsk->sched_task_group = tg;
 
  #ifdef CONFIG_FAIR_GROUP_SCHED
@@ -9047,11 +9108,6 @@ void sched_move_task(struct task_struct *tsk)
  	task_rq_unlock(rq, tsk, &rf);
  }
 
-static inline struct task_group *css_tg(struct cgroup_subsys_state *css)
-{
-	return css ? container_of(css, struct task_group, css) : NULL;
-}
-
  static struct cgroup_subsys_state *
  cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
  {
@@ -9087,6 +9143,18 @@ static int cpu_cgroup_css_online(struct cgroup_subsys_state *css)
  	return 0;
  }
 
+static void cpu_cgroup_css_offline(struct cgroup_subsys_state *css)
+{
+#ifdef CONFIG_SCHED_CORE
+	struct task_group *tg = css_tg(css);
+
+	if (tg->core_tagged) {
+		sched_core_put();
+		tg->core_tagged = 0;
+	}
+#endif
+}
+
  static void cpu_cgroup_css_released(struct cgroup_subsys_state *css)
  {
  	struct task_group *tg = css_tg(css);
@@ -9688,6 +9756,22 @@ static struct cftype cpu_legacy_files[] = {
  		.write_u64 = cpu_rt_period_write_uint,
  	},
  #endif
+#ifdef CONFIG_SCHED_CORE
+	{
+		.name = "core_tag",
+		.flags = CFTYPE_NOT_ON_ROOT,
+		.read_u64 = cpu_core_tag_read_u64,
+		.write_u64 = cpu_core_tag_write_u64,
+	},
+#ifdef CONFIG_SCHED_DEBUG
+	/* Read the group cookie. */
+	{
+		.name = "core_group_cookie",
+		.flags = CFTYPE_NOT_ON_ROOT,
+		.read_u64 = cpu_core_group_cookie_read_u64,
+	},
+#endif
+#endif
  #ifdef CONFIG_UCLAMP_TASK_GROUP
  	{
  		.name = "uclamp.min",
@@ -9861,6 +9945,22 @@ static struct cftype cpu_files[] = {
  		.write_s64 = cpu_weight_nice_write_s64,
  	},
  #endif
+#ifdef CONFIG_SCHED_CORE
+	{
+		.name = "core_tag",
+		.flags = CFTYPE_NOT_ON_ROOT,
+		.read_u64 = cpu_core_tag_read_u64,
+		.write_u64 = cpu_core_tag_write_u64,
+	},
+#ifdef CONFIG_SCHED_DEBUG
+	/* Read the group cookie. */
+	{
+		.name = "core_group_cookie",
+		.flags = CFTYPE_NOT_ON_ROOT,
+		.read_u64 = cpu_core_group_cookie_read_u64,
+	},
+#endif
+#endif
  #ifdef CONFIG_CFS_BANDWIDTH
  	{
  		.name = "max",
@@ -9889,6 +9989,7 @@ static struct cftype cpu_files[] = {
  struct cgroup_subsys cpu_cgrp_subsys = {
  	.css_alloc	= cpu_cgroup_css_alloc,
  	.css_online	= cpu_cgroup_css_online,
+	.css_offline	= cpu_cgroup_css_offline,
  	.css_released	= cpu_cgroup_css_released,
  	.css_free	= cpu_cgroup_css_free,
  	.css_extra_stat_show = cpu_extra_stat_show,
kernel/sched/coretag.c new file +734 -0
new file mode 100644
index 000000000000..4eeb956382ee
@@ -0,0 +1,734 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * kernel/sched/core-tag.c
+ *
+ * Core-scheduling tagging interface support.
+ *
+ * Copyright(C) 2020, Joel Fernandes.
+ * Initial interfacing code  by Peter Ziljstra.
+ */
+
+#include <linux/prctl.h>
+#include "sched.h"
+
+/*
+ * Wrapper representing a complete cookie. The address of the cookie is used as
+ * a unique identifier. Each cookie has a unique permutation of the internal
+ * cookie fields.
+ */
+struct sched_core_cookie {
+	unsigned long task_cookie;
+	unsigned long group_cookie;
+
+	struct rb_node node;
+	refcount_t refcnt;
+};
+
+/*
+ * A simple wrapper around refcount. An allocated sched_core_task_cookie's
+ * address is used to compute the cookie of the task.
+ */
+struct sched_core_task_cookie {
+	refcount_t refcnt;
+	struct work_struct work; /* to free in WQ context. */;
+};
+
+static DEFINE_MUTEX(sched_core_tasks_mutex);
+
+/* All active sched_core_cookies */
+static struct rb_root sched_core_cookies = RB_ROOT;
+static DEFINE_RAW_SPINLOCK(sched_core_cookies_lock);
+
+/*
+ * Returns the following:
+ * a < b  => -1
+ * a == b => 0
+ * a > b  => 1
+ */
+static int sched_core_cookie_cmp(const struct sched_core_cookie *a,
+				 const struct sched_core_cookie *b)
+{
+#define COOKIE_CMP_RETURN(field) do {		\
+	if (a->field < b->field)		\
+		return -1;			\
+	else if (a->field > b->field)		\
+		return 1;			\
+} while (0)					\
+
+	COOKIE_CMP_RETURN(task_cookie);
+	COOKIE_CMP_RETURN(group_cookie);
+
+	/* all cookie fields match */
+	return 0;
+
+#undef COOKIE_CMP_RETURN
+}
+
+static inline void __sched_core_erase_cookie(struct sched_core_cookie *cookie)
+{
+	lockdep_assert_held(&sched_core_cookies_lock);
+
+	/* Already removed */
+	if (RB_EMPTY_NODE(&cookie->node))
+		return;
+
+	rb_erase(&cookie->node, &sched_core_cookies);
+	RB_CLEAR_NODE(&cookie->node);
+}
+
+/* Called when a task no longer points to the cookie in question */
+static void sched_core_put_cookie(struct sched_core_cookie *cookie)
+{
+	unsigned long flags;
+
+	if (!cookie)
+		return;
+
+	if (refcount_dec_and_test(&cookie->refcnt)) {
+		raw_spin_lock_irqsave(&sched_core_cookies_lock, flags);
+		__sched_core_erase_cookie(cookie);
+		raw_spin_unlock_irqrestore(&sched_core_cookies_lock, flags);
+		kfree(cookie);
+	}
+}
+
+/*
+ * A task's core cookie is a compound structure composed of various cookie
+ * fields (task_cookie, group_cookie). The overall core_cookie is
+ * a pointer to a struct containing those values. This function either finds
+ * an existing core_cookie or creates a new one, and then updates the task's
+ * core_cookie to point to it. Additionally, it handles the necessary reference
+ * counting.
+ *
+ * REQUIRES: task_rq(p) lock or called from cpu_stopper.
+ * Doing so ensures that we do not cause races/corruption by modifying/reading
+ * task cookie fields.
+ */
+static void __sched_core_update_cookie(struct task_struct *p)
+{
+	struct rb_node *parent, **node;
+	struct sched_core_cookie *node_core_cookie, *match;
+	static const struct sched_core_cookie zero_cookie;
+	struct sched_core_cookie temp = {
+		.task_cookie	= p->core_task_cookie,
+		.group_cookie	= p->core_group_cookie,
+	};
+	const bool is_zero_cookie =
+		(sched_core_cookie_cmp(&temp, &zero_cookie) == 0);
+	struct sched_core_cookie *const curr_cookie =
+		(struct sched_core_cookie *)p->core_cookie;
+	unsigned long flags;
+
+	/*
+	 * Already have a cookie matching the requested settings? Nothing to
+	 * do.
+	 */
+	if ((curr_cookie && sched_core_cookie_cmp(curr_cookie, &temp) == 0) ||
+	    (!curr_cookie && is_zero_cookie))
+		return;
+
+	raw_spin_lock_irqsave(&sched_core_cookies_lock, flags);
+
+	if (is_zero_cookie) {
+		match = NULL;
+		goto finish;
+	}
+
+retry:
+	match = NULL;
+
+	node = &sched_core_cookies.rb_node;
+	parent = *node;
+	while (*node) {
+		int cmp;
+
+		node_core_cookie =
+			container_of(*node, struct sched_core_cookie, node);
+		parent = *node;
+
+		cmp = sched_core_cookie_cmp(&temp, node_core_cookie);
+		if (cmp < 0) {
+			node = &parent->rb_left;
+		} else if (cmp > 0) {
+			node = &parent->rb_right;
+		} else {
+			match = node_core_cookie;
+			break;
+		}
+	}
+
+	if (!match) {
+		/* No existing cookie; create and insert one */
+		match = kmalloc(sizeof(struct sched_core_cookie), GFP_ATOMIC);
+
+		/* Fall back to zero cookie */
+		if (WARN_ON_ONCE(!match))
+			goto finish;
+
+		match->task_cookie = temp.task_cookie;
+		match->group_cookie = temp.group_cookie;
+		refcount_set(&match->refcnt, 1);
+
+		rb_link_node(&match->node, parent, node);
+		rb_insert_color(&match->node, &sched_core_cookies);
+	} else {
+		/*
+		 * Cookie exists, increment refcnt. If refcnt is currently 0,
+		 * we're racing with a put() (refcnt decremented but cookie not
+		 * yet removed from the tree). In this case, we can simply
+		 * perform the removal ourselves and retry.
+		 * sched_core_put_cookie() will still function correctly.
+		 */
+		if (unlikely(!refcount_inc_not_zero(&match->refcnt))) {
+			__sched_core_erase_cookie(match);
+			goto retry;
+		}
+	}
+
+finish:
+	/*
+	 * Set the core_cookie under the cookies lock. This guarantees that
+	 * p->core_cookie cannot be freed while the cookies lock is held in
+	 * sched_core_fork().
+	 */
+	p->core_cookie = (unsigned long)match;
+
+	raw_spin_unlock_irqrestore(&sched_core_cookies_lock, flags);
+
+	sched_core_put_cookie(curr_cookie);
+}
+
+/*
+ * sched_core_update_cookie - Common helper to update a task's core cookie. This
+ * updates the selected cookie field and then updates the overall cookie.
+ * @p: The task whose cookie should be updated.
+ * @cookie: The new cookie.
+ * @cookie_type: The cookie field to which the cookie corresponds.
+ *
+ * REQUIRES: either task_rq(p)->lock held or called from a stop-machine handler.
+ * Doing so ensures that we do not cause races/corruption by modifying/reading
+ * task cookie fields.
+ */
+static void sched_core_update_cookie(struct task_struct *p, unsigned long cookie,
+				     enum sched_core_cookie_type cookie_type)
+{
+	if (!p)
+		return;
+
+	switch (cookie_type) {
+	case sched_core_no_update:
+		break;
+	case sched_core_task_cookie_type:
+		p->core_task_cookie = cookie;
+		break;
+	case sched_core_group_cookie_type:
+		p->core_group_cookie = cookie;
+		break;
+	default:
+		WARN_ON_ONCE(1);
+	}
+
+	/* Set p->core_cookie, which is the overall cookie */
+	__sched_core_update_cookie(p);
+
+	if (sched_core_enqueued(p)) {
+		sched_core_dequeue(task_rq(p), p);
+		if (!p->core_cookie)
+			return;
+	}
+
+	if (sched_core_enabled(task_rq(p)) &&
+	    p->core_cookie && task_on_rq_queued(p))
+		sched_core_enqueue(task_rq(p), p);
+}
+
+#ifdef CONFIG_CGROUP_SCHED
+void cpu_core_get_group_cookie(struct task_group *tg,
+			       unsigned long *group_cookie_ptr);
+
+void sched_core_change_group(struct task_struct *p, struct task_group *new_tg)
+{
+	unsigned long new_group_cookie;
+
+	cpu_core_get_group_cookie(new_tg, &new_group_cookie);
+
+	if (p->core_group_cookie == new_group_cookie)
+		return;
+
+	p->core_group_cookie = new_group_cookie;
+
+	__sched_core_update_cookie(p);
+}
+#endif
+
+/* Per-task interface: Used by fork(2) and prctl(2). */
+static void sched_core_put_cookie_work(struct work_struct *ws);
+
+/* Caller has to call sched_core_get() if non-zero value is returned. */
+static unsigned long sched_core_alloc_task_cookie(void)
+{
+	struct sched_core_task_cookie *ck =
+		kmalloc(sizeof(struct sched_core_task_cookie), GFP_KERNEL);
+
+	if (!ck)
+		return 0;
+	refcount_set(&ck->refcnt, 1);
+	INIT_WORK(&ck->work, sched_core_put_cookie_work);
+
+	return (unsigned long)ck;
+}
+
+static void sched_core_get_task_cookie(unsigned long cookie)
+{
+	struct sched_core_task_cookie *ptr =
+		(struct sched_core_task_cookie *)cookie;
+
+	refcount_inc(&ptr->refcnt);
+}
+
+static void sched_core_put_task_cookie(unsigned long cookie)
+{
+	struct sched_core_task_cookie *ptr =
+		(struct sched_core_task_cookie *)cookie;
+
+	if (refcount_dec_and_test(&ptr->refcnt))
+		kfree(ptr);
+}
+
+static void sched_core_put_cookie_work(struct work_struct *ws)
+{
+	struct sched_core_task_cookie *ck =
+		container_of(ws, struct sched_core_task_cookie, work);
+
+	sched_core_put_task_cookie((unsigned long)ck);
+	sched_core_put();
+}
+
+struct sched_core_task_write_tag {
+	struct task_struct *tasks[2];
+	unsigned long cookies[2];
+};
+
+/*
+ * Ensure that the task has been requeued. The stopper ensures that the task cannot
+ * be migrated to a different CPU while its core scheduler queue state is being updated.
+ * It also makes sure to requeue a task if it was running actively on another CPU.
+ */
+static int sched_core_task_join_stopper(void *data)
+{
+	struct sched_core_task_write_tag *tag = (struct sched_core_task_write_tag *)data;
+	int i;
+
+	for (i = 0; i < 2; i++)
+		sched_core_update_cookie(tag->tasks[i], tag->cookies[i],
+					 sched_core_task_cookie_type);
+
+	return 0;
+}
+
+int sched_core_share_tasks(struct task_struct *t1, struct task_struct *t2)
+{
+	struct sched_core_task_write_tag wr = {}; /* for stop machine. */
+	bool sched_core_put_after_stopper = false;
+	unsigned long cookie;
+	int ret = -ENOMEM;
+
+	mutex_lock(&sched_core_tasks_mutex);
+
+	if (!t2) {
+		if (t1->core_task_cookie) {
+			sched_core_put_task_cookie(t1->core_task_cookie);
+			sched_core_put_after_stopper = true;
+			wr.tasks[0] = t1; /* Keep wr.cookies[0] reset for t1. */
+		}
+	} else if (t1 == t2) {
+		/* Assign a unique per-task cookie solely for t1. */
+
+		cookie = sched_core_alloc_task_cookie();
+		if (!cookie)
+			goto out_unlock;
+		sched_core_get();
+
+		if (t1->core_task_cookie) {
+			sched_core_put_task_cookie(t1->core_task_cookie);
+			sched_core_put_after_stopper = true;
+		}
+		wr.tasks[0] = t1;
+		wr.cookies[0] = cookie;
+	} else if (!t1->core_task_cookie && !t2->core_task_cookie) {
+		/*
+		 * 		t1		joining		t2
+		 * CASE 1:
+		 * before	0				0
+		 * after	new cookie			new cookie
+		 *
+		 * CASE 2:
+		 * before	X (non-zero)			0
+		 * after	0				0
+		 *
+		 * CASE 3:
+		 * before	0				X (non-zero)
+		 * after	X				X
+		 *
+		 * CASE 4:
+		 * before	Y (non-zero)			X (non-zero)
+		 * after	X				X
+		 */
+
+		/* CASE 1. */
+		cookie = sched_core_alloc_task_cookie();
+		if (!cookie)
+			goto out_unlock;
+		sched_core_get(); /* For the alloc. */
+
+		/* Add another reference for the other task. */
+		sched_core_get_task_cookie(cookie);
+		sched_core_get(); /* For the other task. */
+
+		wr.tasks[0] = t1;
+		wr.tasks[1] = t2;
+		wr.cookies[0] = wr.cookies[1] = cookie;
+
+	} else if (t1->core_task_cookie && !t2->core_task_cookie) {
+		/* CASE 2. */
+		sched_core_put_task_cookie(t1->core_task_cookie);
+		sched_core_put_after_stopper = true;
+
+		wr.tasks[0] = t1; /* Reset cookie for t1. */
+
+	} else if (!t1->core_task_cookie && t2->core_task_cookie) {
+		/* CASE 3. */
+		sched_core_get_task_cookie(t2->core_task_cookie);
+		sched_core_get();
+
+		wr.tasks[0] = t1;
+		wr.cookies[0] = t2->core_task_cookie;
+
+	} else {
+		/* CASE 4. */
+		sched_core_get_task_cookie(t2->core_task_cookie);
+		sched_core_get();
+
+		sched_core_put_task_cookie(t1->core_task_cookie);
+		sched_core_put_after_stopper = true;
+
+		wr.tasks[0] = t1;
+		wr.cookies[0] = t2->core_task_cookie;
+	}
+
+	stop_machine(sched_core_task_join_stopper, (void *)&wr, NULL);
+
+	if (sched_core_put_after_stopper)
+		sched_core_put();
+
+	ret = 0;
+out_unlock:
+	mutex_unlock(&sched_core_tasks_mutex);
+	return ret;
+}
+
+/* Called from prctl interface: PR_SCHED_CORE_SHARE */
+int sched_core_share_pid(unsigned long flags, pid_t pid)
+{
+	struct task_struct *to;
+	struct task_struct *from;
+	struct task_struct *task;
+	int err;
+
+	rcu_read_lock();
+	task = find_task_by_vpid(pid);
+	if (!task) {
+		rcu_read_unlock();
+		return -ESRCH;
+	}
+
+	get_task_struct(task);
+
+	/*
+	 * Check if this process has the right to modify the specified
+	 * process. Use the regular "ptrace_may_access()" checks.
+	 */
+	if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) {
+		rcu_read_unlock();
+		err = -EPERM;
+		goto out;
+	}
+	rcu_read_unlock();
+
+	if (flags == PR_SCHED_CORE_CLEAR) {
+		to = task;
+		from = NULL;
+	} else if (flags == PR_SCHED_CORE_SHARE_TO) {
+		to = task;
+		from = current;
+	} else if (flags == PR_SCHED_CORE_SHARE_FROM) {
+		to = current;
+		from = task;
+	} else {
+		err = -EINVAL;
+		goto out;
+	}
+
+	err = sched_core_share_tasks(to, from);
+out:
+	if (task)
+		put_task_struct(task);
+	return err;
+}
+
+/* CGroup core-scheduling interface support. */
+#ifdef CONFIG_CGROUP_SCHED
+/*
+ * Helper to get the group cookie in a hierarchy. Any ancestor can have a
+ * cookie.
+ *
+ * Sets *group_cookie_ptr to the hierarchical group cookie.
+ */
+void cpu_core_get_group_cookie(struct task_group *tg,
+			       unsigned long *group_cookie_ptr)
+{
+	unsigned long group_cookie = 0UL;
+
+	if (!tg)
+		goto out;
+
+	for (; tg; tg = tg->parent) {
+
+		if (tg->core_tagged) {
+			group_cookie = (unsigned long)tg;
+			break;
+		}
+	}
+
+out:
+	*group_cookie_ptr = group_cookie;
+}
+
+/* Determine if any group in @tg's children are tagged. */
+static bool cpu_core_check_descendants(struct task_group *tg, bool check_tag)
+{
+	struct task_group *child;
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(child, &tg->children, siblings) {
+		if ((child->core_tagged && check_tag)) {
+			rcu_read_unlock();
+			return true;
+		}
+
+		rcu_read_unlock();
+		return cpu_core_check_descendants(child, check_tag);
+	}
+
+	rcu_read_unlock();
+	return false;
+}
+
+u64 cpu_core_tag_read_u64(struct cgroup_subsys_state *css,
+			  struct cftype *cft)
+{
+	struct task_group *tg = css_tg(css);
+
+	return !!tg->core_tagged;
+}
+
+#ifdef CONFIG_SCHED_DEBUG
+u64 cpu_core_group_cookie_read_u64(struct cgroup_subsys_state *css,
+				   struct cftype *cft)
+{
+	unsigned long group_cookie;
+
+	cpu_core_get_group_cookie(css_tg(css), &group_cookie);
+
+	return group_cookie;
+}
+#endif
+
+struct write_core_tag {
+	struct cgroup_subsys_state *css;
+	unsigned long cookie;
+	enum sched_core_cookie_type cookie_type;
+};
+
+static int __sched_write_tag(void *data)
+{
+	struct write_core_tag *tag = (struct write_core_tag *)data;
+	struct task_struct *p;
+	struct cgroup_subsys_state *css;
+
+	rcu_read_lock();
+	css_for_each_descendant_pre(css, tag->css) {
+		struct css_task_iter it;
+
+		css_task_iter_start(css, 0, &it);
+		/*
+		 * Note: css_task_iter_next will skip dying tasks.
+		 * There could still be dying tasks left in the core queue
+		 * when we set cgroup tag to 0 when the loop is done below.
+		 */
+		while ((p = css_task_iter_next(&it)))
+			sched_core_update_cookie(p, tag->cookie,
+						 tag->cookie_type);
+
+		css_task_iter_end(&it);
+	}
+	rcu_read_unlock();
+
+	return 0;
+}
+
+int cpu_core_tag_write_u64(struct cgroup_subsys_state *css, struct cftype *cft,
+			   u64 val)
+{
+	struct task_group *tg = css_tg(css);
+	struct write_core_tag wtag;
+	unsigned long group_cookie;
+
+	if (val > 1)
+		return -ERANGE;
+
+	if (!static_branch_likely(&sched_smt_present))
+		return -EINVAL;
+
+	if (!tg->core_tagged && val) {
+		/* Tag is being set. Check ancestors and descendants. */
+		cpu_core_get_group_cookie(tg, &group_cookie);
+		if (group_cookie ||
+		    cpu_core_check_descendants(tg, true /* tag */))
+			return -EBUSY;
+	} else if (tg->core_tagged && !val) {
+		/* Tag is being reset. Check descendants. */
+		if (cpu_core_check_descendants(tg, true /* tag */))
+			return -EBUSY;
+	} else {
+		return 0;
+	}
+
+	if (!!val)
+		sched_core_get();
+
+	wtag.css = css;
+	wtag.cookie = (unsigned long)tg;
+	wtag.cookie_type = sched_core_group_cookie_type;
+
+	tg->core_tagged = val;
+
+	stop_machine(__sched_write_tag, (void *)&wtag, NULL);
+	if (!val)
+		sched_core_put();
+
+	return 0;
+}
+#endif
+
+/*
+ * Tagging support when fork(2) is called:
+ * If it is a CLONE_THREAD fork, share parent's tag. Otherwise assign a unique per-task tag.
+ */
+static int sched_update_core_tag_stopper(void *data)
+{
+	struct task_struct *p = (struct task_struct *)data;
+
+	/* Recalculate core cookie */
+	sched_core_update_cookie(p, 0, sched_core_no_update);
+
+	return 0;
+}
+
+/* Called from sched_fork() */
+int sched_core_fork(struct task_struct *p, unsigned long clone_flags)
+{
+	struct sched_core_cookie *parent_cookie =
+		(struct sched_core_cookie *)current->core_cookie;
+
+	/*
+	 * core_cookie is ref counted; avoid an uncounted reference.
+	 * If p should have a cookie, it will be set below.
+	 */
+	p->core_cookie = 0UL;
+
+	/*
+	 * If parent is tagged via per-task cookie, tag the child (either with
+	 * the parent's cookie, or a new one).
+	 *
+	 * We can return directly in this case, because sched_core_share_tasks()
+	 * will set the core_cookie (so there is no need to try to inherit from
+	 * the parent). The cookie will have the proper sub-fields (ie. group
+	 * cookie, etc.), because these come from p's task_struct, which is
+	 * dup'd from the parent.
+	 */
+	if (current->core_task_cookie) {
+		int ret;
+
+		/* If it is not CLONE_THREAD fork, assign a unique per-task tag. */
+		if (!(clone_flags & CLONE_THREAD)) {
+			ret = sched_core_share_tasks(p, p);
+		} else {
+			/* Otherwise share the parent's per-task tag. */
+			ret = sched_core_share_tasks(p, current);
+		}
+
+		if (ret)
+			return ret;
+
+		/*
+		 * We expect sched_core_share_tasks() to always update p's
+		 * core_cookie.
+		 */
+		WARN_ON_ONCE(!p->core_cookie);
+
+		return 0;
+	}
+
+	/*
+	 * If parent is tagged, inherit the cookie and ensure that the reference
+	 * count is updated.
+	 *
+	 * Technically, we could instead zero-out the task's group_cookie and
+	 * allow sched_core_change_group() to handle this post-fork, but
+	 * inheriting here has a performance advantage, since we don't
+	 * need to traverse the core_cookies RB tree and can instead grab the
+	 * parent's cookie directly.
+	 */
+	if (parent_cookie) {
+		bool need_stopper = false;
+		unsigned long flags;
+
+		/*
+		 * cookies lock prevents task->core_cookie from changing or
+		 * being freed
+		 */
+		raw_spin_lock_irqsave(&sched_core_cookies_lock, flags);
+
+		if (likely(refcount_inc_not_zero(&parent_cookie->refcnt))) {
+			p->core_cookie = (unsigned long)parent_cookie;
+		} else {
+			/*
+			 * Raced with put(). We'll use stop_machine to get
+			 * a core_cookie
+			 */
+			need_stopper = true;
+		}
+
+		raw_spin_unlock_irqrestore(&sched_core_cookies_lock, flags);
+
+		if (need_stopper)
+			stop_machine(sched_update_core_tag_stopper,
+				     (void *)p, NULL);
+	}
+
+	return 0;
+}
+
+void sched_tsk_free(struct task_struct *tsk)
+{
+	struct sched_core_task_cookie *ck;
+
+	sched_core_put_cookie((struct sched_core_cookie *)tsk->core_cookie);
+
+	if (!tsk->core_task_cookie)
+		return;
+
+	ck = (struct sched_core_task_cookie *)tsk->core_task_cookie;
+	queue_work(system_wq, &ck->work);
+}
kernel/sched/debug.c +4 -0
index 60a922d3f46f..8c452b8010ad 100644
@@ -1024,6 +1024,10 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns,
  		__PS("clock-delta", t1-t0);
  	}
 
+#ifdef CONFIG_SCHED_CORE
+	__PS("core_cookie", p->core_cookie);
+#endif
+
  	sched_show_numa(p, m);
  }
 
kernel/sched/sched.h +57 -4
index b03f261b95b3..94e07c271528 100644
@@ -377,6 +377,10 @@ struct cfs_bandwidth {
  struct task_group {
  	struct cgroup_subsys_state css;
 
+#ifdef CONFIG_SCHED_CORE
+	int			core_tagged;
+#endif
+
  #ifdef CONFIG_FAIR_GROUP_SCHED
  	/* schedulable entities of this group on each CPU */
  	struct sched_entity	**se;
@@ -425,6 +429,11 @@ struct task_group {
 
  };
 
+static inline struct task_group *css_tg(struct cgroup_subsys_state *css)
+{
+	return css ? container_of(css, struct task_group, css) : NULL;
+}
+
  #ifdef CONFIG_FAIR_GROUP_SCHED
  #define ROOT_TASK_GROUP_LOAD	NICE_0_LOAD
 
@@ -1127,6 +1136,12 @@ static inline bool is_migration_disabled(struct task_struct *p)
  DECLARE_STATIC_KEY_FALSE(__sched_core_enabled);
  static inline struct cpumask *sched_group_span(struct sched_group *sg);
 
+enum sched_core_cookie_type {
+	sched_core_no_update = 0,
+	sched_core_task_cookie_type,
+	sched_core_group_cookie_type,
+};
+
  static inline bool sched_core_enabled(struct rq *rq)
  {
  	return static_branch_unlikely(&__sched_core_enabled) && rq->core_enabled;
@@ -1197,12 +1212,53 @@ static inline bool sched_group_cookie_match(struct rq *rq,
  	return false;
  }
 
-extern void queue_core_balance(struct rq *rq);
+void sched_core_change_group(struct task_struct *p, struct task_group *new_tg);
+int sched_core_fork(struct task_struct *p, unsigned long clone_flags);
+
+static inline bool sched_core_enqueued(struct task_struct *task)
+{
+	return !RB_EMPTY_NODE(&task->core_node);
+}
+
+void queue_core_balance(struct rq *rq);
+
+void sched_core_enqueue(struct rq *rq, struct task_struct *p);
+void sched_core_dequeue(struct rq *rq, struct task_struct *p);
+void sched_core_get(void);
+void sched_core_put(void);
+
+int sched_core_share_pid(unsigned long flags, pid_t pid);
+int sched_core_share_tasks(struct task_struct *t1, struct task_struct *t2);
+
+#ifdef CONFIG_CGROUP_SCHED
+u64 cpu_core_tag_read_u64(struct cgroup_subsys_state *css,
+			  struct cftype *cft);
+
+#ifdef CONFIG_SCHED_DEBUG
+u64 cpu_core_group_cookie_read_u64(struct cgroup_subsys_state *css,
+				   struct cftype *cft);
+#endif
+
+int cpu_core_tag_write_u64(struct cgroup_subsys_state *css, struct cftype *cft,
+			   u64 val);
+#endif
+
+#ifndef TIF_UNSAFE_RET
+#define TIF_UNSAFE_RET (0)
+#endif
 
  bool cfs_prio_less(struct task_struct *a, struct task_struct *b, bool fi);
 
  #else /* !CONFIG_SCHED_CORE */
 
+static inline bool sched_core_enqueued(struct task_struct *task) { return false; }
+static inline void sched_core_enqueue(struct rq *rq, struct task_struct *p) { }
+static inline void sched_core_dequeue(struct rq *rq, struct task_struct *p) { }
+static inline int sched_core_share_tasks(struct task_struct *t1, struct task_struct *t2)
+{
+	return -ENOTSUPP;
+}
+
  static inline bool sched_core_enabled(struct rq *rq)
  {
  	return false;
@@ -2899,7 +2955,4 @@ void swake_up_all_locked(struct swait_queue_head *q);
  void __prepare_to_swait(struct swait_queue_head *q, struct swait_queue *wait);
 
  #ifdef CONFIG_SCHED_CORE
-#ifndef TIF_UNSAFE_RET
-#define TIF_UNSAFE_RET (0)
-#endif
  #endif
kernel/sys.c +7 -0
index a730c03ee607..da52a0da24ef 100644
@@ -2530,6 +2530,13 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3,
 
  		error = (current->flags & PR_IO_FLUSHER) == PR_IO_FLUSHER;
  		break;
+#ifdef CONFIG_SCHED_CORE
+	case PR_SCHED_CORE_SHARE:
+		if (arg4 || arg5)
+			return -EINVAL;
+		error = sched_core_share_pid(arg2, arg3);
+		break;
+#endif
  	default:
  		error = -EINVAL;
  		break;
tools/include/uapi/linux/prctl.h +0 -0
index 7f0827705c9a..8fbf9d164ec4 100644

tools/include/uapi/linux/prctl.h struct prctl_mm_map { · line 254

Quoted 8 lines
tools/include/uapi/linux/prctl.h +5 -0
@@ -247,4 +247,10 @@ struct prctl_mm_map {
  #define PR_SET_IO_FLUSHER		57
  #define PR_GET_IO_FLUSHER		58
 
+/* Request the scheduler to share a core */
+#define PR_SCHED_CORE_SHARE		59
+#define PR_SCHED_CORE_CLEAR		0  /* clear core_sched cookie of pid */
+#define PR_SCHED_CORE_SHARE_FROM	1  /* get core_sched cookie from pid */
+#define PR_SCHED_CORE_SHARE_TO		2  /* push core_sched cookie to pid */

My original patch had this:

# define PR_SCHED_CORE_CLEAR            0  /* clear core_sched cookie of pid */
# define PR_SCHED_CORE_SHARE_FROM       1  /* get core_sched cookie from pid */
# define PR_SCHED_CORE_SHARE_TO         2  /* push core_sched cookie to pid */

The space between the '#' and 'define' is the trick apparently used to prevent a script run during 'perf' build that pulls options out of this file to generate a 'beauty' array. Without the space, you can't build perf.

-chrish

Quoted 3 lines
tools/include/uapi/linux/prctl.h +1 -0
+
  #endif /* _LINUX_PRCTL_H */
#122 lore raw

Re: [PATCH -tip 23/32] sched: Add a per-thread core scheduling interface

In reply to Joel Fernandes
Quoted 4 lines · Joel Fernandes
Quoted 1 line

No problem. That was there primarily for debugging.

Ok. I squashed Josh's changes into this patch and several of my fixups. So there'll be 3 patches:
1. CGroup + prctl (single patch as it is hell to split it)

Please don't do that. I am not sure we have thought the cgroup interface through (looking at all the discussions). IMHO, it would be better for us to get a simpler interface (prctl) right and then once we learn the lessons using it, apply it for the cgroup interface. I think we all agree we don't want to maintain a messy interface forever.

Dhaval

#123 lore raw

Re: [PATCH -tip 23/32] sched: Add a per-thread core scheduling interface

Hello Dhaval,

Quoted 11 lines · Dhaval Giani
Quoted 4 lines · Joel Fernandes
Quoted 1 line

No problem. That was there primarily for debugging.

Ok. I squashed Josh's changes into this patch and several of my fixups. So there'll be 3 patches:
1. CGroup + prctl (single patch as it is hell to split it)

Please don't do that.
I am not sure we have thought the cgroup interface through (looking at all the discussions).

Unfortunately, this comment does not provides any information on the issues you are concerned about. If there are specific issues you are concerned about, please don't keep it a secret! What requirement of yours is not being met with the CGroup interface the way it is in v8 series?

thanks,

- Joel

#124 lore raw

PATCH 24/32 sched: Release references to the per-task cookie on exit

In reply to Joel Fernandes (Google)

During exit, we have to free the references to a cookie that might be shared by many tasks. This commit therefore ensures when the task_struct is released, any references to cookies that it holds are also released.

---
 include/linux/sched.h | 3 +++
 kernel/fork.c         | 1 +
 kernel/sched/core.c   | 8 ++++++++
 3 files changed, 12 insertions(+)
include/linux/sched.h +3 -0
index 79d76c78cc8e..6fbdb1a204bf 100644
@@ -2084,11 +2084,14 @@ void sched_core_unsafe_exit(void);
 bool sched_core_wait_till_safe(unsigned long ti_check);
 bool sched_core_kernel_protected(void);
 int sched_core_share_pid(pid_t pid);
+void sched_tsk_free(struct task_struct *tsk);
 #else
 #define sched_core_unsafe_enter(ignore) do { } while (0)
 #define sched_core_unsafe_exit(ignore) do { } while (0)
 #define sched_core_wait_till_safe(ignore) do { } while (0)
 #define sched_core_kernel_protected(ignore) do { } while (0)
+#define sched_core_share_pid(pid) do { } while (0)
+#define sched_tsk_free(tsk) do { } while (0)
 #endif
 
 #endif
kernel/fork.c +1 -0
index 7199d359690c..5468c93829c5 100644
@@ -736,6 +736,7 @@ void __put_task_struct(struct task_struct *tsk)
 	exit_creds(tsk);
 	delayacct_tsk_free(tsk);
 	put_signal_struct(tsk->signal);
+	sched_tsk_free(tsk);
 
 	if (!profile_handoff_task(tsk))
 		free_task(tsk);
kernel/sched/core.c +8 -0
index a95898c75bdf..cc36c384364e 100644
@@ -10066,6 +10066,14 @@ static int cpu_core_tag_write_u64(struct cgroup_subsys_state *css, struct cftype
 
 	return 0;
 }
+
+void sched_tsk_free(struct task_struct *tsk)
+{
+	if (!tsk->core_task_cookie)
+		return;
+	sched_core_put_task_cookie(tsk->core_task_cookie);
+	sched_core_put();
+}
 #endif
 
 static struct cftype cpu_legacy_files[] = {
Signature · 1 line
2.29.2.299.gdc1121823c-goog
maintainer
#125 lore raw

Re: [PATCH -tip 24/32] sched: Release references to the per-task cookie on exit

Quoted 3 lines · Joel Fernandes (Google)

During exit, we have to free the references to a cookie that might be shared by many tasks. This commit therefore ensures when the task_struct is released, any references to cookies that it holds are also released.

This is one of those patches that just shouldn't exist. Squash it into whatever patch that introduces this nonsense.

#126 lore raw

PATCH 25/32 sched: Refactor core cookie into struct

In reply to Joel Fernandes (Google)

From: Josh Don <[email protected]>

The overall core cookie is currently a single unsigned long value. This poses issues as we seek to add additional sub-fields to the cookie. This patch refactors the core_cookie to be a pointer to a struct containing an arbitrary set of cookie fields.

We maintain a sorted RB tree of existing core cookies so that multiple tasks may share the same core_cookie.

This will be especially useful in the next patch, where the concept of cookie color is introduced.

---
 kernel/sched/core.c  | 481 +++++++++++++++++++++++++++++++++++++------
 kernel/sched/sched.h |  11 +-
 2 files changed, 429 insertions(+), 63 deletions(-)
kernel/sched/core.c +419 -62
index cc36c384364e..bd75b3d62a97 100644
@@ -3958,6 +3958,7 @@ static inline void init_schedstats(void) {}
 int sched_fork(unsigned long clone_flags, struct task_struct *p)
 {
 	unsigned long flags;
+	int __maybe_unused ret;
 
 	__sched_fork(clone_flags, p);
 	/*
@@ -4037,20 +4038,9 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p)
 #ifdef CONFIG_SCHED_CORE
 	RB_CLEAR_NODE(&p->core_node);
 
-	/*
-	 * If parent is tagged via per-task cookie, tag the child (either with
-	 * the parent's cookie, or a new one). The final cookie is calculated
-	 * by concatenating the per-task cookie with that of the CGroup's.
-	 */
-	if (current->core_task_cookie) {
-
-		/* If it is not CLONE_THREAD fork, assign a unique per-task tag. */
-		if (!(clone_flags & CLONE_THREAD)) {
-			return sched_core_share_tasks(p, p);
-               }
-		/* Otherwise share the parent's per-task tag. */
-		return sched_core_share_tasks(p, current);
-	}
+	ret = sched_core_fork(p, clone_flags);
+	if (ret)
+		return ret;
 #endif
 	return 0;
 }
@@ -9059,6 +9049,9 @@ void sched_offline_group(struct task_group *tg)
 	spin_unlock_irqrestore(&task_group_lock, flags);
 }
 
+void cpu_core_get_group_cookie(struct task_group *tg,
+			       unsigned long *group_cookie_ptr);
+
 static void sched_change_group(struct task_struct *tsk, int type)
 {
 	struct task_group *tg;
@@ -9073,11 +9066,7 @@ static void sched_change_group(struct task_struct *tsk, int type)
 	tg = autogroup_task_group(tsk, tg);
 
 #ifdef CONFIG_SCHED_CORE
-	if ((unsigned long)tsk->sched_task_group == tsk->core_cookie)
-		tsk->core_cookie = 0UL;
-
-	if (tg->tagged /* && !tsk->core_cookie ? */)
-		tsk->core_cookie = (unsigned long)tg;
+	sched_core_change_group(tsk, tg);
 #endif
 
 	tsk->sched_task_group = tg;
@@ -9177,9 +9166,9 @@ static void cpu_cgroup_css_offline(struct cgroup_subsys_state *css)
 #ifdef CONFIG_SCHED_CORE
 	struct task_group *tg = css_tg(css);
 
-	if (tg->tagged) {
+	if (tg->core_tagged) {
 		sched_core_put();
-		tg->tagged = 0;
+		tg->core_tagged = 0;
 	}
 #endif
 }
@@ -9751,38 +9740,225 @@ static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css,
 
 #ifdef CONFIG_SCHED_CORE
 /*
- * A simple wrapper around refcount. An allocated sched_core_cookie's
- * address is used to compute the cookie of the task.
+ * Wrapper representing a complete cookie. The address of the cookie is used as
+ * a unique identifier. Each cookie has a unique permutation of the internal
+ * cookie fields.
  */
 struct sched_core_cookie {
+	unsigned long task_cookie;
+	unsigned long group_cookie;
+
+	struct rb_node node;
 	refcount_t refcnt;
 };
 
 /*
- * sched_core_tag_requeue - Common helper for all interfaces to set a cookie.
- * @p: The task to assign a cookie to.
- * @cookie: The cookie to assign.
- * @group: is it a group interface or a per-task interface.
+ * A simple wrapper around refcount. An allocated sched_core_task_cookie's
+ * address is used to compute the cookie of the task.
+ */
+struct sched_core_task_cookie {
+	refcount_t refcnt;
+};
+
+/* All active sched_core_cookies */
+static struct rb_root sched_core_cookies = RB_ROOT;
+static DEFINE_RAW_SPINLOCK(sched_core_cookies_lock);
+
+/*
+ * Returns the following:
+ * a < b  => -1
+ * a == b => 0
+ * a > b  => 1
+ */
+static int sched_core_cookie_cmp(const struct sched_core_cookie *a,
+				 const struct sched_core_cookie *b)
+{
+#define COOKIE_CMP_RETURN(field) do {		\
+	if (a->field < b->field)		\
+		return -1;			\
+	else if (a->field > b->field)		\
+		return 1;			\
+} while (0)					\
+
+	COOKIE_CMP_RETURN(task_cookie);
+	COOKIE_CMP_RETURN(group_cookie);
+
+	/* all cookie fields match */
+	return 0;
+
+#undef COOKIE_CMP_RETURN
+}
+
+static inline void __sched_core_erase_cookie(struct sched_core_cookie *cookie)
+{
+	lockdep_assert_held(&sched_core_cookies_lock);
+
+	/* Already removed */
+	if (RB_EMPTY_NODE(&cookie->node))
+		return;
+
+	rb_erase(&cookie->node, &sched_core_cookies);
+	RB_CLEAR_NODE(&cookie->node);
+}
+
+/* Called when a task no longer points to the cookie in question */
+static void sched_core_put_cookie(struct sched_core_cookie *cookie)
+{
+	unsigned long flags;
+
+	if (!cookie)
+		return;
+
+	if (refcount_dec_and_test(&cookie->refcnt)) {
+		raw_spin_lock_irqsave(&sched_core_cookies_lock, flags);
+		__sched_core_erase_cookie(cookie);
+		raw_spin_unlock_irqrestore(&sched_core_cookies_lock, flags);
+		kfree(cookie);
+	}
+}
+
+/*
+ * A task's core cookie is a compound structure composed of various cookie
+ * fields (task_cookie, group_cookie). The overall core_cookie is
+ * a pointer to a struct containing those values. This function either finds
+ * an existing core_cookie or creates a new one, and then updates the task's
+ * core_cookie to point to it. Additionally, it handles the necessary reference
+ * counting.
  *
- * This function is typically called from a stop-machine handler.
+ * REQUIRES: task_rq(p) lock or called from cpu_stopper.
+ * Doing so ensures that we do not cause races/corruption by modifying/reading
+ * task cookie fields.
  */
-void sched_core_tag_requeue(struct task_struct *p, unsigned long cookie, bool group)
+static void __sched_core_update_cookie(struct task_struct *p)
+{
+	struct rb_node *parent, **node;
+	struct sched_core_cookie *node_core_cookie, *match;
+	static const struct sched_core_cookie zero_cookie;
+	struct sched_core_cookie temp = {
+		.task_cookie	= p->core_task_cookie,
+		.group_cookie	= p->core_group_cookie,
+	};
+	const bool is_zero_cookie =
+		(sched_core_cookie_cmp(&temp, &zero_cookie) == 0);
+	struct sched_core_cookie *const curr_cookie =
+		(struct sched_core_cookie *)p->core_cookie;
+	unsigned long flags;
+
+	/*
+	 * Already have a cookie matching the requested settings? Nothing to
+	 * do.
+	 */
+	if ((curr_cookie && sched_core_cookie_cmp(curr_cookie, &temp) == 0) ||
+	    (!curr_cookie && is_zero_cookie))
+		return;
+
+	raw_spin_lock_irqsave(&sched_core_cookies_lock, flags);
+
+	if (is_zero_cookie) {
+		match = NULL;
+		goto finish;
+	}
+
+retry:
+	match = NULL;
+
+	node = &sched_core_cookies.rb_node;
+	parent = *node;
+	while (*node) {
+		int cmp;
+
+		node_core_cookie =
+			container_of(*node, struct sched_core_cookie, node);
+		parent = *node;
+
+		cmp = sched_core_cookie_cmp(&temp, node_core_cookie);
+		if (cmp < 0) {
+			node = &parent->rb_left;
+		} else if (cmp > 0) {
+			node = &parent->rb_right;
+		} else {
+			match = node_core_cookie;
+			break;
+		}
+	}
+
+	if (!match) {
+		/* No existing cookie; create and insert one */
+		match = kmalloc(sizeof(struct sched_core_cookie), GFP_ATOMIC);
+
+		/* Fall back to zero cookie */
+		if (WARN_ON_ONCE(!match))
+			goto finish;
+
+		match->task_cookie = temp.task_cookie;
+		match->group_cookie = temp.group_cookie;
+		refcount_set(&match->refcnt, 1);
+
+		rb_link_node(&match->node, parent, node);
+		rb_insert_color(&match->node, &sched_core_cookies);
+	} else {
+		/*
+		 * Cookie exists, increment refcnt. If refcnt is currently 0,
+		 * we're racing with a put() (refcnt decremented but cookie not
+		 * yet removed from the tree). In this case, we can simply
+		 * perform the removal ourselves and retry.
+		 * sched_core_put_cookie() will still function correctly.
+		 */
+		if (unlikely(!refcount_inc_not_zero(&match->refcnt))) {
+			__sched_core_erase_cookie(match);
+			goto retry;
+		}
+	}
+
+finish:
+	/*
+	 * Set the core_cookie under the cookies lock. This guarantees that
+	 * p->core_cookie cannot be freed while the cookies lock is held in
+	 * sched_core_fork().
+	 */
+	p->core_cookie = (unsigned long)match;
+
+	raw_spin_unlock_irqrestore(&sched_core_cookies_lock, flags);
+
+	sched_core_put_cookie(curr_cookie);
+}
+
+/*
+ * sched_core_update_cookie - Common helper to update a task's core cookie. This
+ * updates the selected cookie field and then updates the overall cookie.
+ * @p: The task whose cookie should be updated.
+ * @cookie: The new cookie.
+ * @cookie_type: The cookie field to which the cookie corresponds.
+ *
+ * REQUIRES: either task_rq(p)->lock held or called from a stop-machine handler.
+ * Doing so ensures that we do not cause races/corruption by modifying/reading
+ * task cookie fields.
+ */
+static void sched_core_update_cookie(struct task_struct *p, unsigned long cookie,
+				     enum sched_core_cookie_type cookie_type)
 {
 	if (!p)
 		return;
 
-	if (group)
-		p->core_group_cookie = cookie;
-	else
+	switch (cookie_type) {
+	case sched_core_no_update:
+		break;
+	case sched_core_task_cookie_type:
 		p->core_task_cookie = cookie;
+		break;
+	case sched_core_group_cookie_type:
+		p->core_group_cookie = cookie;
+		break;
+	default:
+		WARN_ON_ONCE(1);
+	}
 
-	/* Use up half of the cookie's bits for task cookie and remaining for group cookie. */
-	p->core_cookie = (p->core_task_cookie <<
-				(sizeof(unsigned long) * 4)) + p->core_group_cookie;
+	/* Set p->core_cookie, which is the overall cookie */
+	__sched_core_update_cookie(p);
 
 	if (sched_core_enqueued(p)) {
 		sched_core_dequeue(task_rq(p), p);
-		if (!p->core_task_cookie)
+		if (!p->core_cookie)
 			return;
 	}
 
@@ -9791,11 +9967,28 @@ void sched_core_tag_requeue(struct task_struct *p, unsigned long cookie, bool gr
 		sched_core_enqueue(task_rq(p), p);
 }
 
+void cpu_core_get_group_cookie(struct task_group *tg,
+			       unsigned long *group_cookie_ptr);
+
+void sched_core_change_group(struct task_struct *p, struct task_group *new_tg)
+{
+	unsigned long new_group_cookie;
+
+	cpu_core_get_group_cookie(new_tg, &new_group_cookie);
+
+	if (p->core_group_cookie == new_group_cookie)
+		return;
+
+	p->core_group_cookie = new_group_cookie;
+
+	__sched_core_update_cookie(p);
+}
+
 /* Per-task interface */
 static unsigned long sched_core_alloc_task_cookie(void)
 {
-	struct sched_core_cookie *ptr =
-		kmalloc(sizeof(struct sched_core_cookie), GFP_KERNEL);
+	struct sched_core_task_cookie *ptr =
+		kmalloc(sizeof(struct sched_core_task_cookie), GFP_KERNEL);
 
 	if (!ptr)
 		return 0;
@@ -9811,7 +10004,8 @@ static unsigned long sched_core_alloc_task_cookie(void)
 
 static bool sched_core_get_task_cookie(unsigned long cookie)
 {
-	struct sched_core_cookie *ptr = (struct sched_core_cookie *)cookie;
+	struct sched_core_task_cookie *ptr =
+		(struct sched_core_task_cookie *)cookie;
 
 	/*
 	 * NOTE: sched_core_put() is not done by put_task_cookie(). Instead, it
@@ -9823,7 +10017,8 @@ static bool sched_core_get_task_cookie(unsigned long cookie)
 
 static void sched_core_put_task_cookie(unsigned long cookie)
 {
-	struct sched_core_cookie *ptr = (struct sched_core_cookie *)cookie;
+	struct sched_core_task_cookie *ptr =
+		(struct sched_core_task_cookie *)cookie;
 
 	if (refcount_dec_and_test(&ptr->refcnt))
 		kfree(ptr);
@@ -9845,7 +10040,8 @@ static int sched_core_task_join_stopper(void *data)
 	int i;
 
 	for (i = 0; i < 2; i++)
-		sched_core_tag_requeue(tag->tasks[i], tag->cookies[i], false /* !group */);
+		sched_core_update_cookie(tag->tasks[i], tag->cookies[i],
+					 sched_core_task_cookie_type);
 
 	return 0;
 }
@@ -10002,41 +10198,89 @@ int sched_core_share_pid(pid_t pid)
 }
 
 /* CGroup interface */
+
+/*
+ * Helper to get the group cookie and color in a hierarchy.
+ * Any ancestor can have a tag/color. Atmost one color and one
+ * tag is allowed.
+ * Sets *group_cookie_ptr to the hierarchical group cookie.
+ */
+void cpu_core_get_group_cookie(struct task_group *tg,
+			       unsigned long *group_cookie_ptr)
+{
+	unsigned long group_cookie = 0UL;
+
+	if (!tg)
+		goto out;
+
+	for (; tg; tg = tg->parent) {
+
+		if (tg->core_tagged) {
+			group_cookie = (unsigned long)tg;
+			break;
+		}
+	}
+
+out:
+	*group_cookie_ptr = group_cookie;
+}
+
+/* Determine if any group in @tg's children are tagged. */
+static bool cpu_core_check_descendants(struct task_group *tg, bool check_tag)
+{
+	struct task_group *child;
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(child, &tg->children, siblings) {
+		if ((child->core_tagged && check_tag)) {
+			rcu_read_unlock();
+			return true;
+		}
+
+		rcu_read_unlock();
+		return cpu_core_check_descendants(child, check_tag);
+	}
+
+	rcu_read_unlock();
+	return false;
+}
+
 static u64 cpu_core_tag_read_u64(struct cgroup_subsys_state *css, struct cftype *cft)
 {
 	struct task_group *tg = css_tg(css);
 
-	return !!tg->tagged;
+	return !!tg->core_tagged;
 }
 
 struct write_core_tag {
 	struct cgroup_subsys_state *css;
-	int val;
+	unsigned long cookie;
+	enum sched_core_cookie_type cookie_type;
 };
 
 static int __sched_write_tag(void *data)
 {
 	struct write_core_tag *tag = (struct write_core_tag *) data;
-	struct cgroup_subsys_state *css = tag->css;
-	int val = tag->val;
-	struct task_group *tg = css_tg(tag->css);
-	struct css_task_iter it;
 	struct task_struct *p;
+	struct cgroup_subsys_state *css;
 
-	tg->tagged = !!val;
+	rcu_read_lock();
+	css_for_each_descendant_pre(css, tag->css) {
+		struct css_task_iter it;
 
-	css_task_iter_start(css, 0, &it);
-	/*
-	 * Note: css_task_iter_next will skip dying tasks.
-	 * There could still be dying tasks left in the core queue
-	 * when we set cgroup tag to 0 when the loop is done below.
-	 */
-	while ((p = css_task_iter_next(&it))) {
-		unsigned long cookie = !!val ? (unsigned long)tg : 0UL;
+		css_task_iter_start(css, 0, &it);
+		/*
+		 * Note: css_task_iter_next will skip dying tasks.
+		 * There could still be dying tasks left in the core queue
+		 * when we set cgroup tag to 0 when the loop is done below.
+		 */
+		while ((p = css_task_iter_next(&it)))
+			sched_core_update_cookie(p, tag->cookie,
+						 tag->cookie_type);
 
-		sched_core_tag_requeue(p, cookie, true /* group */);
+		css_task_iter_end(&it);
 	}
-	css_task_iter_end(&it);
+	rcu_read_unlock();
 
 	return 0;
 }
@@ -10045,6 +10289,7 @@ static int cpu_core_tag_write_u64(struct cgroup_subsys_state *css, struct cftype
 {
 	struct task_group *tg = css_tg(css);
 	struct write_core_tag wtag;
+	unsigned long group_cookie;
 
 	if (val > 1)
 		return -ERANGE;
@@ -10052,14 +10297,29 @@ static int cpu_core_tag_write_u64(struct cgroup_subsys_state *css, struct cftype
 	if (!static_branch_likely(&sched_smt_present))
 		return -EINVAL;
 
-	if (tg->tagged == !!val)
+	if (!tg->core_tagged && val) {
+		/* Tag is being set. Check ancestors and descendants. */
+		cpu_core_get_group_cookie(tg, &group_cookie);
+		if (group_cookie ||
+		    cpu_core_check_descendants(tg, true /* tag */))
+			return -EBUSY;
+	} else if (tg->core_tagged && !val) {
+		/* Tag is being reset. Check descendants. */
+		if (cpu_core_check_descendants(tg, true /* tag */))
+			return -EBUSY;
+	} else {
 		return 0;
+	}
 
 	if (!!val)
 		sched_core_get();
 
 	wtag.css = css;
-	wtag.val = val;
+	wtag.cookie = (unsigned long)tg;
+	wtag.cookie_type = sched_core_group_cookie_type;
+
+	tg->core_tagged = val;
+
 	stop_machine(__sched_write_tag, (void *) &wtag, NULL);
 	if (!val)
 		sched_core_put();
@@ -10067,8 +10327,105 @@ static int cpu_core_tag_write_u64(struct cgroup_subsys_state *css, struct cftype
 	return 0;
 }
 
+static int sched_update_core_tag_stopper(void *data)
+{
+	struct task_struct *p = (struct task_struct *)data;
+
+	/* Recalculate core cookie */
+	sched_core_update_cookie(p, 0, sched_core_no_update);
+
+	return 0;
+}
+
+/* Called from sched_fork() */
+int sched_core_fork(struct task_struct *p, unsigned long clone_flags)
+{
+	struct sched_core_cookie *parent_cookie =
+		(struct sched_core_cookie *)current->core_cookie;
+
+	/*
+	 * core_cookie is ref counted; avoid an uncounted reference.
+	 * If p should have a cookie, it will be set below.
+	 */
+	p->core_cookie = 0UL;
+
+	/*
+	 * If parent is tagged via per-task cookie, tag the child (either with
+	 * the parent's cookie, or a new one).
+	 *
+	 * We can return directly in this case, because sched_core_share_tasks()
+	 * will set the core_cookie (so there is no need to try to inherit from
+	 * the parent). The cookie will have the proper sub-fields (ie. group
+	 * cookie, etc.), because these come from p's task_struct, which is
+	 * dup'd from the parent.
+	 */
+	if (current->core_task_cookie) {
+		int ret;
+
+		/* If it is not CLONE_THREAD fork, assign a unique per-task tag. */
+		if (!(clone_flags & CLONE_THREAD)) {
+			ret = sched_core_share_tasks(p, p);
+		} else {
+			/* Otherwise share the parent's per-task tag. */
+			ret = sched_core_share_tasks(p, current);
+		}
+
+		if (ret)
+			return ret;
+
+		/*
+		 * We expect sched_core_share_tasks() to always update p's
+		 * core_cookie.
+		 */
+		WARN_ON_ONCE(!p->core_cookie);
+
+		return 0;
+	}
+
+	/*
+	 * If parent is tagged, inherit the cookie and ensure that the reference
+	 * count is updated.
+	 *
+	 * Technically, we could instead zero-out the task's group_cookie and
+	 * allow sched_core_change_group() to handle this post-fork, but
+	 * inheriting here has a performance advantage, since we don't
+	 * need to traverse the core_cookies RB tree and can instead grab the
+	 * parent's cookie directly.
+	 */
+	if (parent_cookie) {
+		bool need_stopper = false;
+		unsigned long flags;
+
+		/*
+		 * cookies lock prevents task->core_cookie from changing or
+		 * being freed
+		 */
+		raw_spin_lock_irqsave(&sched_core_cookies_lock, flags);
+
+		if (likely(refcount_inc_not_zero(&parent_cookie->refcnt))) {
+			p->core_cookie = (unsigned long)parent_cookie;
+		} else {
+			/*
+			 * Raced with put(). We'll use stop_machine to get
+			 * a core_cookie
+			 */
+			need_stopper = true;
+		}
+
+		raw_spin_unlock_irqrestore(&sched_core_cookies_lock, flags);
+
+		if (need_stopper)
+			stop_machine(sched_update_core_tag_stopper,
+				     (void *)p, NULL);
+	}
+
+	return 0;
+}
+
 void sched_tsk_free(struct task_struct *tsk)
 {
+	sched_core_put_cookie((struct sched_core_cookie *)tsk->core_cookie);
+
 	if (!tsk->core_task_cookie)
 		return;
 	sched_core_put_task_cookie(tsk->core_task_cookie);
@@ -10114,7 +10471,7 @@ static struct cftype cpu_legacy_files[] = {
 #endif
 #ifdef CONFIG_SCHED_CORE
 	{
-		.name = "tag",
+		.name = "core_tag",
 		.flags = CFTYPE_NOT_ON_ROOT,
 		.read_u64 = cpu_core_tag_read_u64,
 		.write_u64 = cpu_core_tag_write_u64,
@@ -10295,7 +10652,7 @@ static struct cftype cpu_files[] = {
 #endif
 #ifdef CONFIG_SCHED_CORE
 	{
-		.name = "tag",
+		.name = "core_tag",
 		.flags = CFTYPE_NOT_ON_ROOT,
 		.read_u64 = cpu_core_tag_read_u64,
 		.write_u64 = cpu_core_tag_write_u64,
kernel/sched/sched.h +10 -1
index 3ba08973ed58..042a9d6a3be9 100644
@@ -378,7 +378,7 @@ struct task_group {
 	struct cgroup_subsys_state css;
 
 #ifdef CONFIG_SCHED_CORE
-	int			tagged;
+	int			core_tagged;
 #endif
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
@@ -1130,6 +1130,12 @@ static inline bool is_migration_disabled(struct task_struct *p)
 #ifdef CONFIG_SCHED_CORE
 DECLARE_STATIC_KEY_FALSE(__sched_core_enabled);
 
+enum sched_core_cookie_type {
+	sched_core_no_update = 0,
+	sched_core_task_cookie_type,
+	sched_core_group_cookie_type,
+};
+
 static inline bool sched_core_enabled(struct rq *rq)
 {
 	return static_branch_unlikely(&__sched_core_enabled) && rq->core_enabled;
@@ -1174,6 +1180,9 @@ static inline bool sched_core_cookie_match(struct rq *rq, struct task_struct *p)
 	return idle_core || rq->core->core_cookie == p->core_cookie;
 }
 
+void sched_core_change_group(struct task_struct *p, struct task_group *new_tg);
+int sched_core_fork(struct task_struct *p, unsigned long clone_flags);
+
 extern void queue_core_balance(struct rq *rq);
 
 bool cfs_prio_less(struct task_struct *a, struct task_struct *b, bool fi);
Signature · 1 line
2.29.2.299.gdc1121823c-goog
#127 lore raw

PATCH 26/32 sched: Add a second-level tag for nested CGroup usecase

In reply to Joel Fernandes (Google)

From: Josh Don <[email protected]>

Google has a usecase where the first level tag to tag a CGroup is not sufficient. So, a patch is carried for years where a second tag is added which is writeable by unprivileged users.

Google uses DAC controls to make the 'tag' possible to set only by root while the second-level 'color' can be changed by anyone. The actual names that Google uses is different, but the concept is the same.

The hierarchy looks like:

Root group
/ \
A B (These are created by the root daemon - borglet).
/ \ \
C D E (These are created by AppEngine within the container).

The reason why Google has two parts is that AppEngine wants to allow a subset of subcgroups within a parent tagged cgroup sharing execution. Think of these subcgroups belong to the same customer or project. Because these subcgroups are created by AppEngine, they are not tracked by borglet (the root daemon), therefore borglet won't have a chance to set a color for them. That's where 'color' file comes from. Color could be set by AppEngine, and once set, the normal tasks within the subcgroup would not be able to overwrite it. This is enforced by promoting the permission of the color file in cgroupfs.

---
 include/linux/sched.h |   1 +
 kernel/sched/core.c   | 120 +++++++++++++++++++++++++++++++++++-------
 kernel/sched/sched.h  |   2 +
 3 files changed, 103 insertions(+), 20 deletions(-)
include/linux/sched.h +1 -0
index 6fbdb1a204bf..c9efdf8ccdf3 100644
@@ -690,6 +690,7 @@ struct task_struct {
 	unsigned long			core_cookie;
 	unsigned long			core_task_cookie;
 	unsigned long			core_group_cookie;
+	unsigned long			core_color;
 	unsigned int			core_occupation;
 #endif
 
kernel/sched/core.c +100 -20

248 lines not on this page · raw message

kernel/sched/sched.h +2 -0
index 042a9d6a3be9..0ca22918b69a 100644
@@ -379,6 +379,7 @@ struct task_group {
 
 #ifdef CONFIG_SCHED_CORE
 	int			core_tagged;
+	unsigned long		core_tag_color;
 #endif
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
@@ -1134,6 +1135,7 @@ enum sched_core_cookie_type {
 	sched_core_no_update = 0,
 	sched_core_task_cookie_type,
 	sched_core_group_cookie_type,
+	sched_core_color_type
 };
 
 static inline bool sched_core_enabled(struct rq *rq)
Signature · 1 line
2.29.2.299.gdc1121823c-goog
maintainer
#128 lore raw

Re: [PATCH -tip 26/32] sched: Add a second-level tag for nested CGroup usecase

Quoted 18 lines · Joel Fernandes (Google)

From: Josh Don <[email protected]>

Google has a usecase where the first level tag to tag a CGroup is not sufficient. So, a patch is carried for years where a second tag is added which is writeable by unprivileged users.

Google uses DAC controls to make the 'tag' possible to set only by root while the second-level 'color' can be changed by anyone. The actual names that Google uses is different, but the concept is the same.

The hierarchy looks like:

Root group
/ \
A B (These are created by the root daemon - borglet).
/ \ \
C D E (These are created by AppEngine within the container).

Quoted 8 lines

The reason why Google has two parts is that AppEngine wants to allow a subset of subcgroups within a parent tagged cgroup sharing execution. Think of these subcgroups belong to the same customer or project. Because these subcgroups are created by AppEngine, they are not tracked by borglet (the root daemon), therefore borglet won't have a chance to set a color for them. That's where 'color' file comes from. Color could be set by AppEngine, and once set, the normal tasks within the subcgroup would not be able to overwrite it. This is enforced by promoting the permission of the color file in cgroupfs.

Why can't the above work by setting 'tag' (that's a terrible name, why does that still live) in CDE? Have the most specific tag live. Same with that thread stuff.

All this API stuff here is a complete and utter trainwreck. Please just delete the patches and start over. Hint: if you use stop_machine(), you're doing it wrong.

At best you now have the requirements sorted.

#129 lore raw

Re: [PATCH -tip 26/32] sched: Add a second-level tag for nested CGroup usecase

Quoted 32 lines · Peter Zijlstra
Quoted 18 lines · Joel Fernandes (Google)

From: Josh Don <[email protected]>

Google has a usecase where the first level tag to tag a CGroup is not sufficient. So, a patch is carried for years where a second tag is added which is writeable by unprivileged users.

Google uses DAC controls to make the 'tag' possible to set only by root while the second-level 'color' can be changed by anyone. The actual names that Google uses is different, but the concept is the same.

The hierarchy looks like:

Root group
/ \
A B (These are created by the root daemon - borglet).
/ \ \
C D E (These are created by AppEngine within the container).

Quoted 8 lines

The reason why Google has two parts is that AppEngine wants to allow a subset of subcgroups within a parent tagged cgroup sharing execution. Think of these subcgroups belong to the same customer or project. Because these subcgroups are created by AppEngine, they are not tracked by borglet (the root daemon), therefore borglet won't have a chance to set a color for them. That's where 'color' file comes from. Color could be set by AppEngine, and once set, the normal tasks within the subcgroup would not be able to overwrite it. This is enforced by promoting the permission of the color file in cgroupfs.

Why can't the above work by setting 'tag' (that's a terrible name, why does that still live) in CDE? Have the most specific tag live. Same with that thread stuff.

Quoted 5 lines

All this API stuff here is a complete and utter trainwreck. Please just delete the patches and start over. Hint: if you use stop_machine(), you're doing it wrong.

At best you now have the requirements sorted.

+1, just remove this patch from the series so as to unblock the series.

Balbir Singh.

#130 lore raw

Re: [PATCH -tip 26/32] sched: Add a second-level tag for nested CGroup usecase

In reply to Peter Zijlstra

Hi Peter,

Quoted 28 lines · Peter Zijlstra
Quoted 18 lines · Joel Fernandes (Google)

From: Josh Don <[email protected]>

Google has a usecase where the first level tag to tag a CGroup is not sufficient. So, a patch is carried for years where a second tag is added which is writeable by unprivileged users.

Google uses DAC controls to make the 'tag' possible to set only by root while the second-level 'color' can be changed by anyone. The actual names that Google uses is different, but the concept is the same.

The hierarchy looks like:

Root group
/ \
A B (These are created by the root daemon - borglet).
/ \ \
C D E (These are created by AppEngine within the container).

Quoted 8 lines

The reason why Google has two parts is that AppEngine wants to allow a subset of subcgroups within a parent tagged cgroup sharing execution. Think of these subcgroups belong to the same customer or project. Because these subcgroups are created by AppEngine, they are not tracked by borglet (the root daemon), therefore borglet won't have a chance to set a color for them. That's where 'color' file comes from. Color could be set by AppEngine, and once set, the normal tasks within the subcgroup would not be able to overwrite it. This is enforced by promoting the permission of the color file in cgroupfs.

Quoted 3 lines

Why can't the above work by setting 'tag' (that's a terrible name, why does that still live) in CDE? Have the most specific tag live. Same with that thread stuff.

There's 2 parts that Google's usecase has. The first part is set by a privileged process, and the second part (color) is set within the container. Maybe we can just put the "color" feature behind a CONFIG option for Google to enable?

Quoted 3 lines

All this API stuff here is a complete and utter trainwreck. Please just delete the patches and start over. Hint: if you use stop_machine(), you're doing it wrong.

Ok, the idea was to use stop_machine() as in your initial patch. It works quite well in testing. However I agree with its horrible we ought to do better (or at least try).

Maybe we can do a synchronize_rcu() after changing cookie, to ensure we are no longer using the old cookie value in the scheduler.

Quoted 1 line

At best you now have the requirements sorted.

Yes.

thanks,

- Joel

#131 lore raw

Re: [PATCH -tip 26/32] sched: Add a second-level tag for nested CGroup usecase

In reply to Peter Zijlstra

Hey Peter,

Quoted 4 lines · Peter Zijlstra

Why can't the above work by setting 'tag' (that's a terrible name, why does that still live) in CDE? Have the most specific tag live. Same with that thread stuff.

The motivation is to allow an unprivileged user the ability to configure the trust hierarchy in a way that otherwise wouldn't be possible for a given cgroup hierarchy. For example given a cookie'd hierarchy such as:

A

   /  |  |   \
B  C  D  E

the user might only want subsets of {B, C, D, E} to share. For instance, the user might only want {B,C} and {D, E} to share. One way to solve this would be to allow the user to write the group cookie directly. However, this interface would need to be restricted to privileged users, since otherwise the cookie could be configured to share with any arbitrary cgroup. The purpose of the 'color' field is to expose a portion of the cookie that can be modified by a non-privileged user in order to achieve this sharing goal.

If this doesn't seem like a useful case, I'm happy to drop this patch from the series to unblock it.

Quoted 3 lines

All this API stuff here is a complete and utter trainwreck. Please just delete the patches and start over. Hint: if you use stop_machine(), you're doing it wrong.

Yes, agree on stop_machine(); we'll pull that out of the underlying interface patch.

Thanks,
Josh

maintainer
#132 lore raw

Re: [PATCH -tip 26/32] sched: Add a second-level tag for nested CGroup usecase

Quoted 26 lines · Josh Don

Hey Peter,

Quoted 4 lines · Peter Zijlstra

Why can't the above work by setting 'tag' (that's a terrible name, why does that still live) in CDE? Have the most specific tag live. Same with that thread stuff.

The motivation is to allow an unprivileged user the ability to configure the trust hierarchy in a way that otherwise wouldn't be possible for a given cgroup hierarchy. For example given a cookie'd hierarchy such as:

A

   /  |  |   \
B  C  D  E

the user might only want subsets of {B, C, D, E} to share. For instance, the user might only want {B,C} and {D, E} to share. One way to solve this would be to allow the user to write the group cookie directly. However, this interface would need to be restricted to privileged users, since otherwise the cookie could be configured to share with any arbitrary cgroup. The purpose of the 'color' field is to expose a portion of the cookie that can be modified by a non-privileged user in order to achieve this sharing goal.

Quoted 2 lines

If this doesn't seem like a useful case, I'm happy to drop this patch from the series to unblock it.

Well, the traditional cgroup way of doing that would be to:

         A
	/ \
       T1 T2
      / \
     B   C

And tag T1 if you want B,C to share.

So me the color thing reads like an end-run around the cgroup hierarchy.

#133 lore raw

Re: [PATCH -tip 26/32] sched: Add a second-level tag for nested CGroup usecase

Hello,

Quoted 20 lines · Peter Zijlstra
Quoted 11 lines

the user might only want subsets of {B, C, D, E} to share. For instance, the user might only want {B,C} and {D, E} to share. One way to solve this would be to allow the user to write the group cookie directly. However, this interface would need to be restricted to privileged users, since otherwise the cookie could be configured to share with any arbitrary cgroup. The purpose of the 'color' field is to expose a portion of the cookie that can be modified by a non-privileged user in order to achieve this sharing goal.

If this doesn't seem like a useful case, I'm happy to drop this patch from the series to unblock it.

Well, the traditional cgroup way of doing that would be to:

         A
	/ \
       T1 T2
      / \
     B   C
Quoted 3 lines

And tag T1 if you want B,C to share.

So me the color thing reads like an end-run around the cgroup hierarchy.

+1

and please cc me on cgroup related changes.

Thanks.

Signature · 1 line
tejun
#134 lore raw

Re: [PATCH -tip 26/32] sched: Add a second-level tag for nested CGroup usecase

In reply to Peter Zijlstra
Quoted 39 lines · Peter Zijlstra
Quoted 26 lines · Josh Don

Hey Peter,

Quoted 4 lines · Peter Zijlstra

Why can't the above work by setting 'tag' (that's a terrible name, why does that still live) in CDE? Have the most specific tag live. Same with that thread stuff.

The motivation is to allow an unprivileged user the ability to configure the trust hierarchy in a way that otherwise wouldn't be possible for a given cgroup hierarchy. For example given a cookie'd hierarchy such as:

A

   /  |  |   \
B  C  D  E

the user might only want subsets of {B, C, D, E} to share. For instance, the user might only want {B,C} and {D, E} to share. One way to solve this would be to allow the user to write the group cookie directly. However, this interface would need to be restricted to privileged users, since otherwise the cookie could be configured to share with any arbitrary cgroup. The purpose of the 'color' field is to expose a portion of the cookie that can be modified by a non-privileged user in order to achieve this sharing goal.

Quoted 2 lines

If this doesn't seem like a useful case, I'm happy to drop this patch from the series to unblock it.

Well, the traditional cgroup way of doing that would be to:

         A
        / \
       T1 T2
      / \
     B   C
Quoted 3 lines

And tag T1 if you want B,C to share.

So me the color thing reads like an end-run around the cgroup hierarchy.

Restructuring the cgroup resource hierarchy to incorporate the trust domains is not necessarily trivial (as is the case for us). I agree though that that would be the ideal correct solution from the cgroup hierarchy perspective.

#135 lore raw

Re: [PATCH -tip 26/32] sched: Add a second-level tag for nested CGroup usecase

Hello,

Quoted 6 lines · Josh Don
Quoted 1 line

So me the color thing reads like an end-run around the cgroup hierarchy.

Restructuring the cgroup resource hierarchy to incorporate the trust domains is not necessarily trivial (as is the case for us). I agree though that that would be the ideal correct solution from the cgroup hierarchy perspective.

Yeah, it sure isn't. We (FB) had to go through a couple iterations and it's looking like we need another adjustment, so I fully agree that these are painful but at the same time I don't think it's difficult to see that how piling up workarounds in the lowest layer is not the right thing to do. The workarounds you guys need would be different from what we or others would need. The kernel can't be maintained in any sustainable manner if we keep piling on disjoint workarounds on it. Please consider long term trajectory when proposing interface changes which often boils down to identifying the core features which must be supported by the interface.

Thanks.

Signature · 1 line
tejun
#136 lore raw

PATCH 27/32 sched/debug: Add CGroup node for printing group cookie if SCHED_DEBUG

In reply to Joel Fernandes (Google)

This will be used by kselftest to verify the CGroup cookie value that is set by the CGroup interface.

---
 kernel/sched/core.c | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)
kernel/sched/core.c +31 -0
index 8f17ec8e993e..f1d9762b571a 100644
@@ -10277,6 +10277,21 @@ static u64 cpu_core_tag_color_read_u64(struct cgroup_subsys_state *css, struct c
 	return tg->core_tag_color;
 }
 
+#ifdef CONFIG_SCHED_DEBUG
+static u64 cpu_core_group_cookie_read_u64(struct cgroup_subsys_state *css, struct cftype *cft)
+{
+	unsigned long group_cookie, color;
+
+	cpu_core_get_group_cookie_and_color(css_tg(css), &group_cookie, &color);
+
+	/*
+	 * Combine group_cookie and color into a single 64 bit value, for
+	 * display purposes only.
+	 */
+	return (group_cookie << 32) | (color & 0xffffffff);
+}
+#endif
+
 struct write_core_tag {
 	struct cgroup_subsys_state *css;
 	unsigned long cookie;
@@ -10550,6 +10565,14 @@ static struct cftype cpu_legacy_files[] = {
 		.read_u64 = cpu_core_tag_color_read_u64,
 		.write_u64 = cpu_core_tag_color_write_u64,
 	},
+#ifdef CONFIG_SCHED_DEBUG
+	/* Read the effective cookie (color+tag) of the group. */
+	{
+		.name = "core_group_cookie",
+		.flags = CFTYPE_NOT_ON_ROOT,
+		.read_u64 = cpu_core_group_cookie_read_u64,
+	},
+#endif
 #endif
 #ifdef CONFIG_UCLAMP_TASK_GROUP
 	{
@@ -10737,6 +10760,14 @@ static struct cftype cpu_files[] = {
 		.read_u64 = cpu_core_tag_color_read_u64,
 		.write_u64 = cpu_core_tag_color_write_u64,
 	},
+#ifdef CONFIG_SCHED_DEBUG
+	/* Read the effective cookie (color+tag) of the group. */
+	{
+		.name = "core_group_cookie",
+		.flags = CFTYPE_NOT_ON_ROOT,
+		.read_u64 = cpu_core_group_cookie_read_u64,
+	},
+#endif
 #endif
 #ifdef CONFIG_CFS_BANDWIDTH
 	{
Signature · 1 line
2.29.2.299.gdc1121823c-goog
#137 lore raw

PATCH 28/32 kselftest: Add tests for core-sched interface

In reply to Joel Fernandes (Google)

Add a kselftest test to ensure that the core-sched interface is working correctly.

---
 tools/testing/selftests/sched/.gitignore      |   1 +
 tools/testing/selftests/sched/Makefile        |  14 +
 tools/testing/selftests/sched/config          |   1 +
 .../testing/selftests/sched/test_coresched.c  | 818 ++++++++++++++++++
 4 files changed, 834 insertions(+)
 create mode 100644 tools/testing/selftests/sched/.gitignore
 create mode 100644 tools/testing/selftests/sched/Makefile
 create mode 100644 tools/testing/selftests/sched/config
 create mode 100644 tools/testing/selftests/sched/test_coresched.c
tools/testing/selftests/sched/.gitignore new file +1 -0
new file mode 100644
index 000000000000..4660929b0b9a
@@ -0,0 +1 @@
+test_coresched
tools/testing/selftests/sched/Makefile new file +14 -0
new file mode 100644
index 000000000000..e43b74fc5d7e
@@ -0,0 +1,14 @@
+# SPDX-License-Identifier: GPL-2.0+
+
+ifneq ($(shell $(CC) --version 2>&1 | head -n 1 | grep clang),)
+CLANG_FLAGS += -no-integrated-as
+endif
+
+CFLAGS += -O2 -Wall -g -I./ -I../../../../usr/include/  -Wl,-rpath=./ \
+	  $(CLANG_FLAGS)
+LDLIBS += -lpthread
+
+TEST_GEN_FILES := test_coresched
+TEST_PROGS := test_coresched
+
+include ../lib.mk
tools/testing/selftests/sched/config new file +1 -0
new file mode 100644
index 000000000000..e8b09aa7c0c4
@@ -0,0 +1 @@
+CONFIG_SCHED_DEBUG=y
tools/testing/selftests/sched/test_coresched.c new file +818 -0

818 lines not on this page · raw message

Signature · 1 line
2.29.2.299.gdc1121823c-goog
#138 lore raw

PATCH 29/32 sched: Move core-scheduler interfacing code to a new file

In reply to Joel Fernandes (Google)

core.c is already huge. The core-tagging interface code is largely independent of it. Move it to its own file to make both files easier to maintain.

Also make the following changes:
- Fix SWA bugs found by Chris Hyser.
- Fix refcount underrun caused by not zero'ing new task's cookie.

---
 kernel/sched/Makefile  |   1 +
 kernel/sched/core.c    | 809 +---------------------------------------
 kernel/sched/coretag.c | 819 +++++++++++++++++++++++++++++++++++++++++
 kernel/sched/sched.h   |  51 ++-
 4 files changed, 872 insertions(+), 808 deletions(-)
 create mode 100644 kernel/sched/coretag.c
kernel/sched/Makefile +1 -0
index 5fc9c9b70862..c526c20adf9d 100644
@@ -36,3 +36,4 @@ obj-$(CONFIG_CPU_FREQ_GOV_SCHEDUTIL) += cpufreq_schedutil.o
 obj-$(CONFIG_MEMBARRIER) += membarrier.o
 obj-$(CONFIG_CPU_ISOLATION) += isolation.o
 obj-$(CONFIG_PSI) += psi.o
+obj-$(CONFIG_SCHED_CORE) += coretag.o
kernel/sched/core.c +5 -804

857 lines not on this page · raw message

kernel/sched/coretag.c new file +819 -0

819 lines not on this page · raw message

kernel/sched/sched.h +47 -4

72 lines not on this page · raw message

Signature · 1 line
2.29.2.299.gdc1121823c-goog
#139 lore raw

PATCH 30/32 Documentation: Add core scheduling documentation

In reply to Joel Fernandes (Google)

Document the usecases, design and interfaces for core scheduling.

---
 .../admin-guide/hw-vuln/core-scheduling.rst   | 330 ++++++++++++++++++
 Documentation/admin-guide/hw-vuln/index.rst   |   1 +
 2 files changed, 331 insertions(+)
 create mode 100644 Documentation/admin-guide/hw-vuln/core-scheduling.rst
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +330 -0
new file mode 100644
index 000000000000..01be28d0687a
@@ -0,0 +1,330 @@
+Core Scheduling
+***************
+Core scheduling support allows userspace to define groups of tasks that can
+share a core. These groups can be specified either for security usecases (one
+group of tasks don't trust another), or for performance usecases (some
+workloads may benefit from running on the same core as they don't need the same
+hardware resources of the shared core).
+
+Security usecase
+----------------
+A cross-HT attack involves the attacker and victim running on different
+Hyper Threads of the same core. MDS and L1TF are examples of such attacks.
+Without core scheduling, the only full mitigation of cross-HT attacks is to
+disable Hyper Threading (HT). Core scheduling allows HT to be turned on safely
+by ensuring that trusted tasks can share a core. This increase in core sharing
+can improvement performance, however it is not guaranteed that performance will
+always improve, though that is seen to be the case with a number of real world
+workloads. In theory, core scheduling aims to perform at least as good as when
+Hyper Threading is disabled. In practice, this is mostly the case though not
+always: as synchronizing scheduling decisions across 2 or more CPUs in a core
+involves additional overhead - especially when the system is lightly loaded
+(``total_threads <= N/2``, where N is the total number of CPUs).
+
+Usage
+-----
+Core scheduling support is enabled via the ``CONFIG_SCHED_CORE`` config option.
+Using this feature, userspace defines groups of tasks that trust each other.
+The core scheduler uses this information to make sure that tasks that do not
+trust each other will never run simultaneously on a core, while doing its best
+to satisfy the system's scheduling requirements.
+
+There are 2 ways to use core-scheduling:
+
+CGroup
+######
+Core scheduling adds additional files to the CPU controller CGroup:
+
+* ``cpu.core_tag``
+Writing ``1`` into this file results in all tasks in the group getting tagged.
+This results in all the CGroup's tasks allowed to run concurrently on a core's
+hyperthreads (also called siblings).
+
+The file being a value of ``0`` means the tag state of the CGroup is inherited
+from its parent hierarchy. If any ancestor of the CGroup is tagged, then the
+group is tagged.
+
+.. note:: Once a CGroup is tagged via cpu.core_tag, it is not possible to set this
+          for any descendant of the tagged group. For finer grained control, the
+          ``cpu.core_tag_color`` file described next may be used.
+
+.. note:: When a CGroup is not tagged, all the tasks within the group can share
+          a core with kernel threads and untagged system threads. For this reason,
+          if a group has ``cpu.core_tag`` of 0, it is considered to be trusted.
+
+* ``cpu.core_tag_color``
+For finer grained control over core sharing, a color can also be set in
+addition to the tag. This allows to further control core sharing between child
+CGroups within an already tagged CGroup. The color and the tag are both used to
+generate a `cookie` which is used by the scheduler to identify the group.
+
+Up to 256 different colors can be set (0-255) by writing into this file.
+
+A sample real-world usage of this file follows:
+
+Google uses DAC controls to make ``cpu.core_tag`` writable only by root and the
+``cpu.core_tag_color`` can be changed by anyone.
+
+The hierarchy looks like this:
+::
+  Root group
+     / \
+    A   B    (These are created by the root daemon - borglet).
+   / \   \
+  C   D   E  (These are created by AppEngine within the container).
+
+A and B are containers for 2 different jobs or apps that are created by a root
+daemon called borglet. borglet then tags each of these group with the ``cpu.core_tag``
+file. The job itself can create additional child CGroups which are colored by
+the container's AppEngine with the ``cpu.core_tag_color`` file.
+
+The reason why Google uses this 2-level tagging system is that AppEngine wants to
+allow a subset of child CGroups within a tagged parent CGroup to be co-scheduled on a
+core while not being co-scheduled with other child CGroups. Think of these
+child CGroups as belonging to the same customer or project.  Because these
+child CGroups are created by AppEngine, they are not tracked by borglet (the
+root daemon), therefore borglet won't have a chance to set a color for them.
+That's where cpu.core_tag_color file comes in. A color could be set by AppEngine,
+and once set, the normal tasks within the subcgroup would not be able to
+overwrite it. This is enforced by promoting the permission of the
+``cpu.core_tag_color`` file in cgroupfs.
+
+The color is an 8-bit value allowing for up to 256 unique colors.
+
+.. note:: Once a CGroup is colored, none of its descendants can be re-colored. Also
+          coloring of a CGroup is possible only if either the group or one of its
+          ancestors was tagged via the ``cpu.core_tag`` file.
+
+prctl interface
+###############
+A ``prtcl(2)`` command ``PR_SCHED_CORE_SHARE`` is available to a process to request
+sharing a core with another process.  For example, consider 2 processes ``P1``
+and ``P2`` with PIDs 100 and 200. If process ``P1`` calls
+``prctl(PR_SCHED_CORE_SHARE, 200)``, the kernel makes ``P1`` share a core with ``P2``.
+The kernel performs ptrace access mode checks before granting the request.
+
+.. note:: This operation is not commutative. P1 calling
+          ``prctl(PR_SCHED_CORE_SHARE, pidof(P2)`` is not the same as P2 calling the
+          same for P1. The former case is P1 joining P2's group of processes
+          (which P2 would have joined with ``prctl(2)`` prior to P1's ``prctl(2)``).
+
+.. note:: The core-sharing granted with prctl(2) will be subject to
+          core-sharing restrictions specified by the CGroup interface. For example
+          if P1 and P2 are a part of 2 different tagged CGroups, then they will
+          not share a core even if a prctl(2) call is made. This is analogous
+          to how affinities are set using the cpuset interface.
+
+It is important to note that, on a ``CLONE_THREAD`` ``clone(2)`` syscall, the child
+will be assigned the same tag as its parent and thus be allowed to share a core
+with them. This design choice is because, for the security usecase, a
+``CLONE_THREAD`` child can access its parent's address space anyway, so there's
+no point in not allowing them to share a core. If a different behavior is
+desired, the child thread can call ``prctl(2)`` as needed.  This behavior is
+specific to the ``prctl(2)`` interface. For the CGroup interface, the child of a
+fork always shares a core with its parent.  On the other hand, if a parent
+was previously tagged via ``prctl(2)`` and does a regular ``fork(2)`` syscall, the
+child will receive a unique tag.
+
+Design/Implementation
+---------------------
+Each task that is tagged is assigned a cookie internally in the kernel. As
+mentioned in `Usage`_, tasks with the same cookie value are assumed to trust
+each other and share a core.
+
+The basic idea is that, every schedule event tries to select tasks for all the
+siblings of a core such that all the selected tasks running on a core are
+trusted (same cookie) at any point in time. Kernel threads are assumed trusted.
+The idle task is considered special, as it trusts everything and everything
+trusts it.
+
+During a schedule() event on any sibling of a core, the highest priority task on
+the sibling's core is picked and assigned to the sibling calling schedule(), if
+the sibling has the task enqueued. For rest of the siblings in the core,
+highest priority task with the same cookie is selected if there is one runnable
+in their individual run queues. If a task with same cookie is not available,
+the idle task is selected.  Idle task is globally trusted.
+
+Once a task has been selected for all the siblings in the core, an IPI is sent to
+siblings for whom a new task was selected. Siblings on receiving the IPI will
+switch to the new task immediately. If an idle task is selected for a sibling,
+then the sibling is considered to be in a `forced idle` state. I.e., it may
+have tasks on its on runqueue to run, however it will still have to run idle.
+More on this in the next section.
+
+Forced-idling of tasks
+----------------------
+The scheduler tries its best to find tasks that trust each other such that all
+tasks selected to be scheduled are of the highest priority in a core.  However,
+it is possible that some runqueues had tasks that were incompatible with the
+highest priority ones in the core. Favoring security over fairness, one or more
+siblings could be forced to select a lower priority task if the highest
+priority task is not trusted with respect to the core wide highest priority
+task.  If a sibling does not have a trusted task to run, it will be forced idle
+by the scheduler (idle thread is scheduled to run).
+
+When the highest priority task is selected to run, a reschedule-IPI is sent to
+the sibling to force it into idle. This results in 4 cases which need to be
+considered depending on whether a VM or a regular usermode process was running
+on either HT::
+
+          HT1 (attack)            HT2 (victim)
+   A      idle -> user space      user space -> idle
+   B      idle -> user space      guest -> idle
+   C      idle -> guest           user space -> idle
+   D      idle -> guest           guest -> idle
+
+Note that for better performance, we do not wait for the destination CPU
+(victim) to enter idle mode. This is because the sending of the IPI would bring
+the destination CPU immediately into kernel mode from user space, or VMEXIT
+in the case of guests. At best, this would only leak some scheduler metadata
+which may not be worth protecting. It is also possible that the IPI is received
+too late on some architectures, but this has not been observed in the case of
+x86.
+
+Kernel protection from untrusted tasks
+--------------------------------------
+Entry into the kernel (syscall, IRQ or VMEXIT) needs protection.  The scheduler
+on its own cannot protect the kernel executing concurrently with an untrusted
+task in a core. This is because the scheduler is unaware of interrupts/syscalls
+at scheduling time. To mitigate this, an IPI is sent to siblings on kernel
+entry. This IPI forces the sibling to enter kernel mode and wait before
+returning to user until all siblings of the core have left kernel mode. This
+process is also known as stunning.  For good performance, an IPI is sent only
+to a sibling only if it is running a tagged task. If a sibling is running a
+kernel thread or is idle, no IPI is sent.
+
+The kernel protection feature is on my default and can be turned off on the
+kernel command line by passing ``sched_core_protect_kernel=0``.
+
+Note that an arch has to define the ``TIF_UNSAFE_RET`` thread info flag to be
+able to use kernel protection. Also, if protecting the kernel from a VM is
+desired, an arch should call kvm_exit_to_guest_mode() during ``VMENTER`` and
+kvm_exit_to_guest_mode() during ``VMEXIT``. Currently, x86 support both these.
+
+Other alternative ideas discussed for kernel protection are listed below just
+for completeness. They all have limitations:
+
+1. Changing interrupt affinities to a trusted core which does not execute untrusted tasks
+#########################################################################################
+By changing the interrupt affinities to a designated safe-CPU which runs
+only trusted tasks, IRQ data can be protected. One issue is this involves
+giving up a full CPU core of the system to run safe tasks. Another is that,
+per-cpu interrupts such as the local timer interrupt cannot have their
+affinity changed. Also, sensitive timer callbacks such as the random entropy timer
+can run in softirq on return from these interrupts and expose sensitive
+data. In the future, that could be mitigated by forcing softirqs into threaded
+mode by utilizing a mechanism similar to ``CONFIG_PREEMPT_RT``.
+
+Yet another issue with this is, for multiqueue devices with managed
+interrupts, the IRQ affinities cannot be changed; however it could be
+possible to force a reduced number of queues which would in turn allow to
+shield one or two CPUs from such interrupts and queue handling for the price
+of indirection.
+
+2. Running IRQs as threaded-IRQs
+################################
+This would result in forcing IRQs into the scheduler which would then provide
+the process-context mitigation. However, not all interrupts can be threaded.
+Also this does nothing about syscall entries.
+
+3. Kernel Address Space Isolation
+#################################
+System calls could run in a much restricted address space which is
+guaranteed not to leak any sensitive data. There are practical limitation in
+implementing this - the main concern being how to decide on an address space
+that is guaranteed to not have any sensitive data.
+
+4. Limited cookie-based protection
+##################################
+On a system call, change the cookie to the system trusted cookie and initiate a
+schedule event. This would be better than pausing all the siblings during the
+entire duration for the system call, but still would be a huge hit to the
+performance.
+
+Trust model
+-----------
+Core scheduling maintains trust relationships amongst groups of tasks by
+assigning the tag of them with the same cookie value.
+When a system with core scheduling boots, all tasks are considered to trust
+each other. This is because the core scheduler does not have information about
+trust relationships until userspace uses the above mentioned interfaces, to
+communicate them. In other words, all tasks have a default cookie value of 0.
+and are considered system-wide trusted. The stunning of siblings running
+cookie-0 tasks is also avoided.
+
+Once userspace uses the above mentioned interfaces to group sets of tasks, tasks
+within such groups are considered to trust each other, but do not trust those
+outside. Tasks outside the group also don't trust tasks within.
+
+coresched command line option
+-----------------------------
+The coresched kernel command line option can be used to:
+  - Keep coresched on even if system is not vulnerable x (``coresched=on``).
+  - Keep coresched off even if system is vulnerable (``coresched=off``).
+  - Keep coresched on only if system is vulnerable x (``coresched=secure``).
+
+The default is ``coresched=secure``. However a user who has a usecase that
+needs core-scheduling, such as improving performance of VMs by tagging vCPU
+threads, could pass ``coresched=on`` to force it on.
+
+Limitations in core-scheduling
+------------------------------
+Core scheduling tries to guarantee that only trusted tasks run concurrently on a
+core. But there could be small window of time during which untrusted tasks run
+concurrently or kernel could be running concurrently with a task not trusted by
+kernel.
+
+1. IPI processing delays
+########################
+Core scheduling selects only trusted tasks to run together. IPI is used to notify
+the siblings to switch to the new task. But there could be hardware delays in
+receiving of the IPI on some arch (on x86, this has not been observed). This may
+cause an attacker task to start running on a CPU before its siblings receive the
+IPI. Even though cache is flushed on entry to user mode, victim tasks on siblings
+may populate data in the cache and micro architectural buffers after the attacker
+starts to run and this is a possibility for data leak.
+
+Open cross-HT issues that core scheduling does not solve
+--------------------------------------------------------
+1. For MDS
+##########
+Core scheduling cannot protect against MDS attacks between an HT running in
+user mode and another running in kernel mode. Even though both HTs run tasks
+which trust each other, kernel memory is still considered untrusted. Such
+attacks are possible for any combination of sibling CPU modes (host or guest mode).
+
+2. For L1TF
+###########
+Core scheduling cannot protect against an L1TF guest attacker exploiting a
+guest or host victim. This is because the guest attacker can craft invalid
+PTEs which are not inverted due to a vulnerable guest kernel. The only
+solution is to disable EPT (Extended Page Tables).
+
+For both MDS and L1TF, if the guest vCPU is configured to not trust each
+other (by tagging separately), then the guest to guest attacks would go away.
+Or it could be a system admin policy which considers guest to guest attacks as
+a guest problem.
+
+Another approach to resolve these would be to make every untrusted task on the
+system to not trust every other untrusted task. While this could reduce
+parallelism of the untrusted tasks, it would still solve the above issues while
+allowing system processes (trusted tasks) to share a core.
+
+Use cases
+---------
+The main use case for Core scheduling is mitigating the cross-HT vulnerabilities
+with SMT enabled. There are other use cases where this feature could be used:
+
+- Isolating tasks that needs a whole core: Examples include realtime tasks, tasks
+  that uses SIMD instructions etc.
+- Gang scheduling: Requirements for a group of tasks that needs to be scheduled
+  together could also be realized using core scheduling. One example is vCPUs of
+  a VM.
+
+Future work
+-----------
+Skipping per-HT mitigations if task is trusted
+##############################################
+If core scheduling is enabled, by default all tasks trust each other as
+mentioned above. In such scenario, it may be desirable to skip the same-HT
+mitigations on return to the trusted user-mode to improve performance.
Documentation/admin-guide/hw-vuln/index.rst +1 -0
index 21710f8609fe..361ccbbd9e54 100644
@@ -16,3 +16,4 @@ are configurable at compile, boot or run time.
    multihit.rst
    special-register-buffer-data-sampling.rst
    l1d_flush.rst
+   core-scheduling.rst
Signature · 1 line
2.29.2.299.gdc1121823c-goog
#140 lore raw

PATCH 31/32 sched: Add a coresched command line option

In reply to Joel Fernandes (Google)

Some hardware such as certain AMD variants don't have cross-HT MDS/L1TF issues. Detect this and don't enable core scheduling as it can needlessly slow those device down.

However, some users may want core scheduling even if the hardware is secure. To support them, add a coresched= option which defaults to 'secure' and can be overridden to 'on' if the user wants to enable coresched even if the HW is not vulnerable. 'off' would disable core scheduling in any case.

Also add a sched_debug entry to indicate if core scheduling is turned on or not.

---
 .../admin-guide/kernel-parameters.txt         | 14 ++++++
 arch/x86/kernel/cpu/bugs.c                    | 19 ++++++++
 include/linux/cpu.h                           |  1 +
 include/linux/sched/smt.h                     |  4 ++
 kernel/cpu.c                                  | 43 +++++++++++++++++++
 kernel/sched/core.c                           |  6 +++
 kernel/sched/debug.c                          |  4 ++
 7 files changed, 91 insertions(+)
Documentation/admin-guide/kernel-parameters.txt +14 -0
index b185c6ed4aba..9cd2cf7c18d4 100644
@@ -698,6 +698,20 @@
 			/proc/<pid>/coredump_filter.
 			See also Documentation/filesystems/proc.rst.
 
+	coresched=	[SCHED_CORE] This feature allows the Linux scheduler
+			to force hyperthread siblings of a CPU to only execute tasks
+			concurrently on all hyperthreads that are running within the
+			same core scheduling group.
+			Possible values are:
+			'on' - Enable scheduler capability to core schedule.
+			By default, no tasks will be core scheduled, but the coresched
+			interface can be used to form groups of tasks that are forced
+			to share a core.
+			'off' - Disable scheduler capability to core schedule.
+			'secure' - Like 'on' but only enable on systems affected by
+			MDS or L1TF vulnerabilities. 'off' otherwise.
+			Default: 'secure'.
+
 	coresight_cpu_debug.enable
 			[ARM,ARM64]
 			Format: <bool>
arch/x86/kernel/cpu/bugs.c +19 -0
index dece79e4d1e9..f3163f4a805c 100644
@@ -43,6 +43,7 @@ static void __init mds_select_mitigation(void);
 static void __init mds_print_mitigation(void);
 static void __init taa_select_mitigation(void);
 static void __init srbds_select_mitigation(void);
+static void __init coresched_select(void);
 
 /* The base value of the SPEC_CTRL MSR that always has to be preserved. */
 u64 x86_spec_ctrl_base;
@@ -103,6 +104,9 @@ void __init check_bugs(void)
 	if (boot_cpu_has(X86_FEATURE_STIBP))
 		x86_spec_ctrl_mask |= SPEC_CTRL_STIBP;
 
+	/* Update whether core-scheduling is needed. */
+	coresched_select();
+
 	/* Select the proper CPU mitigations before patching alternatives: */
 	spectre_v1_select_mitigation();
 	spectre_v2_select_mitigation();
@@ -1808,4 +1812,19 @@ ssize_t cpu_show_srbds(struct device *dev, struct device_attribute *attr, char *
 {
 	return cpu_show_common(dev, attr, buf, X86_BUG_SRBDS);
 }
+
+/*
+ * When coresched=secure command line option is passed (default), disable core
+ * scheduling if CPU does not have MDS/L1TF vulnerability.
+ */
+static void __init coresched_select(void)
+{
+#ifdef CONFIG_SCHED_CORE
+	if (coresched_cmd_secure() &&
+	    !boot_cpu_has_bug(X86_BUG_MDS) &&
+	    !boot_cpu_has_bug(X86_BUG_L1TF))
+		static_branch_disable(&sched_coresched_supported);
+#endif
+}
+
 #endif
include/linux/cpu.h +1 -0
index d6428aaf67e7..d1f1e64316d6 100644
@@ -228,4 +228,5 @@ static inline int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval) { return 0;
 extern bool cpu_mitigations_off(void);
 extern bool cpu_mitigations_auto_nosmt(void);
 
+extern bool coresched_cmd_secure(void);
 #endif /* _LINUX_CPU_H_ */
include/linux/sched/smt.h +4 -0
index 59d3736c454c..561064eb3268 100644
@@ -17,4 +17,8 @@ static inline bool sched_smt_active(void) { return false; }
 
 void arch_smt_update(void);
 
+#ifdef CONFIG_SCHED_CORE
+extern struct static_key_true sched_coresched_supported;
+#endif
+
 #endif
kernel/cpu.c +43 -0

46 lines not on this page · raw message

kernel/sched/core.c +6 -0
index 5ef04bdc849f..01938a2154fd 100644
@@ -325,8 +325,12 @@ static void __sched_core_disable(void)
 	static_branch_disable(&__sched_core_enabled);
 }
 
+DEFINE_STATIC_KEY_TRUE(sched_coresched_supported);
+
 void sched_core_get(void)
 {
+	if (!static_branch_likely(&sched_coresched_supported))
+		return;
 	mutex_lock(&sched_core_mutex);
 	if (!sched_core_count++)
 		__sched_core_enable();
@@ -335,6 +339,8 @@ void sched_core_get(void)
 
 void sched_core_put(void)
 {
+	if (!static_branch_likely(&sched_coresched_supported))
+		return;
 	mutex_lock(&sched_core_mutex);
 	if (!--sched_core_count)
 		__sched_core_disable();
kernel/sched/debug.c +4 -0
index 8c452b8010ad..cffdfab7478e 100644
@@ -782,6 +782,10 @@ static void sched_debug_header(struct seq_file *m)
 		"sysctl_sched_tunable_scaling",
 		sysctl_sched_tunable_scaling,
 		sched_tunable_scaling_names[sysctl_sched_tunable_scaling]);
+#ifdef CONFIG_SCHED_CORE
+	SEQ_printf(m, "  .%-40s: %d\n", "core_sched_enabled",
+		   !!static_branch_likely(&__sched_core_enabled));
+#endif
 	SEQ_printf(m, "\n");
 }
 
Signature · 1 line
2.29.2.299.gdc1121823c-goog
reviewer
#141 lore raw

Re: [PATCH -tip 31/32] sched: Add a coresched command line option

Hi Joel,

Documentation/admin-guide/kernel-parameters.txt line 701

Quoted 9 lines · Joel Fernandes (Google)
Documentation/admin-guide/kernel-parameters.txt +1 -0
index b185c6ed4aba..9cd2cf7c18d4 100644
@@ -698,6 +698,20 @@
 			/proc/<pid>/coredump_filter.
 			See also Documentation/filesystems/proc.rst.
 
+	coresched=	[SCHED_CORE] This feature allows the Linux scheduler

Unless I missed it somewhere else, this "SCHED_CORE" string should be added to Documentation/admin-guide/kernel-parameters.rst, where there is a list of "qualifiers" for kernel parameters.

(It looks like you are using it as the name of a Kconfig option, which makes some sense, but that's not how it's [currently] done. :)

Quoted 6 lines
Documentation/admin-guide/kernel-parameters.txt +6 -0
+			to force hyperthread siblings of a CPU to only execute tasks
+			concurrently on all hyperthreads that are running within the
+			same core scheduling group.
+			Possible values are:
+			'on' - Enable scheduler capability to core schedule.
+			By default, no tasks will be core scheduled, but the coresched

Documentation/admin-guide/kernel-parameters.txt line 717

Quoted 10 lines
Documentation/admin-guide/kernel-parameters.txt +7 -0
+			interface can be used to form groups of tasks that are forced
+			to share a core.
+			'off' - Disable scheduler capability to core schedule.
+			'secure' - Like 'on' but only enable on systems affected by
+			MDS or L1TF vulnerabilities. 'off' otherwise.
+			Default: 'secure'.
+
 	coresight_cpu_debug.enable
 			[ARM,ARM64]
 			Format: <bool>

thanks.

Signature · 1 line
~Randy
maintainer
#142 lore raw

Re: [PATCH -tip 31/32] sched: Add a coresched command line option

In reply to Joel Fernandes (Google)
Quoted 9 lines · Joel Fernandes (Google)

Some hardware such as certain AMD variants don't have cross-HT MDS/L1TF issues. Detect this and don't enable core scheduling as it can needlessly slow those device down.

However, some users may want core scheduling even if the hardware is secure. To support them, add a coresched= option which defaults to 'secure' and can be overridden to 'on' if the user wants to enable coresched even if the HW is not vulnerable. 'off' would disable core scheduling in any case.

This is all sorts of wrong, and the reason is because you hard-coded that stupid policy.

Core scheduling should always be available on SMT (provided you did that CONFIG_ thing). Even on AMD systems RT tasks might want to claim the core exclusively.

#143 lore raw

Re: [PATCH -tip 31/32] sched: Add a coresched command line option

Quoted 11 lines · Peter Zijlstra
Quoted 9 lines · Joel Fernandes (Google)

Some hardware such as certain AMD variants don't have cross-HT MDS/L1TF issues. Detect this and don't enable core scheduling as it can needlessly slow those device down.

However, some users may want core scheduling even if the hardware is secure. To support them, add a coresched= option which defaults to 'secure' and can be overridden to 'on' if the user wants to enable coresched even if the HW is not vulnerable. 'off' would disable core scheduling in any case.

Quoted 6 lines

This is all sorts of wrong, and the reason is because you hard-coded that stupid policy.

Core scheduling should always be available on SMT (provided you did that CONFIG_ thing). Even on AMD systems RT tasks might want to claim the core exclusively.

Agreed, specifically if we need to have special cgroup tag/association to enable it.

Balbir Singh.

#144 lore raw

PATCH 32/32 sched: Debug bits...

In reply to Joel Fernandes (Google)
---
 kernel/sched/core.c | 35 ++++++++++++++++++++++++++++++++++-
 kernel/sched/fair.c |  9 +++++++++
 2 files changed, 43 insertions(+), 1 deletion(-)
kernel/sched/core.c +34 -1
index 01938a2154fd..bbeeb18d460e 100644
@@ -127,6 +127,10 @@ static inline bool prio_less(struct task_struct *a, struct task_struct *b, bool
 
 	int pa = __task_prio(a), pb = __task_prio(b);
 
+	trace_printk("(%s/%d;%d,%Lu,%Lu) ?< (%s/%d;%d,%Lu,%Lu)\n",
+		     a->comm, a->pid, pa, a->se.vruntime, a->dl.deadline,
+		     b->comm, b->pid, pb, b->se.vruntime, b->dl.deadline);
+
 	if (-pa < -pb)
 		return true;
 
@@ -317,12 +321,16 @@ static void __sched_core_enable(void)
 
 	static_branch_enable(&__sched_core_enabled);
 	stop_machine(__sched_core_stopper, (void *)true, NULL);
+
+	printk("core sched enabled\n");
 }
 
 static void __sched_core_disable(void)
 {
 	stop_machine(__sched_core_stopper, (void *)false, NULL);
 	static_branch_disable(&__sched_core_enabled);
+
+	printk("core sched disabled\n");
 }
 
 DEFINE_STATIC_KEY_TRUE(sched_coresched_supported);
@@ -5486,6 +5494,13 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
 			set_next_task(rq, next);
 		}
 
+		trace_printk("pick pre selected (%u %u %u): %s/%d %lx\n",
+			     rq->core->core_task_seq,
+			     rq->core->core_pick_seq,
+			     rq->core_sched_seq,
+			     next->comm, next->pid,
+			     next->core_cookie);
+
 		rq->core_pick = NULL;
 		return next;
 	}
@@ -5580,6 +5595,9 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
 					rq->core->core_forceidle_seq++;
 			}
 
+			trace_printk("cpu(%d): selected: %s/%d %lx\n",
+				     i, p->comm, p->pid, p->core_cookie);
+
 			/*
 			 * If this new candidate is of higher priority than the
 			 * previous; and they're incompatible; we need to wipe
@@ -5596,6 +5614,8 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
 				rq->core->core_cookie = p->core_cookie;
 				max = p;
 
+				trace_printk("max: %s/%d %lx\n", max->comm, max->pid, max->core_cookie);
+
 				if (old_max) {
 					rq->core->core_forceidle = false;
 					for_each_cpu(j, smt_mask) {
@@ -5617,6 +5637,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
 
 	/* Something should have been selected for current CPU */
 	WARN_ON_ONCE(!next);
+	trace_printk("picked: %s/%d %lx\n", next->comm, next->pid, next->core_cookie);
 
 	/*
 	 * Reschedule siblings
@@ -5658,13 +5679,21 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
 		}
 
 		/* Did we break L1TF mitigation requirements? */
-		WARN_ON_ONCE(!cookie_match(next, rq_i->core_pick));
+		if (unlikely(!cookie_match(next, rq_i->core_pick))) {
+			trace_printk("[%d]: cookie mismatch. %s/%d/0x%lx/0x%lx\n",
+				     rq_i->cpu, rq_i->core_pick->comm,
+				     rq_i->core_pick->pid,
+				     rq_i->core_pick->core_cookie,
+				     rq_i->core->core_cookie);
+			WARN_ON_ONCE(1);
+		}
 
 		if (rq_i->curr == rq_i->core_pick) {
 			rq_i->core_pick = NULL;
 			continue;
 		}
 
+		trace_printk("IPI(%d)\n", i);
 		resched_curr(rq_i);
 	}
 
@@ -5704,6 +5733,10 @@ static bool try_steal_cookie(int this, int that)
 		if (p->core_occupation > dst->idle->core_occupation)
 			goto next;
 
+		trace_printk("core fill: %s/%d (%d->%d) %d %d %lx\n",
+			     p->comm, p->pid, that, this,
+			     p->core_occupation, dst->idle->core_occupation, cookie);
+
 		p->on_rq = TASK_ON_RQ_MIGRATING;
 		deactivate_task(src, p, 0);
 		set_task_cpu(p, this);
kernel/sched/fair.c +9 -0
index a89c7c917cc6..81c8a50ab4c4 100644
@@ -10798,6 +10798,15 @@ static void se_fi_update(struct sched_entity *se, unsigned int fi_seq, bool forc
 			cfs_rq->forceidle_seq = fi_seq;
 		}
 
+
+		if (root) {
+			old = cfs_rq->min_vruntime_fi;
+			new = cfs_rq->min_vruntime;
+			root = false;
+			trace_printk("cfs_rq(min_vruntime_fi) %lu->%lu\n",
+				     old, new);
+		}
+
 		cfs_rq->min_vruntime_fi = cfs_rq->min_vruntime;
 	}
 }
Signature · 1 line
2.29.2.299.gdc1121823c-goog
#146 lore raw

Re: [PATCH -tip 32/32] sched: Debug bits...

Quoted 7 lines · Balbir Singh
Quoted 3 lines · Joel Fernandes (Google)

---

May be put it under a #ifdef CONFIG_SCHED_CORE_DEBUG, even then please make it more driven by selection via tracing rather than just trace_printk()

This particular patch is only for debug and is not for merging.

Peter is preparing a tree with some patches already applied, once that's done we will send a new series with the remaining patches (mostly interface and docs left).

thanks,

- Joel

maintainer
#147 lore raw
In reply to Joel Fernandes (Google)

Hi Joel,

Quoted 6 lines · Joel Fernandes (Google)

Core-Scheduling
===============
Enclosed is series v9 of core scheduling.
v9 is rebased on tip/master (fe4adf6f92c4 ("Merge branch 'irq/core'")).. I hope that this version is acceptable to be merged (pending any new review

./scripts/get_maintainer.pl is quite useful to make sure that all maintainers are cced and helps a lot to get some reviews

Quoted 255 lines

comments that arise) as the main issues in the past are all resolved:
1. Vruntime comparison.
2. Documentation updates.
3. CGroup and per-task interface developed by Google and Oracle.
4. Hotplug fixes.
Almost all patches also have Reviewed-by or Acked-by tag. See below for full list of changes in v9.

Introduction of feature
=======================
Core scheduling is a feature that allows only trusted tasks to run concurrently on cpus sharing compute resources (eg: hyperthreads on a core). The goal is to mitigate the core-level side-channel attacks without requiring to disable SMT (which has a significant impact on performance in some situations). Core scheduling (as of v7) mitigates user-space to user-space attacks and user to kernel attack when one of the siblings enters the kernel via interrupts or system call.

By default, the feature doesn't change any of the current scheduler behavior. The user decides which tasks can run simultaneously on the same core (for now by having them in the same tagged cgroup). When a tag is enabled in a cgroup and a task from that cgroup is running on a hardware thread, the scheduler ensures that only idle or trusted tasks run on the other sibling(s). Besides security concerns, this feature can also be beneficial for RT and performance applications where we want to control how tasks make use of SMT dynamically.

Both a CGroup and Per-task interface via prctl(2) are provided for configuring core sharing. More details are provided in documentation patch. Kselftests are provided to verify the correctness/rules of the interface.

Testing
=======
ChromeOS testing shows 300% improvement in keypress latency on a Google docs key press with Google hangout test (the maximum latency drops from 150ms to 50ms for keypresses).

Julien: TPCC tests showed improvements with core-scheduling as below. With kernel protection enabled, it does not show any regression. Possibly ASI will improve the performance for those who choose kernel protection (can be toggled through sched_core_protect_kernel sysctl).

                                average         stdev           diff
baseline (SMT on)               1197.272        44.78312824
core sched (   kernel protect)  412.9895        45.42734343     -65.51%
core sched (no kernel protect)  686.6515        71.77756931     -42.65%
nosmt                           408.667         39.39042872     -65.87%

(Note these results are from v8).

Vineeth tested sysbench and does not see any regressions.
Hong and Aubrey tested v9 and see results similar to v8. There is a known issue with uperf that does regress. This appears to be because of ksoftirq heavily contending with other tasks on the core. The consensus is this can be improved in the future.

Changes in v9
=============
- Note that the vruntime snapshot change is written in 2 patches to show the progression of the idea and prevent merge conflicts:
sched/fair: Snapshot the min_vruntime of CPUs on force idle
sched: Improve snapshotting of min_vruntime for CGroups
Same with the RT priority inversion change:
sched: Fix priority inversion of cookied task with sibling
sched: Improve snapshotting of min_vruntime for CGroups
- Disable coresched on certain AMD HW.

Changes in v8
=============
- New interface/API implementation
- Joel
- Revised kernel protection patch
- Joel
- Revised Hotplug fixes
- Joel
- Minor bug fixes and address review comments
- Vineeth

Changes in v7
=============
- Kernel protection from untrusted usermode tasks
- Joel, Vineeth
- Fix for hotplug crashes and hangs
- Joel, Vineeth

Changes in v6
=============
- Documentation
- Joel
- Pause siblings on entering nmi/irq/softirq
- Joel, Vineeth
- Fix for RCU crash
- Joel
- Fix for a crash in pick_next_task
- Yu Chen, Vineeth
- Minor re-write of core-wide vruntime comparison
- Aaron Lu
- Cleanup: Address Review comments
- Cleanup: Remove hotplug support (for now)
- Build fixes: 32 bit, SMT=n, AUTOGROUP=n etc
- Joel, Vineeth

Changes in v5
=============
- Fixes for cgroup/process tagging during corner cases like cgroup destroy, task moving across cgroups etc
- Tim Chen
- Coresched aware task migrations
- Aubrey Li
- Other minor stability fixes.

Changes in v4
=============
- Implement a core wide min_vruntime for vruntime comparison of tasks across cpus in a core.
- Aaron Lu
- Fixes a typo bug in setting the forced_idle cpu.
- Aaron Lu

Changes in v3
=============
- Fixes the issue of sibling picking up an incompatible task
- Aaron Lu
- Vineeth Pillai
- Julien Desfossez
- Fixes the issue of starving threads due to forced idle
- Peter Zijlstra
- Fixes the refcounting issue when deleting a cgroup with tag
- Julien Desfossez
- Fixes a crash during cpu offline/online with coresched enabled
- Vineeth Pillai
- Fixes a comparison logic issue in sched_core_find
- Aaron Lu

Changes in v2
=============
- Fixes for couple of NULL pointer dereference crashes
- Subhra Mazumdar
- Tim Chen
- Improves priority comparison logic for process in different cpus
- Peter Zijlstra
- Aaron Lu
- Fixes a hard lockup in rq locking
- Vineeth Pillai
- Julien Desfossez
- Fixes a performance issue seen on IO heavy workloads
- Vineeth Pillai
- Julien Desfossez
- Fix for 32bit build
- Aubrey Li

Future work
===========
- Load balancing/Migration fixes for core scheduling. With v6, Load balancing is partially coresched aware, but has some issues w.r.t process/taskgroup weights:
https://lwn.net/ml/linux-kernel/20200225034438.GA617271@z...

Aubrey Li (1):
sched: migration changes for core scheduling

Joel Fernandes (Google) (16):
sched/fair: Snapshot the min_vruntime of CPUs on force idle
sched: Enqueue task into core queue only after vruntime is updated
sched: Simplify the core pick loop for optimized case
sched: Improve snapshotting of min_vruntime for CGroups arch/x86: Add a new TIF flag for untrusted tasks kernel/entry: Add support for core-wide protection of kernel-mode entry/idle: Enter and exit kernel protection during idle entry and exit
sched: Split the cookie and setup per-task cookie on fork
sched: Add a per-thread core scheduling interface
sched: Release references to the per-task cookie on exit sched/debug: Add CGroup node for printing group cookie if SCHED_DEBUG
kselftest: Add tests for core-sched interface
sched: Move core-scheduler interfacing code to a new file
Documentation: Add core scheduling documentation
sched: Add a coresched command line option
sched: Debug bits...

Josh Don (2):
sched: Refactor core cookie into struct
sched: Add a second-level tag for nested CGroup usecase

Peter Zijlstra (11):
sched: Wrap rq::lock access
sched: Introduce sched_class::pick_task() sched/fair: Fix pick_task_fair crashes due to empty rbtree
sched: Core-wide rq->lock
sched/fair: Add a few assertions
sched: Basic tracking of matching tasks
sched: Add core wide task selection and scheduling.
sched: Fix priority inversion of cookied task with sibling
sched: Trivial forced-newidle balancer
irq_work: Cleanup
sched: CGroup tagging interface for core scheduling

Vineeth Pillai (2):
sched/fair: Fix forced idle sibling starvation corner case entry/kvm: Protect the kernel when entering from guest

.../admin-guide/hw-vuln/core-scheduling.rst   |  330 +++++
Documentation/admin-guide/hw-vuln/index.rst   |    1 +
.../admin-guide/kernel-parameters.txt         |   25 +
arch/x86/include/asm/thread_info.h            |    2 +
arch/x86/kernel/cpu/bugs.c                    |   19 +
arch/x86/kvm/x86.c                            |    2 +
drivers/gpu/drm/i915/i915_request.c           |    4 +-
include/linux/cpu.h                           |    1 +
include/linux/entry-common.h                  |   30 +-
include/linux/entry-kvm.h                     |   12 +
include/linux/irq_work.h                      |   33 +-
include/linux/irqflags.h                      |    4 +-
include/linux/sched.h                         |   28 +-
include/linux/sched/smt.h                     |    4 +
include/uapi/linux/prctl.h                    |    3 +
kernel/Kconfig.preempt                        |    5 +
kernel/bpf/stackmap.c                         |    2 +-
kernel/cpu.c                                  |   43 +
kernel/entry/common.c                         |   28 +-
kernel/entry/kvm.c                            |   33 +
kernel/fork.c                                 |    1 +
kernel/irq_work.c                             |   18 +-
kernel/printk/printk.c                        |    6 +-
kernel/rcu/tree.c                             |    3 +-
kernel/sched/Makefile                         |    1 +
kernel/sched/core.c                           | 1278 +++++++++++++++--
kernel/sched/coretag.c                        |  819 +++++++++++
kernel/sched/cpuacct.c                        |   12 +-
kernel/sched/deadline.c                       |   38 +-
kernel/sched/debug.c                          |   12 +-
kernel/sched/fair.c                           |  313 +++-
kernel/sched/idle.c                           |   24 +-
kernel/sched/pelt.h                           |    2 +-
kernel/sched/rt.c                             |   31 +-
kernel/sched/sched.h                          |  315 +++-
kernel/sched/stop_task.c                      |   14 +-
kernel/sched/topology.c                       |    4 +-
kernel/sys.c                                  |    3 +
kernel/time/tick-sched.c                      |    6 +-
kernel/trace/bpf_trace.c                      |    2 +-
tools/include/uapi/linux/prctl.h              |    3 +
tools/testing/selftests/sched/.gitignore      |    1 +
tools/testing/selftests/sched/Makefile        |   14 +
tools/testing/selftests/sched/config          |    1 +
.../testing/selftests/sched/test_coresched.c  |  818 +++++++++++
45 files changed, 4033 insertions(+), 315 deletions(-)

create mode 100644 Documentation/admin-guide/hw-vuln/core-scheduling.rst create mode 100644 kernel/sched/coretag.c
create mode 100644 tools/testing/selftests/sched/.gitignore create mode 100644 tools/testing/selftests/sched/Makefile create mode 100644 tools/testing/selftests/sched/config create mode 100644 tools/testing/selftests/sched/test_coresched.c

--
2.29.2.299.gdc1121823c-goog

#148 lore raw
Quoted 6 lines · Vincent Guittot

Hi Joel,

Quoted 1 line · Joel Fernandes (Google)
Quoted 8 lines
Quoted 5 lines

Core-Scheduling
===============
Enclosed is series v9 of core scheduling.
v9 is rebased on tip/master (fe4adf6f92c4 ("Merge branch 'irq/core'")).. I hope that this version is acceptable to be merged (pending any new review

./scripts/get_maintainer.pl is quite useful to make sure that all maintainers are cced and helps a lot to get some reviews

Apologies. I was just going by folks who were CC'd on previous series. I am new to doing this series's postings. Sorry if I missed you and will run get_maintainers henceforth!

- Joel

Quoted 255 lines
Quoted 255 lines

comments that arise) as the main issues in the past are all resolved:
1. Vruntime comparison.
2. Documentation updates.
3. CGroup and per-task interface developed by Google and Oracle.
4. Hotplug fixes.
Almost all patches also have Reviewed-by or Acked-by tag. See below for full list of changes in v9.

Introduction of feature
=======================
Core scheduling is a feature that allows only trusted tasks to run concurrently on cpus sharing compute resources (eg: hyperthreads on a core). The goal is to mitigate the core-level side-channel attacks without requiring to disable SMT (which has a significant impact on performance in some situations). Core scheduling (as of v7) mitigates user-space to user-space attacks and user to kernel attack when one of the siblings enters the kernel via interrupts or system call.

By default, the feature doesn't change any of the current scheduler behavior. The user decides which tasks can run simultaneously on the same core (for now by having them in the same tagged cgroup). When a tag is enabled in a cgroup and a task from that cgroup is running on a hardware thread, the scheduler ensures that only idle or trusted tasks run on the other sibling(s). Besides security concerns, this feature can also be beneficial for RT and performance applications where we want to control how tasks make use of SMT dynamically.

Both a CGroup and Per-task interface via prctl(2) are provided for configuring core sharing. More details are provided in documentation patch. Kselftests are provided to verify the correctness/rules of the interface.

Testing
=======
ChromeOS testing shows 300% improvement in keypress latency on a Google docs key press with Google hangout test (the maximum latency drops from 150ms to 50ms for keypresses).

Julien: TPCC tests showed improvements with core-scheduling as below. With kernel protection enabled, it does not show any regression. Possibly ASI will improve the performance for those who choose kernel protection (can be toggled through sched_core_protect_kernel sysctl).

                                average         stdev           diff
baseline (SMT on)               1197.272        44.78312824
core sched (   kernel protect)  412.9895        45.42734343     -65.51%
core sched (no kernel protect)  686.6515        71.77756931     -42.65%
nosmt                           408.667         39.39042872     -65.87%

(Note these results are from v8).

Vineeth tested sysbench and does not see any regressions.
Hong and Aubrey tested v9 and see results similar to v8. There is a known issue with uperf that does regress. This appears to be because of ksoftirq heavily contending with other tasks on the core. The consensus is this can be improved in the future.

Changes in v9
=============
- Note that the vruntime snapshot change is written in 2 patches to show the progression of the idea and prevent merge conflicts:
sched/fair: Snapshot the min_vruntime of CPUs on force idle
sched: Improve snapshotting of min_vruntime for CGroups
Same with the RT priority inversion change:
sched: Fix priority inversion of cookied task with sibling
sched: Improve snapshotting of min_vruntime for CGroups
- Disable coresched on certain AMD HW.

Changes in v8
=============
- New interface/API implementation
- Joel
- Revised kernel protection patch
- Joel
- Revised Hotplug fixes
- Joel
- Minor bug fixes and address review comments
- Vineeth

Changes in v7
=============
- Kernel protection from untrusted usermode tasks
- Joel, Vineeth
- Fix for hotplug crashes and hangs
- Joel, Vineeth

Changes in v6
=============
- Documentation
- Joel
- Pause siblings on entering nmi/irq/softirq
- Joel, Vineeth
- Fix for RCU crash
- Joel
- Fix for a crash in pick_next_task
- Yu Chen, Vineeth
- Minor re-write of core-wide vruntime comparison
- Aaron Lu
- Cleanup: Address Review comments
- Cleanup: Remove hotplug support (for now)
- Build fixes: 32 bit, SMT=n, AUTOGROUP=n etc
- Joel, Vineeth

Changes in v5
=============
- Fixes for cgroup/process tagging during corner cases like cgroup destroy, task moving across cgroups etc
- Tim Chen
- Coresched aware task migrations
- Aubrey Li
- Other minor stability fixes.

Changes in v4
=============
- Implement a core wide min_vruntime for vruntime comparison of tasks across cpus in a core.
- Aaron Lu
- Fixes a typo bug in setting the forced_idle cpu.
- Aaron Lu

Changes in v3
=============
- Fixes the issue of sibling picking up an incompatible task
- Aaron Lu
- Vineeth Pillai
- Julien Desfossez
- Fixes the issue of starving threads due to forced idle
- Peter Zijlstra
- Fixes the refcounting issue when deleting a cgroup with tag
- Julien Desfossez
- Fixes a crash during cpu offline/online with coresched enabled
- Vineeth Pillai
- Fixes a comparison logic issue in sched_core_find
- Aaron Lu

Changes in v2
=============
- Fixes for couple of NULL pointer dereference crashes
- Subhra Mazumdar
- Tim Chen
- Improves priority comparison logic for process in different cpus
- Peter Zijlstra
- Aaron Lu
- Fixes a hard lockup in rq locking
- Vineeth Pillai
- Julien Desfossez
- Fixes a performance issue seen on IO heavy workloads
- Vineeth Pillai
- Julien Desfossez
- Fix for 32bit build
- Aubrey Li

Future work
===========
- Load balancing/Migration fixes for core scheduling. With v6, Load balancing is partially coresched aware, but has some issues w.r.t process/taskgroup weights:
https://lwn.net/ml/linux-kernel/20200225034438.GA617271@z...

Aubrey Li (1):
sched: migration changes for core scheduling

Joel Fernandes (Google) (16):
sched/fair: Snapshot the min_vruntime of CPUs on force idle
sched: Enqueue task into core queue only after vruntime is updated
sched: Simplify the core pick loop for optimized case
sched: Improve snapshotting of min_vruntime for CGroups arch/x86: Add a new TIF flag for untrusted tasks kernel/entry: Add support for core-wide protection of kernel-mode entry/idle: Enter and exit kernel protection during idle entry and exit
sched: Split the cookie and setup per-task cookie on fork
sched: Add a per-thread core scheduling interface
sched: Release references to the per-task cookie on exit sched/debug: Add CGroup node for printing group cookie if SCHED_DEBUG
kselftest: Add tests for core-sched interface
sched: Move core-scheduler interfacing code to a new file
Documentation: Add core scheduling documentation
sched: Add a coresched command line option
sched: Debug bits...

Josh Don (2):
sched: Refactor core cookie into struct
sched: Add a second-level tag for nested CGroup usecase

Peter Zijlstra (11):
sched: Wrap rq::lock access
sched: Introduce sched_class::pick_task() sched/fair: Fix pick_task_fair crashes due to empty rbtree
sched: Core-wide rq->lock
sched/fair: Add a few assertions
sched: Basic tracking of matching tasks
sched: Add core wide task selection and scheduling.
sched: Fix priority inversion of cookied task with sibling
sched: Trivial forced-newidle balancer
irq_work: Cleanup
sched: CGroup tagging interface for core scheduling

Vineeth Pillai (2):
sched/fair: Fix forced idle sibling starvation corner case entry/kvm: Protect the kernel when entering from guest

.../admin-guide/hw-vuln/core-scheduling.rst   |  330 +++++
Documentation/admin-guide/hw-vuln/index.rst   |    1 +
.../admin-guide/kernel-parameters.txt         |   25 +
arch/x86/include/asm/thread_info.h            |    2 +
arch/x86/kernel/cpu/bugs.c                    |   19 +
arch/x86/kvm/x86.c                            |    2 +
drivers/gpu/drm/i915/i915_request.c           |    4 +-
include/linux/cpu.h                           |    1 +
include/linux/entry-common.h                  |   30 +-
include/linux/entry-kvm.h                     |   12 +
include/linux/irq_work.h                      |   33 +-
include/linux/irqflags.h                      |    4 +-
include/linux/sched.h                         |   28 +-
include/linux/sched/smt.h                     |    4 +
include/uapi/linux/prctl.h                    |    3 +
kernel/Kconfig.preempt                        |    5 +
kernel/bpf/stackmap.c                         |    2 +-
kernel/cpu.c                                  |   43 +
kernel/entry/common.c                         |   28 +-
kernel/entry/kvm.c                            |   33 +
kernel/fork.c                                 |    1 +
kernel/irq_work.c                             |   18 +-
kernel/printk/printk.c                        |    6 +-
kernel/rcu/tree.c                             |    3 +-
kernel/sched/Makefile                         |    1 +
kernel/sched/core.c                           | 1278 +++++++++++++++--
kernel/sched/coretag.c                        |  819 +++++++++++
kernel/sched/cpuacct.c                        |   12 +-
kernel/sched/deadline.c                       |   38 +-
kernel/sched/debug.c                          |   12 +-
kernel/sched/fair.c                           |  313 +++-
kernel/sched/idle.c                           |   24 +-
kernel/sched/pelt.h                           |    2 +-
kernel/sched/rt.c                             |   31 +-
kernel/sched/sched.h                          |  315 +++-
kernel/sched/stop_task.c                      |   14 +-
kernel/sched/topology.c                       |    4 +-
kernel/sys.c                                  |    3 +
kernel/time/tick-sched.c                      |    6 +-
kernel/trace/bpf_trace.c                      |    2 +-
tools/include/uapi/linux/prctl.h              |    3 +
tools/testing/selftests/sched/.gitignore      |    1 +
tools/testing/selftests/sched/Makefile        |   14 +
tools/testing/selftests/sched/config          |    1 +
.../testing/selftests/sched/test_coresched.c  |  818 +++++++++++
45 files changed, 4033 insertions(+), 315 deletions(-)

create mode 100644 Documentation/admin-guide/hw-vuln/core-scheduling.rst create mode 100644 kernel/sched/coretag.c
create mode 100644 tools/testing/selftests/sched/.gitignore create mode 100644 tools/testing/selftests/sched/Makefile create mode 100644 tools/testing/selftests/sched/config create mode 100644 tools/testing/selftests/sched/test_coresched.c

--
2.29.2.299.gdc1121823c-goog

#149 lore raw
Quoted 60 lines · Joel Fernandes
Quoted 60 lines
Quoted 60 lines

Core-Scheduling
===============
Enclosed is series v9 of core scheduling.
v9 is rebased on tip/master (fe4adf6f92c4 ("Merge branch 'irq/core'")).. I hope that this version is acceptable to be merged (pending any new review comments that arise) as the main issues in the past are all resolved:
1. Vruntime comparison.
2. Documentation updates.
3. CGroup and per-task interface developed by Google and Oracle.
4. Hotplug fixes.
Almost all patches also have Reviewed-by or Acked-by tag. See below for full list of changes in v9.

Introduction of feature
=======================
Core scheduling is a feature that allows only trusted tasks to run concurrently on cpus sharing compute resources (eg: hyperthreads on a core). The goal is to mitigate the core-level side-channel attacks without requiring to disable SMT (which has a significant impact on performance in some situations). Core scheduling (as of v7) mitigates user-space to user-space attacks and user to kernel attack when one of the siblings enters the kernel via interrupts or system call.

By default, the feature doesn't change any of the current scheduler behavior. The user decides which tasks can run simultaneously on the same core (for now by having them in the same tagged cgroup). When a tag is enabled in a cgroup and a task from that cgroup is running on a hardware thread, the scheduler ensures that only idle or trusted tasks run on the other sibling(s). Besides security concerns, this feature can also be beneficial for RT and performance applications where we want to control how tasks make use of SMT dynamically.

Both a CGroup and Per-task interface via prctl(2) are provided for configuring core sharing. More details are provided in documentation patch. Kselftests are provided to verify the correctness/rules of the interface.

Testing
=======
ChromeOS testing shows 300% improvement in keypress latency on a Google docs key press with Google hangout test (the maximum latency drops from 150ms to 50ms for keypresses).

Julien: TPCC tests showed improvements with core-scheduling as below. With kernel protection enabled, it does not show any regression. Possibly ASI will improve the performance for those who choose kernel protection (can be toggled through sched_core_protect_kernel sysctl).

                                average         stdev           diff
baseline (SMT on)               1197.272        44.78312824
core sched (   kernel protect)  412.9895        45.42734343     -65.51%
core sched (no kernel protect)  686.6515        71.77756931     -42.65%
nosmt                           408.667         39.39042872     -65.87%

(Note these results are from v8).

Vineeth tested sysbench and does not see any regressions.
Hong and Aubrey tested v9 and see results similar to v8. There is a known issue with uperf that does regress. This appears to be because of ksoftirq heavily contending with other tasks on the core. The consensus is this can be improved in the future.

Quoted 11 lines
Quoted 11 lines
Quoted 11 lines

Changes in v9
=============
- Note that the vruntime snapshot change is written in 2 patches to show the progression of the idea and prevent merge conflicts:
sched/fair: Snapshot the min_vruntime of CPUs on force idle
sched: Improve snapshotting of min_vruntime for CGroups
Same with the RT priority inversion change:
sched: Fix priority inversion of cookied task with sibling
sched: Improve snapshotting of min_vruntime for CGroups
- Disable coresched on certain AMD HW.

Adding workloads and negative case test results for core scheduling v9 posted:

- kernel under test:
-- coresched community v9 posted from https://git.kernel.org/pub/scm/linux/kernel/git/jfern/linux.git/log/?h=sched/coresched-v9-posted (tag: sched/coresched-v9-posted) -- latest commit: d48636e429de (HEAD -> coresched-v9-posted, tag: sched/coresched-v9-posted) sched: Debug bits...
-- coresched=on kernel parameter applied
- workloads:
-- A. sysbench cpu (192 threads) + sysbench cpu (192 threads)
-- B. sysbench cpu (192 threads) + sysbench mysql (192 threads, mysqld forced into the same cgroup)
-- C. uperf netperf.xml (192 threads over TCP or UDP protocol separately)
-- D. will-it-scale context_switch via pipe (192 threads)
- negative case:
-- A. continuously toggle cpu.core_tag, during full loading uperf workload running with cs_on
-- B. continuously toggle smt setting via /sys/devices/system/cpu/smt/control, during full loading uperf workload running with cs_on
-- C. continuously cgroup switch between cs_on cgroup and cs_off cgroup via cgclassify, during full loading uperf workload running
- test machine setup:
CPU(s): 192
On-line CPU(s) list: 0-191
Thread(s) per core: 2
Core(s) per socket: 48
Socket(s): 2
NUMA node(s): 4
- test results of workloads, no obvious performance drop compared to community v8 build:
-- workload A:
+----------------------+------+----------------------+------------------------+
| workloads | ** | sysbench cpu * 192 | sysbench cpu * 192 |
+======================+======+======================+========================+
| cgroup | ** | cg_sysbench_cpu_0 | cg_sysbench_cpu_1 |
+----------------------+------+----------------------+------------------------+
| record_item | ** | Tput_avg (events/s) | Tput_avg (events/s) |
+----------------------+------+----------------------+------------------------+
| coresched_normalized | ** | 0.97 | 1.02 |
+----------------------+------+----------------------+------------------------+
| default_normalized | ** | 1.00 | 1.00 |
+----------------------+------+----------------------+------------------------+
| smtoff_normalized | ** | 0.60 | 0.60 |
+----------------------+------+----------------------+------------------------+

-- workload B:
+----------------------+------+----------------------+------------------------+
| workloads | ** | sysbench cpu * 192 | sysbench mysql * 192 |
+======================+======+======================+========================+
| cgroup | ** | cg_sysbench_cpu_0 | cg_sysbench_mysql_0 |
+----------------------+------+----------------------+------------------------+
| record_item | ** | Tput_avg (events/s) | Tput_avg (events/s) |
+----------------------+------+----------------------+------------------------+
| coresched_normalized | ** | 0.94 | 0.88 |
+----------------------+------+----------------------+------------------------+
| default_normalized | ** | 1.00 | 1.00 |
+----------------------+------+----------------------+------------------------+
| smtoff_normalized | ** | 0.56 | 0.84 |
+----------------------+------+----------------------+------------------------+

-- workload C:
+----------------------+------+---------------------------+---------------------------+
| workloads | ** | uperf netperf TCP * 192 | uperf netperf UDP * 192 |
+======================+======+===========================+===========================+
| cgroup | ** | cg_uperf | cg_uperf |
+----------------------+------+---------------------------+---------------------------+
| record_item | ** | Tput_avg (Gb/s) | Tput_avg (Gb/s) |
+----------------------+------+---------------------------+---------------------------+
| coresched_normalized | ** | 0.64 | 0.68 |
+----------------------+------+---------------------------+---------------------------+
| default_normalized | ** | 1.00 | 1.00 |
+----------------------+------+---------------------------+---------------------------+
| smtoff_normalized | ** | 0.92 | 0.89 |
+----------------------+------+---------------------------+---------------------------+

-- workload D:
+----------------------+------+-------------------------------+
| workloads | ** | will-it-scale * 192 |
| | | (pipe based context_switch) |
+======================+======+===============================+
| cgroup | ** | cg_will-it-scale |
+----------------------+------+-------------------------------+
| record_item | ** | threads_avg |
+----------------------+------+-------------------------------+
| coresched_normalized | ** | 0.30 |
+----------------------+------+-------------------------------+
| default_normalized | ** | 1.00 |
+----------------------+------+-------------------------------+
| smtoff_normalized | ** | 0.87 |
+----------------------+------+-------------------------------+

-- notes on record_item:
* coresched_normalized: smton, cs enabled, test result normalized by default value
* default_normalized: smton, cs disabled, test result normalized by default value
* smtoff_normalized: smtoff, test result normalized by default value

- test results of negative case, all as expected, no kernel panic or system hang observed

Hongyu