Skip to contentSkip to conversation

Core scheduling

PATCH Applied Joel Fernandes (Google) 10 participants 98 messages Oct 20, 2020 – Nov 16, 2020 1 message not in the archive Page 2 of 2

Subsystems (18)
lore
Share & tools
lore
#82 lore raw

PATCH v8 22/26 sched/debug: Add CGroup node for printing group cookie if SCHED_DEBUG

In reply to Joel Fernandes (Google)

This will be used by kselftest to verify the CGroup cookie value that is set by the CGroup interface.

---
 kernel/sched/core.c | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)
kernel/sched/core.c +23 -0
index 1321c26a8385..b3afbba5abe1 100644
@@ -9520,6 +9520,13 @@ static u64 cpu_core_tag_color_read_u64(struct cgroup_subsys_state *css, struct c
 	return tg->core_tag_color;
 }
 
+#ifdef CONFIG_SCHED_DEBUG
+static u64 cpu_core_group_cookie_read_u64(struct cgroup_subsys_state *css, struct cftype *cft)
+{
+	return cpu_core_get_group_cookie(css_tg(css));
+}
+#endif
+
 struct write_core_tag {
 	struct cgroup_subsys_state *css;
 	unsigned long cookie;
@@ -9695,6 +9702,14 @@ static struct cftype cpu_legacy_files[] = {
 		.read_u64 = cpu_core_tag_color_read_u64,
 		.write_u64 = cpu_core_tag_color_write_u64,
 	},
+#ifdef CONFIG_SCHED_DEBUG
+	/* Read the effective cookie (color+tag) of the group. */
+	{
+		.name = "core_group_cookie",
+		.flags = CFTYPE_NOT_ON_ROOT,
+		.read_u64 = cpu_core_group_cookie_read_u64,
+	},
+#endif
 #endif
 #ifdef CONFIG_UCLAMP_TASK_GROUP
 	{
@@ -9882,6 +9897,14 @@ static struct cftype cpu_files[] = {
 		.read_u64 = cpu_core_tag_color_read_u64,
 		.write_u64 = cpu_core_tag_color_write_u64,
 	},
+#ifdef CONFIG_SCHED_DEBUG
+	/* Read the effective cookie (color+tag) of the group. */
+	{
+		.name = "core_group_cookie",
+		.flags = CFTYPE_NOT_ON_ROOT,
+		.read_u64 = cpu_core_group_cookie_read_u64,
+	},
+#endif
 #endif
 #ifdef CONFIG_CFS_BANDWIDTH
 	{
Signature · 1 line
2.29.0.rc1.297.gfa9743e501-goog
#83 lore raw

PATCH v8 23/26 kselftest: Add tests for core-sched interface

In reply to Joel Fernandes (Google)
---
 tools/testing/selftests/sched/.gitignore      |   1 +
 tools/testing/selftests/sched/Makefile        |  14 +
 tools/testing/selftests/sched/config          |   1 +
 .../testing/selftests/sched/test_coresched.c  | 840 ++++++++++++++++++
 4 files changed, 856 insertions(+)
 create mode 100644 tools/testing/selftests/sched/.gitignore
 create mode 100644 tools/testing/selftests/sched/Makefile
 create mode 100644 tools/testing/selftests/sched/config
 create mode 100644 tools/testing/selftests/sched/test_coresched.c
tools/testing/selftests/sched/.gitignore new file +1 -0
new file mode 100644
index 000000000000..4660929b0b9a
@@ -0,0 +1 @@
+test_coresched
tools/testing/selftests/sched/Makefile new file +14 -0
new file mode 100644
index 000000000000..e43b74fc5d7e
@@ -0,0 +1,14 @@
+# SPDX-License-Identifier: GPL-2.0+
+
+ifneq ($(shell $(CC) --version 2>&1 | head -n 1 | grep clang),)
+CLANG_FLAGS += -no-integrated-as
+endif
+
+CFLAGS += -O2 -Wall -g -I./ -I../../../../usr/include/  -Wl,-rpath=./ \
+	  $(CLANG_FLAGS)
+LDLIBS += -lpthread
+
+TEST_GEN_FILES := test_coresched
+TEST_PROGS := test_coresched
+
+include ../lib.mk
tools/testing/selftests/sched/config new file +1 -0
new file mode 100644
index 000000000000..e8b09aa7c0c4
@@ -0,0 +1 @@
+CONFIG_SCHED_DEBUG=y
tools/testing/selftests/sched/test_coresched.c new file +840 -0

840 lines not on this page · raw message

Signature · 1 line
2.29.0.rc1.297.gfa9743e501-goog
#84 lore raw

PATCH 1/1 sched: Change all 4 space tabs to actual tabs

This patch solves over a hundred style warnings caused by 4 space tabs from patch 23 in the coresched-v8-posted-to-list patch set.

Please review. A script was written to convert all 4 spaces blocks to tabs and some manual editing was used to fix the any remaining alignment issues caused by the conversion to tabs.

Issues reported by checkpatch.

---
 .../testing/selftests/sched/test_coresched.c  | 998 +++++++++---------
 1 file changed, 499 insertions(+), 499 deletions(-)
tools/testing/selftests/sched/test_coresched.c +499 -499
index 2fdefb843115..44a33ef88435 100644
@@ -33,20 +33,20 @@
 
 void print_banner(char *s)
 {
-    printf("coresched: %s:  ", s);
+	printf("coresched: %s:  ", s);
 }
 
 void print_pass(void)
 {
-    printf("PASS\n");
+	printf("PASS\n");
 }
 
 void assert_cond(int cond, char *str)
 {
-    if (!cond) {
-	printf("Error: %s\n", str);
-	abort();
-    }
+	if (!cond) {
+		printf("Error: %s\n", str);
+		abort();
+	}
 }
 
 char *make_group_root(void)
@@ -56,8 +56,8 @@ char *make_group_root(void)
 
 	mntpath = malloc(50);
 	if (!mntpath) {
-	    perror("Failed to allocate mntpath\n");
-	    abort();
+		perror("Failed to allocate mntpath\n");
+		abort();
 	}
 
 	sprintf(mntpath, "/tmp/coresched-test-XXXXXX");
@@ -78,82 +78,82 @@ char *make_group_root(void)
 
 char *read_group_cookie(char *cgroup_path)
 {
-    char path[50] = {}, *val;
-    int fd;
+	char path[50] = {}, *val;
+	int fd;
 
-    sprintf(path, "%s/cpu.core_group_cookie", cgroup_path);
-    fd = open(path, O_RDONLY, 0666);
-    if (fd == -1) {
-	perror("Open of cgroup tag path failed: ");
-	abort();
-    }
+	sprintf(path, "%s/cpu.core_group_cookie", cgroup_path);
+	fd = open(path, O_RDONLY, 0666);
+	if (fd == -1) {
+		perror("Open of cgroup tag path failed: ");
+		abort();
+	}
 
-    val = calloc(1, 50);
-    if (read(fd, val, 50) == -1) {
-	perror("Failed to read group cookie: ");
-	abort();
-    }
+	val = calloc(1, 50);
+	if (read(fd, val, 50) == -1) {
+		perror("Failed to read group cookie: ");
+		abort();
+	}
 
-    val[strcspn(val, "\r\n")] = 0;
+	val[strcspn(val, "\r\n")] = 0;
 
-    close(fd);
-    return val;
+	close(fd);
+	return val;
 }
 
 void assert_group_tag(char *cgroup_path, char *tag)
 {
-    char tag_path[50] = {}, rdbuf[8] = {};
-    int tfd;
+	char tag_path[50] = {}, rdbuf[8] = {};
+	int tfd;
 
-    sprintf(tag_path, "%s/cpu.core_tag", cgroup_path);
-    tfd = open(tag_path, O_RDONLY, 0666);
-    if (tfd == -1) {
-	perror("Open of cgroup tag path failed: ");
-	abort();
-    }
+	sprintf(tag_path, "%s/cpu.core_tag", cgroup_path);
+	tfd = open(tag_path, O_RDONLY, 0666);
+	if (tfd == -1) {
+		perror("Open of cgroup tag path failed: ");
+		abort();
+	}
 
-    if (read(tfd, rdbuf, 1) != 1) {
-	perror("Failed to enable coresched on cgroup: ");
-	abort();
-    }
+	if (read(tfd, rdbuf, 1) != 1) {
+		perror("Failed to enable coresched on cgroup: ");
+		abort();
+	}
 
-    if (strcmp(rdbuf, tag)) {
-	printf("Group tag does not match (exp: %s, act: %s)\n", tag, rdbuf);
-	abort();
-    }
+	if (strcmp(rdbuf, tag)) {
+		printf("Group tag does not match (exp: %s, act: %s)\n", tag, rdbuf);
+		abort();
+	}
 
-    if (close(tfd) == -1) {
-	perror("Failed to close tag fd: ");
-	abort();
-    }
+	if (close(tfd) == -1) {
+		perror("Failed to close tag fd: ");
+		abort();
+	}
 }
 
 void assert_group_color(char *cgroup_path, const char *color)
 {
-    char tag_path[50] = {}, rdbuf[8] = {};
-    int tfd;
+	char tag_path[50] = {}, rdbuf[8] = {};
+	int tfd;
 
-    sprintf(tag_path, "%s/cpu.core_tag_color", cgroup_path);
-    tfd = open(tag_path, O_RDONLY, 0666);
-    if (tfd == -1) {
-	perror("Open of cgroup tag path failed: ");
-	abort();
-    }
+	sprintf(tag_path, "%s/cpu.core_tag_color", cgroup_path);
+	tfd = open(tag_path, O_RDONLY, 0666);
+	if (tfd == -1) {
+		perror("Open of cgroup tag path failed: ");
+		abort();
+	}
 
-    if (read(tfd, rdbuf, 8) == -1) {
-	perror("Failed to read group color\n");
-	abort();
-    }
+	if (read(tfd, rdbuf, 8) == -1) {
+		perror("Failed to read group color\n");
+		abort();
+	}
 
-    if (strncmp(color, rdbuf, strlen(color))) {
-	printf("Group color does not match (exp: %s, act: %s)\n", color, rdbuf);
-	abort();
-    }
+	if (strncmp(color, rdbuf, strlen(color))) {
+		printf("Group color does not match (exp: %s, act: %s)\n", color, rdbuf);
+		abort();
+	}
 
-    if (close(tfd) == -1) {
-	perror("Failed to close color fd: ");
-	abort();
-    }
+	if (close(tfd) == -1) {
+		perror("Failed to close color fd: ");
+		abort();
+	}
 }
 
 void color_group(char *cgroup_path, const char *color_str)
@@ -172,11 +172,11 @@ void color_group(char *cgroup_path, const char *color_str)
 
 	ret = write(tfd, color_str, strlen(color_str));
 	assert_cond(color < 256 || ret == -1,
-		    "Writing invalid range color should have failed!");
+			"Writing invalid range color should have failed!");
 
 	if (color < 1 || color > 255) {
-	    close(tfd);
-	    return;
+		close(tfd);
+		return;
 	}
 
 	if (ret == -1) {
@@ -248,12 +248,12 @@ char *make_group(char *parent, char *name)
 	int ret;
 
 	if (!parent && !name)
-	    return make_group_root();
+		return make_group_root();
 
 	cgroup_path = malloc(50);
 	if (!cgroup_path) {
-	    perror("Failed to allocate cgroup_path\n");
-	    abort();
+		perror("Failed to allocate cgroup_path\n");
+		abort();
 	}
 
 	/* Make the cgroup node for this group */
@@ -269,278 +269,278 @@ char *make_group(char *parent, char *name)
 
 static void del_group(char *path)
 {
-    if (rmdir(path) != 0) {
-	printf("Removal of group failed\n");
-	abort();
-    }
+	if (rmdir(path) != 0) {
+		printf("Removal of group failed\n");
+		abort();
+	}
 
-    free(path);
+	free(path);
 }
 
 static void del_root_group(char *path)
 {
-    if (umount(path) != 0) {
-	perror("umount of cgroup failed\n");
-	abort();
-    }
+	if (umount(path) != 0) {
+		perror("umount of cgroup failed\n");
+		abort();
+	}
 
-    if (rmdir(path) != 0) {
-	printf("Removal of group failed\n");
-	abort();
-    }
+	if (rmdir(path) != 0) {
+		printf("Removal of group failed\n");
+		abort();
+	}
 
-    free(path);
+	free(path);
 }
 
 void assert_group_cookie_equal(char *c1, char *c2)
 {
-    char *v1, *v2;
+	char *v1, *v2;
 
-    v1 = read_group_cookie(c1);
-    v2 = read_group_cookie(c2);
-    if (strcmp(v1, v2)) {
-	printf("Group cookies not equal\n");
-	abort();
-    }
+	v1 = read_group_cookie(c1);
+	v2 = read_group_cookie(c2);
+	if (strcmp(v1, v2)) {
+		printf("Group cookies not equal\n");
+		abort();
+	}
 
-    free(v1);
-    free(v2);
+	free(v1);
+	free(v2);
 }
 
 void assert_group_cookie_not_equal(char *c1, char *c2)
 {
-    char *v1, *v2;
+	char *v1, *v2;
 
-    v1 = read_group_cookie(c1);
-    v2 = read_group_cookie(c2);
-    if (!strcmp(v1, v2)) {
-	printf("Group cookies not equal\n");
-	abort();
-    }
+	v1 = read_group_cookie(c1);
+	v2 = read_group_cookie(c2);
+	if (!strcmp(v1, v2)) {
+		printf("Group cookies not equal\n");
+		abort();
+	}
 
-    free(v1);
-    free(v2);
+	free(v1);
+	free(v2);
 }
 
 void assert_group_cookie_not_zero(char *c1)
 {
-    char *v1 = read_group_cookie(c1);
+	char *v1 = read_group_cookie(c1);
 
-    v1[1] = 0;
-    if (!strcmp(v1, "0")) {
-	printf("Group cookie zero\n");
-	abort();
-    }
-    free(v1);
+	v1[1] = 0;
+	if (!strcmp(v1, "0")) {
+		printf("Group cookie zero\n");
+		abort();
+	}
+	free(v1);
 }
 
 void assert_group_cookie_zero(char *c1)
 {
-    char *v1 = read_group_cookie(c1);
+	char *v1 = read_group_cookie(c1);
 
-    v1[1] = 0;
-    if (strcmp(v1, "0")) {
-	printf("Group cookie not zero");
-	abort();
-    }
-    free(v1);
+	v1[1] = 0;
+	if (strcmp(v1, "0")) {
+		printf("Group cookie not zero");
+		abort();
+	}
+	free(v1);
 }
 
 struct task_state {
-    int pid_share;
-    char pid_str[50];
-    pthread_mutex_t m;
-    pthread_cond_t cond;
-    pthread_cond_t cond_par;
+	int pid_share;
+	char pid_str[50];
+	pthread_mutex_t m;
+	pthread_cond_t cond;
+	pthread_cond_t cond_par;
 };
 
 struct task_state *add_task(char *p)
 {
-    struct task_state *mem;
-    pthread_mutexattr_t am;
-    pthread_condattr_t a;
-    char tasks_path[50];
-    int tfd, pid, ret;
-
-    sprintf(tasks_path, "%s/tasks", p);
-    tfd = open(tasks_path, O_WRONLY, 0666);
-    if (tfd == -1) {
-	perror("Open of cgroup tasks path failed: ");
-	abort();
-    }
-
-    mem = mmap(NULL, sizeof *mem, PROT_READ | PROT_WRITE,
-	    MAP_SHARED | MAP_ANONYMOUS, -1, 0);
-    memset(mem, 0, sizeof(*mem));
-
-    pthread_condattr_init(&a);
-    pthread_condattr_setpshared(&a, PTHREAD_PROCESS_SHARED);
-    pthread_mutexattr_init(&am);
-    pthread_mutexattr_setpshared(&am, PTHREAD_PROCESS_SHARED);
-
-    pthread_cond_init(&mem->cond, &a);
-    pthread_cond_init(&mem->cond_par, &a);
-    pthread_mutex_init(&mem->m, &am);
-
-    pid = fork();
-    if (pid == 0) {
-	while(1) {
-	    pthread_mutex_lock(&mem->m);
-	    while(!mem->pid_share)
-		pthread_cond_wait(&mem->cond, &mem->m);
-
-	    pid = mem->pid_share;
-	    mem->pid_share = 0;
-	    if (pid == -1)
-		pid = 0;
-	    prctl(PR_SCHED_CORE_SHARE, pid);
-	    pthread_mutex_unlock(&mem->m);
-	    pthread_cond_signal(&mem->cond_par);
-	}
-    }
-
-    sprintf(mem->pid_str, "%d", pid);
-    dprint("add task %d to group %s", pid, p);
-
-    ret = write(tfd, mem->pid_str, strlen(mem->pid_str));
-    assert_cond(ret != -1,
-	    "Failed to write pid into tasks");
-
-    close(tfd);
-    return mem;
+	struct task_state *mem;
+	pthread_mutexattr_t am;
+	pthread_condattr_t a;
+	char tasks_path[50];
+	int tfd, pid, ret;
+
+	sprintf(tasks_path, "%s/tasks", p);
+	tfd = open(tasks_path, O_WRONLY, 0666);
+	if (tfd == -1) {
+		perror("Open of cgroup tasks path failed: ");
+		abort();
+	}
+
+	mem = mmap(NULL, sizeof *mem, PROT_READ | PROT_WRITE,
+		MAP_SHARED | MAP_ANONYMOUS, -1, 0);
+	memset(mem, 0, sizeof(*mem));
+
+	pthread_condattr_init(&a);
+	pthread_condattr_setpshared(&a, PTHREAD_PROCESS_SHARED);
+	pthread_mutexattr_init(&am);
+	pthread_mutexattr_setpshared(&am, PTHREAD_PROCESS_SHARED);
+
+	pthread_cond_init(&mem->cond, &a);
+	pthread_cond_init(&mem->cond_par, &a);
+	pthread_mutex_init(&mem->m, &am);
+
+	pid = fork();
+	if (pid == 0) {
+		while(1) {
+			pthread_mutex_lock(&mem->m);
+			while(!mem->pid_share)
+				pthread_cond_wait(&mem->cond, &mem->m);
+
+			pid = mem->pid_share;
+			mem->pid_share = 0;
+			if (pid == -1)
+				pid = 0;
+			prctl(PR_SCHED_CORE_SHARE, pid);
+			pthread_mutex_unlock(&mem->m);
+			pthread_cond_signal(&mem->cond_par);
+		}
+	}
+
+	sprintf(mem->pid_str, "%d", pid);
+	dprint("add task %d to group %s", pid, p);
+
+	ret = write(tfd, mem->pid_str, strlen(mem->pid_str));
+	assert_cond(ret != -1,
+		"Failed to write pid into tasks");
+
+	close(tfd);
+	return mem;
 }
 
 /* Make t1 share with t2 */
 void make_tasks_share(struct task_state *t1, struct task_state *t2)
 {
-    int p2 = atoi(t2->pid_str);
-    dprint("task %s %s", t1->pid_str, t2->pid_str);
+	int p2 = atoi(t2->pid_str);
+	dprint("task %s %s", t1->pid_str, t2->pid_str);
 
-    pthread_mutex_lock(&t1->m);
-    t1->pid_share = p2;
-    pthread_mutex_unlock(&t1->m);
+	pthread_mutex_lock(&t1->m);
+	t1->pid_share = p2;
+	pthread_mutex_unlock(&t1->m);
 
-    pthread_cond_signal(&t1->cond);
+	pthread_cond_signal(&t1->cond);
 
-    pthread_mutex_lock(&t1->m);
-    while (t1->pid_share)
-	pthread_cond_wait(&t1->cond_par, &t1->m);
-    pthread_mutex_unlock(&t1->m);
+	pthread_mutex_lock(&t1->m);
+	while (t1->pid_share)
+		pthread_cond_wait(&t1->cond_par, &t1->m);
+	pthread_mutex_unlock(&t1->m);
 }
 
 /* Make t1 share with t2 */
 void reset_task_cookie(struct task_state *t1)
 {
-    dprint("task %s", t1->pid_str);
+	dprint("task %s", t1->pid_str);
 
-    pthread_mutex_lock(&t1->m);
-    t1->pid_share = -1;
-    pthread_mutex_unlock(&t1->m);
+	pthread_mutex_lock(&t1->m);
+	t1->pid_share = -1;
+	pthread_mutex_unlock(&t1->m);
 
-    pthread_cond_signal(&t1->cond);
+	pthread_cond_signal(&t1->cond);
 
-    pthread_mutex_lock(&t1->m);
-    while (t1->pid_share)
-	pthread_cond_wait(&t1->cond_par, &t1->m);
-    pthread_mutex_unlock(&t1->m);
+	pthread_mutex_lock(&t1->m);
+	while (t1->pid_share)
+		pthread_cond_wait(&t1->cond_par, &t1->m);
+	pthread_mutex_unlock(&t1->m);
 }
 
 char *get_task_core_cookie(char *pid)
 {
-    char proc_path[50];
-    int found = 0;
-    char *line;
-    int i, j;
-    FILE *fp;
-
-    line = malloc(1024);
-    assert_cond(!!line, "Failed to alloc memory");
-
-    sprintf(proc_path, "/proc/%s/sched", pid);
-
-    fp = fopen(proc_path, "r");
-    while ((fgets(line, 1024, fp)) != NULL)
-    {
-        if(!strstr(line, "core_cookie"))
-            continue;
-
-        for (j = 0, i = 0; i < 1024 && line[i] != '\0'; i++)
-            if (line[i] >= '0' && line[i] <= '9')
-                line[j++] = line[i];
-        line[j] = '\0';
-        found = 1;
-        break;
-    }
-
-    fclose(fp);
-
-    if (found) {
-        return line;
-    } else {
-        free(line);
+	char proc_path[50];
+	int found = 0;
+	char *line;
+	int i, j;
+	FILE *fp;
+
+	line = malloc(1024);
+	assert_cond(!!line, "Failed to alloc memory");
+
+	sprintf(proc_path, "/proc/%s/sched", pid);
+
+	fp = fopen(proc_path, "r");
+	while ((fgets(line, 1024, fp)) != NULL)
+	{
+		if(!strstr(line, "core_cookie"))
+			continue;
+
+		for (j = 0, i = 0; i < 1024 && line[i] != '\0'; i++)
+			if (line[i] >= '0' && line[i] <= '9')
+				line[j++] = line[i];
+		line[j] = '\0';
+		found = 1;
+		break;
+	}
+
+	fclose(fp);
+
+	if (found) {
+		return line;
+	} else {
+		free(line);
 	printf("core_cookie not found. Enable SCHED_DEBUG?\n");
 	abort();
-        return NULL;
-    }
+		return NULL;
+	}
 }
 
 void assert_tasks_share(struct task_state *t1, struct task_state *t2)
 {
-    char *c1, *c2;
-
-    c1 = get_task_core_cookie(t1->pid_str);
-    c2 = get_task_core_cookie(t2->pid_str);
-    dprint("check task (%s) cookie (%s) == task (%s) cookie (%s)",
-	    t1->pid_str, c1, t2->pid_str, c2);
-    assert_cond(!strcmp(c1, c2), "Tasks don't share cookie");
-    free(c1); free(c2);
+	char *c1, *c2;
+
+	c1 = get_task_core_cookie(t1->pid_str);
+	c2 = get_task_core_cookie(t2->pid_str);
+	dprint("check task (%s) cookie (%s) == task (%s) cookie (%s)",
+		t1->pid_str, c1, t2->pid_str, c2);
+	assert_cond(!strcmp(c1, c2), "Tasks don't share cookie");
+	free(c1); free(c2);
 }
 
 void assert_tasks_dont_share(struct task_state *t1,  struct task_state *t2)
 {
-    char *c1, *c2;
-    c1 = get_task_core_cookie(t1->pid_str);
-    c2 = get_task_core_cookie(t2->pid_str);
-    dprint("check task (%s) cookie (%s) != task (%s) cookie (%s)",
-	    t1->pid_str, c1, t2->pid_str, c2);
-    assert_cond(strcmp(c1, c2), "Tasks don't share cookie");
-    free(c1); free(c2);
+	char *c1, *c2;
+	c1 = get_task_core_cookie(t1->pid_str);
+	c2 = get_task_core_cookie(t2->pid_str);
+	dprint("check task (%s) cookie (%s) != task (%s) cookie (%s)",
+		t1->pid_str, c1, t2->pid_str, c2);
+	assert_cond(strcmp(c1, c2), "Tasks don't share cookie");
+	free(c1); free(c2);
 }
 
 void assert_group_cookie_equals_task_cookie(char *g, char *pid)
 {
-    char *gk;
-    char *tk;
+	char *gk;
+	char *tk;
 
-    gk = read_group_cookie(g);
-    tk = get_task_core_cookie(pid);
+	gk = read_group_cookie(g);
+	tk = get_task_core_cookie(pid);
 
-    assert_cond(!strcmp(gk, tk), "Group cookie not equal to tasks'");
+	assert_cond(!strcmp(gk, tk), "Group cookie not equal to tasks'");
 
-    free(gk);
-    free(tk);
+	free(gk);
+	free(tk);
 }
 
 void assert_group_cookie_not_equals_task_cookie(char *g, char *pid)
 {
-    char *gk;
-    char *tk;
+	char *gk;
+	char *tk;
 
-    gk = read_group_cookie(g);
-    tk = get_task_core_cookie(pid);
+	gk = read_group_cookie(g);
+	tk = get_task_core_cookie(pid);
 
-    assert_cond(strcmp(gk, tk), "Group cookie not equal to tasks'");
+	assert_cond(strcmp(gk, tk), "Group cookie not equal to tasks'");
 
-    free(gk);
-    free(tk);
+	free(gk);
+	free(tk);
 }
 
 void kill_task(struct task_state *t)
 {
-    int pid = atoi(t->pid_str);
+	int pid = atoi(t->pid_str);
 
-    kill(pid, SIGKILL);
-    waitpid(pid, NULL, 0);
+	kill(pid, SIGKILL);
+	waitpid(pid, NULL, 0);
 }
 
 /*
@@ -556,51 +556,51 @@ void kill_task(struct task_state *t)
  */
 static void test_cgroup_coloring(char *root)
 {
-    char *y1, *y2, *y22, *r1, *r11, *b3, *r4;
+	char *y1, *y2, *y22, *r1, *r11, *b3, *r4;
 
-    print_banner("TEST-CGROUP-COLORING");
+	print_banner("TEST-CGROUP-COLORING");
 
-    y1 = make_group(root, "y1");
-    tag_group(y1);
+	y1 = make_group(root, "y1");
+	tag_group(y1);
 
-    y2 = make_group(y1, "y2");
-    y22 = make_group(y2, "y22");
+	y2 = make_group(y1, "y2");
+	y22 = make_group(y2, "y22");
 
-    r1 = make_group(y1, "y1");
-    r11 = make_group(r1, "r11");
+	r1 = make_group(y1, "y1");
+	r11 = make_group(r1, "r11");
 
-    color_group(r1, "256"); /* Wouldn't succeed. */
-    color_group(r1, "0");   /* Wouldn't succeed. */
-    color_group(r1, "254");
+	color_group(r1, "256"); /* Wouldn't succeed. */
+	color_group(r1, "0");   /* Wouldn't succeed. */
+	color_group(r1, "254");
 
-    b3 = make_group(y1, "b3");
-    color_group(b3, "8");
+	b3 = make_group(y1, "b3");
+	color_group(b3, "8");
 
-    r4 = make_group(y1, "r4");
-    color_group(r4, "254");
+	r4 = make_group(y1, "r4");
+	color_group(r4, "254");
 
-    /* Check that all yellows share the same cookie. */
-    assert_group_cookie_not_zero(y1);
-    assert_group_cookie_equal(y1, y2);
-    assert_group_cookie_equal(y1, y22);
+	/* Check that all yellows share the same cookie. */
+	assert_group_cookie_not_zero(y1);
+	assert_group_cookie_equal(y1, y2);
+	assert_group_cookie_equal(y1, y22);
 
-    /* Check that all reds share the same cookie. */
-    assert_group_cookie_not_zero(r1);
-    assert_group_cookie_equal(r1, r11);
-    assert_group_cookie_equal(r11, r4);
+	/* Check that all reds share the same cookie. */
+	assert_group_cookie_not_zero(r1);
+	assert_group_cookie_equal(r1, r11);
+	assert_group_cookie_equal(r11, r4);
 
-    /* Check that blue, red and yellow are different cookie. */
-    assert_group_cookie_not_equal(r1, b3);
-    assert_group_cookie_not_equal(b3, y1);
+	/* Check that blue, red and yellow are different cookie. */
+	assert_group_cookie_not_equal(r1, b3);
+	assert_group_cookie_not_equal(b3, y1);
 
-    del_group(r11);
-    del_group(r1);
-    del_group(y22);
-    del_group(y2);
-    del_group(b3);
-    del_group(r4);
-    del_group(y1);
-    print_pass();
+	del_group(r11);
+	del_group(r1);
+	del_group(y22);
+	del_group(y2);
+	del_group(b3);
+	del_group(r4);
+	del_group(y1);
+	print_pass();
 }
 
 /*
@@ -608,68 +608,68 @@ static void test_cgroup_coloring(char *root)
  * from their parent group _after_ the parent was tagged.
  *
  *   p ----- c1 - c11
- *     \ c2 - c22
+ *	 \ c2 - c22
  */
 static void test_cgroup_parent_child_tag_inherit(char *root)
 {
-    char *p, *c1, *c11, *c2, *c22;
-
-    print_banner("TEST-CGROUP-PARENT-CHILD-TAG");
-
-    p = make_group(root, "p");
-    assert_group_cookie_zero(p);
-
-    c1 = make_group(p, "c1");
-    assert_group_tag(c1, "0"); /* Child tag is "0" but inherits cookie from parent. */
-    assert_group_cookie_zero(c1);
-    assert_group_cookie_equal(c1, p);
-
-    c11 = make_group(c1, "c11");
-    assert_group_tag(c11, "0");
-    assert_group_cookie_zero(c11);
-    assert_group_cookie_equal(c11, p);
-
-    c2 = make_group(p, "c2");
-    assert_group_tag(c2, "0");
-    assert_group_cookie_zero(c2);
-    assert_group_cookie_equal(c2, p);
-
-    tag_group(p);
-
-    /* Verify c1 got the cookie */
-    assert_group_tag(c1, "0");
-    assert_group_cookie_not_zero(c1);
-    assert_group_cookie_equal(c1, p);
-
-    /* Verify c2 got the cookie */
-    assert_group_tag(c2, "0");
-    assert_group_cookie_not_zero(c2);
-    assert_group_cookie_equal(c2, p);
-
-    /* Verify c11 got the cookie */
-    assert_group_tag(c11, "0");
-    assert_group_cookie_not_zero(c11);
-    assert_group_cookie_equal(c11, p);
-
-    /*
-     * Verify c22 which is a nested group created
-     * _after_ tagging got the cookie.
-     */
-    c22 = make_group(c2, "c22");
-
-    assert_group_tag(c22, "0");
-    assert_group_cookie_not_zero(c22);
-    assert_group_cookie_equal(c22, c1);
-    assert_group_cookie_equal(c22, c11);
-    assert_group_cookie_equal(c22, c2);
-    assert_group_cookie_equal(c22, p);
-
-    del_group(c22);
-    del_group(c11);
-    del_group(c1);
-    del_group(c2);
-    del_group(p);
-    print_pass();
+	char *p, *c1, *c11, *c2, *c22;
+
+	print_banner("TEST-CGROUP-PARENT-CHILD-TAG");
+
+	p = make_group(root, "p");
+	assert_group_cookie_zero(p);
+
+	c1 = make_group(p, "c1");
+	assert_group_tag(c1, "0"); /* Child tag is "0" but inherits cookie from parent. */
+	assert_group_cookie_zero(c1);
+	assert_group_cookie_equal(c1, p);
+
+	c11 = make_group(c1, "c11");
+	assert_group_tag(c11, "0");
+	assert_group_cookie_zero(c11);
+	assert_group_cookie_equal(c11, p);
+
+	c2 = make_group(p, "c2");
+	assert_group_tag(c2, "0");
+	assert_group_cookie_zero(c2);
+	assert_group_cookie_equal(c2, p);
+
+	tag_group(p);
+
+	/* Verify c1 got the cookie */
+	assert_group_tag(c1, "0");
+	assert_group_cookie_not_zero(c1);
+	assert_group_cookie_equal(c1, p);
+
+	/* Verify c2 got the cookie */
+	assert_group_tag(c2, "0");
+	assert_group_cookie_not_zero(c2);
+	assert_group_cookie_equal(c2, p);
+
+	/* Verify c11 got the cookie */
+	assert_group_tag(c11, "0");
+	assert_group_cookie_not_zero(c11);
+	assert_group_cookie_equal(c11, p);
+
+	/*
+	 * Verify c22 which is a nested group created
+	 * _after_ tagging got the cookie.
+	 */
+	c22 = make_group(c2, "c22");
+
+	assert_group_tag(c22, "0");
+	assert_group_cookie_not_zero(c22);
+	assert_group_cookie_equal(c22, c1);
+	assert_group_cookie_equal(c22, c11);
+	assert_group_cookie_equal(c22, c2);
+	assert_group_cookie_equal(c22, p);
+
+	del_group(c22);
+	del_group(c11);
+	del_group(c1);
+	del_group(c2);
+	del_group(p);
+	print_pass();
 }
 
 /*
@@ -678,163 +678,163 @@ static void test_cgroup_parent_child_tag_inherit(char *root)
  */
 static void test_cgroup_parent_tag_child_inherit(char *root)
 {
-    char *p, *c1, *c2, *c3;
-
-    print_banner("TEST-CGROUP-PARENT-TAG-CHILD-INHERIT");
-
-    p = make_group(root, "p");
-    assert_group_cookie_zero(p);
-    tag_group(p);
-    assert_group_cookie_not_zero(p);
-
-    c1 = make_group(p, "c1");
-    assert_group_cookie_not_zero(c1);
-    /* Child tag is "0" but it inherits cookie from parent. */
-    assert_group_tag(c1, "0");
-    assert_group_cookie_equal(c1, p);
-
-    c2 = make_group(p, "c2");
-    assert_group_tag(c2, "0");
-    assert_group_cookie_equal(c2, p);
-    assert_group_cookie_equal(c1, c2);
-
-    c3 = make_group(c1, "c3");
-    assert_group_tag(c3, "0");
-    assert_group_cookie_equal(c3, p);
-    assert_group_cookie_equal(c1, c3);
-
-    del_group(c3);
-    del_group(c1);
-    del_group(c2);
-    del_group(p);
-    print_pass();
+	char *p, *c1, *c2, *c3;
+
+	print_banner("TEST-CGROUP-PARENT-TAG-CHILD-INHERIT");
+
+	p = make_group(root, "p");
+	assert_group_cookie_zero(p);
+	tag_group(p);
+	assert_group_cookie_not_zero(p);
+
+	c1 = make_group(p, "c1");
+	assert_group_cookie_not_zero(c1);
+	/* Child tag is "0" but it inherits cookie from parent. */
+	assert_group_tag(c1, "0");
+	assert_group_cookie_equal(c1, p);
+
+	c2 = make_group(p, "c2");
+	assert_group_tag(c2, "0");
+	assert_group_cookie_equal(c2, p);
+	assert_group_cookie_equal(c1, c2);
+
+	c3 = make_group(c1, "c3");
+	assert_group_tag(c3, "0");
+	assert_group_cookie_equal(c3, p);
+	assert_group_cookie_equal(c1, c3);
+
+	del_group(c3);
+	del_group(c1);
+	del_group(c2);
+	del_group(p);
+	print_pass();
 }
 
 static void test_prctl_in_group(char *root)
 {
-    char *p;
-    struct task_state *tsk1, *tsk2, *tsk3;
-
-    print_banner("TEST-PRCTL-IN-GROUP");
-
-    p = make_group(root, "p");
-    assert_group_cookie_zero(p);
-    tag_group(p);
-    assert_group_cookie_not_zero(p);
-
-    tsk1 = add_task(p);
-    assert_group_cookie_equals_task_cookie(p, tsk1->pid_str);
-
-    tsk2 = add_task(p);
-    assert_group_cookie_equals_task_cookie(p, tsk2->pid_str);
-
-    tsk3 = add_task(p);
-    assert_group_cookie_equals_task_cookie(p, tsk3->pid_str);
-
-    /* tsk2 share with tsk3 -- both get disconnected from CGroup. */
-    make_tasks_share(tsk2, tsk3);
-    assert_tasks_share(tsk2, tsk3);
-    assert_tasks_dont_share(tsk1, tsk2);
-    assert_tasks_dont_share(tsk1, tsk3);
-    assert_group_cookie_equals_task_cookie(p, tsk1->pid_str);
-    assert_group_cookie_not_equals_task_cookie(p, tsk2->pid_str);
-    assert_group_cookie_not_equals_task_cookie(p, tsk3->pid_str);
-
-    /* now reset tsk3 -- get connected back to CGroup. */
-    reset_task_cookie(tsk3);
-    assert_tasks_dont_share(tsk2, tsk3);
-    assert_tasks_share(tsk1, tsk3);      // tsk3 is back.
-    assert_tasks_dont_share(tsk1, tsk2); // but tsk2 is still zombie
-    assert_group_cookie_equals_task_cookie(p, tsk1->pid_str);
-    assert_group_cookie_not_equals_task_cookie(p, tsk2->pid_str);
-    assert_group_cookie_equals_task_cookie(p, tsk3->pid_str); // tsk3 is back.
-
-    /* now reset tsk2 as well to get it connected back to CGroup. */
-    reset_task_cookie(tsk2);
-    assert_tasks_share(tsk2, tsk3);
-    assert_tasks_share(tsk1, tsk3);
-    assert_tasks_share(tsk1, tsk2);
-    assert_group_cookie_equals_task_cookie(p, tsk1->pid_str);
-    assert_group_cookie_equals_task_cookie(p, tsk2->pid_str);
-    assert_group_cookie_equals_task_cookie(p, tsk3->pid_str);
-
-    /* Test the rest of the cases (2 to 4)
-     *
-     *		t1		joining		t2
-     * CASE 1:
-     * before	0				0
-     * after	new cookie			new cookie
-     *
-     * CASE 2:
-     * before	X (non-zero)			0
-     * after	0				0
-     *
-     * CASE 3:
-     * before	0				X (non-zero)
-     * after	X				X
-     *
-     * CASE 4:
-     * before	Y (non-zero)			X (non-zero)
-     * after	X				X
-     */
-
-    /* case 2: */
-    dprint("case 2");
-    make_tasks_share(tsk1, tsk1);
-    assert_tasks_dont_share(tsk1, tsk2);
-    assert_tasks_dont_share(tsk1, tsk3);
-    assert_group_cookie_not_equals_task_cookie(p, tsk1->pid_str);
-    make_tasks_share(tsk1, tsk2); /* Will reset the task cookie. */
-    assert_group_cookie_equals_task_cookie(p, tsk1->pid_str);
-
-    /* case 3: */
-    dprint("case 3");
-    assert_group_cookie_equals_task_cookie(p, tsk2->pid_str);
-    make_tasks_share(tsk2, tsk2);
-    assert_group_cookie_not_equals_task_cookie(p, tsk2->pid_str);
-    assert_tasks_dont_share(tsk2, tsk1);
-    assert_tasks_dont_share(tsk2, tsk3);
-    assert_group_cookie_equals_task_cookie(p, tsk1->pid_str);
-    make_tasks_share(tsk1, tsk2);
-    assert_group_cookie_not_equals_task_cookie(p, tsk1->pid_str);
-    assert_tasks_share(tsk1, tsk2);
-    assert_tasks_dont_share(tsk1, tsk3);
-    reset_task_cookie(tsk1);
-    reset_task_cookie(tsk2);
-
-    /* case 4: */
-    dprint("case 4");
-    assert_tasks_share(tsk1, tsk2);
-    assert_group_cookie_equals_task_cookie(p, tsk1->pid_str);
-    make_tasks_share(tsk1, tsk1);
-    assert_group_cookie_not_equals_task_cookie(p, tsk1->pid_str);
-    assert_group_cookie_equals_task_cookie(p, tsk2->pid_str);
-    make_tasks_share(tsk2, tsk2);
-    assert_group_cookie_not_equals_task_cookie(p, tsk2->pid_str);
-    assert_tasks_dont_share(tsk1, tsk2);
-    make_tasks_share(tsk1, tsk2);
-    assert_group_cookie_not_equals_task_cookie(p, tsk1->pid_str);
-    assert_tasks_share(tsk1, tsk2);
-    assert_tasks_dont_share(tsk1, tsk3);
-    reset_task_cookie(tsk1);
-    reset_task_cookie(tsk2);
-
-    kill_task(tsk1);
-    kill_task(tsk2);
-    kill_task(tsk3);
-    del_group(p);
-    print_pass();
+	char *p;
+	struct task_state *tsk1, *tsk2, *tsk3;
+
+	print_banner("TEST-PRCTL-IN-GROUP");
+
+	p = make_group(root, "p");
+	assert_group_cookie_zero(p);
+	tag_group(p);
+	assert_group_cookie_not_zero(p);
+
+	tsk1 = add_task(p);
+	assert_group_cookie_equals_task_cookie(p, tsk1->pid_str);
+
+	tsk2 = add_task(p);
+	assert_group_cookie_equals_task_cookie(p, tsk2->pid_str);
+
+	tsk3 = add_task(p);
+	assert_group_cookie_equals_task_cookie(p, tsk3->pid_str);
+
+	/* tsk2 share with tsk3 -- both get disconnected from CGroup. */
+	make_tasks_share(tsk2, tsk3);
+	assert_tasks_share(tsk2, tsk3);
+	assert_tasks_dont_share(tsk1, tsk2);
+	assert_tasks_dont_share(tsk1, tsk3);
+	assert_group_cookie_equals_task_cookie(p, tsk1->pid_str);
+	assert_group_cookie_not_equals_task_cookie(p, tsk2->pid_str);
+	assert_group_cookie_not_equals_task_cookie(p, tsk3->pid_str);
+
+	/* now reset tsk3 -- get connected back to CGroup. */
+	reset_task_cookie(tsk3);
+	assert_tasks_dont_share(tsk2, tsk3);
+	assert_tasks_share(tsk1, tsk3);	  // tsk3 is back.
+	assert_tasks_dont_share(tsk1, tsk2); // but tsk2 is still zombie
+	assert_group_cookie_equals_task_cookie(p, tsk1->pid_str);
+	assert_group_cookie_not_equals_task_cookie(p, tsk2->pid_str);
+	assert_group_cookie_equals_task_cookie(p, tsk3->pid_str); // tsk3 is back.
+
+	/* now reset tsk2 as well to get it connected back to CGroup. */
+	reset_task_cookie(tsk2);
+	assert_tasks_share(tsk2, tsk3);
+	assert_tasks_share(tsk1, tsk3);
+	assert_tasks_share(tsk1, tsk2);
+	assert_group_cookie_equals_task_cookie(p, tsk1->pid_str);
+	assert_group_cookie_equals_task_cookie(p, tsk2->pid_str);
+	assert_group_cookie_equals_task_cookie(p, tsk3->pid_str);
+
+	/* Test the rest of the cases (2 to 4)
+	 *
+	 *		t1		joining		t2
+	 * CASE 1:
+	 * before	0				0
+	 * after	new cookie			new cookie
+	 *
+	 * CASE 2:
+	 * before	X (non-zero)			0
+	 * after	0				0
+	 *
+	 * CASE 3:
+	 * before	0				X (non-zero)
+	 * after	X				X
+	 *
+	 * CASE 4:
+	 * before	Y (non-zero)			X (non-zero)
+	 * after	X				X
+	 */
+
+	/* case 2: */
+	dprint("case 2");
+	make_tasks_share(tsk1, tsk1);
+	assert_tasks_dont_share(tsk1, tsk2);
+	assert_tasks_dont_share(tsk1, tsk3);
+	assert_group_cookie_not_equals_task_cookie(p, tsk1->pid_str);
+	make_tasks_share(tsk1, tsk2); /* Will reset the task cookie. */
+	assert_group_cookie_equals_task_cookie(p, tsk1->pid_str);
+
+	/* case 3: */
+	dprint("case 3");
+	assert_group_cookie_equals_task_cookie(p, tsk2->pid_str);
+	make_tasks_share(tsk2, tsk2);
+	assert_group_cookie_not_equals_task_cookie(p, tsk2->pid_str);
+	assert_tasks_dont_share(tsk2, tsk1);
+	assert_tasks_dont_share(tsk2, tsk3);
+	assert_group_cookie_equals_task_cookie(p, tsk1->pid_str);
+	make_tasks_share(tsk1, tsk2);
+	assert_group_cookie_not_equals_task_cookie(p, tsk1->pid_str);
+	assert_tasks_share(tsk1, tsk2);
+	assert_tasks_dont_share(tsk1, tsk3);
+	reset_task_cookie(tsk1);
+	reset_task_cookie(tsk2);
+
+	/* case 4: */
+	dprint("case 4");
+	assert_tasks_share(tsk1, tsk2);
+	assert_group_cookie_equals_task_cookie(p, tsk1->pid_str);
+	make_tasks_share(tsk1, tsk1);
+	assert_group_cookie_not_equals_task_cookie(p, tsk1->pid_str);
+	assert_group_cookie_equals_task_cookie(p, tsk2->pid_str);
+	make_tasks_share(tsk2, tsk2);
+	assert_group_cookie_not_equals_task_cookie(p, tsk2->pid_str);
+	assert_tasks_dont_share(tsk1, tsk2);
+	make_tasks_share(tsk1, tsk2);
+	assert_group_cookie_not_equals_task_cookie(p, tsk1->pid_str);
+	assert_tasks_share(tsk1, tsk2);
+	assert_tasks_dont_share(tsk1, tsk3);
+	reset_task_cookie(tsk1);
+	reset_task_cookie(tsk2);
+
+	kill_task(tsk1);
+	kill_task(tsk2);
+	kill_task(tsk3);
+	del_group(p);
+	print_pass();
 }
 
 int main() {
-    char *root = make_group(NULL, NULL);
+	char *root = make_group(NULL, NULL);
 
-    test_cgroup_parent_tag_child_inherit(root);
-    test_cgroup_parent_child_tag_inherit(root);
-    test_cgroup_coloring(root);
-    test_prctl_in_group(root);
+	test_cgroup_parent_tag_child_inherit(root);
+	test_cgroup_parent_child_tag_inherit(root);
+	test_cgroup_coloring(root);
+	test_prctl_in_group(root);
 
-    del_root_group(root);
-    return 0;
+	del_root_group(root);
+	return 0;
 }
 
Signature · 1 line
2.28.0
#85 lore raw

PATCH v8 24/26 sched: Move core-scheduler interfacing code to a new file

In reply to Joel Fernandes (Google)

core.c is already huge. The core-tagging interface code is largely independent of it. Move it to its own file to make both files easier to maintain.

---
 kernel/sched/Makefile  |   1 +
 kernel/sched/core.c    | 481 +----------------------------------------
 kernel/sched/coretag.c | 468 +++++++++++++++++++++++++++++++++++++++
 kernel/sched/sched.h   |  56 ++++-
 4 files changed, 523 insertions(+), 483 deletions(-)
 create mode 100644 kernel/sched/coretag.c
kernel/sched/Makefile +1 -0
index 5fc9c9b70862..c526c20adf9d 100644
@@ -36,3 +36,4 @@ obj-$(CONFIG_CPU_FREQ_GOV_SCHEDUTIL) += cpufreq_schedutil.o
 obj-$(CONFIG_MEMBARRIER) += membarrier.o
 obj-$(CONFIG_CPU_ISOLATION) += isolation.o
 obj-$(CONFIG_PSI) += psi.o
+obj-$(CONFIG_SCHED_CORE) += coretag.o
kernel/sched/core.c +2 -479

529 lines not on this page · raw message

kernel/sched/coretag.c new file +468 -0

468 lines not on this page · raw message

kernel/sched/sched.h +52 -4

81 lines not on this page · raw message

Signature · 1 line
2.29.0.rc1.297.gfa9743e501-goog
#86 lore raw

Re: [PATCH v8 -tip 24/26] sched: Move core-scheduler interfacing code to a new file

Quoted 736 lines · Joel Fernandes (Google)

core.c is already huge. The core-tagging interface code is largely independent of it. Move it to its own file to make both files easier to maintain.

---
 kernel/sched/Makefile  |   1 +
 kernel/sched/core.c    | 481 +----------------------------------------
 kernel/sched/coretag.c | 468 +++++++++++++++++++++++++++++++++++++++
 kernel/sched/sched.h   |  56 ++++-
 4 files changed, 523 insertions(+), 483 deletions(-)
 create mode 100644 kernel/sched/coretag.c
kernel/sched/Makefile +1 -0
index 5fc9c9b70862..c526c20adf9d 100644
@@ -36,3 +36,4 @@ obj-$(CONFIG_CPU_FREQ_GOV_SCHEDUTIL) += cpufreq_schedutil.o
 obj-$(CONFIG_MEMBARRIER) += membarrier.o
 obj-$(CONFIG_CPU_ISOLATION) += isolation.o
 obj-$(CONFIG_PSI) += psi.o
+obj-$(CONFIG_SCHED_CORE) += coretag.o
kernel/sched/core.c +2 -479
index b3afbba5abe1..211e0784675f 100644
@@ -162,11 +162,6 @@ static bool sched_core_empty(struct rq *rq)
 	return RB_EMPTY_ROOT(&rq->core_tree);
 }
 
-static bool sched_core_enqueued(struct task_struct *task)
-{
-	return !RB_EMPTY_NODE(&task->core_node);
-}
-
 static struct task_struct *sched_core_first(struct rq *rq)
 {
 	struct task_struct *task;
@@ -188,7 +183,7 @@ static void sched_core_flush(int cpu)
 	rq->core->core_task_seq++;
 }
 
-static void sched_core_enqueue(struct rq *rq, struct task_struct *p)
+void sched_core_enqueue(struct rq *rq, struct task_struct *p)
 {
 	struct rb_node *parent, **node;
 	struct task_struct *node_task;
@@ -215,7 +210,7 @@ static void sched_core_enqueue(struct rq *rq, struct task_struct *p)
 	rb_insert_color(&p->core_node, &rq->core_tree);
 }
 
-static void sched_core_dequeue(struct rq *rq, struct task_struct *p)
+void sched_core_dequeue(struct rq *rq, struct task_struct *p)
 {
 	rq->core->core_task_seq++;
 
@@ -310,7 +305,6 @@ static int __sched_core_stopper(void *data)
 }
 
 static DEFINE_MUTEX(sched_core_mutex);
-static DEFINE_MUTEX(sched_core_tasks_mutex);
 static int sched_core_count;
 
 static void __sched_core_enable(void)
@@ -346,16 +340,6 @@ void sched_core_put(void)
 		__sched_core_disable();
 	mutex_unlock(&sched_core_mutex);
 }
-
-static int sched_core_share_tasks(struct task_struct *t1, struct task_struct *t2);
-
-#else /* !CONFIG_SCHED_CORE */
-
-static inline void sched_core_enqueue(struct rq *rq, struct task_struct *p) { }
-static inline void sched_core_dequeue(struct rq *rq, struct task_struct *p) { }
-static bool sched_core_enqueued(struct task_struct *task) { return false; }
-static int sched_core_share_tasks(struct task_struct *t1, struct task_struct *t2) { }
-
 #endif /* CONFIG_SCHED_CORE */
 
 /*
@@ -8505,9 +8489,6 @@ void sched_offline_group(struct task_group *tg)
 	spin_unlock_irqrestore(&task_group_lock, flags);
 }
 
-#define SCHED_CORE_GROUP_COOKIE_MASK ((1UL << (sizeof(unsigned long) * 4)) - 1)
-static unsigned long cpu_core_get_group_cookie(struct task_group *tg);
-
 static void sched_change_group(struct task_struct *tsk, int type)
 {
 	struct task_group *tg;
@@ -8583,11 +8564,6 @@ void sched_move_task(struct task_struct *tsk)
 	task_rq_unlock(rq, tsk, &rf);
 }
 
-static inline struct task_group *css_tg(struct cgroup_subsys_state *css)
-{
-	return css ? container_of(css, struct task_group, css) : NULL;
-}
-
 static struct cgroup_subsys_state *
 cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
 {
@@ -9200,459 +9176,6 @@ static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css,
 }
 #endif /* CONFIG_RT_GROUP_SCHED */
 
-#ifdef CONFIG_SCHED_CORE
-/*
- * A simple wrapper around refcount. An allocated sched_core_cookie's
- * address is used to compute the cookie of the task.
- */
-struct sched_core_cookie {
-	refcount_t refcnt;
-};
-
-/*
- * sched_core_tag_requeue - Common helper for all interfaces to set a cookie.
- * @p: The task to assign a cookie to.
- * @cookie: The cookie to assign.
- * @group: is it a group interface or a per-task interface.
- *
- * This function is typically called from a stop-machine handler.
- */
-void sched_core_tag_requeue(struct task_struct *p, unsigned long cookie, bool group)
-{
-	if (!p)
-		return;
-
-	if (group)
-		p->core_group_cookie = cookie;
-	else
-		p->core_task_cookie = cookie;
-
-	/* Use up half of the cookie's bits for task cookie and remaining for group cookie. */
-	p->core_cookie = (p->core_task_cookie <<
-				(sizeof(unsigned long) * 4)) + p->core_group_cookie;
-
-	if (sched_core_enqueued(p)) {
-		sched_core_dequeue(task_rq(p), p);
-		if (!p->core_cookie)
-			return;
-	}
-
-	if (sched_core_enabled(task_rq(p)) &&
-			p->core_cookie && task_on_rq_queued(p))
-		sched_core_enqueue(task_rq(p), p);
-}
-
-/* Per-task interface */
-static unsigned long sched_core_alloc_task_cookie(void)
-{
-	struct sched_core_cookie *ptr =
-		kmalloc(sizeof(struct sched_core_cookie), GFP_KERNEL);
-
-	if (!ptr)
-		return 0;
-	refcount_set(&ptr->refcnt, 1);
-
-	/*
-	 * NOTE: sched_core_put() is not done by put_task_cookie(). Instead, it
-	 * is done after the stopper runs.
-	 */
-	sched_core_get();
-	return (unsigned long)ptr;
-}
-
-static bool sched_core_get_task_cookie(unsigned long cookie)
-{
-	struct sched_core_cookie *ptr = (struct sched_core_cookie *)cookie;
-
-	/*
-	 * NOTE: sched_core_put() is not done by put_task_cookie(). Instead, it
-	 * is done after the stopper runs.
-	 */
-	sched_core_get();
-	return refcount_inc_not_zero(&ptr->refcnt);
-}
-
-static void sched_core_put_task_cookie(unsigned long cookie)
-{
-	struct sched_core_cookie *ptr = (struct sched_core_cookie *)cookie;
-
-	if (refcount_dec_and_test(&ptr->refcnt))
-		kfree(ptr);
-}
-
-struct sched_core_task_write_tag {
-	struct task_struct *tasks[2];
-	unsigned long cookies[2];
-};
-
-/*
- * Ensure that the task has been requeued. The stopper ensures that the task cannot
- * be migrated to a different CPU while its core scheduler queue state is being updated.
- * It also makes sure to requeue a task if it was running actively on another CPU.
- */
-static int sched_core_task_join_stopper(void *data)
-{
-	struct sched_core_task_write_tag *tag = (struct sched_core_task_write_tag *)data;
-	int i;
-
-	for (i = 0; i < 2; i++)
-		sched_core_tag_requeue(tag->tasks[i], tag->cookies[i], false /* !group */);
-
-	return 0;
-}
-
-static int sched_core_share_tasks(struct task_struct *t1, struct task_struct *t2)
-{
-	struct sched_core_task_write_tag wr = {}; /* for stop machine. */
-	bool sched_core_put_after_stopper = false;
-	unsigned long cookie;
-	int ret = -ENOMEM;
-
-	mutex_lock(&sched_core_tasks_mutex);
-
-	/*
-	 * NOTE: sched_core_get() is done by sched_core_alloc_task_cookie() or
-	 *       sched_core_put_task_cookie(). However, sched_core_put() is done
-	 *       by this function *after* the stopper removes the tasks from the
-	 *       core queue, and not before. This is just to play it safe.
-	 */
-	if (t2 == NULL) {
-		if (t1->core_task_cookie) {
-			sched_core_put_task_cookie(t1->core_task_cookie);
-			sched_core_put_after_stopper = true;
-			wr.tasks[0] = t1; /* Keep wr.cookies[0] reset for t1. */
-		}
-	} else if (t1 == t2) {
-		/* Assign a unique per-task cookie solely for t1. */
-
-		cookie = sched_core_alloc_task_cookie();
-		if (!cookie)
-			goto out_unlock;
-
-		if (t1->core_task_cookie) {
-			sched_core_put_task_cookie(t1->core_task_cookie);
-			sched_core_put_after_stopper = true;
-		}
-		wr.tasks[0] = t1;
-		wr.cookies[0] = cookie;
-	} else
-	/*
-	 * 		t1		joining		t2
-	 * CASE 1:
-	 * before	0				0
-	 * after	new cookie			new cookie
-	 *
-	 * CASE 2:
-	 * before	X (non-zero)			0
-	 * after	0				0
-	 *
-	 * CASE 3:
-	 * before	0				X (non-zero)
-	 * after	X				X
-	 *
-	 * CASE 4:
-	 * before	Y (non-zero)			X (non-zero)
-	 * after	X				X
-	 */
-	if (!t1->core_task_cookie && !t2->core_task_cookie) {
-		/* CASE 1. */
-		cookie = sched_core_alloc_task_cookie();
-		if (!cookie)
-			goto out_unlock;
-
-		/* Add another reference for the other task. */
-		if (!sched_core_get_task_cookie(cookie)) {
-			return -EINVAL;
-			goto out_unlock;
-		}
-
-		wr.tasks[0] = t1;
-		wr.tasks[1] = t2;
-		wr.cookies[0] = wr.cookies[1] = cookie;
-
-	} else if (t1->core_task_cookie && !t2->core_task_cookie) {
-		/* CASE 2. */
-		sched_core_put_task_cookie(t1->core_task_cookie);
-		sched_core_put_after_stopper = true;
-
-		wr.tasks[0] = t1; /* Reset cookie for t1. */
-
-	} else if (!t1->core_task_cookie && t2->core_task_cookie) {
-		/* CASE 3. */
-		if (!sched_core_get_task_cookie(t2->core_task_cookie)) {
-			ret = -EINVAL;
-			goto out_unlock;
-		}
-
-		wr.tasks[0] = t1;
-		wr.cookies[0] = t2->core_task_cookie;
-
-	} else {
-		/* CASE 4. */
-		if (!sched_core_get_task_cookie(t2->core_task_cookie)) {
-			ret = -EINVAL;
-			goto out_unlock;
-		}
-		sched_core_put_task_cookie(t1->core_task_cookie);
-		sched_core_put_after_stopper = true;
-
-		wr.tasks[0] = t1;
-		wr.cookies[0] = t2->core_task_cookie;
-	}
-
-	stop_machine(sched_core_task_join_stopper, (void *)&wr, NULL);
-
-	if (sched_core_put_after_stopper)
-		sched_core_put();
-
-	ret = 0;
-out_unlock:
-	mutex_unlock(&sched_core_tasks_mutex);
-	return ret;
-}
-
-/* Called from prctl interface: PR_SCHED_CORE_SHARE */
-int sched_core_share_pid(pid_t pid)
-{
-	struct task_struct *task;
-	int err;
-
-	if (pid == 0) { /* Recent current task's cookie. */
-		/* Resetting a cookie requires privileges. */
-		if (current->core_task_cookie)
-			if (!capable(CAP_SYS_ADMIN))
-				return -EPERM;
-		task = NULL;
-	} else {
-		rcu_read_lock();
-		task = pid ? find_task_by_vpid(pid) : current;
-		if (!task) {
-			rcu_read_unlock();
-			return -ESRCH;
-		}
-
-		get_task_struct(task);
-
-		/*
-		 * Check if this process has the right to modify the specified
-		 * process. Use the regular "ptrace_may_access()" checks.
-		 */
-		if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) {
-			rcu_read_unlock();
-			err = -EPERM;
-			goto out_put;
-		}
-		rcu_read_unlock();
-	}
-
-	err = sched_core_share_tasks(current, task);
-out_put:
-	if (task)
-		put_task_struct(task);
-	return err;
-}
-
-/* CGroup interface */
-
-/*
- * Helper to get the cookie in a hierarchy.
- * The cookie is a combination of a tag and color. Any ancestor
- * can have a tag/color. tag is the first-level cookie setting
- * with color being the second. Atmost one color and one tag is
- * allowed.
- */
-static unsigned long cpu_core_get_group_cookie(struct task_group *tg)
-{
-	unsigned long color = 0;
-
-	if (!tg)
-		return 0;
-
-	for (; tg; tg = tg->parent) {
-		if (tg->core_tag_color) {
-			WARN_ON_ONCE(color);
-			color = tg->core_tag_color;
-		}
-
-		if (tg->core_tagged) {
-			unsigned long cookie = ((unsigned long)tg << 8) | color;
-			cookie &= SCHED_CORE_GROUP_COOKIE_MASK;
-			return cookie;
-		}
-	}
-
-	return 0;
-}
-
-/* Determine if any group in @tg's children are tagged or colored. */
-static bool cpu_core_check_descendants(struct task_group *tg, bool check_tag,
-					bool check_color)
-{
-	struct task_group *child;
-
-	rcu_read_lock();
-	list_for_each_entry_rcu(child, &tg->children, siblings) {
-		if ((child->core_tagged && check_tag) ||
-		    (child->core_tag_color && check_color)) {
-			rcu_read_unlock();
-			return true;
-		}
-
-		rcu_read_unlock();
-		return cpu_core_check_descendants(child, check_tag, check_color);
-	}
-
-	rcu_read_unlock();
-	return false;
-}
-
-static u64 cpu_core_tag_read_u64(struct cgroup_subsys_state *css, struct cftype *cft)
-{
-	struct task_group *tg = css_tg(css);
-
-	return !!tg->core_tagged;
-}
-
-static u64 cpu_core_tag_color_read_u64(struct cgroup_subsys_state *css, struct cftype *cft)
-{
-	struct task_group *tg = css_tg(css);
-
-	return tg->core_tag_color;
-}
-
-#ifdef CONFIG_SCHED_DEBUG
-static u64 cpu_core_group_cookie_read_u64(struct cgroup_subsys_state *css, struct cftype *cft)
-{
-	return cpu_core_get_group_cookie(css_tg(css));
-}
-#endif
-
-struct write_core_tag {
-	struct cgroup_subsys_state *css;
-	unsigned long cookie;
-};
-
-static int __sched_write_tag(void *data)
-{
-	struct write_core_tag *tag = (struct write_core_tag *) data;
-	struct task_struct *p;
-	struct cgroup_subsys_state *css;
-
-	rcu_read_lock();
-	css_for_each_descendant_pre(css, tag->css) {
-		struct css_task_iter it;
-
-		css_task_iter_start(css, 0, &it);
-		/*
-		 * Note: css_task_iter_next will skip dying tasks.
-		 * There could still be dying tasks left in the core queue
-		 * when we set cgroup tag to 0 when the loop is done below.
-		 */
-		while ((p = css_task_iter_next(&it)))
-			sched_core_tag_requeue(p, tag->cookie, true /* group */);
-
-		css_task_iter_end(&it);
-	}
-	rcu_read_unlock();
-
-	return 0;
-}
-
-static int cpu_core_tag_write_u64(struct cgroup_subsys_state *css, struct cftype *cft, u64 val)
-{
-	struct task_group *tg = css_tg(css);
-	struct write_core_tag wtag;
-
-	if (val > 1)
-		return -ERANGE;
-
-	if (!static_branch_likely(&sched_smt_present))
-		return -EINVAL;
-
-	if (!tg->core_tagged && val) {
-		/* Tag is being set. Check ancestors and descendants. */
-		if (cpu_core_get_group_cookie(tg) ||
-		    cpu_core_check_descendants(tg, true /* tag */, true /* color */))
-			return -EBUSY;
-	} else if (tg->core_tagged && !val) {
-		/* Tag is being reset. Check descendants. */
-		if (cpu_core_check_descendants(tg, true /* tag */, true /* color */))
-			return -EBUSY;
-	} else {
-		return 0;
-	}
-
-	if (!!val)
-		sched_core_get();
-
-	wtag.css = css;
-	wtag.cookie = (unsigned long)tg << 8; /* Reserve lower 8 bits for color. */
-
-	/* Truncate the upper 32-bits - those are used by the per-task cookie. */
-	wtag.cookie &= (1UL << (sizeof(unsigned long) * 4)) - 1;
-
-	tg->core_tagged = val;
-
-	stop_machine(__sched_write_tag, (void *) &wtag, NULL);
-	if (!val)
-		sched_core_put();
-
-	return 0;
-}
-
-static int cpu_core_tag_color_write_u64(struct cgroup_subsys_state *css,
-					struct cftype *cft, u64 val)
-{
-	struct task_group *tg = css_tg(css);
-	struct write_core_tag wtag;
-	u64 cookie;
-
-	if (val > 255)
-		return -ERANGE;
-
-	if (!static_branch_likely(&sched_smt_present))
-		return -EINVAL;
-
-	cookie = cpu_core_get_group_cookie(tg);
-	/* Can't set color if nothing in the ancestors were tagged. */
-	if (!cookie)
-		return -EINVAL;
-
-	/*
-	 * Something in the ancestors already colors us. Can't change the color
-	 * at this level.
-	 */
-	if (!tg->core_tag_color && (cookie & 255))
-		return -EINVAL;
-
-	/*
-	 * Check if any descendants are colored. If so, we can't recolor them.
-	 * Don't need to check if descendants are tagged, since we don't allow
-	 * tagging when already tagged.
-	 */
-	if (cpu_core_check_descendants(tg, false /* tag */, true /* color */))
-		return -EINVAL;
-
-	cookie &= ~255;
-	cookie |= val;
-	wtag.css = css;
-	wtag.cookie = cookie;
-	tg->core_tag_color = val;
-
-	stop_machine(__sched_write_tag, (void *) &wtag, NULL);
-
-	return 0;
-}
-
-void sched_tsk_free(struct task_struct *tsk)
-{
-	if (!tsk->core_task_cookie)
-		return;
-	sched_core_put_task_cookie(tsk->core_task_cookie);
-	sched_core_put();
-}
-#endif
-
 static struct cftype cpu_legacy_files[] = {
 #ifdef CONFIG_FAIR_GROUP_SCHED
 	{
kernel/sched/coretag.c new file +166 -0
new file mode 100644
index 000000000000..3333c9b0afc5
@@ -0,0 +1,468 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * kernel/sched/core-tag.c
+ *
+ * Core-scheduling tagging interface support.
+ *
+ * Copyright(C) 2020, Joel Fernandes.
+ * Initial interfacing code  by Peter Ziljstra.
+ */
+
+#include "sched.h"
+
+/*
+ * A simple wrapper around refcount. An allocated sched_core_cookie's
+ * address is used to compute the cookie of the task.
+ */
+struct sched_core_cookie {
+	refcount_t refcnt;
+};
+
+static DEFINE_MUTEX(sched_core_tasks_mutex);
+
+/*
+ * sched_core_tag_requeue - Common helper for all interfaces to set a cookie.
+ * @p: The task to assign a cookie to.
+ * @cookie: The cookie to assign.
+ * @group: is it a group interface or a per-task interface.
+ *
+ * This function is typically called from a stop-machine handler.
+ */
+void sched_core_tag_requeue(struct task_struct *p, unsigned long cookie, bool group)
+{
+	if (!p)
+		return;
+
+	if (group)
+		p->core_group_cookie = cookie;
+	else
+		p->core_task_cookie = cookie;
+
+	/* Use up half of the cookie's bits for task cookie and remaining for group cookie. */
+	p->core_cookie = (p->core_task_cookie <<
+				(sizeof(unsigned long) * 4)) + p->core_group_cookie;
+
+	if (sched_core_enqueued(p)) {
+		sched_core_dequeue(task_rq(p), p);
+		if (!p->core_cookie)
+			return;
+	}
+
+	if (sched_core_enabled(task_rq(p)) &&
+			p->core_cookie && task_on_rq_queued(p))
+		sched_core_enqueue(task_rq(p), p);
+}
+
+/* Per-task interface: Used by fork(2) and prctl(2). */
+static unsigned long sched_core_alloc_task_cookie(void)
+{
+	struct sched_core_cookie *ptr =
+		kmalloc(sizeof(struct sched_core_cookie), GFP_KERNEL);
+
+	if (!ptr)
+		return 0;
+	refcount_set(&ptr->refcnt, 1);
+
+	/*
+	 * NOTE: sched_core_put() is not done by put_task_cookie(). Instead, it
+	 * is done after the stopper runs.
+	 */
+	sched_core_get();
+	return (unsigned long)ptr;
+}
+
+static bool sched_core_get_task_cookie(unsigned long cookie)
+{
+	struct sched_core_cookie *ptr = (struct sched_core_cookie *)cookie;
+
+	/*
+	 * NOTE: sched_core_put() is not done by put_task_cookie(). Instead, it
+	 * is done after the stopper runs.
+	 */
+	sched_core_get();
+	return refcount_inc_not_zero(&ptr->refcnt);
+}
+
+static void sched_core_put_task_cookie(unsigned long cookie)
+{
+	struct sched_core_cookie *ptr = (struct sched_core_cookie *)cookie;
+
+	if (refcount_dec_and_test(&ptr->refcnt))
+		kfree(ptr);
+}
+
+struct sched_core_task_write_tag {
+	struct task_struct *tasks[2];
+	unsigned long cookies[2];
+};
+
+/*
+ * Ensure that the task has been requeued. The stopper ensures that the task cannot
+ * be migrated to a different CPU while its core scheduler queue state is being updated.
+ * It also makes sure to requeue a task if it was running actively on another CPU.
+ */
+static int sched_core_task_join_stopper(void *data)
+{
+	struct sched_core_task_write_tag *tag = (struct sched_core_task_write_tag *)data;
+	int i;
+
+	for (i = 0; i < 2; i++)
+		sched_core_tag_requeue(tag->tasks[i], tag->cookies[i], false /* !group */);
+
+	return 0;
+}
+
+int sched_core_share_tasks(struct task_struct *t1, struct task_struct *t2)
+{
+	struct sched_core_task_write_tag wr = {}; /* for stop machine. */
+	bool sched_core_put_after_stopper = false;
+	unsigned long cookie;
+	int ret = -ENOMEM;
+
+	mutex_lock(&sched_core_tasks_mutex);
+
+	/*
+	 * NOTE: sched_core_get() is done by sched_core_alloc_task_cookie() or
+	 *       sched_core_put_task_cookie(). However, sched_core_put() is done
+	 *       by this function *after* the stopper removes the tasks from the
+	 *       core queue, and not before. This is just to play it safe.
+	 */
+	if (t2 == NULL) {
+		if (t1->core_task_cookie) {
+			sched_core_put_task_cookie(t1->core_task_cookie);
+			sched_core_put_after_stopper = true;
+			wr.tasks[0] = t1; /* Keep wr.cookies[0] reset for t1. */
+		}
+	} else if (t1 == t2) {
+		/* Assign a unique per-task cookie solely for t1. */
+
+		cookie = sched_core_alloc_task_cookie();
+		if (!cookie)
+			goto out_unlock;
+
+		if (t1->core_task_cookie) {
+			sched_core_put_task_cookie(t1->core_task_cookie);
+			sched_core_put_after_stopper = true;
+		}
+		wr.tasks[0] = t1;
+		wr.cookies[0] = cookie;
+	} else
+	/*
+	 * 		t1		joining		t2
+	 * CASE 1:
+	 * before	0				0
+	 * after	new cookie			new cookie
+	 *
+	 * CASE 2:
+	 * before	X (non-zero)			0
+	 * after	0				0
+	 *
+	 * CASE 3:
+	 * before	0				X (non-zero)
+	 * after	X				X
+	 *
+	 * CASE 4:
+	 * before	Y (non-zero)			X (non-zero)
+	 * after	X				X

kernel/sched/coretag.c line 176

Quoted 10 lines
kernel/sched/coretag.c new file +10 -0
+	 */
+	if (!t1->core_task_cookie && !t2->core_task_cookie) {
+		/* CASE 1. */
+		cookie = sched_core_alloc_task_cookie();
+		if (!cookie)
+			goto out_unlock;
+
+		/* Add another reference for the other task. */
+		if (!sched_core_get_task_cookie(cookie)) {
+			return -EINVAL;

ret = -EINVAL; mutex is not released otherwise...

Quoted 382 lines
kernel/sched/coretag.c new file +292 -0
+			goto out_unlock;
+		}
+
+		wr.tasks[0] = t1;
+		wr.tasks[1] = t2;
+		wr.cookies[0] = wr.cookies[1] = cookie;
+
+	} else if (t1->core_task_cookie && !t2->core_task_cookie) {
+		/* CASE 2. */
+		sched_core_put_task_cookie(t1->core_task_cookie);
+		sched_core_put_after_stopper = true;
+
+		wr.tasks[0] = t1; /* Reset cookie for t1. */
+
+	} else if (!t1->core_task_cookie && t2->core_task_cookie) {
+		/* CASE 3. */
+		if (!sched_core_get_task_cookie(t2->core_task_cookie)) {
+			ret = -EINVAL;
+			goto out_unlock;
+		}
+
+		wr.tasks[0] = t1;
+		wr.cookies[0] = t2->core_task_cookie;
+
+	} else {
+		/* CASE 4. */
+		if (!sched_core_get_task_cookie(t2->core_task_cookie)) {
+			ret = -EINVAL;
+			goto out_unlock;
+		}
+		sched_core_put_task_cookie(t1->core_task_cookie);
+		sched_core_put_after_stopper = true;
+
+		wr.tasks[0] = t1;
+		wr.cookies[0] = t2->core_task_cookie;
+	}
+
+	stop_machine(sched_core_task_join_stopper, (void *)&wr, NULL);
+
+	if (sched_core_put_after_stopper)
+		sched_core_put();
+
+	ret = 0;
+out_unlock:
+	mutex_unlock(&sched_core_tasks_mutex);
+	return ret;
+}
+
+/* Called from prctl interface: PR_SCHED_CORE_SHARE */
+int sched_core_share_pid(pid_t pid)
+{
+	struct task_struct *task;
+	int err;
+
+	if (pid == 0) { /* Recent current task's cookie. */
+		/* Resetting a cookie requires privileges. */
+		if (current->core_task_cookie)
+			if (!capable(CAP_SYS_ADMIN))
+				return -EPERM;
+		task = NULL;
+	} else {
+		rcu_read_lock();
+		task = pid ? find_task_by_vpid(pid) : current;
+		if (!task) {
+			rcu_read_unlock();
+			return -ESRCH;
+		}
+
+		get_task_struct(task);
+
+		/*
+		 * Check if this process has the right to modify the specified
+		 * process. Use the regular "ptrace_may_access()" checks.
+		 */
+		if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) {
+			rcu_read_unlock();
+			err = -EPERM;
+			goto out_put;
+		}
+		rcu_read_unlock();
+	}
+
+	err = sched_core_share_tasks(current, task);
+out_put:
+	if (task)
+		put_task_struct(task);
+	return err;
+}
+
+/* CGroup core-scheduling interface support. */
+
+/*
+ * Helper to get the cookie in a hierarchy.
+ * The cookie is a combination of a tag and color. Any ancestor
+ * can have a tag/color. tag is the first-level cookie setting
+ * with color being the second. Atmost one color and one tag is
+ * allowed.
+ */
+unsigned long cpu_core_get_group_cookie(struct task_group *tg)
+{
+	unsigned long color = 0;
+
+	if (!tg)
+		return 0;
+
+	for (; tg; tg = tg->parent) {
+		if (tg->core_tag_color) {
+			WARN_ON_ONCE(color);
+			color = tg->core_tag_color;
+		}
+
+		if (tg->core_tagged) {
+			unsigned long cookie = ((unsigned long)tg << 8) | color;
+			cookie &= SCHED_CORE_GROUP_COOKIE_MASK;
+			return cookie;
+		}
+	}
+
+	return 0;
+}
+
+/* Determine if any group in @tg's children are tagged or colored. */
+static bool cpu_core_check_descendants(struct task_group *tg, bool check_tag,
+				       bool check_color)
+{
+	struct task_group *child;
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(child, &tg->children, siblings) {
+		if ((child->core_tagged && check_tag) ||
+		    (child->core_tag_color && check_color)) {
+			rcu_read_unlock();
+			return true;
+		}
+
+		rcu_read_unlock();
+		return cpu_core_check_descendants(child, check_tag, check_color);
+	}
+
+	rcu_read_unlock();
+	return false;
+}
+
+u64 cpu_core_tag_read_u64(struct cgroup_subsys_state *css,
+			  struct cftype *cft)
+{
+	struct task_group *tg = css_tg(css);
+
+	return !!tg->core_tagged;
+}
+
+u64 cpu_core_tag_color_read_u64(struct cgroup_subsys_state *css,
+				struct cftype *cft)
+{
+	struct task_group *tg = css_tg(css);
+
+	return tg->core_tag_color;
+}
+
+#ifdef CONFIG_SCHED_DEBUG
+u64 cpu_core_group_cookie_read_u64(struct cgroup_subsys_state *css,
+				   struct cftype *cft)
+{
+	return cpu_core_get_group_cookie(css_tg(css));
+}
+#endif
+
+struct write_core_tag {
+	struct cgroup_subsys_state *css;
+	unsigned long cookie;
+};
+
+static int __sched_write_tag(void *data)
+{
+	struct write_core_tag *tag = (struct write_core_tag *) data;
+	struct task_struct *p;
+	struct cgroup_subsys_state *css;
+
+	rcu_read_lock();
+	css_for_each_descendant_pre(css, tag->css) {
+		struct css_task_iter it;
+
+		css_task_iter_start(css, 0, &it);
+		/*
+		 * Note: css_task_iter_next will skip dying tasks.
+		 * There could still be dying tasks left in the core queue
+		 * when we set cgroup tag to 0 when the loop is done below.
+		 */
+		while ((p = css_task_iter_next(&it)))
+			sched_core_tag_requeue(p, tag->cookie, true /* group */);
+
+		css_task_iter_end(&it);
+	}
+	rcu_read_unlock();
+
+	return 0;
+}
+
+int cpu_core_tag_write_u64(struct cgroup_subsys_state *css, struct cftype *cft,
+			   u64 val)
+{
+	struct task_group *tg = css_tg(css);
+	struct write_core_tag wtag;
+
+	if (val > 1)
+		return -ERANGE;
+
+	if (!static_branch_likely(&sched_smt_present))
+		return -EINVAL;
+
+	if (!tg->core_tagged && val) {
+		/* Tag is being set. Check ancestors and descendants. */
+		if (cpu_core_get_group_cookie(tg) ||
+		    cpu_core_check_descendants(tg, true /* tag */, true /* color */))
+			return -EBUSY;
+	} else if (tg->core_tagged && !val) {
+		/* Tag is being reset. Check descendants. */
+		if (cpu_core_check_descendants(tg, true /* tag */, true /* color */))
+			return -EBUSY;
+	} else {
+		return 0;
+	}
+
+	if (!!val)
+		sched_core_get();
+
+	wtag.css = css;
+	wtag.cookie = (unsigned long)tg << 8; /* Reserve lower 8 bits for color. */
+
+	/* Truncate the upper 32-bits - those are used by the per-task cookie. */
+	wtag.cookie &= (1UL << (sizeof(unsigned long) * 4)) - 1;
+
+	tg->core_tagged = val;
+
+	stop_machine(__sched_write_tag, (void *) &wtag, NULL);
+	if (!val)
+		sched_core_put();
+
+	return 0;
+}
+
+int cpu_core_tag_color_write_u64(struct cgroup_subsys_state *css,
+				 struct cftype *cft, u64 val)
+{
+	struct task_group *tg = css_tg(css);
+	struct write_core_tag wtag;
+	u64 cookie;
+
+	if (val > 255)
+		return -ERANGE;
+
+	if (!static_branch_likely(&sched_smt_present))
+		return -EINVAL;
+
+	cookie = cpu_core_get_group_cookie(tg);
+	/* Can't set color if nothing in the ancestors were tagged. */
+	if (!cookie)
+		return -EINVAL;
+
+	/*
+	 * Something in the ancestors already colors us. Can't change the color
+	 * at this level.
+	 */
+	if (!tg->core_tag_color && (cookie & 255))
+		return -EINVAL;
+
+	/*
+	 * Check if any descendants are colored. If so, we can't recolor them.
+	 * Don't need to check if descendants are tagged, since we don't allow
+	 * tagging when already tagged.
+	 */
+	if (cpu_core_check_descendants(tg, false /* tag */, true /* color */))
+		return -EINVAL;
+
+	cookie &= ~255;
+	cookie |= val;
+	wtag.css = css;
+	wtag.cookie = cookie;
+	tg->core_tag_color = val;
+
+	stop_machine(__sched_write_tag, (void *) &wtag, NULL);
+
+	return 0;
+}
+
+void sched_tsk_free(struct task_struct *tsk)
+{
+	if (!tsk->core_task_cookie)
+		return;
+	sched_core_put_task_cookie(tsk->core_task_cookie);
+	sched_core_put();
+}
kernel/sched/sched.h +52 -4
index aebeb91c4a0f..290a3b8be3d3 100644
@@ -437,6 +437,11 @@ struct task_group {
 
 };
 
+static inline struct task_group *css_tg(struct cgroup_subsys_state *css)
+{
+	return css ? container_of(css, struct task_group, css) : NULL;
+}
+
 #ifdef CONFIG_FAIR_GROUP_SCHED
 #define ROOT_TASK_GROUP_LOAD	NICE_0_LOAD
 
@@ -1104,6 +1109,8 @@ static inline int cpu_of(struct rq *rq)
 #ifdef CONFIG_SCHED_CORE
 DECLARE_STATIC_KEY_FALSE(__sched_core_enabled);
 
+#define SCHED_CORE_GROUP_COOKIE_MASK ((1UL << (sizeof(unsigned long) * 4)) - 1)
+
 static inline bool sched_core_enabled(struct rq *rq)
 {
 	return static_branch_unlikely(&__sched_core_enabled) && rq->core_enabled;
@@ -1148,10 +1155,54 @@ static inline bool sched_core_cookie_match(struct rq *rq, struct task_struct *p)
 	return idle_core || rq->core->core_cookie == p->core_cookie;
 }
 
-extern void queue_core_balance(struct rq *rq);
+static inline bool sched_core_enqueued(struct task_struct *task)
+{
+	return !RB_EMPTY_NODE(&task->core_node);
+}
+
+void queue_core_balance(struct rq *rq);
+
+void sched_core_enqueue(struct rq *rq, struct task_struct *p);
+void sched_core_dequeue(struct rq *rq, struct task_struct *p);
+void sched_core_get(void);
+void sched_core_put(void);
+
+void sched_core_tag_requeue(struct task_struct *p, unsigned long cookie,
+			    bool group);
+
+int sched_core_share_pid(pid_t pid);
+int sched_core_share_tasks(struct task_struct *t1, struct task_struct *t2);
+
+unsigned long cpu_core_get_group_cookie(struct task_group *tg);
+
+u64 cpu_core_tag_read_u64(struct cgroup_subsys_state *css,
+			  struct cftype *cft);
+
+u64 cpu_core_tag_color_read_u64(struct cgroup_subsys_state *css,
+				struct cftype *cft);
+
+#ifdef CONFIG_SCHED_DEBUG
+u64 cpu_core_group_cookie_read_u64(struct cgroup_subsys_state *css,
+				   struct cftype *cft);
+#endif
+
+int cpu_core_tag_write_u64(struct cgroup_subsys_state *css, struct cftype *cft,
+			   u64 val);
+
+int cpu_core_tag_color_write_u64(struct cgroup_subsys_state *css,
+				 struct cftype *cft, u64 val);
+
+#ifndef TIF_UNSAFE_RET
+#define TIF_UNSAFE_RET (0)
+#endif
 
 #else /* !CONFIG_SCHED_CORE */
 
+static inline bool sched_core_enqueued(struct task_struct *task) { return false; }
+static inline void sched_core_enqueue(struct rq *rq, struct task_struct *p) { }
+static inline void sched_core_dequeue(struct rq *rq, struct task_struct *p) { }
+static inline int sched_core_share_tasks(struct task_struct *t1, struct task_struct *t2) { }
+
 static inline bool sched_core_enabled(struct rq *rq)
 {
 	return false;
@@ -2779,7 +2830,4 @@ void swake_up_all_locked(struct swait_queue_head *q);
 void __prepare_to_swait(struct swait_queue_head *q, struct swait_queue *wait);
 
 #ifdef CONFIG_SCHED_CORE
-#ifndef TIF_UNSAFE_RET
-#define TIF_UNSAFE_RET (0)
-#endif
 #endif
#87 lore raw

Re: [PATCH v8 -tip 24/26] sched: Move core-scheduler interfacing code to a new file

On Mon, Oct 26, 2020 at 09:05:52AM +0800, Li, Aubrey wrote:
[..]

Quoted 56 lines
Quoted 56 lines
kernel/sched/coretag.c new file +56 -0
+int sched_core_share_tasks(struct task_struct *t1, struct task_struct *t2)
+{
+	struct sched_core_task_write_tag wr = {}; /* for stop machine. */
+	bool sched_core_put_after_stopper = false;
+	unsigned long cookie;
+	int ret = -ENOMEM;
+
+	mutex_lock(&sched_core_tasks_mutex);
+
+	/*
+	 * NOTE: sched_core_get() is done by sched_core_alloc_task_cookie() or
+	 *       sched_core_put_task_cookie(). However, sched_core_put() is done
+	 *       by this function *after* the stopper removes the tasks from the
+	 *       core queue, and not before. This is just to play it safe.
+	 */
+	if (t2 == NULL) {
+		if (t1->core_task_cookie) {
+			sched_core_put_task_cookie(t1->core_task_cookie);
+			sched_core_put_after_stopper = true;
+			wr.tasks[0] = t1; /* Keep wr.cookies[0] reset for t1. */
+		}
+	} else if (t1 == t2) {
+		/* Assign a unique per-task cookie solely for t1. */
+
+		cookie = sched_core_alloc_task_cookie();
+		if (!cookie)
+			goto out_unlock;
+
+		if (t1->core_task_cookie) {
+			sched_core_put_task_cookie(t1->core_task_cookie);
+			sched_core_put_after_stopper = true;
+		}
+		wr.tasks[0] = t1;
+		wr.cookies[0] = cookie;
+	} else
+	/*
+	 * 		t1		joining		t2
+	 * CASE 1:
+	 * before	0				0
+	 * after	new cookie			new cookie
+	 *
+	 * CASE 2:
+	 * before	X (non-zero)			0
+	 * after	0				0
+	 *
+	 * CASE 3:
+	 * before	0				X (non-zero)
+	 * after	X				X
+	 *
+	 * CASE 4:
+	 * before	Y (non-zero)			X (non-zero)
+	 * after	X				X
+	 */
+	if (!t1->core_task_cookie && !t2->core_task_cookie) {
+		/* CASE 1. */
+		cookie = sched_core_alloc_task_cookie();
Quoted 8 lines

kernel/sched/coretag.c line 176

Quoted 6 lines
kernel/sched/coretag.c new file +6 -0
+		if (!cookie)
+			goto out_unlock;
+
+		/* Add another reference for the other task. */
+		if (!sched_core_get_task_cookie(cookie)) {
+			return -EINVAL;

ret = -EINVAL; mutex is not released otherwise...

Good find and will fix.

Minor suggestion: Could you truncate your emails when replying?

thanks,

- Joel

#88 lore raw

PATCH v8 25/26 Documentation: Add core scheduling documentation

In reply to Joel Fernandes (Google)

Document the usecases, design and interfaces for core scheduling.

---
 .../admin-guide/hw-vuln/core-scheduling.rst   | 312 ++++++++++++++++++
 Documentation/admin-guide/hw-vuln/index.rst   |   1 +
 2 files changed, 313 insertions(+)
 create mode 100644 Documentation/admin-guide/hw-vuln/core-scheduling.rst
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +312 -0
new file mode 100644
index 000000000000..eacafbb8fa3f
@@ -0,0 +1,312 @@
+Core Scheduling
+***************
+Core scheduling support allows userspace to define groups of tasks that can
+share a core. These groups can be specified either for security usecases (one
+group of tasks don't trust another), or for performance usecases (some
+workloads may benefit from running on the same core as they don't need the same
+hardware resources of the shared core).
+
+Security usecase
+----------------
+A cross-HT attack involves the attacker and victim running on different
+Hyper Threads of the same core. MDS and L1TF are examples of such attacks.
+Without core scheduling, the only full mitigation of cross-HT attacks is to
+disable Hyper Threading (HT). Core scheduling allows HT to be turned on safely
+by ensuring that trusted tasks can share a core. This increase in core sharing
+can improvement performance, however it is not guaranteed that performance will
+always improve, though that is seen to be the case with a number of real world
+workloads. In theory, core scheduling aims to perform at least as good as when
+Hyper Threading is disabled. In practise, this is mostly the case though not
+always: as synchronizing scheduling decisions across 2 or more CPUs in a core
+involves additional overhead - especially when the system is lightly loaded
+(``total_threads <= N/2``).
+
+Usage
+-----
+Core scheduling support is enabled via the ``CONFIG_SCHED_CORE`` config option.
+Using this feature, userspace defines groups of tasks that trust each other.
+The core scheduler uses this information to make sure that tasks that do not
+trust each other will never run simultaneously on a core, while doing its best
+to satisfy the system's scheduling requirements.
+
+There are 2 ways to use core-scheduling:
+
+CGroup
+######
+Core scheduling adds additional files to the CPU controller CGroup:
+
+* ``cpu.tag``
+Writing ``1`` into this file results in all tasks in the group get tagged. This
+results in all the CGroup's tasks allowed to run concurrently on a core's
+hyperthreads (also called siblings).
+
+The file being a value of ``0`` means the tag state of the CGroup is inheritted
+from its parent hierarchy. If any ancestor of the CGroup is tagged, then the
+group is tagged.
+
+.. note:: Once a CGroup is tagged via cpu.tag, it is not possible to set this
+          for any descendant of the tagged group. For finer grained control, the
+          ``cpu.tag_color`` file described next may be used.
+
+.. note:: When a CGroup is not tagged, all the tasks within the group can share
+          a core with kernel threads and untagged system threads. For this reason,
+          if a group has ``cpu.tag`` of 0, it is considered to be trusted.
+
+* ``cpu.tag_color``
+For finer grained control over core sharing, a color can also be set in
+addition to the tag. This allows to further control core sharing between child
+CGroups within an already tagged CGroup. The color and the tag are both used to
+generate a `cookie` which is used by the scheduler to identify the group.
+
+Upto 256 different colors can be set (0-255) by writing into this file.
+
+A sample real-world usage of this file follows:
+
+Google uses DAC controls to make ``cpu.tag`` writeable only by root and the
+``cpu.tag_color`` can be changed by anyone.
+
+The hierarchy looks like this:
+::
+  Root group
+     / \
+    A   B    (These are created by the root daemon - borglet).
+   / \   \
+  C   D   E  (These are created by AppEngine within the container).
+
+A and B are containers for 2 different jobs or apps that are created by a root
+daemon called borglet. borglet then tags each of these group with the ``cpu.tag``
+file. The job itself can create additional child CGroups which are colored by
+the container's AppEngine with the ``cpu.tag_color`` file.
+
+The reason why Google uses this 2-level tagging system is that AppEngine wants to
+allow a subset of child CGroups within a tagged parent CGroup to be co-scheduled on a
+core while not being co-scheduled with other child CGroups. Think of these
+child CGroups as belonging to the same customer or project.  Because these
+child CGroups are created by AppEngine, they are not tracked by borglet (the
+root daemon), therefore borglet won't have a chance to set a color for them.
+That's where cpu.tag_color file comes in. A color could be set by AppEngine,
+and once set, the normal tasks within the subcgroup would not be able to
+overwrite it. This is enforced by promoting the permission of the
+``cpu.tag_color`` file in cgroupfs.
+
+The color is an 8-bit value allowing for upto 256 unique colors.
+
+.. note:: Once a CGroup is colored, none of its descendants can be re-colored. Also
+          coloring of a CGroup is possible only if either the group or one of its
+          ancestors were tagged via the ``cpu.tag`` file.
+
+prctl interface
+###############
+A ``prtcl(2)`` command ``PR_SCHED_CORE_SHARE`` is available to a process to request
+sharing a core with another process.  For example, consider 2 processes ``P1``
+and ``P2`` with PIDs 100 and 200. If process ``P1`` calls
+``prctl(PR_SCHED_CORE_SHARE, 200)``, the kernel makes ``P1`` share a core with ``P2``.
+The kernel performs ptrace access mode checks before granting the request.
+
+.. note:: This operation is not commutative. P1 calling
+          ``prctl(PR_SCHED_CORE_SHARE, pidof(P2)`` is not the same as P2 calling the
+          same for P1. The former case is P1 joining P2's group of processes
+          (which P2 would have joined with ``prctl(2)`` prior to P1's ``prctl(2)``).
+
+.. note:: The core-sharing granted with prctl(2) will be subject to
+          core-sharing restrictions specified by the CGroup interface. For example
+          if P1 and P2 are a part of 2 different tagged CGroups, then they will
+          not share a core even if a prctl(2) call is made. This is analogous
+          to how affinities are set using the cpuset interface.
+
+It is important to note that, on a ``CLONE_THREAD`` ``clone(2)`` syscall, the child
+will be assigned the same tag as its parent and thus be allowed to share a core
+with them. is design choice is because, for the security usecase, a
+``CLONE_THREAD`` child can access its parent's address space anyway, so there's
+no point in not allowing them to share a core. If a different behavior is
+desired, the child thread can call ``prctl(2)`` as needed.  This behavior is
+specific to the ``prctl(2)`` interface. For the CGroup interface, the child of a
+fork always share's a core with its parent's.  On the other hand, if a parent
+was previously tagged via ``prctl(2)`` and does a regular ``fork(2)`` syscall, the
+child will receive a unique tag.
+
+Design/Implementation
+---------------------
+Each task that is tagged is assigned a cookie internally in the kernel. As
+mentioned in `Usage`_, tasks with the same cookie value are assumed to trust
+each other and share a core.
+
+The basic idea is that, every schedule event tries to select tasks for all the
+siblings of a core such that all the selected tasks running on a core are
+trusted (same cookie) at any point in time. Kernel threads are assumed trusted.
+The idle task is considered special, in that it trusts every thing.
+
+During a ``schedule()`` event on any sibling of a core, the highest priority task for
+that core is picked and assigned to the sibling calling ``schedule()`` if it has it
+enqueued. For rest of the siblings in the core, highest priority task with the
+same cookie is selected if there is one runnable in their individual run
+queues. If a task with same cookie is not available, the idle task is selected.
+Idle task is globally trusted.
+
+Once a task has been selected for all the siblings in the core, an IPI is sent to
+siblings for whom a new task was selected. Siblings on receiving the IPI, will
+switch to the new task immediately. If an idle task is selected for a sibling,
+then the sibling is considered to be in a `forced idle` state. i.e., it may
+have tasks on its on runqueue to run, however it will still have to run idle.
+More on this in the next section.
+
+Forced-idling of tasks
+----------------------
+The scheduler tries its best to find tasks that trust each other such that all
+tasks selected to be scheduled are of the highest priority in a core.  However,
+it is possible that some runqueues had tasks that were incompatibile with the
+highest priority ones in the core. Favoring security over fairness, one or more
+siblings could be forced to select a lower priority task if the highest
+priority task is not trusted with respect to the core wide highest priority
+task.  If a sibling does not have a trusted task to run, it will be forced idle
+by the scheduler(idle thread is scheduled to run).
+
+When the highest priorty task is selected to run, a reschedule-IPI is sent to
+the sibling to force it into idle. This results in 4 cases which need to be
+considered depending on whether a VM or a regular usermode process was running
+on either HT::
+
+          HT1 (attack)            HT2 (victim)
+   A      idle -> user space      user space -> idle
+   B      idle -> user space      guest -> idle
+   C      idle -> guest           user space -> idle
+   D      idle -> guest           guest -> idle
+
+Note that for better performance, we do not wait for the destination CPU
+(victim) to enter idle mode. This is because the sending of the IPI would bring
+the destination CPU immediately into kernel mode from user space, or VMEXIT
+in the case of  guests. At best, this would only leak some scheduler metadata
+which may not be worth protecting. It is also possible that the IPI is received
+too late on some architectures, but this has not been observed in the case of
+x86.
+
+Kernel protection from untrusted tasks
+--------------------------------------
+The scheduler on its own cannot protect the kernel executing concurrently with
+an untrusted task in a core. This is because the scheduler is unaware of
+interrupts/syscalls at scheduling time. To mitigate this, an IPI is sent to
+siblings on kernel entry (syscall and IRQ). This IPI forces the sibling to enter
+kernel mode and wait before returning to user until all siblings of the
+core have left kernel mode. This process is also known as stunning.  For good
+performance, an IPI is sent only to a sibling only if it is running a tagged
+task. If a sibling is running a kernel thread or is idle, no IPI is sent.
+
+The kernel protection feature can be turned off on the kernel command line by
+passing ``sched_core_protect_kernel=0``.
+
+Other alternative ideas discussed for kernel protection are listed below just
+for completeness. They all have limitations:
+
+1. Changing interrupt affinities to a trusted core which does not execute untrusted tasks
+#########################################################################################
+By changing the interrupt affinities to a designated safe-CPU which runs
+only trusted tasks, IRQ data can be protected. One issue is this involves
+giving up a full CPU core of the system to run safe tasks. Another is that,
+per-cpu interrupts such as the local timer interrupt cannot have their
+affinity changed. also, sensitive timer callbacks such as the random entropy timer
+can run in softirq on return from these interrupts and expose sensitive
+data. In the future, that could be mitigated by forcing softirqs into threaded
+mode by utilizing a mechanism similar to ``CONFIG_PREEMPT_RT``.
+
+Yet another issue with this is, for multiqueue devices with managed
+interrupts, the IRQ affinities cannot be changed however it could be
+possible to force a reduced number of queues which would in turn allow to
+shield one or two CPUs from such interrupts and queue handling for the price
+of indirection.
+
+2. Running IRQs as threaded-IRQs
+################################
+This would result in forcing IRQs into the scheduler which would then provide
+the process-context mitigation. However, not all interrupts can be threaded.
+Also this does nothing about syscall entries.
+
+3. Kernel Address Space Isolation
+#################################
+System calls could run in a much restricted address space which is
+guarenteed not to leak any sensitive data. There are practical limitation in
+implementing this - the main concern being how to decide on an address space
+that is guarenteed to not have any sensitive data.
+
+4. Limited cookie-based protection
+##################################
+On a system call, change the cookie to the system trusted cookie and initiate a
+schedule event. This would be better than pausing all the siblings during the
+entire duration for the system call, but still would be a huge hit to the
+performance.
+
+Trust model
+-----------
+Core scheduling maintains trust relationships amongst groups of tasks by
+assigning the tag of them with the same cookie value.
+When a system with core scheduling boots, all tasks are considered to trust
+each other. This is because the core scheduler does not have information about
+trust relationships until userspace uses the above mentioned interfaces, to
+communicate them. In other words, all tasks have a default cookie value of 0.
+and are considered system-wide trusted. The stunning of siblings running
+cookie-0 tasks is also avoided.
+
+Once userspace uses the above mentioned interfaces to group sets of tasks, tasks
+within such groups are considered to trust each other, but do not trust those
+outside. Tasks outside the group also don't trust tasks within.
+
+Limitations
+-----------
+Core scheduling tries to guarentee that only trusted tasks run concurrently on a
+core. But there could be small window of time during which untrusted tasks run
+concurrently or kernel could be running concurrently with a task not trusted by
+kernel.
+
+1. IPI processing delays
+########################
+Core scheduling selects only trusted tasks to run together. IPI is used to notify
+the siblings to switch to the new task. But there could be hardware delays in
+receiving of the IPI on some arch (on x86, this has not been observed). This may
+cause an attacker task to start running on a cpu before its siblings receive the
+IPI. Even though cache is flushed on entry to user mode, victim tasks on siblings
+may populate data in the cache and micro acrhitectural buffers after the attacker
+starts to run and this is a possibility for data leak.
+
+Open cross-HT issues that core scheduling does not solve
+--------------------------------------------------------
+1. For MDS
+##########
+Core scheduling cannot protect against MDS attacks between an HT running in
+user mode and another running in kernel mode. Even though both HTs run tasks
+which trust each other, kernel memory is still considered untrusted. Such
+attacks are possible for any combination of sibling CPU modes (host or guest mode).
+
+2. For L1TF
+###########
+Core scheduling cannot protect against a L1TF guest attackers exploiting a
+guest or host victim. This is because the guest attacker can craft invalid
+PTEs which are not inverted due to a vulnerable guest kernel. The only
+solution is to disable EPT.
+
+For both MDS and L1TF, if the guest vCPU is configured to not trust each
+other (by tagging separately), then the guest to guest attacks would go away.
+Or it could be a system admin policy which considers guest to guest attacks as
+a guest problem.
+
+Another approach to resolve these would be to make every untrusted task on the
+system to not trust every other untrusted task. While this could reduce
+parallelism of the untrusted tasks, it would still solve the above issues while
+allowing system processes (trusted tasks) to share a core.
+
+Use cases
+---------
+The main use case for Core scheduling is mitigating the cross-HT vulnerabilities
+with SMT enabled. There are other use cases where this feature could be used:
+
+- Isolating tasks that needs a whole core: Examples include realtime tasks, tasks
+  that uses SIMD instructions etc.
+- Gang scheduling: Requirements for a group of tasks that needs to be scheduled
+  together could also be realized using core scheduling. One example is vcpus of
+  a VM.
+
+Future work
+-----------
+Skipping per-HT mitigations if task is trusted
+##############################################
+If core scheduling is enabled, by default all tasks trust each other as
+mentioned above. In such scenario, it may be desirable to skip the same-HT
+mitigations on return to the trusted user-mode to improve performance.
Documentation/admin-guide/hw-vuln/index.rst +1 -0
index 21710f8609fe..361ccbbd9e54 100644
@@ -16,3 +16,4 @@ are configurable at compile, boot or run time.
    multihit.rst
    special-register-buffer-data-sampling.rst
    l1d_flush.rst
+   core-scheduling.rst
Signature · 1 line
2.29.0.rc1.297.gfa9743e501-goog
reviewer
#89 lore raw

Re: [PATCH v8 -tip 25/26] Documentation: Add core scheduling documentation

Hi Joel,

Quoted 29 lines · Joel Fernandes (Google)

Document the usecases, design and interfaces for core scheduling.

---
  .../admin-guide/hw-vuln/core-scheduling.rst   | 312 ++++++++++++++++++
  Documentation/admin-guide/hw-vuln/index.rst   |   1 +
  2 files changed, 313 insertions(+)

create mode 100644 Documentation/admin-guide/hw-vuln/core-scheduling.rst

Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +12 -0
new file mode 100644
index 000000000000..eacafbb8fa3f
@@ -0,0 +1,312 @@
+Core Scheduling
+***************
+Core scheduling support allows userspace to define groups of tasks that can
+share a core. These groups can be specified either for security usecases (one
+group of tasks don't trust another), or for performance usecases (some
+workloads may benefit from running on the same core as they don't need the same
+hardware resources of the shared core).
+
+Security usecase
+----------------
+A cross-HT attack involves the attacker and victim running on different
+Hyper Threads of the same core. MDS and L1TF are examples of such attacks.

Documentation/admin-guide/hw-vuln/core-scheduling.rst line 22

Quoted 10 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +10 -0
+Without core scheduling, the only full mitigation of cross-HT attacks is to
+disable Hyper Threading (HT). Core scheduling allows HT to be turned on safely
+by ensuring that trusted tasks can share a core. This increase in core sharing
+can improvement performance, however it is not guaranteed that performance will
+always improve, though that is seen to be the case with a number of real world
+workloads. In theory, core scheduling aims to perform at least as good as when
+Hyper Threading is disabled. In practise, this is mostly the case though not
+always: as synchronizing scheduling decisions across 2 or more CPUs in a core
+involves additional overhead - especially when the system is lightly loaded
+(``total_threads <= N/2``).

N is number of CPUs?

Quoted 7 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +7 -0
+
+Usage
+-----
+Core scheduling support is enabled via the ``CONFIG_SCHED_CORE`` config option.
+Using this feature, userspace defines groups of tasks that trust each other.
+The core scheduler uses this information to make sure that tasks that do not
+trust each other will never run simultaneously on a core, while doing its best

Documentation/admin-guide/hw-vuln/core-scheduling.rst line 39

Quoted 10 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +10 -0
+to satisfy the system's scheduling requirements.
+
+There are 2 ways to use core-scheduling:
+
+CGroup
+######
+Core scheduling adds additional files to the CPU controller CGroup:
+
+* ``cpu.tag``
+Writing ``1`` into this file results in all tasks in the group get tagged. This

getting
or being

Documentation/admin-guide/hw-vuln/core-scheduling.rst line 43

Quoted 4 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +4 -0
+results in all the CGroup's tasks allowed to run concurrently on a core's
+hyperthreads (also called siblings).
+
+The file being a value of ``0`` means the tag state of the CGroup is inheritted

inherited

Quoted 8 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +8 -0
+from its parent hierarchy. If any ancestor of the CGroup is tagged, then the
+group is tagged.
+
+.. note:: Once a CGroup is tagged via cpu.tag, it is not possible to set this
+          for any descendant of the tagged group. For finer grained control, the
+          ``cpu.tag_color`` file described next may be used.
+
+.. note:: When a CGroup is not tagged, all the tasks within the group can share

Documentation/admin-guide/hw-vuln/core-scheduling.rst line 61

Quoted 10 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +10 -0
+          a core with kernel threads and untagged system threads. For this reason,
+          if a group has ``cpu.tag`` of 0, it is considered to be trusted.
+
+* ``cpu.tag_color``
+For finer grained control over core sharing, a color can also be set in
+addition to the tag. This allows to further control core sharing between child
+CGroups within an already tagged CGroup. The color and the tag are both used to
+generate a `cookie` which is used by the scheduler to identify the group.
+
+Upto 256 different colors can be set (0-255) by writing into this file.

Up to

Documentation/admin-guide/hw-vuln/core-scheduling.rst line 65

Quoted 4 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +4 -0
+
+A sample real-world usage of this file follows:
+
+Google uses DAC controls to make ``cpu.tag`` writeable only by root and the

$search tells me "writable".

Quoted 17 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +17 -0
+``cpu.tag_color`` can be changed by anyone.
+
+The hierarchy looks like this:
+::
+  Root group
+     / \
+    A   B    (These are created by the root daemon - borglet).
+   / \   \
+  C   D   E  (These are created by AppEngine within the container).
+
+A and B are containers for 2 different jobs or apps that are created by a root
+daemon called borglet. borglet then tags each of these group with the ``cpu.tag``
+file. The job itself can create additional child CGroups which are colored by
+the container's AppEngine with the ``cpu.tag_color`` file.
+
+The reason why Google uses this 2-level tagging system is that AppEngine wants to
+allow a subset of child CGroups within a tagged parent CGroup to be co-scheduled on a

Documentation/admin-guide/hw-vuln/core-scheduling.rst line 92

Quoted 10 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +10 -0
+core while not being co-scheduled with other child CGroups. Think of these
+child CGroups as belonging to the same customer or project.  Because these
+child CGroups are created by AppEngine, they are not tracked by borglet (the
+root daemon), therefore borglet won't have a chance to set a color for them.
+That's where cpu.tag_color file comes in. A color could be set by AppEngine,
+and once set, the normal tasks within the subcgroup would not be able to
+overwrite it. This is enforced by promoting the permission of the
+``cpu.tag_color`` file in cgroupfs.
+
+The color is an 8-bit value allowing for upto 256 unique colors.

up to

Documentation/admin-guide/hw-vuln/core-scheduling.rst line 96

Quoted 4 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +4 -0
+
+.. note:: Once a CGroup is colored, none of its descendants can be re-colored. Also
+          coloring of a CGroup is possible only if either the group or one of its
+          ancestors were tagged via the ``cpu.tag`` file.

was

Quoted 13 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +13 -0
+
+prctl interface
+###############
+A ``prtcl(2)`` command ``PR_SCHED_CORE_SHARE`` is available to a process to request
+sharing a core with another process.  For example, consider 2 processes ``P1``
+and ``P2`` with PIDs 100 and 200. If process ``P1`` calls
+``prctl(PR_SCHED_CORE_SHARE, 200)``, the kernel makes ``P1`` share a core with ``P2``.
+The kernel performs ptrace access mode checks before granting the request.
+
+.. note:: This operation is not commutative. P1 calling
+          ``prctl(PR_SCHED_CORE_SHARE, pidof(P2)`` is not the same as P2 calling the
+          same for P1. The former case is P1 joining P2's group of processes
+          (which P2 would have joined with ``prctl(2)`` prior to P1's ``prctl(2)``).

Documentation/admin-guide/hw-vuln/core-scheduling.rst line 119

Quoted 10 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +10 -0
+
+.. note:: The core-sharing granted with prctl(2) will be subject to
+          core-sharing restrictions specified by the CGroup interface. For example
+          if P1 and P2 are a part of 2 different tagged CGroups, then they will
+          not share a core even if a prctl(2) call is made. This is analogous
+          to how affinities are set using the cpuset interface.
+
+It is important to note that, on a ``CLONE_THREAD`` ``clone(2)`` syscall, the child
+will be assigned the same tag as its parent and thus be allowed to share a core
+with them. is design choice is because, for the security usecase, a
              ^^^ missing subject ...

Documentation/admin-guide/hw-vuln/core-scheduling.rst line 124

Quoted 5 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +5 -0
+``CLONE_THREAD`` child can access its parent's address space anyway, so there's
+no point in not allowing them to share a core. If a different behavior is
+desired, the child thread can call ``prctl(2)`` as needed.  This behavior is
+specific to the ``prctl(2)`` interface. For the CGroup interface, the child of a
+fork always share's a core with its parent's.  On the other hand, if a parent

shares with its parent's what? "parent's" is possessive.

Documentation/admin-guide/hw-vuln/core-scheduling.rst line 137

Quoted 13 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +13 -0
+was previously tagged via ``prctl(2)`` and does a regular ``fork(2)`` syscall, the
+child will receive a unique tag.
+
+Design/Implementation
+---------------------
+Each task that is tagged is assigned a cookie internally in the kernel. As
+mentioned in `Usage`_, tasks with the same cookie value are assumed to trust
+each other and share a core.
+
+The basic idea is that, every schedule event tries to select tasks for all the
+siblings of a core such that all the selected tasks running on a core are
+trusted (same cookie) at any point in time. Kernel threads are assumed trusted.
+The idle task is considered special, in that it trusts every thing.

or everything.

Documentation/admin-guide/hw-vuln/core-scheduling.rst line 140

Quoted 3 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +3 -0
+
+During a ``schedule()`` event on any sibling of a core, the highest priority task for
+that core is picked and assigned to the sibling calling ``schedule()`` if it has it

too many it it
They are not the same "it,", are they?

Documentation/admin-guide/hw-vuln/core-scheduling.rst line 147

Quoted 7 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +7 -0
+enqueued. For rest of the siblings in the core, highest priority task with the
+same cookie is selected if there is one runnable in their individual run
+queues. If a task with same cookie is not available, the idle task is selected.
+Idle task is globally trusted.
+
+Once a task has been selected for all the siblings in the core, an IPI is sent to
+siblings for whom a new task was selected. Siblings on receiving the IPI, will

no comma^

Documentation/admin-guide/hw-vuln/core-scheduling.rst line 149

Quoted 2 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +2 -0
+switch to the new task immediately. If an idle task is selected for a sibling,
+then the sibling is considered to be in a `forced idle` state. i.e., it may
                                                                  I.e.,

Documentation/admin-guide/hw-vuln/core-scheduling.rst line 157

Quoted 8 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +8 -0
+have tasks on its on runqueue to run, however it will still have to run idle.
+More on this in the next section.
+
+Forced-idling of tasks
+----------------------
+The scheduler tries its best to find tasks that trust each other such that all
+tasks selected to be scheduled are of the highest priority in a core.  However,
+it is possible that some runqueues had tasks that were incompatibile with the

incompatible

Documentation/admin-guide/hw-vuln/core-scheduling.rst line 162

Quoted 5 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +5 -0
+highest priority ones in the core. Favoring security over fairness, one or more
+siblings could be forced to select a lower priority task if the highest
+priority task is not trusted with respect to the core wide highest priority
+task.  If a sibling does not have a trusted task to run, it will be forced idle
+by the scheduler(idle thread is scheduled to run).

scheduler (idle

Documentation/admin-guide/hw-vuln/core-scheduling.rst line 164

Quoted 2 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +2 -0
+
+When the highest priorty task is selected to run, a reschedule-IPI is sent to

priority

Quoted 4 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +4 -0
+the sibling to force it into idle. This results in 4 cases which need to be
+considered depending on whether a VM or a regular usermode process was running
+on either HT::
+

Documentation/admin-guide/hw-vuln/core-scheduling.rst line 178

Quoted 10 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +10 -0
+          HT1 (attack)            HT2 (victim)
+   A      idle -> user space      user space -> idle
+   B      idle -> user space      guest -> idle
+   C      idle -> guest           user space -> idle
+   D      idle -> guest           guest -> idle
+
+Note that for better performance, we do not wait for the destination CPU
+(victim) to enter idle mode. This is because the sending of the IPI would bring
+the destination CPU immediately into kernel mode from user space, or VMEXIT
+in the case of  guests. At best, this would only leak some scheduler metadata

drop one space ^^

Quoted 18 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +18 -0
+which may not be worth protecting. It is also possible that the IPI is received
+too late on some architectures, but this has not been observed in the case of
+x86.
+
+Kernel protection from untrusted tasks
+--------------------------------------
+The scheduler on its own cannot protect the kernel executing concurrently with
+an untrusted task in a core. This is because the scheduler is unaware of
+interrupts/syscalls at scheduling time. To mitigate this, an IPI is sent to
+siblings on kernel entry (syscall and IRQ). This IPI forces the sibling to enter
+kernel mode and wait before returning to user until all siblings of the
+core have left kernel mode. This process is also known as stunning.  For good
+performance, an IPI is sent only to a sibling only if it is running a tagged
+task. If a sibling is running a kernel thread or is idle, no IPI is sent.
+
+The kernel protection feature can be turned off on the kernel command line by
+passing ``sched_core_protect_kernel=0``.
+

Documentation/admin-guide/hw-vuln/core-scheduling.rst line 206

Quoted 10 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +10 -0
+Other alternative ideas discussed for kernel protection are listed below just
+for completeness. They all have limitations:
+
+1. Changing interrupt affinities to a trusted core which does not execute untrusted tasks
+#########################################################################################
+By changing the interrupt affinities to a designated safe-CPU which runs
+only trusted tasks, IRQ data can be protected. One issue is this involves
+giving up a full CPU core of the system to run safe tasks. Another is that,
+per-cpu interrupts such as the local timer interrupt cannot have their
+affinity changed. also, sensitive timer callbacks such as the random entropy timer

Also,

Documentation/admin-guide/hw-vuln/core-scheduling.rst line 212

Quoted 6 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +6 -0
+can run in softirq on return from these interrupts and expose sensitive
+data. In the future, that could be mitigated by forcing softirqs into threaded
+mode by utilizing a mechanism similar to ``CONFIG_PREEMPT_RT``.
+
+Yet another issue with this is, for multiqueue devices with managed
+interrupts, the IRQ affinities cannot be changed however it could be

changed; however,

Quoted 4 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +4 -0
+possible to force a reduced number of queues which would in turn allow to
+shield one or two CPUs from such interrupts and queue handling for the price
+of indirection.
+

Documentation/admin-guide/hw-vuln/core-scheduling.rst line 226

Quoted 10 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +10 -0
+2. Running IRQs as threaded-IRQs
+################################
+This would result in forcing IRQs into the scheduler which would then provide
+the process-context mitigation. However, not all interrupts can be threaded.
+Also this does nothing about syscall entries.
+
+3. Kernel Address Space Isolation
+#################################
+System calls could run in a much restricted address space which is
+guarenteed not to leak any sensitive data. There are practical limitation in

guaranteed limitations in

Documentation/admin-guide/hw-vuln/core-scheduling.rst line 228

Quoted 2 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +2 -0
+implementing this - the main concern being how to decide on an address space
+that is guarenteed to not have any sensitive data.

guaranteed

Quoted 16 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +16 -0
+
+4. Limited cookie-based protection
+##################################
+On a system call, change the cookie to the system trusted cookie and initiate a
+schedule event. This would be better than pausing all the siblings during the
+entire duration for the system call, but still would be a huge hit to the
+performance.
+
+Trust model
+-----------
+Core scheduling maintains trust relationships amongst groups of tasks by
+assigning the tag of them with the same cookie value.
+When a system with core scheduling boots, all tasks are considered to trust
+each other. This is because the core scheduler does not have information about
+trust relationships until userspace uses the above mentioned interfaces, to
+communicate them. In other words, all tasks have a default cookie value of 0.

Documentation/admin-guide/hw-vuln/core-scheduling.rst line 254

Quoted 10 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +10 -0
+and are considered system-wide trusted. The stunning of siblings running
+cookie-0 tasks is also avoided.
+
+Once userspace uses the above mentioned interfaces to group sets of tasks, tasks
+within such groups are considered to trust each other, but do not trust those
+outside. Tasks outside the group also don't trust tasks within.
+
+Limitations
+-----------
+Core scheduling tries to guarentee that only trusted tasks run concurrently on a

guarantee

Documentation/admin-guide/hw-vuln/core-scheduling.rst line 264

Quoted 10 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +10 -0
+core. But there could be small window of time during which untrusted tasks run
+concurrently or kernel could be running concurrently with a task not trusted by
+kernel.
+
+1. IPI processing delays
+########################
+Core scheduling selects only trusted tasks to run together. IPI is used to notify
+the siblings to switch to the new task. But there could be hardware delays in
+receiving of the IPI on some arch (on x86, this has not been observed). This may
+cause an attacker task to start running on a cpu before its siblings receive the

CPU

Documentation/admin-guide/hw-vuln/core-scheduling.rst line 266

Quoted 2 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +2 -0
+IPI. Even though cache is flushed on entry to user mode, victim tasks on siblings
+may populate data in the cache and micro acrhitectural buffers after the attacker

architectural

Quoted 4 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +4 -0
+starts to run and this is a possibility for data leak.
+
+Open cross-HT issues that core scheduling does not solve
+--------------------------------------------------------

Documentation/admin-guide/hw-vuln/core-scheduling.rst line 280

Quoted 10 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +10 -0
+1. For MDS
+##########
+Core scheduling cannot protect against MDS attacks between an HT running in
+user mode and another running in kernel mode. Even though both HTs run tasks
+which trust each other, kernel memory is still considered untrusted. Such
+attacks are possible for any combination of sibling CPU modes (host or guest mode).
+
+2. For L1TF
+###########
+Core scheduling cannot protect against a L1TF guest attackers exploiting a

an attacker

Documentation/admin-guide/hw-vuln/core-scheduling.rst line 283

Quoted 3 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +3 -0
+guest or host victim. This is because the guest attacker can craft invalid
+PTEs which are not inverted due to a vulnerable guest kernel. The only
+solution is to disable EPT.

huh? what is EPT? where is it documented/discussed?

Quoted 10 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +10 -0
+
+For both MDS and L1TF, if the guest vCPU is configured to not trust each
+other (by tagging separately), then the guest to guest attacks would go away.
+Or it could be a system admin policy which considers guest to guest attacks as
+a guest problem.
+
+Another approach to resolve these would be to make every untrusted task on the
+system to not trust every other untrusted task. While this could reduce
+parallelism of the untrusted tasks, it would still solve the above issues while
+allowing system processes (trusted tasks) to share a core.

Documentation/admin-guide/hw-vuln/core-scheduling.rst line 303

Quoted 10 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +10 -0
+
+Use cases
+---------
+The main use case for Core scheduling is mitigating the cross-HT vulnerabilities
+with SMT enabled. There are other use cases where this feature could be used:
+
+- Isolating tasks that needs a whole core: Examples include realtime tasks, tasks
+  that uses SIMD instructions etc.
+- Gang scheduling: Requirements for a group of tasks that needs to be scheduled
+  together could also be realized using core scheduling. One example is vcpus of

vCPUs

Quoted 15 lines
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +9 -0
+  a VM.
+
+Future work
+-----------
+Skipping per-HT mitigations if task is trusted
+##############################################
+If core scheduling is enabled, by default all tasks trust each other as
+mentioned above. In such scenario, it may be desirable to skip the same-HT
+mitigations on return to the trusted user-mode to improve performance.
Documentation/admin-guide/hw-vuln/index.rst +0 -0
index 21710f8609fe..361ccbbd9e54 100644

Documentation/admin-guide/hw-vuln/index.rst are configurable at compile, boot or run time. · line 19

Quoted 4 lines
Documentation/admin-guide/hw-vuln/index.rst +1 -0
@@ -16,3 +16,4 @@ are configurable at compile, boot or run time.
     multihit.rst
     special-register-buffer-data-sampling.rst
     l1d_flush.rst
+   core-scheduling.rst

Might be an indentation problem there? Can't tell for sure.

thanks.

author Tested-by Julien Desfossez
#90 lore raw

Re: [PATCH v8 -tip 25/26] Documentation: Add core scheduling documentation

Quoted 8 lines · Randy Dunlap

Hi Joel,

Quoted 5 lines · Joel Fernandes (Google)

Document the usecases, design and interfaces for core scheduling.

All fixed as below, updated and thanks!

---8<-----------------------

From: "Joel Fernandes (Google)" <[email protected]>
Subject: [PATCH] Documentation: Add core scheduling documentation

Document the usecases, design and interfaces for core scheduling.

---
 .../admin-guide/hw-vuln/core-scheduling.rst   | 313 ++++++++++++++++++
 Documentation/admin-guide/hw-vuln/index.rst   |   1 +
 2 files changed, 314 insertions(+)
 create mode 100644 Documentation/admin-guide/hw-vuln/core-scheduling.rst
Documentation/admin-guide/hw-vuln/core-scheduling.rst new file +313 -0
new file mode 100644
index 000000000000..c7399809c74d
@@ -0,0 +1,313 @@
+Core Scheduling
+***************
+Core scheduling support allows userspace to define groups of tasks that can
+share a core. These groups can be specified either for security usecases (one
+group of tasks don't trust another), or for performance usecases (some
+workloads may benefit from running on the same core as they don't need the same
+hardware resources of the shared core).
+
+Security usecase
+----------------
+A cross-HT attack involves the attacker and victim running on different
+Hyper Threads of the same core. MDS and L1TF are examples of such attacks.
+Without core scheduling, the only full mitigation of cross-HT attacks is to
+disable Hyper Threading (HT). Core scheduling allows HT to be turned on safely
+by ensuring that trusted tasks can share a core. This increase in core sharing
+can improvement performance, however it is not guaranteed that performance will
+always improve, though that is seen to be the case with a number of real world
+workloads. In theory, core scheduling aims to perform at least as good as when
+Hyper Threading is disabled. In practice, this is mostly the case though not
+always: as synchronizing scheduling decisions across 2 or more CPUs in a core
+involves additional overhead - especially when the system is lightly loaded
+(``total_threads <= N/2``, where N is the total number of CPUs).
+
+Usage
+-----
+Core scheduling support is enabled via the ``CONFIG_SCHED_CORE`` config option.
+Using this feature, userspace defines groups of tasks that trust each other.
+The core scheduler uses this information to make sure that tasks that do not
+trust each other will never run simultaneously on a core, while doing its best
+to satisfy the system's scheduling requirements.
+
+There are 2 ways to use core-scheduling:
+
+CGroup
+######
+Core scheduling adds additional files to the CPU controller CGroup:
+
+* ``cpu.tag``
+Writing ``1`` into this file results in all tasks in the group getting tagged.
+This results in all the CGroup's tasks allowed to run concurrently on a core's
+hyperthreads (also called siblings).
+
+The file being a value of ``0`` means the tag state of the CGroup is inherited
+from its parent hierarchy. If any ancestor of the CGroup is tagged, then the
+group is tagged.
+
+.. note:: Once a CGroup is tagged via cpu.tag, it is not possible to set this
+          for any descendant of the tagged group. For finer grained control, the
+          ``cpu.tag_color`` file described next may be used.
+
+.. note:: When a CGroup is not tagged, all the tasks within the group can share
+          a core with kernel threads and untagged system threads. For this reason,
+          if a group has ``cpu.tag`` of 0, it is considered to be trusted.
+
+* ``cpu.tag_color``
+For finer grained control over core sharing, a color can also be set in
+addition to the tag. This allows to further control core sharing between child
+CGroups within an already tagged CGroup. The color and the tag are both used to
+generate a `cookie` which is used by the scheduler to identify the group.
+
+Up to 256 different colors can be set (0-255) by writing into this file.
+
+A sample real-world usage of this file follows:
+
+Google uses DAC controls to make ``cpu.tag`` writable only by root and the
+``cpu.tag_color`` can be changed by anyone.
+
+The hierarchy looks like this:
+::
+  Root group
+     / \
+    A   B    (These are created by the root daemon - borglet).
+   / \   \
+  C   D   E  (These are created by AppEngine within the container).
+
+A and B are containers for 2 different jobs or apps that are created by a root
+daemon called borglet. borglet then tags each of these group with the ``cpu.tag``
+file. The job itself can create additional child CGroups which are colored by
+the container's AppEngine with the ``cpu.tag_color`` file.
+
+The reason why Google uses this 2-level tagging system is that AppEngine wants to
+allow a subset of child CGroups within a tagged parent CGroup to be co-scheduled on a
+core while not being co-scheduled with other child CGroups. Think of these
+child CGroups as belonging to the same customer or project.  Because these
+child CGroups are created by AppEngine, they are not tracked by borglet (the
+root daemon), therefore borglet won't have a chance to set a color for them.
+That's where cpu.tag_color file comes in. A color could be set by AppEngine,
+and once set, the normal tasks within the subcgroup would not be able to
+overwrite it. This is enforced by promoting the permission of the
+``cpu.tag_color`` file in cgroupfs.
+
+The color is an 8-bit value allowing for up to 256 unique colors.
+
+.. note:: Once a CGroup is colored, none of its descendants can be re-colored. Also
+          coloring of a CGroup is possible only if either the group or one of its
+          ancestors was tagged via the ``cpu.tag`` file.
+
+prctl interface
+###############
+A ``prtcl(2)`` command ``PR_SCHED_CORE_SHARE`` is available to a process to request
+sharing a core with another process.  For example, consider 2 processes ``P1``
+and ``P2`` with PIDs 100 and 200. If process ``P1`` calls
+``prctl(PR_SCHED_CORE_SHARE, 200)``, the kernel makes ``P1`` share a core with ``P2``.
+The kernel performs ptrace access mode checks before granting the request.
+
+.. note:: This operation is not commutative. P1 calling
+          ``prctl(PR_SCHED_CORE_SHARE, pidof(P2)`` is not the same as P2 calling the
+          same for P1. The former case is P1 joining P2's group of processes
+          (which P2 would have joined with ``prctl(2)`` prior to P1's ``prctl(2)``).
+
+.. note:: The core-sharing granted with prctl(2) will be subject to
+          core-sharing restrictions specified by the CGroup interface. For example
+          if P1 and P2 are a part of 2 different tagged CGroups, then they will
+          not share a core even if a prctl(2) call is made. This is analogous
+          to how affinities are set using the cpuset interface.
+
+It is important to note that, on a ``CLONE_THREAD`` ``clone(2)`` syscall, the child
+will be assigned the same tag as its parent and thus be allowed to share a core
+with them. This design choice is because, for the security usecase, a
+``CLONE_THREAD`` child can access its parent's address space anyway, so there's
+no point in not allowing them to share a core. If a different behavior is
+desired, the child thread can call ``prctl(2)`` as needed.  This behavior is
+specific to the ``prctl(2)`` interface. For the CGroup interface, the child of a
+fork always shares a core with its parent.  On the other hand, if a parent
+was previously tagged via ``prctl(2)`` and does a regular ``fork(2)`` syscall, the
+child will receive a unique tag.
+
+Design/Implementation
+---------------------
+Each task that is tagged is assigned a cookie internally in the kernel. As
+mentioned in `Usage`_, tasks with the same cookie value are assumed to trust
+each other and share a core.
+
+The basic idea is that, every schedule event tries to select tasks for all the
+siblings of a core such that all the selected tasks running on a core are
+trusted (same cookie) at any point in time. Kernel threads are assumed trusted.
+The idle task is considered special, as it trusts everything and everything
+trusts it.
+
+During a schedule() event on any sibling of a core, the highest priority task on
+the sibling's core is picked and assigned to the sibling calling schedule(), if
+the sibling has the task enqueued. For rest of the siblings in the core,
+highest priority task with the same cookie is selected if there is one runnable
+in their individual run queues. If a task with same cookie is not available,
+the idle task is selected.  Idle task is globally trusted.
+
+Once a task has been selected for all the siblings in the core, an IPI is sent to
+siblings for whom a new task was selected. Siblings on receiving the IPI will
+switch to the new task immediately. If an idle task is selected for a sibling,
+then the sibling is considered to be in a `forced idle` state. I.e., it may
+have tasks on its on runqueue to run, however it will still have to run idle.
+More on this in the next section.
+
+Forced-idling of tasks
+----------------------
+The scheduler tries its best to find tasks that trust each other such that all
+tasks selected to be scheduled are of the highest priority in a core.  However,
+it is possible that some runqueues had tasks that were incompatible with the
+highest priority ones in the core. Favoring security over fairness, one or more
+siblings could be forced to select a lower priority task if the highest
+priority task is not trusted with respect to the core wide highest priority
+task.  If a sibling does not have a trusted task to run, it will be forced idle
+by the scheduler (idle thread is scheduled to run).
+
+When the highest priority task is selected to run, a reschedule-IPI is sent to
+the sibling to force it into idle. This results in 4 cases which need to be
+considered depending on whether a VM or a regular usermode process was running
+on either HT::
+
+          HT1 (attack)            HT2 (victim)
+   A      idle -> user space      user space -> idle
+   B      idle -> user space      guest -> idle
+   C      idle -> guest           user space -> idle
+   D      idle -> guest           guest -> idle
+
+Note that for better performance, we do not wait for the destination CPU
+(victim) to enter idle mode. This is because the sending of the IPI would bring
+the destination CPU immediately into kernel mode from user space, or VMEXIT
+in the case of guests. At best, this would only leak some scheduler metadata
+which may not be worth protecting. It is also possible that the IPI is received
+too late on some architectures, but this has not been observed in the case of
+x86.
+
+Kernel protection from untrusted tasks
+--------------------------------------
+The scheduler on its own cannot protect the kernel executing concurrently with
+an untrusted task in a core. This is because the scheduler is unaware of
+interrupts/syscalls at scheduling time. To mitigate this, an IPI is sent to
+siblings on kernel entry (syscall and IRQ). This IPI forces the sibling to enter
+kernel mode and wait before returning to user until all siblings of the
+core have left kernel mode. This process is also known as stunning.  For good
+performance, an IPI is sent only to a sibling only if it is running a tagged
+task. If a sibling is running a kernel thread or is idle, no IPI is sent.
+
+The kernel protection feature can be turned off on the kernel command line by
+passing ``sched_core_protect_kernel=0``.
+
+Other alternative ideas discussed for kernel protection are listed below just
+for completeness. They all have limitations:
+
+1. Changing interrupt affinities to a trusted core which does not execute untrusted tasks
+#########################################################################################
+By changing the interrupt affinities to a designated safe-CPU which runs
+only trusted tasks, IRQ data can be protected. One issue is this involves
+giving up a full CPU core of the system to run safe tasks. Another is that,
+per-cpu interrupts such as the local timer interrupt cannot have their
+affinity changed. Also, sensitive timer callbacks such as the random entropy timer
+can run in softirq on return from these interrupts and expose sensitive
+data. In the future, that could be mitigated by forcing softirqs into threaded
+mode by utilizing a mechanism similar to ``CONFIG_PREEMPT_RT``.
+
+Yet another issue with this is, for multiqueue devices with managed
+interrupts, the IRQ affinities cannot be changed; however it could be
+possible to force a reduced number of queues which would in turn allow to
+shield one or two CPUs from such interrupts and queue handling for the price
+of indirection.
+
+2. Running IRQs as threaded-IRQs
+################################
+This would result in forcing IRQs into the scheduler which would then provide
+the process-context mitigation. However, not all interrupts can be threaded.
+Also this does nothing about syscall entries.
+
+3. Kernel Address Space Isolation
+#################################
+System calls could run in a much restricted address space which is
+guaranteed not to leak any sensitive data. There are practical limitation in
+implementing this - the main concern being how to decide on an address space
+that is guaranteed to not have any sensitive data.
+
+4. Limited cookie-based protection
+##################################
+On a system call, change the cookie to the system trusted cookie and initiate a
+schedule event. This would be better than pausing all the siblings during the
+entire duration for the system call, but still would be a huge hit to the
+performance.
+
+Trust model
+-----------
+Core scheduling maintains trust relationships amongst groups of tasks by
+assigning the tag of them with the same cookie value.
+When a system with core scheduling boots, all tasks are considered to trust
+each other. This is because the core scheduler does not have information about
+trust relationships until userspace uses the above mentioned interfaces, to
+communicate them. In other words, all tasks have a default cookie value of 0.
+and are considered system-wide trusted. The stunning of siblings running
+cookie-0 tasks is also avoided.
+
+Once userspace uses the above mentioned interfaces to group sets of tasks, tasks
+within such groups are considered to trust each other, but do not trust those
+outside. Tasks outside the group also don't trust tasks within.
+
+Limitations
+-----------
+Core scheduling tries to guarantee that only trusted tasks run concurrently on a
+core. But there could be small window of time during which untrusted tasks run
+concurrently or kernel could be running concurrently with a task not trusted by
+kernel.
+
+1. IPI processing delays
+########################
+Core scheduling selects only trusted tasks to run together. IPI is used to notify
+the siblings to switch to the new task. But there could be hardware delays in
+receiving of the IPI on some arch (on x86, this has not been observed). This may
+cause an attacker task to start running on a CPU before its siblings receive the
+IPI. Even though cache is flushed on entry to user mode, victim tasks on siblings
+may populate data in the cache and micro architectural buffers after the attacker
+starts to run and this is a possibility for data leak.
+
+Open cross-HT issues that core scheduling does not solve
+--------------------------------------------------------
+1. For MDS
+##########
+Core scheduling cannot protect against MDS attacks between an HT running in
+user mode and another running in kernel mode. Even though both HTs run tasks
+which trust each other, kernel memory is still considered untrusted. Such
+attacks are possible for any combination of sibling CPU modes (host or guest mode).
+
+2. For L1TF
+###########
+Core scheduling cannot protect against an L1TF guest attacker exploiting a
+guest or host victim. This is because the guest attacker can craft invalid
+PTEs which are not inverted due to a vulnerable guest kernel. The only
+solution is to disable EPT (Extended Page Tables).
+
+For both MDS and L1TF, if the guest vCPU is configured to not trust each
+other (by tagging separately), then the guest to guest attacks would go away.
+Or it could be a system admin policy which considers guest to guest attacks as
+a guest problem.
+
+Another approach to resolve these would be to make every untrusted task on the
+system to not trust every other untrusted task. While this could reduce
+parallelism of the untrusted tasks, it would still solve the above issues while
+allowing system processes (trusted tasks) to share a core.
+
+Use cases
+---------
+The main use case for Core scheduling is mitigating the cross-HT vulnerabilities
+with SMT enabled. There are other use cases where this feature could be used:
+
+- Isolating tasks that needs a whole core: Examples include realtime tasks, tasks
+  that uses SIMD instructions etc.
+- Gang scheduling: Requirements for a group of tasks that needs to be scheduled
+  together could also be realized using core scheduling. One example is vCPUs of
+  a VM.
+
+Future work
+-----------
+Skipping per-HT mitigations if task is trusted
+##############################################
+If core scheduling is enabled, by default all tasks trust each other as
+mentioned above. In such scenario, it may be desirable to skip the same-HT
+mitigations on return to the trusted user-mode to improve performance.
Documentation/admin-guide/hw-vuln/index.rst +1 -0
index 21710f8609fe..361ccbbd9e54 100644
@@ -16,3 +16,4 @@ are configurable at compile, boot or run time.
    multihit.rst
    special-register-buffer-data-sampling.rst
    l1d_flush.rst
+   core-scheduling.rst
Signature · 1 line
2.29.2.222.g5d2a92d10f8-goog
#91 lore raw

PATCH v8 26/26 sched: Debug bits...

In reply to Joel Fernandes (Google)

From: Peter Zijlstra <[email protected]>

---
 kernel/sched/core.c | 37 ++++++++++++++++++++++++++++++++++++-
 1 file changed, 36 insertions(+), 1 deletion(-)
kernel/sched/core.c +36 -1
index 211e0784675f..61758b5478d8 100644
@@ -127,6 +127,10 @@ static inline bool prio_less(struct task_struct *a, struct task_struct *b)
 
 	int pa = __task_prio(a), pb = __task_prio(b);
 
+	trace_printk("(%s/%d;%d,%Lu,%Lu) ?< (%s/%d;%d,%Lu,%Lu)\n",
+		     a->comm, a->pid, pa, a->se.vruntime, a->dl.deadline,
+		     b->comm, b->pid, pb, b->se.vruntime, b->dl.deadline);
+
 	if (-pa < -pb)
 		return true;
 
@@ -317,12 +321,16 @@ static void __sched_core_enable(void)
 
 	static_branch_enable(&__sched_core_enabled);
 	stop_machine(__sched_core_stopper, (void *)true, NULL);
+
+	printk("core sched enabled\n");
 }
 
 static void __sched_core_disable(void)
 {
 	stop_machine(__sched_core_stopper, (void *)false, NULL);
 	static_branch_disable(&__sched_core_enabled);
+
+	printk("core sched disabled\n");
 }
 
 void sched_core_get(void)
@@ -4978,6 +4986,13 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
 			set_next_task(rq, next);
 		}
 
+		trace_printk("pick pre selected (%u %u %u): %s/%d %lx\n",
+			     rq->core->core_task_seq,
+			     rq->core->core_pick_seq,
+			     rq->core_sched_seq,
+			     next->comm, next->pid,
+			     next->core_cookie);
+
 		rq->core_pick = NULL;
 		return next;
 	}
@@ -5066,6 +5081,8 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
 			 */
 			if (i == cpu && !need_sync && !p->core_cookie) {
 				next = p;
+				trace_printk("unconstrained pick: %s/%d %lx\n",
+					     next->comm, next->pid, next->core_cookie);
 				goto done;
 			}
 
@@ -5074,6 +5091,9 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
 
 			rq_i->core_pick = p;
 
+			trace_printk("cpu(%d): selected: %s/%d %lx\n",
+				     i, p->comm, p->pid, p->core_cookie);
+
 			/*
 			 * If this new candidate is of higher priority than the
 			 * previous; and they're incompatible; we need to wipe
@@ -5090,6 +5110,8 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
 				rq->core->core_cookie = p->core_cookie;
 				max = p;
 
+				trace_printk("max: %s/%d %lx\n", max->comm, max->pid, max->core_cookie);
+
 				if (old_max) {
 					for_each_cpu(j, smt_mask) {
 						if (j == i)
@@ -5120,6 +5142,7 @@ next_class:;
 
 	/* Something should have been selected for current CPU */
 	WARN_ON_ONCE(!next);
+	trace_printk("picked: %s/%d %lx\n", next->comm, next->pid, next->core_cookie);
 
 	/*
 	 * Reschedule siblings
@@ -5155,13 +5178,21 @@ next_class:;
 		}
 
 		/* Did we break L1TF mitigation requirements? */
-		WARN_ON_ONCE(!cookie_match(next, rq_i->core_pick));
+		if (unlikely(!cookie_match(next, rq_i->core_pick))) {
+			trace_printk("[%d]: cookie mismatch. %s/%d/0x%lx/0x%lx\n",
+				     rq_i->cpu, rq_i->core_pick->comm,
+				     rq_i->core_pick->pid,
+				     rq_i->core_pick->core_cookie,
+				     rq_i->core->core_cookie);
+			WARN_ON_ONCE(1);
+		}
 
 		if (rq_i->curr == rq_i->core_pick) {
 			rq_i->core_pick = NULL;
 			continue;
 		}
 
+		trace_printk("IPI(%d)\n", i);
 		resched_curr(rq_i);
 	}
 
@@ -5209,6 +5240,10 @@ static bool try_steal_cookie(int this, int that)
 		if (p->core_occupation > dst->idle->core_occupation)
 			goto next;
 
+		trace_printk("core fill: %s/%d (%d->%d) %d %d %lx\n",
+			     p->comm, p->pid, that, this,
+			     p->core_occupation, dst->idle->core_occupation, cookie);
+
 		p->on_rq = TASK_ON_RQ_MIGRATING;
 		deactivate_task(src, p, 0);
 		set_task_cpu(p, this);
Signature · 1 line
2.29.0.rc1.297.gfa9743e501-goog
#92 lore raw
In reply to Joel Fernandes (Google)
Quoted 61 lines · Joel Fernandes (Google)

Eighth iteration of the Core-Scheduling feature.

Core scheduling is a feature that allows only trusted tasks to run concurrently on cpus sharing compute resources (eg: hyperthreads on a core). The goal is to mitigate the core-level side-channel attacks without requiring to disable SMT (which has a significant impact on performance in some situations). Core scheduling (as of v7) mitigates user-space to user-space attacks and user to kernel attack when one of the siblings enters the kernel via interrupts or system call.

By default, the feature doesn't change any of the current scheduler behavior. The user decides which tasks can run simultaneously on the same core (for now by having them in the same tagged cgroup). When a tag is enabled in a cgroup and a task from that cgroup is running on a hardware thread, the scheduler ensures that only idle or trusted tasks run on the other sibling(s). Besides security concerns, this feature can also be beneficial for RT and performance applications where we want to control how tasks make use of SMT dynamically.

This iteration focuses on the the following stuff:
- Redesigned API.
- Rework of Kernel Protection feature based on Thomas's entry work.
- Rework of hotplug fixes.
- Address review comments in v7

Joel: Both a CGroup and Per-task interface via prctl(2) are provided for configuring core sharing. More details are provided in documentation patch. Kselftests are provided to verify the correctness/rules of the interface.

Julien: TPCC tests showed improvements with core-scheduling. With kernel protection enabled, it does not show any regression. Possibly ASI will improve the performance for those who choose kernel protection (can be toggled through sched_core_protect_kernel sysctl). Results:
v8 average stdev diff
baseline (SMT on) 1197.272 44.78312824
core sched ( kernel protect) 412.9895 45.42734343 -65.51%
core sched (no kernel protect) 686.6515 71.77756931 -42.65%
nosmt 408.667 39.39042872 -65.87%

v8 is rebased on tip/master.

Future work
===========
- Load balancing/Migration fixes for core scheduling. With v6, Load balancing is partially coresched aware, but has some issues w.r.t process/taskgroup weights:
https://lwn.net/ml/linux-kernel/20200225034438.GA617271@z...
- Core scheduling test framework: kselftests, torture tests etc

Changes in v8
=============
- New interface/API implementation
- Joel
- Revised kernel protection patch
- Joel
- Revised Hotplug fixes
- Joel
- Minor bug fixes and address review comments
- Vineeth

Quoted 3 lines

create mode 100644 tools/testing/selftests/sched/config create mode 100644 tools/testing/selftests/sched/test_coresched.c

Adding 4 workloads test results for Core Scheduling v8:

- kernel under test: coresched community v8 from https://git.kernel.org/pub/scm/linux/kernel/git/jfern/linux.git/log/?h=coresched-v5.9
- workloads:
-- A. sysbench cpu (192 threads) + sysbench cpu (192 threads)
-- B. sysbench cpu (192 threads) + sysbench mysql (192 threads, mysqld forced into the same cgroup)
-- C. uperf netperf.xml (192 threads over TCP or UDP protocol separately)
-- D. will-it-scale context_switch via pipe (192 threads)
- test machine setup:
CPU(s): 192
On-line CPU(s) list: 0-191
Thread(s) per core: 2
Core(s) per socket: 48
Socket(s): 2
NUMA node(s): 4
- test results:
-- workload A, no obvious performance drop in cs_on:
+----------------------+------+----------------------+------------------------+
| | ** | sysbench cpu * 192 | sysbench mysql * 192 |
+======================+======+======================+========================+
| cgroup | ** | cg_sysbench_cpu_0 | cg_sysbench_mysql_0 |
+----------------------+------+----------------------+------------------------+
| record_item | ** | Tput_avg (events/s) | Tput_avg (events/s) |
+----------------------+------+----------------------+------------------------+
| coresched_normalized | ** | 1.01 | 0.87 |
+----------------------+------+----------------------+------------------------+
| default_normalized | ** | 1 | 1 |
+----------------------+------+----------------------+------------------------+
| smtoff_normalized | ** | 0.59 | 0.82 |
+----------------------+------+----------------------+------------------------+

-- workload B, no obvious performance drop in cs_on:
+----------------------+------+----------------------+------------------------+
| | ** | sysbench cpu * 192 | sysbench cpu * 192 |
+======================+======+======================+========================+
| cgroup | ** | cg_sysbench_cpu_0 | cg_sysbench_cpu_1 |
+----------------------+------+----------------------+------------------------+
| record_item | ** | Tput_avg (events/s) | Tput_avg (events/s) |
+----------------------+------+----------------------+------------------------+
| coresched_normalized | ** | 1.01 | 0.98 |
+----------------------+------+----------------------+------------------------+
| default_normalized | ** | 1 | 1 |
+----------------------+------+----------------------+------------------------+
| smtoff_normalized | ** | 0.6 | 0.6 |
+----------------------+------+----------------------+------------------------+

-- workload C, known performance drop in cs_on since Core Scheduling v6:
+----------------------+------+---------------------------+---------------------------+
| | ** | uperf netperf TCP * 192 | uperf netperf UDP * 192 |
+======================+======+===========================+===========================+
| cgroup | ** | cg_uperf | cg_uperf |
+----------------------+------+---------------------------+---------------------------+
| record_item | ** | Tput_avg (Gb/s) | Tput_avg (Gb/s) |
+----------------------+------+---------------------------+---------------------------+
| coresched_normalized | ** | 0.46 | 0.48 |
+----------------------+------+---------------------------+---------------------------+
| default_normalized | ** | 1 | 1 |
+----------------------+------+---------------------------+---------------------------+
| smtoff_normalized | ** | 0.82 | 0.79 |
+----------------------+------+---------------------------+---------------------------+

-- workload D, new added syscall workload, performance drop in cs_on:
+----------------------+------+-------------------------------+
| | ** | will-it-scale * 192 |
| | | (pipe based context_switch) |
+======================+======+===============================+
| cgroup | ** | cg_will-it-scale |
+----------------------+------+-------------------------------+
| record_item | ** | threads_avg |
+----------------------+------+-------------------------------+
| coresched_normalized | ** | 0.2 |
+----------------------+------+-------------------------------+
| default_normalized | ** | 1 |
+----------------------+------+-------------------------------+
| smtoff_normalized | ** | 0.89 |
+----------------------+------+-------------------------------+

comments: per internal analyzing, suspected huge amount of spin_lock contention in cs_on, may lead to significant performance drop

- notes on test results record_item:
* coresched_normalized: smton, cs enabled, test result normalized by default value
* default_normalized: smton, cs disabled, test result normalized by default value
* smtoff_normalized: smtoff, test result normalized by default value

#93 lore raw
Quoted 118 lines · Hongyu
Quoted 61 lines · Joel Fernandes (Google)

Eighth iteration of the Core-Scheduling feature.

Core scheduling is a feature that allows only trusted tasks to run concurrently on cpus sharing compute resources (eg: hyperthreads on a core). The goal is to mitigate the core-level side-channel attacks without requiring to disable SMT (which has a significant impact on performance in some situations). Core scheduling (as of v7) mitigates user-space to user-space attacks and user to kernel attack when one of the siblings enters the kernel via interrupts or system call.

By default, the feature doesn't change any of the current scheduler behavior. The user decides which tasks can run simultaneously on the same core (for now by having them in the same tagged cgroup). When a tag is enabled in a cgroup and a task from that cgroup is running on a hardware thread, the scheduler ensures that only idle or trusted tasks run on the other sibling(s). Besides security concerns, this feature can also be beneficial for RT and performance applications where we want to control how tasks make use of SMT dynamically.

This iteration focuses on the the following stuff:
- Redesigned API.
- Rework of Kernel Protection feature based on Thomas's entry work.
- Rework of hotplug fixes.
- Address review comments in v7

Joel: Both a CGroup and Per-task interface via prctl(2) are provided for configuring core sharing. More details are provided in documentation patch. Kselftests are provided to verify the correctness/rules of the interface.

Julien: TPCC tests showed improvements with core-scheduling. With kernel protection enabled, it does not show any regression. Possibly ASI will improve the performance for those who choose kernel protection (can be toggled through sched_core_protect_kernel sysctl). Results:
v8 average stdev diff
baseline (SMT on) 1197.272 44.78312824
core sched ( kernel protect) 412.9895 45.42734343 -65.51%
core sched (no kernel protect) 686.6515 71.77756931 -42.65%
nosmt 408.667 39.39042872 -65.87%

v8 is rebased on tip/master.

Future work
===========
- Load balancing/Migration fixes for core scheduling. With v6, Load balancing is partially coresched aware, but has some issues w.r.t process/taskgroup weights:
https://lwn.net/ml/linux-kernel/20200225034438.GA617271@z...
- Core scheduling test framework: kselftests, torture tests etc

Changes in v8
=============
- New interface/API implementation
- Joel
- Revised kernel protection patch
- Joel
- Revised Hotplug fixes
- Joel
- Minor bug fixes and address review comments
- Vineeth

Quoted 3 lines

create mode 100644 tools/testing/selftests/sched/config create mode 100644 tools/testing/selftests/sched/test_coresched.c

Adding 4 workloads test results for Core Scheduling v8:

- kernel under test: coresched community v8 from https://git.kernel.org/pub/scm/linux/kernel/git/jfern/linux.git/log/?h=coresched-v5.9
- workloads:
-- A. sysbench cpu (192 threads) + sysbench cpu (192 threads)
-- B. sysbench cpu (192 threads) + sysbench mysql (192 threads, mysqld forced into the same cgroup)
-- C. uperf netperf.xml (192 threads over TCP or UDP protocol separately)
-- D. will-it-scale context_switch via pipe (192 threads)
- test machine setup:
CPU(s): 192
On-line CPU(s) list: 0-191
Thread(s) per core: 2
Core(s) per socket: 48
Socket(s): 2
NUMA node(s): 4
- test results:

	-- workload A, no obvious performance drop in cs_on:
	+----------------------+------+----------------------+------------------------+
	|                      | **   | sysbench cpu * 192   | sysbench mysql * 192   |
	+======================+======+======================+========================+
	| cgroup               | **   | cg_sysbench_cpu_0    | cg_sysbench_mysql_0    |
	+----------------------+------+----------------------+------------------------+
	| record_item          | **   | Tput_avg (events/s)  | Tput_avg (events/s)    |
	+----------------------+------+----------------------+------------------------+
	| coresched_normalized | **   | 1.01                 | 0.87                   |
	+----------------------+------+----------------------+------------------------+
	| default_normalized   | **   | 1                    | 1                      |
	+----------------------+------+----------------------+------------------------+
	| smtoff_normalized    | **   | 0.59                 | 0.82                   |
	+----------------------+------+----------------------+------------------------+

	-- workload B, no obvious performance drop in cs_on:
	+----------------------+------+----------------------+------------------------+
	|                      | **   | sysbench cpu * 192   | sysbench cpu * 192     |
	+======================+======+======================+========================+
	| cgroup               | **   | cg_sysbench_cpu_0    | cg_sysbench_cpu_1      |
	+----------------------+------+----------------------+------------------------+
	| record_item          | **   | Tput_avg (events/s)  | Tput_avg (events/s)    |
	+----------------------+------+----------------------+------------------------+
	| coresched_normalized | **   | 1.01                 | 0.98                   |
	+----------------------+------+----------------------+------------------------+
	| default_normalized   | **   | 1                    | 1                      |
	+----------------------+------+----------------------+------------------------+
	| smtoff_normalized    | **   | 0.6                  | 0.6                    |
	+----------------------+------+----------------------+------------------------+

	-- workload C, known performance drop in cs_on since Core Scheduling v6:
	+----------------------+------+---------------------------+---------------------------+
	|                      | **   | uperf netperf TCP * 192   | uperf netperf UDP * 192   |
	+======================+======+===========================+===========================+
	| cgroup               | **   | cg_uperf                  | cg_uperf                  |
	+----------------------+------+---------------------------+---------------------------+
Quoted 8 lines
	| record_item          | **   | Tput_avg (Gb/s)           | Tput_avg (Gb/s)           |
	+----------------------+------+---------------------------+---------------------------+
	| coresched_normalized | **   | 0.46                      | 0.48                      |
	+----------------------+------+---------------------------+---------------------------+
	| default_normalized   | **   | 1                         | 1                         |
	+----------------------+------+---------------------------+---------------------------+
	| smtoff_normalized    | **   | 0.82                      | 0.79                      |
	+----------------------+------+---------------------------+---------------------------+

This is known that when coresched is on, uperf offloads softirq service to ksoftirqd, and the cookie of ksoftirqd is different from the cookie of uperf. As a result, ksoftirqd can run concurrently with uperf previous but not now.

Quoted 16 lines

-- workload D, new added syscall workload, performance drop in cs_on:
+----------------------+------+-------------------------------+
| | ** | will-it-scale * 192 |
| | | (pipe based context_switch) |
+======================+======+===============================+
| cgroup | ** | cg_will-it-scale |
+----------------------+------+-------------------------------+
| record_item | ** | threads_avg |
+----------------------+------+-------------------------------+
| coresched_normalized | ** | 0.2 |
+----------------------+------+-------------------------------+
| default_normalized | ** | 1 |
+----------------------+------+-------------------------------+
| smtoff_normalized | ** | 0.89 |
+----------------------+------+-------------------------------+

will-it-scale may be a very extreme case. The story here is,
- On one sibling reader/writer gets blocked and tries to schedule another reader/writer in.
- The other sibling tries to wake up reader/writer.

Both CPUs are acquiring rq->__lock,

So when coresched off, they are two different locks, lock stat(1 second delta) below:

class name con-bounces contentions waittime-min waittime-max waittime-total waittime-avg acq-bounces acquisitions holdtime-min holdtime-max holdtime-total holdtime-avg
&rq->__lock: 210 210 0.10 3.04 180.87 0.86 797 79165021 0.03 20.69 60650198.34 0.77

But when coresched on, they are actually one same lock, lock stat(1 second delta) below:

class name con-bounces contentions waittime-min waittime-max waittime-total waittime-avg acq-bounces acquisitions holdtime-min holdtime-max holdtime-total holdtime-avg
&rq->__lock: 6479459 6484857 0.05 216.46 60829776.85 9.38 8346319 15399739 0.03 95.56 81119515.38 5.27

This nature of core scheduling may degrade the performance of similar workloads with frequent context switching.

Any thoughts?

Thanks,
-Aubrey

#94 lore raw
Quoted 28 lines · Aubrey
Quoted 16 lines

-- workload D, new added syscall workload, performance drop in cs_on:
+----------------------+------+-------------------------------+
| | ** | will-it-scale * 192 |
| | | (pipe based context_switch) |
+======================+======+===============================+
| cgroup | ** | cg_will-it-scale |
+----------------------+------+-------------------------------+
| record_item | ** | threads_avg |
+----------------------+------+-------------------------------+
| coresched_normalized | ** | 0.2 |
+----------------------+------+-------------------------------+
| default_normalized | ** | 1 |
+----------------------+------+-------------------------------+
| smtoff_normalized | ** | 0.89 |
+----------------------+------+-------------------------------+

will-it-scale may be a very extreme case. The story here is,
- On one sibling reader/writer gets blocked and tries to schedule another reader/writer in.
- The other sibling tries to wake up reader/writer.

Both CPUs are acquiring rq->__lock,

So when coresched off, they are two different locks, lock stat(1 second delta) below:

class name con-bounces contentions waittime-min waittime-max waittime-total waittime-avg acq-bounces acquisitions holdtime-min holdtime-max holdtime-total holdtime-avg
&rq->__lock: 210 210 0.10 3.04 180.87 0.86 797 79165021 0.03 20.69 60650198.34 0.77

Quoted 6 lines

But when coresched on, they are actually one same lock, lock stat(1 second delta) below:

class name con-bounces contentions waittime-min waittime-max waittime-total waittime-avg acq-bounces acquisitions holdtime-min holdtime-max holdtime-total holdtime-avg
&rq->__lock: 6479459 6484857 0.05 216.46 60829776.85 9.38 8346319 15399739 0.03 95.56 81119515.38 5.27

This nature of core scheduling may degrade the performance of similar workloads with frequent context switching.

When core sched is off, is SMT off as well? From the above table, it seems to be. So even for core sched off, there will be a single lock per physical CPU core (assuming SMT is also off) right? Or did I miss something?

thanks,

- Joel

#95 lore raw
Quoted 32 lines · Joel Fernandes
Quoted 30 lines · Aubrey
Quoted 16 lines

-- workload D, new added syscall workload, performance drop in cs_on:
+----------------------+------+-------------------------------+
| | ** | will-it-scale * 192 |
| | | (pipe based context_switch) |
+======================+======+===============================+
| cgroup | ** | cg_will-it-scale |
+----------------------+------+-------------------------------+
| record_item | ** | threads_avg |
+----------------------+------+-------------------------------+
| coresched_normalized | ** | 0.2 |
+----------------------+------+-------------------------------+
| default_normalized | ** | 1 |
+----------------------+------+-------------------------------+
| smtoff_normalized | ** | 0.89 |
+----------------------+------+-------------------------------+

will-it-scale may be a very extreme case. The story here is,
- On one sibling reader/writer gets blocked and tries to schedule another reader/writer in.
- The other sibling tries to wake up reader/writer.

Both CPUs are acquiring rq->__lock,

So when coresched off, they are two different locks, lock stat(1 second delta) below:

class name con-bounces contentions waittime-min waittime-max waittime-total waittime-avg acq-bounces acquisitions holdtime-min holdtime-max holdtime-total holdtime-avg
&rq->__lock: 210 210 0.10 3.04 180.87 0.86 797 79165021 0.03 20.69 60650198.34 0.77

But when coresched on, they are actually one same lock, lock stat(1 second delta) below:

Quoted 9 lines
Quoted 4 lines

class name con-bounces contentions waittime-min waittime-max waittime-total waittime-avg acq-bounces acquisitions holdtime-min holdtime-max holdtime-total holdtime-avg
&rq->__lock: 6479459 6484857 0.05 216.46 60829776.85 9.38 8346319 15399739 0.03 95.56 81119515.38 5.27

This nature of core scheduling may degrade the performance of similar workloads with frequent context switching.

When core sched is off, is SMT off as well? From the above table, it seems to be. So even for core sched off, there will be a single lock per physical CPU core (assuming SMT is also off) right? Or did I miss something?

The table includes 3 cases:
- default: SMT on, coresched off
- coresched: SMT on, coresched on
- smtoff: SMT off, coresched off

I was comparing the default(coresched off & SMT on) case with (coresched on & SMT on) case.

If SMT off, then reader and writer on the different cores have different rq->lock, so the lock contention is not that serious.

class name con-bounces contentions waittime-min waittime-max waittime-total waittime-avg acq-bounces acquisitions holdtime-min holdtime-max holdtime-total holdtime-avg
&rq->__lock: 60 60 0.11 1.92 41.33 0.69 127 67184172 0.03 22.95 33160428.37 0.49

Does this address your concern?

Thanks,
-Aubrey

#96 lore raw

PATCH v9 1/1 (Was Re: [PATCH v8 -tip 00/26] Core scheduling)

In reply to Joel Fernandes (Google)

All,

I am getting ready to send the next v9 series based on tip/master branch. Could you please give the below tree a try and report any results in your testing?
git tree:
https://git.kernel.org/pub/scm/linux/kernel/git/jfern/linux.git (branch coresched) git log:
https://git.kernel.org/pub/scm/linux/kernel/git/jfern/linux.git/log/?h=coresched

The major changes in this series are the improvements:
(1)
"sched: Make snapshotting of min_vruntime more CGroup-friendly"
https://git.kernel.org/pub/scm/linux/kernel/git/jfern/linux.git/commit/?h=coresched-v9-for-test&id=9a20a6652b3c50fd51faa829f7947004239a04eb

(2)
"sched: Simplify the core pick loop for optimized case"
https://git.kernel.org/pub/scm/linux/kernel/git/jfern/linux.git/commit/?h=coresched-v9-for-test&id=0370117b4fd418cdaaa6b1489bfc14f305691152

And a bug fix:
(1)
"sched: Enqueue task into core queue only after vruntime is updated"
https://git.kernel.org/pub/scm/linux/kernel/git/jfern/linux.git/commit/?h=coresched-v9-for-test&id=401dad5536e7e05d1299d0864e6fc5072029f492

There are also 2 more bug fixes that I squashed-in related to kernel protection and a crash seen on the tip/master branch.

Hoping to send the series next week out to the list.

Have a great weekend, and Thanks!

- Joel

Quoted 220 lines · Joel Fernandes (Google)

Eighth iteration of the Core-Scheduling feature.

Core scheduling is a feature that allows only trusted tasks to run concurrently on cpus sharing compute resources (eg: hyperthreads on a core). The goal is to mitigate the core-level side-channel attacks without requiring to disable SMT (which has a significant impact on performance in some situations). Core scheduling (as of v7) mitigates user-space to user-space attacks and user to kernel attack when one of the siblings enters the kernel via interrupts or system call.

By default, the feature doesn't change any of the current scheduler behavior. The user decides which tasks can run simultaneously on the same core (for now by having them in the same tagged cgroup). When a tag is enabled in a cgroup and a task from that cgroup is running on a hardware thread, the scheduler ensures that only idle or trusted tasks run on the other sibling(s). Besides security concerns, this feature can also be beneficial for RT and performance applications where we want to control how tasks make use of SMT dynamically.

This iteration focuses on the the following stuff:
- Redesigned API.
- Rework of Kernel Protection feature based on Thomas's entry work.
- Rework of hotplug fixes.
- Address review comments in v7

Joel: Both a CGroup and Per-task interface via prctl(2) are provided for configuring core sharing. More details are provided in documentation patch. Kselftests are provided to verify the correctness/rules of the interface.

Julien: TPCC tests showed improvements with core-scheduling. With kernel protection enabled, it does not show any regression. Possibly ASI will improve the performance for those who choose kernel protection (can be toggled through sched_core_protect_kernel sysctl). Results:
v8 average stdev diff
baseline (SMT on) 1197.272 44.78312824
core sched ( kernel protect) 412.9895 45.42734343 -65.51%
core sched (no kernel protect) 686.6515 71.77756931 -42.65%
nosmt 408.667 39.39042872 -65.87%

v8 is rebased on tip/master.

Future work
===========
- Load balancing/Migration fixes for core scheduling. With v6, Load balancing is partially coresched aware, but has some issues w.r.t process/taskgroup weights:
https://lwn.net/ml/linux-kernel/20200225034438.GA617271@z...
- Core scheduling test framework: kselftests, torture tests etc

Changes in v8
=============
- New interface/API implementation
- Joel
- Revised kernel protection patch
- Joel
- Revised Hotplug fixes
- Joel
- Minor bug fixes and address review comments
- Vineeth

Changes in v7
=============
- Kernel protection from untrusted usermode tasks
- Joel, Vineeth
- Fix for hotplug crashes and hangs
- Joel, Vineeth

Changes in v6
=============
- Documentation
- Joel
- Pause siblings on entering nmi/irq/softirq
- Joel, Vineeth
- Fix for RCU crash
- Joel
- Fix for a crash in pick_next_task
- Yu Chen, Vineeth
- Minor re-write of core-wide vruntime comparison
- Aaron Lu
- Cleanup: Address Review comments
- Cleanup: Remove hotplug support (for now)
- Build fixes: 32 bit, SMT=n, AUTOGROUP=n etc
- Joel, Vineeth

Changes in v5
=============
- Fixes for cgroup/process tagging during corner cases like cgroup destroy, task moving across cgroups etc
- Tim Chen
- Coresched aware task migrations
- Aubrey Li
- Other minor stability fixes.

Changes in v4
=============
- Implement a core wide min_vruntime for vruntime comparison of tasks across cpus in a core.
- Aaron Lu
- Fixes a typo bug in setting the forced_idle cpu.
- Aaron Lu

Changes in v3
=============
- Fixes the issue of sibling picking up an incompatible task
- Aaron Lu
- Vineeth Pillai
- Julien Desfossez
- Fixes the issue of starving threads due to forced idle
- Peter Zijlstra
- Fixes the refcounting issue when deleting a cgroup with tag
- Julien Desfossez
- Fixes a crash during cpu offline/online with coresched enabled
- Vineeth Pillai
- Fixes a comparison logic issue in sched_core_find
- Aaron Lu

Changes in v2
=============
- Fixes for couple of NULL pointer dereference crashes
- Subhra Mazumdar
- Tim Chen
- Improves priority comparison logic for process in different cpus
- Peter Zijlstra
- Aaron Lu
- Fixes a hard lockup in rq locking
- Vineeth Pillai
- Julien Desfossez
- Fixes a performance issue seen on IO heavy workloads
- Vineeth Pillai
- Julien Desfossez
- Fix for 32bit build
- Aubrey Li

Aubrey Li (1):
sched: migration changes for core scheduling

Joel Fernandes (Google) (13):
sched/fair: Snapshot the min_vruntime of CPUs on force idle arch/x86: Add a new TIF flag for untrusted tasks kernel/entry: Add support for core-wide protection of kernel-mode entry/idle: Enter and exit kernel protection during idle entry and exit
sched: Split the cookie and setup per-task cookie on fork
sched: Add a per-thread core scheduling interface
sched: Add a second-level tag for nested CGroup usecase
sched: Release references to the per-task cookie on exit
sched: Handle task addition to CGroup
sched/debug: Add CGroup node for printing group cookie if SCHED_DEBUG
kselftest: Add tests for core-sched interface
sched: Move core-scheduler interfacing code to a new file
Documentation: Add core scheduling documentation

Peter Zijlstra (10):
sched: Wrap rq::lock access
sched: Introduce sched_class::pick_task()
sched: Core-wide rq->lock
sched/fair: Add a few assertions
sched: Basic tracking of matching tasks
sched: Add core wide task selection and scheduling.
sched: Trivial forced-newidle balancer
irq_work: Cleanup
sched: cgroup tagging interface for core scheduling
sched: Debug bits...

Vineeth Pillai (2):
sched/fair: Fix forced idle sibling starvation corner case entry/kvm: Protect the kernel when entering from guest

.../admin-guide/hw-vuln/core-scheduling.rst   |  312 +++++
Documentation/admin-guide/hw-vuln/index.rst   |    1 +
.../admin-guide/kernel-parameters.txt         |    7 +
arch/x86/include/asm/thread_info.h            |    2 +
arch/x86/kvm/x86.c                            |    3 +
drivers/gpu/drm/i915/i915_request.c           |    4 +-
include/linux/entry-common.h                  |   20 +-
include/linux/entry-kvm.h                     |   12 +
include/linux/irq_work.h                      |   33 +-
include/linux/irqflags.h                      |    4 +-
include/linux/sched.h                         |   27 +-
include/uapi/linux/prctl.h                    |    3 +
kernel/Kconfig.preempt                        |    6 +
kernel/bpf/stackmap.c                         |    2 +-
kernel/entry/common.c                         |   25 +-
kernel/entry/kvm.c                            |   13 +
kernel/fork.c                                 |    1 +
kernel/irq_work.c                             |   18 +-
kernel/printk/printk.c                        |    6 +-
kernel/rcu/tree.c                             |    3 +-
kernel/sched/Makefile                         |    1 +
kernel/sched/core.c                           | 1135 ++++++++++++++++-
kernel/sched/coretag.c                        |  468 +++++++
kernel/sched/cpuacct.c                        |   12 +-
kernel/sched/deadline.c                       |   34 +-
kernel/sched/debug.c                          |    8 +-
kernel/sched/fair.c                           |  272 ++--
kernel/sched/idle.c                           |   24 +-
kernel/sched/pelt.h                           |    2 +-
kernel/sched/rt.c                             |   22 +-
kernel/sched/sched.h                          |  302 ++++-
kernel/sched/stop_task.c                      |   13 +-
kernel/sched/topology.c                       |    4 +-
kernel/sys.c                                  |    3 +
kernel/time/tick-sched.c                      |    6 +-
kernel/trace/bpf_trace.c                      |    2 +-
tools/include/uapi/linux/prctl.h              |    3 +
tools/testing/selftests/sched/.gitignore      |    1 +
tools/testing/selftests/sched/Makefile        |   14 +
tools/testing/selftests/sched/config          |    1 +
.../testing/selftests/sched/test_coresched.c  |  840 ++++++++++++
41 files changed, 3437 insertions(+), 232 deletions(-)

create mode 100644 Documentation/admin-guide/hw-vuln/core-scheduling.rst create mode 100644 kernel/sched/coretag.c
create mode 100644 tools/testing/selftests/sched/.gitignore create mode 100644 tools/testing/selftests/sched/Makefile create mode 100644 tools/testing/selftests/sched/config create mode 100644 tools/testing/selftests/sched/test_coresched.c

--
2.29.0.rc1.297.gfa9743e501-goog

#97 lore raw

Re: [RFT for v9] (Was Re: [PATCH v8 -tip 00/26] Core scheduling)

Quoted 27 lines · Joel Fernandes

All,

I am getting ready to send the next v9 series based on tip/master branch. Could you please give the below tree a try and report any results in your testing?
git tree:
https://git.kernel.org/pub/scm/linux/kernel/git/jfern/linux.git (branch coresched) git log:
https://git.kernel.org/pub/scm/linux/kernel/git/jfern/linux.git/log/?h=coresched

The major changes in this series are the improvements:
(1)
"sched: Make snapshotting of min_vruntime more CGroup-friendly"
https://git.kernel.org/pub/scm/linux/kernel/git/jfern/linux.git/commit/?h=coresched-v9-for-test&id=9a20a6652b3c50fd51faa829f7947004239a04eb

(2)
"sched: Simplify the core pick loop for optimized case"
https://git.kernel.org/pub/scm/linux/kernel/git/jfern/linux.git/commit/?h=coresched-v9-for-test&id=0370117b4fd418cdaaa6b1489bfc14f305691152

And a bug fix:
(1)
"sched: Enqueue task into core queue only after vruntime is updated"
https://git.kernel.org/pub/scm/linux/kernel/git/jfern/linux.git/commit/?h=coresched-v9-for-test&id=401dad5536e7e05d1299d0864e6fc5072029f492

There are also 2 more bug fixes that I squashed-in related to kernel protection and a crash seen on the tip/master branch.

Quoted 8 lines

Hoping to send the series next week out to the list.

Have a great weekend, and Thanks!

- Joel

On Mon, Oct 19, 2020 at 09:43:10PM -0400, Joel Fernandes (Google) wrote:

Adding 4 workloads test results for core scheduling v9 candidate:

- kernel under test:
-- coresched community v9 candidate from https://git.kernel.org/pub/scm/linux/kernel/git/jfern/linux.git (branch coresched) -- latest commit: 2e8591a330ff (HEAD -> coresched, origin/coresched) NEW: sched: Add a coresched command line option -- coresched=on kernel parameter applied
- workloads:
-- A. sysbench cpu (192 threads) + sysbench cpu (192 threads)
-- B. sysbench cpu (192 threads) + sysbench mysql (192 threads, mysqld forced into the same cgroup)
-- C. uperf netperf.xml (192 threads over TCP or UDP protocol separately)
-- D. will-it-scale context_switch via pipe (192 threads)
- test machine setup:
CPU(s): 192
On-line CPU(s) list: 0-191
Thread(s) per core: 2
Core(s) per socket: 48
Socket(s): 2
NUMA node(s): 4
- test results, no obvious performance drop compared to community v8 build:
-- workload A:
+----------------------+------+----------------------+------------------------+
| | ** | sysbench cpu * 192 | sysbench cpu * 192 |
+======================+======+======================+========================+
| cgroup | ** | cg_sysbench_cpu_0 | cg_sysbench_cpu_1 |
+----------------------+------+----------------------+------------------------+
| record_item | ** | Tput_avg (events/s) | Tput_avg (events/s) |
+----------------------+------+----------------------+------------------------+
| coresched_normalized | ** | 0.98 | 1.01 |
+----------------------+------+----------------------+------------------------+
| default_normalized | ** | 1 | 1 |
+----------------------+------+----------------------+------------------------+
| smtoff_normalized | ** | 0.59 | 0.6 |
+----------------------+------+----------------------+------------------------+

-- workload B:
+----------------------+------+----------------------+------------------------+
| | ** | sysbench cpu * 192 | sysbench mysql * 192 |
+======================+======+======================+========================+
| cgroup | ** | cg_sysbench_cpu_0 | cg_sysbench_mysql_0 |
+----------------------+------+----------------------+------------------------+
| record_item | ** | Tput_avg (events/s) | Tput_avg (events/s) |
+----------------------+------+----------------------+------------------------+
| coresched_normalized | ** | 1.02 | 0.78 |
+----------------------+------+----------------------+------------------------+
| default_normalized | ** | 1 | 1 |
+----------------------+------+----------------------+------------------------+
| smtoff_normalized | ** | 0.59 | 0.75 |
+----------------------+------+----------------------+------------------------+

-- workload C:
+----------------------+------+---------------------------+---------------------------+
| | ** | uperf netperf TCP * 192 | uperf netperf UDP * 192 |
+======================+======+===========================+===========================+
| cgroup | ** | cg_uperf | cg_uperf |
+----------------------+------+---------------------------+---------------------------+
| record_item | ** | Tput_avg (Gb/s) | Tput_avg (Gb/s) |
+----------------------+------+---------------------------+---------------------------+
| coresched_normalized | ** | 0.65 | 0.67 |
+----------------------+------+---------------------------+---------------------------+
| default_normalized | ** | 1 | 1 |
+----------------------+------+---------------------------+---------------------------+
| smtoff_normalized | ** | 0.83 | 0.91 |
+----------------------+------+---------------------------+---------------------------+

-- workload D:
+----------------------+------+-------------------------------+
| | ** | will-it-scale * 192 |
| | | (pipe based context_switch) |
+======================+======+===============================+
| cgroup | ** | cg_will-it-scale |
+----------------------+------+-------------------------------+
| record_item | ** | threads_avg |
+----------------------+------+-------------------------------+
| coresched_normalized | ** | 0.29 |
+----------------------+------+-------------------------------+
| default_normalized | ** | 1.00 |
+----------------------+------+-------------------------------+
| smtoff_normalized | ** | 0.87 |
+----------------------+------+-------------------------------+

- notes on test results record_item:
* coresched_normalized: smton, cs enabled, test result normalized by default value
* default_normalized: smton, cs disabled, test result normalized by default value
* smtoff_normalized: smtoff, test result normalized by default value

Hongyu

#98 lore raw

Re: [RFT for v9] (Was Re: [PATCH v8 -tip 00/26] Core scheduling)

Quoted 117 lines · Hongyu
Quoted 27 lines · Joel Fernandes

All,

I am getting ready to send the next v9 series based on tip/master branch. Could you please give the below tree a try and report any results in your testing?
git tree:
https://git.kernel.org/pub/scm/linux/kernel/git/jfern/linux.git (branch coresched) git log:
https://git.kernel.org/pub/scm/linux/kernel/git/jfern/linux.git/log/?h=coresched

The major changes in this series are the improvements:
(1)
"sched: Make snapshotting of min_vruntime more CGroup-friendly"
https://git.kernel.org/pub/scm/linux/kernel/git/jfern/linux.git/commit/?h=coresched-v9-for-test&id=9a20a6652b3c50fd51faa829f7947004239a04eb

(2)
"sched: Simplify the core pick loop for optimized case"
https://git.kernel.org/pub/scm/linux/kernel/git/jfern/linux.git/commit/?h=coresched-v9-for-test&id=0370117b4fd418cdaaa6b1489bfc14f305691152

And a bug fix:
(1)
"sched: Enqueue task into core queue only after vruntime is updated"
https://git.kernel.org/pub/scm/linux/kernel/git/jfern/linux.git/commit/?h=coresched-v9-for-test&id=401dad5536e7e05d1299d0864e6fc5072029f492

There are also 2 more bug fixes that I squashed-in related to kernel protection and a crash seen on the tip/master branch.

Quoted 8 lines

Hoping to send the series next week out to the list.

Have a great weekend, and Thanks!

- Joel

On Mon, Oct 19, 2020 at 09:43:10PM -0400, Joel Fernandes (Google) wrote:

Adding 4 workloads test results for core scheduling v9 candidate:

- kernel under test:
-- coresched community v9 candidate from https://git.kernel.org/pub/scm/linux/kernel/git/jfern/linux.git (branch coresched) -- latest commit: 2e8591a330ff (HEAD -> coresched, origin/coresched) NEW: sched: Add a coresched command line option -- coresched=on kernel parameter applied
- workloads:
-- A. sysbench cpu (192 threads) + sysbench cpu (192 threads)
-- B. sysbench cpu (192 threads) + sysbench mysql (192 threads, mysqld forced into the same cgroup)
-- C. uperf netperf.xml (192 threads over TCP or UDP protocol separately)
-- D. will-it-scale context_switch via pipe (192 threads)
- test machine setup:
CPU(s): 192
On-line CPU(s) list: 0-191
Thread(s) per core: 2
Core(s) per socket: 48
Socket(s): 2
NUMA node(s): 4
- test results, no obvious performance drop compared to community v8 build:
-- workload A:
+----------------------+------+----------------------+------------------------+
| | ** | sysbench cpu * 192 | sysbench cpu * 192 |
+======================+======+======================+========================+
| cgroup | ** | cg_sysbench_cpu_0 | cg_sysbench_cpu_1 |
+----------------------+------+----------------------+------------------------+
| record_item | ** | Tput_avg (events/s) | Tput_avg (events/s) |
+----------------------+------+----------------------+------------------------+
| coresched_normalized | ** | 0.98 | 1.01 |
+----------------------+------+----------------------+------------------------+
| default_normalized | ** | 1 | 1 |
+----------------------+------+----------------------+------------------------+
| smtoff_normalized | ** | 0.59 | 0.6 |
+----------------------+------+----------------------+------------------------+

-- workload B:
+----------------------+------+----------------------+------------------------+
| | ** | sysbench cpu * 192 | sysbench mysql * 192 |
+======================+======+======================+========================+
| cgroup | ** | cg_sysbench_cpu_0 | cg_sysbench_mysql_0 |
+----------------------+------+----------------------+------------------------+
| record_item | ** | Tput_avg (events/s) | Tput_avg (events/s) |
+----------------------+------+----------------------+------------------------+
| coresched_normalized | ** | 1.02 | 0.78 |
+----------------------+------+----------------------+------------------------+
| default_normalized | ** | 1 | 1 |
+----------------------+------+----------------------+------------------------+
| smtoff_normalized | ** | 0.59 | 0.75 |
+----------------------+------+----------------------+------------------------+

-- workload C:
+----------------------+------+---------------------------+---------------------------+
| | ** | uperf netperf TCP * 192 | uperf netperf UDP * 192 |
+======================+======+===========================+===========================+
| cgroup | ** | cg_uperf | cg_uperf |
+----------------------+------+---------------------------+---------------------------+
| record_item | ** | Tput_avg (Gb/s) | Tput_avg (Gb/s) |
+----------------------+------+---------------------------+---------------------------+
| coresched_normalized | ** | 0.65 | 0.67 |
+----------------------+------+---------------------------+---------------------------+
| default_normalized | ** | 1 | 1 |
+----------------------+------+---------------------------+---------------------------+
| smtoff_normalized | ** | 0.83 | 0.91 |
+----------------------+------+---------------------------+---------------------------+

-- workload D:
+----------------------+------+-------------------------------+
| | ** | will-it-scale * 192 |
| | | (pipe based context_switch) |
+======================+======+===============================+
| cgroup | ** | cg_will-it-scale |
+----------------------+------+-------------------------------+
| record_item | ** | threads_avg |
+----------------------+------+-------------------------------+
| coresched_normalized | ** | 0.29 |
+----------------------+------+-------------------------------+
| default_normalized | ** | 1.00 |
+----------------------+------+-------------------------------+
| smtoff_normalized | ** | 0.87 |
+----------------------+------+-------------------------------+

Quoted 8 lines

- notes on test results record_item:
* coresched_normalized: smton, cs enabled, test result normalized by default value
* default_normalized: smton, cs disabled, test result normalized by default value
* smtoff_normalized: smtoff, test result normalized by default value

Hongyu

Add 2 more negative test case:

- continuously toggle cpu.core_tag, during workload running with cs_on
- continuously toggle smt setting via /sys/devices/system/cpu/smt/control, during workload running with cs_on

no kernel panic or platform hang observed.