Skip to contentSkip to conversation

RFC: sched: UMCG: episode IV: A New Hope

PATCH New Peter Oskolkov 2 participants 3 messages Feb 11, 2022 – Feb 12, 2022

Subsystems (8)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 RFC: sched: UMCG: episode IV: A New Hope

A lot of effort has been put into making UMCG based on userspace TLS data work, and it gets ugly very fast because it is very hard to guarantee that the pages are present when needed; and they are needed in non-preemptible (sched) contexts. The last attempt here:
https://lore.kernel.org/lkml/[email protected]/
is a good example: a lot of mm-related work, a lot of extra stuff added to struct task_struct just to deal with kernel-to-userspace writes in sched contexts.

Here I propose a different approach (actually, it was my first approach, before we pivoted to userspace TLS). Keep everything the kernel needs in a kernel-side struct umcg_task, and copy relevant data out to the userspace when the server's sys_umcg_wait() returns.

Before I go too deep down into implementing and testing this, I'd like to get some feedback re: if this approach is acceptable.

Please review.

=====================

User Managed Concurrency Groups is an M:N threading toolkit that allows constructing user space schedulers designed to efficiently manage heterogeneous in-process workloads while maintaining high CPU utilization (95%+).

Add UMCG syscall stubs, Kconfig, as well as stubs for hooks into sched, execve, etc., as this boilerplate is more or less stable, comparing to various approaches attempted at implementing UMCG.

---
 arch/x86/Kconfig                       |   1 +
 arch/x86/entry/syscalls/syscall_64.tbl |   2 +
 fs/exec.c                              |   1 +
 include/linux/sched.h                  |  49 ++++++
 include/linux/syscalls.h               |   7 +-
 include/uapi/asm-generic/unistd.h      |   7 +-
 include/uapi/linux/umcg.h              | 232 +++++++++++++++++++++++++
 init/Kconfig                           |  15 ++
 kernel/entry/common.c                  |   4 +-
 kernel/exit.c                          |   5 +
 kernel/sched/Makefile                  |   1 +
 kernel/sched/core.c                    |  11 +-
 kernel/sched/umcg.c                    | 109 ++++++++++++
 kernel/sys_ni.c                        |   4 +
 14 files changed, 442 insertions(+), 6 deletions(-)
 create mode 100644 include/uapi/linux/umcg.h
 create mode 100644 kernel/sched/umcg.c
arch/x86/Kconfig +1 -0
index ebe8fc76949a..bcc97d7f0145 100644
@@ -249,6 +249,7 @@ config X86
 	select HAVE_RSEQ
 	select HAVE_SYSCALL_TRACEPOINTS
 	select HAVE_UNSTABLE_SCHED_CLOCK
+	select HAVE_UMCG			if X86_64
 	select HAVE_USER_RETURN_NOTIFIER
 	select HAVE_GENERIC_VDSO
 	select HOTPLUG_SMT			if SMP
arch/x86/entry/syscalls/syscall_64.tbl +2 -0
index c84d12608cd2..0d266f4a3dd6 100644
@@ -372,6 +372,8 @@
 448	common	process_mrelease	sys_process_mrelease
 449	common	futex_waitv		sys_futex_waitv
 450	common	set_mempolicy_home_node	sys_set_mempolicy_home_node
+451	common	umcg_wait		sys_umcg_wait
+452	common	umcg_kick		sys_umcg_kick
 
 #
 # Due to a historical design error, certain syscalls are numbered differently
fs/exec.c +1 -0
index 79f2c9483302..6c42749c1c58 100644
@@ -1842,6 +1842,7 @@ static int bprm_execve(struct linux_binprm *bprm,
 	current->fs->in_exec = 0;
 	current->in_execve = 0;
 	rseq_execve(current);
+	umcg_execve(current);
 	acct_update_integrals(current);
 	task_numa_free(current, false);
 	return retval;
include/linux/sched.h +49 -0

73 lines not on this page · raw message

include/linux/syscalls.h +5 -2
index 819c0cb00b6d..3ed99aa49bb8 100644
@@ -178,9 +178,9 @@ extern struct trace_event_functions exit_syscall_print_funcs;
 	SYSCALL_TRACE_EXIT_EVENT(sname);			\
 	static struct syscall_metadata __used			\
 	  __syscall_meta_##sname = {				\
-		.name 		= "sys"#sname,			\
+		.name		= "sys"#sname,			\
 		.syscall_nr	= -1,	/* Filled in at boot */	\
-		.nb_args 	= nb,				\
+		.nb_args	= nb,				\
 		.types		= nb ? types_##sname : NULL,	\
 		.args		= nb ? args_##sname : NULL,	\
 		.enter_event	= &event_enter_##sname,		\
@@ -1060,6 +1060,9 @@ asmlinkage long sys_memfd_secret(unsigned int flags);
 asmlinkage long sys_set_mempolicy_home_node(unsigned long start, unsigned long len,
 					    unsigned long home_node,
 					    unsigned long flags);
+asmlinkage long sys_umcg_wait(u64 flags, pid_t next_tid, u64 abs_timeout,
+			      u64 __user *workers, u64 worker_id_or_sz);
+asmlinkage long sys_umcg_kick(u32 flags, pid_t tid);
 
 /*
  * Architecture-specific system calls
include/uapi/asm-generic/unistd.h +6 -1
index 1c48b0ae3ba3..68406e99d0a7 100644
@@ -886,8 +886,13 @@ __SYSCALL(__NR_futex_waitv, sys_futex_waitv)
 #define __NR_set_mempolicy_home_node 450
 __SYSCALL(__NR_set_mempolicy_home_node, sys_set_mempolicy_home_node)
 
+#define __NR_umcg_wait 451
+__SYSCALL(__NR_umcg_wait, sys_umcg_wait)
+#define __NR_umcg_kick 452
+__SYSCALL(__NR_umcg_kick, sys_umcg_kick)
+
 #undef __NR_syscalls
-#define __NR_syscalls 451
+#define __NR_syscalls 453
 
 /*
  * 32 bit systems traditionally used different
include/uapi/linux/umcg.h new file +232 -0

232 lines not on this page · raw message

init/Kconfig +15 -0
index e9119bf54b1f..75a623deb8cc 100644
@@ -1685,6 +1685,21 @@ config MEMBARRIER
 
 	  If unsure, say Y.
 
+config HAVE_UMCG
+	bool
+
+config UMCG
+	bool "Enable User Managed Concurrency Groups API"
+	depends on 64BIT
+	depends on GENERIC_ENTRY
+	depends on HAVE_UMCG
+	default n
+	help
+	  Enable User Managed Concurrency Groups API, which form the basis
+	  for an in-process M:N userspace scheduling framework.
+	  At the moment this is an experimental/RFC feature that is not
+	  guaranteed to be backward-compatible.
+
 config KALLSYMS
 	bool "Load all symbols for debugging/ksymoops" if EXPERT
 	default y
kernel/entry/common.c +3 -1
index bad713684c2e..081c3d5b4197 100644
@@ -171,8 +171,10 @@ static unsigned long exit_to_user_mode_loop(struct pt_regs *regs,
 		if (ti_work & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL))
 			handle_signal_work(regs, ti_work);
 
-		if (ti_work & _TIF_NOTIFY_RESUME)
+		if (ti_work & _TIF_NOTIFY_RESUME) {
+			umcg_notify_resume();
 			tracehook_notify_resume(regs);
+		}
 
 		/* Architecture specific TIF work */
 		arch_exit_to_user_mode_work(regs, ti_work);
kernel/exit.c +5 -0
index b00a25bb4ab9..bf0e4c95fb77 100644
@@ -737,6 +737,10 @@ void __noreturn do_exit(long code)
 
 	WARN_ON(blk_needs_flush_plug(tsk));
 
+	/* Turn off UMCG sched hooks. */
+	if (unlikely(tsk->flags & PF_UMCG_WORKER))
+		tsk->flags &= ~PF_UMCG_WORKER;
+
 	/*
 	 * If do_dead is called because this processes oopsed, it's possible
 	 * that get_fs() was left as KERNEL_DS, so reset it to USER_DS before
@@ -760,6 +764,7 @@ void __noreturn do_exit(long code)
 
 	io_uring_files_cancel();
 	exit_signals(tsk);  /* sets PF_EXITING */
+	umcg_handle_exit();
 
 	/* sync mm's RSS info before statistics gathering */
 	if (tsk->mm)
kernel/sched/Makefile +1 -0
index c83b37af155b..0da21064aa71 100644
@@ -40,3 +40,4 @@ obj-$(CONFIG_MEMBARRIER) += membarrier.o
 obj-$(CONFIG_CPU_ISOLATION) += isolation.o
 obj-$(CONFIG_PSI) += psi.o
 obj-$(CONFIG_SCHED_CORE) += core_sched.o
+obj-$(CONFIG_UMCG) += umcg.o
kernel/sched/core.c +9 -2
index 0fded9fd1cd3..3a58592fe391 100644
@@ -4273,6 +4273,9 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
 	p->wake_entry.u_flags = CSD_TYPE_TTWU;
 	p->migration_pending = NULL;
 #endif
+#ifdef CONFIG_UMCG
+	p->umcg_task = NULL;
+#endif
 }
 
 DEFINE_STATIC_KEY_FALSE(sched_numa_balancing);
@@ -6331,9 +6334,11 @@ static inline void sched_submit_work(struct task_struct *tsk)
 	 * If a worker goes to sleep, notify and ask workqueue whether it
 	 * wants to wake up a task to maintain concurrency.
 	 */
-	if (task_flags & (PF_WQ_WORKER | PF_IO_WORKER)) {
+	if (task_flags & (PF_WQ_WORKER | PF_IO_WORKER | PF_UMCG_WORKER)) {
 		if (task_flags & PF_WQ_WORKER)
 			wq_worker_sleeping(tsk);
+		else if (task_flags & PF_UMCG_WORKER)
+			umcg_wq_worker_sleeping(tsk);
 		else
 			io_wq_worker_sleeping(tsk);
 	}
@@ -6351,9 +6356,11 @@ static inline void sched_submit_work(struct task_struct *tsk)
 
 static void sched_update_worker(struct task_struct *tsk)
 {
-	if (tsk->flags & (PF_WQ_WORKER | PF_IO_WORKER)) {
+	if (tsk->flags & (PF_WQ_WORKER | PF_IO_WORKER | PF_UMCG_WORKER)) {
 		if (tsk->flags & PF_WQ_WORKER)
 			wq_worker_running(tsk);
+		else if (tsk->flags & PF_UMCG_WORKER)
+			umcg_wq_worker_running(tsk);
 		else
 			io_wq_worker_running(tsk);
 	}
kernel/sched/umcg.c new file +109 -0

109 lines not on this page · raw message

kernel/sys_ni.c +4 -0
index a492f159624f..122f79a2ccb9 100644
@@ -275,6 +275,10 @@ COND_SYSCALL(landlock_restrict_self);
 
 /* arch/example/kernel/sys_example.c */
 
+/* kernel/sched/umcg.c */
+COND_SYSCALL(umcg_wait);
+COND_SYSCALL(umcg_kick);
+
 /* mm/fadvise.c */
 COND_SYSCALL(fadvise64_64);
 
Signature · 1 line
2.25.1
#1 lore raw
[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in https://git-scm.com/docs/git-format-patch]

Hi Peter,

[FYI, it's a private test report for your RFC patch.]
[auto build test ERROR on tip/sched/core]
[also build test ERROR on tip/master linus/master v5.17-rc3]
[cannot apply to tip/x86/core arnd-asm-generic/master tip/x86/asm tip/core/entry next-20220211] [If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Peter-Oskolkov/RFC-sched-UMCG-episode-IV-A-New-Hope/20220212-031635
base:   https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git c8eaf6ac76f40f6c59fc7d056e2e08c4a57ea9c7
config: i386-randconfig-a013 (https://download.01.org/0day-ci/archive/20220212/202202121124.uAuI6YJb-lkp(a)intel.com/config)
compiler: clang version 15.0.0 (https://github.com/llvm/llvm-project f6685f774697c85d6a352dcea013f46a99f9fe31)

reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# https://github.com/0day-ci/linux/commit/64a46b86ef2bb15a0a1bc7566b6ca87f5e0492dd
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Peter-Oskolkov/RFC-sched-UMCG-episode-IV-A-New-Hope/20220212-031635 git checkout 64a46b86ef2bb15a0a1bc7566b6ca87f5e0492dd
# save the config file to linux build tree
mkdir build_dir
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross W=1 O=build_dir ARCH=i386 SHELL=/bin/bash

If you fix the issue, kindly add following tag as appropriate

All errors (new ones prefixed by >>):

In file included from <built-in>:1:

>> ./usr/include/linux/umcg.h:110:2: error: unknown type name 'u64'
           u64     worker_id;
           ^
>> ./usr/include/linux/umcg.h:111:2: error: unknown type name 'u32'
           u32     worker_event_type;
           ^
   ./usr/include/linux/umcg.h:112:2: error: unknown type name 'u32'
           u32     counter;
           ^

3 errors generated.

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org

#2 lore raw
In reply to Peter Oskolkov
[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in https://git-scm.com/docs/git-format-patch]

Hi Peter,

[FYI, it's a private test report for your RFC patch.]
[auto build test ERROR on tip/sched/core]
[also build test ERROR on tip/master linus/master v5.17-rc3]
[cannot apply to tip/x86/core arnd-asm-generic/master tip/x86/asm tip/core/entry next-20220211] [If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Peter-Oskolkov/RFC-sched-UMCG-episode-IV-A-New-Hope/20220212-031635
base:   https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git c8eaf6ac76f40f6c59fc7d056e2e08c4a57ea9c7
config: x86_64-randconfig-a015 (https://download.01.org/0day-ci/archive/20220212/202202121149.e0CZvI0s-lkp(a)intel.com/config)
compiler: gcc-9 (Debian 9.3.0-22) 9.3.0
reproduce (this is a W=1 build):
        # https://github.com/0day-ci/linux/commit/64a46b86ef2bb15a0a1bc7566b6ca87f5e0492dd
        git remote add linux-review https://github.com/0day-ci/linux
        git fetch --no-tags linux-review Peter-Oskolkov/RFC-sched-UMCG-episode-IV-A-New-Hope/20220212-031635
        git checkout 64a46b86ef2bb15a0a1bc7566b6ca87f5e0492dd
        # save the config file to linux build tree
        mkdir build_dir
        make W=1 O=build_dir ARCH=x86_64 SHELL=/bin/bash

If you fix the issue, kindly add following tag as appropriate

All errors (new ones prefixed by >>):

In file included from <command-line>:32:

>> ./usr/include/linux/umcg.h:110:2: error: unknown type name 'u64'
     110 |  u64 worker_id;
         |  ^~~
>> ./usr/include/linux/umcg.h:111:2: error: unknown type name 'u32'
     111 |  u32 worker_event_type;
         |  ^~~
   ./usr/include/linux/umcg.h:112:2: error: unknown type name 'u32'
     112 |  u32 counter;
         |  ^~~

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org