Skip to contentSkip to conversation

sched: User Managed Concurrency Groups

RFC PATCH In review Peter Zijlstra 9 participants 47 messages Jan 20, 2022 – Jan 28, 2022

Subsystems (12)
Memory management x86 VFS Scheduler +8 more subsystems
lore
Share & tools
lore

Peter Zijlstra

author maintainer
#0 lore raw

v2 cover sched: User Managed Concurrency Groups

Latest version, many changes since last time, still under heavy discussion.

Seems to work with the test-case I have (below), but that still has a few gaps, coverage wise.

Still haven't done the SMP wakeup thing, finally get the idea with worker-timeouts but haven't yet implemented that.

Happy hacking..

----

#define _GNU_SOURCE
#include <unistd.h>
#include <sys/types.h>
#include <sys/syscall.h>
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <signal.h>

#ifndef __NR_umcg_ctl
#define __NR_umcg_ctl 450
#define __NR_umcg_wait 451
#define __NR_umcg_kick 452
#endif

#include <linux/list.h>
#include "include/uapi/linux/umcg.h"

/* syscall wrappers */

static inline int
sys_umcg_ctl(u32 flags, struct umcg_task *self, clockid_t which_clock)

{
	return syscall(__NR_umcg_ctl, flags, self, which_clock);
}

static inline int
sys_umcg_wait(u32 flags, u64 timo)

{
	return syscall(__NR_umcg_wait, flags, timo);
}

static inline int
sys_umcg_kick(u32 flags, pid_t tid)

{
	return syscall(__NR_umcg_kick, flags, tid);
}

/* the 'foo' scheduler */

struct foo_task {
	struct umcg_task	task;
	struct list_head	node;
	pid_t			tid;
};
struct foo_server {
	struct umcg_task	task;
	struct list_head	node;
	pid_t			tid;
	struct foo_task		*cur;
	int			workers;
};

void foo_add(struct foo_server *server, struct umcg_task *t)

{
	struct foo_task *foo = container_of(t, struct foo_task, task);

	t->runnable_workers_ptr = 0ULL;
	list_add_tail(&foo->node, &server->node);
}

struct foo_task *foo_pick_next(struct foo_server *server)

{
	struct foo_task *first = NULL;

	if (list_empty(&server->node))
		return first;

	first = list_first_entry(&server->node, struct foo_task, node);
	list_del(&first->node);
	return first;
}

#define NSEC_PER_SEC 1000000000ULL

u64 foo_time(void)

{
	struct timespec ts;
	clock_gettime(CLOCK_MONOTONIC, &ts);
	return (unsigned long long)ts.tv_sec * NSEC_PER_SEC + ts.tv_nsec;
}

void foo_yield(struct umcg_task *self)

{
	self->state = UMCG_TASK_RUNNABLE | UMCG_TF_COND_WAIT;
	sys_umcg_wait(0, 0);
}

#define TICK_NSEC NSEC_PER_SEC

static volatile bool foo_preemptible = false;

/* our workers */

static volatile bool done = false;

static void umcg_signal(int signr)

{
	done = true;
}

/* always running worker */
void *worker_fn0(void *arg)

{
	struct foo_server *server = arg;
	struct foo_task task = { };
	unsigned long i;
	int ret;

	task.tid = gettid();
	task.task.server_tid = server->tid;
	task.task.state = UMCG_TASK_BLOCKED;

	printf("A == %d\n", gettid());

	ret = sys_umcg_ctl(UMCG_CTL_REGISTER|UMCG_CTL_WORKER, &task.task, CLOCK_MONOTONIC);
	if (ret) {
		perror("umcg_ctl(A): ");
		exit(-1);
	}

	__atomic_add_fetch(&server->workers, 1, __ATOMIC_RELAXED);

	while (!done) {
		int x = i++;

		if (!(x % 1000000)) {
			putchar('.');
			fflush(stdout);
		}

		/* co-operative or preemptible */
		if (!foo_preemptible && !(x % 10000000))
			foo_yield(&task.task);
	}

	printf("A == done\n");

	__atomic_add_fetch(&server->workers, -1, __ATOMIC_RELAXED);

	ret = sys_umcg_ctl(UMCG_CTL_UNREGISTER|UMCG_CTL_WORKER, &task.task, 0);
	if (ret) {
		perror("umcg_ctl(~A): ");
		exit(-1);
	}

	return NULL;
}

/* event driven worker */
void *worker_fn1(void *arg)

{
	struct foo_server *server = arg;
	struct foo_task task = { };
	int ret;

	task.tid = gettid();
	task.task.server_tid = server->tid;
	task.task.state = UMCG_TASK_BLOCKED;

	printf("B == %d\n", gettid());

	ret = sys_umcg_ctl(UMCG_CTL_REGISTER|UMCG_CTL_WORKER, &task.task, CLOCK_MONOTONIC);
	if (ret) {
		perror("umcg_ctl(B): ");
		exit(-1);
	}

	__atomic_add_fetch(&server->workers, 1, __ATOMIC_RELAXED);

	while (!done) {
		printf("B\n");
		fflush(stdout);

		sleep(1);
	}

	printf("B == done\n");

	__atomic_add_fetch(&server->workers, -1, __ATOMIC_RELAXED);

	ret = sys_umcg_ctl(UMCG_CTL_UNREGISTER|UMCG_CTL_WORKER, &task.task, 0);
	if (ret) {
		perror("umcg_ctl(~B): ");
		exit(-1);
	}

	return NULL;
}

void *worker_fn2(void *arg)

{
	struct foo_server *server = arg;
	struct foo_task task = { };
	int ret;

	task.tid = gettid();
	task.task.server_tid = server->tid;
	task.task.state = UMCG_TASK_BLOCKED;

	printf("C == %d\n", gettid());

	ret = sys_umcg_ctl(UMCG_CTL_REGISTER|UMCG_CTL_WORKER, &task.task, CLOCK_MONOTONIC);
	if (ret) {
		perror("umcg_ctl(C): ");
		exit(-1);
	}

	__atomic_add_fetch(&server->workers, 1, __ATOMIC_RELAXED);

	while (!done) {
		printf("C\n");
		fflush(stdout);

		sleep(2);
	}

	printf("C == done\n");

	__atomic_add_fetch(&server->workers, -1, __ATOMIC_RELAXED);

	ret = sys_umcg_ctl(UMCG_CTL_UNREGISTER|UMCG_CTL_WORKER, &task.task, 0);
	if (ret) {
		perror("umcg_ctl(~C): ");
		exit(-1);
	}

	return NULL;
}

/* the server */

int main(int argc, char **argv)

{
	struct umcg_task *runnable_ptr, *next;
	struct foo_server server = { };
	pthread_t worker[3];
	u64 timeout = 0;
	u32 tid;
	int ret;

	struct sigaction sa = {
		.sa_handler = umcg_signal,
	};

	sigaction(SIGINT, &sa, NULL);

	printf("server == %d\n", gettid());
	fflush(stdout);

	server.tid = gettid();
	INIT_LIST_HEAD(&server.node);
	server.task.server_tid = gettid();
	server.task.state = UMCG_TASK_RUNNING;

	ret = sys_umcg_ctl(UMCG_CTL_REGISTER, &server.task, CLOCK_MONOTONIC);
	if (ret) {
		perror("umcg_ctl: ");
		exit(-1);
	}

	pthread_create(&worker[0], NULL, worker_fn0, &server);
	pthread_create(&worker[1], NULL, worker_fn1, &server);
	pthread_create(&worker[2], NULL, worker_fn2, &server);

	if (argc > 1) {
		foo_preemptible = true;
		/*
		 * setup preemption tick
		 */
		timeout = foo_time() + TICK_NSEC;
	}

	while (!(done && !__atomic_load_n(&server.workers, __ATOMIC_RELAXED))) {
		/*
		 * Mark the server as runnable first, so we can detect
		 * additions to the runnable list after we read it.
		 */
		__atomic_store_n(&server.task.state,
				 UMCG_TASK_RUNNABLE | UMCG_TF_COND_WAIT,
				 __ATOMIC_RELAXED);

		/*
		 * comsume the runnable notification list and add
		 * the tasks to our local runqueue.
		 */
		runnable_ptr = (void*)__atomic_exchange_n(&server.task.runnable_workers_ptr,
							  NULL, __ATOMIC_SEQ_CST);
		while (runnable_ptr) {
			next = (void *)runnable_ptr->runnable_workers_ptr;
			foo_add(&server, runnable_ptr);
			runnable_ptr = next;
		}

		if (server.cur && server.cur->task.state == UMCG_TASK_RUNNING) {
			/*
			 * Assert ::next_tid still points there and has RUNNING bit on
			 */
			if (server.task.next_tid != (server.cur->tid | UMCG_TID_RUNNING)) {
				printf("current not running: %d %x\n",
				       server.task.next_tid & UMCG_TID_MASK,
				       server.task.next_tid & ~UMCG_TID_MASK);
				exit(-1);
			}

			putchar('x');
		} else {
			tid = 0;
			server.cur = foo_pick_next(&server);
			if (server.cur)
				tid = server.cur->tid;

			__atomic_store_n(&server.task.next_tid, tid, __ATOMIC_RELAXED);

			printf("pick: %d\n", tid);
		}
		fflush(stdout);

		ret = sys_umcg_wait(0, timeout);

		/*
		 * If we set ::next_tid but it hasn't been consumed by the
		 * syscall due to failure, make sure to put the task back on
		 * the queue, lest we leak it.
		 */
		tid = __atomic_load_n(&server.task.next_tid, __ATOMIC_RELAXED);
		if (tid && !(tid & UMCG_TID_RUNNING)) {
			foo_add(&server, &server.cur->task);
			server.cur = NULL;
			putchar('*');
		}

		if (!ret)
			continue;

		switch (errno) {
		case EAGAIN:
			/*
			 * Got a wakeup, try again.
			 */
			continue;

		case ETIMEDOUT:
			/*
			 * timeout: drive preemption
			 */
			putchar('t');
			fflush(stdout);

			/*
			 * Next tick..
			 */
			timeout += TICK_NSEC;

			/*
			 * If we have a current, cmpxchg set TF_PREEMPT and on success
			 * send it a signal to kick it into the kernel such that
			 * it might re-report itself runnable.
			 */
			if (server.cur) {
				struct foo_task *t = server.cur;
				u32 val = UMCG_TASK_RUNNING;
				u32 new = UMCG_TASK_RUNNING | UMCG_TF_PREEMPT;

				if (__atomic_compare_exchange_n(&t->task.state, &val, new,
								false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) {
					sys_umcg_kick(0, t->tid);
				}
			}
			/*
			 * Either way around, if the cmpxchg
			 * failed the task will have blocked
			 * and we should re-start the loop.
			 */
			continue;

		default:
			printf("errno: %d\n", errno);
			perror("wait:");
			exit(-1);
		}
	}

	pthread_join(worker[0], NULL);
	pthread_join(worker[1], NULL);
	pthread_join(worker[2], NULL);

	return 0;
}
author maintainer
#1 lore raw

RFC PATCH v2 1/5 mm: Avoid unmapping pinned pages

Add a guarantee for Anon pages that pin_user_page*() ensures the user-mapping of these pages stay preserved. In order to ensure this all rmap users have been audited:

vmscan: already fails eviction due to page_maybe_dma_pinned()

migrate: migration will fail on pinned pages due to
expected_page_refs() not matching, however that is *after* try_to_migrate() has already destroyed the user mapping of these pages. Add an early exit for this case.

numa-balance: as per the above, pinned pages cannot be migrated,
however numa balancing scanning will happily PROT_NONE them to get usage information on these pages. Avoid this for pinned pages.

None of the other rmap users (damon,page-idle,mlock,..) unmap the page, they mostly just muck about with reference,dirty flags etc.

This same guarantee cannot be provided for Shared (file) pages due to dirty page tracking.

---
 mm/migrate.c  |   10 +++++++++-
 mm/mprotect.c |    6 ++++++
 2 files changed, 15 insertions(+), 1 deletion(-)
mm/migrate.c +9 -1
@@ -1472,7 +1472,15 @@ int migrate_pages(struct list_head *from
 			nr_subpages = thp_nr_pages(page);
 			cond_resched();
 
-			if (PageHuge(page))
+			/*
+			 * If the page has a pin then expected_page_refs() will
+			 * not match and the whole migration will fail later
+			 * anyway, fail early and preserve the mappings.
+			 */
+			if (page_maybe_dma_pinned(page))
+				rc = -EAGAIN;
+
+			else if (PageHuge(page))
 				rc = unmap_and_move_huge_page(get_new_page,
 						put_new_page, private, page,
 						pass > 2, mode, reason,
mm/mprotect.c +6 -0
@@ -106,6 +106,12 @@ static unsigned long change_pte_range(st
 					continue;
 
 				/*
+				 * Can't migrate pinned pages, avoid touching them.
+				 */
+				if (page_maybe_dma_pinned(page))
+					continue;
+
+				/*
 				 * Don't mess with PTEs if page is already on the node
 				 * a single-threaded process is running on.
 				 */
#2 lore raw

Re: [RFC][PATCH v2 1/5] mm: Avoid unmapping pinned pages

Quoted 28 lines · Peter Zijlstra

Add a guarantee for Anon pages that pin_user_page*() ensures the user-mapping of these pages stay preserved. In order to ensure this all rmap users have been audited:

vmscan: already fails eviction due to page_maybe_dma_pinned()

migrate: migration will fail on pinned pages due to
expected_page_refs() not matching, however that is *after* try_to_migrate() has already destroyed the user mapping of these pages. Add an early exit for this case.

numa-balance: as per the above, pinned pages cannot be migrated,
however numa balancing scanning will happily PROT_NONE them to get usage information on these pages. Avoid this for pinned pages.

None of the other rmap users (damon,page-idle,mlock,..) unmap the page, they mostly just muck about with reference,dirty flags etc.

This same guarantee cannot be provided for Shared (file) pages due to dirty page tracking.

[ snip ]

mm/mprotect.c static unsigned long change_pte_range(st · line 115

Quoted 13 lines
mm/mprotect.c +6 -0
@@ -106,6 +106,12 @@ static unsigned long change_pte_range(st
 					continue;
 
 				/*
+				 * Can't migrate pinned pages, avoid touching them.
+				 */
+				if (page_maybe_dma_pinned(page))
+					continue;
+
+				/*
 

I have a similar problem with userfaultfd changing protection for DMA-pinned pages. For userfaultfd it is important to know how many pages were actually modified.

I am working on a vectored UFFDIO_WRITEPROTECTV that aborts once a pinned page is encountered, but also returns the number of pages that were properly protected. I still need to do some work to send patches for that as it requires further changes (to return the number of pages that were handled).

But for the matter of your patch, is it possible to make this test generic (not migration specific) and rely on a new flag in cp_flags? I can of course make this change later if you prefer it this way.

author maintainer
#3 lore raw

Re: [RFC][PATCH v2 1/5] mm: Avoid unmapping pinned pages

Quoted 24 lines · Nadav Amit

mm/mprotect.c static unsigned long change_pte_range(st · line 115

Quoted 13 lines
mm/mprotect.c +6 -0
@@ -106,6 +106,12 @@ static unsigned long change_pte_range(st
 					continue;
 
 				/*
+				 * Can't migrate pinned pages, avoid touching them.
+				 */
+				if (page_maybe_dma_pinned(page))
+					continue;
+
+				/*
 

I have a similar problem with userfaultfd changing protection for DMA-pinned pages. For userfaultfd it is important to know how many pages were actually modified.

I am working on a vectored UFFDIO_WRITEPROTECTV that aborts once a pinned page is encountered, but also returns the number of pages that were properly protected. I still need to do some work to send patches for that as it requires further changes (to return the number of pages that were handled).

Quoted 4 lines

But for the matter of your patch, is it possible to make this test generic (not migration specific) and rely on a new flag in cp_flags? I can of course make this change later if you prefer it this way.

I have no objection to making it apply more widely, but I'm currently only interested in the rmap users. If userspace does mprotect() on it's own pages, it gets to keep whatever pieces are the result of that.

#4 lore raw

Re: [RFC][PATCH v2 1/5] mm: Avoid unmapping pinned pages

In reply to Peter Zijlstra
Quoted 12 lines · Peter Zijlstra

Add a guarantee for Anon pages that pin_user_page*() ensures the user-mapping of these pages stay preserved. In order to ensure this all rmap users have been audited:

vmscan: already fails eviction due to page_maybe_dma_pinned()

migrate: migration will fail on pinned pages due to
expected_page_refs() not matching, however that is *after* try_to_migrate() has already destroyed the user mapping of these pages. Add an early exit for this case.

Quoted 4 lines

numa-balance: as per the above, pinned pages cannot be migrated,
however numa balancing scanning will happily PROT_NONE them to get usage information on these pages. Avoid this for pinned pages.

page_maybe_dma_pinned() can race with GUP-fast without mm->write_protect_seq. This is a real problem for vmscan() with concurrent GUP-fast as it can result in R/O mappings of pinned pages and GUP will lose synchronicity to the page table on write faults due to wrong COW.

If you're just using it as an optimization, that should work just fine. I assume all migration will freeze the refcount and consequently bail out at that point. In that case, LGTM.

Signature · 3 lines
Thanks,

David / dhildenb
author maintainer
#5 lore raw

Re: [RFC][PATCH v2 1/5] mm: Avoid unmapping pinned pages

Quoted 18 lines · David Hildenbrand
Quoted 12 lines · Peter Zijlstra

Add a guarantee for Anon pages that pin_user_page*() ensures the user-mapping of these pages stay preserved. In order to ensure this all rmap users have been audited:

vmscan: already fails eviction due to page_maybe_dma_pinned()

migrate: migration will fail on pinned pages due to
expected_page_refs() not matching, however that is *after* try_to_migrate() has already destroyed the user mapping of these pages. Add an early exit for this case.

Quoted 4 lines

numa-balance: as per the above, pinned pages cannot be migrated,
however numa balancing scanning will happily PROT_NONE them to get usage information on these pages. Avoid this for pinned pages.

Quoted 5 lines

page_maybe_dma_pinned() can race with GUP-fast without mm->write_protect_seq. This is a real problem for vmscan() with concurrent GUP-fast as it can result in R/O mappings of pinned pages and GUP will lose synchronicity to the page table on write faults due to wrong COW.

Urgh, so yeah, that might be a problem. Follow up code uses it like this:

+/*
+ * Pinning a page inhibits rmap based unmap for Anon pages. Doing a load
+ * through the user mapping ensures the user mapping exists.
+ */

+#define umcg_pin_and_load(_self, _pagep, _member)                              \
+({                                                                             \
+       __label__ __out;                                                        \
+       int __ret = -EFAULT;                                                    \
+                                                                               \
+       if (pin_user_pages_fast((unsigned long)(_self), 1, 0, &(_pagep)) != 1)  \
+               goto __out;                                                     \
+                                                                               \
+       if (!PageAnon(_pagep) ||                                                \
+           get_user(_member, &(_self)->_member)) {                             \
+               unpin_user_page(_pagep);                                        \
+               goto __out;                                                     \
+       }                                                                       \
+       __ret = 0;                                                              \
+__out: __ret;                                                                  \

+})

And after that hard assumes (on the penalty of SIGKILL) that direct user access works. Specifically it does RmW ops on it. So I suppose I'd better upgrade that load to a RmW at the very least.

But is that sufficient? Let me go find that race you mention...

#6 lore raw

Re: [RFC][PATCH v2 1/5] mm: Avoid unmapping pinned pages

Quoted 49 lines · Peter Zijlstra
Quoted 18 lines · David Hildenbrand
Quoted 12 lines · Peter Zijlstra

Add a guarantee for Anon pages that pin_user_page*() ensures the user-mapping of these pages stay preserved. In order to ensure this all rmap users have been audited:

vmscan: already fails eviction due to page_maybe_dma_pinned()

migrate: migration will fail on pinned pages due to
expected_page_refs() not matching, however that is *after* try_to_migrate() has already destroyed the user mapping of these pages. Add an early exit for this case.

Quoted 4 lines

numa-balance: as per the above, pinned pages cannot be migrated,
however numa balancing scanning will happily PROT_NONE them to get usage information on these pages. Avoid this for pinned pages.

Quoted 5 lines

page_maybe_dma_pinned() can race with GUP-fast without mm->write_protect_seq. This is a real problem for vmscan() with concurrent GUP-fast as it can result in R/O mappings of pinned pages and GUP will lose synchronicity to the page table on write faults due to wrong COW.

Urgh, so yeah, that might be a problem. Follow up code uses it like this:

(quoted diff) +20 -0
+/*
+ * Pinning a page inhibits rmap based unmap for Anon pages. Doing a load
+ * through the user mapping ensures the user mapping exists.
+ */
+#define umcg_pin_and_load(_self, _pagep, _member)                              \
+({                                                                             \
+       __label__ __out;                                                        \
+       int __ret = -EFAULT;                                                    \
+                                                                               \
+       if (pin_user_pages_fast((unsigned long)(_self), 1, 0, &(_pagep)) != 1)  \
+               goto __out;                                                     \
+                                                                               \
+       if (!PageAnon(_pagep) ||                                                \
+           get_user(_member, &(_self)->_member)) {                             \
+               unpin_user_page(_pagep);                                        \
+               goto __out;                                                     \
+       }                                                                       \
+       __ret = 0;                                                              \
+__out: __ret;                                                                  \
+})
Quoted 6 lines

And after that hard assumes (on the penalty of SIGKILL) that direct user access works. Specifically it does RmW ops on it. So I suppose I'd better upgrade that load to a RmW at the very least.

But is that sufficient? Let me go find that race you mention...

It's described in [1] under point 3.

After we put the page into the swapcache, it's still mapped into the page tables, where GUP can find it. Only after that, we try to unmap the page (placing swap entries). So it's racy.

Note also point 2. in [1], which is related to O_DIRECT that does currently not yet use FOLL_PIN but uses FOLL_GET.

[1]
https://lore.kernel.org/r/[email protected]

Signature · 3 lines
Thanks,

David / dhildenb
author maintainer
#7 lore raw

Re: [RFC][PATCH v2 1/5] mm: Avoid unmapping pinned pages

In reply to Peter Zijlstra
Quoted 49 lines · Peter Zijlstra
Quoted 18 lines · David Hildenbrand
Quoted 12 lines · Peter Zijlstra

Add a guarantee for Anon pages that pin_user_page*() ensures the user-mapping of these pages stay preserved. In order to ensure this all rmap users have been audited:

vmscan: already fails eviction due to page_maybe_dma_pinned()

migrate: migration will fail on pinned pages due to
expected_page_refs() not matching, however that is *after* try_to_migrate() has already destroyed the user mapping of these pages. Add an early exit for this case.

Quoted 4 lines

numa-balance: as per the above, pinned pages cannot be migrated,
however numa balancing scanning will happily PROT_NONE them to get usage information on these pages. Avoid this for pinned pages.

Quoted 5 lines

page_maybe_dma_pinned() can race with GUP-fast without mm->write_protect_seq. This is a real problem for vmscan() with concurrent GUP-fast as it can result in R/O mappings of pinned pages and GUP will lose synchronicity to the page table on write faults due to wrong COW.

Urgh, so yeah, that might be a problem. Follow up code uses it like this:

(quoted diff) +20 -0
+/*
+ * Pinning a page inhibits rmap based unmap for Anon pages. Doing a load
+ * through the user mapping ensures the user mapping exists.
+ */
+#define umcg_pin_and_load(_self, _pagep, _member)                              \
+({                                                                             \
+       __label__ __out;                                                        \
+       int __ret = -EFAULT;                                                    \
+                                                                               \
+       if (pin_user_pages_fast((unsigned long)(_self), 1, 0, &(_pagep)) != 1)  \
+               goto __out;                                                     \
+                                                                               \
+       if (!PageAnon(_pagep) ||                                                \
+           get_user(_member, &(_self)->_member)) {                             \
+               unpin_user_page(_pagep);                                        \
+               goto __out;                                                     \
+       }                                                                       \
+       __ret = 0;                                                              \
+__out: __ret;                                                                  \
+})
Quoted 5 lines

And after that hard assumes (on the penalty of SIGKILL) that direct user access works. Specifically it does RmW ops on it. So I suppose I'd better upgrade that load to a RmW at the very least.

But is that sufficient? Let me go find that race you mention...

OK, so copy_page_range() vs lockless_pages_from_mm(). Since I use FOLL_PIN that should be sorted, it'll fall back the slow path and use mmap_sem and serialize against the fork().

(Also, can I express my hate for __gup_longterm_unlocked(), that function name is utter garbage)

However, I'm not quite sure what fork() does with pages that have a pin. There's been a number of GUP vs fork() problems over the years, but I'm afraid I have lost track of that and I can't quickly find anything in the code..

Naively, a page that has async DMA activity should not be CoW'ed, or if it is, care must be taken to ensure the original pages stays in the original process, but I realize that's somewhat hard.

Let me dig in a bit more.

#8 lore raw

Re: [RFC][PATCH v2 1/5] mm: Avoid unmapping pinned pages

Quoted 54 lines · Peter Zijlstra
Quoted 53 lines · Peter Zijlstra
Quoted 18 lines · David Hildenbrand
Quoted 12 lines · Peter Zijlstra

Add a guarantee for Anon pages that pin_user_page*() ensures the user-mapping of these pages stay preserved. In order to ensure this all rmap users have been audited:

vmscan: already fails eviction due to page_maybe_dma_pinned()

migrate: migration will fail on pinned pages due to
expected_page_refs() not matching, however that is *after* try_to_migrate() has already destroyed the user mapping of these pages. Add an early exit for this case.

Quoted 4 lines

numa-balance: as per the above, pinned pages cannot be migrated,
however numa balancing scanning will happily PROT_NONE them to get usage information on these pages. Avoid this for pinned pages.

Quoted 5 lines

page_maybe_dma_pinned() can race with GUP-fast without mm->write_protect_seq. This is a real problem for vmscan() with concurrent GUP-fast as it can result in R/O mappings of pinned pages and GUP will lose synchronicity to the page table on write faults due to wrong COW.

Urgh, so yeah, that might be a problem. Follow up code uses it like this:

(quoted diff) +20 -0
+/*
+ * Pinning a page inhibits rmap based unmap for Anon pages. Doing a load
+ * through the user mapping ensures the user mapping exists.
+ */
+#define umcg_pin_and_load(_self, _pagep, _member)                              \
+({                                                                             \
+       __label__ __out;                                                        \
+       int __ret = -EFAULT;                                                    \
+                                                                               \
+       if (pin_user_pages_fast((unsigned long)(_self), 1, 0, &(_pagep)) != 1)  \
+               goto __out;                                                     \
+                                                                               \
+       if (!PageAnon(_pagep) ||                                                \
+           get_user(_member, &(_self)->_member)) {                             \
+               unpin_user_page(_pagep);                                        \
+               goto __out;                                                     \
+       }                                                                       \
+       __ret = 0;                                                              \
+__out: __ret;                                                                  \
+})

And after that hard assumes (on the penalty of SIGKILL) that direct user access works. Specifically it does RmW ops on it. So I suppose I'd better upgrade that load to a RmW at the very least.

Quoted 8 lines
Quoted 1 line

But is that sufficient? Let me go find that race you mention...

OK, so copy_page_range() vs lockless_pages_from_mm(). Since I use FOLL_PIN that should be sorted, it'll fall back the slow path and use mmap_sem and serialize against the fork().

(Also, can I express my hate for __gup_longterm_unlocked(), that function name is utter garbage)

Absolutely, the "_unlocked_ also caused a lot of confusion on my end in the past.

Quoted 2 lines

However, I'm not quite sure what fork() does with pages that have a pin.

We COW the anon pages always, and we protect against concurrent GUP using the
* mmap_lock in exclusive mode for ordinary GUP
* mm->write_protect_seq for GUP-fast

Quoted 4 lines

Naively, a page that has async DMA activity should not be CoW'ed, or if it is, care must be taken to ensure the original pages stays in the original process, but I realize that's somewhat hard.

That's precisely what I'm working on fixing ... and yes, it's hard.

Let me know if you need any other information, I've spent way too much time on this than I ever panned.

Signature · 3 lines
Thanks,

David / dhildenb
author maintainer
#9 lore raw

Re: [RFC][PATCH v2 1/5] mm: Avoid unmapping pinned pages

Quoted 8 lines · David Hildenbrand
Quoted 1 line · Peter Zijlstra

However, I'm not quite sure what fork() does with pages that have a pin.

We COW the anon pages always, and we protect against concurrent GUP using the
* mmap_lock in exclusive mode for ordinary GUP
* mm->write_protect_seq for GUP-fast

Right, but neither the mmap_sem nor the write_protect_seq help anything at all vs already extant page pins.

But I just found copy_present_page()'s page_needs_cow_for_dma(), which I think deals with exactly that case, it avoids doing CoW on pinned pages and instead feeds the child a full copy while keeping the pinned page in the original process.

Quoted 8 lines
Quoted 3 lines

Naively, a page that has async DMA activity should not be CoW'ed, or if it is, care must be taken to ensure the original pages stays in the original process, but I realize that's somewhat hard.

That's precisely what I'm working on fixing ... and yes, it's hard.

Let me know if you need any other information, I've spent way too much time on this than I ever panned.

So let me try and get this right:

- GUP post-fork breaks CoW for FOLL_WRITE/FOLL_PIN, without either there's a problem where one task might observe changes by another.

- GUP pre-fork prevents CoW and does a full copy.

And that all mostly works, except for a fair amount of 'fun' cases?

#10 lore raw

Re: [RFC][PATCH v2 1/5] mm: Avoid unmapping pinned pages

Quoted 10 lines · Peter Zijlstra
Quoted 8 lines · David Hildenbrand
Quoted 1 line · Peter Zijlstra

However, I'm not quite sure what fork() does with pages that have a pin.

We COW the anon pages always, and we protect against concurrent GUP using the
* mmap_lock in exclusive mode for ordinary GUP
* mm->write_protect_seq for GUP-fast

Quoted 7 lines

Right, but neither the mmap_sem nor the write_protect_seq help anything at all vs already extant page pins.

But I just found copy_present_page()'s page_needs_cow_for_dma(), which I think deals with exactly that case, it avoids doing CoW on pinned pages and instead feeds the child a full copy while keeping the pinned page in the original process.

Yes, page_needs_cow_for_dma() is the magic bit. The locking I explained keep its output "reliable".

Quoted 10 lines
Quoted 9 lines
Quoted 4 lines

Naively, a page that has async DMA activity should not be CoW'ed, or if it is, care must be taken to ensure the original pages stays in the original process, but I realize that's somewhat hard.

That's precisely what I'm working on fixing ... and yes, it's hard.

Let me know if you need any other information, I've spent way too much time on this than I ever panned.

Quoted 6 lines

So let me try and get this right:

- GUP post-fork breaks CoW for FOLL_WRITE/FOLL_PIN, without either there's a problem where one task might observe changes by another.

- GUP pre-fork prevents CoW and does a full copy.

Yes, pretty much.

Quoted 2 lines

And that all mostly works, except for a fair amount of 'fun' cases?

I'd say some obviously broken cases, some racy cases, some fun cases :)

We have three main cases. And usually, trying to tackle one triggers another.

(1) Missed CoW

If the child R/O pins and unmaps the page, the parent might miss to CoW and reuse the page. Security issue. Once CVE in that area is currently still applicable for THP (well, and hugetlb).

(2) Unnecessary CoW

We CoW instead of reusing the page, but there are no relevant pins, so it's unnecessary.

(3) Wrong CoW

We CoW a page that has relevant pins, losing synchronicity between GUP and the page tables.

The "criticality" is (1), (3), (2).

(3) can currently get triggered by anything that can map a pinned page R/O. The racy case is what I described about the swapcache. Other broken cases are mprotect() and friends (we cannot differentiate between R/O and R/W pins ...).

Signature · 3 lines
Thanks,

David / dhildenb
author maintainer
#11 lore raw

RFC PATCH v2 2/5 entry,x86: Create common IRQ operations for exceptions

In reply to Peter Zijlstra

A number of exceptions can re-enable IRQs and schedule (provided the context taking the exception has IRQs enabled etc.). Currently this isn't standardized and each architecture does it slightly different.

(Notably, ARM64 restores/inherits more than just the IRQ state, suggesting at least some form of arch_ hook might be appropriate on top of this).

Create a single set of functions for this and convert x86 to use these. The purpose is to have a single (common) place to hook in order to cover all the exceptions that can schedule().

---
 arch/x86/kernel/traps.c      |   48 ++++++++++++++++---------------------------
 arch/x86/mm/fault.c          |   28 ++++++++++++-------------
 include/linux/entry-common.h |   24 +++++++++++++++++++++
 3 files changed, 56 insertions(+), 44 deletions(-)
arch/x86/kernel/traps.c +18 -30
@@ -73,18 +73,6 @@
 
 DECLARE_BITMAP(system_vectors, NR_VECTORS);
 
-static inline void cond_local_irq_enable(struct pt_regs *regs)
-{
-	if (regs->flags & X86_EFLAGS_IF)
-		local_irq_enable();
-}
-
-static inline void cond_local_irq_disable(struct pt_regs *regs)
-{
-	if (regs->flags & X86_EFLAGS_IF)
-		local_irq_disable();
-}
-
 __always_inline int is_valid_bugaddr(unsigned long addr)
 {
 	if (addr < TASK_SIZE_MAX)
@@ -177,9 +165,9 @@ static void do_error_trap(struct pt_regs
 
 	if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) !=
 			NOTIFY_STOP) {
-		cond_local_irq_enable(regs);
+		irqentry_irq_enable(regs);
 		do_trap(trapnr, signr, str, regs, error_code, sicode, addr);
-		cond_local_irq_disable(regs);
+		irqentry_irq_disable(regs);
 	}
 }
 
@@ -300,7 +288,7 @@ DEFINE_IDTENTRY_ERRORCODE(exc_alignment_
 	if (!user_mode(regs))
 		die("Split lock detected\n", regs, error_code);
 
-	local_irq_enable();
+	irqentry_irq_enable(regs);
 
 	if (handle_user_split_lock(regs, error_code))
 		goto out;
@@ -309,7 +297,7 @@ DEFINE_IDTENTRY_ERRORCODE(exc_alignment_
 		error_code, BUS_ADRALN, NULL);
 
 out:
-	local_irq_disable();
+	irqentry_irq_disable(regs);
 }
 
 #ifdef CONFIG_VMAP_STACK
@@ -473,14 +461,14 @@ DEFINE_IDTENTRY(exc_bounds)
 	if (notify_die(DIE_TRAP, "bounds", regs, 0,
 			X86_TRAP_BR, SIGSEGV) == NOTIFY_STOP)
 		return;
-	cond_local_irq_enable(regs);
+	irqentry_irq_enable(regs);
 
 	if (!user_mode(regs))
 		die("bounds", regs, 0);
 
 	do_trap(X86_TRAP_BR, SIGSEGV, "bounds", regs, 0, 0, NULL);
 
-	cond_local_irq_disable(regs);
+	irqentry_irq_disable(regs);
 }
 
 enum kernel_gp_hint {
@@ -567,7 +555,7 @@ DEFINE_IDTENTRY_ERRORCODE(exc_general_pr
 	unsigned long gp_addr;
 	int ret;
 
-	cond_local_irq_enable(regs);
+	irqentry_irq_enable(regs);
 
 	if (static_cpu_has(X86_FEATURE_UMIP)) {
 		if (user_mode(regs) && fixup_umip_exception(regs))
@@ -638,7 +626,7 @@ DEFINE_IDTENTRY_ERRORCODE(exc_general_pr
 	die_addr(desc, regs, error_code, gp_addr);
 
 exit:
-	cond_local_irq_disable(regs);
+	irqentry_irq_disable(regs);
 }
 
 static bool do_int3(struct pt_regs *regs)
@@ -665,9 +653,9 @@ static void do_int3_user(struct pt_regs
 	if (do_int3(regs))
 		return;
 
-	cond_local_irq_enable(regs);
+	irqentry_irq_enable(regs);
 	do_trap(X86_TRAP_BP, SIGTRAP, "int3", regs, 0, 0, NULL);
-	cond_local_irq_disable(regs);
+	irqentry_irq_disable(regs);
 }
 
 DEFINE_IDTENTRY_RAW(exc_int3)
@@ -1003,7 +991,7 @@ static __always_inline void exc_debug_us
 		goto out;
 
 	/* It's safe to allow irq's after DR6 has been saved */
-	local_irq_enable();
+	irqentry_irq_enable(regs);
 
 	if (v8086_mode(regs)) {
 		handle_vm86_trap((struct kernel_vm86_regs *)regs, 0, X86_TRAP_DB);
@@ -1020,7 +1008,7 @@ static __always_inline void exc_debug_us
 		send_sigtrap(regs, 0, get_si_code(dr6));
 
 out_irq:
-	local_irq_disable();
+	irqentry_irq_disable(regs);
 out:
 	instrumentation_end();
 	irqentry_exit_to_user_mode(regs);
@@ -1064,7 +1052,7 @@ static void math_error(struct pt_regs *r
 	char *str = (trapnr == X86_TRAP_MF) ? "fpu exception" :
 						"simd exception";
 
-	cond_local_irq_enable(regs);
+	irqentry_irq_enable(regs);
 
 	if (!user_mode(regs)) {
 		if (fixup_exception(regs, trapnr, 0, 0))
@@ -1099,7 +1087,7 @@ static void math_error(struct pt_regs *r
 	force_sig_fault(SIGFPE, si_code,
 			(void __user *)uprobe_get_trap_addr(regs));
 exit:
-	cond_local_irq_disable(regs);
+	irqentry_irq_disable(regs);
 }
 
 DEFINE_IDTENTRY(exc_coprocessor_error)
@@ -1160,7 +1148,7 @@ static bool handle_xfd_event(struct pt_r
 	if (WARN_ON(!user_mode(regs)))
 		return false;
 
-	local_irq_enable();
+	irqentry_irq_enable(regs);
 
 	err = xfd_enable_feature(xfd_err);
 
@@ -1173,7 +1161,7 @@ static bool handle_xfd_event(struct pt_r
 		break;
 	}
 
-	local_irq_disable();
+	irqentry_irq_disable(regs);
 	return true;
 }
 
@@ -1188,12 +1176,12 @@ DEFINE_IDTENTRY(exc_device_not_available
 	if (!boot_cpu_has(X86_FEATURE_FPU) && (cr0 & X86_CR0_EM)) {
 		struct math_emu_info info = { };
 
-		cond_local_irq_enable(regs);
+		irqentry_irq_enable(regs);
 
 		info.regs = regs;
 		math_emulate(&info);
 
-		cond_local_irq_disable(regs);
+		irqentry_irq_disable(regs);
 		return;
 	}
 #endif
arch/x86/mm/fault.c +14 -14
@@ -1209,6 +1209,12 @@ do_kern_addr_fault(struct pt_regs *regs,
 NOKPROBE_SYMBOL(do_kern_addr_fault);
 
 /*
+ * EFLAGS[3] is unused and ABI defined to be 0, use it to store IRQ state,
+ * because do_user_addr_fault() is too convoluted to track things.
+ */
+#define X86_EFLAGS_MISC		(1UL << 3)
+
+/*
  * Handle faults in the user portion of the address space.  Nothing in here
  * should check X86_PF_USER without a specific justification: for almost
  * all purposes, we should treat a normal kernel access to user memory
@@ -1290,13 +1296,11 @@ void do_user_addr_fault(struct pt_regs *
 	 * User-mode registers count as a user access even for any
 	 * potential system fault or CPU buglet:
 	 */
-	if (user_mode(regs)) {
-		local_irq_enable();
+	if (user_mode(regs))
 		flags |= FAULT_FLAG_USER;
-	} else {
-		if (regs->flags & X86_EFLAGS_IF)
-			local_irq_enable();
-	}
+
+	irqentry_irq_enable(regs);
+	regs->flags |= X86_EFLAGS_MISC;
 
 	perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
 
@@ -1483,14 +1487,10 @@ handle_page_fault(struct pt_regs *regs,
 		do_kern_addr_fault(regs, error_code, address);
 	} else {
 		do_user_addr_fault(regs, error_code, address);
-		/*
-		 * User address page fault handling might have reenabled
-		 * interrupts. Fixing up all potential exit points of
-		 * do_user_addr_fault() and its leaf functions is just not
-		 * doable w/o creating an unholy mess or turning the code
-		 * upside down.
-		 */
-		local_irq_disable();
+		if (regs->flags & X86_EFLAGS_MISC) {
+			regs->flags &= ~X86_EFLAGS_MISC;
+			irqentry_irq_disable(regs);
+		}
 	}
 }
 
include/linux/entry-common.h +24 -0
@@ -7,6 +7,7 @@
 #include <linux/syscalls.h>
 #include <linux/seccomp.h>
 #include <linux/sched.h>
+#include <asm/ptrace.h>
 
 #include <asm/entry-common.h>
 
@@ -213,6 +214,29 @@ static inline void local_irq_disable_exi
 #endif
 
 /**
+ * irqentry_irq_enable - Conditionally enable IRQs from exceptions
+ *
+ * Common code for exceptions to (re)enable IRQs, typically done to allow
+ * from-user exceptions to schedule (since they run on the task stack).
+ */
+static inline void irqentry_irq_enable(struct pt_regs *regs)
+{
+	if (!regs_irqs_disabled(regs))
+		local_irq_enable();
+}
+
+/**
+ * irqentry_irq_disable - Conditionally disable IRQs from exceptions
+ *
+ * Counterpart of irqentry_irq_enable().
+ */
+static inline void irqentry_irq_disable(struct pt_regs *regs)
+{
+	if (!regs_irqs_disabled(regs))
+		local_irq_disable();
+}
+
+/**
  * arch_exit_to_user_mode_work - Architecture specific TIF work for exit
  *				 to user mode.
  * @regs:	Pointer to currents pt_regs
#12 lore raw

Re: [RFC][PATCH v2 2/5] entry,x86: Create common IRQ operations for exceptions

Quoted 7 lines · Peter Zijlstra

A number of exceptions can re-enable IRQs and schedule (provided the context taking the exception has IRQs enabled etc.). Currently this isn't standardized and each architecture does it slightly different.

(Notably, ARM64 restores/inherits more than just the IRQ state, suggesting at least some form of arch_ hook might be appropriate on top of this).

For arm64, I suspect we want the arch code to call the umcg hooks directly, rather than adding arch_* hooks that get called by the generic irqentry_irq_{enable,disable}() functions, which doesn't really fit the way we want to conditionally inherit some mask bits upon entry (depending on the specific exception taken) and unconditionally mask everything prior to exit.

Ignoring that, arm64 also has separate vectors for exceptions taken from EL0 (userspace) and EL1 (kernel), so we don't need to look at the regs to know that an exception has been taken from userspace (which we always run with IRQs unmasked). For patch 5, where we actually add hooks, it'd be cleaner for us to directly invoke UMCG hooks in our enter_from_user_mode() and exit_to_user_mode() paths.

That said, if this is useful for other architectures, I don't have a problem for adding them as generic helpers; I just don't think we should be required to use them directly, and should make sure the underlying primitives are available and clearly documented.

Thanks,
Mark.

Quoted 272 lines

Create a single set of functions for this and convert x86 to use these. The purpose is to have a single (common) place to hook in order to cover all the exceptions that can schedule().

---
 arch/x86/kernel/traps.c      |   48 ++++++++++++++++---------------------------
 arch/x86/mm/fault.c          |   28 ++++++++++++-------------
 include/linux/entry-common.h |   24 +++++++++++++++++++++
 3 files changed, 56 insertions(+), 44 deletions(-)
arch/x86/kernel/traps.c +18 -30
@@ -73,18 +73,6 @@
 
 DECLARE_BITMAP(system_vectors, NR_VECTORS);
 
-static inline void cond_local_irq_enable(struct pt_regs *regs)
-{
-	if (regs->flags & X86_EFLAGS_IF)
-		local_irq_enable();
-}
-
-static inline void cond_local_irq_disable(struct pt_regs *regs)
-{
-	if (regs->flags & X86_EFLAGS_IF)
-		local_irq_disable();
-}
-
 __always_inline int is_valid_bugaddr(unsigned long addr)
 {
 	if (addr < TASK_SIZE_MAX)
@@ -177,9 +165,9 @@ static void do_error_trap(struct pt_regs
 
 	if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) !=
 			NOTIFY_STOP) {
-		cond_local_irq_enable(regs);
+		irqentry_irq_enable(regs);
 		do_trap(trapnr, signr, str, regs, error_code, sicode, addr);
-		cond_local_irq_disable(regs);
+		irqentry_irq_disable(regs);
 	}
 }
 
@@ -300,7 +288,7 @@ DEFINE_IDTENTRY_ERRORCODE(exc_alignment_
 	if (!user_mode(regs))
 		die("Split lock detected\n", regs, error_code);
 
-	local_irq_enable();
+	irqentry_irq_enable(regs);
 
 	if (handle_user_split_lock(regs, error_code))
 		goto out;
@@ -309,7 +297,7 @@ DEFINE_IDTENTRY_ERRORCODE(exc_alignment_
 		error_code, BUS_ADRALN, NULL);
 
 out:
-	local_irq_disable();
+	irqentry_irq_disable(regs);
 }
 
 #ifdef CONFIG_VMAP_STACK
@@ -473,14 +461,14 @@ DEFINE_IDTENTRY(exc_bounds)
 	if (notify_die(DIE_TRAP, "bounds", regs, 0,
 			X86_TRAP_BR, SIGSEGV) == NOTIFY_STOP)
 		return;
-	cond_local_irq_enable(regs);
+	irqentry_irq_enable(regs);
 
 	if (!user_mode(regs))
 		die("bounds", regs, 0);
 
 	do_trap(X86_TRAP_BR, SIGSEGV, "bounds", regs, 0, 0, NULL);
 
-	cond_local_irq_disable(regs);
+	irqentry_irq_disable(regs);
 }
 
 enum kernel_gp_hint {
@@ -567,7 +555,7 @@ DEFINE_IDTENTRY_ERRORCODE(exc_general_pr
 	unsigned long gp_addr;
 	int ret;
 
-	cond_local_irq_enable(regs);
+	irqentry_irq_enable(regs);
 
 	if (static_cpu_has(X86_FEATURE_UMIP)) {
 		if (user_mode(regs) && fixup_umip_exception(regs))
@@ -638,7 +626,7 @@ DEFINE_IDTENTRY_ERRORCODE(exc_general_pr
 	die_addr(desc, regs, error_code, gp_addr);
 
 exit:
-	cond_local_irq_disable(regs);
+	irqentry_irq_disable(regs);
 }
 
 static bool do_int3(struct pt_regs *regs)
@@ -665,9 +653,9 @@ static void do_int3_user(struct pt_regs
 	if (do_int3(regs))
 		return;
 
-	cond_local_irq_enable(regs);
+	irqentry_irq_enable(regs);
 	do_trap(X86_TRAP_BP, SIGTRAP, "int3", regs, 0, 0, NULL);
-	cond_local_irq_disable(regs);
+	irqentry_irq_disable(regs);
 }
 
 DEFINE_IDTENTRY_RAW(exc_int3)
@@ -1003,7 +991,7 @@ static __always_inline void exc_debug_us
 		goto out;
 
 	/* It's safe to allow irq's after DR6 has been saved */
-	local_irq_enable();
+	irqentry_irq_enable(regs);
 
 	if (v8086_mode(regs)) {
 		handle_vm86_trap((struct kernel_vm86_regs *)regs, 0, X86_TRAP_DB);
@@ -1020,7 +1008,7 @@ static __always_inline void exc_debug_us
 		send_sigtrap(regs, 0, get_si_code(dr6));
 
 out_irq:
-	local_irq_disable();
+	irqentry_irq_disable(regs);
 out:
 	instrumentation_end();
 	irqentry_exit_to_user_mode(regs);
@@ -1064,7 +1052,7 @@ static void math_error(struct pt_regs *r
 	char *str = (trapnr == X86_TRAP_MF) ? "fpu exception" :
 						"simd exception";
 
-	cond_local_irq_enable(regs);
+	irqentry_irq_enable(regs);
 
 	if (!user_mode(regs)) {
 		if (fixup_exception(regs, trapnr, 0, 0))
@@ -1099,7 +1087,7 @@ static void math_error(struct pt_regs *r
 	force_sig_fault(SIGFPE, si_code,
 			(void __user *)uprobe_get_trap_addr(regs));
 exit:
-	cond_local_irq_disable(regs);
+	irqentry_irq_disable(regs);
 }
 
 DEFINE_IDTENTRY(exc_coprocessor_error)
@@ -1160,7 +1148,7 @@ static bool handle_xfd_event(struct pt_r
 	if (WARN_ON(!user_mode(regs)))
 		return false;
 
-	local_irq_enable();
+	irqentry_irq_enable(regs);
 
 	err = xfd_enable_feature(xfd_err);
 
@@ -1173,7 +1161,7 @@ static bool handle_xfd_event(struct pt_r
 		break;
 	}
 
-	local_irq_disable();
+	irqentry_irq_disable(regs);
 	return true;
 }
 
@@ -1188,12 +1176,12 @@ DEFINE_IDTENTRY(exc_device_not_available
 	if (!boot_cpu_has(X86_FEATURE_FPU) && (cr0 & X86_CR0_EM)) {
 		struct math_emu_info info = { };
 
-		cond_local_irq_enable(regs);
+		irqentry_irq_enable(regs);
 
 		info.regs = regs;
 		math_emulate(&info);
 
-		cond_local_irq_disable(regs);
+		irqentry_irq_disable(regs);
 		return;
 	}
 #endif
arch/x86/mm/fault.c +14 -14
@@ -1209,6 +1209,12 @@ do_kern_addr_fault(struct pt_regs *regs,
 NOKPROBE_SYMBOL(do_kern_addr_fault);
 
 /*
+ * EFLAGS[3] is unused and ABI defined to be 0, use it to store IRQ state,
+ * because do_user_addr_fault() is too convoluted to track things.
+ */
+#define X86_EFLAGS_MISC		(1UL << 3)
+
+/*
  * Handle faults in the user portion of the address space.  Nothing in here
  * should check X86_PF_USER without a specific justification: for almost
  * all purposes, we should treat a normal kernel access to user memory
@@ -1290,13 +1296,11 @@ void do_user_addr_fault(struct pt_regs *
 	 * User-mode registers count as a user access even for any
 	 * potential system fault or CPU buglet:
 	 */
-	if (user_mode(regs)) {
-		local_irq_enable();
+	if (user_mode(regs))
 		flags |= FAULT_FLAG_USER;
-	} else {
-		if (regs->flags & X86_EFLAGS_IF)
-			local_irq_enable();
-	}
+
+	irqentry_irq_enable(regs);
+	regs->flags |= X86_EFLAGS_MISC;
 
 	perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
 
@@ -1483,14 +1487,10 @@ handle_page_fault(struct pt_regs *regs,
 		do_kern_addr_fault(regs, error_code, address);
 	} else {
 		do_user_addr_fault(regs, error_code, address);
-		/*
-		 * User address page fault handling might have reenabled
-		 * interrupts. Fixing up all potential exit points of
-		 * do_user_addr_fault() and its leaf functions is just not
-		 * doable w/o creating an unholy mess or turning the code
-		 * upside down.
-		 */
-		local_irq_disable();
+		if (regs->flags & X86_EFLAGS_MISC) {
+			regs->flags &= ~X86_EFLAGS_MISC;
+			irqentry_irq_disable(regs);
+		}
 	}
 }
 
include/linux/entry-common.h +24 -0
@@ -7,6 +7,7 @@
 #include <linux/syscalls.h>
 #include <linux/seccomp.h>
 #include <linux/sched.h>
+#include <asm/ptrace.h>
 
 #include <asm/entry-common.h>
 
@@ -213,6 +214,29 @@ static inline void local_irq_disable_exi
 #endif
 
 /**
+ * irqentry_irq_enable - Conditionally enable IRQs from exceptions
+ *
+ * Common code for exceptions to (re)enable IRQs, typically done to allow
+ * from-user exceptions to schedule (since they run on the task stack).
+ */
+static inline void irqentry_irq_enable(struct pt_regs *regs)
+{
+	if (!regs_irqs_disabled(regs))
+		local_irq_enable();
+}
+
+/**
+ * irqentry_irq_disable - Conditionally disable IRQs from exceptions
+ *
+ * Counterpart of irqentry_irq_enable().
+ */
+static inline void irqentry_irq_disable(struct pt_regs *regs)
+{
+	if (!regs_irqs_disabled(regs))
+		local_irq_disable();
+}
+
+/**
  * arch_exit_to_user_mode_work - Architecture specific TIF work for exit
  *				 to user mode.
  * @regs:	Pointer to currents pt_regs
author maintainer
#13 lore raw

RFC PATCH v2 3/5 sched/umcg: add WF_CURRENT_CPU and externise ttwu

In reply to Peter Zijlstra

From: Peter Oskolkov <[email protected]>

Add WF_CURRENT_CPU wake flag that advices the scheduler to move the wakee to the current CPU. This is useful for fast on-CPU context switching use cases such as UMCG.

In addition, make ttwu external rather than static so that the flag could be passed to it from outside of sched/core.c.

---
 kernel/sched/core.c  |    3 +--
 kernel/sched/fair.c  |    4 ++++
 kernel/sched/sched.h |   17 ++++++++++-------
 3 files changed, 15 insertions(+), 9 deletions(-)
kernel/sched/core.c +1 -2
@@ -3980,8 +3980,7 @@ bool ttwu_state_match(struct task_struct
  * Return: %true if @p->state changes (an actual wakeup was done),
  *	   %false otherwise.
  */
-static int
-try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
+int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
 {
 	unsigned long flags;
 	int cpu, success = 0;
kernel/sched/fair.c +4 -0
@@ -6838,6 +6838,10 @@ select_task_rq_fair(struct task_struct *
 	if (wake_flags & WF_TTWU) {
 		record_wakee(p);
 
+		if ((wake_flags & WF_CURRENT_CPU) &&
+		    cpumask_test_cpu(cpu, p->cpus_ptr))
+			return cpu;
+
 		if (sched_energy_enabled()) {
 			new_cpu = find_energy_efficient_cpu(p, prev_cpu);
 			if (new_cpu >= 0)
kernel/sched/sched.h +10 -7
@@ -2052,13 +2052,14 @@ static inline int task_on_rq_migrating(s
 }
 
 /* Wake flags. The first three directly map to some SD flag value */
-#define WF_EXEC     0x02 /* Wakeup after exec; maps to SD_BALANCE_EXEC */
-#define WF_FORK     0x04 /* Wakeup after fork; maps to SD_BALANCE_FORK */
-#define WF_TTWU     0x08 /* Wakeup;            maps to SD_BALANCE_WAKE */
-
-#define WF_SYNC     0x10 /* Waker goes to sleep after wakeup */
-#define WF_MIGRATED 0x20 /* Internal use, task got migrated */
-#define WF_ON_CPU   0x40 /* Wakee is on_cpu */
+#define WF_EXEC         0x02 /* Wakeup after exec; maps to SD_BALANCE_EXEC */
+#define WF_FORK         0x04 /* Wakeup after fork; maps to SD_BALANCE_FORK */
+#define WF_TTWU         0x08 /* Wakeup;            maps to SD_BALANCE_WAKE */
+
+#define WF_SYNC         0x10 /* Waker goes to sleep after wakeup */
+#define WF_MIGRATED     0x20 /* Internal use, task got migrated */
+#define WF_ON_CPU       0x40 /* Wakee is on_cpu */
+#define WF_CURRENT_CPU  0x80 /* Prefer to move the wakee to the current CPU. */
 
 #ifdef CONFIG_SMP
 static_assert(WF_EXEC == SD_BALANCE_EXEC);
@@ -3112,6 +3113,8 @@ static inline bool is_per_cpu_kthread(st
 extern void swake_up_all_locked(struct swait_queue_head *q);
 extern void __prepare_to_swait(struct swait_queue_head *q, struct swait_queue *wait);
 
+extern int try_to_wake_up(struct task_struct *tsk, unsigned int state, int wake_flags);
+
 #ifdef CONFIG_PREEMPT_DYNAMIC
 extern int preempt_dynamic_mode;
 extern int sched_dynamic_mode(const char *str);
author maintainer
#14 lore raw

RFC PATCH v2 4/5 x86/uaccess: Implement unsafe_try_cmpxchg_user()

In reply to Peter Zijlstra

Do try_cmpxchg() loops on userspace addresses.

---
 arch/x86/include/asm/uaccess.h |   67 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)
arch/x86/include/asm/uaccess.h +67 -0
@@ -342,6 +342,24 @@ do {									\
 		     : [umem] "m" (__m(addr))				\
 		     : : label)
 
+#define __try_cmpxchg_user_asm(itype, ltype, _ptr, _pold, _new, label)	({ \
+	bool success;							\
+	__typeof__(_ptr) _old = (__typeof__(_ptr))(_pold);		\
+	__typeof__(*(_ptr)) __old = *_old;				\
+	__typeof__(*(_ptr)) __new = (_new);				\
+	asm_volatile_goto("\n"						\
+		     "1: " LOCK_PREFIX "cmpxchg"itype" %[new], %[ptr]\n"\
+		     _ASM_EXTABLE_UA(1b, %l[label])			\
+		     : CC_OUT(z) (success),				\
+		       [ptr] "+m" (*_ptr),				\
+		       [old] "+a" (__old)				\
+		     : [new] ltype (__new)				\
+		     : "memory", "cc"					\
+		     : label);						\
+	if (unlikely(!success))						\
+		*_old = __old;						\
+	likely(success);					})
+
 #else // !CONFIG_CC_HAS_ASM_GOTO_OUTPUT
 
 #ifdef CONFIG_X86_32
@@ -407,6 +425,30 @@ do {									\
 		     : [umem] "m" (__m(addr)),				\
 		       "0" (err))
 
+#define __try_cmpxchg_user_asm(itype, ltype, _ptr, _pold, _new, label)	({ \
+	int __err = 0;							\
+	bool success;							\
+	__typeof__(_ptr) _old = (__typeof__(_ptr))(_pold);		\
+	__typeof__(*(_ptr)) __old = *_old;				\
+	__typeof__(*(_ptr)) __new = (_new);				\
+	asm volatile("\n"						\
+		     "1: " LOCK_PREFIX "cmpxchg"itype" %[new], %[ptr]\n"\
+		     CC_SET(z)						\
+		     "2:\n"						\
+		     _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG,	\
+					   %[errout])			\
+		     : CC_OUT(z) (success),				\
+		       [errout] "+r" (__err),				\
+		       [ptr] "+m" (*_ptr),				\
+		       [old] "+a" (__old)				\
+		     : [new] ltype (__new)				\
+		     : "memory", "cc");					\
+	if (unlikely(__err))						\
+		goto label;						\
+	if (unlikely(!success))						\
+		*_old = __old;						\
+	likely(success);					})
+
 #endif // CONFIG_CC_HAS_ASM_GOTO_OUTPUT
 
 /* FIXME: this hack is definitely wrong -AK */
@@ -501,6 +543,31 @@ do {										\
 } while (0)
 #endif // CONFIG_CC_HAS_ASM_GOTO_OUTPUT
 
+extern void __try_cmpxchg_user_wrong_size(void);
+
+#define unsafe_try_cmpxchg_user(_ptr, _oldp, _nval, _label) ({		\
+	__typeof__(*(_ptr)) __ret;					\
+	switch (sizeof(__ret)) {					\
+	case 1:	__ret = __try_cmpxchg_user_asm("b", "q",		\
+					       (_ptr), (_oldp),		\
+					       (_nval), _label);	\
+		break;							\
+	case 2:	__ret = __try_cmpxchg_user_asm("w", "r",		\
+					       (_ptr), (_oldp),		\
+					       (_nval), _label);	\
+		break;							\
+	case 4:	__ret = __try_cmpxchg_user_asm("l", "r",		\
+					       (_ptr), (_oldp),		\
+					       (_nval), _label);	\
+		break;							\
+	case 8:	__ret = __try_cmpxchg_user_asm("q", "r",		\
+					       (_ptr), (_oldp),		\
+					       (_nval), _label);	\
+		break;							\
+	default: __try_cmpxchg_user_wrong_size();			\
+	}								\
+	__ret;						})
+
 /*
  * We want the unsafe accessors to always be inlined and use
  * the error labels - thus the macro games.
#15 lore raw

Re: [RFC][PATCH v2 4/5] x86/uaccess: Implement unsafe_try_cmpxchg_user()

Quoted 17 lines · Peter Zijlstra

Do try_cmpxchg() loops on userspace addresses.

---
 arch/x86/include/asm/uaccess.h |   67 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)
arch/x86/include/asm/uaccess.h +3 -0
@@ -342,6 +342,24 @@ do {									\
 		     : [umem] "m" (__m(addr))				\
 		     : : label)
 
+#define __try_cmpxchg_user_asm(itype, ltype, _ptr, _pold, _new, label)	({ \
+	bool success;							\
+	__typeof__(_ptr) _old = (__typeof__(_ptr))(_pold);		\

arch/x86/include/asm/uaccess.h do { \ · line 357

Quoted 10 lines
arch/x86/include/asm/uaccess.h +10 -0
+	__typeof__(*(_ptr)) __old = *_old;				\
+	__typeof__(*(_ptr)) __new = (_new);				\
+	asm_volatile_goto("\n"						\
+		     "1: " LOCK_PREFIX "cmpxchg"itype" %[new], %[ptr]\n"\
+		     _ASM_EXTABLE_UA(1b, %l[label])			\
+		     : CC_OUT(z) (success),				\
+		       [ptr] "+m" (*_ptr),				\
+		       [old] "+a" (__old)				\
+		     : [new] ltype (__new)				\
+		     : "memory", "cc"					\

IIUC, the "cc" clobber is unnecessary as CONFIG_CC_HAS_ASM_GOTO_OUTPUT=y implies __GCC_ASM_FLAG_OUTPUTS__=y, i.e. CC_OUT() will resolve to "=@cc".

Quoted 6 lines
arch/x86/include/asm/uaccess.h +5 -0
+		     : label);						\
+	if (unlikely(!success))						\
+		*_old = __old;						\
+	likely(success);					})
+
 #else // !CONFIG_CC_HAS_ASM_GOTO_OUTPUT

...

arch/x86/include/asm/uaccess.h do { \ · line 549

Quoted 4 lines
arch/x86/include/asm/uaccess.h +4 -0
+extern void __try_cmpxchg_user_wrong_size(void);
+
+#define unsafe_try_cmpxchg_user(_ptr, _oldp, _nval, _label) ({		\
+	__typeof__(*(_ptr)) __ret;					\

This should probably be a bool, the return from the lower level helpers is a bool that's true if the exchange succeed. Declaring the type of the target implies that they return the raw result, which is confusing.

Quoted 6 lines
arch/x86/include/asm/uaccess.h +6 -0
+	switch (sizeof(__ret)) {					\
+	case 1:	__ret = __try_cmpxchg_user_asm("b", "q",		\
+					       (_ptr), (_oldp),		\
+					       (_nval), _label);	\
+		break;							\
+	case 2:	__ret = __try_cmpxchg_user_asm("w", "r",		\

arch/x86/include/asm/uaccess.h do { \ · line 565

Quoted 10 lines
arch/x86/include/asm/uaccess.h +10 -0
+					       (_ptr), (_oldp),		\
+					       (_nval), _label);	\
+		break;							\
+	case 4:	__ret = __try_cmpxchg_user_asm("l", "r",		\
+					       (_ptr), (_oldp),		\
+					       (_nval), _label);	\
+		break;							\
+	case 8:	__ret = __try_cmpxchg_user_asm("q", "r",		\
+					       (_ptr), (_oldp),		\
+					       (_nval), _label);	\

Doh, I should have specified that KVM needs 8-byte CMPXCHG on 32-bit kernels due to using it to atomically update guest PAE PTEs and LTR descriptors (yay).

Also, KVM's use case isn't a tight loop, how gross would it be to add a slightly less unsafe version that does __uaccess_begin_nospec()? KVM pre-checks the address way ahead of time, so the access_ok() check can be omitted. Alternatively, KVM could add its own macro, but that seems a little silly. E.g. somethign like this, though I don't think this is correct (something is getting inverted somewhere and the assembly output is a nightmare):

/* "Returns" 0 on success, 1 on failure, -EFAULT if the access faults. */

#define ___try_cmpxchg_user(_ptr, _oldp, _nval, _label)	({		\
	int ____ret = -EFAULT;						\
	__uaccess_begin_nospec();					\
	____ret = !unsafe_try_cmpxchg_user(_ptr, _oldp, _nval, _label);	\
_label:									\
	__uaccess_end();						\
	____ret;							\
						})

Lastly, assuming I get my crap working, mind if I post a variant (Cc'd to stable@) in the context of KVM series? Turns out KVM has an ugly bug where it completely botches the pfn calculation of memory it remaps and accesses[*], the easiest fix is to switch to __try_cmpxchg_user() and purge the nastiness.

[*] https://lore.kernel.org/all/[email protected]

#16 lore raw

Re: [RFC][PATCH v2 4/5] x86/uaccess: Implement unsafe_try_cmpxchg_user()

Quoted 8 lines · Sean Christopherson

Doh, I should have specified that KVM needs 8-byte CMPXCHG on 32-bit kernels due to using it to atomically update guest PAE PTEs and LTR descriptors (yay).

Also, KVM's use case isn't a tight loop, how gross would it be to add a slightly less unsafe version that does __uaccess_begin_nospec()? KVM pre-checks the address way ahead of time, so the access_ok() check can be omitted. Alternatively, KVM could add its own macro, but that seems a little silly. E.g. somethign like this, though I don't think this is correct

*sigh*

Finally realized I forgot to add back the page offset after converting from guest page frame to host virtual address. Anyways, this is what I ended up with, will test more tomorrow.

From: Peter Zijlstra <[email protected]>
Date: Thu, 20 Jan 2022 16:55:21 +0100
Subject: [PATCH] x86/uaccess: Implement unsafe_try_cmpxchg_user()

Do try_cmpxchg() loops on userspace addresses. Provide 8-byte versions for 32-bit kernels so that KVM can do cmpxchg on guest PAE PTEs, which are accessed via userspace addresses.

---
 arch/x86/include/asm/uaccess.h | 129 +++++++++++++++++++++++++++++++++
 1 file changed, 129 insertions(+)
arch/x86/include/asm/uaccess.h +129 -0
index ac96f9b2d64b..b706008aed28 100644
@@ -342,6 +342,45 @@ do {									\
 		     : [umem] "m" (__m(addr))				\
 		     : : label)
 
+#define __try_cmpxchg_user_asm(itype, ltype, _ptr, _pold, _new, label)	({ \
+	bool success;							\
+	__typeof__(_ptr) _old = (__typeof__(_ptr))(_pold);		\
+	__typeof__(*(_ptr)) __old = *_old;				\
+	__typeof__(*(_ptr)) __new = (_new);				\
+	asm_volatile_goto("\n"						\
+		     "1: " LOCK_PREFIX "cmpxchg"itype" %[new], %[ptr]\n"\
+		     _ASM_EXTABLE_UA(1b, %l[label])			\
+		     : CC_OUT(z) (success),				\
+		       [ptr] "+m" (*_ptr),				\
+		       [old] "+a" (__old)				\
+		     : [new] ltype (__new)				\
+		     : "memory"						\
+		     : label);						\
+	if (unlikely(!success))						\
+		*_old = __old;						\
+	likely(success);					})
+
+#ifdef CONFIG_X86_32
+#define __try_cmpxchg64_user_asm(_ptr, _pold, _new, label)	({ \
+	bool success;							\
+	__typeof__(_ptr) _old = (__typeof__(_ptr))(_pold);		\
+	__typeof__(*(_ptr)) __old = *_old;				\
+	__typeof__(*(_ptr)) __new = (_new);				\
+	asm_volatile_goto("\n"						\
+		     "1: " LOCK_PREFIX "cmpxchg8b %[ptr]\n"		\
+		     _ASM_EXTABLE_UA(1b, %l[label])			\
+		     : CC_OUT(z) (success),				\
+		       "+A" (__old),					\
+		       [ptr] "+m" (*_ptr)				\
+		     : "b" ((u32)__new),				\
+		       "c" ((u32)((u64)__new >> 32))			\
+		     : "memory"						\
+		     : label);						\
+	if (unlikely(!success))						\
+		*_old = __old;						\
+	likely(success);					})
+#endif // CONFIG_X86_32
+
 #else // !CONFIG_CC_HAS_ASM_GOTO_OUTPUT
 
 #ifdef CONFIG_X86_32
@@ -407,6 +446,57 @@ do {									\
 		     : [umem] "m" (__m(addr)),				\
 		       "0" (err))
 
+#define __try_cmpxchg_user_asm(itype, ltype, _ptr, _pold, _new, label)	({ \
+	int __err = 0;							\
+	bool success;							\
+	__typeof__(_ptr) _old = (__typeof__(_ptr))(_pold);		\
+	__typeof__(*(_ptr)) __old = *_old;				\
+	__typeof__(*(_ptr)) __new = (_new);				\
+	asm volatile("\n"						\
+		     "1: " LOCK_PREFIX "cmpxchg"itype" %[new], %[ptr]\n"\
+		     CC_SET(z)						\
+		     "2:\n"						\
+		     _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG,	\
+					   %[errout])			\
+		     : CC_OUT(z) (success),				\
+		       [errout] "+r" (__err),				\
+		       [ptr] "+m" (*_ptr),				\
+		       [old] "+a" (__old)				\
+		     : [new] ltype (__new)				\
+		     : "memory", "cc");					\
+	if (unlikely(__err))						\
+		goto label;						\
+	if (unlikely(!success))						\
+		*_old = __old;						\
+	likely(success);					})
+
+#ifdef CONFIG_X86_32
+#define __try_cmpxchg64_user_asm(_ptr, _pold, _new, label)	({	\
+	int __err = 0;							\
+	bool success;							\
+	__typeof__(_ptr) _old = (__typeof__(_ptr))(_pold);		\
+	__typeof__(*(_ptr)) __old = *_old;				\
+	__typeof__(*(_ptr)) __new = (_new);				\
+	asm volatile("\n"						\
+		     "1: " LOCK_PREFIX "cmpxchg8b %[ptr]\n"		\
+		     CC_SET(z)						\
+		     "2:\n"						\
+		     _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG,	\
+					   %[errout])			\
+		     : CC_OUT(z) (success),				\
+		       [errout] "+r" (__err),				\
+		       "+A" (__old),					\
+		       [ptr] "+m" (*_ptr)				\
+		     : "b" ((u32)__new),				\
+		       "c" ((u32)((u64)__new >> 32))			\
+		     : "memory", "cc");					\
+	if (unlikely(__err))						\
+		goto label;						\
+	if (unlikely(!success))						\
+		*_old = __old;						\
+	likely(success);					})
+#endif // CONFIG_X86_32
+
 #endif // CONFIG_CC_HAS_ASM_GOTO_OUTPUT
 
 /* FIXME: this hack is definitely wrong -AK */
@@ -501,6 +591,45 @@ do {										\
 } while (0)
 #endif // CONFIG_CC_HAS_ASM_GOTO_OUTPUT
 
+extern void __try_cmpxchg_user_wrong_size(void);
+
+#ifndef CONFIG_X86_32
+#define __try_cmpxchg64_user_asm(_ptr, _oldp, _nval, _label)		\
+	__try_cmpxchg_user_asm("q", "r", (_ptr), (_oldp), (_nval), _label)
+#endif
+
+#define unsafe_try_cmpxchg_user(_ptr, _oldp, _nval, _label) ({		\
+	bool __ret;							\
+	switch (sizeof(*(_ptr))) {					\
+	case 1:	__ret = __try_cmpxchg_user_asm("b", "q",		\
+					       (_ptr), (_oldp),		\
+					       (_nval), _label);	\
+		break;							\
+	case 2:	__ret = __try_cmpxchg_user_asm("w", "r",		\
+					       (_ptr), (_oldp),		\
+					       (_nval), _label);	\
+		break;							\
+	case 4:	__ret = __try_cmpxchg_user_asm("l", "r",		\
+					       (_ptr), (_oldp),		\
+					       (_nval), _label);	\
+		break;							\
+	case 8:	__ret = __try_cmpxchg64_user_asm((_ptr), (_oldp),	\
+						 (_nval), _label);	\
+		break;							\
+	default: __try_cmpxchg_user_wrong_size();			\
+	}								\
+	__ret;						})
+
+/* "Returns" 0 on success, 1 on failure, -EFAULT if the access faults. */
+#define __try_cmpxchg_user(_ptr, _oldp, _nval, _label)	({		\
+	int __ret = -EFAULT;						\
+	__uaccess_begin_nospec();					\
+	__ret = !unsafe_try_cmpxchg_user(_ptr, _oldp, _nval, _label);	\
+_label:									\
+	__uaccess_end();						\
+	__ret;								\
+							})
+
 /*
  * We want the unsafe accessors to always be inlined and use
  * the error labels - thus the macro games.

--

author maintainer
#17 lore raw

Re: [RFC][PATCH v2 4/5] x86/uaccess: Implement unsafe_try_cmpxchg_user()

Quoted 10 lines · Sean Christopherson
Quoted 8 lines · Sean Christopherson

Doh, I should have specified that KVM needs 8-byte CMPXCHG on 32-bit kernels due to using it to atomically update guest PAE PTEs and LTR descriptors (yay).

Also, KVM's use case isn't a tight loop, how gross would it be to add a slightly less unsafe version that does __uaccess_begin_nospec()? KVM pre-checks the address way ahead of time, so the access_ok() check can be omitted. Alternatively, KVM could add its own macro, but that seems a little silly. E.g. somethign like this, though I don't think this is correct

Quoted 5 lines

*sigh*

Finally realized I forgot to add back the page offset after converting from guest page frame to host virtual address. Anyways, this is what I ended up with, will test more tomorrow.

Looks about right :-) (famous last words etc..)

#18 lore raw

Re: [RFC][PATCH v2 4/5] x86/uaccess: Implement unsafe_try_cmpxchg_user()

+Nick

Quoted 11 lines · Peter Zijlstra
Quoted 10 lines · Sean Christopherson
Quoted 8 lines · Sean Christopherson

Doh, I should have specified that KVM needs 8-byte CMPXCHG on 32-bit kernels due to using it to atomically update guest PAE PTEs and LTR descriptors (yay).

Also, KVM's use case isn't a tight loop, how gross would it be to add a slightly less unsafe version that does __uaccess_begin_nospec()? KVM pre-checks the address way ahead of time, so the access_ok() check can be omitted. Alternatively, KVM could add its own macro, but that seems a little silly. E.g. somethign like this, though I don't think this is correct

Quoted 7 lines
Quoted 5 lines

*sigh*

Finally realized I forgot to add back the page offset after converting from guest page frame to host virtual address. Anyways, this is what I ended up with, will test more tomorrow.

Looks about right :-) (famous last words etc..)

And it was right, but clang-13 ruined the party :-/

clang barfs on asm goto with a "+m" input/output. Change the "+m" to "=m" and clang is happy. Remove usage of the label, clang is happy.

I tried a bunch of different variants to see if anything would squeak by, but clang found a way to die on everything I threw at it.

$ clang --version

Debian clang version 13.0.0-9+build1
Target: x86_64-pc-linux-gnu
Thread model: posix
InstalledDir: /usr/bin

As written, with a named label param, clang yields:

  $ echo 'int foo(int *x) { asm goto (".long (%l[bar]) - .\n": "+m"(*x) ::: bar); return *x; bar: return 0; }' | clang -x c - -c -o /dev/null
  <stdin>:1:29: error: invalid operand in inline asm: '.long (${1:l}) - .'
  int foo(int *x) { asm goto (".long (%l[bar]) - .\n": "+m"(*x) ::: bar); return *x; bar: return 0; }
                            ^
  <stdin>:1:29: error: unknown token in expression
  <inline asm>:1:9: note: instantiated into assembly here
          .long () - .
               ^
  2 errors generated.

While clang is perfectly happy switching "+m" to "=m":

  $ echo 'int foo(int *x) { asm goto (".long (%l[bar]) - .\n": "=m"(*x) ::: bar); return *x; bar: return 0; }' | clang -x c - -c -o /dev/null

Referencing the label with a numbered param yields either the original error:

  $ echo 'int foo(int *x) { asm goto (".long (%l1) - .\n": "+m"(*x) ::: bar); return *x; bar: return 0; }' | clang -x c - -c -o /dev/null
  <stdin>:1:29: error: invalid operand in inline asm: '.long (${1:l}) - .'
  int foo(int *x) { asm goto (".long (%l1) - .\n": "+m"(*x) ::: bar); return *x; bar: return 0; }
                            ^
  <stdin>:1:29: error: unknown token in expression
  <inline asm>:1:9: note: instantiated into assembly here
          .long () - .
                 ^
   2 errors generated.

Bumping the param number (more below) yields a different error (I tried defining tmp1, that didn't work :-) ).

  $ echo 'int foo(int *x) { asm goto (".long (%l2) - .\n": "+m"(*x) ::: bar); return *x; bar: return 0; }' | clang -x c - -c -o /dev/null
  error: Undefined temporary symbol .Ltmp1
  1 error generated.

Regarding the param number, gcc also appears to have a goof with asm goto and "+m", but bumping the param number in that case remedies its woes.

$echo 'int foo(int *x) { asm goto (".long (%l1) - .\n": "+m"(*x) ::: bar); return *x; bar: return 0; }' | gcc -x c - -c -o /dev/null <stdin>: In function ‘foo’:
<stdin>:1:19: error: invalid 'asm': '%l' operand isn't a label

  $ echo 'int foo(int *x) { asm goto (".long (%l2) - .\n": "+m"(*x) ::: bar); return *x; bar: return 0; }' | gcc -x c - -c -o /dev/null

So my immediate question: how do we want to we deal with this in the kernel? Keeping in mind that I'd really like to send this to stable@ to fix the KVM mess.

I can think of few options that are varying degrees of gross.

1) Use a more complex sequence for probing CC_HAS_ASM_GOTO_OUTPUT.

2) Use an output-only "=m" operand.

3) Use an input register param.

Option #1 has the obvious downside of the fancier asm goto for __get_user_asm() and friends being collateral damage. The biggest benefit is it'd reduce the likelihood of someone else having to debug similar errors, which was quite painful.

Options #2 and #3 are quite gross, but I _think_ would be ok since the sequence is tagged as clobbering memory anyways?

#19 lore raw

Re: [RFC][PATCH v2 4/5] x86/uaccess: Implement unsafe_try_cmpxchg_user()

Quoted 2 lines · Sean Christopherson

+Nick

(well, you asked the right person; you probably wont like the answer much though)

Quoted 19 lines · Peter Zijlstra
Quoted 17 lines · Sean Christopherson
Quoted 15 lines · Sean Christopherson
Quoted 8 lines

Doh, I should have specified that KVM needs 8-byte CMPXCHG on 32-bit kernels due to using it to atomically update guest PAE PTEs and LTR descriptors (yay).

Also, KVM's use case isn't a tight loop, how gross would it be to add a slightly less unsafe version that does __uaccess_begin_nospec()? KVM pre-checks the address way ahead of time, so the access_ok() check can be omitted. Alternatively, KVM could add its own macro, but that seems a little silly. E.g. somethign like this, though I don't think this is correct

*sigh*

Finally realized I forgot to add back the page offset after converting from guest page frame to host virtual address. Anyways, this is what I ended up with, will test more tomorrow.

Quoted 6 lines
Quoted 1 line

Looks about right :-) (famous last words etc..)

And it was right, but clang-13 ruined the party :-/

clang barfs on asm goto with a "+m" input/output. Change the "+m" to "=m" and clang is happy. Remove usage of the label, clang is happy.

Yep, sorry, we only recently noticed that this was broken. I fixed this very recently (over the holidays) in clang-14, and is too risky and late to backport to clang-13 IMO.
https://reviews.llvm.org/rG4edb9983cb8c8b850083ee5941468f5d0ef6fe2c

Quoted 23 lines

I tried a bunch of different variants to see if anything would squeak by, but clang found a way to die on everything I threw at it.

$ clang --version

Debian clang version 13.0.0-9+build1
Target: x86_64-pc-linux-gnu
Thread model: posix
InstalledDir: /usr/bin

As written, with a named label param, clang yields:

  $ echo 'int foo(int *x) { asm goto (".long (%l[bar]) - .\n": "+m"(*x) ::: bar); return *x; bar: return 0; }' | clang -x c - -c -o /dev/null
  <stdin>:1:29: error: invalid operand in inline asm: '.long (${1:l}) - .'
  int foo(int *x) { asm goto (".long (%l[bar]) - .\n": "+m"(*x) ::: bar); return *x; bar: return 0; }
                            ^
  <stdin>:1:29: error: unknown token in expression
  <inline asm>:1:9: note: instantiated into assembly here
          .long () - .
               ^
  2 errors generated.
Quoted 3 lines

While clang is perfectly happy switching "+m" to "=m":

  $ echo 'int foo(int *x) { asm goto (".long (%l[bar]) - .\n": "=m"(*x) ::: bar); return *x; bar: return 0; }' | clang -x c - -c -o /dev/null

= constraints should work with older clang releases; + constraints are what was broken (Not generally, only when using asm goto with outputs), fixed in clang-14.

Quoted 12 lines

Referencing the label with a numbered param yields either the original error:

  $ echo 'int foo(int *x) { asm goto (".long (%l1) - .\n": "+m"(*x) ::: bar); return *x; bar: return 0; }' | clang -x c - -c -o /dev/null
  <stdin>:1:29: error: invalid operand in inline asm: '.long (${1:l}) - .'
  int foo(int *x) { asm goto (".long (%l1) - .\n": "+m"(*x) ::: bar); return *x; bar: return 0; }
                            ^
  <stdin>:1:29: error: unknown token in expression
  <inline asm>:1:9: note: instantiated into assembly here
          .long () - .
                 ^
   2 errors generated.

^ That case should not work in either compilers, more info below...

Quoted 7 lines

Bumping the param number (more below) yields a different error (I tried defining tmp1, that didn't work :-) ).

  $ echo 'int foo(int *x) { asm goto (".long (%l2) - .\n": "+m"(*x) ::: bar); return *x; bar: return 0; }' | clang -x c - -c -o /dev/null
  error: Undefined temporary symbol .Ltmp1
  1 error generated.

"Bumping the param number" will be required when using numbered references, more info below...

Quoted 9 lines

Regarding the param number, gcc also appears to have a goof with asm goto and "+m", but bumping the param number in that case remedies its woes.

$echo 'int foo(int *x) { asm goto (".long (%l1) - .\n": "+m"(*x) ::: bar); return *x; bar: return 0; }' | gcc -x c - -c -o /dev/null <stdin>: In function ‘foo’:
<stdin>:1:19: error: invalid 'asm': '%l' operand isn't a label

  $ echo 'int foo(int *x) { asm goto (".long (%l2) - .\n": "+m"(*x) ::: bar); return *x; bar: return 0; }' | gcc -x c - -c -o /dev/null

Right, so in fixing the above issue with tied outputs, I noticed that the GCC implementation of asm goto with outputs had different behavior. I changed clang's implementation in clang-14 (same patch series) to match:
https://reviews.llvm.org/rG5c562f62a4ee15592f5d764d0710553a4b07a6f2
This comment summarizes most of my thoughts on the issue:
https://gcc.gnu.org/bugzilla/show_bug.cgi?id=98096#c7
Specifically the quote "It appears to me that the GCC decision here was accidental, and that when pointed out, the bug was simply documented rather than fixed."
Though I think compatibility between compilers is ultimately more important. There's no standards bodies involved in these extension, which is simultaneously more flexible, yet can also lead to differences in implementations like this. Thanks for attending my TED talk.

Quoted 9 lines

So my immediate question: how do we want to we deal with this in the kernel? Keeping in mind that I'd really like to send this to stable@ to fix the KVM mess.

I can think of few options that are varying degrees of gross.

1) Use a more complex sequence for probing CC_HAS_ASM_GOTO_OUTPUT.

Quoted 7 lines

2) Use an output-only "=m" operand.

3) Use an input register param.

Option #1 has the obvious downside of the fancier asm goto for __get_user_asm() and friends being collateral damage. The biggest benefit is it'd reduce the likelihood of someone else having to debug similar errors, which was quite painful.

Right; I assumed we'd hit this at some point, as soon as people wanted to used tied outputs with asm goto. I'd rather have a different Kconfig test for working tied outputs, and that all uses in the kernels used the symbolic references which are much more readable and less confusing than the rules for numbered references (which are bug prone IMO).

Quoted 3 lines

Options #2 and #3 are quite gross, but I _think_ would be ok since the sequence is tagged as clobbering memory anyways?

Signature · 2 lines
Thanks,
~Nick Desaulniers
#20 lore raw

Re: [RFC][PATCH v2 4/5] x86/uaccess: Implement unsafe_try_cmpxchg_user()

Quoted 10 lines · Nick Desaulniers
Quoted 8 lines · Sean Christopherson

Regarding the param number, gcc also appears to have a goof with asm goto and "+m", but bumping the param number in that case remedies its woes.

$echo 'int foo(int *x) { asm goto (".long (%l1) - .\n": "+m"(*x) ::: bar); return *x; bar: return 0; }' | gcc -x c - -c -o /dev/null <stdin>: In function ‘foo’:
<stdin>:1:19: error: invalid 'asm': '%l' operand isn't a label

  $ echo 'int foo(int *x) { asm goto (".long (%l2) - .\n": "+m"(*x) ::: bar); return *x; bar: return 0; }' | gcc -x c - -c -o /dev/null
Quoted 10 lines

Right, so in fixing the above issue with tied outputs, I noticed that the GCC implementation of asm goto with outputs had different behavior. I changed clang's implementation in clang-14 (same patch series) to match:
https://reviews.llvm.org/rG5c562f62a4ee15592f5d764d0710553a4b07a6f2
This comment summarizes most of my thoughts on the issue:
https://gcc.gnu.org/bugzilla/show_bug.cgi?id=98096#c7
Specifically the quote "It appears to me that the GCC decision here was accidental, and that when pointed out, the bug was simply documented rather than fixed."

I guess that makes a certain amount of sense, but wow that is subtle, confusing, and potentially dangerous. Looks like the hidden inputs are numbered after all explicit inputs, otherwise there would be broken code left and right, but it means that a typo like so:

echo 'int foo(int x) { asm ("xor %0, %0; xor %2, %2" : "+a"(x) : "b"(x)); return x; return 0; }' | clang -x c - -c -o tmp.o

will compile cleanly.

Quoted 19 lines

Though I think compatibility between compilers is ultimately more important. There's no standards bodies involved in these extension, which is simultaneously more flexible, yet can also lead to differences in implementations like this. Thanks for attending my TED talk.

Quoted 13 lines

So my immediate question: how do we want to we deal with this in the kernel? Keeping in mind that I'd really like to send this to stable@ to fix the KVM mess.

I can think of few options that are varying degrees of gross.

1) Use a more complex sequence for probing CC_HAS_ASM_GOTO_OUTPUT.

2) Use an output-only "=m" operand.

3) Use an input register param.

Quoted 7 lines
Quoted 3 lines

Option #1 has the obvious downside of the fancier asm goto for __get_user_asm() and friends being collateral damage. The biggest benefit is it'd reduce the likelihood of someone else having to debug similar errors, which was quite painful.

Right; I assumed we'd hit this at some point, as soon as people wanted to used tied outputs with asm goto. I'd rather have a different Kconfig test for working tied outputs,

Is it all tied outputs, or just "+m"? E.g. using "+r" compiles cleanly.

echo 'int foo(int x) { asm goto ("xor %0, %0;.long (%l[bar]) - .\n": "+r"(x) ::: bar); return x; bar: return 0; }' | clang -x c - -c -o /dev/null

It might be a moot point as I can't find any instances of "+"-anything in conjunction with asm_volatile_goto, i.e. adding CC_HAS_ASM_GOTO_OUTPUT_TIED_OUTPUTS won't create an inconsistency with existing code.

Regardless, I like that idea.

Quoted 3 lines

and that all uses in the kernels used the symbolic references which are much more readable and less confusing than the rules for numbered references (which are bug prone IMO).

100% agree, even though it takes me twice as long to write because I can never remember the syntax :-) Converting all existing usage is probably a fools errand, but adding a checkpatch rule would get us going in the right direction.

author maintainer
#21 lore raw

Re: [RFC][PATCH v2 4/5] x86/uaccess: Implement unsafe_try_cmpxchg_user()

In reply to Sean Christopherson
Quoted 15 lines · Sean Christopherson
Quoted 14 lines · Peter Zijlstra

Do try_cmpxchg() loops on userspace addresses.

---
 arch/x86/include/asm/uaccess.h |   67 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)
arch/x86/include/asm/uaccess.h +0 -0
@@ -342,6 +342,24 @@ do {									\
 		     : [umem] "m" (__m(addr))				\
 		     : : label)
 
Quoted 16 lines

arch/x86/include/asm/uaccess.h do { \ · line 357

Quoted 13 lines
arch/x86/include/asm/uaccess.h +13 -0
+#define __try_cmpxchg_user_asm(itype, ltype, _ptr, _pold, _new, label)	({ \
+	bool success;							\
+	__typeof__(_ptr) _old = (__typeof__(_ptr))(_pold);		\
+	__typeof__(*(_ptr)) __old = *_old;				\
+	__typeof__(*(_ptr)) __new = (_new);				\
+	asm_volatile_goto("\n"						\
+		     "1: " LOCK_PREFIX "cmpxchg"itype" %[new], %[ptr]\n"\
+		     _ASM_EXTABLE_UA(1b, %l[label])			\
+		     : CC_OUT(z) (success),				\
+		       [ptr] "+m" (*_ptr),				\
+		       [old] "+a" (__old)				\
+		     : [new] ltype (__new)				\
+		     : "memory", "cc"					\

IIUC, the "cc" clobber is unnecessary as CONFIG_CC_HAS_ASM_GOTO_OUTPUT=y implies __GCC_ASM_FLAG_OUTPUTS__=y, i.e. CC_OUT() will resolve to "=@cc".

Yeah, even without that GCC always assumes 'cc' is clobbered due to hysterical raisins.

Quoted 9 lines
Quoted 6 lines
arch/x86/include/asm/uaccess.h +5 -0
+		     : label);						\
+	if (unlikely(!success))						\
+		*_old = __old;						\
+	likely(success);					})
+
 #else // !CONFIG_CC_HAS_ASM_GOTO_OUTPUT

...

Quoted 8 lines

arch/x86/include/asm/uaccess.h do { \ · line 549

Quoted 4 lines
arch/x86/include/asm/uaccess.h +4 -0
+extern void __try_cmpxchg_user_wrong_size(void);
+
+#define unsafe_try_cmpxchg_user(_ptr, _oldp, _nval, _label) ({		\
+	__typeof__(*(_ptr)) __ret;					\

This should probably be a bool, the return from the lower level helpers is a bool that's true if the exchange succeed. Declaring the type of the target implies that they return the raw result, which is confusing.

Fair enough.

Quoted 11 lines
Quoted 11 lines
arch/x86/include/asm/uaccess.h +11 -0
+	switch (sizeof(__ret)) {					\
+	case 1:	__ret = __try_cmpxchg_user_asm("b", "q",		\
+					       (_ptr), (_oldp),		\
+					       (_nval), _label);	\
+		break;							\
+	case 2:	__ret = __try_cmpxchg_user_asm("w", "r",		\
+					       (_ptr), (_oldp),		\
+					       (_nval), _label);	\
+		break;							\
+	case 4:	__ret = __try_cmpxchg_user_asm("l", "r",		\
+					       (_ptr), (_oldp),		\
Quoted 8 lines

arch/x86/include/asm/uaccess.h do { \ · line 565

Quoted 5 lines
arch/x86/include/asm/uaccess.h +5 -0
+					       (_nval), _label);	\
+		break;							\
+	case 8:	__ret = __try_cmpxchg_user_asm("q", "r",		\
+					       (_ptr), (_oldp),		\
+					       (_nval), _label);	\

Doh, I should have specified that KVM needs 8-byte CMPXCHG on 32-bit kernels due to using it to atomically update guest PAE PTEs and LTR descriptors (yay).

:-) I'm so trying to de-feature 32bit.

Quoted 8 lines

Also, KVM's use case isn't a tight loop, how gross would it be to add a slightly less unsafe version that does __uaccess_begin_nospec()? KVM pre-checks the address way ahead of time, so the access_ok() check can be omitted. Alternatively, KVM could add its own macro, but that seems a little silly. E.g. somethign like this, though I don't think this is correct (something is getting inverted somewhere and the assembly output is a nightmare):

/* "Returns" 0 on success, 1 on failure, -EFAULT if the access faults. */

Quoted 8 lines
#define ___try_cmpxchg_user(_ptr, _oldp, _nval, _label)	({		\
	int ____ret = -EFAULT;						\
	__uaccess_begin_nospec();					\
	____ret = !unsafe_try_cmpxchg_user(_ptr, _oldp, _nval, _label);	\
_label:									\
	__uaccess_end();						\
	____ret;							\
						})

Works for me I suppose, but we really ought to keep usage of that in arch code.

Quoted 2 lines

Lastly, assuming I get my crap working, mind if I post a variant (Cc'd to stable@) in the context of KVM series?

Not at all.

author maintainer
#22 lore raw

RFC PATCH v2 5/5 sched: User Mode Concurency Groups

In reply to Peter Zijlstra

User Managed Concurrency Groups is an M:N threading toolkit that allows constructing user space schedulers designed to efficiently manage heterogeneous in-process workloads while maintaining high CPU utilization (95%+).

XXX moar changelog explaining how this is moar awesome than traditional user-space threading.

The big thing that's still missing is the SMP wake-to-remote-idle.

---
 arch/x86/Kconfig                       |    1 
 arch/x86/entry/syscalls/syscall_64.tbl |    3 
 arch/x86/include/asm/thread_info.h     |    2 
 fs/exec.c                              |    1 
 include/linux/entry-common.h           |   16 
 include/linux/sched.h                  |   82 ++
 include/linux/syscalls.h               |    4 
 include/linux/thread_info.h            |    2 
 include/uapi/asm-generic/unistd.h      |    9 
 include/uapi/linux/umcg.h              |  141 ++++
 init/Kconfig                           |   15 
 kernel/entry/common.c                  |   18 
 kernel/exit.c                          |    5 
 kernel/sched/Makefile                  |    1 
 kernel/sched/core.c                    |    9 
 kernel/sched/umcg.c                    |  954 +++++++++++++++++++++++++++++++++
 kernel/sys_ni.c                        |    5 
 17 files changed, 1259 insertions(+), 9 deletions(-)
arch/x86/Kconfig +1 -0
@@ -248,6 +248,7 @@ config X86
 	select HAVE_RSEQ
 	select HAVE_SYSCALL_TRACEPOINTS
 	select HAVE_UNSTABLE_SCHED_CLOCK
+	select HAVE_UMCG			if X86_64
 	select HAVE_USER_RETURN_NOTIFIER
 	select HAVE_GENERIC_VDSO
 	select HOTPLUG_SMT			if SMP
arch/x86/entry/syscalls/syscall_64.tbl +3 -0
@@ -371,6 +371,9 @@
 447	common	memfd_secret		sys_memfd_secret
 448	common	process_mrelease	sys_process_mrelease
 449	common	futex_waitv		sys_futex_waitv
+450	common	umcg_ctl		sys_umcg_ctl
+451	common	umcg_wait		sys_umcg_wait
+452	common	umcg_kick		sys_umcg_kick
 
 #
 # Due to a historical design error, certain syscalls are numbered differently
arch/x86/include/asm/thread_info.h +2 -0
@@ -83,6 +83,7 @@ struct thread_info {
 #define TIF_NEED_RESCHED	3	/* rescheduling necessary */
 #define TIF_SINGLESTEP		4	/* reenable singlestep on user return*/
 #define TIF_SSBD		5	/* Speculative store bypass disable */
+#define TIF_UMCG		6	/* UMCG return to user hook */
 #define TIF_SPEC_IB		9	/* Indirect branch speculation mitigation */
 #define TIF_SPEC_L1D_FLUSH	10	/* Flush L1D on mm switches (processes) */
 #define TIF_USER_RETURN_NOTIFY	11	/* notify kernel of userspace return */
@@ -107,6 +108,7 @@ struct thread_info {
 #define _TIF_NEED_RESCHED	(1 << TIF_NEED_RESCHED)
 #define _TIF_SINGLESTEP		(1 << TIF_SINGLESTEP)
 #define _TIF_SSBD		(1 << TIF_SSBD)
+#define _TIF_UMCG		(1 << TIF_UMCG)
 #define _TIF_SPEC_IB		(1 << TIF_SPEC_IB)
 #define _TIF_SPEC_L1D_FLUSH	(1 << TIF_SPEC_L1D_FLUSH)
 #define _TIF_USER_RETURN_NOTIFY	(1 << TIF_USER_RETURN_NOTIFY)
fs/exec.c +1 -0
@@ -1838,6 +1838,7 @@ static int bprm_execve(struct linux_binp
 	current->fs->in_exec = 0;
 	current->in_execve = 0;
 	rseq_execve(current);
+	umcg_execve(current);
 	acct_update_integrals(current);
 	task_numa_free(current, false);
 	return retval;
include/linux/entry-common.h +14 -2
@@ -23,6 +23,10 @@
 # define _TIF_UPROBE			(0)
 #endif
 
+#ifndef _TIF_UMCG
+# define _TIF_UMCG			(0)
+#endif
+
 /*
  * SYSCALL_WORK flags handled in syscall_enter_from_user_mode()
  */
@@ -43,11 +47,13 @@
 				 SYSCALL_WORK_SYSCALL_EMU |		\
 				 SYSCALL_WORK_SYSCALL_AUDIT |		\
 				 SYSCALL_WORK_SYSCALL_USER_DISPATCH |	\
+				 SYSCALL_WORK_SYSCALL_UMCG |		\
 				 ARCH_SYSCALL_WORK_ENTER)
 #define SYSCALL_WORK_EXIT	(SYSCALL_WORK_SYSCALL_TRACEPOINT |	\
 				 SYSCALL_WORK_SYSCALL_TRACE |		\
 				 SYSCALL_WORK_SYSCALL_AUDIT |		\
 				 SYSCALL_WORK_SYSCALL_USER_DISPATCH |	\
+				 SYSCALL_WORK_SYSCALL_UMCG |		\
 				 SYSCALL_WORK_SYSCALL_EXIT_TRAP	|	\
 				 ARCH_SYSCALL_WORK_EXIT)
 
@@ -221,8 +227,11 @@ static inline void local_irq_disable_exi
  */
 static inline void irqentry_irq_enable(struct pt_regs *regs)
 {
-	if (!regs_irqs_disabled(regs))
+	if (!regs_irqs_disabled(regs)) {
 		local_irq_enable();
+		if (user_mode(regs) && (current->flags & PF_UMCG_WORKER))
+			umcg_sys_enter(regs, -1);
+	}
 }
 
 /**
@@ -232,8 +241,11 @@ static inline void irqentry_irq_enable(s
  */
 static inline void irqentry_irq_disable(struct pt_regs *regs)
 {
-	if (!regs_irqs_disabled(regs))
+	if (!regs_irqs_disabled(regs)) {
+		if (user_mode(regs) && (current->flags & PF_UMCG_WORKER))
+			umcg_sys_exit(regs);
 		local_irq_disable();
+	}
 }
 
 /**
include/linux/sched.h +82 -0

106 lines not on this page · raw message

include/linux/syscalls.h +4 -0
@@ -72,6 +72,7 @@ struct open_how;
 struct mount_attr;
 struct landlock_ruleset_attr;
 enum landlock_rule_type;
+struct umcg_task;
 
 #include <linux/types.h>
 #include <linux/aio_abi.h>
@@ -1057,6 +1058,9 @@ asmlinkage long sys_landlock_add_rule(in
 		const void __user *rule_attr, __u32 flags);
 asmlinkage long sys_landlock_restrict_self(int ruleset_fd, __u32 flags);
 asmlinkage long sys_memfd_secret(unsigned int flags);
+asmlinkage long sys_umcg_ctl(u32 flags, struct umcg_task __user *self, clockid_t which_clock);
+asmlinkage long sys_umcg_wait(u32 flags, u64 abs_timeout);
+asmlinkage long sys_umcg_kick(u32 flags, pid_t tid);
 
 /*
  * Architecture-specific system calls
include/linux/thread_info.h +2 -0
@@ -46,6 +46,7 @@ enum syscall_work_bit {
 	SYSCALL_WORK_BIT_SYSCALL_AUDIT,
 	SYSCALL_WORK_BIT_SYSCALL_USER_DISPATCH,
 	SYSCALL_WORK_BIT_SYSCALL_EXIT_TRAP,
+	SYSCALL_WORK_BIT_SYSCALL_UMCG,
 };
 
 #define SYSCALL_WORK_SECCOMP		BIT(SYSCALL_WORK_BIT_SECCOMP)
@@ -55,6 +56,7 @@ enum syscall_work_bit {
 #define SYSCALL_WORK_SYSCALL_AUDIT	BIT(SYSCALL_WORK_BIT_SYSCALL_AUDIT)
 #define SYSCALL_WORK_SYSCALL_USER_DISPATCH BIT(SYSCALL_WORK_BIT_SYSCALL_USER_DISPATCH)
 #define SYSCALL_WORK_SYSCALL_EXIT_TRAP	BIT(SYSCALL_WORK_BIT_SYSCALL_EXIT_TRAP)
+#define SYSCALL_WORK_SYSCALL_UMCG	BIT(SYSCALL_WORK_BIT_SYSCALL_UMCG)
 #endif
 
 #include <asm/thread_info.h>
include/uapi/asm-generic/unistd.h +8 -1
@@ -883,8 +883,15 @@ __SYSCALL(__NR_process_mrelease, sys_pro
 #define __NR_futex_waitv 449
 __SYSCALL(__NR_futex_waitv, sys_futex_waitv)
 
+#define __NR_umcg_ctl 450
+__SYSCALL(__NR_umcg_ctl, sys_umcg_ctl)
+#define __NR_umcg_wait 451
+__SYSCALL(__NR_umcg_wait, sys_umcg_wait)
+#define __NR_umcg_kick 452
+__SYSCALL(__NR_umcg_kick, sys_umcg_kick)
+
 #undef __NR_syscalls
-#define __NR_syscalls 450
+#define __NR_syscalls 453
 
 /*
  * 32 bit systems traditionally used different
include/uapi/linux/umcg.h new file +141 -0

141 lines not on this page · raw message

init/Kconfig +15 -0
@@ -1686,6 +1686,21 @@ config MEMBARRIER
 
 	  If unsure, say Y.
 
+config HAVE_UMCG
+	bool
+
+config UMCG
+	bool "Enable User Managed Concurrency Groups API"
+	depends on 64BIT
+	depends on GENERIC_ENTRY
+	depends on HAVE_UMCG
+	default n
+	help
+	  Enable User Managed Concurrency Groups API, which form the basis
+	  for an in-process M:N userspace scheduling framework.
+	  At the moment this is an experimental/RFC feature that is not
+	  guaranteed to be backward-compatible.
+
 config KALLSYMS
 	bool "Load all symbols for debugging/ksymoops" if EXPERT
 	default y
kernel/entry/common.c +14 -4
@@ -6,6 +6,7 @@
 #include <linux/livepatch.h>
 #include <linux/audit.h>
 #include <linux/tick.h>
+#include <linux/sched.h>
 
 #include "common.h"
 
@@ -76,6 +77,9 @@ static long syscall_trace_enter(struct p
 	if (unlikely(work & SYSCALL_WORK_SYSCALL_TRACEPOINT))
 		trace_sys_enter(regs, syscall);
 
+	if (work & SYSCALL_WORK_SYSCALL_UMCG)
+		umcg_sys_enter(regs, syscall);
+
 	syscall_enter_audit(regs, syscall);
 
 	return ret ? : syscall;
@@ -155,8 +159,7 @@ static unsigned long exit_to_user_mode_l
 	 * Before returning to user space ensure that all pending work
 	 * items have been completed.
 	 */
-	while (ti_work & EXIT_TO_USER_MODE_WORK) {
-
+	do {
 		local_irq_enable_exit_to_user(ti_work);
 
 		if (ti_work & _TIF_NEED_RESCHED)
@@ -168,6 +171,10 @@ static unsigned long exit_to_user_mode_l
 		if (ti_work & _TIF_PATCH_PENDING)
 			klp_update_patch_state(current);
 
+		/* must be before handle_signal_work(); terminates on sigpending */
+		if (ti_work & _TIF_UMCG)
+			umcg_notify_resume(regs);
+
 		if (ti_work & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL))
 			handle_signal_work(regs, ti_work);
 
@@ -188,7 +195,7 @@ static unsigned long exit_to_user_mode_l
 		tick_nohz_user_enter_prepare();
 
 		ti_work = read_thread_flags();
-	}
+	} while (ti_work & EXIT_TO_USER_MODE_WORK);
 
 	/* Return the latest work state for arch_exit_to_user_mode() */
 	return ti_work;
@@ -203,7 +210,7 @@ static void exit_to_user_mode_prepare(st
 	/* Flush pending rcuog wakeup before the last need_resched() check */
 	tick_nohz_user_enter_prepare();
 
-	if (unlikely(ti_work & EXIT_TO_USER_MODE_WORK))
+	if (unlikely(ti_work & (EXIT_TO_USER_MODE_WORK | _TIF_UMCG)))
 		ti_work = exit_to_user_mode_loop(regs, ti_work);
 
 	arch_exit_to_user_mode_prepare(regs, ti_work);
@@ -253,6 +260,9 @@ static void syscall_exit_work(struct pt_
 	step = report_single_step(work);
 	if (step || work & SYSCALL_WORK_SYSCALL_TRACE)
 		arch_syscall_exit_tracehook(regs, step);
+
+	if (work & SYSCALL_WORK_SYSCALL_UMCG)
+		umcg_sys_exit(regs);
 }
 
 /*
kernel/exit.c +5 -0
@@ -749,6 +749,10 @@ void __noreturn do_exit(long code)
 	if (unlikely(!tsk->pid))
 		panic("Attempted to kill the idle task!");
 
+	/* Turn off UMCG sched hooks. */
+	if (unlikely(tsk->flags & PF_UMCG_WORKER))
+		tsk->flags &= ~PF_UMCG_WORKER;
+
 	/*
 	 * If do_exit is called because this processes oopsed, it's possible
 	 * that get_fs() was left as KERNEL_DS, so reset it to USER_DS before
@@ -786,6 +790,7 @@ void __noreturn do_exit(long code)
 
 	io_uring_files_cancel();
 	exit_signals(tsk);  /* sets PF_EXITING */
+	umcg_handle_exit();
 
 	/* sync mm's RSS info before statistics gathering */
 	if (tsk->mm)
kernel/sched/Makefile +1 -0
@@ -41,3 +41,4 @@ obj-$(CONFIG_MEMBARRIER) += membarrier.o
 obj-$(CONFIG_CPU_ISOLATION) += isolation.o
 obj-$(CONFIG_PSI) += psi.o
 obj-$(CONFIG_SCHED_CORE) += core_sched.o
+obj-$(CONFIG_UMCG) += umcg.o
kernel/sched/core.c +7 -2
@@ -4272,6 +4272,7 @@ static void __sched_fork(unsigned long c
 	p->wake_entry.u_flags = CSD_TYPE_TTWU;
 	p->migration_pending = NULL;
 #endif
+	umcg_clear_child(p);
 }
 
 DEFINE_STATIC_KEY_FALSE(sched_numa_balancing);
@@ -6330,9 +6331,11 @@ static inline void sched_submit_work(str
 	 * If a worker goes to sleep, notify and ask workqueue whether it
 	 * wants to wake up a task to maintain concurrency.
 	 */
-	if (task_flags & (PF_WQ_WORKER | PF_IO_WORKER)) {
+	if (task_flags & (PF_WQ_WORKER | PF_IO_WORKER | PF_UMCG_WORKER)) {
 		if (task_flags & PF_WQ_WORKER)
 			wq_worker_sleeping(tsk);
+		else if (task_flags & PF_UMCG_WORKER)
+			umcg_wq_worker_sleeping(tsk);
 		else
 			io_wq_worker_sleeping(tsk);
 	}
@@ -6350,9 +6353,11 @@ static inline void sched_submit_work(str
 
 static void sched_update_worker(struct task_struct *tsk)
 {
-	if (tsk->flags & (PF_WQ_WORKER | PF_IO_WORKER)) {
+	if (tsk->flags & (PF_WQ_WORKER | PF_IO_WORKER | PF_UMCG_WORKER)) {
 		if (tsk->flags & PF_WQ_WORKER)
 			wq_worker_running(tsk);
+		else if (tsk->flags & PF_UMCG_WORKER)
+			umcg_wq_worker_running(tsk);
 		else
 			io_wq_worker_running(tsk);
 	}
kernel/sched/umcg.c new file +954 -0

954 lines not on this page · raw message

kernel/sys_ni.c +5 -0
@@ -273,6 +273,11 @@ COND_SYSCALL(landlock_create_ruleset);
 COND_SYSCALL(landlock_add_rule);
 COND_SYSCALL(landlock_restrict_self);
 
+/* kernel/sched/umcg.c */
+COND_SYSCALL(umcg_ctl);
+COND_SYSCALL(umcg_wait);
+COND_SYSCALL(umcg_kick);
+
 /* arch/example/kernel/sys_example.c */
 
 /* mm/fadvise.c */
author maintainer
#23 lore raw

Re: [RFC][PATCH v2 5/5] sched: User Mode Concurency Groups

Quoted 43 lines · Peter Zijlstra
kernel/sched/umcg.c new file +43 -0
+SYSCALL_DEFINE2(umcg_wait, u32, flags, u64, timo)
+{
+	struct task_struct *tsk = current;
+	struct umcg_task __user *self = READ_ONCE(tsk->umcg_task);
+	bool worker = tsk->flags & PF_UMCG_WORKER;
+	int ret;
+
+	if (!self || flags)
+		return -EINVAL;
+
+	if (worker) {
+		tsk->flags &= ~PF_UMCG_WORKER;
+		if (timo)
+			return -ERANGE;
+	}
+
+	/* see umcg_sys_{enter,exit}() syscall exceptions */
+	ret = umcg_pin_pages();
+	if (ret)
+		goto unblock;
+
+	/*
+	 * Clear UMCG_TF_COND_WAIT *and* check state == RUNNABLE.
+	 */
+	ret = umcg_update_state(tsk, self, UMCG_TASK_RUNNABLE, UMCG_TASK_RUNNABLE);
+	if (ret)
+		goto unpin;
+
+	ret = umcg_wake_next(tsk, self);
+	if (ret)
+		goto unpin;
+
+	if (worker) {
+		/*
+		 * If this fails it is possible ::next_tid is already running
+		 * while this task is not going to block. This violates our
+		 * constraints.
+		 *
+		 * That said, pretty much the only way to make this fail is by
+		 * force munmap()'ing things. In which case one is most welcome
+		 * to the pieces.
+		 */
+		ret = umcg_enqueue_and_wake(tsk);

kernel/sched/umcg.c line 757

Quoted 10 lines
kernel/sched/umcg.c new file +10 -0
+		if (ret)
+			goto unpin;
+	}
+
+	umcg_unpin_pages();
+
+	ret = umcg_wait(timo);
+	switch (ret) {
+	case 0:		/* all done */
+	case -EINTR:	/* umcg_notify_resume() will continue the wait */

So I was playing with the whole worker timeout thing last night and realized this is broken. If we get a signal while we have a timeout, the timeout gets lost.

I think the easiest solution is to have umcg_notify_resume() also resume the timeout, but the first pass of that was yuck, so I need to try again.

Related, by moving the whole enqueue-and-wake thing into the timeout, we get more 'fun' failure cases :-(

Oh well..

Quoted 17 lines
kernel/sched/umcg.c new file +17 -0
+		ret = 0;
+		break;
+
+	default:
+		goto unblock;
+	}
+out:
+	if (worker)
+		tsk->flags |= PF_UMCG_WORKER;
+	return ret;
+
+unpin:
+	umcg_unpin_pages();
+unblock:
+	umcg_update_state(tsk, self, UMCG_TASK_RUNNABLE, UMCG_TASK_RUNNING);
+	goto out;
+}
author maintainer
#24 lore raw

Re: [RFC][PATCH v2 5/5] sched: User Mode Concurency Groups

Quoted 60 lines · Peter Zijlstra
Quoted 43 lines · Peter Zijlstra
(quoted diff) +43 -0
+SYSCALL_DEFINE2(umcg_wait, u32, flags, u64, timo)
+{
+	struct task_struct *tsk = current;
+	struct umcg_task __user *self = READ_ONCE(tsk->umcg_task);
+	bool worker = tsk->flags & PF_UMCG_WORKER;
+	int ret;
+
+	if (!self || flags)
+		return -EINVAL;
+
+	if (worker) {
+		tsk->flags &= ~PF_UMCG_WORKER;
+		if (timo)
+			return -ERANGE;
+	}
+
+	/* see umcg_sys_{enter,exit}() syscall exceptions */
+	ret = umcg_pin_pages();
+	if (ret)
+		goto unblock;
+
+	/*
+	 * Clear UMCG_TF_COND_WAIT *and* check state == RUNNABLE.
+	 */
+	ret = umcg_update_state(tsk, self, UMCG_TASK_RUNNABLE, UMCG_TASK_RUNNABLE);
+	if (ret)
+		goto unpin;
+
+	ret = umcg_wake_next(tsk, self);
+	if (ret)
+		goto unpin;
+
+	if (worker) {
+		/*
+		 * If this fails it is possible ::next_tid is already running
+		 * while this task is not going to block. This violates our
+		 * constraints.
+		 *
+		 * That said, pretty much the only way to make this fail is by
+		 * force munmap()'ing things. In which case one is most welcome
+		 * to the pieces.
+		 */
+		ret = umcg_enqueue_and_wake(tsk);

Review

Quoted 10 lines
(quoted diff) +10 -0
+		if (ret)
+			goto unpin;
+	}
+
+	umcg_unpin_pages();
+
+	ret = umcg_wait(timo);
+	switch (ret) {
+	case 0:		/* all done */
+	case -EINTR:	/* umcg_notify_resume() will continue the wait */

So I was playing with the whole worker timeout thing last night and realized this is broken. If we get a signal while we have a timeout, the timeout gets lost.

Quoted 3 lines

I think the easiest solution is to have umcg_notify_resume() also resume the timeout, but the first pass of that was yuck, so I need to try again.

Something like this, still yuck though. Also still need to write me a test for this.

include/linux/sched.h +3 -1
@@ -1300,12 +1300,14 @@ struct task_struct {
 	clockid_t		umcg_clock;
 	struct umcg_task __user	*umcg_task;
 
-	/* setup by umcg_pin_enter() */
+	/* setup by umcg_pin_pages() */
 	struct page		*umcg_page;
 
 	struct task_struct	*umcg_server;
 	struct umcg_task __user *umcg_server_task;
 	struct page		*umcg_server_page;
+
+	u64			umcg_timeout;
 #endif
 
 	struct tlbflush_unmap_batch	tlb_ubc;
kernel/sched/umcg.c +10 -1
@@ -232,6 +232,8 @@ static int umcg_update_state(struct task
 /* Called from syscall enter path and exceptions that can schedule */
 void umcg_sys_enter(struct pt_regs *regs, long syscall)
 {
+	current->umcg_timeout = 0;
+
 	/* avoid recursion vs our own syscalls */
 	if (syscall == __NR_umcg_wait ||
 	    syscall == __NR_umcg_ctl)
@@ -519,6 +521,7 @@ void umcg_notify_resume(struct pt_regs *
 	struct umcg_task __user *self = tsk->umcg_task;
 	bool worker = tsk->flags & PF_UMCG_WORKER;
 	u32 state;
+	int ret;
 
 	/* avoid recursion vs schedule() */
 	if (worker)
@@ -554,12 +557,17 @@ void umcg_notify_resume(struct pt_regs *
 		umcg_unpin_pages();
 	}
 
-	switch (umcg_wait(0)) {
+	ret = umcg_wait(tsk->umcg_timeout);
+	switch (ret) {
 	case 0:
 	case -EINTR:
 		/* we will resume the wait after the signal */
 		break;
 
+	case -ETIMEDOUT:
+		regs_set_return_value(regs, ret);
+		break;
+
 	default:
 		UMCG_DIE("wait");
 	}
@@ -759,6 +767,7 @@ SYSCALL_DEFINE2(umcg_wait, u32, flags, u
 	switch (ret) {
 	case 0:		/* all done */
 	case -EINTR:	/* umcg_notify_resume() will continue the wait */
+		tsk->umcg_timeout = timo;
 		ret = 0;
 		break;
 
author maintainer
#25 lore raw

Re: [RFC][PATCH v2 5/5] sched: User Mode Concurency Groups

Quoted 41 lines · Peter Zijlstra

Something like this, still yuck though. Also still need to write me a test for this.

include/linux/sched.h +3 -1
@@ -1300,12 +1300,14 @@ struct task_struct {
 	clockid_t		umcg_clock;
 	struct umcg_task __user	*umcg_task;
 
-	/* setup by umcg_pin_enter() */
+	/* setup by umcg_pin_pages() */
 	struct page		*umcg_page;
 
 	struct task_struct	*umcg_server;
 	struct umcg_task __user *umcg_server_task;
 	struct page		*umcg_server_page;
+
+	u64			umcg_timeout;
 #endif
 
 	struct tlbflush_unmap_batch	tlb_ubc;
kernel/sched/umcg.c +3 -0
@@ -232,6 +232,8 @@ static int umcg_update_state(struct task
 /* Called from syscall enter path and exceptions that can schedule */
 void umcg_sys_enter(struct pt_regs *regs, long syscall)
 {
+	current->umcg_timeout = 0;
+
 	/* avoid recursion vs our own syscalls */
 	if (syscall == __NR_umcg_wait ||
 	    syscall == __NR_umcg_ctl)
@@ -519,6 +521,7 @@ void umcg_notify_resume(struct pt_regs *
 	struct umcg_task __user *self = tsk->umcg_task;
 	bool worker = tsk->flags & PF_UMCG_WORKER;
 	u32 state;
+	int ret;
 
 	/* avoid recursion vs schedule() */
 	if (worker)

kernel/sched/umcg.c void umcg_notify_resume(struct pt_regs * · line 560

Quoted 5 lines
kernel/sched/umcg.c +1 -1
@@ -554,12 +557,17 @@ void umcg_notify_resume(struct pt_regs *
 		umcg_unpin_pages();
 	}
 
-	switch (umcg_wait(0)) {
+	ret = umcg_wait(tsk->umcg_timeout);

Oh how I hate signals... this can get scribbled by a syscall/fault from sigcontext :/

Maybe I can recover the timo argument from the original syscall pt_regs.. let me try.

Quoted 21 lines
kernel/sched/umcg.c +6 -0
+	switch (ret) {
 	case 0:
 	case -EINTR:
 		/* we will resume the wait after the signal */
 		break;
 
+	case -ETIMEDOUT:
+		regs_set_return_value(regs, ret);
+		break;
+
 	default:
 		UMCG_DIE("wait");
 	}
@@ -759,6 +767,7 @@ SYSCALL_DEFINE2(umcg_wait, u32, flags, u
 	switch (ret) {
 	case 0:		/* all done */
 	case -EINTR:	/* umcg_notify_resume() will continue the wait */
+		tsk->umcg_timeout = timo;
 		ret = 0;
 		break;
 
author maintainer
#26 lore raw

Re: [RFC][PATCH v2 5/5] sched: User Mode Concurency Groups

Quoted 28 lines · Peter Zijlstra
Quoted 27 lines · Peter Zijlstra

Something like this, still yuck though. Also still need to write me a test for this.

kernel/sched/umcg.c +3 -0
@@ -232,6 +232,8 @@ static int umcg_update_state(struct task
 /* Called from syscall enter path and exceptions that can schedule */
 void umcg_sys_enter(struct pt_regs *regs, long syscall)
 {
+	current->umcg_timeout = 0;
+
 	/* avoid recursion vs our own syscalls */
 	if (syscall == __NR_umcg_wait ||
 	    syscall == __NR_umcg_ctl)
@@ -519,6 +521,7 @@ void umcg_notify_resume(struct pt_regs *
 	struct umcg_task __user *self = tsk->umcg_task;
 	bool worker = tsk->flags & PF_UMCG_WORKER;
 	u32 state;
+	int ret;
 
 	/* avoid recursion vs schedule() */
 	if (worker)
@@ -554,12 +557,17 @@ void umcg_notify_resume(struct pt_regs *
 		umcg_unpin_pages();
 	}
 
Quoted 8 lines

kernel/sched/umcg.c void umcg_notify_resume(struct pt_regs * · line 560

Quoted 2 lines
kernel/sched/umcg.c +1 -1
-	switch (umcg_wait(0)) {
+	ret = umcg_wait(tsk->umcg_timeout);

Oh how I hate signals... this can get scribbled by a syscall/fault from sigcontext :/

Maybe I can recover the timo argument from the original syscall pt_regs.. let me try.

Urgh, recursive hell... If the signal does *anything* that tickles notify-resume it'll find RUNNABLE and go wait there --- ad infinitum.

I need to go cook dinner, I'll prod more at this later

#27 lore raw

Re: [RFC][PATCH v2 5/5] sched: User Mode Concurency Groups

On Mon, Jan 24, 2022 at 8:44 AM Peter Zijlstra <[email protected]> wrote:

[...]

Quoted 6 lines
Quoted 6 lines

Oh how I hate signals... this can get scribbled by a syscall/fault from sigcontext :/

Maybe I can recover the timo argument from the original syscall pt_regs.. let me try.

I don't think we need to do anything special with timeouts if a signal happens - just normally return -EINTR (after a resume from a server) and let the userspace figure things out.

Alternatively, in my version of the patchset UMCG tasks ignore non-fatal signals. :)

[...]

author maintainer
#28 lore raw

Re: [RFC][PATCH v2 5/5] sched: User Mode Concurency Groups

In reply to Peter Zijlstra
Quoted 2 lines · Peter Zijlstra

Oh how I hate signals... this can get scribbled by a syscall/fault from sigcontext :/

OK, the below seems to work. I'll see if I can clean it up some.

arch/x86/include/asm/syscall.h +33 -17
@@ -94,28 +94,44 @@ static inline int syscall_get_arch(struc
 
 #else	 /* CONFIG_X86_64 */
 
-static inline void syscall_get_arguments(struct task_struct *task,
-					 struct pt_regs *regs,
-					 unsigned long *args)
+static inline unsigned long
+syscall_get_argument(struct task_struct *task, struct pt_regs *regs, int nr)
 {
-# ifdef CONFIG_IA32_EMULATION
+#ifdef CONFIG_IA32_EMULATION
 	if (task->thread_info.status & TS_COMPAT) {
-		*args++ = regs->bx;
-		*args++ = regs->cx;
-		*args++ = regs->dx;
-		*args++ = regs->si;
-		*args++ = regs->di;
-		*args   = regs->bp;
+		switch (nr) {
+		case 0: return regs->bx;
+		case 1: return regs->cx;
+		case 2: return regs->dx;
+		case 3: return regs->si;
+		case 4: return regs->di;
+		case 5: return regs->bp;
+		}
 	} else
-# endif
+#endif
 	{
-		*args++ = regs->di;
-		*args++ = regs->si;
-		*args++ = regs->dx;
-		*args++ = regs->r10;
-		*args++ = regs->r8;
-		*args   = regs->r9;
+		switch (nr) {
+		case 0: return regs->di;
+		case 1: return regs->si;
+		case 2: return regs->dx;
+		case 3: return regs->r10;
+		case 4: return regs->r8;
+		case 5: return regs->r9;
+		}
 	}
+
+	WARN_ON_ONCE(1);
+	return 0;
+}
+
+static inline void syscall_get_arguments(struct task_struct *task,
+					 struct pt_regs *regs,
+					 unsigned long *args)
+{
+	int i;
+
+	for (i = 0; i < 6; i++)
+		*args++ = syscall_get_argument(task, regs, i);
 }
 
 static inline int syscall_get_arch(struct task_struct *task)
include/linux/sched.h +3 -0
@@ -1307,6 +1307,9 @@ struct task_struct {
 	struct task_struct	*umcg_server;
 	struct umcg_task __user *umcg_server_task;
 	struct page		*umcg_server_page;
+
+	unsigned long		umcg_stack_pointer;
+	unsigned int		umcg_worker;
 #endif
 
 	struct tlbflush_unmap_batch	tlb_ubc;
kernel/sched/umcg.c +65 -25

145 lines not on this page · raw message

author maintainer
#29 lore raw

Re: [RFC][PATCH v2 5/5] sched: User Mode Concurency Groups

In reply to Peter Zijlstra
Quoted 60 lines · Peter Zijlstra
Quoted 43 lines · Peter Zijlstra
(quoted diff) +43 -0
+SYSCALL_DEFINE2(umcg_wait, u32, flags, u64, timo)
+{
+	struct task_struct *tsk = current;
+	struct umcg_task __user *self = READ_ONCE(tsk->umcg_task);
+	bool worker = tsk->flags & PF_UMCG_WORKER;
+	int ret;
+
+	if (!self || flags)
+		return -EINVAL;
+
+	if (worker) {
+		tsk->flags &= ~PF_UMCG_WORKER;
+		if (timo)
+			return -ERANGE;
+	}
+
+	/* see umcg_sys_{enter,exit}() syscall exceptions */
+	ret = umcg_pin_pages();
+	if (ret)
+		goto unblock;
+
+	/*
+	 * Clear UMCG_TF_COND_WAIT *and* check state == RUNNABLE.
+	 */
+	ret = umcg_update_state(tsk, self, UMCG_TASK_RUNNABLE, UMCG_TASK_RUNNABLE);
+	if (ret)
+		goto unpin;
+
+	ret = umcg_wake_next(tsk, self);
+	if (ret)
+		goto unpin;
+
+	if (worker) {
+		/*
+		 * If this fails it is possible ::next_tid is already running
+		 * while this task is not going to block. This violates our
+		 * constraints.
+		 *
+		 * That said, pretty much the only way to make this fail is by
+		 * force munmap()'ing things. In which case one is most welcome
+		 * to the pieces.
+		 */
+		ret = umcg_enqueue_and_wake(tsk);

Review

Quoted 10 lines
(quoted diff) +10 -0
+		if (ret)
+			goto unpin;
+	}
+
+	umcg_unpin_pages();
+
+	ret = umcg_wait(timo);
+	switch (ret) {
+	case 0:		/* all done */
+	case -EINTR:	/* umcg_notify_resume() will continue the wait */

So I was playing with the whole worker timeout thing last night and realized this is broken. If we get a signal while we have a timeout, the timeout gets lost.

Quoted 6 lines

I think the easiest solution is to have umcg_notify_resume() also resume the timeout, but the first pass of that was yuck, so I need to try again.

Related, by moving the whole enqueue-and-wake thing into the timeout, we get more 'fun' failure cases :-(

This is the best I can come up with,... but it's a hot mess :-(

Still, let me go try this.

---

include/uapi/linux/umcg.h +8 -0
@@ -127,6 +127,14 @@ struct umcg_task {
 } __attribute__((packed, aligned(UMCG_TASK_ALIGN)));
 
 /**
+ * enum umcg_wait_flag - flags to pass to sys_umcg_wait
+ * @UMCG_WAIT_ENQUEUE:	Enqueue the task on runnable_workers_ptr before waiting
+ */
+enum umcg_wait_flag {
+	UMCG_WAIT_ENQUEUE	= 0x0001,
+};
+
+/**
  * enum umcg_ctl_flag - flags to pass to sys_umcg_ctl
  * @UMCG_CTL_REGISTER:   register the current task as a UMCG task
  * @UMCG_CTL_UNREGISTER: unregister the current task as a UMCG task
kernel/sched/umcg.c +56 -43

187 lines not on this page · raw message

author maintainer
#30 lore raw

Re: [RFC][PATCH v2 5/5] sched: User Mode Concurency Groups

In reply to Peter Zijlstra
Quoted 10 lines · Peter Zijlstra
(quoted diff) +10 -0
+/*
+ * Pinning a page inhibits rmap based unmap for Anon pages. Doing a load
+ * through the user mapping ensures the user mapping exists.
+ */
+#define umcg_pin_and_load(_self, _pagep, _member)				\
+({										\
+	__label__ __out;							\
+	int __ret = -EFAULT;							\
+										\
+	if (pin_user_pages_fast((unsigned long)(_self), 1, 0, &(_pagep)) != 1)	\

Review

Quoted 10 lines
(quoted diff) +10 -0
+		goto __out;							\
+										\
+	if (!PageAnon(_pagep) ||						\
+	    get_user(_member, &(_self)->_member)) {				\
+		unpin_user_page(_pagep);					\
+		goto __out;							\
+	}									\
+	__ret = 0;								\
+__out:	__ret;									\
+})

Per the thread with David, this wants changing like so.

---

kernel/sched/umcg.c +25 -21
@@ -34,25 +34,26 @@ static struct task_struct *umcg_get_task
 }
 
 /*
- * Pinning a page inhibits rmap based unmap for Anon pages. Doing a load
- * through the user mapping ensures the user mapping exists.
+ * Pinning a page inhibits rmap based unmap for Anon pages. Doing a store
+ * through the user mapping ensures the user mapping exists and is writable.
  */
-#define umcg_pin_and_load(_self, _pagep, _member)				\
-({										\
-	__label__ __out;							\
-	int __ret = -EFAULT;							\
-										\
-	if (pin_user_pages_fast((unsigned long)(_self), 1, 0, &(_pagep)) != 1)	\
-		goto __out;							\
-										\
-	if (!PageAnon(_pagep) ||						\
-	    get_user(_member, &(_self)->_member)) {				\
-		unpin_user_page(_pagep);					\
-		goto __out;							\
-	}									\
-	__ret = 0;								\
-__out:	__ret;									\
-})
+static int umcg_pin_page(struct umcg_task __user *self, struct page **pagep)
+{
+	int ret = -EFAULT;
+
+	if (pin_user_pages_fast((unsigned long)self, 1, FOLL_WRITE, pagep) != 1)
+		goto out;
+
+	if (!PageAnon(*pagep) ||
+	    put_user(0ULL, &self->__zero[0])) {
+		unpin_user_page(*pagep);
+		goto out;
+	}
+
+	ret = 0;
+out:
+	return ret;
+}
 
 /**
  * umcg_pin_pages: pin pages containing struct umcg_task of
@@ -72,10 +73,13 @@ static int umcg_pin_pages(void)
 			 tsk->umcg_server))
 		return -EBUSY;
 
-	ret = umcg_pin_and_load(self, tsk->umcg_page, server_tid);
+	ret = umcg_pin_page(self, &tsk->umcg_page);
 	if (ret)
 		goto clear_self;
 
+	if (get_user(server_tid, &self->server_tid))
+		goto unpin_self;
+
 	ret = -ESRCH;
 	server = umcg_get_task(server_tid);
 	if (!server)
@@ -83,7 +87,7 @@ static int umcg_pin_pages(void)
 
 	/* must cache due to possible concurrent change */
 	tsk->umcg_server_task = READ_ONCE(server->umcg_task);
-	ret = umcg_pin_and_load(tsk->umcg_server_task, tsk->umcg_server_page, server_tid);
+	ret = umcg_pin_page(tsk->umcg_server_task, &tsk->umcg_server_page);
 	if (ret)
 		goto clear_server;
 
@@ -414,7 +418,7 @@ static int umcg_wait(u64 timo)
 				break;
 			}
 
-			ret = umcg_pin_and_load(self, page, state);
+			ret = umcg_pin_page(self, &page);
 			if (ret) {
 				page = NULL;
 				break;
#31 lore raw

Re: [RFC][PATCH v2 5/5] sched: User Mode Concurency Groups

In reply to Peter Zijlstra
Quoted 7 lines · Peter Zijlstra

User Managed Concurrency Groups is an M:N threading toolkit that allows constructing user space schedulers designed to efficiently manage heterogeneous in-process workloads while maintaining high CPU utilization (95%+).

XXX moar changelog explaining how this is moar awesome than traditional user-space threading.

Awaiting a commit message that I can parse, I'm just looking at the entry bits for now. TBH I have no idea what this is actually trying to do...

[...]

Quoted 28 lines
include/linux/entry-common.h +6 -0
@@ -23,6 +23,10 @@
 # define _TIF_UPROBE			(0)
 #endif
 
+#ifndef _TIF_UMCG
+# define _TIF_UMCG			(0)
+#endif
+
 /*
  * SYSCALL_WORK flags handled in syscall_enter_from_user_mode()
  */
@@ -43,11 +47,13 @@
 				 SYSCALL_WORK_SYSCALL_EMU |		\
 				 SYSCALL_WORK_SYSCALL_AUDIT |		\
 				 SYSCALL_WORK_SYSCALL_USER_DISPATCH |	\
+				 SYSCALL_WORK_SYSCALL_UMCG |		\
 				 ARCH_SYSCALL_WORK_ENTER)
 #define SYSCALL_WORK_EXIT	(SYSCALL_WORK_SYSCALL_TRACEPOINT |	\
 				 SYSCALL_WORK_SYSCALL_TRACE |		\
 				 SYSCALL_WORK_SYSCALL_AUDIT |		\
 				 SYSCALL_WORK_SYSCALL_USER_DISPATCH |	\
+				 SYSCALL_WORK_SYSCALL_UMCG |		\
 				 SYSCALL_WORK_SYSCALL_EXIT_TRAP	|	\
 				 ARCH_SYSCALL_WORK_EXIT)
 

include/linux/entry-common.h static inline void local_irq_disable_exi · line 235

Quoted 10 lines
include/linux/entry-common.h +4 -1
@@ -221,8 +227,11 @@ static inline void local_irq_disable_exi
  */
 static inline void irqentry_irq_enable(struct pt_regs *regs)
 {
-	if (!regs_irqs_disabled(regs))
+	if (!regs_irqs_disabled(regs)) {
 		local_irq_enable();
+		if (user_mode(regs) && (current->flags & PF_UMCG_WORKER))
+			umcg_sys_enter(regs, -1);
+	}
 }

Perhaps it would make sense to have separate umcg_sys_enter(regs) and umcg_sys_enter_syscall(regs, syscallno)? Even if the former is just a wrapper, to make the entry/exit bits clearly correspond for all the !syscall cases?

Also, is the syscall case meant to nest within this, or syscall entry paths not supposed to call irqentry_irq_enable() ?

include/linux/entry-common.h static inline void irqentry_irq_enable(s · line 249

Quoted 13 lines
include/linux/entry-common.h +4 -1
 /**
@@ -232,8 +241,11 @@ static inline void irqentry_irq_enable(s
  */
 static inline void irqentry_irq_disable(struct pt_regs *regs)
 {
-	if (!regs_irqs_disabled(regs))
+	if (!regs_irqs_disabled(regs)) {
+		if (user_mode(regs) && (current->flags & PF_UMCG_WORKER))
+			umcg_sys_exit(regs);
 		local_irq_disable();
+	}
 }

Do the umcg_sys_{enter,exit}() calls need to happen with IRQs unmasked?

* If not (and this nests): for arm64 these can live in our enter_from_user_mode() and exit_to_user_mode() helpers.

* If so (or this doesn't nest): for arm64 we'd need to rework our local_daif_{inherit,restore,mask}() calls to handle this, though I've been meaning to do that anyway to handle pseudo-NMI better.

Either way, it looks like we'd need helpers along the lines of:

| static __always_inline void umcg_enter_from_user(struct pt_regs *regs)
| {
| 	if (current->flags & PF_UMCG_WORKER)
| 		umcg_sys_enter(regs, -1);
| }
| 
| static __always_inline void umcg_exit_to_user(struct pt_regs *regs)
| {
| 	if (current->flags & PF_UMCG_WORKER)
| 		umcg_sys_exit(regs);
| }

Thanks,
Mark.

author maintainer
#32 lore raw

Re: [RFC][PATCH v2 5/5] sched: User Mode Concurency Groups

Quoted 11 lines · Mark Rutland
Quoted 7 lines · Peter Zijlstra

User Managed Concurrency Groups is an M:N threading toolkit that allows constructing user space schedulers designed to efficiently manage heterogeneous in-process workloads while maintaining high CPU utilization (95%+).

XXX moar changelog explaining how this is moar awesome than traditional user-space threading.

Awaiting a commit message that I can parse, I'm just looking at the entry bits for now. TBH I have no idea what this is actually trying to do...

Ha! yes.. I knew I was going to have to do that eventually :-)

It's basically a user-space scheduler that is subservient to the kernel scheduler (hierarchical scheduling were a user task is a server for other user tasks), where a server thread is in charge of selecting which of it's worker threads gets to run. The original idea was that each server only ever runs a single worker, but PeterO is currently trying to reconsider that.

The *big* feature here, over traditional N:M scheduling, is that threads can block, while traditional userspace threading is limited to non-blocking system calls (and per later, page-faults).

In order to make that happen we must ovbiously hook schedule() for these worker threads and inform userspace (the server thread) when this happens such that it can select another worker thread to go vroom.

Meanwhile, a worker task getting woken from schedule() must not continue running; instead it must enter the server's ready-queue and await it's turn again. Instead of dealing with arbitrary delays deep inside the kernel block chain, we punt and let the task complete until return-to-user and block it there. The time between schedule() and return-to-user is unmanaged time.

Now, since we can't readily poke at userspace memory from schedule(), we could be holding mmap_sem etc., we pin the worker and server page on sys-enter such that when we hit schedule() we can update state and then unpin the pages such that page pin time is from sys-enter to first schedule(), or sys-exit which ever comes first. This ensures the page-pin is *short* term.

Additionally we must deal with signals :-(, the currnt approach is to let them bust boundaries and run them as unmanaged time. UMCG userspace can obviously control this by using pthread_sigmask() and friends.

Now, the reason for irqentry_irq_enable() is mostly #PF. When a worker faults and blocks we want the same things to happen.

Anyway, so workers have 3 layers of hooks:

sys_enter
schedule()
sys_exit

return-to-user

There's a bunch of paths through this:

- sys_enter -> sys_exit:

no blocking; nothing changes:
- sys_enter:
* pin pages

- sys_exit:
* unpin pages

- sys_enter -> schedule() -> sys_exit:

we did block:
- sys_enter:
* pin pages

- schedule():
* mark worker BLOCKED
* wake server (it will observe it's current worker !RUNNING and select a new worker or idles)
* unpin pages

- sys_exit():
* mark worker RUNNABLE
* enqueue worker on server's runnable_list
* wake server (which will observe a new runnable task, add it to whatever and if it was idle goes run, otherwise goes back to sleep to let it's current worker finish)
* block until RUNNING

- sys_enter -> schedule() -> sys_exit -> return_to_user:

As above; except now we got a signal while !RUNNING. sys_exit() terminates and return-to-user takes over running the signal and on return from the signal we'll again block until RUNNING, or do the whole signal dance again if so required.

Does this clarify things a little?

author maintainer
#33 lore raw

Re: [RFC][PATCH v2 5/5] sched: User Mode Concurency Groups

In reply to Mark Rutland
Quoted 7 lines · Mark Rutland
Quoted 7 lines
include/linux/entry-common.h +1 -1
@@ -221,8 +227,11 @@ static inline void local_irq_disable_exi
  */
 static inline void irqentry_irq_enable(struct pt_regs *regs)
 {
-	if (!regs_irqs_disabled(regs))
+	if (!regs_irqs_disabled(regs)) {
 		local_irq_enable();
Quoted 8 lines

include/linux/entry-common.h static inline void local_irq_disable_exi · line 235

Quoted 4 lines
include/linux/entry-common.h +3 -0
+		if (user_mode(regs) && (current->flags & PF_UMCG_WORKER))
+			umcg_sys_enter(regs, -1);
+	}
 }

Perhaps it would make sense to have separate umcg_sys_enter(regs) and umcg_sys_enter_syscall(regs, syscallno)? Even if the former is just a wrapper, to make the entry/exit bits clearly correspond for all the !syscall cases?

Can do I suppose.

Quoted 2 lines

Also, is the syscall case meant to nest within this, or syscall entry paths not supposed to call irqentry_irq_enable() ?

No nesting, syscall_ vs irqentry_. And you can't have a syscall and an exception both be from user at the same time :-)

Quoted 6 lines
Quoted 6 lines
include/linux/entry-common.h +0 -1
 /**
@@ -232,8 +241,11 @@ static inline void irqentry_irq_enable(s
  */
 static inline void irqentry_irq_disable(struct pt_regs *regs)
 {
-	if (!regs_irqs_disabled(regs))
Quoted 8 lines

include/linux/entry-common.h static inline void irqentry_irq_enable(s · line 249

Quoted 6 lines
include/linux/entry-common.h +4 -0
+	if (!regs_irqs_disabled(regs)) {
+		if (user_mode(regs) && (current->flags & PF_UMCG_WORKER))
+			umcg_sys_exit(regs);
 		local_irq_disable();
+	}
 }

Do the umcg_sys_{enter,exit}() calls need to happen with IRQs unmasked?

Yes; both can end up blocking.

Quoted 9 lines

* If not (and this nests): for arm64 these can live in our enter_from_user_mode() and exit_to_user_mode() helpers.

* If so (or this doesn't nest): for arm64 we'd need to rework our local_daif_{inherit,restore,mask}() calls to handle this, though I've been meaning to do that anyway to handle pseudo-NMI better.

Either way, it looks like we'd need helpers along the lines of:

Quoted 11 lines
| static __always_inline void umcg_enter_from_user(struct pt_regs *regs)
| {
| 	if (current->flags & PF_UMCG_WORKER)
| 		umcg_sys_enter(regs, -1);
| }
| 
| static __always_inline void umcg_exit_to_user(struct pt_regs *regs)
| {
| 	if (current->flags & PF_UMCG_WORKER)
| 		umcg_sys_exit(regs);
| }

Would something like:

#ifndef arch_irqentry_irq_enter
static __always_inline bool arch_irqentry_irq_enter(struct pt_regs *regs)

{
	if (!regs_irqs_disabled(regs)) {
		local_irq_enable();
		return true;
	}
	return false;
}

#endif

static __always_inline void irqentry_irq_enter(struct pt_regs *regs)

{
	if (arch_irqentry_irq_inherit(regs)) {
		if (user_mode(regs) && (current->flags & PF_UMCG_WORKER))
			umcg_sys_enter(regs, -1);
	}
}

Work? Then arm64 can do:

static __always_inline bool arch_irqentry_irq_enter(struct pt_regs *regs)

{
	local_daif_inherit();
	return interrupts_enabled(regs);
}

or somesuch...

author maintainer
#34 lore raw

Re: [RFC][PATCH v2 5/5] sched: User Mode Concurency Groups

Quoted 37 lines · Peter Zijlstra
Quoted 13 lines

Either way, it looks like we'd need helpers along the lines of:

| static __always_inline void umcg_enter_from_user(struct pt_regs *regs)
| {
| 	if (current->flags & PF_UMCG_WORKER)
| 		umcg_sys_enter(regs, -1);
| }
| 
| static __always_inline void umcg_exit_to_user(struct pt_regs *regs)
| {
| 	if (current->flags & PF_UMCG_WORKER)
| 		umcg_sys_exit(regs);
| }

Would something like:

#ifndef arch_irqentry_irq_enter
static __always_inline bool arch_irqentry_irq_enter(struct pt_regs *regs)

{
	if (!regs_irqs_disabled(regs)) {
		local_irq_enable();
		return true;
	}
	return false;
}

#endif

static __always_inline void irqentry_irq_enter(struct pt_regs *regs)

{
	if (arch_irqentry_irq_inherit(regs)) {
		if (user_mode(regs) && (current->flags & PF_UMCG_WORKER))
			umcg_sys_enter(regs, -1);
	}
}

Work? Then arm64 can do:

Quoted 7 lines

static __always_inline bool arch_irqentry_irq_enter(struct pt_regs *regs)

{
	local_daif_inherit();
	return interrupts_enabled(regs);
}

or somesuch...

Ah,.. just read your other email, so your concern is about the user_mode() thing due to ARM64 taking a different exception path for from-user vs from-kernel ?

I don't mind too much if arm64 decides to open-code the umcg hooks, but please do it such that's hard to forget a spot.

#35 lore raw

Re: [RFC][PATCH v2 5/5] sched: User Mode Concurency Groups

Quoted 45 lines · Peter Zijlstra
Quoted 43 lines · Peter Zijlstra
Quoted 13 lines

Either way, it looks like we'd need helpers along the lines of:

| static __always_inline void umcg_enter_from_user(struct pt_regs *regs)
| {
| 	if (current->flags & PF_UMCG_WORKER)
| 		umcg_sys_enter(regs, -1);
| }
| 
| static __always_inline void umcg_exit_to_user(struct pt_regs *regs)
| {
| 	if (current->flags & PF_UMCG_WORKER)
| 		umcg_sys_exit(regs);
| }

Would something like:

#ifndef arch_irqentry_irq_enter
static __always_inline bool arch_irqentry_irq_enter(struct pt_regs *regs)

{
	if (!regs_irqs_disabled(regs)) {
		local_irq_enable();
		return true;
	}
	return false;
}

#endif

static __always_inline void irqentry_irq_enter(struct pt_regs *regs)

{
	if (arch_irqentry_irq_inherit(regs)) {
		if (user_mode(regs) && (current->flags & PF_UMCG_WORKER))
			umcg_sys_enter(regs, -1);
	}
}

Work? Then arm64 can do:

static __always_inline bool arch_irqentry_irq_enter(struct pt_regs *regs)

{
	local_daif_inherit();
	return interrupts_enabled(regs);
}
Quoted 5 lines
Quoted 1 line

or somesuch...

Ah,.. just read your other email, so your concern is about the user_mode() thing due to ARM64 taking a different exception path for from-user vs from-kernel ?

Yup; it's two-fold:

1) We have separate vectors for entry from-user and from-kernel, and I'd like to avoid the conditionality (e.g. the user_mode(regs) checks) where possible. Having that unconditional and explicit in the from-user code avoids redundant work and is much easier to see that it's correct and balanced.

We have separate irqentry_from_user() and irqentry_from_kernel() helpers today for this.

2) Due to the way we nest classes of exception, on the entry path we manipulate the flags differently depending on which specific exception we've taken. On the return path we always mask everything (necessary due to the way exception return works architecturally).

Luckily exceptions from-user don't nest, so those cases are simpler than exceptions from-kernel.

Quoted 2 lines

I don't mind too much if arm64 decides to open-code the umcg hooks, but please do it such that's hard to forget a spot.

I'll see what I can do. :)

Thanks,
Mark.

#36 lore raw

Re: [RFC][PATCH v2 5/5] sched: User Mode Concurency Groups

In reply to Peter Zijlstra

Hi Peter,

On Thu, Jan 20, 2022 at 04:55:22PM +0100, Peter Zijlstra wrote:

[...]

Quoted 17 lines
kernel/sched/umcg.c new file +17 -0
+/* pre-schedule() */
+void umcg_wq_worker_sleeping(struct task_struct *tsk)
+{
+	struct umcg_task __user *self = READ_ONCE(tsk->umcg_task);
+	int ret;
+
+	if (!tsk->umcg_server) {
+		/*
+		 * Already blocked before, the pages are unpinned.
+		 */
+		return;
+	}
+
+	/* Must not fault, mmap_sem might be held. */
+	pagefault_disable();
+
+	ret = umcg_update_state(tsk, self, UMCG_TASK_RUNNING, UMCG_TASK_BLOCKED);

kernel/sched/umcg.c line 300

Quoted 10 lines
kernel/sched/umcg.c new file +10 -0
+	if (ret == -EAGAIN) {
+		/*
+		 * Consider:
+		 *
+		 *   self->state = UMCG_TASK_RUNNABLE | UMCG_TF_COND_WAIT;
+		 *   ...
+		 *   sys_umcg_wait();
+		 *
+		 * and the '...' code doing a blocking syscall/fault. This
+		 * ensures that returns with UMCG_TASK_RUNNING, which will make

/UMCG_TASK_RUNNING/UMCG_TASK_RUNNABLE/

Quoted 18 lines
kernel/sched/umcg.c new file +18 -0
+		 * sys_umcg_wait() return with -EAGAIN.
+		 */
+		ret = umcg_update_state(tsk, self, UMCG_TASK_RUNNABLE, UMCG_TASK_BLOCKED);
+	}
+	if (ret)
+		UMCG_DIE_PF("state");
+
+	if (umcg_wake_server(tsk))
+		UMCG_DIE_PF("wake");
+
+	pagefault_enable();
+
+	/*
+	 * We're going to sleep, make sure to unpin the pages, this ensures
+	 * the pins are temporary. Also see umcg_sys_exit().
+	 */
+	umcg_unpin_pages();
+}

[...]

Quoted 9 lines
kernel/sched/umcg.c new file +9 -0
+/* Called from syscall exit path and exceptions that can schedule */
+void umcg_sys_exit(struct pt_regs *regs)
+{
+	struct task_struct *tsk = current;
+	long syscall = syscall_get_nr(tsk, regs);
+
+	if (syscall == __NR_umcg_wait ||
+	    syscall == __NR_umcg_ctl)
+		return;

kernel/sched/umcg.c line 508

Quoted 10 lines
kernel/sched/umcg.c new file +10 -0
+
+	if (tsk->umcg_server) {
+		/*
+		 * Didn't block, we done.
+		 */
+		umcg_unpin_pages();
+		return;
+	}
+
+	umcg_unblock_and_wait();

umcg_unblock_and_wait() -> umcg_enqueue_and_wake() -> umcg_wake_server() -> umcg_wake_task(tsk->umcg_server, ...)

tsk->umcg_server is NULL here and umcg_wake_task() use it to update state in umcg_update_state(NULL, ...), that means tsk->umcg_clock will happen something i do not know.

There are two places to call umcg_unblock_and_wait(). One is in umcg_register() where the server is set. Another one is in umcg_sys_exit() where the server is not set. May use a bool to indicate if the server is set.

Quoted 1 line

+}

[...]

Quoted 10 lines

+/**
+ * sys_umcg_wait: transfer running context
+ *
+ * Called like:
+ *
+ * self->state = UMCG_TASK_RUNNABLE | UMCG_TF_COND_WAIT;
+ * ...
+ * sys_umcg_wait(0, time);
+ *
+ * The syscall will clear TF_COND_WAIT and wait until state becomes RUNNING.

Quoted 8 lines

+ * The code '...' must not contain syscalls
+ *
+ * If self->next_tid is set and indicates a valid UMCG task with RUNNABLE state
+ * that task will be made RUNNING and woken -- transfering the running context
+ * to that task. In this case self->next_tid is modified with TID_RUNNING to
+ * indicate self->next_tid is consumed.
+ *
+ * If self->next has TID_RUNNING set, it is validated the related task has

/self->next/self->next_tid/
Things are not clear to me even they are clear now. Nice.

Thanks,
Tao

author maintainer
#37 lore raw

Re: [RFC][PATCH v2 5/5] sched: User Mode Concurency Groups

Quoted 27 lines · Tao Zhou

Hi Peter,

On Thu, Jan 20, 2022 at 04:55:22PM +0100, Peter Zijlstra wrote:

[...]

Quoted 21 lines
kernel/sched/umcg.c new file +21 -0
+/* pre-schedule() */
+void umcg_wq_worker_sleeping(struct task_struct *tsk)
+{
+	struct umcg_task __user *self = READ_ONCE(tsk->umcg_task);
+	int ret;
+
+	if (!tsk->umcg_server) {
+		/*
+		 * Already blocked before, the pages are unpinned.
+		 */
+		return;
+	}
+
+	/* Must not fault, mmap_sem might be held. */
+	pagefault_disable();
+
+	ret = umcg_update_state(tsk, self, UMCG_TASK_RUNNING, UMCG_TASK_BLOCKED);
+	if (ret == -EAGAIN) {
+		/*
+		 * Consider:
+		 *
Quoted 8 lines

kernel/sched/umcg.c line 300

Quoted 6 lines
kernel/sched/umcg.c new file +6 -0
+		 *   self->state = UMCG_TASK_RUNNABLE | UMCG_TF_COND_WAIT;
+		 *   ...
+		 *   sys_umcg_wait();
+		 *
+		 * and the '...' code doing a blocking syscall/fault. This
+		 * ensures that returns with UMCG_TASK_RUNNING, which will make

/UMCG_TASK_RUNNING/UMCG_TASK_RUNNABLE/

So the issue is that:

	self->state = UMCG_TASK_RUNNABLE | UMCG_TF_COND_WAIT;

	<#PF>
	  umcg_sys_enter()
	    umcg_pin_user_page()
	  schedule()
	    sched_submit_work()
	      umcg_wq_worker_sleeping()
	        umcg_update_state(tsk, self, UMCG_TASK_RUNNING, UMCG_TASK_BLOCKED) // -EAGAIN
		UMCG_DIE()

Which is clearly not desirable.

So this additinoal thing ensures that:

		umcg_update_state(tsk, self, UMCG_TASK_RUNNABLE, UMCG_TASK_BLOCKED) // 0

	  umcg_sys_exit()
	    umcg_update_state(tsk, self, UMCG_TASK_BLOCKED, UMCG_TASK_RUNNABLE);
	    umcg_enqueue_and_wake()

	  umcg_notify_resume()
	    umcg_wait()

	// must be UMCG_TASK_RUNNING here
	</#PF>

So when the pagefault finally does return, it will have:
UMCG_TASK_RUNNING.

Which will then make sys_umcg_wait() return -EAGAIN and around we go.

Quoted 18 lines
Quoted 18 lines
kernel/sched/umcg.c new file +18 -0
+		 * sys_umcg_wait() return with -EAGAIN.
+		 */
+		ret = umcg_update_state(tsk, self, UMCG_TASK_RUNNABLE, UMCG_TASK_BLOCKED);
+	}
+	if (ret)
+		UMCG_DIE_PF("state");
+
+	if (umcg_wake_server(tsk))
+		UMCG_DIE_PF("wake");
+
+	pagefault_enable();
+
+	/*
+	 * We're going to sleep, make sure to unpin the pages, this ensures
+	 * the pins are temporary. Also see umcg_sys_exit().
+	 */
+	umcg_unpin_pages();
+}
#38 lore raw

Re: [RFC][PATCH v2 5/5] sched: User Mode Concurency Groups

Quoted 66 lines · Peter Zijlstra
Quoted 27 lines · Tao Zhou

Hi Peter,

On Thu, Jan 20, 2022 at 04:55:22PM +0100, Peter Zijlstra wrote:

[...]

Quoted 21 lines
kernel/sched/umcg.c new file +21 -0
+/* pre-schedule() */
+void umcg_wq_worker_sleeping(struct task_struct *tsk)
+{
+	struct umcg_task __user *self = READ_ONCE(tsk->umcg_task);
+	int ret;
+
+	if (!tsk->umcg_server) {
+		/*
+		 * Already blocked before, the pages are unpinned.
+		 */
+		return;
+	}
+
+	/* Must not fault, mmap_sem might be held. */
+	pagefault_disable();
+
+	ret = umcg_update_state(tsk, self, UMCG_TASK_RUNNING, UMCG_TASK_BLOCKED);
+	if (ret == -EAGAIN) {
+		/*
+		 * Consider:
+		 *
Quoted 8 lines

kernel/sched/umcg.c line 300

Quoted 6 lines
kernel/sched/umcg.c new file +6 -0
+		 *   self->state = UMCG_TASK_RUNNABLE | UMCG_TF_COND_WAIT;
+		 *   ...
+		 *   sys_umcg_wait();
+		 *
+		 * and the '...' code doing a blocking syscall/fault. This
+		 * ensures that returns with UMCG_TASK_RUNNING, which will make

/UMCG_TASK_RUNNING/UMCG_TASK_RUNNABLE/

So the issue is that:

	self->state = UMCG_TASK_RUNNABLE | UMCG_TF_COND_WAIT;

	<#PF>
	  umcg_sys_enter()
	    umcg_pin_user_page()
	  schedule()
	    sched_submit_work()
	      umcg_wq_worker_sleeping()
	        umcg_update_state(tsk, self, UMCG_TASK_RUNNING, UMCG_TASK_BLOCKED) // -EAGAIN
		UMCG_DIE()

Which is clearly not desirable.

So this additinoal thing ensures that:

		umcg_update_state(tsk, self, UMCG_TASK_RUNNABLE, UMCG_TASK_BLOCKED) // 0

	  umcg_sys_exit()
	    umcg_update_state(tsk, self, UMCG_TASK_BLOCKED, UMCG_TASK_RUNNABLE);
	    umcg_enqueue_and_wake()

	  umcg_notify_resume()
	    umcg_wait()

	// must be UMCG_TASK_RUNNING here
	</#PF>
Quoted 4 lines

So when the pagefault finally does return, it will have:
UMCG_TASK_RUNNING.

Which will then make sys_umcg_wait() return -EAGAIN and around we go.

Thank you, Peter.

Quoted 18 lines
Quoted 18 lines
Quoted 18 lines
kernel/sched/umcg.c new file +18 -0
+		 * sys_umcg_wait() return with -EAGAIN.
+		 */
+		ret = umcg_update_state(tsk, self, UMCG_TASK_RUNNABLE, UMCG_TASK_BLOCKED);
+	}
+	if (ret)
+		UMCG_DIE_PF("state");
+
+	if (umcg_wake_server(tsk))
+		UMCG_DIE_PF("wake");
+
+	pagefault_enable();
+
+	/*
+	 * We're going to sleep, make sure to unpin the pages, this ensures
+	 * the pins are temporary. Also see umcg_sys_exit().
+	 */
+	umcg_unpin_pages();
+}
author maintainer
#39 lore raw

Re: [RFC][PATCH v2 5/5] sched: User Mode Concurency Groups

In reply to Tao Zhou
Quoted 23 lines · Tao Zhou
Quoted 9 lines
kernel/sched/umcg.c new file +9 -0
+/* Called from syscall exit path and exceptions that can schedule */
+void umcg_sys_exit(struct pt_regs *regs)
+{
+	struct task_struct *tsk = current;
+	long syscall = syscall_get_nr(tsk, regs);
+
+	if (syscall == __NR_umcg_wait ||
+	    syscall == __NR_umcg_ctl)
+		return;

kernel/sched/umcg.c line 508

Quoted 10 lines
kernel/sched/umcg.c new file +10 -0
+
+	if (tsk->umcg_server) {
+		/*
+		 * Didn't block, we done.
+		 */
+		umcg_unpin_pages();
+		return;
+	}
+
+	umcg_unblock_and_wait();

umcg_unblock_and_wait() -> umcg_enqueue_and_wake() -> umcg_wake_server() -> umcg_wake_task(tsk->umcg_server, ...)

Quoted 3 lines

tsk->umcg_server is NULL here and umcg_wake_task() use it to update state in umcg_update_state(NULL, ...), that means tsk->umcg_clock will happen something i do not know.

I think umcg_unblock_and_wait() will repin, at which point we should have tsk->umcg_server again.

Quoted 4 lines

There are two places to call umcg_unblock_and_wait(). One is in umcg_register() where the server is set. Another one is in umcg_sys_exit() where the server is not set. May use a bool to indicate if the server is set.

I'm not sure what you're on about, but I absolutely hate redundant state, that only leads to problems.

#40 lore raw

Re: [RFC][PATCH v2 5/5] sched: User Mode Concurency Groups

Quoted 25 lines · Peter Zijlstra
Quoted 23 lines · Tao Zhou
Quoted 9 lines
kernel/sched/umcg.c new file +9 -0
+/* Called from syscall exit path and exceptions that can schedule */
+void umcg_sys_exit(struct pt_regs *regs)
+{
+	struct task_struct *tsk = current;
+	long syscall = syscall_get_nr(tsk, regs);
+
+	if (syscall == __NR_umcg_wait ||
+	    syscall == __NR_umcg_ctl)
+		return;

kernel/sched/umcg.c line 508

Quoted 10 lines
kernel/sched/umcg.c new file +10 -0
+
+	if (tsk->umcg_server) {
+		/*
+		 * Didn't block, we done.
+		 */
+		umcg_unpin_pages();
+		return;
+	}
+
+	umcg_unblock_and_wait();

umcg_unblock_and_wait() -> umcg_enqueue_and_wake() -> umcg_wake_server() -> umcg_wake_task(tsk->umcg_server, ...)

Quoted 6 lines
Quoted 3 lines

tsk->umcg_server is NULL here and umcg_wake_task() use it to update state in umcg_update_state(NULL, ...), that means tsk->umcg_clock will happen something i do not know.

I think umcg_unblock_and_wait() will repin, at which point we should have tsk->umcg_server again.

That's right, I miss that.

Quoted 7 lines
Quoted 4 lines

There are two places to call umcg_unblock_and_wait(). One is in umcg_register() where the server is set. Another one is in umcg_sys_exit() where the server is not set. May use a bool to indicate if the server is set.

I'm not sure what you're on about, but I absolutely hate redundant state, that only leads to problems.

But, it's my noise though.

author maintainer
#41 lore raw

Re: [RFC][PATCH v2 5/5] sched: User Mode Concurency Groups

In reply to Tao Zhou
Quoted 19 lines · Tao Zhou
Quoted 10 lines

+/**
+ * sys_umcg_wait: transfer running context
+ *
+ * Called like:
+ *
+ * self->state = UMCG_TASK_RUNNABLE | UMCG_TF_COND_WAIT;
+ * ...
+ * sys_umcg_wait(0, time);
+ *
+ * The syscall will clear TF_COND_WAIT and wait until state becomes RUNNING.

Quoted 8 lines

+ * The code '...' must not contain syscalls
+ *
+ * If self->next_tid is set and indicates a valid UMCG task with RUNNABLE state
+ * that task will be made RUNNING and woken -- transfering the running context
+ * to that task. In this case self->next_tid is modified with TID_RUNNING to
+ * indicate self->next_tid is consumed.
+ *
+ * If self->next has TID_RUNNING set, it is validated the related task has

Quoted 1 line

/self->next/self->next_tid/

Yeah, there's more of that, I'll be sure to go re-read all the comments.

#42 lore raw

Re: [RFC][PATCH v2 5/5] sched: User Mode Concurency Groups

In reply to Peter Zijlstra

On Thu, Jan 20, 2022 at 04:55:22PM +0100, Peter Zijlstra wrote:

Another iterator of the patch, some nits below.

[...]

Quoted 4 lines
kernel/sched/umcg.c new file +4 -0
+/*
+ * Pinning a page inhibits rmap based unmap for Anon pages. Doing a load
+ * through the user mapping ensures the user mapping exists.
+ */

kernel/sched/umcg.c line 49

Quoted 10 lines
kernel/sched/umcg.c new file +10 -0
+#define umcg_pin_and_load(_self, _pagep, _member)				\
+({										\
+	__label__ __out;							\
+	int __ret = -EFAULT;							\
+										\
+	if (pin_user_pages_fast((unsigned long)(_self), 1, 0, &(_pagep)) != 1)	\
+		goto __out;							\
+										\
+	if (!PageAnon(_pagep) ||						\
+	    get_user(_member, &(_self)->_member)) {				\

Here should be 'get_user(_member, &(_self)->##_member))' if I am not wrong.

Quoted 6 lines
kernel/sched/umcg.c new file +6 -0
+		unpin_user_page(_pagep);					\
+		goto __out;							\
+	}									\
+	__ret = 0;								\
+__out:	__ret;									\
+})

[...]

Quoted 10 lines
kernel/sched/umcg.c new file +10 -0
+
+/*
+ * Enqueue @tsk on it's server's runnable list
+ *
+ * Must be called in umcg_pin_pages() context, relies on tsk->umcg_server.
+ *
+ * cmpxchg based single linked list add such that list integrity is never
+ * violated.  Userspace *MUST* remove it from the list before changing ->state.
+ * As such, we must change state to RUNNABLE before enqueue.
+ *

kernel/sched/umcg.c line 344

Quoted 10 lines
kernel/sched/umcg.c new file +10 -0
+ * Returns:
+ *   0: success
+ *   -EFAULT
+ */
+static int umcg_enqueue_runnable(struct task_struct *tsk)
+{
+	struct umcg_task __user *server = tsk->umcg_server_task;
+	struct umcg_task __user *self = tsk->umcg_task;
+	u64 first_ptr, *head = &server->runnable_workers_ptr;
+	u64 self_ptr = (unsigned long)self;

Why not 'u64 self_ptr = (u64)self;' ?

Quoted 19 lines
kernel/sched/umcg.c new file +19 -0
+	/*
+	 * umcg_pin_pages() did access_ok() on both pointers, use self here
+	 * only because __user_access_begin() isn't available in generic code.
+	 */
+	if (!user_access_begin(self, sizeof(*self)))
+		return -EFAULT;
+
+	unsafe_get_user(first_ptr, head, Efault);
+	do {
+		unsafe_put_user(first_ptr, &self->runnable_workers_ptr, Efault);
+	} while (!unsafe_try_cmpxchg_user(head, &first_ptr, self_ptr, Efault));
+
+	user_access_end();
+	return 0;
+
+Efault:
+	user_access_end();
+	return -EFAULT;
+}

[...]

Quoted 11 lines
kernel/sched/umcg.c new file +11 -0
+/*
+ * Handles ::next_tid as per sys_umcg_wait().
+ *
+ * ::next_tid		- return
+ * -----------------------------
+ * 0			- 0 (success)
+ *
+ * tid			- -ESRCH (no such task, or not of this UMCG)
+ *			- -EAGAIN (next::state != RUNNABLE)
+ *			- 0 (success, ::next_tid |= RUNNING)
+ *

kernel/sched/umcg.c line 612

Quoted 10 lines
kernel/sched/umcg.c new file +10 -0
+ * tid|RUNNING		- -EAGAIN (next::state != RUNNING)
+ *			- 0 (success)
+ *
+ * Returns:
+ *  0: success
+ *  -EFAULT
+ *  -ESRCH
+ *  -EAGAIN
+ */
+static int umcg_wake_next(struct task_struct *tsk, struct umcg_task __user *self)

@tsk is not used in function.

Quoted 50 lines
kernel/sched/umcg.c new file +50 -0
+{
+	struct umcg_task __user *next_task;
+	struct task_struct *next;
+	u32 next_tid, state;
+	int ret;
+
+	if (get_user(next_tid, &self->next_tid))
+		return -EFAULT;
+
+	if (!next_tid)
+		return 0;
+
+	next = umcg_get_task(next_tid);
+	if (!next)
+		return -ESRCH;
+
+	next_task = READ_ONCE(next->umcg_task);
+
+	if (next_tid & UMCG_TID_RUNNING) {
+		ret = -EFAULT;
+		if (get_user(state, &next_task->state))
+			goto put_next;
+
+		ret = 0;
+		if ((state & UMCG_TASK_MASK) != UMCG_TASK_RUNNING)
+			ret = -EAGAIN;
+
+	} else {
+		ret = umcg_wake_task(next, next_task);
+		if (ret)
+			goto put_next;
+
+		ret = -EFAULT;
+		if (put_user(next_tid | UMCG_TID_RUNNING, &self->next_tid))
+			goto put_next;
+
+		/*
+		 * If this is a worker doing sys_umcg_wait() switching to
+		 * another worker, userspace has the responsibility to update
+		 * server::next_tid.
+		 */
+
+		ret = 0;
+	}
+
+put_next:
+	put_task_struct(next);
+	return ret;
+}
+
author maintainer
#44 lore raw
Quoted 7 lines · Peter Oskolkov
Quoted 2 lines · Peter Zijlstra

Latest version, many changes since last time, still under heavy discussion.

Thanks! I'll work on testing/integrating this patchset. I'll also assume that my changes that added blocked worker list
(https://lore.kernel.org/lkml/[email protected]/)

So the blocked list I don't mind, but I'm not sure about the use-case you mentioned... it gets a mess vs PE, but I see why you'd want it, so i need to think about it more.

Quoted 3 lines

and server kicking
(https://lore.kernel.org/lkml/[email protected]/)
are acceptable.

Still need to look at the server kicking thing...

#45 lore raw
In reply to Peter Zijlstra
Quoted 4 lines · On Thu, 20 Jan 2022 16:55:17 +0100 Peter Zijlstra <[email protected]> wrote:

Latest version, many changes since last time, still under heavy discussion.

Seems to work with the test-case I have (below), but that still has a few gaps, coverage wise.

Nice work, Peter.

Quoted 8 lines

Still haven't done the SMP wakeup thing, finally get the idea with worker-timeouts but haven't yet implemented that.

Happy hacking..

----

Should the below be added to samples/ ?

-- Steve

Quoted 9 lines

#define _GNU_SOURCE
#include <unistd.h>
#include <sys/types.h>
#include <sys/syscall.h>
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <signal.h>

[..]

author maintainer
#46 lore raw
Quoted 1 line · Steven Rostedt

Should the below be added to samples/ ?

Maybe eventually, it needs a little TLC to build with clang at the very least.