Skip to contentSkip to conversation

rseq/membarrier: add MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU

PATCH Superseded Peter Oskolkov 5 participants 10 messages Aug 6, 2020 – Aug 7, 2020

Subsystems (4)

v2 of 8 Superseded Latest is v8 →

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/2 rseq/membarrier: add MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU

This patchset is based on Google-internal RSEQ work done by Paul Turner and Andrew Hunter.

When working with per-CPU RSEQ-based memory allocations, it is sometimes important to make sure that a global memory location is no longer accessed from RSEQ critical sections. For example, there can be two per-CPU lists, one is "active" and accessed per-CPU, while another one is inactive and worked on asynchronously "off CPU" (e.g. garbage collection is performed). Then at some point the two lists are swapped, and a fast RCU-like mechanism is required to make sure that the previously active list is no longer accessed.

This patch introduces such a mechanism: in short, membarrier() syscall issues an IPI to a CPU, restarting a potentially active RSEQ critical section on the CPU.

v1->v2:
- removed the ability to IPI all CPUs in a single sycall;
- use task->mm rather than task->group_leader to identify tasks belonging to the same process.

The second patch in the patchset adds a selftest of this feature.

---
 include/uapi/linux/membarrier.h |  5 ++++
 kernel/sched/membarrier.c       | 43 +++++++++++++++++++++++++++++++--
 2 files changed, 46 insertions(+), 2 deletions(-)
include/uapi/linux/membarrier.h +5 -0
index 5891d7614c8c..ce4628ea17fa 100644
@@ -114,6 +114,10 @@
  *                          If this command is not implemented by an
  *                          architecture, -EINVAL is returned.
  *                          Returns 0 on success.
+ * @MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU:
+ *                          If a thread belonging to the current process
+ *                          is currently in an RSEQ critical section on the
+ *                          CPU identified by flags parameter, restart it.
  * @MEMBARRIER_CMD_SHARED:
  *                          Alias to MEMBARRIER_CMD_GLOBAL. Provided for
  *                          header backward compatibility.
@@ -131,6 +135,7 @@ enum membarrier_cmd {
 	MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED		= (1 << 4),
 	MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE		= (1 << 5),
 	MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE	= (1 << 6),
+	MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU		= (1 << 7),
 
 	/* Alias for header backward compatibility. */
 	MEMBARRIER_CMD_SHARED			= MEMBARRIER_CMD_GLOBAL,
kernel/sched/membarrier.c +41 -2

75 lines not on this page · raw message

Signature · 1 line
2.28.0.163.g6104cc2f0b6-goog
#1 lore raw

PATCH v2 2/2 rseq/selftests: test MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU

Based on Google-internal RSEQ work done by
Paul Turner and Andrew Hunter.

This patch adds a selftest for MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU. The test quite often fails without the previous patch in this patchset, but consistently passes with it.

---
 .../selftests/rseq/basic_percpu_ops_test.c    | 181 ++++++++++++++++++
 1 file changed, 181 insertions(+)
tools/testing/selftests/rseq/basic_percpu_ops_test.c +181 -0
index eb3f6db36d36..147c80deac19 100644
@@ -3,16 +3,21 @@
 #include <assert.h>
 #include <pthread.h>
 #include <sched.h>
+#include <stdatomic.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <stddef.h>
+#include <syscall.h>
+#include <unistd.h>
 
 #include "rseq.h"
 
 #define ARRAY_SIZE(arr)	(sizeof(arr) / sizeof((arr)[0]))
 
+#define MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU	(1<<7)
+
 struct percpu_lock_entry {
 	intptr_t v;
 } __attribute__((aligned(128)));
@@ -289,6 +294,180 @@ void test_percpu_list(void)
 	assert(sum == expected_sum);
 }
 
+struct test_membarrier_thread_args {
+	int stop;
+	intptr_t percpu_list_ptr;
+};
+
+/* Worker threads modify data in their "active" percpu lists. */
+void *test_membarrier_worker_thread(void *arg)
+{
+	struct test_membarrier_thread_args *args =
+		(struct test_membarrier_thread_args *)arg;
+	const int iters = 10 * 1000 * 1000;
+	int i;
+
+	if (rseq_register_current_thread()) {
+		fprintf(stderr, "Error: rseq_register_current_thread(...) failed(%d): %s\n",
+			errno, strerror(errno));
+		abort();
+	}
+
+	for (i = 0; i < iters; ++i) {
+		while (true) {
+			int cpu, ret;
+			struct percpu_list *list_ptr = (struct percpu_list *)
+				atomic_load(&args->percpu_list_ptr);
+
+			if (!list_ptr)
+				continue;  /* Not yet initialized. */
+
+			cpu = rseq_cpu_start();
+			struct percpu_list_node *node = list_ptr->c[cpu].head;
+			const intptr_t prev = node->data;
+
+			ret = rseq_cmpeqv_cmpeqv_storev(&node->data, prev,
+					&args->percpu_list_ptr,
+					(intptr_t)list_ptr, prev + 1, cpu);
+			if (!ret)
+				break;  /* Success. */
+		}
+	}
+
+	if (rseq_unregister_current_thread()) {
+		fprintf(stderr, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n",
+			errno, strerror(errno));
+		abort();
+	}
+	return NULL;
+}
+
+void test_membarrier_init_percpu_list(struct percpu_list *list)
+{
+	int i;
+
+	memset(list, 0, sizeof(*list));
+	for (i = 0; i < CPU_SETSIZE; i++) {
+		struct percpu_list_node *node;
+
+		node = malloc(sizeof(*node));
+		assert(node);
+		node->data = 0;
+		node->next = NULL;
+		list->c[i].head = node;
+	}
+}
+
+void test_membarrier_free_percpu_list(struct percpu_list *list)
+{
+	int i;
+
+	for (i = 0; i < CPU_SETSIZE; i++)
+		free(list->c[i].head);
+}
+
+static int sys_membarrier(int cmd, int flags)
+{
+	return syscall(__NR_membarrier, cmd, flags);
+}
+
+/*
+ * The manager thread swaps per-cpu lists that worker threads see,
+ * and validates that there are no unexpected modifications.
+ */
+void *test_membarrier_manager_thread(void *arg)
+{
+	struct test_membarrier_thread_args *args =
+		(struct test_membarrier_thread_args *)arg;
+	struct percpu_list list_a, list_b;
+	intptr_t expect_a = 0, expect_b = 0;
+	int cpu_a = 0, cpu_b = 0;
+
+	if (rseq_register_current_thread()) {
+		fprintf(stderr, "Error: rseq_register_current_thread(...) failed(%d): %s\n",
+			errno, strerror(errno));
+		abort();
+	}
+
+	/* Init lists. */
+	test_membarrier_init_percpu_list(&list_a);
+	test_membarrier_init_percpu_list(&list_b);
+
+	atomic_store(&args->percpu_list_ptr, (intptr_t)&list_a);
+
+	while (!atomic_load(&args->stop)) {
+		/* list_a is "active". */
+		cpu_a = rand() % CPU_SETSIZE;
+		/*
+		 * As list_b is "inactive", we should never see changes
+		 * to list_b.
+		 */
+		if (expect_b != atomic_load(&list_b.c[cpu_b].head->data)) {
+			fprintf(stderr, "Membarrier test failed\n");
+			abort();
+		}
+
+		/* Make list_b "active". */
+		atomic_store(&args->percpu_list_ptr, (intptr_t)&list_b);
+		sys_membarrier(MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU, cpu_a);
+		/*
+		 * Cpu A should now only modify list_b, so we values
+		 * in list_a should be stable.
+		 */
+		expect_a = atomic_load(&list_a.c[cpu_a].head->data);
+
+		cpu_b = rand() % CPU_SETSIZE;
+		/*
+		 * As list_a is "inactive", we should never see changes
+		 * to list_a.
+		 */
+		if (expect_a != atomic_load(&list_a.c[cpu_a].head->data)) {
+			fprintf(stderr, "Membarrier test failed\n");
+			abort();
+		}
+
+		/* Make list_a "active". */
+		atomic_store(&args->percpu_list_ptr, (intptr_t)&list_a);
+		sys_membarrier(MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU, cpu_b);
+		/* Remember a value from list_b. */
+		expect_b = atomic_load(&list_b.c[cpu_b].head->data);
+	}
+
+	test_membarrier_free_percpu_list(&list_a);
+	test_membarrier_free_percpu_list(&list_b);
+
+	if (rseq_unregister_current_thread()) {
+		fprintf(stderr, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n",
+			errno, strerror(errno));
+		abort();
+	}
+	return NULL;
+}
+
+/* Test MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU membarrier command. */
+void test_membarrier(void)
+{
+	struct test_membarrier_thread_args thread_args;
+	pthread_t worker_threads[CPU_SETSIZE];
+	pthread_t manager_thread;
+	int i;
+
+	thread_args.stop = 0;
+	thread_args.percpu_list_ptr = 0;
+	pthread_create(&manager_thread, NULL,
+		       test_membarrier_manager_thread, &thread_args);
+
+	for (i = 0; i < CPU_SETSIZE; i++)
+		pthread_create(&worker_threads[i], NULL,
+		       test_membarrier_worker_thread, &thread_args);
+
+	for (i = 0; i < CPU_SETSIZE; i++)
+		pthread_join(worker_threads[i], NULL);
+
+	atomic_store(&thread_args.stop, 1);
+	pthread_join(manager_thread, NULL);
+}
+
 int main(int argc, char **argv)
 {
 	if (rseq_register_current_thread()) {
@@ -300,6 +479,8 @@ int main(int argc, char **argv)
 	test_percpu_spinlock();
 	printf("percpu_list\n");
 	test_percpu_list();
+	printf("membarrier\n");
+	test_membarrier();
 	if (rseq_unregister_current_thread()) {
 		fprintf(stderr, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n",
 			errno, strerror(errno));
Signature · 1 line
2.28.0.163.g6104cc2f0b6-goog
#2 lore raw

Re: [PATCH 2/2 v2] rseq/selftests: test MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU

Quoted 57 lines · Peter Oskolkov

Based on Google-internal RSEQ work done by
Paul Turner and Andrew Hunter.

This patch adds a selftest for MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU. The test quite often fails without the previous patch in this patchset, but consistently passes with it.

---
 .../selftests/rseq/basic_percpu_ops_test.c    | 181 ++++++++++++++++++
 1 file changed, 181 insertions(+)
tools/testing/selftests/rseq/basic_percpu_ops_test.c +20 -0
index eb3f6db36d36..147c80deac19 100644
@@ -3,16 +3,21 @@
 #include <assert.h>
 #include <pthread.h>
 #include <sched.h>
+#include <stdatomic.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <stddef.h>
+#include <syscall.h>
+#include <unistd.h>
 
 #include "rseq.h"
 
 #define ARRAY_SIZE(arr)	(sizeof(arr) / sizeof((arr)[0]))
 
+#define MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU	(1<<7)
+
 struct percpu_lock_entry {
 	intptr_t v;
 } __attribute__((aligned(128)));
@@ -289,6 +294,180 @@ void test_percpu_list(void)
 	assert(sum == expected_sum);
 }
 
+struct test_membarrier_thread_args {
+	int stop;
+	intptr_t percpu_list_ptr;
+};
+
+/* Worker threads modify data in their "active" percpu lists. */
+void *test_membarrier_worker_thread(void *arg)
+{
+	struct test_membarrier_thread_args *args =
+		(struct test_membarrier_thread_args *)arg;
+	const int iters = 10 * 1000 * 1000;
+	int i;
+
+	if (rseq_register_current_thread()) {
+		fprintf(stderr, "Error: rseq_register_current_thread(...) failed(%d): %s\n",

tools/testing/selftests/rseq/basic_percpu_ops_test.c void test_percpu_list(void) · line 321

Quoted 10 lines
tools/testing/selftests/rseq/basic_percpu_ops_test.c +10 -0
+			errno, strerror(errno));
+		abort();
+	}
+
+	for (i = 0; i < iters; ++i) {
+		while (true) {
+			int cpu, ret;
+			struct percpu_list *list_ptr = (struct percpu_list *)
+				atomic_load(&args->percpu_list_ptr);
+

What if the manager thread update ->percpu_list_ptr and call membarrier() here? I.e.

	CPU0			CPU1
				list_ptr = atomic_load(&args->percpu_list_ptr); // read list_b
	
	atomic_store(&args->percpu_list_ptr, list_a);
	sys_membarrier(MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU, 1); // send ipi to restart rseq.cs on CPU1

				<got IPI, but not in a rseq.cs, so nothing to do>
				cpu = rseq_cpu_start(); // start a rseq.cs and accessing list_b!

The thing is, atomic_load() is an reference to ->percpu_list_ptr, which is outside the rseq.cs, simply restarting rseq doesn't kill this reference.

Am I missing something subtle?

Regards,
Boqun

Quoted 164 lines
tools/testing/selftests/rseq/basic_percpu_ops_test.c +151 -0
+			if (!list_ptr)
+				continue;  /* Not yet initialized. */
+
+			cpu = rseq_cpu_start();
+			struct percpu_list_node *node = list_ptr->c[cpu].head;
+			const intptr_t prev = node->data;
+
+			ret = rseq_cmpeqv_cmpeqv_storev(&node->data, prev,
+					&args->percpu_list_ptr,
+					(intptr_t)list_ptr, prev + 1, cpu);
+			if (!ret)
+				break;  /* Success. */
+		}
+	}
+
+	if (rseq_unregister_current_thread()) {
+		fprintf(stderr, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n",
+			errno, strerror(errno));
+		abort();
+	}
+	return NULL;
+}
+
+void test_membarrier_init_percpu_list(struct percpu_list *list)
+{
+	int i;
+
+	memset(list, 0, sizeof(*list));
+	for (i = 0; i < CPU_SETSIZE; i++) {
+		struct percpu_list_node *node;
+
+		node = malloc(sizeof(*node));
+		assert(node);
+		node->data = 0;
+		node->next = NULL;
+		list->c[i].head = node;
+	}
+}
+
+void test_membarrier_free_percpu_list(struct percpu_list *list)
+{
+	int i;
+
+	for (i = 0; i < CPU_SETSIZE; i++)
+		free(list->c[i].head);
+}
+
+static int sys_membarrier(int cmd, int flags)
+{
+	return syscall(__NR_membarrier, cmd, flags);
+}
+
+/*
+ * The manager thread swaps per-cpu lists that worker threads see,
+ * and validates that there are no unexpected modifications.
+ */
+void *test_membarrier_manager_thread(void *arg)
+{
+	struct test_membarrier_thread_args *args =
+		(struct test_membarrier_thread_args *)arg;
+	struct percpu_list list_a, list_b;
+	intptr_t expect_a = 0, expect_b = 0;
+	int cpu_a = 0, cpu_b = 0;
+
+	if (rseq_register_current_thread()) {
+		fprintf(stderr, "Error: rseq_register_current_thread(...) failed(%d): %s\n",
+			errno, strerror(errno));
+		abort();
+	}
+
+	/* Init lists. */
+	test_membarrier_init_percpu_list(&list_a);
+	test_membarrier_init_percpu_list(&list_b);
+
+	atomic_store(&args->percpu_list_ptr, (intptr_t)&list_a);
+
+	while (!atomic_load(&args->stop)) {
+		/* list_a is "active". */
+		cpu_a = rand() % CPU_SETSIZE;
+		/*
+		 * As list_b is "inactive", we should never see changes
+		 * to list_b.
+		 */
+		if (expect_b != atomic_load(&list_b.c[cpu_b].head->data)) {
+			fprintf(stderr, "Membarrier test failed\n");
+			abort();
+		}
+
+		/* Make list_b "active". */
+		atomic_store(&args->percpu_list_ptr, (intptr_t)&list_b);
+		sys_membarrier(MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU, cpu_a);
+		/*
+		 * Cpu A should now only modify list_b, so we values
+		 * in list_a should be stable.
+		 */
+		expect_a = atomic_load(&list_a.c[cpu_a].head->data);
+
+		cpu_b = rand() % CPU_SETSIZE;
+		/*
+		 * As list_a is "inactive", we should never see changes
+		 * to list_a.
+		 */
+		if (expect_a != atomic_load(&list_a.c[cpu_a].head->data)) {
+			fprintf(stderr, "Membarrier test failed\n");
+			abort();
+		}
+
+		/* Make list_a "active". */
+		atomic_store(&args->percpu_list_ptr, (intptr_t)&list_a);
+		sys_membarrier(MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU, cpu_b);
+		/* Remember a value from list_b. */
+		expect_b = atomic_load(&list_b.c[cpu_b].head->data);
+	}
+
+	test_membarrier_free_percpu_list(&list_a);
+	test_membarrier_free_percpu_list(&list_b);
+
+	if (rseq_unregister_current_thread()) {
+		fprintf(stderr, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n",
+			errno, strerror(errno));
+		abort();
+	}
+	return NULL;
+}
+
+/* Test MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU membarrier command. */
+void test_membarrier(void)
+{
+	struct test_membarrier_thread_args thread_args;
+	pthread_t worker_threads[CPU_SETSIZE];
+	pthread_t manager_thread;
+	int i;
+
+	thread_args.stop = 0;
+	thread_args.percpu_list_ptr = 0;
+	pthread_create(&manager_thread, NULL,
+		       test_membarrier_manager_thread, &thread_args);
+
+	for (i = 0; i < CPU_SETSIZE; i++)
+		pthread_create(&worker_threads[i], NULL,
+		       test_membarrier_worker_thread, &thread_args);
+
+	for (i = 0; i < CPU_SETSIZE; i++)
+		pthread_join(worker_threads[i], NULL);
+
+	atomic_store(&thread_args.stop, 1);
+	pthread_join(manager_thread, NULL);
+}
+
 int main(int argc, char **argv)
 {
 	if (rseq_register_current_thread()) {
@@ -300,6 +479,8 @@ int main(int argc, char **argv)
 	test_percpu_spinlock();
 	printf("percpu_list\n");
 	test_percpu_list();
+	printf("membarrier\n");
+	test_membarrier();
 	if (rseq_unregister_current_thread()) {
 		fprintf(stderr, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n",
 			errno, strerror(errno));
Signature · 1 line
2.28.0.163.g6104cc2f0b6-goog
#3 lore raw

Re: [PATCH 2/2 v2] rseq/selftests: test MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU

Quoted 83 lines · ----- On Aug 6, 2020, at 8:27 PM, Boqun Feng [email protected] wrote:
Quoted 58 lines · Peter Oskolkov

Based on Google-internal RSEQ work done by
Paul Turner and Andrew Hunter.

This patch adds a selftest for MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU. The test quite often fails without the previous patch in this patchset, but consistently passes with it.

---
 .../selftests/rseq/basic_percpu_ops_test.c    | 181 ++++++++++++++++++
 1 file changed, 181 insertions(+)
tools/testing/selftests/rseq/basic_percpu_ops_test.c +20 -0
index eb3f6db36d36..147c80deac19 100644
@@ -3,16 +3,21 @@
 #include <assert.h>
 #include <pthread.h>
 #include <sched.h>
+#include <stdatomic.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <stddef.h>
+#include <syscall.h>
+#include <unistd.h>
 
 #include "rseq.h"
 
 #define ARRAY_SIZE(arr)	(sizeof(arr) / sizeof((arr)[0]))
 
+#define MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU	(1<<7)
+
 struct percpu_lock_entry {
 	intptr_t v;
 } __attribute__((aligned(128)));
@@ -289,6 +294,180 @@ void test_percpu_list(void)
 	assert(sum == expected_sum);
 }
 
+struct test_membarrier_thread_args {
+	int stop;
+	intptr_t percpu_list_ptr;
+};
+
+/* Worker threads modify data in their "active" percpu lists. */
+void *test_membarrier_worker_thread(void *arg)
+{
+	struct test_membarrier_thread_args *args =
+		(struct test_membarrier_thread_args *)arg;
+	const int iters = 10 * 1000 * 1000;
+	int i;
+
+	if (rseq_register_current_thread()) {
+		fprintf(stderr, "Error: rseq_register_current_thread(...) failed(%d): %s\n",

tools/testing/selftests/rseq/basic_percpu_ops_test.c void test_percpu_list(void) · line 321

Quoted 10 lines
tools/testing/selftests/rseq/basic_percpu_ops_test.c +10 -0
+			errno, strerror(errno));
+		abort();
+	}
+
+	for (i = 0; i < iters; ++i) {
+		while (true) {
+			int cpu, ret;
+			struct percpu_list *list_ptr = (struct percpu_list *)
+				atomic_load(&args->percpu_list_ptr);
+

What if the manager thread update ->percpu_list_ptr and call membarrier() here? I.e.

	CPU0			CPU1
				list_ptr = atomic_load(&args->percpu_list_ptr); // read list_b
	
	atomic_store(&args->percpu_list_ptr, list_a);
	sys_membarrier(MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU, 1); // send ipi to
	restart rseq.cs on CPU1

				<got IPI, but not in a rseq.cs, so nothing to do>
				cpu = rseq_cpu_start(); // start a rseq.cs and accessing list_b!
Quoted 5 lines

The thing is, atomic_load() is an reference to ->percpu_list_ptr, which is outside the rseq.cs, simply restarting rseq doesn't kill this reference.

Am I missing something subtle?

I'm with you on this, something looks fishy. It would be good to use delay-inducing testing methods like rseq parametrized selftests to increase the odds of hitting this race more reliably.

Thanks,

Mathieu

Quoted 169 lines

Regards,
Boqun

Quoted 165 lines
tools/testing/selftests/rseq/basic_percpu_ops_test.c +149 -0
+			if (!list_ptr)
+				continue;  /* Not yet initialized. */
+
+			cpu = rseq_cpu_start();
+			struct percpu_list_node *node = list_ptr->c[cpu].head;
+			const intptr_t prev = node->data;
+
+			ret = rseq_cmpeqv_cmpeqv_storev(&node->data, prev,
+					&args->percpu_list_ptr,
+					(intptr_t)list_ptr, prev + 1, cpu);
+			if (!ret)
+				break;  /* Success. */
+		}
+	}
+
+	if (rseq_unregister_current_thread()) {
+		fprintf(stderr, "Error: rseq_unregister_current_thread(...) failed(%d):
 %s\n",
+			errno, strerror(errno));
+		abort();
+	}
+	return NULL;
+}
+
+void test_membarrier_init_percpu_list(struct percpu_list *list)
+{
+	int i;
+
+	memset(list, 0, sizeof(*list));
+	for (i = 0; i < CPU_SETSIZE; i++) {
+		struct percpu_list_node *node;
+
+		node = malloc(sizeof(*node));
+		assert(node);
+		node->data = 0;
+		node->next = NULL;
+		list->c[i].head = node;
+	}
+}
+
+void test_membarrier_free_percpu_list(struct percpu_list *list)
+{
+	int i;
+
+	for (i = 0; i < CPU_SETSIZE; i++)
+		free(list->c[i].head);
+}
+
+static int sys_membarrier(int cmd, int flags)
+{
+	return syscall(__NR_membarrier, cmd, flags);
+}
+
+/*
+ * The manager thread swaps per-cpu lists that worker threads see,
+ * and validates that there are no unexpected modifications.
+ */
+void *test_membarrier_manager_thread(void *arg)
+{
+	struct test_membarrier_thread_args *args =
+		(struct test_membarrier_thread_args *)arg;
+	struct percpu_list list_a, list_b;
+	intptr_t expect_a = 0, expect_b = 0;
+	int cpu_a = 0, cpu_b = 0;
+
+	if (rseq_register_current_thread()) {
+		fprintf(stderr, "Error: rseq_register_current_thread(...) failed(%d): %s\n",
+			errno, strerror(errno));
+		abort();
+	}
+
+	/* Init lists. */
+	test_membarrier_init_percpu_list(&list_a);
+	test_membarrier_init_percpu_list(&list_b);
+
+	atomic_store(&args->percpu_list_ptr, (intptr_t)&list_a);
+
+	while (!atomic_load(&args->stop)) {
+		/* list_a is "active". */
+		cpu_a = rand() % CPU_SETSIZE;
+		/*
+		 * As list_b is "inactive", we should never see changes
+		 * to list_b.
+		 */
+		if (expect_b != atomic_load(&list_b.c[cpu_b].head->data)) {
+			fprintf(stderr, "Membarrier test failed\n");
+			abort();
+		}
+
+		/* Make list_b "active". */
+		atomic_store(&args->percpu_list_ptr, (intptr_t)&list_b);
+		sys_membarrier(MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU, cpu_a);
+		/*
+		 * Cpu A should now only modify list_b, so we values
+		 * in list_a should be stable.
+		 */
+		expect_a = atomic_load(&list_a.c[cpu_a].head->data);
+
+		cpu_b = rand() % CPU_SETSIZE;
+		/*
+		 * As list_a is "inactive", we should never see changes
+		 * to list_a.
+		 */
+		if (expect_a != atomic_load(&list_a.c[cpu_a].head->data)) {
+			fprintf(stderr, "Membarrier test failed\n");
+			abort();
+		}
+
+		/* Make list_a "active". */
+		atomic_store(&args->percpu_list_ptr, (intptr_t)&list_a);
+		sys_membarrier(MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU, cpu_b);
+		/* Remember a value from list_b. */
+		expect_b = atomic_load(&list_b.c[cpu_b].head->data);
+	}
+
+	test_membarrier_free_percpu_list(&list_a);
+	test_membarrier_free_percpu_list(&list_b);
+
+	if (rseq_unregister_current_thread()) {
+		fprintf(stderr, "Error: rseq_unregister_current_thread(...) failed(%d):
 %s\n",
+			errno, strerror(errno));
+		abort();
+	}
+	return NULL;
+}
+
+/* Test MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU membarrier command. */
+void test_membarrier(void)
+{
+	struct test_membarrier_thread_args thread_args;
+	pthread_t worker_threads[CPU_SETSIZE];
+	pthread_t manager_thread;
+	int i;
+
+	thread_args.stop = 0;
+	thread_args.percpu_list_ptr = 0;
+	pthread_create(&manager_thread, NULL,
+		       test_membarrier_manager_thread, &thread_args);
+
+	for (i = 0; i < CPU_SETSIZE; i++)
+		pthread_create(&worker_threads[i], NULL,
+		       test_membarrier_worker_thread, &thread_args);
+
+	for (i = 0; i < CPU_SETSIZE; i++)
+		pthread_join(worker_threads[i], NULL);
+
+	atomic_store(&thread_args.stop, 1);
+	pthread_join(manager_thread, NULL);
+}
+
 int main(int argc, char **argv)

{

tools/testing/selftests/rseq/basic_percpu_ops_test.c +2 -1
 	if (rseq_register_current_thread()) {
@@ -300,6 +479,8 @@ int main(int argc, char **argv)
 	test_percpu_spinlock();
 	printf("percpu_list\n");
 	test_percpu_list();
+	printf("membarrier\n");
+	test_membarrier();
 	if (rseq_unregister_current_thread()) {
 		fprintf(stderr, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n",
 			errno, strerror(errno));
--

2.28.0.163.g6104cc2f0b6-goog

Signature · 3 lines
Mathieu Desnoyers
EfficiOS Inc.
http://www.efficios.com
#4 lore raw

Re: [PATCH 2/2 v2] rseq/selftests: test MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU

In reply to Boqun Feng
Quoted 82 lines · Boqun Feng
Quoted 57 lines · Peter Oskolkov

Based on Google-internal RSEQ work done by
Paul Turner and Andrew Hunter.

This patch adds a selftest for MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU. The test quite often fails without the previous patch in this patchset, but consistently passes with it.

---
 .../selftests/rseq/basic_percpu_ops_test.c    | 181 ++++++++++++++++++
 1 file changed, 181 insertions(+)
tools/testing/selftests/rseq/basic_percpu_ops_test.c +20 -0
index eb3f6db36d36..147c80deac19 100644
@@ -3,16 +3,21 @@
 #include <assert.h>
 #include <pthread.h>
 #include <sched.h>
+#include <stdatomic.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <stddef.h>
+#include <syscall.h>
+#include <unistd.h>
 
 #include "rseq.h"
 
 #define ARRAY_SIZE(arr)      (sizeof(arr) / sizeof((arr)[0]))
 
+#define MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU   (1<<7)
+
 struct percpu_lock_entry {
      intptr_t v;
 } __attribute__((aligned(128)));
@@ -289,6 +294,180 @@ void test_percpu_list(void)
      assert(sum == expected_sum);
 }
 
+struct test_membarrier_thread_args {
+     int stop;
+     intptr_t percpu_list_ptr;
+};
+
+/* Worker threads modify data in their "active" percpu lists. */
+void *test_membarrier_worker_thread(void *arg)
+{
+     struct test_membarrier_thread_args *args =
+             (struct test_membarrier_thread_args *)arg;
+     const int iters = 10 * 1000 * 1000;
+     int i;
+
+     if (rseq_register_current_thread()) {
+             fprintf(stderr, "Error: rseq_register_current_thread(...) failed(%d): %s\n",

tools/testing/selftests/rseq/basic_percpu_ops_test.c void test_percpu_list(void) · line 321

Quoted 10 lines
tools/testing/selftests/rseq/basic_percpu_ops_test.c +10 -0
+                     errno, strerror(errno));
+             abort();
+     }
+
+     for (i = 0; i < iters; ++i) {
+             while (true) {
+                     int cpu, ret;
+                     struct percpu_list *list_ptr = (struct percpu_list *)
+                             atomic_load(&args->percpu_list_ptr);
+

What if the manager thread update ->percpu_list_ptr and call membarrier() here? I.e.

        CPU0                    CPU1
                                list_ptr = atomic_load(&args->percpu_list_ptr); // read list_b

        atomic_store(&args->percpu_list_ptr, list_a);
        sys_membarrier(MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU, 1); // send ipi to restart rseq.cs on CPU1

                                <got IPI, but not in a rseq.cs, so nothing to do>
                                cpu = rseq_cpu_start(); // start a rseq.cs and accessing list_b!
Quoted 5 lines

The thing is, atomic_load() is an reference to ->percpu_list_ptr, which is outside the rseq.cs, simply restarting rseq doesn't kill this reference.

Am I missing something subtle?

rseq_cmpeqv_cmpeqv_store is used below to make sure the reference is the one that should be used; if it is no longer "active", the iteration is restarted.

Quoted 168 lines

Regards,
Boqun

Quoted 164 lines
tools/testing/selftests/rseq/basic_percpu_ops_test.c +151 -1
+                     if (!list_ptr)
+                             continue;  /* Not yet initialized. */
+
+                     cpu = rseq_cpu_start();
+                     struct percpu_list_node *node = list_ptr->c[cpu].head;
+                     const intptr_t prev = node->data;
+
+                     ret = rseq_cmpeqv_cmpeqv_storev(&node->data, prev,
+                                     &args->percpu_list_ptr,
+                                     (intptr_t)list_ptr, prev + 1, cpu);
+                     if (!ret)
+                             break;  /* Success. */
+             }
+     }
+
+     if (rseq_unregister_current_thread()) {
+             fprintf(stderr, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n",
+                     errno, strerror(errno));
+             abort();
+     }
+     return NULL;
+}
+
+void test_membarrier_init_percpu_list(struct percpu_list *list)
+{
+     int i;
+
+     memset(list, 0, sizeof(*list));
+     for (i = 0; i < CPU_SETSIZE; i++) {
+             struct percpu_list_node *node;
+
+             node = malloc(sizeof(*node));
+             assert(node);
+             node->data = 0;
+             node->next = NULL;
+             list->c[i].head = node;
+     }
+}
+
+void test_membarrier_free_percpu_list(struct percpu_list *list)
+{
+     int i;
+
+     for (i = 0; i < CPU_SETSIZE; i++)
+             free(list->c[i].head);
+}
+
+static int sys_membarrier(int cmd, int flags)
+{
+     return syscall(__NR_membarrier, cmd, flags);
+}
+
+/*
+ * The manager thread swaps per-cpu lists that worker threads see,
+ * and validates that there are no unexpected modifications.
+ */
+void *test_membarrier_manager_thread(void *arg)
+{
+     struct test_membarrier_thread_args *args =
+             (struct test_membarrier_thread_args *)arg;
+     struct percpu_list list_a, list_b;
+     intptr_t expect_a = 0, expect_b = 0;
+     int cpu_a = 0, cpu_b = 0;
+
+     if (rseq_register_current_thread()) {
+             fprintf(stderr, "Error: rseq_register_current_thread(...) failed(%d): %s\n",
+                     errno, strerror(errno));
+             abort();
+     }
+
+     /* Init lists. */
+     test_membarrier_init_percpu_list(&list_a);
+     test_membarrier_init_percpu_list(&list_b);
+
+     atomic_store(&args->percpu_list_ptr, (intptr_t)&list_a);
+
+     while (!atomic_load(&args->stop)) {
+             /* list_a is "active". */
+             cpu_a = rand() % CPU_SETSIZE;
+             /*
+              * As list_b is "inactive", we should never see changes
+              * to list_b.
+              */
+             if (expect_b != atomic_load(&list_b.c[cpu_b].head->data)) {
+                     fprintf(stderr, "Membarrier test failed\n");
+                     abort();
+             }
+
+             /* Make list_b "active". */
+             atomic_store(&args->percpu_list_ptr, (intptr_t)&list_b);
+             sys_membarrier(MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU, cpu_a);
+             /*
+              * Cpu A should now only modify list_b, so we values
+              * in list_a should be stable.
+              */
+             expect_a = atomic_load(&list_a.c[cpu_a].head->data);
+
+             cpu_b = rand() % CPU_SETSIZE;
+             /*
+              * As list_a is "inactive", we should never see changes
+              * to list_a.
+              */
+             if (expect_a != atomic_load(&list_a.c[cpu_a].head->data)) {
+                     fprintf(stderr, "Membarrier test failed\n");
+                     abort();
+             }
+
+             /* Make list_a "active". */
+             atomic_store(&args->percpu_list_ptr, (intptr_t)&list_a);
+             sys_membarrier(MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU, cpu_b);
+             /* Remember a value from list_b. */
+             expect_b = atomic_load(&list_b.c[cpu_b].head->data);
+     }
+
+     test_membarrier_free_percpu_list(&list_a);
+     test_membarrier_free_percpu_list(&list_b);
+
+     if (rseq_unregister_current_thread()) {
+             fprintf(stderr, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n",
+                     errno, strerror(errno));
+             abort();
+     }
+     return NULL;
+}
+
+/* Test MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU membarrier command. */
+void test_membarrier(void)
+{
+     struct test_membarrier_thread_args thread_args;
+     pthread_t worker_threads[CPU_SETSIZE];
+     pthread_t manager_thread;
+     int i;
+
+     thread_args.stop = 0;
+     thread_args.percpu_list_ptr = 0;
+     pthread_create(&manager_thread, NULL,
+                    test_membarrier_manager_thread, &thread_args);
+
+     for (i = 0; i < CPU_SETSIZE; i++)
+             pthread_create(&worker_threads[i], NULL,
+                    test_membarrier_worker_thread, &thread_args);
+
+     for (i = 0; i < CPU_SETSIZE; i++)
+             pthread_join(worker_threads[i], NULL);
+
+     atomic_store(&thread_args.stop, 1);
+     pthread_join(manager_thread, NULL);
+}
+
 int main(int argc, char **argv)
 {
      if (rseq_register_current_thread()) {
@@ -300,6 +479,8 @@ int main(int argc, char **argv)
      test_percpu_spinlock();
      printf("percpu_list\n");
      test_percpu_list();
+     printf("membarrier\n");
+     test_membarrier();
      if (rseq_unregister_current_thread()) {
              fprintf(stderr, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n",
                      errno, strerror(errno));
--

2.28.0.163.g6104cc2f0b6-goog

#5 lore raw

Re: [PATCH 2/2 v2] rseq/selftests: test MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU

Quoted 1 line · ----- On Aug 7, 2020, at 1:55 PM, Peter Oskolkov [email protected] wrote:

On Thu, Aug 6, 2020 at 5:27 PM Boqun Feng <[email protected]> wrote:

[...]

Quoted 17 lines
Quoted 17 lines

What if the manager thread update ->percpu_list_ptr and call membarrier() here? I.e.

CPU0 CPU1
list_ptr = atomic_load(&args->percpu_list_ptr); // read list_b

atomic_store(&args->percpu_list_ptr, list_a); sys_membarrier(MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU, 1); // send ipi to restart rseq.cs on CPU1

<got IPI, but not in a rseq.cs, so nothing to do> cpu = rseq_cpu_start(); // start a rseq.cs and accessing list_b!

The thing is, atomic_load() is an reference to ->percpu_list_ptr, which is outside the rseq.cs, simply restarting rseq doesn't kill this reference.

Quoted 5 lines
Quoted 1 line

Am I missing something subtle?

rseq_cmpeqv_cmpeqv_store is used below to make sure the reference is the one that should be used; if it is no longer "active", the iteration is restarted.

I suspect it "works" because the manager thread does not free and repurpose the memory where list_a is allocated, nor does it store to its list head (which would corrupt the pointer dereferenced by CPU 1 in the scenario above). This shares similarities with type-safe memory allocation (see SLAB_TYPESAFE_BY_RCU).

Even though it is not documented as such (or otherwise) in the example code, I feel this example looks like it guarantees that the manager thread "owns" list_a after the rseq-fence, when in fact it can still be read by the rseq critical sections.

AFAIU moving the atomic_load(&args->percpu_list_ptr) into the critical section should entirely solve this and guarantee exclusive access to the old list after the manager's rseq-fence. I wonder why this simpler approach is not favored ?

Thanks,

Mathieu

Signature · 3 lines
Mathieu Desnoyers
EfficiOS Inc.
http://www.efficios.com
#6 lore raw

Re: [PATCH 2/2 v2] rseq/selftests: test MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU

Quoted 39 lines · Mathieu Desnoyers
Quoted 1 line · ----- On Aug 7, 2020, at 1:55 PM, Peter Oskolkov [email protected] wrote:

On Thu, Aug 6, 2020 at 5:27 PM Boqun Feng <[email protected]> wrote:

[...]

Quoted 17 lines
Quoted 17 lines

What if the manager thread update ->percpu_list_ptr and call membarrier() here? I.e.

CPU0 CPU1
list_ptr = atomic_load(&args->percpu_list_ptr); // read list_b

atomic_store(&args->percpu_list_ptr, list_a); sys_membarrier(MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU, 1); // send ipi to restart rseq.cs on CPU1

<got IPI, but not in a rseq.cs, so nothing to do> cpu = rseq_cpu_start(); // start a rseq.cs and accessing list_b!

The thing is, atomic_load() is an reference to ->percpu_list_ptr, which is outside the rseq.cs, simply restarting rseq doesn't kill this reference.

Quoted 5 lines
Quoted 1 line

Am I missing something subtle?

rseq_cmpeqv_cmpeqv_store is used below to make sure the reference is the one that should be used; if it is no longer "active", the iteration is restarted.

I suspect it "works" because the manager thread does not free and repurpose the memory where list_a is allocated, nor does it store to its list head (which would corrupt the pointer dereferenced by CPU 1 in the scenario above). This shares similarities with type-safe memory allocation (see SLAB_TYPESAFE_BY_RCU).

Even though it is not documented as such (or otherwise) in the example code, I feel this example looks like it guarantees that the manager thread "owns" list_a after the rseq-fence, when in fact it can still be read by the rseq critical sections.

Quoted 4 lines

AFAIU moving the atomic_load(&args->percpu_list_ptr) into the critical section should entirely solve this and guarantee exclusive access to the old list after the manager's rseq-fence. I wonder why this simpler approach is not favored ?

I think the test code mimics our actual production code, where the concerns you outlined are not particularly relevant. I'll see if the test can be simplified
in v3 along the lines you suggested.

Thanks,
Peter

Quoted 9 lines

Thanks,

Mathieu

--
Mathieu Desnoyers
EfficiOS Inc.
http://www.efficios.com

#7 lore raw

Re: [PATCH 2/2 v2] rseq/selftests: test MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU

Quoted 46 lines · ----- On Aug 7, 2020, at 2:47 PM, Peter Oskolkov [email protected] wrote:
Quoted 39 lines · Mathieu Desnoyers
Quoted 1 line · ----- On Aug 7, 2020, at 1:55 PM, Peter Oskolkov [email protected] wrote:

On Thu, Aug 6, 2020 at 5:27 PM Boqun Feng <[email protected]> wrote:

[...]

Quoted 17 lines
Quoted 17 lines

What if the manager thread update ->percpu_list_ptr and call membarrier() here? I.e.

CPU0 CPU1
list_ptr = atomic_load(&args->percpu_list_ptr); // read list_b

atomic_store(&args->percpu_list_ptr, list_a); sys_membarrier(MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU, 1); // send ipi to restart rseq.cs on CPU1

<got IPI, but not in a rseq.cs, so nothing to do> cpu = rseq_cpu_start(); // start a rseq.cs and accessing list_b!

The thing is, atomic_load() is an reference to ->percpu_list_ptr, which is outside the rseq.cs, simply restarting rseq doesn't kill this reference.

Quoted 5 lines
Quoted 1 line

Am I missing something subtle?

rseq_cmpeqv_cmpeqv_store is used below to make sure the reference is the one that should be used; if it is no longer "active", the iteration is restarted.

I suspect it "works" because the manager thread does not free and repurpose the memory where list_a is allocated, nor does it store to its list head (which would corrupt the pointer dereferenced by CPU 1 in the scenario above). This shares similarities with type-safe memory allocation (see SLAB_TYPESAFE_BY_RCU).

Even though it is not documented as such (or otherwise) in the example code, I feel this example looks like it guarantees that the manager thread "owns" list_a after the rseq-fence, when in fact it can still be read by the rseq critical sections.

Quoted 4 lines

AFAIU moving the atomic_load(&args->percpu_list_ptr) into the critical section should entirely solve this and guarantee exclusive access to the old list after the manager's rseq-fence. I wonder why this simpler approach is not favored ?

Quoted 4 lines

I think the test code mimics our actual production code, where the concerns you outlined are not particularly relevant. I'll see if the test can be simplified
in v3 along the lines you suggested.

In order to implement that, you'll need to extend the rseq per-arch macros. Here is one I did for x86 (but not all other arch) which dereferences a pointer, adds an offset that the resulting address, and loads the contents of that memory location, all within a rseq critical section. See
https://git.kernel.org/pub/scm/libs/librseq/librseq.git/tree/include/rseq/rseq-x86.h#n1292

int rseq_deref_loadoffp(intptr_t *p, off_t voffp, intptr_t *load, int cpu)

I did that following a discussion with Paul Turner about the requirements for the rseq fence.

For the use-case you have in this example, you will probably want to create a new

int rseq_deref_offset_addv(intptr_t *p, off_t voffp, intptr_t count, int cpu)

Which dereferences the list pointer and adds an offset within the critical section, and then increments the value at that memory location as a commit.

offsetof() is very useful to generate the voffp argument.

Thanks,

Mathieu

Signature · 3 lines
Mathieu Desnoyers
EfficiOS Inc.
http://www.efficios.com
#8 lore raw
In reply to Peter Oskolkov
Quoted 15 lines · Peter Oskolkov
(quoted diff) +15 -0
+#ifdef CONFIG_RSEQ
+static void membarrier_rseq_ipi(void *arg)
+{
+	if (current->mm != arg)  /* Not our process. */
+		return;
+	if (!current->rseq)  /* RSEQ not set up for the current task/thread. */
+		return;
+
+	rseq_preempt(current);
+}
+#endif
+
+static int membarrier_private_restart_rseq_on_cpu(int cpu_id)
+{
+#ifdef CONFIG_RSEQ

Review

Quoted 10 lines
(quoted diff) +10 -0
+	/* syscalls are not allowed inside rseq critical sections. */
+	if (cpu_id == raw_smp_processor_id())
+		return 0;
+
+	return smp_call_function_single(cpu_id, membarrier_rseq_ipi,
+					current->mm, true);
+#else
+	return 0;
+#endif
+}

I'm thinking even this is a problem, we can end up sending IPIs to CPUs outside out partition (they might be NOHZ_FULL) and that's a no-no too.

Something like so perhaps... that really limits it to CPUs that match our mm.

include/linux/sched/mm.h +1 -0
index 6be66f52a2ad..bee5e98e6774 100644
@@ -356,6 +356,7 @@ enum {
 
 enum {
 	MEMBARRIER_FLAG_SYNC_CORE	= (1U << 0),
+	MEMBARRIER_FLAG_RSEQ		= (1U << 1),
 };
 
 #ifdef CONFIG_ARCH_HAS_MEMBARRIER_CALLBACKS
kernel/sched/membarrier.c +17 -6
index 168479a7d61b..4d9b22c2f5e2 100644
@@ -27,6 +27,11 @@
 
 static void ipi_mb(void *info)
 {
+	int *flags = info;
+
+	if (flags && (*flags & MEMBARRIER_FLAG_RSEQ))
+		rseq_preempt(current);
+
 	smp_mb();	/* IPIs should be serializing but paranoid. */
 }
 
@@ -129,11 +134,11 @@ static int membarrier_global_expedited(void)
 	return 0;
 }
 
-static int membarrier_private_expedited(int flags)
+static int membarrier_private_expedited(int flags, int cpu_id)
 {
-	int cpu;
-	cpumask_var_t tmpmask;
 	struct mm_struct *mm = current->mm;
+	cpumask_var_t tmpmask;
+	int cpu;
 
 	if (flags & MEMBARRIER_FLAG_SYNC_CORE) {
 		if (!IS_ENABLED(CONFIG_ARCH_HAS_MEMBARRIER_SYNC_CORE))
@@ -174,6 +179,10 @@ static int membarrier_private_expedited(int flags)
 		 */
 		if (cpu == raw_smp_processor_id())
 			continue;
+
+		if (cpu_id >= 0 && cpu != cpu_id)
+			continue;
+
 		p = rcu_dereference(cpu_rq(cpu)->curr);
 		if (p && p->mm == mm)
 			__cpumask_set_cpu(cpu, tmpmask);
@@ -181,7 +190,7 @@ static int membarrier_private_expedited(int flags)
 	rcu_read_unlock();
 
 	preempt_disable();
-	smp_call_function_many(tmpmask, ipi_mb, NULL, 1);
+	smp_call_function_many(tmpmask, ipi_mb, &flags, 1);
 	preempt_enable();
 
 	free_cpumask_var(tmpmask);
@@ -362,11 +371,13 @@ SYSCALL_DEFINE2(membarrier, int, cmd, int, flags)
 	case MEMBARRIER_CMD_REGISTER_GLOBAL_EXPEDITED:
 		return membarrier_register_global_expedited();
 	case MEMBARRIER_CMD_PRIVATE_EXPEDITED:
-		return membarrier_private_expedited(0);
+		return membarrier_private_expedited(0, -1);
 	case MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED:
 		return membarrier_register_private_expedited(0);
 	case MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE:
-		return membarrier_private_expedited(MEMBARRIER_FLAG_SYNC_CORE);
+		return membarrier_private_expedited(MEMBARRIER_FLAG_SYNC_CORE, -1);
+	case MEMBERRIER_CMD_PRIVATE_EXPEDITED_RSEQ:
+		return membarrier_private_expedited(MEMBARRIER_FLAG_RSEQ, flags);
 	case MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE:
 		return membarrier_register_private_expedited(MEMBARRIER_FLAG_SYNC_CORE);
 	default:
#9 lore raw
Quoted 1 line · On Fri, Aug 7, 2020 at 6:38 AM <[email protected]> wrote:

[...]

Quoted 5 lines

I'm thinking even this is a problem, we can end up sending IPIs to CPUs outside out partition (they might be NOHZ_FULL) and that's a no-no too.

Something like so perhaps... that really limits it to CPUs that match our mm.

Thanks for the suggestion - I'll prepare a v3 based on your and Mathieu's feedback.

Quoted 80 lines
include/linux/sched/mm.h +1 -0
index 6be66f52a2ad..bee5e98e6774 100644
@@ -356,6 +356,7 @@ enum {
 
 enum {
        MEMBARRIER_FLAG_SYNC_CORE       = (1U << 0),
+       MEMBARRIER_FLAG_RSEQ            = (1U << 1),
 };
 
 #ifdef CONFIG_ARCH_HAS_MEMBARRIER_CALLBACKS
kernel/sched/membarrier.c +17 -6
index 168479a7d61b..4d9b22c2f5e2 100644
@@ -27,6 +27,11 @@
 
 static void ipi_mb(void *info)
 {
+       int *flags = info;
+
+       if (flags && (*flags & MEMBARRIER_FLAG_RSEQ))
+               rseq_preempt(current);
+
        smp_mb();       /* IPIs should be serializing but paranoid. */
 }
 
@@ -129,11 +134,11 @@ static int membarrier_global_expedited(void)
        return 0;
 }
 
-static int membarrier_private_expedited(int flags)
+static int membarrier_private_expedited(int flags, int cpu_id)
 {
-       int cpu;
-       cpumask_var_t tmpmask;
        struct mm_struct *mm = current->mm;
+       cpumask_var_t tmpmask;
+       int cpu;
 
        if (flags & MEMBARRIER_FLAG_SYNC_CORE) {
                if (!IS_ENABLED(CONFIG_ARCH_HAS_MEMBARRIER_SYNC_CORE))
@@ -174,6 +179,10 @@ static int membarrier_private_expedited(int flags)
                 */
                if (cpu == raw_smp_processor_id())
                        continue;
+
+               if (cpu_id >= 0 && cpu != cpu_id)
+                       continue;
+
                p = rcu_dereference(cpu_rq(cpu)->curr);
                if (p && p->mm == mm)
                        __cpumask_set_cpu(cpu, tmpmask);
@@ -181,7 +190,7 @@ static int membarrier_private_expedited(int flags)
        rcu_read_unlock();
 
        preempt_disable();
-       smp_call_function_many(tmpmask, ipi_mb, NULL, 1);
+       smp_call_function_many(tmpmask, ipi_mb, &flags, 1);
        preempt_enable();
 
        free_cpumask_var(tmpmask);
@@ -362,11 +371,13 @@ SYSCALL_DEFINE2(membarrier, int, cmd, int, flags)
        case MEMBARRIER_CMD_REGISTER_GLOBAL_EXPEDITED:
                return membarrier_register_global_expedited();
        case MEMBARRIER_CMD_PRIVATE_EXPEDITED:
-               return membarrier_private_expedited(0);
+               return membarrier_private_expedited(0, -1);
        case MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED:
                return membarrier_register_private_expedited(0);
        case MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE:
-               return membarrier_private_expedited(MEMBARRIER_FLAG_SYNC_CORE);
+               return membarrier_private_expedited(MEMBARRIER_FLAG_SYNC_CORE, -1);
+       case MEMBERRIER_CMD_PRIVATE_EXPEDITED_RSEQ:
+               return membarrier_private_expedited(MEMBARRIER_FLAG_RSEQ, flags);
        case MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE:
                return membarrier_register_private_expedited(MEMBARRIER_FLAG_SYNC_CORE);
        default: