Skip to contentSkip to conversation

rseq/membarrier: add MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ

PATCH Superseded Peter Oskolkov 3 participants 5 messages Sep 15, 2020 – Sep 16, 2020

Subsystems (5)

v7 of 8 Superseded Latest is v8 →

lore
Share & tools
lore
#0 lore raw

PATCH v7 1/2 rseq/membarrier: add MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ

This patchset is based on Google-internal RSEQ work done by Paul Turner and Andrew Hunter.

When working with per-CPU RSEQ-based memory allocations, it is sometimes important to make sure that a global memory location is no longer accessed from RSEQ critical sections. For example, there can be two per-CPU lists, one is "active" and accessed per-CPU, while another one is inactive and worked on asynchronously "off CPU" (e.g. garbage collection is performed). Then at some point the two lists are swapped, and a fast RCU-like mechanism is required to make sure that the previously active list is no longer accessed.

This patch introduces such a mechanism: in short, membarrier() syscall issues an IPI to a CPU, restarting a potentially active RSEQ critical section on the CPU.

v1->v2:
- removed the ability to IPI all CPUs in a single sycall;
- use task->mm rather than task->group_leader to identify tasks belonging to the same process.
v2->v3:
- re-added the ability to IPI all CPUs in a single syscall;
- integrated with membarrier_private_expedited() to make sure only CPUs running tasks with the same mm as the current task are interrupted;
- also added MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ;
- flags in membarrier_private_expedited are never actually bit flags but always distinct values (i.e. never two flags are combined), so I modified bit testing to full equation comparison for simplicity (otherwise the code needs to work when several bits are set, for example).
v3->v4:
- added the third parameter to membarrier syscall: @cpu_id:
if @flags == MEMBARRIER_CMD_FLAG_CPU, then @cpu_id indicates the cpu on which RSEQ CS should be restarted.
v4->v5:
- added @cpu_id parameter to sys_membarrier in syscalls.h.
v5->v6:
- made membarrier_private_expedited more efficient in a

    single-cpu case;

- a couple of minor refactorings.
v6->v7:
- made @flags an unsigned int in sys_membarrier;
- a couple of minor refactorings.

The second patch in the patchset adds a selftest of this feature.

---
 include/linux/sched/mm.h        |   3 +
 include/linux/syscalls.h        |   2 +-
 include/uapi/linux/membarrier.h |  26 ++++++
 kernel/sched/membarrier.c       | 136 +++++++++++++++++++++++++-------
 4 files changed, 136 insertions(+), 31 deletions(-)
include/linux/sched/mm.h +3 -0
index f889e332912f..15bfb06f2884 100644
@@ -348,10 +348,13 @@ enum {
 	MEMBARRIER_STATE_GLOBAL_EXPEDITED			= (1U << 3),
 	MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE_READY	= (1U << 4),
 	MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE		= (1U << 5),
+	MEMBARRIER_STATE_PRIVATE_EXPEDITED_RSEQ_READY		= (1U << 6),
+	MEMBARRIER_STATE_PRIVATE_EXPEDITED_RSEQ			= (1U << 7),
 };
 
 enum {
 	MEMBARRIER_FLAG_SYNC_CORE	= (1U << 0),
+	MEMBARRIER_FLAG_RSEQ		= (1U << 1),
 };
 
 #ifdef CONFIG_ARCH_HAS_MEMBARRIER_CALLBACKS
include/linux/syscalls.h +1 -1
index 75ac7f8ae93c..466c993e52bf 100644
@@ -974,7 +974,7 @@ asmlinkage long sys_execveat(int dfd, const char __user *filename,
 			const char __user *const __user *argv,
 			const char __user *const __user *envp, int flags);
 asmlinkage long sys_userfaultfd(int flags);
-asmlinkage long sys_membarrier(int cmd, int flags);
+asmlinkage long sys_membarrier(int cmd, int flags, int cpu_id);
 asmlinkage long sys_mlock2(unsigned long start, size_t len, int flags);
 asmlinkage long sys_copy_file_range(int fd_in, loff_t __user *off_in,
 				    int fd_out, loff_t __user *off_out,
include/uapi/linux/membarrier.h +26 -0
index 5891d7614c8c..737605897f36 100644
@@ -114,6 +114,26 @@
  *                          If this command is not implemented by an
  *                          architecture, -EINVAL is returned.
  *                          Returns 0 on success.
+ * @MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ:
+ *                          Ensure the caller thread, upon return from
+ *                          system call, that all its running thread
+ *                          siblings have any currently running rseq
+ *                          critical sections restarted if @flags
+ *                          parameter is 0; if @flags parameter is
+ *                          MEMBARRIER_CMD_FLAG_CPU,
+ *                          then this operation is performed only
+ *                          on CPU indicated by @cpu_id. If this command is
+ *                          not implemented by an architecture, -EINVAL
+ *                          is returned. A process needs to register its
+ *                          intent to use the private expedited rseq
+ *                          command prior to using it, otherwise
+ *                          this command returns -EPERM.
+ * @MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ:
+ *                          Register the process intent to use
+ *                          MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ.
+ *                          If this command is not implemented by an
+ *                          architecture, -EINVAL is returned.
+ *                          Returns 0 on success.
  * @MEMBARRIER_CMD_SHARED:
  *                          Alias to MEMBARRIER_CMD_GLOBAL. Provided for
  *                          header backward compatibility.
@@ -131,9 +151,15 @@ enum membarrier_cmd {
 	MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED		= (1 << 4),
 	MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE		= (1 << 5),
 	MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE	= (1 << 6),
+	MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ			= (1 << 7),
+	MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ		= (1 << 8),
 
 	/* Alias for header backward compatibility. */
 	MEMBARRIER_CMD_SHARED			= MEMBARRIER_CMD_GLOBAL,
 };
 
+enum membarrier_cmd_flag {
+	MEMBARRIER_CMD_FLAG_CPU		= (1 << 0),
+};
+
 #endif /* _UAPI_LINUX_MEMBARRIER_H */
kernel/sched/membarrier.c +106 -30

220 lines not on this page · raw message

Signature · 1 line
2.28.0.618.gf4bc123cb7-goog
#1 lore raw

PATCH v7 2/2 rseq/selftests: test MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ

Based on Google-internal RSEQ work done by
Paul Turner and Andrew Hunter.

This patch adds a selftest for MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ. The test quite often fails without the previous patch in this patchset, but consistently passes with it.

v3: added rseq_offset_deref_addv() to x86_64 to make the test
more explicit; on other architectures I kept using existing rseq_cmpeqv_cmpeqv_storev() as I have no easy way to test there. Added a comment explaining why the test works this way.
v4: skipped the test if rseq_offset_deref_addv() is not present
(that is, on all architectures other than x86_64).

---
 .../selftests/rseq/basic_percpu_ops_test.c    | 187 ++++++++++++++++++
 tools/testing/selftests/rseq/rseq-x86.h       |  57 ++++++
 2 files changed, 244 insertions(+)
tools/testing/selftests/rseq/basic_percpu_ops_test.c +187 -0
index eb3f6db36d36..e6e10ba4b9ed 100644
@@ -3,16 +3,24 @@
 #include <assert.h>
 #include <pthread.h>
 #include <sched.h>
+#include <stdatomic.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <stddef.h>
+#include <syscall.h>
+#include <unistd.h>
 
 #include "rseq.h"
 
 #define ARRAY_SIZE(arr)	(sizeof(arr) / sizeof((arr)[0]))
 
+/* The local <linux/membarrier.h> may not contain the commands below. */
+#define MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ	(1<<7)
+#define MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ	(1<<8)
+#define MEMBARRIER_CMD_FLAG_CPU		(1<<0)
+
 struct percpu_lock_entry {
 	intptr_t v;
 } __attribute__((aligned(128)));
@@ -289,6 +297,183 @@ void test_percpu_list(void)
 	assert(sum == expected_sum);
 }
 
+struct test_membarrier_thread_args {
+	int stop;
+	intptr_t percpu_list_ptr;
+};
+
+/* Worker threads modify data in their "active" percpu lists. */
+void *test_membarrier_worker_thread(void *arg)
+{
+	struct test_membarrier_thread_args *args =
+		(struct test_membarrier_thread_args *)arg;
+	const int iters = 10 * 1000 * 1000;
+	int i;
+
+	if (rseq_register_current_thread()) {
+		fprintf(stderr, "Error: rseq_register_current_thread(...) failed(%d): %s\n",
+			errno, strerror(errno));
+		abort();
+	}
+
+	/* Wait for initialization. */
+	while (!atomic_load(&args->percpu_list_ptr)) {}
+
+	for (i = 0; i < iters; ++i) {
+		int ret;
+
+		do {
+			int cpu = rseq_cpu_start();
+
+			ret = rseq_offset_deref_addv(&args->percpu_list_ptr,
+							128 * cpu, 1, cpu);
+		} while (rseq_unlikely(ret));
+	}
+
+	if (rseq_unregister_current_thread()) {
+		fprintf(stderr, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n",
+			errno, strerror(errno));
+		abort();
+	}
+	return NULL;
+}
+
+void test_membarrier_init_percpu_list(struct percpu_list *list)
+{
+	int i;
+
+	memset(list, 0, sizeof(*list));
+	for (i = 0; i < CPU_SETSIZE; i++) {
+		struct percpu_list_node *node;
+
+		node = malloc(sizeof(*node));
+		assert(node);
+		node->data = 0;
+		node->next = NULL;
+		list->c[i].head = node;
+	}
+}
+
+void test_membarrier_free_percpu_list(struct percpu_list *list)
+{
+	int i;
+
+	for (i = 0; i < CPU_SETSIZE; i++)
+		free(list->c[i].head);
+}
+
+static int sys_membarrier(int cmd, int flags, int cpu_id)
+{
+	return syscall(__NR_membarrier, cmd, flags, cpu_id);
+}
+
+/*
+ * The manager thread swaps per-cpu lists that worker threads see,
+ * and validates that there are no unexpected modifications.
+ */
+void *test_membarrier_manager_thread(void *arg)
+{
+	struct test_membarrier_thread_args *args =
+		(struct test_membarrier_thread_args *)arg;
+	struct percpu_list list_a, list_b;
+	intptr_t expect_a = 0, expect_b = 0;
+	int cpu_a = 0, cpu_b = 0;
+
+	if (rseq_register_current_thread()) {
+		fprintf(stderr, "Error: rseq_register_current_thread(...) failed(%d): %s\n",
+			errno, strerror(errno));
+		abort();
+	}
+
+	/* Init lists. */
+	test_membarrier_init_percpu_list(&list_a);
+	test_membarrier_init_percpu_list(&list_b);
+
+	atomic_store(&args->percpu_list_ptr, (intptr_t)&list_a);
+
+	while (!atomic_load(&args->stop)) {
+		/* list_a is "active". */
+		cpu_a = rand() % CPU_SETSIZE;
+		/*
+		 * As list_b is "inactive", we should never see changes
+		 * to list_b.
+		 */
+		if (expect_b != atomic_load(&list_b.c[cpu_b].head->data)) {
+			fprintf(stderr, "Membarrier test failed\n");
+			abort();
+		}
+
+		/* Make list_b "active". */
+		atomic_store(&args->percpu_list_ptr, (intptr_t)&list_b);
+		sys_membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ,
+			       MEMBARRIER_CMD_FLAG_CPU, cpu_a);
+		/*
+		 * Cpu A should now only modify list_b, so we values
+		 * in list_a should be stable.
+		 */
+		expect_a = atomic_load(&list_a.c[cpu_a].head->data);
+
+		cpu_b = rand() % CPU_SETSIZE;
+		/*
+		 * As list_a is "inactive", we should never see changes
+		 * to list_a.
+		 */
+		if (expect_a != atomic_load(&list_a.c[cpu_a].head->data)) {
+			fprintf(stderr, "Membarrier test failed\n");
+			abort();
+		}
+
+		/* Make list_a "active". */
+		atomic_store(&args->percpu_list_ptr, (intptr_t)&list_a);
+		sys_membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ,
+			       MEMBARRIER_CMD_FLAG_CPU, cpu_b);
+		/* Remember a value from list_b. */
+		expect_b = atomic_load(&list_b.c[cpu_b].head->data);
+	}
+
+	test_membarrier_free_percpu_list(&list_a);
+	test_membarrier_free_percpu_list(&list_b);
+
+	if (rseq_unregister_current_thread()) {
+		fprintf(stderr, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n",
+			errno, strerror(errno));
+		abort();
+	}
+	return NULL;
+}
+
+/* Test MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU membarrier command. */
+void test_membarrier(void)
+{
+#ifndef RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
+	fprintf(stderr, "rseq_offset_deref_addv is not implemented on this architecture. "
+			"Skipping membarrier test.\n");
+	return;
+#else
+	struct test_membarrier_thread_args thread_args;
+	pthread_t worker_threads[CPU_SETSIZE];
+	pthread_t manager_thread;
+	int i;
+
+	sys_membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ, 0, 0);
+
+	thread_args.stop = 0;
+	thread_args.percpu_list_ptr = 0;
+	pthread_create(&manager_thread, NULL,
+		       test_membarrier_manager_thread, &thread_args);
+
+	for (i = 0; i < CPU_SETSIZE; i++)
+		pthread_create(&worker_threads[i], NULL,
+		       test_membarrier_worker_thread, &thread_args);
+
+	for (i = 0; i < CPU_SETSIZE; i++)
+		pthread_join(worker_threads[i], NULL);
+
+	atomic_store(&thread_args.stop, 1);
+	pthread_join(manager_thread, NULL);
+#endif
+}
+
 int main(int argc, char **argv)
 {
 	if (rseq_register_current_thread()) {
@@ -300,6 +485,8 @@ int main(int argc, char **argv)
 	test_percpu_spinlock();
 	printf("percpu_list\n");
 	test_percpu_list();
+	printf("membarrier\n");
+	test_membarrier();
 	if (rseq_unregister_current_thread()) {
 		fprintf(stderr, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n",
 			errno, strerror(errno));
tools/testing/selftests/rseq/rseq-x86.h +57 -0

63 lines not on this page · raw message

Signature · 1 line
2.28.0.618.gf4bc123cb7-goog
#2 lore raw

Re: [PATCH 2/2 v7] rseq/selftests: test MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ

[ Adding selftests maintainer and mailing list in CC. You should add them to the CC list
of the selftests patches for the next round. ]

Quoted 6 lines · ----- On Sep 15, 2020, at 2:55 PM, Peter Oskolkov [email protected] wrote:

Based on Google-internal RSEQ work done by
Paul Turner and Andrew Hunter.

This patch adds a selftest for MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ. The test quite often fails without the previous patch in this patchset, but consistently passes with it.

Did you consider moving this test into tools/testing/selftests/rseq/param_test.c instead, and update the script "run_param_test.sh" accordingly ? You would leverage for free all the work I have done to insert configurable delay loops into the critical sections, which will very likely increase the likelihood of failure tremendously.

Reproducible frequent and fast failure is really something we want to aim for here when a bug is hiding.

Quoted 20 lines

v3: added rseq_offset_deref_addv() to x86_64 to make the test
more explicit; on other architectures I kept using existing rseq_cmpeqv_cmpeqv_storev() as I have no easy way to test there. Added a comment explaining why the test works this way.
v4: skipped the test if rseq_offset_deref_addv() is not present
(that is, on all architectures other than x86_64).

---
.../selftests/rseq/basic_percpu_ops_test.c | 187 ++++++++++++++++++
tools/testing/selftests/rseq/rseq-x86.h | 57 ++++++

2 files changed, 244 insertions(+)
tools/testing/selftests/rseq/basic_percpu_ops_test.c +0 -0
index eb3f6db36d36..e6e10ba4b9ed 100644

tools/testing/selftests/rseq/basic_percpu_ops_test.c line 6

Quoted 4 lines
tools/testing/selftests/rseq/basic_percpu_ops_test.c +1 -0
@@ -3,16 +3,24 @@
 #include <assert.h>
 #include <pthread.h>
 #include <sched.h>
+#include <stdatomic.h>

That would be the first time stdatomic.h is included in the kernel selftests. Do we want this dependency ?

Quoted 7 lines
tools/testing/selftests/rseq/basic_percpu_ops_test.c +2 -0
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <stddef.h>
+#include <syscall.h>
+#include <unistd.h>

tools/testing/selftests/rseq/basic_percpu_ops_test.c line 23

Quoted 10 lines
tools/testing/selftests/rseq/basic_percpu_ops_test.c +5 -0
 
 #include "rseq.h"
 
 #define ARRAY_SIZE(arr)	(sizeof(arr) / sizeof((arr)[0]))
 
+/* The local <linux/membarrier.h> may not contain the commands below. */
+#define MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ	(1<<7)
+#define MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ	(1<<8)
+#define MEMBARRIER_CMD_FLAG_CPU		(1<<0)
+

The usual way to build and run tests AFAIK is to do "make headers_install" first, and then build the tests against those headers. Therefore, when building the tests, the additional membarrier commands and flags should always be there. Please remove those duplicated preprocessor defines.

Quoted 27 lines
tools/testing/selftests/rseq/basic_percpu_ops_test.c +20 -0
 struct percpu_lock_entry {
 	intptr_t v;
 } __attribute__((aligned(128)));
@@ -289,6 +297,183 @@ void test_percpu_list(void)
 	assert(sum == expected_sum);
 }
 
+struct test_membarrier_thread_args {
+	int stop;
+	intptr_t percpu_list_ptr;
+};
+
+/* Worker threads modify data in their "active" percpu lists. */
+void *test_membarrier_worker_thread(void *arg)
+{
+	struct test_membarrier_thread_args *args =
+		(struct test_membarrier_thread_args *)arg;
+	const int iters = 10 * 1000 * 1000;
+	int i;
+
+	if (rseq_register_current_thread()) {
+		fprintf(stderr, "Error: rseq_register_current_thread(...) failed(%d): %s\n",
+			errno, strerror(errno));
+		abort();
+	}
+
+	/* Wait for initialization. */

tools/testing/selftests/rseq/basic_percpu_ops_test.c void test_percpu_list(void) · line 329

Quoted 10 lines
tools/testing/selftests/rseq/basic_percpu_ops_test.c +10 -0
+	while (!atomic_load(&args->percpu_list_ptr)) {}
+
+	for (i = 0; i < iters; ++i) {
+		int ret;
+
+		do {
+			int cpu = rseq_cpu_start();
+
+			ret = rseq_offset_deref_addv(&args->percpu_list_ptr,
+							128 * cpu, 1, cpu);

That 128 happens to be related to:

struct percpu_list_entry {
        struct percpu_list_node *head;
} __attribute__((aligned(128)));
struct percpu_list {
        struct percpu_list_entry c[CPU_SETSIZE];
};

Please don't hardcode numbers like that. Instead:

+ ret = rseq_offset_deref_addv(&args->percpu_list_ptr,
+ sizeof(struct percpu_list_entry) * cpu, 1, cpu);

Quoted 71 lines
tools/testing/selftests/rseq/basic_percpu_ops_test.c +70 -0
+		} while (rseq_unlikely(ret));
+	}
+
+	if (rseq_unregister_current_thread()) {
+		fprintf(stderr, "Error: rseq_unregister_current_thread(...) failed(%d):
 %s\n",
+			errno, strerror(errno));
+		abort();
+	}
+	return NULL;
+}
+
+void test_membarrier_init_percpu_list(struct percpu_list *list)
+{
+	int i;
+
+	memset(list, 0, sizeof(*list));
+	for (i = 0; i < CPU_SETSIZE; i++) {
+		struct percpu_list_node *node;
+
+		node = malloc(sizeof(*node));
+		assert(node);
+		node->data = 0;
+		node->next = NULL;
+		list->c[i].head = node;
+	}
+}
+
+void test_membarrier_free_percpu_list(struct percpu_list *list)
+{
+	int i;
+
+	for (i = 0; i < CPU_SETSIZE; i++)
+		free(list->c[i].head);
+}
+
+static int sys_membarrier(int cmd, int flags, int cpu_id)
+{
+	return syscall(__NR_membarrier, cmd, flags, cpu_id);
+}
+
+/*
+ * The manager thread swaps per-cpu lists that worker threads see,
+ * and validates that there are no unexpected modifications.
+ */
+void *test_membarrier_manager_thread(void *arg)
+{
+	struct test_membarrier_thread_args *args =
+		(struct test_membarrier_thread_args *)arg;
+	struct percpu_list list_a, list_b;
+	intptr_t expect_a = 0, expect_b = 0;
+	int cpu_a = 0, cpu_b = 0;
+
+	if (rseq_register_current_thread()) {
+		fprintf(stderr, "Error: rseq_register_current_thread(...) failed(%d): %s\n",
+			errno, strerror(errno));
+		abort();
+	}
+
+	/* Init lists. */
+	test_membarrier_init_percpu_list(&list_a);
+	test_membarrier_init_percpu_list(&list_b);
+
+	atomic_store(&args->percpu_list_ptr, (intptr_t)&list_a);
+
+	while (!atomic_load(&args->stop)) {
+		/* list_a is "active". */
+		cpu_a = rand() % CPU_SETSIZE;
+		/*
+		 * As list_b is "inactive", we should never see changes
+		 * to list_b.

tools/testing/selftests/rseq/basic_percpu_ops_test.c void test_percpu_list(void) · line 409

Quoted 10 lines
tools/testing/selftests/rseq/basic_percpu_ops_test.c +10 -0
+		 */
+		if (expect_b != atomic_load(&list_b.c[cpu_b].head->data)) {
+			fprintf(stderr, "Membarrier test failed\n");
+			abort();
+		}
+
+		/* Make list_b "active". */
+		atomic_store(&args->percpu_list_ptr, (intptr_t)&list_b);
+		sys_membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ,
+			       MEMBARRIER_CMD_FLAG_CPU, cpu_a);

missing error check.

tools/testing/selftests/rseq/basic_percpu_ops_test.c void test_percpu_list(void) · line 412

Quoted 2 lines
tools/testing/selftests/rseq/basic_percpu_ops_test.c +2 -0
+		/*
+		 * Cpu A should now only modify list_b, so we values

we -> the

Quoted 8 lines
tools/testing/selftests/rseq/basic_percpu_ops_test.c +8 -0
+		 * in list_a should be stable.
+		 */
+		expect_a = atomic_load(&list_a.c[cpu_a].head->data);
+
+		cpu_b = rand() % CPU_SETSIZE;
+		/*
+		 * As list_a is "inactive", we should never see changes
+		 * to list_a.

tools/testing/selftests/rseq/basic_percpu_ops_test.c void test_percpu_list(void) · line 429

Quoted 10 lines
tools/testing/selftests/rseq/basic_percpu_ops_test.c +10 -0
+		 */
+		if (expect_a != atomic_load(&list_a.c[cpu_a].head->data)) {
+			fprintf(stderr, "Membarrier test failed\n");
+			abort();
+		}
+
+		/* Make list_a "active". */
+		atomic_store(&args->percpu_list_ptr, (intptr_t)&list_a);
+		sys_membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ,
+			       MEMBARRIER_CMD_FLAG_CPU, cpu_b);

missing error check.

Quoted 10 lines
tools/testing/selftests/rseq/basic_percpu_ops_test.c +9 -0
+		/* Remember a value from list_b. */
+		expect_b = atomic_load(&list_b.c[cpu_b].head->data);
+	}
+
+	test_membarrier_free_percpu_list(&list_a);
+	test_membarrier_free_percpu_list(&list_b);
+
+	if (rseq_unregister_current_thread()) {
+		fprintf(stderr, "Error: rseq_unregister_current_thread(...) failed(%d):
 %s\n",

tools/testing/selftests/rseq/basic_percpu_ops_test.c void test_percpu_list(void) · line 448

Quoted 10 lines
tools/testing/selftests/rseq/basic_percpu_ops_test.c +10 -0
+			errno, strerror(errno));
+		abort();
+	}
+	return NULL;
+}
+
+/* Test MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU membarrier command. */
+void test_membarrier(void)
+{
+#ifndef RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV

Please lift the preprocessor conditional outside of the function, e.g.:

#ifdef RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
void test_membarrier(void)
{
[... code goes here...]
}
#else /* RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV /*
void test_membarrier(void)
{
}
#endif /* RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV */

tools/testing/selftests/rseq/basic_percpu_ops_test.c void test_percpu_list(void) · line 458

Quoted 11 lines
tools/testing/selftests/rseq/basic_percpu_ops_test.c +1 -0
+	fprintf(stderr, "rseq_offset_deref_addv is not implemented on this

architecture. "

tools/testing/selftests/rseq/basic_percpu_ops_test.c +9 -0
+			"Skipping membarrier test.\n");
+	return;
+#else
+	struct test_membarrier_thread_args thread_args;
+	pthread_t worker_threads[CPU_SETSIZE];
+	pthread_t manager_thread;
+	int i;
+
+	sys_membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ, 0, 0);

Missing error handling.

Quoted 5 lines
tools/testing/selftests/rseq/basic_percpu_ops_test.c +5 -0
+
+	thread_args.stop = 0;
+	thread_args.percpu_list_ptr = 0;
+	pthread_create(&manager_thread, NULL,
+		       test_membarrier_manager_thread, &thread_args);

tools/testing/selftests/rseq/basic_percpu_ops_test.c void test_percpu_list(void) · line 473

Quoted 10 lines
tools/testing/selftests/rseq/basic_percpu_ops_test.c +10 -0
+
+	for (i = 0; i < CPU_SETSIZE; i++)
+		pthread_create(&worker_threads[i], NULL,
+		       test_membarrier_worker_thread, &thread_args);
+
+	for (i = 0; i < CPU_SETSIZE; i++)
+		pthread_join(worker_threads[i], NULL);
+
+	atomic_store(&thread_args.stop, 1);
+	pthread_join(manager_thread, NULL);

Arguably the existing tests do not check pthread_* errors, but I guess it would not hurt to do so.

Quoted 21 lines
tools/testing/selftests/rseq/basic_percpu_ops_test.c +3 -0
+#endif
+}
+

int main(int argc, char **argv)

{
	if (rseq_register_current_thread()) {
tools/testing/selftests/rseq/basic_percpu_ops_test.c +2 -0
@@ -300,6 +485,8 @@ int main(int argc, char **argv)
 	test_percpu_spinlock();
 	printf("percpu_list\n");
 	test_percpu_list();
+	printf("membarrier\n");
+	test_membarrier();
 	if (rseq_unregister_current_thread()) {
 		fprintf(stderr, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n",
 			errno, strerror(errno));
tools/testing/selftests/rseq/rseq-x86.h +0 -0
index b2da6004fe30..640411518e46 100644

tools/testing/selftests/rseq/rseq-x86.h int rseq_addv(intptr_t *v, intptr_t count, int cpu) · line 287

Quoted 9 lines
tools/testing/selftests/rseq/rseq-x86.h +6 -0
@@ -279,6 +279,63 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
 #endif
 }
 
+#define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
+
+/*
+ *   pval = *(ptr+off)
+ *  *pval += inc;
+ */

Addition to rseq-x86.h should be split into its own commit.

Thanks,

Mathieu

Quoted 56 lines
tools/testing/selftests/rseq/rseq-x86.h +51 -1
+static inline __attribute__((always_inline))
+int rseq_offset_deref_addv(intptr_t *ptr, off_t off, intptr_t inc, int cpu)
+{
+	RSEQ_INJECT_C(9)
+
+	__asm__ __volatile__ goto (
+		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
+#ifdef RSEQ_COMPARE_TWICE
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
+#endif
+		/* Start rseq by storing table entry pointer into rseq_cs. */
+		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
+		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
+		RSEQ_INJECT_ASM(3)
+#ifdef RSEQ_COMPARE_TWICE
+		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
+#endif
+		/* get p+v */
+		"movq %[ptr], %%rbx\n\t"
+		"addq %[off], %%rbx\n\t"
+		/* get pv */
+		"movq (%%rbx), %%rcx\n\t"
+		/* *pv += inc */
+		"addq %[inc], (%%rcx)\n\t"
+		"2:\n\t"
+		RSEQ_INJECT_ASM(4)
+		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
+		: /* gcc asm goto does not allow outputs */
+		: [cpu_id]		"r" (cpu),
+		  [rseq_abi]		"r" (&__rseq_abi),
+		  /* final store input */
+		  [ptr]			"m" (*ptr),
+		  [off]			"er" (off),
+		  [inc]			"er" (inc)
+		: "memory", "cc", "rax", "rbx", "rcx"
+		  RSEQ_INJECT_CLOBBER
+		: abort
+#ifdef RSEQ_COMPARE_TWICE
+		  , error1
+#endif
+	);
+	return 0;
+abort:
+	RSEQ_INJECT_FAILED
+	return -1;
+#ifdef RSEQ_COMPARE_TWICE
+error1:
+	rseq_bug("cpu_id comparison failed");
+#endif
+}
+
 static inline __attribute__((always_inline))
 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
 				 intptr_t *v2, intptr_t newv2,
--

2.28.0.618.gf4bc123cb7-goog

Signature · 3 lines
Mathieu Desnoyers
EfficiOS Inc.
http://www.efficios.com
maintainer Reviewed-by on v7 1/2
#3 lore raw
In reply to Peter Oskolkov

----- On Sep 15, 2020, at 2:55 PM, Peter Oskolkov [email protected] wrote:
[...]

Quoted 24 lines
kernel/sched/membarrier.c +1 -1
-static int membarrier_private_expedited(int flags)
+static int membarrier_private_expedited(int flags, int cpu_id)

{

kernel/sched/membarrier.c +0 -1
-	int cpu;
	cpumask_var_t tmpmask;
	struct mm_struct *mm = current->mm;
kernel/sched/membarrier.c +2 -1
+	smp_call_func_t ipi_func = ipi_mb;
 
-	if (flags & MEMBARRIER_FLAG_SYNC_CORE) {
+	if (flags == MEMBARRIER_FLAG_SYNC_CORE) {
		if (!IS_ENABLED(CONFIG_ARCH_HAS_MEMBARRIER_SYNC_CORE))
			return -EINVAL;
		if (!(atomic_read(&mm->membarrier_state) &
		      MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE_READY))
			return -EPERM;
kernel/sched/membarrier.c +7 -0
+	} else if (flags == MEMBARRIER_FLAG_RSEQ) {
+		if (!IS_ENABLED(CONFIG_RSEQ))
+			return -EINVAL;
+		if (!(atomic_read(&mm->membarrier_state) &
+		      MEMBARRIER_STATE_PRIVATE_EXPEDITED_RSEQ_READY))
+			return -EPERM;
+		ipi_func = ipi_rseq;
	} else {

Review

Quoted 1 line
(quoted diff) +1 -0
+		BUG_ON(flags != 0);

Usually BUG_ON() is really for utterly unrecoverable situations, which is not the case here. I am tempted to just use WARN_ON_ONCE(flags) instead to make dmesg yell (once) if an unexpected flags parameter is received. This is not a user-space input, so it should never trigger unless we change the code.

The rest looks good to me, thanks!

Mathieu

Signature · 3 lines
Mathieu Desnoyers
EfficiOS Inc.
http://www.efficios.com
#4 lore raw
In reply to Peter Oskolkov
Thank you for the patch! Yet something to improve:

Hi Peter,

Thank you for the patch! Yet something to improve:

[auto build test ERROR on kselftest/next]
[also build test ERROR on tip/sched/core linux/master linus/master v5.9-rc5 next-20200915] [If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Peter-Oskolkov/rseq-membarrier-add-MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ/20200916-025708
base:   https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git next
config: m68k-randconfig-r015-20200916 (attached as .config)
compiler: m68k-linux-gcc (GCC) 9.3.0

reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=m68k

If you fix the issue, kindly add following tag as appropriate

All errors (new ones prefixed by >>):

In file included from include/linux/kernel.h:11,
from include/linux/list.h:9,
from include/linux/module.h:12,
from drivers/mmc/host/wbsd.c:20:
include/linux/scatterlist.h: In function 'sg_set_buf':
arch/m68k/include/asm/page_no.h:33:50: warning: ordered comparison of pointer with null pointer [-Wextra]

      33 | #define virt_addr_valid(kaddr) (((void *)(kaddr) >= (void *)PAGE_OFFSET) && \
         |                                                  ^~
   include/linux/compiler.h:78:42: note: in definition of macro 'unlikely'
      78 | # define unlikely(x) __builtin_expect(!!(x), 0)
         |                                          ^
   include/linux/scatterlist.h:143:2: note: in expansion of macro 'BUG_ON'
     143 |  BUG_ON(!virt_addr_valid(buf));
         |  ^~~~~~
   include/linux/scatterlist.h:143:10: note: in expansion of macro 'virt_addr_valid'
     143 |  BUG_ON(!virt_addr_valid(buf));
         |          ^~~~~~~~~~~~~~~
   drivers/mmc/host/wbsd.c: In function 'wbsd_request_end':
   drivers/mmc/host/wbsd.c:212:14: error: implicit declaration of function 'claim_dma_lock' [-Werror=implicit-function-declaration]
     212 |   dmaflags = claim_dma_lock();
         |              ^~~~~~~~~~~~~~
>> drivers/mmc/host/wbsd.c:213:3: error: implicit declaration of function 'disable_dma'; did you mean 'disable_irq'? [-Werror=implicit-function-declaration]
     213 |   disable_dma(host->dma);
         |   ^~~~~~~~~~~
         |   disable_irq
>> drivers/mmc/host/wbsd.c:214:3: error: implicit declaration of function 'clear_dma_ff' [-Werror=implicit-function-declaration]
     214 |   clear_dma_ff(host->dma);
         |   ^~~~~~~~~~~~
   drivers/mmc/host/wbsd.c:215:3: error: implicit declaration of function 'release_dma_lock'; did you mean 'release_task'? [-Werror=implicit-function-declaration]
     215 |   release_dma_lock(dmaflags);
         |   ^~~~~~~~~~~~~~~~
         |   release_task
   drivers/mmc/host/wbsd.c: In function 'wbsd_prepare_data':
>> drivers/mmc/host/wbsd.c:618:4: error: implicit declaration of function 'set_dma_mode'; did you mean 'set_dev_node'? [-Werror=implicit-function-declaration]
     618 |    set_dma_mode(host->dma, DMA_MODE_READ & ~0x40);
         |    ^~~~~~~~~~~~
         |    set_dev_node
>> drivers/mmc/host/wbsd.c:618:28: error: 'DMA_MODE_READ' undeclared (first use in this function); did you mean 'FMODE_READ'?
     618 |    set_dma_mode(host->dma, DMA_MODE_READ & ~0x40);
         |                            ^~~~~~~~~~~~~
         |                            FMODE_READ
   drivers/mmc/host/wbsd.c:618:28: note: each undeclared identifier is reported only once for each function it appears in
>> drivers/mmc/host/wbsd.c:620:28: error: 'DMA_MODE_WRITE' undeclared (first use in this function); did you mean 'FMODE_WRITE'?
     620 |    set_dma_mode(host->dma, DMA_MODE_WRITE & ~0x40);
         |                            ^~~~~~~~~~~~~~
         |                            FMODE_WRITE
>> drivers/mmc/host/wbsd.c:621:3: error: implicit declaration of function 'set_dma_addr'; did you mean 'print_vma_addr'? [-Werror=implicit-function-declaration]
     621 |   set_dma_addr(host->dma, host->dma_addr);
         |   ^~~~~~~~~~~~
         |   print_vma_addr
>> drivers/mmc/host/wbsd.c:622:3: error: implicit declaration of function 'set_dma_count'; did you mean 'head_mapcount'? [-Werror=implicit-function-declaration]
     622 |   set_dma_count(host->dma, size);
         |   ^~~~~~~~~~~~~
         |   head_mapcount
>> drivers/mmc/host/wbsd.c:624:3: error: implicit declaration of function 'enable_dma'; did you mean 'enable_nmi'? [-Werror=implicit-function-declaration]
     624 |   enable_dma(host->dma);
         |   ^~~~~~~~~~
         |   enable_nmi
   drivers/mmc/host/wbsd.c: In function 'wbsd_finish_data':
>> drivers/mmc/host/wbsd.c:702:11: error: implicit declaration of function 'get_dma_residue'; did you mean 'set_dma_reserve'? [-Werror=implicit-function-declaration]
     702 |   count = get_dma_residue(host->dma);
         |           ^~~~~~~~~~~~~~~
         |           set_dma_reserve
   cc1: some warnings being treated as errors
--
   In file included from include/asm-generic/bug.h:5,
                    from arch/m68k/include/asm/bug.h:32,
                    from include/linux/bug.h:5,
                    from include/linux/thread_info.h:12,
                    from arch/m68k/include/asm/current.h:16,
                    from include/linux/sched.h:12,
                    from kernel/sched/sched.h:5,
                    from kernel/sched/membarrier.c:7:
   include/linux/scatterlist.h: In function 'sg_set_buf':
   arch/m68k/include/asm/page_no.h:33:50: warning: ordered comparison of pointer with null pointer [-Wextra]
      33 | #define virt_addr_valid(kaddr) (((void *)(kaddr) >= (void *)PAGE_OFFSET) && \
         |                                                  ^~
   include/linux/compiler.h:78:42: note: in definition of macro 'unlikely'
      78 | # define unlikely(x) __builtin_expect(!!(x), 0)
         |                                          ^
   include/linux/scatterlist.h:143:2: note: in expansion of macro 'BUG_ON'
     143 |  BUG_ON(!virt_addr_valid(buf));
         |  ^~~~~~
   include/linux/scatterlist.h:143:10: note: in expansion of macro 'virt_addr_valid'
     143 |  BUG_ON(!virt_addr_valid(buf));
         |          ^~~~~~~~~~~~~~~
   In file included from kernel/sched/sched.h:65,
                    from kernel/sched/membarrier.c:7:
   kernel/sched/membarrier.c: At top level:
>> include/linux/syscalls.h:238:18: error: conflicting types for 'sys_membarrier'
     238 |  asmlinkage long sys##name(__MAP(x,__SC_DECL,__VA_ARGS__)) \
         |                  ^~~
   include/linux/syscalls.h:224:2: note: in expansion of macro '__SYSCALL_DEFINEx'
     224 |  __SYSCALL_DEFINEx(x, sname, __VA_ARGS__)
         |  ^~~~~~~~~~~~~~~~~
   include/linux/syscalls.h:215:36: note: in expansion of macro 'SYSCALL_DEFINEx'
     215 | #define SYSCALL_DEFINE3(name, ...) SYSCALL_DEFINEx(3, _##name, __VA_ARGS__)
         |                                    ^~~~~~~~~~~~~~~
   kernel/sched/membarrier.c:401:1: note: in expansion of macro 'SYSCALL_DEFINE3'
     401 | SYSCALL_DEFINE3(membarrier, int, cmd, unsigned int, flags, int, cpu_id)
         | ^~~~~~~~~~~~~~~
   In file included from kernel/sched/sched.h:65,
                    from kernel/sched/membarrier.c:7:
   include/linux/syscalls.h:977:17: note: previous declaration of 'sys_membarrier' was here
     977 | asmlinkage long sys_membarrier(int cmd, int flags, int cpu_id);
         |                 ^~~~~~~~~~~~~~

# https://github.com/0day-ci/linux/commit/eb21b18c9bbbbb6f280319c4ead2be5c52dade77
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Peter-Oskolkov/rseq-membarrier-add-MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ/20200916-025708 git checkout eb21b18c9bbbbb6f280319c4ead2be5c52dade77
vim +/sys_membarrier +238 include/linux/syscalls.h

1bd21c6c21e8489 Dominik Brodowski   2018-04-05  227  
e145242ea0df6b7 Dominik Brodowski   2018-04-09  228  /*
e145242ea0df6b7 Dominik Brodowski   2018-04-09  229   * The asmlinkage stub is aliased to a function named __se_sys_*() which
e145242ea0df6b7 Dominik Brodowski   2018-04-09  230   * sign-extends 32-bit ints to longs whenever needed. The actual work is
e145242ea0df6b7 Dominik Brodowski   2018-04-09  231   * done within __do_sys_*().
e145242ea0df6b7 Dominik Brodowski   2018-04-09  232   */
1bd21c6c21e8489 Dominik Brodowski   2018-04-05  233  #ifndef __SYSCALL_DEFINEx
bed1ffca022cc87 Frederic Weisbecker 2009-03-13  234  #define __SYSCALL_DEFINEx(x, name, ...)					\
bee20031772af3d Arnd Bergmann       2018-06-19  235  	__diag_push();							\
bee20031772af3d Arnd Bergmann       2018-06-19  236  	__diag_ignore(GCC, 8, "-Wattribute-alias",			\
bee20031772af3d Arnd Bergmann       2018-06-19  237  		      "Type aliasing is used to sanitize syscall arguments");\
83460ec8dcac141 Andi Kleen          2013-11-12 @238  	asmlinkage long sys##name(__MAP(x,__SC_DECL,__VA_ARGS__))	\
e145242ea0df6b7 Dominik Brodowski   2018-04-09  239  		__attribute__((alias(__stringify(__se_sys##name))));	\
c9a211951c7c79c Howard McLauchlan   2018-03-21  240  	ALLOW_ERROR_INJECTION(sys##name, ERRNO);			\
e145242ea0df6b7 Dominik Brodowski   2018-04-09  241  	static inline long __do_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__));\
e145242ea0df6b7 Dominik Brodowski   2018-04-09  242  	asmlinkage long __se_sys##name(__MAP(x,__SC_LONG,__VA_ARGS__));	\
e145242ea0df6b7 Dominik Brodowski   2018-04-09  243  	asmlinkage long __se_sys##name(__MAP(x,__SC_LONG,__VA_ARGS__))	\
1a94bc34768e463 Heiko Carstens      2009-01-14  244  	{								\
e145242ea0df6b7 Dominik Brodowski   2018-04-09  245  		long ret = __do_sys##name(__MAP(x,__SC_CAST,__VA_ARGS__));\
07fe6e00f6cca6f Al Viro             2013-01-21  246  		__MAP(x,__SC_TEST,__VA_ARGS__);				\
2cf0966683430b6 Al Viro             2013-01-21  247  		__PROTECT(x, ret,__MAP(x,__SC_ARGS,__VA_ARGS__));	\
2cf0966683430b6 Al Viro             2013-01-21  248  		return ret;						\
1a94bc34768e463 Heiko Carstens      2009-01-14  249  	}								\
bee20031772af3d Arnd Bergmann       2018-06-19  250  	__diag_pop();							\
e145242ea0df6b7 Dominik Brodowski   2018-04-09  251  	static inline long __do_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__))
1bd21c6c21e8489 Dominik Brodowski   2018-04-05  252  #endif /* __SYSCALL_DEFINEx */
1a94bc34768e463 Heiko Carstens      2009-01-14  253  

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org