mm, sparse: do not swamp log with huge vmemmap allocation failures

From: Michal Hocko <[email protected]>

While doing a memory hotplug tests under a heavy memory pressure we have noticed too many page allocation failures when allocating vmemmap memmap backed by huge page

[146792.281354] kworker/u3072:1: page allocation failure: order:9, mode:0x24084c0(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO)
[...]
[146792.281394] Call Trace:
[146792.281430]  [<ffffffff81019a99>] dump_trace+0x59/0x310
[146792.281436]  [<ffffffff81019e3a>] show_stack_log_lvl+0xea/0x170
[146792.281440]  [<ffffffff8101abc1>] show_stack+0x21/0x40
[146792.281448]  [<ffffffff8130f040>] dump_stack+0x5c/0x7c
[146792.281464]  [<ffffffff8118c982>] warn_alloc_failed+0xe2/0x150
[146792.281471]  [<ffffffff8118cddd>] __alloc_pages_nodemask+0x3ed/0xb20
[146792.281489]  [<ffffffff811d3aaf>] alloc_pages_current+0x7f/0x100
[146792.281503]  [<ffffffff815dfa2c>] vmemmap_alloc_block+0x79/0xb6
[146792.281510]  [<ffffffff815dfbd3>] __vmemmap_alloc_block_buf+0x136/0x145
[146792.281524]  [<ffffffff815dd0c5>] vmemmap_populate+0xd2/0x2b9
[146792.281529]  [<ffffffff815dffd9>] sparse_mem_map_populate+0x23/0x30
[146792.281532]  [<ffffffff815df88d>] sparse_add_one_section+0x68/0x18e
[146792.281537]  [<ffffffff815d9f5a>] __add_pages+0x10a/0x1d0
[146792.281553]  [<ffffffff8106249a>] arch_add_memory+0x4a/0xc0
[146792.281559]  [<ffffffff815da1f9>] add_memory_resource+0x89/0x160
[146792.281564]  [<ffffffff815da33d>] add_memory+0x6d/0xd0
[146792.281585]  [<ffffffff813d36c4>] acpi_memory_device_add+0x181/0x251
[146792.281597]  [<ffffffff813946e5>] acpi_bus_attach+0xfd/0x19b
[146792.281602]  [<ffffffff81394866>] acpi_bus_scan+0x59/0x69
[146792.281604]  [<ffffffff813949de>] acpi_device_hotplug+0xd2/0x41f
[146792.281608]  [<ffffffff8138db67>] acpi_hotplug_work_fn+0x1a/0x23
[146792.281623]  [<ffffffff81093cee>] process_one_work+0x14e/0x410
[146792.281630]  [<ffffffff81094546>] worker_thread+0x116/0x490
[146792.281637]  [<ffffffff810999ed>] kthread+0xbd/0xe0
[146792.281651]  [<ffffffff815e4e7f>] ret_from_fork+0x3f/0x70

and we do see many of those because essentially every the allocation failes for each memory section. This is overly excessive way to tell user that there is nothing to really worry about because we do have a fallback mechanism to use base pages. The only downside might be a performance degradation due to TLB pressure.

This patch changes vmemmap_alloc_block to use __GFP_NOWARN and warn explicitly once on the first allocation failure. This will reduce the noise in the kernel log considerably, while we still have an indication that a performance might be impacted.

---
Hi,
this has somehow fell of my radar completely. The patch is essentially what Johannes suggested [1] so I have added his s-o-b and added the changelog into it.

Can we have this merged?

[1] http://lkml.kernel.org/r/[email protected]

 arch/x86/mm/init_64.c |  1 -
 mm/sparse-vmemmap.c   | 11 +++++++++--
 2 files changed, 9 insertions(+), 3 deletions(-)

index 126e09625979..5eb954f930be 100644
@@ -1405,7 +1405,6 @@ static int __meminit vmemmap_populate_hugepages(unsigned long start,
 			vmemmap_verify((pte_t *)pmd, node, addr, next);
 			continue;
 		}
-		pr_warn_once("vmemmap: falling back to regular page backing\n");
 		if (vmemmap_populate_basepages(addr, next, node))
 			return -ENOMEM;
 	}

index d0860aab1c89..3f85084cb8bb 100644
@@ -52,12 +52,19 @@ void * __meminit vmemmap_alloc_block(unsigned long size, int node)
 {
 	/* If the main allocator is up use that, fallback to bootmem. */
 	if (slab_is_available()) {
+		gfp_t gfp_mask = GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN;
+		int order = get_order(size);
+		static bool warned;
 		struct page *page;
 
-		page = alloc_pages_node(node, GFP_KERNEL | __GFP_RETRY_MAYFAIL,
-					get_order(size));
+		page = alloc_pages_node(node, gfp_mask, order);
 		if (page)
 			return page_address(page);
+
+		if (!warned) {
+			warn_alloc(gfp_mask, NULL, "vmemmap alloc failure: order:%u", order);
+			warned = true;
+		}
 		return NULL;
 	} else
 		return __earlyonly_bootmem_alloc(node, size, size,

2.14.2

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Looks good to me.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

I think it'd be better to change the ratelimit state to something like once a minute

---
 mm/page_alloc.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

index 82e6d2c914ab..af3f92beec04 100644
@@ -3269,8 +3269,7 @@ void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...)
 {
 	struct va_format vaf;
 	va_list args;
-	static DEFINE_RATELIMIT_STATE(nopage_rs, DEFAULT_RATELIMIT_INTERVAL,
-				      DEFAULT_RATELIMIT_BURST);
+	static DEFINE_RATELIMIT_STATE(nopage_rs, HZ * 60, 1);
 
 	if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs))
 		return;

 mm/page_alloc.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

index 82e6d2c914ab..af3f92beec04 100644
@@ -3269,8 +3269,7 @@ void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...)
 {
 	struct va_format vaf;
 	va_list args;
-	static DEFINE_RATELIMIT_STATE(nopage_rs, DEFAULT_RATELIMIT_INTERVAL,
-				      DEFAULT_RATELIMIT_BURST);
+	static DEFINE_RATELIMIT_STATE(nopage_rs, HZ * 60, 1);
 
 	if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs))
 		return;

This will warn once and only once after a kernel is booted. This condition may happen repeatedly over a long period of time with significant time span between two such events and it can be useful to know if this is happening repeatedly. There might be better ways to throttle the rate of warnings, something like warn once and then suppress warnings for the next 15 minutes (or pick any other time frame). If this condition happens again later, there will be another warning.

--
Khalid

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

While this is all true I am not sure we care all that much. The failure mode is basically not using an optimization. This is not something we warn normally about. Even the performance degradation is a theoretical concern which nobody has backed by real life numbers AFAIR.

If we want to make it more sophisticated I would expect some numbers to back such a change.

Michal Hocko
SUSE Labs

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

That is reasonable enough.

--
Khalid

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Dohh, forgot to git add the follow up fix on top of Johannes' original diff so it didn't make it into the finall commit. Could you fold this into the patch Andrew, please?

Sorry about that.
---

index 3f85084cb8bb..9a745e2a6f9a 100644
@@ -62,7 +62,7 @@ void * __meminit vmemmap_alloc_block(unsigned long size, int node)
 			return page_address(page);
 
 		if (!warned) {
-			warn_alloc(gfp_mask, NULL, "vmemmap alloc failure: order:%u", order);
+			warn_alloc(gfp_mask & ~__GFP_NOWARN, NULL, "vmemmap alloc failure: order:%u", order);
 			warned = true;
 		}
 		return NULL;

Michal Hocko
SUSE Labs

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Here is an updated rseq patchset taking into account the feedback received at kernel summit and afterwards.

Use-cases explanation and benchmarks can be found in patch 01 "Restartable sequences system call".

This is still submitted as RFC. I'm keeping a linux-rseq tree with this patchset at:

https://git.kernel.org/pub/scm/linux/kernel/git/rseq/linux-rseq.git/tag/?h=v4.14-rc8-rseq-20171106.1

If everyone is happy with it, I plan to submit this officially when the merge window opens.

Thanks,

Mathieu

Boqun Feng (2):
Restartable sequences: powerpc architecture support Restartable sequences: Wire up powerpc system call

Mathieu Desnoyers (12):
Restartable sequences system call (v10)
Restartable sequences: ARM 32 architecture support Restartable sequences: wire up ARM 32 system call Restartable sequences: x86 32/64 architecture support Restartable sequences: wire up x86 32/64 system call Provide cpu_opv system call (v2)
cpu_opv: Wire up x86 32/64 system call
cpu_opv: Wire up powerpc system call
cpu_opv: Wire up ARM32 system call
cpu_opv: Implement selftests (v2)
Restartable sequences: Provide self-tests (v2) Restartable sequences selftests: arm: workaround gcc asm size guess

 MAINTAINERS                                        |   20 +
 arch/Kconfig                                       |    7 +
 arch/arm/Kconfig                                   |    1 +
 arch/arm/kernel/signal.c                           |    7 +
 arch/arm/tools/syscall.tbl                         |    2 +
 arch/powerpc/Kconfig                               |    1 +
 arch/powerpc/include/asm/systbl.h                  |    2 +
 arch/powerpc/include/asm/unistd.h                  |    2 +-
 arch/powerpc/include/uapi/asm/unistd.h             |    2 +
 arch/powerpc/kernel/signal.c                       |    3 +
 arch/x86/Kconfig                                   |    1 +
 arch/x86/entry/common.c                            |    1 +
 arch/x86/entry/syscalls/syscall_32.tbl             |    2 +
 arch/x86/entry/syscalls/syscall_64.tbl             |    2 +
 arch/x86/kernel/signal.c                           |    6 +
 fs/exec.c                                          |    1 +
 include/linux/sched.h                              |   89 ++
 include/trace/events/rseq.h                        |   60 +
 include/uapi/linux/cpu_opv.h                       |  117 ++
 include/uapi/linux/rseq.h                          |  138 +++
 init/Kconfig                                       |   28 +
 kernel/Makefile                                    |    2 +
 kernel/cpu_opv.c                                   |  952 +++++++++++++++
 kernel/fork.c                                      |    2 +
 kernel/rseq.c                                      |  329 +++++
 kernel/sched/core.c                                |   41 +
 kernel/sched/sched.h                               |    2 +
 kernel/sys_ni.c                                    |    4 +
 tools/testing/selftests/Makefile                   |    2 +
 tools/testing/selftests/cpu-opv/.gitignore         |    1 +
 tools/testing/selftests/cpu-opv/Makefile           |   15 +
 .../testing/selftests/cpu-opv/basic_cpu_opv_test.c | 1157 ++++++++++++++++++
 tools/testing/selftests/cpu-opv/cpu-op.c           |  348 ++++++
 tools/testing/selftests/cpu-opv/cpu-op.h           |   68 ++
 tools/testing/selftests/rseq/.gitignore            |    4 +
 tools/testing/selftests/rseq/Makefile              |   22 +
 .../testing/selftests/rseq/basic_percpu_ops_test.c |  333 +++++
 tools/testing/selftests/rseq/basic_test.c          |   55 +
 tools/testing/selftests/rseq/param_test.c          | 1285 ++++++++++++++++++++
 tools/testing/selftests/rseq/rseq-arm.h            |  568 +++++++++
 tools/testing/selftests/rseq/rseq-ppc.h            |  567 +++++++++
 tools/testing/selftests/rseq/rseq-x86.h            |  898 ++++++++++++++
 tools/testing/selftests/rseq/rseq.c                |  116 ++
 tools/testing/selftests/rseq/rseq.h                |  154 +++
 tools/testing/selftests/rseq/run_param_test.sh     |  124 ++
 45 files changed, 7540 insertions(+), 1 deletion(-)
 create mode 100644 include/trace/events/rseq.h
 create mode 100644 include/uapi/linux/cpu_opv.h
 create mode 100644 include/uapi/linux/rseq.h
 create mode 100644 kernel/cpu_opv.c
 create mode 100644 kernel/rseq.c
 create mode 100644 tools/testing/selftests/cpu-opv/.gitignore
 create mode 100644 tools/testing/selftests/cpu-opv/Makefile
 create mode 100644 tools/testing/selftests/cpu-opv/basic_cpu_opv_test.c
 create mode 100644 tools/testing/selftests/cpu-opv/cpu-op.c
 create mode 100644 tools/testing/selftests/cpu-opv/cpu-op.h
 create mode 100644 tools/testing/selftests/rseq/.gitignore
 create mode 100644 tools/testing/selftests/rseq/Makefile
 create mode 100644 tools/testing/selftests/rseq/basic_percpu_ops_test.c
 create mode 100644 tools/testing/selftests/rseq/basic_test.c
 create mode 100644 tools/testing/selftests/rseq/param_test.c
 create mode 100644 tools/testing/selftests/rseq/rseq-arm.h
 create mode 100644 tools/testing/selftests/rseq/rseq-ppc.h
 create mode 100644 tools/testing/selftests/rseq/rseq-x86.h
 create mode 100644 tools/testing/selftests/rseq/rseq.c
 create mode 100644 tools/testing/selftests/rseq/rseq.h
 create mode 100755 tools/testing/selftests/rseq/run_param_test.sh

2.11.0

Expose a new system call allowing each thread to register one userspace memory area to be used as an ABI between kernel and user-space for two
purposes: user-space restartable sequences and quick access to read the current CPU number value from user-space.

* Restartable sequences (per-cpu atomics)

Restartables sequences allow user-space to perform update operations on per-cpu data without requiring heavy-weight atomic operations.

The restartable critical sections (percpu atomics) work has been started by Paul Turner and Andrew Hunter. It lets the kernel handle restart of critical sections. [1] [2] The re-implementation proposed here brings a few simplifications to the ABI which facilitates porting to other architectures and speeds up the user-space fast path. A second system call, cpu_opv(), is proposed as fallback to deal with debugger single-stepping. cpu_opv() executes a sequence of operations on behalf of user-space with preemption disabled.

Here are benchmarks of various rseq use-cases.

Test hardware:

arm32: ARMv7 Processor rev 4 (v7l) "Cubietruck", 2-core
x86-64: Intel E5-2630 [email protected], 16-core, hyperthreading

The following benchmarks were all performed on a single thread.

* Per-CPU statistic counter increment

getcpu+atomic (ns/op) rseq (ns/op) speedup

arm32:                344.0                 31.4          11.0
x86-64:                15.3                  2.0           7.7

* LTTng-UST: write event 32-bit header, 32-bit payload into tracer per-cpu buffer

getcpu+atomic (ns/op) rseq (ns/op) speedup

arm32:               2502.0                 2250.0         1.1
x86-64:               117.4                   98.0         1.2

* liburcu percpu: lock-unlock pair, dereference, read/compare word

getcpu+atomic (ns/op) rseq (ns/op) speedup

arm32:                751.0                 128.5          5.8
x86-64:                53.4                  28.6          1.9

* jemalloc memory allocator adapted to use rseq

Using rseq with per-cpu memory pools in jemalloc at Facebook (based on rseq 2016 implementation):

The production workload response-time has 1-2% gain avg. latency, and the P99 overall latency drops by 2-3%.

* Reading the current CPU number

Speeding up reading the current CPU number on which the caller thread is running is done by keeping the current CPU number up do date within the cpu_id field of the memory area registered by the thread. This is done by making scheduler preemption set the TIF_NOTIFY_RESUME flag on the current thread. Upon return to user-space, a notify-resume handler updates the current CPU value within the registered user-space memory area. User-space can then read the current CPU number directly from memory.

Keeping the current cpu id in a memory area shared between kernel and user-space is an improvement over current mechanisms available to read the current CPU number, which has the following benefits over alternative approaches:

- 35x speedup on ARM vs system call through glibc
- 20x speedup on x86 compared to calling glibc, which calls vdso executing a "lsl" instruction,
- 14x speedup on x86 compared to inlined "lsl" instruction,
- Unlike vdso approaches, this cpu_id value can be read from an inline assembly, which makes it a useful building block for restartable sequences.
- The approach of reading the cpu id through memory mapping shared between kernel and user-space is portable (e.g. ARM), which is not the case for the lsl-based x86 vdso.

On x86, yet another possible approach would be to use the gs segment selector to point to user-space per-cpu data. This approach performs similarly to the cpu id cache, but it has two disadvantages: it is not portable, and it is incompatible with existing applications already using the gs segment selector for other purposes.

Benchmarking various approaches for reading the current CPU number:

ARMv7 Processor rev 4 (v7l)
Machine model: Cubietruck
- Baseline (empty loop): 8.4 ns
- Read CPU from rseq cpu_id: 16.7 ns
- Read CPU from rseq cpu_id (lazy register): 19.8 ns
- glibc 2.19-0ubuntu6.6 getcpu: 301.8 ns
- getcpu system call: 234.9 ns

x86-64 Intel(R) Xeon(R) CPU E5-2630 v3 @ 2.40GHz:

- Baseline (empty loop):                                    0.8 ns
- Read CPU from rseq cpu_id:                                0.8 ns
- Read CPU from rseq cpu_id (lazy register):                0.8 ns
- Read using gs segment selector:                           0.8 ns
- "lsl" inline assembly:                                   13.0 ns
- glibc 2.19-0ubuntu6 getcpu:                              16.6 ns
- getcpu system call:                                      53.9 ns

- Speed (benchmark taken on v8 of patchset)

Running 10 runs of hackbench -l 100000 seems to indicate, contrary to expectations, that enabling CONFIG_RSEQ slightly accelerates the scheduler:

Configuration: 2 sockets * 8-core Intel(R) Xeon(R) CPU E5-2630 v3 @ 2.40GHz (directly on hardware, hyperthreading disabled in BIOS, energy saving disabled in BIOS, turboboost disabled in BIOS, cpuidle.off=1 kernel parameter), with a Linux v4.6 defconfig+localyesconfig, restartable sequences series applied.

* CONFIG_RSEQ=n

avg.:      41.37 s
std.dev.:   0.36 s

* CONFIG_RSEQ=y

avg.:      40.46 s
std.dev.:   0.33 s

- Size

On x86-64, between CONFIG_RSEQ=n/y, the text size increase of vmlinux is 567 bytes, and the data size increase of vmlinux is 5696 bytes.

On x86-64, between CONFIG_CPU_OPV=n/y, the text size increase of vmlinux is 5576 bytes, and the data size increase of vmlinux is 6164 bytes.

[1] https://lwn.net/Articles/650333/
[2] http://www.linuxplumbersconf.org/2013/ocw/system/presentations/1695/original/LPC%20-%20PerCpu%20Atomics.pdf

---

Changes since v1:
- Return -1, errno=EINVAL if cpu_cache pointer is not aligned on sizeof(int32_t).
- Update man page to describe the pointer alignement requirements and update atomicity guarantees.
- Add MAINTAINERS file GETCPU_CACHE entry.
- Remove dynamic memory allocation: go back to having a single getcpu_cache entry per thread. Update documentation accordingly.
- Rebased on Linux 4.4.

Changes since v2:
- Introduce a "cmd" argument, along with an enum with GETCPU_CACHE_GET and GETCPU_CACHE_SET. Introduce a uapi header linux/getcpu_cache.h defining this enumeration.
- Split resume notifier architecture implementation from the system call wire up in the following arch-specific patches.
- Man pages updates.
- Handle 32-bit compat pointers.
- Simplify handling of getcpu_cache GETCPU_CACHE_SET compiler barrier:
set the current cpu cache pointer before doing the cache update, and set it back to NULL if the update fails. Setting it back to NULL on error ensures that no resume notifier will trigger a SIGSEGV if a migration happened concurrently.

Changes since v3:
- Fix __user annotations in compat code,
- Update memory ordering comments.
- Rebased on kernel v4.5-rc5.

Changes since v4:
- Inline getcpu_cache_fork, getcpu_cache_execve, and getcpu_cache_exit.
- Add new line between if() and switch() to improve readability.
- Added sched switch benchmarks (hackbench) and size overhead comparison to change log.

Changes since v5:
- Rename "getcpu_cache" to "thread_local_abi", allowing to extend this system call to cover future features such as restartable critical sections. Generalizing this system call ensures that we can add features similar to the cpu_id field within the same cache-line without having to track one pointer per feature within the task struct.
- Add a tlabi_nr parameter to the system call, thus allowing to extend the ABI beyond the initial 64-byte structure by registering structures with tlabi_nr greater than 0. The initial ABI structure is associated with tlabi_nr 0.
- Rebased on kernel v4.5.

Changes since v6:
- Integrate "restartable sequences" v2 patchset from Paul Turner.
- Add handling of single-stepping purely in user-space, with a fallback to locking after 2 rseq failures to ensure progress, and by exposing a __rseq_table section to debuggers so they know where to put breakpoints when dealing with rseq assembly blocks which can be aborted at any point.
- make the code and ABI generic: porting the kernel implementation simply requires to wire up the signal handler and return to user-space hooks, and allocate the syscall number.
- extend testing with a fully configurable test program. See param_spinlock_test -h for details.
- handling of rseq ENOSYS in user-space, also with a fallback to locking.
- modify Paul Turner's rseq ABI to only require a single TLS store on the user-space fast-path, removing the need to populate two additional registers. This is made possible by introducing struct rseq_cs into the ABI to describe a critical section start_ip, post_commit_ip, and abort_ip.
- Rebased on kernel v4.7-rc7.

Changes since v7:
- Documentation updates.
- Integrated powerpc architecture support.
- Compare rseq critical section start_ip, allows shriking the user-space fast-path code size.
- Added Peter Zijlstra, Paul E. McKenney and Boqun Feng as co-maintainers.
- Added do_rseq2 and do_rseq_memcpy to test program helper library.
- Code cleanup based on review from Peter Zijlstra, Andy Lutomirski and Boqun Feng.
- Rebase on kernel v4.8-rc2.

Changes since v8:
- clear rseq_cs even if non-nested. Speeds up user-space fast path by removing the final "rseq_cs=NULL" assignment.
- add enum rseq_flags: critical sections and threads can set migration, preemption and signal "disable" flags to inhibit rseq behavior.
- rseq_event_counter needs to be updated with a pre-increment: Otherwise misses an increment after exec (when TLS and in-kernel states are initially 0).

Changes since v9:
- Update changelog.
- Fold instrumentation patch.
- check abort-ip signature: Add a signature before the abort-ip landing address. This signature is also received as a new parameter to the rseq system call. The kernel uses it ensures that rseq cannot be used as an exploit vector to redirect execution to arbitrary code.
- Use rseq pointer for both register and unregister. This is more symmetric, and eventually allow supporting a linked list of rseq struct per thread if needed in the future.
- Unregistration of a rseq structure is now done with RSEQ_FLAG_UNREGISTER.
- Remove reference counting. Return "EBUSY" to the caller if rseq is already registered for the current thread. This simplifies implementation while still allowing user-space to perform lazy registration in multi-lib use-cases. (suggested by Ben Maurer)
- Clear rseq_cs upon unregister.
- Set cpu_id back to -1 on unregister, so if rseq user libraries follow an unregister, and they expect to lazily register rseq, they can do so.
- Document rseq_cs clear requirement: JIT should reset the rseq_cs pointer before reclaiming memory of rseq_cs structure.
- Introduce rseq_len syscall parameter, rseq_cs version field:
Allow keeping track of the registered rseq struct length, for future extensions. Add rseq_cs version as first field. Will allow future extensions.
- Use offset and unsigned arithmetic to save a branch: Save a conditional branch when comparing instruction pointer against a rseq_cs descriptor's address range by having post_commit_ip as an offset from start_ip, and using unsigned integer comparison. Suggested by Ben Maurer.
- Remove event counter from ABI. Suggested by Andy Lutomirski.
- Add INIT_ONSTACK macro: Introduce the RSEQ_FIELD_u32_u64_INIT_ONSTACK() macros to ensure that users correctly initialize the upper bits of RSEQ_FIELD_u32_u64() on their stack to 0 on 32-bit architectures.
- Select MEMBARRIER: Allows user-space rseq fast-paths to use the value of cpu_id field (inherently required by the rseq algorithm) to figure out whether membarrier can be expected to be available. This effectively allows user-space fast-paths to remove extra comparisons and branch testing whether membarrier is enabled, and thus whether a full barrier is required (e.g. in userspace RCU implementation after rcu_read_lock/before rcu_read_unlock).
- Expose cpu_id_start field: Checking whether the (cpu_id < 0) in the C preparation part of the rseq fast-path brings significant overhead at least on arm32. We can remove this extra comparison by exposing two distinct cpu_id fields in the rseq TLS:

The field cpu_id_start always contain a *possible* cpu number, although it may not be the current one if, for instance, rseq is not initialized for the current thread. cpu_id_start is meant to be used in the C code for the pointer chasing to figure out which per-cpu data structure should be passed to the rseq asm sequence.

The field cpu_id values -1 means rseq is not initialized, and -2 means initialization failed. That field is used in the rseq asm sequence to confirm that the cpu_id_start value was indeed the current cpu number. It also ends up confirming that rseq is initialized for the current thread, because values -1 and -2 will never match the cpu_id_start possible cpu number values.

This allows checking the current CPU number and rseq initialization state with a single comparison on the fast-path.

Man page associated:

RSEQ(2) Linux Programmer's Manual RSEQ(2)

NAME
rseq - Restartable sequences and cpu number cache

SYNOPSIS

       #include <linux/rseq.h>

       int rseq(struct rseq * rseq, uint32_t rseq_len, int flags, uint32_t sig);

DESCRIPTION
The rseq() ABI accelerates user-space operations on per-cpu data by defining a shared data structure ABI between each user- space thread and the kernel.

It allows user-space to perform update operations on per-cpu data without requiring heavy-weight atomic operations.

Restartable sequences are atomic with respect to preemption (making it atomic with respect to other threads running on the same CPU), as well as signal delivery (user-space execution contexts nested over the same thread).

It is suited for update operations on per-cpu data.

It can be used on data structures shared between threads within a process, and on data structures shared between threads across different processes.

Some examples of operations that can be accelerated or improved by this ABI:

· Memory allocator per-cpu free-lists,

· Querying the current CPU number,

· Incrementing per-CPU counters,

· Modifying data protected by per-CPU spinlocks,

· Inserting/removing elements in per-CPU linked-lists,

· Writing/reading per-CPU ring buffers content.

· Accurately reading performance monitoring unit counters with
respect to thread migration.

The rseq argument is a pointer to the thread-local rseq struc‐ ture to be shared between kernel and user-space. A NULL rseq value unregisters the current thread rseq structure.

The layout of struct rseq is as follows:

Structure alignment
This structure is aligned on multiples of 32 bytes.

Structure size
This structure is extensible. Its size is passed as parameter to the rseq system call.

Fields

cpu_id_start
Optimistic cache of the CPU number on which the current thread is running. Its value is guaranteed to always be a possible CPU number, even when rseq is not initial‐ ized. The value it contains should always be confirmed by reading the cpu_id field.

cpu_id
Cache of the CPU number on which the current thread is running. -1 if uninitialized.

rseq_cs
The rseq_cs field is a pointer to a struct rseq_cs. Is is NULL when no rseq assembly block critical section is active for the current thread. Setting it to point to a critical section descriptor (struct rseq_cs) marks the beginning of the critical section.

flags
Flags indicating the restart behavior for the current thread. This is mainly used for debugging purposes. Can be either:

· RSEQ_CS_FLAG_NO_RESTART_ON_PREEMPT

· RSEQ_CS_FLAG_NO_RESTART_ON_SIGNAL

· RSEQ_CS_FLAG_NO_RESTART_ON_MIGRATE

The layout of struct rseq_cs version 0 is as follows:

Structure alignment
This structure is aligned on multiples of 32 bytes.

Structure size
This structure has a fixed size of 32 bytes.

Fields

version
Version of this structure.

flags
Flags indicating the restart behavior of this structure. Can be either:

· RSEQ_CS_FLAG_NO_RESTART_ON_PREEMPT

· RSEQ_CS_FLAG_NO_RESTART_ON_SIGNAL

· RSEQ_CS_FLAG_NO_RESTART_ON_MIGRATE

start_ip
Instruction pointer address of the first instruction of the sequence of consecutive assembly instructions.

post_commit_offset
Offset (from start_ip address) of the address after the last instruction of the sequence of consecutive assembly instructions.

abort_ip
Instruction pointer address where to move the execution flow in case of abort of the sequence of consecutive assembly instructions.

The rseq_len argument is the size of the struct rseq to regis‐ ter.

The flags argument is 0 for registration, and RSEQ_FLAG_UNREG‐ ISTER for unregistration.

The sig argument is the 32-bit signature to be expected before the abort handler code.

A single library per process should keep the rseq structure in a thread-local storage variable. The cpu_id field should be initialized to -1, and the cpu_id_start field should be ini‐ tialized to a possible CPU value (typically 0).

Each thread is responsible for registering and unregistering its rseq structure. No more than one rseq structure address can be registered per thread at a given time.

In a typical usage scenario, the thread registering the rseq structure will be performing loads and stores from/to that structure. It is however also allowed to read that structure from other threads. The rseq field updates performed by the kernel provide relaxed atomicity semantics, which guarantee that other threads performing relaxed atomic reads of the cpu number cache will always observe a consistent value.

RETURN VALUE
A return value of 0 indicates success. On error, -1 is returned, and errno is set appropriately.

ERRORS
EINVAL Either flags contains an invalid value, or rseq contains
an address which is not appropriately aligned, or rseq_len contains a size that does not match the size received on registration.

ENOSYS The rseq() system call is not implemented by this ker‐
nel.

EFAULT rseq is an invalid address.

EBUSY Restartable sequence is already registered for this
thread.

EPERM The sig argument on unregistration does not match the
signature received on registration.

VERSIONS
The rseq() system call was added in Linux 4.X (TODO).

CONFORMING TO
rseq() is Linux-specific.

SEE ALSO

       sched_getcpu(3)

Linux 2017-11-06 RSEQ(2)

---
 MAINTAINERS                 |  11 ++
 arch/Kconfig                |   7 +
 fs/exec.c                   |   1 +
 include/linux/sched.h       |  89 ++++++++++++
 include/trace/events/rseq.h |  60 ++++++++
 include/uapi/linux/rseq.h   | 138 +++++++++++++++++++
 init/Kconfig                |  14 ++
 kernel/Makefile             |   1 +
 kernel/fork.c               |   2 +
 kernel/rseq.c               | 329 ++++++++++++++++++++++++++++++++++++++++++++
 kernel/sched/core.c         |   4 +
 kernel/sys_ni.c             |   3 +
 12 files changed, 659 insertions(+)
 create mode 100644 include/trace/events/rseq.h
 create mode 100644 include/uapi/linux/rseq.h
 create mode 100644 kernel/rseq.c

index 2f4e462aa4a2..353366928ae8 100644
@@ -11484,6 +11484,17 @@ F:	include/dt-bindings/reset/
 F:	include/linux/reset.h
 F:	include/linux/reset-controller.h
 
+RESTARTABLE SEQUENCES SUPPORT
+M:	Mathieu Desnoyers <[email protected]>
+M:	Peter Zijlstra <[email protected]>
+M:	"Paul E. McKenney" <[email protected]>
+M:	Boqun Feng <[email protected]>
+L:	[email protected]
+S:	Supported
+F:	kernel/rseq.c
+F:	include/uapi/linux/rseq.h
+F:	include/trace/events/rseq.h
+
 RFKILL
 M:	Johannes Berg <[email protected]>
 L:	[email protected]

index 057370a0ac4e..b5e7f977fc29 100644
@@ -258,6 +258,13 @@ config HAVE_REGS_AND_STACK_ACCESS_API
 	  declared in asm/ptrace.h
 	  For example the kprobes-based event tracer needs this API.
 
+config HAVE_RSEQ
+	bool
+	depends on HAVE_REGS_AND_STACK_ACCESS_API
+	help
+	  This symbol should be selected by an architecture if it
+	  supports an implementation of restartable sequences.
+
 config HAVE_CLK
 	bool
 	help

index 3e14ba25f678..3faf8ff0fc6d 100644
@@ -1803,6 +1803,7 @@ static int do_execveat_common(int fd, struct filename *filename,
 	current->fs->in_exec = 0;
 	current->in_execve = 0;
 	membarrier_execve(current);
+	rseq_execve(current);
 	acct_update_integrals(current);
 	task_numa_free(current);
 	free_bprm(bprm);

index 3c1faaa2af4a..cbedfb91b40a 100644
@@ -1400,6 +1400,20 @@ config MEMBARRIER
 
 	  If unsure, say Y.
 
+config RSEQ
+	bool "Enable rseq() system call" if EXPERT
+	default y
+	depends on HAVE_RSEQ
+	select MEMBARRIER
+	help
+	  Enable the restartable sequences system call. It provides a
+	  user-space cache for the current CPU number value, which
+	  speeds up getting the current CPU number from user-space,
+	  as well as an ABI to speed up user-space operations on
+	  per-CPU data.
+
+	  If unsure, say Y.
+
 config EMBEDDED
 	bool "Embedded system"
 	option allnoconfig_y

index 172d151d429c..3574669dafd9 100644
@@ -112,6 +112,7 @@ obj-$(CONFIG_CONTEXT_TRACKING) += context_tracking.o
 obj-$(CONFIG_TORTURE_TEST) += torture.o
 
 obj-$(CONFIG_HAS_IOMEM) += memremap.o
+obj-$(CONFIG_RSEQ) += rseq.o
 
 $(obj)/configs.o: $(obj)/config_data.h
 

index 07cc743698d3..1f3c25e28742 100644
@@ -1862,6 +1862,8 @@ static __latent_entropy struct task_struct *copy_process(
 	 */
 	copy_seccomp(p);
 
+	rseq_fork(p, clone_flags);
+
 	/*
 	 * Process group and session signals need to be delivered to just the
 	 * parent before the fork or both the parent and the child after the

index d17c5da523a0..6bba05f47e51 100644
@@ -1179,6 +1179,8 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
 	WARN_ON_ONCE(!cpu_online(new_cpu));
 #endif
 
+	rseq_migrate(p);
+
 	trace_sched_migrate_task(p, new_cpu);
 
 	if (task_cpu(p) != new_cpu) {

@@ -2581,6 +2583,7 @@ prepare_task_switch(struct rq *rq, struct task_struct *prev,
 {
 	sched_info_switch(rq, prev, next);
 	perf_event_task_sched_out(prev, next);
+	rseq_sched_out(prev);
 	fire_sched_out_preempt_notifiers(prev, next);
 	prepare_lock_switch(rq, next);
 	prepare_arch_switch(next);

@@ -3341,6 +3344,7 @@ static void __sched notrace __schedule(bool preempt)
 	clear_preempt_need_resched();
 
 	if (likely(prev != next)) {
+		rseq_preempt(prev);
 		rq->nr_switches++;
 		rq->curr = next;
 		/*

index b5189762d275..bfa1ee1bf669 100644
@@ -259,3 +259,6 @@ cond_syscall(sys_membarrier);
 cond_syscall(sys_pkey_mprotect);
 cond_syscall(sys_pkey_alloc);
 cond_syscall(sys_pkey_free);
+
+/* restartable sequence */
+cond_syscall(sys_rseq);

2.11.0

On Mon, Nov 06, 2017 at 03:56:31PM -0500, Mathieu Desnoyers wrote:
[...]

[...]

[...]

I guess you mean "cpu_id_start".

Good point. v11 will include that fix. Meanwhile, it is available
at https://git.kernel.org/pub/scm/linux/kernel/git/rseq/linux-rseq.git/log/?h=rseq/dev

Thanks,

Mathieu

Mathieu Desnoyers
EfficiOS Inc.
http://www.efficios.com

Call the rseq_handle_notify_resume() function on return to userspace if TIF_NOTIFY_RESUME thread flag is set.

Increment the event counter and perform fixup on the pre-signal frame when a signal is delivered on top of a restartable sequence critical section.

---
 arch/arm/Kconfig         | 1 +
 arch/arm/kernel/signal.c | 7 +++++++
 2 files changed, 8 insertions(+)

index d1346a160760..1469f3f39475 100644
@@ -87,6 +87,7 @@ config ARM
 	select HAVE_PERF_USER_STACK_DUMP
 	select HAVE_RCU_TABLE_FREE if (SMP && ARM_LPAE)
 	select HAVE_REGS_AND_STACK_ACCESS_API
+	select HAVE_RSEQ
 	select HAVE_SYSCALL_TRACEPOINTS
 	select HAVE_UID16
 	select HAVE_VIRT_CPU_ACCOUNTING_GEN

index b67ae12503f3..cc3260f475b0 100644
@@ -518,6 +518,12 @@ static void handle_signal(struct ksignal *ksig, struct pt_regs *regs)
 	int ret;
 
 	/*
+	 * Increment event counter and perform fixup for the pre-signal
+	 * frame.
+	 */
+	rseq_signal_deliver(regs);
+
+	/*
 	 * Set up the stack frame
 	 */
 	if (ksig->ka.sa.sa_flags & SA_SIGINFO)

@@ -637,6 +643,7 @@ do_work_pending(struct pt_regs *regs, unsigned int thread_flags, int syscall)
 			} else {
 				clear_thread_flag(TIF_NOTIFY_RESUME);
 				tracehook_notify_resume(regs);
+				rseq_handle_notify_resume(regs);
 			}
 		}
 		local_irq_disable();

2.11.0

Wire up the rseq system call on 32-bit ARM.

This provides an ABI improving the speed of a user-space getcpu operation on ARM by skipping the getcpu system call on the fast path, as well as improving the speed of user-space operations on per-cpu data compared to using load-linked/store-conditional.

---
 arch/arm/tools/syscall.tbl | 1 +
 1 file changed, 1 insertion(+)

index 0bb0e9c6376c..fbc74b5fa3ed 100644
@@ -412,3 +412,4 @@
 395	common	pkey_alloc		sys_pkey_alloc
 396	common	pkey_free		sys_pkey_free
 397	common	statx			sys_statx
+398	common	rseq			sys_rseq

2.11.0

Call the rseq_handle_notify_resume() function on return to userspace if TIF_NOTIFY_RESUME thread flag is set.

Increment the event counter and perform fixup on the pre-signal frame when a signal is delivered on top of a restartable sequence critical section.

---
 arch/x86/Kconfig         | 1 +
 arch/x86/entry/common.c  | 1 +
 arch/x86/kernel/signal.c | 6 ++++++
 3 files changed, 8 insertions(+)

index 2fdb23313dd5..01f78c1d40b5 100644
@@ -173,6 +173,7 @@ config X86
 	select HAVE_REGS_AND_STACK_ACCESS_API
 	select HAVE_RELIABLE_STACKTRACE		if X86_64 && FRAME_POINTER_UNWINDER && STACK_VALIDATION
 	select HAVE_STACK_VALIDATION		if X86_64
+	select HAVE_RSEQ
 	select HAVE_SYSCALL_TRACEPOINTS
 	select HAVE_UNSTABLE_SCHED_CLOCK
 	select HAVE_USER_RETURN_NOTIFIER

index 03505ffbe1b6..4c717bdd1139 100644
@@ -160,6 +160,7 @@ static void exit_to_usermode_loop(struct pt_regs *regs, u32 cached_flags)
 		if (cached_flags & _TIF_NOTIFY_RESUME) {
 			clear_thread_flag(TIF_NOTIFY_RESUME);
 			tracehook_notify_resume(regs);
+			rseq_handle_notify_resume(regs);
 		}
 
 		if (cached_flags & _TIF_USER_RETURN_NOTIFY)

index b9e00e8f1c9b..991017d26d00 100644
@@ -687,6 +687,12 @@ setup_rt_frame(struct ksignal *ksig, struct pt_regs *regs)
 	sigset_t *set = sigmask_to_save();
 	compat_sigset_t *cset = (compat_sigset_t *) set;
 
+	/*
+	 * Increment event counter and perform fixup for the pre-signal
+	 * frame.
+	 */
+	rseq_signal_deliver(regs);
+
 	/* Set up the stack frame */
 	if (is_ia32_frame(ksig)) {
 		if (ksig->ka.sa.sa_flags & SA_SIGINFO)

2.11.0

Wire up the rseq system call on x86 32/64.

This provides an ABI improving the speed of a user-space getcpu operation on x86 by removing the need to perform a function call, "lsl" instruction, or system call on the fast path, as well as improving the speed of user-space operations on per-cpu data.

---
 arch/x86/entry/syscalls/syscall_32.tbl | 1 +
 arch/x86/entry/syscalls/syscall_64.tbl | 1 +
 2 files changed, 2 insertions(+)

index 448ac2161112..ba43ee75e425 100644
@@ -391,3 +391,4 @@
 382	i386	pkey_free		sys_pkey_free
 383	i386	statx			sys_statx
 384	i386	arch_prctl		sys_arch_prctl			compat_sys_arch_prctl
+385	i386	rseq			sys_rseq

index 5aef183e2f85..3ad03495bbb9 100644
@@ -339,6 +339,7 @@
 330	common	pkey_alloc		sys_pkey_alloc
 331	common	pkey_free		sys_pkey_free
 332	common	statx			sys_statx
+333	common	rseq			sys_rseq
 
 #
 # x32-specific system call numbers start at 512 to avoid cache impact

2.11.0

From: Boqun Feng <[email protected]>

Call the rseq_handle_notify_resume() function on return to userspace if TIF_NOTIFY_RESUME thread flag is set.

Increment the event counter and perform fixup on the pre-signal when a signal is delivered on top of a restartable sequence critical section.

---
 arch/powerpc/Kconfig         | 1 +
 arch/powerpc/kernel/signal.c | 3 +++
 2 files changed, 4 insertions(+)

index cb782ac1c35d..41d1dae3b1b5 100644
@@ -219,6 +219,7 @@ config PPC
 	select HAVE_SYSCALL_TRACEPOINTS
 	select HAVE_VIRT_CPU_ACCOUNTING
 	select HAVE_IRQ_TIME_ACCOUNTING
+	select HAVE_RSEQ
 	select IRQ_DOMAIN
 	select IRQ_FORCED_THREADING
 	select MODULES_USE_ELF_RELA

index e9436c5e1e09..17a994b801b1 100644
@@ -133,6 +133,8 @@ static void do_signal(struct task_struct *tsk)
 	/* Re-enable the breakpoints for the signal stack */
 	thread_change_pc(tsk, tsk->thread.regs);
 
+	rseq_signal_deliver(tsk->thread.regs);
+
 	if (is32) {
         	if (ksig.ka.sa.sa_flags & SA_SIGINFO)
 			ret = handle_rt_signal32(&ksig, oldset, tsk);

@@ -161,6 +163,7 @@ void do_notify_resume(struct pt_regs *regs, unsigned long thread_info_flags)
 	if (thread_info_flags & _TIF_NOTIFY_RESUME) {
 		clear_thread_flag(TIF_NOTIFY_RESUME);
 		tracehook_notify_resume(regs);
+		rseq_handle_notify_resume(regs);
 	}
 
 	if (thread_info_flags & _TIF_PATCH_PENDING)

2.11.0

From: Boqun Feng <[email protected]>

Wire up the rseq system call on powerpc.

This provides an ABI improving the speed of a user-space getcpu operation on powerpc by skipping the getcpu system call on the fast path, as well as improving the speed of user-space operations on per-cpu data compared to using load-reservation/store-conditional atomics.

---
 arch/powerpc/include/asm/systbl.h      | 1 +
 arch/powerpc/include/asm/unistd.h      | 2 +-
 arch/powerpc/include/uapi/asm/unistd.h | 1 +
 3 files changed, 3 insertions(+), 1 deletion(-)

index 449912f057f6..964321a5799c 100644
@@ -389,3 +389,4 @@ COMPAT_SYS_SPU(preadv2)
 COMPAT_SYS_SPU(pwritev2)
 SYSCALL(kexec_file_load)
 SYSCALL(statx)
+SYSCALL(rseq)

index 9ba11dbcaca9..e76bd5601ea4 100644
@@ -12,7 +12,7 @@
 #include <uapi/asm/unistd.h>
 
 
-#define NR_syscalls		384
+#define NR_syscalls		385
 
 #define __NR__exit __NR_exit
 

index df8684f31919..b1980fcd56d5 100644
@@ -395,5 +395,6 @@
 #define __NR_pwritev2		381
 #define __NR_kexec_file_load	382
 #define __NR_statx		383
+#define __NR_rseq		384
 
 #endif /* _UAPI_ASM_POWERPC_UNISTD_H_ */

2.11.0

This new cpu_opv system call executes a vector of operations on behalf of user-space on a specific CPU with preemption disabled. It is inspired from readv() and writev() system calls which take a "struct iovec" array as argument.

The operations available are: comparison, memcpy, add, or, and, xor, left shift, right shift, and mb. The system call receives a CPU number from user-space as argument, which is the CPU on which those operations need to be performed. All preparation steps such as loading pointers, and applying offsets to arrays, need to be performed by user-space before invoking the system call. The "comparison" operation can be used to check that the data used in the preparation step did not change between preparation of system call inputs and operation execution within the preempt-off critical section.

The reason why we require all pointer offsets to be calculated by user-space beforehand is because we need to use get_user_pages_fast() to first pin all pages touched by each operation. This takes care of faulting-in the pages. Then, preemption is disabled, and the operations are performed atomically with respect to other thread execution on that CPU, without generating any page fault.

A maximum limit of 16 operations per cpu_opv syscall invocation is enforced, so user-space cannot generate a too long preempt-off critical section. Each operation is also limited a length of PAGE_SIZE bytes, meaning that an operation can touch a maximum of 4 pages (memcpy: 2 pages for source, 2 pages for destination if addresses are not aligned on page boundaries). Moreover, a total limit of 4216 bytes is applied to operation lengths.

If the thread is not running on the requested CPU, a new push_task_to_cpu() is invoked to migrate the task to the requested CPU. If the requested CPU is not part of the cpus allowed mask of the thread, the system call fails with EINVAL. After the migration has been performed, preemption is disabled, and the current CPU number is checked again and compared to the requested CPU number. If it still differs, it means the scheduler migrated us away from that CPU. Return EAGAIN to user-space in that case, and let user-space retry (either requesting the same CPU number, or a different one, depending on the user-space algorithm constraints).

---

Changes since v1:
- handle CPU hotplug,
- cleanup implementation using function pointers: We can use function pointers to implement the operations rather than duplicating all the user-access code.
- refuse device pages: Performing cpu_opv operations on io map'd pages with preemption disabled could generate long preempt-off critical sections, which leads to unwanted scheduler latency. Return EFAULT if a device page is received as parameter
- restrict op vector to 4216 bytes length sum: Restrict the operation vector to length sum of:
- 4096 bytes (typical page size on most architectures, should be enough for a string, or structures)
- 15 * 8 bytes (typical operations on integers or pointers). The goal here is to keep the duration of preempt off critical section short, so we don't add significant scheduler latency.
- Add INIT_ONSTACK macro: Introduce the CPU_OP_FIELD_u32_u64_INIT_ONSTACK() macros to ensure that users correctly initialize the upper bits of CPU_OP_FIELD_u32_u64() on their stack to 0 on 32-bit architectures.
- Add CPU_MB_OP operation:
Use-cases with:
- two consecutive stores,
- a mempcy followed by a store,
require a memory barrier before the final store operation. A typical use-case is a store-release on the final store. Given that this is a slow path, just providing an explicit full barrier instruction should be sufficient.
- Add expect fault field:
The use-case of list_pop brings interesting challenges. With rseq, we can use rseq_cmpnev_storeoffp_load(), and therefore load a pointer, compare it against NULL, add an offset, and load the target "next" pointer from the object, all within a single req critical section.

Life is not so easy for cpu_opv in this use-case, mainly because we need to pin all pages we are going to touch in the preempt-off critical section beforehand. So we need to know the target object (in which we apply an offset to fetch the next pointer) when we pin pages before disabling preemption.

So the approach is to load the head pointer and compare it against NULL in user-space, before doing the cpu_opv syscall. User-space can then compute the address of the head->next field, *without loading it*.

The cpu_opv system call will first need to pin all pages associated with input data. This includes the page backing the head->next object, which may have been concurrently deallocated and unmapped. Therefore, in this case, getting -EFAULT when trying to pin those pages may
happen: it just means they have been concurrently unmapped. This is an expected situation, and should just return -EAGAIN to user-space, to user-space can distinguish between "should retry" type of situations and actual errors that should be handled with extreme prejudice to the program (e.g. abort()).

Therefore, add "expect_fault" fields along with op input address pointers, so user-space can identify whether a fault when getting a field should return EAGAIN rather than EFAULT.
- Add compiler barrier between operations: Adding a compiler barrier between store operations in a cpu_opv sequence can be useful when paired with membarrier system call.

An algorithm with a paired slow path and fast path can use sys_membarrier on the slow path to replace fast-path memory barriers by compiler barrier.

Adding an explicit compiler barrier between operations allows cpu_opv to be used as fallback for operations meant to match the membarrier system call.

---
 MAINTAINERS                  |   7 +
 include/uapi/linux/cpu_opv.h | 117 ++++++
 init/Kconfig                 |  14 +
 kernel/Makefile              |   1 +
 kernel/cpu_opv.c             | 952 +++++++++++++++++++++++++++++++++++++++++++
 kernel/sched/core.c          |  37 ++
 kernel/sched/sched.h         |   2 +
 kernel/sys_ni.c              |   1 +
 8 files changed, 1131 insertions(+)
 create mode 100644 include/uapi/linux/cpu_opv.h
 create mode 100644 kernel/cpu_opv.c

index 353366928ae8..6a428d6cf494 100644
@@ -3675,6 +3675,13 @@ B:	https://bugzilla.kernel.org
 F:	drivers/cpuidle/*
 F:	include/linux/cpuidle.h
 
+CPU NON-PREEMPTIBLE OPERATION VECTOR SUPPORT
+M:	Mathieu Desnoyers <[email protected]>
+L:	[email protected]
+S:	Supported
+F:	kernel/cpu_opv.c
+F:	include/uapi/linux/cpu_opv.h
+
 CRAMFS FILESYSTEM
 W:	http://sourceforge.net/projects/cramfs/
 S:	Orphan / Obsolete

index cbedfb91b40a..e4fbb5dd6a24 100644
@@ -1404,6 +1404,7 @@ config RSEQ
 	bool "Enable rseq() system call" if EXPERT
 	default y
 	depends on HAVE_RSEQ
+	select CPU_OPV
 	select MEMBARRIER
 	help
 	  Enable the restartable sequences system call. It provides a

@@ -1414,6 +1415,19 @@ config RSEQ
 
 	  If unsure, say Y.
 
+config CPU_OPV
+	bool "Enable cpu_opv() system call" if EXPERT
+	default y
+	help
+	  Enable the CPU preempt-off operation vector system call.
+	  It allows user-space to perform a sequence of operations on
+	  per-cpu data with preemption disabled. Useful as
+	  single-stepping fall-back for restartable sequences, and for
+	  performing more complex operations on per-cpu data that would
+	  not be otherwise possible to do with restartable sequences.
+
+	  If unsure, say Y.
+
 config EMBEDDED
 	bool "Embedded system"
 	option allnoconfig_y

index 3574669dafd9..cac8855196ff 100644
@@ -113,6 +113,7 @@ obj-$(CONFIG_TORTURE_TEST) += torture.o
 
 obj-$(CONFIG_HAS_IOMEM) += memremap.o
 obj-$(CONFIG_RSEQ) += rseq.o
+obj-$(CONFIG_CPU_OPV) += cpu_opv.o
 
 $(obj)/configs.o: $(obj)/config_data.h
 

index 6bba05f47e51..e547f93a46c2 100644
@@ -1052,6 +1052,43 @@ void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
 		set_curr_task(rq, p);
 }
 
+int push_task_to_cpu(struct task_struct *p, unsigned int dest_cpu)
+{
+	struct rq_flags rf;
+	struct rq *rq;
+	int ret = 0;
+
+	rq = task_rq_lock(p, &rf);
+	update_rq_clock(rq);
+
+	if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed)) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	if (task_cpu(p) == dest_cpu)
+		goto out;
+
+	if (task_running(rq, p) || p->state == TASK_WAKING) {
+		struct migration_arg arg = { p, dest_cpu };
+		/* Need help from migration thread: drop lock and wait. */
+		task_rq_unlock(rq, p, &rf);
+		stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
+		tlb_migrate_finish(p->mm);
+		return 0;
+	} else if (task_on_rq_queued(p)) {
+		/*
+		 * OK, since we're going to drop the lock immediately
+		 * afterwards anyway.
+		 */
+		rq = move_queued_task(rq, &rf, p, dest_cpu);
+	}
+out:
+	task_rq_unlock(rq, p, &rf);
+
+	return ret;
+}
+
 /*
  * Change a given task's CPU affinity. Migrate the thread to a
  * proper CPU and schedule it away if the CPU it's executing on

index 3b448ba82225..cab256c1720a 100644
@@ -1209,6 +1209,8 @@ static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
 #endif
 }
 
+int push_task_to_cpu(struct task_struct *p, unsigned int dest_cpu);
+
 /*
  * Tunables that become constants when CONFIG_SCHED_DEBUG is off:
  */

index bfa1ee1bf669..59e622296dc3 100644
@@ -262,3 +262,4 @@ cond_syscall(sys_pkey_free);
 
 /* restartable sequence */
 cond_syscall(sys_rseq);
+cond_syscall(sys_cpu_opv);

2.11.0

On Mon, Nov 06, 2017 at 03:56:38PM -0500, Mathieu Desnoyers wrote:
[...]

	struct cpu_opv_page_pinner {
		int nr_pinned;
		bool is_kmalloc;
		struct page **pinned_pages;
	};

[...]

Good catch !

How about the attached diff ? I'll fold it into the rseq/dev tree.

Thanks,

Mathieu

Mathieu Desnoyers
EfficiOS Inc.
http://www.efficios.com

index 09754bbe6a4f..3d8fd66416a0 100644
@@ -46,6 +46,12 @@ union op_fn_data {
 #endif
 };
 
+struct cpu_opv_pinned_pages {
+	struct page **pages;
+	size_t nr;
+	bool is_kmalloc;
+};
+
 typedef int (*op_fn_t)(union op_fn_data *data, uint64_t v, uint32_t len);
 
 static DEFINE_MUTEX(cpu_opv_offline_lock);

@@ -217,8 +223,7 @@ static int cpu_op_check_pages(struct page **pages,
 }
 
 static int cpu_op_pin_pages(unsigned long addr, unsigned long len,
-		struct page ***pinned_pages_ptr, size_t *nr_pinned,
-		int write)
+		struct cpu_opv_pinned_pages *pin_pages, int write)
 {
 	struct page *pages[2];
 	int ret, nr_pages;

@@ -227,15 +232,17 @@ static int cpu_op_pin_pages(unsigned long addr, unsigned long len,
 		return 0;
 	nr_pages = cpu_op_range_nr_pages(addr, len);
 	BUG_ON(nr_pages > 2);
-	if (*nr_pinned + nr_pages > NR_PINNED_PAGES_ON_STACK) {
+	if (!pin_pages->is_kmalloc && pin_pages->nr + nr_pages
+			> NR_PINNED_PAGES_ON_STACK) {
 		struct page **pinned_pages =
 			kzalloc(CPU_OP_VEC_LEN_MAX * CPU_OP_MAX_PAGES
 				* sizeof(struct page *), GFP_KERNEL);
 		if (!pinned_pages)
 			return -ENOMEM;
-		memcpy(pinned_pages, *pinned_pages_ptr,
-			*nr_pinned * sizeof(struct page *));
-		*pinned_pages_ptr = pinned_pages;
+		memcpy(pinned_pages, pin_pages->pages,
+			pin_pages->nr * sizeof(struct page *));
+		pin_pages->pages = pinned_pages;
+		pin_pages->is_kmalloc = true;
 	}
 again:
 	ret = get_user_pages_fast(addr, nr_pages, write, pages);

@@ -257,9 +264,9 @@ static int cpu_op_pin_pages(unsigned long addr, unsigned long len,
 	}
 	if (ret)
 		goto error;
-	(*pinned_pages_ptr)[(*nr_pinned)++] = pages[0];
+	pin_pages->pages[pin_pages->nr++] = pages[0];
 	if (nr_pages > 1)
-		(*pinned_pages_ptr)[(*nr_pinned)++] = pages[1];
+		pin_pages->pages[pin_pages->nr++] = pages[1];
 	return 0;
 
 error:

@@ -270,7 +277,7 @@ static int cpu_op_pin_pages(unsigned long addr, unsigned long len,
 }
 
 static int cpu_opv_pin_pages(struct cpu_op *cpuop, int cpuopcnt,
-		struct page ***pinned_pages_ptr, size_t *nr_pinned)
+		struct cpu_opv_pinned_pages *pin_pages)
 {
 	int ret, i;
 	bool expect_fault = false;

@@ -289,7 +296,7 @@ static int cpu_opv_pin_pages(struct cpu_op *cpuop, int cpuopcnt,
 				goto error;
 			ret = cpu_op_pin_pages(
 					(unsigned long)op->u.compare_op.a,
-					op->len, pinned_pages_ptr, nr_pinned, 0);
+					op->len, pin_pages, 0);
 			if (ret)
 				goto error;
 			ret = -EFAULT;

@@ -299,7 +306,7 @@ static int cpu_opv_pin_pages(struct cpu_op *cpuop, int cpuopcnt,
 				goto error;
 			ret = cpu_op_pin_pages(
 					(unsigned long)op->u.compare_op.b,
-					op->len, pinned_pages_ptr, nr_pinned, 0);
+					op->len, pin_pages, 0);
 			if (ret)
 				goto error;
 			break;

@@ -311,7 +318,7 @@ static int cpu_opv_pin_pages(struct cpu_op *cpuop, int cpuopcnt,
 				goto error;
 			ret = cpu_op_pin_pages(
 					(unsigned long)op->u.memcpy_op.dst,
-					op->len, pinned_pages_ptr, nr_pinned, 1);
+					op->len, pin_pages, 1);
 			if (ret)
 				goto error;
 			ret = -EFAULT;

@@ -321,7 +328,7 @@ static int cpu_opv_pin_pages(struct cpu_op *cpuop, int cpuopcnt,
 				goto error;
 			ret = cpu_op_pin_pages(
 					(unsigned long)op->u.memcpy_op.src,
-					op->len, pinned_pages_ptr, nr_pinned, 0);
+					op->len, pin_pages, 0);
 			if (ret)
 				goto error;
 			break;

@@ -333,7 +340,7 @@ static int cpu_opv_pin_pages(struct cpu_op *cpuop, int cpuopcnt,
 				goto error;
 			ret = cpu_op_pin_pages(
 					(unsigned long)op->u.arithmetic_op.p,
-					op->len, pinned_pages_ptr, nr_pinned, 1);
+					op->len, pin_pages, 1);
 			if (ret)
 				goto error;
 			break;

@@ -347,7 +354,7 @@ static int cpu_opv_pin_pages(struct cpu_op *cpuop, int cpuopcnt,
 				goto error;
 			ret = cpu_op_pin_pages(
 					(unsigned long)op->u.bitwise_op.p,
-					op->len, pinned_pages_ptr, nr_pinned, 1);
+					op->len, pin_pages, 1);
 			if (ret)
 				goto error;
 			break;

@@ -360,7 +367,7 @@ static int cpu_opv_pin_pages(struct cpu_op *cpuop, int cpuopcnt,
 				goto error;
 			ret = cpu_op_pin_pages(
 					(unsigned long)op->u.shift_op.p,
-					op->len, pinned_pages_ptr, nr_pinned, 1);
+					op->len, pin_pages, 1);
 			if (ret)
 				goto error;
 			break;

@@ -373,9 +380,9 @@ static int cpu_opv_pin_pages(struct cpu_op *cpuop, int cpuopcnt,
 	return 0;
 
 error:
-	for (i = 0; i < *nr_pinned; i++)
-		put_page((*pinned_pages_ptr)[i]);
-	*nr_pinned = 0;
+	for (i = 0; i < pin_pages->nr; i++)
+		put_page(pin_pages->pages[i]);
+	pin_pages->nr = 0;
 	/*
 	 * If faulting access is expected, return EAGAIN to user-space.
 	 * It allows user-space to distinguish between a fault caused by

@@ -923,9 +930,12 @@ SYSCALL_DEFINE4(cpu_opv, struct cpu_op __user *, ucpuopv, int, cpuopcnt,
 {
 	struct cpu_op cpuopv[CPU_OP_VEC_LEN_MAX];
 	struct page *pinned_pages_on_stack[NR_PINNED_PAGES_ON_STACK];
-	struct page **pinned_pages = pinned_pages_on_stack;
+	struct cpu_opv_pinned_pages pin_pages = {
+		.pages = pinned_pages_on_stack,
+		.nr = 0,
+		.is_kmalloc = false,
+	};
 	int ret, i;
-	size_t nr_pinned = 0;
 
 	if (unlikely(flags))
 		return -EINVAL;

@@ -938,15 +948,14 @@ SYSCALL_DEFINE4(cpu_opv, struct cpu_op __user *, ucpuopv, int, cpuopcnt,
 	ret = cpu_opv_check(cpuopv, cpuopcnt);
 	if (ret)
 		return ret;
-	ret = cpu_opv_pin_pages(cpuopv, cpuopcnt,
-				&pinned_pages, &nr_pinned);
+	ret = cpu_opv_pin_pages(cpuopv, cpuopcnt, &pin_pages);
 	if (ret)
 		goto end;
 	ret = do_cpu_opv(cpuopv, cpuopcnt, cpu);
-	for (i = 0; i < nr_pinned; i++)
-		put_page(pinned_pages[i]);
+	for (i = 0; i < pin_pages.nr; i++)
+		put_page(pin_pages.pages[i]);
 end:
-	if (pinned_pages != pinned_pages_on_stack)
-		kfree(pinned_pages);
+	if (pin_pages.is_kmalloc)
+		kfree(pin_pages.pages);
 	return ret;
 }

Looks good to me ;-)

Regards,
Boqun

---
 arch/x86/entry/syscalls/syscall_32.tbl | 1 +
 arch/x86/entry/syscalls/syscall_64.tbl | 1 +
 2 files changed, 2 insertions(+)

index ba43ee75e425..afc6988fb2c8 100644
@@ -392,3 +392,4 @@
 383	i386	statx			sys_statx
 384	i386	arch_prctl		sys_arch_prctl			compat_sys_arch_prctl
 385	i386	rseq			sys_rseq
+386	i386	cpu_opv			sys_cpu_opv

index 3ad03495bbb9..ab5d1f9f9396 100644
@@ -340,6 +340,7 @@
 331	common	pkey_free		sys_pkey_free
 332	common	statx			sys_statx
 333	common	rseq			sys_rseq
+334	common	cpu_opv			sys_cpu_opv
 
 #
 # x32-specific system call numbers start at 512 to avoid cache impact

2.11.0

---
 arch/powerpc/include/asm/systbl.h      | 1 +
 arch/powerpc/include/asm/unistd.h      | 2 +-
 arch/powerpc/include/uapi/asm/unistd.h | 1 +
 3 files changed, 3 insertions(+), 1 deletion(-)

index 964321a5799c..f9cdb896fbaa 100644
@@ -390,3 +390,4 @@ COMPAT_SYS_SPU(pwritev2)
 SYSCALL(kexec_file_load)
 SYSCALL(statx)
 SYSCALL(rseq)
+SYSCALL(cpu_opv)

index e76bd5601ea4..48f80f452e31 100644
@@ -12,7 +12,7 @@
 #include <uapi/asm/unistd.h>
 
 
-#define NR_syscalls		385
+#define NR_syscalls		386
 
 #define __NR__exit __NR_exit
 

index b1980fcd56d5..972a7d68c143 100644
@@ -396,5 +396,6 @@
 #define __NR_kexec_file_load	382
 #define __NR_statx		383
 #define __NR_rseq		384
+#define __NR_cpu_opv		385
 
 #endif /* _UAPI_ASM_POWERPC_UNISTD_H_ */

2.11.0

Whenever I try to make variables or function names more explicit, I can literally feel my consciousness (taking the form of an angry Peter Zijlstra) breathing down my neck asking me to make them shorter. So I guess this is where it becomes a question of taste.

I think the "rseq" syscall name is short, to the point, and should be mostly fine.

For "cpu_opv", it was just a short name that fit the bill until a better idea would come.

I'm open to suggestions. Any color preference ? ;-)

Thanks,

Mathieu

Mathieu Desnoyers
EfficiOS Inc.
http://www.efficios.com

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Specialist syscall is a bit different than a common function or variable though.

I'm not sure if it's really "to the point". I think kexec_file_load is better than kfload, for example :)

What can you do within 16 characters?

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

---
 arch/arm/tools/syscall.tbl | 1 +
 1 file changed, 1 insertion(+)

index fbc74b5fa3ed..213ccfc2c437 100644
@@ -413,3 +413,4 @@
 396	common	pkey_free		sys_pkey_free
 397	common	statx			sys_statx
 398	common	rseq			sys_rseq
+399	common	cpu_opv			sys_cpu_opv

2.11.0

---

Changes since v1:

- Expose similar library API as rseq: Expose library API closely matching the rseq APIs, following removal of the event counter from the rseq kernel API.

---
 MAINTAINERS                                        |    1 +
 tools/testing/selftests/Makefile                   |    1 +
 tools/testing/selftests/cpu-opv/.gitignore         |    1 +
 tools/testing/selftests/cpu-opv/Makefile           |   15 +
 .../testing/selftests/cpu-opv/basic_cpu_opv_test.c | 1154 ++++++++++++++++++++
 tools/testing/selftests/cpu-opv/cpu-op.c           |  348 ++++++
 tools/testing/selftests/cpu-opv/cpu-op.h           |   68 ++
 7 files changed, 1588 insertions(+)
 create mode 100644 tools/testing/selftests/cpu-opv/.gitignore
 create mode 100644 tools/testing/selftests/cpu-opv/Makefile
 create mode 100644 tools/testing/selftests/cpu-opv/basic_cpu_opv_test.c
 create mode 100644 tools/testing/selftests/cpu-opv/cpu-op.c
 create mode 100644 tools/testing/selftests/cpu-opv/cpu-op.h

index 6a428d6cf494..54e11f0569e0 100644
@@ -3681,6 +3681,7 @@ L:	[email protected]
 S:	Supported
 F:	kernel/cpu_opv.c
 F:	include/uapi/linux/cpu_opv.h
+F:	tools/testing/selftests/cpu-opv/
 
 CRAMFS FILESYSTEM
 W:	http://sourceforge.net/projects/cramfs/

index 3c9c0bbe7dbb..c66e5e67cfab 100644
@@ -4,6 +4,7 @@ TARGETS += breakpoints
 TARGETS += capabilities
 TARGETS += cpufreq
 TARGETS += cpu-hotplug
+TARGETS += cpu-opv
 TARGETS += efivarfs
 TARGETS += exec
 TARGETS += firmware

new file mode 100644
index 000000000000..c7186eb95cf5
@@ -0,0 +1 @@
+basic_cpu_opv_test

new file mode 100644
index 000000000000..afb02f301567
@@ -0,0 +1,15 @@
+CFLAGS += -O2 -Wall -g -I./ -I../../../../usr/include/ -L./ -Wl,-rpath=./
+
+TEST_GEN_PROGS = basic_cpu_opv_test libcpu-op.so
+
+ALL: $(TEST_GEN_PROGS)
+
+libcpu-op.so: cpu-op.c cpu-op.h
+	$(CC) $(CFLAGS) -shared -fPIC $< -o $@
+
+# Own recipe because we only want to build against 1st prerequisite, but
+# still track changes to header files.
+%: %.c libcpu-op.so cpu-op.h
+	$(CC) $(CFLAGS) $< -lcpu-op -o $@
+
+include ../lib.mk

2.11.0

Implements two basic tests of RSEQ functionality, and one more exhaustive parameterizable test.

The first, "basic_test" only asserts that RSEQ works moderately correctly. E.g. that the CPUID pointer works.

"basic_percpu_ops_test" is a slightly more "realistic" variant, implementing a few simple per-cpu operations and testing their correctness.

"param_test" is a parametrizable restartable sequences test. See the "--help" output for usage.

A run_param_test.sh script runs many variants of the parametrizable tests.

As part of those tests, a helper library "rseq" implements a user-space API around restartable sequences. It uses the cpu_opv system call as fallback when single-stepped by a debugger. It exposes the instruction pointer addresses where the rseq assembly blocks begin and end, as well as the associated abort instruction pointer, in the __rseq_table section. This section allows debuggers may know where to place breakpoints when single-stepping through assembly blocks which may be aborted at any point by the kernel.

The rseq library expose APIs that present the fast-path operations. The new from userspace is, e.g. for a counter increment:

    cpu = rseq_cpu_start();
    ret = rseq_addv(&data->c[cpu].count, 1, cpu);
    if (likely(!ret))
        return 0;        /* Success. */
    do {
        cpu = rseq_current_cpu();
        ret = cpu_op_addv(&data->c[cpu].count, 1, cpu);
        if (likely(!ret))
            return 0;    /* Success. */
    } while (ret > 0 || errno == EAGAIN);
    perror("cpu_op_addv");
    return -1;           /* Unexpected error. */

PowerPC tests have been implemented by Boqun Feng.

---

Changes since v1:
- Provide abort-ip signature: The abort-ip signature is located just before the abort-ip target. It is currently hardcoded, but a user-space application could use the __rseq_table to iterate on all abort-ip targets and use a random value as signature if needed in the future.
- Add rseq_prepare_unload(): Libraries and JIT code using rseq critical sections need to issue rseq_prepare_unload() on each thread at least once before reclaim of struct rseq_cs.
- Use initial-exec TLS model, non-weak symbol: The initial-exec model is signal-safe, whereas the global-dynamic model is not. Remove the "weak" symbol attribute from the __rseq_abi in rseq.c. The rseq.so library will have ownership of that symbol, and there is not reason for an application or user library to try to define that symbol. The expected use is to link against libreq.so, which owns and provide that symbol.
- Set cpu_id to -2 on register error
- Add rseq_len syscall parameter, rseq_cs version
- Ensure disassember-friendly signature: x86 32/64 disassembler have a hard time decoding the instruction stream after a bad instruction. Use a nopl instruction to encode the signature. Suggested by Andy Lutomirski.
- Exercise parametrized tests variants in a shell scripts.
- Restartable sequences selftests: Remove use of event counter.
- Use cpu_id_start field: With the cpu_id_start field, the C preparation phase of the fast-path does not need to compare cpu_id < 0 anymore.
- Signal-safe registration and refcounting: Allow libraries using librseq.so to register it from signal handlers.

---
 MAINTAINERS                                        |    1 +
 tools/testing/selftests/Makefile                   |    1 +
 .../testing/selftests/cpu-opv/basic_cpu_opv_test.c |   13 +-
 tools/testing/selftests/rseq/.gitignore            |    4 +
 tools/testing/selftests/rseq/Makefile              |   22 +
 .../testing/selftests/rseq/basic_percpu_ops_test.c |  333 +++++
 tools/testing/selftests/rseq/basic_test.c          |   55 +
 tools/testing/selftests/rseq/param_test.c          | 1285 ++++++++++++++++++++
 tools/testing/selftests/rseq/rseq-arm.h            |  535 ++++++++
 tools/testing/selftests/rseq/rseq-ppc.h            |  567 +++++++++
 tools/testing/selftests/rseq/rseq-x86.h            |  898 ++++++++++++++
 tools/testing/selftests/rseq/rseq.c                |  116 ++
 tools/testing/selftests/rseq/rseq.h                |  154 +++
 tools/testing/selftests/rseq/run_param_test.sh     |  124 ++
 14 files changed, 4103 insertions(+), 5 deletions(-)
 create mode 100644 tools/testing/selftests/rseq/.gitignore
 create mode 100644 tools/testing/selftests/rseq/Makefile
 create mode 100644 tools/testing/selftests/rseq/basic_percpu_ops_test.c
 create mode 100644 tools/testing/selftests/rseq/basic_test.c
 create mode 100644 tools/testing/selftests/rseq/param_test.c
 create mode 100644 tools/testing/selftests/rseq/rseq-arm.h
 create mode 100644 tools/testing/selftests/rseq/rseq-ppc.h
 create mode 100644 tools/testing/selftests/rseq/rseq-x86.h
 create mode 100644 tools/testing/selftests/rseq/rseq.c
 create mode 100644 tools/testing/selftests/rseq/rseq.h
 create mode 100755 tools/testing/selftests/rseq/run_param_test.sh

index 54e11f0569e0..1022b5f51cd1 100644
@@ -11502,6 +11502,7 @@ S:	Supported
 F:	kernel/rseq.c
 F:	include/uapi/linux/rseq.h
 F:	include/trace/events/rseq.h
+F:	tools/testing/selftests/rseq/
 
 RFKILL
 M:	Johannes Berg <[email protected]>

index c66e5e67cfab..b7fcd7bcb87e 100644
@@ -25,6 +25,7 @@ TARGETS += nsfs
 TARGETS += powerpc
 TARGETS += pstore
 TARGETS += ptrace
+TARGETS += rseq
 TARGETS += seccomp
 TARGETS += sigaltstack
 TARGETS += size

index 23072dcf5612..6b624f1939ea 100644
@@ -19,6 +19,8 @@
 #define TESTBUFLEN	4096
 #define TESTBUFLEN_CMP	16
 
+#define TESTBUFLEN_PAGE_MAX	65536
+
 static int test_compare_eq_op(char *a, char *b, size_t len)
 {
 	struct cpu_op opvec[] = {

@@ -1047,20 +1049,21 @@ static int test_too_many_ops(void)
 	return 0;
 }
 
+/* Use 64kB len, largest page size known on Linux. */
 static int test_memcpy_single_too_large(void)
 {
 	int i, ret;
-	char buf1[TESTBUFLEN + 1];
-	char buf2[TESTBUFLEN + 1];
+	char buf1[TESTBUFLEN_PAGE_MAX + 1];
+	char buf2[TESTBUFLEN_PAGE_MAX + 1];
 	const char *test_name = "test_memcpy_single_too_large";
 
 	printf("Testing %s\n", test_name);
 
 	/* Test memcpy */
-	for (i = 0; i < TESTBUFLEN + 1; i++)
+	for (i = 0; i < TESTBUFLEN_PAGE_MAX + 1; i++)
 		buf1[i] = (char)i;
-	memset(buf2, 0, TESTBUFLEN + 1);
-	ret = test_memcpy_op(buf2, buf1, TESTBUFLEN + 1);
+	memset(buf2, 0, TESTBUFLEN_PAGE_MAX + 1);
+	ret = test_memcpy_op(buf2, buf1, TESTBUFLEN_PAGE_MAX + 1);
 	if (!ret || (ret < 0 && errno != EINVAL)) {
 		printf("%s returned with %d, errno: %s\n",
 			test_name, ret, strerror(errno));

new file mode 100644
index 000000000000..9409c3db99b2
@@ -0,0 +1,4 @@
+basic_percpu_ops_test
+basic_test
+basic_rseq_op_test
+param_test

new file mode 100644
index 000000000000..e9b0562dd450
@@ -0,0 +1,22 @@
+CFLAGS += -O2 -Wall -g -I./ -I../cpu-opv/ -I../../../../usr/include/ -L./ -Wl,-rpath=./
+LDLIBS += -lpthread
+
+TEST_GEN_PROGS = basic_test basic_percpu_ops_test param_test \
+		librseq.so libcpu-op.so
+
+ALL: $(TEST_GEN_PROGS)
+
+librseq.so: rseq.c rseq.h rseq-*.h
+	$(CC) $(CFLAGS) -shared -fPIC $< $(LDLIBS) -o $@
+
+libcpu-op.so: ../cpu-opv/cpu-op.c ../cpu-opv/cpu-op.h
+	$(CC) $(CFLAGS) -shared -fPIC $< $(LDLIBS) -o $@
+
+# Own recipe because we only want to build against 1st prerequisite, but
+# still track changes to header files.
+%: %.c librseq.so libcpu-op.so rseq.h rseq-*.h ../cpu-opv/cpu-op.h
+	$(CC) $(CFLAGS) $< $(LDLIBS) -lrseq -lcpu-op -o $@
+
+TEST_PROGS = run_param_test.sh
+
+include ../lib.mk

2.11.0

Part of the diff is not shown here. View the raw message.

Fixes assembler errors:
/tmp/cceKwI9a.s: Assembler messages:
/tmp/cceKwI9a.s:849: Error: co-processor offset out of range

with gcc prior to gcc-7. This can trigger if multiple rseq inline asm are used within the same function.

My best guess on the cause of this issue is that gcc has a hard time figuring out the actual size of the inline asm, and therefore does not compute the offsets at which literal values can be placed from the program counter accurately.

---
 tools/testing/selftests/rseq/rseq-arm.h | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

index d2e9f07d569a..75371e4dfbfb 100644
@@ -79,12 +79,15 @@ do {									\
 		teardown						\
 		"b %l[" __rseq_str(cmpfail_label) "]\n\t"
 
+#define rseq_workaround_gcc_asm_size_guess()	__asm__ __volatile__("")
+
 static inline __attribute__((always_inline))
 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv,
 		int cpu)
 {
 	RSEQ_INJECT_C(9)
 
+	rseq_workaround_gcc_asm_size_guess();
 	__asm__ __volatile__ goto (
 		RSEQ_ASM_DEFINE_TABLE(__rseq_table, 0x0, 0x0, 1f, 2f-1f, 4f)
 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)

@@ -115,11 +118,14 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv,
 		  RSEQ_INJECT_CLOBBER
 		: abort, cmpfail
 	);
+	rseq_workaround_gcc_asm_size_guess();
 	return 0;
 abort:
+	rseq_workaround_gcc_asm_size_guess();
 	RSEQ_INJECT_FAILED
 	return -1;
 cmpfail:
+	rseq_workaround_gcc_asm_size_guess();
 	return 1;
 }
 

@@ -129,6 +135,7 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
 {
 	RSEQ_INJECT_C(9)
 
+	rseq_workaround_gcc_asm_size_guess();
 	__asm__ __volatile__ goto (
 		RSEQ_ASM_DEFINE_TABLE(__rseq_table, 0x0, 0x0, 1f, 2f-1f, 4f)
 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)

@@ -164,11 +171,14 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
 		  RSEQ_INJECT_CLOBBER
 		: abort, cmpfail
 	);
+	rseq_workaround_gcc_asm_size_guess();
 	return 0;
 abort:
+	rseq_workaround_gcc_asm_size_guess();
 	RSEQ_INJECT_FAILED
 	return -1;
 cmpfail:
+	rseq_workaround_gcc_asm_size_guess();
 	return 1;
 }
 

@@ -177,6 +187,7 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
 {
 	RSEQ_INJECT_C(9)
 
+	rseq_workaround_gcc_asm_size_guess();
 	__asm__ __volatile__ goto (
 		RSEQ_ASM_DEFINE_TABLE(__rseq_table, 0x0, 0x0, 1f, 2f-1f, 4f)
 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)

@@ -203,8 +214,10 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
 		  RSEQ_INJECT_CLOBBER
 		: abort
 	);
+	rseq_workaround_gcc_asm_size_guess();
 	return 0;
 abort:
+	rseq_workaround_gcc_asm_size_guess();
 	RSEQ_INJECT_FAILED
 	return -1;
 }

@@ -216,6 +229,7 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
 {
 	RSEQ_INJECT_C(9)
 
+	rseq_workaround_gcc_asm_size_guess();
 	__asm__ __volatile__ goto (
 		RSEQ_ASM_DEFINE_TABLE(__rseq_table, 0x0, 0x0, 1f, 2f-1f, 4f)
 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)

@@ -253,11 +267,14 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
 		  RSEQ_INJECT_CLOBBER
 		: abort, cmpfail
 	);
+	rseq_workaround_gcc_asm_size_guess();
 	return 0;
 abort:
+	rseq_workaround_gcc_asm_size_guess();
 	RSEQ_INJECT_FAILED
 	return -1;
 cmpfail:
+	rseq_workaround_gcc_asm_size_guess();
 	return 1;
 }
 

@@ -268,6 +285,7 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
 {
 	RSEQ_INJECT_C(9)
 
+	rseq_workaround_gcc_asm_size_guess();
 	__asm__ __volatile__ goto (
 		RSEQ_ASM_DEFINE_TABLE(__rseq_table, 0x0, 0x0, 1f, 2f-1f, 4f)
 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)

@@ -306,11 +324,14 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
 		  RSEQ_INJECT_CLOBBER
 		: abort, cmpfail
 	);
+	rseq_workaround_gcc_asm_size_guess();
 	return 0;
 abort:
+	rseq_workaround_gcc_asm_size_guess();
 	RSEQ_INJECT_FAILED
 	return -1;
 cmpfail:
+	rseq_workaround_gcc_asm_size_guess();
 	return 1;
 }
 

@@ -321,6 +342,7 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
 {
 	RSEQ_INJECT_C(9)
 
+	rseq_workaround_gcc_asm_size_guess();
 	__asm__ __volatile__ goto (
 		RSEQ_ASM_DEFINE_TABLE(__rseq_table, 0x0, 0x0, 1f, 2f-1f, 4f)
 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)

@@ -359,11 +381,14 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
 		  RSEQ_INJECT_CLOBBER
 		: abort, cmpfail
 	);
+	rseq_workaround_gcc_asm_size_guess();
 	return 0;
 abort:
+	rseq_workaround_gcc_asm_size_guess();
 	RSEQ_INJECT_FAILED
 	return -1;
 cmpfail:
+	rseq_workaround_gcc_asm_size_guess();
 	return 1;
 }
 

@@ -376,6 +401,7 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
 
 	RSEQ_INJECT_C(9)
 
+	rseq_workaround_gcc_asm_size_guess();
 	__asm__ __volatile__ goto (
 		RSEQ_ASM_DEFINE_TABLE(__rseq_table, 0x0, 0x0, 1f, 2f-1f, 4f)
 		"str %[src], %[rseq_scratch0]\n\t"

@@ -442,11 +468,14 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
 		  RSEQ_INJECT_CLOBBER
 		: abort, cmpfail
 	);
+	rseq_workaround_gcc_asm_size_guess();
 	return 0;
 abort:
+	rseq_workaround_gcc_asm_size_guess();
 	RSEQ_INJECT_FAILED
 	return -1;
 cmpfail:
+	rseq_workaround_gcc_asm_size_guess();
 	return 1;
 }
 

@@ -459,6 +488,7 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
 
 	RSEQ_INJECT_C(9)
 
+	rseq_workaround_gcc_asm_size_guess();
 	__asm__ __volatile__ goto (
 		RSEQ_ASM_DEFINE_TABLE(__rseq_table, 0x0, 0x0, 1f, 2f-1f, 4f)
 		"str %[src], %[rseq_scratch0]\n\t"

@@ -526,10 +556,13 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
 		  RSEQ_INJECT_CLOBBER
 		: abort, cmpfail
 	);
+	rseq_workaround_gcc_asm_size_guess();
 	return 0;
 abort:
+	rseq_workaround_gcc_asm_size_guess();
 	RSEQ_INJECT_FAILED
 	return -1;
 cmpfail:
+	rseq_workaround_gcc_asm_size_guess();
 	return 1;
 }

2.11.0