membarrier: Provide register expedited private command
Mathieu Desnoyers
maintainerRFC PATCH v2 1/2 membarrier: Provide register expedited private command
Provide a new command allowing processes to register their intent to use the private expedited command.
This allows PowerPC to skip the full memory barrier in switch_mm(), and only issue the barrier when scheduling into a task belonging to a process that has registered to use expedited private.
Processes are now required to register before using MEMBARRIER_CMD_PRIVATE_EXPEDITED, otherwise that command returns EPERM.
[ Build tested on PowerPC. Waiting on test machine to come back to life
for runtime testing. ]
Changes since v1:
- Use test_ti_thread_flag(next, ...) instead of test_thread_flag() in powerpc membarrier_arch_sched_in(), given that we want to specifically check the next thread state.
- Add missing ARCH_HAS_MEMBARRIER_HOOKS in Kconfig.
- Use task_thread_info() to pass thread_info from task to *_ti_thread_flag().
--- MAINTAINERS | 2 ++ arch/powerpc/Kconfig | 1 + arch/powerpc/include/asm/membarrier.h | 40 ++++++++++++++++++++++++++++++ arch/powerpc/include/asm/thread_info.h | 3 +++ arch/powerpc/kernel/Makefile | 2 ++ arch/powerpc/kernel/membarrier.c | 45 ++++++++++++++++++++++++++++++++++ fs/exec.c | 1 + include/linux/mm_types.h | 3 +++ include/linux/sched/mm.h | 45 ++++++++++++++++++++++++++++++++++ include/uapi/linux/membarrier.h | 23 +++++++++++------ init/Kconfig | 3 +++ kernel/fork.c | 2 ++ kernel/sched/core.c | 16 +++--------- kernel/sched/membarrier.c | 25 ++++++++++++++++--- 14 files changed, 187 insertions(+), 24 deletions(-) create mode 100644 arch/powerpc/include/asm/membarrier.h create mode 100644 arch/powerpc/kernel/membarrier.c
MAINTAINERS +2 -0
@@ -8805,6 +8805,8 @@ L: [email protected] S: Supported F: kernel/sched/membarrier.c F: include/uapi/linux/membarrier.h +F: arch/powerpc/kernel/membarrier.c +F: arch/powerpc/include/asm/membarrier.h MEMORY MANAGEMENT L: [email protected]
arch/powerpc/Kconfig +1 -0
@@ -138,6 +138,7 @@ config PPC select ARCH_HAS_ELF_RANDOMIZE select ARCH_HAS_FORTIFY_SOURCE select ARCH_HAS_GCOV_PROFILE_ALL + select ARCH_HAS_MEMBARRIER_HOOKS select ARCH_HAS_SCALED_CPUTIME if VIRT_CPU_ACCOUNTING_NATIVE select ARCH_HAS_SG_CHAIN select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
arch/powerpc/include/asm/membarrier.h new file +40 -0
@@ -0,0 +1,40 @@ +#ifndef _ASM_POWERPC_MEMBARRIER_H +#define _ASM_POWERPC_MEMBARRIER_H + +static inline void membarrier_arch_sched_in (struct task_struct *prev, +struct task_struct *next) +{ +/* +* Only need the full barrier when switching between processes. +*/ +if (likely (!test_ti_thread_flag (task_thread_info (next), +TIF_MEMBARRIER_PRIVATE_EXPEDITED ) + || prev->mm == next->mm)) +return ; + +/* +* The membarrier system call requires a full memory barrier +* after storing to rq->curr, before going back to user-space. +*/ +smp_mb (); +} +static inline void membarrier_arch_fork (struct task_struct *t, +unsigned long clone_flags) +{ +/* +* Coherence of TIF_MEMBARRIER_PRIVATE_EXPEDITED against thread +* fork is protected by siglock. membarrier_arch_fork is called +* with siglock held. +*/ +if (t->mm->membarrier_private_expedited) +set_ti_thread_flag (task_thread_info (t), +TIF_MEMBARRIER_PRIVATE_EXPEDITED ); +} +static inline void membarrier_arch_execve (struct task_struct *t) +{ +clear_ti_thread_flag (task_thread_info (t), +TIF_MEMBARRIER_PRIVATE_EXPEDITED ); +} +void membarrier_arch_register_private_expedited (struct task_struct *t); + +#endif /* _ASM_POWERPC_MEMBARRIER_H */
arch/powerpc/include/asm/thread_info.h +3 -0
@@ -100,6 +100,7 @@ static inline struct thread_info *current_thread_info(void)#if defined(CONFIG_PPC64 )#define TIF_ELF2ABI 18/* function descriptors must die! */ #endif +#define TIF_MEMBARRIER_PRIVATE_EXPEDITED 19/* membarrier */ /* as above, but as bit values */ #define _TIF_SYSCALL_TRACE (1<<TIF_SYSCALL_TRACE)
@@ -119,6 +120,8 @@ static inline struct thread_info *current_thread_info(void)#define _TIF_SYSCALL_TRACEPOINT (1<<TIF_SYSCALL_TRACEPOINT)#define _TIF_EMULATE_STACK_STORE (1<<TIF_EMULATE_STACK_STORE)#define _TIF_NOHZ (1<<TIF_NOHZ) +#define _TIF_MEMBARRIER_PRIVATE_EXPEDITED \ + (1<<TIF_MEMBARRIER_PRIVATE_EXPEDITED)#define _TIF_SYSCALL_DOTRACE (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \ _TIF_SECCOMP | _TIF_SYSCALL_TRACEPOINT | \ _TIF_NOHZ)
arch/powerpc/kernel/Makefile +2 -0
@@ -135,6 +135,8 @@ endif obj-$(CONFIG_EPAPR_PARAVIRT) += epapr_paravirt.o epapr_hcalls.o obj-$(CONFIG_KVM_GUEST) += kvm.o kvm_emul.o +obj-$(CONFIG_MEMBARRIER) += membarrier.o + # Disable GCOV & sanitizers in odd or sensitive code GCOV_PROFILE_prom_init.o := n UBSAN_SANITIZE_prom_init.o := n
arch/powerpc/kernel/membarrier.c new file +45 -0
45 lines not on this page · raw message
fs/exec.c +1 -0
@@ -1802,6 +1802,7 @@ static int do_execveat_common(int fd, struct filename *filename,/* execve succeeded */ current->fs->in_exec =0 ; current->in_execve =0 ; +membarrier_execve (current);acct_update_integrals (current);task_numa_free (current);free_bprm (bprm);
include/linux/mm_types.h +3 -0
@@ -445,6 +445,9 @@ struct mm_struct {unsigned long flags;/* Must use atomic bitops to access the bits */ struct core_state *core_state;/* coredumping support */ +#ifdef CONFIG_MEMBARRIER +int membarrier_private_expedited; +#endif #ifdef CONFIG_AIO spinlock_t ioctx_lock;struct kioctx_table __rcu *ioctx_table;
include/linux/sched/mm.h +45 -0
49 lines not on this page · raw message
include/uapi/linux/membarrier.h +16 -7
@@ -52,21 +52,30 @@* (non-running threads are de facto in such a * state). This only covers threads from the * same processes as the caller thread. This -* command returns 0. The "expedited" commands -* complete faster than the non-expedited ones, -* they never block, but have the downside of -* causing extra overhead. +* command returns 0 on success. The +* "expedited" commands complete faster than +* the non-expedited ones, they never block, +* but have the downside of causing extra +* overhead. A process needs to register its +* intent to use the private expedited command +* prior to using it, otherwise this command +* returns -EPERM. +* @MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED: +* Register the process intent to use +* MEMBARRIER_CMD_PRIVATE_EXPEDITED. Always +* returns 0. * * Command to be passed to the membarrier system call. The commands need to * be a single bit each, except for MEMBARRIER_CMD_QUERY which is assigned to * the value 0. */ enum membarrier_cmd { -MEMBARRIER_CMD_QUERY =0 , -MEMBARRIER_CMD_SHARED = (1 <<0 ), +MEMBARRIER_CMD_QUERY =0 , +MEMBARRIER_CMD_SHARED = (1 <<0 ),/* reserved for MEMBARRIER_CMD_SHARED_EXPEDITED (1 << 1) */ /* reserved for MEMBARRIER_CMD_PRIVATE (1 << 2) */ -MEMBARRIER_CMD_PRIVATE_EXPEDITED = (1 <<3 ), +MEMBARRIER_CMD_PRIVATE_EXPEDITED = (1 <<3 ), +MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED = (1 <<4 ), };#endif /* _UAPI_LINUX_MEMBARRIER_H */
init/Kconfig +3 -0
@@ -1400,6 +1400,9 @@ config MEMBARRIER If unsure, say Y. +config ARCH_HAS_MEMBARRIER_HOOKS + bool + config EMBEDDED bool "Embedded system" option allnoconfig_y
kernel/fork.c +2 -0
@@ -1840,6 +1840,8 @@ static __latent_entropy struct task_struct *copy_process(*/ copy_seccomp (p); +membarrier_fork (p, clone_flags); +/* * Process group and session signals need to be delivered to just the * parent before the fork or both the parent and the child after the
kernel/sched/core.c +3 -13
@@ -2644,16 +2644,6 @@ static struct rq *finish_task_switch(struct task_struct *prev) prev_state = prev->state;vtime_task_switch (prev);perf_event_task_sched_in (prev, current); -/* -* The membarrier system call requires a full memory barrier -* after storing to rq->curr, before going back to user-space. -* -* TODO: This smp_mb__after_unlock_lock can go away if PPC end -* up adding a full barrier to switch_mm(), or we should figure -* out if a smp_mb__after_unlock_lock is really the proper API -* to use. -*/ -smp_mb__after_unlock_lock ();finish_lock_switch (rq, prev);finish_arch_post_lock_switch ();
@@ -3363,9 +3353,8 @@ static void __sched notrace __schedule(bool preempt)* care of this barrier. For weakly ordered machines for * which spin_unlock() acts as a RELEASE barrier (only * arm64 and PowerPC), arm64 has a full barrier in -* switch_to(), and PowerPC has -* smp_mb__after_unlock_lock() before -* finish_lock_switch(). +* switch_to(), and PowerPC has a full barrier in +* membarrier_arch_sched_in(). */ ++*switch_count;
@@ -3373,6 +3362,7 @@ static void __sched notrace __schedule(bool preempt)/* Also unlocks the rq: */ rq =context_switch (rq, prev, next, &rf); +membarrier_arch_sched_in (prev, next); }else { rq->clock_update_flags &= ~(RQCF_ACT_SKIP |RQCF_REQ_SKIP );rq_unlock_irq (rq, &rf);
kernel/sched/membarrier.c +21 -4
@@ -26,21 +26,25 @@* except MEMBARRIER_CMD_QUERY. */ #define MEMBARRIER_CMD_BITMASK \ - (MEMBARRIER_CMD_SHARED | MEMBARRIER_CMD_PRIVATE_EXPEDITED) + (MEMBARRIER_CMD_SHARED | MEMBARRIER_CMD_PRIVATE_EXPEDITED \ + | MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED)static void ipi_mb (void *info) {smp_mb ();/* IPIs should be serializing but paranoid. */ } -static void membarrier_private_expedited (void ) +static int membarrier_private_expedited (void ) {int cpu;bool fallback = false;cpumask_var_t tmpmask; +if (!current->mm->membarrier_private_expedited) +return -EPERM ; +if (num_online_cpus () ==1 ) -return ; +return 0 ;/* * Matches memory barriers around rq->curr modification in
@@ -94,6 +98,17 @@ static void membarrier_private_expedited(void)* rq->curr modification in scheduler. */ smp_mb ();/* exit from system call is not a mb */ +return 0 ; +} + +static void membarrier_register_private_expedited (void ) +{ +struct task_struct *p = current; + +if (READ_ONCE (p->mm->membarrier_private_expedited)) +return ; +WRITE_ONCE (p->mm->membarrier_private_expedited,1 ); +membarrier_arch_register_private_expedited (p); }/**
@@ -144,7 +159,9 @@ SYSCALL_DEFINE2(membarrier, int, cmd, int, flags)synchronize_sched ();return 0 ;case MEMBARRIER_CMD_PRIVATE_EXPEDITED : -membarrier_private_expedited (); +return membarrier_private_expedited (); +case MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED : +membarrier_register_private_expedited ();return 0 ;default :return -EINVAL ;
Signature · 1 line
2.11.0
Mathieu Desnoyers
maintainerRFC PATCH 2/2 membarrier: selftest: Test private expedited cmd
Test the new MEMBARRIER_CMD_PRIVATE_EXPEDITED and MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED commands.
Add checks expecting specific error values on system calls expected to fail.
--- .../testing/selftests/membarrier/membarrier_test.c | 109 ++++++++++++++++++--- 1 file changed, 94 insertions(+), 15 deletions(-)
tools/testing/selftests/membarrier/membarrier_test.c +94 -15
@@ -15,49 +15,119 @@ static int sys_membarrier(int cmd, int flags)static int test_membarrier_cmd_fail (void ) {int cmd =-1 , flags =0 ; +const char *test_name ="sys membarrier invalid command" ;if (sys_membarrier (cmd, flags) !=-1 ) {ksft_exit_fail_msg ( -"sys membarrier invalid command test: command = %d, flags = %d. Should fail, but passed\n" , - cmd, flags); +"%s test: command = %d, flags = %d. Should fail, but passed\n" , + test_name, cmd, flags); + } +if (errno !=EINVAL ) { +ksft_exit_fail_msg ( +"%s test: flags = %d. Should return (%d: \"%s\"), but returned (%d: \"%s\").\n" , + test_name, flags,EINVAL ,strerror (EINVAL ), + errno,strerror (errno)); }ksft_test_result_pass ( -"sys membarrier invalid command test: command = %d, flags = %d. Failed as expected\n" , - cmd, flags); +"%s test: command = %d, flags = %d, errno = %d. Failed as expected\n" , + test_name, cmd, flags, errno);return 0 ; }static int test_membarrier_flags_fail (void ) {int cmd =MEMBARRIER_CMD_QUERY , flags =1 ; +const char *test_name ="sys membarrier MEMBARRIER_CMD_QUERY invalid flags" ;if (sys_membarrier (cmd, flags) !=-1 ) {ksft_exit_fail_msg ( -"sys membarrier MEMBARRIER_CMD_QUERY invalid flags test: flags = %d. Should fail, but passed\n" , - flags); +"%s test: flags = %d. Should fail, but passed\n" , + test_name, flags); + } +if (errno !=EINVAL ) { +ksft_exit_fail_msg ( +"%s test: flags = %d. Should return (%d: \"%s\"), but returned (%d: \"%s\").\n" , + test_name, flags,EINVAL ,strerror (EINVAL ), + errno,strerror (errno)); }ksft_test_result_pass ( -"sys membarrier MEMBARRIER_CMD_QUERY invalid flags test: flags = %d. Failed as expected\n" , - flags); +"%s test: flags = %d, errno = %d. Failed as expected\n" , + test_name, flags, errno);return 0 ; } -static int test_membarrier_success (void ) +static int test_membarrier_shared_success (void ) {int cmd =MEMBARRIER_CMD_SHARED , flags =0 ; -const char *test_name ="sys membarrier MEMBARRIER_CMD_SHARED\n" ; +const char *test_name ="sys membarrier MEMBARRIER_CMD_SHARED" ; + +if (sys_membarrier (cmd, flags) !=0 ) { +ksft_exit_fail_msg ( +"%s test: flags = %d, errno = %d\n" , + test_name, flags, errno); + } + +ksft_test_result_pass ( +"%s test: flags = %d\n" , test_name, flags); +return 0 ; +} + +static int test_membarrier_private_expedited_fail (void ) +{ +int cmd =MEMBARRIER_CMD_PRIVATE_EXPEDITED , flags =0 ; +const char *test_name ="sys membarrier MEMBARRIER_CMD_PRIVATE_EXPEDITED not registered failure" ; + +if (sys_membarrier (cmd, flags) !=-1 ) { +ksft_exit_fail_msg ( +"%s test: flags = %d. Should fail, but passed\n" , + test_name, flags); + } +if (errno !=EPERM ) { +ksft_exit_fail_msg ( +"%s test: flags = %d. Should return (%d: \"%s\"), but returned (%d: \"%s\").\n" , + test_name, flags,EPERM ,strerror (EPERM ), + errno,strerror (errno)); + } + +ksft_test_result_pass ( +"%s test: flags = %d, errno = %d\n" , + test_name, flags, errno); +return 0 ; +} + +static int test_membarrier_register_private_expedited_success (void ) +{ +int cmd =MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED , flags =0 ; +const char *test_name ="sys membarrier MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED" ;if (sys_membarrier (cmd, flags) !=0 ) {ksft_exit_fail_msg ( -"sys membarrier MEMBARRIER_CMD_SHARED test: flags = %d\n" , - flags); +"%s test: flags = %d, errno = %d\n" , + test_name, flags, errno); }ksft_test_result_pass ( -"sys membarrier MEMBARRIER_CMD_SHARED test: flags = %d\n" , - flags); +"%s test: flags = %d\n" , + test_name, flags); +return 0 ; +} + +static int test_membarrier_private_expedited_success (void ) +{ +int cmd =MEMBARRIER_CMD_PRIVATE_EXPEDITED , flags =0 ; +const char *test_name ="sys membarrier MEMBARRIER_CMD_PRIVATE_EXPEDITED" ; + +if (sys_membarrier (cmd, flags) !=0 ) { +ksft_exit_fail_msg ( +"%s test: flags = %d, errno = %d\n" , + test_name, flags, errno); + } + +ksft_test_result_pass ( +"%s test: flags = %d\n" , + test_name, flags);return 0 ; }
@@ -71,7 +141,16 @@ static int test_membarrier(void) status =test_membarrier_flags_fail ();if (status)return status; - status =test_membarrier_success (); + status =test_membarrier_shared_success (); +if (status) +return status; + status =test_membarrier_private_expedited_fail (); +if (status) +return status; + status =test_membarrier_register_private_expedited_success (); +if (status) +return status; + status =test_membarrier_private_expedited_success ();if (status)return status;return 0 ;
Signature · 1 line
2.11.0
Peter Zijlstra
maintainerkernel/sched/core.c static void __sched notrace __schedule(bool preempt) · line 3365
Quoted 5 lines · Mathieu Desnoyers
kernel/sched/core.c +1
-0@@ -3373,6 +3362,7 @@ static void __sched notrace __schedule(bool preempt)/* Also unlocks the rq: */ rq =context_switch (rq, prev, next, &rf); +membarrier_arch_sched_in (prev, next);
That's wrong iirc; we just switched stacks, you cannot use @prev and @next.
Quoted 3 lines
kernel/sched/core.c +0
-0}else { rq->clock_update_flags &= ~(RQCF_ACT_SKIP |RQCF_REQ_SKIP );rq_unlock_irq (rq, &rf);
Mathieu Desnoyers
maintainerQuoted 9 lines · ----- On Sep 21, 2017, at 8:13 AM, Peter Zijlstra [email protected] wrote:
kernel/sched/core.c static void __sched notrace __schedule(bool preempt) · line 3365
Quoted 5 lines · Mathieu Desnoyers
kernel/sched/core.c +1
-0@@ -3373,6 +3362,7 @@ static void __sched notrace __schedule(bool preempt)/* Also unlocks the rq: */ rq =context_switch (rq, prev, next, &rf); +membarrier_arch_sched_in (prev, next);That's wrong iirc; we just switched stacks, you cannot use @prev and @next.
Yes, it did indeed explode spectacularly at boot in testing. RFC v3 fixes that.
Thanks,
Mathieu
Quoted 4 lines
Quoted 4 lines
kernel/sched/core.c +0
-0}else { rq->clock_update_flags &= ~(RQCF_ACT_SKIP |RQCF_REQ_SKIP );rq_unlock_irq (rq, &rf);
Signature · 3 lines
Mathieu Desnoyers EfficiOS Inc. http://www.efficios.com
Quoted 5 lines · Mathieu Desnoyers
Quoted 3 lines · ----- On Sep 21, 2017, at 8:13 AM, Peter Zijlstra [email protected] wrote:
Quoted 2 lines · Mathieu Desnoyers
(quoted diff) +0
-0@@ -3373,6 +3362,7 @@ static void __sched notrace __schedule(bool preempt)
Quoted 8 lines
Quoted 6 lines
kernel/sched/core.c static void __sched notrace __schedule(bool preempt) · line 3365
Quoted 3 lines
kernel/sched/core.c +1
-0/* Also unlocks the rq: */ rq =context_switch (rq, prev, next, &rf); +membarrier_arch_sched_in (prev, next);That's wrong iirc; we just switched stacks, you cannot use @prev and @next.
Yes, it did indeed explode spectacularly at boot in testing. RFC v3 fixes that.
Good to hear that there is a useful test. ;-)
Thanx, Paul
Quoted 15 lines
Thanks,
Mathieu
Quoted 4 lines
Quoted 4 lines
kernel/sched/core.c +0
-0}else { rq->clock_update_flags &= ~(RQCF_ACT_SKIP |RQCF_REQ_SKIP );rq_unlock_irq (rq, &rf);Signature · 3 lines
Mathieu Desnoyers EfficiOS Inc. http://www.efficios.com
Mathieu Desnoyers
maintainerQuoted 13 lines · ----- On Sep 21, 2017, at 11:27 AM, Paul E. McKenney…
Quoted 12 lines · Mathieu Desnoyers
Quoted 9 lines · ----- On Sep 21, 2017, at 8:13 AM, Peter Zijlstra [email protected] wrote:
kernel/sched/core.c static void __sched notrace __schedule(bool preempt) · line 3365
Quoted 5 lines · Mathieu Desnoyers
kernel/sched/core.c +1
-0@@ -3373,6 +3362,7 @@ static void __sched notrace __schedule(bool preempt)/* Also unlocks the rq: */ rq =context_switch (rq, prev, next, &rf); +membarrier_arch_sched_in (prev, next);That's wrong iirc; we just switched stacks, you cannot use @prev and @next.
Quoted 3 lines
Quoted 1 line
Yes, it did indeed explode spectacularly at boot in testing. RFC v3 fixes that.
Good to hear that there is a useful test. ;-)
For the issue identified by PeterZ, just booting the machine was enough to touch a task_struct after it was put, which exploded. After fixing this, I ran through the membarrier selftests: they run fine.
Then I did a test branch for liburcu [1] which uses the private cmd with registration. Both 'make check' and 'make regtest' run fine on Power8.
Thanks,
Mathieu
[1] https://github.com/compudj/userspace-rcu-dev/tree/test-urcu-reg-priv
Quoted 17 lines
Thanx, Paul
Quoted 14 lines
Thanks,
Mathieu
Quoted 4 lines
kernel/sched/core.c static void __sched notrace __schedule(bool preempt) · line 3368
Quoted 4 lines
kernel/sched/core.c +0
-0}else { rq->clock_update_flags &= ~(RQCF_ACT_SKIP |RQCF_REQ_SKIP );rq_unlock_irq (rq, &rf);--
Mathieu Desnoyers
EfficiOS Inc.
http://www.efficios.com
Signature · 3 lines
Mathieu Desnoyers EfficiOS Inc. http://www.efficios.com