context_tracking: Fixes and improvements
Subsystems (7)
cover context_tracking: Fixes and improvements
Hi,
Here is a bunch of patches preview for 3.10
Thanks.
Frederic Weisbecker (5):
context_tracking: Move exception handling to generic code
context_tracking: Restore correct previous context state on exception
exit
context_tracking: Restore preempted context state after
preempt_schedule_irq()
cputime: Dynamically scale cputime for full dynticks accounting
context_tracking: Enable probes by default for selftesting
arch/x86/include/asm/context_tracking.h | 21 ---- arch/x86/kernel/kvm.c | 8 +- arch/x86/kernel/traps.c | 68 +++++++++----- arch/x86/mm/fault.c | 8 +- include/linux/context_tracking.h | 24 +++++- include/linux/sched.h | 4 +- init/Kconfig | 1 + kernel/fork.c | 2 +- kernel/sched/core.c | 6 +- kernel/sched/cputime.c | 154 ++++++++++++++++--------------- 10 files changed, 163 insertions(+), 133 deletions(-)
Signature · 1 line
1.7.5.4
PATCH 1/5 context_tracking: Move exception handling to generic code
Exceptions handling on context tracking should share common
treatment: on entry we exit user mode if the exception triggered in that context. Then on exception exit we return to that previous context.
Generalize this to avoid duplication across archs.
--- arch/x86/include/asm/context_tracking.h | 21 --------------------- arch/x86/kernel/kvm.c | 2 +- arch/x86/kernel/traps.c | 3 +-- arch/x86/mm/fault.c | 2 +- include/linux/context_tracking.h | 17 ++++++++++++++++- 5 files changed, 19 insertions(+), 26 deletions(-)
arch/x86/include/asm/context_tracking.h +0 -21
@@ -1,31 +1,10 @@#ifndef _ASM_X86_CONTEXT_TRACKING_H#define _ASM_X86_CONTEXT_TRACKING_H -#ifndef __ASSEMBLY__ -#include <linux/context_tracking.h> -#include <asm/ptrace.h> - -static inline void exception_enter (struct pt_regs *regs) -{ -user_exit (); -} - -static inline void exception_exit (struct pt_regs *regs) -{ -#ifdef CONFIG_CONTEXT_TRACKING -if (user_mode (regs)) -user_enter (); -#endif -} - -#else /* __ASSEMBLY__ */ -#ifdef CONFIG_CONTEXT_TRACKING # define SCHEDULE_USER call schedule_user#else # define SCHEDULE_USER call schedule#endif -#endif /* !__ASSEMBLY__ */ -#endif
arch/x86/kernel/kvm.c +1 -1
@@ -20,6 +20,7 @@* Authors: Anthony Liguori <[email protected]> */ +#include <linux/context_tracking.h> #include <linux/module.h> #include <linux/kernel.h> #include <linux/kvm_para.h>
@@ -43,7 +44,6 @@#include <asm/apicdef.h> #include <asm/hypervisor.h> #include <asm/kvm_guest.h> -#include <asm/context_tracking.h> static int kvmapf =1 ;
arch/x86/kernel/traps.c +1 -2
@@ -12,6 +12,7 @@#define pr_fmt (fmt) KBUILD_MODNAME ": " fmt +#include <linux/context_tracking.h> #include <linux/interrupt.h> #include <linux/kallsyms.h> #include <linux/spinlock.h>
@@ -55,8 +56,6 @@#include <asm/i387.h> #include <asm/fpu-internal.h> #include <asm/mce.h> -#include <asm/context_tracking.h> -#include <asm/mach_traps.h> #ifdef CONFIG_X86_64
arch/x86/mm/fault.c +1 -1
@@ -13,12 +13,12 @@#include <linux/perf_event.h> /* perf_sw_event */ #include <linux/hugetlb.h> /* hstate_index_to_shift */ #include <linux/prefetch.h> /* prefetchw */ +#include <linux/context_tracking.h> /* exception_enter(), ... */ #include <asm/traps.h> /* dotraplinkage, ... */ #include <asm/pgalloc.h> /* pgd_*(), ... */ #include <asm/kmemcheck.h> /* kmemcheck_*(), ... */ #include <asm/fixmap.h> /* VSYSCALL_START */ -#include <asm/context_tracking.h> /* exception_enter(), ... */ /* * Page fault error code bits:
include/linux/context_tracking.h +16 -1
@@ -1,10 +1,11 @@#ifndef _LINUX_CONTEXT_TRACKING_H#define _LINUX_CONTEXT_TRACKING_H -#ifdef CONFIG_CONTEXT_TRACKING #include <linux/sched.h> #include <linux/percpu.h> +#include <asm/ptrace.h> +#ifdef CONFIG_CONTEXT_TRACKING struct context_tracking {/* * When active is false, probes are unset in order
@@ -33,12 +34,26 @@ static inline bool context_tracking_active(void)extern void user_enter (void );extern void user_exit (void ); + +static inline void exception_enter (struct pt_regs *regs) +{ +user_exit (); +} + +static inline void exception_exit (struct pt_regs *regs) +{ +if (user_mode (regs)) +user_enter (); +} +extern void context_tracking_task_switch (struct task_struct *prev,struct task_struct *next);#else static inline bool context_tracking_in_user (void ) {return false; }static inline void user_enter (void ) { }static inline void user_exit (void ) { } +static inline void exception_enter (struct pt_regs *regs) { } +static inline void exception_exit (struct pt_regs *regs) { }static inline void context_tracking_task_switch (struct task_struct *prev,struct task_struct *next) { }#endif /* !CONFIG_CONTEXT_TRACKING */
Signature · 1 line
1.7.5.4
PATCH 2/5 context_tracking: Restore correct previous context state on exception exit
In reply to Frederic WeisbeckerOn exception exit, we restore the previous context tracking state based on the regs of the interrupted frame. Iff that frame is in user mode as stated by user_mode() helper, we restore the context tracking user mode.
However there is a tiny chunck of low level arch code after we pass through user_enter() and until the CPU eventually resumes userspace.
If an exception happens in this tiny area, exception_enter() correctly exits the context tracking user mode but exception_exit() won't restore it because of the value returned by user_mode(regs).
As a result we may return to userspace with the wrong context tracking state.
To fix this, change exception_enter() to return the context tracking state prior to its call and pass this saved state to exception_exit(). This restores the real context tracking state of the interrupted frame.
(May be this patch was suggested to me, I don't recall exactly. If so, sorry for the missing credit).
--- arch/x86/kernel/kvm.c | 6 ++- arch/x86/kernel/traps.c | 65 ++++++++++++++++++++++++-------------- arch/x86/mm/fault.c | 6 ++- include/linux/context_tracking.h | 19 +++++++---- 4 files changed, 61 insertions(+), 35 deletions(-)
arch/x86/kernel/kvm.c +4 -2
@@ -254,16 +254,18 @@ EXPORT_SYMBOL_GPL(kvm_read_and_reset_pf_reason); dotraplinkage void __kprobesdo_async_page_fault (struct pt_regs *regs,unsigned long error_code) { +enum ctx_state prev_state; +switch (kvm_read_and_reset_pf_reason ()) {default :do_page_fault (regs, error_code);break ;case KVM_PV_REASON_PAGE_NOT_PRESENT :/* page is swapped out by the host. */ -exception_enter (regs); + prev_state =exception_enter ();exit_idle ();kvm_async_pf_task_wait ((u32 )read_cr2 ()); -exception_exit (regs); +exception_exit (prev_state);break ;case KVM_PV_REASON_PAGE_READY :rcu_irq_enter ();
arch/x86/kernel/traps.c +41 -24
201 lines not on this page · raw message
arch/x86/mm/fault.c +4 -2
@@ -1228,7 +1228,9 @@ good_area: dotraplinkage void __kprobesdo_page_fault (struct pt_regs *regs,unsigned long error_code) { -exception_enter (regs); +enum ctx_state prev_state; + + prev_state =exception_enter ();__do_page_fault (regs, error_code); -exception_exit (regs); +exception_exit (prev_state); }
include/linux/context_tracking.h +12 -7
@@ -5,7 +5,6 @@#include <linux/percpu.h> #include <asm/ptrace.h> -#ifdef CONFIG_CONTEXT_TRACKING struct context_tracking {/* * When active is false, probes are unset in order
@@ -14,12 +13,13 @@ struct context_tracking {* may be further optimized using static keys. */ bool active; -enum { +enum ctx_state {IN_KERNEL =0 ,IN_USER , } state; }; +#ifdef CONFIG_CONTEXT_TRACKING DECLARE_PER_CPU(struct context_tracking, context_tracking);static inline bool context_tracking_in_user (void )
@@ -35,14 +35,19 @@ static inline bool context_tracking_active(void)extern void user_enter (void );extern void user_exit (void ); -static inline void exception_enter (struct pt_regs *regs) +static inline enum ctx_state exception_enter (void ) { +enum ctx_state prev_ctx; + + prev_ctx =this_cpu_read (context_tracking.state);user_exit (); + +return prev_ctx; } -static inline void exception_exit (struct pt_regs *regs) +static inline void exception_exit (enum ctx_state prev_ctx) { -if (user_mode (regs)) +if (prev_ctx ==IN_USER )user_enter (); }
@@ -52,8 +57,8 @@ extern void context_tracking_task_switch(struct task_struct *prev,static inline bool context_tracking_in_user (void ) {return false; }static inline void user_enter (void ) { }static inline void user_exit (void ) { } -static inline void exception_enter (struct pt_regs *regs) { } -static inline void exception_exit (struct pt_regs *regs) { } +static inline enum ctx_state exception_enter (void ) {return 0 ; } +static inline void exception_exit (enum ctx_state prev_ctx) { }static inline void context_tracking_task_switch (struct task_struct *prev,struct task_struct *next) { }#endif /* !CONFIG_CONTEXT_TRACKING */
Signature · 1 line
1.7.5.4
PATCH 3/5 context_tracking: Restore preempted context state after preempt_schedule_irq()
In reply to Frederic WeisbeckerQuoted 1 line
From the context tracking POV,
preempt_schedule_irq()behaves pretty much
like an exception: It can be called anytime and schedule another task.
But currently it doesn't restore the context tracking state of the preempted code on preempt_schedule_irq() return.
As a result, if preempt_schedule_irq() is called in the tiny frame between user_enter() and the actual return to userspace, we resume userspace with the wrong context tracking state.
Fix this by using exception_enter/exit() which are a perfect fit for this kind of issue.
--- kernel/sched/core.c | 6 +++++- 1 files changed, 5 insertions(+), 1 deletions(-)
kernel/sched/core.c +5 -1
@@ -3074,11 +3074,13 @@ EXPORT_SYMBOL(preempt_schedule); asmlinkage void __sched preempt_schedule_irq(void) {struct thread_info *ti =current_thread_info (); +enum ctx_state prev_state;/* Catch callers which need to be fixed */ BUG_ON (ti->preempt_count || !irqs_disabled ()); -user_exit (); + prev_state =exception_enter (); + do { add_preempt_count(PREEMPT_ACTIVE); local_irq_enable();
@@ -3092,6 +3094,8 @@ asmlinkage void __sched preempt_schedule_irq(void)*/ barrier(); }while (need_resched ()); + +exception_exit (prev_state); }#endif /* CONFIG_PREEMPT */
Signature · 1 line
1.7.5.4
PATCH 4/5 cputime: Dynamically scale cputime for full dynticks accounting
In reply to Frederic WeisbeckerThe full dynticks cputime accounting is able to account either using the tick or the context tracking subsystem. This way the housekeeping CPU can keep the low overhead tick based solution.
This latter mode has a low jiffies resolution granularity and need to be scaled against CFS precise runtime accounting to improve its result. We are doing this for CONFIG_TICK_CPU_ACCOUNTING, now we also need to expand it to full dynticks accounting dynamic off-case as well.
--- include/linux/sched.h | 4 +- kernel/fork.c | 2 +- kernel/sched/cputime.c | 154 +++++++++++++++++++++++++----------------------- 3 files changed, 83 insertions(+), 77 deletions(-)
include/linux/sched.h +2 -2
@@ -574,7 +574,7 @@ struct signal_struct {cputime_t utime, stime, cutime, cstime;cputime_t gtime;cputime_t cgtime; -#ifndef CONFIG_VIRT_CPU_ACCOUNTING +#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE struct cputime prev_cputime;#endif unsigned long nvcsw, nivcsw, cnvcsw, cnivcsw;
@@ -1331,7 +1331,7 @@ struct task_struct {cputime_t utime, stime, utimescaled, stimescaled;cputime_t gtime; -#ifndef CONFIG_VIRT_CPU_ACCOUNTING +#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE struct cputime prev_cputime;#endif #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
kernel/fork.c +1 -1
@@ -1230,7 +1230,7 @@ static struct task_struct *copy_process(unsigned long clone_flags, p->utime = p->stime = p->gtime =0 ; p->utimescaled = p->stimescaled =0 ; -#ifndef CONFIG_VIRT_CPU_ACCOUNTING +#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE p->prev_cputime.utime = p->prev_cputime.stime =0 ;#endif #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
kernel/sched/cputime.c +80 -74
183 lines not on this page · raw message
Signature · 1 line
1.7.5.4
PATCH 5/5 context_tracking: Enable probes by default for selftesting
In reply to Frederic WeisbeckerUntil we provide the nohz_mask boot parameter, keeping the context tracking probes disabled by default is pointless since what we want is to runtime test this code anyway.
It's furthermore confusing for the users which don't expect the probes to be off when they select RCU user mode or full dynticks cputime accounting.
Let's enable these probes selftests by default for now.
Suggested: Steven Rostedt <[email protected]>
--- init/Kconfig | 1 + 1 files changed, 1 insertions(+), 0 deletions(-)
init/Kconfig +1 -0
@@ -536,6 +536,7 @@ config RCU_USER_QS config CONTEXT_TRACKING_FORCE bool "Force context tracking" depends on CONTEXT_TRACKING + default CONTEXT_TRACKING help Probe on user/kernel boundaries by default in order to test the features that rely on it such as userspace RCU extended
Signature · 1 line
1.7.5.4