User return notifiers / just-in-time MSR switching for KVM
cover User return notifiers / just-in-time MSR switching for KVM
The various x86 syscall related MSRs (MSR_LSTAR and friends, EFER when SCE needs to be updated) are fairly expensive to read or write. Since different operating systems can set different values for these MSRs, KVM needs to reload them when switching to a different guest or to the host.
Switching on every guest entry/exit is too expensive, so KVM reloads on guest preemption, which is a lot rarer. Even so, preemption based reload is suboptimal:
- if we're switching to a kernel thread and back, there's no need to reload the MSRs. Examples of kernel threads we're likely to switch to are:
- the idle task
- a threaded interrupt handler
- a kernel-mode virtio server (vhost-net)
- if we're switching to a guest running the same OS, the MSRs will have the same values and there's no need to reload them
- if the guest and host run the same OS, again the MSRs need not be reloaded.
This patchset implements just-in-time reloads to defer them to the last possible instant. When we do reload, we check whether the values have in fact changed and reload conditionally.
For the just-in-time reloads the first patch implements "user return notifiers", a callback invoked just before return to userspace. This has been written so that there is no code impact if KVM is not configured, and no runtime impact if KVM is not running.
The patchset improves guest/idle/guest switches by about 2000 cycles.
Avi Kivity (4):
core, x86: Add user return notifiers
KVM: VMX: Move MSR_KERNEL_GS_BASE out of the vmx autoload msr area
KVM: x86 shared msr infrastructure
KVM: VMX: Use shared msr infrastructure
arch/Kconfig | 10 +++ arch/x86/Kconfig | 1 + arch/x86/include/asm/kvm_host.h | 3 + arch/x86/include/asm/thread_info.h | 7 +- arch/x86/kernel/process.c | 2 + arch/x86/kernel/signal.c | 3 + arch/x86/kvm/Kconfig | 1 + arch/x86/kvm/vmx.c | 142 +++++++++++++++-------------------- arch/x86/kvm/x86.c | 79 ++++++++++++++++++++ kernel/Makefile | 1 + 10 files changed, 166 insertions(+), 83 deletions(-)
PATCH 1/4 core, x86: Add user return notifiers
Add a general per-cpu notifier that is called whenever the kernel is about to return to userspace. The notifier uses a thread_info flag and existing checks, so there is no impact on user return or context switch fast paths.
--- arch/Kconfig | 10 ++++++++++ arch/x86/Kconfig | 1 + arch/x86/include/asm/thread_info.h | 7 +++++-- arch/x86/kernel/process.c | 2 ++ arch/x86/kernel/signal.c | 3 +++ kernel/Makefile | 1 + 6 files changed, 22 insertions(+), 2 deletions(-)
arch/Kconfig +10 -0
@@ -70,6 +70,13 @@ config KRETPROBES def_bool y depends on KPROBES && HAVE_KRETPROBES +config USER_RETURN_NOTIFIER + bool + depends on HAVE_USER_RETURN_NOTIFIER + help + Provide a kernel-internal notification when a cpu is about to + switch to user mode. + config HAVE_IOREMAP_PROT bool
@@ -113,4 +120,7 @@ config HAVE_DMA_API_DEBUG config HAVE_DEFAULT_NO_SPIN_MUTEXES bool +config HAVE_USER_RETURN_NOTIFIER + bool + source "kernel/gcov/Kconfig"
arch/x86/Kconfig +1 -0
@@ -50,6 +50,7 @@ config X86 select HAVE_KERNEL_BZIP2 select HAVE_KERNEL_LZMA select HAVE_ARCH_KMEMCHECK + select HAVE_USER_RETURN_NOTIFIER config OUTPUT_FORMAT string
arch/x86/include/asm/thread_info.h +5 -2
@@ -83,6 +83,7 @@ struct thread_info {#define TIF_SYSCALL_AUDIT 7/* syscall auditing active */ #define TIF_SECCOMP 8/* secure computing */ #define TIF_MCE_NOTIFY 10/* notify userspace of an MCE */ +#define TIF_USER_RETURN_NOTIFY 11/* notify kernel of userspace return */ #define TIF_NOTSC 16/* TSC is not accessible in userland */ #define TIF_IA32 17/* 32bit process */ #define TIF_FORK 18/* ret_from_fork */
@@ -107,6 +108,7 @@ struct thread_info {#define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT)#define _TIF_SECCOMP (1 << TIF_SECCOMP)#define _TIF_MCE_NOTIFY (1 << TIF_MCE_NOTIFY) +#define _TIF_USER_RETURN_NOTIFY (1 << TIF_USER_RETURN_NOTIFY)#define _TIF_NOTSC (1 << TIF_NOTSC)#define _TIF_IA32 (1 << TIF_IA32)#define _TIF_FORK (1 << TIF_FORK)
@@ -141,13 +143,14 @@ struct thread_info {/* Only used for 64 bit */ #define _TIF_DO_NOTIFY_MASK \ - (_TIF_SIGPENDING|_TIF_MCE_NOTIFY|_TIF_NOTIFY_RESUME) + (_TIF_SIGPENDING | _TIF_MCE_NOTIFY | _TIF_NOTIFY_RESUME | \ + _TIF_USER_RETURN_NOTIFY)/* flags to check in __switch_to() */ #define _TIF_WORK_CTXSW \ (_TIF_IO_BITMAP|_TIF_DEBUGCTLMSR|_TIF_DS_AREA_MSR|_TIF_NOTSC) -#define _TIF_WORK_CTXSW_PREV _TIF_WORK_CTXSW +#define _TIF_WORK_CTXSW_PREV (_TIF_WORK_CTXSW|_TIF_USER_RETURN_NOTIFY)#define _TIF_WORK_CTXSW_NEXT (_TIF_WORK_CTXSW|_TIF_DEBUG)#define PREEMPT_ACTIVE 0x10000000
arch/x86/kernel/process.c +2 -0
@@ -9,6 +9,7 @@#include <linux/pm.h> #include <linux/clockchips.h> #include <linux/random.h> +#include <linux/user-return-notifier.h> #include <trace/power.h> #include <asm/system.h> #include <asm/apic.h>
@@ -227,6 +228,7 @@ void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p,*/ memset (tss->io_bitmap,0xff , prev->io_bitmap_max); } +propagate_user_return_notify (prev_p, next_p); }int sys_fork (struct pt_regs *regs)
arch/x86/kernel/signal.c +3 -0
@@ -19,6 +19,7 @@#include <linux/stddef.h> #include <linux/personality.h> #include <linux/uaccess.h> +#include <linux/user-return-notifier.h> #include <asm/processor.h> #include <asm/ucontext.h>
@@ -870,6 +871,8 @@ do_notify_resume(struct pt_regs *regs, void *unused, __u32 thread_info_flags)clear_thread_flag (TIF_NOTIFY_RESUME );tracehook_notify_resume (regs); } +if (thread_info_flags & _TIF_USER_RETURN_NOTIFY) +fire_user_return_notifiers ();#ifdef CONFIG_X86_32 clear_thread_flag (TIF_IRET );
kernel/Makefile +1 -0
@@ -100,6 +100,7 @@ obj-$(CONFIG_RING_BUFFER) += trace/ obj-$(CONFIG_SMP) += sched_cpupri.o obj-$(CONFIG_SLOW_WORK) += slow-work.o obj-$(CONFIG_PERF_COUNTERS) += perf_counter.o +obj-$(CONFIG_USER_RETURN_NOTIFIER) += user-return-notifier.o ifneq ($(CONFIG_SCHED_OMIT_FRAME_POINTER),y) # According to Alan Modra <[email protected]>, the -fno-omit-frame-pointer is
Signature · 1 line
1.6.4.1
H. Peter Anvin
reviewerRe: [PATCH 1/4] core, x86: Add user return notifiers
Quoted 6 lines · Avi Kivity
Add a general per-cpu notifier that is called whenever the kernel is about to return to userspace. The notifier uses a
thread_infoflag and existing checks, so there is no impact on user return or context switch fast paths.Signed-off-by Avi Kivity <[email protected]>
The concept of this patch looks entirely sane, as does the code, *except* for:
+#include <linux/user-return-notifier.h>
... missing header file, and ...
+obj-$(CONFIG_USER_RETURN_NOTIFIER) += user-return-notifier.o
... missing source file.
Could you fix the patch, please?
-hpa
Signature · 2 lines
H. Peter Anvin, Intel Open Source Technology Center I work for Intel. I don't speak on their behalf.
PATCH v2 1/1 core, x86: Add user return notifiers
Add a general per-cpu notifier that is called whenever the kernel is about to return to userspace. The notifier uses a thread_info flag and existing checks, so there is no impact on user return or context switch fast paths.
---
v2: include new files in patch
arch/Kconfig | 10 +++++++ arch/x86/Kconfig | 1 + arch/x86/include/asm/thread_info.h | 7 +++- arch/x86/kernel/process.c | 2 + arch/x86/kernel/signal.c | 3 ++ include/linux/user-return-notifier.h | 42 +++++++++++++++++++++++++++++++ kernel/Makefile | 1 + kernel/user-return-notifier.c | 46 ++++++++++++++++++++++++++++++++++ 8 files changed, 110 insertions(+), 2 deletions(-) create mode 100644 include/linux/user-return-notifier.h create mode 100644 kernel/user-return-notifier.c
arch/Kconfig +10 -0
@@ -82,6 +82,13 @@ config KRETPROBES def_bool y depends on KPROBES && HAVE_KRETPROBES +config USER_RETURN_NOTIFIER + bool + depends on HAVE_USER_RETURN_NOTIFIER + help + Provide a kernel-internal notification when a cpu is about to + switch to user mode. + config HAVE_IOREMAP_PROT bool
@@ -125,4 +132,7 @@ config HAVE_DMA_API_DEBUG config HAVE_DEFAULT_NO_SPIN_MUTEXES bool +config HAVE_USER_RETURN_NOTIFIER + bool + source "kernel/gcov/Kconfig"
arch/x86/Kconfig +1 -0
@@ -50,6 +50,7 @@ config X86 select HAVE_KERNEL_BZIP2 select HAVE_KERNEL_LZMA select HAVE_ARCH_KMEMCHECK + select HAVE_USER_RETURN_NOTIFIER config OUTPUT_FORMAT string
arch/x86/include/asm/thread_info.h +5 -2
@@ -83,6 +83,7 @@ struct thread_info {#define TIF_SYSCALL_AUDIT 7/* syscall auditing active */ #define TIF_SECCOMP 8/* secure computing */ #define TIF_MCE_NOTIFY 10/* notify userspace of an MCE */ +#define TIF_USER_RETURN_NOTIFY 11/* notify kernel of userspace return */ #define TIF_NOTSC 16/* TSC is not accessible in userland */ #define TIF_IA32 17/* 32bit process */ #define TIF_FORK 18/* ret_from_fork */
@@ -107,6 +108,7 @@ struct thread_info {#define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT)#define _TIF_SECCOMP (1 << TIF_SECCOMP)#define _TIF_MCE_NOTIFY (1 << TIF_MCE_NOTIFY) +#define _TIF_USER_RETURN_NOTIFY (1 << TIF_USER_RETURN_NOTIFY)#define _TIF_NOTSC (1 << TIF_NOTSC)#define _TIF_IA32 (1 << TIF_IA32)#define _TIF_FORK (1 << TIF_FORK)
@@ -142,13 +144,14 @@ struct thread_info {/* Only used for 64 bit */ #define _TIF_DO_NOTIFY_MASK \ - (_TIF_SIGPENDING|_TIF_MCE_NOTIFY|_TIF_NOTIFY_RESUME) + (_TIF_SIGPENDING | _TIF_MCE_NOTIFY | _TIF_NOTIFY_RESUME | \ + _TIF_USER_RETURN_NOTIFY)/* flags to check in __switch_to() */ #define _TIF_WORK_CTXSW \ (_TIF_IO_BITMAP|_TIF_DEBUGCTLMSR|_TIF_DS_AREA_MSR|_TIF_NOTSC) -#define _TIF_WORK_CTXSW_PREV _TIF_WORK_CTXSW +#define _TIF_WORK_CTXSW_PREV (_TIF_WORK_CTXSW|_TIF_USER_RETURN_NOTIFY)#define _TIF_WORK_CTXSW_NEXT (_TIF_WORK_CTXSW|_TIF_DEBUG)#define PREEMPT_ACTIVE 0x10000000
arch/x86/kernel/process.c +2 -0
@@ -9,6 +9,7 @@#include <linux/pm.h> #include <linux/clockchips.h> #include <linux/random.h> +#include <linux/user-return-notifier.h> #include <trace/power.h> #include <asm/system.h> #include <asm/apic.h>
@@ -227,6 +228,7 @@ void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p,*/ memset (tss->io_bitmap,0xff , prev->io_bitmap_max); } +propagate_user_return_notify (prev_p, next_p); }int sys_fork (struct pt_regs *regs)
arch/x86/kernel/signal.c +3 -0
@@ -19,6 +19,7 @@#include <linux/stddef.h> #include <linux/personality.h> #include <linux/uaccess.h> +#include <linux/user-return-notifier.h> #include <asm/processor.h> #include <asm/ucontext.h>
@@ -872,6 +873,8 @@ do_notify_resume(struct pt_regs *regs, void *unused, __u32 thread_info_flags)if (current->replacement_session_keyring)key_replace_session_keyring (); } +if (thread_info_flags & _TIF_USER_RETURN_NOTIFY) +fire_user_return_notifiers ();#ifdef CONFIG_X86_32 clear_thread_flag (TIF_IRET );
include/linux/user-return-notifier.h new file +42 -0
42 lines not on this page · raw message
kernel/Makefile +1 -0
@@ -98,6 +98,7 @@ obj-$(CONFIG_RING_BUFFER) += trace/ obj-$(CONFIG_SMP) += sched_cpupri.o obj-$(CONFIG_SLOW_WORK) += slow-work.o obj-$(CONFIG_PERF_COUNTERS) += perf_counter.o +obj-$(CONFIG_USER_RETURN_NOTIFIER) += user-return-notifier.o ifneq ($(CONFIG_SCHED_OMIT_FRAME_POINTER),y) # According to Alan Modra <[email protected]>, the -fno-omit-frame-pointer is
kernel/user-return-notifier.c new file +46 -0
46 lines not on this page · raw message
Signature · 1 line
1.6.4.1
Re: [PATCH v2] core, x86: Add user return notifiers
Quoted 5 lines · Avi Kivity
Add a general per-cpu notifier that is called whenever the kernel is about to return to userspace. The notifier uses a
thread_infoflag and existing checks, so there is no impact on user return or context switch fast paths.
Ingo/Peter?
Signature · 1 line
error compiling committee.c: too many arguments to function
Re: [PATCH v2] core, x86: Add user return notifiers
Quoted 8 lines · On Tue, 22 Sep 2009 12:25:33 +0300 Avi Kivity <[email protected]> wrote:
Quoted 5 lines · Avi Kivity
Add a general per-cpu notifier that is called whenever the kernel is about to return to userspace. The notifier uses a
thread_infoflag and existing checks, so there is no impact on user return or context switch fast paths.Ingo/Peter?
isn't this like really expensive when used ?
Signature · 3 lines
Arjan van de Ven Intel Open Source Technology Centre For development, discussion and tips for power savings, visit http://www.lesswatts.org
Re: [PATCH v2] core, x86: Add user return notifiers
Quoted 12 lines · Arjan van de Ven
Quoted 8 lines · On Tue, 22 Sep 2009 12:25:33 +0300 Avi Kivity<[email protected]> wrote:
Quoted 6 lines · Avi Kivity
Add a general per-cpu notifier that is called whenever the kernel is about to return to userspace. The notifier uses a
thread_infoflag and existing checks, so there is no impact on user return or context switch fast paths.
Quoted 4 lines
Quoted 2 lines
Ingo/Peter?
isn't this like really expensive when used ?
No, why? It triggers a call to do_notify_resume() and a walks a list of length 1.
Signature · 1 line
error compiling committee.c: too many arguments to function
Re: [PATCH v2] core, x86: Add user return notifiers
In reply to Avi KivityQuoted 8 lines · Avi Kivity
Quoted 5 lines · Avi Kivity
Add a general per-cpu notifier that is called whenever the kernel is about to return to userspace. The notifier uses a
thread_infoflag and existing checks, so there is no impact on user return or context switch fast paths.Ingo/Peter?
Would be nice to convert some existing open-coded return-to-user-space logic to this facility. One such candidate would be lockdep_sys_exit?
Ingo
Re: [PATCH v2] core, x86: Add user return notifiers
Quoted 11 lines · Ingo Molnar
Quoted 8 lines · Avi Kivity
Quoted 6 lines · Avi Kivity
Add a general per-cpu notifier that is called whenever the kernel is about to return to userspace. The notifier uses a
thread_infoflag and existing checks, so there is no impact on user return or context switch fast paths.
Quoted 5 lines
Quoted 2 lines
Ingo/Peter?
Would be nice to convert some existing open-coded return-to-user-space logic to this facility. One such candidate would be
lockdep_sys_exit?
I only implemented this for x86, while lockdep is arch independent. If arch support is added, it should be trivial.
I think perf counters could use preempt notifiers though, these are arch independent.
Signature · 1 line
error compiling committee.c: too many arguments to function
Re: [PATCH v2] core, x86: Add user return notifiers
Quoted 5 lines · Avi Kivity
Quoted 2 lines
Would be nice to convert some existing open-coded return-to-user-space logic to this facility. One such candidate would be
lockdep_sys_exit?I only implemented this for x86, while lockdep is arch independent. If arch support is added, it should be trivial.
The lockdep_sys_exit bit is actually x86/s390 only, and can easily be adapted to use the new functionality on x86 only. I'll try it out.
Signature · 1 line
error compiling committee.c: too many arguments to function
Re: [PATCH v2] core, x86: Add user return notifiers
Quoted 8 lines · Avi Kivity
Quoted 5 lines · Avi Kivity
Quoted 2 lines
Would be nice to convert some existing open-coded return-to-user-space logic to this facility. One such candidate would be
lockdep_sys_exit?I only implemented this for x86, while lockdep is arch independent. If arch support is added, it should be trivial.
The
lockdep_sys_exitbit is actually x86/s390 only, and can easily be adapted to use the new functionality on x86 only. I'll try it out.
Unfortunately it doesn't work out well. The notifier is called until explicitly unregistered (since it relies on a bit in TIF_NOTIFY_MASK), so we have to disarm it on the first return to usersspace or it spins forever. We could re-arm it on the next kernel entry, but we don't have a kernel entry notifier so we'll just be moving hooks from one point to another.
Signature · 1 line
error compiling committee.c: too many arguments to function
H. Peter Anvin
reviewerRe: [PATCH v2] core, x86: Add user return notifiers
In reply to Ingo MolnarQuoted 8 lines · Ingo Molnar
Quoted 6 lines · Avi Kivity
Quoted 5 lines · Avi Kivity
Add a general per-cpu notifier that is called whenever the kernel is about to return to userspace. The notifier uses a
thread_infoflag and existing checks, so there is no impact on user return or context switch fast paths.
Quoted 5 lines
Quoted 1 line
Ingo/Peter?
Would be nice to convert some existing open-coded return-to-user-space logic to this facility. One such candidate would be
lockdep_sys_exit?Ingo
Sorry, limited bandwidth due to LinuxCon, but I like the concept, and the previous (partial) patch was really clean. I agree with Ingo that arch support so we can use this as a general facility would be nice, but I don't consider that as a prerequisite for merging.
-hpa
Re: [PATCH v2] core, x86: Add user return notifiers
Quoted 8 lines · H. Peter Anvin
Quoted 7 lines · Ingo Molnar
Quoted 5 lines · Avi Kivity
Quoted 4 lines · Avi Kivity
Add a general per-cpu notifier that is called whenever the kernel is about to return to userspace. The notifier uses a
thread_infoflag and existing checks, so there is no impact on user return or context switch fast paths.
Quoted 7 lines
Quoted 5 lines
Quoted 1 line
Ingo/Peter?
Would be nice to convert some existing open-coded return-to-user-space logic to this facility. One such candidate would be
lockdep_sys_exit?Ingo
Sorry, limited bandwidth due to LinuxCon, but I like the concept, and the previous (partial) patch was really clean. I agree with Ingo that arch support so we can use this as a general facility would be nice, but I don't consider that as a prerequisite for merging.
Re-ping?
If accepted, please merge just the core patch and I will carry all of them in parallel. Once tip is merged I'll drop my copy of the first patch.
Signature · 1 line
Do not meddle in the internals of kernels, for they are subtle and quick to panic.
H. Peter Anvin
reviewerRe: [PATCH v2] core, x86: Add user return notifiers
Quoted 6 lines · Avi Kivity
Re-ping?
If accepted, please merge just the core patch and I will carry all of them in parallel. Once tip is merged I'll drop my copy of the first patch.
I was just talking to Ingo about this... would you consider this .32 or .33 material at his time? It came in awfully late for .32...
-hpa
Signature · 2 lines
H. Peter Anvin, Intel Open Source Technology Center I work for Intel. I don't speak on their behalf.
Re: [PATCH v2] core, x86: Add user return notifiers
Quoted 11 lines · H. Peter Anvin
Quoted 6 lines · Avi Kivity
Re-ping?
If accepted, please merge just the core patch and I will carry all of them in parallel. Once tip is merged I'll drop my copy of the first patch.
I was just talking to Ingo about this... would you consider this .32 or .33 material at his time? It came in awfully late for .32...
While I'd prefer it to be in .32, as there's much churn in both trees, I guess we should follow the process and schedule it for .33.
Signature · 1 line
Do not meddle in the internals of kernels, for they are subtle and quick to panic.
H. Peter Anvin
reviewerRe: [PATCH v2] core, x86: Add user return notifiers
Quoted 4 lines · Avi Kivity
While I'd prefer it to be in .32, as there's much churn in both trees, I guess we should follow the process and schedule it for .33.
OK, sounds like a plan. I'll pull it into -tip.
-hpa
Re: [PATCH v2] core, x86: Add user return notifiers
In reply to Ingo MolnarQuoted 9 lines · Ingo Molnar
Quoted 7 lines · Avi Kivity
Quoted 5 lines · Avi Kivity
Add a general per-cpu notifier that is called whenever the kernel is about to return to userspace. The notifier uses a
thread_infoflag and existing checks, so there is no impact on user return or context switch fast paths.
Quoted 4 lines
Quoted 1 line
Ingo/Peter?
Would be nice to convert some existing open-coded return-to-user-space logic to this facility. One such candidate would be
lockdep_sys_exit?
And here I was thinking this was one of the hottest code paths in the whole kernel...
Re: [PATCH v2] core, x86: Add user return notifiers
Quoted 6 lines · Peter Zijlstra
Quoted 3 lines
Would be nice to convert some existing open-coded return-to-user-space logic to this facility. One such candidate would be
lockdep_sys_exit?And here I was thinking this was one of the hottest code paths in the whole kernel...
If you're using lockdep, surely that's not your biggest worry?
Signature · 1 line
error compiling committee.c: too many arguments to function
Re: [PATCH v2] core, x86: Add user return notifiers
Quoted 9 lines · Avi Kivity
Quoted 6 lines · Peter Zijlstra
Quoted 3 lines
Would be nice to convert some existing open-coded return-to-user-space logic to this facility. One such candidate would be
lockdep_sys_exit?And here I was thinking this was one of the hottest code paths in the whole kernel...
If you're using lockdep, surely that's not your biggest worry?
No, but that's all under #ifdef and fully disappears when not enabled. Generic return-tu-user notifiers don't sound like they will though.
Re: [PATCH v2] core, x86: Add user return notifiers
Quoted 5 lines · Peter Zijlstra
Quoted 2 lines
If you're using lockdep, surely that's not your biggest worry?
No, but that's all under #ifdef and fully disappears when not enabled. Generic return-tu-user notifiers don't sound like they will though.
They will if not selected. If selected and not armed, they will have zero runtime impact since they piggyback on existing branches (_TIF_DO_NOTIFY_MASK and near relatives). If selected and armed they'll cause __switch_to_xtra() on every context switch and do_notity_resume() on syscall exit until disarmed, but then you've asked for it.
Signature · 1 line
error compiling committee.c: too many arguments to function
Re: [PATCH v2] core, x86: Add user return notifiers
In reply to Ingo MolnarQuoted 2 lines · Ingo Molnar
Would be nice to convert some existing open-coded return-to-user-space logic to this facility. One such candidate would be
lockdep_sys_exit?
I don't really like lockdep_sys_exit() in such a call, lockdep_sys_exit() is currently placed such that it will execute as the last C code before returning to user-space.
Put it any earlier and you've got a window where people can leak a lock to userspace undetected.
[tip:x86/entry] core, x86: Add user return notifiers
In reply to Avi KivityCommit-ID: 7c68af6e32c73992bad24107311f3433c89016e2 Gitweb: http://git.kernel.org/tip/7c68af6e32c73992bad24107311f3433c89016e2 Author: Avi Kivity <[email protected]> AuthorDate: Sat, 19 Sep 2009 09:40:22 +0300 Committer: H. Peter Anvin <[email protected]> CommitDate: Thu, 1 Oct 2009 12:12:18 -0700
core, x86: Add user return notifiers
Add a general per-cpu notifier that is called whenever the kernel is about to return to userspace. The notifier uses a thread_info flag and existing checks, so there is no impact on user return or context switch fast paths.
This will be used initially to speed up KVM task switching by lazily updating MSRs.
LKML-Reference: <[email protected]>
--- arch/Kconfig | 10 +++++++ arch/x86/Kconfig | 1 + arch/x86/include/asm/thread_info.h | 7 +++- arch/x86/kernel/process.c | 2 + arch/x86/kernel/signal.c | 3 ++ include/linux/user-return-notifier.h | 42 +++++++++++++++++++++++++++++++ kernel/user-return-notifier.c | 46 ++++++++++++++++++++++++++++++++++ 7 files changed, 109 insertions(+), 2 deletions(-)
arch/Kconfig +10 -0
@@ -83,6 +83,13 @@ config KRETPROBES def_bool y depends on KPROBES && HAVE_KRETPROBES +config USER_RETURN_NOTIFIER + bool + depends on HAVE_USER_RETURN_NOTIFIER + help + Provide a kernel-internal notification when a cpu is about to + switch to user mode. + config HAVE_IOREMAP_PROT bool
@@ -126,4 +133,7 @@ config HAVE_DMA_API_DEBUG config HAVE_DEFAULT_NO_SPIN_MUTEXES bool +config HAVE_USER_RETURN_NOTIFIER + bool + source "kernel/gcov/Kconfig"
arch/x86/Kconfig +1 -0
@@ -50,6 +50,7 @@ config X86 select HAVE_KERNEL_BZIP2 select HAVE_KERNEL_LZMA select HAVE_ARCH_KMEMCHECK + select HAVE_USER_RETURN_NOTIFIER config OUTPUT_FORMAT string
arch/x86/include/asm/thread_info.h +5 -2
@@ -83,6 +83,7 @@ struct thread_info {#define TIF_SYSCALL_AUDIT 7/* syscall auditing active */ #define TIF_SECCOMP 8/* secure computing */ #define TIF_MCE_NOTIFY 10/* notify userspace of an MCE */ +#define TIF_USER_RETURN_NOTIFY 11/* notify kernel of userspace return */ #define TIF_NOTSC 16/* TSC is not accessible in userland */ #define TIF_IA32 17/* 32bit process */ #define TIF_FORK 18/* ret_from_fork */
@@ -107,6 +108,7 @@ struct thread_info {#define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT)#define _TIF_SECCOMP (1 << TIF_SECCOMP)#define _TIF_MCE_NOTIFY (1 << TIF_MCE_NOTIFY) +#define _TIF_USER_RETURN_NOTIFY (1 << TIF_USER_RETURN_NOTIFY)#define _TIF_NOTSC (1 << TIF_NOTSC)#define _TIF_IA32 (1 << TIF_IA32)#define _TIF_FORK (1 << TIF_FORK)
@@ -142,13 +144,14 @@ struct thread_info {/* Only used for 64 bit */ #define _TIF_DO_NOTIFY_MASK \ - (_TIF_SIGPENDING|_TIF_MCE_NOTIFY|_TIF_NOTIFY_RESUME) + (_TIF_SIGPENDING | _TIF_MCE_NOTIFY | _TIF_NOTIFY_RESUME | \ + _TIF_USER_RETURN_NOTIFY)/* flags to check in __switch_to() */ #define _TIF_WORK_CTXSW \ (_TIF_IO_BITMAP|_TIF_DEBUGCTLMSR|_TIF_DS_AREA_MSR|_TIF_NOTSC) -#define _TIF_WORK_CTXSW_PREV _TIF_WORK_CTXSW +#define _TIF_WORK_CTXSW_PREV (_TIF_WORK_CTXSW|_TIF_USER_RETURN_NOTIFY)#define _TIF_WORK_CTXSW_NEXT (_TIF_WORK_CTXSW|_TIF_DEBUG)#define PREEMPT_ACTIVE 0x10000000
arch/x86/kernel/process.c +2 -0
@@ -9,6 +9,7 @@#include <linux/pm.h> #include <linux/clockchips.h> #include <linux/random.h> +#include <linux/user-return-notifier.h> #include <trace/events/power.h> #include <asm/system.h> #include <asm/apic.h>
@@ -224,6 +225,7 @@ void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p,*/ memset (tss->io_bitmap,0xff , prev->io_bitmap_max); } +propagate_user_return_notify (prev_p, next_p); }int sys_fork (struct pt_regs *regs)
arch/x86/kernel/signal.c +3 -0
@@ -19,6 +19,7 @@#include <linux/stddef.h> #include <linux/personality.h> #include <linux/uaccess.h> +#include <linux/user-return-notifier.h> #include <asm/processor.h> #include <asm/ucontext.h>
@@ -872,6 +873,8 @@ do_notify_resume(struct pt_regs *regs, void *unused, __u32 thread_info_flags)if (current->replacement_session_keyring)key_replace_session_keyring (); } +if (thread_info_flags & _TIF_USER_RETURN_NOTIFY) +fire_user_return_notifiers ();#ifdef CONFIG_X86_32 clear_thread_flag (TIF_IRET );
include/linux/user-return-notifier.h new file +42 -0
42 lines not on this page · raw message
kernel/user-return-notifier.c new file +46 -0
46 lines not on this page · raw message
Re: [PATCH 1/4] core, x86: Add user return notifiers
In reply to H. Peter AnvinQuoted 14 lines · H. Peter Anvin
Quoted 7 lines · Avi Kivity
Add a general per-cpu notifier that is called whenever the kernel is about to return to userspace. The notifier uses a
thread_infoflag and existing checks, so there is no impact on user return or context switch fast paths.Signed-off-by Avi Kivity<[email protected]>The concept of this patch looks entirely sane, as does the code, *except* for:
+#include<linux/user-return-notifier.h>
Quoted 9 lines
... missing header file, and ...
+obj-$(CONFIG_USER_RETURN_NOTIFIER) += user-return-notifier.o
... missing source file.
Could you fix the patch, please?
Doh. Sent out fixed patch.
Signature · 1 line
Do not meddle in the internals of kernels, for they are subtle and quick to panic.
PATCH 2/4 KVM: VMX: Move MSR_KERNEL_GS_BASE out of the vmx autoload msr area
In reply to Avi KivityCurrently MSR_KERNEL_GS_BASE is saved and restored as part of the guest/host msr reloading. Since we wish to lazy-restore all the other msrs, save and reload MSR_KERNEL_GS_BASE explicitly instead of using the common code.
--- arch/x86/kvm/vmx.c | 34 +++++++++++++++++++++------------- 1 files changed, 21 insertions(+), 13 deletions(-)
arch/x86/kvm/vmx.c +21 -13
@@ -80,7 +80,8 @@ struct vcpu_vmx {int save_nmsrs;int msr_offset_efer;#ifdef CONFIG_X86_64 -int msr_offset_kernel_gs_base; +u64 msr_host_kernel_gs_base; +u64 msr_guest_kernel_gs_base;#endif struct vmcs *vmcs;struct {
@@ -183,7 +184,7 @@ static void ept_save_pdptrs(struct kvm_vcpu *vcpu);*/ static const u32 vmx_msr_index[] = { #ifdef CONFIG_X86_64 -MSR_SYSCALL_MASK ,MSR_LSTAR ,MSR_CSTAR ,MSR_KERNEL_GS_BASE , +MSR_SYSCALL_MASK ,MSR_LSTAR ,MSR_CSTAR , #endifMSR_EFER ,MSR_K6_STAR , };
@@ -649,10 +650,10 @@ static void vmx_save_host_state(struct kvm_vcpu *vcpu)#endif #ifdef CONFIG_X86_64 -if (is_long_mode (&vmx->vcpu)) -save_msrs (vmx->host_msrs + - vmx->msr_offset_kernel_gs_base,1 ); - +if (is_long_mode (&vmx->vcpu)) { +rdmsrl (MSR_KERNEL_GS_BASE , vmx->msr_host_kernel_gs_base); +wrmsrl (MSR_KERNEL_GS_BASE , vmx->msr_guest_kernel_gs_base); + }#endif load_msrs (vmx->guest_msrs, vmx->save_nmsrs);load_transition_efer (vmx);
@@ -686,6 +687,12 @@ static void __vmx_load_host_state(struct vcpu_vmx *vmx)save_msrs (vmx->guest_msrs, vmx->save_nmsrs);load_msrs (vmx->host_msrs, vmx->save_nmsrs);reload_host_efer (vmx); +#ifdef CONFIG_X86_64 +if (is_long_mode (&vmx->vcpu)) { +rdmsrl (MSR_KERNEL_GS_BASE , vmx->msr_guest_kernel_gs_base); +wrmsrl (MSR_KERNEL_GS_BASE , vmx->msr_host_kernel_gs_base); + } +#endif }static void vmx_load_host_state (struct vcpu_vmx *vmx)
@@ -915,9 +922,6 @@ static void setup_msrs(struct vcpu_vmx *vmx) index =__find_msr_index (vmx,MSR_CSTAR );if (index >=0 )move_msr_up (vmx, index, save_nmsrs++); - index =__find_msr_index (vmx,MSR_KERNEL_GS_BASE ); -if (index >=0 ) -move_msr_up (vmx, index, save_nmsrs++);/* * MSR_K6_STAR is only needed on long mode guests, and only * if efer.sce is enabled.
@@ -929,10 +933,6 @@ static void setup_msrs(struct vcpu_vmx *vmx)#endif vmx->save_nmsrs = save_nmsrs; -#ifdef CONFIG_X86_64 - vmx->msr_offset_kernel_gs_base = -__find_msr_index (vmx,MSR_KERNEL_GS_BASE ); -#endif vmx->msr_offset_efer =__find_msr_index (vmx,MSR_EFER );if (cpu_has_vmx_msr_bitmap ()) {
@@ -990,6 +990,10 @@ static int vmx_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)case MSR_GS_BASE : data =vmcs_readl (GUEST_GS_BASE );break ; +case MSR_KERNEL_GS_BASE : +vmx_load_host_state (to_vmx (vcpu)); + data =to_vmx (vcpu)->msr_guest_kernel_gs_base; +break ;case MSR_EFER :return kvm_get_msr_common (vcpu, msr_index, pdata);#endif
@@ -1043,6 +1047,10 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)case MSR_GS_BASE :vmcs_writel (GUEST_GS_BASE , data);break ; +case MSR_KERNEL_GS_BASE : +vmx_load_host_state (vmx); + vmx->msr_guest_kernel_gs_base = data; +break ;#endif case MSR_IA32_SYSENTER_CS :vmcs_write32 (GUEST_SYSENTER_CS , data);
Signature · 1 line
1.6.4.1
PATCH 3/4 KVM: x86 shared msr infrastructure
In reply to Avi KivityThe various syscall-related MSRs are fairly expensive to switch. Currently we switch them on every vcpu preemption, which is far too often:
- if we're switching to a kernel thread (idle task, threaded interrupt, kernel-mode virtio server (vhost-net), for example) and back, then there's no need to switch those MSRs since kernel threasd won't be exiting to userspace.
- if we're switching to another guest running an identical OS, most likely those MSRs will have the same value, so there's little point in reloading them.
- if we're running the same OS on the guest and host, the MSRs will have identical values and reloading is unnecessary.
This patch uses the new user return notifiers to implement last-minute switching, and checks the msr values to avoid unnecessary reloading.
--- arch/x86/include/asm/kvm_host.h | 3 + arch/x86/kvm/Kconfig | 1 + arch/x86/kvm/x86.c | 79 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 0 deletions(-)
arch/x86/include/asm/kvm_host.h +3 -0
@@ -799,4 +799,7 @@ int kvm_cpu_has_interrupt(struct kvm_vcpu *vcpu);int kvm_arch_interrupt_allowed (struct kvm_vcpu *vcpu);int kvm_cpu_get_interrupt (struct kvm_vcpu *v); +void kvm_define_shared_msr (unsigned index,u32 msr); +void kvm_set_shared_msr (unsigned index,u64 val); +#endif /* _ASM_X86_KVM_HOST_H */
arch/x86/kvm/Kconfig +1 -0
@@ -28,6 +28,7 @@ config KVM select HAVE_KVM_IRQCHIP select HAVE_KVM_EVENTFD select KVM_APIC_ARCHITECTURE + select USER_RETURN_NOTIFIER ---help--- Support hosting fully virtualized guest machines using hardware virtualization extensions. You will need a fairly recent
arch/x86/kvm/x86.c +79 -0
103 lines not on this page · raw message
Signature · 1 line
1.6.4.1
Re: [PATCH 3/4] KVM: x86 shared msr infrastructure
Quoted 72 lines · Avi Kivity
The various syscall-related MSRs are fairly expensive to switch. Currently we switch them on every vcpu preemption, which is far too often:
- if we're switching to a kernel thread (idle task, threaded interrupt, kernel-mode virtio server (vhost-net), for example) and back, then there's no need to switch those MSRs since kernel threasd won't be exiting to userspace.
- if we're switching to another guest running an identical OS, most likely those MSRs will have the same value, so there's little point in reloading them.
- if we're running the same OS on the guest and host, the MSRs will have identical values and reloading is unnecessary.
This patch uses the new user return notifiers to implement last-minute switching, and checks the msr values to avoid unnecessary reloading.
Signed-off-by Avi Kivity <[email protected]>--- arch/x86/include/asm/kvm_host.h | 3 + arch/x86/kvm/Kconfig | 1 + arch/x86/kvm/x86.c | 79 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 0 deletions(-)arch/x86/include/asm/kvm_host.h +3
-0@@ -799,4 +799,7 @@ int kvm_cpu_has_interrupt(struct kvm_vcpu *vcpu);int kvm_arch_interrupt_allowed (struct kvm_vcpu *vcpu);int kvm_cpu_get_interrupt (struct kvm_vcpu *v); +void kvm_define_shared_msr (unsigned index,u32 msr); +void kvm_set_shared_msr (unsigned index,u64 val); +#endif /* _ASM_X86_KVM_HOST_H */ arch/x86/kvm/Kconfig +1
-0@@ -28,6 +28,7 @@ config KVM select HAVE_KVM_IRQCHIP select HAVE_KVM_EVENTFD select KVM_APIC_ARCHITECTURE + select USER_RETURN_NOTIFIER ---help--- Support hosting fully virtualized guest machines using hardware virtualization extensions. You will need a fairly recentarch/x86/kvm/x86.c +8
-0@@ -37,6 +37,7 @@#include <linux/iommu.h> #include <linux/intel-iommu.h> #include <linux/cpufreq.h> +#include <linux/user-return-notifier.h> #include <trace/events/kvm.h> #undef TRACE_INCLUDE_FILE#define CREATE_TRACE_POINTS @@ -87,6 +88,25 @@ EXPORT_SYMBOL_GPL(kvm_x86_ops);int ignore_msrs =0 ;module_param_named (ignore_msrs, ignore_msrs, bool,S_IRUGO |S_IWUSR ); +#define KVM_NR_SHARED_MSRS 16 + +struct kvm_shared_msrs_global { +int nr; +struct kvm_shared_msr { +u32 msr; +u64 value;
arch/x86/kvm/x86.c EXPORT_SYMBOL_GPL(kvm_x86_ops); · line 107
Quoted 10 lines
arch/x86/kvm/x86.c +10
-0+ } msrs[KVM_NR_SHARED_MSRS ]; +}; + +struct kvm_shared_msrs { +struct user_return_notifier urn; +bool registered; +u64 current_value[KVM_NR_SHARED_MSRS ]; +}; + +static struct kvm_shared_msrs_global __read_mostly shared_msrs_global;
Does this assume the MSRs in question are consistent across CPUs?
I guess that is not true with arch_prctl(ARCH_SET_GS/ARCH_GET_GS) ?
Re: [PATCH 3/4] KVM: x86 shared msr infrastructure
Quoted 4 lines · Marcelo Tosatti
Quoted 2 lines
+static struct
kvm_shared_msrs_global__read_mostlyshared_msrs_global;Does this assume the MSRs in question are consistent across CPUs?
Yes. And they are.
Quoted 2 lines
I guess that is not true with
arch_prctl(ARCH_SET_GS/ARCH_GET_GS)?
These aren't shared MSRs and are saved and restored independently.
Signature · 2 lines
I have a truly marvellous patch that fixes the bug which this signature is too narrow to contain.
PATCH 4/4 KVM: VMX: Use shared msr infrastructure
In reply to Avi KivityInstead of reloading syscall MSRs on every preemption, use the new shared msr infrastructure to reload them at the last possible minute (just before exit to userspace).
Improves vcpu/idle/vcpu switches by about 2000 cycles (when EFER needs to be reloaded as well).
--- arch/x86/kvm/vmx.c | 108 +++++++++++++++++++-------------------------------- 1 files changed, 40 insertions(+), 68 deletions(-)
arch/x86/kvm/vmx.c +40 -68
@@ -67,6 +67,11 @@ struct vmcs {char data[0 ]; }; +struct shared_msr_entry { +unsigned index; +u64 data; +}; +struct vcpu_vmx {struct kvm_vcpu vcpu;struct list_head local_vcpus_link;
@@ -74,8 +79,7 @@ struct vcpu_vmx {int launched;u8 fail;u32 idt_vectoring_info; -struct kvm_msr_entry *guest_msrs; -struct kvm_msr_entry *host_msrs; +struct shared_msr_entry *guest_msrs;int nmsrs;int save_nmsrs;int msr_offset_efer;
@@ -89,7 +93,6 @@ struct vcpu_vmx {u16 fs_sel, gs_sel, ldt_sel;int gs_ldt_reload_needed;int fs_reload_needed; -int guest_efer_loaded; } host_state;struct {int vm86_active;
@@ -176,6 +179,8 @@ static struct kvm_vmx_segment_field { VMX_SEGMENT_FIELD(LDTR), }; +static u64 host_efer; +static void ept_save_pdptrs (struct kvm_vcpu *vcpu);/*
@@ -190,22 +195,6 @@ static const u32 vmx_msr_index[] = { };#define NR_VMX_MSR ARRAY_SIZE(vmx_msr_index) -static void load_msrs (struct kvm_msr_entry *e,int n) -{ -int i; - -for (i =0 ; i < n; ++i) -wrmsrl (e[i].index, e[i].data); -} - -static void save_msrs (struct kvm_msr_entry *e,int n) -{ -int i; - -for (i =0 ; i < n; ++i) -rdmsrl (e[i].index, e[i].data); -} -static inline int is_page_fault (u32 intr_info) { return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
@@ -348,7 +337,7 @@ static int __find_msr_index(struct vcpu_vmx *vmx, u32 msr)int i;for (i =0 ; i < vmx->nmsrs; ++i) -if (vmx->guest_msrs[i].index == msr) +if (vmx_msr_index[vmx->guest_msrs[i].index] == msr)return i;return -1 ; }
@@ -379,7 +368,7 @@ static inline void __invept(int ext, u64 eptp, gpa_t gpa) : : "a" (&operand), "c" (ext) : "cc", "memory"); } -static struct kvm_msr_entry *find_msr_entry (struct vcpu_vmx *vmx,u32 msr) +static struct shared_msr_entry *find_msr_entry (struct vcpu_vmx *vmx,u32 msr) {int i;
@@ -570,17 +559,15 @@ static void reload_tss(void)load_TR_desc (); } -static void load_transition_efer (struct vcpu_vmx *vmx) +static bool update_transition_efer (struct vcpu_vmx *vmx) {int efer_offset = vmx->msr_offset_efer; -u64 host_efer;u64 guest_efer;u64 ignore_bits;if (efer_offset <0 ) -return ; - host_efer = vmx->host_msrs[efer_offset].data; - guest_efer = vmx->guest_msrs[efer_offset].data; +return false; + guest_efer = vmx->vcpu.arch.shadow_efer;/* * NX is emulated; LMA and LME handled by hardware; SCE meaninless
@@ -594,26 +581,18 @@ static void load_transition_efer(struct vcpu_vmx *vmx) ignore_bits &= ~(u64 )EFER_SCE ;#endif if ((guest_efer & ~ignore_bits) == (host_efer & ~ignore_bits)) -return ; +return false; - vmx->host_state.guest_efer_loaded =1 ; guest_efer &= ~ignore_bits; guest_efer |= host_efer & ignore_bits; -wrmsrl (MSR_EFER , guest_efer); - vmx->vcpu.stat.efer_reload++; -} - -static void reload_host_efer (struct vcpu_vmx *vmx) -{ -if (vmx->host_state.guest_efer_loaded) { - vmx->host_state.guest_efer_loaded =0 ; -load_msrs (vmx->host_msrs + vmx->msr_offset_efer,1 ); - } + vmx->guest_msrs[efer_offset].data = guest_efer; +return true; }static void vmx_save_host_state (struct kvm_vcpu *vcpu) {struct vcpu_vmx *vmx =to_vmx (vcpu); +int i;if (vmx->host_state.loaded)return ;
@@ -655,8 +634,8 @@ static void vmx_save_host_state(struct kvm_vcpu *vcpu) wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base); }#endif -load_msrs (vmx->guest_msrs, vmx->save_nmsrs); -load_transition_efer (vmx); +for (i =0 ; i < vmx->save_nmsrs; ++i) +kvm_set_shared_msr (i, vmx->guest_msrs[i].data); }static void __vmx_load_host_state (struct vcpu_vmx *vmx)
@@ -684,9 +663,6 @@ static void __vmx_load_host_state(struct vcpu_vmx *vmx)local_irq_restore (flags); }reload_tss (); -save_msrs (vmx->guest_msrs, vmx->save_nmsrs); -load_msrs (vmx->host_msrs, vmx->save_nmsrs); -reload_host_efer (vmx);#ifdef CONFIG_X86_64 if (is_long_mode (&vmx->vcpu)) {rdmsrl (MSR_KERNEL_GS_BASE , vmx->msr_guest_kernel_gs_base);
@@ -886,14 +862,11 @@ static void vmx_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,#ifdef CONFIG_X86_64 static void move_msr_up (struct vcpu_vmx *vmx,int from,int to) { -struct kvm_msr_entry tmp; +struct shared_msr_entry tmp; tmp = vmx->guest_msrs[to]; vmx->guest_msrs[to] = vmx->guest_msrs[from]; vmx->guest_msrs[from] = tmp; - tmp = vmx->host_msrs[to]; - vmx->host_msrs[to] = vmx->host_msrs[from]; - vmx->host_msrs[from] = tmp; }#endif
@@ -904,15 +877,13 @@ static void move_msr_up(struct vcpu_vmx *vmx, int from, int to)*/ static void setup_msrs (struct vcpu_vmx *vmx) { -int save_nmsrs; +int save_nmsrs, index;unsigned long *msr_bitmap;vmx_load_host_state (vmx); save_nmsrs =0 ;#ifdef CONFIG_X86_64 if (is_long_mode (&vmx->vcpu)) { -int index; - index =__find_msr_index (vmx,MSR_SYSCALL_MASK );if (index >=0 )move_msr_up (vmx, index, save_nmsrs++);
@@ -931,9 +902,11 @@ static void setup_msrs(struct vcpu_vmx *vmx) move_msr_up(vmx, index, save_nmsrs++); }#endif - vmx->save_nmsrs = save_nmsrs; + vmx->msr_offset_efer = index =__find_msr_index (vmx,MSR_EFER ); +if (index >=0 &&update_transition_efer (vmx)) +move_msr_up (vmx, index, save_nmsrs++); - vmx->msr_offset_efer =__find_msr_index (vmx,MSR_EFER ); + vmx->save_nmsrs = save_nmsrs;if (cpu_has_vmx_msr_bitmap ()) {if (is_long_mode (&vmx->vcpu))
@@ -975,7 +948,7 @@ static void guest_write_tsc(u64 guest_tsc, u64 host_tsc)static int vmx_get_msr (struct kvm_vcpu *vcpu,u32 msr_index,u64 *pdata) {u64 data; -struct kvm_msr_entry *msr; +struct shared_msr_entry *msr;if (!pdata) {printk (KERN_ERR "BUG: get_msr called with NULL pdata\n" );
@@ -994,9 +967,9 @@ static int vmx_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)vmx_load_host_state (to_vmx (vcpu)); data =to_vmx (vcpu)->msr_guest_kernel_gs_base;break ; +#endif case MSR_EFER :return kvm_get_msr_common (vcpu, msr_index, pdata); -#endif case MSR_IA32_TSC : data =guest_read_tsc ();break ;
@@ -1010,6 +983,7 @@ static int vmx_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata) data =vmcs_readl (GUEST_SYSENTER_ESP );break ;default : +vmx_load_host_state (to_vmx (vcpu)); msr =find_msr_entry (to_vmx (vcpu), msr_index);if (msr) {vmx_load_host_state (to_vmx (vcpu));
@@ -1031,7 +1005,7 @@ static int vmx_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)static int vmx_set_msr (struct kvm_vcpu *vcpu,u32 msr_index,u64 data) {struct vcpu_vmx *vmx =to_vmx (vcpu); -struct kvm_msr_entry *msr; +struct shared_msr_entry *msr;u64 host_tsc;int ret =0 ;
@@ -1543,8 +1517,10 @@ continue_rmode:static void vmx_set_efer (struct kvm_vcpu *vcpu,u64 efer) {struct vcpu_vmx *vmx =to_vmx (vcpu); -struct kvm_msr_entry *msr =find_msr_entry (vmx,MSR_EFER ); +struct shared_msr_entry *msr =find_msr_entry (vmx,MSR_EFER ); +if (!msr) +return ; vcpu->arch.shadow_efer = efer;if (!msr)return ;
@@ -2383,10 +2359,8 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx)if (wrmsr_safe (index, data_low, data_high) <0 )continue ; data = data_low | ((u64 )data_high <<32 ); - vmx->host_msrs[j].index = index; - vmx->host_msrs[j].reserved =0 ; - vmx->host_msrs[j].data = data; - vmx->guest_msrs[j] = vmx->host_msrs[j]; + vmx->guest_msrs[j].index = i; + vmx->guest_msrs[j].data =0 ; ++vmx->nmsrs; }
@@ -3774,7 +3748,6 @@ static void vmx_free_vcpu(struct kvm_vcpu *vcpu)__clear_bit (vmx->vpid, vmx_vpid_bitmap);spin_unlock (&vmx_vpid_lock);vmx_free_vmcs (vcpu); -kfree (vmx->host_msrs);kfree (vmx->guest_msrs);kvm_vcpu_uninit (vcpu);kmem_cache_free (kvm_vcpu_cache, vmx);
@@ -3801,10 +3774,6 @@ static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id) goto uninit_vcpu; } - vmx->host_msrs =kmalloc (PAGE_SIZE ,GFP_KERNEL ); -if (!vmx->host_msrs) - goto free_guest_msrs; - vmx->vmcs =alloc_vmcs ();if (!vmx->vmcs) goto free_msrs;
@@ -3835,8 +3804,6 @@ static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id) free_vmcs:free_vmcs (vmx->vmcs); free_msrs: -kfree (vmx->host_msrs); -free_guest_msrs:kfree (vmx->guest_msrs); uninit_vcpu:kvm_vcpu_uninit (&vmx->vcpu);
@@ -3986,7 +3953,12 @@ static struct kvm_x86_ops vmx_x86_ops = {static int __init vmx_init (void ) { -int r; +int r, i; + +rdmsrl_safe (MSR_EFER , &host_efer); + +for (i =0 ; i <NR_VMX_MSR ; ++i) +kvm_define_shared_msr (i, vmx_msr_index[i]); vmx_io_bitmap_a = (unsigned long *)__get_free_page (GFP_KERNEL );if (!vmx_io_bitmap_a)
Signature · 1 line
1.6.4.1
Avi Kivity
AppliedQuoted 25 lines · Avi Kivity
The various x86 syscall related MSRs (
MSR_LSTARand friends, EFER when SCE needs to be updated) are fairly expensive to read or write. Since different operating systems can set different values for these MSRs, KVM needs to reload them when switching to a different guest or to the host.Switching on every guest entry/exit is too expensive, so KVM reloads on guest preemption, which is a lot rarer. Even so, preemption based reload is suboptimal:
- if we're switching to a kernel thread and back, there's no need to reload the MSRs. Examples of kernel threads we're likely to switch to are:
- the idle task
- a threaded interrupt handler
- a kernel-mode virtio server (vhost-net)- if we're switching to a guest running the same OS, the MSRs will have the same values and there's no need to reload them
- if the guest and host run the same OS, again the MSRs need not be reloaded.
This patchset implements just-in-time reloads to defer them to the last possible instant. When we do reload, we check whether the values have in fact changed and reload conditionally.
Quoted 8 lines
For the just-in-time reloads the first patch implements "user return notifiers", a callback invoked just before return to userspace. This has been written so that there is no code impact if KVM is not configured, and no runtime impact if KVM is not running.
The patchset improves guest/idle/guest switches by about 2000 cycles.
I've applied this to kvm.git master.
Signature · 1 line
error compiling committee.c: too many arguments to function