x86/spinlocks changes for v3.12
Subsystems (9)
Linus,
Please pull the latest x86-spinlocks-for-linus git tree from:
git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git x86-spinlocks-for-linus
HEAD: 36bd621337c91a1ecda588e5bbbae8dd9698bae7 x86/kvm/guest: Fix sparse warning: "symbol 'klock_waiting' was not declared as static"
The biggest change here are paravirtualized ticket spinlocks (PV spinlocks), which bring a nice speedup on various benchmarks. (The KVM host side will come to you via the KVM tree.)
out-of-topic modifications in x86-spinlocks-for-linus:
------------------------------------------------------include/linux/jump_label.h # 851cf6e: jump_label: Split jumplabel ratel
include/linux/jump_label_ratelimit.h# 851cf6e: jump_label: Split jumplabel ratel
include/linux/perf_event.h # 851cf6e: jump_label: Split jumplabel ratel include/uapi/linux/kvm_para.h # 3a3bb00: kvm uapi: Add KICK_CPU and PV_UNH kernel/jump_label.c # 851cf6e: jump_label: Split jumplabel ratel
Thanks,
Ingo
------------------>
Andrew Jones (1):
jump_label: Split jumplabel ratelimit
Jeremy Fitzhardinge (9):
x86, spinlock: Replace pv spinlocks with pv ticketlocks
x86, ticketlock: Collapse a layer of functions
xen: Defer spinlock setup until boot CPU setup
xen, pvticketlock: Xen implementation for PV ticket locks
xen, pvticketlocks: Add xen_nopvspin parameter to disable xen pv ticketlocks x86, pvticketlock: Use callee-save for lock_spinning
x86, pvticketlock: When paravirtualizing ticket locks, increment by 2 x86, ticketlock: Add slowpath logic
xen, pvticketlock: Allow interrupts to be enabled while blocking
Raghavendra K T (3):
x86, ticketlock: Don't inline _spin_unlock when using paravirt spinlocks kvm uapi: Add KICK_CPU and PV_UNHALT definition to uapi
x86/kvm/guest: Fix sparse warning: "symbol 'klock_waiting' was not declared as static"
Srivatsa Vaddagiri (2):
kvm guest: Add configuration support to enable debug information for KVM Guests
kvm: Paravirtual ticketlocks support for linux guests running on KVM hypervisor
arch/x86/Kconfig | 10 + arch/x86/include/asm/kvm_para.h | 14 +- arch/x86/include/asm/paravirt.h | 32 +-- arch/x86/include/asm/paravirt_types.h | 14 +- arch/x86/include/asm/spinlock.h | 128 +++++++---- arch/x86/include/asm/spinlock_types.h | 16 +- arch/x86/include/uapi/asm/kvm_para.h | 1 + arch/x86/kernel/kvm.c | 262 +++++++++++++++++++++++ arch/x86/kernel/paravirt-spinlocks.c | 18 +- arch/x86/xen/smp.c | 2 +- arch/x86/xen/spinlock.c | 387 +++++++++++----------------------- include/linux/jump_label.h | 28 +-- include/linux/jump_label_ratelimit.h | 34 +++ include/linux/perf_event.h | 1 + include/uapi/linux/kvm_para.h | 1 + kernel/jump_label.c | 1 + 16 files changed, 569 insertions(+), 380 deletions(-) create mode 100644 include/linux/jump_label_ratelimit.h
arch/x86/Kconfig +10 -0
@@ -632,6 +632,7 @@ config PARAVIRT_DEBUG config PARAVIRT_SPINLOCKS bool "Paravirtualization layer for spinlocks" depends on PARAVIRT && SMP + select UNINLINE_SPIN_UNLOCK ---help--- Paravirtualized spinlocks allow a pvops backend to replace the spinlock implementation with something virtualization-friendly
@@ -656,6 +657,15 @@ config KVM_GUEST underlying device model, the host provides the guest with timing infrastructure such as time of day, and system time +config KVM_DEBUG_FS + bool "Enable debug information for KVM Guests in debugfs" + depends on KVM_GUEST && DEBUG_FS + default n + ---help--- + This option enables collection of various statistics for KVM guest. + Statistics are displayed in debugfs filesystem. Enabling this option + may incur significant overhead. + source "arch/x86/lguest/Kconfig" config PARAVIRT_TIME_ACCOUNTING
arch/x86/include/asm/kvm_para.h +12 -2
@@ -118,10 +118,20 @@ void kvm_async_pf_task_wait(u32 token);void kvm_async_pf_task_wake (u32 token);u32 kvm_read_and_reset_pf_reason (void );extern void kvm_disable_steal_time (void ); -#else -#define kvm_guest_init () do { } while (0) + +#ifdef CONFIG_PARAVIRT_SPINLOCKS +void __init kvm_spinlock_init (void ); +#else /* !CONFIG_PARAVIRT_SPINLOCKS */ +static inline void kvm_spinlock_init (void ) +{ +} +#endif /* CONFIG_PARAVIRT_SPINLOCKS */ + +#else /* CONFIG_KVM_GUEST */ +#define kvm_guest_init () do {} while (0)#define kvm_async_pf_task_wait (T ) do {} while(0)#define kvm_async_pf_task_wake (T ) do {} while(0) +static inline u32 kvm_read_and_reset_pf_reason (void ) {return 0 ;
arch/x86/include/asm/paravirt.h +6 -26
@@ -712,36 +712,16 @@ static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx,#if defined(CONFIG_SMP ) && defined(CONFIG_PARAVIRT_SPINLOCKS ) -static inline int arch_spin_is_locked (struct arch_spinlock *lock) +static __always_inline void __ticket_lock_spinning(struct arch_spinlock *lock, +__ticket_t ticket) { -return PVOP_CALL1 (int, pv_lock_ops.spin_is_locked, lock); +PVOP_VCALLEE2 (pv_lock_ops.lock_spinning, lock, ticket); } -static inline int arch_spin_is_contended (struct arch_spinlock *lock) +static __always_inline void __ticket_unlock_kick(struct arch_spinlock *lock, +__ticket_t ticket) { -return PVOP_CALL1 (int, pv_lock_ops.spin_is_contended, lock); -} -#define arch_spin_is_contended arch_spin_is_contended - -static __always_inline void arch_spin_lock(struct arch_spinlock *lock) -{ -PVOP_VCALL1 (pv_lock_ops.spin_lock, lock); -} - -static __always_inline void arch_spin_lock_flags(struct arch_spinlock *lock, -unsigned long flags) -{ -PVOP_VCALL2 (pv_lock_ops.spin_lock_flags, lock, flags); -} - -static __always_inline int arch_spin_trylock(struct arch_spinlock *lock) -{ -return PVOP_CALL1 (int, pv_lock_ops.spin_trylock, lock); -} - -static __always_inline void arch_spin_unlock(struct arch_spinlock *lock) -{ -PVOP_VCALL1 (pv_lock_ops.spin_unlock, lock); +PVOP_VCALL2 (pv_lock_ops.unlock_kick, lock, ticket); }#endif
arch/x86/include/asm/paravirt_types.h +8 -6
@@ -327,13 +327,15 @@ struct pv_mmu_ops { };struct arch_spinlock ; +#ifdef CONFIG_SMP +#include <asm/spinlock_types.h> +#else +typedef u16 __ticket_t ; +#endif +struct pv_lock_ops { -int (*spin_is_locked)(struct arch_spinlock *lock); -int (*spin_is_contended)(struct arch_spinlock *lock); -void (*spin_lock)(struct arch_spinlock *lock); -void (*spin_lock_flags)(struct arch_spinlock *lock,unsigned long flags); -int (*spin_trylock)(struct arch_spinlock *lock); -void (*spin_unlock)(struct arch_spinlock *lock); +struct paravirt_callee_save lock_spinning; +void (*unlock_kick)(struct arch_spinlock *lock,__ticket_t ticket); };/* This contains all the paravirt structures: we get a convenient
arch/x86/include/asm/spinlock.h +87 -41
193 lines not on this page · raw message
arch/x86/include/asm/spinlock_types.h +11 -5
@@ -1,13 +1,17 @@#ifndef _ASM_X86_SPINLOCK_TYPES_H#define _ASM_X86_SPINLOCK_TYPES_H -#ifndef __LINUX_SPINLOCK_TYPES_H -# error "please don't include this file directly" -#endif -#include <linux/types.h> -#if (CONFIG_NR_CPUS <256 ) +#ifdef CONFIG_PARAVIRT_SPINLOCKS +#define __TICKET_LOCK_INC 2 +#define TICKET_SLOWPATH_FLAG ((__ticket_t)1) +#else +#define __TICKET_LOCK_INC 1 +#define TICKET_SLOWPATH_FLAG ((__ticket_t)0) +#endif + +#if (CONFIG_NR_CPUS < (256 / __TICKET_LOCK_INC))typedef u8 __ticket_t ;typedef u16 __ticketpair_t ;#else
@@ -15,6 +19,8 @@ typedef u16 __ticket_t;typedef u32 __ticketpair_t ;#endif +#define TICKET_LOCK_INC ((__ticket_t)__TICKET_LOCK_INC) +#define TICKET_SHIFT (sizeof(__ticket_t) * 8) typedef struct arch_spinlock {
arch/x86/include/uapi/asm/kvm_para.h +1 -0
@@ -23,6 +23,7 @@#define KVM_FEATURE_ASYNC_PF 4#define KVM_FEATURE_STEAL_TIME 5#define KVM_FEATURE_PV_EOI 6 +#define KVM_FEATURE_PV_UNHALT 7/* The last 8 bits are used to indicate how to interpret the flags field * in pvclock structure. If no bits are set, all flags are ignored.
arch/x86/kernel/kvm.c +262 -0
277 lines not on this page · raw message
arch/x86/kernel/paravirt-spinlocks.c +5 -13
@@ -4,25 +4,17 @@*/ #include <linux/spinlock.h> #include <linux/module.h> +#include <linux/jump_label.h> #include <asm/paravirt.h> -static inline void -default_spin_lock_flags (arch_spinlock_t *lock,unsigned long flags) -{ -arch_spin_lock (lock); -} -struct pv_lock_ops pv_lock_ops = { #ifdef CONFIG_SMP - .spin_is_locked = __ticket_spin_is_locked, - .spin_is_contended = __ticket_spin_is_contended, - - .spin_lock = __ticket_spin_lock, - .spin_lock_flags = default_spin_lock_flags, - .spin_trylock = __ticket_spin_trylock, - .spin_unlock = __ticket_spin_unlock, + .lock_spinning =__PV_IS_CALLEE_SAVE (paravirt_nop), + .unlock_kick = paravirt_nop, #endif };EXPORT_SYMBOL (pv_lock_ops); +struct static_key paravirt_ticketlocks_enabled =STATIC_KEY_INIT_FALSE ; +EXPORT_SYMBOL (paravirt_ticketlocks_enabled);
arch/x86/xen/smp.c +1 -1
@@ -279,6 +279,7 @@ static void __init xen_smp_prepare_boot_cpu(void)xen_filter_cpu_maps ();xen_setup_vcpu_info_placement (); +xen_init_spinlocks (); }static void __init xen_smp_prepare_cpus (unsigned int max_cpus)
@@ -680,7 +681,6 @@ void __init xen_smp_init(void) { smp_ops = xen_smp_ops;xen_fill_possible_map (); -xen_init_spinlocks (); }static void __init xen_hvm_smp_prepare_cpus (unsigned int max_cpus)
arch/x86/xen/spinlock.c +128 -259
499 lines not on this page · raw message
include/linux/jump_label.h +1 -27
@@ -48,7 +48,6 @@#include <linux/types.h> #include <linux/compiler.h> -#include <linux/workqueue.h> #if defined(CC_HAVE_ASM_GOTO ) && defined(CONFIG_JUMP_LABEL )
@@ -61,12 +60,6 @@ struct static_key { #endif }; -struct static_key_deferred { -struct static_key key; -unsigned long timeout; -struct delayed_work work; -}; -# include <asm/jump_label.h> # define HAVE_JUMP_LABEL #endif /* CC_HAVE_ASM_GOTO && CONFIG_JUMP_LABEL */
@@ -78,6 +71,7 @@ enum jump_label_type {struct module ; +#include <linux/atomic.h> #ifdef HAVE_JUMP_LABEL #define JUMP_LABEL_TRUE_BRANCH 1UL
@@ -119,10 +113,7 @@ extern void arch_jump_label_transform_static(struct jump_entry *entry,extern int jump_label_text_reserved (void *start,void *end);extern void static_key_slow_inc (struct static_key *key);extern void static_key_slow_dec (struct static_key *key); -extern void static_key_slow_dec_deferred (struct static_key_deferred *key);extern void jump_label_apply_nops (struct module *mod); -extern void -jump_label_rate_limit (struct static_key_deferred *key,unsigned long rl);#define STATIC_KEY_INIT_TRUE ((struct static_key) \ { .enabled = ATOMIC_INIT(1), .entries = (void *)1 })
@@ -131,8 +122,6 @@ jump_label_rate_limit(struct static_key_deferred *key, unsigned long rl);#else /* !HAVE_JUMP_LABEL */ -#include <linux/atomic.h> -struct static_key {atomic_t enabled; };
@@ -141,10 +130,6 @@ static __always_inline void jump_label_init(void) { } -struct static_key_deferred { -struct static_key key; -}; - static __always_inline bool static_key_false(struct static_key *key) {if (unlikely (atomic_read (&key->enabled)) >0 )
@@ -169,11 +154,6 @@ static inline void static_key_slow_dec(struct static_key *key)atomic_dec (&key->enabled); } -static inline void static_key_slow_dec_deferred (struct static_key_deferred *key) -{ -static_key_slow_dec (&key->key); -} -static inline int jump_label_text_reserved (void *start,void *end) {return 0 ;
@@ -187,12 +167,6 @@ static inline int jump_label_apply_nops(struct module *mod)return 0 ; } -static inline void -jump_label_rate_limit (struct static_key_deferred *key, -unsigned long rl) -{ -} -#define STATIC_KEY_INIT_TRUE ((struct static_key) \ { .enabled = ATOMIC_INIT(1) })#define STATIC_KEY_INIT_FALSE ((struct static_key) \
include/linux/jump_label_ratelimit.h new file +34 -0
@@ -0,0 +1,34 @@ +#ifndef _LINUX_JUMP_LABEL_RATELIMIT_H +#define _LINUX_JUMP_LABEL_RATELIMIT_H + +#include <linux/jump_label.h> +#include <linux/workqueue.h> + +#if defined(CC_HAVE_ASM_GOTO ) && defined(CONFIG_JUMP_LABEL ) +struct static_key_deferred { +struct static_key key; +unsigned long timeout; +struct delayed_work work; +}; +#endif + +#ifdef HAVE_JUMP_LABEL +extern void static_key_slow_dec_deferred (struct static_key_deferred *key); +extern void +jump_label_rate_limit (struct static_key_deferred *key,unsigned long rl); + +#else /* !HAVE_JUMP_LABEL */ +struct static_key_deferred { +struct static_key key; +}; +static inline void static_key_slow_dec_deferred (struct static_key_deferred *key) +{ +static_key_slow_dec (&key->key); +} +static inline void +jump_label_rate_limit (struct static_key_deferred *key, +unsigned long rl) +{ +} +#endif /* HAVE_JUMP_LABEL */ +#endif /* _LINUX_JUMP_LABEL_RATELIMIT_H */
include/linux/perf_event.h +1 -0
@@ -48,6 +48,7 @@ struct perf_guest_info_callbacks {#include <linux/cpu.h> #include <linux/irq_work.h> #include <linux/static_key.h> +#include <linux/jump_label_ratelimit.h> #include <linux/atomic.h> #include <linux/sysfs.h> #include <linux/perf_regs.h>
include/uapi/linux/kvm_para.h +1 -0
@@ -19,6 +19,7 @@#define KVM_HC_MMU_OP 2#define KVM_HC_FEATURES 3#define KVM_HC_PPC_MAP_MAGIC_PAGE 4 +#define KVM_HC_KICK_CPU 5/* * hypercalls use architecture specific
kernel/jump_label.c +1 -0
@@ -13,6 +13,7 @@#include <linux/sort.h> #include <linux/err.h> #include <linux/static_key.h> +#include <linux/jump_label_ratelimit.h> #ifdef HAVE_JUMP_LABEL