KVM: PPC: Book3S HV P9: Reduce guest entry/exit
Subsystems (2)
Nicholas Piggin
reviewercover KVM: PPC: Book3S HV P9: Reduce guest entry/exit
This goes on top of the previous speedup series. The previous series is mostly invovled with reducing the cost of SPR accesses. This one starts to look beyond those, to atomics, barriers, and other logic that can be reduced. After this series the P9 path uses very few things from the vcore structure. This saves several hundred cycles for guest entry/exit on a POWER9.
Thanks,
Nick
Nicholas Piggin (6):
KVM: PPC: Book3S HV P9: Add unlikely annotation for !mmu_ready
KVM: PPC: Book3S HV P9: Avoid cpu_in_guest atomics on entry and exit
KVM: PPC: Book3S HV P9: Remove most of the vcore logic
KVM: PPC: Book3S HV P9: Tidy kvmppc_create_dtl_entry
KVM: PPC: Book3S HV P9: Stop using vc->dpdes
KVM: PPC: Book3S HV P9: Remove subcore HMI handling
arch/powerpc/include/asm/kvm_book3s_64.h | 1 - arch/powerpc/include/asm/kvm_host.h | 1 - arch/powerpc/kvm/book3s_hv.c | 250 +++++++++++++---------- arch/powerpc/kvm/book3s_hv_builtin.c | 2 + arch/powerpc/kvm/book3s_hv_hmi.c | 7 +- arch/powerpc/kvm/book3s_hv_p9_entry.c | 35 +++- arch/powerpc/kvm/book3s_hv_ras.c | 4 + 7 files changed, 185 insertions(+), 115 deletions(-)
Signature · 1 line
2.23.0
Nicholas Piggin
reviewercover KVM: PPC: Book3S HV P9: Reduce guest entry/exit
This goes on top of the previous speedup series. The previous series is mostly invovled with reducing the cost of SPR accesses. This one starts to look beyond those, to atomics, barriers, and other logic that can be reduced. After this series the P9 path uses very few things from the vcore structure. This saves several hundred cycles for guest entry/exit on a POWER9.
Thanks,
Nick
Nicholas Piggin (6):
KVM: PPC: Book3S HV P9: Add unlikely annotation for !mmu_ready
KVM: PPC: Book3S HV P9: Avoid cpu_in_guest atomics on entry and exit
KVM: PPC: Book3S HV P9: Remove most of the vcore logic
KVM: PPC: Book3S HV P9: Tidy kvmppc_create_dtl_entry
KVM: PPC: Book3S HV P9: Stop using vc->dpdes
KVM: PPC: Book3S HV P9: Remove subcore HMI handling
arch/powerpc/include/asm/kvm_book3s_64.h | 1 - arch/powerpc/include/asm/kvm_host.h | 1 - arch/powerpc/kvm/book3s_hv.c | 250 +++++++++++++---------- arch/powerpc/kvm/book3s_hv_builtin.c | 2 + arch/powerpc/kvm/book3s_hv_hmi.c | 7 +- arch/powerpc/kvm/book3s_hv_p9_entry.c | 35 +++- arch/powerpc/kvm/book3s_hv_ras.c | 4 + 7 files changed, 185 insertions(+), 115 deletions(-)
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerRFC PATCH 1/6 KVM: PPC: Book3S HV P9: Add unlikely annotation for !mmu_ready
The mmu will almost always be ready.
--- arch/powerpc/kvm/book3s_hv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/powerpc/kvm/book3s_hv.c +1 -1
@@ -4367,7 +4367,7 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit, vc->runner = vcpu;/* See if the MMU is ready to go */ -if (!kvm->arch.mmu_ready) { +if (unlikely (!kvm->arch.mmu_ready)) { r =kvmhv_setup_mmu (vcpu);if (r) { run->exit_reason =KVM_EXIT_FAIL_ENTRY ;
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerRFC PATCH 2/6 KVM: PPC: Book3S HV P9: Avoid cpu_in_guest atomics on entry and exit
In reply to Nicholas Piggincpu_in_guest is set to determine if a CPU needs to be IPI'ed to exit the guest and notice the need_tlb_flush bit.
This can be implemented as a global per-CPU pointer to the currently running guest instead of per-guest cpumasks, saving 2 atomics per entry/exit. P7/8 doesn't require cpu_in_guest, nor does a nested HV (only the L0 does), so move it to the P9 HV path.
--- arch/powerpc/include/asm/kvm_book3s_64.h | 1 - arch/powerpc/include/asm/kvm_host.h | 1 - arch/powerpc/kvm/book3s_hv.c | 38 +++++++++++++----------- 3 files changed, 21 insertions(+), 19 deletions(-)
arch/powerpc/include/asm/kvm_book3s_64.h +0 -1
@@ -43,7 +43,6 @@ struct kvm_nested_guest {struct mutex tlb_lock;/* serialize page faults and tlbies */ struct kvm_nested_guest *next;cpumask_t need_tlb_flush; -cpumask_t cpu_in_guest;short prev_cpu[NR_CPUS ];u8 radix;/* is this nested guest radix */ };
arch/powerpc/include/asm/kvm_host.h +0 -1
@@ -288,7 +288,6 @@ struct kvm_arch {u32 online_vcores;atomic_t hpte_mod_interest;cpumask_t need_tlb_flush; -cpumask_t cpu_in_guest;u8 radix;u8 fwnmi_enabled;u8 secure_guest;
arch/powerpc/kvm/book3s_hv.c +21 -17
@@ -2981,30 +2981,33 @@ static void kvmppc_release_hwthread(int cpu) tpaca->kvm_hstate.kvm_split_mode =NULL ; } +static DEFINE_PER_CPU(struct kvm *, cpu_in_guest); +static void radix_flush_cpu (struct kvm *kvm,int cpu,struct kvm_vcpu *vcpu) {struct kvm_nested_guest *nested = vcpu->arch.nested; -cpumask_t *cpu_in_guest;int i; cpu =cpu_first_tlb_thread_sibling (cpu); -if (nested) { +if (nested)cpumask_set_cpu (cpu, &nested->need_tlb_flush); - cpu_in_guest = &nested->cpu_in_guest; - }else { +else cpumask_set_cpu (cpu, &kvm->arch.need_tlb_flush); - cpu_in_guest = &kvm->arch.cpu_in_guest; - }/* -* Make sure setting of bit in need_tlb_flush precedes -* testing of cpu_in_guest bits. The matching barrier on -* the other side is the first smp_mb() in kvmppc_run_core(). +* Make sure setting of bit in need_tlb_flush precedes testing of +* cpu_in_guest. The matching barrier on the other side is hwsync +* when switching to guest MMU mode, which happens between +* cpu_in_guest being set to the guest kvm, and need_tlb_flush bit +* being tested. */ smp_mb ();for (i = cpu; i <=cpu_last_tlb_thread_sibling (cpu); - i +=cpu_tlb_thread_sibling_step ()) -if (cpumask_test_cpu (i, cpu_in_guest)) + i +=cpu_tlb_thread_sibling_step ()) { +struct kvm *running = *per_cpu_ptr (&cpu_in_guest, i); + +if (running = kvm)smp_call_function_single (i, do_nothing,NULL ,1 ); + } }static void do_migrate_away_vcpu (void *arg)
@@ -3072,7 +3075,6 @@ static void kvmppc_start_thread(struct kvm_vcpu *vcpu, struct kvmppc_vcore *vc) {int cpu;struct paca_struct *tpaca; -struct kvm *kvm = vc->kvm; cpu = vc->pcpu;if (vcpu) {
@@ -3083,7 +3085,6 @@ static void kvmppc_start_thread(struct kvm_vcpu *vcpu, struct kvmppc_vcore *vc) cpu += vcpu->arch.ptid; vcpu->cpu = vc->pcpu; vcpu->arch.thread_cpu = cpu; -cpumask_set_cpu (cpu, &kvm->arch.cpu_in_guest); } tpaca = paca_ptrs[cpu]; tpaca->kvm_hstate.kvm_vcpu = vcpu;
@@ -3801,7 +3802,6 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc)kvmppc_release_hwthread (pcpu + i);if (sip && sip->napped[i])kvmppc_ipi_thread (pcpu + i); -cpumask_clear_cpu (pcpu + i, &vc->kvm->arch.cpu_in_guest); }spin_unlock (&vc->lock);
@@ -3968,8 +3968,14 @@ static int kvmhv_p9_guest_entry(struct kvm_vcpu *vcpu, u64 time_limit, } }else { +struct kvm *kvm = vcpu->kvm; +kvmppc_xive_push_vcpu (vcpu); + +__this_cpu_write (cpu_in_guest, kvm); trap =kvmhv_vcpu_entry_p9 (vcpu, time_limit, lpcr, tb); +__this_cpu_write (cpu_in_guest,NULL ); +if (trap =BOOK3S_INTERRUPT_SYSCALL && !vcpu->arch.nested && !(vcpu->arch.shregs.msr &MSR_PR )) {unsigned long req =kvmppc_get_gpr (vcpu,3 );
@@ -3994,7 +4000,7 @@ static int kvmhv_p9_guest_entry(struct kvm_vcpu *vcpu, u64 time_limit, }kvmppc_xive_pull_vcpu (vcpu); -if (kvm_is_radix (vcpu->kvm)) +if (kvm_is_radix (kvm)) vcpu->arch.slb_max =0 ; }
@@ -4459,8 +4465,6 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit,powerpc_local_irq_pmu_restore (flags); -cpumask_clear_cpu (pcpu, &kvm->arch.cpu_in_guest); -preempt_enable ();/*
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerRFC PATCH 3/6 KVM: PPC: Book3S HV P9: Remove most of the vcore logic
In reply to Nicholas PigginThe P9 path always uses one vcpu per vcore, so none of the the vcore, locks, stolen time, blocking logic, shared waitq, etc., is required.
Remove most of it.
--- arch/powerpc/kvm/book3s_hv.c | 147 ++++++++++++++++++++--------------- 1 file changed, 85 insertions(+), 62 deletions(-)
arch/powerpc/kvm/book3s_hv.c +85 -62
@@ -281,6 +281,8 @@ static void kvmppc_core_start_stolen(struct kvmppc_vcore *vc, u64 tb) {unsigned long flags; +WARN_ON_ONCE (cpu_has_feature (CPU_FTR_ARCH_300 )); +spin_lock_irqsave (&vc->stoltb_lock, flags); vc->preempt_tb = tb;spin_unlock_irqrestore (&vc->stoltb_lock, flags);
@@ -290,6 +292,8 @@ static void kvmppc_core_end_stolen(struct kvmppc_vcore *vc, u64 tb) {unsigned long flags; +WARN_ON_ONCE (cpu_has_feature (CPU_FTR_ARCH_300 )); +spin_lock_irqsave (&vc->stoltb_lock, flags);if (vc->preempt_tb !=TB_NIL ) { vc->stolen_tb += tb - vc->preempt_tb;
@@ -302,7 +306,12 @@ static void kvmppc_core_vcpu_load_hv(struct kvm_vcpu *vcpu, int cpu) {struct kvmppc_vcore *vc = vcpu->arch.vcore;unsigned long flags; -u64 now =mftb (); +u64 now; + +if (cpu_has_feature (CPU_FTR_ARCH_300 )) +return ; + + now =mftb ();/* * We can test vc->runner without taking the vcore lock,
@@ -326,7 +335,12 @@ static void kvmppc_core_vcpu_put_hv(struct kvm_vcpu *vcpu) {struct kvmppc_vcore *vc = vcpu->arch.vcore;unsigned long flags; -u64 now =mftb (); +u64 now; + +if (cpu_has_feature (CPU_FTR_ARCH_300 )) +return ; + + now =mftb ();if (vc->runner = vcpu && vc->vcore_state >=VCORE_SLEEPING )kvmppc_core_start_stolen (vc, now);
@@ -678,6 +692,8 @@ static u64 vcore_stolen_time(struct kvmppc_vcore *vc, u64 now)u64 p;unsigned long flags; +WARN_ON_ONCE (cpu_has_feature (CPU_FTR_ARCH_300 )); +spin_lock_irqsave (&vc->stoltb_lock, flags); p = vc->stolen_tb; if (vc->vcore_state != VCORE_INACTIVE &&
@@ -700,13 +716,19 @@ static void kvmppc_create_dtl_entry(struct kvm_vcpu *vcpu, dt = vcpu->arch.dtl_ptr; vpa = vcpu->arch.vpa.pinned_addr; now = tb; - core_stolen =vcore_stolen_time (vc, now); - stolen = core_stolen - vcpu->arch.stolen_logged; - vcpu->arch.stolen_logged = core_stolen; -spin_lock_irqsave (&vcpu->arch.tbacct_lock, flags); - stolen += vcpu->arch.busy_stolen; - vcpu->arch.busy_stolen =0 ; -spin_unlock_irqrestore (&vcpu->arch.tbacct_lock, flags); + +if (cpu_has_feature (CPU_FTR_ARCH_300 )) { + stolen =0 ; + }else { + core_stolen =vcore_stolen_time (vc, now); + stolen = core_stolen - vcpu->arch.stolen_logged; + vcpu->arch.stolen_logged = core_stolen; +spin_lock_irqsave (&vcpu->arch.tbacct_lock, flags); + stolen += vcpu->arch.busy_stolen; + vcpu->arch.busy_stolen =0 ; +spin_unlock_irqrestore (&vcpu->arch.tbacct_lock, flags); + } +if (!dt || !vpa)return ;memset (dt,0 ,sizeof (struct dtl_entry ));
@@ -903,13 +925,14 @@ static int kvm_arch_vcpu_yield_to(struct kvm_vcpu *target)* mode handler is not called but no other threads are in the * source vcore. */ - -spin_lock (&vcore->lock); -if (target->arch.state =KVMPPC_VCPU_RUNNABLE && - vcore->vcore_state !=VCORE_INACTIVE && - vcore->runner) - target = vcore->runner; -spin_unlock (&vcore->lock); +if (!cpu_has_feature (CPU_FTR_ARCH_300 )) { +spin_lock (&vcore->lock); +if (target->arch.state =KVMPPC_VCPU_RUNNABLE && + vcore->vcore_state !=VCORE_INACTIVE && + vcore->runner) + target = vcore->runner; +spin_unlock (&vcore->lock); + }return kvm_vcpu_yield_to (target); }
@@ -3097,13 +3120,6 @@ static void kvmppc_start_thread(struct kvm_vcpu *vcpu, struct kvmppc_vcore *vc)kvmppc_ipi_thread (cpu); } -/* Old path does this in asm */ -static void kvmppc_stop_thread (struct kvm_vcpu *vcpu) -{ - vcpu->cpu =-1 ; - vcpu->arch.thread_cpu =-1 ; -} -static void kvmppc_wait_for_nap (int n_threads) {int cpu =smp_processor_id ();
@@ -3192,6 +3208,8 @@ static void kvmppc_vcore_preempt(struct kvmppc_vcore *vc) {struct preempted_vcore_list *lp =this_cpu_ptr (&preempted_vcores); +WARN_ON_ONCE (cpu_has_feature (CPU_FTR_ARCH_300 )); + vc->vcore_state =VCORE_PREEMPT ; vc->pcpu =smp_processor_id ();if (vc->num_threads <threads_per_vcore (vc->kvm)) {
@@ -3208,6 +3226,8 @@ static void kvmppc_vcore_end_preempt(struct kvmppc_vcore *vc) {struct preempted_vcore_list *lp; +WARN_ON_ONCE (cpu_has_feature (CPU_FTR_ARCH_300 )); +kvmppc_core_end_stolen (vc,mftb ());if (!list_empty (&vc->preempt_list)) { lp = &per_cpu (preempted_vcores, vc->pcpu);
@@ -3935,7 +3955,6 @@ static int kvmhv_vcpu_entry_p9_nested(struct kvm_vcpu *vcpu, u64 time_limit, unsstatic int kvmhv_p9_guest_entry (struct kvm_vcpu *vcpu,u64 time_limit,unsigned long lpcr,u64 *tb) { -struct kvmppc_vcore *vc = vcpu->arch.vcore;u64 next_timer;int trap;
@@ -3951,9 +3970,6 @@ static int kvmhv_p9_guest_entry(struct kvm_vcpu *vcpu, u64 time_limit,kvmppc_subcore_enter_guest (); - vc->entry_exit_map =1 ; - vc->in_guest =1 ; -vcpu_vpa_increment_dispatch (vcpu);if (kvmhv_on_pseries ()) {
@@ -4006,9 +4022,6 @@ static int kvmhv_p9_guest_entry(struct kvm_vcpu *vcpu, u64 time_limit,vcpu_vpa_increment_dispatch (vcpu); - vc->entry_exit_map =0x101 ; - vc->in_guest =0 ; -kvmppc_subcore_exit_guest ();return trap;
@@ -4074,6 +4087,13 @@ static bool kvmppc_vcpu_woken(struct kvm_vcpu *vcpu)return false; } +static bool kvmppc_vcpu_check_block (struct kvm_vcpu *vcpu) +{ +if (!vcpu->arch.ceded ||kvmppc_vcpu_woken (vcpu)) +return true; +return false; +} +/* * Check to see if any of the runnable vcpus on the vcore have pending * exceptions or are no longer ceded
@@ -4084,7 +4104,7 @@ static int kvmppc_vcore_check_block(struct kvmppc_vcore *vc)int i;for_each_runnable_thread (i, vcpu, vc) { -if (!vcpu->arch.ceded ||kvmppc_vcpu_woken (vcpu)) +if (kvmppc_vcpu_check_block (vcpu))return 1 ; }
@@ -4101,6 +4121,8 @@ static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc)int do_sleep =1 ;u64 block_ns; +WARN_ON_ONCE (cpu_has_feature (CPU_FTR_ARCH_300 )); +/* Poll for pending exceptions and ceded state */ cur = start_poll =ktime_get ();if (vc->halt_poll_ns) {
@@ -4366,11 +4388,7 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit, vcpu->arch.ceded =0 ; vcpu->arch.run_task = current; vcpu->arch.state =KVMPPC_VCPU_RUNNABLE ; - vcpu->arch.busy_preempt =TB_NIL ; vcpu->arch.last_inst =KVM_INST_FETCH_FAILED ; - vc->runnable_threads[0 ] = vcpu; - vc->n_runnable =1 ; - vc->runner = vcpu;/* See if the MMU is ready to go */ if (unlikely (!kvm->arch.mmu_ready)) {
@@ -4388,11 +4406,8 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit,kvmppc_update_vpas (vcpu); -init_vcore_to_run (vc); -preempt_disable (); pcpu =smp_processor_id (); - vc->pcpu = pcpu;if (kvm_is_radix (kvm))kvmppc_prepare_radix_vcpu (vcpu, pcpu);
@@ -4421,21 +4436,23 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit, goto out; } - tb =mftb (); +if (vcpu->arch.timer_running) { +hrtimer_try_to_cancel (&vcpu->arch.dec_timer); + vcpu->arch.timer_running =0 ; + } - vcpu->arch.stolen_logged =vcore_stolen_time (vc, tb); - vc->preempt_tb =TB_NIL ; + tb =mftb (); -kvmppc_clear_host_core (pcpu); + vcpu->cpu = pcpu; + vcpu->arch.thread_cpu = pcpu; + local_paca->kvm_hstate.kvm_vcpu = vcpu; + local_paca->kvm_hstate.ptid =0 ; + local_paca->kvm_hstate.fake_suspend =0 ; - local_paca->kvm_hstate.napping =0 ; - local_paca->kvm_hstate.kvm_split_mode =NULL ; -kvmppc_start_thread (vcpu, vc); + vc->pcpu = pcpu;// for kvmppc_create_dtl_entry kvmppc_create_dtl_entry (vcpu, vc, tb); -trace_kvm_guest_enter (vcpu); - vc->vcore_state =VCORE_RUNNING ; -trace_kvmppc_run_core (vc,0 ); +trace_kvm_guest_enter (vcpu);guest_enter_irqoff ();
@@ -4457,11 +4474,10 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit,set_irq_happened (trap); -kvmppc_set_host_core (pcpu); -guest_exit_irqoff (); -kvmppc_stop_thread (vcpu); + vcpu->cpu =-1 ; + vcpu->arch.thread_cpu =-1 ;powerpc_local_irq_pmu_restore (flags);
@@ -4488,28 +4504,31 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit, } vcpu->arch.ret = r; -if (is_kvmppc_resume_guest (r) && vcpu->arch.ceded && - !kvmppc_vcpu_woken (vcpu)) { +if (is_kvmppc_resume_guest (r) && !kvmppc_vcpu_check_block (vcpu)) {kvmppc_set_timer (vcpu); -while (vcpu->arch.ceded && !kvmppc_vcpu_woken (vcpu)) { + +prepare_to_rcuwait (&vcpu->wait); +for (;;) { +set_current_state (TASK_INTERRUPTIBLE );if (signal_pending (current)) { vcpu->stat.signal_exits++; run->exit_reason =KVM_EXIT_INTR ; vcpu->arch.ret = -EINTR ;break ; } -spin_lock (&vc->lock); -kvmppc_vcore_blocked (vc); -spin_unlock (&vc->lock); + +if (kvmppc_vcpu_check_block (vcpu)) +break ; + +trace_kvmppc_vcore_blocked (vc,0 ); +schedule (); +trace_kvmppc_vcore_blocked (vc,1 ); } +finish_rcuwait (&vcpu->wait); } vcpu->arch.ceded =0 ; - vc->vcore_state =VCORE_INACTIVE ; -trace_kvmppc_run_core (vc,1 ); - done: -kvmppc_remove_runnable (vc, vcpu, tb);trace_kvmppc_run_vcpu_exit (vcpu);return vcpu->arch.ret;
@@ -4593,7 +4612,8 @@ static int kvmppc_vcpu_run_hv(struct kvm_vcpu *vcpu)kvmppc_save_current_sprs (); - vcpu->arch.waitp = &vcpu->arch.vcore->wait; +if (!cpu_has_feature (CPU_FTR_ARCH_300 )) + vcpu->arch.waitp = &vcpu->arch.vcore->wait; vcpu->arch.pgdir = kvm->mm->pgd; vcpu->arch.state =KVMPPC_VCPU_BUSY_IN_HOST ;
@@ -5055,6 +5075,9 @@ void kvmppc_alloc_host_rm_ops(void)int cpu, core;int size; +if (cpu_has_feature (CPU_FTR_ARCH_300 )) +return ; +/* Not the first time here ? */ if (kvmppc_host_rm_ops_hv !=NULL )return ;
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerRFC PATCH 4/6 KVM: PPC: Book3S HV P9: Tidy kvmppc_create_dtl_entry
In reply to Nicholas PigginThis goes further to removing vcores from the P9 path. Also avoid the memset in favour of explicitly initialising all fields.
--- arch/powerpc/kvm/book3s_hv.c | 61 +++++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 26 deletions(-)
arch/powerpc/kvm/book3s_hv.c +35 -26
@@ -703,41 +703,30 @@ static u64 vcore_stolen_time(struct kvmppc_vcore *vc, u64 now)return p; } -static void kvmppc_create_dtl_entry (struct kvm_vcpu *vcpu, -struct kvmppc_vcore *vc,u64 tb) +static void __kvmppc_create_dtl_entry (struct kvm_vcpu *vcpu, +unsigned int pcpu,u64 now, +unsigned long stolen) {struct dtl_entry *dt;struct lppaca *vpa; -unsigned long stolen; -unsigned long core_stolen; -u64 now; -unsigned long flags; dt = vcpu->arch.dtl_ptr; vpa = vcpu->arch.vpa.pinned_addr; - now = tb; - -if (cpu_has_feature (CPU_FTR_ARCH_300 )) { - stolen =0 ; - }else { - core_stolen =vcore_stolen_time (vc, now); - stolen = core_stolen - vcpu->arch.stolen_logged; - vcpu->arch.stolen_logged = core_stolen; -spin_lock_irqsave (&vcpu->arch.tbacct_lock, flags); - stolen += vcpu->arch.busy_stolen; - vcpu->arch.busy_stolen =0 ; -spin_unlock_irqrestore (&vcpu->arch.tbacct_lock, flags); - }if (!dt || !vpa)return ; -memset (dt,0 ,sizeof (struct dtl_entry )); + dt->dispatch_reason =7 ; - dt->processor_id =cpu_to_be16 (vc->pcpu + vcpu->arch.ptid); - dt->timebase =cpu_to_be64 (now + vc->tb_offset); + dt->preempt_reason =0 ; + dt->processor_id =cpu_to_be16 (pcpu + vcpu->arch.ptid); dt->enqueue_to_dispatch_time =cpu_to_be32 (stolen); + dt->ready_to_enqueue_time =0 ; + dt->waiting_to_ready_time =0 ; + dt->timebase =cpu_to_be64 (now); + dt->fault_addr =0 ; dt->srr0 =cpu_to_be64 (kvmppc_get_pc (vcpu)); dt->srr1 =cpu_to_be64 (vcpu->arch.shregs.msr); + ++dt;if (dt = vcpu->arch.dtl.pinned_end) dt = vcpu->arch.dtl.pinned_addr;
@@ -748,6 +737,27 @@ static void kvmppc_create_dtl_entry(struct kvm_vcpu *vcpu, vcpu->arch.dtl.dirty = true; } +static void kvmppc_create_dtl_entry (struct kvm_vcpu *vcpu, +struct kvmppc_vcore *vc) +{ +unsigned long stolen; +unsigned long core_stolen; +u64 now; +unsigned long flags; + + now =mftb (); + + core_stolen =vcore_stolen_time (vc, now); + stolen = core_stolen - vcpu->arch.stolen_logged; + vcpu->arch.stolen_logged = core_stolen; +spin_lock_irqsave (&vcpu->arch.tbacct_lock, flags); + stolen += vcpu->arch.busy_stolen; + vcpu->arch.busy_stolen =0 ; +spin_unlock_irqrestore (&vcpu->arch.tbacct_lock, flags); + +__kvmppc_create_dtl_entry (vcpu, vc->pcpu, now + vc->tb_offset, stolen); +} +/* See if there is a doorbell interrupt pending for a vcpu */ static bool kvmppc_doorbell_pending (struct kvm_vcpu *vcpu) {
@@ -3722,7 +3732,7 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) pvc->pcpu = pcpu + thr;for_each_runnable_thread (i, vcpu, pvc) {kvmppc_start_thread (vcpu, pvc); -kvmppc_create_dtl_entry (vcpu, pvc,mftb ()); +kvmppc_create_dtl_entry (vcpu, pvc);trace_kvm_guest_enter (vcpu);if (!vcpu->arch.ptid) thr0_done = true;
@@ -4272,7 +4282,7 @@ static int kvmppc_run_vcpu(struct kvm_vcpu *vcpu)if ((vc->vcore_state =VCORE_PIGGYBACK || vc->vcore_state =VCORE_RUNNING ) && !VCORE_IS_EXITING (vc)) { -kvmppc_create_dtl_entry (vcpu, vc,mftb ()); +kvmppc_create_dtl_entry (vcpu, vc);kvmppc_start_thread (vcpu, vc);trace_kvm_guest_enter (vcpu); }else if (vc->vcore_state =VCORE_SLEEPING ) {
@@ -4449,8 +4459,7 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit, local_paca->kvm_hstate.ptid =0 ; local_paca->kvm_hstate.fake_suspend =0 ; - vc->pcpu = pcpu;// for kvmppc_create_dtl_entry -kvmppc_create_dtl_entry (vcpu, vc, tb); +__kvmppc_create_dtl_entry (vcpu, pcpu, tb + vc->tb_offset,0 );trace_kvm_guest_enter (vcpu);
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerRFC PATCH 5/6 KVM: PPC: Book3S HV P9: Stop using vc->dpdes
In reply to Nicholas PigginThe P9 path uses vc->dpdes only for msgsndp / SMT emulation. This adds an ordering requirement between vcpu->doorbell_request and vc->dpdes for no real benefit. Use vcpu->doorbell_request directly.
XXX: verify msgsndp / DPDES emulation works properly.
--- arch/powerpc/kvm/book3s_hv.c | 18 ++++++++++-------- arch/powerpc/kvm/book3s_hv_builtin.c | 2 ++ arch/powerpc/kvm/book3s_hv_p9_entry.c | 14 ++++++++++---- 3 files changed, 22 insertions(+), 12 deletions(-)
arch/powerpc/kvm/book3s_hv.c +10 -8
@@ -766,6 +766,8 @@ static bool kvmppc_doorbell_pending(struct kvm_vcpu *vcpu)if (vcpu->arch.doorbell_request)return true; +if (cpu_has_feature (CPU_FTR_ARCH_300 )) +return false;/* * Ensure that the read of vcore->dpdes comes after the read * of vcpu->doorbell_request. This barrier matches the
@@ -2165,8 +2167,10 @@ static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id,* either vcore->dpdes or doorbell_request. * On POWER8, doorbell_request is 0. */ - *val =get_reg_val (id, vcpu->arch.vcore->dpdes | - vcpu->arch.doorbell_request); +if (cpu_has_feature (CPU_FTR_ARCH_300 )) + *val =get_reg_val (id, vcpu->arch.doorbell_request); +else + *val =get_reg_val (id, vcpu->arch.vcore->dpdes);break ;case KVM_REG_PPC_VTB : *val =get_reg_val (id, vcpu->arch.vcore->vtb);
@@ -2403,7 +2407,10 @@ static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, vcpu->arch.pspb =set_reg_val (id, *val);break ;case KVM_REG_PPC_DPDES : - vcpu->arch.vcore->dpdes =set_reg_val (id, *val); +if (cpu_has_feature (CPU_FTR_ARCH_300 )) + vcpu->arch.doorbell_request =set_reg_val (id, *val) &1 ; +else + vcpu->arch.vcore->dpdes =set_reg_val (id, *val);break ;case KVM_REG_PPC_VTB : vcpu->arch.vcore->vtb =set_reg_val (id, *val);
@@ -4431,11 +4438,6 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit,if (!nested) {kvmppc_core_prepare_to_enter (vcpu); -if (vcpu->arch.doorbell_request) { - vc->dpdes =1 ; -smp_wmb (); - vcpu->arch.doorbell_request =0 ; - }if (test_bit (BOOK3S_IRQPRIO_EXTERNAL , &vcpu->arch.pending_exceptions)) lpcr |=LPCR_MER ;
arch/powerpc/kvm/book3s_hv_builtin.c +2 -0
@@ -649,6 +649,8 @@ void kvmppc_guest_entry_inject_int(struct kvm_vcpu *vcpu)int ext;unsigned long lpcr; +WARN_ON_ONCE (cpu_has_feature (CPU_FTR_ARCH_300 )); +/* Insert EXTERNAL bit into LPCR at the MER bit position */ ext = (vcpu->arch.pending_exceptions >>BOOK3S_IRQPRIO_EXTERNAL ) &1 ; lpcr =mfspr (SPRN_LPCR );
arch/powerpc/kvm/book3s_hv_p9_entry.c +10 -4
@@ -677,6 +677,7 @@ int kvmhv_vcpu_entry_p9(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpcunsigned long host_pidr;unsigned long host_dawr1;unsigned long host_dawrx1; +unsigned long dpdes; hdec = time_limit - *tb;if (hdec <0 )
@@ -736,8 +737,10 @@ int kvmhv_vcpu_entry_p9(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpcif (vc->pcr)mtspr (SPRN_PCR , vc->pcr |PCR_MASK ); -if (vc->dpdes) -mtspr (SPRN_DPDES , vc->dpdes); +if (vcpu->arch.doorbell_request) { + vcpu->arch.doorbell_request =0 ; +mtspr (SPRN_DPDES ,1 ); + }if (dawr_enabled ()) {if (vcpu->arch.dawr0 != host_dawr0)
@@ -968,7 +971,10 @@ int kvmhv_vcpu_entry_p9(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpc vcpu->arch.shregs.sprg2 =mfspr (SPRN_SPRG2 ); vcpu->arch.shregs.sprg3 =mfspr (SPRN_SPRG3 ); - vc->dpdes =mfspr (SPRN_DPDES ); + dpdes =mfspr (SPRN_DPDES ); +if (dpdes) + vcpu->arch.doorbell_request =1 ; + vc->vtb =mfspr (SPRN_VTB ); dec =mfspr (SPRN_DEC );
@@ -1030,7 +1036,7 @@ int kvmhv_vcpu_entry_p9(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpc } } -if (vc->dpdes) +if (dpdes)mtspr (SPRN_DPDES ,0 );if (vc->pcr)mtspr (SPRN_PCR ,PCR_MASK );
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerRFC PATCH 6/6 KVM: PPC: Book3S HV P9: Remove subcore HMI handling
In reply to Nicholas PigginOn POWER9 and newer, rather than the complex HMI synchronisation and subcore state, have each thread un-apply the guest TB offset before calling into the early HMI handler.
This allows the subcore state to be avoided, including subcore enter / exit guest, which includes an expensive divide that shows up slightly in profiles.
--- arch/powerpc/kvm/book3s_hv.c | 12 +++++------- arch/powerpc/kvm/book3s_hv_hmi.c | 7 ++++++- arch/powerpc/kvm/book3s_hv_p9_entry.c | 21 ++++++++++++++++++++- arch/powerpc/kvm/book3s_hv_ras.c | 4 ++++ 4 files changed, 35 insertions(+), 9 deletions(-)
arch/powerpc/kvm/book3s_hv.c +5 -7
@@ -3985,8 +3985,6 @@ static int kvmhv_p9_guest_entry(struct kvm_vcpu *vcpu, u64 time_limit, vcpu->arch.ceded =0 ; -kvmppc_subcore_enter_guest (); -vcpu_vpa_increment_dispatch (vcpu);if (kvmhv_on_pseries ()) {
@@ -4039,8 +4037,6 @@ static int kvmhv_p9_guest_entry(struct kvm_vcpu *vcpu, u64 time_limit,vcpu_vpa_increment_dispatch (vcpu); -kvmppc_subcore_exit_guest (); -return trap; }
@@ -6022,9 +6018,11 @@ static int kvmppc_book3s_init_hv(void)if (r)return r; - r =kvm_init_subcore_bitmap (); -if (r) -return r; +if (!cpu_has_feature (CPU_FTR_ARCH_300 )) { + r =kvm_init_subcore_bitmap (); +if (r) +return r; + }/* * We need a way of accessing the XICS interrupt controller,
arch/powerpc/kvm/book3s_hv_hmi.c +6 -1
@@ -20,10 +20,15 @@ void wait_for_subcore_guest_exit(void)/* * NULL bitmap pointer indicates that KVM module hasn't -* been loaded yet and hence no guests are running. +* been loaded yet and hence no guests are running, or running +* on POWER9 or newer CPU. +* * If no KVM is in use, no need to co-ordinate among threads * as all of them will always be in host and no one is going * to modify TB other than the opal hmi handler. +* +* POWER9 and newer don't need this synchronisation. +* * Hence, just return from here. */ if (!local_paca->sibling_subcore_state)
arch/powerpc/kvm/book3s_hv_p9_entry.c +20 -1
@@ -3,6 +3,7 @@#include <linux/kvm_host.h> #include <asm/asm-prototypes.h> #include <asm/dbell.h> +#include <asm/interrupt.h> #include <asm/kvm_ppc.h> #include <asm/pmc.h> #include <asm/ppc-opcode.h>
@@ -900,7 +901,25 @@ int kvmhv_vcpu_entry_p9(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpckvmppc_realmode_machine_check (vcpu); }else if (unlikely (trap =BOOK3S_INTERRUPT_HMI )) { -kvmppc_realmode_hmi_handler (); +/* +* Unapply and clear the offset first. That way, if the TB +* was fine then no harm done, if it is corrupted then the +* HMI resync will bring it back to host mode. This way, we +* don't need to actualy know whether not OPAL resynced the +* timebase. Although it would be cleaner if we could rely +* on that, early POWER9 OPAL did not support the +* OPAL_HANDLE_HMI2 call. +*/ +if (vc->tb_offset_applied) { +u64 new_tb =mftb () - vc->tb_offset_applied; +mtspr (SPRN_TBU40 , new_tb); +if ((mftb () &0xffffff ) < (new_tb &0xffffff )) { + new_tb +=0x1000000 ; +mtspr (SPRN_TBU40 , new_tb); + } + vc->tb_offset_applied =0 ; + } +hmi_exception_realmode (NULL ); }else if (trap =BOOK3S_INTERRUPT_H_EMUL_ASSIST ) { vcpu->arch.emul_inst =mfspr (SPRN_HEIR );
arch/powerpc/kvm/book3s_hv_ras.c +4 -0
@@ -136,6 +136,10 @@ void kvmppc_realmode_machine_check(struct kvm_vcpu *vcpu) vcpu->arch.mce_evt = mce_evt; } +/* +* This subcore HMI handling is all only for pre-POWER9 CPUs. +*/ +/* Check if dynamic split is in force and return subcore size accordingly. */ static inline int kvmppc_cur_subcore_size (void ) {
Signature · 1 line
2.23.0