KVM RISC-V Sv57x4 support and HFENCE improvements
Subsystems (3)
cover KVM RISC-V Sv57x4 support and HFENCE improvements
This series adds Sv57x4 support for KVM RISC-V G-stage and various HFENCE related improvements.
These patches can also be found in riscv_kvm_sv57_plus_v1 branch at:
https://github.com/avpatel/linux.git
Anup Patel (6):
RISC-V: KVM: Use G-stage name for hypervisor page table
RISC-V: KVM: Add Sv57x4 mode support for G-stage
RISC-V: KVM: Treat SBI HFENCE calls as NOPs
RISC-V: KVM: Introduce range based local HFENCE functions
RISC-V: KVM: Reduce KVM_MAX_VCPUS value
RISC-V: KVM: Add remote HFENCE functions based on VCPU requests
arch/riscv/include/asm/csr.h | 1 + arch/riscv/include/asm/kvm_host.h | 119 ++++++-- arch/riscv/kvm/main.c | 11 +- arch/riscv/kvm/mmu.c | 264 ++++++++--------- arch/riscv/kvm/tlb.S | 74 ----- arch/riscv/kvm/tlb.c | 456 ++++++++++++++++++++++++++++++ arch/riscv/kvm/vcpu.c | 34 ++- arch/riscv/kvm/vcpu_exit.c | 6 +- arch/riscv/kvm/vcpu_sbi_replace.c | 40 ++- arch/riscv/kvm/vcpu_sbi_v01.c | 35 ++- arch/riscv/kvm/vm.c | 8 +- arch/riscv/kvm/vmid.c | 30 +- 12 files changed, 791 insertions(+), 287 deletions(-) delete mode 100644 arch/riscv/kvm/tlb.S create mode 100644 arch/riscv/kvm/tlb.c
Signature · 1 line
2.25.1
PATCH 1/6 RISC-V: KVM: Use G-stage name for hypervisor page table
The two-stage address translation defined by the RISC-V privileged specification defines: VS-stage (guest virtual address to guest physical address) programmed by the Guest OS and G-stage (guest physical addree to host physical address) programmed by the hypervisor.
To align with above terminology, we replace "stage2" with "gstage" and "Stage2" with "G-stage" name everywhere in KVM RISC-V sources.
--- arch/riscv/include/asm/kvm_host.h | 30 ++-- arch/riscv/kvm/main.c | 8 +- arch/riscv/kvm/mmu.c | 222 +++++++++++++++--------------- arch/riscv/kvm/vcpu.c | 10 +- arch/riscv/kvm/vcpu_exit.c | 6 +- arch/riscv/kvm/vm.c | 8 +- arch/riscv/kvm/vmid.c | 18 +-- 7 files changed, 151 insertions(+), 151 deletions(-)
arch/riscv/include/asm/kvm_host.h +15 -15
@@ -54,10 +54,10 @@ struct kvm_vmid { };struct kvm_arch { -/* stage2 vmid */ +/* G-stage vmid */ struct kvm_vmid vmid; -/* stage2 page table */ +/* G-stage page table */ pgd_t *pgd;phys_addr_t pgd_phys;
@@ -210,21 +210,21 @@ void __kvm_riscv_hfence_gvma_vmid(unsigned long vmid);void __kvm_riscv_hfence_gvma_gpa (unsigned long gpa_divby_4);void __kvm_riscv_hfence_gvma_all (void ); -int kvm_riscv_stage2_map (struct kvm_vcpu *vcpu, +int kvm_riscv_gstage_map (struct kvm_vcpu *vcpu,struct kvm_memory_slot *memslot,gpa_t gpa,unsigned long hva,bool is_write); -int kvm_riscv_stage2_alloc_pgd (struct kvm *kvm); -void kvm_riscv_stage2_free_pgd (struct kvm *kvm); -void kvm_riscv_stage2_update_hgatp (struct kvm_vcpu *vcpu); -void kvm_riscv_stage2_mode_detect (void ); -unsigned long kvm_riscv_stage2_mode (void ); -int kvm_riscv_stage2_gpa_bits (void ); - -void kvm_riscv_stage2_vmid_detect (void ); -unsigned long kvm_riscv_stage2_vmid_bits (void ); -int kvm_riscv_stage2_vmid_init (struct kvm *kvm); -bool kvm_riscv_stage2_vmid_ver_changed (struct kvm_vmid *vmid); -void kvm_riscv_stage2_vmid_update (struct kvm_vcpu *vcpu); +int kvm_riscv_gstage_alloc_pgd (struct kvm *kvm); +void kvm_riscv_gstage_free_pgd (struct kvm *kvm); +void kvm_riscv_gstage_update_hgatp (struct kvm_vcpu *vcpu); +void kvm_riscv_gstage_mode_detect (void ); +unsigned long kvm_riscv_gstage_mode (void ); +int kvm_riscv_gstage_gpa_bits (void ); + +void kvm_riscv_gstage_vmid_detect (void ); +unsigned long kvm_riscv_gstage_vmid_bits (void ); +int kvm_riscv_gstage_vmid_init (struct kvm *kvm); +bool kvm_riscv_gstage_vmid_ver_changed (struct kvm_vmid *vmid); +void kvm_riscv_gstage_vmid_update (struct kvm_vcpu *vcpu);void __kvm_riscv_unpriv_trap (void );
arch/riscv/kvm/main.c +4 -4
@@ -89,13 +89,13 @@ int kvm_arch_init(void *opaque)return -ENODEV ; } -kvm_riscv_stage2_mode_detect (); +kvm_riscv_gstage_mode_detect (); -kvm_riscv_stage2_vmid_detect (); +kvm_riscv_gstage_vmid_detect ();kvm_info ("hypervisor extension available\n" ); -switch (kvm_riscv_stage2_mode ()) { +switch (kvm_riscv_gstage_mode ()) {case HGATP_MODE_SV32X4 : str ="Sv32x4" ;break ;
@@ -110,7 +110,7 @@ int kvm_arch_init(void *opaque) }kvm_info ("using %s G-stage page table format\n" , str); -kvm_info ("VMID %ld bits available\n" ,kvm_riscv_stage2_vmid_bits ()); +kvm_info ("VMID %ld bits available\n" ,kvm_riscv_gstage_vmid_bits ());return 0 ; }
arch/riscv/kvm/mmu.c +111 -111
573 lines not on this page · raw message
arch/riscv/kvm/vcpu.c +5 -5
@@ -135,7 +135,7 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)/* Cleanup VCPU timer */ kvm_riscv_vcpu_timer_deinit (vcpu); -/* Free unused pages pre-allocated for Stage2 page table mappings */ +/* Free unused pages pre-allocated for G-stage page table mappings */ kvm_mmu_free_memory_cache (&vcpu->arch.mmu_page_cache); }
@@ -632,7 +632,7 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)csr_write (CSR_HVIP , csr->hvip);csr_write (CSR_VSATP , csr->vsatp); -kvm_riscv_stage2_update_hgatp (vcpu); +kvm_riscv_gstage_update_hgatp (vcpu);kvm_riscv_vcpu_timer_restore (vcpu);
@@ -687,7 +687,7 @@ static void kvm_riscv_check_vcpu_requests(struct kvm_vcpu *vcpu)kvm_riscv_reset_vcpu (vcpu);if (kvm_check_request (KVM_REQ_UPDATE_HGATP , vcpu)) -kvm_riscv_stage2_update_hgatp (vcpu); +kvm_riscv_gstage_update_hgatp (vcpu);if (kvm_check_request (KVM_REQ_TLB_FLUSH , vcpu))__kvm_riscv_hfence_gvma_all ();
@@ -759,7 +759,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)/* Check conditions before entering the guest */ cond_resched (); -kvm_riscv_stage2_vmid_update (vcpu); +kvm_riscv_gstage_vmid_update (vcpu);kvm_riscv_check_vcpu_requests (vcpu);
@@ -797,7 +797,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)kvm_riscv_update_hvip (vcpu);if (ret <=0 || -kvm_riscv_stage2_vmid_ver_changed (&vcpu->kvm->arch.vmid) || +kvm_riscv_gstage_vmid_ver_changed (&vcpu->kvm->arch.vmid) ||kvm_request_pending (vcpu)) { vcpu->mode =OUTSIDE_GUEST_MODE ;local_irq_enable ();
arch/riscv/kvm/vcpu_exit.c +3 -3
@@ -412,7 +412,7 @@ static int emulate_store(struct kvm_vcpu *vcpu, struct kvm_run *run,return 0 ; } -static int stage2_page_fault (struct kvm_vcpu *vcpu,struct kvm_run *run, +static int gstage_page_fault (struct kvm_vcpu *vcpu,struct kvm_run *run,struct kvm_cpu_trap *trap) {struct kvm_memory_slot *memslot;
@@ -440,7 +440,7 @@ static int stage2_page_fault(struct kvm_vcpu *vcpu, struct kvm_run *run, }; } - ret =kvm_riscv_stage2_map (vcpu, memslot, fault_addr, hva, + ret =kvm_riscv_gstage_map (vcpu, memslot, fault_addr, hva, (trap->scause ==EXC_STORE_GUEST_PAGE_FAULT ) ? true : false);if (ret <0 )return ret;
@@ -686,7 +686,7 @@ int kvm_riscv_vcpu_exit(struct kvm_vcpu *vcpu, struct kvm_run *run,case EXC_LOAD_GUEST_PAGE_FAULT :case EXC_STORE_GUEST_PAGE_FAULT :if (vcpu->arch.guest_context.hstatus &HSTATUS_SPV ) - ret =stage2_page_fault (vcpu, run, trap); + ret =gstage_page_fault (vcpu, run, trap);break ;case EXC_SUPERVISOR_SYSCALL :if (vcpu->arch.guest_context.hstatus &HSTATUS_SPV )
arch/riscv/kvm/vm.c +4 -4
@@ -31,13 +31,13 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) {int r; - r =kvm_riscv_stage2_alloc_pgd (kvm); + r =kvm_riscv_gstage_alloc_pgd (kvm);if (r)return r; - r =kvm_riscv_stage2_vmid_init (kvm); + r =kvm_riscv_gstage_vmid_init (kvm);if (r) { -kvm_riscv_stage2_free_pgd (kvm); +kvm_riscv_gstage_free_pgd (kvm);return r; }
@@ -75,7 +75,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) r =KVM_USER_MEM_SLOTS ;break ;case KVM_CAP_VM_GPA_BITS : - r =kvm_riscv_stage2_gpa_bits (); + r =kvm_riscv_gstage_gpa_bits ();break ;default : r =0 ;
arch/riscv/kvm/vmid.c +9 -9
@@ -20,7 +20,7 @@ static unsigned long vmid_next;static unsigned long vmid_bits;static DEFINE_SPINLOCK (vmid_lock); -void kvm_riscv_stage2_vmid_detect (void ) +void kvm_riscv_gstage_vmid_detect (void ) {unsigned long old;
@@ -40,12 +40,12 @@ void kvm_riscv_stage2_vmid_detect(void) vmid_bits =0 ; } -unsigned long kvm_riscv_stage2_vmid_bits (void ) +unsigned long kvm_riscv_gstage_vmid_bits (void ) {return vmid_bits; } -int kvm_riscv_stage2_vmid_init (struct kvm *kvm) +int kvm_riscv_gstage_vmid_init (struct kvm *kvm) {/* Mark the initial VMID and VMID version invalid */ kvm->arch.vmid.vmid_version =0 ;
@@ -54,7 +54,7 @@ int kvm_riscv_stage2_vmid_init(struct kvm *kvm)return 0 ; } -bool kvm_riscv_stage2_vmid_ver_changed (struct kvm_vmid *vmid) +bool kvm_riscv_gstage_vmid_ver_changed (struct kvm_vmid *vmid) {if (!vmid_bits)return false;
@@ -63,13 +63,13 @@ bool kvm_riscv_stage2_vmid_ver_changed(struct kvm_vmid *vmid) READ_ONCE(vmid_version)); } -void kvm_riscv_stage2_vmid_update (struct kvm_vcpu *vcpu) +void kvm_riscv_gstage_vmid_update (struct kvm_vcpu *vcpu) {unsigned long i;struct kvm_vcpu *v;struct kvm_vmid *vmid = &vcpu->kvm->arch.vmid; -if (!kvm_riscv_stage2_vmid_ver_changed (vmid)) +if (!kvm_riscv_gstage_vmid_ver_changed (vmid))return ;spin_lock (&vmid_lock);
@@ -78,7 +78,7 @@ void kvm_riscv_stage2_vmid_update(struct kvm_vcpu *vcpu)* We need to re-check the vmid_version here to ensure that if * another vcpu already allocated a valid vmid for this vm. */ -if (!kvm_riscv_stage2_vmid_ver_changed (vmid)) { +if (!kvm_riscv_gstage_vmid_ver_changed (vmid)) {spin_unlock (&vmid_lock);return ; }
@@ -96,7 +96,7 @@ void kvm_riscv_stage2_vmid_update(struct kvm_vcpu *vcpu)* instances is invalid and we have force VMID re-assignement * for all Guest instances. The Guest instances that were not * running will automatically pick-up new VMIDs because will -* call kvm_riscv_stage2_vmid_update() whenever they enter +* call kvm_riscv_gstage_vmid_update() whenever they enter * in-kernel run loop. For Guest instances that are already * running, we force VM exits on all host CPUs using IPI and * flush all Guest TLBs.
@@ -112,7 +112,7 @@ void kvm_riscv_stage2_vmid_update(struct kvm_vcpu *vcpu)spin_unlock (&vmid_lock); -/* Request stage2 page table update for all VCPUs */ +/* Request G-stage page table update for all VCPUs */ kvm_for_each_vcpu (i, v, vcpu->kvm)kvm_make_request (KVM_REQ_UPDATE_HGATP , v); }
Signature · 1 line
2.25.1
PATCH 2/6 RISC-V: KVM: Add Sv57x4 mode support for G-stage
In reply to Anup PatelLatest QEMU supports G-stage Sv57x4 mode so this patch extends KVM RISC-V G-stage handling to detect and use Sv57x4 mode when available.
--- arch/riscv/include/asm/csr.h | 1 + arch/riscv/kvm/main.c | 3 +++ arch/riscv/kvm/mmu.c | 11 ++++++++++- 3 files changed, 14 insertions(+), 1 deletion(-)
arch/riscv/include/asm/csr.h +1 -0
@@ -117,6 +117,7 @@#define HGATP_MODE_SV32X4 _AC(1, UL)#define HGATP_MODE_SV39X4 _AC(8, UL)#define HGATP_MODE_SV48X4 _AC(9, UL) +#define HGATP_MODE_SV57X4 _AC(10, UL)#define HGATP32_MODE_SHIFT 31#define HGATP32_VMID_SHIFT 22
arch/riscv/kvm/main.c +3 -0
@@ -105,6 +105,9 @@ int kvm_arch_init(void *opaque)case HGATP_MODE_SV48X4 : str ="Sv48x4" ;break ; +case HGATP_MODE_SV57X4 : + str ="Sv57x4" ; +break ;default :return -ENODEV ; }
arch/riscv/kvm/mmu.c +10 -1
@@ -751,14 +751,23 @@ void kvm_riscv_gstage_update_hgatp(struct kvm_vcpu *vcpu)void kvm_riscv_gstage_mode_detect (void ) {#ifdef CONFIG_64BIT +/* Try Sv57x4 G-stage mode */ +csr_write (CSR_HGATP ,HGATP_MODE_SV57X4 <<HGATP_MODE_SHIFT ); +if ((csr_read (CSR_HGATP ) >>HGATP_MODE_SHIFT ) ==HGATP_MODE_SV57X4 ) { + gstage_mode = (HGATP_MODE_SV57X4 <<HGATP_MODE_SHIFT ); + gstage_pgd_levels =5 ; + goto skip_sv48x4_test; + } +/* Try Sv48x4 G-stage mode */ csr_write (CSR_HGATP ,HGATP_MODE_SV48X4 <<HGATP_MODE_SHIFT );if ((csr_read (CSR_HGATP ) >>HGATP_MODE_SHIFT ) ==HGATP_MODE_SV48X4 ) { gstage_mode = (HGATP_MODE_SV48X4 <<HGATP_MODE_SHIFT ); gstage_pgd_levels =4 ; } -csr_write (CSR_HGATP ,0 ); +skip_sv48x4_test: +csr_write (CSR_HGATP ,0 );__kvm_riscv_hfence_gvma_all ();#endif }
Signature · 1 line
2.25.1
PATCH 3/6 RISC-V: KVM: Treat SBI HFENCE calls as NOPs
In reply to Anup PatelWe should treat SBI HFENCE calls as NOPs until nested virtualization is supported by KVM RISC-V. This will help us test booting a hypervisor under KVM RISC-V.
--- arch/riscv/kvm/vcpu_sbi_replace.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-)
arch/riscv/kvm/vcpu_sbi_replace.c +5 -1
@@ -117,7 +117,11 @@ static int kvm_sbi_ext_rfence_handler(struct kvm_vcpu *vcpu, struct kvm_run *runcase SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID :case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA :case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID : -/* TODO: implement for nested hypervisor case */ +/* +* Until nested virtualization is implemented, the +* SBI HFENCE calls should be treated as NOPs +*/ +break ;default : ret = -EOPNOTSUPP ; }
Signature · 1 line
2.25.1
PATCH 4/6 RISC-V: KVM: Introduce range based local HFENCE functions
In reply to Anup PatelVarious __kvm_riscv_hfence_xyz() functions implemented in the kvm/tlb.S are equivalent to corresponding HFENCE.GVMA instructions and we don't have range based local HFENCE functions.
This patch provides complete set of local HFENCE functions which supports range based TLB invalidation and supports HFENCE.VVMA based functions. This is also a preparatory patch for upcoming Svinval support in KVM RISC-V.
--- arch/riscv/include/asm/kvm_host.h | 25 +++- arch/riscv/kvm/mmu.c | 4 +- arch/riscv/kvm/tlb.S | 74 ---------- arch/riscv/kvm/tlb.c | 231 ++++++++++++++++++++++++++++++ arch/riscv/kvm/vcpu.c | 2 +- arch/riscv/kvm/vmid.c | 2 +- 6 files changed, 255 insertions(+), 83 deletions(-) delete mode 100644 arch/riscv/kvm/tlb.S create mode 100644 arch/riscv/kvm/tlb.c
arch/riscv/include/asm/kvm_host.h +20 -5
@@ -204,11 +204,26 @@ static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {}#define KVM_ARCH_WANT_MMU_NOTIFIER -void __kvm_riscv_hfence_gvma_vmid_gpa (unsigned long gpa_divby_4, -unsigned long vmid); -void __kvm_riscv_hfence_gvma_vmid (unsigned long vmid); -void __kvm_riscv_hfence_gvma_gpa (unsigned long gpa_divby_4); -void __kvm_riscv_hfence_gvma_all (void ); +#define KVM_RISCV_GSTAGE_TLB_MIN_ORDER 12 + +void kvm_riscv_local_hfence_gvma_vmid_gpa (unsigned long vmid, +gpa_t gpa,gpa_t gpsz, +unsigned long order); +void kvm_riscv_local_hfence_gvma_vmid_all (unsigned long vmid); +void kvm_riscv_local_hfence_gvma_gpa (gpa_t gpa,gpa_t gpsz, +unsigned long order); +void kvm_riscv_local_hfence_gvma_all (void ); +void kvm_riscv_local_hfence_vvma_asid_gva (unsigned long vmid, +unsigned long asid, +unsigned long gva, +unsigned long gvsz, +unsigned long order); +void kvm_riscv_local_hfence_vvma_asid_all (unsigned long vmid, +unsigned long asid); +void kvm_riscv_local_hfence_vvma_gva (unsigned long vmid, +unsigned long gva,unsigned long gvsz, +unsigned long order); +void kvm_riscv_local_hfence_vvma_all (unsigned long vmid); int kvm_riscv_gstage_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot,
arch/riscv/kvm/mmu.c +2 -2
@@ -745,7 +745,7 @@ void kvm_riscv_gstage_update_hgatp(struct kvm_vcpu *vcpu)csr_write (CSR_HGATP , hgatp);if (!kvm_riscv_gstage_vmid_bits ()) -__kvm_riscv_hfence_gvma_all (); +kvm_riscv_local_hfence_gvma_all (); }void kvm_riscv_gstage_mode_detect (void )
@@ -768,7 +768,7 @@ void kvm_riscv_gstage_mode_detect(void) skip_sv48x4_test:csr_write (CSR_HGATP ,0 ); -__kvm_riscv_hfence_gvma_all (); +kvm_riscv_local_hfence_gvma_all ();#endif }
arch/riscv/kvm/tlb.S deleted +0 -74
74 lines not on this page · raw message
arch/riscv/kvm/tlb.c new file +231 -0
231 lines not on this page · raw message
arch/riscv/kvm/vcpu.c +1 -1
@@ -690,7 +690,7 @@ static void kvm_riscv_check_vcpu_requests(struct kvm_vcpu *vcpu)kvm_riscv_gstage_update_hgatp (vcpu);if (kvm_check_request (KVM_REQ_TLB_FLUSH , vcpu)) -__kvm_riscv_hfence_gvma_all (); +kvm_riscv_local_hfence_gvma_all (); } }
arch/riscv/kvm/vmid.c +1 -1
@@ -33,7 +33,7 @@ void kvm_riscv_gstage_vmid_detect(void)csr_write (CSR_HGATP , old);/* We polluted local TLB so flush all guest TLB */ -__kvm_riscv_hfence_gvma_all (); +kvm_riscv_local_hfence_gvma_all ();/* We don't use VMID bits if they are not sufficient */ if ((1UL << vmid_bits) <num_possible_cpus ())
Signature · 1 line
2.25.1
PATCH 5/6 RISC-V: KVM: Reduce KVM_MAX_VCPUS value
In reply to Anup PatelCurrently, the KVM_MAX_VCPUS value is 16384 for RV64 and 128 for RV32.
The KVM_MAX_VCPUS value is too high for RV64 and too low for RV32 compared to other architectures (e.g. x86 sets it to 1024 and ARM64 sets it to 512). The too high value of KVM_MAX_VCPUS on RV64 also leads to VCPU mask on stack consuming 2KB.
We set KVM_MAX_VCPUS to 1024 for both RV64 and RV32 to be aligned other architectures.
--- arch/riscv/include/asm/kvm_host.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
arch/riscv/include/asm/kvm_host.h +1 -2
@@ -16,8 +16,7 @@#include <asm/kvm_vcpu_fp.h> #include <asm/kvm_vcpu_timer.h> -#define KVM_MAX_VCPUS \ - ((HGATP_VMID_MASK >> HGATP_VMID_SHIFT) + 1) +#define KVM_MAX_VCPUS 1024#define KVM_HALT_POLL_NS_DEFAULT 500000
Signature · 1 line
2.25.1
PATCH 6/6 RISC-V: KVM: Add remote HFENCE functions based on VCPU requests
In reply to Anup PatelThe generic KVM has support for VCPU requests which can be used to do arch-specific work in the run-loop. We introduce remote HFENCE functions which will internally use VCPU requests instead of host SBI calls.
Advantages of doing remote HFENCEs as VCPU requests are:
1) Multiple VCPUs of a Guest may be running on different Host CPUs so it is not always possible to determine the Host CPU mask for doing Host SBI call. For example, when VCPU X wants to do HFENCE on VCPU Y, it is possible that VCPU Y is blocked or in user-space (i.e. vcpu->cpu < 0).
2) To support nested virtualization, we will be having a separate shadow G-stage for each VCPU and a common host G-stage for the entire Guest/VM. The VCPU requests based remote HFENCEs helps us easily synchronize the common host G-stage and shadow G-stage of each VCPU without any additional IPI calls.
This is also a preparatory patch for upcoming nested virtualization support where we will be having a shadow G-stage page table for each Guest VCPU.
--- arch/riscv/include/asm/kvm_host.h | 59 ++++++++ arch/riscv/kvm/mmu.c | 33 +++-- arch/riscv/kvm/tlb.c | 227 +++++++++++++++++++++++++++++- arch/riscv/kvm/vcpu.c | 24 +++- arch/riscv/kvm/vcpu_sbi_replace.c | 34 ++--- arch/riscv/kvm/vcpu_sbi_v01.c | 35 +++-- arch/riscv/kvm/vmid.c | 10 +- 7 files changed, 369 insertions(+), 53 deletions(-)
arch/riscv/include/asm/kvm_host.h +59 -0
@@ -12,6 +12,7 @@#include <linux/types.h> #include <linux/kvm.h> #include <linux/kvm_types.h> +#include <linux/spinlock.h> #include <asm/csr.h> #include <asm/kvm_vcpu_fp.h> #include <asm/kvm_vcpu_timer.h>
@@ -26,6 +27,31 @@ KVM_ARCH_REQ_FLAGS(0, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)#define KVM_REQ_VCPU_RESET KVM_ARCH_REQ(1)#define KVM_REQ_UPDATE_HGATP KVM_ARCH_REQ(2) +#define KVM_REQ_FENCE_I \ + KVM_ARCH_REQ_FLAGS(3, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) +#define KVM_REQ_HFENCE_GVMA_VMID_ALL KVM_REQ_TLB_FLUSH +#define KVM_REQ_HFENCE_VVMA_ALL \ + KVM_ARCH_REQ_FLAGS(4, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) +#define KVM_REQ_HFENCE \ + KVM_ARCH_REQ_FLAGS(5, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) + +enum kvm_riscv_hfence_type { +KVM_RISCV_HFENCE_UNKNOWN =0 , +KVM_RISCV_HFENCE_GVMA_VMID_GPA , +KVM_RISCV_HFENCE_VVMA_ASID_GVA , +KVM_RISCV_HFENCE_VVMA_ASID_ALL , +KVM_RISCV_HFENCE_VVMA_GVA , +}; + +struct kvm_riscv_hfence { +enum kvm_riscv_hfence_type type; +unsigned long asid; +unsigned long order; +gpa_t addr; +gpa_t size; +}; + +#define KVM_RISCV_VCPU_MAX_HFENCE 64struct kvm_vm_stat {struct kvm_vm_stat_generic generic;
@@ -178,6 +204,12 @@ struct kvm_vcpu_arch {/* VCPU Timer */ struct kvm_vcpu_timer timer; +/* HFENCE request queue */ +spinlock_t hfence_lock; +unsigned long hfence_head; +unsigned long hfence_tail; +struct kvm_riscv_hfence hfence_queue[KVM_RISCV_VCPU_MAX_HFENCE ]; +/* MMIO instruction details */ struct kvm_mmio_decode mmio_decode;
@@ -224,6 +256,33 @@ void kvm_riscv_local_hfence_vvma_gva(unsigned long vmid, unsigned long order);void kvm_riscv_local_hfence_vvma_all (unsigned long vmid); +void kvm_riscv_fence_i_process (struct kvm_vcpu *vcpu); +void kvm_riscv_hfence_gvma_vmid_all_process (struct kvm_vcpu *vcpu); +void kvm_riscv_hfence_vvma_all_process (struct kvm_vcpu *vcpu); +void kvm_riscv_hfence_process (struct kvm_vcpu *vcpu); + +void kvm_riscv_fence_i (struct kvm *kvm, +unsigned long hbase,unsigned long hmask); +void kvm_riscv_hfence_gvma_vmid_gpa (struct kvm *kvm, +unsigned long hbase,unsigned long hmask, +gpa_t gpa,gpa_t gpsz, +unsigned long order); +void kvm_riscv_hfence_gvma_vmid_all (struct kvm *kvm, +unsigned long hbase,unsigned long hmask); +void kvm_riscv_hfence_vvma_asid_gva (struct kvm *kvm, +unsigned long hbase,unsigned long hmask, +unsigned long gva,unsigned long gvsz, +unsigned long order,unsigned long asid); +void kvm_riscv_hfence_vvma_asid_all (struct kvm *kvm, +unsigned long hbase,unsigned long hmask, +unsigned long asid); +void kvm_riscv_hfence_vvma_gva (struct kvm *kvm, +unsigned long hbase,unsigned long hmask, +unsigned long gva,unsigned long gvsz, +unsigned long order); +void kvm_riscv_hfence_vvma_all (struct kvm *kvm, +unsigned long hbase,unsigned long hmask); +int kvm_riscv_gstage_map (struct kvm_vcpu *vcpu,struct kvm_memory_slot *memslot,gpa_t gpa,unsigned long hva,bool is_write);
arch/riscv/kvm/mmu.c +18 -15
@@ -18,7 +18,6 @@#include <asm/csr.h> #include <asm/page.h> #include <asm/pgtable.h> -#include <asm/sbi.h> #ifdef CONFIG_64BIT static unsigned long gstage_mode = (HGATP_MODE_SV39X4 <<HGATP_MODE_SHIFT );
@@ -73,13 +72,25 @@ static int gstage_page_size_to_level(unsigned long page_size, u32 *out_level)return -EINVAL ; } -static int gstage_level_to_page_size (u32 level,unsigned long *out_pgsize) +static int gstage_level_to_page_order (u32 level,unsigned long *out_pgorder) {if (gstage_pgd_levels < level)return -EINVAL ; - *out_pgsize =1UL << (12 + (level * gstage_index_bits)); + *out_pgorder =12 + (level * gstage_index_bits); +return 0 ; +} +static int gstage_level_to_page_size (u32 level,unsigned long *out_pgsize) +{ +int rc; +unsigned long page_order =PAGE_SHIFT ; + + rc =gstage_level_to_page_order (level, &page_order); +if (rc) +return rc; + + *out_pgsize =BIT (page_order);return 0 ; }
@@ -114,21 +125,13 @@ static bool gstage_get_leaf_entry(struct kvm *kvm, gpa_t addr,static void gstage_remote_tlb_flush (struct kvm *kvm,u32 level,gpa_t addr) { -unsigned long size =PAGE_SIZE ; -struct kvm_vmid *vmid = &kvm->arch.vmid; +unsigned long order =PAGE_SHIFT ; -if (gstage_level_to_page_size (level, &size)) +if (gstage_level_to_page_order (level, &order))return ; - addr &= ~(size -1 ); + addr &= ~(BIT (order) -1 ); -/* -* TODO: Instead of cpu_online_mask, we should only target CPUs -* where the Guest/VM is running. -*/ -preempt_disable (); -sbi_remote_hfence_gvma_vmid (cpu_online_mask, addr, size, -READ_ONCE (vmid->vmid)); -preempt_enable (); +kvm_riscv_hfence_gvma_vmid_gpa (kvm,-1UL ,0 , addr,BIT (order), order); } static int gstage_set_pte(struct kvm *kvm, u32 level,
arch/riscv/kvm/tlb.c +226 -1
240 lines not on this page · raw message
arch/riscv/kvm/vcpu.c +22 -2
@@ -76,6 +76,10 @@ static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu)WRITE_ONCE (vcpu->arch.irqs_pending,0 );WRITE_ONCE (vcpu->arch.irqs_pending_mask,0 ); + vcpu->arch.hfence_head =0 ; + vcpu->arch.hfence_tail =0 ; +memset (vcpu->arch.hfence_queue,0 ,sizeof (vcpu->arch.hfence_queue)); +/* Reset the guest CSRs for hotplug usecase */ if (loaded)kvm_arch_vcpu_load (vcpu,smp_processor_id ());
@@ -99,6 +103,9 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)/* Setup ISA features available to VCPU */ vcpu->arch.isa =riscv_isa_extension_base (NULL ) &KVM_RISCV_ISA_ALLOWED ; +/* Setup VCPU hfence queue */ +spin_lock_init (&vcpu->arch.hfence_lock); +/* Setup reset state of shadow SSTATUS and HSTATUS CSRs */ cntx = &vcpu->arch.guest_reset_context; cntx->sstatus =SR_SPP |SR_SPIE ;
@@ -689,8 +696,21 @@ static void kvm_riscv_check_vcpu_requests(struct kvm_vcpu *vcpu)if (kvm_check_request (KVM_REQ_UPDATE_HGATP , vcpu))kvm_riscv_gstage_update_hgatp (vcpu); -if (kvm_check_request (KVM_REQ_TLB_FLUSH , vcpu)) -kvm_riscv_local_hfence_gvma_all (); +if (kvm_check_request (KVM_REQ_FENCE_I , vcpu)) +kvm_riscv_fence_i_process (vcpu); + +/* +* The generic KVM_REQ_TLB_FLUSH is same as +* KVM_REQ_HFENCE_GVMA_VMID_ALL +*/ +if (kvm_check_request (KVM_REQ_HFENCE_GVMA_VMID_ALL , vcpu)) +kvm_riscv_hfence_gvma_vmid_all_process (vcpu); + +if (kvm_check_request (KVM_REQ_HFENCE_VVMA_ALL , vcpu)) +kvm_riscv_hfence_vvma_all_process (vcpu); + +if (kvm_check_request (KVM_REQ_HFENCE , vcpu)) +kvm_riscv_hfence_process (vcpu); } }
arch/riscv/kvm/vcpu_sbi_replace.c +14 -20
@@ -81,37 +81,31 @@ static int kvm_sbi_ext_rfence_handler(struct kvm_vcpu *vcpu, struct kvm_run *run struct kvm_cpu_trap *utrap, bool *exit) { int ret = 0; -unsigned long i; -struct cpumask cm; -struct kvm_vcpu *tmp;struct kvm_cpu_context *cp = &vcpu->arch.guest_context;unsigned long hmask = cp->a0;unsigned long hbase = cp->a1;unsigned long funcid = cp->a6; -cpumask_clear (&cm); -kvm_for_each_vcpu (i, tmp, vcpu->kvm) { -if (hbase !=-1UL ) { -if (tmp->vcpu_id < hbase) -continue ; -if (!(hmask & (1UL << (tmp->vcpu_id - hbase)))) -continue ; - } -if (tmp->cpu <0 ) -continue ; -cpumask_set_cpu (tmp->cpu, &cm); - } -switch (funcid) {case SBI_EXT_RFENCE_REMOTE_FENCE_I : - ret =sbi_remote_fence_i (&cm); +kvm_riscv_fence_i (vcpu->kvm, hbase, hmask);break ;case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA : - ret =sbi_remote_hfence_vvma (&cm, cp->a2, cp->a3); +if (cp->a2 ==0 && cp->a3 ==0 ) +kvm_riscv_hfence_vvma_all (vcpu->kvm, hbase, hmask); +else +kvm_riscv_hfence_vvma_gva (vcpu->kvm, hbase, hmask, + cp->a2, cp->a3,PAGE_SHIFT );break ;case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID : - ret =sbi_remote_hfence_vvma_asid (&cm, cp->a2, - cp->a3, cp->a4); +if (cp->a2 ==0 && cp->a3 ==0 ) +kvm_riscv_hfence_vvma_asid_all (vcpu->kvm, + hbase, hmask, cp->a4); +else +kvm_riscv_hfence_vvma_asid_gva (vcpu->kvm, + hbase, hmask, + cp->a2, cp->a3, +PAGE_SHIFT , cp->a4);break ;case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA :case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID :
arch/riscv/kvm/vcpu_sbi_v01.c +22 -13
@@ -23,7 +23,6 @@ static int kvm_sbi_ext_v01_handler(struct kvm_vcpu *vcpu, struct kvm_run *run,int i, ret =0 ;u64 next_cycle;struct kvm_vcpu *rvcpu; -struct cpumask cm;struct kvm *kvm = vcpu->kvm;struct kvm_cpu_context *cp = &vcpu->arch.guest_context;
@@ -80,19 +79,29 @@ static int kvm_sbi_ext_v01_handler(struct kvm_vcpu *vcpu, struct kvm_run *run,if (utrap->scause)break ; -cpumask_clear (&cm); -for_each_set_bit (i, &hmask,BITS_PER_LONG ) { - rvcpu =kvm_get_vcpu_by_id (vcpu->kvm, i); -if (rvcpu->cpu <0 ) -continue ; -cpumask_set_cpu (rvcpu->cpu, &cm); - }if (cp->a7 ==SBI_EXT_0_1_REMOTE_FENCE_I ) - ret =sbi_remote_fence_i (&cm); -else if (cp->a7 ==SBI_EXT_0_1_REMOTE_SFENCE_VMA ) - ret =sbi_remote_hfence_vvma (&cm, cp->a1, cp->a2); -else - ret =sbi_remote_hfence_vvma_asid (&cm, cp->a1, cp->a2, cp->a3); +kvm_riscv_fence_i (vcpu->kvm,0 , hmask); +else if (cp->a7 ==SBI_EXT_0_1_REMOTE_SFENCE_VMA ) { +if (cp->a1 ==0 && cp->a2 ==0 ) +kvm_riscv_hfence_vvma_all (vcpu->kvm, +0 , hmask); +else +kvm_riscv_hfence_vvma_gva (vcpu->kvm, +0 , hmask, + cp->a1, cp->a2, +PAGE_SHIFT ); + }else { +if (cp->a1 ==0 && cp->a2 ==0 ) +kvm_riscv_hfence_vvma_asid_all (vcpu->kvm, +0 , hmask, + cp->a3); +else +kvm_riscv_hfence_vvma_asid_gva (vcpu->kvm, +0 , hmask, + cp->a1, cp->a2, +PAGE_SHIFT , + cp->a3); + }break ;default : ret = -EINVAL ;
arch/riscv/kvm/vmid.c +8 -2
@@ -11,9 +11,9 @@#include <linux/errno.h> #include <linux/err.h> #include <linux/module.h> +#include <linux/smp.h> #include <linux/kvm_host.h> #include <asm/csr.h> -#include <asm/sbi.h> static unsigned long vmid_version =1 ;static unsigned long vmid_next;
@@ -63,6 +63,11 @@ bool kvm_riscv_gstage_vmid_ver_changed(struct kvm_vmid *vmid) READ_ONCE(vmid_version)); } +static void __local_hfence_gvma_all (void *info) +{ +kvm_riscv_local_hfence_gvma_all (); +} +void kvm_riscv_gstage_vmid_update (struct kvm_vcpu *vcpu) {unsigned long i;
@@ -101,7 +106,8 @@ void kvm_riscv_gstage_vmid_update(struct kvm_vcpu *vcpu)* running, we force VM exits on all host CPUs using IPI and * flush all Guest TLBs. */ -sbi_remote_hfence_gvma (cpu_online_mask,0 ,0 ); +on_each_cpu_mask (cpu_online_mask, __local_hfence_gvma_all, +NULL ,1 ); } vmid->vmid = vmid_next;
Signature · 1 line
2.25.1