arm64: fix VTTBR_BADDR_MASK
Subsystems (4)
PATCH v5 1/1 arm64: fix VTTBR_BADDR_MASK
The current VTTBR_BADDR_MASK only masks 39 bits, which is broken on current systems. Rather than just add a bit it seems like a good time to also set things at run-time instead of compile time to accomodate more hardware.
This patch sets TCR_EL2.PS, VTCR_EL2.T0SZ and vttbr_baddr_mask in runtime, not compile time.
In ARMv8, EL2 physical address size (TCR_EL2.PS) and stage2 input address size (VTCR_EL2.T0SZE) cannot be determined in compile time since they depend on hardware capability.
According to Table D4-23 and Table D4-25 in ARM DDI 0487A.b document, vttbr_x is calculated using different fixed values with consideration of T0SZ, granule size and the level of translation tables. Therefore, vttbr_baddr_mask should be determined dynamically.
Changes since v4:
More minor cleanups from review
Moved some functions into headers
Changes since v3:
Another rebase
Addressed minor comments from v2
Changes since v2:
Rebased on https://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm.git next branch
Changes since v1:
Rebased fix on Jungseok Lee's patch https://lkml.org/lkml/2014/5/12/189 to provide better long term fix. Updated that patch to log error instead of silently fail on unaligned vttbr.
--- arch/arm/include/asm/kvm_mmu.h | 12 ++++++ arch/arm/kvm/arm.c | 17 +++++++- arch/arm64/include/asm/kvm_arm.h | 17 +------- arch/arm64/include/asm/kvm_mmu.h | 78 ++++++++++++++++++++++++++++++++++++++ arch/arm64/kvm/hyp-init.S | 20 +++++++--- 5 files changed, 122 insertions(+), 22 deletions(-)
arch/arm/include/asm/kvm_mmu.h +12 -0
@@ -166,6 +166,18 @@ static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva,void stage2_flush_vm (struct kvm *kvm); +static inline int kvm_get_phys_addr_shift (void ) +{ +return KVM_PHYS_SHIFT ; +} + +static inline int set_vttbr_baddr_mask (void ) +{ + vttbr_baddr_mask =VTTBR_BADDR_MASK ; +return 0 ; +} + +#endif /* !__ASSEMBLY__ */ #endif /* __ARM_KVM_MMU_H__ */
arch/arm/kvm/arm.c +15 -2
@@ -37,6 +37,7 @@#include <asm/mman.h> #include <asm/tlbflush.h> #include <asm/cacheflush.h> +#include <asm/cputype.h> #include <asm/virt.h> #include <asm/kvm_arm.h> #include <asm/kvm_asm.h>
@@ -466,8 +467,14 @@ static void update_vttbr(struct kvm *kvm)/* update vttbr to be used with the new vmid */ pgd_phys =virt_to_phys (kvm->arch.pgd); vmid = ((u64)(kvm->arch.vmid) <<VTTBR_VMID_SHIFT ) &VTTBR_VMID_MASK ; - kvm->arch.vttbr = pgd_phys &VTTBR_BADDR_MASK ; - kvm->arch.vttbr |= vmid; + +/* +* If the VTTBR isn't aligned there is something wrong with the system +* or kernel. +*/ +BUG_ON (pgd_phys & ~vttbr_baddr_mask); + + kvm->arch.vttbr = pgd_phys | vmid;spin_unlock (&kvm_vmid_lock); }
@@ -1052,6 +1059,12 @@ int kvm_arch_init(void *opaque) } } + err =set_vttbr_baddr_mask (); +if (err) { +kvm_err ("Cannot set vttbr_baddr_mask\n" ); +return -EINVAL ; + } +cpu_notifier_register_begin (); err =init_hyp_mode ();
arch/arm64/include/asm/kvm_arm.h +3 -14
@@ -94,7 +94,6 @@/* TCR_EL2 Registers bits */ #define TCR_EL2_TBI (1 << 20)#define TCR_EL2_PS (7 << 16) -#define TCR_EL2_PS_40B (2 << 16)#define TCR_EL2_TG0 (1 << 14)#define TCR_EL2_SH0 (3 << 12)#define TCR_EL2_ORGN0 (3 << 10)
@@ -103,8 +102,6 @@#define TCR_EL2_MASK (TCR_EL2_TG0 | TCR_EL2_SH0 | \ TCR_EL2_ORGN0 | TCR_EL2_IRGN0 | TCR_EL2_T0SZ) -#define TCR_EL2_FLAGS (TCR_EL2_PS_40B) -/* VTCR_EL2 Registers bits */ #define VTCR_EL2_PS_MASK (7 << 16)#define VTCR_EL2_TG0_MASK (1 << 14)
@@ -119,36 +116,28 @@#define VTCR_EL2_SL0_MASK (3 << 6)#define VTCR_EL2_SL0_LVL1 (1 << 6)#define VTCR_EL2_T0SZ_MASK 0x3f -#define VTCR_EL2_T0SZ_40B 24 +#define VTCR_EL2_T0SZ (bits) (64 - (bits))#ifdef CONFIG_ARM64_64K_PAGES /* * Stage2 translation configuration: -* 40bits output (PS = 2) -* 40bits input (T0SZ = 24) * 64kB pages (TG0 = 1) * 2 level page tables (SL = 1) */ #define VTCR_EL2_FLAGS (VTCR_EL2_TG0_64K | VTCR_EL2_SH0_INNER | \ VTCR_EL2_ORGN0_WBWA | VTCR_EL2_IRGN0_WBWA | \ - VTCR_EL2_SL0_LVL1 | VTCR_EL2_T0SZ_40B) -#define VTTBR_X (38 - VTCR_EL2_T0SZ_40B) + VTCR_EL2_SL0_LVL1)#else /* * Stage2 translation configuration: -* 40bits output (PS = 2) -* 40bits input (T0SZ = 24) * 4kB pages (TG0 = 0) * 3 level page tables (SL = 1) */ #define VTCR_EL2_FLAGS (VTCR_EL2_TG0_4K | VTCR_EL2_SH0_INNER | \ VTCR_EL2_ORGN0_WBWA | VTCR_EL2_IRGN0_WBWA | \ - VTCR_EL2_SL0_LVL1 | VTCR_EL2_T0SZ_40B) -#define VTTBR_X (37 - VTCR_EL2_T0SZ_40B) + VTCR_EL2_SL0_LVL1)#endif -#define VTTBR_BADDR_SHIFT (VTTBR_X - 1) -#define VTTBR_BADDR_MASK (((1LLU << (40 - VTTBR_X)) - 1) << VTTBR_BADDR_SHIFT)#define VTTBR_VMID_SHIFT (48LLU)#define VTTBR_VMID_MASK (0xffLLU << VTTBR_VMID_SHIFT)
arch/arm64/include/asm/kvm_mmu.h +78 -0
83 lines not on this page · raw message
arch/arm64/kvm/hyp-init.S +14 -6
@@ -63,17 +63,21 @@ __do_hyp_init: mrs x4, tcr_el1 ldr x5, =TCR_EL2_MASK and x4, x4, x5 - ldr x5, =TCR_EL2_FLAGS - orr x4, x4, x5 - msr tcr_el2, x4 - - ldr x4, =VTCR_EL2_FLAGS /* * Read the PARange bits from ID_AA64MMFR0_EL1 and set the PS bits in - * VTCR_EL2. + * TCR_EL2 and both PS bits and T0SZ bits in VTCR_EL2. */ mrs x5, ID_AA64MMFR0_EL1 bfi x4, x5, #16, #3 + msr tcr_el2, x4 + + ldr x4, =VTCR_EL2_FLAGS + bfi x4, x5, #16, #3 + and x5, x5, #0xf + adr x6, t0sz + add x6, x6, x5, lsl #2 + ldr w5, [x6] + orr x4, x4, x5 msr vtcr_el2, x4 mrs x4, mair_el1
@@ -109,6 +113,10 @@ target: /* We're now in the trampoline code, switch page tables */ /* Hello, World! */ eret + +t0sz: + .word VTCR_EL2_T0SZ(32), VTCR_EL2_T0SZ(36), VTCR_EL2_T0SZ(40) + .word VTCR_EL2_T0SZ(42), VTCR_EL2_T0SZ(44), VTCR_EL2_T0SZ(48) ENDPROC(__kvm_hyp_init) .ltorg
Quoted 202 lines · Joel Schopp
The current
VTTBR_BADDR_MASKonly masks 39 bits, which is broken on current systems. Rather than just add a bit it seems like a good time to also set things at run-time instead of compile time to accomodate more hardware.This patch sets TCR_EL2.PS, VTCR_EL2.T0SZ and
vttbr_baddr_maskin runtime, not compile time.In ARMv8, EL2 physical address size (TCR_EL2.PS) and stage2 input address size (VTCR_EL2.T0SZE) cannot be determined in compile time since they depend on hardware capability.
According to Table D4-23 and Table D4-25 in ARM DDI 0487A.b document,
vttbr_xis calculated using different fixed values with consideration of T0SZ, granule size and the level of translation tables. Therefore,vttbr_baddr_maskshould be determined dynamically.Changes since v4:
More minor cleanups from review
Moved some functions into headersChanges since v3:
Another rebase
Addressed minor comments from v2Changes since v2:
Rebased on https://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm.git next branchChanges since v1:
Rebased fix on Jungseok Lee's patch https://lkml.org/lkml/2014/5/12/189 to provide better long term fix. Updated that patch to log error instead of silently fail on unaligned vttbr.Cc Christoffer Dall <[email protected]>Cc Sungjinn Chung <[email protected]>Signed-off-by Jungseok Lee <[email protected]>Signed-off-by Joel Schopp <[email protected]>--- arch/arm/include/asm/kvm_mmu.h | 12 ++++++ arch/arm/kvm/arm.c | 17 +++++++- arch/arm64/include/asm/kvm_arm.h | 17 +------- arch/arm64/include/asm/kvm_mmu.h | 78 ++++++++++++++++++++++++++++++++++++++ arch/arm64/kvm/hyp-init.S | 20 +++++++--- 5 files changed, 122 insertions(+), 22 deletions(-)arch/arm/include/asm/kvm_mmu.h +12
-0@@ -166,6 +166,18 @@ static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva,void stage2_flush_vm (struct kvm *kvm); +static inline int kvm_get_phys_addr_shift (void ) +{ +return KVM_PHYS_SHIFT ; +} + +static inline int set_vttbr_baddr_mask (void ) +{ + vttbr_baddr_mask =VTTBR_BADDR_MASK ; +return 0 ; +} + +#endif /* !__ASSEMBLY__ */ #endif /* __ARM_KVM_MMU_H__ */ arch/arm/kvm/arm.c +15
-2@@ -37,6 +37,7 @@#include <asm/mman.h> #include <asm/tlbflush.h> #include <asm/cacheflush.h> +#include <asm/cputype.h> #include <asm/virt.h> #include <asm/kvm_arm.h> #include <asm/kvm_asm.h> @@ -466,8 +467,14 @@ static void update_vttbr(struct kvm *kvm)/* update vttbr to be used with the new vmid */ pgd_phys =virt_to_phys (kvm->arch.pgd); vmid = ((u64)(kvm->arch.vmid) <<VTTBR_VMID_SHIFT ) &VTTBR_VMID_MASK ; - kvm->arch.vttbr = pgd_phys &VTTBR_BADDR_MASK ; - kvm->arch.vttbr |= vmid; + +/* +* If the VTTBR isn't aligned there is something wrong with the system +* or kernel. +*/ +BUG_ON (pgd_phys & ~vttbr_baddr_mask); + + kvm->arch.vttbr = pgd_phys | vmid;spin_unlock (&kvm_vmid_lock); }@@ -1052,6 +1059,12 @@ int kvm_arch_init(void *opaque) } } + err =set_vttbr_baddr_mask (); +if (err) { +kvm_err ("Cannot set vttbr_baddr_mask\n" ); +return -EINVAL ; + } +cpu_notifier_register_begin (); err =init_hyp_mode ();arch/arm64/include/asm/kvm_arm.h +3
-14@@ -94,7 +94,6 @@/* TCR_EL2 Registers bits */ #define TCR_EL2_TBI (1 << 20)#define TCR_EL2_PS (7 << 16) -#define TCR_EL2_PS_40B (2 << 16)#define TCR_EL2_TG0 (1 << 14)#define TCR_EL2_SH0 (3 << 12)#define TCR_EL2_ORGN0 (3 << 10)@@ -103,8 +102,6 @@#define TCR_EL2_MASK (TCR_EL2_TG0 | TCR_EL2_SH0 | \ TCR_EL2_ORGN0 | TCR_EL2_IRGN0 | TCR_EL2_T0SZ) -#define TCR_EL2_FLAGS (TCR_EL2_PS_40B) -/* VTCR_EL2 Registers bits */ #define VTCR_EL2_PS_MASK (7 << 16)#define VTCR_EL2_TG0_MASK (1 << 14)@@ -119,36 +116,28 @@#define VTCR_EL2_SL0_MASK (3 << 6)#define VTCR_EL2_SL0_LVL1 (1 << 6)#define VTCR_EL2_T0SZ_MASK 0x3f -#define VTCR_EL2_T0SZ_40B 24 +#define VTCR_EL2_T0SZ (bits) (64 - (bits))#ifdef CONFIG_ARM64_64K_PAGES /* * Stage2 translation configuration: -* 40bits output (PS = 2) -* 40bits input (T0SZ = 24) * 64kB pages (TG0 = 1) * 2 level page tables (SL = 1) */ #define VTCR_EL2_FLAGS (VTCR_EL2_TG0_64K | VTCR_EL2_SH0_INNER | \ VTCR_EL2_ORGN0_WBWA | VTCR_EL2_IRGN0_WBWA | \ - VTCR_EL2_SL0_LVL1 | VTCR_EL2_T0SZ_40B) -#define VTTBR_X (38 - VTCR_EL2_T0SZ_40B) + VTCR_EL2_SL0_LVL1)#else /* * Stage2 translation configuration: -* 40bits output (PS = 2) -* 40bits input (T0SZ = 24) * 4kB pages (TG0 = 0) * 3 level page tables (SL = 1) */ #define VTCR_EL2_FLAGS (VTCR_EL2_TG0_4K | VTCR_EL2_SH0_INNER | \ VTCR_EL2_ORGN0_WBWA | VTCR_EL2_IRGN0_WBWA | \ - VTCR_EL2_SL0_LVL1 | VTCR_EL2_T0SZ_40B) -#define VTTBR_X (37 - VTCR_EL2_T0SZ_40B) + VTCR_EL2_SL0_LVL1)#endif -#define VTTBR_BADDR_SHIFT (VTTBR_X - 1) -#define VTTBR_BADDR_MASK (((1LLU << (40 - VTTBR_X)) - 1) << VTTBR_BADDR_SHIFT)#define VTTBR_VMID_SHIFT (48LLU)#define VTTBR_VMID_MASK (0xffLLU << VTTBR_VMID_SHIFT)arch/arm64/include/asm/kvm_mmu.h +24
-0@@ -152,5 +152,83 @@ static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva,void stage2_flush_vm (struct kvm *kvm); +/* +* ARMv8 64K architecture limitations: +* 16 <= T0SZ <= 21 is valid under 3 level of translation tables +* 18 <= T0SZ <= 34 is valid under 2 level of translation tables +* 31 <= T0SZ <= 39 is valid under 1 level of transltaion tables +* +* ARMv8 4K architecture limitations: +* 16 <= T0SZ <= 24 is valid under 4 level of translation tables +* 21 <= T0SZ <= 33 is valid under 3 level of translation tables +* 30 <= T0SZ <= 39 is valid under 2 level of translation tables +* +* For 4K pages we only support 3 or 4 level, giving T0SZ a range of 16 to 33. +* For 64K pages we only support 2 or 3 level, giving T0SZ a range of 16 to 34. +* +* See Table D4-23 and Table D4-25 in ARM DDI 0487A.b to figure out +* the origin of the hardcoded values, 38 and 37. +*/ + +#ifdef CONFIG_ARM64_64K_PAGES +static inline int t0sz_to_vttbr_x (int t0sz) +{ +if (t0sz <16 || t0sz >34 ) { +kvm_err ("Cannot support %d-bit address space\n" ,64 - t0sz); +return 0 ;
arch/arm64/include/asm/kvm_mmu.h static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva, · line 188
Quoted 10 lines
arch/arm64/include/asm/kvm_mmu.h +10
-0+ } + +return 38 - t0sz; +} +#else /* 4K pages */ +static inline int t0sz_to_vttbr_x (int t0sz) +{ +if (t0sz <16 || t0sz >33 ) { +kvm_err ("Cannot support %d-bit address space\n" ,64 - t0sz); +return 0 ;
why not just return an error code directly here? These are now returning 0 on error which usually means everything was ok...
Quoted 90 lines
arch/arm64/include/asm/kvm_mmu.h +44
-0+ } +return 37 - t0sz; +} +#endif +static inline int kvm_get_phys_addr_shift (void ) +{ +int pa_range =read_cpuid (ID_AA64MMFR0_EL1 ) &0xf ; + +switch (pa_range) { +case 0 :return 32 ; +case 1 :return 36 ; +case 2 :return 40 ; +case 3 :return 42 ; +case 4 :return 44 ; +case 5 :return 48 ; +default : +BUG (); +return 0 ; + } +} + +static u64 vttbr_baddr_mask; + +/** +* set_vttbr_baddr_mask - set mask value for vttbr base address +* +* In ARMv8, vttbr_baddr_mask cannot be determined in compile time since the +* stage2 input address size depends on hardware capability. Thus, we first +* need to read ID_AA64MMFR0_EL1.PARange and then set vttbr_baddr_mask with +* consideration of both the granule size and the level of translation tables. +*/ +static inline int set_vttbr_baddr_mask (void ) +{ +int t0sz, vttbr_x; + + t0sz =VTCR_EL2_T0SZ (kvm_get_phys_addr_shift ()); + vttbr_x =t0sz_to_vttbr_x (t0sz); +if (!vttbr_x) +return -EINVAL ; + vttbr_baddr_mask = (((1LLU << (48 - vttbr_x)) -1 ) << (vttbr_x -1 )); + +return 0 ; +} +#endif /* __ASSEMBLY__ */ #endif /* __ARM64_KVM_MMU_H__ */ arch/arm64/kvm/hyp-init.S +14
-6@@ -63,17 +63,21 @@ __do_hyp_init: mrs x4, tcr_el1 ldr x5, =TCR_EL2_MASK and x4, x4, x5 - ldr x5, =TCR_EL2_FLAGS - orr x4, x4, x5 - msr tcr_el2, x4 - - ldr x4, =VTCR_EL2_FLAGS /* * Read the PARange bits from ID_AA64MMFR0_EL1 and set the PS bits in - * VTCR_EL2. + * TCR_EL2 and both PS bits and T0SZ bits in VTCR_EL2. */ mrs x5, ID_AA64MMFR0_EL1 bfi x4, x5, #16, #3 + msr tcr_el2, x4 + + ldr x4, =VTCR_EL2_FLAGS + bfi x4, x5, #16, #3 + and x5, x5, #0xf + adr x6, t0sz + add x6, x6, x5, lsl #2 + ldr w5, [x6] + orr x4, x4, x5 msr vtcr_el2, x4 mrs x4, mair_el1@@ -109,6 +113,10 @@ target: /* We're now in the trampoline code, switch page tables */ /* Hello, World! */ eret + +t0sz: + .word VTCR_EL2_T0SZ(32), VTCR_EL2_T0SZ(36), VTCR_EL2_T0SZ(40) + .word VTCR_EL2_T0SZ(42), VTCR_EL2_T0SZ(44), VTCR_EL2_T0SZ(48) ENDPROC(__kvm_hyp_init) .ltorg
Quoted 207 lines · Christoffer Dall
Quoted 206 lines · Joel Schopp
The current
VTTBR_BADDR_MASKonly masks 39 bits, which is broken on current systems. Rather than just add a bit it seems like a good time to also set things at run-time instead of compile time to accomodate more hardware.This patch sets TCR_EL2.PS, VTCR_EL2.T0SZ and
vttbr_baddr_maskin runtime, not compile time.In ARMv8, EL2 physical address size (TCR_EL2.PS) and stage2 input address size (VTCR_EL2.T0SZE) cannot be determined in compile time since they depend on hardware capability.
According to Table D4-23 and Table D4-25 in ARM DDI 0487A.b document,
vttbr_xis calculated using different fixed values with consideration of T0SZ, granule size and the level of translation tables. Therefore,vttbr_baddr_maskshould be determined dynamically.Changes since v4:
More minor cleanups from review
Moved some functions into headersChanges since v3:
Another rebase
Addressed minor comments from v2Changes since v2:
Rebased on https://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm.git next branchChanges since v1:
Rebased fix on Jungseok Lee's patch https://lkml.org/lkml/2014/5/12/189 to provide better long term fix. Updated that patch to log error instead of silently fail on unaligned vttbr.Cc Christoffer Dall <[email protected]>Cc Sungjinn Chung <[email protected]>Signed-off-by Jungseok Lee <[email protected]>Signed-off-by Joel Schopp <[email protected]>--- arch/arm/include/asm/kvm_mmu.h | 12 ++++++ arch/arm/kvm/arm.c | 17 +++++++- arch/arm64/include/asm/kvm_arm.h | 17 +------- arch/arm64/include/asm/kvm_mmu.h | 78 ++++++++++++++++++++++++++++++++++++++ arch/arm64/kvm/hyp-init.S | 20 +++++++--- 5 files changed, 122 insertions(+), 22 deletions(-)arch/arm/include/asm/kvm_mmu.h +12
-0@@ -166,6 +166,18 @@ static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva,void stage2_flush_vm (struct kvm *kvm); +static inline int kvm_get_phys_addr_shift (void ) +{ +return KVM_PHYS_SHIFT ; +} + +static inline int set_vttbr_baddr_mask (void ) +{ + vttbr_baddr_mask =VTTBR_BADDR_MASK ; +return 0 ; +} + +#endif /* !__ASSEMBLY__ */ #endif /* __ARM_KVM_MMU_H__ */ arch/arm/kvm/arm.c +15
-2@@ -37,6 +37,7 @@#include <asm/mman.h> #include <asm/tlbflush.h> #include <asm/cacheflush.h> +#include <asm/cputype.h> #include <asm/virt.h> #include <asm/kvm_arm.h> #include <asm/kvm_asm.h> @@ -466,8 +467,14 @@ static void update_vttbr(struct kvm *kvm)/* update vttbr to be used with the new vmid */ pgd_phys =virt_to_phys (kvm->arch.pgd); vmid = ((u64)(kvm->arch.vmid) <<VTTBR_VMID_SHIFT ) &VTTBR_VMID_MASK ; - kvm->arch.vttbr = pgd_phys &VTTBR_BADDR_MASK ; - kvm->arch.vttbr |= vmid; + +/* +* If the VTTBR isn't aligned there is something wrong with the system +* or kernel. +*/ +BUG_ON (pgd_phys & ~vttbr_baddr_mask); + + kvm->arch.vttbr = pgd_phys | vmid;spin_unlock (&kvm_vmid_lock); }@@ -1052,6 +1059,12 @@ int kvm_arch_init(void *opaque) } } + err =set_vttbr_baddr_mask (); +if (err) { +kvm_err ("Cannot set vttbr_baddr_mask\n" ); +return -EINVAL ; + } +cpu_notifier_register_begin (); err =init_hyp_mode ();arch/arm64/include/asm/kvm_arm.h +3
-14@@ -94,7 +94,6 @@/* TCR_EL2 Registers bits */ #define TCR_EL2_TBI (1 << 20)#define TCR_EL2_PS (7 << 16) -#define TCR_EL2_PS_40B (2 << 16)#define TCR_EL2_TG0 (1 << 14)#define TCR_EL2_SH0 (3 << 12)#define TCR_EL2_ORGN0 (3 << 10)@@ -103,8 +102,6 @@#define TCR_EL2_MASK (TCR_EL2_TG0 | TCR_EL2_SH0 | \ TCR_EL2_ORGN0 | TCR_EL2_IRGN0 | TCR_EL2_T0SZ) -#define TCR_EL2_FLAGS (TCR_EL2_PS_40B) -/* VTCR_EL2 Registers bits */ #define VTCR_EL2_PS_MASK (7 << 16)#define VTCR_EL2_TG0_MASK (1 << 14)@@ -119,36 +116,28 @@#define VTCR_EL2_SL0_MASK (3 << 6)#define VTCR_EL2_SL0_LVL1 (1 << 6)#define VTCR_EL2_T0SZ_MASK 0x3f -#define VTCR_EL2_T0SZ_40B 24 +#define VTCR_EL2_T0SZ (bits) (64 - (bits))#ifdef CONFIG_ARM64_64K_PAGES /* * Stage2 translation configuration: -* 40bits output (PS = 2) -* 40bits input (T0SZ = 24) * 64kB pages (TG0 = 1) * 2 level page tables (SL = 1) */ #define VTCR_EL2_FLAGS (VTCR_EL2_TG0_64K | VTCR_EL2_SH0_INNER | \ VTCR_EL2_ORGN0_WBWA | VTCR_EL2_IRGN0_WBWA | \ - VTCR_EL2_SL0_LVL1 | VTCR_EL2_T0SZ_40B) -#define VTTBR_X (38 - VTCR_EL2_T0SZ_40B) + VTCR_EL2_SL0_LVL1)#else /* * Stage2 translation configuration: -* 40bits output (PS = 2) -* 40bits input (T0SZ = 24) * 4kB pages (TG0 = 0) * 3 level page tables (SL = 1) */ #define VTCR_EL2_FLAGS (VTCR_EL2_TG0_4K | VTCR_EL2_SH0_INNER | \ VTCR_EL2_ORGN0_WBWA | VTCR_EL2_IRGN0_WBWA | \ - VTCR_EL2_SL0_LVL1 | VTCR_EL2_T0SZ_40B) -#define VTTBR_X (37 - VTCR_EL2_T0SZ_40B) + VTCR_EL2_SL0_LVL1)#endif -#define VTTBR_BADDR_SHIFT (VTTBR_X - 1) -#define VTTBR_BADDR_MASK (((1LLU << (40 - VTTBR_X)) - 1) << VTTBR_BADDR_SHIFT)#define VTTBR_VMID_SHIFT (48LLU)#define VTTBR_VMID_MASK (0xffLLU << VTTBR_VMID_SHIFT)arch/arm64/include/asm/kvm_mmu.h +28
-0@@ -152,5 +152,83 @@ static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva,void stage2_flush_vm (struct kvm *kvm); +/* +* ARMv8 64K architecture limitations: +* 16 <= T0SZ <= 21 is valid under 3 level of translation tables +* 18 <= T0SZ <= 34 is valid under 2 level of translation tables +* 31 <= T0SZ <= 39 is valid under 1 level of transltaion tables +* +* ARMv8 4K architecture limitations: +* 16 <= T0SZ <= 24 is valid under 4 level of translation tables +* 21 <= T0SZ <= 33 is valid under 3 level of translation tables +* 30 <= T0SZ <= 39 is valid under 2 level of translation tables +* +* For 4K pages we only support 3 or 4 level, giving T0SZ a range of 16 to 33. +* For 64K pages we only support 2 or 3 level, giving T0SZ a range of 16 to 34. +* +* See Table D4-23 and Table D4-25 in ARM DDI 0487A.b to figure out +* the origin of the hardcoded values, 38 and 37. +*/ + +#ifdef CONFIG_ARM64_64K_PAGES +static inline int t0sz_to_vttbr_x (int t0sz) +{ +if (t0sz <16 || t0sz >34 ) { +kvm_err ("Cannot support %d-bit address space\n" ,64 - t0sz); +return 0 ; + } + +return 38 - t0sz; +}
Quoted 8 lines
arch/arm64/include/asm/kvm_mmu.h static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva, · line 188
Quoted 6 lines
arch/arm64/include/asm/kvm_mmu.h +6
-0+#else /* 4K pages */ +static inline int t0sz_to_vttbr_x (int t0sz) +{ +if (t0sz <16 || t0sz >33 ) { +kvm_err ("Cannot support %d-bit address space\n" ,64 - t0sz); +return 0 ;why not just return an error code directly here? These are now returning 0 on error which usually means everything was ok...
The return is a value,not just an error code. Because of this returning an error overloads that value. 0 just seemed like a convenient invalid value to check since a vttbr_x of 0 is invalid, but returning a negative error code would be as equally invalid. If this is the only issue it doesn't seem worth respinning the patch for, but I'll change it to -EINVAL if for some reason a v6 is needed.
Quoted 217 lines · Joel Schopp
Quoted 207 lines · Christoffer Dall
Quoted 206 lines · Joel Schopp
The current
VTTBR_BADDR_MASKonly masks 39 bits, which is broken on current systems. Rather than just add a bit it seems like a good time to also set things at run-time instead of compile time to accomodate more hardware.This patch sets TCR_EL2.PS, VTCR_EL2.T0SZ and
vttbr_baddr_maskin runtime, not compile time.In ARMv8, EL2 physical address size (TCR_EL2.PS) and stage2 input address size (VTCR_EL2.T0SZE) cannot be determined in compile time since they depend on hardware capability.
According to Table D4-23 and Table D4-25 in ARM DDI 0487A.b document,
vttbr_xis calculated using different fixed values with consideration of T0SZ, granule size and the level of translation tables. Therefore,vttbr_baddr_maskshould be determined dynamically.Changes since v4:
More minor cleanups from review
Moved some functions into headersChanges since v3:
Another rebase
Addressed minor comments from v2Changes since v2:
Rebased on https://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm.git next branchChanges since v1:
Rebased fix on Jungseok Lee's patch https://lkml.org/lkml/2014/5/12/189 to provide better long term fix. Updated that patch to log error instead of silently fail on unaligned vttbr.Cc Christoffer Dall <[email protected]>Cc Sungjinn Chung <[email protected]>Signed-off-by Jungseok Lee <[email protected]>Signed-off-by Joel Schopp <[email protected]>--- arch/arm/include/asm/kvm_mmu.h | 12 ++++++ arch/arm/kvm/arm.c | 17 +++++++- arch/arm64/include/asm/kvm_arm.h | 17 +------- arch/arm64/include/asm/kvm_mmu.h | 78 ++++++++++++++++++++++++++++++++++++++ arch/arm64/kvm/hyp-init.S | 20 +++++++--- 5 files changed, 122 insertions(+), 22 deletions(-)arch/arm/include/asm/kvm_mmu.h +12
-0@@ -166,6 +166,18 @@ static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva,void stage2_flush_vm (struct kvm *kvm); +static inline int kvm_get_phys_addr_shift (void ) +{ +return KVM_PHYS_SHIFT ; +} + +static inline int set_vttbr_baddr_mask (void ) +{ + vttbr_baddr_mask =VTTBR_BADDR_MASK ; +return 0 ; +} + +#endif /* !__ASSEMBLY__ */ #endif /* __ARM_KVM_MMU_H__ */ arch/arm/kvm/arm.c +15
-2@@ -37,6 +37,7 @@#include <asm/mman.h> #include <asm/tlbflush.h> #include <asm/cacheflush.h> +#include <asm/cputype.h> #include <asm/virt.h> #include <asm/kvm_arm.h> #include <asm/kvm_asm.h> @@ -466,8 +467,14 @@ static void update_vttbr(struct kvm *kvm)/* update vttbr to be used with the new vmid */ pgd_phys =virt_to_phys (kvm->arch.pgd); vmid = ((u64)(kvm->arch.vmid) <<VTTBR_VMID_SHIFT ) &VTTBR_VMID_MASK ; - kvm->arch.vttbr = pgd_phys &VTTBR_BADDR_MASK ; - kvm->arch.vttbr |= vmid; + +/* +* If the VTTBR isn't aligned there is something wrong with the system +* or kernel. +*/ +BUG_ON (pgd_phys & ~vttbr_baddr_mask); + + kvm->arch.vttbr = pgd_phys | vmid;spin_unlock (&kvm_vmid_lock); }@@ -1052,6 +1059,12 @@ int kvm_arch_init(void *opaque) } } + err =set_vttbr_baddr_mask (); +if (err) { +kvm_err ("Cannot set vttbr_baddr_mask\n" ); +return -EINVAL ; + } +cpu_notifier_register_begin (); err =init_hyp_mode ();arch/arm64/include/asm/kvm_arm.h +3
-14@@ -94,7 +94,6 @@/* TCR_EL2 Registers bits */ #define TCR_EL2_TBI (1 << 20)#define TCR_EL2_PS (7 << 16) -#define TCR_EL2_PS_40B (2 << 16)#define TCR_EL2_TG0 (1 << 14)#define TCR_EL2_SH0 (3 << 12)#define TCR_EL2_ORGN0 (3 << 10)@@ -103,8 +102,6 @@#define TCR_EL2_MASK (TCR_EL2_TG0 | TCR_EL2_SH0 | \ TCR_EL2_ORGN0 | TCR_EL2_IRGN0 | TCR_EL2_T0SZ) -#define TCR_EL2_FLAGS (TCR_EL2_PS_40B) -/* VTCR_EL2 Registers bits */ #define VTCR_EL2_PS_MASK (7 << 16)#define VTCR_EL2_TG0_MASK (1 << 14)@@ -119,36 +116,28 @@#define VTCR_EL2_SL0_MASK (3 << 6)#define VTCR_EL2_SL0_LVL1 (1 << 6)#define VTCR_EL2_T0SZ_MASK 0x3f -#define VTCR_EL2_T0SZ_40B 24 +#define VTCR_EL2_T0SZ (bits) (64 - (bits))#ifdef CONFIG_ARM64_64K_PAGES /* * Stage2 translation configuration: -* 40bits output (PS = 2) -* 40bits input (T0SZ = 24) * 64kB pages (TG0 = 1) * 2 level page tables (SL = 1) */ #define VTCR_EL2_FLAGS (VTCR_EL2_TG0_64K | VTCR_EL2_SH0_INNER | \ VTCR_EL2_ORGN0_WBWA | VTCR_EL2_IRGN0_WBWA | \ - VTCR_EL2_SL0_LVL1 | VTCR_EL2_T0SZ_40B) -#define VTTBR_X (38 - VTCR_EL2_T0SZ_40B) + VTCR_EL2_SL0_LVL1)#else /* * Stage2 translation configuration: -* 40bits output (PS = 2) -* 40bits input (T0SZ = 24) * 4kB pages (TG0 = 0) * 3 level page tables (SL = 1) */ #define VTCR_EL2_FLAGS (VTCR_EL2_TG0_4K | VTCR_EL2_SH0_INNER | \ VTCR_EL2_ORGN0_WBWA | VTCR_EL2_IRGN0_WBWA | \ - VTCR_EL2_SL0_LVL1 | VTCR_EL2_T0SZ_40B) -#define VTTBR_X (37 - VTCR_EL2_T0SZ_40B) + VTCR_EL2_SL0_LVL1)#endif -#define VTTBR_BADDR_SHIFT (VTTBR_X - 1) -#define VTTBR_BADDR_MASK (((1LLU << (40 - VTTBR_X)) - 1) << VTTBR_BADDR_SHIFT)#define VTTBR_VMID_SHIFT (48LLU)#define VTTBR_VMID_MASK (0xffLLU << VTTBR_VMID_SHIFT)arch/arm64/include/asm/kvm_mmu.h +28
-0@@ -152,5 +152,83 @@ static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva,void stage2_flush_vm (struct kvm *kvm); +/* +* ARMv8 64K architecture limitations: +* 16 <= T0SZ <= 21 is valid under 3 level of translation tables +* 18 <= T0SZ <= 34 is valid under 2 level of translation tables +* 31 <= T0SZ <= 39 is valid under 1 level of transltaion tables +* +* ARMv8 4K architecture limitations: +* 16 <= T0SZ <= 24 is valid under 4 level of translation tables +* 21 <= T0SZ <= 33 is valid under 3 level of translation tables +* 30 <= T0SZ <= 39 is valid under 2 level of translation tables +* +* For 4K pages we only support 3 or 4 level, giving T0SZ a range of 16 to 33. +* For 64K pages we only support 2 or 3 level, giving T0SZ a range of 16 to 34. +* +* See Table D4-23 and Table D4-25 in ARM DDI 0487A.b to figure out +* the origin of the hardcoded values, 38 and 37. +*/ + +#ifdef CONFIG_ARM64_64K_PAGES +static inline int t0sz_to_vttbr_x (int t0sz) +{ +if (t0sz <16 || t0sz >34 ) { +kvm_err ("Cannot support %d-bit address space\n" ,64 - t0sz); +return 0 ; + } + +return 38 - t0sz; +}Quoted 8 lines
arch/arm64/include/asm/kvm_mmu.h static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva, · line 188
Quoted 6 lines
arch/arm64/include/asm/kvm_mmu.h +6
-0+#else /* 4K pages */ +static inline int t0sz_to_vttbr_x (int t0sz) +{ +if (t0sz <16 || t0sz >33 ) { +kvm_err ("Cannot support %d-bit address space\n" ,64 - t0sz); +return 0 ;why not just return an error code directly here? These are now returning 0 on error which usually means everything was ok...
Quoted 6 lines
The return is a value,not just an error code. Because of this returning an error overloads that value. 0 just seemed like a convenient invalid value to check since a
vttbr_xof 0 is invalid, but returning a negative error code would be as equally invalid. If this is the only issue it doesn't seem worth respinning the patch for, but I'll change it to-EINVALif for some reason a v6 is needed.
Have you given up on doing the alignment check with the proper size on the pgd allocation for this patch?
-Christoffer
Quoted 9 lines
Quoted 7 lines
The return is a value,not just an error code. Because of this returning an error overloads that value. 0 just seemed like a convenient invalid value to check since a
vttbr_xof 0 is invalid, but returning a negative error code would be as equally invalid. If this is the only issue it doesn't seem worth respinning the patch for, but I'll change it to-EINVALif for some reason a v6 is needed.Have you given up on doing the alignment check with the proper size on the pgd allocation for this patch?
Yes, I'd rather leave the extra check out of this patch. If I were changing the pgd allocation code I would make sure to add a check, or if there were a static check there now I would update it for the dynamic value from the hardware, but it seems unrelated to add several checks to other parts of the code beyond those already in the patch. I did leave the functions in the headers such that checks like this could be added when someone is updating the code for other reasons, say 4 level page tables.
-Joel
Quoted 9 lines · Joel Schopp
Quoted 9 lines
Quoted 7 lines
The return is a value,not just an error code. Because of this returning an error overloads that value. 0 just seemed like a convenient invalid value to check since a
vttbr_xof 0 is invalid, but returning a negative error code would be as equally invalid. If this is the only issue it doesn't seem worth respinning the patch for, but I'll change it to-EINVALif for some reason a v6 is needed.Have you given up on doing the alignment check with the proper size on the pgd allocation for this patch?
Quoted 9 lines
Yes, I'd rather leave the extra check out of this patch. If I were changing the pgd allocation code I would make sure to add a check, or if there were a static check there now I would update it for the dynamic value from the hardware, but it seems unrelated to add several checks to other parts of the code beyond those already in the patch. I did leave the functions in the headers such that checks like this could be added when someone is updating the code for other reasons, say 4 level page tables.
hmmm, the point is that we need to ensure that we have a properly aligned allocated PGD, that's what this patch currently addresses, and as you pointed out, the BUG_ON() just before trying to run a VM is not the nicest solution - we should really be dealing with this properly at allocation time.
But, if you don't have time to look at that, then ok, I'll have to pick it up myself.
However, you are hinting that we do not support 4 levels of page tables, yet you do allow the t0sz_to_vttbr_x funciton to pass even when using t0sz values only supported under 4 levels of page tables.... What is the rationale for that?
-Christoffer
Quoted 7 lines
hmmm, the point is that we need to ensure that we have a properly aligned allocated PGD, that's what this patch currently addresses, and as you pointed out, the
BUG_ON()just before trying to run a VM is not the nicest solution - we should really be dealing with this properly at allocation time.
Quoted 7 lines
But, if you don't have time to look at that, then ok, I'll have to pick it up myself.
However, you are hinting that we do not support 4 levels of page tables, yet you do allow the
t0sz_to_vttbr_xfunciton to pass even when using t0sz values only supported under 4 levels of page tables.... What is the rationale for that?
Minimizing merge conflicts. I figure both are going in within the next month or two.
Hi Joel,
Quoted 50 lines · Joel Schopp
The current
VTTBR_BADDR_MASKonly masks 39 bits, which is broken on current systems. Rather than just add a bit it seems like a good time to also set things at run-time instead of compile time to accomodate more hardware.This patch sets TCR_EL2.PS, VTCR_EL2.T0SZ and
vttbr_baddr_maskin runtime, not compile time.In ARMv8, EL2 physical address size (TCR_EL2.PS) and stage2 input address size (VTCR_EL2.T0SZE) cannot be determined in compile time since they depend on hardware capability.
According to Table D4-23 and Table D4-25 in ARM DDI 0487A.b document,
vttbr_xis calculated using different fixed values with consideration of T0SZ, granule size and the level of translation tables. Therefore,vttbr_baddr_maskshould be determined dynamically.Changes since v4:
More minor cleanups from review
Moved some functions into headersChanges since v3:
Another rebase
Addressed minor comments from v2Changes since v2:
Rebased on https://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm.git next branchChanges since v1:
Rebased fix on Jungseok Lee's patch https://lkml.org/lkml/2014/5/12/189 to provide better long term fix. Updated that patch to log error instead of silently fail on unaligned vttbr.Cc Christoffer Dall <[email protected]>Cc Sungjinn Chung <[email protected]>Signed-off-by Jungseok Lee <[email protected]>Signed-off-by Joel Schopp <[email protected]>--- arch/arm/include/asm/kvm_mmu.h | 12 ++++++ arch/arm/kvm/arm.c | 17 +++++++- arch/arm64/include/asm/kvm_arm.h | 17 +------- arch/arm64/include/asm/kvm_mmu.h | 78 ++++++++++++++++++++++++++++++++++++++ arch/arm64/kvm/hyp-init.S | 20 +++++++--- 5 files changed, 122 insertions(+), 22 deletions(-)arch/arm/include/asm/kvm_mmu.h +0
-0@@ -166,6 +166,18 @@ static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva,
arch/arm/include/asm/kvm_mmu.h static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva, · line 176
Quoted 10 lines
arch/arm/include/asm/kvm_mmu.h +8
-0void stage2_flush_vm (struct kvm *kvm); +static inline int kvm_get_phys_addr_shift (void ) +{ +return KVM_PHYS_SHIFT ; +} + +static inline int set_vttbr_baddr_mask (void ) +{ + vttbr_baddr_mask =VTTBR_BADDR_MASK ;
Have you tried compiling this?
Apart from the obvious missing definition of the variable, I'm not fond of functions with side-effects hidden in an include file. What is wrong with just returning the mask and letting the common code setting it?
Quoted 132 lines
arch/arm/include/asm/kvm_mmu.h +4
-0+return 0 ; +} + +#endif /* !__ASSEMBLY__ */ #endif /* __ARM_KVM_MMU_H__ */ arch/arm/kvm/arm.c +15
-2@@ -37,6 +37,7 @@#include <asm/mman.h> #include <asm/tlbflush.h> #include <asm/cacheflush.h> +#include <asm/cputype.h> #include <asm/virt.h> #include <asm/kvm_arm.h> #include <asm/kvm_asm.h> @@ -466,8 +467,14 @@ static void update_vttbr(struct kvm *kvm)/* update vttbr to be used with the new vmid */ pgd_phys =virt_to_phys (kvm->arch.pgd); vmid = ((u64)(kvm->arch.vmid) <<VTTBR_VMID_SHIFT ) &VTTBR_VMID_MASK ; - kvm->arch.vttbr = pgd_phys &VTTBR_BADDR_MASK ; - kvm->arch.vttbr |= vmid; + +/* +* If the VTTBR isn't aligned there is something wrong with the system +* or kernel. +*/ +BUG_ON (pgd_phys & ~vttbr_baddr_mask); + + kvm->arch.vttbr = pgd_phys | vmid;spin_unlock (&kvm_vmid_lock); }@@ -1052,6 +1059,12 @@ int kvm_arch_init(void *opaque) } } + err =set_vttbr_baddr_mask (); +if (err) { +kvm_err ("Cannot set vttbr_baddr_mask\n" ); +return -EINVAL ; + } +cpu_notifier_register_begin (); err =init_hyp_mode ();arch/arm64/include/asm/kvm_arm.h +3
-14@@ -94,7 +94,6 @@/* TCR_EL2 Registers bits */ #define TCR_EL2_TBI (1 << 20)#define TCR_EL2_PS (7 << 16) -#define TCR_EL2_PS_40B (2 << 16)#define TCR_EL2_TG0 (1 << 14)#define TCR_EL2_SH0 (3 << 12)#define TCR_EL2_ORGN0 (3 << 10)@@ -103,8 +102,6 @@#define TCR_EL2_MASK (TCR_EL2_TG0 | TCR_EL2_SH0 | \ TCR_EL2_ORGN0 | TCR_EL2_IRGN0 | TCR_EL2_T0SZ) -#define TCR_EL2_FLAGS (TCR_EL2_PS_40B) -/* VTCR_EL2 Registers bits */ #define VTCR_EL2_PS_MASK (7 << 16)#define VTCR_EL2_TG0_MASK (1 << 14)@@ -119,36 +116,28 @@#define VTCR_EL2_SL0_MASK (3 << 6)#define VTCR_EL2_SL0_LVL1 (1 << 6)#define VTCR_EL2_T0SZ_MASK 0x3f -#define VTCR_EL2_T0SZ_40B 24 +#define VTCR_EL2_T0SZ (bits) (64 - (bits))#ifdef CONFIG_ARM64_64K_PAGES /* * Stage2 translation configuration: -* 40bits output (PS = 2) -* 40bits input (T0SZ = 24) * 64kB pages (TG0 = 1) * 2 level page tables (SL = 1) */ #define VTCR_EL2_FLAGS (VTCR_EL2_TG0_64K | VTCR_EL2_SH0_INNER | \ VTCR_EL2_ORGN0_WBWA | VTCR_EL2_IRGN0_WBWA | \ - VTCR_EL2_SL0_LVL1 | VTCR_EL2_T0SZ_40B) -#define VTTBR_X (38 - VTCR_EL2_T0SZ_40B) + VTCR_EL2_SL0_LVL1)#else /* * Stage2 translation configuration: -* 40bits output (PS = 2) -* 40bits input (T0SZ = 24) * 4kB pages (TG0 = 0) * 3 level page tables (SL = 1) */ #define VTCR_EL2_FLAGS (VTCR_EL2_TG0_4K | VTCR_EL2_SH0_INNER | \ VTCR_EL2_ORGN0_WBWA | VTCR_EL2_IRGN0_WBWA | \ - VTCR_EL2_SL0_LVL1 | VTCR_EL2_T0SZ_40B) -#define VTTBR_X (37 - VTCR_EL2_T0SZ_40B) + VTCR_EL2_SL0_LVL1)#endif -#define VTTBR_BADDR_SHIFT (VTTBR_X - 1) -#define VTTBR_BADDR_MASK (((1LLU << (40 - VTTBR_X)) - 1) << VTTBR_BADDR_SHIFT)#define VTTBR_VMID_SHIFT (48LLU)#define VTTBR_VMID_MASK (0xffLLU << VTTBR_VMID_SHIFT)arch/arm64/include/asm/kvm_mmu.h +14
-0@@ -152,5 +152,83 @@ static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva,void stage2_flush_vm (struct kvm *kvm); +/* +* ARMv8 64K architecture limitations: +* 16 <= T0SZ <= 21 is valid under 3 level of translation tables +* 18 <= T0SZ <= 34 is valid under 2 level of translation tables +* 31 <= T0SZ <= 39 is valid under 1 level of transltaion tables +* +* ARMv8 4K architecture limitations: +* 16 <= T0SZ <= 24 is valid under 4 level of translation tables +* 21 <= T0SZ <= 33 is valid under 3 level of translation tables +* 30 <= T0SZ <= 39 is valid under 2 level of translation tables +* +* For 4K pages we only support 3 or 4 level, giving T0SZ a range of 16 to 33. +* For 64K pages we only support 2 or 3 level, giving T0SZ a range of 16 to 34. +*
arch/arm64/include/asm/kvm_mmu.h static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva, · line 178
Quoted 10 lines
arch/arm64/include/asm/kvm_mmu.h +10
-0+* See Table D4-23 and Table D4-25 in ARM DDI 0487A.b to figure out +* the origin of the hardcoded values, 38 and 37. +*/ + +#ifdef CONFIG_ARM64_64K_PAGES +static inline int t0sz_to_vttbr_x (int t0sz) +{ +if (t0sz <16 || t0sz >34 ) { +kvm_err ("Cannot support %d-bit address space\n" ,64 - t0sz); +return 0 ;
0 is definitely a bad value for something that is an error case. Consider -EINVAL instead.
Also, what if we're in a range that only deals with more levels of page tables than the kernel can deal with (remember we use the kernel page table accessors)? See the new ARM64_VA_BITS and ARM64_PGTABLE_LEVELS symbols that are now available, and use them to validate the range you have.
arch/arm64/include/asm/kvm_mmu.h static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva, · line 188
Quoted 10 lines
arch/arm64/include/asm/kvm_mmu.h +10
-0+ } + +return 38 - t0sz; +} +#else /* 4K pages */ +static inline int t0sz_to_vttbr_x (int t0sz) +{ +if (t0sz <16 || t0sz >33 ) { +kvm_err ("Cannot support %d-bit address space\n" ,64 - t0sz); +return 0 ;
Same here.
Quoted 12 lines
arch/arm64/include/asm/kvm_mmu.h +12
-0+ } +return 37 - t0sz; +} +#endif +static inline int kvm_get_phys_addr_shift (void ) +{ +int pa_range =read_cpuid (ID_AA64MMFR0_EL1 ) &0xf ; + +switch (pa_range) { +case 0 :return 32 ; +case 1 :return 36 ; +case 2 :return 40 ;
arch/arm64/include/asm/kvm_mmu.h static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva, · line 210
Quoted 10 lines
arch/arm64/include/asm/kvm_mmu.h +10
-0+case 3 :return 42 ; +case 4 :return 44 ; +case 5 :return 48 ; +default : +BUG (); +return 0 ; + } +} + +static u64 vttbr_baddr_mask;
Now every compilation unit that includes kvm_mmu.h has an instance of this variable. I doubt that it is the intended effect.
Quoted 8 lines
arch/arm64/include/asm/kvm_mmu.h +8
-0+ +/** +* set_vttbr_baddr_mask - set mask value for vttbr base address +* +* In ARMv8, vttbr_baddr_mask cannot be determined in compile time since the +* stage2 input address size depends on hardware capability. Thus, we first +* need to read ID_AA64MMFR0_EL1.PARange and then set vttbr_baddr_mask with +* consideration of both the granule size and the level of translation tables.
arch/arm64/include/asm/kvm_mmu.h static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva, · line 228
Quoted 10 lines
arch/arm64/include/asm/kvm_mmu.h +10
-0+*/ +static inline int set_vttbr_baddr_mask (void ) +{ +int t0sz, vttbr_x; + + t0sz =VTCR_EL2_T0SZ (kvm_get_phys_addr_shift ()); + vttbr_x =t0sz_to_vttbr_x (t0sz); +if (!vttbr_x) +return -EINVAL ; + vttbr_baddr_mask = (((1LLU << (48 - vttbr_x)) -1 ) << (vttbr_x -1 ));
I think this can now be written as GENMASK_ULL(48, (vttbr_x - 1)).
Quoted 24 lines
arch/arm64/include/asm/kvm_mmu.h +3
-0+return 0 ; +} +#endif /* __ASSEMBLY__ */ #endif /* __ARM64_KVM_MMU_H__ */ arch/arm64/kvm/hyp-init.S +1
-6@@ -63,17 +63,21 @@ __do_hyp_init: mrs x4, tcr_el1 ldr x5, =TCR_EL2_MASK and x4, x4, x5 - ldr x5, =TCR_EL2_FLAGS - orr x4, x4, x5 - msr tcr_el2, x4 - - ldr x4, =VTCR_EL2_FLAGS /* * Read the PARange bits from ID_AA64MMFR0_EL1 and set the PS bits in - * VTCR_EL2. + * TCR_EL2 and both PS bits and T0SZ bits in VTCR_EL2. */ mrs x5, ID_AA64MMFR0_EL1
arch/arm64/kvm/hyp-init.S __do_hyp_init: · line 80
Quoted 10 lines
arch/arm64/kvm/hyp-init.S +9
-0bfi x4, x5, #16, #3 + msr tcr_el2, x4 + + ldr x4, =VTCR_EL2_FLAGS + bfi x4, x5, #16, #3 + and x5, x5, #0xf + adr x6, t0sz + add x6, x6, x5, lsl #2 + ldr w5, [x6] + orr x4, x4, x5
You'll need to validate the T0SZ value, and possibly adjust it so that it is compatible with the addressing capability of the kernel. That probably require a slight change of the hyp-init API.
Quoted 5 lines
arch/arm64/kvm/hyp-init.S +0
-0msr vtcr_el2, x4 mrs x4, mair_el1
arch/arm64/kvm/hyp-init.S target: /* We're now in the trampoline code, switch page tables */ · line 122
Quoted 10 lines
arch/arm64/kvm/hyp-init.S +4
-0@@ -109,6 +113,10 @@ target: /* We're now in the trampoline code, switch page tables */ /* Hello, World! */ eret + +t0sz: + .word VTCR_EL2_T0SZ(32), VTCR_EL2_T0SZ(36), VTCR_EL2_T0SZ(40) + .word VTCR_EL2_T0SZ(42), VTCR_EL2_T0SZ(44), VTCR_EL2_T0SZ(48) ENDPROC(__kvm_hyp_init) .ltorg
Another element that doesn't appear in this patch is that we need a way for the kernel to expose the maximum input address to userspace (and validate that noone puts memory outside of that range). This should be a separate patch, but it is conceptually tied to the same problem.
Thanks,
M.
Signature · 1 line
Jazz is not dead. It just smells funny.
Quoted 19 lines
Quoted 7 lines
arch/arm/include/asm/kvm_mmu.h +0
-0@@ -166,6 +166,18 @@ static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva,arch/arm/include/asm/kvm_mmu.h static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva, · line 176
Quoted 10 lines
arch/arm/include/asm/kvm_mmu.h +8
-0void stage2_flush_vm (struct kvm *kvm); +static inline int kvm_get_phys_addr_shift (void ) +{ +return KVM_PHYS_SHIFT ; +} + +static inline int set_vttbr_baddr_mask (void ) +{ + vttbr_baddr_mask =VTTBR_BADDR_MASK ;Have you tried compiling this?
Quoted 3 lines
Apart from the obvious missing definition of the variable, I'm not fond of functions with side-effects hidden in an include file. What is wrong with just returning the mask and letting the common code setting it?
I like that change, will do in v6.
Quoted 8 lines
arch/arm/include/asm/kvm_mmu.h static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva, · line 182
Quoted 6 lines
arch/arm/include/asm/kvm_mmu.h +6
-0+#ifdef CONFIG_ARM64_64K_PAGES +static inline int t0sz_to_vttbr_x (int t0sz) +{ +if (t0sz <16 || t0sz >34 ) { +kvm_err ("Cannot support %d-bit address space\n" ,64 - t0sz); +return 0 ;0 is definitely a bad value for something that is an error case. Consider
-EINVALinstead.
OK.
Quoted 6 lines
Also, what if we're in a range that only deals with more levels of page tables than the kernel can deal with (remember we use the kernel page table accessors)? See the new
ARM64_VA_BITSandARM64_PGTABLE_LEVELSsymbols that are now available, and use them to validate the range you have.
With the simple current tests I can look at them and see they are correct, even if I can't make a scenario to test that they would fail. However, if I add in more complicated checks I'd really like to test them catching the failure cases. Can you describe a case where we can boot a kernel and then have the checks still fail in kvm?
Quoted 17 lines
Quoted 17 lines
arch/arm/include/asm/kvm_mmu.h +16
-0+ } +return 37 - t0sz; +} +#endif +static inline int kvm_get_phys_addr_shift (void ) +{ +int pa_range =read_cpuid (ID_AA64MMFR0_EL1 ) &0xf ; + +switch (pa_range) { +case 0 :return 32 ; +case 1 :return 36 ; +case 2 :return 40 ; +case 3 :return 42 ; +case 4 :return 44 ; +case 5 :return 48 ; +default :
Quoted 8 lines
arch/arm/include/asm/kvm_mmu.h
Quoted 6 lines
arch/arm/include/asm/kvm_mmu.h +6
-0+BUG (); +return 0 ; + } +} + +static u64 vttbr_baddr_mask;Now every compilation unit that includes
kvm_mmu.hhas an instance of this variable. I doubt that it is the intended effect.
The change for the comment farther above to just return the mask and have the common code set it should resolve this as well.
Quoted 12 lines
Quoted 12 lines
arch/arm/include/asm/kvm_mmu.h +11
-0+ +/** +* set_vttbr_baddr_mask - set mask value for vttbr base address +* +* In ARMv8, vttbr_baddr_mask cannot be determined in compile time since the +* stage2 input address size depends on hardware capability. Thus, we first +* need to read ID_AA64MMFR0_EL1.PARange and then set vttbr_baddr_mask with +* consideration of both the granule size and the level of translation tables. +*/ +static inline int set_vttbr_baddr_mask (void ) +{
Quoted 8 lines
arch/arm/include/asm/kvm_mmu.h
Quoted 7 lines
arch/arm/include/asm/kvm_mmu.h +7
-0+int t0sz, vttbr_x; + + t0sz =VTCR_EL2_T0SZ (kvm_get_phys_addr_shift ()); + vttbr_x =t0sz_to_vttbr_x (t0sz); +if (!vttbr_x) +return -EINVAL ; + vttbr_baddr_mask = (((1LLU << (48 - vttbr_x)) -1 ) << (vttbr_x -1 ));I think this can now be written as
GENMASK_ULL(48, (vttbr_x - 1)).
That does improve readability, I like it.
Quoted 30 lines
Quoted 30 lines
arch/arm/include/asm/kvm_mmu.h +3
-0+return 0 ; +} +#endif /* __ASSEMBLY__ */ #endif /* __ARM64_KVM_MMU_H__ */ arch/arm64/kvm/hyp-init.S +5
-6@@ -63,17 +63,21 @@ __do_hyp_init: mrs x4, tcr_el1 ldr x5, =TCR_EL2_MASK and x4, x4, x5 - ldr x5, =TCR_EL2_FLAGS - orr x4, x4, x5 - msr tcr_el2, x4 - - ldr x4, =VTCR_EL2_FLAGS /* * Read the PARange bits from ID_AA64MMFR0_EL1 and set the PS bits in - * VTCR_EL2. + * TCR_EL2 and both PS bits and T0SZ bits in VTCR_EL2. */ mrs x5, ID_AA64MMFR0_EL1 bfi x4, x5, #16, #3 + msr tcr_el2, x4 + + ldr x4, =VTCR_EL2_FLAGS + bfi x4, x5, #16, #3
Quoted 8 lines
arch/arm64/kvm/hyp-init.S __do_hyp_init: · line 80
Quoted 5 lines
arch/arm64/kvm/hyp-init.S +5
-0+ and x5, x5, #0xf + adr x6, t0sz + add x6, x6, x5, lsl #2 + ldr w5, [x6] + orr x4, x4, x5You'll need to validate the T0SZ value, and possibly adjust it so that it is compatible with the addressing capability of the kernel. That probably require a slight change of the hyp-init API.
In order to do that I really should test that path, can you think of a way to generate a t0sz value that is incompatible with the kernel, but the kernel still boots so I can load kvm and test it?
Quoted 14 lines
Quoted 14 lines
arch/arm64/kvm/hyp-init.S +4
-0msr vtcr_el2, x4 mrs x4, mair_el1@@ -109,6 +113,10 @@ target: /* We're now in the trampoline code, switch page tables */ /* Hello, World! */ eret + +t0sz: + .word VTCR_EL2_T0SZ(32), VTCR_EL2_T0SZ(36), VTCR_EL2_T0SZ(40) + .word VTCR_EL2_T0SZ(42), VTCR_EL2_T0SZ(44), VTCR_EL2_T0SZ(48) ENDPROC(__kvm_hyp_init)
Quoted 6 lines
arch/arm64/kvm/hyp-init.S target: /* We're now in the trampoline code, switch page tables */ · line 122
Quoted 2 lines
arch/arm64/kvm/hyp-init.S +0
-0.ltorgAnother element that doesn't appear in this patch is that we need a way for the kernel to expose the maximum input address to userspace (and validate that noone puts memory outside of that range). This should be a separate patch, but it is conceptually tied to the same problem.
I do think that separate patch would be a nice addition and probably dependent on this, but I do think this patch is useful on its own. Especially since the current VTTBR_BADDR_MASK is broken.
Quoted 39 lines · Joel Schopp
Quoted 19 lines
Quoted 7 lines
arch/arm/include/asm/kvm_mmu.h +0
-0@@ -166,6 +166,18 @@ static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva,arch/arm/include/asm/kvm_mmu.h static inline void · line 177
Quoted 10 lines
arch/arm/include/asm/kvm_mmu.h +8
-0void stage2_flush_vm (struct kvm *kvm); +static inline int kvm_get_phys_addr_shift (void ) +{ +return KVM_PHYS_SHIFT ; +} + +static inline int set_vttbr_baddr_mask (void ) +{ + vttbr_baddr_mask =VTTBR_BADDR_MASK ;Have you tried compiling this?
Quoted 3 lines
Apart from the obvious missing definition of the variable, I'm not fond of functions with side-effects hidden in an include file. What is wrong with just returning the mask and letting the common code setting it?
I like that change, will do in v6.
Quoted 8 lines
arch/arm/include/asm/kvm_mmu.h static inline void · line 183
Quoted 6 lines
arch/arm/include/asm/kvm_mmu.h +6
-0+#ifdef CONFIG_ARM64_64K_PAGES +static inline int t0sz_to_vttbr_x (int t0sz) +{ +if (t0sz <16 || t0sz >34 ) { +kvm_err ("Cannot support %d-bit address space\n" ,64 - t0sz); +return 0 ;0 is definitely a bad value for something that is an error case. Consider
-EINVALinstead.OK.
Quoted 6 lines
Also, what if we're in a range that only deals with more levels of page tables than the kernel can deal with (remember we use the kernel page table accessors)? See the new
ARM64_VA_BITSandARM64_PGTABLE_LEVELSsymbols that are now available, and use them to validate the range you have.
Quoted 5 lines
With the simple current tests I can look at them and see they are correct, even if I can't make a scenario to test that they would fail. However, if I add in more complicated checks I'd really like to test them catching the failure cases. Can you describe a case where we can boot a kernel and then have the checks still fail in kvm?
You're looking at T0SZ values that are outside of what a given configuration of the kernel can handle (T0SZ == 16, for example, is only valid with 3 levels of page tables, and the current implementation only deals with 2). This is a case where things may explode, as you allow input addresses that we simply cannot support, and in this case you want to cap T0SZ to something sensible that is compatible with what the kernel (and thus KVM) can express in terms of translations.
Testing is another thing, and I don't expect you to be able to test every possible combinaison. Nonetheless, I do expect some reasonable effort in the code to avoid deadly situations.
Quoted 113 lines
Quoted 17 lines
Quoted 17 lines
arch/arm/include/asm/kvm_mmu.h +16
-0+ } +return 37 - t0sz; +} +#endif +static inline int kvm_get_phys_addr_shift (void ) +{ +int pa_range =read_cpuid (ID_AA64MMFR0_EL1 ) &0xf ; + +switch (pa_range) { +case 0 :return 32 ; +case 1 :return 36 ; +case 2 :return 40 ; +case 3 :return 42 ; +case 4 :return 44 ; +case 5 :return 48 ; +default :Quoted 8 lines
arch/arm/include/asm/kvm_mmu.h
Quoted 6 lines
arch/arm/include/asm/kvm_mmu.h +6
-0+BUG (); +return 0 ; + } +} + +static u64 vttbr_baddr_mask;Now every compilation unit that includes
kvm_mmu.hhas an instance of this variable. I doubt that it is the intended effect.The change for the comment farther above to just return the mask and have the common code set it should resolve this as well.
Quoted 13 lines
Quoted 13 lines
arch/arm/include/asm/kvm_mmu.h +8
-0+ +/** +* set_vttbr_baddr_mask - set mask value for vttbr base address +* +* In ARMv8, vttbr_baddr_mask cannot be determined in compile time since the +* stage2 input address size depends on hardware capability. Thus, we first +* need to read ID_AA64MMFR0_EL1.PARange and then set vttbr_baddr_mask with +* consideration of both the granule size and the level of translation tables.
+ */arch/arm/include/asm/kvm_mmu.h +2
-0+static inline int set_vttbr_baddr_mask (void ) +{Quoted 8 lines
arch/arm/include/asm/kvm_mmu.h
Quoted 7 lines
arch/arm/include/asm/kvm_mmu.h +7
-0+int t0sz, vttbr_x; + + t0sz =VTCR_EL2_T0SZ (kvm_get_phys_addr_shift ()); + vttbr_x =t0sz_to_vttbr_x (t0sz); +if (!vttbr_x) +return -EINVAL ; + vttbr_baddr_mask = (((1LLU << (48 - vttbr_x)) -1 ) << (vttbr_x -1 ));I think this can now be written as
GENMASK_ULL(48, (vttbr_x - 1)).That does improve readability, I like it.
Quoted 30 lines
Quoted 30 lines
arch/arm/include/asm/kvm_mmu.h +3
-0+return 0 ; +} +#endif /* __ASSEMBLY__ */ #endif /* __ARM64_KVM_MMU_H__ */ arch/arm64/kvm/hyp-init.S +5
-6@@ -63,17 +63,21 @@ __do_hyp_init: mrs x4, tcr_el1 ldr x5, =TCR_EL2_MASK and x4, x4, x5 - ldr x5, =TCR_EL2_FLAGS - orr x4, x4, x5 - msr tcr_el2, x4 - - ldr x4, =VTCR_EL2_FLAGS /* * Read the PARange bits from ID_AA64MMFR0_EL1 and set the PS bits in - * VTCR_EL2. + * TCR_EL2 and both PS bits and T0SZ bits in VTCR_EL2. */ mrs x5, ID_AA64MMFR0_EL1 bfi x4, x5, #16, #3 + msr tcr_el2, x4 + + ldr x4, =VTCR_EL2_FLAGS + bfi x4, x5, #16, #3Quoted 8 lines
arch/arm64/kvm/hyp-init.S __do_hyp_init: · line 80
Quoted 5 lines
arch/arm64/kvm/hyp-init.S +5
-0+ and x5, x5, #0xf + adr x6, t0sz + add x6, x6, x5, lsl #2 + ldr w5, [x6] + orr x4, x4, x5You'll need to validate the T0SZ value, and possibly adjust it so that it is compatible with the addressing capability of the kernel. That probably require a slight change of the hyp-init API.
In order to do that I really should test that path, can you think of a way to generate a t0sz value that is incompatible with the kernel, but the kernel still boots so I can load kvm and test it?
Quoted 14 lines
Quoted 14 lines
arch/arm64/kvm/hyp-init.S +4
-0msr vtcr_el2, x4 mrs x4, mair_el1@@ -109,6 +113,10 @@ target: /* We're now in the trampoline code, switch page tables */ /* Hello, World! */ eret + +t0sz: + .word VTCR_EL2_T0SZ(32), VTCR_EL2_T0SZ(36), VTCR_EL2_T0SZ(40) + .word VTCR_EL2_T0SZ(42), VTCR_EL2_T0SZ(44), VTCR_EL2_T0SZ(48) ENDPROC(__kvm_hyp_init)Quoted 6 lines
arch/arm64/kvm/hyp-init.S target: /* We're now in the trampoline code, switch page tables */ · line 122
Quoted 2 lines
arch/arm64/kvm/hyp-init.S +0
-0.ltorgAnother element that doesn't appear in this patch is that we need a way for the kernel to expose the maximum input address to userspace (and validate that noone puts memory outside of that range). This should be a separate patch, but it is conceptually tied to the same problem.
Quoted 3 lines
I do think that separate patch would be a nice addition and probably dependent on this, but I do think this patch is useful on its own. Especially since the current
VTTBR_BADDR_MASKis broken.
This patch is indeed useful on its own, but I want to see a complete solution, not a set of point fixes.
Thanks,
M.
Signature · 1 line
Jazz is not dead. It just smells funny.