KVM: SVM: improve the code readability for ASID management
PATCH v2 1/1 KVM: SVM: improve the code readability for ASID management
KVM SEV code uses bitmaps to manage ASID states. ASID 0 was always skipped because it is never used by VM. Thus, in existing code, ASID value and its bitmap postion always has an 'offset-by-1' relationship.
Both SEV and SEV-ES shares the ASID space, thus KVM uses a dynamic range [min_asid, max_asid] to handle SEV and SEV-ES ASIDs separately.
Existing code mixes the usage of ASID value and its bitmap position by using the same variable called 'min_asid'.
Fix the min_asid usage: ensure that its usage is consistent with its name; allocate extra size for ASID 0 to ensure that each ASID has the same value with its bitmap position. Add comments on ASID bitmap allocation to clarify the size change.
v1 -> v2:
- change ASID bitmap size to incorporate ASID 0 [sean]
- remove the 'fixes' line in commit message. [sean/joerg]
--- arch/x86/kvm/svm/sev.c | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-)
arch/x86/kvm/svm/sev.c +21 -15
@@ -63,6 +63,7 @@ static DEFINE_MUTEX(sev_bitmap_lock);unsigned int max_sev_asid;static unsigned int min_sev_asid;static unsigned long sev_me_mask; +static unsigned int nr_asids;static unsigned long *sev_asid_bitmap;static unsigned long *sev_reclaim_asid_bitmap;
@@ -77,11 +78,11 @@ struct enc_region {/* Called with the sev_bitmap_lock held, or on shutdown */ static int sev_flush_asids (int min_asid,int max_asid) { -int ret, pos, error =0 ; +int ret, asid, error =0 ;/* Check if there are any ASIDs to reclaim before performing a flush */ - pos =find_next_bit (sev_reclaim_asid_bitmap, max_asid, min_asid); -if (pos >= max_asid) + asid =find_next_bit (sev_reclaim_asid_bitmap, nr_asids, min_asid); +if (asid > max_asid)return -EBUSY ;/*
@@ -114,15 +115,15 @@ static bool __sev_recycle_asids(int min_asid, int max_asid)/* The flush process will flush all reclaimable SEV and SEV-ES ASIDs */ bitmap_xor (sev_asid_bitmap, sev_asid_bitmap, sev_reclaim_asid_bitmap, - max_sev_asid); -bitmap_zero (sev_reclaim_asid_bitmap, max_sev_asid); + nr_asids); +bitmap_zero (sev_reclaim_asid_bitmap, nr_asids);return true; }static int sev_asid_new (struct kvm_sev_info *sev) { -int pos, min_asid, max_asid, ret; +int asid, min_asid, max_asid, ret;bool retry = true;enum misc_res_type type;
@@ -142,11 +143,11 @@ static int sev_asid_new(struct kvm_sev_info *sev)* SEV-enabled guests must use asid from min_sev_asid to max_sev_asid. * SEV-ES-enabled guest can use from 1 to min_sev_asid - 1. */ - min_asid = sev->es_active ?0 : min_sev_asid -1 ; + min_asid = sev->es_active ?1 : min_sev_asid; max_asid = sev->es_active ? min_sev_asid -1 : max_sev_asid; again: - pos =find_next_zero_bit (sev_asid_bitmap, max_sev_asid, min_asid); -if (pos >= max_asid) { + asid =find_next_zero_bit (sev_asid_bitmap, max_sev_asid, min_asid); +if (asid > max_asid) {if (retry &&__sev_recycle_asids (min_asid, max_asid)) { retry = false; goto again;
@@ -156,11 +157,11 @@ static int sev_asid_new(struct kvm_sev_info *sev) goto e_uncharge; } -__set_bit (pos, sev_asid_bitmap); +__set_bit (asid, sev_asid_bitmap);mutex_unlock (&sev_bitmap_lock); -return pos +1 ; +return asid; e_uncharge:misc_cg_uncharge (type, sev->misc_cg,1 );put_misc_cg (sev->misc_cg);
@@ -1854,12 +1855,17 @@ void __init sev_hardware_setup(void) min_sev_asid = edx; sev_me_mask =1UL << (ebx &0x3f ); -/* Initialize SEV ASID bitmaps */ - sev_asid_bitmap =bitmap_zalloc (max_sev_asid,GFP_KERNEL ); +/* +* Initialize SEV ASID bitmaps. Allocate space for ASID 0 in the bitmap, +* even though it's never used, so that the bitmap is indexed by the +* actual ASID. +*/ + nr_asids = max_sev_asid +1 ; + sev_asid_bitmap =bitmap_zalloc (nr_asids,GFP_KERNEL );if (!sev_asid_bitmap) goto out; - sev_reclaim_asid_bitmap =bitmap_zalloc (max_sev_asid,GFP_KERNEL ); + sev_reclaim_asid_bitmap =bitmap_zalloc (nr_asids,GFP_KERNEL );if (!sev_reclaim_asid_bitmap) {bitmap_free (sev_asid_bitmap); sev_asid_bitmap =NULL ;
@@ -1904,7 +1910,7 @@ void sev_hardware_teardown(void)return ;/* No need to take sev_bitmap_lock, all VMs have been destroyed. */ -sev_flush_asids (0 , max_sev_asid); +sev_flush_asids (1 , max_sev_asid);bitmap_free (sev_asid_bitmap);bitmap_free (sev_reclaim_asid_bitmap);
Signature · 1 line
2.32.0.554.ge1b32706d8-goog
Quoted 131 lines · Mingwei Zhang
KVM SEV code uses bitmaps to manage ASID states. ASID 0 was always skipped because it is never used by VM. Thus, in existing code, ASID value and its bitmap postion always has an 'offset-by-1' relationship.
Both SEV and SEV-ES shares the ASID space, thus KVM uses a dynamic range [
min_asid,max_asid] to handle SEV and SEV-ES ASIDs separately.Existing code mixes the usage of ASID value and its bitmap position by using the same variable called '
min_asid'.Fix the
min_asidusage: ensure that its usage is consistent with its name; allocate extra size for ASID 0 to ensure that each ASID has the same value with its bitmap position. Add comments on ASID bitmap allocation to clarify the size change.v1 -> v2:
- change ASID bitmap size to incorporate ASID 0 [sean]
- remove the 'fixes' line in commit message. [sean/joerg]Signed-off-by Mingwei Zhang <[email protected]>Cc Tom Lendacky <[email protected]>Cc Marc Orr <[email protected]>Cc David Rientjes <[email protected]>Cc Alper Gun <[email protected]>Cc Dionna Glaze <[email protected]>Cc Sean Christopherson <[email protected]>Cc Vipin Sharma <[email protected]>Cc Peter Gonda <[email protected]>Cc Joerg Roedel <[email protected]>--- arch/x86/kvm/svm/sev.c | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-)arch/x86/kvm/svm/sev.c +20
-14@@ -63,6 +63,7 @@ static DEFINE_MUTEX(sev_bitmap_lock);unsigned int max_sev_asid;static unsigned int min_sev_asid;static unsigned long sev_me_mask; +static unsigned int nr_asids;static unsigned long *sev_asid_bitmap;static unsigned long *sev_reclaim_asid_bitmap;@@ -77,11 +78,11 @@ struct enc_region {/* Called with the sev_bitmap_lock held, or on shutdown */ static int sev_flush_asids (int min_asid,int max_asid) { -int ret, pos, error =0 ; +int ret, asid, error =0 ;/* Check if there are any ASIDs to reclaim before performing a flush */ - pos =find_next_bit (sev_reclaim_asid_bitmap, max_asid, min_asid); -if (pos >= max_asid) + asid =find_next_bit (sev_reclaim_asid_bitmap, nr_asids, min_asid); +if (asid > max_asid)return -EBUSY ;/* @@ -114,15 +115,15 @@ static bool __sev_recycle_asids(int min_asid, int max_asid)/* The flush process will flush all reclaimable SEV and SEV-ES ASIDs */ bitmap_xor (sev_asid_bitmap, sev_asid_bitmap, sev_reclaim_asid_bitmap, - max_sev_asid); -bitmap_zero (sev_reclaim_asid_bitmap, max_sev_asid); + nr_asids); +bitmap_zero (sev_reclaim_asid_bitmap, nr_asids);return true; }static int sev_asid_new (struct kvm_sev_info *sev) { -int pos, min_asid, max_asid, ret; +int asid, min_asid, max_asid, ret;bool retry = true;enum misc_res_type type;@@ -142,11 +143,11 @@ static int sev_asid_new(struct kvm_sev_info *sev)* SEV-enabled guests must use asid from min_sev_asid to max_sev_asid. * SEV-ES-enabled guest can use from 1 to min_sev_asid - 1. */ - min_asid = sev->es_active ?0 : min_sev_asid -1 ; + min_asid = sev->es_active ?1 : min_sev_asid; max_asid = sev->es_active ? min_sev_asid -1 : max_sev_asid; again: - pos =find_next_zero_bit (sev_asid_bitmap, max_sev_asid, min_asid); -if (pos >= max_asid) { + asid =find_next_zero_bit (sev_asid_bitmap, max_sev_asid, min_asid); +if (asid > max_asid) {if (retry &&__sev_recycle_asids (min_asid, max_asid)) { retry = false; goto again;@@ -156,11 +157,11 @@ static int sev_asid_new(struct kvm_sev_info *sev) goto e_uncharge; } -__set_bit (pos, sev_asid_bitmap); +__set_bit (asid, sev_asid_bitmap);mutex_unlock (&sev_bitmap_lock); -return pos +1 ; +return asid; e_uncharge:misc_cg_uncharge (type, sev->misc_cg,1 );put_misc_cg (sev->misc_cg);@@ -1854,12 +1855,17 @@ void __init sev_hardware_setup(void) min_sev_asid = edx; sev_me_mask =1UL << (ebx &0x3f ); -/* Initialize SEV ASID bitmaps */ - sev_asid_bitmap =bitmap_zalloc (max_sev_asid,GFP_KERNEL ); +/* +* Initialize SEV ASID bitmaps. Allocate space for ASID 0 in the bitmap, +* even though it's never used, so that the bitmap is indexed by the +* actual ASID. +*/ + nr_asids = max_sev_asid +1 ; + sev_asid_bitmap =bitmap_zalloc (nr_asids,GFP_KERNEL );if (!sev_asid_bitmap) goto out; - sev_reclaim_asid_bitmap =bitmap_zalloc (max_sev_asid,GFP_KERNEL ); + sev_reclaim_asid_bitmap =bitmap_zalloc (nr_asids,GFP_KERNEL );if (!sev_reclaim_asid_bitmap) {bitmap_free (sev_asid_bitmap); sev_asid_bitmap =NULL ;
arch/x86/kvm/svm/sev.c void sev_hardware_teardown(void) · line 1916
Quoted 8 lines
arch/x86/kvm/svm/sev.c +1
-1@@ -1904,7 +1910,7 @@ void sev_hardware_teardown(void)return ;/* No need to take sev_bitmap_lock, all VMs have been destroyed. */ -sev_flush_asids (0 , max_sev_asid); +sev_flush_asids (1 , max_sev_asid);bitmap_free (sev_asid_bitmap);bitmap_free (sev_reclaim_asid_bitmap);
Queued, thanks.
Paolo
Sean Christopherson
maintainer+Tom and Brijesh
Quoted 30 lines · Mingwei Zhang
KVM SEV code uses bitmaps to manage ASID states. ASID 0 was always skipped because it is never used by VM. Thus, in existing code, ASID value and its bitmap postion always has an 'offset-by-1' relationship.
Both SEV and SEV-ES shares the ASID space, thus KVM uses a dynamic range [
min_asid,max_asid] to handle SEV and SEV-ES ASIDs separately.Existing code mixes the usage of ASID value and its bitmap position by using the same variable called '
min_asid'.Fix the
min_asidusage: ensure that its usage is consistent with its name; allocate extra size for ASID 0 to ensure that each ASID has the same value with its bitmap position. Add comments on ASID bitmap allocation to clarify the size change.v1 -> v2:
- change ASID bitmap size to incorporate ASID 0 [sean]
- remove the 'fixes' line in commit message. [sean/joerg]Signed-off-by Mingwei Zhang <[email protected]>Cc Tom Lendacky <[email protected]>Cc Marc Orr <[email protected]>Cc David Rientjes <[email protected]>Cc Alper Gun <[email protected]>Cc Dionna Glaze <[email protected]>Cc Sean Christopherson <[email protected]>Cc Vipin Sharma <[email protected]>Cc Peter Gonda <[email protected]>Cc Joerg Roedel <[email protected]>---
...
Review static int sev_asid_new(struct kvm_sev_info *sev) · line 160
Quoted 6 lines
(quoted diff) +1
-1@@ -156,11 +157,11 @@ static int sev_asid_new(struct kvm_sev_info *sev) goto e_uncharge; } - __set_bit(pos, sev_asid_bitmap); + __set_bit(asid, sev_asid_bitmap);
This patch missed sev_asid_free().
And on a very related topic, I'm pretty sure the VMCB+ASID invalidation logic indexes sev_vmcbs incorrectly. pre_sev_run() indexes sev_vmcbs by the ASID, whereas sev_asid_free() indexes by ASID-1, i.e. on free KVM nullifies the wrong sev_vmcb entry. sev_cpu_init() allocates for max_sev_asid+1, so indexing by ASID appears to be the intended behavior. That code is also a good candidate for conversion to nr_asids in this patch.
For the sev_vmcbs bug:
From 78c334121adaa51a2a84942ec65dceefd3752590 Mon Sep 17 00:00:00 2001
From: Sean Christopherson <[email protected]>
Date: Tue, 3 Aug 2021 09:27:46 -0700
Subject: [PATCH] KVM: SVM: Fix off-by-one indexing when nullifying last used
SEV VMCB
Use the raw ASID, not ASID-1, when nullifying the last used VMCB when freeing an SEV ASID. The consumer, pre_sev_run(), indexes the array by the raw ASID, thus KVM could get a false negative when checking for a different VMCB if KVM manages to reallocate the same ASID+VMCB combo for a new VM.
Note, this cannot cause a functional issue _in the current code_, as pre_sev_run() also checks which pCPU last did VMRUN for the vCPU, and last_vmentry_cpu is initialized to -1 during vCPU creation, i.e. is guaranteed to mismatch on the first VMRUN. However, prior to commit 8a14fe4f0c54 ("kvm: x86: Move last_cpu into kvm_vcpu_arch as last_vmentry_cpu"), SVM tracked pCPU on its own and zero-initialized the last_cpu variable. Thus it's theoretically possible that older versions of KVM could miss a TLB flush if the first VMRUN is on pCPU0 and the ASID and VMCB exactly match those of a prior VM.
--- arch/x86/kvm/svm/sev.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/x86/kvm/svm/sev.c +1 -1
@@ -189,7 +189,7 @@ static void sev_asid_free(struct kvm_sev_info *sev)for_each_possible_cpu (cpu) { sd =per_cpu (svm_data, cpu); - sd->sev_vmcbs[pos] =NULL ; + sd->sev_vmcbs[sev->asid] =NULL ; }mutex_unlock (&sev_bitmap_lock);
--
2.32.0.554.ge1b32706d8-goog
And fixup for this patch:
arch/x86/kvm/svm/sev.c +3 -4
@@ -180,13 +180,12 @@ static int sev_get_asid(struct kvm *kvm)static void sev_asid_free (struct kvm_sev_info *sev) {struct svm_cpu_data *sd; -int cpu, pos; +int cpu;enum misc_res_type type;mutex_lock (&sev_bitmap_lock); - pos = sev->asid -1 ; -__set_bit (pos, sev_reclaim_asid_bitmap); +__set_bit (sev->asid, sev_reclaim_asid_bitmap);for_each_possible_cpu (cpu) { sd =per_cpu (svm_data, cpu);
@@ -1928,7 +1927,7 @@ int sev_cpu_init(struct svm_cpu_data *sd)if (!sev_enabled)return 0 ; - sd->sev_vmcbs =kcalloc (max_sev_asid +1 ,sizeof (void *),GFP_KERNEL ); + sd->sev_vmcbs =kcalloc (nr_asids,sizeof (void *),GFP_KERNEL );if (!sd->sev_vmcbs)return -ENOMEM ;
--
Quoted 8 lines · Sean Christopherson
This patch missed
sev_asid_free().And on a very related topic, I'm pretty sure the VMCB+ASID invalidation logic indexes
sev_vmcbsincorrectly.pre_sev_run()indexessev_vmcbsby the ASID, whereassev_asid_free()indexes by ASID-1, i.e. on free KVM nullifies the wrongsev_vmcbentry.sev_cpu_init()allocates for max_sev_asid+1, so indexing by ASID appears to be the intended behavior. That code is also a good candidate for conversion tonr_asidsin this patch.
It's also missing this (off by one for SEV guests, pointless extra work for SEV-ES):
index da5b9515a47b..7fbce342eec4 100644
arch/x86/kvm/svm/sev.c +1 -1
@@ -147,7 +147,7 @@ static int sev_asid_new(struct kvm_sev_info *sev) min_asid = sev->es_active ?1 : min_sev_asid; max_asid = sev->es_active ? min_sev_asid -1 : max_sev_asid; again: - asid =find_next_zero_bit (sev_asid_bitmap, max_sev_asid, min_asid); + asid =find_next_zero_bit (sev_asid_bitmap, max_asid +1 , min_asid);if (asid > max_asid) {if (retry &&__sev_recycle_asids (min_asid, max_asid)) { retry = false;
Queued both Mingwei's patch and yours, thanks (to 5.14-rc in order to avoid conflicts).
Paolo