Skip to contentSkip to conversation

KVM: SVM: improve the code readability for ASID management

PATCH Applied Mingwei Zhang 3 participants 4 messages Aug 2, 2021 – Aug 4, 2021

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 KVM: SVM: improve the code readability for ASID management

KVM SEV code uses bitmaps to manage ASID states. ASID 0 was always skipped because it is never used by VM. Thus, in existing code, ASID value and its bitmap postion always has an 'offset-by-1' relationship.

Both SEV and SEV-ES shares the ASID space, thus KVM uses a dynamic range [min_asid, max_asid] to handle SEV and SEV-ES ASIDs separately.

Existing code mixes the usage of ASID value and its bitmap position by using the same variable called 'min_asid'.

Fix the min_asid usage: ensure that its usage is consistent with its name; allocate extra size for ASID 0 to ensure that each ASID has the same value with its bitmap position. Add comments on ASID bitmap allocation to clarify the size change.

v1 -> v2:
- change ASID bitmap size to incorporate ASID 0 [sean]
- remove the 'fixes' line in commit message. [sean/joerg]

Cc Tom Lendacky <[email protected]>
Cc Marc Orr <[email protected]>
Cc David Rientjes <[email protected]>
Cc Alper Gun <[email protected]>
Cc Dionna Glaze <[email protected]>
Cc Sean Christopherson <[email protected]>
Cc Vipin Sharma <[email protected]>
Cc Peter Gonda <[email protected]>
Cc Joerg Roedel <[email protected]>
---
 arch/x86/kvm/svm/sev.c | 36 +++++++++++++++++++++---------------
 1 file changed, 21 insertions(+), 15 deletions(-)
arch/x86/kvm/svm/sev.c +21 -15
index 8d36f0c73071..42d46c30f313 100644
@@ -63,6 +63,7 @@ static DEFINE_MUTEX(sev_bitmap_lock);
 unsigned int max_sev_asid;
 static unsigned int min_sev_asid;
 static unsigned long sev_me_mask;
+static unsigned int nr_asids;
 static unsigned long *sev_asid_bitmap;
 static unsigned long *sev_reclaim_asid_bitmap;
 
@@ -77,11 +78,11 @@ struct enc_region {
 /* Called with the sev_bitmap_lock held, or on shutdown  */
 static int sev_flush_asids(int min_asid, int max_asid)
 {
-	int ret, pos, error = 0;
+	int ret, asid, error = 0;
 
 	/* Check if there are any ASIDs to reclaim before performing a flush */
-	pos = find_next_bit(sev_reclaim_asid_bitmap, max_asid, min_asid);
-	if (pos >= max_asid)
+	asid = find_next_bit(sev_reclaim_asid_bitmap, nr_asids, min_asid);
+	if (asid > max_asid)
 		return -EBUSY;
 
 	/*
@@ -114,15 +115,15 @@ static bool __sev_recycle_asids(int min_asid, int max_asid)
 
 	/* The flush process will flush all reclaimable SEV and SEV-ES ASIDs */
 	bitmap_xor(sev_asid_bitmap, sev_asid_bitmap, sev_reclaim_asid_bitmap,
-		   max_sev_asid);
-	bitmap_zero(sev_reclaim_asid_bitmap, max_sev_asid);
+		   nr_asids);
+	bitmap_zero(sev_reclaim_asid_bitmap, nr_asids);
 
 	return true;
 }
 
 static int sev_asid_new(struct kvm_sev_info *sev)
 {
-	int pos, min_asid, max_asid, ret;
+	int asid, min_asid, max_asid, ret;
 	bool retry = true;
 	enum misc_res_type type;
 
@@ -142,11 +143,11 @@ static int sev_asid_new(struct kvm_sev_info *sev)
 	 * SEV-enabled guests must use asid from min_sev_asid to max_sev_asid.
 	 * SEV-ES-enabled guest can use from 1 to min_sev_asid - 1.
 	 */
-	min_asid = sev->es_active ? 0 : min_sev_asid - 1;
+	min_asid = sev->es_active ? 1 : min_sev_asid;
 	max_asid = sev->es_active ? min_sev_asid - 1 : max_sev_asid;
 again:
-	pos = find_next_zero_bit(sev_asid_bitmap, max_sev_asid, min_asid);
-	if (pos >= max_asid) {
+	asid = find_next_zero_bit(sev_asid_bitmap, max_sev_asid, min_asid);
+	if (asid > max_asid) {
 		if (retry && __sev_recycle_asids(min_asid, max_asid)) {
 			retry = false;
 			goto again;
@@ -156,11 +157,11 @@ static int sev_asid_new(struct kvm_sev_info *sev)
 		goto e_uncharge;
 	}
 
-	__set_bit(pos, sev_asid_bitmap);
+	__set_bit(asid, sev_asid_bitmap);
 
 	mutex_unlock(&sev_bitmap_lock);
 
-	return pos + 1;
+	return asid;
 e_uncharge:
 	misc_cg_uncharge(type, sev->misc_cg, 1);
 	put_misc_cg(sev->misc_cg);
@@ -1854,12 +1855,17 @@ void __init sev_hardware_setup(void)
 	min_sev_asid = edx;
 	sev_me_mask = 1UL << (ebx & 0x3f);
 
-	/* Initialize SEV ASID bitmaps */
-	sev_asid_bitmap = bitmap_zalloc(max_sev_asid, GFP_KERNEL);
+	/*
+	 * Initialize SEV ASID bitmaps. Allocate space for ASID 0 in the bitmap,
+	 * even though it's never used, so that the bitmap is indexed by the
+	 * actual ASID.
+	 */
+	nr_asids = max_sev_asid + 1;
+	sev_asid_bitmap = bitmap_zalloc(nr_asids, GFP_KERNEL);
 	if (!sev_asid_bitmap)
 		goto out;
 
-	sev_reclaim_asid_bitmap = bitmap_zalloc(max_sev_asid, GFP_KERNEL);
+	sev_reclaim_asid_bitmap = bitmap_zalloc(nr_asids, GFP_KERNEL);
 	if (!sev_reclaim_asid_bitmap) {
 		bitmap_free(sev_asid_bitmap);
 		sev_asid_bitmap = NULL;
@@ -1904,7 +1910,7 @@ void sev_hardware_teardown(void)
 		return;
 
 	/* No need to take sev_bitmap_lock, all VMs have been destroyed. */
-	sev_flush_asids(0, max_sev_asid);
+	sev_flush_asids(1, max_sev_asid);
 
 	bitmap_free(sev_asid_bitmap);
 	bitmap_free(sev_reclaim_asid_bitmap);
Signature · 1 line
2.32.0.554.ge1b32706d8-goog
maintainer Applied
#1 lore raw
Quoted 131 lines · Mingwei Zhang

KVM SEV code uses bitmaps to manage ASID states. ASID 0 was always skipped because it is never used by VM. Thus, in existing code, ASID value and its bitmap postion always has an 'offset-by-1' relationship.

Both SEV and SEV-ES shares the ASID space, thus KVM uses a dynamic range [min_asid, max_asid] to handle SEV and SEV-ES ASIDs separately.

Existing code mixes the usage of ASID value and its bitmap position by using the same variable called 'min_asid'.

Fix the min_asid usage: ensure that its usage is consistent with its name; allocate extra size for ASID 0 to ensure that each ASID has the same value with its bitmap position. Add comments on ASID bitmap allocation to clarify the size change.

v1 -> v2:
- change ASID bitmap size to incorporate ASID 0 [sean]
- remove the 'fixes' line in commit message. [sean/joerg]

Cc Tom Lendacky <[email protected]>
Cc Marc Orr <[email protected]>
Cc David Rientjes <[email protected]>
Cc Alper Gun <[email protected]>
Cc Dionna Glaze <[email protected]>
Cc Sean Christopherson <[email protected]>
Cc Vipin Sharma <[email protected]>
Cc Peter Gonda <[email protected]>
Cc Joerg Roedel <[email protected]>
---
  arch/x86/kvm/svm/sev.c | 36 +++++++++++++++++++++---------------
  1 file changed, 21 insertions(+), 15 deletions(-)
arch/x86/kvm/svm/sev.c +20 -14
index 8d36f0c73071..42d46c30f313 100644
@@ -63,6 +63,7 @@ static DEFINE_MUTEX(sev_bitmap_lock);
  unsigned int max_sev_asid;
  static unsigned int min_sev_asid;
  static unsigned long sev_me_mask;
+static unsigned int nr_asids;
  static unsigned long *sev_asid_bitmap;
  static unsigned long *sev_reclaim_asid_bitmap;
 
@@ -77,11 +78,11 @@ struct enc_region {
  /* Called with the sev_bitmap_lock held, or on shutdown  */
  static int sev_flush_asids(int min_asid, int max_asid)
  {
-	int ret, pos, error = 0;
+	int ret, asid, error = 0;
 
  	/* Check if there are any ASIDs to reclaim before performing a flush */
-	pos = find_next_bit(sev_reclaim_asid_bitmap, max_asid, min_asid);
-	if (pos >= max_asid)
+	asid = find_next_bit(sev_reclaim_asid_bitmap, nr_asids, min_asid);
+	if (asid > max_asid)
  		return -EBUSY;
 
  	/*
@@ -114,15 +115,15 @@ static bool __sev_recycle_asids(int min_asid, int max_asid)
 
  	/* The flush process will flush all reclaimable SEV and SEV-ES ASIDs */
  	bitmap_xor(sev_asid_bitmap, sev_asid_bitmap, sev_reclaim_asid_bitmap,
-		   max_sev_asid);
-	bitmap_zero(sev_reclaim_asid_bitmap, max_sev_asid);
+		   nr_asids);
+	bitmap_zero(sev_reclaim_asid_bitmap, nr_asids);
 
  	return true;
  }
 
  static int sev_asid_new(struct kvm_sev_info *sev)
  {
-	int pos, min_asid, max_asid, ret;
+	int asid, min_asid, max_asid, ret;
  	bool retry = true;
  	enum misc_res_type type;
 
@@ -142,11 +143,11 @@ static int sev_asid_new(struct kvm_sev_info *sev)
  	 * SEV-enabled guests must use asid from min_sev_asid to max_sev_asid.
  	 * SEV-ES-enabled guest can use from 1 to min_sev_asid - 1.
  	 */
-	min_asid = sev->es_active ? 0 : min_sev_asid - 1;
+	min_asid = sev->es_active ? 1 : min_sev_asid;
  	max_asid = sev->es_active ? min_sev_asid - 1 : max_sev_asid;
  again:
-	pos = find_next_zero_bit(sev_asid_bitmap, max_sev_asid, min_asid);
-	if (pos >= max_asid) {
+	asid = find_next_zero_bit(sev_asid_bitmap, max_sev_asid, min_asid);
+	if (asid > max_asid) {
  		if (retry && __sev_recycle_asids(min_asid, max_asid)) {
  			retry = false;
  			goto again;
@@ -156,11 +157,11 @@ static int sev_asid_new(struct kvm_sev_info *sev)
  		goto e_uncharge;
  	}
 
-	__set_bit(pos, sev_asid_bitmap);
+	__set_bit(asid, sev_asid_bitmap);
 
  	mutex_unlock(&sev_bitmap_lock);
 
-	return pos + 1;
+	return asid;
  e_uncharge:
  	misc_cg_uncharge(type, sev->misc_cg, 1);
  	put_misc_cg(sev->misc_cg);
@@ -1854,12 +1855,17 @@ void __init sev_hardware_setup(void)
  	min_sev_asid = edx;
  	sev_me_mask = 1UL << (ebx & 0x3f);
 
-	/* Initialize SEV ASID bitmaps */
-	sev_asid_bitmap = bitmap_zalloc(max_sev_asid, GFP_KERNEL);
+	/*
+	 * Initialize SEV ASID bitmaps. Allocate space for ASID 0 in the bitmap,
+	 * even though it's never used, so that the bitmap is indexed by the
+	 * actual ASID.
+	 */
+	nr_asids = max_sev_asid + 1;
+	sev_asid_bitmap = bitmap_zalloc(nr_asids, GFP_KERNEL);
  	if (!sev_asid_bitmap)
  		goto out;
 
-	sev_reclaim_asid_bitmap = bitmap_zalloc(max_sev_asid, GFP_KERNEL);
+	sev_reclaim_asid_bitmap = bitmap_zalloc(nr_asids, GFP_KERNEL);
  	if (!sev_reclaim_asid_bitmap) {
  		bitmap_free(sev_asid_bitmap);
  		sev_asid_bitmap = NULL;

arch/x86/kvm/svm/sev.c void sev_hardware_teardown(void) · line 1916

Quoted 8 lines
arch/x86/kvm/svm/sev.c +1 -1
@@ -1904,7 +1910,7 @@ void sev_hardware_teardown(void)
  		return;
 
  	/* No need to take sev_bitmap_lock, all VMs have been destroyed. */
-	sev_flush_asids(0, max_sev_asid);
+	sev_flush_asids(1, max_sev_asid);
 
  	bitmap_free(sev_asid_bitmap);
  	bitmap_free(sev_reclaim_asid_bitmap);

Queued, thanks.

Paolo

#2 lore raw
In reply to Mingwei Zhang

+Tom and Brijesh

Quoted 30 lines · Mingwei Zhang

KVM SEV code uses bitmaps to manage ASID states. ASID 0 was always skipped because it is never used by VM. Thus, in existing code, ASID value and its bitmap postion always has an 'offset-by-1' relationship.

Both SEV and SEV-ES shares the ASID space, thus KVM uses a dynamic range [min_asid, max_asid] to handle SEV and SEV-ES ASIDs separately.

Existing code mixes the usage of ASID value and its bitmap position by using the same variable called 'min_asid'.

Fix the min_asid usage: ensure that its usage is consistent with its name; allocate extra size for ASID 0 to ensure that each ASID has the same value with its bitmap position. Add comments on ASID bitmap allocation to clarify the size change.

v1 -> v2:
- change ASID bitmap size to incorporate ASID 0 [sean]
- remove the 'fixes' line in commit message. [sean/joerg]

Cc Tom Lendacky <[email protected]>
Cc Marc Orr <[email protected]>
Cc David Rientjes <[email protected]>
Cc Alper Gun <[email protected]>
Cc Dionna Glaze <[email protected]>
Cc Sean Christopherson <[email protected]>
Cc Vipin Sharma <[email protected]>
Cc Peter Gonda <[email protected]>
Cc Joerg Roedel <[email protected]>

---

...

Review static int sev_asid_new(struct kvm_sev_info *sev) · line 160

Quoted 6 lines
(quoted diff) +1 -1
@@ -156,11 +157,11 @@ static int sev_asid_new(struct kvm_sev_info *sev)
 		goto e_uncharge;
 	}
 
-	__set_bit(pos, sev_asid_bitmap);
+	__set_bit(asid, sev_asid_bitmap);

This patch missed sev_asid_free().

And on a very related topic, I'm pretty sure the VMCB+ASID invalidation logic indexes sev_vmcbs incorrectly. pre_sev_run() indexes sev_vmcbs by the ASID, whereas sev_asid_free() indexes by ASID-1, i.e. on free KVM nullifies the wrong sev_vmcb entry. sev_cpu_init() allocates for max_sev_asid+1, so indexing by ASID appears to be the intended behavior. That code is also a good candidate for conversion to nr_asids in this patch.

For the sev_vmcbs bug:

From 78c334121adaa51a2a84942ec65dceefd3752590 Mon Sep 17 00:00:00 2001
From: Sean Christopherson <[email protected]>
Date: Tue, 3 Aug 2021 09:27:46 -0700
Subject: [PATCH] KVM: SVM: Fix off-by-one indexing when nullifying last used
SEV VMCB

Use the raw ASID, not ASID-1, when nullifying the last used VMCB when freeing an SEV ASID. The consumer, pre_sev_run(), indexes the array by the raw ASID, thus KVM could get a false negative when checking for a different VMCB if KVM manages to reallocate the same ASID+VMCB combo for a new VM.

Note, this cannot cause a functional issue _in the current code_, as pre_sev_run() also checks which pCPU last did VMRUN for the vCPU, and last_vmentry_cpu is initialized to -1 during vCPU creation, i.e. is guaranteed to mismatch on the first VMRUN. However, prior to commit 8a14fe4f0c54 ("kvm: x86: Move last_cpu into kvm_vcpu_arch as last_vmentry_cpu"), SVM tracked pCPU on its own and zero-initialized the last_cpu variable. Thus it's theoretically possible that older versions of KVM could miss a TLB flush if the first VMRUN is on pCPU0 and the ASID and VMCB exactly match those of a prior VM.

Fixes 70cd94e60c73 ("KVM: SVM: VMRUN should use associated ASID when SEV is enabled")
Cc Tom Lendacky <[email protected]>
Cc Brijesh Singh <[email protected]>
---
 arch/x86/kvm/svm/sev.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/x86/kvm/svm/sev.c +1 -1
index 9f1585f40c85..f4f5d554eaaa 100644
@@ -189,7 +189,7 @@ static void sev_asid_free(struct kvm_sev_info *sev)
 
 	for_each_possible_cpu(cpu) {
 		sd = per_cpu(svm_data, cpu);
-		sd->sev_vmcbs[pos] = NULL;
+		sd->sev_vmcbs[sev->asid] = NULL;
 	}
 
 	mutex_unlock(&sev_bitmap_lock);

--
2.32.0.554.ge1b32706d8-goog

And fixup for this patch:

arch/x86/kvm/svm/sev.c +3 -4
index f3df1eba0c72..416ae0b687fc 100644
@@ -180,13 +180,12 @@ static int sev_get_asid(struct kvm *kvm)
 static void sev_asid_free(struct kvm_sev_info *sev)
 {
 	struct svm_cpu_data *sd;
-	int cpu, pos;
+	int cpu;
 	enum misc_res_type type;
 
 	mutex_lock(&sev_bitmap_lock);
 
-	pos = sev->asid - 1;
-	__set_bit(pos, sev_reclaim_asid_bitmap);
+	__set_bit(sev->asid, sev_reclaim_asid_bitmap);
 
 	for_each_possible_cpu(cpu) {
 		sd = per_cpu(svm_data, cpu);
@@ -1928,7 +1927,7 @@ int sev_cpu_init(struct svm_cpu_data *sd)
 	if (!sev_enabled)
 		return 0;
 
-	sd->sev_vmcbs = kcalloc(max_sev_asid + 1, sizeof(void *), GFP_KERNEL);
+	sd->sev_vmcbs = kcalloc(nr_asids, sizeof(void *), GFP_KERNEL);
 	if (!sd->sev_vmcbs)
 		return -ENOMEM;
 

--

maintainer Applied
#3 lore raw
Quoted 8 lines · Sean Christopherson

This patch missed sev_asid_free().

And on a very related topic, I'm pretty sure the VMCB+ASID invalidation logic indexes sev_vmcbs incorrectly. pre_sev_run() indexes sev_vmcbs by the ASID, whereas sev_asid_free() indexes by ASID-1, i.e. on free KVM nullifies the wrong sev_vmcb entry. sev_cpu_init() allocates for max_sev_asid+1, so indexing by ASID appears to be the intended behavior. That code is also a good candidate for conversion to nr_asids in this patch.

It's also missing this (off by one for SEV guests, pointless extra work for SEV-ES):

index da5b9515a47b..7fbce342eec4 100644

arch/x86/kvm/svm/sev.c +1 -1
@@ -147,7 +147,7 @@ static int sev_asid_new(struct kvm_sev_info *sev)
 	min_asid = sev->es_active ? 1 : min_sev_asid;
 	max_asid = sev->es_active ? min_sev_asid - 1 : max_sev_asid;
 again:
-	asid = find_next_zero_bit(sev_asid_bitmap, max_sev_asid, min_asid);
+	asid = find_next_zero_bit(sev_asid_bitmap, max_asid + 1, min_asid);
 	if (asid > max_asid) {
 		if (retry && __sev_recycle_asids(min_asid, max_asid)) {
 			retry = false;

Queued both Mingwei's patch and yours, thanks (to 5.14-rc in order to avoid conflicts).

Paolo