Skip to contentSkip to conversation

KVM: Allocate small memslot metadata with kvzalloc()

PATCH Applied Fred Griffoul 2 participants 4 messages Nov 5 – Nov 13

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

v2 cover KVM: Allocate small memslot metadata with kvzalloc()

From: Fred Griffoul <[email protected]>

__vcalloc() makes every memslot metadata allocation at least one page, so small memslots waste most of their lpage_info and dirty bitmap, and memslots below 512 GiB waste most of a page on their 1 GiB lpage_info. Mushahid's patch [1] switched those arrays to kvcalloc(), and Sean suggested dedicated helpers for memslot metadata with a pivot that makes sense for KVM instead [2].

Patch 1 adds the helpers and routes the dirty bitmap, rmaps, lpage_info and write-tracking arrays through them, with no functional change. Patch 2 makes the helpers use kvzalloc() up to KMALLOC_MAX_CACHE_SIZE, where kmalloc() never uses more memory than vmalloc(), and keeps __vcalloc() above.

I ran the memslot and dirty logging selftests with the TDP MMU and with kvm.tdp_mmu=0, plus a check that one-page memslots no longer take a vmalloc area each and that a 1 TiB memslot still gets its rmaps from vmalloc.

Fred

Changes in v2:
- Split the helpers and the allocator change into two patches
- Pivot on KMALLOC_MAX_CACHE_SIZE instead of INT_MAX

v1: https://lore.kernel.org/kvm/[email protected]/

[1] https://lore.kernel.org/all/[email protected]/
[2] https://lore.kernel.org/all/[email protected]/

Fred Griffoul (2):
KVM: Add helpers to allocate memslot metadata
KVM: Use kvzalloc() for memslot metadata that fits in a slab bucket

 arch/x86/kvm/mmu/page_track.c |  6 +++---
 arch/x86/kvm/x86.c            | 10 +++++-----
 include/linux/kvm_host.h      |  2 ++
 virt/kvm/kvm_main.c           | 22 ++++++++++++++++++++--
 4 files changed, 30 insertions(+), 10 deletions(-)

base-commit: d21eef2084a9af87ac2a783bc7c2b175ba114e4e

Signature · 1 line
2.47.3
#1 lore raw

PATCH v2 1/2 KVM: Add helpers to allocate memslot metadata

From: Fred Griffoul <[email protected]>

Add and use helpers to allocate and free memslot metadata, i.e. the dirty bitmap, rmaps, lpage_info and write-tracking arrays, in anticipation of allocating small arrays with kvzalloc() instead of vmalloc().

No functional change intended.

---
 arch/x86/kvm/mmu/page_track.c |  6 +++---
 arch/x86/kvm/x86.c            | 10 +++++-----
 include/linux/kvm_host.h      |  2 ++
 virt/kvm/kvm_main.c           | 14 ++++++++++++--
 4 files changed, 22 insertions(+), 10 deletions(-)
arch/x86/kvm/mmu/page_track.c +3 -3
index f12558dfcd81..f33547964b98 100644
@@ -43,7 +43,7 @@ bool kvm_page_track_write_tracking_enabled(struct kvm *kvm)
 
 void kvm_page_track_free_memslot(struct kvm_memory_slot *slot)
 {
-	vfree(slot->arch.gfn_write_track);
+	kvm_memslot_metadata_free(slot->arch.gfn_write_track);
 	slot->arch.gfn_write_track = NULL;
 }
 
@@ -53,8 +53,8 @@ static int __kvm_page_track_write_tracking_alloc(struct kvm_memory_slot *slot,
 	const size_t size = sizeof(*slot->arch.gfn_write_track);
 
 	if (!slot->arch.gfn_write_track)
-		slot->arch.gfn_write_track = __vcalloc(npages, size,
-						       GFP_KERNEL_ACCOUNT);
+		slot->arch.gfn_write_track =
+			kvm_memslot_metadata_alloc(npages, size);
 
 	return slot->arch.gfn_write_track ? 0 : -ENOMEM;
 }
arch/x86/kvm/x86.c +5 -5
index 26e7beaff8a0..dc716ad83c8f 100644
@@ -10146,7 +10146,7 @@ static void memslot_rmap_free(struct kvm_memory_slot *slot)
 	int i;
 
 	for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
-		vfree(slot->arch.rmap[i]);
+		kvm_memslot_metadata_free(slot->arch.rmap[i]);
 		slot->arch.rmap[i] = NULL;
 	}
 }
@@ -10158,7 +10158,7 @@ void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot)
 	memslot_rmap_free(slot);
 
 	for (i = 1; i < KVM_NR_PAGE_SIZES; ++i) {
-		vfree(slot->arch.lpage_info[i - 1]);
+		kvm_memslot_metadata_free(slot->arch.lpage_info[i - 1]);
 		slot->arch.lpage_info[i - 1] = NULL;
 	}
 
@@ -10177,7 +10177,7 @@ int memslot_rmap_alloc(struct kvm_memory_slot *slot, unsigned long npages)
 		if (slot->arch.rmap[i])
 			continue;
 
-		slot->arch.rmap[i] = __vcalloc(lpages, sz, GFP_KERNEL_ACCOUNT);
+		slot->arch.rmap[i] = kvm_memslot_metadata_alloc(lpages, sz);
 		if (!slot->arch.rmap[i]) {
 			memslot_rmap_free(slot);
 			return -ENOMEM;
@@ -10214,7 +10214,7 @@ static int kvm_alloc_memslot_metadata(struct kvm *kvm,
 
 		lpages = __kvm_mmu_slot_lpages(slot, npages, level);
 
-		linfo = __vcalloc(lpages, sizeof(*linfo), GFP_KERNEL_ACCOUNT);
+		linfo = kvm_memslot_metadata_alloc(lpages, sizeof(*linfo));
 		if (!linfo)
 			goto out_free;
 
@@ -10250,7 +10250,7 @@ static int kvm_alloc_memslot_metadata(struct kvm *kvm,
 	memslot_rmap_free(slot);
 
 	for (i = 1; i < KVM_NR_PAGE_SIZES; ++i) {
-		vfree(slot->arch.lpage_info[i - 1]);
+		kvm_memslot_metadata_free(slot->arch.lpage_info[i - 1]);
 		slot->arch.lpage_info[i - 1] = NULL;
 	}
 	return -ENOMEM;
include/linux/kvm_host.h +2 -0
index e7fc94e2910a..11a6f501a78b 100644
@@ -1253,6 +1253,8 @@ enum kvm_mr_change {
 
 int kvm_set_internal_memslot(struct kvm *kvm,
 			     const struct kvm_userspace_memory_region2 *mem);
+void *kvm_memslot_metadata_alloc(size_t n, size_t size);
+void kvm_memslot_metadata_free(void *p);
 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot);
 void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen);
 int kvm_arch_prepare_memory_region(struct kvm *kvm,
virt/kvm/kvm_main.c +12 -2
index cf4cb586f140..ebdf9308d06c 100644
@@ -941,12 +941,22 @@ static void kvm_destroy_pm_notifier(struct kvm *kvm)
 }
 #endif /* CONFIG_HAVE_KVM_PM_NOTIFIER */
 
+void *kvm_memslot_metadata_alloc(size_t n, size_t size)
+{
+	return __vcalloc(n, size, GFP_KERNEL_ACCOUNT);
+}
+
+void kvm_memslot_metadata_free(void *p)
+{
+	vfree(p);
+}
+
 static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
 {
 	if (!memslot->dirty_bitmap)
 		return;
 
-	vfree(memslot->dirty_bitmap);
+	kvm_memslot_metadata_free(memslot->dirty_bitmap);
 	memslot->dirty_bitmap = NULL;
 }
 
@@ -1449,7 +1459,7 @@ static int kvm_alloc_dirty_bitmap(struct kvm_memory_slot *memslot)
 {
 	unsigned long dirty_bytes = kvm_dirty_bitmap_bytes(memslot);
 
-	memslot->dirty_bitmap = __vcalloc(2, dirty_bytes, GFP_KERNEL_ACCOUNT);
+	memslot->dirty_bitmap = kvm_memslot_metadata_alloc(2, dirty_bytes);
 	if (!memslot->dirty_bitmap)
 		return -ENOMEM;
 

base-commit: d21eef2084a9af87ac2a783bc7c2b175ba114e4e

Signature · 1 line
2.47.3
#2 lore raw

PATCH v2 2/2 KVM: Use kvzalloc() for memslot metadata that fits in a slab bucket

In reply to Fred Griffoul

From: Fred Griffoul <[email protected]>

Use kvzalloc() instead of __vcalloc() for memslot metadata up to KMALLOC_MAX_CACHE_SIZE, as vmalloc() rounds every allocation up to a page, e.g. a memslot that spans a single page uses 8 KiB for 8 bytes of lpage_info. The waste grows with the number of memslots and VMs.

Up to KMALLOC_MAX_CACHE_SIZE, kmalloc() serves the request from a slab bucket that is never larger than the pages vmalloc() would use, and above a page kvzalloc() falls back to vmalloc() if the slab allocation fails. Above KMALLOC_MAX_CACHE_SIZE, kmalloc() rounds to a power-of-two number of pages, so keep __vcalloc() there.

Free the metadata with kvfree(), which handles both kmalloc() and vmalloc() memory.

---
 virt/kvm/kvm_main.c | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)
virt/kvm/kvm_main.c +9 -1
index ebdf9308d06c..1854e1b075c5 100644
@@ -943,12 +943,20 @@ static void kvm_destroy_pm_notifier(struct kvm *kvm)
 
 void *kvm_memslot_metadata_alloc(size_t n, size_t size)
 {
+	size_t bytes;
+
+	if (unlikely(check_mul_overflow(n, size, &bytes)))
+		return NULL;
+
+	if (bytes <= KMALLOC_MAX_CACHE_SIZE)
+		return kvzalloc(bytes, GFP_KERNEL_ACCOUNT);
+
 	return __vcalloc(n, size, GFP_KERNEL_ACCOUNT);
 }
 
 void kvm_memslot_metadata_free(void *p)
 {
-	vfree(p);
+	kvfree(p);
 }
 
 static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
Signature · 1 line
2.47.3
maintainer Applied
#3 lore raw
In reply to Fred Griffoul
Quoted 10 lines · Fred Griffoul

From: Fred Griffoul <[email protected]>

__vcalloc() makes every memslot metadata allocation at least one page, so small memslots waste most of their lpage_info and dirty bitmap, and memslots below 512 GiB waste most of a page on their 1 GiB lpage_info. Mushahid's patch [1] switched those arrays to kvcalloc(), and Sean suggested dedicated helpers for memslot metadata with a pivot that makes sense for KVM instead [2].

[...]

Applied to kvm-x86 generic, thanks!

[1/2] KVM: Add helpers to allocate memslot metadata

      https://github.com/kvm-x86/linux/commit/bd0ac62e682f
[2/2] KVM: Use kvzalloc() for memslot metadata that fits in a slab bucket
      https://github.com/kvm-x86/linux/commit/7a6d1004c2c5

--
https://github.com/kvm-x86/linux/tree/next