KVM: Allocate small memslot metadata with kvzalloc()
v2 cover KVM: Allocate small memslot metadata with kvzalloc()
From: Fred Griffoul <[email protected]>
__vcalloc() makes every memslot metadata allocation at least one page, so small memslots waste most of their lpage_info and dirty bitmap, and memslots below 512 GiB waste most of a page on their 1 GiB lpage_info. Mushahid's patch [1] switched those arrays to kvcalloc(), and Sean suggested dedicated helpers for memslot metadata with a pivot that makes sense for KVM instead [2].
Patch 1 adds the helpers and routes the dirty bitmap, rmaps, lpage_info and write-tracking arrays through them, with no functional change. Patch 2 makes the helpers use kvzalloc() up to KMALLOC_MAX_CACHE_SIZE, where kmalloc() never uses more memory than vmalloc(), and keeps __vcalloc() above.
I ran the memslot and dirty logging selftests with the TDP MMU and with kvm.tdp_mmu=0, plus a check that one-page memslots no longer take a vmalloc area each and that a 1 TiB memslot still gets its rmaps from vmalloc.
Fred
Changes in v2:
- Split the helpers and the allocator change into two patches
- Pivot on KMALLOC_MAX_CACHE_SIZE instead of INT_MAX
v1: https://lore.kernel.org/kvm/[email protected]/
[1] https://lore.kernel.org/all/[email protected]/
[2] https://lore.kernel.org/all/[email protected]/
Fred Griffoul (2):
KVM: Add helpers to allocate memslot metadata
KVM: Use kvzalloc() for memslot metadata that fits in a slab bucket
arch/x86/kvm/mmu/page_track.c | 6 +++--- arch/x86/kvm/x86.c | 10 +++++----- include/linux/kvm_host.h | 2 ++ virt/kvm/kvm_main.c | 22 ++++++++++++++++++++-- 4 files changed, 30 insertions(+), 10 deletions(-)
base-commit: d21eef2084a9af87ac2a783bc7c2b175ba114e4e
Signature · 1 line
2.47.3
PATCH v2 1/2 KVM: Add helpers to allocate memslot metadata
From: Fred Griffoul <[email protected]>
Add and use helpers to allocate and free memslot metadata, i.e. the dirty bitmap, rmaps, lpage_info and write-tracking arrays, in anticipation of allocating small arrays with kvzalloc() instead of vmalloc().
No functional change intended.
--- arch/x86/kvm/mmu/page_track.c | 6 +++--- arch/x86/kvm/x86.c | 10 +++++----- include/linux/kvm_host.h | 2 ++ virt/kvm/kvm_main.c | 14 ++++++++++++-- 4 files changed, 22 insertions(+), 10 deletions(-)
arch/x86/kvm/mmu/page_track.c +3 -3
@@ -43,7 +43,7 @@ bool kvm_page_track_write_tracking_enabled(struct kvm *kvm)void kvm_page_track_free_memslot (struct kvm_memory_slot *slot) { -vfree (slot->arch.gfn_write_track); +kvm_memslot_metadata_free (slot->arch.gfn_write_track); slot->arch.gfn_write_track =NULL ; }
@@ -53,8 +53,8 @@ static int __kvm_page_track_write_tracking_alloc(struct kvm_memory_slot *slot,const size_t size =sizeof (*slot->arch.gfn_write_track);if (!slot->arch.gfn_write_track) - slot->arch.gfn_write_track =__vcalloc (npages, size, -GFP_KERNEL_ACCOUNT ); + slot->arch.gfn_write_track = +kvm_memslot_metadata_alloc (npages, size);return slot->arch.gfn_write_track ?0 : -ENOMEM ; }
arch/x86/kvm/x86.c +5 -5
@@ -10146,7 +10146,7 @@ static void memslot_rmap_free(struct kvm_memory_slot *slot)int i;for (i =0 ; i <KVM_NR_PAGE_SIZES ; ++i) { -vfree (slot->arch.rmap[i]); +kvm_memslot_metadata_free (slot->arch.rmap[i]); slot->arch.rmap[i] =NULL ; } }
@@ -10158,7 +10158,7 @@ void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot)memslot_rmap_free (slot);for (i =1 ; i <KVM_NR_PAGE_SIZES ; ++i) { -vfree (slot->arch.lpage_info[i -1 ]); +kvm_memslot_metadata_free (slot->arch.lpage_info[i -1 ]); slot->arch.lpage_info[i -1 ] =NULL ; }
@@ -10177,7 +10177,7 @@ int memslot_rmap_alloc(struct kvm_memory_slot *slot, unsigned long npages)if (slot->arch.rmap[i])continue ; - slot->arch.rmap[i] =__vcalloc (lpages, sz,GFP_KERNEL_ACCOUNT ); + slot->arch.rmap[i] =kvm_memslot_metadata_alloc (lpages, sz);if (!slot->arch.rmap[i]) {memslot_rmap_free (slot);return -ENOMEM ;
@@ -10214,7 +10214,7 @@ static int kvm_alloc_memslot_metadata(struct kvm *kvm, lpages =__kvm_mmu_slot_lpages (slot, npages, level); - linfo =__vcalloc (lpages,sizeof (*linfo),GFP_KERNEL_ACCOUNT ); + linfo =kvm_memslot_metadata_alloc (lpages,sizeof (*linfo));if (!linfo) goto out_free;
@@ -10250,7 +10250,7 @@ static int kvm_alloc_memslot_metadata(struct kvm *kvm,memslot_rmap_free (slot);for (i =1 ; i <KVM_NR_PAGE_SIZES ; ++i) { -vfree (slot->arch.lpage_info[i -1 ]); +kvm_memslot_metadata_free (slot->arch.lpage_info[i -1 ]); slot->arch.lpage_info[i -1 ] =NULL ; }return -ENOMEM ;
include/linux/kvm_host.h +2 -0
@@ -1253,6 +1253,8 @@ enum kvm_mr_change {int kvm_set_internal_memslot (struct kvm *kvm,const struct kvm_userspace_memory_region2 *mem); +void *kvm_memslot_metadata_alloc (size_t n,size_t size); +void kvm_memslot_metadata_free (void *p);void kvm_arch_free_memslot (struct kvm *kvm,struct kvm_memory_slot *slot);void kvm_arch_memslots_updated (struct kvm *kvm,u64 gen); int kvm_arch_prepare_memory_region(struct kvm *kvm,
virt/kvm/kvm_main.c +12 -2
@@ -941,12 +941,22 @@ static void kvm_destroy_pm_notifier(struct kvm *kvm) }#endif /* CONFIG_HAVE_KVM_PM_NOTIFIER */ +void *kvm_memslot_metadata_alloc (size_t n,size_t size) +{ +return __vcalloc (n, size,GFP_KERNEL_ACCOUNT ); +} + +void kvm_memslot_metadata_free (void *p) +{ +vfree (p); +} +static void kvm_destroy_dirty_bitmap (struct kvm_memory_slot *memslot) {if (!memslot->dirty_bitmap)return ; -vfree (memslot->dirty_bitmap); +kvm_memslot_metadata_free (memslot->dirty_bitmap); memslot->dirty_bitmap =NULL ; }
@@ -1449,7 +1459,7 @@ static int kvm_alloc_dirty_bitmap(struct kvm_memory_slot *memslot) {unsigned long dirty_bytes =kvm_dirty_bitmap_bytes (memslot); - memslot->dirty_bitmap =__vcalloc (2 , dirty_bytes,GFP_KERNEL_ACCOUNT ); + memslot->dirty_bitmap =kvm_memslot_metadata_alloc (2 , dirty_bytes);if (!memslot->dirty_bitmap)return -ENOMEM ;
base-commit: d21eef2084a9af87ac2a783bc7c2b175ba114e4e
Signature · 1 line
2.47.3
PATCH v2 2/2 KVM: Use kvzalloc() for memslot metadata that fits in a slab bucket
In reply to Fred GriffoulFrom: Fred Griffoul <[email protected]>
Use kvzalloc() instead of __vcalloc() for memslot metadata up to KMALLOC_MAX_CACHE_SIZE, as vmalloc() rounds every allocation up to a page, e.g. a memslot that spans a single page uses 8 KiB for 8 bytes of lpage_info. The waste grows with the number of memslots and VMs.
Up to KMALLOC_MAX_CACHE_SIZE, kmalloc() serves the request from a slab bucket that is never larger than the pages vmalloc() would use, and above a page kvzalloc() falls back to vmalloc() if the slab allocation fails. Above KMALLOC_MAX_CACHE_SIZE, kmalloc() rounds to a power-of-two number of pages, so keep __vcalloc() there.
Free the metadata with kvfree(), which handles both kmalloc() and vmalloc() memory.
--- virt/kvm/kvm_main.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-)
virt/kvm/kvm_main.c +9 -1
@@ -943,12 +943,20 @@ static void kvm_destroy_pm_notifier(struct kvm *kvm)void *kvm_memslot_metadata_alloc (size_t n,size_t size) { +size_t bytes; + +if (unlikely (check_mul_overflow (n, size, &bytes))) +return NULL ; + +if (bytes <=KMALLOC_MAX_CACHE_SIZE ) +return kvzalloc (bytes,GFP_KERNEL_ACCOUNT ); +return __vcalloc (n, size,GFP_KERNEL_ACCOUNT ); }void kvm_memslot_metadata_free (void *p) { -vfree (p); +kvfree (p); }static void kvm_destroy_dirty_bitmap (struct kvm_memory_slot *memslot)
Signature · 1 line
2.47.3
Quoted 10 lines · Fred Griffoul
From: Fred Griffoul <[email protected]>
__vcalloc()makes every memslot metadata allocation at least one page, so small memslots waste most of theirlpage_infoand dirty bitmap, and memslots below 512 GiB waste most of a page on their 1 GiBlpage_info. Mushahid's patch [1] switched those arrays tokvcalloc(), and Sean suggested dedicated helpers for memslot metadata with a pivot that makes sense for KVM instead [2].[...]
Applied to kvm-x86 generic, thanks!
[1/2] KVM: Add helpers to allocate memslot metadata
https://github.com/kvm-x86/linux/commit/bd0ac62e682f [2/2] KVM: Use kvzalloc() for memslot metadata that fits in a slab bucket https://github.com/kvm-x86/linux/commit/7a6d1004c2c5