KVM: x86/mmu: Prevent installing hugepages when mem attributes are changing
Sean Christopherson
maintainerPATCH 1/1 KVM: x86/mmu: Prevent installing hugepages when mem attributes are changing
When changing memory attributes on a subset of a potential hugepage, add the hugepage to the invalidation range tracking to prevent installing a hugepage until the attributes are fully updated. Like the actual hugepage tracking updates in kvm_arch_post_set_memory_attributes(), process only the head and tail pages, as any potential hugepages that are entirely covered by the range will already be tracked.
Note, only hugepage chunks whose current attributes are NOT mixed need to be added to the invalidation set, as mixed attributes already prevent installing a hugepage, and it's perfectly safe to install a smaller mapping for a gfn whose attributes aren't changing.
--- arch/x86/kvm/mmu/mmu.c | 68 ++++++++++++++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 16 deletions(-)
arch/x86/kvm/mmu/mmu.c +52 -16
@@ -7669,9 +7669,30 @@ void kvm_mmu_pre_destroy_vm(struct kvm *kvm) }#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +static bool hugepage_test_mixed (struct kvm_memory_slot *slot,gfn_t gfn, +int level) +{ +return lpage_info_slot (gfn, slot, level)->disallow_lpage &KVM_LPAGE_MIXED_FLAG ; +} + +static void hugepage_clear_mixed (struct kvm_memory_slot *slot,gfn_t gfn, +int level) +{ +lpage_info_slot (gfn, slot, level)->disallow_lpage &= ~KVM_LPAGE_MIXED_FLAG ; +} + +static void hugepage_set_mixed (struct kvm_memory_slot *slot,gfn_t gfn, +int level) +{ +lpage_info_slot (gfn, slot, level)->disallow_lpage |=KVM_LPAGE_MIXED_FLAG ; +} +bool kvm_arch_pre_set_memory_attributes (struct kvm *kvm,struct kvm_gfn_range *range) { +struct kvm_memory_slot *slot = range->slot; +int level; +/* * Zap SPTEs even if the slot can't be mapped PRIVATE. KVM x86 only * supports KVM_MEMORY_ATTRIBUTE_PRIVATE, and so it *seems* like KVM
@@ -7686,6 +7707,37 @@ bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm,if (WARN_ON_ONCE (!kvm_arch_has_private_mem (kvm)))return false; +if (WARN_ON_ONCE (range->end <= range->start)) +return false; + +/* +* If the head and tail pages of the range currently allow a hugepage, +* i.e. reside fully in the slot and don't have mixed attributes, then +* add each corresponding hugepage range to the ongoing invalidation, +* e.g. to prevent KVM from creating a hugepage in response to a fault +* for a gfn whose attributes aren't changing. Note, only the range +* of gfns whose attributes are being modified needs to be explicitly +* unmapped, as that will unmap any existing hugepages. +*/ +for (level =PG_LEVEL_2M ; level <=KVM_MAX_HUGEPAGE_LEVEL ; level++) { +gfn_t start =gfn_round_for_level (range->start, level); +gfn_t end =gfn_round_for_level (range->end -1 , level); +gfn_t nr_pages =KVM_PAGES_PER_HPAGE (level); + +if ((start != range->start || start + nr_pages > range->end) && + start >= slot->base_gfn && + start + nr_pages <= slot->base_gfn + slot->npages && + !hugepage_test_mixed (slot, start, level)) +kvm_mmu_invalidate_range_add (kvm, start, start + nr_pages); + +if (end == start) +continue ; + +if ((end + nr_pages) <= (slot->base_gfn + slot->npages) && + !hugepage_test_mixed (slot, end, level)) +kvm_mmu_invalidate_range_add (kvm, end, end + nr_pages); + } +/* Unmap the old attribute page. */ if (range->arg.attributes &KVM_MEMORY_ATTRIBUTE_PRIVATE ) range->attr_filter =KVM_FILTER_SHARED ;
@@ -7695,23 +7747,7 @@ bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm,return kvm_unmap_gfn_range (kvm, range); } -static bool hugepage_test_mixed (struct kvm_memory_slot *slot,gfn_t gfn, -int level) -{ -return lpage_info_slot (gfn, slot, level)->disallow_lpage &KVM_LPAGE_MIXED_FLAG ; -} -static void hugepage_clear_mixed (struct kvm_memory_slot *slot,gfn_t gfn, -int level) -{ -lpage_info_slot (gfn, slot, level)->disallow_lpage &= ~KVM_LPAGE_MIXED_FLAG ; -} - -static void hugepage_set_mixed (struct kvm_memory_slot *slot,gfn_t gfn, -int level) -{ -lpage_info_slot (gfn, slot, level)->disallow_lpage |=KVM_LPAGE_MIXED_FLAG ; -}static bool hugepage_has_attrs (struct kvm *kvm,struct kvm_memory_slot *slot,gfn_t gfn,int level,unsigned long attrs)
base-commit: 2d7124941a273c7233849a7a2bbfbeb7e28f1caa
Signature · 1 line
2.49.0.850.g28803427d3-goog
Quoted 71 lines · Sean Christopherson
When changing memory attributes on a subset of a potential hugepage, add the hugepage to the invalidation range tracking to prevent installing a hugepage until the attributes are fully updated. Like the actual hugepage tracking updates in
kvm_arch_post_set_memory_attributes(), process only the head and tail pages, as any potential hugepages that are entirely covered by the range will already be tracked.Note, only hugepage chunks whose current attributes are NOT mixed need to be added to the invalidation set, as mixed attributes already prevent installing a hugepage, and it's perfectly safe to install a smaller mapping for a gfn whose attributes aren't changing.
Fixes 8dd2eee9d526 ("KVM: x86/mmu: Handle page fault for private memory")Reported-by Michael Roth <[email protected]>Tested-by Michael Roth <[email protected]>Signed-off-by Sean Christopherson <[email protected]>--- arch/x86/kvm/mmu/mmu.c | 68 ++++++++++++++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 16 deletions(-)arch/x86/kvm/mmu/mmu.c +32
-0@@ -7669,9 +7669,30 @@ void kvm_mmu_pre_destroy_vm(struct kvm *kvm) }#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +static bool hugepage_test_mixed (struct kvm_memory_slot *slot,gfn_t gfn, +int level) +{ +return lpage_info_slot (gfn, slot, level)->disallow_lpage &KVM_LPAGE_MIXED_FLAG ; +} + +static void hugepage_clear_mixed (struct kvm_memory_slot *slot,gfn_t gfn, +int level) +{ +lpage_info_slot (gfn, slot, level)->disallow_lpage &= ~KVM_LPAGE_MIXED_FLAG ; +} + +static void hugepage_set_mixed (struct kvm_memory_slot *slot,gfn_t gfn, +int level) +{ +lpage_info_slot (gfn, slot, level)->disallow_lpage |=KVM_LPAGE_MIXED_FLAG ; +} +bool kvm_arch_pre_set_memory_attributes (struct kvm *kvm,struct kvm_gfn_range *range) { +struct kvm_memory_slot *slot = range->slot; +int level; +/* * Zap SPTEs even if the slot can't be mapped PRIVATE. KVM x86 only * supports KVM_MEMORY_ATTRIBUTE_PRIVATE, and so it *seems* like KVM @@ -7686,6 +7707,37 @@ bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm,if (WARN_ON_ONCE (!kvm_arch_has_private_mem (kvm)))return false; +if (WARN_ON_ONCE (range->end <= range->start)) +return false; + +/* +* If the head and tail pages of the range currently allow a hugepage, +* i.e. reside fully in the slot and don't have mixed attributes, then +* add each corresponding hugepage range to the ongoing invalidation, +* e.g. to prevent KVM from creating a hugepage in response to a fault +* for a gfn whose attributes aren't changing. Note, only the range +* of gfns whose attributes are being modified needs to be explicitly +* unmapped, as that will unmap any existing hugepages.
arch/x86/kvm/mmu/mmu.c bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm, · line 7730
Quoted 10 lines
arch/x86/kvm/mmu/mmu.c +10
-0+*/ +for (level =PG_LEVEL_2M ; level <=KVM_MAX_HUGEPAGE_LEVEL ; level++) { +gfn_t start =gfn_round_for_level (range->start, level); +gfn_t end =gfn_round_for_level (range->end -1 , level); +gfn_t nr_pages =KVM_PAGES_PER_HPAGE (level); + +if ((start != range->start || start + nr_pages > range->end) && + start >= slot->base_gfn && + start + nr_pages <= slot->base_gfn + slot->npages && + !hugepage_test_mixed (slot, start, level))
Instead of checking mixed flag in disallow_lpage, could we check disallow_lpage directly?
So, if mixed flag is not set but disallow_lpage is 1, there's no need to update the invalidate range.
arch/x86/kvm/mmu/mmu.c bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm, · line 7737
Quoted 7 lines
arch/x86/kvm/mmu/mmu.c +7
-0+kvm_mmu_invalidate_range_add (kvm, start, start + nr_pages); + +if (end == start) +continue ; + +if ((end + nr_pages) <= (slot->base_gfn + slot->npages) && + !hugepage_test_mixed (slot, end, level))
if ((end + nr_pages > range->end) &&
((end + nr_pages) <= (slot->base_gfn + slot->npages)) &&
!lpage_info_slot(gfn, slot, level)->disallow_lpage)?
Quoted 36 lines
arch/x86/kvm/mmu/mmu.c +3
-16+kvm_mmu_invalidate_range_add (kvm, end, end + nr_pages); + } +/* Unmap the old attribute page. */ if (range->arg.attributes &KVM_MEMORY_ATTRIBUTE_PRIVATE ) range->attr_filter =KVM_FILTER_SHARED ;@@ -7695,23 +7747,7 @@ bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm,return kvm_unmap_gfn_range (kvm, range); } -static bool hugepage_test_mixed (struct kvm_memory_slot *slot,gfn_t gfn, -int level) -{ -return lpage_info_slot (gfn, slot, level)->disallow_lpage &KVM_LPAGE_MIXED_FLAG ; -} -static void hugepage_clear_mixed (struct kvm_memory_slot *slot,gfn_t gfn, -int level) -{ -lpage_info_slot (gfn, slot, level)->disallow_lpage &= ~KVM_LPAGE_MIXED_FLAG ; -} - -static void hugepage_set_mixed (struct kvm_memory_slot *slot,gfn_t gfn, -int level) -{ -lpage_info_slot (gfn, slot, level)->disallow_lpage |=KVM_LPAGE_MIXED_FLAG ; -}static bool hugepage_has_attrs (struct kvm *kvm,struct kvm_memory_slot *slot,gfn_t gfn,int level,unsigned long attrs)base-commit:
2d7124941a273c7233849a7a2bbfbeb7e28f1caaSignature · 1 line
2.49.0.850.g28803427d3-goog
Sean Christopherson
maintainerQuoted 40 lines · Yan Zhao
Quoted 15 lines · Sean Christopherson
arch/x86/kvm/mmu/mmu.c +11
-0@@ -7686,6 +7707,37 @@ bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm,if (WARN_ON_ONCE (!kvm_arch_has_private_mem (kvm)))return false; +if (WARN_ON_ONCE (range->end <= range->start)) +return false; + +/* +* If the head and tail pages of the range currently allow a hugepage, +* i.e. reside fully in the slot and don't have mixed attributes, then +* add each corresponding hugepage range to the ongoing invalidation, +* e.g. to prevent KVM from creating a hugepage in response to a fault +* for a gfn whose attributes aren't changing. Note, only the range +* of gfns whose attributes are being modified needs to be explicitly +* unmapped, as that will unmap any existing hugepages. arch/x86/kvm/mmu/mmu.c bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm, · line 7730
Quoted 10 lines
arch/x86/kvm/mmu/mmu.c +10
-0+*/ +for (level =PG_LEVEL_2M ; level <=KVM_MAX_HUGEPAGE_LEVEL ; level++) { +gfn_t start =gfn_round_for_level (range->start, level); +gfn_t end =gfn_round_for_level (range->end -1 , level); +gfn_t nr_pages =KVM_PAGES_PER_HPAGE (level); + +if ((start != range->start || start + nr_pages > range->end) && + start >= slot->base_gfn && + start + nr_pages <= slot->base_gfn + slot->npages && + !hugepage_test_mixed (slot, start, level))Instead of checking mixed flag in
disallow_lpage, could we checkdisallow_lpagedirectly?So, if mixed flag is not set but
disallow_lpageis 1, there's no need to update the invalidate range.arch/x86/kvm/mmu/mmu.c bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm, · line 7737
Quoted 7 lines
arch/x86/kvm/mmu/mmu.c +7
-0+kvm_mmu_invalidate_range_add (kvm, start, start + nr_pages); + +if (end == start) +continue ; + +if ((end + nr_pages) <= (slot->base_gfn + slot->npages) && + !hugepage_test_mixed (slot, end, level))if ((end +
nr_pages>range->end) &&
Quoted 4 lines
((end + nr_pages) <= (slot->base_gfn + slot->npages)) && !lpage_info_slot(gfn, slot, level)->disallow_lpage)?
No, disallow_lpage is used by write-tracking and shadow paging to prevent creating huge pages for a write-protected gfn. mmu_lock is dropped after the pre_set_range call to kvm_handle_gfn_range(), and so disallow_lpage could go to zero if the last shadow page for the affected range is zapped. In practice, KVM isn't going to be doing write-tracking or shadow paging for CoCo VMs, so there's no missed optimization on that front.
And if disallow_lpage is non-zero due to a misaligned memslot base/size, then the start/end checks will skip this level anyways.
Quoted 44 lines · Sean Christopherson
Quoted 43 lines · Yan Zhao
Quoted 15 lines · Sean Christopherson
arch/x86/kvm/mmu/mmu.c +11
-0@@ -7686,6 +7707,37 @@ bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm,if (WARN_ON_ONCE (!kvm_arch_has_private_mem (kvm)))return false; +if (WARN_ON_ONCE (range->end <= range->start)) +return false; + +/* +* If the head and tail pages of the range currently allow a hugepage, +* i.e. reside fully in the slot and don't have mixed attributes, then +* add each corresponding hugepage range to the ongoing invalidation, +* e.g. to prevent KVM from creating a hugepage in response to a fault +* for a gfn whose attributes aren't changing. Note, only the range +* of gfns whose attributes are being modified needs to be explicitly +* unmapped, as that will unmap any existing hugepages. arch/x86/kvm/mmu/mmu.c bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm, · line 7730
Quoted 10 lines
arch/x86/kvm/mmu/mmu.c +10
-0+*/ +for (level =PG_LEVEL_2M ; level <=KVM_MAX_HUGEPAGE_LEVEL ; level++) { +gfn_t start =gfn_round_for_level (range->start, level); +gfn_t end =gfn_round_for_level (range->end -1 , level); +gfn_t nr_pages =KVM_PAGES_PER_HPAGE (level); + +if ((start != range->start || start + nr_pages > range->end) && + start >= slot->base_gfn && + start + nr_pages <= slot->base_gfn + slot->npages && + !hugepage_test_mixed (slot, start, level))Instead of checking mixed flag in
disallow_lpage, could we checkdisallow_lpagedirectly?So, if mixed flag is not set but
disallow_lpageis 1, there's no need to update the invalidate range.arch/x86/kvm/mmu/mmu.c bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm, · line 7737
Quoted 7 lines
arch/x86/kvm/mmu/mmu.c +7
-0+kvm_mmu_invalidate_range_add (kvm, start, start + nr_pages); + +if (end == start) +continue ; + +if ((end + nr_pages) <= (slot->base_gfn + slot->npages) && + !hugepage_test_mixed (slot, end, level))if ((end +
nr_pages>range->end) &&((end + nr_pages) <= (slot->base_gfn + slot->npages)) && !lpage_info_slot(gfn, slot, level)->disallow_lpage)
Quoted 6 lines
Quoted 1 line
?
No,
disallow_lpageis used by write-tracking and shadow paging to prevent creating huge pages for a write-protected gfn.mmu_lockis dropped after thepre_set_rangecall tokvm_handle_gfn_range(), and sodisallow_lpagecould go to zero if the last shadow page for the affected range is zapped. In practice, KVM isn't going to be
That's a good point. I missed it.
Quoted 5 lines
doing write-tracking or shadow paging for CoCo VMs, so there's no missed optimization on that front.
And if
disallow_lpageis non-zero due to a misaligned memslot base/size, then the start/end checks will skip this level anyways.
If the gfn and userspace address are not aligned wrt each other at a certain level, the disallow_lpage for that level is set to 1 for the entire slot. This is often the case at the 1G level.
But as kvm_vm_set_mem_attributes() holds write mmu_lock for most of the time, preventing fault over a larger range for another short period looks no harm.
Quoted 33 lines · Sean Christopherson
Quoted 32 lines · Yan Zhao
Quoted 15 lines · Sean Christopherson
arch/x86/kvm/mmu/mmu.c +11
-0@@ -7686,6 +7707,37 @@ bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm,if (WARN_ON_ONCE (!kvm_arch_has_private_mem (kvm)))return false; +if (WARN_ON_ONCE (range->end <= range->start)) +return false; + +/* +* If the head and tail pages of the range currently allow a hugepage, +* i.e. reside fully in the slot and don't have mixed attributes, then +* add each corresponding hugepage range to the ongoing invalidation, +* e.g. to prevent KVM from creating a hugepage in response to a fault +* for a gfn whose attributes aren't changing. Note, only the range +* of gfns whose attributes are being modified needs to be explicitly +* unmapped, as that will unmap any existing hugepages. arch/x86/kvm/mmu/mmu.c bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm, · line 7730
Quoted 10 lines
arch/x86/kvm/mmu/mmu.c +10
-0+*/ +for (level =PG_LEVEL_2M ; level <=KVM_MAX_HUGEPAGE_LEVEL ; level++) { +gfn_t start =gfn_round_for_level (range->start, level); +gfn_t end =gfn_round_for_level (range->end -1 , level); +gfn_t nr_pages =KVM_PAGES_PER_HPAGE (level); + +if ((start != range->start || start + nr_pages > range->end) && + start >= slot->base_gfn && + start + nr_pages <= slot->base_gfn + slot->npages && + !hugepage_test_mixed (slot, start, level))Instead of checking mixed flag in
disallow_lpage, could we checkdisallow_lpagedirectly?So, if mixed flag is not set but
disallow_lpageis 1, there's no need to update the invalidate range.
Quoted 8 lines
Quoted 8 lines
arch/x86/kvm/mmu/mmu.c bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm, · line 7737
Quoted 7 lines
arch/x86/kvm/mmu/mmu.c +7
-0+kvm_mmu_invalidate_range_add (kvm, start, start + nr_pages); + +if (end == start) +continue ; + +if ((end + nr_pages) <= (slot->base_gfn + slot->npages) && + !hugepage_test_mixed (slot, end, level))if ((end +
nr_pages>range->end) &&
Checking "end + nr_pages > range->end" is necessary?
if range->end equals to
"gfn_round_for_level(range->end - 1, level) + KVM_PAGES_PER_HPAGE(level)", there's no need to do other checks to update the invalidate range.
Quoted 4 lines
Quoted 4 lines
((end + nr_pages) <= (slot->base_gfn + slot->npages)) && !lpage_info_slot(gfn, slot, level)->disallow_lpage)?
Quoted 42 lines · Yan Zhao
Quoted 33 lines · Sean Christopherson
Quoted 32 lines · Yan Zhao
Quoted 15 lines · Sean Christopherson
arch/x86/kvm/mmu/mmu.c +11
-0@@ -7686,6 +7707,37 @@ bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm,if (WARN_ON_ONCE (!kvm_arch_has_private_mem (kvm)))return false; +if (WARN_ON_ONCE (range->end <= range->start)) +return false; + +/* +* If the head and tail pages of the range currently allow a hugepage, +* i.e. reside fully in the slot and don't have mixed attributes, then +* add each corresponding hugepage range to the ongoing invalidation, +* e.g. to prevent KVM from creating a hugepage in response to a fault +* for a gfn whose attributes aren't changing. Note, only the range +* of gfns whose attributes are being modified needs to be explicitly +* unmapped, as that will unmap any existing hugepages. arch/x86/kvm/mmu/mmu.c bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm, · line 7730
Quoted 10 lines
arch/x86/kvm/mmu/mmu.c +10
-0+*/ +for (level =PG_LEVEL_2M ; level <=KVM_MAX_HUGEPAGE_LEVEL ; level++) { +gfn_t start =gfn_round_for_level (range->start, level); +gfn_t end =gfn_round_for_level (range->end -1 , level); +gfn_t nr_pages =KVM_PAGES_PER_HPAGE (level); + +if ((start != range->start || start + nr_pages > range->end) && + start >= slot->base_gfn && + start + nr_pages <= slot->base_gfn + slot->npages && + !hugepage_test_mixed (slot, start, level))Instead of checking mixed flag in
disallow_lpage, could we checkdisallow_lpagedirectly?So, if mixed flag is not set but
disallow_lpageis 1, there's no need to update the invalidate range.Quoted 8 lines
Quoted 8 lines
arch/x86/kvm/mmu/mmu.c bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm, · line 7737
Quoted 7 lines
arch/x86/kvm/mmu/mmu.c +7
-0+kvm_mmu_invalidate_range_add (kvm, start, start + nr_pages); + +if (end == start) +continue ; + +if ((end + nr_pages) <= (slot->base_gfn + slot->npages) && + !hugepage_test_mixed (slot, end, level))if ((end +
nr_pages>range->end) &&
Quoted 5 lines
Checking "end +
nr_pages>range->end" is necessary?if
range->endequals to
"gfn_round_for_level(range->end - 1, level)+KVM_PAGES_PER_HPAGE(level)", there's no need to do other checks to update the invalidate range.
It's not really necessary, but yah, it would avoid an uneeded call to kvm_mmu_invalidate_range_add(), and it would more closely match the logic for the 'start' hugepage which similarly skips the call in that case.
-Mike
Quoted 7 lines
Quoted 5 lines
Quoted 5 lines
((end + nr_pages) <= (slot->base_gfn + slot->npages)) && !lpage_info_slot(gfn, slot, level)->disallow_lpage)?