KVM: x86/mmu: Defer TDP MMU RCU callback barrier
PATCH 1/1 KVM: x86/mmu: Defer TDP MMU RCU callback barrier
TDP MMU pages are freed from RCU callbacks, which return their headers to MMU slab caches. Waiting for an rcu_barrier() during every VM teardown makes each VM wait for all pending RCU callbacks, including callbacks queued by unrelated VMs.
Move the barrier to kvm_mmu_vendor_module_exit(), immediately before the MMU caches are destroyed. Set the owner of KVM stats and per-VM debugfs files to the vendor module so open files pin the module and prevent its exit while those files are active.
This keeps the caches alive until queued callbacks finish while removing the callback drain from individual VM teardown.
--- arch/x86/kvm/debugfs.c | 7 ++++++- arch/x86/kvm/mmu/mmu.c | 2 ++ arch/x86/kvm/mmu/tdp_mmu.c | 7 ------- include/linux/kvm_host.h | 1 + virt/kvm/kvm_main.c | 16 ++++++++++++---- 5 files changed, 21 insertions(+), 12 deletions(-)
arch/x86/kvm/debugfs.c +6 -1
@@ -181,7 +181,7 @@ static int kvm_mmu_rmaps_stat_release(struct inode *inode, struct file *file)return single_release (inode, file); } -static const struct file_operations mmu_rmaps_stat_fops = { +static struct file_operations mmu_rmaps_stat_fops = { .owner =THIS_MODULE , .open = kvm_mmu_rmaps_stat_open, .read = seq_read,
@@ -189,6 +189,11 @@ static const struct file_operations mmu_rmaps_stat_fops = { .release = kvm_mmu_rmaps_stat_release, }; +void kvm_arch_init_debugfs (struct module *module) +{ + mmu_rmaps_stat_fops.owner = module; +} +void kvm_arch_create_vm_debugfs (struct kvm *kvm) { debugfs_create_file("mmu_rmaps_stat", 0644, kvm->debugfs_dentry, kvm,
arch/x86/kvm/mmu/mmu.c +2 -0
@@ -7936,6 +7936,8 @@ void kvm_mmu_destroy(struct kvm_vcpu *vcpu)void kvm_mmu_vendor_module_exit (void ) { +/* Drain TDP callbacks before destroying the cache they free pages into. */ +rcu_barrier ();mmu_destroy_caches (); }
arch/x86/kvm/mmu/tdp_mmu.c +0 -7
@@ -44,13 +44,6 @@ void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm)KVM_MMU_WARN_ON (atomic64_read (&kvm->arch.tdp_mmu_pages));#endif WARN_ON (!list_empty (&kvm->arch.tdp_mmu_roots)); - -/* -* Ensure that all the outstanding RCU callbacks to free shadow pages -* can run before the VM is torn down. Putting the last reference to -* zapped roots will create new callbacks. -*/ -rcu_barrier (); }static void __tdp_mmu_free_sp (struct kvm_mmu_page *sp)
include/linux/kvm_host.h +1 -0
@@ -1700,6 +1700,7 @@ bool kvm_arch_dy_has_pending_interrupt(struct kvm_vcpu *vcpu);bool kvm_arch_vcpu_preempted_in_kernel (struct kvm_vcpu *vcpu);void kvm_arch_pre_destroy_vm (struct kvm *kvm);void kvm_arch_create_vm_debugfs (struct kvm *kvm); +void kvm_arch_init_debugfs (struct module *module);#ifndef __KVM_HAVE_ARCH_VM_ALLOC/*
virt/kvm/kvm_main.c +12 -4
@@ -126,7 +126,7 @@ static DEFINE_PER_CPU(struct kvm_vcpu *, kvm_running_vcpu);static struct dentry *kvm_debugfs_dir; -static const struct file_operations stat_fops_per_vm; +static struct file_operations stat_fops_per_vm;static long kvm_vcpu_ioctl (struct file *file,unsigned int ioctl,unsigned long arg);
@@ -1103,6 +1103,10 @@ void __weak kvm_arch_create_vm_debugfs(struct kvm *kvm) { } +void __weak kvm_arch_init_debugfs (struct module *module) +{ +} +/* Called only on cleanup and destruction paths when there are no users. */ static inline struct kvm_io_bus *kvm_get_bus_for_destruction (struct kvm *kvm,enum kvm_bus idx)
@@ -4395,7 +4399,7 @@ static int kvm_vcpu_stats_release(struct inode *inode, struct file *file)return 0 ; } -static const struct file_operations kvm_vcpu_stats_fops = { +static struct file_operations kvm_vcpu_stats_fops = { .owner =THIS_MODULE , .read = kvm_vcpu_stats_read, .release = kvm_vcpu_stats_release,
@@ -5218,7 +5222,7 @@ static int kvm_vm_stats_release(struct inode *inode, struct file *file)return 0 ; } -static const struct file_operations kvm_vm_stats_fops = { +static struct file_operations kvm_vm_stats_fops = { .owner =THIS_MODULE , .read = kvm_vm_stats_read, .release = kvm_vm_stats_release,
@@ -6322,7 +6326,7 @@ static int kvm_stat_data_open(struct inode *inode, struct file *file) kvm_stat_data_clear, "%llu\n"); } -static const struct file_operations stat_fops_per_vm = { +static struct file_operations stat_fops_per_vm = { .owner =THIS_MODULE , .open = kvm_stat_data_open, .release = kvm_debugfs_release,
@@ -6633,6 +6637,10 @@ int kvm_init(unsigned vcpu_size, unsigned vcpu_align, struct module *module) kvm_vm_fops.owner = module; kvm_vcpu_fops.owner = module; kvm_device_fops.owner = module; +kvm_arch_init_debugfs (module); + kvm_vm_stats_fops.owner = module; + kvm_vcpu_stats_fops.owner = module; + stat_fops_per_vm.owner = module; kvm_preempt_ops.sched_in = kvm_sched_in; kvm_preempt_ops.sched_out = kvm_sched_out;
base-commit: 7475ea8b58616b33c04afb976cf0fbaad9dd63fb
Signature · 1 line
2.53.0
Sean Christopherson
maintainerQuoted 4 lines · Mark Street
TDP MMU pages are freed from RCU callbacks, which return their headers to MMU slab caches. Waiting for an
rcu_barrier()during every VM teardown makes each VM wait for all pending RCU callbacks, including callbacks queued by unrelated VMs.
Wrap your changelogs.
Quoted 2 lines
Move the barrier to
kvm_mmu_vendor_module_exit(), immediately before the MMU caches are destroyed.
This was *just* posted.
https://lore.kernel.org/all/20261118-feat-kvm-tdp-mmu-rcu-barrier-v1-1-24106947c616@linux.dev
Quoted 2 lines
Set the owner of KVM stats and per-VM debugfs files to the vendor module so open files pin the module and prevent its exit while those files are active.
What does this have to do with anything? AFAICT, this is a completely unrelated bug fix.
Thanks for the review. I've been sitting on this for about a month, waiting for Steffen's file-based-reference-counting change to merge before sending my patch. If someone has already posted the same RCU-barrier change, that's great; I'll leave it to that patch. You're also right that the stats/debugfs .owner changes are unrelated and shouldn't have been included.