VMX CR3 cache

Hi,

The CR3 cache feature of VMX CPU's does not seem to increase context switch performance significantly as it did in the original implementation (http://lkml.org/lkml/2007/1/5/205).

The following is similar to the original, but it also caches roots for 4-level pagetables on x86-64, and clearing the cache is only performed in zap_page() instead of on every pagefault.

Tests performed on a 4-way guest.

lat_ctx numbers (output is "nr-procs overhead-in-us"):

cr3-cache:
"size=0k ovr=1.30
2 6.63
"size=0k ovr=1.31
4 7.43
"size=0k ovr=1.32
8 11.02

standard guest:
"size=0k ovr=1.28
2 10.02
"size=0k ovr=1.29
4 10.79
"size=0k ovr=1.30
8 11.20

hackbench numbers (hackbench 1 process 50000)

cr3-cache:
Time: 48.531
Time: 50.665
Time: 51.000

standard guest:
Time: 48.712
Time: 52.653
Time: 55.376

Nowhere near the results achieved earlier (and kernel compilation and httperf seems slightly slower, probably due to paravirt overhead).

So the questions are, why the benefit is not similar (vm-exit's are now way cheaper?) and is it worth to merge this given the results?

In addition to this it is necessary to comment out the

PATCH_SITE(pv_mmu_ops, write_cr3);

line in paravirt_patch_{32,64}.c, otherwise a native write_cr3() will be used.

Index: linux-2.6-x86-kvm/arch/x86/kernel/kvm.c
===================================================================
@@ -26,6 +26,15 @@
 #include <linux/cpu.h>
 #include <linux/mm.h>
 
+#include <asm/tlbflush.h>
+
+struct kvm_para_state {
+	struct kvm_cr3_cache cr3_cache;
+	char pad[PAGE_SIZE];
+} __attribute__ ((aligned(PAGE_SIZE)));
+
+static DEFINE_PER_CPU(struct kvm_para_state, para_state);
+
 /*
  * No need for any "IO delay" on KVM
  */

@@ -33,20 +42,110 @@ static void kvm_io_delay(void)
 {
 }
 
+/*
+ * Special, register-to-cr3 instruction based hypercall API
+ * variant to the KVM host. This utilizes the cr3 filter capability
+ * of the hardware - if this works out then no VM exit happens,
+ * if a VM exit happens then KVM will get the virtual address too.
+ */
+static void kvm_write_cr3(unsigned long guest_cr3)
+{
+	struct kvm_para_state *para_state = &get_cpu_var(para_state);
+	struct kvm_cr3_cache *cache = &para_state->cr3_cache;
+	int idx;
+
+	/*
+	 * Check the cache (maintained by the host) for a matching
+	 * guest_cr3 => host_cr3 mapping. Use it if found:
+	 */
+	for (idx = 0; idx < cache->max_idx; idx++) {
+		if (cache->entry[idx].guest_cr3 == guest_cr3) {
+			/*
+			 * Cache-hit: we load the cached host-CR3 value.
+			 * This never causes any VM exit. (if it does then the
+			 * hypervisor could do nothing with this instruction
+			 * and the guest OS would be aborted)
+			 */
+			native_write_cr3(cache->entry[idx].host_cr3);
+			goto out;
+		}
+	}
+
+	/*
+	 * Cache-miss. Load the guest-cr3 value into cr3, which will
+	 * cause a VM exit to the hypervisor, which then loads the
+	 * host cr3 value and updates the cr3_cache.
+	 */
+	native_write_cr3(guest_cr3);
+out:
+	put_cpu_var(para_state);
+}
+
+/*
+ * Avoid the VM exit upon cr3 load by using the cached
+ * ->active_mm->pgd value:
+ */
+static void kvm_flush_tlb_user(void)
+{
+	kvm_write_cr3(__pa(current->active_mm->pgd));
+}
+
+static void kvm_flush_tlb_single(unsigned long addr)
+{
+	__flush_tlb_one(addr);
+}
+/*
+ * Disable global pages, do a flush, then enable global pages:
+ */
+static fastcall void kvm_flush_tlb_kernel(void)
+{
+	unsigned long orig_cr4 = read_cr4();
+
+	write_cr4(orig_cr4 & ~X86_CR4_PGE);
+	kvm_flush_tlb_user();
+	write_cr4(orig_cr4);
+}
+
+static void register_cr3_cache(void *cache)
+{
+	struct kvm_para_state *state;
+
+	state = &per_cpu(para_state, raw_smp_processor_id());
+	wrmsrl(KVM_MSR_SET_CR3_CACHE, &state->cr3_cache);
+}
+
+static void setup_guest_cr3_cache(void)
+{
+	on_each_cpu(register_cr3_cache, NULL, 0, 1);
+
+	pv_mmu_ops.write_cr3 = kvm_write_cr3;
+	pv_mmu_ops.flush_tlb_user = kvm_flush_tlb_user;
+	pv_mmu_ops.flush_tlb_single = kvm_flush_tlb_single;
+	pv_mmu_ops.flush_tlb_kernel = kvm_flush_tlb_kernel;
+}
+
 static void paravirt_ops_setup(void)
 {
+	unsigned long flags;
+
 	pv_info.name = "KVM";
 	pv_info.paravirt_enabled = 1;
 
 	if (kvm_para_has_feature(KVM_FEATURE_NOP_IO_DELAY))
 		pv_cpu_ops.io_delay = kvm_io_delay;
-
+	if (kvm_para_has_feature(KVM_FEATURE_CR3_CACHE))
+		setup_guest_cr3_cache();
+	local_irq_save(flags);
+	apply_paravirt(__parainstructions, __parainstructions_end);
+	local_irq_restore(flags);
 }
 
-void __init kvm_guest_init(void)
+int __init kvm_guest_init(void)
 {
 	if (!kvm_para_available())
-		return;
+		return -ENOSYS;
 
 	paravirt_ops_setup();
+	return 0;
 }
+module_init(kvm_guest_init);

Index: linux-2.6-x86-kvm/include/asm-x86/kvm_host.h
===================================================================
@@ -140,6 +140,7 @@ union kvm_mmu_page_role {
 		unsigned pad_for_nice_hex_output : 6;
 		unsigned metaphysical : 1;
 		unsigned access : 3;
+		unsigned invalid : 1;
 	};
 };
 

@@ -180,11 +181,11 @@ struct kvm_mmu {
 	gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t gva);
 	void (*prefetch_page)(struct kvm_vcpu *vcpu,
 			      struct kvm_mmu_page *page);
-	hpa_t root_hpa;
+	hpa_t root_hpa[KVM_CR3_CACHE_SIZE];
 	int root_level;
 	int shadow_root_level;
 
-	u64 *pae_root;
+	u64 *pae_root[KVM_CR3_CACHE_SIZE];
 };
 
 struct kvm_vcpu_arch {

@@ -198,6 +199,10 @@ struct kvm_vcpu_arch {
 	unsigned long cr0;
 	unsigned long cr2;
 	unsigned long cr3;
+	struct kvm_cr3_cache *cr3_cache;
+	unsigned int cr3_cache_idx;
+	unsigned int cr3_cache_limit;
+	gpa_t guest_cr3_gpa[KVM_CR3_CACHE_SIZE];
 	unsigned long cr4;
 	unsigned long cr8;
 	u64 pdptrs[4]; /* pae */

@@ -320,6 +325,7 @@ struct kvm_vcpu_stat {
 	u32 fpu_reload;
 	u32 insn_emulation;
 	u32 insn_emulation_fail;
+	u32 cr3_cache_synced;
 };
 
 struct descriptor_table {

Index: linux-2.6-x86-kvm/include/asm-x86/kvm_para.h
===================================================================
@@ -6,6 +6,7 @@
  */
 #define KVM_CPUID_SIGNATURE	0x40000000
 #define KVM_FEATURE_NOP_IO_DELAY	0
+#define KVM_FEATURE_CR3_CACHE		1
 
 /* This CPUID returns a feature bitmap in eax.  Before enabling a particular
  * paravirtualization, the appropriate feature bit should be checked.

@@ -15,7 +16,22 @@
 #ifdef __KERNEL__
 #include <asm/processor.h>
 
-#define KVM_PARA_FEATURES (1UL << KVM_FEATURE_NOP_IO_DELAY)
+#define KVM_PARA_FEATURES ((1UL << KVM_FEATURE_NOP_IO_DELAY) | 	\
+			   (1UL << KVM_FEATURE_CR3_CACHE))
+
+#define KVM_MSR_SET_CR3_CACHE 0x87655678
+
+#define KVM_CR3_CACHE_SIZE 4
+
+struct kvm_cr3_cache_entry {
+	u64 guest_cr3;
+	u64 host_cr3;
+};
+
+struct kvm_cr3_cache {
+	struct kvm_cr3_cache_entry entry[KVM_CR3_CACHE_SIZE];
+	u32 max_idx;
+};
 
 /* This instruction is vmcall.  On non-VT architectures, it will generate a
  * trap that we will then rewrite to the appropriate instruction.

Index: linux-2.6-x86-kvm/arch/x86/kvm/paging_tmpl.h
===================================================================
@@ -140,7 +140,7 @@ walk:
 	}
 #endif
 	ASSERT((!is_long_mode(vcpu) && is_pae(vcpu)) ||
-	       (vcpu->cr3 & CR3_NONPAE_RESERVED_BITS) == 0);
+	       (vcpu->arch.cr3 & CR3_NONPAE_RESERVED_BITS) == 0);
 
 	pt_access = ACC_ALL;
 

@@ -280,10 +280,10 @@ static u64 *FNAME(fetch)(struct kvm_vcpu
 	if (!is_present_pte(walker->ptes[walker->level - 1]))
 		return NULL;
 
-	shadow_addr = vcpu->arch.mmu.root_hpa;
+	shadow_addr = vcpu->arch.mmu.root_hpa[vcpu->arch.cr3_cache_idx];
 	level = vcpu->arch.mmu.shadow_root_level;
 	if (level == PT32E_ROOT_LEVEL) {
-		shadow_addr = vcpu->arch.mmu.pae_root[(addr >> 30) & 3];
+		shadow_addr = vcpu->arch.mmu.pae_root[vcpu->arch.cr3_cache_idx][(addr >> 30) & 3];
 		shadow_addr &= PT64_BASE_ADDR_MASK;
 		--level;
 	}

Index: linux-2.6-x86-kvm/arch/x86/kvm/x86.c
===================================================================
@@ -67,6 +67,7 @@ struct kvm_stats_debugfs_item debugfs_en
 	{ "fpu_reload", VCPU_STAT(fpu_reload) },
 	{ "insn_emulation", VCPU_STAT(insn_emulation) },
 	{ "insn_emulation_fail", VCPU_STAT(insn_emulation_fail) },
+	{ "cr3_cache_synced", VCPU_STAT(cr3_cache_synced) },
 	{ "mmu_shadow_zapped", VM_STAT(mmu_shadow_zapped) },
 	{ "mmu_pte_write", VM_STAT(mmu_pte_write) },
 	{ "mmu_pte_updated", VM_STAT(mmu_pte_updated) },

Index: linux-2.6-x86-kvm/include/linux/kvm_para.h
===================================================================
@@ -21,7 +21,7 @@
 
 #ifdef __KERNEL__
 #ifdef CONFIG_KVM_GUEST
-void __init kvm_guest_init(void);
+int __init kvm_guest_init(void);
 #else
 #define kvm_guest_init() do { } while (0)
 #endif

-------------------------------------------------------------------------
This SF.net email is sponsored by: Microsoft Defy all challenges. Microsoft(R) Visual Studio 2008.
http://clk.atdmt.com/MRT/go/vse0120000070mrt/direct/01/

when i did the testing then i was able to get zero VM exits in the lat_ctx hotpath and get the same performance as on native. The above numbers you got suggest that this is not the case. (your lat_ctx numbers on native are probably around 1 usec, right?) If that is the case, could you check via profile=kvm (or whatever other method you use to profile KVM) where the VM exits come from?

Ingo

-------------------------------------------------------------------------
This SF.net email is sponsored by: Microsoft Defy all challenges. Microsoft(R) Visual Studio 2008.
http://clk.atdmt.com/MRT/go/vse0120000070mrt/direct/01/

Actually, native is around 5usec:

# bin/x86_64-linux-gnu/lat_ctx -N 10 2

"size=0k ovr=1.09
2 5.21

So it does get pretty close to native, but the problem is that its difficult to see improvements on macrobenchmarks (even hackbench which remotely resembles a real scenario is not improving).

-------------------------------------------------------------------------
This SF.net email is sponsored by: Microsoft Defy all challenges. Microsoft(R) Visual Studio 2008.
http://clk.atdmt.com/MRT/go/vse0120000070mrt/direct/01/

try running your shell on one CPU:

taskset -p 1 $$

that should give you the 'raw' single-CPU context switch results.

Ingo

-------------------------------------------------------------------------
This SF.net email is sponsored by: Microsoft Defy all challenges. Microsoft(R) Visual Studio 2008.
http://clk.atdmt.com/MRT/go/vse0120000070mrt/direct/01/

With pinning ~= 3.30, and on an UP guest still ~= 6. I'll check the vmexits later, thanks.

-------------------------------------------------------------------------
This SF.net email is sponsored by: Microsoft Defy all challenges. Microsoft(R) Visual Studio 2008.
http://clk.atdmt.com/MRT/go/vse0120000070mrt/direct/01/

Yes, vmexits are now about 5 times faster compared to older kvm.

error compiling committee.c: too many arguments to function


-------------------------------------------------------------------------
This SF.net email is sponsored by: Microsoft
Defy all challenges. Microsoft(R) Visual Studio 2008.
http://clk.atdmt.com/MRT/go/vse0120000070mrt/direct/01/

Hmm, what kvm version is this against? latest git I guess? After applying to kvm-60 (and fixing up some trivial rejects) it doesn't build.

Even if it it doesn't help much on native: With xenner it probably gives a nice speedup especially on 64 bit where each guest syscall involves a cr3 switch (not benchmarked yet though).

Looks like the mmu.h chunk is missing in the patch.

cheers,
Gerd

-------------------------------------------------------------------------
This SF.net email is sponsored by: Microsoft Defy all challenges. Microsoft(R) Visual Studio 2008.
http://clk.atdmt.com/MRT/go/vse0120000070mrt/direct/01/

Hmm, and x86.c looks incomplete too. vcpu->arch.mmu.root_hpa becomes an array, but mmu.h and x86.c still use it the old way.

Can you double-check and resend the patch please?

thanks,
Gerd

-------------------------------------------------------------------------
This SF.net email is sponsored by: Microsoft Defy all challenges. Microsoft(R) Visual Studio 2008.
http://clk.atdmt.com/MRT/go/vse0120000070mrt/direct/01/

Yes it is, sorry. It also misses the kernel/kvm.c guest bits.

And this is against a changed x86.git -mm tree (with pvops64 patches). I'll send the PTE-write-via-hypercall patches soon and will rebase on top of that (the CR3 cache needs more testing/tuning apparently).

-------------------------------------------------------------------------
This SF.net email is sponsored by: Microsoft Defy all challenges. Microsoft(R) Visual Studio 2008.
http://clk.atdmt.com/MRT/go/vse0120000070mrt/direct/01/

Oops for sale ;)

Triggered by guests wrmsr, looks like some error checks are missing.

I've passed in a physical address. The vmx_cr3_cache_msr() function has a gva_to_page() call which makes me suspect it expects a virtual address. First it should not Oops when a invalid virtual address is passed in, and second I think it better shouldn't expect a virtual address in the first place.

What is the reason to expect the cr3 cache being page aligned btw? It should be enougth to require the struct not cross a page border, right?

cheers,
Gerd

MSR_IA32_VMX_MISC: 00000000000403c0
cr3 target values: 4
device xenner0 entered promiscuous mode audit(1201680208.401:28): dev=xenner0 prom=256 old_prom=0 auid=4294967295
br0: port 1(xenner0) entering learning state

Unable to handle kernel NULL pointer dereference at 0000000000000000 RIP: 
 [<ffffffff8847a9ec>] :kvm_intel:vmx_cr3_cache_msr+0x76/0xef
PGD 102d0067 PUD f816067 PMD 0 
Oops: 0000 [1] SMP 
CPU 1 
Modules linked in: i915 drm nls_utf8 ipt_LOG xt_TCPMSS xt_mark xt_MARK iptable_mangle kvm_intel(U) kvm(U) ipt_MASQUERADE iptable_nat nf_nat nfsd exportfs lockd nfs_acl auth_rpcgss autofs4 tun sunrpc bridge nf_conntrack_ipv4 ipt_REJECT iptable_filter ip_tables nf_conntrack_netbios_ns nf_conntrack_ipv6 xt_state nf_conntrack nfnetlink xt_tcpudp ip6t_ipv6header ip6t_REJECT ip6table_filter ip6_tables x_tables ipv6 cpufreq_ondemand acpi_cpufreq loop dm_multipath sr_mod cdrom ata_generic snd_hda_intel snd_seq_dummy snd_seq_oss snd_seq_midi_event snd_seq snd_seq_device snd_pcm_oss arc4 snd_mixer_oss ecb blkcipher snd_pcm iTCO_wdt iTCO_vendor_support video snd_timer snd_page_alloc output i2c_i801 i2c_core ata_piix iwl3945 snd_hwdep snd battery ac nsc_ircc mac80211 button cfg80211 sg e1000 pcspkr irda soundcore thinkpad_acpi crc_ccitt hwmon joydev dm_snapshot dm_zero dm_mirror dm_mod ahci libata sd_mod scsi_mod ext3 jbd mbcache uhci_hcd ohci_hcd ehci_hcd
Pid: 7680, comm: xenner Not tainted 2.6.23.14-107.fc8 #1
RIP: 0010:[<ffffffff8847a9ec>]  [<ffffffff8847a9ec>] :kvm_intel:vmx_cr3_cache_msr+0x76/0xef
RSP: 0018:ffff810011509ca8  EFLAGS: 00010296
RAX: ffff810008e48770 RBX: 0000000000000000 RCX: 0000000000000000
RDX: ffffffffffffffff RSI: 0000000000000296 RDI: ffff810008e4876c
RBP: ffff81001648a000 R08: 0000000087655678 R09: 0000000000000000
R10: 0000000000000034 R11: ffffffff8847afcd R12: ffff81007cc99000
R13: 0000000000000000 R14: 000000000000000a R15: 0000000000000000
FS:  00002aaaaaad2b20(0000) GS:ffff810037c21300(0000) knlGS:0000000000000000
CS:  0010 DS: 002b ES: 002b CR0: 000000008005003b
CR2: 0000000000000000 CR3: 000000007ad72000 CR4: 00000000000026e0
DR0: ffffffff8125b0e0 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff1 DR7: 0000000000000701
Process xenner (pid: 7680, threadinfo ffff810011508000, task ffff81000a83b040)
Stack:  ffff81001648a000 ffff81001648a000 0000000000000000 ffffffff8847aff8
 ffff81001648a000 ffffffff88460c3f 000000008090ae81 ffff810011509e68
 ffff810011509ee8 ffff81001648a000 0000000000000000 000000000000ae80
Call Trace:
 [<ffffffff8847aff8>] :kvm_intel:handle_wrmsr+0x2b/0x4f
 [<ffffffff88460c3f>] :kvm:kvm_arch_vcpu_ioctl_run+0x3a7/0x4fb
 [<ffffffff8845d3ff>] :kvm:kvm_vcpu_ioctl+0xda/0x2dd
 [<ffffffff81176325>] n_tty_receive_buf+0xd49/0xdc9
 [<ffffffff81074717>] generic_file_aio_write+0x6c/0xc1
 [<ffffffff810f4b72>] avc_has_perm+0x49/0x5b
 [<ffffffff880361ae>] :ext3:ext3_file_write+0x16/0x94
 [<ffffffff810f57b1>] inode_has_perm+0x65/0x72
 [<ffffffff8102f7d2>] __wake_up+0x38/0x4f
 [<ffffffff810f5852>] file_has_perm+0x94/0xa3
 [<ffffffff810a7625>] do_ioctl+0x21/0x6b
 [<ffffffff810a78b2>] vfs_ioctl+0x243/0x25c
 [<ffffffff810a7924>] sys_ioctl+0x59/0x79
 [<ffffffff8100bbce>] system_call+0x7e/0x83


Code: 48 8b 13 31 c9 48 c1 ea 33 48 89 d0 48 c1 e8 09 48 8b 04 c5 
RIP  [<ffffffff8847a9ec>] :kvm_intel:vmx_cr3_cache_msr+0x76/0xef
 RSP <ffff810011509ca8>

CR2: 0000000000000000
xenner0: no IPv6 routers present
br0: topology change detected, propagating
br0: port 1(xenner0) entering forwarding state

-------------------------------------------------------------------------
This SF.net email is sponsored by: Microsoft Defy all challenges. Microsoft(R) Visual Studio 2008.
http://clk.atdmt.com/MRT/go/vse0120000070mrt/direct/01/

_______________________________________________
kvm-devel mailing list [email protected]
https://lists.sourceforge.net/lists/listinfo/kvm-devel

Confirmed. When passing in a virtual address it works. And it gives me a nice speedup for kernel builds:

rhel5-64 kraxel ~# cat kbench-cr3-*

date:    Wed Jan 30 09:50:03 CET 2008
host:    2.6.18-53.el5xen x86_64 (cr3-cache)
target:  linux-2.6.21 i386 vmlinux
config:  allnoconfig
   1: real 184.03 user 96.66 sys 41.30
   2: real 137.23 user 91.57 sys 37.93
   3: real 136.53 user 90.98 sys 38.39
date:    Wed Jan 30 09:37:10 CET 2008
host:    2.6.18-53.el5xen x86_64 (cr3-nocache)
target:  linux-2.6.21 i386 vmlinux
config:  allnoconfig

1: real 182.47 user 112.33 sys 41.56
2: real 175.75 user 109.45 sys 40.53
3: real 173.54 user 108.49 sys 41.22

xen pv guests on 64bit do two cr3 switches per syscall due to kernel/userspace separation being done by having two different page table sets. Thus cr3 caching eliminates two vmexits per xen guest system call, and the pgd caching probably helps too. Watching statistics confirms that the number of vmexits goes down significantly.

I strongly support cr3 caching being merged (after being cleaned up of course).

cheers,
Gerd

-------------------------------------------------------------------------
This SF.net email is sponsored by: Microsoft Defy all challenges. Microsoft(R) Visual Studio 2008.
http://clk.atdmt.com/MRT/go/vse0120000070mrt/direct/01/

[fairly amazing results. how do they compare to xen?]

Yes, the improvement for xenner alone justifies this. I still doubt
anything else would benefit though.

One of the cleanups being using physical addresses in msrs.

error compiling committee.c: too many arguments to function


-------------------------------------------------------------------------
This SF.net email is sponsored by: Microsoft
Defy all challenges. Microsoft(R) Visual Studio 2008.
http://clk.atdmt.com/MRT/go/vse0120000070mrt/direct/01/

Didn't benchmark it side-by-side yet. Most likely xenner is still noticeable slower on 64bit (32bit should be roughly comparable). I also wouldn't surprised if you see different results on different workloads.

xen mangles page table flags alot to make guests run fast despite the frequent cr3 switches. It sets the global flag for userspace mappings (which are identical in kernel/userspace page table tree) to avoid them being flushed from tlb on every syscall enter/exit. I havn't tried to do similar things in xenner because I don't track page tables in the first place. I hope to get it to speed comparable to xen using virtualization hardware features instead.

cheers,
Gerd

http://kraxel.fedorapeople.org/xenner/

-------------------------------------------------------------------------
This SF.net email is sponsored by: Microsoft
Defy all challenges. Microsoft(R) Visual Studio 2008.
http://clk.atdmt.com/MRT/go/vse0120000070mrt/direct/01/

I'll switch to a physical address and make sure proper error handling is in place.

Right, page aligning it on the guest seems the easier way to avoid it from crossing a page boundary.

irda soundcore thinkpad_acpi crc_ccitt hwmon joydev dm_snapshot dm_zero dm_mirror dm_mod ahci libata sd_mod scsi_mod ext3 jbd mbcache uhci_hcd ohci_hcd ehci_hcd

-------------------------------------------------------------------------
This SF.net email is sponsored by: Microsoft Defy all challenges. Microsoft(R) Visual Studio 2008.
http://clk.atdmt.com/MRT/go/vse0120000070mrt/direct/01/