KVM shadow scalability enhancements

Index: kvm.quilt/arch/x86/kvm/mmu.c
===================================================================
@@ -1026,6 +1026,7 @@ static void mmu_free_roots(struct kvm_vc
 
 	if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
 		return;
+	mutex_lock(&vcpu->kvm->lock);
 #ifdef CONFIG_X86_64
 	if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
 		hpa_t root = vcpu->arch.mmu.root_hpa;

@@ -1033,6 +1034,7 @@ static void mmu_free_roots(struct kvm_vc
 		sp = page_header(root);
 		--sp->root_count;
 		vcpu->arch.mmu.root_hpa = INVALID_PAGE;
+		mutex_unlock(&vcpu->kvm->lock);
 		return;
 	}
 #endif

@@ -1046,6 +1048,7 @@ static void mmu_free_roots(struct kvm_vc
 		}
 		vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
 	}
+	mutex_unlock(&vcpu->kvm->lock);
 	vcpu->arch.mmu.root_hpa = INVALID_PAGE;
 }
 

@@ -1245,15 +1248,15 @@ int kvm_mmu_load(struct kvm_vcpu *vcpu)
 {
 	int r;
 
-	mutex_lock(&vcpu->kvm->lock);
 	r = mmu_topup_memory_caches(vcpu);
 	if (r)
 		goto out;
+	mutex_lock(&vcpu->kvm->lock);
 	mmu_alloc_roots(vcpu);
+	mutex_unlock(&vcpu->kvm->lock);
 	kvm_x86_ops->set_cr3(vcpu, vcpu->arch.mmu.root_hpa);
 	kvm_mmu_flush_tlb(vcpu);
 out:
-	mutex_unlock(&vcpu->kvm->lock);
 	return r;
 }
 EXPORT_SYMBOL_GPL(kvm_mmu_load);

@@ -1420,13 +1423,22 @@ void kvm_mmu_pte_write(struct kvm_vcpu *
 
 int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
 {
-	gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gva);
+	gpa_t gpa;
+	int r;
+
+	down_read(&current->mm->mmap_sem);
+	gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gva);
+	up_read(&current->mm->mmap_sem);
 
-	return kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
+	mutex_lock(&vcpu->kvm->lock);
+	r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
+	mutex_unlock(&vcpu->kvm->lock);
+	return r;
 }
 
 void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu)
 {
+	mutex_lock(&vcpu->kvm->lock);
 	while (vcpu->kvm->arch.n_free_mmu_pages < KVM_REFILL_PAGES) {
 		struct kvm_mmu_page *sp;
 

@@ -1435,6 +1447,7 @@ void __kvm_mmu_free_some_pages(struct kv
 		kvm_mmu_zap_page(vcpu->kvm, sp);
 		++vcpu->kvm->stat.mmu_recycled;
 	}
+	mutex_unlock(&vcpu->kvm->lock);
 }
 
 int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code)

@@ -1442,7 +1455,6 @@ int kvm_mmu_page_fault(struct kvm_vcpu *
 	int r;
 	enum emulation_result er;
 
-	mutex_lock(&vcpu->kvm->lock);
 	r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code);
 	if (r < 0)
 		goto out;

@@ -1457,7 +1469,6 @@ int kvm_mmu_page_fault(struct kvm_vcpu *
 		goto out;
 
 	er = emulate_instruction(vcpu, vcpu->run, cr2, error_code, 0);
-	mutex_unlock(&vcpu->kvm->lock);
 
 	switch (er) {
 	case EMULATE_DONE:

@@ -1472,7 +1483,6 @@ int kvm_mmu_page_fault(struct kvm_vcpu *
 		BUG();
 	}
 out:
-	mutex_unlock(&vcpu->kvm->lock);
 	return r;
 }
 EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);

@@ -1569,8 +1579,10 @@ void kvm_mmu_zap_all(struct kvm *kvm)
 {
 	struct kvm_mmu_page *sp, *node;
 
+	mutex_lock(&kvm->lock);
 	list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link)
 		kvm_mmu_zap_page(kvm, sp);
+	mutex_unlock(&kvm->lock);
 
 	kvm_flush_remote_tlbs(kvm);
 }

Index: kvm.quilt/arch/x86/kvm/paging_tmpl.h
===================================================================
@@ -212,7 +212,9 @@ walk:
 		if (ret)
 			goto walk;
 		pte |= PT_DIRTY_MASK;
+		mutex_lock(&vcpu->kvm->lock);
 		kvm_mmu_pte_write(vcpu, pte_gpa, (u8 *)&pte, sizeof(pte));
+		mutex_unlock(&vcpu->kvm->lock);
 		walker->ptes[walker->level - 1] = pte;
 	}
 

@@ -368,11 +370,13 @@ static int FNAME(page_fault)(struct kvm_
 	if (r)
 		return r;
 
+	down_read(&current->mm->mmap_sem);
 	/*
 	 * Look up the shadow pte for the faulting address.
 	 */
 	r = FNAME(walk_addr)(&walker, vcpu, addr, write_fault, user_fault,
 			     fetch_fault);
+	up_read(&current->mm->mmap_sem);
 
 	/*
 	 * The page is not mapped by the guest.  Let the guest handle it.

@@ -384,6 +388,7 @@ static int FNAME(page_fault)(struct kvm_
 		return 0;
 	}
 
+	mutex_lock(&vcpu->kvm->lock);
 	shadow_pte = FNAME(fetch)(vcpu, addr, &walker, user_fault, write_fault,
 				  &write_pt);
 	pgprintk("%s: shadow pte %p %llx ptwrite %d\n", __FUNCTION__,

@@ -395,11 +400,14 @@ static int FNAME(page_fault)(struct kvm_
 	/*
 	 * mmio: emulate if accessible, otherwise its a guest fault.
 	 */
-	if (shadow_pte && is_io_pte(*shadow_pte))
+	if (shadow_pte && is_io_pte(*shadow_pte)) {
+		mutex_unlock(&vcpu->kvm->lock);
 		return 1;
+	}
 
 	++vcpu->stat.pf_fixed;
 	kvm_mmu_audit(vcpu, "post page fault (fixed)");
+	mutex_unlock(&vcpu->kvm->lock);
 
 	return write_pt;
 }

Index: kvm.quilt/arch/x86/kvm/vmx.c
===================================================================
@@ -1431,27 +1431,34 @@ static int init_rmode_tss(struct kvm *kv
 {
 	gfn_t fn = rmode_tss_base(kvm) >> PAGE_SHIFT;
 	u16 data = 0;
+	int ret = 0;
 	int r;
 
+	down_read(&current->mm->mmap_sem);
 	r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
 	if (r < 0)
-		return 0;
+		goto out;
 	data = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE;
 	r = kvm_write_guest_page(kvm, fn++, &data, 0x66, sizeof(u16));
 	if (r < 0)
-		return 0;
+		goto out;
 	r = kvm_clear_guest_page(kvm, fn++, 0, PAGE_SIZE);
 	if (r < 0)
-		return 0;
+		goto out;
 	r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
 	if (r < 0)
-		return 0;
+		goto out;
 	data = ~0;
-	r = kvm_write_guest_page(kvm, fn, &data, RMODE_TSS_SIZE - 2 * PAGE_SIZE - 1,
-			sizeof(u8));
+	r = kvm_write_guest_page(kvm, fn, &data,
+				 RMODE_TSS_SIZE - 2 * PAGE_SIZE - 1,
+				 sizeof(u8));
 	if (r < 0)
-		return 0;
-	return 1;
+		goto out;
+
+	ret = 1;
+out:
+	up_read(&current->mm->mmap_sem);
+	return ret;
 }
 
 static void seg_setup(int seg)

@@ -1470,6 +1477,7 @@ static int alloc_apic_access_page(struct
 	int r = 0;
 
 	mutex_lock(&kvm->lock);
+	down_write(&current->mm->mmap_sem);
 	if (kvm->arch.apic_access_page)
 		goto out;
 	kvm_userspace_mem.slot = APIC_ACCESS_PAGE_PRIVATE_MEMSLOT;

@@ -1481,6 +1489,7 @@ static int alloc_apic_access_page(struct
 		goto out;
 	kvm->arch.apic_access_page = gfn_to_page(kvm, 0xfee00);
 out:
+	up_write(&current->mm->mmap_sem);
 	mutex_unlock(&kvm->lock);
 	return r;
 }

302 lines not on this page · raw message

Index: kvm.quilt/virt/kvm/kvm_main.c
===================================================================
@@ -227,7 +227,7 @@ static int kvm_vm_release(struct inode *
  *
  * Discontiguous memory is allowed, mostly for framebuffers.
  *
- * Must be called holding kvm->lock.
+ * Must be called holding mmap_sem for write.
  */
 int __kvm_set_memory_region(struct kvm *kvm,
 			    struct kvm_userspace_memory_region *mem,

@@ -338,9 +338,9 @@ int kvm_set_memory_region(struct kvm *kv
 {
 	int r;
 
-	mutex_lock(&kvm->lock);
+	down_write(&current->mm->mmap_sem);
 	r = __kvm_set_memory_region(kvm, mem, user_alloc);
-	mutex_unlock(&kvm->lock);
+	up_write(&current->mm->mmap_sem);
 	return r;
 }
 EXPORT_SYMBOL_GPL(kvm_set_memory_region);

@@ -456,7 +456,7 @@ static unsigned long gfn_to_hva(struct k
 /*
  * Requires current->mm->mmap_sem to be held
  */
-static struct page *__gfn_to_page(struct kvm *kvm, gfn_t gfn)
+struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
 {
 	struct page *page[1];
 	unsigned long addr;

@@ -481,17 +481,6 @@ static struct page *__gfn_to_page(struct
 	return page[0];
 }
 
-struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
-{
-	struct page *page;
-
-	down_read(&current->mm->mmap_sem);
-	page = __gfn_to_page(kvm, gfn);
-	up_read(&current->mm->mmap_sem);
-
-	return page;
-}
-
 EXPORT_SYMBOL_GPL(gfn_to_page);
 
 void kvm_release_page_clean(struct page *page)

@@ -977,8 +966,7 @@ static int kvm_vm_fault(struct vm_area_s
 
 	if (!kvm_is_visible_gfn(kvm, vmf->pgoff))
 		return VM_FAULT_SIGBUS;
-	/* current->mm->mmap_sem is already held so call lockless version */
-	page = __gfn_to_page(kvm, vmf->pgoff);
+	page = gfn_to_page(kvm, vmf->pgoff);
 	if (is_error_page(page)) {
 		kvm_release_page_clean(page);
 		return VM_FAULT_SIGBUS;

Index: kvm.quilt/arch/x86/kvm/paging_tmpl.h
===================================================================
@@ -318,10 +318,12 @@ static u64 *FNAME(fetch)(struct kvm_vcpu
 					       metaphysical, access,
 					       shadow_ent, &new_page);
 		if (new_page && !metaphysical) {
+			int r;
 			pt_element_t curr_pte;
-			kvm_read_guest(vcpu->kvm, walker->pte_gpa[level - 2],
-				       &curr_pte, sizeof(curr_pte));
-			if (curr_pte != walker->ptes[level - 2])
+			r = kvm_read_guest_atomic(vcpu->kvm,
+						  walker->pte_gpa[level - 2],
+				       		  &curr_pte, sizeof(curr_pte));
+			if (r || curr_pte != walker->ptes[level - 2])
 				return NULL;
 		}
 		shadow_addr = __pa(shadow_page->spt);

@@ -431,9 +433,8 @@ static gpa_t FNAME(gva_to_gpa)(struct kv
 static void FNAME(prefetch_page)(struct kvm_vcpu *vcpu,
 				 struct kvm_mmu_page *sp)
 {
-	int i, offset = 0;
-	pt_element_t *gpt;
-	struct page *page;
+	int i, offset = 0, r = 0;
+	pt_element_t pt;
 
 	if (sp->role.metaphysical
 	    || (PTTYPE == 32 && sp->role.level > PT_PAGE_TABLE_LEVEL)) {

@@ -443,15 +444,18 @@ static void FNAME(prefetch_page)(struct 
 
 	if (PTTYPE == 32)
 		offset = sp->role.quadrant << PT64_LEVEL_BITS;
-	page = gfn_to_page(vcpu->kvm, sp->gfn);
-	gpt = kmap_atomic(page, KM_USER0);
-	for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
-		if (is_present_pte(gpt[offset + i]))
+
+	for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
+		gpa_t pte_gpa = gfn_to_gpa(sp->gfn);
+		pte_gpa += (i+offset) * sizeof(pt_element_t);
+
+		r = kvm_read_guest_atomic(vcpu->kvm, pte_gpa, &pt,
+					  sizeof(pt_element_t));
+		if (r || is_present_pte(pt))
 			sp->spt[i] = shadow_trap_nonpresent_pte;
 		else
 			sp->spt[i] = shadow_notrap_nonpresent_pte;
-	kunmap_atomic(gpt, KM_USER0);
-	kvm_release_page_clean(page);
+	}
 }
 
 #undef pt_element_t

Index: kvm.quilt/include/linux/kvm_host.h
===================================================================
@@ -167,6 +167,8 @@ void kvm_release_page_clean(struct page 
 void kvm_release_page_dirty(struct page *page);
 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
 			int len);
+int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
+			  unsigned long len);
 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len);
 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
 			 int offset, int len);

Index: kvm.quilt/virt/kvm/kvm_main.c
===================================================================
@@ -541,6 +541,26 @@ int kvm_read_guest(struct kvm *kvm, gpa_
 }
 EXPORT_SYMBOL_GPL(kvm_read_guest);
 
+int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
+			  unsigned long len)
+{
+	int r;
+	unsigned long addr;
+	gfn_t gfn = gpa >> PAGE_SHIFT;
+	int offset = offset_in_page(gpa);
+
+	addr = gfn_to_hva(kvm, gfn);
+	if (kvm_is_error_hva(addr))
+		return -EFAULT;
+	pagefault_disable();
+	r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
+	pagefault_enable();
+	if (r)
+		return -EFAULT;
+	return 0;
+}
+EXPORT_SYMBOL(kvm_read_guest_atomic);
+
 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
 			 int offset, int len)
 {

Index: kvm.quilt/arch/x86/kvm/mmu.c
===================================================================
@@ -907,7 +907,9 @@ static void mmu_set_spte(struct kvm_vcpu
 	if (!(pte_access & ACC_EXEC_MASK))
 		spte |= PT64_NX_MASK;
 
-	page = gfn_to_page(vcpu->kvm, gfn);
+	page = kvm_follow_page(vcpu->kvm, gfn);
+	if (!page)
+		return;
 
 	spte |= PT_PRESENT_MASK;
 	if (pte_access & ACC_USER_MASK)

@@ -983,8 +985,11 @@ static int nonpaging_map(struct kvm_vcpu
 		table = __va(table_addr);
 
 		if (level == 1) {
+			struct page *page;
+			page = gfn_to_page(vcpu->kvm, gfn);
 			mmu_set_spte(vcpu, &table[index], ACC_ALL, ACC_ALL,
 				     0, write, 1, &pt_write, gfn);
+			kvm_release_page_clean(page);
 			return pt_write || is_io_pte(table[index]);
 		}
 

Index: kvm.quilt/include/linux/kvm_host.h
===================================================================
@@ -163,6 +163,7 @@ int kvm_arch_set_memory_region(struct kv
 				int user_alloc);
 gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn);
 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn);
+struct page *kvm_follow_page(struct kvm *kvm, gfn_t gfn);
 void kvm_release_page_clean(struct page *page);
 void kvm_release_page_dirty(struct page *page);
 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,

Index: kvm.quilt/virt/kvm/kvm_main.c
===================================================================
@@ -453,6 +453,25 @@ static unsigned long gfn_to_hva(struct k
 	return (slot->userspace_addr + (gfn - slot->base_gfn) * PAGE_SIZE);
 }
 
+struct page *kvm_follow_page(struct kvm *kvm, gfn_t gfn)
+{
+	unsigned long addr;
+	struct vm_area_struct *vma;
+
+	addr = gfn_to_hva(kvm, gfn);
+	/* MMIO access */
+	if (kvm_is_error_hva(addr)) {
+		get_page(bad_page);
+		return bad_page;
+	}
+
+	vma = find_vma(current->mm, addr);
+	if (!vma)
+		return NULL;
+
+	return follow_page(vma, addr, FOLL_GET|FOLL_TOUCH);
+}
+
 /*
  * Requires current->mm->mmap_sem to be held
  */

Index: kvm.quilt/arch/x86/kvm/paging_tmpl.h
===================================================================
@@ -67,6 +67,7 @@ struct guest_walker {
 	gfn_t table_gfn[PT_MAX_FULL_LEVELS];
 	pt_element_t ptes[PT_MAX_FULL_LEVELS];
 	gpa_t pte_gpa[PT_MAX_FULL_LEVELS];
+	struct page *page;
 	unsigned pt_access;
 	unsigned pte_access;
 	gfn_t gfn;

@@ -203,14 +204,18 @@ walk:
 		--walker->level;
 	}
 
+	walker->page = gfn_to_page(vcpu->kvm, walker->gfn);
+
 	if (write_fault && !is_dirty_pte(pte)) {
 		bool ret;
 
 		mark_page_dirty(vcpu->kvm, table_gfn);
 		ret = FNAME(cmpxchg_gpte)(vcpu->kvm, table_gfn, index, pte,
 			    pte|PT_DIRTY_MASK);
-		if (ret)
+		if (ret) {
+			kvm_release_page_clean(walker->page);
 			goto walk;
+		}
 		pte |= PT_DIRTY_MASK;
 		mutex_lock(&vcpu->kvm->lock);
 		kvm_mmu_pte_write(vcpu, pte_gpa, (u8 *)&pte, sizeof(pte));

@@ -323,8 +328,10 @@ static u64 *FNAME(fetch)(struct kvm_vcpu
 			r = kvm_read_guest_atomic(vcpu->kvm,
 						  walker->pte_gpa[level - 2],
 				       		  &curr_pte, sizeof(curr_pte));
-			if (r || curr_pte != walker->ptes[level - 2])
-				return NULL;
+			if (r || curr_pte != walker->ptes[level - 2]) {
+				shadow_ent = NULL;
+				goto out;
+			}
 		}
 		shadow_addr = __pa(shadow_page->spt);
 		shadow_pte = shadow_addr | PT_PRESENT_MASK | PT_ACCESSED_MASK

@@ -336,7 +343,8 @@ static u64 *FNAME(fetch)(struct kvm_vcpu
 		     user_fault, write_fault,
 		     walker->ptes[walker->level-1] & PT_DIRTY_MASK,
 		     ptwrite, walker->gfn);
-
+out:
+	kvm_release_page_clean(walker->page);
 	return shadow_ent;
 }
 

@@ -425,6 +433,7 @@ static gpa_t FNAME(gva_to_gpa)(struct kv
 	if (r) {
 		gpa = gfn_to_gpa(walker.gfn);
 		gpa |= vaddr & ~PAGE_MASK;
+		kvm_release_page_clean(walker.page);
 	}
 
 	return gpa;

Index: kvm.quilt/mm/pagewalk.c
===================================================================
@@ -1,6 +1,7 @@
 #include <linux/mm.h>
 #include <linux/highmem.h>
 #include <linux/sched.h>
+#include <linux/module.h>
 
 static int walk_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
 			  const struct mm_walk *walk, void *private)

@@ -129,3 +130,4 @@ int walk_page_range(const struct mm_stru
 
 	return err;
 }
+EXPORT_SYMBOL(walk_page_range);

Index: kvm.quilt/mm/memory.c
===================================================================
@@ -412,6 +412,7 @@ struct page *vm_normal_page(struct vm_ar
 	 */
 	return pfn_to_page(pfn);
 }
+EXPORT_SYMBOL(vm_normal_page);
 
 /*
  * copy one vm_area from one task to the other. Assumes the page tables

Index: kvm.quilt/arch/x86/kvm/mmu.c
===================================================================
@@ -882,14 +882,18 @@ struct page *gva_to_page(struct kvm_vcpu
 	return gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
 }
 
+/*
+ * mmu_set_spte must be called with an additional reference on
+ * struct page *page, and it will release that if necessary (so
+ * the callers should not worry about it).
+ */
 static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte,
 			 unsigned pt_access, unsigned pte_access,
 			 int user_fault, int write_fault, int dirty,
-			 int *ptwrite, gfn_t gfn)
+			 int *ptwrite, gfn_t gfn, struct page *page)
 {
 	u64 spte;
 	int was_rmapped = is_rmap_pte(*shadow_pte);
-	struct page *page;
 
 	pgprintk("%s: spte %llx access %x write_fault %d"
 		 " user_fault %d gfn %lx\n",

@@ -907,8 +911,6 @@ static void mmu_set_spte(struct kvm_vcpu
 	if (!(pte_access & ACC_EXEC_MASK))
 		spte |= PT64_NX_MASK;
 
-	page = gfn_to_page(vcpu->kvm, gfn);
-
 	spte |= PT_PRESENT_MASK;
 	if (pte_access & ACC_USER_MASK)
 		spte |= PT_USER_MASK;

@@ -983,8 +985,10 @@ static int nonpaging_map(struct kvm_vcpu
 		table = __va(table_addr);
 
 		if (level == 1) {
+			struct page *page;
+			page = gfn_to_page(vcpu->kvm, gfn);
 			mmu_set_spte(vcpu, &table[index], ACC_ALL, ACC_ALL,
-				     0, write, 1, &pt_write, gfn);
+				     0, write, 1, &pt_write, gfn, page);
 			return pt_write || is_io_pte(table[index]);
 		}
 

Index: kvm.quilt/include/linux/kvm_host.h
===================================================================
@@ -163,6 +163,7 @@ int kvm_arch_set_memory_region(struct kv
 				int user_alloc);
 gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn);
 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn);
+struct page *kvm_find_page(struct kvm *kvm, gfn_t gfn);
 void kvm_release_page_clean(struct page *page);
 void kvm_release_page_dirty(struct page *page);
 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,

54 lines not on this page · raw message

Index: kvm.quilt/arch/x86/kvm/paging_tmpl.h
===================================================================
@@ -67,6 +67,7 @@ struct guest_walker {
 	gfn_t table_gfn[PT_MAX_FULL_LEVELS];
 	pt_element_t ptes[PT_MAX_FULL_LEVELS];
 	gpa_t pte_gpa[PT_MAX_FULL_LEVELS];
+	struct page *page;
 	unsigned pt_access;
 	unsigned pte_access;
 	gfn_t gfn;

@@ -203,14 +204,18 @@ walk:
 		--walker->level;
 	}
 
+	walker->page = gfn_to_page(vcpu->kvm, walker->gfn);
+
 	if (write_fault && !is_dirty_pte(pte)) {
 		bool ret;
 
 		mark_page_dirty(vcpu->kvm, table_gfn);
 		ret = FNAME(cmpxchg_gpte)(vcpu->kvm, table_gfn, index, pte,
 			    pte|PT_DIRTY_MASK);
-		if (ret)
+		if (ret) {
+			kvm_release_page_clean(walker->page);
 			goto walk;
+		}
 		pte |= PT_DIRTY_MASK;
 		mutex_lock(&vcpu->kvm->lock);
 		kvm_mmu_pte_write(vcpu, pte_gpa, (u8 *)&pte, sizeof(pte));

@@ -241,12 +246,13 @@ err:
 	return 0;
 }
 
-static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page,
+static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *spage,
 			      u64 *spte, const void *pte, int bytes,
 			      int offset_in_pte)
 {
 	pt_element_t gpte;
 	unsigned pte_access;
+	struct page *page;
 
 	gpte = *(const pt_element_t *)pte;
 	if (~gpte & (PT_PRESENT_MASK | PT_ACCESSED_MASK)) {

@@ -256,10 +262,15 @@ static void FNAME(update_pte)(struct kvm
 	}
 	if (bytes < sizeof(pt_element_t))
 		return;
+
+	page = kvm_find_page(vcpu->kvm, gpte_to_gfn(gpte));
+	if (!page)
+		return;
+
 	pgprintk("%s: gpte %llx spte %p\n", __FUNCTION__, (u64)gpte, spte);
-	pte_access = page->role.access & FNAME(gpte_access)(vcpu, gpte);
-	mmu_set_spte(vcpu, spte, page->role.access, pte_access, 0, 0,
-		     gpte & PT_DIRTY_MASK, NULL, gpte_to_gfn(gpte));
+	pte_access = spage->role.access & FNAME(gpte_access)(vcpu, gpte);
+	mmu_set_spte(vcpu, spte, spage->role.access, pte_access, 0, 0,
+		     gpte & PT_DIRTY_MASK, NULL, gpte_to_gfn(gpte), page);
 }
 
 /*

@@ -323,8 +334,10 @@ static u64 *FNAME(fetch)(struct kvm_vcpu
 			r = kvm_read_guest_atomic(vcpu->kvm,
 						  walker->pte_gpa[level - 2],
 				       		  &curr_pte, sizeof(curr_pte));
-			if (r || curr_pte != walker->ptes[level - 2])
+			if (r || curr_pte != walker->ptes[level - 2]) {
+				kvm_release_page_clean(walker->page);
 				return NULL;
+			}
 		}
 		shadow_addr = __pa(shadow_page->spt);
 		shadow_pte = shadow_addr | PT_PRESENT_MASK | PT_ACCESSED_MASK

@@ -335,7 +348,7 @@ static u64 *FNAME(fetch)(struct kvm_vcpu
 	mmu_set_spte(vcpu, shadow_ent, access, walker->pte_access & access,
 		     user_fault, write_fault,
 		     walker->ptes[walker->level-1] & PT_DIRTY_MASK,
-		     ptwrite, walker->gfn);
+		     ptwrite, walker->gfn, walker->page);
 
 	return shadow_ent;
 }

@@ -425,6 +438,7 @@ static gpa_t FNAME(gva_to_gpa)(struct kv
 	if (r) {
 		gpa = gfn_to_gpa(walker.gfn);
 		gpa |= vaddr & ~PAGE_MASK;
+		kvm_release_page_clean(walker.page);
 	}
 
 	return gpa;

Index: kvm.quilt/mm/memory.c
===================================================================
@@ -970,6 +970,7 @@ no_page_table:
 	}
 	return page;
 }
+EXPORT_SYMBOL(follow_page);
 
 int get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
 		unsigned long start, int len, int write, int force,

Index: kvm.quilt/arch/x86/kvm/mmu.c
===================================================================
@@ -1031,7 +1031,7 @@ static void mmu_free_roots(struct kvm_vc
 
 	if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
 		return;
-	mutex_lock(&vcpu->kvm->lock);
+	spin_lock(&vcpu->kvm->mmu_lock);
 #ifdef CONFIG_X86_64
 	if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
 		hpa_t root = vcpu->arch.mmu.root_hpa;

@@ -1039,7 +1039,7 @@ static void mmu_free_roots(struct kvm_vc
 		sp = page_header(root);
 		--sp->root_count;
 		vcpu->arch.mmu.root_hpa = INVALID_PAGE;
-		mutex_unlock(&vcpu->kvm->lock);
+		spin_unlock(&vcpu->kvm->mmu_lock);
 		return;
 	}
 #endif

@@ -1053,7 +1053,7 @@ static void mmu_free_roots(struct kvm_vc
 		}
 		vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
 	}
-	mutex_unlock(&vcpu->kvm->lock);
+	spin_unlock(&vcpu->kvm->mmu_lock);
 	vcpu->arch.mmu.root_hpa = INVALID_PAGE;
 }
 

@@ -1256,9 +1256,9 @@ int kvm_mmu_load(struct kvm_vcpu *vcpu)
 	r = mmu_topup_memory_caches(vcpu);
 	if (r)
 		goto out;
-	mutex_lock(&vcpu->kvm->lock);
+	spin_lock(&vcpu->kvm->mmu_lock);
 	mmu_alloc_roots(vcpu);
-	mutex_unlock(&vcpu->kvm->lock);
+	spin_unlock(&vcpu->kvm->mmu_lock);
 	kvm_x86_ops->set_cr3(vcpu, vcpu->arch.mmu.root_hpa);
 	kvm_mmu_flush_tlb(vcpu);
 out:

@@ -1435,15 +1435,15 @@ int kvm_mmu_unprotect_page_virt(struct k
 	gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gva);
 	up_read(&current->mm->mmap_sem);
 
-	mutex_lock(&vcpu->kvm->lock);
+	spin_lock(&vcpu->kvm->mmu_lock);
 	r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
-	mutex_unlock(&vcpu->kvm->lock);
+	spin_unlock(&vcpu->kvm->mmu_lock);
 	return r;
 }
 
 void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu)
 {
-	mutex_lock(&vcpu->kvm->lock);
+	spin_lock(&vcpu->kvm->mmu_lock);
 	while (vcpu->kvm->arch.n_free_mmu_pages < KVM_REFILL_PAGES) {
 		struct kvm_mmu_page *sp;
 

@@ -1452,7 +1452,7 @@ void __kvm_mmu_free_some_pages(struct kv
 		kvm_mmu_zap_page(vcpu->kvm, sp);
 		++vcpu->kvm->stat.mmu_recycled;
 	}
-	mutex_unlock(&vcpu->kvm->lock);
+	spin_unlock(&vcpu->kvm->mmu_lock);
 }
 
 int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code)

@@ -1584,10 +1584,10 @@ void kvm_mmu_zap_all(struct kvm *kvm)
 {
 	struct kvm_mmu_page *sp, *node;
 
-	mutex_lock(&kvm->lock);
+	spin_lock(&kvm->mmu_lock);
 	list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link)
 		kvm_mmu_zap_page(kvm, sp);
-	mutex_unlock(&kvm->lock);
+	spin_unlock(&kvm->mmu_lock);
 
 	kvm_flush_remote_tlbs(kvm);
 }

Index: kvm.quilt/arch/x86/kvm/paging_tmpl.h
===================================================================
@@ -217,9 +217,9 @@ walk:
 			goto walk;
 		}
 		pte |= PT_DIRTY_MASK;
-		mutex_lock(&vcpu->kvm->lock);
+		spin_lock(&vcpu->kvm->mmu_lock);
 		kvm_mmu_pte_write(vcpu, pte_gpa, (u8 *)&pte, sizeof(pte));
-		mutex_unlock(&vcpu->kvm->lock);
+		spin_unlock(&vcpu->kvm->mmu_lock);
 		walker->ptes[walker->level - 1] = pte;
 	}
 

@@ -386,7 +386,6 @@ static int FNAME(page_fault)(struct kvm_
 	 */
 	r = FNAME(walk_addr)(&walker, vcpu, addr, write_fault, user_fault,
 			     fetch_fault);
-	up_read(&current->mm->mmap_sem);
 
 	/*
 	 * The page is not mapped by the guest.  Let the guest handle it.

@@ -395,10 +394,11 @@ static int FNAME(page_fault)(struct kvm_
 		pgprintk("%s: guest page fault\n", __FUNCTION__);
 		inject_page_fault(vcpu, addr, walker.error_code);
 		vcpu->arch.last_pt_write_count = 0; /* reset fork detector */
+		up_read(&current->mm->mmap_sem);
 		return 0;
 	}
 
-	mutex_lock(&vcpu->kvm->lock);
+	spin_lock(&vcpu->kvm->mmu_lock);
 	shadow_pte = FNAME(fetch)(vcpu, addr, &walker, user_fault, write_fault,
 				  &write_pt);
 	pgprintk("%s: shadow pte %p %llx ptwrite %d\n", __FUNCTION__,

@@ -411,13 +411,15 @@ static int FNAME(page_fault)(struct kvm_
 	 * mmio: emulate if accessible, otherwise its a guest fault.
 	 */
 	if (shadow_pte && is_io_pte(*shadow_pte)) {
-		mutex_unlock(&vcpu->kvm->lock);
+		spin_unlock(&vcpu->kvm->mmu_lock);
+		up_read(&current->mm->mmap_sem);
 		return 1;
 	}
 
 	++vcpu->stat.pf_fixed;
 	kvm_mmu_audit(vcpu, "post page fault (fixed)");
-	mutex_unlock(&vcpu->kvm->lock);
+	spin_unlock(&vcpu->kvm->mmu_lock);
+	up_read(&current->mm->mmap_sem);
 
 	return write_pt;
 }

Index: kvm.quilt/arch/x86/kvm/x86.c
===================================================================
@@ -1609,9 +1609,9 @@ static int emulator_write_phys(struct kv
 		up_read(&current->mm->mmap_sem);
 		return 0;
 	}
-	mutex_lock(&vcpu->kvm->lock);
+	spin_lock(&vcpu->kvm->mmu_lock);
 	kvm_mmu_pte_write(vcpu, gpa, val, bytes);
-	mutex_unlock(&vcpu->kvm->lock);
+	spin_unlock(&vcpu->kvm->mmu_lock);
 	up_read(&current->mm->mmap_sem);
 	return 1;
 }

Index: kvm.quilt/include/linux/kvm_host.h
===================================================================
@@ -104,7 +104,8 @@ struct kvm_memory_slot {
 };
 
 struct kvm {
-	struct mutex lock; /* protects everything except vcpus */
+	struct mutex lock; /* protects the vcpus array and APIC accesses */
+	spinlock_t mmu_lock;
 	struct mm_struct *mm; /* userspace tied to this vm */
 	int nmemslots;
 	struct kvm_memory_slot memslots[KVM_MEMORY_SLOTS +

Index: kvm.quilt/virt/kvm/kvm_main.c
===================================================================
@@ -165,6 +165,7 @@ static struct kvm *kvm_create_vm(void)
 
 	kvm->mm = current->mm;
 	atomic_inc(&kvm->mm->mm_count);
+	spin_lock_init(&kvm->mmu_lock);
 	kvm_io_bus_init(&kvm->pio_bus);
 	mutex_init(&kvm->lock);
 	kvm_io_bus_init(&kvm->mmio_bus);

@@ -571,9 +572,7 @@ int kvm_read_guest_atomic(struct kvm *kv
 	addr = gfn_to_hva(kvm, gfn);
 	if (kvm_is_error_hva(addr))
 		return -EFAULT;
-	pagefault_disable();
 	r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
-	pagefault_enable();
 	if (r)
 		return -EFAULT;
 	return 0;

Index: kvm.quilt/arch/x86/kvm/vmx.c
===================================================================
@@ -1476,7 +1476,6 @@ static int alloc_apic_access_page(struct
 	struct kvm_userspace_memory_region kvm_userspace_mem;
 	int r = 0;
 
-	mutex_lock(&kvm->lock);
 	down_write(&current->mm->mmap_sem);
 	if (kvm->arch.apic_access_page)
 		goto out;

@@ -1490,7 +1489,6 @@ static int alloc_apic_access_page(struct
 	kvm->arch.apic_access_page = gfn_to_page(kvm, 0xfee00);
 out:
 	up_write(&current->mm->mmap_sem);
-	mutex_unlock(&kvm->lock);
 	return r;
 }