KVM: Dirty ring interface

Cache the address space ID just like the slot ID. It will be used in order to fill in the dirty ring entries.

Suggested-by Paolo Bonzini <[email protected]>

Suggested-by Sean Christopherson <[email protected]>

Signed-off-by Peter Xu <[email protected]>

---
 include/linux/kvm_host.h | 1 +
 virt/kvm/kvm_main.c      | 2 ++
 2 files changed, 3 insertions(+)

include/linux/kvm_host.h +0 -0

index bcb9b2ac0791..afa0e9034881 100644

include/linux/kvm_host.h +0 -0

 };
 
 static inline unsigned long kvm_dirty_bitmap_bytes(struct kvm_memory_slot *memslot)

virt/kvm/kvm_main.c +2 -0

index 70f03ce0e5c1..e6484dabfc59 100644
@@ -1036,6 +1036,8 @@ int __kvm_set_memory_region(struct kvm *kvm,
 
 	new = old = *slot;
 
+	BUILD_BUG_ON(U8_MAX < KVM_ADDRESS_SPACE_NUM);
+	new.as_id = as_id;
 	new.id = id;
 	new.base_gfn = base_gfn;
 	new.npages = npages;

Signature · 1 line

2.24.1

Quoted 11 lines · Peter Xu

Cache the address space ID just like the slot ID. It will be used in order to fill in the dirty ring entries.

Suggested-by Paolo Bonzini <[email protected]>

Suggested-by Sean Christopherson <[email protected]>

Signed-off-by Peter Xu <[email protected]>

---
 include/linux/kvm_host.h | 1 +
 virt/kvm/kvm_main.c      | 2 ++
 2 files changed, 3 insertions(+)

include/linux/kvm_host.h struct kvm_memory_slot { · line 349

Quoted 9 lines

include/linux/kvm_host.h +1 -0

index bcb9b2ac0791..afa0e9034881 100644
@@ -346,6 +346,7 @@ struct kvm_memory_slot {
 	unsigned long userspace_addr;
 	u32 flags;
 	short id;
+	u8 as_id;

My vote is to store this as a u16 and remove the BUILD_BUG_ON. Using a u8 doesn't save any memory since the compiler will pad out the struct.

Quoted 20 lines

include/linux/kvm_host.h +0 -0

 };
 
 static inline unsigned long kvm_dirty_bitmap_bytes(struct kvm_memory_slot *memslot)

virt/kvm/kvm_main.c +2 -0

index 70f03ce0e5c1..e6484dabfc59 100644
@@ -1036,6 +1036,8 @@ int __kvm_set_memory_region(struct kvm *kvm,
 
 	new = old = *slot;
 
+	BUILD_BUG_ON(U8_MAX < KVM_ADDRESS_SPACE_NUM);
+	new.as_id = as_id;
 	new.id = id;
 	new.base_gfn = base_gfn;
 	new.npages = npages;

Signature · 1 line

2.24.1

arch/x86/kvm/vmx/vmx.c +0 -0

index 40b1e6138cd5..fc638a164e03 100644

arch/x86/kvm/vmx/vmx.c +32 -32

+
 	data = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE;
-	r = kvm_write_guest_page(kvm, fn++, &data,
-			TSS_IOPB_BASE_OFFSET, sizeof(u16));
-	if (r < 0)
-		goto out;
-	r = kvm_clear_guest_page(kvm, fn++, 0, PAGE_SIZE);
-	if (r < 0)
-		goto out;
-	r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
-	if (r < 0)
-		goto out;
+	r = __copy_to_user(ua + TSS_IOPB_BASE_OFFSET, &data, sizeof(u16));
+	if (r)
+		return -EFAULT;
+
 	data = ~0;
-	r = kvm_write_guest_page(kvm, fn, &data,
-				 RMODE_TSS_SIZE - 2 * PAGE_SIZE - 1,
-				 sizeof(u8));
-out:
-	srcu_read_unlock(&kvm->srcu, idx);
+	r = __copy_to_user(ua + RMODE_TSS_SIZE - 1, &data, sizeof(u8));
+
 	return r;
 }
 

@@ -3503,6 +3495,7 @@ static int init_rmode_identity_map(struct kvm *kvm)
 	struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm);
 	int i, r = 0;
 	kvm_pfn_t identity_map_pfn;
+	void __user *uaddr;
 	u32 tmp;
 
 	/* Protect kvm_vmx->ept_identity_pagetable_done. */

@@ -3515,22 +3508,24 @@ static int init_rmode_identity_map(struct kvm *kvm)
 		kvm_vmx->ept_identity_map_addr = VMX_EPT_IDENTITY_PAGETABLE_ADDR;
 	identity_map_pfn = kvm_vmx->ept_identity_map_addr >> PAGE_SHIFT;
 
-	r = __x86_set_memory_region(kvm, IDENTITY_PAGETABLE_PRIVATE_MEMSLOT,
-				    kvm_vmx->ept_identity_map_addr, PAGE_SIZE);
-	if (r < 0)
+	uaddr = __x86_set_memory_region(kvm,
+					IDENTITY_PAGETABLE_PRIVATE_MEMSLOT,
+					kvm_vmx->ept_identity_map_addr,
+					PAGE_SIZE);
+	if (IS_ERR(uaddr)) {
+		r = PTR_ERR(uaddr);
 		goto out;
+	}
 
-	r = kvm_clear_guest_page(kvm, identity_map_pfn, 0, PAGE_SIZE);
-	if (r < 0)
-		goto out;
 	/* Set up identity-mapping pagetable for EPT in real mode */
 	for (i = 0; i < PT32_ENT_PER_PAGE; i++) {
 		tmp = (i << 22) + (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER |
 			_PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_PSE);
-		r = kvm_write_guest_page(kvm, identity_map_pfn,
-				&tmp, i * sizeof(tmp), sizeof(tmp));
-		if (r < 0)
+		r = __copy_to_user(uaddr + i * sizeof(tmp), &tmp, sizeof(tmp));
+		if (r) {
+			r = -EFAULT;
 			goto out;
+		}
 	}
 	kvm_vmx->ept_identity_pagetable_done = true;
 

@@ -3557,19 +3552,22 @@ static void seg_setup(int seg)
 static int alloc_apic_access_page(struct kvm *kvm)
 {
 	struct page *page;
-	int r = 0;
+	void __user *r;
+	int ret = 0;
 
 	mutex_lock(&kvm->slots_lock);
 	if (kvm->arch.apic_access_page_done)
 		goto out;
 	r = __x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
 				    APIC_DEFAULT_PHYS_BASE, PAGE_SIZE);
-	if (r)
+	if (IS_ERR(r)) {
+		ret = PTR_ERR(r);
 		goto out;
+	}
 
 	page = gfn_to_page(kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT);
 	if (is_error_page(page)) {
-		r = -EFAULT;
+		ret = -EFAULT;
 		goto out;
 	}
 

@@ -3581,7 +3579,7 @@ static int alloc_apic_access_page(struct kvm *kvm)
 	kvm->arch.apic_access_page_done = true;
 out:
 	mutex_unlock(&kvm->slots_lock);
-	return r;
+	return ret;
 }
 
 int allocate_vpid(void)

@@ -4503,7 +4501,7 @@ static int vmx_interrupt_allowed(struct kvm_vcpu *vcpu)
 
 static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr)
 {
-	int ret;
+	void __user *ret;
 
 	if (enable_unrestricted_guest)
 		return 0;

@@ -4513,10 +4511,12 @@ static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr)
 				      PAGE_SIZE * 3);
 	mutex_unlock(&kvm->slots_lock);
 
-	if (ret)
-		return ret;
+	if (IS_ERR(ret))
+		return PTR_ERR(ret);
+
 	to_kvm_vmx(kvm)->tss_addr = addr;
-	return init_rmode_tss(kvm);
+
+	return init_rmode_tss(kvm, ret);
 }
 
 static int vmx_set_identity_map_addr(struct kvm *kvm, u64 ident_addr)

arch/x86/kvm/x86.c +5 -1

index 5de200663f51..fe485d4ba6c7 100644
@@ -9756,7 +9756,33 @@ void kvm_arch_sync_events(struct kvm *kvm)
 	kvm_free_pit(kvm);
 }
 
-int __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size)
+/**
+ * __x86_set_memory_region: Setup KVM internal memory slot
+ *
+ * @kvm: the kvm pointer to the VM.
+ * @id: the slot ID to setup.

arch/x86/kvm/x86.c +18 -6

+ *
+ * The caller should always use IS_ERR() to check the return value
+ * before use.  NOTE: KVM internal memory slots are guaranteed and
+ * won't change until the VM is destroyed. This is also true to the
+ * returned HVA when installing a new memory slot.  The HVA can be
+ * invalidated by either an errornous userspace program or a VM under
+ * destruction, however as long as we use __copy_{to|from}_user()
+ * properly upon the HVAs and handle the failure paths always then
+ * we're safe.
+ */
+void __user * __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa,
+				      u32 size)
 {
 	int i, r;
 	unsigned long hva;

@@ -9765,12 +9791,12 @@ int __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size)
 
 	/* Called with kvm->slots_lock held.  */
 	if (WARN_ON(id >= KVM_MEM_SLOTS_NUM))
-		return -EINVAL;
+		return ERR_PTR(-EINVAL);
 
 	slot = id_to_memslot(slots, id);
 	if (size) {
 		if (slot->npages)
-			return -EEXIST;
+			return ERR_PTR(-EEXIST);
 
 		/*
 		 * MAP_SHARED to prevent internal slot pages from being moved

@@ -9779,10 +9805,10 @@ int __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size)
 		hva = vm_mmap(NULL, 0, size, PROT_READ | PROT_WRITE,
 			      MAP_SHARED | MAP_ANONYMOUS, 0);
 		if (IS_ERR((void *)hva))
-			return PTR_ERR((void *)hva);
+			return (void __user *)hva;
 	} else {
 		if (!slot->npages)
-			return 0;
+			return ERR_PTR(0);
 
 		hva = 0;
 	}

@@ -9798,13 +9824,13 @@ int __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size)
 		m.memory_size = size;
 		r = __kvm_set_memory_region(kvm, &m);
 		if (r < 0)
-			return r;
+			return ERR_PTR(r);
 	}
 
 	if (!size)
 		vm_munmap(old.userspace_addr, old.npages * PAGE_SIZE);
 
-	return 0;
+	return (void __user *)hva;
 }
 EXPORT_SYMBOL_GPL(__x86_set_memory_region);
 

Signature · 1 line

2.24.1

Quoted 8 lines · Peter Xu

arch/x86/kvm/vmx/vmx.c +0 -0

index 40b1e6138cd5..fc638a164e03 100644
@@ -3467,34 +3467,26 @@ static bool guest_state_valid(struct kvm_vcpu *vcpu)
 	return true;
 }
 

arch/x86/kvm/vmx/vmx.c static bool guest_state_valid(struct kvm_vcpu *vcpu) · line 3473

Quoted 6 lines

arch/x86/kvm/vmx/vmx.c +2 -2

-static int init_rmode_tss(struct kvm *kvm)
+static int init_rmode_tss(struct kvm *kvm, void __user *ua)
 {
-	gfn_t fn;
+	const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0)));
 	u16 data = 0;

"data" doesn't need to be intialized to zero, it's set below before it's used.

arch/x86/kvm/vmx/vmx.c static bool guest_state_valid(struct kvm_vcpu *vcpu) · line 3474

Quoted 2 lines

arch/x86/kvm/vmx/vmx.c +0 -0

 	int idx, r;

nit: I'd prefer to rename "idx" to "i" to make it more obvious it's a plain ole loop counter. Reusing the srcu index made me do a double take :-)

arch/x86/kvm/vmx/vmx.c static bool guest_state_valid(struct kvm_vcpu *vcpu) · line 3480

Quoted 12 lines

arch/x86/kvm/vmx/vmx.c +5 -5

-	idx = srcu_read_lock(&kvm->srcu);
-	fn = to_kvm_vmx(kvm)->tss_addr >> PAGE_SHIFT;
-	r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
-	if (r < 0)
-		goto out;
+	for (idx = 0; idx < 3; idx++) {
+		r = __copy_to_user(ua + PAGE_SIZE * idx, zero_page, PAGE_SIZE);
+		if (r)
+			return -EFAULT;
+	}

Can this be done in a single __copy_to_user(), or do those helpers not like crossing page boundaries?

Quoted 6 lines

arch/x86/kvm/vmx/vmx.c +5 -1

-int __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size)
+/**
+ * __x86_set_memory_region: Setup KVM internal memory slot
+ *
+ * @kvm: the kvm pointer to the VM.
+ * @id: the slot ID to setup.

arch/x86/kvm/vmx/vmx.c static bool guest_state_valid(struct kvm_vcpu *vcpu)

Quoted 10 lines

arch/x86/kvm/vmx/vmx.c +10 -0

+ * @gpa: the GPA to install the slot (unused when @size == 0).
+ * @size: the size of the slot. Set to zero to uninstall a slot.
+ *
+ * This function helps to setup a KVM internal memory slot.  Specify
+ * @size > 0 to install a new slot, while @size == 0 to uninstall a
+ * slot.  The return code can be one of the following:
+ *
+ *   - An error number if error happened, or,
+ *   - For installation: the HVA of the newly mapped memory slot, or,
+ *   - For uninstallation: zero if we successfully uninstall a slot.

Maybe tweak this so the return it stands out? And returning zero on uninstallation is no longer true in kvm/queue, at least not without further modifications (as is it'll return 0xdead000000000000 on 64-bit). The 0xdead shenanigans won't trigger IS_ERR(), so I think this can simply be:

* Returns:
* hva: on success
* -errno: on error

With the blurb below calling out that hva is bogus uninstallation.

Quoted 14 lines · Sean Christopherson

arch/x86/kvm/vmx/vmx.c static bool guest_state_valid(struct kvm_vcpu *vcpu) · line 3480

Quoted 10 lines · Peter Xu

arch/x86/kvm/vmx/vmx.c +5 -5

-	idx = srcu_read_lock(&kvm->srcu);
-	fn = to_kvm_vmx(kvm)->tss_addr >> PAGE_SHIFT;
-	r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
-	if (r < 0)
-		goto out;
+	for (idx = 0; idx < 3; idx++) {
+		r = __copy_to_user(ua + PAGE_SIZE * idx, zero_page, PAGE_SIZE);
+		if (r)
+			return -EFAULT;
+	}

Can this be done in a single __copy_to_user(), or do those helpers not like crossing page boundaries?

Maybe because the zero_page is only PAGE_SIZE long? :)

[...]

Hi Peter,

Thank you for the patch! Perhaps something to improve:

[auto build test WARNING on tip/auto-latest]
[also build test WARNING on vhost/linux-next linus/master v5.6-rc5 next-20200310] [cannot apply to kvm/linux-next linux/master]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system. BTW, we also suggest to use '--base' option to specify the base tree in git format-patch, please see https://stackoverflow.com/a/37406982]

url:    https://github.com/0day-ci/linux/commits/Peter-Xu/KVM-Dirty-ring-interface/20200310-070637
base:   https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git 12481c76713078054f2d043b3ce946e4814ac29f
reproduce:
        # apt-get install sparse
        # sparse version: v0.6.1-174-g094d5a94-dirty
        make ARCH=x86_64 allmodconfig
        make C=1 CF='-fdiagnostic-prefix -D__CHECK_ENDIAN__'

If you fix the issue, kindly add following tag

Reported-by kbuild test robot <[email protected]>

sparse warnings: (new ones prefixed by >>)

   arch/x86/kvm/x86.c:2599:38: sparse: sparse: incorrect type in argument 1 (different address spaces) @@    expected void const [noderef] <asn:1> * @@    got  const [noderef] <asn:1> * @@
   arch/x86/kvm/x86.c:2599:38: sparse:    expected void const [noderef] <asn:1> *
   arch/x86/kvm/x86.c:2599:38: sparse:    got unsigned char [usertype] *
   arch/x86/kvm/x86.c:7501:15: sparse: sparse: incompatible types in comparison expression (different address spaces):
   arch/x86/kvm/x86.c:7501:15: sparse:    struct kvm_apic_map [noderef] <asn:4> *
   arch/x86/kvm/x86.c:7501:15: sparse:    struct kvm_apic_map *

Quoted 1 line

Quoted 1 line

arch/x86/kvm/x86.c:9794:31: sparse: sparse: incorrect type in return expression (different address spaces) @@ expected void [noderef] <asn:1> * @@ got n:1> * @@

   arch/x86/kvm/x86.c:9794:31: sparse:    expected void [noderef] <asn:1> *
   arch/x86/kvm/x86.c:9794:31: sparse:    got void *
   arch/x86/kvm/x86.c:9799:39: sparse: sparse: incorrect type in return expression (different address spaces) @@    expected void [noderef] <asn:1> * @@    got n:1> * @@
   arch/x86/kvm/x86.c:9799:39: sparse:    expected void [noderef] <asn:1> *
   arch/x86/kvm/x86.c:9799:39: sparse:    got void *
   arch/x86/kvm/x86.c:9811:39: sparse: sparse: incorrect type in return expression (different address spaces) @@    expected void [noderef] <asn:1> * @@    got n:1> * @@
   arch/x86/kvm/x86.c:9811:39: sparse:    expected void [noderef] <asn:1> *
   arch/x86/kvm/x86.c:9811:39: sparse:    got void *
   arch/x86/kvm/x86.c:9827:39: sparse: sparse: incorrect type in return expression (different address spaces) @@    expected void [noderef] <asn:1> * @@    got n:1> * @@
   arch/x86/kvm/x86.c:9827:39: sparse:    expected void [noderef] <asn:1> *
   arch/x86/kvm/x86.c:9827:39: sparse:    got void *
   arch/x86/kvm/x86.c:9863:16: sparse: sparse: incompatible types in comparison expression (different address spaces):
   arch/x86/kvm/x86.c:9863:16: sparse:    struct kvm_apic_map [noderef] <asn:4> *
   arch/x86/kvm/x86.c:9863:16: sparse:    struct kvm_apic_map *
   arch/x86/kvm/x86.c:9864:15: sparse: sparse: incompatible types in comparison expression (different address spaces):
   arch/x86/kvm/x86.c:9864:15: sparse:    struct kvm_pmu_event_filter [noderef] <asn:4> *
   arch/x86/kvm/x86.c:9864:15: sparse:    struct kvm_pmu_event_filter *
   include/linux/srcu.h:179:9: sparse: sparse: context imbalance in 'vcpu_enter_guest' - unexpected unlock

vim +9794 arch/x86/kvm/x86.c

9758
9759 /**
9760 * __x86_set_memory_region: Setup KVM internal memory slot
9761 *
9762 * @kvm: the kvm pointer to the VM.
9763 * @id: the slot ID to setup.
9764 * @gpa: the GPA to install the slot (unused when @size == 0).
9765 * @size: the size of the slot. Set to zero to uninstall a slot.
9766 *
9767 * This function helps to setup a KVM internal memory slot. Specify
9768 * @size > 0 to install a new slot, while @size == 0 to uninstall a
9769 * slot. The return code can be one of the following:
9770 *

  9771	 *   - An error number if error happened, or,
  9772	 *   - For installation: the HVA of the newly mapped memory slot, or,
  9773	 *   - For uninstallation: zero if we successfully uninstall a slot.

9774 *
9775 * The caller should always use IS_ERR() to check the return value
9776 * before use. NOTE: KVM internal memory slots are guaranteed and
9777 * won't change until the VM is destroyed. This is also true to the
9778 * returned HVA when installing a new memory slot. The HVA can be
9779 * invalidated by either an errornous userspace program or a VM under
9780 * destruction, however as long as we use __copy_{to|from}_user()
9781 * properly upon the HVAs and handle the failure paths always then
9782 * we're safe.
9783 */
9784 void __user * __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa,
9785 u32 size)
9786 {
9787 int i, r;
9788 unsigned long hva;
9789 struct kvm_memslots *slots = kvm_memslots(kvm);
9790 struct kvm_memory_slot *slot, old;
9791
9792 /* Called with kvm->slots_lock held. */
9793 if (WARN_ON(id >= KVM_MEM_SLOTS_NUM))

Quoted 1 line

9794 return ERR_PTR(-EINVAL);

9795
9796 slot = id_to_memslot(slots, id);
9797 if (size) {
9798 if (slot->npages)
9799 return ERR_PTR(-EEXIST);
9800
9801 /*
9802 * MAP_SHARED to prevent internal slot pages from being moved
9803 * by fork()/COW.
9804 */
9805 hva = vm_mmap(NULL, 0, size, PROT_READ | PROT_WRITE,
9806 MAP_SHARED | MAP_ANONYMOUS, 0);
9807 if (IS_ERR((void *)hva))
9808 return (void __user *)hva;
9809 } else {
9810 if (!slot->npages)
9811 return ERR_PTR(0);
9812
9813 hva = 0;
9814 }
9815
9816 old = *slot;
9817 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
9818 struct kvm_userspace_memory_region m;
9819
9820 m.slot = id | (i << 16);
9821 m.flags = 0;
9822 m.guest_phys_addr = gpa;
9823 m.userspace_addr = hva;
9824 m.memory_size = size;
9825 r = __kvm_set_memory_region(kvm, &m);
9826 if (r < 0)
9827 return ERR_PTR(r);
9828 }
9829
9830 if (!size)
9831 vm_munmap(old.userspace_addr, old.npages * PAGE_SIZE);
9832
9833 return (void __user *)hva;
9834 }
9835 EXPORT_SYMBOL_GPL(__x86_set_memory_region);
9836

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org

Quoted 31 lines · kbuild test robot

Hi Peter,

Thank you for the patch! Perhaps something to improve:

[auto build test WARNING on tip/auto-latest]
[also build test WARNING on vhost/linux-next linus/master v5.6-rc5 next-20200310] [cannot apply to kvm/linux-next linux/master]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system. BTW, we also suggest to use '--base' option to specify the base tree in git format-patch, please see https://stackoverflow.com/a/37406982]

url:    https://github.com/0day-ci/linux/commits/Peter-Xu/KVM-Dirty-ring-interface/20200310-070637
base:   https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git 12481c76713078054f2d043b3ce946e4814ac29f
reproduce:
        # apt-get install sparse
        # sparse version: v0.6.1-174-g094d5a94-dirty
        make ARCH=x86_64 allmodconfig
        make C=1 CF='-fdiagnostic-prefix -D__CHECK_ENDIAN__'

If you fix the issue, kindly add following tag

Reported-by kbuild test robot <[email protected]>

sparse warnings: (new ones prefixed by >>)

   arch/x86/kvm/x86.c:2599:38: sparse: sparse: incorrect type in argument 1 (different address spaces) @@    expected void const [noderef] <asn:1> * @@    got  const [noderef] <asn:1> * @@
   arch/x86/kvm/x86.c:2599:38: sparse:    expected void const [noderef] <asn:1> *
   arch/x86/kvm/x86.c:2599:38: sparse:    got unsigned char [usertype] *
   arch/x86/kvm/x86.c:7501:15: sparse: sparse: incompatible types in comparison expression (different address spaces):
   arch/x86/kvm/x86.c:7501:15: sparse:    struct kvm_apic_map [noderef] <asn:4> *
   arch/x86/kvm/x86.c:7501:15: sparse:    struct kvm_apic_map *

Quoted 1 line

Quoted 1 line

Quoted 1 line

arch/x86/kvm/x86.c:9794:31: sparse: sparse: incorrect type in return expression (different address spaces) @@ expected void [noderef] <asn:1> * @@ got n:1> * @@

I'm not sure on how I can reproduce this locally, and also I'm not very sure I understand this warning. I'd be glad to know if anyone knows...

If without further hints, I'll try to remove the __user for __x86_set_memory_region() and use a cast on the callers next.

Quoted 32 lines · Peter Xu

Quoted 31 lines · kbuild test robot

Hi Peter,

Thank you for the patch! Perhaps something to improve:

[auto build test WARNING on tip/auto-latest]
[also build test WARNING on vhost/linux-next linus/master v5.6-rc5 next-20200310] [cannot apply to kvm/linux-next linux/master]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system. BTW, we also suggest to use '--base' option to specify the base tree in git format-patch, please see https://stackoverflow.com/a/37406982]

url:    https://github.com/0day-ci/linux/commits/Peter-Xu/KVM-Dirty-ring-interface/20200310-070637
base:   https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git 12481c76713078054f2d043b3ce946e4814ac29f
reproduce:
        # apt-get install sparse
        # sparse version: v0.6.1-174-g094d5a94-dirty
        make ARCH=x86_64 allmodconfig
        make C=1 CF='-fdiagnostic-prefix -D__CHECK_ENDIAN__'

If you fix the issue, kindly add following tag

Reported-by kbuild test robot <[email protected]>

sparse warnings: (new ones prefixed by >>)

   arch/x86/kvm/x86.c:2599:38: sparse: sparse: incorrect type in argument 1 (different address spaces) @@    expected void const [noderef] <asn:1> * @@    got  const [noderef] <asn:1> * @@
   arch/x86/kvm/x86.c:2599:38: sparse:    expected void const [noderef] <asn:1> *
   arch/x86/kvm/x86.c:2599:38: sparse:    got unsigned char [usertype] *
   arch/x86/kvm/x86.c:7501:15: sparse: sparse: incompatible types in comparison expression (different address spaces):
   arch/x86/kvm/x86.c:7501:15: sparse:    struct kvm_apic_map [noderef] <asn:4> *
   arch/x86/kvm/x86.c:7501:15: sparse:    struct kvm_apic_map *

Quoted 8 lines

Quoted 1 line

Quoted 1 line

Quoted 1 line

arch/x86/kvm/x86.c:9794:31: sparse: sparse: incorrect type in return expression (different address spaces) @@ expected void [noderef] <asn:1> * @@ got n:1> * @@

I'm not sure on how I can reproduce this locally, and also I'm not very sure I understand this warning. I'd be glad to know if anyone knows...

If without further hints, I'll try to remove the __user for __x86_set_memory_region() and use a cast on the callers next.

Ah, it's complaining that the ERR_PTR() returns in __x86_set_memory_region() aren't explicitly casting to a __user pointer.

Part of me wonders if something along the lines of your original approach of keeping the "int" return and passing a "void __user **p_hva" would be cleaner overall, as opposed to having to cast everywhere. The diff would certainly be smaller. E.g.

int __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size,

			    void __user **p_hva)
{
	...

	if (p_hva)
		*p_hva = (void __user *)hva;

        return 0;
}

Remove the clear_dirty_log test, instead merge it into the existing dirty_log_test. It should be cleaner to use this single binary to do both tests, also it's a preparation for the upcoming dirty ring test.

The default behavior will run all the modes in sequence.

Signed-off-by Peter Xu <[email protected]>

---
 tools/testing/selftests/kvm/Makefile          |   2 -
 .../selftests/kvm/clear_dirty_log_test.c      |   2 -
 tools/testing/selftests/kvm/dirty_log_test.c  | 169 +++++++++++++++---
 3 files changed, 146 insertions(+), 27 deletions(-)
 delete mode 100644 tools/testing/selftests/kvm/clear_dirty_log_test.c

tools/testing/selftests/kvm/Makefile +0 -2

index d91c53b726e6..941bfcd48eaa 100644
@@ -27,11 +27,9 @@ TEST_GEN_PROGS_x86_64 += x86_64/vmx_set_nested_state_test
 TEST_GEN_PROGS_x86_64 += x86_64/vmx_tsc_adjust_test
 TEST_GEN_PROGS_x86_64 += x86_64/xss_msr_test
 TEST_GEN_PROGS_x86_64 += x86_64/svm_vmcall_test
-TEST_GEN_PROGS_x86_64 += clear_dirty_log_test
 TEST_GEN_PROGS_x86_64 += dirty_log_test
 TEST_GEN_PROGS_x86_64 += kvm_create_max_vcpus
 
-TEST_GEN_PROGS_aarch64 += clear_dirty_log_test
 TEST_GEN_PROGS_aarch64 += dirty_log_test
 TEST_GEN_PROGS_aarch64 += kvm_create_max_vcpus
 

tools/testing/selftests/kvm/clear_dirty_log_test.c deleted +0 -2

deleted file mode 100644
index 749336937d37..000000000000
@@ -1,2 +0,0 @@
-#define USE_CLEAR_DIRTY_LOG
-#include "dirty_log_test.c"

tools/testing/selftests/kvm/dirty_log_test.c +67 -0

index 3c0ffd34b3b0..642886394e34 100644
@@ -128,6 +128,73 @@ static uint64_t host_dirty_count;
 static uint64_t host_clear_count;
 static uint64_t host_track_next_count;
 
+enum log_mode_t {
+	/* Only use KVM_GET_DIRTY_LOG for logging */
+	LOG_MODE_DIRTY_LOG = 0,
+
+	/* Use both KVM_[GET|CLEAR]_DIRTY_LOG for logging */
+	LOG_MODE_CLEAR_LOG = 1,
+
+	LOG_MODE_NUM,
+
+	/* Run all supported modes */
+	LOG_MODE_ALL = LOG_MODE_NUM,
+};
+
+/* Mode of logging to test.  Default is to run all supported modes */
+static enum log_mode_t host_log_mode_option = LOG_MODE_ALL;
+/* Logging mode for current run */
+static enum log_mode_t host_log_mode;
+
+static bool clear_log_supported(void)
+{
+	return kvm_check_cap(KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2);
+}
+
+static void clear_log_create_vm_done(struct kvm_vm *vm)
+{
+	struct kvm_enable_cap cap = {};
+
+	cap.cap = KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2;
+	cap.args[0] = 1;
+	vm_enable_cap(vm, &cap);
+}
+
+static void dirty_log_collect_dirty_pages(struct kvm_vm *vm, int slot,
+					  void *bitmap, uint32_t num_pages)
+{
+	kvm_vm_get_dirty_log(vm, slot, bitmap);
+}
+
+static void clear_log_collect_dirty_pages(struct kvm_vm *vm, int slot,
+					  void *bitmap, uint32_t num_pages)
+{
+	kvm_vm_get_dirty_log(vm, slot, bitmap);
+	kvm_vm_clear_dirty_log(vm, slot, bitmap, 0, num_pages);
+}
+
+struct log_mode {
+	const char *name;
+	/* Return true if this mode is supported, otherwise false */
+	bool (*supported)(void);
+	/* Hook when the vm creation is done (before vcpu creation) */
+	void (*create_vm_done)(struct kvm_vm *vm);
+	/* Hook to collect the dirty pages into the bitmap provided */
+	void (*collect_dirty_pages) (struct kvm_vm *vm, int slot,
+				     void *bitmap, uint32_t num_pages);
+} log_modes[LOG_MODE_NUM] = {
+	{
+		.name = "dirty-log",
+		.collect_dirty_pages = dirty_log_collect_dirty_pages,
+	},
+	{
+		.name = "clear-log",
+		.supported = clear_log_supported,
+		.create_vm_done = clear_log_create_vm_done,
+		.collect_dirty_pages = clear_log_collect_dirty_pages,
+	},
+};
+
 /*
  * We use this bitmap to track some pages that should have its dirty
  * bit set in the _next_ iteration.  For example, if we detected the

tools/testing/selftests/kvm/dirty_log_test.c +31 -0

+}
+
+static bool log_mode_supported(void)
+{
+	struct log_mode *mode = &log_modes[host_log_mode];
+
+	if (mode->supported)
+		return mode->supported();
+
+	return true;
+}
+
+static void log_mode_create_vm_done(struct kvm_vm *vm)
+{
+	struct log_mode *mode = &log_modes[host_log_mode];
+
+	if (mode->create_vm_done)
+		mode->create_vm_done(vm);
+}
+
+static void log_mode_collect_dirty_pages(struct kvm_vm *vm, int slot,
+					 void *bitmap, uint32_t num_pages)
+{
+	struct log_mode *mode = &log_modes[host_log_mode];
+
+	TEST_ASSERT(mode->collect_dirty_pages != NULL,
+		    "collect_dirty_pages() is required for any log mode!");
+	mode->collect_dirty_pages(vm, slot, bitmap, num_pages);
+}
+
 static void generate_random_array(uint64_t *guest_array, uint64_t size)
 {
 	uint64_t i;

@@ -257,6 +361,7 @@ static struct kvm_vm *create_vm(enum vm_guest_mode mode, uint32_t vcpuid,
 #ifdef __x86_64__
 	vm_create_irqchip(vm);
 #endif
+	log_mode_create_vm_done(vm);
 	vm_vcpu_add_default(vm, vcpuid, guest_code);
 	return vm;
 }

tools/testing/selftests/kvm/dirty_log_test.c +10 -22

+		return;
+	}
+
 	/*
 	 * We reserve page table for 2 times of extra dirty mem which
 	 * will definitely cover the original (1G+) test range.  Here

@@ -316,14 +427,6 @@ static void run_test(enum vm_guest_mode mode, unsigned long iterations,
 	bmap = bitmap_alloc(host_num_pages);
 	host_bmap_track = bitmap_alloc(host_num_pages);
 
-#ifdef USE_CLEAR_DIRTY_LOG
-	struct kvm_enable_cap cap = {};
-
-	cap.cap = KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2;
-	cap.args[0] = 1;
-	vm_enable_cap(vm, &cap);
-#endif
-
 	/* Add an extra memory slot for testing dirty logging */
 	vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS,
 				    guest_test_phys_mem,

@@ -364,11 +467,8 @@ static void run_test(enum vm_guest_mode mode, unsigned long iterations,
 	while (iteration < iterations) {
 		/* Give the vcpu thread some time to dirty some pages */
 		usleep(interval * 1000);
-		kvm_vm_get_dirty_log(vm, TEST_MEM_SLOT_INDEX, bmap);
-#ifdef USE_CLEAR_DIRTY_LOG
-		kvm_vm_clear_dirty_log(vm, TEST_MEM_SLOT_INDEX, bmap, 0,
-				       host_num_pages);
-#endif
+		log_mode_collect_dirty_pages(vm, TEST_MEM_SLOT_INDEX,
+					     bmap, host_num_pages);
 		vm_dirty_log_verify(bmap);
 		iteration++;
 		sync_global_to_guest(vm, iteration);

@@ -413,6 +513,9 @@ static void help(char *name)
 	       TEST_HOST_LOOP_INTERVAL);
 	printf(" -p: specify guest physical test memory offset\n"
 	       "     Warning: a low offset can conflict with the loaded test code.\n");
+	printf(" -M: specify the host logging mode "
+	       "(default: run all log modes).  Supported modes: \n\t");
+	log_modes_dump();
 	printf(" -m: specify the guest mode ID to test "
 	       "(default: test all supported modes)\n"
 	       "     This option may be used multiple times.\n"

@@ -432,18 +535,11 @@ int main(int argc, char *argv[])
 	bool mode_selected = false;
 	uint64_t phys_offset = 0;
 	unsigned int mode;
-	int opt, i;
+	int opt, i, j;
 #ifdef __aarch64__
 	unsigned int host_ipa_limit;
 #endif
 
-#ifdef USE_CLEAR_DIRTY_LOG
-	if (!kvm_check_cap(KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2)) {
-		fprintf(stderr, "KVM_CLEAR_DIRTY_LOG not available, skipping tests\n");
-		exit(KSFT_SKIP);
-	}
-#endif
-
 #ifdef __x86_64__
 	vm_guest_mode_params_init(VM_MODE_PXXV48_4K, true, true);
 #endif

@@ -463,7 +559,7 @@ int main(int argc, char *argv[])
 	vm_guest_mode_params_init(VM_MODE_P40V48_4K, true, true);
 #endif
 
-	while ((opt = getopt(argc, argv, "hi:I:p:m:")) != -1) {
+	while ((opt = getopt(argc, argv, "hi:I:p:m:M:")) != -1) {
 		switch (opt) {
 		case 'i':
 			iterations = strtol(optarg, NULL, 10);

tools/testing/selftests/kvm/dirty_log_test.c +14 -1

+			}
+			break;
 		case 'h':
 		default:
 			help(argv[0]);

@@ -506,7 +618,18 @@ int main(int argc, char *argv[])
 		TEST_ASSERT(vm_guest_mode_params[i].supported,
 			    "Guest mode ID %d (%s) not supported.",
 			    i, vm_guest_mode_string(i));
-		run_test(i, iterations, interval, phys_offset);
+		if (host_log_mode_option == LOG_MODE_ALL) {
+			/* Run each log mode */
+			for (j = 0; j < LOG_MODE_NUM; j++) {
+				DEBUG("Testing Log Mode '%s'\n",
+				      log_modes[j].name);
+				host_log_mode = j;
+				run_test(i, iterations, interval, phys_offset);
+			}
+		} else {
+			host_log_mode = host_log_mode_option;
+			run_test(i, iterations, interval, phys_offset);
+		}
 	}
 
 	return 0;

Signature · 1 line

2.24.1

Quoted 117 lines · Peter Xu

Remove the clear_dirty_log test, instead merge it into the existing dirty_log_test. It should be cleaner to use this single binary to do both tests, also it's a preparation for the upcoming dirty ring test.

The default behavior will run all the modes in sequence.

Signed-off-by Peter Xu <[email protected]>

---
 tools/testing/selftests/kvm/Makefile          |   2 -
 .../selftests/kvm/clear_dirty_log_test.c      |   2 -
 tools/testing/selftests/kvm/dirty_log_test.c  | 169 +++++++++++++++---
 3 files changed, 146 insertions(+), 27 deletions(-)
 delete mode 100644 tools/testing/selftests/kvm/clear_dirty_log_test.c

tools/testing/selftests/kvm/Makefile +0 -2

index d91c53b726e6..941bfcd48eaa 100644
@@ -27,11 +27,9 @@ TEST_GEN_PROGS_x86_64 += x86_64/vmx_set_nested_state_test
 TEST_GEN_PROGS_x86_64 += x86_64/vmx_tsc_adjust_test
 TEST_GEN_PROGS_x86_64 += x86_64/xss_msr_test
 TEST_GEN_PROGS_x86_64 += x86_64/svm_vmcall_test
-TEST_GEN_PROGS_x86_64 += clear_dirty_log_test
 TEST_GEN_PROGS_x86_64 += dirty_log_test
 TEST_GEN_PROGS_x86_64 += kvm_create_max_vcpus
 
-TEST_GEN_PROGS_aarch64 += clear_dirty_log_test
 TEST_GEN_PROGS_aarch64 += dirty_log_test
 TEST_GEN_PROGS_aarch64 += kvm_create_max_vcpus
 

tools/testing/selftests/kvm/clear_dirty_log_test.c deleted +0 -2

deleted file mode 100644
index 749336937d37..000000000000
@@ -1,2 +0,0 @@
-#define USE_CLEAR_DIRTY_LOG
-#include "dirty_log_test.c"

tools/testing/selftests/kvm/dirty_log_test.c +67 -0

index 3c0ffd34b3b0..642886394e34 100644
@@ -128,6 +128,73 @@ static uint64_t host_dirty_count;
 static uint64_t host_clear_count;
 static uint64_t host_track_next_count;
 
+enum log_mode_t {
+	/* Only use KVM_GET_DIRTY_LOG for logging */
+	LOG_MODE_DIRTY_LOG = 0,
+
+	/* Use both KVM_[GET|CLEAR]_DIRTY_LOG for logging */
+	LOG_MODE_CLEAR_LOG = 1,
+
+	LOG_MODE_NUM,
+
+	/* Run all supported modes */
+	LOG_MODE_ALL = LOG_MODE_NUM,
+};
+
+/* Mode of logging to test.  Default is to run all supported modes */
+static enum log_mode_t host_log_mode_option = LOG_MODE_ALL;
+/* Logging mode for current run */
+static enum log_mode_t host_log_mode;
+
+static bool clear_log_supported(void)
+{
+	return kvm_check_cap(KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2);
+}
+
+static void clear_log_create_vm_done(struct kvm_vm *vm)
+{
+	struct kvm_enable_cap cap = {};
+
+	cap.cap = KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2;
+	cap.args[0] = 1;
+	vm_enable_cap(vm, &cap);
+}
+
+static void dirty_log_collect_dirty_pages(struct kvm_vm *vm, int slot,
+					  void *bitmap, uint32_t num_pages)
+{
+	kvm_vm_get_dirty_log(vm, slot, bitmap);
+}
+
+static void clear_log_collect_dirty_pages(struct kvm_vm *vm, int slot,
+					  void *bitmap, uint32_t num_pages)
+{
+	kvm_vm_get_dirty_log(vm, slot, bitmap);
+	kvm_vm_clear_dirty_log(vm, slot, bitmap, 0, num_pages);
+}
+
+struct log_mode {
+	const char *name;
+	/* Return true if this mode is supported, otherwise false */
+	bool (*supported)(void);
+	/* Hook when the vm creation is done (before vcpu creation) */
+	void (*create_vm_done)(struct kvm_vm *vm);
+	/* Hook to collect the dirty pages into the bitmap provided */
+	void (*collect_dirty_pages) (struct kvm_vm *vm, int slot,
+				     void *bitmap, uint32_t num_pages);
+} log_modes[LOG_MODE_NUM] = {
+	{
+		.name = "dirty-log",
+		.collect_dirty_pages = dirty_log_collect_dirty_pages,
+	},
+	{
+		.name = "clear-log",
+		.supported = clear_log_supported,
+		.create_vm_done = clear_log_create_vm_done,
+		.collect_dirty_pages = clear_log_collect_dirty_pages,
+	},
+};
+
 /*
  * We use this bitmap to track some pages that should have its dirty
  * bit set in the _next_ iteration.  For example, if we detected the

tools/testing/selftests/kvm/dirty_log_test.c static uint64_t host_track_next_count; · line 213

Quoted 10 lines

tools/testing/selftests/kvm/dirty_log_test.c +7 -0

@@ -137,6 +204,43 @@ static uint64_t host_track_next_count;
  */
 static unsigned long *host_bmap_track;
 
+static void log_modes_dump(void)
+{
+	int i;
+
+	for (i = 0; i < LOG_MODE_NUM; i++)
+		printf("%s, ", log_modes[i].name);
+	puts("\b\b  \b\b");

This will be ugly when the output is redirected to a file. How about just

printf("%s", log_modes[0].name);
for (i = 1; i < LOG_MODE_NUM; i++)
printf(", %s", log_modes[i].name);
printf("\n");

Quoted 46 lines

tools/testing/selftests/kvm/dirty_log_test.c +31 -0

+}
+
+static bool log_mode_supported(void)
+{
+	struct log_mode *mode = &log_modes[host_log_mode];
+
+	if (mode->supported)
+		return mode->supported();
+
+	return true;
+}
+
+static void log_mode_create_vm_done(struct kvm_vm *vm)
+{
+	struct log_mode *mode = &log_modes[host_log_mode];
+
+	if (mode->create_vm_done)
+		mode->create_vm_done(vm);
+}
+
+static void log_mode_collect_dirty_pages(struct kvm_vm *vm, int slot,
+					 void *bitmap, uint32_t num_pages)
+{
+	struct log_mode *mode = &log_modes[host_log_mode];
+
+	TEST_ASSERT(mode->collect_dirty_pages != NULL,
+		    "collect_dirty_pages() is required for any log mode!");
+	mode->collect_dirty_pages(vm, slot, bitmap, num_pages);
+}
+
 static void generate_random_array(uint64_t *guest_array, uint64_t size)
 {
 	uint64_t i;

@@ -257,6 +361,7 @@ static struct kvm_vm *create_vm(enum vm_guest_mode mode, uint32_t vcpuid,
 #ifdef __x86_64__
 	vm_create_irqchip(vm);
 #endif
+	log_mode_create_vm_done(vm);
 	vm_vcpu_add_default(vm, vcpuid, guest_code);
 	return vm;
 }

@@ -271,6 +376,12 @@ static void run_test(enum vm_guest_mode mode, unsigned long iterations,
 	struct kvm_vm *vm;
 	unsigned long *bmap;
 

tools/testing/selftests/kvm/dirty_log_test.c static void run_test(enum vm_guest_mode mode, unsigned long iterations, · line 381

Quoted 3 lines

tools/testing/selftests/kvm/dirty_log_test.c +3 -0

+	if (!log_mode_supported()) {
+		fprintf(stderr, "Log mode '%s' not supported, skip\n",
+			log_modes[host_log_mode].name);

I think kvm selftests needs a skip_test() function that outputs a more consistent test skip message. It seems we mostly do

fprintf(stderr, "%s, skipping test\n", custom_message);

but here we have ', skip'. Also, I see a few places were we output skipping to stderr and others to stdout. I think I like stdout better.

Quoted 76 lines

tools/testing/selftests/kvm/dirty_log_test.c +10 -22

+		return;
+	}
+
 	/*
 	 * We reserve page table for 2 times of extra dirty mem which
 	 * will definitely cover the original (1G+) test range.  Here

@@ -316,14 +427,6 @@ static void run_test(enum vm_guest_mode mode, unsigned long iterations,
 	bmap = bitmap_alloc(host_num_pages);
 	host_bmap_track = bitmap_alloc(host_num_pages);
 
-#ifdef USE_CLEAR_DIRTY_LOG
-	struct kvm_enable_cap cap = {};
-
-	cap.cap = KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2;
-	cap.args[0] = 1;
-	vm_enable_cap(vm, &cap);
-#endif
-
 	/* Add an extra memory slot for testing dirty logging */
 	vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS,
 				    guest_test_phys_mem,

@@ -364,11 +467,8 @@ static void run_test(enum vm_guest_mode mode, unsigned long iterations,
 	while (iteration < iterations) {
 		/* Give the vcpu thread some time to dirty some pages */
 		usleep(interval * 1000);
-		kvm_vm_get_dirty_log(vm, TEST_MEM_SLOT_INDEX, bmap);
-#ifdef USE_CLEAR_DIRTY_LOG
-		kvm_vm_clear_dirty_log(vm, TEST_MEM_SLOT_INDEX, bmap, 0,
-				       host_num_pages);
-#endif
+		log_mode_collect_dirty_pages(vm, TEST_MEM_SLOT_INDEX,
+					     bmap, host_num_pages);
 		vm_dirty_log_verify(bmap);
 		iteration++;
 		sync_global_to_guest(vm, iteration);

@@ -413,6 +513,9 @@ static void help(char *name)
 	       TEST_HOST_LOOP_INTERVAL);
 	printf(" -p: specify guest physical test memory offset\n"
 	       "     Warning: a low offset can conflict with the loaded test code.\n");
+	printf(" -M: specify the host logging mode "
+	       "(default: run all log modes).  Supported modes: \n\t");
+	log_modes_dump();
 	printf(" -m: specify the guest mode ID to test "
 	       "(default: test all supported modes)\n"
 	       "     This option may be used multiple times.\n"

@@ -432,18 +535,11 @@ int main(int argc, char *argv[])
 	bool mode_selected = false;
 	uint64_t phys_offset = 0;
 	unsigned int mode;
-	int opt, i;
+	int opt, i, j;
 #ifdef __aarch64__
 	unsigned int host_ipa_limit;
 #endif
 
-#ifdef USE_CLEAR_DIRTY_LOG
-	if (!kvm_check_cap(KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2)) {
-		fprintf(stderr, "KVM_CLEAR_DIRTY_LOG not available, skipping tests\n");
-		exit(KSFT_SKIP);
-	}
-#endif
-
 #ifdef __x86_64__
 	vm_guest_mode_params_init(VM_MODE_PXXV48_4K, true, true);
 #endif

@@ -463,7 +559,7 @@ int main(int argc, char *argv[])
 	vm_guest_mode_params_init(VM_MODE_P40V48_4K, true, true);
 #endif
 
-	while ((opt = getopt(argc, argv, "hi:I:p:m:")) != -1) {
+	while ((opt = getopt(argc, argv, "hi:I:p:m:M:")) != -1) {
 		switch (opt) {
 		case 'i':
 			iterations = strtol(optarg, NULL, 10);

tools/testing/selftests/kvm/dirty_log_test.c int main(int argc, char *argv[]) · line 584

Quoted 4 lines

tools/testing/selftests/kvm/dirty_log_test.c +1 -0

@@ -485,6 +581,22 @@ int main(int argc, char *argv[])
 				    "Guest mode ID %d too big", mode);
 			vm_guest_mode_params[mode].enabled = true;
 			break;
+		case 'M':

Can also add

if (!strcmp(optarg, "all"))
host_log_mode_option = LOG_MODE_ALL;

tools/testing/selftests/kvm/dirty_log_test.c int main(int argc, char *argv[]) · line 588

Quoted 5 lines

tools/testing/selftests/kvm/dirty_log_test.c +4 -0

+			for (i = 0; i < LOG_MODE_NUM; i++) {
+				if (!strcmp(optarg, log_modes[i].name)) {
+					DEBUG("Setting log mode to: '%s'\n",
+					      optarg);

Basing this on kvm/queue won't work as DEBUG() no longer exists. This looks like a pr_info().

tools/testing/selftests/kvm/dirty_log_test.c int main(int argc, char *argv[]) · line 597

Quoted 10 lines

tools/testing/selftests/kvm/dirty_log_test.c +9 -0

+					host_log_mode_option = i;
+					break;
+				}
+			}
+			if (i == LOG_MODE_NUM) {
+				printf("Log mode '%s' is invalid.  "
+				       "Please choose from: ", optarg);
+				log_modes_dump();
+				exit(-1);

Exit code of 255? Probably just want exit(1);

Quoted 7 lines

tools/testing/selftests/kvm/dirty_log_test.c static void run_test(enum vm_guest_mode mode, unsigned long iterations, · line 381

Quoted 4 lines

tools/testing/selftests/kvm/dirty_log_test.c +3 -0

+	if (!log_mode_supported()) {
+		fprintf(stderr, "Log mode '%s' not supported, skip\n",
+			log_modes[host_log_mode].name);

I think kvm selftests needs a skip_test() function that outputs a more consistent test skip message. It seems we mostly do

Yep, I can introduce one.

tools/testing/selftests/kvm/lib/kvm_util.c +17 -0

+void *vcpu_map_dirty_ring(struct kvm_vm *vm, uint32_t vcpuid)
+{
+	struct vcpu *vcpu;
+	uint32_t size = vm->dirty_ring_size;
+
+	TEST_ASSERT(size > 0, "Should enable dirty ring first");
+
+	vcpu = vcpu_find(vm, vcpuid);
+
+	TEST_ASSERT(vcpu, "Cannot find vcpu %u", vcpuid);
+
+	if (!vcpu->dirty_gfns) {
+		void *addr;
+
+		addr = mmap(NULL, size, PROT_READ,
+			    MAP_PRIVATE, vcpu->fd,
+			    vm->page_size * KVM_DIRTY_LOG_PAGE_OFFSET);

tools/testing/selftests/kvm/lib/kvm_util.c +7 -0

+
+		vcpu->dirty_gfns = addr;
+		vcpu->dirty_gfns_count = size / sizeof(struct kvm_dirty_gfn);
+	}
+
+	return vcpu->dirty_gfns;
+}

Quoted 21 lines · Peter Xu

tools/testing/selftests/kvm/lib/kvm_util.c +21 -0

+void *vcpu_map_dirty_ring(struct kvm_vm *vm, uint32_t vcpuid)
+{
+	struct vcpu *vcpu;
+	uint32_t size = vm->dirty_ring_size;
+
+	TEST_ASSERT(size > 0, "Should enable dirty ring first");
+
+	vcpu = vcpu_find(vm, vcpuid);
+
+	TEST_ASSERT(vcpu, "Cannot find vcpu %u", vcpuid);
+
+	if (!vcpu->dirty_gfns) {
+		void *addr;
+
+		addr = mmap(NULL, size, PROT_READ,
+			    MAP_PRIVATE, vcpu->fd,
+			    vm->page_size * KVM_DIRTY_LOG_PAGE_OFFSET);
+		TEST_ASSERT(addr == MAP_FAILED, "Dirty ring mapped private");
+
+		addr = mmap(NULL, size, PROT_READ | PROT_EXEC,
+			    MAP_PRIVATE, vcpu->fd,

tools/testing/selftests/kvm/lib/kvm_util.c int _vcpu_ioctl(struct kvm_vm *vm, uint32_t vcpuid, · line 1496

Quoted 6 lines

tools/testing/selftests/kvm/lib/kvm_util.c +6 -0

+			    vm->page_size * KVM_DIRTY_LOG_PAGE_OFFSET);
+		TEST_ASSERT(addr == MAP_FAILED, "Dirty ring mapped exec");
+
+		addr = mmap(NULL, size, PROT_READ | PROT_WRITE,
+			    MAP_SHARED, vcpu->fd,
+			    vm->page_size * KVM_DIRTY_LOG_PAGE_OFFSET);

No TEST_ASSERT for this mmap?

Quoted 8 lines

tools/testing/selftests/kvm/lib/kvm_util.c +7 -0

+
+		vcpu->dirty_gfns = addr;
+		vcpu->dirty_gfns_count = size / sizeof(struct kvm_dirty_gfn);
+	}
+
+	return vcpu->dirty_gfns;
+}

Previously the dirty ring test was working in synchronous way, because only with a vmexit (with that it was the ring full event) we'll know the hardware dirty bits will be flushed to the dirty ring.

With this patch we first introduced the vcpu kick mechanism by using SIGUSR1, meanwhile we can have a guarantee of vmexit and also the flushing of hardware dirty bits. With all these, we can keep the vcpu dirty work asynchronous of the whole collection procedure now. Still, we need to be very careful that we can only do it async if the vcpu is not reaching soft limit (no KVM_EXIT_DIRTY_RING_FULL). Otherwise we must collect the dirty bits before continuing the vcpu.

Further increase the dirty ring size to current maximum to make sure we torture more on the no-ring-full case, which should be the major scenario when the hypervisors like QEMU would like to use this feature.

Signed-off-by Peter Xu <[email protected]>

---
 tools/testing/selftests/kvm/dirty_log_test.c  | 126 +++++++++++++-----
 .../testing/selftests/kvm/include/kvm_util.h  |   1 +
 tools/testing/selftests/kvm/lib/kvm_util.c    |   9 ++
 3 files changed, 106 insertions(+), 30 deletions(-)

tools/testing/selftests/kvm/dirty_log_test.c +12 -1

index 134637267af4..b07e52858e87 100644
@@ -13,6 +13,9 @@
 #include <time.h>
 #include <pthread.h>
 #include <semaphore.h>
+#include <sys/types.h>
+#include <signal.h>
+#include <errno.h>
 #include <linux/bitmap.h>
 #include <linux/bitops.h>
 #include <asm/barrier.h>

@@ -59,7 +62,9 @@
 # define test_and_clear_bit_le	test_and_clear_bit
 #endif
 
-#define TEST_DIRTY_RING_COUNT		1024
+#define TEST_DIRTY_RING_COUNT		65536
+
+#define SIG_IPI SIGUSR1
 
 /*
  * Guest/Host shared variables. Ensure addr_gva2hva() and/or

@@ -135,6 +140,12 @@ static uint64_t host_track_next_count;
 /* Whether dirty ring reset is requested, or finished */
 static sem_t dirty_ring_vcpu_stop;
 static sem_t dirty_ring_vcpu_cont;
+/*
+ * This is updated by the vcpu thread to tell the host whether it's a
+ * ring-full event.  It should only be read until a sem_wait() of
+ * dirty_ring_vcpu_stop and before vcpu continues to run.
+ */
+static bool dirty_ring_vcpu_ring_full;
 
 enum log_mode_t {
 	/* Only use KVM_GET_DIRTY_LOG for logging */

Quoted 4 lines

enum log_mode_t {

tools/testing/selftests/kvm/dirty_log_test.c +0 -0

 	/* Only use KVM_GET_DIRTY_LOG for logging */

@@ -156,6 +167,33 @@ enum log_mode_t {
 static enum log_mode_t host_log_mode_option = LOG_MODE_ALL;

tools/testing/selftests/kvm/dirty_log_test.c enum log_mode_t { · line 173

Quoted 6 lines

tools/testing/selftests/kvm/dirty_log_test.c +4 -0

 /* Logging mode for current run */
 static enum log_mode_t host_log_mode;
+pthread_t vcpu_thread;
+
+/* Only way to pass this to the signal handler */
+struct kvm_vm *current_vm;

nit: above two new globals could be static