Add AMD SEV guest live migration support

From: Brijesh Singh <[email protected]>

The command is used for encrypting the guest memory region using the encryption context created with KVM_SEV_SEND_START.

Cc Thomas Gleixner <[email protected]>

Cc Ingo Molnar <[email protected]>

Cc "H. Peter Anvin" <[email protected]>

Cc Paolo Bonzini <[email protected]>

Cc "Radim Krčmář" <[email protected]>

Cc Joerg Roedel <[email protected]>

Cc Borislav Petkov <[email protected]>

Cc Tom Lendacky <[email protected]>

Cc [email protected]

Cc [email protected]

Cc [email protected]

Reviewed-by Steve Rutherford <[email protected]>

Signed-off-by Brijesh Singh <[email protected]>

Signed-off-by Ashish Kalra <[email protected]>

---
 .../virt/kvm/amd-memory-encryption.rst        |  24 ++++
 arch/x86/kvm/svm/sev.c                        | 135 +++++++++++++++++-
 include/uapi/linux/kvm.h                      |   9 ++
 3 files changed, 164 insertions(+), 4 deletions(-)

Documentation/virt/kvm/amd-memory-encryption.rst +24 -0

index 59cb59bd4675..d0dfa5b54e4f 100644
@@ -290,6 +290,30 @@ Returns: 0 on success, -negative on error
                 __u32 session_len;
         };
 
+11. KVM_SEV_SEND_UPDATE_DATA
+----------------------------
+
+The KVM_SEV_SEND_UPDATE_DATA command can be used by the hypervisor to encrypt the
+outgoing guest memory region with the encryption context creating using
+KVM_SEV_SEND_START.
+
+Parameters (in): struct kvm_sev_send_update_data
+
+Returns: 0 on success, -negative on error
+
+::
+
+        struct kvm_sev_launch_send_update_data {
+                __u64 hdr_uaddr;        /* userspace address containing the packet header */
+                __u32 hdr_len;
+
+                __u64 guest_uaddr;      /* the source memory region to be encrypted */
+                __u32 guest_len;
+
+                __u64 trans_uaddr;      /* the destition memory region  */
+                __u32 trans_len;
+        };
+
 References
 ==========
 

arch/x86/kvm/svm/sev.c +131 -4

index 5a15b43b4349..7031b660f64d 100644
@@ -23,6 +23,7 @@ static DECLARE_RWSEM(sev_deactivate_lock);
 static DEFINE_MUTEX(sev_bitmap_lock);
 unsigned int max_sev_asid;
 static unsigned int min_sev_asid;
+static unsigned long sev_me_mask;
 static unsigned long *sev_asid_bitmap;
 static unsigned long *sev_reclaim_asid_bitmap;
 #define __sme_page_pa(x) __sme_set(page_to_pfn(x) << PAGE_SHIFT)

@@ -1035,6 +1036,123 @@ static int sev_send_start(struct kvm *kvm, struct kvm_sev_cmd *argp)
 	return ret;
 }
 
+/* Userspace wants to query either header or trans length. */
+static int
+__sev_send_update_data_query_lengths(struct kvm *kvm, struct kvm_sev_cmd *argp,
+				     struct kvm_sev_send_update_data *params)
+{
+	struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
+	struct sev_data_send_update_data *data;
+	int ret;
+
+	data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
+	if (!data)
+		return -ENOMEM;
+
+	data->handle = sev->handle;
+	ret = sev_issue_cmd(kvm, SEV_CMD_SEND_UPDATE_DATA, data, &argp->error);
+
+	params->hdr_len = data->hdr_len;
+	params->trans_len = data->trans_len;
+
+	if (copy_to_user((void __user *)(uintptr_t)argp->data, params,
+			 sizeof(struct kvm_sev_send_update_data)))
+		ret = -EFAULT;
+
+	kfree(data);
+	return ret;
+}
+
+static int sev_send_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
+{
+	struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
+	struct sev_data_send_update_data *data;
+	struct kvm_sev_send_update_data params;
+	void *hdr, *trans_data;
+	struct page **guest_page;
+	unsigned long n;
+	int ret, offset;
+
+	if (!sev_guest(kvm))
+		return -ENOTTY;
+
+	if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data,
+			sizeof(struct kvm_sev_send_update_data)))
+		return -EFAULT;
+
+	/* userspace wants to query either header or trans length */
+	if (!params.trans_len || !params.hdr_len)
+		return __sev_send_update_data_query_lengths(kvm, argp, &params);
+
+	if (!params.trans_uaddr || !params.guest_uaddr ||
+	    !params.guest_len || !params.hdr_uaddr)
+		return -EINVAL;
+
+	/* Check if we are crossing the page boundary */
+	offset = params.guest_uaddr & (PAGE_SIZE - 1);
+	if ((params.guest_len + offset > PAGE_SIZE))
+		return -EINVAL;
+
+	/* Pin guest memory */
+	guest_page = sev_pin_memory(kvm, params.guest_uaddr & PAGE_MASK,
+				    PAGE_SIZE, &n, 0);
+	if (!guest_page)
+		return -EFAULT;
+
+	/* allocate memory for header and transport buffer */
+	ret = -ENOMEM;
+	hdr = kmalloc(params.hdr_len, GFP_KERNEL_ACCOUNT);
+	if (!hdr)
+		goto e_unpin;
+
+	trans_data = kmalloc(params.trans_len, GFP_KERNEL_ACCOUNT);
+	if (!trans_data)
+		goto e_free_hdr;
+
+	data = kzalloc(sizeof(*data), GFP_KERNEL);
+	if (!data)
+		goto e_free_trans_data;
+
+	data->hdr_address = __psp_pa(hdr);
+	data->hdr_len = params.hdr_len;
+	data->trans_address = __psp_pa(trans_data);
+	data->trans_len = params.trans_len;
+
+	/* The SEND_UPDATE_DATA command requires C-bit to be always set. */
+	data->guest_address = (page_to_pfn(guest_page[0]) << PAGE_SHIFT) +
+				offset;
+	data->guest_address |= sev_me_mask;
+	data->guest_len = params.guest_len;
+	data->handle = sev->handle;
+
+	ret = sev_issue_cmd(kvm, SEV_CMD_SEND_UPDATE_DATA, data, &argp->error);
+
+	if (ret)
+		goto e_free;
+
+	/* copy transport buffer to user space */
+	if (copy_to_user((void __user *)(uintptr_t)params.trans_uaddr,
+			 trans_data, params.trans_len)) {
+		ret = -EFAULT;
+		goto e_free;
+	}
+
+	/* Copy packet header to userspace. */
+	ret = copy_to_user((void __user *)(uintptr_t)params.hdr_uaddr, hdr,
+				params.hdr_len);
+
+e_free:
+	kfree(data);
+e_free_trans_data:
+	kfree(trans_data);
+e_free_hdr:
+	kfree(hdr);
+e_unpin:
+	sev_unpin_memory(kvm, guest_page, n);
+
+	return ret;
+}
+
 int svm_mem_enc_op(struct kvm *kvm, void __user *argp)
 {
 	struct kvm_sev_cmd sev_cmd;

@@ -1082,6 +1200,9 @@ int svm_mem_enc_op(struct kvm *kvm, void __user *argp)
 	case KVM_SEV_SEND_START:
 		r = sev_send_start(kvm, &sev_cmd);
 		break;
+	case KVM_SEV_SEND_UPDATE_DATA:
+		r = sev_send_update_data(kvm, &sev_cmd);
+		break;
 	default:
 		r = -EINVAL;
 		goto out;

@@ -1238,16 +1359,22 @@ void sev_vm_destroy(struct kvm *kvm)
 int __init sev_hardware_setup(void)
 {
 	struct sev_user_data_status *status;
+	u32 eax, ebx;
 	int rc;
 
-	/* Maximum number of encrypted guests supported simultaneously */
-	max_sev_asid = cpuid_ecx(0x8000001F);
+	/*
+	 * Query the memory encryption information.
+	 *  EBX:  Bit 0:5 Pagetable bit position used to indicate encryption
+	 *  (aka Cbit).
+	 *  ECX:  Maximum number of encrypted guests supported simultaneously.
+	 *  EDX:  Minimum ASID value that should be used for SEV guest.
+	 */
+	cpuid(0x8000001f, &eax, &ebx, &max_sev_asid, &min_sev_asid);
 
 	if (!svm_sev_enabled())
 		return 1;
 
-	/* Minimum ASID value that should be used for SEV guest */
-	min_sev_asid = cpuid_edx(0x8000001F);
+	sev_me_mask = 1UL << (ebx & 0x3f);
 
 	/* Initialize SEV ASID bitmaps */
 	sev_asid_bitmap = bitmap_zalloc(max_sev_asid, GFP_KERNEL);

include/uapi/linux/kvm.h +9 -0

index 8827d43e2684..7aaed8ee33cf 100644
@@ -1610,6 +1610,15 @@ struct kvm_sev_send_start {
 	__u32 session_len;
 };
 
+struct kvm_sev_send_update_data {
+	__u64 hdr_uaddr;
+	__u32 hdr_len;
+	__u64 guest_uaddr;
+	__u32 guest_len;
+	__u64 trans_uaddr;
+	__u32 trans_len;
+};
+
 #define KVM_DEV_ASSIGN_ENABLE_IOMMU	(1 << 0)
 #define KVM_DEV_ASSIGN_PCI_2_3		(1 << 1)
 #define KVM_DEV_ASSIGN_MASK_INTX	(1 << 2)

Signature · 1 line

2.17.1

From: Brijesh Singh <[email protected]>

The command is used to finailize the encryption context created with KVM_SEV_SEND_START command.

Cc Thomas Gleixner <[email protected]>

Cc Ingo Molnar <[email protected]>

Cc "H. Peter Anvin" <[email protected]>

Cc Paolo Bonzini <[email protected]>

Cc "Radim Krčmář" <[email protected]>

Cc Joerg Roedel <[email protected]>

Cc Borislav Petkov <[email protected]>

Cc Tom Lendacky <[email protected]>

Cc [email protected]

Cc [email protected]

Cc [email protected]

Reviewed-by Steve Rutherford <[email protected]>

Signed-off-by Brijesh Singh <[email protected]>

Signed-off-by Ashish Kalra <[email protected]>

---
 .../virt/kvm/amd-memory-encryption.rst        |  8 +++++++
 arch/x86/kvm/svm/sev.c                        | 23 +++++++++++++++++++
 2 files changed, 31 insertions(+)

Documentation/virt/kvm/amd-memory-encryption.rst +8 -0

index d0dfa5b54e4f..93884ec8918e 100644
@@ -314,6 +314,14 @@ Returns: 0 on success, -negative on error
                 __u32 trans_len;
         };
 
+12. KVM_SEV_SEND_FINISH
+------------------------
+
+After completion of the migration flow, the KVM_SEV_SEND_FINISH command can be
+issued by the hypervisor to delete the encryption context.
+
+Returns: 0 on success, -negative on error
+
 References
 ==========
 

arch/x86/kvm/svm/sev.c +23 -0

index 7031b660f64d..4d3031c9fdcf 100644
@@ -1153,6 +1153,26 @@ static int sev_send_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
 	return ret;
 }
 
+static int sev_send_finish(struct kvm *kvm, struct kvm_sev_cmd *argp)
+{
+	struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
+	struct sev_data_send_finish *data;
+	int ret;
+
+	if (!sev_guest(kvm))
+		return -ENOTTY;
+
+	data = kzalloc(sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	data->handle = sev->handle;
+	ret = sev_issue_cmd(kvm, SEV_CMD_SEND_FINISH, data, &argp->error);
+
+	kfree(data);
+	return ret;
+}
+
 int svm_mem_enc_op(struct kvm *kvm, void __user *argp)
 {
 	struct kvm_sev_cmd sev_cmd;

@@ -1203,6 +1223,9 @@ int svm_mem_enc_op(struct kvm *kvm, void __user *argp)
 	case KVM_SEV_SEND_UPDATE_DATA:
 		r = sev_send_update_data(kvm, &sev_cmd);
 		break;
+	case KVM_SEV_SEND_FINISH:
+		r = sev_send_finish(kvm, &sev_cmd);
+		break;
 	default:
 		r = -EINVAL;
 		goto out;

Signature · 1 line

2.17.1

From: Brijesh Singh <[email protected]>

The command is used to create the encryption context for an incoming SEV guest. The encryption context can be later used by the hypervisor to import the incoming data into the SEV guest memory space.

Cc Thomas Gleixner <[email protected]>

Cc Ingo Molnar <[email protected]>

Cc "H. Peter Anvin" <[email protected]>

Cc Paolo Bonzini <[email protected]>

Cc "Radim Krčmář" <[email protected]>

Cc Joerg Roedel <[email protected]>

Cc Borislav Petkov <[email protected]>

Cc Tom Lendacky <[email protected]>

Cc [email protected]

Cc [email protected]

Cc [email protected]

Reviewed-by Steve Rutherford <[email protected]>

Signed-off-by Brijesh Singh <[email protected]>

Signed-off-by Ashish Kalra <[email protected]>

---
 .../virt/kvm/amd-memory-encryption.rst        | 29 +++++++
 arch/x86/kvm/svm/sev.c                        | 81 +++++++++++++++++++
 include/uapi/linux/kvm.h                      |  9 +++
 3 files changed, 119 insertions(+)

Documentation/virt/kvm/amd-memory-encryption.rst +29 -0

index 93884ec8918e..337bf6a8a3ee 100644
@@ -322,6 +322,35 @@ issued by the hypervisor to delete the encryption context.
 
 Returns: 0 on success, -negative on error
 
+13. KVM_SEV_RECEIVE_START
+------------------------
+
+The KVM_SEV_RECEIVE_START command is used for creating the memory encryption
+context for an incoming SEV guest. To create the encryption context, the user must
+provide a guest policy, the platform public Diffie-Hellman (PDH) key and session
+information.
+
+Parameters: struct  kvm_sev_receive_start (in/out)
+
+Returns: 0 on success, -negative on error
+
+::
+
+        struct kvm_sev_receive_start {
+                __u32 handle;           /* if zero then firmware creates a new handle */
+                __u32 policy;           /* guest's policy */
+
+                __u64 pdh_uaddr;        /* userspace address pointing to the PDH key */
+                __u32 pdh_len;
+
+                __u64 session_uaddr;    /* userspace address which points to the guest session information */
+                __u32 session_len;
+        };
+
+On success, the 'handle' field contains a new handle and on error, a negative value.
+
+For more details, see SEV spec Section 6.12.
+
 References
 ==========
 

arch/x86/kvm/svm/sev.c +81 -0

index 4d3031c9fdcf..b575aa8e27af 100644
@@ -1173,6 +1173,84 @@ static int sev_send_finish(struct kvm *kvm, struct kvm_sev_cmd *argp)
 	return ret;
 }
 
+static int sev_receive_start(struct kvm *kvm, struct kvm_sev_cmd *argp)
+{
+	struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
+	struct sev_data_receive_start *start;
+	struct kvm_sev_receive_start params;
+	int *error = &argp->error;
+	void *session_data;
+	void *pdh_data;
+	int ret;
+
+	if (!sev_guest(kvm))
+		return -ENOTTY;
+
+	/* Get parameter from the userspace */
+	if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data,
+			sizeof(struct kvm_sev_receive_start)))
+		return -EFAULT;
+
+	/* some sanity checks */
+	if (!params.pdh_uaddr || !params.pdh_len ||
+	    !params.session_uaddr || !params.session_len)
+		return -EINVAL;
+
+	pdh_data = psp_copy_user_blob(params.pdh_uaddr, params.pdh_len);
+	if (IS_ERR(pdh_data))
+		return PTR_ERR(pdh_data);
+
+	session_data = psp_copy_user_blob(params.session_uaddr,
+			params.session_len);
+	if (IS_ERR(session_data)) {
+		ret = PTR_ERR(session_data);
+		goto e_free_pdh;
+	}
+
+	ret = -ENOMEM;
+	start = kzalloc(sizeof(*start), GFP_KERNEL);
+	if (!start)
+		goto e_free_session;
+
+	start->handle = params.handle;
+	start->policy = params.policy;
+	start->pdh_cert_address = __psp_pa(pdh_data);
+	start->pdh_cert_len = params.pdh_len;
+	start->session_address = __psp_pa(session_data);
+	start->session_len = params.session_len;
+
+	/* create memory encryption context */
+	ret = __sev_issue_cmd(argp->sev_fd, SEV_CMD_RECEIVE_START, start,
+				error);
+	if (ret)
+		goto e_free;
+
+	/* Bind ASID to this guest */
+	ret = sev_bind_asid(kvm, start->handle, error);
+	if (ret)
+		goto e_free;
+
+	params.handle = start->handle;
+	if (copy_to_user((void __user *)(uintptr_t)argp->data,
+			 &params, sizeof(struct kvm_sev_receive_start))) {
+		ret = -EFAULT;
+		sev_unbind_asid(kvm, start->handle);
+		goto e_free;
+	}
+
+	sev->handle = start->handle;
+	sev->fd = argp->sev_fd;
+
+e_free:
+	kfree(start);
+e_free_session:
+	kfree(session_data);
+e_free_pdh:
+	kfree(pdh_data);
+
+	return ret;
+}
+
 int svm_mem_enc_op(struct kvm *kvm, void __user *argp)
 {
 	struct kvm_sev_cmd sev_cmd;

@@ -1226,6 +1304,9 @@ int svm_mem_enc_op(struct kvm *kvm, void __user *argp)
 	case KVM_SEV_SEND_FINISH:
 		r = sev_send_finish(kvm, &sev_cmd);
 		break;
+	case KVM_SEV_RECEIVE_START:
+		r = sev_receive_start(kvm, &sev_cmd);
+		break;
 	default:
 		r = -EINVAL;
 		goto out;

include/uapi/linux/kvm.h +9 -0

index 7aaed8ee33cf..24ac57151d53 100644
@@ -1619,6 +1619,15 @@ struct kvm_sev_send_update_data {
 	__u32 trans_len;
 };
 
+struct kvm_sev_receive_start {
+	__u32 handle;
+	__u32 policy;
+	__u64 pdh_uaddr;
+	__u32 pdh_len;
+	__u64 session_uaddr;
+	__u32 session_len;
+};
+
 #define KVM_DEV_ASSIGN_ENABLE_IOMMU	(1 << 0)
 #define KVM_DEV_ASSIGN_PCI_2_3		(1 << 1)
 #define KVM_DEV_ASSIGN_MASK_INTX	(1 << 2)

Signature · 1 line

2.17.1

From: Brijesh Singh <[email protected]>

This hypercall is used by the SEV guest to notify a change in the page encryption status to the hypervisor. The hypercall should be invoked only when the encryption attribute is changed from encrypted -> decrypted and vice versa. By default all guest pages are considered encrypted.

Cc Thomas Gleixner <[email protected]>

Cc Ingo Molnar <[email protected]>

Cc "H. Peter Anvin" <[email protected]>

Cc Paolo Bonzini <[email protected]>

Cc "Radim Krčmář" <[email protected]>

Cc Joerg Roedel <[email protected]>

Cc Borislav Petkov <[email protected]>

Cc Tom Lendacky <[email protected]>

Cc [email protected]

Cc [email protected]

Cc [email protected]

Reviewed-by Venu Busireddy <[email protected]>

Signed-off-by Brijesh Singh <[email protected]>

Signed-off-by Ashish Kalra <[email protected]>

---
 Documentation/virt/kvm/hypercalls.rst | 15 +++++
 arch/x86/include/asm/kvm_host.h       |  2 +
 arch/x86/kvm/svm/sev.c                | 90 +++++++++++++++++++++++++++
 arch/x86/kvm/svm/svm.c                |  2 +
 arch/x86/kvm/svm/svm.h                |  4 ++
 arch/x86/kvm/vmx/vmx.c                |  1 +
 arch/x86/kvm/x86.c                    |  6 ++
 include/uapi/linux/kvm_para.h         |  1 +
 8 files changed, 121 insertions(+)

Documentation/virt/kvm/hypercalls.rst +15 -0

index dbaf207e560d..ff5287e68e81 100644
@@ -169,3 +169,18 @@ a0: destination APIC ID
 
 :Usage example: When sending a call-function IPI-many to vCPUs, yield if
                any of the IPI target vCPUs was preempted.
+
+
+8. KVM_HC_PAGE_ENC_STATUS
+-------------------------
+:Architecture: x86
+:Status: active
+:Purpose: Notify the encryption status changes in guest page table (SEV guest)
+
+a0: the guest physical address of the start page
+a1: the number of pages
+a2: encryption attribute
+
+   Where:
+       * 1: Encryption attribute is set
+       * 0: Encryption attribute is cleared

arch/x86/include/asm/kvm_host.h +2 -0

index 42a2d0d3984a..4a8ee22f4f5b 100644
@@ -1254,6 +1254,8 @@ struct kvm_x86_ops {
 
        bool (*apic_init_signal_blocked)(struct kvm_vcpu *vcpu);
        int (*enable_direct_tlbflush)(struct kvm_vcpu *vcpu);
+       int (*page_enc_status_hc)(struct kvm *kvm, unsigned long gpa,
+                                 unsigned long sz, unsigned long mode);
 };
 
 struct kvm_x86_init_ops {

arch/x86/kvm/svm/sev.c +90 -0

index 698704defbcd..f088467708f0 100644
@@ -1347,6 +1347,93 @@ static int sev_receive_finish(struct kvm *kvm, struct kvm_sev_cmd *argp)
        return ret;
 }
 
+static int sev_resize_page_enc_bitmap(struct kvm *kvm, unsigned long new_size)
+{
+       struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
+       unsigned long *map;
+       unsigned long sz;
+
+       if (sev->page_enc_bmap_size >= new_size)
+               return 0;
+
+       sz = ALIGN(new_size, BITS_PER_LONG) / 8;
+
+       map = vmalloc(sz);
+       if (!map) {
+               pr_err_once("Failed to allocate encrypted bitmap size %lx\n",
+                               sz);
+               return -ENOMEM;
+       }
+
+       /* mark the page encrypted (by default) */
+       memset(map, 0xff, sz);
+
+       bitmap_copy(map, sev->page_enc_bmap, sev->page_enc_bmap_size);
+       kvfree(sev->page_enc_bmap);
+
+       sev->page_enc_bmap = map;
+       sev->page_enc_bmap_size = new_size;
+
+       return 0;
+}
+
+int svm_page_enc_status_hc(struct kvm *kvm, unsigned long gpa,
+                                 unsigned long npages, unsigned long enc)
+{
+       struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
+       kvm_pfn_t pfn_start, pfn_end;
+       gfn_t gfn_start, gfn_end;
+
+       if (!sev_guest(kvm))
+               return -EINVAL;
+
+       if (!npages)
+               return 0;
+
+       gfn_start = gpa_to_gfn(gpa);
+       gfn_end = gfn_start + npages;
+
+       /* out of bound access error check */
+       if (gfn_end <= gfn_start)
+               return -EINVAL;
+
+       /* lets make sure that gpa exist in our memslot */
+       pfn_start = gfn_to_pfn(kvm, gfn_start);
+       pfn_end = gfn_to_pfn(kvm, gfn_end);
+
+       if (is_error_noslot_pfn(pfn_start) && !is_noslot_pfn(pfn_start)) {
+               /*
+                * Allow guest MMIO range(s) to be added
+                * to the page encryption bitmap.
+                */
+               return -EINVAL;
+       }
+
+       if (is_error_noslot_pfn(pfn_end) && !is_noslot_pfn(pfn_end)) {
+               /*
+                * Allow guest MMIO range(s) to be added
+                * to the page encryption bitmap.
+                */
+               return -EINVAL;
+       }
+
+       mutex_lock(&kvm->lock);
+
+       if (sev->page_enc_bmap_size < gfn_end)
+               goto unlock;
+
+       if (enc)
+               __bitmap_set(sev->page_enc_bmap, gfn_start,
+                               gfn_end - gfn_start);
+       else
+               __bitmap_clear(sev->page_enc_bmap, gfn_start,
+                               gfn_end - gfn_start);
+
+unlock:
+       mutex_unlock(&kvm->lock);
+       return 0;
+}
+
 int svm_mem_enc_op(struct kvm *kvm, void __user *argp)
 {
        struct kvm_sev_cmd sev_cmd;

@@ -1560,6 +1647,9 @@ void sev_vm_destroy(struct kvm *kvm)
 
        sev_unbind_asid(kvm, sev->handle);
        sev_asid_free(sev->asid);
+
+       kvfree(sev->page_enc_bmap);
+       sev->page_enc_bmap = NULL;
 }
 
 int __init sev_hardware_setup(void)

arch/x86/kvm/svm/svm.c +2 -0

index 2f379bacbb26..1013ef0f4ce2 100644
@@ -4014,6 +4014,8 @@ static struct kvm_x86_ops svm_x86_ops __initdata = {
        .apic_init_signal_blocked = svm_apic_init_signal_blocked,
 
        .check_nested_events = svm_check_nested_events,
+
+       .page_enc_status_hc = svm_page_enc_status_hc,
 };
 
 static struct kvm_x86_init_ops svm_init_ops __initdata = {

arch/x86/kvm/svm/svm.h +4 -0

index df3474f4fb02..6a562f5928a2 100644
@@ -65,6 +65,8 @@ struct kvm_sev_info {
        int fd;                 /* SEV device fd */
        unsigned long pages_locked; /* Number of pages locked */
        struct list_head regions_list;  /* List of registered regions */
+       unsigned long *page_enc_bmap;
+       unsigned long page_enc_bmap_size;
 };
 
 struct kvm_svm {

@@ -400,6 +402,8 @@ int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
                               bool has_error_code, u32 error_code);
 int svm_check_nested_events(struct kvm_vcpu *vcpu);
 int nested_svm_exit_special(struct vcpu_svm *svm);
+int svm_page_enc_status_hc(struct kvm *kvm, unsigned long gpa,
+                                 unsigned long npages, unsigned long enc);
 
 /* avic.c */
 

arch/x86/kvm/vmx/vmx.c +1 -0

index c2c6335a998c..7d01d3aa6461 100644
@@ -7838,6 +7838,7 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = {
        .nested_get_evmcs_version = NULL,
        .need_emulation_on_page_fault = vmx_need_emulation_on_page_fault,
        .apic_init_signal_blocked = vmx_apic_init_signal_blocked,
+       .page_enc_status_hc = NULL,
 };
 
 static __init int hardware_setup(void)

arch/x86/kvm/x86.c +6 -0

index c5835f9cb9ad..5f5ddb5765e2 100644
@@ -7605,6 +7605,12 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
                kvm_sched_yield(vcpu->kvm, a0);
                ret = 0;
                break;
+       case KVM_HC_PAGE_ENC_STATUS:
+               ret = -KVM_ENOSYS;
+               if (kvm_x86_ops.page_enc_status_hc)
+                       ret = kvm_x86_ops.page_enc_status_hc(vcpu->kvm,
+                                       a0, a1, a2);
+               break;
        default:
                ret = -KVM_ENOSYS;
                break;

include/uapi/linux/kvm_para.h +1 -1

index 8b86609849b9..847b83b75dc8 100644
@@ -29,6 +29,7 @@
 #define KVM_HC_CLOCK_PAIRING           9
 #define KVM_HC_SEND_IPI                10
 #define KVM_HC_SCHED_YIELD             11
+#define KVM_HC_PAGE_ENC_STATUS         12
 
 /*
  * hypercalls use architecture specific
--

2.17.1

From: Brijesh Singh <[email protected]>

The ioctl can be used to retrieve page encryption bitmap for a given gfn range.

Return the correct bitmap as per the number of pages being requested by the user. Ensure that we only copy bmap->num_pages bytes in the userspace buffer, if bmap->num_pages is not byte aligned we read the trailing bits from the userspace and copy those bits as is.

Cc Thomas Gleixner <[email protected]>

Cc Ingo Molnar <[email protected]>

Cc "H. Peter Anvin" <[email protected]>

Cc Paolo Bonzini <[email protected]>

Cc "Radim Krčmář" <[email protected]>

Cc Joerg Roedel <[email protected]>

Cc Borislav Petkov <[email protected]>

Cc Tom Lendacky <[email protected]>

Cc [email protected]

Cc [email protected]

Cc [email protected]

Reviewed-by Venu Busireddy <[email protected]>

Signed-off-by Brijesh Singh <[email protected]>

Signed-off-by Ashish Kalra <[email protected]>

---
 Documentation/virt/kvm/api.rst  | 27 +++++++++++++
 arch/x86/include/asm/kvm_host.h |  2 +
 arch/x86/kvm/svm/sev.c          | 70 +++++++++++++++++++++++++++++++++
 arch/x86/kvm/svm/svm.c          |  1 +
 arch/x86/kvm/svm/svm.h          |  1 +
 arch/x86/kvm/x86.c              | 12 ++++++
 include/uapi/linux/kvm.h        | 12 ++++++
 7 files changed, 125 insertions(+)

Documentation/virt/kvm/api.rst +27 -0

index efbbe570aa9b..ecad84086892 100644
@@ -4636,6 +4636,33 @@ This ioctl resets VCPU registers and control structures according to
 the clear cpu reset definition in the POP. However, the cpu is not put
 into ESA mode. This reset is a superset of the initial reset.
 
+4.125 KVM_GET_PAGE_ENC_BITMAP (vm ioctl)
+---------------------------------------
+
+:Capability: basic
+:Architectures: x86
+:Type: vm ioctl
+:Parameters: struct kvm_page_enc_bitmap (in/out)
+:Returns: 0 on success, -1 on error
+
+/* for KVM_GET_PAGE_ENC_BITMAP */
+struct kvm_page_enc_bitmap {
+       __u64 start_gfn;
+       __u64 num_pages;
+       union {
+               void __user *enc_bitmap; /* one bit per page */
+               __u64 padding2;
+       };
+};
+
+The encrypted VMs have the concept of private and shared pages. The private
+pages are encrypted with the guest-specific key, while the shared pages may
+be encrypted with the hypervisor key. The KVM_GET_PAGE_ENC_BITMAP can
+be used to get the bitmap indicating whether the guest page is private
+or shared. The bitmap can be used during the guest migration. If the page
+is private then the userspace need to use SEV migration commands to transmit
+the page.
+
 
 4.125 KVM_S390_PV_COMMAND
 -------------------------

arch/x86/include/asm/kvm_host.h +2 -0

index 4a8ee22f4f5b..9e428befb6a4 100644
@@ -1256,6 +1256,8 @@ struct kvm_x86_ops {
        int (*enable_direct_tlbflush)(struct kvm_vcpu *vcpu);
        int (*page_enc_status_hc)(struct kvm *kvm, unsigned long gpa,
                                  unsigned long sz, unsigned long mode);
+       int (*get_page_enc_bitmap)(struct kvm *kvm,
+                               struct kvm_page_enc_bitmap *bmap);
 };
 
 struct kvm_x86_init_ops {

arch/x86/kvm/svm/sev.c +18 -0

index f088467708f0..387045902470 100644
@@ -1434,6 +1434,76 @@ int svm_page_enc_status_hc(struct kvm *kvm, unsigned long gpa,
        return 0;
 }
 
+int svm_get_page_enc_bitmap(struct kvm *kvm,
+                                  struct kvm_page_enc_bitmap *bmap)
+{
+       struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
+       unsigned long gfn_start, gfn_end;
+       unsigned long sz, i, sz_bytes;
+       unsigned long *bitmap;
+       int ret, n;
+
+       if (!sev_guest(kvm))
+               return -ENOTTY;
+
+       gfn_start = bmap->start_gfn;
+       gfn_end = gfn_start + bmap->num_pages;
+
+       sz = ALIGN(bmap->num_pages, BITS_PER_LONG) / BITS_PER_BYTE;
+       bitmap = kmalloc(sz, GFP_KERNEL);
+       if (!bitmap)

arch/x86/kvm/svm/sev.c +4 -0

+                       clear_bit(i - gfn_start, bitmap);
+       }
+       mutex_unlock(&kvm->lock);
+

arch/x86/kvm/svm/sev.c +28 -0

+        * those bits as is.
+        */
+
+       if (n) {
+               unsigned char *bitmap_kernel = (unsigned char *)bitmap;
+               unsigned char bitmap_user;
+               unsigned long offset, mask;
+
+               offset = bmap->num_pages / BITS_PER_BYTE;
+               if (copy_from_user(&bitmap_user, bmap->enc_bitmap + offset,
+                               sizeof(unsigned char)))
+                       goto out;
+
+               mask = GENMASK(n - 1, 0);
+               bitmap_user &= ~mask;
+               bitmap_kernel[offset] &= mask;
+               bitmap_kernel[offset] |= bitmap_user;
+       }
+
+       if (copy_to_user(bmap->enc_bitmap, bitmap, sz_bytes))
+               goto out;
+
+       ret = 0;
+out:
+       kfree(bitmap);
+       return ret;
+}
+
 int svm_mem_enc_op(struct kvm *kvm, void __user *argp)
 {
        struct kvm_sev_cmd sev_cmd;

arch/x86/kvm/svm/svm.c +1 -0

index 1013ef0f4ce2..588709a9f68e 100644
@@ -4016,6 +4016,7 @@ static struct kvm_x86_ops svm_x86_ops __initdata = {
        .check_nested_events = svm_check_nested_events,
 
        .page_enc_status_hc = svm_page_enc_status_hc,
+       .get_page_enc_bitmap = svm_get_page_enc_bitmap,
 };
 
 static struct kvm_x86_init_ops svm_init_ops __initdata = {

arch/x86/kvm/svm/svm.h +1 -0

index 6a562f5928a2..f087fa7b380c 100644
@@ -404,6 +404,7 @@ int svm_check_nested_events(struct kvm_vcpu *vcpu);
 int nested_svm_exit_special(struct vcpu_svm *svm);
 int svm_page_enc_status_hc(struct kvm *kvm, unsigned long gpa,
                                  unsigned long npages, unsigned long enc);
+int svm_get_page_enc_bitmap(struct kvm *kvm, struct kvm_page_enc_bitmap *bmap);
 
 /* avic.c */
 

arch/x86/kvm/x86.c +12 -0

index 5f5ddb5765e2..937797cfaf9a 100644
@@ -5208,6 +5208,18 @@ long kvm_arch_vm_ioctl(struct file *filp,
        case KVM_SET_PMU_EVENT_FILTER:
                r = kvm_vm_ioctl_set_pmu_event_filter(kvm, argp);
                break;
+       case KVM_GET_PAGE_ENC_BITMAP: {
+               struct kvm_page_enc_bitmap bitmap;
+
+               r = -EFAULT;
+               if (copy_from_user(&bitmap, argp, sizeof(bitmap)))
+                       goto out;
+
+               r = -ENOTTY;
+               if (kvm_x86_ops.get_page_enc_bitmap)
+                       r = kvm_x86_ops.get_page_enc_bitmap(kvm, &bitmap);
+               break;
+       }
        default:
                r = -ENOTTY;
        }

include/uapi/linux/kvm.h +12 -1

index 0fe1d206d750..af62f2afaa5d 100644
@@ -505,6 +505,16 @@ struct kvm_dirty_log {
        };
 };
 
+/* for KVM_GET_PAGE_ENC_BITMAP */
+struct kvm_page_enc_bitmap {
+       __u64 start_gfn;
+       __u64 num_pages;
+       union {
+               void __user *enc_bitmap; /* one bit per page */
+               __u64 padding2;
+       };
+};
+
 /* for KVM_CLEAR_DIRTY_LOG */
 struct kvm_clear_dirty_log {
        __u32 slot;

@@ -1518,6 +1528,8 @@ struct kvm_pv_cmd {
 /* Available with KVM_CAP_S390_PROTECTED */
 #define KVM_S390_PV_COMMAND            _IOWR(KVMIO, 0xc5, struct kvm_pv_cmd)
 
+#define KVM_GET_PAGE_ENC_BITMAP        _IOW(KVMIO, 0xc6, struct kvm_page_enc_bitmap)
+
 /* Secure Encrypted Virtualization command */
 enum sev_cmd_id {
        /* Guest initialization commands */
--

2.17.1

From: Brijesh Singh <[email protected]>

Invoke a hypercall when a memory region is changed from encrypted -> decrypted and vice versa. Hypervisor needs to know the page encryption status during the guest migration.

Cc Thomas Gleixner <[email protected]>

Cc Ingo Molnar <[email protected]>

Cc "H. Peter Anvin" <[email protected]>

Cc Paolo Bonzini <[email protected]>

Cc "Radim Krčmář" <[email protected]>

Cc Joerg Roedel <[email protected]>

Cc Borislav Petkov <[email protected]>

Cc Tom Lendacky <[email protected]>

Cc [email protected]

Cc [email protected]

Cc [email protected]

Reviewed-by Venu Busireddy <[email protected]>

Signed-off-by Brijesh Singh <[email protected]>

Signed-off-by Ashish Kalra <[email protected]>

---
 arch/x86/include/asm/paravirt.h       | 10 +++++
 arch/x86/include/asm/paravirt_types.h |  2 +
 arch/x86/kernel/paravirt.c            |  1 +
 arch/x86/mm/mem_encrypt.c             | 57 ++++++++++++++++++++++++++-
 arch/x86/mm/pat/set_memory.c          |  7 ++++
 5 files changed, 76 insertions(+), 1 deletion(-)

arch/x86/include/asm/paravirt.h +10 -0

index 694d8daf4983..8127b9c141bf 100644
@@ -78,6 +78,12 @@ static inline void paravirt_arch_exit_mmap(struct mm_struct *mm)
        PVOP_VCALL1(mmu.exit_mmap, mm);
 }
 
+static inline void page_encryption_changed(unsigned long vaddr, int npages,
+                                               bool enc)
+{
+       PVOP_VCALL3(mmu.page_encryption_changed, vaddr, npages, enc);
+}
+
 #ifdef CONFIG_PARAVIRT_XXL
 static inline void load_sp0(unsigned long sp0)
 {

@@ -946,6 +952,10 @@ static inline void paravirt_arch_dup_mmap(struct mm_struct *oldmm,
 static inline void paravirt_arch_exit_mmap(struct mm_struct *mm)
 {
 }
+
+static inline void page_encryption_changed(unsigned long vaddr, int npages, bool enc)
+{
+}
 #endif
 #endif /* __ASSEMBLY__ */
 #endif /* _ASM_X86_PARAVIRT_H */

arch/x86/include/asm/paravirt_types.h +2 -0

index 732f62e04ddb..03bfd515c59c 100644
@@ -215,6 +215,8 @@ struct pv_mmu_ops {
 
        /* Hook for intercepting the destruction of an mm_struct. */
        void (*exit_mmap)(struct mm_struct *mm);
+       void (*page_encryption_changed)(unsigned long vaddr, int npages,
+                                       bool enc);
 
 #ifdef CONFIG_PARAVIRT_XXL
        struct paravirt_callee_save read_cr2;

arch/x86/kernel/paravirt.c +1 -0

index c131ba4e70ef..840c02b23aeb 100644
@@ -367,6 +367,7 @@ struct paravirt_patch_template pv_ops = {
                        (void (*)(struct mmu_gather *, void *))tlb_remove_page,
 
        .mmu.exit_mmap          = paravirt_nop,
+       .mmu.page_encryption_changed    = paravirt_nop,
 
 #ifdef CONFIG_PARAVIRT_XXL
        .mmu.read_cr2           = __PV_IS_CALLEE_SAVE(native_read_cr2),

arch/x86/mm/mem_encrypt.c +56 -1

index f4bd4b431ba1..c9800fa811f6 100644
@@ -19,6 +19,7 @@
 #include <linux/kernel.h>
 #include <linux/bitops.h>
 #include <linux/dma-mapping.h>
+#include <linux/kvm_para.h>
 
 #include <asm/tlbflush.h>
 #include <asm/fixmap.h>

@@ -29,6 +30,7 @@
 #include <asm/processor-flags.h>
 #include <asm/msr.h>
 #include <asm/cmdline.h>
+#include <asm/kvm_para.h>
 
 #include "mm_internal.h"
 

@@ -196,6 +198,47 @@ void __init sme_early_init(void)
                swiotlb_force = SWIOTLB_FORCE;
 }
 
+static void set_memory_enc_dec_hypercall(unsigned long vaddr, int npages,
+                                       bool enc)
+{
+       unsigned long sz = npages << PAGE_SHIFT;
+       unsigned long vaddr_end, vaddr_next;
+
+       vaddr_end = vaddr + sz;
+
+       for (; vaddr < vaddr_end; vaddr = vaddr_next) {
+               int psize, pmask, level;
+               unsigned long pfn;
+               pte_t *kpte;
+
+               kpte = lookup_address(vaddr, &level);
+               if (!kpte || pte_none(*kpte))
+                       return;
+
+               switch (level) {
+               case PG_LEVEL_4K:
+                       pfn = pte_pfn(*kpte);
+                       break;
+               case PG_LEVEL_2M:
+                       pfn = pmd_pfn(*(pmd_t *)kpte);
+                       break;
+               case PG_LEVEL_1G:
+                       pfn = pud_pfn(*(pud_t *)kpte);
+                       break;
+               default:
+                       return;
+               }
+
+               psize = page_level_size(level);
+               pmask = page_level_mask(level);
+
+               kvm_sev_hypercall3(KVM_HC_PAGE_ENC_STATUS,
+                                  pfn << PAGE_SHIFT, psize >> PAGE_SHIFT, enc);
+
+               vaddr_next = (vaddr & pmask) + psize;
+       }
+}
+
 static void __init __set_clr_pte_enc(pte_t *kpte, int level, bool enc)
 {
        pgprot_t old_prot, new_prot;

@@ -253,12 +296,13 @@ static void __init __set_clr_pte_enc(pte_t *kpte, int level, bool enc)
 static int __init early_set_memory_enc_dec(unsigned long vaddr,
                                           unsigned long size, bool enc)
 {
-       unsigned long vaddr_end, vaddr_next;
+       unsigned long vaddr_end, vaddr_next, start;
        unsigned long psize, pmask;
        int split_page_size_mask;
        int level, ret;
        pte_t *kpte;
 
+       start = vaddr;
        vaddr_next = vaddr;
        vaddr_end = vaddr + size;
 

@@ -313,6 +357,8 @@ static int __init early_set_memory_enc_dec(unsigned long vaddr,
 
        ret = 0;
 
+       set_memory_enc_dec_hypercall(start, PAGE_ALIGN(size) >> PAGE_SHIFT,
+                                       enc);
 out:
        __flush_tlb_all();
        return ret;

@@ -451,6 +497,15 @@ void __init mem_encrypt_init(void)
        if (sev_active())
                static_branch_enable(&sev_enable_key);
 
+#ifdef CONFIG_PARAVIRT
+       /*
+        * With SEV, we need to make a hypercall when page encryption state is
+        * changed.
+        */
+       if (sev_active())
+               pv_ops.mmu.page_encryption_changed = set_memory_enc_dec_hypercall;
+#endif
+
        pr_info("AMD %s active\n",
                sev_active() ? "Secure Encrypted Virtualization (SEV)"
                             : "Secure Memory Encryption (SME)");

arch/x86/mm/pat/set_memory.c +7 -1

index 59eca6a94ce7..9aaf1b6f5a1b 100644
@@ -27,6 +27,7 @@
 #include <asm/proto.h>
 #include <asm/memtype.h>
 #include <asm/set_memory.h>
+#include <asm/paravirt.h>
 
 #include "../mm_internal.h"
 

@@ -2003,6 +2004,12 @@ static int __set_memory_enc_dec(unsigned long addr, int numpages, bool enc)
         */
        cpa_flush(&cpa, 0);
 
+       /* Notify hypervisor that a given memory range is mapped encrypted
+        * or decrypted. The hypervisor will use this information during the
+        * VM migration.
+        */
+       page_encryption_changed(addr, numpages, enc);
+
        return ret;
 }
 
--

2.17.1

From: Brijesh Singh <[email protected]>

The ioctl can be used to set page encryption bitmap for an incoming guest.

Cc Thomas Gleixner <[email protected]>

Cc Ingo Molnar <[email protected]>

Cc "H. Peter Anvin" <[email protected]>

Cc Paolo Bonzini <[email protected]>

Cc "Radim Krčmář" <[email protected]>

Cc Joerg Roedel <[email protected]>

Cc Borislav Petkov <[email protected]>

Cc Tom Lendacky <[email protected]>

Cc [email protected]

Cc [email protected]

Cc [email protected]

Reviewed-by Venu Busireddy <[email protected]>

Signed-off-by Brijesh Singh <[email protected]>

Signed-off-by Ashish Kalra <[email protected]>

---
 Documentation/virt/kvm/api.rst  | 44 +++++++++++++++++++++++++++++
 arch/x86/include/asm/kvm_host.h |  2 ++
 arch/x86/kvm/svm/sev.c          | 50 +++++++++++++++++++++++++++++++++
 arch/x86/kvm/svm/svm.c          |  1 +
 arch/x86/kvm/svm/svm.h          |  1 +
 arch/x86/kvm/x86.c              | 12 ++++++++
 include/uapi/linux/kvm.h        |  1 +
 7 files changed, 111 insertions(+)

Documentation/virt/kvm/api.rst +44 -0

index ecad84086892..fa70017ee693 100644
@@ -4663,6 +4663,28 @@ or shared. The bitmap can be used during the guest migration. If the page
 is private then the userspace need to use SEV migration commands to transmit
 the page.
 
+4.126 KVM_SET_PAGE_ENC_BITMAP (vm ioctl)
+---------------------------------------
+
+:Capability: basic
+:Architectures: x86
+:Type: vm ioctl
+:Parameters: struct kvm_page_enc_bitmap (in/out)
+:Returns: 0 on success, -1 on error
+
+/* for KVM_SET_PAGE_ENC_BITMAP */
+struct kvm_page_enc_bitmap {
+       __u64 start_gfn;
+       __u64 num_pages;
+       union {
+               void __user *enc_bitmap; /* one bit per page */
+               __u64 padding2;
+       };
+};
+
+During the guest live migration the outgoing guest exports its page encryption
+bitmap, the KVM_SET_PAGE_ENC_BITMAP can be used to build the page encryption
+bitmap for an incoming guest.
 
 4.125 KVM_S390_PV_COMMAND
 -------------------------

@@ -4717,6 +4739,28 @@ KVM_PV_VM_VERIFY
   Verify the integrity of the unpacked image. Only if this succeeds,
   KVM is allowed to start protected VCPUs.
 
+4.126 KVM_SET_PAGE_ENC_BITMAP (vm ioctl)
+---------------------------------------
+
+:Capability: basic
+:Architectures: x86
+:Type: vm ioctl
+:Parameters: struct kvm_page_enc_bitmap (in/out)
+:Returns: 0 on success, -1 on error
+
+/* for KVM_SET_PAGE_ENC_BITMAP */
+struct kvm_page_enc_bitmap {
+       __u64 start_gfn;
+       __u64 num_pages;
+       union {
+               void __user *enc_bitmap; /* one bit per page */
+               __u64 padding2;
+       };
+};
+
+During the guest live migration the outgoing guest exports its page encryption
+bitmap, the KVM_SET_PAGE_ENC_BITMAP can be used to build the page encryption
+bitmap for an incoming guest.
 
 5. The kvm_run structure
 ========================

arch/x86/include/asm/kvm_host.h +2 -0

index 9e428befb6a4..fc74144d5ab0 100644
@@ -1258,6 +1258,8 @@ struct kvm_x86_ops {
                                  unsigned long sz, unsigned long mode);
        int (*get_page_enc_bitmap)(struct kvm *kvm,
                                struct kvm_page_enc_bitmap *bmap);
+       int (*set_page_enc_bitmap)(struct kvm *kvm,
+                               struct kvm_page_enc_bitmap *bmap);
 };
 
 struct kvm_x86_init_ops {

arch/x86/kvm/svm/sev.c +31 -0

index 387045902470..30efc1068707 100644
@@ -1504,6 +1504,56 @@ int svm_get_page_enc_bitmap(struct kvm *kvm,
        return ret;
 }
 
+int svm_set_page_enc_bitmap(struct kvm *kvm,
+                                  struct kvm_page_enc_bitmap *bmap)
+{
+       struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
+       unsigned long gfn_start, gfn_end;
+       unsigned long *bitmap;
+       unsigned long sz;
+       int ret;
+
+       if (!sev_guest(kvm))
+               return -ENOTTY;
+       /* special case of resetting the complete bitmap */
+       if (!bmap->enc_bitmap) {
+               mutex_lock(&kvm->lock);
+               /* by default all pages are marked encrypted */
+               if (sev->page_enc_bmap_size)
+                       bitmap_fill(sev->page_enc_bmap,
+                                   sev->page_enc_bmap_size);
+               mutex_unlock(&kvm->lock);
+               return 0;
+       }
 
+
+       gfn_start = bmap->start_gfn;
+       gfn_end = gfn_start + bmap->num_pages;
+
+       sz = ALIGN(bmap->num_pages, BITS_PER_LONG) / 8;
+       bitmap = kmalloc(sz, GFP_KERNEL);
+       if (!bitmap)
+               return -ENOMEM;
+
+       ret = -EFAULT;

arch/x86/kvm/svm/sev.c +9 -0

+
+       ret = 0;
+unlock:
+       mutex_unlock(&kvm->lock);
+out:
+       kfree(bitmap);
+       return ret;
+}
+
 int svm_mem_enc_op(struct kvm *kvm, void __user *argp)
 {

        struct kvm_sev_cmd sev_cmd;

arch/x86/kvm/svm/svm.c +1 -0

index 588709a9f68e..501e82f5593c 100644
@@ -4017,6 +4017,7 @@ static struct kvm_x86_ops svm_x86_ops __initdata = {
 
        .page_enc_status_hc = svm_page_enc_status_hc,
        .get_page_enc_bitmap = svm_get_page_enc_bitmap,
+       .set_page_enc_bitmap = svm_set_page_enc_bitmap,
 };
 
 static struct kvm_x86_init_ops svm_init_ops __initdata = {

arch/x86/kvm/svm/svm.h +1 -0

index f087fa7b380c..2ebdcce50312 100644
@@ -405,6 +405,7 @@ int nested_svm_exit_special(struct vcpu_svm *svm);
 int svm_page_enc_status_hc(struct kvm *kvm, unsigned long gpa,
                                  unsigned long npages, unsigned long enc);
 int svm_get_page_enc_bitmap(struct kvm *kvm, struct kvm_page_enc_bitmap *bmap);
+int svm_set_page_enc_bitmap(struct kvm *kvm, struct kvm_page_enc_bitmap *bmap);
 
 /* avic.c */
 

arch/x86/kvm/x86.c +12 -0

index 937797cfaf9a..c4166d7a0493 100644
@@ -5220,6 +5220,18 @@ long kvm_arch_vm_ioctl(struct file *filp,
                        r = kvm_x86_ops.get_page_enc_bitmap(kvm, &bitmap);
                break;
        }
+       case KVM_SET_PAGE_ENC_BITMAP: {
+               struct kvm_page_enc_bitmap bitmap;
+
+               r = -EFAULT;
+               if (copy_from_user(&bitmap, argp, sizeof(bitmap)))
+                       goto out;
+
+               r = -ENOTTY;
+               if (kvm_x86_ops.set_page_enc_bitmap)
+                       r = kvm_x86_ops.set_page_enc_bitmap(kvm, &bitmap);
+               break;
+       }
        default:
                r = -ENOTTY;
        }

include/uapi/linux/kvm.h +1 -1

index af62f2afaa5d..2798b17484d0 100644
@@ -1529,6 +1529,7 @@ struct kvm_pv_cmd {
 #define KVM_S390_PV_COMMAND            _IOWR(KVMIO, 0xc5, struct kvm_pv_cmd)
 
 #define KVM_GET_PAGE_ENC_BITMAP        _IOW(KVMIO, 0xc6, struct kvm_page_enc_bitmap)
+#define KVM_SET_PAGE_ENC_BITMAP        _IOW(KVMIO, 0xc7, struct kvm_page_enc_bitmap)
 
 /* Secure Encrypted Virtualization command */
 enum sev_cmd_id {
--

2.17.1

From: Ashish Kalra <[email protected]>

Add support for static allocation of the unified Page encryption bitmap by extending kvm_arch_commit_memory_region() callack to add svm specific x86_ops which can read the userspace provided memory region/memslots and calculate the amount of guest RAM managed by the KVM and grow the bitmap based on that information, i.e. the highest guest PA that is mapped by a memslot.

Signed-off-by Ashish Kalra <[email protected]>

---
 arch/x86/include/asm/kvm_host.h |  1 +
 arch/x86/kvm/svm/sev.c          | 35 +++++++++++++++++++++++++++++++++
 arch/x86/kvm/svm/svm.c          |  1 +
 arch/x86/kvm/svm/svm.h          |  1 +
 arch/x86/kvm/x86.c              |  5 +++++
 5 files changed, 43 insertions(+)

arch/x86/include/asm/kvm_host.h +1 -0

index fc74144d5ab0..b573ea85b57e 100644
@@ -1254,6 +1254,7 @@ struct kvm_x86_ops {
 
        bool (*apic_init_signal_blocked)(struct kvm_vcpu *vcpu);
        int (*enable_direct_tlbflush)(struct kvm_vcpu *vcpu);
+       void (*commit_memory_region)(struct kvm *kvm, enum kvm_mr_change change);
        int (*page_enc_status_hc)(struct kvm *kvm, unsigned long gpa,
                                  unsigned long sz, unsigned long mode);
        int (*get_page_enc_bitmap)(struct kvm *kvm,

arch/x86/kvm/svm/sev.c +35 -0

index 30efc1068707..c0d7043a0627 100644
@@ -1377,6 +1377,41 @@ static int sev_resize_page_enc_bitmap(struct kvm *kvm, unsigned long new_size)
        return 0;
 }
 
+void svm_commit_memory_region(struct kvm *kvm, enum kvm_mr_change change)
+{
+       struct kvm_memslots *slots;
+       struct kvm_memory_slot *memslot;
+       gfn_t start, end = 0;
+
+       spin_lock(&kvm->mmu_lock);
+       if (change == KVM_MR_CREATE) {
+               slots = kvm_memslots(kvm);
+               kvm_for_each_memslot(memslot, slots) {
+                       start = memslot->base_gfn;
+                       end = memslot->base_gfn + memslot->npages;
+                       /*
+                        * KVM memslots is a sorted list, starting with
+                        * the highest mapped guest PA, so pick the topmost
+                        * valid guest PA.
+                        */
+                       if (memslot->npages)
+                               break;
+               }
+       }
+       spin_unlock(&kvm->mmu_lock);
+
+       if (end) {
+               /*
+                * NORE: This callback is invoked in vm ioctl
+                * set_user_memory_region, hence we can use a
+                * mutex here.
+                */
+               mutex_lock(&kvm->lock);
+               sev_resize_page_enc_bitmap(kvm, end);
+               mutex_unlock(&kvm->lock);
+       }
+}
+
 int svm_page_enc_status_hc(struct kvm *kvm, unsigned long gpa,
                                  unsigned long npages, unsigned long enc)
 {

arch/x86/kvm/svm/svm.c +1 -0

index 501e82f5593c..442adbbb0641 100644
@@ -4015,6 +4015,7 @@ static struct kvm_x86_ops svm_x86_ops __initdata = {
 
        .check_nested_events = svm_check_nested_events,
 
+       .commit_memory_region = svm_commit_memory_region,
        .page_enc_status_hc = svm_page_enc_status_hc,
        .get_page_enc_bitmap = svm_get_page_enc_bitmap,
        .set_page_enc_bitmap = svm_set_page_enc_bitmap,

arch/x86/kvm/svm/svm.h +1 -0

index 2ebdcce50312..fd99e0a5417a 100644
@@ -406,6 +406,7 @@ int svm_page_enc_status_hc(struct kvm *kvm, unsigned long gpa,
                                  unsigned long npages, unsigned long enc);
 int svm_get_page_enc_bitmap(struct kvm *kvm, struct kvm_page_enc_bitmap *bmap);
 int svm_set_page_enc_bitmap(struct kvm *kvm, struct kvm_page_enc_bitmap *bmap);
+void svm_commit_memory_region(struct kvm *kvm, enum kvm_mr_change change);
 
 /* avic.c */
 

arch/x86/kvm/x86.c +0 -0

index c4166d7a0493..8938de868d42 100644

arch/x86/kvm/x86.c +2 -1

+       }
+
        /*
         * Dirty logging tracks sptes in 4k granularity, meaning that large
         * sptes have to be split.  If live migration is successful, the guest
--

2.17.1

Quoted 108 lines · Ashish Kalra

From: Ashish Kalra <[email protected]>

Add support for static allocation of the unified Page encryption bitmap by extending kvm_arch_commit_memory_region() callack to add svm specific x86_ops which can read the userspace provided memory region/memslots and calculate the amount of guest RAM managed by the KVM and grow the bitmap based on that information, i.e. the highest guest PA that is mapped by a memslot.

Signed-off-by Ashish Kalra <[email protected]>

---
 arch/x86/include/asm/kvm_host.h |  1 +
 arch/x86/kvm/svm/sev.c          | 35 +++++++++++++++++++++++++++++++++
 arch/x86/kvm/svm/svm.c          |  1 +
 arch/x86/kvm/svm/svm.h          |  1 +
 arch/x86/kvm/x86.c              |  5 +++++
 5 files changed, 43 insertions(+)

arch/x86/include/asm/kvm_host.h +1 -0

index fc74144d5ab0..b573ea85b57e 100644
@@ -1254,6 +1254,7 @@ struct kvm_x86_ops {
 
        bool (*apic_init_signal_blocked)(struct kvm_vcpu *vcpu);
        int (*enable_direct_tlbflush)(struct kvm_vcpu *vcpu);
+       void (*commit_memory_region)(struct kvm *kvm, enum kvm_mr_change change);
        int (*page_enc_status_hc)(struct kvm *kvm, unsigned long gpa,
                                  unsigned long sz, unsigned long mode);
        int (*get_page_enc_bitmap)(struct kvm *kvm,

arch/x86/kvm/svm/sev.c +35 -0

index 30efc1068707..c0d7043a0627 100644
@@ -1377,6 +1377,41 @@ static int sev_resize_page_enc_bitmap(struct kvm *kvm, unsigned long new_size)
        return 0;
 }
 
+void svm_commit_memory_region(struct kvm *kvm, enum kvm_mr_change change)
+{
+       struct kvm_memslots *slots;
+       struct kvm_memory_slot *memslot;
+       gfn_t start, end = 0;
+
+       spin_lock(&kvm->mmu_lock);
+       if (change == KVM_MR_CREATE) {
+               slots = kvm_memslots(kvm);
+               kvm_for_each_memslot(memslot, slots) {
+                       start = memslot->base_gfn;
+                       end = memslot->base_gfn + memslot->npages;
+                       /*
+                        * KVM memslots is a sorted list, starting with
+                        * the highest mapped guest PA, so pick the topmost
+                        * valid guest PA.
+                        */
+                       if (memslot->npages)
+                               break;
+               }
+       }
+       spin_unlock(&kvm->mmu_lock);
+
+       if (end) {
+               /*
+                * NORE: This callback is invoked in vm ioctl
+                * set_user_memory_region, hence we can use a
+                * mutex here.
+                */
+               mutex_lock(&kvm->lock);
+               sev_resize_page_enc_bitmap(kvm, end);
+               mutex_unlock(&kvm->lock);
+       }
+}
+
 int svm_page_enc_status_hc(struct kvm *kvm, unsigned long gpa,
                                  unsigned long npages, unsigned long enc)
 {

arch/x86/kvm/svm/svm.c +1 -0

index 501e82f5593c..442adbbb0641 100644
@@ -4015,6 +4015,7 @@ static struct kvm_x86_ops svm_x86_ops __initdata = {
 
        .check_nested_events = svm_check_nested_events,
 
+       .commit_memory_region = svm_commit_memory_region,
        .page_enc_status_hc = svm_page_enc_status_hc,
        .get_page_enc_bitmap = svm_get_page_enc_bitmap,
        .set_page_enc_bitmap = svm_set_page_enc_bitmap,

arch/x86/kvm/svm/svm.h +1 -0

index 2ebdcce50312..fd99e0a5417a 100644
@@ -406,6 +406,7 @@ int svm_page_enc_status_hc(struct kvm *kvm, unsigned long gpa,
                                  unsigned long npages, unsigned long enc);
 int svm_get_page_enc_bitmap(struct kvm *kvm, struct kvm_page_enc_bitmap *bmap);
 int svm_set_page_enc_bitmap(struct kvm *kvm, struct kvm_page_enc_bitmap *bmap);
+void svm_commit_memory_region(struct kvm *kvm, enum kvm_mr_change change);
 
 /* avic.c */
 

arch/x86/kvm/x86.c +0 -0

index c4166d7a0493..8938de868d42 100644
@@ -10133,6 +10133,11 @@ void kvm_arch_commit_memory_region(struct kvm *kvm,
                kvm_mmu_change_mmu_pages(kvm,
                                kvm_mmu_calculate_default_mmu_pages(kvm));
 

arch/x86/kvm/x86.c void kvm_arch_commit_memory_region(struct kvm *kvm, · line 10138

Quoted 3 lines

arch/x86/kvm/x86.c +3 -0

+       if (change == KVM_MR_CREATE || change == KVM_MR_DELETE) {
+               if (kvm_x86_ops.commit_memory_region)
+                       kvm_x86_ops.commit_memory_region(kvm, change);

Why not just call this every time (if it exists) and have the kvm_x86_op determine if it should do anything?

It seems like it's a nop anyway unless you are doing a create.

Quoted 8 lines

arch/x86/kvm/x86.c +2 -1

+       }
+
        /*
         * Dirty logging tracks sptes in 4k granularity, meaning that large
         * sptes have to be split.  If live migration is successful, the guest
--

2.17.1

Add support for static allocation of the unified Page encryption bitmap by extending kvm_arch_commit_memory_region() callack to add svm specific x86_ops which can read the userspace provided memory region/memslots and calculate the amount of guest RAM managed by the KVM and grow the bitmap based on that information, i.e. the highest guest PA that is mapped by a memslot.

Quoted 6 lines · Ashish Kalra

Add support for static allocation of the unified Page encryption bitmap by extending kvm_arch_commit_memory_region() callack to add svm specific x86_ops which can read the userspace provided memory region/memslots and calculate the amount of guest RAM managed by the KVM and grow the bitmap based on that information, i.e. the highest guest PA that is mapped by a memslot.

Hi Ashish,

the commit message should explain why this is needed or useful.

Earlier we used to dynamic resizing of the page encryption bitmap based on the guest hypercall, but potentially a malicious guest can make a hypercall which can trigger a really large memory allocation on the host side and may eventually cause denial of service.

Hence now we don't do dynamic resizing of the page encryption bitmap as per the hypercall and allocate it statically based on guest memory allocation by walking through memslots and computing it's size.

I will add the above comment to the fresh series of the patch-set i am going to post.

From: Ashish Kalra <[email protected]>

Add new KVM_FEATURE_SEV_LIVE_MIGRATION feature for guest to check for host-side support for SEV live migration. Also add a new custom MSR_KVM_SEV_LIVE_MIG_EN for guest to enable the SEV live migration feature.

Signed-off-by Ashish Kalra <[email protected]>

---
 Documentation/virt/kvm/cpuid.rst     |  5 +++++
 Documentation/virt/kvm/msr.rst       | 10 ++++++++++
 arch/x86/include/uapi/asm/kvm_para.h |  5 +++++
 arch/x86/kvm/svm/sev.c               | 14 ++++++++++++++
 arch/x86/kvm/svm/svm.c               | 16 ++++++++++++++++
 arch/x86/kvm/svm/svm.h               |  2 ++
 6 files changed, 52 insertions(+)

Documentation/virt/kvm/cpuid.rst +5 -0

index 01b081f6e7ea..0514523e00cd 100644
@@ -86,6 +86,11 @@ KVM_FEATURE_PV_SCHED_YIELD        13          guest checks this feature bit
                                               before using paravirtualized
                                               sched yield.
 
+KVM_FEATURE_SEV_LIVE_MIGRATION    14          guest checks this feature bit before
+                                              using the page encryption state
+                                              hypercall to notify the page state
+                                              change
+
 KVM_FEATURE_CLOCSOURCE_STABLE_BIT 24          host will warn if no guest-side
                                               per-cpu warps are expeced in
                                               kvmclock

Documentation/virt/kvm/msr.rst +10 -0

index 33892036672d..7cd7786bbb03 100644
@@ -319,3 +319,13 @@ data:
 
        KVM guests can request the host not to poll on HLT, for example if
        they are performing polling themselves.
+
+MSR_KVM_SEV_LIVE_MIG_EN:
+        0x4b564d06
+
+       Control SEV Live Migration features.
+
+data:
+        Bit 0 enables (1) or disables (0) host-side SEV Live Migration feature.
+        Bit 1 enables (1) or disables (0) support for SEV Live Migration extensions.
+        All other bits are reserved.

arch/x86/include/uapi/asm/kvm_para.h +5 -0

index 2a8e0b6b9805..d9d4953b42ad 100644
@@ -31,6 +31,7 @@
 #define KVM_FEATURE_PV_SEND_IPI        11
 #define KVM_FEATURE_POLL_CONTROL       12
 #define KVM_FEATURE_PV_SCHED_YIELD     13
+#define KVM_FEATURE_SEV_LIVE_MIGRATION 14
 
 #define KVM_HINTS_REALTIME      0
 

@@ -50,6 +51,7 @@
 #define MSR_KVM_STEAL_TIME  0x4b564d03
 #define MSR_KVM_PV_EOI_EN      0x4b564d04
 #define MSR_KVM_POLL_CONTROL   0x4b564d05
+#define MSR_KVM_SEV_LIVE_MIG_EN        0x4b564d06
 
 struct kvm_steal_time {
        __u64 steal;

@@ -122,4 +124,7 @@ struct kvm_vcpu_pv_apf_data {
 #define KVM_PV_EOI_ENABLED KVM_PV_EOI_MASK
 #define KVM_PV_EOI_DISABLED 0x0
 
+#define KVM_SEV_LIVE_MIGRATION_ENABLED                 (1 << 0)
+#define KVM_SEV_LIVE_MIGRATION_EXTENSIONS_SUPPORTED    (1 << 1)
+
 #endif /* _UAPI_ASM_X86_KVM_PARA_H */

arch/x86/kvm/svm/sev.c +14 -0

index c0d7043a0627..6f69c3a47583 100644
@@ -1469,6 +1469,17 @@ int svm_page_enc_status_hc(struct kvm *kvm, unsigned long gpa,
        return 0;
 }
 
+void sev_update_migration_flags(struct kvm *kvm, u64 data)
+{
+       struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
+
+       if (!sev_guest(kvm))
+               return;
+
+       if (data & KVM_SEV_LIVE_MIGRATION_ENABLED)
+               sev->live_migration_enabled = true;
+}
+
 int svm_get_page_enc_bitmap(struct kvm *kvm,
                                   struct kvm_page_enc_bitmap *bmap)
 {

@@ -1481,6 +1492,9 @@ int svm_get_page_enc_bitmap(struct kvm *kvm,
        if (!sev_guest(kvm))
                return -ENOTTY;
 
+       if (!sev->live_migration_enabled)
+               return -EINVAL;
+
        gfn_start = bmap->start_gfn;
        gfn_end = gfn_start + bmap->num_pages;
 

arch/x86/kvm/svm/svm.c +16 -0

index 442adbbb0641..a99f5457f244 100644
@@ -2633,6 +2633,9 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
                svm->msr_decfg = data;
                break;
        }
+       case MSR_KVM_SEV_LIVE_MIG_EN:
+               sev_update_migration_flags(vcpu->kvm, data);
+               break;
        case MSR_IA32_APICBASE:
                if (kvm_vcpu_apicv_active(vcpu))
                        avic_update_vapic_bar(to_svm(vcpu), data);

@@ -3493,6 +3496,19 @@ static void svm_cpuid_update(struct kvm_vcpu *vcpu)
        svm->nrips_enabled = kvm_cpu_cap_has(X86_FEATURE_NRIPS) &&
                             guest_cpuid_has(&svm->vcpu, X86_FEATURE_NRIPS);
 
+        /*
+         * If SEV guest then enable the Live migration feature.
+         */
+        if (sev_guest(vcpu->kvm)) {
+              struct kvm_cpuid_entry2 *best;
+
+              best = kvm_find_cpuid_entry(vcpu, KVM_CPUID_FEATURES, 0);
+              if (!best)
+                      return;
+
+              best->eax |= (1 << KVM_FEATURE_SEV_LIVE_MIGRATION);
+        }
+
        if (!kvm_vcpu_apicv_active(vcpu))
                return;
 

arch/x86/kvm/svm/svm.h +2 -1

index fd99e0a5417a..77f132a6fead 100644
@@ -65,6 +65,7 @@ struct kvm_sev_info {
        int fd;                 /* SEV device fd */
        unsigned long pages_locked; /* Number of pages locked */
        struct list_head regions_list;  /* List of registered regions */
+       bool live_migration_enabled;
        unsigned long *page_enc_bmap;
        unsigned long page_enc_bmap_size;
 };

@@ -494,5 +495,6 @@ int svm_unregister_enc_region(struct kvm *kvm,
 void pre_sev_run(struct vcpu_svm *svm, int cpu);
 int __init sev_hardware_setup(void);
 void sev_hardware_teardown(void);
+void sev_update_migration_flags(struct kvm *kvm, u64 data);
 
 #endif
--

2.17.1

From: Ashish Kalra <[email protected]>

Add new KVM_FEATURE_SEV_LIVE_MIGRATION feature for guest to check for host-side support for SEV live migration. Also add a new custom MSR_KVM_SEV_LIVE_MIG_EN for guest to enable the SEV live migration feature.

Signed-off-by Ashish Kalra <[email protected]>

---
  Documentation/virt/kvm/cpuid.rst     |  5 +++++
  Documentation/virt/kvm/msr.rst       | 10 ++++++++++
  arch/x86/include/uapi/asm/kvm_para.h |  5 +++++
  arch/x86/kvm/svm/sev.c               | 14 ++++++++++++++
  arch/x86/kvm/svm/svm.c               | 16 ++++++++++++++++
  arch/x86/kvm/svm/svm.h               |  2 ++
  6 files changed, 52 insertions(+)

Documentation/virt/kvm/cpuid.rst +5 -0

index 01b081f6e7ea..0514523e00cd 100644
@@ -86,6 +86,11 @@ KVM_FEATURE_PV_SCHED_YIELD        13          guest checks this feature bit
                                                before using paravirtualized
                                                sched yield.
 
+KVM_FEATURE_SEV_LIVE_MIGRATION    14          guest checks this feature bit before
+                                              using the page encryption state
+                                              hypercall to notify the page state
+                                              change
+
  KVM_FEATURE_CLOCSOURCE_STABLE_BIT 24          host will warn if no guest-side
                                                per-cpu warps are expeced in
                                                kvmclock

Documentation/virt/kvm/msr.rst +0 -0

index 33892036672d..7cd7786bbb03 100644
@@ -319,3 +319,13 @@ data:
 
  	KVM guests can request the host not to poll on HLT, for example if
  	they are performing polling themselves.

I applied patches -13, this one a bit changed as follows.

An immediate response, actually the SEV live migration patches are preferred over the Page encryption bitmap patches, in other words, if SEV live migration patches are applied then we don't need the Page encryption bitmap patches and we prefer the live migration series to be applied.

It is not that page encryption bitmap series supersede the live migration patches, they are just cut of the live migration patches.

Quoted 5 lines · Ashish Kalra

An immediate response, actually the SEV live migration patches are preferred over the Page encryption bitmap patches, in other words, if SEV live migration patches are applied then we don't need the Page encryption bitmap patches and we prefer the live migration series to be applied.

Quoted 2 lines

It is not that page encryption bitmap series supersede the live migration patches, they are just cut of the live migration patches.

In that case, can you post a fresh version of the live migration series? Paolo is obviously willing to take a big chunk of that series, and it will likely be easier to review with the full context, e.g. one of my comments on the standalone encryption bitmap series was going to be that it's hard to review without seeing the live migration aspect.

Quoted 8 lines · Sean Christopherson

Quoted 8 lines · Ashish Kalra

An immediate response, actually the SEV live migration patches are preferred over the Page encryption bitmap patches, in other words, if SEV live migration patches are applied then we don't need the Page encryption bitmap patches and we prefer the live migration series to be applied.

It is not that page encryption bitmap series supersede the live migration patches, they are just cut of the live migration patches.

Quoted 5 lines

In that case, can you post a fresh version of the live migration series? Paolo is obviously willing to take a big chunk of that series, and it will likely be easier to review with the full context, e.g. one of my comments on the standalone encryption bitmap series was going to be that it's hard to review without seeing the live migration aspect.

It still applies without change. For now I'll only keep the series queued in my (n)SVM branch, but will hold on applying it to kvm.git's queue and next branches.

Yes i will post a fresh version of the live migration patches.

Also, can you please check your email settings, we are only able to see your response on the mailing list but we are not getting your direct responses.

Quoted 5 lines · Ashish Kalra

Yes i will post a fresh version of the live migration patches.

Also, can you please check your email settings, we are only able to see your response on the mailing list but we are not getting your direct responses.

Hrm, as in you don't get the email?

Is this email any different? Sending via gmail instead of mutt...

Quoted 5 lines · Ashish Kalra

Yes i will post a fresh version of the live migration patches.

Also, can you please check your email settings, we are only able to see your response on the mailing list but we are not getting your direct responses.

Hrm, as in you don't get the email?

Is this email any different? Sending via gmail instead of mutt...