Skip to contentSkip to conversation

Add AMD SEV guest live migration support

PATCH Superseded Ashish Kalra 7 participants 107 messages Mar 30, 2020 – Apr 11, 2020 4 messages not in the archive Page 2 of 2

Subsystems (9)
Documentation x86 KVM KVM x86 +5 more subsystems

v6 of 13 Superseded Latest is v13 →

lore
Share & tools
lore
#72 lore raw

Re: [PATCH v6 12/14] KVM: x86: Introduce KVM_PAGE_ENC_BITMAP_RESET ioctl

In reply to Krish Sadhukhan
Quoted 77 lines · Krish Sadhukhan
Quoted 75 lines · Ashish Kalra

From: Ashish Kalra <[email protected]>

This ioctl can be used by the application to reset the page encryption bitmap managed by the KVM driver. A typical usage for this ioctl is on VM reboot, on reboot, we must reinitialize the bitmap.

---
  Documentation/virt/kvm/api.rst  | 13 +++++++++++++
  arch/x86/include/asm/kvm_host.h |  1 +
  arch/x86/kvm/svm.c              | 16 ++++++++++++++++
  arch/x86/kvm/x86.c              |  6 ++++++
  include/uapi/linux/kvm.h        |  1 +
  5 files changed, 37 insertions(+)
Documentation/virt/kvm/api.rst +13 -0
index 4d1004a154f6..a11326ccc51d 100644
@@ -4698,6 +4698,19 @@ During the guest live migration the outgoing guest exports its page encryption
  bitmap, the KVM_SET_PAGE_ENC_BITMAP can be used to build the page encryption
  bitmap for an incoming guest.
+4.127 KVM_PAGE_ENC_BITMAP_RESET (vm ioctl)
+-----------------------------------------
+
+:Capability: basic
+:Architectures: x86
+:Type: vm ioctl
+:Parameters: none
+:Returns: 0 on success, -1 on error
+
+The KVM_PAGE_ENC_BITMAP_RESET is used to reset the guest's page encryption
+bitmap during guest reboot and this is only done on the guest's boot vCPU.
+
+
  5. The kvm_run structure
  ========================
arch/x86/include/asm/kvm_host.h +1 -0
index d30f770aaaea..a96ef6338cd2 100644
@@ -1273,6 +1273,7 @@ struct kvm_x86_ops {
  				struct kvm_page_enc_bitmap *bmap);
  	int (*set_page_enc_bitmap)(struct kvm *kvm,
  				struct kvm_page_enc_bitmap *bmap);
+	int (*reset_page_enc_bitmap)(struct kvm *kvm);
  };
  struct kvm_arch_async_pf {
arch/x86/kvm/svm.c +15 -0
index 313343a43045..c99b0207a443 100644
@@ -7797,6 +7797,21 @@ static int svm_set_page_enc_bitmap(struct kvm *kvm,
  	return ret;
  }
+static int svm_reset_page_enc_bitmap(struct kvm *kvm)
+{
+	struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
+
+	if (!sev_guest(kvm))
+		return -ENOTTY;
+
+	mutex_lock(&kvm->lock);
+	/* by default all pages should be marked encrypted */
+	if (sev->page_enc_bmap_size)
+		bitmap_fill(sev->page_enc_bmap, sev->page_enc_bmap_size);
+	mutex_unlock(&kvm->lock);
+	return 0;
+}
+
  static int svm_mem_enc_op(struct kvm *kvm, void __user *argp)
  {
  	struct kvm_sev_cmd sev_cmd;
@@ -8203,6 +8218,7 @@ static struct kvm_x86_ops svm_x86_ops __ro_after_init = {
Quoted 10 lines

arch/x86/kvm/svm.c static struct kvm_x86_ops svm_x86_ops __ro_after_init = { · line 8221

Quoted 4 lines
arch/x86/kvm/svm.c +1 -0
  	.page_enc_status_hc = svm_page_enc_status_hc,
  	.get_page_enc_bitmap = svm_get_page_enc_bitmap,
  	.set_page_enc_bitmap = svm_set_page_enc_bitmap,
+	.reset_page_enc_bitmap = svm_reset_page_enc_bitmap,

We don't need to initialize the intel ops to NULL ? It's not initialized in the previous patch either.

Quoted 1 line
arch/x86/kvm/svm.c +0 -0
  };

This struct is declared as "static storage", so won't the non-initialized members be 0 ?

Quoted 29 lines
Quoted 23 lines
arch/x86/kvm/svm.c +0 -0
  static int __init svm_init(void)
arch/x86/kvm/x86.c +6 -0
index 05e953b2ec61..2127ed937f53 100644
@@ -5250,6 +5250,12 @@ long kvm_arch_vm_ioctl(struct file *filp,
  			r = kvm_x86_ops->set_page_enc_bitmap(kvm, &bitmap);
  		break;
  	}
+	case KVM_PAGE_ENC_BITMAP_RESET: {
+		r = -ENOTTY;
+		if (kvm_x86_ops->reset_page_enc_bitmap)
+			r = kvm_x86_ops->reset_page_enc_bitmap(kvm);
+		break;
+	}
  	default:
  		r = -ENOTTY;
  	}
include/uapi/linux/kvm.h +0 -0
index b4b01d47e568..0884a581fc37 100644

include/uapi/linux/kvm.h struct kvm_enc_region { · line 1494

Quoted 5 lines
include/uapi/linux/kvm.h +1 -0
@@ -1490,6 +1490,7 @@ struct kvm_enc_region {
  #define KVM_GET_PAGE_ENC_BITMAP	_IOW(KVMIO, 0xc5, struct kvm_page_enc_bitmap)
  #define KVM_SET_PAGE_ENC_BITMAP	_IOW(KVMIO, 0xc6, struct kvm_page_enc_bitmap)
+#define KVM_PAGE_ENC_BITMAP_RESET	_IO(KVMIO, 0xc7)
  /* Secure Encrypted Virtualization command */
  enum sev_cmd_id {
#73 lore raw

Re: [PATCH v6 12/14] KVM: x86: Introduce KVM_PAGE_ENC_BITMAP_RESET ioctl

Quoted 85 lines · Ashish Kalra
Quoted 84 lines · Krish Sadhukhan
Quoted 75 lines · Ashish Kalra

From: Ashish Kalra <[email protected]>

This ioctl can be used by the application to reset the page encryption bitmap managed by the KVM driver. A typical usage for this ioctl is on VM reboot, on reboot, we must reinitialize the bitmap.

---
   Documentation/virt/kvm/api.rst  | 13 +++++++++++++
   arch/x86/include/asm/kvm_host.h |  1 +
   arch/x86/kvm/svm.c              | 16 ++++++++++++++++
   arch/x86/kvm/x86.c              |  6 ++++++
   include/uapi/linux/kvm.h        |  1 +
   5 files changed, 37 insertions(+)
Documentation/virt/kvm/api.rst +13 -0
index 4d1004a154f6..a11326ccc51d 100644
@@ -4698,6 +4698,19 @@ During the guest live migration the outgoing guest exports its page encryption
   bitmap, the KVM_SET_PAGE_ENC_BITMAP can be used to build the page encryption
   bitmap for an incoming guest.
+4.127 KVM_PAGE_ENC_BITMAP_RESET (vm ioctl)
+-----------------------------------------
+
+:Capability: basic
+:Architectures: x86
+:Type: vm ioctl
+:Parameters: none
+:Returns: 0 on success, -1 on error
+
+The KVM_PAGE_ENC_BITMAP_RESET is used to reset the guest's page encryption
+bitmap during guest reboot and this is only done on the guest's boot vCPU.
+
+
   5. The kvm_run structure
   ========================
arch/x86/include/asm/kvm_host.h +1 -0
index d30f770aaaea..a96ef6338cd2 100644
@@ -1273,6 +1273,7 @@ struct kvm_x86_ops {
   				struct kvm_page_enc_bitmap *bmap);
   	int (*set_page_enc_bitmap)(struct kvm *kvm,
   				struct kvm_page_enc_bitmap *bmap);
+	int (*reset_page_enc_bitmap)(struct kvm *kvm);
   };
   struct kvm_arch_async_pf {
arch/x86/kvm/svm.c +15 -0
index 313343a43045..c99b0207a443 100644
@@ -7797,6 +7797,21 @@ static int svm_set_page_enc_bitmap(struct kvm *kvm,
   	return ret;
   }
+static int svm_reset_page_enc_bitmap(struct kvm *kvm)
+{
+	struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
+
+	if (!sev_guest(kvm))
+		return -ENOTTY;
+
+	mutex_lock(&kvm->lock);
+	/* by default all pages should be marked encrypted */
+	if (sev->page_enc_bmap_size)
+		bitmap_fill(sev->page_enc_bmap, sev->page_enc_bmap_size);
+	mutex_unlock(&kvm->lock);
+	return 0;
+}
+
   static int svm_mem_enc_op(struct kvm *kvm, void __user *argp)
   {
   	struct kvm_sev_cmd sev_cmd;

arch/x86/kvm/svm.c static struct kvm_x86_ops svm_x86_ops __ro_after_init = { · line 8221

Quoted 4 lines
arch/x86/kvm/svm.c +1 -0
@@ -8203,6 +8218,7 @@ static struct kvm_x86_ops svm_x86_ops __ro_after_init = {
   	.page_enc_status_hc = svm_page_enc_status_hc,
   	.get_page_enc_bitmap = svm_get_page_enc_bitmap,
   	.set_page_enc_bitmap = svm_set_page_enc_bitmap,
+	.reset_page_enc_bitmap = svm_reset_page_enc_bitmap,

We don't need to initialize the intel ops to NULL ? It's not initialized in the previous patch either.

Quoted 3 lines
Quoted 1 line

arch/x86/kvm/svm.c static struct kvm_x86_ops svm_x86_ops __ro_after_init = { · line 8222

Quoted 1 line
arch/x86/kvm/svm.c +0 -0
   };

This struct is declared as "static storage", so won't the non-initialized members be 0 ?

Correct. Although, I see that 'nested_enable_evmcs' is explicitly initialized. We should maintain the convention, perhaps.

Quoted 30 lines
Quoted 30 lines
Quoted 24 lines
arch/x86/kvm/svm.c +0 -0
   static int __init svm_init(void)
arch/x86/kvm/x86.c +6 -0
index 05e953b2ec61..2127ed937f53 100644
@@ -5250,6 +5250,12 @@ long kvm_arch_vm_ioctl(struct file *filp,
   			r = kvm_x86_ops->set_page_enc_bitmap(kvm, &bitmap);
   		break;
   	}
+	case KVM_PAGE_ENC_BITMAP_RESET: {
+		r = -ENOTTY;
+		if (kvm_x86_ops->reset_page_enc_bitmap)
+			r = kvm_x86_ops->reset_page_enc_bitmap(kvm);
+		break;
+	}
   	default:
   		r = -ENOTTY;
   	}
include/uapi/linux/kvm.h +0 -0
index b4b01d47e568..0884a581fc37 100644

include/uapi/linux/kvm.h struct kvm_enc_region { · line 1494

Quoted 5 lines
include/uapi/linux/kvm.h +1 -0
@@ -1490,6 +1490,7 @@ struct kvm_enc_region {
   #define KVM_GET_PAGE_ENC_BITMAP	_IOW(KVMIO, 0xc5, struct kvm_page_enc_bitmap)
   #define KVM_SET_PAGE_ENC_BITMAP	_IOW(KVMIO, 0xc6, struct kvm_page_enc_bitmap)
+#define KVM_PAGE_ENC_BITMAP_RESET	_IO(KVMIO, 0xc7)
   /* Secure Encrypted Virtualization command */
   enum sev_cmd_id {
#74 lore raw

Re: [PATCH v6 12/14] KVM: x86: Introduce KVM_PAGE_ENC_BITMAP_RESET ioctl

Quoted 88 lines · Krish Sadhukhan
Quoted 85 lines · Ashish Kalra
Quoted 84 lines · Krish Sadhukhan
Quoted 75 lines · Ashish Kalra

From: Ashish Kalra <[email protected]>

This ioctl can be used by the application to reset the page encryption bitmap managed by the KVM driver. A typical usage for this ioctl is on VM reboot, on reboot, we must reinitialize the bitmap.

---
   Documentation/virt/kvm/api.rst  | 13 +++++++++++++
   arch/x86/include/asm/kvm_host.h |  1 +
   arch/x86/kvm/svm.c              | 16 ++++++++++++++++
   arch/x86/kvm/x86.c              |  6 ++++++
   include/uapi/linux/kvm.h        |  1 +
   5 files changed, 37 insertions(+)
Documentation/virt/kvm/api.rst +13 -0
index 4d1004a154f6..a11326ccc51d 100644
@@ -4698,6 +4698,19 @@ During the guest live migration the outgoing guest exports its page encryption
   bitmap, the KVM_SET_PAGE_ENC_BITMAP can be used to build the page encryption
   bitmap for an incoming guest.
+4.127 KVM_PAGE_ENC_BITMAP_RESET (vm ioctl)
+-----------------------------------------
+
+:Capability: basic
+:Architectures: x86
+:Type: vm ioctl
+:Parameters: none
+:Returns: 0 on success, -1 on error
+
+The KVM_PAGE_ENC_BITMAP_RESET is used to reset the guest's page encryption
+bitmap during guest reboot and this is only done on the guest's boot vCPU.
+
+
   5. The kvm_run structure
   ========================
arch/x86/include/asm/kvm_host.h +1 -0
index d30f770aaaea..a96ef6338cd2 100644
@@ -1273,6 +1273,7 @@ struct kvm_x86_ops {
                            struct kvm_page_enc_bitmap *bmap);
    int (*set_page_enc_bitmap)(struct kvm *kvm,
                            struct kvm_page_enc_bitmap *bmap);
+   int (*reset_page_enc_bitmap)(struct kvm *kvm);
   };
   struct kvm_arch_async_pf {
arch/x86/kvm/svm.c +15 -0
index 313343a43045..c99b0207a443 100644
@@ -7797,6 +7797,21 @@ static int svm_set_page_enc_bitmap(struct kvm *kvm,
    return ret;
   }
+static int svm_reset_page_enc_bitmap(struct kvm *kvm)
+{
+   struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
+
+   if (!sev_guest(kvm))
+           return -ENOTTY;
+
+   mutex_lock(&kvm->lock);
+   /* by default all pages should be marked encrypted */
+   if (sev->page_enc_bmap_size)
+           bitmap_fill(sev->page_enc_bmap, sev->page_enc_bmap_size);
+   mutex_unlock(&kvm->lock);
+   return 0;
+}
+
   static int svm_mem_enc_op(struct kvm *kvm, void __user *argp)
   {
    struct kvm_sev_cmd sev_cmd;

arch/x86/kvm/svm.c static struct kvm_x86_ops svm_x86_ops __ro_after_init = { · line 8221

Quoted 4 lines
arch/x86/kvm/svm.c +1 -0
@@ -8203,6 +8218,7 @@ static struct kvm_x86_ops svm_x86_ops __ro_after_init = {
    .page_enc_status_hc = svm_page_enc_status_hc,
    .get_page_enc_bitmap = svm_get_page_enc_bitmap,
    .set_page_enc_bitmap = svm_set_page_enc_bitmap,
+   .reset_page_enc_bitmap = svm_reset_page_enc_bitmap,

We don't need to initialize the intel ops to NULL ? It's not initialized in the previous patch either.

Quoted 38 lines
Quoted 3 lines
Quoted 1 line

arch/x86/kvm/svm.c static struct kvm_x86_ops svm_x86_ops __ro_after_init = { · line 8222

Quoted 1 line
arch/x86/kvm/svm.c +0 -0
   };

This struct is declared as "static storage", so won't the non-initialized members be 0 ?

Correct. Although, I see that 'nested_enable_evmcs' is explicitly initialized. We should maintain the convention, perhaps.

Quoted 30 lines
Quoted 30 lines
Quoted 24 lines
arch/x86/kvm/svm.c +0 -0
   static int __init svm_init(void)
arch/x86/kvm/x86.c +6 -0
index 05e953b2ec61..2127ed937f53 100644
@@ -5250,6 +5250,12 @@ long kvm_arch_vm_ioctl(struct file *filp,
                    r = kvm_x86_ops->set_page_enc_bitmap(kvm, &bitmap);
            break;
    }
+   case KVM_PAGE_ENC_BITMAP_RESET: {
+           r = -ENOTTY;
+           if (kvm_x86_ops->reset_page_enc_bitmap)
+                   r = kvm_x86_ops->reset_page_enc_bitmap(kvm);
+           break;
+   }
    default:
            r = -ENOTTY;
    }
include/uapi/linux/kvm.h +0 -0
index b4b01d47e568..0884a581fc37 100644

include/uapi/linux/kvm.h struct kvm_enc_region { · line 1494

Quoted 5 lines
include/uapi/linux/kvm.h +1 -0
@@ -1490,6 +1490,7 @@ struct kvm_enc_region {
   #define KVM_GET_PAGE_ENC_BITMAP  _IOW(KVMIO, 0xc5, struct kvm_page_enc_bitmap)
   #define KVM_SET_PAGE_ENC_BITMAP  _IOW(KVMIO, 0xc6, struct kvm_page_enc_bitmap)
+#define KVM_PAGE_ENC_BITMAP_RESET  _IO(KVMIO, 0xc7)
   /* Secure Encrypted Virtualization command */
   enum sev_cmd_id {

Doesn't this overlap with the set ioctl? Yes, obviously, you have to copy the new value down and do a bit more work, but I don't think resetting the bitmap is going to be the bottleneck on reboot. Seems excessive to add another ioctl for this.

#75 lore raw

Re: [PATCH v6 12/14] KVM: x86: Introduce KVM_PAGE_ENC_BITMAP_RESET ioctl

Hello Steve,

Quoted 130 lines · Steve Rutherford
Quoted 88 lines · Krish Sadhukhan
Quoted 85 lines · Ashish Kalra
Quoted 84 lines · Krish Sadhukhan
Quoted 75 lines · Ashish Kalra

From: Ashish Kalra <[email protected]>

This ioctl can be used by the application to reset the page encryption bitmap managed by the KVM driver. A typical usage for this ioctl is on VM reboot, on reboot, we must reinitialize the bitmap.

---
   Documentation/virt/kvm/api.rst  | 13 +++++++++++++
   arch/x86/include/asm/kvm_host.h |  1 +
   arch/x86/kvm/svm.c              | 16 ++++++++++++++++
   arch/x86/kvm/x86.c              |  6 ++++++
   include/uapi/linux/kvm.h        |  1 +
   5 files changed, 37 insertions(+)
Documentation/virt/kvm/api.rst +13 -0
index 4d1004a154f6..a11326ccc51d 100644
@@ -4698,6 +4698,19 @@ During the guest live migration the outgoing guest exports its page encryption
   bitmap, the KVM_SET_PAGE_ENC_BITMAP can be used to build the page encryption
   bitmap for an incoming guest.
+4.127 KVM_PAGE_ENC_BITMAP_RESET (vm ioctl)
+-----------------------------------------
+
+:Capability: basic
+:Architectures: x86
+:Type: vm ioctl
+:Parameters: none
+:Returns: 0 on success, -1 on error
+
+The KVM_PAGE_ENC_BITMAP_RESET is used to reset the guest's page encryption
+bitmap during guest reboot and this is only done on the guest's boot vCPU.
+
+
   5. The kvm_run structure
   ========================
arch/x86/include/asm/kvm_host.h +1 -0
index d30f770aaaea..a96ef6338cd2 100644
@@ -1273,6 +1273,7 @@ struct kvm_x86_ops {
                            struct kvm_page_enc_bitmap *bmap);
    int (*set_page_enc_bitmap)(struct kvm *kvm,
                            struct kvm_page_enc_bitmap *bmap);
+   int (*reset_page_enc_bitmap)(struct kvm *kvm);
   };
   struct kvm_arch_async_pf {
arch/x86/kvm/svm.c +15 -0
index 313343a43045..c99b0207a443 100644
@@ -7797,6 +7797,21 @@ static int svm_set_page_enc_bitmap(struct kvm *kvm,
    return ret;
   }
+static int svm_reset_page_enc_bitmap(struct kvm *kvm)
+{
+   struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
+
+   if (!sev_guest(kvm))
+           return -ENOTTY;
+
+   mutex_lock(&kvm->lock);
+   /* by default all pages should be marked encrypted */
+   if (sev->page_enc_bmap_size)
+           bitmap_fill(sev->page_enc_bmap, sev->page_enc_bmap_size);
+   mutex_unlock(&kvm->lock);
+   return 0;
+}
+
   static int svm_mem_enc_op(struct kvm *kvm, void __user *argp)
   {
    struct kvm_sev_cmd sev_cmd;

arch/x86/kvm/svm.c static struct kvm_x86_ops svm_x86_ops __ro_after_init = { · line 8221

Quoted 4 lines
arch/x86/kvm/svm.c +1 -0
@@ -8203,6 +8218,7 @@ static struct kvm_x86_ops svm_x86_ops __ro_after_init = {
    .page_enc_status_hc = svm_page_enc_status_hc,
    .get_page_enc_bitmap = svm_get_page_enc_bitmap,
    .set_page_enc_bitmap = svm_set_page_enc_bitmap,
+   .reset_page_enc_bitmap = svm_reset_page_enc_bitmap,

We don't need to initialize the intel ops to NULL ? It's not initialized in the previous patch either.

Quoted 38 lines
Quoted 3 lines
Quoted 1 line

arch/x86/kvm/svm.c static struct kvm_x86_ops svm_x86_ops __ro_after_init = { · line 8222

Quoted 1 line
arch/x86/kvm/svm.c +0 -0
   };

This struct is declared as "static storage", so won't the non-initialized members be 0 ?

Correct. Although, I see that 'nested_enable_evmcs' is explicitly initialized. We should maintain the convention, perhaps.

Quoted 30 lines
Quoted 30 lines
Quoted 24 lines
arch/x86/kvm/svm.c +0 -0
   static int __init svm_init(void)
arch/x86/kvm/x86.c +6 -0
index 05e953b2ec61..2127ed937f53 100644
@@ -5250,6 +5250,12 @@ long kvm_arch_vm_ioctl(struct file *filp,
                    r = kvm_x86_ops->set_page_enc_bitmap(kvm, &bitmap);
            break;
    }
+   case KVM_PAGE_ENC_BITMAP_RESET: {
+           r = -ENOTTY;
+           if (kvm_x86_ops->reset_page_enc_bitmap)
+                   r = kvm_x86_ops->reset_page_enc_bitmap(kvm);
+           break;
+   }
    default:
            r = -ENOTTY;
    }
include/uapi/linux/kvm.h +0 -0
index b4b01d47e568..0884a581fc37 100644

include/uapi/linux/kvm.h struct kvm_enc_region { · line 1494

Quoted 5 lines
include/uapi/linux/kvm.h +1 -0
@@ -1490,6 +1490,7 @@ struct kvm_enc_region {
   #define KVM_GET_PAGE_ENC_BITMAP  _IOW(KVMIO, 0xc5, struct kvm_page_enc_bitmap)
   #define KVM_SET_PAGE_ENC_BITMAP  _IOW(KVMIO, 0xc6, struct kvm_page_enc_bitmap)
+#define KVM_PAGE_ENC_BITMAP_RESET  _IO(KVMIO, 0xc7)
   /* Secure Encrypted Virtualization command */
   enum sev_cmd_id {
Quoted 4 lines

Doesn't this overlap with the set ioctl? Yes, obviously, you have to copy the new value down and do a bit more work, but I don't think resetting the bitmap is going to be the bottleneck on reboot. Seems excessive to add another ioctl for this.

The set ioctl is generally available/provided for the incoming VM to setup the page encryption bitmap, this reset ioctl is meant for the source VM as a simple interface to reset the whole page encryption bitmap.

Thanks,
Ashish

#76 lore raw

Re: [PATCH v6 12/14] KVM: x86: Introduce KVM_PAGE_ENC_BITMAP_RESET ioctl

Quoted 139 lines · Ashish Kalra

Hello Steve,

Quoted 130 lines · Steve Rutherford
Quoted 88 lines · Krish Sadhukhan
Quoted 85 lines · Ashish Kalra
Quoted 84 lines · Krish Sadhukhan
Quoted 75 lines · Ashish Kalra

From: Ashish Kalra <[email protected]>

This ioctl can be used by the application to reset the page encryption bitmap managed by the KVM driver. A typical usage for this ioctl is on VM reboot, on reboot, we must reinitialize the bitmap.

---
   Documentation/virt/kvm/api.rst  | 13 +++++++++++++
   arch/x86/include/asm/kvm_host.h |  1 +
   arch/x86/kvm/svm.c              | 16 ++++++++++++++++
   arch/x86/kvm/x86.c              |  6 ++++++
   include/uapi/linux/kvm.h        |  1 +
   5 files changed, 37 insertions(+)
Documentation/virt/kvm/api.rst +13 -0
index 4d1004a154f6..a11326ccc51d 100644
@@ -4698,6 +4698,19 @@ During the guest live migration the outgoing guest exports its page encryption
   bitmap, the KVM_SET_PAGE_ENC_BITMAP can be used to build the page encryption
   bitmap for an incoming guest.
+4.127 KVM_PAGE_ENC_BITMAP_RESET (vm ioctl)
+-----------------------------------------
+
+:Capability: basic
+:Architectures: x86
+:Type: vm ioctl
+:Parameters: none
+:Returns: 0 on success, -1 on error
+
+The KVM_PAGE_ENC_BITMAP_RESET is used to reset the guest's page encryption
+bitmap during guest reboot and this is only done on the guest's boot vCPU.
+
+
   5. The kvm_run structure
   ========================
arch/x86/include/asm/kvm_host.h +1 -0
index d30f770aaaea..a96ef6338cd2 100644
@@ -1273,6 +1273,7 @@ struct kvm_x86_ops {
                            struct kvm_page_enc_bitmap *bmap);
    int (*set_page_enc_bitmap)(struct kvm *kvm,
                            struct kvm_page_enc_bitmap *bmap);
+   int (*reset_page_enc_bitmap)(struct kvm *kvm);
   };
   struct kvm_arch_async_pf {
arch/x86/kvm/svm.c +15 -0
index 313343a43045..c99b0207a443 100644
@@ -7797,6 +7797,21 @@ static int svm_set_page_enc_bitmap(struct kvm *kvm,
    return ret;
   }
+static int svm_reset_page_enc_bitmap(struct kvm *kvm)
+{
+   struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
+
+   if (!sev_guest(kvm))
+           return -ENOTTY;
+
+   mutex_lock(&kvm->lock);
+   /* by default all pages should be marked encrypted */
+   if (sev->page_enc_bmap_size)
+           bitmap_fill(sev->page_enc_bmap, sev->page_enc_bmap_size);
+   mutex_unlock(&kvm->lock);
+   return 0;
+}
+
   static int svm_mem_enc_op(struct kvm *kvm, void __user *argp)
   {
    struct kvm_sev_cmd sev_cmd;

arch/x86/kvm/svm.c static struct kvm_x86_ops svm_x86_ops __ro_after_init = { · line 8221

Quoted 4 lines
arch/x86/kvm/svm.c +1 -0
@@ -8203,6 +8218,7 @@ static struct kvm_x86_ops svm_x86_ops __ro_after_init = {
    .page_enc_status_hc = svm_page_enc_status_hc,
    .get_page_enc_bitmap = svm_get_page_enc_bitmap,
    .set_page_enc_bitmap = svm_set_page_enc_bitmap,
+   .reset_page_enc_bitmap = svm_reset_page_enc_bitmap,

We don't need to initialize the intel ops to NULL ? It's not initialized in the previous patch either.

Quoted 38 lines
Quoted 3 lines
Quoted 1 line

arch/x86/kvm/svm.c static struct kvm_x86_ops svm_x86_ops __ro_after_init = { · line 8222

Quoted 1 line
arch/x86/kvm/svm.c +0 -0
   };

This struct is declared as "static storage", so won't the non-initialized members be 0 ?

Correct. Although, I see that 'nested_enable_evmcs' is explicitly initialized. We should maintain the convention, perhaps.

Quoted 30 lines
Quoted 30 lines
Quoted 24 lines
arch/x86/kvm/svm.c +0 -0
   static int __init svm_init(void)
arch/x86/kvm/x86.c +6 -0
index 05e953b2ec61..2127ed937f53 100644
@@ -5250,6 +5250,12 @@ long kvm_arch_vm_ioctl(struct file *filp,
                    r = kvm_x86_ops->set_page_enc_bitmap(kvm, &bitmap);
            break;
    }
+   case KVM_PAGE_ENC_BITMAP_RESET: {
+           r = -ENOTTY;
+           if (kvm_x86_ops->reset_page_enc_bitmap)
+                   r = kvm_x86_ops->reset_page_enc_bitmap(kvm);
+           break;
+   }
    default:
            r = -ENOTTY;
    }
include/uapi/linux/kvm.h +0 -0
index b4b01d47e568..0884a581fc37 100644

include/uapi/linux/kvm.h struct kvm_enc_region { · line 1494

Quoted 5 lines
include/uapi/linux/kvm.h +1 -0
@@ -1490,6 +1490,7 @@ struct kvm_enc_region {
   #define KVM_GET_PAGE_ENC_BITMAP  _IOW(KVMIO, 0xc5, struct kvm_page_enc_bitmap)
   #define KVM_SET_PAGE_ENC_BITMAP  _IOW(KVMIO, 0xc6, struct kvm_page_enc_bitmap)
+#define KVM_PAGE_ENC_BITMAP_RESET  _IO(KVMIO, 0xc7)
   /* Secure Encrypted Virtualization command */
   enum sev_cmd_id {
Quoted 4 lines

Doesn't this overlap with the set ioctl? Yes, obviously, you have to copy the new value down and do a bit more work, but I don't think resetting the bitmap is going to be the bottleneck on reboot. Seems excessive to add another ioctl for this.

Quoted 6 lines

The set ioctl is generally available/provided for the incoming VM to setup the page encryption bitmap, this reset ioctl is meant for the source VM as a simple interface to reset the whole page encryption bitmap.

Thanks,
Ashish

Hey Ashish,

These seem very overlapping. I think this API should be refactored a bit.

1) Use kvm_vm_ioctl_enable_cap to control whether or not this hypercall (and related feature bit) is offered to the VM, and also the size of the buffer.
2) Use set for manipulating values in the bitmap, including resetting the bitmap. Set the bitmap pointer to null if you want to reset to all 0xFFs. When the bitmap pointer is set, it should set the values to exactly what is pointed at, instead of only clearing bits, as is done currently.
3) Use get for fetching values from the kernel. Personally, I'd require alignment of the base GFN to a multiple of 8 (but the number of pages could be whatever), so you can just use a memcpy. Optionally, you may want some way to tell userspace the size of the existing buffer, so it can ensure that it can ask for the entire buffer without having to track the size in usermode (not strictly necessary, but nice to have since it ensures that there is only one place that has to manage this value).

If you want to expand or contract the bitmap, you can use enable cap to adjust the size.
If you don't want to offer the hypercall to the guest, don't call the enable cap.
This API avoids using up another ioctl. Ioctl space is somewhat scarce. It also gives userspace fine grained control over the buffer, so it can support both hot-plug and hot-unplug (or at the very least it is not obviously incompatible with those). It also gives userspace control over whether or not the feature is offered. The hypercall isn't free, and being able to tell guests to not call when the host wasn't going to migrate it anyway will be useful.

Thanks,
--Steve

#77 lore raw

Re: [PATCH v6 12/14] KVM: x86: Introduce KVM_PAGE_ENC_BITMAP_RESET ioctl

Hello Steve,

Quoted 148 lines · Steve Rutherford
Quoted 139 lines · Ashish Kalra

Hello Steve,

Quoted 130 lines · Steve Rutherford
Quoted 88 lines · Krish Sadhukhan
Quoted 85 lines · Ashish Kalra
Quoted 84 lines · Krish Sadhukhan
Quoted 75 lines · Ashish Kalra

From: Ashish Kalra <[email protected]>

This ioctl can be used by the application to reset the page encryption bitmap managed by the KVM driver. A typical usage for this ioctl is on VM reboot, on reboot, we must reinitialize the bitmap.

---
   Documentation/virt/kvm/api.rst  | 13 +++++++++++++
   arch/x86/include/asm/kvm_host.h |  1 +
   arch/x86/kvm/svm.c              | 16 ++++++++++++++++
   arch/x86/kvm/x86.c              |  6 ++++++
   include/uapi/linux/kvm.h        |  1 +
   5 files changed, 37 insertions(+)
Documentation/virt/kvm/api.rst +13 -0
index 4d1004a154f6..a11326ccc51d 100644
@@ -4698,6 +4698,19 @@ During the guest live migration the outgoing guest exports its page encryption
   bitmap, the KVM_SET_PAGE_ENC_BITMAP can be used to build the page encryption
   bitmap for an incoming guest.
+4.127 KVM_PAGE_ENC_BITMAP_RESET (vm ioctl)
+-----------------------------------------
+
+:Capability: basic
+:Architectures: x86
+:Type: vm ioctl
+:Parameters: none
+:Returns: 0 on success, -1 on error
+
+The KVM_PAGE_ENC_BITMAP_RESET is used to reset the guest's page encryption
+bitmap during guest reboot and this is only done on the guest's boot vCPU.
+
+
   5. The kvm_run structure
   ========================
arch/x86/include/asm/kvm_host.h +1 -0
index d30f770aaaea..a96ef6338cd2 100644
@@ -1273,6 +1273,7 @@ struct kvm_x86_ops {
                            struct kvm_page_enc_bitmap *bmap);
    int (*set_page_enc_bitmap)(struct kvm *kvm,
                            struct kvm_page_enc_bitmap *bmap);
+   int (*reset_page_enc_bitmap)(struct kvm *kvm);
   };
   struct kvm_arch_async_pf {
arch/x86/kvm/svm.c +15 -0
index 313343a43045..c99b0207a443 100644
@@ -7797,6 +7797,21 @@ static int svm_set_page_enc_bitmap(struct kvm *kvm,
    return ret;
   }
+static int svm_reset_page_enc_bitmap(struct kvm *kvm)
+{
+   struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
+
+   if (!sev_guest(kvm))
+           return -ENOTTY;
+
+   mutex_lock(&kvm->lock);
+   /* by default all pages should be marked encrypted */
+   if (sev->page_enc_bmap_size)
+           bitmap_fill(sev->page_enc_bmap, sev->page_enc_bmap_size);
+   mutex_unlock(&kvm->lock);
+   return 0;
+}
+
   static int svm_mem_enc_op(struct kvm *kvm, void __user *argp)
   {
    struct kvm_sev_cmd sev_cmd;

arch/x86/kvm/svm.c static struct kvm_x86_ops svm_x86_ops __ro_after_init = { · line 8221

Quoted 4 lines
arch/x86/kvm/svm.c +1 -0
@@ -8203,6 +8218,7 @@ static struct kvm_x86_ops svm_x86_ops __ro_after_init = {
    .page_enc_status_hc = svm_page_enc_status_hc,
    .get_page_enc_bitmap = svm_get_page_enc_bitmap,
    .set_page_enc_bitmap = svm_set_page_enc_bitmap,
+   .reset_page_enc_bitmap = svm_reset_page_enc_bitmap,

We don't need to initialize the intel ops to NULL ? It's not initialized in the previous patch either.

Quoted 38 lines
Quoted 3 lines
Quoted 1 line

arch/x86/kvm/svm.c static struct kvm_x86_ops svm_x86_ops __ro_after_init = { · line 8222

Quoted 1 line
arch/x86/kvm/svm.c +0 -0
   };

This struct is declared as "static storage", so won't the non-initialized members be 0 ?

Correct. Although, I see that 'nested_enable_evmcs' is explicitly initialized. We should maintain the convention, perhaps.

Quoted 30 lines
Quoted 30 lines
Quoted 24 lines
arch/x86/kvm/svm.c +0 -0
   static int __init svm_init(void)
arch/x86/kvm/x86.c +6 -0
index 05e953b2ec61..2127ed937f53 100644
@@ -5250,6 +5250,12 @@ long kvm_arch_vm_ioctl(struct file *filp,
                    r = kvm_x86_ops->set_page_enc_bitmap(kvm, &bitmap);
            break;
    }
+   case KVM_PAGE_ENC_BITMAP_RESET: {
+           r = -ENOTTY;
+           if (kvm_x86_ops->reset_page_enc_bitmap)
+                   r = kvm_x86_ops->reset_page_enc_bitmap(kvm);
+           break;
+   }
    default:
            r = -ENOTTY;
    }
include/uapi/linux/kvm.h +0 -0
index b4b01d47e568..0884a581fc37 100644

include/uapi/linux/kvm.h struct kvm_enc_region { · line 1494

Quoted 5 lines
include/uapi/linux/kvm.h +1 -0
@@ -1490,6 +1490,7 @@ struct kvm_enc_region {
   #define KVM_GET_PAGE_ENC_BITMAP  _IOW(KVMIO, 0xc5, struct kvm_page_enc_bitmap)
   #define KVM_SET_PAGE_ENC_BITMAP  _IOW(KVMIO, 0xc6, struct kvm_page_enc_bitmap)
+#define KVM_PAGE_ENC_BITMAP_RESET  _IO(KVMIO, 0xc7)
   /* Secure Encrypted Virtualization command */
   enum sev_cmd_id {
Quoted 4 lines

Doesn't this overlap with the set ioctl? Yes, obviously, you have to copy the new value down and do a bit more work, but I don't think resetting the bitmap is going to be the bottleneck on reboot. Seems excessive to add another ioctl for this.

Quoted 6 lines

The set ioctl is generally available/provided for the incoming VM to setup the page encryption bitmap, this reset ioctl is meant for the source VM as a simple interface to reset the whole page encryption bitmap.

Thanks,
Ashish

Quoted 7 lines

Hey Ashish,

These seem very overlapping. I think this API should be refactored a bit.

1) Use kvm_vm_ioctl_enable_cap to control whether or not this hypercall (and related feature bit) is offered to the VM, and also the size of the buffer.

If you look at patch 13/14, i have added a new kvm para feature called "KVM_FEATURE_SEV_LIVE_MIGRATION" which indicates host support for SEV Live Migration and a new Custom MSR which the guest does a wrmsr to enable the Live Migration feature, so this is like the enable cap support.

There are further extensions to this support i am adding, so patch 13/14 of this patch-set is still being enhanced and will have full support when i repost next.

Quoted 5 lines

2) Use set for manipulating values in the bitmap, including resetting the bitmap. Set the bitmap pointer to null if you want to reset to all 0xFFs. When the bitmap pointer is set, it should set the values to exactly what is pointed at, instead of only clearing bits, as is done currently.

As i mentioned in my earlier email, the set api is supposed to be for the incoming VM, but if you really need to use it for the outgoing VM then it can be modified.

Quoted 11 lines

3) Use get for fetching values from the kernel. Personally, I'd require alignment of the base GFN to a multiple of 8 (but the number of pages could be whatever), so you can just use a memcpy. Optionally, you may want some way to tell userspace the size of the existing buffer, so it can ensure that it can ask for the entire buffer without having to track the size in usermode (not strictly necessary, but nice to have since it ensures that there is only one place that has to manage this value).

If you want to expand or contract the bitmap, you can use enable cap to adjust the size.

As being discussed on the earlier mail thread, we are doing this dynamically now by computing the guest RAM size when the set_user_memory_region ioctl is invoked. I believe that should handle the hot-plug and hot-unplug events too, as any hot memory updates will need KVM memslots to be updated.

Quoted 10 lines

If you don't want to offer the hypercall to the guest, don't call the enable cap.
This API avoids using up another ioctl. Ioctl space is somewhat scarce. It also gives userspace fine grained control over the buffer, so it can support both hot-plug and hot-unplug (or at the very least it is not obviously incompatible with those). It also gives userspace control over whether or not the feature is offered. The hypercall isn't free, and being able to tell guests to not call when the host wasn't going to migrate it anyway will be useful.

As i mentioned above, now the host indicates if it supports the Live Migration feature and the feature and the hypercall are only enabled on the host when the guest checks for this support and does a wrmsr() to enable the feature. Also the guest will not make the hypercall if the host does not indicate support for it.

Thanks,
Ashish

#78 lore raw

Re: [PATCH v6 12/14] KVM: x86: Introduce KVM_PAGE_ENC_BITMAP_RESET ioctl

Quoted 189 lines · Ashish Kalra

Hello Steve,

Quoted 148 lines · Steve Rutherford
Quoted 139 lines · Ashish Kalra

Hello Steve,

Quoted 130 lines · Steve Rutherford
Quoted 88 lines · Krish Sadhukhan
Quoted 85 lines · Ashish Kalra
Quoted 84 lines · Krish Sadhukhan
Quoted 75 lines · Ashish Kalra

From: Ashish Kalra <[email protected]>

This ioctl can be used by the application to reset the page encryption bitmap managed by the KVM driver. A typical usage for this ioctl is on VM reboot, on reboot, we must reinitialize the bitmap.

---
   Documentation/virt/kvm/api.rst  | 13 +++++++++++++
   arch/x86/include/asm/kvm_host.h |  1 +
   arch/x86/kvm/svm.c              | 16 ++++++++++++++++
   arch/x86/kvm/x86.c              |  6 ++++++
   include/uapi/linux/kvm.h        |  1 +
   5 files changed, 37 insertions(+)
Documentation/virt/kvm/api.rst +13 -0
index 4d1004a154f6..a11326ccc51d 100644
@@ -4698,6 +4698,19 @@ During the guest live migration the outgoing guest exports its page encryption
   bitmap, the KVM_SET_PAGE_ENC_BITMAP can be used to build the page encryption
   bitmap for an incoming guest.
+4.127 KVM_PAGE_ENC_BITMAP_RESET (vm ioctl)
+-----------------------------------------
+
+:Capability: basic
+:Architectures: x86
+:Type: vm ioctl
+:Parameters: none
+:Returns: 0 on success, -1 on error
+
+The KVM_PAGE_ENC_BITMAP_RESET is used to reset the guest's page encryption
+bitmap during guest reboot and this is only done on the guest's boot vCPU.
+
+
   5. The kvm_run structure
   ========================
arch/x86/include/asm/kvm_host.h +1 -0
index d30f770aaaea..a96ef6338cd2 100644
@@ -1273,6 +1273,7 @@ struct kvm_x86_ops {
                            struct kvm_page_enc_bitmap *bmap);
    int (*set_page_enc_bitmap)(struct kvm *kvm,
                            struct kvm_page_enc_bitmap *bmap);
+   int (*reset_page_enc_bitmap)(struct kvm *kvm);
   };
   struct kvm_arch_async_pf {
arch/x86/kvm/svm.c +15 -0
index 313343a43045..c99b0207a443 100644
@@ -7797,6 +7797,21 @@ static int svm_set_page_enc_bitmap(struct kvm *kvm,
    return ret;
   }
+static int svm_reset_page_enc_bitmap(struct kvm *kvm)
+{
+   struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
+
+   if (!sev_guest(kvm))
+           return -ENOTTY;
+
+   mutex_lock(&kvm->lock);
+   /* by default all pages should be marked encrypted */
+   if (sev->page_enc_bmap_size)
+           bitmap_fill(sev->page_enc_bmap, sev->page_enc_bmap_size);
+   mutex_unlock(&kvm->lock);
+   return 0;
+}
+
   static int svm_mem_enc_op(struct kvm *kvm, void __user *argp)
   {
    struct kvm_sev_cmd sev_cmd;

arch/x86/kvm/svm.c static struct kvm_x86_ops svm_x86_ops __ro_after_init = { · line 8221

Quoted 4 lines
arch/x86/kvm/svm.c +1 -0
@@ -8203,6 +8218,7 @@ static struct kvm_x86_ops svm_x86_ops __ro_after_init = {
    .page_enc_status_hc = svm_page_enc_status_hc,
    .get_page_enc_bitmap = svm_get_page_enc_bitmap,
    .set_page_enc_bitmap = svm_set_page_enc_bitmap,
+   .reset_page_enc_bitmap = svm_reset_page_enc_bitmap,

We don't need to initialize the intel ops to NULL ? It's not initialized in the previous patch either.

Quoted 38 lines
Quoted 3 lines
Quoted 1 line

arch/x86/kvm/svm.c static struct kvm_x86_ops svm_x86_ops __ro_after_init = { · line 8222

Quoted 1 line
arch/x86/kvm/svm.c +0 -0
   };

This struct is declared as "static storage", so won't the non-initialized members be 0 ?

Correct. Although, I see that 'nested_enable_evmcs' is explicitly initialized. We should maintain the convention, perhaps.

Quoted 30 lines
Quoted 30 lines
Quoted 24 lines
arch/x86/kvm/svm.c +0 -0
   static int __init svm_init(void)
arch/x86/kvm/x86.c +6 -0
index 05e953b2ec61..2127ed937f53 100644
@@ -5250,6 +5250,12 @@ long kvm_arch_vm_ioctl(struct file *filp,
                    r = kvm_x86_ops->set_page_enc_bitmap(kvm, &bitmap);
            break;
    }
+   case KVM_PAGE_ENC_BITMAP_RESET: {
+           r = -ENOTTY;
+           if (kvm_x86_ops->reset_page_enc_bitmap)
+                   r = kvm_x86_ops->reset_page_enc_bitmap(kvm);
+           break;
+   }
    default:
            r = -ENOTTY;
    }
include/uapi/linux/kvm.h +0 -0
index b4b01d47e568..0884a581fc37 100644

include/uapi/linux/kvm.h struct kvm_enc_region { · line 1494

Quoted 5 lines
include/uapi/linux/kvm.h +1 -0
@@ -1490,6 +1490,7 @@ struct kvm_enc_region {
   #define KVM_GET_PAGE_ENC_BITMAP  _IOW(KVMIO, 0xc5, struct kvm_page_enc_bitmap)
   #define KVM_SET_PAGE_ENC_BITMAP  _IOW(KVMIO, 0xc6, struct kvm_page_enc_bitmap)
+#define KVM_PAGE_ENC_BITMAP_RESET  _IO(KVMIO, 0xc7)
   /* Secure Encrypted Virtualization command */
   enum sev_cmd_id {
Quoted 4 lines

Doesn't this overlap with the set ioctl? Yes, obviously, you have to copy the new value down and do a bit more work, but I don't think resetting the bitmap is going to be the bottleneck on reboot. Seems excessive to add another ioctl for this.

Quoted 6 lines

The set ioctl is generally available/provided for the incoming VM to setup the page encryption bitmap, this reset ioctl is meant for the source VM as a simple interface to reset the whole page encryption bitmap.

Thanks,
Ashish

Quoted 7 lines

Hey Ashish,

These seem very overlapping. I think this API should be refactored a bit.

1) Use kvm_vm_ioctl_enable_cap to control whether or not this hypercall (and related feature bit) is offered to the VM, and also the size of the buffer.

If you look at patch 13/14, i have added a new kvm para feature called "KVM_FEATURE_SEV_LIVE_MIGRATION" which indicates host support for SEV Live Migration and a new Custom MSR which the guest does a wrmsr to enable the Live Migration feature, so this is like the enable cap support.

There are further extensions to this support i am adding, so patch 13/14 of this patch-set is still being enhanced and will have full support when i repost next.

Quoted 5 lines

2) Use set for manipulating values in the bitmap, including resetting the bitmap. Set the bitmap pointer to null if you want to reset to all 0xFFs. When the bitmap pointer is set, it should set the values to exactly what is pointed at, instead of only clearing bits, as is done currently.

As i mentioned in my earlier email, the set api is supposed to be for the incoming VM, but if you really need to use it for the outgoing VM then it can be modified.

Quoted 9 lines

3) Use get for fetching values from the kernel. Personally, I'd require alignment of the base GFN to a multiple of 8 (but the number of pages could be whatever), so you can just use a memcpy. Optionally, you may want some way to tell userspace the size of the existing buffer, so it can ensure that it can ask for the entire buffer without having to track the size in usermode (not strictly necessary, but nice to have since it ensures that there is only one place that has to manage this value).

Quoted 8 lines
Quoted 2 lines

If you want to expand or contract the bitmap, you can use enable cap to adjust the size.

As being discussed on the earlier mail thread, we are doing this dynamically now by computing the guest RAM size when the set_user_memory_region ioctl is invoked. I believe that should handle the hot-plug and hot-unplug events too, as any hot memory updates will need KVM memslots to be updated.

Ahh, sorry, forgot you mentioned this: yes this can work. Host needs to be able to decide not to allocate, but this should be workable.

Quoted 12 lines
Quoted 11 lines

If you don't want to offer the hypercall to the guest, don't call the enable cap.
This API avoids using up another ioctl. Ioctl space is somewhat scarce. It also gives userspace fine grained control over the buffer, so it can support both hot-plug and hot-unplug (or at the very least it is not obviously incompatible with those). It also gives userspace control over whether or not the feature is offered. The hypercall isn't free, and being able to tell guests to not call when the host wasn't going to migrate it anyway will be useful.

Quoted 5 lines

As i mentioned above, now the host indicates if it supports the Live Migration feature and the feature and the hypercall are only enabled on the host when the guest checks for this support and does a wrmsr() to enable the feature. Also the guest will not make the hypercall if the host does not indicate support for it.

If my read of those patches was correct, the host will always advertise support for the hypercall. And the only bit controlling whether or not the hypercall is advertised is essentially the kernel version. You need to rollout a new kernel to disable the hypercall.

An enable cap could give the host control of this feature bit. It could also give the host control of whether or not the bitmaps are allocated. This is important since I assume not every SEV VM is planned to be migrated. And if the host does not plan to migrate it probably doesn't want to waste memory or cycles managing this bitmap.

Thanks,
Steve

#79 lore raw

Re: [PATCH v6 12/14] KVM: x86: Introduce KVM_PAGE_ENC_BITMAP_RESET ioctl

Quoted 218 lines · Steve Rutherford
Quoted 189 lines · Ashish Kalra

Hello Steve,

Quoted 148 lines · Steve Rutherford
Quoted 139 lines · Ashish Kalra

Hello Steve,

Quoted 130 lines · Steve Rutherford
Quoted 88 lines · Krish Sadhukhan
Quoted 85 lines · Ashish Kalra
Quoted 84 lines · Krish Sadhukhan
Quoted 75 lines · Ashish Kalra

From: Ashish Kalra <[email protected]>

This ioctl can be used by the application to reset the page encryption bitmap managed by the KVM driver. A typical usage for this ioctl is on VM reboot, on reboot, we must reinitialize the bitmap.

---
   Documentation/virt/kvm/api.rst  | 13 +++++++++++++
   arch/x86/include/asm/kvm_host.h |  1 +
   arch/x86/kvm/svm.c              | 16 ++++++++++++++++
   arch/x86/kvm/x86.c              |  6 ++++++
   include/uapi/linux/kvm.h        |  1 +
   5 files changed, 37 insertions(+)
Documentation/virt/kvm/api.rst +13 -0
index 4d1004a154f6..a11326ccc51d 100644
@@ -4698,6 +4698,19 @@ During the guest live migration the outgoing guest exports its page encryption
   bitmap, the KVM_SET_PAGE_ENC_BITMAP can be used to build the page encryption
   bitmap for an incoming guest.
+4.127 KVM_PAGE_ENC_BITMAP_RESET (vm ioctl)
+-----------------------------------------
+
+:Capability: basic
+:Architectures: x86
+:Type: vm ioctl
+:Parameters: none
+:Returns: 0 on success, -1 on error
+
+The KVM_PAGE_ENC_BITMAP_RESET is used to reset the guest's page encryption
+bitmap during guest reboot and this is only done on the guest's boot vCPU.
+
+
   5. The kvm_run structure
   ========================
arch/x86/include/asm/kvm_host.h +1 -0
index d30f770aaaea..a96ef6338cd2 100644
@@ -1273,6 +1273,7 @@ struct kvm_x86_ops {
                            struct kvm_page_enc_bitmap *bmap);
    int (*set_page_enc_bitmap)(struct kvm *kvm,
                            struct kvm_page_enc_bitmap *bmap);
+   int (*reset_page_enc_bitmap)(struct kvm *kvm);
   };
   struct kvm_arch_async_pf {
arch/x86/kvm/svm.c +15 -0
index 313343a43045..c99b0207a443 100644
@@ -7797,6 +7797,21 @@ static int svm_set_page_enc_bitmap(struct kvm *kvm,
    return ret;
   }
+static int svm_reset_page_enc_bitmap(struct kvm *kvm)
+{
+   struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
+
+   if (!sev_guest(kvm))
+           return -ENOTTY;
+
+   mutex_lock(&kvm->lock);
+   /* by default all pages should be marked encrypted */
+   if (sev->page_enc_bmap_size)
+           bitmap_fill(sev->page_enc_bmap, sev->page_enc_bmap_size);
+   mutex_unlock(&kvm->lock);
+   return 0;
+}
+
   static int svm_mem_enc_op(struct kvm *kvm, void __user *argp)
   {
    struct kvm_sev_cmd sev_cmd;

arch/x86/kvm/svm.c static struct kvm_x86_ops svm_x86_ops __ro_after_init = { · line 8221

Quoted 4 lines
arch/x86/kvm/svm.c +1 -0
@@ -8203,6 +8218,7 @@ static struct kvm_x86_ops svm_x86_ops __ro_after_init = {
    .page_enc_status_hc = svm_page_enc_status_hc,
    .get_page_enc_bitmap = svm_get_page_enc_bitmap,
    .set_page_enc_bitmap = svm_set_page_enc_bitmap,
+   .reset_page_enc_bitmap = svm_reset_page_enc_bitmap,

We don't need to initialize the intel ops to NULL ? It's not initialized in the previous patch either.

Quoted 38 lines
Quoted 3 lines
Quoted 1 line

arch/x86/kvm/svm.c static struct kvm_x86_ops svm_x86_ops __ro_after_init = { · line 8222

Quoted 1 line
arch/x86/kvm/svm.c +0 -0
   };

This struct is declared as "static storage", so won't the non-initialized members be 0 ?

Correct. Although, I see that 'nested_enable_evmcs' is explicitly initialized. We should maintain the convention, perhaps.

Quoted 30 lines
Quoted 30 lines
Quoted 24 lines
arch/x86/kvm/svm.c +0 -0
   static int __init svm_init(void)
arch/x86/kvm/x86.c +6 -0
index 05e953b2ec61..2127ed937f53 100644
@@ -5250,6 +5250,12 @@ long kvm_arch_vm_ioctl(struct file *filp,
                    r = kvm_x86_ops->set_page_enc_bitmap(kvm, &bitmap);
            break;
    }
+   case KVM_PAGE_ENC_BITMAP_RESET: {
+           r = -ENOTTY;
+           if (kvm_x86_ops->reset_page_enc_bitmap)
+                   r = kvm_x86_ops->reset_page_enc_bitmap(kvm);
+           break;
+   }
    default:
            r = -ENOTTY;
    }
include/uapi/linux/kvm.h +0 -0
index b4b01d47e568..0884a581fc37 100644

include/uapi/linux/kvm.h struct kvm_enc_region { · line 1494

Quoted 5 lines
include/uapi/linux/kvm.h +1 -0
@@ -1490,6 +1490,7 @@ struct kvm_enc_region {
   #define KVM_GET_PAGE_ENC_BITMAP  _IOW(KVMIO, 0xc5, struct kvm_page_enc_bitmap)
   #define KVM_SET_PAGE_ENC_BITMAP  _IOW(KVMIO, 0xc6, struct kvm_page_enc_bitmap)
+#define KVM_PAGE_ENC_BITMAP_RESET  _IO(KVMIO, 0xc7)
   /* Secure Encrypted Virtualization command */
   enum sev_cmd_id {
Quoted 4 lines

Doesn't this overlap with the set ioctl? Yes, obviously, you have to copy the new value down and do a bit more work, but I don't think resetting the bitmap is going to be the bottleneck on reboot. Seems excessive to add another ioctl for this.

Quoted 6 lines

The set ioctl is generally available/provided for the incoming VM to setup the page encryption bitmap, this reset ioctl is meant for the source VM as a simple interface to reset the whole page encryption bitmap.

Thanks,
Ashish

Quoted 7 lines

Hey Ashish,

These seem very overlapping. I think this API should be refactored a bit.

1) Use kvm_vm_ioctl_enable_cap to control whether or not this hypercall (and related feature bit) is offered to the VM, and also the size of the buffer.

If you look at patch 13/14, i have added a new kvm para feature called "KVM_FEATURE_SEV_LIVE_MIGRATION" which indicates host support for SEV Live Migration and a new Custom MSR which the guest does a wrmsr to enable the Live Migration feature, so this is like the enable cap support.

There are further extensions to this support i am adding, so patch 13/14 of this patch-set is still being enhanced and will have full support when i repost next.

Quoted 5 lines

2) Use set for manipulating values in the bitmap, including resetting the bitmap. Set the bitmap pointer to null if you want to reset to all 0xFFs. When the bitmap pointer is set, it should set the values to exactly what is pointed at, instead of only clearing bits, as is done currently.

As i mentioned in my earlier email, the set api is supposed to be for the incoming VM, but if you really need to use it for the outgoing VM then it can be modified.

Quoted 9 lines

3) Use get for fetching values from the kernel. Personally, I'd require alignment of the base GFN to a multiple of 8 (but the number of pages could be whatever), so you can just use a memcpy. Optionally, you may want some way to tell userspace the size of the existing buffer, so it can ensure that it can ask for the entire buffer without having to track the size in usermode (not strictly necessary, but nice to have since it ensures that there is only one place that has to manage this value).

Quoted 8 lines
Quoted 2 lines

If you want to expand or contract the bitmap, you can use enable cap to adjust the size.

As being discussed on the earlier mail thread, we are doing this dynamically now by computing the guest RAM size when the set_user_memory_region ioctl is invoked. I believe that should handle the hot-plug and hot-unplug events too, as any hot memory updates will need KVM memslots to be updated.

Ahh, sorry, forgot you mentioned this: yes this can work. Host needs to be able to decide not to allocate, but this should be workable.

Quoted 12 lines
Quoted 11 lines

If you don't want to offer the hypercall to the guest, don't call the enable cap.
This API avoids using up another ioctl. Ioctl space is somewhat scarce. It also gives userspace fine grained control over the buffer, so it can support both hot-plug and hot-unplug (or at the very least it is not obviously incompatible with those). It also gives userspace control over whether or not the feature is offered. The hypercall isn't free, and being able to tell guests to not call when the host wasn't going to migrate it anyway will be useful.

Quoted 5 lines

As i mentioned above, now the host indicates if it supports the Live Migration feature and the feature and the hypercall are only enabled on the host when the guest checks for this support and does a wrmsr() to enable the feature. Also the guest will not make the hypercall if the host does not indicate support for it.

Quoted 4 lines

If my read of those patches was correct, the host will always advertise support for the hypercall. And the only bit controlling whether or not the hypercall is advertised is essentially the kernel version. You need to rollout a new kernel to disable the hypercall.

Ahh, awesome, I see I misunderstood how the CPUID bits get passed
through: usermode can still override them. Forgot about the back and forth for CPUID with usermode. My point about informing the guest kernel is clearly moot. The host still needs the ability to prevent allocations, but that is more minor. Maybe use a flag on the memslots directly?

#80 lore raw

Re: [PATCH v6 12/14] KVM: x86: Introduce KVM_PAGE_ENC_BITMAP_RESET ioctl

Quoted 226 lines · Steve Rutherford
Quoted 218 lines · Steve Rutherford
Quoted 189 lines · Ashish Kalra

Hello Steve,

Quoted 148 lines · Steve Rutherford
Quoted 139 lines · Ashish Kalra

Hello Steve,

Quoted 130 lines · Steve Rutherford
Quoted 88 lines · Krish Sadhukhan
Quoted 85 lines · Ashish Kalra
Quoted 84 lines · Krish Sadhukhan
Quoted 75 lines · Ashish Kalra

From: Ashish Kalra <[email protected]>

This ioctl can be used by the application to reset the page encryption bitmap managed by the KVM driver. A typical usage for this ioctl is on VM reboot, on reboot, we must reinitialize the bitmap.

---
   Documentation/virt/kvm/api.rst  | 13 +++++++++++++
   arch/x86/include/asm/kvm_host.h |  1 +
   arch/x86/kvm/svm.c              | 16 ++++++++++++++++
   arch/x86/kvm/x86.c              |  6 ++++++
   include/uapi/linux/kvm.h        |  1 +
   5 files changed, 37 insertions(+)
Documentation/virt/kvm/api.rst +13 -0
index 4d1004a154f6..a11326ccc51d 100644
@@ -4698,6 +4698,19 @@ During the guest live migration the outgoing guest exports its page encryption
   bitmap, the KVM_SET_PAGE_ENC_BITMAP can be used to build the page encryption
   bitmap for an incoming guest.
+4.127 KVM_PAGE_ENC_BITMAP_RESET (vm ioctl)
+-----------------------------------------
+
+:Capability: basic
+:Architectures: x86
+:Type: vm ioctl
+:Parameters: none
+:Returns: 0 on success, -1 on error
+
+The KVM_PAGE_ENC_BITMAP_RESET is used to reset the guest's page encryption
+bitmap during guest reboot and this is only done on the guest's boot vCPU.
+
+
   5. The kvm_run structure
   ========================
arch/x86/include/asm/kvm_host.h +1 -0
index d30f770aaaea..a96ef6338cd2 100644
@@ -1273,6 +1273,7 @@ struct kvm_x86_ops {
                            struct kvm_page_enc_bitmap *bmap);
    int (*set_page_enc_bitmap)(struct kvm *kvm,
                            struct kvm_page_enc_bitmap *bmap);
+   int (*reset_page_enc_bitmap)(struct kvm *kvm);
   };
   struct kvm_arch_async_pf {
arch/x86/kvm/svm.c +15 -0
index 313343a43045..c99b0207a443 100644
@@ -7797,6 +7797,21 @@ static int svm_set_page_enc_bitmap(struct kvm *kvm,
    return ret;
   }
+static int svm_reset_page_enc_bitmap(struct kvm *kvm)
+{
+   struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
+
+   if (!sev_guest(kvm))
+           return -ENOTTY;
+
+   mutex_lock(&kvm->lock);
+   /* by default all pages should be marked encrypted */
+   if (sev->page_enc_bmap_size)
+           bitmap_fill(sev->page_enc_bmap, sev->page_enc_bmap_size);
+   mutex_unlock(&kvm->lock);
+   return 0;
+}
+
   static int svm_mem_enc_op(struct kvm *kvm, void __user *argp)
   {
    struct kvm_sev_cmd sev_cmd;

arch/x86/kvm/svm.c static struct kvm_x86_ops svm_x86_ops __ro_after_init = { · line 8221

Quoted 4 lines
arch/x86/kvm/svm.c +1 -0
@@ -8203,6 +8218,7 @@ static struct kvm_x86_ops svm_x86_ops __ro_after_init = {
    .page_enc_status_hc = svm_page_enc_status_hc,
    .get_page_enc_bitmap = svm_get_page_enc_bitmap,
    .set_page_enc_bitmap = svm_set_page_enc_bitmap,
+   .reset_page_enc_bitmap = svm_reset_page_enc_bitmap,

We don't need to initialize the intel ops to NULL ? It's not initialized in the previous patch either.

Quoted 38 lines
Quoted 3 lines
Quoted 1 line

arch/x86/kvm/svm.c static struct kvm_x86_ops svm_x86_ops __ro_after_init = { · line 8222

Quoted 1 line
arch/x86/kvm/svm.c +0 -0
   };

This struct is declared as "static storage", so won't the non-initialized members be 0 ?

Correct. Although, I see that 'nested_enable_evmcs' is explicitly initialized. We should maintain the convention, perhaps.

Quoted 30 lines
Quoted 30 lines
Quoted 24 lines
arch/x86/kvm/svm.c +0 -0
   static int __init svm_init(void)
arch/x86/kvm/x86.c +6 -0
index 05e953b2ec61..2127ed937f53 100644
@@ -5250,6 +5250,12 @@ long kvm_arch_vm_ioctl(struct file *filp,
                    r = kvm_x86_ops->set_page_enc_bitmap(kvm, &bitmap);
            break;
    }
+   case KVM_PAGE_ENC_BITMAP_RESET: {
+           r = -ENOTTY;
+           if (kvm_x86_ops->reset_page_enc_bitmap)
+                   r = kvm_x86_ops->reset_page_enc_bitmap(kvm);
+           break;
+   }
    default:
            r = -ENOTTY;
    }
include/uapi/linux/kvm.h +0 -0
index b4b01d47e568..0884a581fc37 100644

include/uapi/linux/kvm.h struct kvm_enc_region { · line 1494

Quoted 5 lines
include/uapi/linux/kvm.h +1 -0
@@ -1490,6 +1490,7 @@ struct kvm_enc_region {
   #define KVM_GET_PAGE_ENC_BITMAP  _IOW(KVMIO, 0xc5, struct kvm_page_enc_bitmap)
   #define KVM_SET_PAGE_ENC_BITMAP  _IOW(KVMIO, 0xc6, struct kvm_page_enc_bitmap)
+#define KVM_PAGE_ENC_BITMAP_RESET  _IO(KVMIO, 0xc7)
   /* Secure Encrypted Virtualization command */
   enum sev_cmd_id {
Quoted 4 lines

Doesn't this overlap with the set ioctl? Yes, obviously, you have to copy the new value down and do a bit more work, but I don't think resetting the bitmap is going to be the bottleneck on reboot. Seems excessive to add another ioctl for this.

Quoted 6 lines

The set ioctl is generally available/provided for the incoming VM to setup the page encryption bitmap, this reset ioctl is meant for the source VM as a simple interface to reset the whole page encryption bitmap.

Thanks,
Ashish

Quoted 7 lines

Hey Ashish,

These seem very overlapping. I think this API should be refactored a bit.

1) Use kvm_vm_ioctl_enable_cap to control whether or not this hypercall (and related feature bit) is offered to the VM, and also the size of the buffer.

If you look at patch 13/14, i have added a new kvm para feature called "KVM_FEATURE_SEV_LIVE_MIGRATION" which indicates host support for SEV Live Migration and a new Custom MSR which the guest does a wrmsr to enable the Live Migration feature, so this is like the enable cap support.

There are further extensions to this support i am adding, so patch 13/14 of this patch-set is still being enhanced and will have full support when i repost next.

Quoted 5 lines

2) Use set for manipulating values in the bitmap, including resetting the bitmap. Set the bitmap pointer to null if you want to reset to all 0xFFs. When the bitmap pointer is set, it should set the values to exactly what is pointed at, instead of only clearing bits, as is done currently.

As i mentioned in my earlier email, the set api is supposed to be for the incoming VM, but if you really need to use it for the outgoing VM then it can be modified.

Quoted 9 lines

3) Use get for fetching values from the kernel. Personally, I'd require alignment of the base GFN to a multiple of 8 (but the number of pages could be whatever), so you can just use a memcpy. Optionally, you may want some way to tell userspace the size of the existing buffer, so it can ensure that it can ask for the entire buffer without having to track the size in usermode (not strictly necessary, but nice to have since it ensures that there is only one place that has to manage this value).

Quoted 8 lines
Quoted 2 lines

If you want to expand or contract the bitmap, you can use enable cap to adjust the size.

As being discussed on the earlier mail thread, we are doing this dynamically now by computing the guest RAM size when the set_user_memory_region ioctl is invoked. I believe that should handle the hot-plug and hot-unplug events too, as any hot memory updates will need KVM memslots to be updated.

Ahh, sorry, forgot you mentioned this: yes this can work. Host needs to be able to decide not to allocate, but this should be workable.

Quoted 12 lines
Quoted 11 lines

If you don't want to offer the hypercall to the guest, don't call the enable cap.
This API avoids using up another ioctl. Ioctl space is somewhat scarce. It also gives userspace fine grained control over the buffer, so it can support both hot-plug and hot-unplug (or at the very least it is not obviously incompatible with those). It also gives userspace control over whether or not the feature is offered. The hypercall isn't free, and being able to tell guests to not call when the host wasn't going to migrate it anyway will be useful.

Quoted 5 lines

As i mentioned above, now the host indicates if it supports the Live Migration feature and the feature and the hypercall are only enabled on the host when the guest checks for this support and does a wrmsr() to enable the feature. Also the guest will not make the hypercall if the host does not indicate support for it.

Quoted 4 lines

If my read of those patches was correct, the host will always advertise support for the hypercall. And the only bit controlling whether or not the hypercall is advertised is essentially the kernel version. You need to rollout a new kernel to disable the hypercall.

Quoted 6 lines

Ahh, awesome, I see I misunderstood how the CPUID bits get passed
through: usermode can still override them. Forgot about the back and forth for CPUID with usermode. My point about informing the guest kernel is clearly moot. The host still needs the ability to prevent allocations, but that is more minor. Maybe use a flag on the memslots directly?

On second thought: burning the memslot flag for 30mb per tb of VM seems like a waste.

#81 lore raw

RE: [PATCH v6 12/14] KVM: x86: Introduce KVM_PAGE_ENC_BITMAP_RESET ioctl

[AMD Official Use Only - Internal Distribution Only]

Hello Steve,

-----Original Message-----
From: Steve Rutherford <[email protected]>
Sent: Friday, April 10, 2020 3:19 PM
To: Kalra, Ashish <[email protected]>
Cc: Krish Sadhukhan <[email protected]>; Paolo Bonzini <[email protected]>; Thomas Gleixner <[email protected]>; Ingo Molnar <[email protected]>; H. Peter Anvin <[email protected]>; Joerg Roedel <[email protected]>; Borislav Petkov <[email protected]>; Lendacky, Thomas <[email protected]>; X86 ML <[email protected]>; KVM list <[email protected]>; LKML <[email protected]>; David Rientjes <[email protected]>; Andy Lutomirski <[email protected]>; Singh, Brijesh <[email protected]>
Subject: Re: [PATCH v6 12/14] KVM: x86: Introduce KVM_PAGE_ENC_BITMAP_RESET ioctl

Quoted 234 lines · Steve Rutherford
Quoted 226 lines · Steve Rutherford
Quoted 197 lines · Ashish Kalra

Hello Steve,

Quoted 156 lines · Steve Rutherford
Quoted 147 lines · Ashish Kalra

Hello Steve,

Quoted 138 lines · Steve Rutherford
Quoted 93 lines · Krish Sadhukhan
Quoted 90 lines · Ashish Kalra
Quoted 89 lines · Krish Sadhukhan
Quoted 78 lines · Ashish Kalra

From: Ashish Kalra <[email protected]>

This ioctl can be used by the application to reset the page encryption bitmap managed by the KVM driver. A typical usage for this ioctl is on VM reboot, on reboot, we must reinitialize the bitmap.

---
   Documentation/virt/kvm/api.rst  | 13 +++++++++++++
   arch/x86/include/asm/kvm_host.h |  1 +
   arch/x86/kvm/svm.c              | 16 ++++++++++++++++
   arch/x86/kvm/x86.c              |  6 ++++++
   include/uapi/linux/kvm.h        |  1 +
   5 files changed, 37 insertions(+)
(quoted diff) +0 -0

b/Documentation/virt/kvm/api.rst index
4d1004a154f6..a11326ccc51d 100644

Documentation/virt/kvm/api.rst +13 -0
@@ -4698,6 +4698,19 @@ During the guest live migration the outgoing guest exports its page encryption
   bitmap, the KVM_SET_PAGE_ENC_BITMAP can be used to build the page encryption
   bitmap for an incoming guest.
+4.127 KVM_PAGE_ENC_BITMAP_RESET (vm ioctl)
+-----------------------------------------
+
+:Capability: basic
+:Architectures: x86
+:Type: vm ioctl
+:Parameters: none
+:Returns: 0 on success, -1 on error
+
+The KVM_PAGE_ENC_BITMAP_RESET is used to reset the 
+guest's page encryption bitmap during guest reboot and this is only done on the guest's boot vCPU.
+
+
   5. The kvm_run structure
   ======================== diff --git 

a/arch/x86/include/asm/kvm_host.h
b/arch/x86/include/asm/kvm_host.h index
d30f770aaaea..a96ef6338cd2 100644

arch/x86/include/asm/kvm_host.h +1 -0
@@ -1273,6 +1273,7 @@ struct kvm_x86_ops {
                            struct kvm_page_enc_bitmap *bmap);
    int (*set_page_enc_bitmap)(struct kvm *kvm,
                            struct kvm_page_enc_bitmap 
 *bmap);
+   int (*reset_page_enc_bitmap)(struct kvm *kvm);
   };
   struct kvm_arch_async_pf { diff --git 

a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c index 313343a43045..c99b0207a443 100644

arch/x86/kvm/svm.c +16 -0
@@ -7797,6 +7797,21 @@ static int svm_set_page_enc_bitmap(struct kvm *kvm,
    return ret;
   }
+static int svm_reset_page_enc_bitmap(struct kvm *kvm) 
+{
+   struct kvm_sev_info *sev = 
+&to_kvm_svm(kvm)->sev_info;
+
+   if (!sev_guest(kvm))
+           return -ENOTTY;
+
+   mutex_lock(&kvm->lock);
+   /* by default all pages should be marked encrypted */
+   if (sev->page_enc_bmap_size)
+           bitmap_fill(sev->page_enc_bmap, sev->page_enc_bmap_size);
+   mutex_unlock(&kvm->lock);
+   return 0;
+}
+
   static int svm_mem_enc_op(struct kvm *kvm, void __user *argp)
   {
    struct kvm_sev_cmd sev_cmd; @@ -8203,6 +8218,7 @@ 
Quoted 6 lines

static struct kvm_x86_ops svm_x86_ops __ro_after_init = {
.page_enc_status_hc = svm_page_enc_status_hc, .get_page_enc_bitmap = svm_get_page_enc_bitmap, .set_page_enc_bitmap = svm_set_page_enc_bitmap,
+ .reset_page_enc_bitmap =
+ svm_reset_page_enc_bitmap,

We don't need to initialize the intel ops to NULL ? It's not initialized in the previous patch either.

Quoted 41 lines
Quoted 3 lines
Quoted 1 line
Quoted 1 line
   };

This struct is declared as "static storage", so won't the non-initialized members be 0 ?

Correct. Although, I see that 'nested_enable_evmcs' is explicitly initialized. We should maintain the convention, perhaps.

Quoted 33 lines
Quoted 33 lines
Quoted 25 lines
   static int __init svm_init(void) diff --git 

a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 05e953b2ec61..2127ed937f53 100644

arch/x86/kvm/x86.c +6 -0
@@ -5250,6 +5250,12 @@ long kvm_arch_vm_ioctl(struct file *filp,
                    r = kvm_x86_ops->set_page_enc_bitmap(kvm, &bitmap);
            break;
    }
+   case KVM_PAGE_ENC_BITMAP_RESET: {
+           r = -ENOTTY;
+           if (kvm_x86_ops->reset_page_enc_bitmap)
+                   r = kvm_x86_ops->reset_page_enc_bitmap(kvm);
+           break;
+   }
    default:
            r = -ENOTTY;
    }
(quoted diff) +0 -0

b/include/uapi/linux/kvm.h index
b4b01d47e568..0884a581fc37 100644

include/uapi/linux/kvm.h struct kvm_enc_region { · line 1495

Quoted 6 lines
include/uapi/linux/kvm.h +1 -0
@@ -1490,6 +1490,7 @@ struct kvm_enc_region {
   #define KVM_GET_PAGE_ENC_BITMAP  _IOW(KVMIO, 0xc5, struct kvm_page_enc_bitmap)
   #define KVM_SET_PAGE_ENC_BITMAP  _IOW(KVMIO, 0xc6, 
 struct kvm_page_enc_bitmap)
+#define KVM_PAGE_ENC_BITMAP_RESET  _IO(KVMIO, 0xc7)
   /* Secure Encrypted Virtualization command */
   enum sev_cmd_id {

Reviewed-by: Krish Sadhukhan
<[email protected]>

Quoted 4 lines

Doesn't this overlap with the set ioctl? Yes, obviously, you have to copy the new value down and do a bit more work, but I don't think resetting the bitmap is going to be the bottleneck on reboot. Seems excessive to add another ioctl for this.

Quoted 6 lines

The set ioctl is generally available/provided for the incoming
VM to setup the page encryption bitmap, this reset ioctl is
meant for the source VM as a simple interface to reset the whole page encryption bitmap.

Thanks,
Ashish

Quoted 7 lines

Hey Ashish,

These seem very overlapping. I think this API should be refactored a bit.

1) Use kvm_vm_ioctl_enable_cap to control whether or not this hypercall (and related feature bit) is offered to the VM, and also the size of the buffer.

If you look at patch 13/14, i have added a new kvm para feature called "KVM_FEATURE_SEV_LIVE_MIGRATION" which indicates host support for SEV Live Migration and a new Custom MSR which the guest does a wrmsr to enable the Live Migration feature, so this is like the enable cap support.

There are further extensions to this support i am adding, so patch 13/14 of this patch-set is still being enhanced and will have full support when i repost next.

Quoted 5 lines

2) Use set for manipulating values in the bitmap, including resetting the bitmap. Set the bitmap pointer to null if you want to reset to all 0xFFs. When the bitmap pointer is set, it should set the values to exactly what is pointed at, instead of only clearing bits, as is done currently.

As i mentioned in my earlier email, the set api is supposed to be for the incoming VM, but if you really need to use it for the outgoing VM then it can be modified.

Quoted 9 lines

3) Use get for fetching values from the kernel. Personally, I'd require alignment of the base GFN to a multiple of 8 (but the number of pages could be whatever), so you can just use a memcpy. Optionally, you may want some way to tell userspace the size of the existing buffer, so it can ensure that it can ask for the entire buffer without having to track the size in usermode (not strictly necessary, but nice to have since it ensures that there is only one place that has to manage this value).

Quoted 8 lines
Quoted 2 lines

If you want to expand or contract the bitmap, you can use enable cap to adjust the size.

As being discussed on the earlier mail thread, we are doing this dynamically now by computing the guest RAM size when the set_user_memory_region ioctl is invoked. I believe that should handle the hot-plug and hot-unplug events too, as any hot memory updates will need KVM memslots to be updated.

Ahh, sorry, forgot you mentioned this: yes this can work. Host needs to be able to decide not to allocate, but this should be workable.

Quoted 12 lines
Quoted 11 lines

If you don't want to offer the hypercall to the guest, don't call the enable cap.
This API avoids using up another ioctl. Ioctl space is somewhat scarce. It also gives userspace fine grained control over the buffer, so it can support both hot-plug and hot-unplug (or at the very least it is not obviously incompatible with those). It also gives userspace control over whether or not the feature is offered. The hypercall isn't free, and being able to tell guests to not call when the host wasn't going to migrate it anyway will be useful.

Quoted 5 lines

As i mentioned above, now the host indicates if it supports the Live Migration feature and the feature and the hypercall are only enabled on the host when the guest checks for this support and does a wrmsr() to enable the feature. Also the guest will not make the hypercall if the host does not indicate support for it.

Quoted 4 lines

If my read of those patches was correct, the host will always advertise support for the hypercall. And the only bit controlling whether or not the hypercall is advertised is essentially the kernel version. You need to rollout a new kernel to disable the hypercall.

Quoted 7 lines

Ahh, awesome, I see I misunderstood how the CPUID bits get passed
through: usermode can still override them. Forgot about the back and forth for CPUID with usermode. My point about informing the guest kernel is clearly moot. The host still needs the ability to prevent allocations, but that is more minor. Maybe use a flag on the memslots directly?
On second thought: burning the memslot flag for 30mb per tb of VM seems like a waste.

Currently, I am still using the approach of a "unified" page encryption bitmap instead of a bitmap per memslot, with the main change being that the resizing is only done whenever there are any updates in memslots, when memslots are updated using the kvm_arch_commit_memory_region() interface.

Thanks,
Ashish

#82 lore raw

Re: [PATCH v6 12/14] KVM: x86: Introduce KVM_PAGE_ENC_BITMAP_RESET ioctl

Quoted 236 lines · Ashish

[AMD Official Use Only - Internal Distribution Only]

Hello Steve,

-----Original Message-----
From: Steve Rutherford <[email protected]>
Sent: Friday, April 10, 2020 3:19 PM
To: Kalra, Ashish <[email protected]>
Cc: Krish Sadhukhan <[email protected]>; Paolo Bonzini <[email protected]>; Thomas Gleixner <[email protected]>; Ingo Molnar <[email protected]>; H. Peter Anvin <[email protected]>; Joerg Roedel <[email protected]>; Borislav Petkov <[email protected]>; Lendacky, Thomas <[email protected]>; X86 ML <[email protected]>; KVM list <[email protected]>; LKML <[email protected]>; David Rientjes <[email protected]>; Andy Lutomirski <[email protected]>; Singh, Brijesh <[email protected]>
Subject: Re: [PATCH v6 12/14] KVM: x86: Introduce KVM_PAGE_ENC_BITMAP_RESET ioctl

Quoted 217 lines · Steve Rutherford
Quoted 211 lines · Steve Rutherford
Quoted 186 lines · Ashish Kalra

Hello Steve,

Quoted 148 lines · Steve Rutherford
Quoted 140 lines · Ashish Kalra

Hello Steve,

Quoted 133 lines · Steve Rutherford
Quoted 91 lines · Krish Sadhukhan
Quoted 89 lines · Ashish Kalra
Quoted 88 lines · Krish Sadhukhan
Quoted 78 lines · Ashish Kalra

From: Ashish Kalra <[email protected]>

This ioctl can be used by the application to reset the page encryption bitmap managed by the KVM driver. A typical usage for this ioctl is on VM reboot, on reboot, we must reinitialize the bitmap.

---
   Documentation/virt/kvm/api.rst  | 13 +++++++++++++
   arch/x86/include/asm/kvm_host.h |  1 +
   arch/x86/kvm/svm.c              | 16 ++++++++++++++++
   arch/x86/kvm/x86.c              |  6 ++++++
   include/uapi/linux/kvm.h        |  1 +
   5 files changed, 37 insertions(+)
(quoted diff) +0 -0

b/Documentation/virt/kvm/api.rst index
4d1004a154f6..a11326ccc51d 100644

Documentation/virt/kvm/api.rst +13 -0
@@ -4698,6 +4698,19 @@ During the guest live migration the outgoing guest exports its page encryption
   bitmap, the KVM_SET_PAGE_ENC_BITMAP can be used to build the page encryption
   bitmap for an incoming guest.
+4.127 KVM_PAGE_ENC_BITMAP_RESET (vm ioctl)
+-----------------------------------------
+
+:Capability: basic
+:Architectures: x86
+:Type: vm ioctl
+:Parameters: none
+:Returns: 0 on success, -1 on error
+
+The KVM_PAGE_ENC_BITMAP_RESET is used to reset the 
+guest's page encryption bitmap during guest reboot and this is only done on the guest's boot vCPU.
+
+
   5. The kvm_run structure
   ======================== diff --git 

a/arch/x86/include/asm/kvm_host.h
b/arch/x86/include/asm/kvm_host.h index
d30f770aaaea..a96ef6338cd2 100644

arch/x86/include/asm/kvm_host.h +1 -0
@@ -1273,6 +1273,7 @@ struct kvm_x86_ops {
                            struct kvm_page_enc_bitmap *bmap);
    int (*set_page_enc_bitmap)(struct kvm *kvm,
                            struct kvm_page_enc_bitmap 
 *bmap);
+   int (*reset_page_enc_bitmap)(struct kvm *kvm);
   };
   struct kvm_arch_async_pf { diff --git 

a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c index 313343a43045..c99b0207a443 100644

arch/x86/kvm/svm.c +16 -0
@@ -7797,6 +7797,21 @@ static int svm_set_page_enc_bitmap(struct kvm *kvm,
    return ret;
   }
+static int svm_reset_page_enc_bitmap(struct kvm *kvm) 
+{
+   struct kvm_sev_info *sev = 
+&to_kvm_svm(kvm)->sev_info;
+
+   if (!sev_guest(kvm))
+           return -ENOTTY;
+
+   mutex_lock(&kvm->lock);
+   /* by default all pages should be marked encrypted */
+   if (sev->page_enc_bmap_size)
+           bitmap_fill(sev->page_enc_bmap, sev->page_enc_bmap_size);
+   mutex_unlock(&kvm->lock);
+   return 0;
+}
+
   static int svm_mem_enc_op(struct kvm *kvm, void __user *argp)
   {
    struct kvm_sev_cmd sev_cmd; @@ -8203,6 +8218,7 @@ 
Quoted 6 lines

static struct kvm_x86_ops svm_x86_ops __ro_after_init = {
.page_enc_status_hc = svm_page_enc_status_hc, .get_page_enc_bitmap = svm_get_page_enc_bitmap, .set_page_enc_bitmap = svm_set_page_enc_bitmap,
+ .reset_page_enc_bitmap =
+ svm_reset_page_enc_bitmap,

We don't need to initialize the intel ops to NULL ? It's not initialized in the previous patch either.

Quoted 39 lines
Quoted 3 lines
Quoted 1 line
Quoted 1 line
   };

This struct is declared as "static storage", so won't the non-initialized members be 0 ?

Correct. Although, I see that 'nested_enable_evmcs' is explicitly initialized. We should maintain the convention, perhaps.

Quoted 32 lines
Quoted 32 lines
Quoted 24 lines
   static int __init svm_init(void) diff --git 

a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 05e953b2ec61..2127ed937f53 100644

arch/x86/kvm/x86.c +6 -0
@@ -5250,6 +5250,12 @@ long kvm_arch_vm_ioctl(struct file *filp,
                    r = kvm_x86_ops->set_page_enc_bitmap(kvm, &bitmap);
            break;
    }
+   case KVM_PAGE_ENC_BITMAP_RESET: {
+           r = -ENOTTY;
+           if (kvm_x86_ops->reset_page_enc_bitmap)
+                   r = kvm_x86_ops->reset_page_enc_bitmap(kvm);
+           break;
+   }
    default:
            r = -ENOTTY;
    }
(quoted diff) +0 -0

b/include/uapi/linux/kvm.h index
b4b01d47e568..0884a581fc37 100644

include/uapi/linux/kvm.h struct kvm_enc_region { · line 1495

Quoted 6 lines
include/uapi/linux/kvm.h +1 -0
@@ -1490,6 +1490,7 @@ struct kvm_enc_region {
   #define KVM_GET_PAGE_ENC_BITMAP  _IOW(KVMIO, 0xc5, struct kvm_page_enc_bitmap)
   #define KVM_SET_PAGE_ENC_BITMAP  _IOW(KVMIO, 0xc6, 
 struct kvm_page_enc_bitmap)
+#define KVM_PAGE_ENC_BITMAP_RESET  _IO(KVMIO, 0xc7)
   /* Secure Encrypted Virtualization command */
   enum sev_cmd_id {

Reviewed-by: Krish Sadhukhan
<[email protected]>

Quoted 4 lines

Doesn't this overlap with the set ioctl? Yes, obviously, you have to copy the new value down and do a bit more work, but I don't think resetting the bitmap is going to be the bottleneck on reboot. Seems excessive to add another ioctl for this.

Quoted 6 lines

The set ioctl is generally available/provided for the incoming
VM to setup the page encryption bitmap, this reset ioctl is
meant for the source VM as a simple interface to reset the whole page encryption bitmap.

Thanks,
Ashish

Quoted 7 lines

Hey Ashish,

These seem very overlapping. I think this API should be refactored a bit.

1) Use kvm_vm_ioctl_enable_cap to control whether or not this hypercall (and related feature bit) is offered to the VM, and also the size of the buffer.

If you look at patch 13/14, i have added a new kvm para feature called "KVM_FEATURE_SEV_LIVE_MIGRATION" which indicates host support for SEV Live Migration and a new Custom MSR which the guest does a wrmsr to enable the Live Migration feature, so this is like the enable cap support.

There are further extensions to this support i am adding, so patch 13/14 of this patch-set is still being enhanced and will have full support when i repost next.

Quoted 5 lines

2) Use set for manipulating values in the bitmap, including resetting the bitmap. Set the bitmap pointer to null if you want to reset to all 0xFFs. When the bitmap pointer is set, it should set the values to exactly what is pointed at, instead of only clearing bits, as is done currently.

As i mentioned in my earlier email, the set api is supposed to be for the incoming VM, but if you really need to use it for the outgoing VM then it can be modified.

Quoted 9 lines

3) Use get for fetching values from the kernel. Personally, I'd require alignment of the base GFN to a multiple of 8 (but the number of pages could be whatever), so you can just use a memcpy. Optionally, you may want some way to tell userspace the size of the existing buffer, so it can ensure that it can ask for the entire buffer without having to track the size in usermode (not strictly necessary, but nice to have since it ensures that there is only one place that has to manage this value).

Quoted 7 lines
Quoted 2 lines

If you want to expand or contract the bitmap, you can use enable cap to adjust the size.

As being discussed on the earlier mail thread, we are doing this dynamically now by computing the guest RAM size when the set_user_memory_region ioctl is invoked. I believe that should handle the hot-plug and hot-unplug events too, as any hot memory updates will need KVM memslots to be updated.

Ahh, sorry, forgot you mentioned this: yes this can work. Host needs to be able to decide not to allocate, but this should be workable.

Quoted 10 lines
Quoted 10 lines

If you don't want to offer the hypercall to the guest, don't call the enable cap.
This API avoids using up another ioctl. Ioctl space is somewhat scarce. It also gives userspace fine grained control over the buffer, so it can support both hot-plug and hot-unplug (or at the very least it is not obviously incompatible with those). It also gives userspace control over whether or not the feature is offered. The hypercall isn't free, and being able to tell guests to not call when the host wasn't going to migrate it anyway will be useful.

Quoted 5 lines

As i mentioned above, now the host indicates if it supports the Live Migration feature and the feature and the hypercall are only enabled on the host when the guest checks for this support and does a wrmsr() to enable the feature. Also the guest will not make the hypercall if the host does not indicate support for it.

Quoted 4 lines

If my read of those patches was correct, the host will always advertise support for the hypercall. And the only bit controlling whether or not the hypercall is advertised is essentially the kernel version. You need to rollout a new kernel to disable the hypercall.

Quoted 7 lines

Ahh, awesome, I see I misunderstood how the CPUID bits get passed
through: usermode can still override them. Forgot about the back and forth for CPUID with usermode. My point about informing the guest kernel is clearly moot. The host still needs the ability to prevent allocations, but that is more minor. Maybe use a flag on the memslots directly?
On second thought: burning the memslot flag for 30mb per tb of VM seems like a waste.

Quoted 4 lines

Currently, I am still using the approach of a "unified" page encryption bitmap instead of a bitmap per memslot, with the main change being that the resizing is only done whenever there are any updates in memslots, when memslots are updated using the kvm_arch_commit_memory_region() interface.

Just a note, I believe kvm_arch_commit_memory_region() maybe getting called every time there is a change in the memory region (e.g add, update, delete etc). So your architecture specific hook now need to be well aware of all those changes and act accordingly. This basically means that the svm.c will probably need to understand all those memory slot flags etc. IMO, having a separate ioctl to hint the size makes more sense if you are doing a unified bitmap but if the bitmap is per memslot then calculating the size based on the memslot information makes more sense.

#83 lore raw

Re: [PATCH v6 12/14] KVM: x86: Introduce KVM_PAGE_ENC_BITMAP_RESET ioctl

Quoted 3 lines · Brijesh Singh
Quoted 1 line · Ashish

[AMD Official Use Only - Internal Distribution Only]

Can you please resend the original mail without the above header, so us non-AMD folks can follow along? :-)

#84 lore raw

Re: [PATCH v6 12/14] KVM: x86: Introduce KVM_PAGE_ENC_BITMAP_RESET ioctl

Quoted 5 lines · Sean Christopherson
Quoted 2 lines · Brijesh Singh
Quoted 1 line · Ashish

[AMD Official Use Only - Internal Distribution Only]

Can you please resend the original mail without the above header, so us non-AMD folks can follow along? :-)

Haha :) looks like one of us probably used outlook for responding. These days IT have been auto add some of those tags. Let me resend with it removed.

#85 lore raw

Re: [PATCH v6 12/14] KVM: x86: Introduce KVM_PAGE_ENC_BITMAP_RESET ioctl

In reply to Kalra, Ashish

resend with internal distribution tag removed.

On 4/10/20 3:55 PM, Kalra, Ashish wrote:
[snip]
..

Quoted 235 lines

Hello Steve,

-----Original Message-----
From: Steve Rutherford <[email protected]>
Sent: Friday, April 10, 2020 3:19 PM
To: Kalra, Ashish <[email protected]>
Cc: Krish Sadhukhan <[email protected]>; Paolo Bonzini <[email protected]>; Thomas Gleixner <[email protected]>; Ingo Molnar <[email protected]>; H. Peter Anvin <[email protected]>; Joerg Roedel <[email protected]>; Borislav Petkov <[email protected]>; Lendacky, Thomas <[email protected]>; X86 ML <[email protected]>; KVM list <[email protected]>; LKML <[email protected]>; David Rientjes <[email protected]>; Andy Lutomirski <[email protected]>; Singh, Brijesh <[email protected]>
Subject: Re: [PATCH v6 12/14] KVM: x86: Introduce KVM_PAGE_ENC_BITMAP_RESET ioctl

Quoted 217 lines · Steve Rutherford
Quoted 211 lines · Steve Rutherford
Quoted 186 lines · Ashish Kalra

Hello Steve,

Quoted 148 lines · Steve Rutherford
Quoted 140 lines · Ashish Kalra

Hello Steve,

Quoted 133 lines · Steve Rutherford
Quoted 91 lines · Krish Sadhukhan
Quoted 89 lines · Ashish Kalra
Quoted 88 lines · Krish Sadhukhan
Quoted 78 lines · Ashish Kalra

From: Ashish Kalra <[email protected]>

This ioctl can be used by the application to reset the page encryption bitmap managed by the KVM driver. A typical usage for this ioctl is on VM reboot, on reboot, we must reinitialize the bitmap.

---
   Documentation/virt/kvm/api.rst  | 13 +++++++++++++
   arch/x86/include/asm/kvm_host.h |  1 +
   arch/x86/kvm/svm.c              | 16 ++++++++++++++++
   arch/x86/kvm/x86.c              |  6 ++++++
   include/uapi/linux/kvm.h        |  1 +
   5 files changed, 37 insertions(+)
(quoted diff) +0 -0

b/Documentation/virt/kvm/api.rst index
4d1004a154f6..a11326ccc51d 100644

Documentation/virt/kvm/api.rst +13 -0
@@ -4698,6 +4698,19 @@ During the guest live migration the outgoing guest exports its page encryption
   bitmap, the KVM_SET_PAGE_ENC_BITMAP can be used to build the page encryption
   bitmap for an incoming guest.
+4.127 KVM_PAGE_ENC_BITMAP_RESET (vm ioctl)
+-----------------------------------------
+
+:Capability: basic
+:Architectures: x86
+:Type: vm ioctl
+:Parameters: none
+:Returns: 0 on success, -1 on error
+
+The KVM_PAGE_ENC_BITMAP_RESET is used to reset the 
+guest's page encryption bitmap during guest reboot and this is only done on the guest's boot vCPU.
+
+
   5. The kvm_run structure
   ======================== diff --git 

a/arch/x86/include/asm/kvm_host.h
b/arch/x86/include/asm/kvm_host.h index
d30f770aaaea..a96ef6338cd2 100644

arch/x86/include/asm/kvm_host.h +1 -0
@@ -1273,6 +1273,7 @@ struct kvm_x86_ops {
                            struct kvm_page_enc_bitmap *bmap);
    int (*set_page_enc_bitmap)(struct kvm *kvm,
                            struct kvm_page_enc_bitmap 
 *bmap);
+   int (*reset_page_enc_bitmap)(struct kvm *kvm);
   };
   struct kvm_arch_async_pf { diff --git 

a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c index 313343a43045..c99b0207a443 100644

arch/x86/kvm/svm.c +16 -0
@@ -7797,6 +7797,21 @@ static int svm_set_page_enc_bitmap(struct kvm *kvm,
    return ret;
   }
+static int svm_reset_page_enc_bitmap(struct kvm *kvm) 
+{
+   struct kvm_sev_info *sev = 
+&to_kvm_svm(kvm)->sev_info;
+
+   if (!sev_guest(kvm))
+           return -ENOTTY;
+
+   mutex_lock(&kvm->lock);
+   /* by default all pages should be marked encrypted */
+   if (sev->page_enc_bmap_size)
+           bitmap_fill(sev->page_enc_bmap, sev->page_enc_bmap_size);
+   mutex_unlock(&kvm->lock);
+   return 0;
+}
+
   static int svm_mem_enc_op(struct kvm *kvm, void __user *argp)
   {
    struct kvm_sev_cmd sev_cmd; @@ -8203,6 +8218,7 @@ 
Quoted 6 lines

static struct kvm_x86_ops svm_x86_ops __ro_after_init = {
.page_enc_status_hc = svm_page_enc_status_hc, .get_page_enc_bitmap = svm_get_page_enc_bitmap, .set_page_enc_bitmap = svm_set_page_enc_bitmap,
+ .reset_page_enc_bitmap =
+ svm_reset_page_enc_bitmap,

We don't need to initialize the intel ops to NULL ? It's not initialized in the previous patch either.

Quoted 39 lines
Quoted 3 lines
Quoted 1 line
Quoted 1 line
   };

This struct is declared as "static storage", so won't the non-initialized members be 0 ?

Correct. Although, I see that 'nested_enable_evmcs' is explicitly initialized. We should maintain the convention, perhaps.

Quoted 32 lines
Quoted 32 lines
Quoted 24 lines
   static int __init svm_init(void) diff --git 

a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 05e953b2ec61..2127ed937f53 100644

arch/x86/kvm/x86.c +6 -0
@@ -5250,6 +5250,12 @@ long kvm_arch_vm_ioctl(struct file *filp,
                    r = kvm_x86_ops->set_page_enc_bitmap(kvm, &bitmap);
            break;
    }
+   case KVM_PAGE_ENC_BITMAP_RESET: {
+           r = -ENOTTY;
+           if (kvm_x86_ops->reset_page_enc_bitmap)
+                   r = kvm_x86_ops->reset_page_enc_bitmap(kvm);
+           break;
+   }
    default:
            r = -ENOTTY;
    }
(quoted diff) +0 -0

b/include/uapi/linux/kvm.h index
b4b01d47e568..0884a581fc37 100644

include/uapi/linux/kvm.h struct kvm_enc_region { · line 1495

Quoted 6 lines
include/uapi/linux/kvm.h +1 -0
@@ -1490,6 +1490,7 @@ struct kvm_enc_region {
   #define KVM_GET_PAGE_ENC_BITMAP  _IOW(KVMIO, 0xc5, struct kvm_page_enc_bitmap)
   #define KVM_SET_PAGE_ENC_BITMAP  _IOW(KVMIO, 0xc6, 
 struct kvm_page_enc_bitmap)
+#define KVM_PAGE_ENC_BITMAP_RESET  _IO(KVMIO, 0xc7)
   /* Secure Encrypted Virtualization command */
   enum sev_cmd_id {

Reviewed-by: Krish Sadhukhan
<[email protected]>

Quoted 4 lines

Doesn't this overlap with the set ioctl? Yes, obviously, you have to copy the new value down and do a bit more work, but I don't think resetting the bitmap is going to be the bottleneck on reboot. Seems excessive to add another ioctl for this.

Quoted 6 lines

The set ioctl is generally available/provided for the incoming
VM to setup the page encryption bitmap, this reset ioctl is
meant for the source VM as a simple interface to reset the whole page encryption bitmap.

Thanks,
Ashish

Quoted 7 lines

Hey Ashish,

These seem very overlapping. I think this API should be refactored a bit.

1) Use kvm_vm_ioctl_enable_cap to control whether or not this hypercall (and related feature bit) is offered to the VM, and also the size of the buffer.

If you look at patch 13/14, i have added a new kvm para feature called "KVM_FEATURE_SEV_LIVE_MIGRATION" which indicates host support for SEV Live Migration and a new Custom MSR which the guest does a wrmsr to enable the Live Migration feature, so this is like the enable cap support.

There are further extensions to this support i am adding, so patch 13/14 of this patch-set is still being enhanced and will have full support when i repost next.

Quoted 5 lines

2) Use set for manipulating values in the bitmap, including resetting the bitmap. Set the bitmap pointer to null if you want to reset to all 0xFFs. When the bitmap pointer is set, it should set the values to exactly what is pointed at, instead of only clearing bits, as is done currently.

As i mentioned in my earlier email, the set api is supposed to be for the incoming VM, but if you really need to use it for the outgoing VM then it can be modified.

Quoted 9 lines

3) Use get for fetching values from the kernel. Personally, I'd require alignment of the base GFN to a multiple of 8 (but the number of pages could be whatever), so you can just use a memcpy. Optionally, you may want some way to tell userspace the size of the existing buffer, so it can ensure that it can ask for the entire buffer without having to track the size in usermode (not strictly necessary, but nice to have since it ensures that there is only one place that has to manage this value).

Quoted 7 lines
Quoted 2 lines

If you want to expand or contract the bitmap, you can use enable cap to adjust the size.

As being discussed on the earlier mail thread, we are doing this dynamically now by computing the guest RAM size when the set_user_memory_region ioctl is invoked. I believe that should handle the hot-plug and hot-unplug events too, as any hot memory updates will need KVM memslots to be updated.

Ahh, sorry, forgot you mentioned this: yes this can work. Host needs to be able to decide not to allocate, but this should be workable.

Quoted 10 lines
Quoted 10 lines

If you don't want to offer the hypercall to the guest, don't call the enable cap.
This API avoids using up another ioctl. Ioctl space is somewhat scarce. It also gives userspace fine grained control over the buffer, so it can support both hot-plug and hot-unplug (or at the very least it is not obviously incompatible with those). It also gives userspace control over whether or not the feature is offered. The hypercall isn't free, and being able to tell guests to not call when the host wasn't going to migrate it anyway will be useful.

Quoted 5 lines

As i mentioned above, now the host indicates if it supports the Live Migration feature and the feature and the hypercall are only enabled on the host when the guest checks for this support and does a wrmsr() to enable the feature. Also the guest will not make the hypercall if the host does not indicate support for it.

Quoted 4 lines

If my read of those patches was correct, the host will always advertise support for the hypercall. And the only bit controlling whether or not the hypercall is advertised is essentially the kernel version. You need to rollout a new kernel to disable the hypercall.

Quoted 7 lines

Ahh, awesome, I see I misunderstood how the CPUID bits get passed
through: usermode can still override them. Forgot about the back and forth for CPUID with usermode. My point about informing the guest kernel is clearly moot. The host still needs the ability to prevent allocations, but that is more minor. Maybe use a flag on the memslots directly?
On second thought: burning the memslot flag for 30mb per tb of VM seems like a waste.

Quoted 4 lines

Currently, I am still using the approach of a "unified" page encryption bitmap instead of a bitmap per memslot, with the main change being that the resizing is only done whenever there are any updates in memslots, when memslots are updated using the kvm_arch_commit_memory_region() interface.

Just a note, I believe kvm_arch_commit_memory_region() maybe getting called every time there is a change in the memory region (e.g add, update, delete etc). So your architecture specific hook now need to be well aware of all those changes and act accordingly. This basically means that the svm.c will probably need to understand all those memory slot flags etc. IMO, having a separate ioctl to hint the size makes more sense if you are doing a unified bitmap but if the bitmap is per memslot then calculating the size based on the memslot information makes more sense.

#86 lore raw

Re: [PATCH v6 12/14] KVM: x86: Introduce KVM_PAGE_ENC_BITMAP_RESET ioctl

Hello Brijesh,

Quoted 247 lines · Brijesh Singh

resend with internal distribution tag removed.

On 4/10/20 3:55 PM, Kalra, Ashish wrote:
[snip]
..

Quoted 235 lines

Hello Steve,

-----Original Message-----
From: Steve Rutherford <[email protected]>
Sent: Friday, April 10, 2020 3:19 PM
To: Kalra, Ashish <[email protected]>
Cc: Krish Sadhukhan <[email protected]>; Paolo Bonzini <[email protected]>; Thomas Gleixner <[email protected]>; Ingo Molnar <[email protected]>; H. Peter Anvin <[email protected]>; Joerg Roedel <[email protected]>; Borislav Petkov <[email protected]>; Lendacky, Thomas <[email protected]>; X86 ML <[email protected]>; KVM list <[email protected]>; LKML <[email protected]>; David Rientjes <[email protected]>; Andy Lutomirski <[email protected]>; Singh, Brijesh <[email protected]>
Subject: Re: [PATCH v6 12/14] KVM: x86: Introduce KVM_PAGE_ENC_BITMAP_RESET ioctl

Quoted 217 lines · Steve Rutherford
Quoted 211 lines · Steve Rutherford
Quoted 186 lines · Ashish Kalra

Hello Steve,

Quoted 148 lines · Steve Rutherford
Quoted 140 lines · Ashish Kalra

Hello Steve,

Quoted 133 lines · Steve Rutherford
Quoted 91 lines · Krish Sadhukhan
Quoted 89 lines · Ashish Kalra
Quoted 88 lines · Krish Sadhukhan
Quoted 78 lines · Ashish Kalra

From: Ashish Kalra <[email protected]>

This ioctl can be used by the application to reset the page encryption bitmap managed by the KVM driver. A typical usage for this ioctl is on VM reboot, on reboot, we must reinitialize the bitmap.

---
   Documentation/virt/kvm/api.rst  | 13 +++++++++++++
   arch/x86/include/asm/kvm_host.h |  1 +
   arch/x86/kvm/svm.c              | 16 ++++++++++++++++
   arch/x86/kvm/x86.c              |  6 ++++++
   include/uapi/linux/kvm.h        |  1 +
   5 files changed, 37 insertions(+)
(quoted diff) +0 -0

b/Documentation/virt/kvm/api.rst index
4d1004a154f6..a11326ccc51d 100644

Documentation/virt/kvm/api.rst +13 -0
@@ -4698,6 +4698,19 @@ During the guest live migration the outgoing guest exports its page encryption
   bitmap, the KVM_SET_PAGE_ENC_BITMAP can be used to build the page encryption
   bitmap for an incoming guest.
+4.127 KVM_PAGE_ENC_BITMAP_RESET (vm ioctl)
+-----------------------------------------
+
+:Capability: basic
+:Architectures: x86
+:Type: vm ioctl
+:Parameters: none
+:Returns: 0 on success, -1 on error
+
+The KVM_PAGE_ENC_BITMAP_RESET is used to reset the 
+guest's page encryption bitmap during guest reboot and this is only done on the guest's boot vCPU.
+
+
   5. The kvm_run structure
   ======================== diff --git 

a/arch/x86/include/asm/kvm_host.h
b/arch/x86/include/asm/kvm_host.h index
d30f770aaaea..a96ef6338cd2 100644

arch/x86/include/asm/kvm_host.h +1 -0
@@ -1273,6 +1273,7 @@ struct kvm_x86_ops {
                            struct kvm_page_enc_bitmap *bmap);
    int (*set_page_enc_bitmap)(struct kvm *kvm,
                            struct kvm_page_enc_bitmap 
 *bmap);
+   int (*reset_page_enc_bitmap)(struct kvm *kvm);
   };
   struct kvm_arch_async_pf { diff --git 

a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c index 313343a43045..c99b0207a443 100644

arch/x86/kvm/svm.c +16 -0
@@ -7797,6 +7797,21 @@ static int svm_set_page_enc_bitmap(struct kvm *kvm,
    return ret;
   }
+static int svm_reset_page_enc_bitmap(struct kvm *kvm) 
+{
+   struct kvm_sev_info *sev = 
+&to_kvm_svm(kvm)->sev_info;
+
+   if (!sev_guest(kvm))
+           return -ENOTTY;
+
+   mutex_lock(&kvm->lock);
+   /* by default all pages should be marked encrypted */
+   if (sev->page_enc_bmap_size)
+           bitmap_fill(sev->page_enc_bmap, sev->page_enc_bmap_size);
+   mutex_unlock(&kvm->lock);
+   return 0;
+}
+
   static int svm_mem_enc_op(struct kvm *kvm, void __user *argp)
   {
    struct kvm_sev_cmd sev_cmd; @@ -8203,6 +8218,7 @@ 
Quoted 6 lines

static struct kvm_x86_ops svm_x86_ops __ro_after_init = {
.page_enc_status_hc = svm_page_enc_status_hc, .get_page_enc_bitmap = svm_get_page_enc_bitmap, .set_page_enc_bitmap = svm_set_page_enc_bitmap,
+ .reset_page_enc_bitmap =
+ svm_reset_page_enc_bitmap,

We don't need to initialize the intel ops to NULL ? It's not initialized in the previous patch either.

Quoted 39 lines
Quoted 3 lines
Quoted 1 line
Quoted 1 line
   };

This struct is declared as "static storage", so won't the non-initialized members be 0 ?

Correct. Although, I see that 'nested_enable_evmcs' is explicitly initialized. We should maintain the convention, perhaps.

Quoted 32 lines
Quoted 32 lines
Quoted 24 lines
   static int __init svm_init(void) diff --git 

a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 05e953b2ec61..2127ed937f53 100644

arch/x86/kvm/x86.c +6 -0
@@ -5250,6 +5250,12 @@ long kvm_arch_vm_ioctl(struct file *filp,
                    r = kvm_x86_ops->set_page_enc_bitmap(kvm, &bitmap);
            break;
    }
+   case KVM_PAGE_ENC_BITMAP_RESET: {
+           r = -ENOTTY;
+           if (kvm_x86_ops->reset_page_enc_bitmap)
+                   r = kvm_x86_ops->reset_page_enc_bitmap(kvm);
+           break;
+   }
    default:
            r = -ENOTTY;
    }
(quoted diff) +0 -0

b/include/uapi/linux/kvm.h index
b4b01d47e568..0884a581fc37 100644

include/uapi/linux/kvm.h struct kvm_enc_region { · line 1495

Quoted 6 lines
include/uapi/linux/kvm.h +1 -0
@@ -1490,6 +1490,7 @@ struct kvm_enc_region {
   #define KVM_GET_PAGE_ENC_BITMAP  _IOW(KVMIO, 0xc5, struct kvm_page_enc_bitmap)
   #define KVM_SET_PAGE_ENC_BITMAP  _IOW(KVMIO, 0xc6, 
 struct kvm_page_enc_bitmap)
+#define KVM_PAGE_ENC_BITMAP_RESET  _IO(KVMIO, 0xc7)
   /* Secure Encrypted Virtualization command */
   enum sev_cmd_id {

Reviewed-by: Krish Sadhukhan
<[email protected]>

Quoted 4 lines

Doesn't this overlap with the set ioctl? Yes, obviously, you have to copy the new value down and do a bit more work, but I don't think resetting the bitmap is going to be the bottleneck on reboot. Seems excessive to add another ioctl for this.

Quoted 6 lines

The set ioctl is generally available/provided for the incoming
VM to setup the page encryption bitmap, this reset ioctl is
meant for the source VM as a simple interface to reset the whole page encryption bitmap.

Thanks,
Ashish

Quoted 7 lines

Hey Ashish,

These seem very overlapping. I think this API should be refactored a bit.

1) Use kvm_vm_ioctl_enable_cap to control whether or not this hypercall (and related feature bit) is offered to the VM, and also the size of the buffer.

If you look at patch 13/14, i have added a new kvm para feature called "KVM_FEATURE_SEV_LIVE_MIGRATION" which indicates host support for SEV Live Migration and a new Custom MSR which the guest does a wrmsr to enable the Live Migration feature, so this is like the enable cap support.

There are further extensions to this support i am adding, so patch 13/14 of this patch-set is still being enhanced and will have full support when i repost next.

Quoted 5 lines

2) Use set for manipulating values in the bitmap, including resetting the bitmap. Set the bitmap pointer to null if you want to reset to all 0xFFs. When the bitmap pointer is set, it should set the values to exactly what is pointed at, instead of only clearing bits, as is done currently.

As i mentioned in my earlier email, the set api is supposed to be for the incoming VM, but if you really need to use it for the outgoing VM then it can be modified.

Quoted 9 lines

3) Use get for fetching values from the kernel. Personally, I'd require alignment of the base GFN to a multiple of 8 (but the number of pages could be whatever), so you can just use a memcpy. Optionally, you may want some way to tell userspace the size of the existing buffer, so it can ensure that it can ask for the entire buffer without having to track the size in usermode (not strictly necessary, but nice to have since it ensures that there is only one place that has to manage this value).

Quoted 7 lines
Quoted 2 lines

If you want to expand or contract the bitmap, you can use enable cap to adjust the size.

As being discussed on the earlier mail thread, we are doing this dynamically now by computing the guest RAM size when the set_user_memory_region ioctl is invoked. I believe that should handle the hot-plug and hot-unplug events too, as any hot memory updates will need KVM memslots to be updated.

Ahh, sorry, forgot you mentioned this: yes this can work. Host needs to be able to decide not to allocate, but this should be workable.

Quoted 10 lines
Quoted 10 lines

If you don't want to offer the hypercall to the guest, don't call the enable cap.
This API avoids using up another ioctl. Ioctl space is somewhat scarce. It also gives userspace fine grained control over the buffer, so it can support both hot-plug and hot-unplug (or at the very least it is not obviously incompatible with those). It also gives userspace control over whether or not the feature is offered. The hypercall isn't free, and being able to tell guests to not call when the host wasn't going to migrate it anyway will be useful.

Quoted 5 lines

As i mentioned above, now the host indicates if it supports the Live Migration feature and the feature and the hypercall are only enabled on the host when the guest checks for this support and does a wrmsr() to enable the feature. Also the guest will not make the hypercall if the host does not indicate support for it.

Quoted 4 lines

If my read of those patches was correct, the host will always advertise support for the hypercall. And the only bit controlling whether or not the hypercall is advertised is essentially the kernel version. You need to rollout a new kernel to disable the hypercall.

Quoted 7 lines

Ahh, awesome, I see I misunderstood how the CPUID bits get passed
through: usermode can still override them. Forgot about the back and forth for CPUID with usermode. My point about informing the guest kernel is clearly moot. The host still needs the ability to prevent allocations, but that is more minor. Maybe use a flag on the memslots directly?
On second thought: burning the memslot flag for 30mb per tb of VM seems like a waste.

Quoted 4 lines

Currently, I am still using the approach of a "unified" page encryption bitmap instead of a bitmap per memslot, with the main change being that the resizing is only done whenever there are any updates in memslots, when memslots are updated using the kvm_arch_commit_memory_region() interface.

Quoted 9 lines

Just a note, I believe kvm_arch_commit_memory_region() maybe getting called every time there is a change in the memory region (e.g add, update, delete etc). So your architecture specific hook now need to be well aware of all those changes and act accordingly. This basically means that the svm.c will probably need to understand all those memory slot flags etc. IMO, having a separate ioctl to hint the size makes more sense if you are doing a unified bitmap but if the bitmap is per memslot then calculating the size based on the memslot information makes more sense.

If instead of unified bitmap, i use a bitmap per memslot approach, even then the svm/sev code will still need to to have knowledge of memslot flags etc., that information will be required as svm/sev code will be responsible for syncing the page encryption bitmap to userspace and not the generic KVM x86 code which gets invoked for the dirty page bitmap sync.

Currently, the architecture hooks need to be aware of KVM_MR_ADD/KVM_MR_DELETE flags and the resize will only happen if the highest guest PA that is mapped by a memslot gets modified, otherwise, typically, there will mainly one resize at the initial guest launch.

Thanks,
Ashish

Reviewed-by on v6 12/14
#87 lore raw

Re: [PATCH v6 12/14] KVM: x86: Introduce KVM_PAGE_ENC_BITMAP_RESET ioctl

In reply to Ashish Kalra
Quoted 8 lines · Ashish Kalra

From: Ashish Kalra <[email protected]>

This ioctl can be used by the application to reset the page encryption bitmap managed by the KVM driver. A typical usage for this ioctl is on VM reboot, on reboot, we must reinitialize the bitmap.

Quoted 110 lines
---
 Documentation/virt/kvm/api.rst  | 13 +++++++++++++
 arch/x86/include/asm/kvm_host.h |  1 +
 arch/x86/kvm/svm.c              | 16 ++++++++++++++++
 arch/x86/kvm/x86.c              |  6 ++++++
 include/uapi/linux/kvm.h        |  1 +
 5 files changed, 37 insertions(+)
Documentation/virt/kvm/api.rst +13 -0
index 4d1004a154f6..a11326ccc51d 100644
@@ -4698,6 +4698,19 @@ During the guest live migration the outgoing guest exports its page encryption
 bitmap, the KVM_SET_PAGE_ENC_BITMAP can be used to build the page encryption
 bitmap for an incoming guest.
 
+4.127 KVM_PAGE_ENC_BITMAP_RESET (vm ioctl)
+-----------------------------------------
+
+:Capability: basic
+:Architectures: x86
+:Type: vm ioctl
+:Parameters: none
+:Returns: 0 on success, -1 on error
+
+The KVM_PAGE_ENC_BITMAP_RESET is used to reset the guest's page encryption
+bitmap during guest reboot and this is only done on the guest's boot vCPU.
+
+
 5. The kvm_run structure
 ========================
 
arch/x86/include/asm/kvm_host.h +1 -0
index d30f770aaaea..a96ef6338cd2 100644
@@ -1273,6 +1273,7 @@ struct kvm_x86_ops {
 				struct kvm_page_enc_bitmap *bmap);
 	int (*set_page_enc_bitmap)(struct kvm *kvm,
 				struct kvm_page_enc_bitmap *bmap);
+	int (*reset_page_enc_bitmap)(struct kvm *kvm);
 };
 
 struct kvm_arch_async_pf {
arch/x86/kvm/svm.c +16 -0
index 313343a43045..c99b0207a443 100644
@@ -7797,6 +7797,21 @@ static int svm_set_page_enc_bitmap(struct kvm *kvm,
 	return ret;
 }
 
+static int svm_reset_page_enc_bitmap(struct kvm *kvm)
+{
+	struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
+
+	if (!sev_guest(kvm))
+		return -ENOTTY;
+
+	mutex_lock(&kvm->lock);
+	/* by default all pages should be marked encrypted */
+	if (sev->page_enc_bmap_size)
+		bitmap_fill(sev->page_enc_bmap, sev->page_enc_bmap_size);
+	mutex_unlock(&kvm->lock);
+	return 0;
+}
+
 static int svm_mem_enc_op(struct kvm *kvm, void __user *argp)
 {
 	struct kvm_sev_cmd sev_cmd;
@@ -8203,6 +8218,7 @@ static struct kvm_x86_ops svm_x86_ops __ro_after_init = {
 	.page_enc_status_hc = svm_page_enc_status_hc,
 	.get_page_enc_bitmap = svm_get_page_enc_bitmap,
 	.set_page_enc_bitmap = svm_set_page_enc_bitmap,
+	.reset_page_enc_bitmap = svm_reset_page_enc_bitmap,
 };
 
 static int __init svm_init(void)
arch/x86/kvm/x86.c +6 -0
index 05e953b2ec61..2127ed937f53 100644
@@ -5250,6 +5250,12 @@ long kvm_arch_vm_ioctl(struct file *filp,
 			r = kvm_x86_ops->set_page_enc_bitmap(kvm, &bitmap);
 		break;
 	}
+	case KVM_PAGE_ENC_BITMAP_RESET: {
+		r = -ENOTTY;
+		if (kvm_x86_ops->reset_page_enc_bitmap)
+			r = kvm_x86_ops->reset_page_enc_bitmap(kvm);
+		break;
+	}
 	default:
 		r = -ENOTTY;
 	}
include/uapi/linux/kvm.h +1 -0
index b4b01d47e568..0884a581fc37 100644
@@ -1490,6 +1490,7 @@ struct kvm_enc_region {
 
 #define KVM_GET_PAGE_ENC_BITMAP	_IOW(KVMIO, 0xc5, struct kvm_page_enc_bitmap)
 #define KVM_SET_PAGE_ENC_BITMAP	_IOW(KVMIO, 0xc6, struct kvm_page_enc_bitmap)
+#define KVM_PAGE_ENC_BITMAP_RESET	_IO(KVMIO, 0xc7)
 
 /* Secure Encrypted Virtualization command */
 enum sev_cmd_id {
Signature · 1 line
2.17.1
#88 lore raw

PATCH v6 13/14 KVM: x86: Introduce new KVM_FEATURE_SEV_LIVE_MIGRATION feature & Custom MSR.

In reply to Ashish Kalra

From: Ashish Kalra <[email protected]>

Add new KVM_FEATURE_SEV_LIVE_MIGRATION feature for guest to check for host-side support for SEV live migration. Also add a new custom MSR_KVM_SEV_LIVE_MIG_EN for guest to enable the SEV live migration feature.

Also, ensure that _bss_decrypted section is marked as decrypted in the page encryption bitmap.

---
 Documentation/virt/kvm/cpuid.rst     |  4 ++++
 Documentation/virt/kvm/msr.rst       | 10 ++++++++++
 arch/x86/include/asm/kvm_host.h      |  3 +++
 arch/x86/include/uapi/asm/kvm_para.h |  5 +++++
 arch/x86/kernel/kvm.c                |  4 ++++
 arch/x86/kvm/cpuid.c                 |  3 ++-
 arch/x86/kvm/svm.c                   |  5 +++++
 arch/x86/kvm/x86.c                   |  7 +++++++
 arch/x86/mm/mem_encrypt.c            | 14 +++++++++++++-
 9 files changed, 53 insertions(+), 2 deletions(-)
Documentation/virt/kvm/cpuid.rst +4 -0
index 01b081f6e7ea..fcb191bb3016 100644
@@ -86,6 +86,10 @@ KVM_FEATURE_PV_SCHED_YIELD        13          guest checks this feature bit
                                               before using paravirtualized
                                               sched yield.
 
+KVM_FEATURE_SEV_LIVE_MIGRATION    14          guest checks this feature bit
+                                              before enabling SEV live
+                                              migration feature.
+
 KVM_FEATURE_CLOCSOURCE_STABLE_BIT 24          host will warn if no guest-side
                                               per-cpu warps are expeced in
                                               kvmclock
Documentation/virt/kvm/msr.rst +10 -0
index 33892036672d..7cd7786bbb03 100644
@@ -319,3 +319,13 @@ data:
 
 	KVM guests can request the host not to poll on HLT, for example if
 	they are performing polling themselves.
+
+MSR_KVM_SEV_LIVE_MIG_EN:
+        0x4b564d06
+
+	Control SEV Live Migration features.
+
+data:
+        Bit 0 enables (1) or disables (0) host-side SEV Live Migration feature.
+        Bit 1 enables (1) or disables (0) support for SEV Live Migration extensions.
+        All other bits are reserved.
arch/x86/include/asm/kvm_host.h +3 -0
index a96ef6338cd2..ad5faaed43c0 100644
@@ -780,6 +780,9 @@ struct kvm_vcpu_arch {
 
 	u64 msr_kvm_poll_control;
 
+	/* SEV Live Migration MSR (AMD only) */
+	u64 msr_kvm_sev_live_migration_flag;
+
 	/*
 	 * Indicates the guest is trying to write a gfn that contains one or
 	 * more of the PTEs used to translate the write itself, i.e. the access
arch/x86/include/uapi/asm/kvm_para.h +5 -0
index 2a8e0b6b9805..d9d4953b42ad 100644
@@ -31,6 +31,7 @@
 #define KVM_FEATURE_PV_SEND_IPI	11
 #define KVM_FEATURE_POLL_CONTROL	12
 #define KVM_FEATURE_PV_SCHED_YIELD	13
+#define KVM_FEATURE_SEV_LIVE_MIGRATION	14
 
 #define KVM_HINTS_REALTIME      0
 
@@ -50,6 +51,7 @@
 #define MSR_KVM_STEAL_TIME  0x4b564d03
 #define MSR_KVM_PV_EOI_EN      0x4b564d04
 #define MSR_KVM_POLL_CONTROL	0x4b564d05
+#define MSR_KVM_SEV_LIVE_MIG_EN	0x4b564d06
 
 struct kvm_steal_time {
 	__u64 steal;
@@ -122,4 +124,7 @@ struct kvm_vcpu_pv_apf_data {
 #define KVM_PV_EOI_ENABLED KVM_PV_EOI_MASK
 #define KVM_PV_EOI_DISABLED 0x0
 
+#define KVM_SEV_LIVE_MIGRATION_ENABLED			(1 << 0)
+#define KVM_SEV_LIVE_MIGRATION_EXTENSIONS_SUPPORTED	(1 << 1)
+
 #endif /* _UAPI_ASM_X86_KVM_PARA_H */
arch/x86/kernel/kvm.c +4 -0
index 6efe0410fb72..8fcee0b45231 100644
@@ -418,6 +418,10 @@ static void __init sev_map_percpu_data(void)
 	if (!sev_active())
 		return;
 
+	if (kvm_para_has_feature(KVM_FEATURE_SEV_LIVE_MIGRATION)) {
+		wrmsrl(MSR_KVM_SEV_LIVE_MIG_EN, KVM_SEV_LIVE_MIGRATION_ENABLED);
+	}
+
 	for_each_possible_cpu(cpu) {
 		__set_percpu_decrypted(&per_cpu(apf_reason, cpu), sizeof(apf_reason));
 		__set_percpu_decrypted(&per_cpu(steal_time, cpu), sizeof(steal_time));
arch/x86/kvm/cpuid.c +2 -1
index b1c469446b07..74c8b2a7270c 100644
@@ -716,7 +716,8 @@ static inline int __do_cpuid_func(struct kvm_cpuid_entry2 *entry, u32 function,
 			     (1 << KVM_FEATURE_ASYNC_PF_VMEXIT) |
 			     (1 << KVM_FEATURE_PV_SEND_IPI) |
 			     (1 << KVM_FEATURE_POLL_CONTROL) |
-			     (1 << KVM_FEATURE_PV_SCHED_YIELD);
+			     (1 << KVM_FEATURE_PV_SCHED_YIELD) |
+			     (1 << KVM_FEATURE_SEV_LIVE_MIGRATION);
 
 		if (sched_info_on())
 			entry->eax |= (1 << KVM_FEATURE_STEAL_TIME);
arch/x86/kvm/svm.c +5 -0
index c99b0207a443..60ddc242a133 100644
@@ -7632,6 +7632,7 @@ static int svm_page_enc_status_hc(struct kvm *kvm, unsigned long gpa,
 				  unsigned long npages, unsigned long enc)
 {
 	struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
+	struct kvm_vcpu *vcpu = kvm->vcpus[0];
 	kvm_pfn_t pfn_start, pfn_end;
 	gfn_t gfn_start, gfn_end;
 	int ret;
@@ -7639,6 +7640,10 @@ static int svm_page_enc_status_hc(struct kvm *kvm, unsigned long gpa,
 	if (!sev_guest(kvm))
 		return -EINVAL;
 
+	if (!(vcpu->arch.msr_kvm_sev_live_migration_flag &
+		KVM_SEV_LIVE_MIGRATION_ENABLED))
+		return -ENOTTY;
+
 	if (!npages)
 		return 0;
 
arch/x86/kvm/x86.c +7 -0
index 2127ed937f53..82867b8798f8 100644
@@ -2880,6 +2880,10 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 		vcpu->arch.msr_kvm_poll_control = data;
 		break;
 
+	case MSR_KVM_SEV_LIVE_MIG_EN:
+		vcpu->arch.msr_kvm_sev_live_migration_flag = data;
+		break;
+
 	case MSR_IA32_MCG_CTL:
 	case MSR_IA32_MCG_STATUS:
 	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
@@ -3126,6 +3130,9 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 	case MSR_KVM_POLL_CONTROL:
 		msr_info->data = vcpu->arch.msr_kvm_poll_control;
 		break;
+	case MSR_KVM_SEV_LIVE_MIG_EN:
+		msr_info->data = vcpu->arch.msr_kvm_sev_live_migration_flag;
+		break;
 	case MSR_IA32_P5_MC_ADDR:
 	case MSR_IA32_P5_MC_TYPE:
 	case MSR_IA32_MCG_CAP:
arch/x86/mm/mem_encrypt.c +13 -1
index c9800fa811f6..f6a841494845 100644
@@ -502,8 +502,20 @@ void __init mem_encrypt_init(void)
 	 * With SEV, we need to make a hypercall when page encryption state is
 	 * changed.
 	 */
-	if (sev_active())
+	if (sev_active()) {
+		unsigned long nr_pages;
+
 		pv_ops.mmu.page_encryption_changed = set_memory_enc_dec_hypercall;
+
+		/*
+		 * Ensure that _bss_decrypted section is marked as decrypted in the
+		 * page encryption bitmap.
+		 */
+		nr_pages = DIV_ROUND_UP(__end_bss_decrypted - __start_bss_decrypted,
+			PAGE_SIZE);
+		set_memory_enc_dec_hypercall((unsigned long)__start_bss_decrypted,
+			nr_pages, 0);
+	}
 #endif
 
 	pr_info("AMD %s active\n",
Signature · 1 line
2.17.1
#89 lore raw

Re: [PATCH v6 13/14] KVM: x86: Introduce new KVM_FEATURE_SEV_LIVE_MIGRATION feature & Custom MSR.

Quoted 11 lines · Ashish Kalra

From: Ashish Kalra <[email protected]>

Add new KVM_FEATURE_SEV_LIVE_MIGRATION feature for guest to check for host-side support for SEV live migration. Also add a new custom MSR_KVM_SEV_LIVE_MIG_EN for guest to enable the SEV live migration feature.

Also, ensure that _bss_decrypted section is marked as decrypted in the page encryption bitmap.

Quoted 11 lines
---
 Documentation/virt/kvm/cpuid.rst     |  4 ++++
 Documentation/virt/kvm/msr.rst       | 10 ++++++++++
 arch/x86/include/asm/kvm_host.h      |  3 +++
 arch/x86/include/uapi/asm/kvm_para.h |  5 +++++
 arch/x86/kernel/kvm.c                |  4 ++++
 arch/x86/kvm/cpuid.c                 |  3 ++-
 arch/x86/kvm/svm.c                   |  5 +++++
 arch/x86/kvm/x86.c                   |  7 +++++++
 arch/x86/mm/mem_encrypt.c            | 14 +++++++++++++-
 9 files changed, 53 insertions(+), 2 deletions(-)

IMHO, this patch should be broken into multiple patches as it touches guest, and hypervisor at the same time. The first patch can introduce the feature flag in the kvm, second patch can make the changes specific to svm,  and third patch can focus on how to make use of that feature inside the guest. Additionally invoking the HC to clear the __bss_decrypted section should be either squash in Patch 10/14 or be a separate patch itself.

Quoted 95 lines
Documentation/virt/kvm/cpuid.rst +4 -0
index 01b081f6e7ea..fcb191bb3016 100644
@@ -86,6 +86,10 @@ KVM_FEATURE_PV_SCHED_YIELD        13          guest checks this feature bit
                                               before using paravirtualized
                                               sched yield.
 
+KVM_FEATURE_SEV_LIVE_MIGRATION    14          guest checks this feature bit
+                                              before enabling SEV live
+                                              migration feature.
+
 KVM_FEATURE_CLOCSOURCE_STABLE_BIT 24          host will warn if no guest-side
                                               per-cpu warps are expeced in
                                               kvmclock
Documentation/virt/kvm/msr.rst +10 -0
index 33892036672d..7cd7786bbb03 100644
@@ -319,3 +319,13 @@ data:
 
 	KVM guests can request the host not to poll on HLT, for example if
 	they are performing polling themselves.
+
+MSR_KVM_SEV_LIVE_MIG_EN:
+        0x4b564d06
+
+	Control SEV Live Migration features.
+
+data:
+        Bit 0 enables (1) or disables (0) host-side SEV Live Migration feature.
+        Bit 1 enables (1) or disables (0) support for SEV Live Migration extensions.
+        All other bits are reserved.
arch/x86/include/asm/kvm_host.h +3 -0
index a96ef6338cd2..ad5faaed43c0 100644
@@ -780,6 +780,9 @@ struct kvm_vcpu_arch {
 
 	u64 msr_kvm_poll_control;
 
+	/* SEV Live Migration MSR (AMD only) */
+	u64 msr_kvm_sev_live_migration_flag;
+
 	/*
 	 * Indicates the guest is trying to write a gfn that contains one or
 	 * more of the PTEs used to translate the write itself, i.e. the access
arch/x86/include/uapi/asm/kvm_para.h +5 -0
index 2a8e0b6b9805..d9d4953b42ad 100644
@@ -31,6 +31,7 @@
 #define KVM_FEATURE_PV_SEND_IPI	11
 #define KVM_FEATURE_POLL_CONTROL	12
 #define KVM_FEATURE_PV_SCHED_YIELD	13
+#define KVM_FEATURE_SEV_LIVE_MIGRATION	14
 
 #define KVM_HINTS_REALTIME      0
 
@@ -50,6 +51,7 @@
 #define MSR_KVM_STEAL_TIME  0x4b564d03
 #define MSR_KVM_PV_EOI_EN      0x4b564d04
 #define MSR_KVM_POLL_CONTROL	0x4b564d05
+#define MSR_KVM_SEV_LIVE_MIG_EN	0x4b564d06
 
 struct kvm_steal_time {
 	__u64 steal;
@@ -122,4 +124,7 @@ struct kvm_vcpu_pv_apf_data {
 #define KVM_PV_EOI_ENABLED KVM_PV_EOI_MASK
 #define KVM_PV_EOI_DISABLED 0x0
 
+#define KVM_SEV_LIVE_MIGRATION_ENABLED			(1 << 0)
+#define KVM_SEV_LIVE_MIGRATION_EXTENSIONS_SUPPORTED	(1 << 1)
+
 #endif /* _UAPI_ASM_X86_KVM_PARA_H */
arch/x86/kernel/kvm.c +4 -0
index 6efe0410fb72..8fcee0b45231 100644
@@ -418,6 +418,10 @@ static void __init sev_map_percpu_data(void)
 	if (!sev_active())
 		return;
 
+	if (kvm_para_has_feature(KVM_FEATURE_SEV_LIVE_MIGRATION)) {
+		wrmsrl(MSR_KVM_SEV_LIVE_MIG_EN, KVM_SEV_LIVE_MIGRATION_ENABLED);
+	}
+
 	for_each_possible_cpu(cpu) {
 		__set_percpu_decrypted(&per_cpu(apf_reason, cpu), sizeof(apf_reason));
 		__set_percpu_decrypted(&per_cpu(steal_time, cpu), sizeof(steal_time));
arch/x86/kvm/cpuid.c +0 -0
index b1c469446b07..74c8b2a7270c 100644

arch/x86/kvm/cpuid.c static inline int __do_cpuid_func(struct kvm_cpuid_entry2 *entry, u32 function, · line 720

Quoted 6 lines
arch/x86/kvm/cpuid.c +2 -1
@@ -716,7 +716,8 @@ static inline int __do_cpuid_func(struct kvm_cpuid_entry2 *entry, u32 function,
 			     (1 << KVM_FEATURE_ASYNC_PF_VMEXIT) |
 			     (1 << KVM_FEATURE_PV_SEND_IPI) |
 			     (1 << KVM_FEATURE_POLL_CONTROL) |
-			     (1 << KVM_FEATURE_PV_SCHED_YIELD);
+			     (1 << KVM_FEATURE_PV_SCHED_YIELD) |
+			     (1 << KVM_FEATURE_SEV_LIVE_MIGRATION);

Do we want to enable this feature unconditionally ? Who will clear the feature flags for the non-SEV guest ?

Quoted 64 lines
arch/x86/kvm/cpuid.c +0 -0
 		if (sched_info_on())
 			entry->eax |= (1 << KVM_FEATURE_STEAL_TIME);
arch/x86/kvm/svm.c +5 -0
index c99b0207a443..60ddc242a133 100644
@@ -7632,6 +7632,7 @@ static int svm_page_enc_status_hc(struct kvm *kvm, unsigned long gpa,
 				  unsigned long npages, unsigned long enc)
 {
 	struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
+	struct kvm_vcpu *vcpu = kvm->vcpus[0];
 	kvm_pfn_t pfn_start, pfn_end;
 	gfn_t gfn_start, gfn_end;
 	int ret;
@@ -7639,6 +7640,10 @@ static int svm_page_enc_status_hc(struct kvm *kvm, unsigned long gpa,
 	if (!sev_guest(kvm))
 		return -EINVAL;
 
+	if (!(vcpu->arch.msr_kvm_sev_live_migration_flag &
+		KVM_SEV_LIVE_MIGRATION_ENABLED))
+		return -ENOTTY;
+
 	if (!npages)
 		return 0;
 
arch/x86/kvm/x86.c +7 -0
index 2127ed937f53..82867b8798f8 100644
@@ -2880,6 +2880,10 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 		vcpu->arch.msr_kvm_poll_control = data;
 		break;
 
+	case MSR_KVM_SEV_LIVE_MIG_EN:
+		vcpu->arch.msr_kvm_sev_live_migration_flag = data;
+		break;
+
 	case MSR_IA32_MCG_CTL:
 	case MSR_IA32_MCG_STATUS:
 	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
@@ -3126,6 +3130,9 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 	case MSR_KVM_POLL_CONTROL:
 		msr_info->data = vcpu->arch.msr_kvm_poll_control;
 		break;
+	case MSR_KVM_SEV_LIVE_MIG_EN:
+		msr_info->data = vcpu->arch.msr_kvm_sev_live_migration_flag;
+		break;
 	case MSR_IA32_P5_MC_ADDR:
 	case MSR_IA32_P5_MC_TYPE:
 	case MSR_IA32_MCG_CAP:
arch/x86/mm/mem_encrypt.c +3 -1
index c9800fa811f6..f6a841494845 100644
@@ -502,8 +502,20 @@ void __init mem_encrypt_init(void)
 	 * With SEV, we need to make a hypercall when page encryption state is
 	 * changed.
 	 */
-	if (sev_active())
+	if (sev_active()) {
+		unsigned long nr_pages;
+
 		pv_ops.mmu.page_encryption_changed = set_memory_enc_dec_hypercall;

arch/x86/mm/mem_encrypt.c void __init mem_encrypt_init(void) · line 518

Quoted 10 lines
arch/x86/mm/mem_encrypt.c +10 -0
+
+		/*
+		 * Ensure that _bss_decrypted section is marked as decrypted in the
+		 * page encryption bitmap.
+		 */
+		nr_pages = DIV_ROUND_UP(__end_bss_decrypted - __start_bss_decrypted,
+			PAGE_SIZE);
+		set_memory_enc_dec_hypercall((unsigned long)__start_bss_decrypted,
+			nr_pages, 0);
+	}

Isn't this too late, should we be making hypercall at the same time we clear the encryption bit ?

Quoted 3 lines
arch/x86/mm/mem_encrypt.c +0 -0
 #endif
 
 	pr_info("AMD %s active\n",
#90 lore raw

Re: [PATCH v6 13/14] KVM: x86: Introduce new KVM_FEATURE_SEV_LIVE_MIGRATION feature & Custom MSR.

Hello Brijesh,

Quoted 26 lines · Brijesh Singh
Quoted 11 lines · Ashish Kalra

From: Ashish Kalra <[email protected]>

Add new KVM_FEATURE_SEV_LIVE_MIGRATION feature for guest to check for host-side support for SEV live migration. Also add a new custom MSR_KVM_SEV_LIVE_MIG_EN for guest to enable the SEV live migration feature.

Also, ensure that _bss_decrypted section is marked as decrypted in the page encryption bitmap.

Quoted 11 lines
---
 Documentation/virt/kvm/cpuid.rst     |  4 ++++
 Documentation/virt/kvm/msr.rst       | 10 ++++++++++
 arch/x86/include/asm/kvm_host.h      |  3 +++
 arch/x86/include/uapi/asm/kvm_para.h |  5 +++++
 arch/x86/kernel/kvm.c                |  4 ++++
 arch/x86/kvm/cpuid.c                 |  3 ++-
 arch/x86/kvm/svm.c                   |  5 +++++
 arch/x86/kvm/x86.c                   |  7 +++++++
 arch/x86/mm/mem_encrypt.c            | 14 +++++++++++++-
 9 files changed, 53 insertions(+), 2 deletions(-)
Quoted 9 lines

IMHO, this patch should be broken into multiple patches as it touches guest, and hypervisor at the same time. The first patch can introduce the feature flag in the kvm, second patch can make the changes specific to svm,  and third patch can focus on how to make use of that feature inside the guest. Additionally invoking the HC to clear the __bss_decrypted section should be either squash in Patch 10/14 or be a separate patch itself.

Ok.

I will also move the __bss_decrypted section HC to a separate patch.

Quoted 97 lines
Quoted 97 lines
Documentation/virt/kvm/cpuid.rst +4 -0
index 01b081f6e7ea..fcb191bb3016 100644
@@ -86,6 +86,10 @@ KVM_FEATURE_PV_SCHED_YIELD        13          guest checks this feature bit
                                               before using paravirtualized
                                               sched yield.
 
+KVM_FEATURE_SEV_LIVE_MIGRATION    14          guest checks this feature bit
+                                              before enabling SEV live
+                                              migration feature.
+
 KVM_FEATURE_CLOCSOURCE_STABLE_BIT 24          host will warn if no guest-side
                                               per-cpu warps are expeced in
                                               kvmclock
Documentation/virt/kvm/msr.rst +10 -0
index 33892036672d..7cd7786bbb03 100644
@@ -319,3 +319,13 @@ data:
 
 	KVM guests can request the host not to poll on HLT, for example if
 	they are performing polling themselves.
+
+MSR_KVM_SEV_LIVE_MIG_EN:
+        0x4b564d06
+
+	Control SEV Live Migration features.
+
+data:
+        Bit 0 enables (1) or disables (0) host-side SEV Live Migration feature.
+        Bit 1 enables (1) or disables (0) support for SEV Live Migration extensions.
+        All other bits are reserved.
arch/x86/include/asm/kvm_host.h +3 -0
index a96ef6338cd2..ad5faaed43c0 100644
@@ -780,6 +780,9 @@ struct kvm_vcpu_arch {
 
 	u64 msr_kvm_poll_control;
 
+	/* SEV Live Migration MSR (AMD only) */
+	u64 msr_kvm_sev_live_migration_flag;
+
 	/*
 	 * Indicates the guest is trying to write a gfn that contains one or
 	 * more of the PTEs used to translate the write itself, i.e. the access
arch/x86/include/uapi/asm/kvm_para.h +5 -0
index 2a8e0b6b9805..d9d4953b42ad 100644
@@ -31,6 +31,7 @@
 #define KVM_FEATURE_PV_SEND_IPI	11
 #define KVM_FEATURE_POLL_CONTROL	12
 #define KVM_FEATURE_PV_SCHED_YIELD	13
+#define KVM_FEATURE_SEV_LIVE_MIGRATION	14
 
 #define KVM_HINTS_REALTIME      0
 
@@ -50,6 +51,7 @@
 #define MSR_KVM_STEAL_TIME  0x4b564d03
 #define MSR_KVM_PV_EOI_EN      0x4b564d04
 #define MSR_KVM_POLL_CONTROL	0x4b564d05
+#define MSR_KVM_SEV_LIVE_MIG_EN	0x4b564d06
 
 struct kvm_steal_time {
 	__u64 steal;
@@ -122,4 +124,7 @@ struct kvm_vcpu_pv_apf_data {
 #define KVM_PV_EOI_ENABLED KVM_PV_EOI_MASK
 #define KVM_PV_EOI_DISABLED 0x0
 
+#define KVM_SEV_LIVE_MIGRATION_ENABLED			(1 << 0)
+#define KVM_SEV_LIVE_MIGRATION_EXTENSIONS_SUPPORTED	(1 << 1)
+
 #endif /* _UAPI_ASM_X86_KVM_PARA_H */
arch/x86/kernel/kvm.c +4 -0
index 6efe0410fb72..8fcee0b45231 100644
@@ -418,6 +418,10 @@ static void __init sev_map_percpu_data(void)
 	if (!sev_active())
 		return;
 
+	if (kvm_para_has_feature(KVM_FEATURE_SEV_LIVE_MIGRATION)) {
+		wrmsrl(MSR_KVM_SEV_LIVE_MIG_EN, KVM_SEV_LIVE_MIGRATION_ENABLED);
+	}
+
 	for_each_possible_cpu(cpu) {
 		__set_percpu_decrypted(&per_cpu(apf_reason, cpu), sizeof(apf_reason));
 		__set_percpu_decrypted(&per_cpu(steal_time, cpu), sizeof(steal_time));
arch/x86/kvm/cpuid.c +0 -0
index b1c469446b07..74c8b2a7270c 100644
@@ -716,7 +716,8 @@ static inline int __do_cpuid_func(struct kvm_cpuid_entry2 *entry, u32 function,
 			     (1 << KVM_FEATURE_ASYNC_PF_VMEXIT) |
 			     (1 << KVM_FEATURE_PV_SEND_IPI) |
Quoted 9 lines

arch/x86/kvm/cpuid.c static inline int __do_cpuid_func(struct kvm_cpuid_entry2 *entry, u32 function, · line 720

Quoted 4 lines
arch/x86/kvm/cpuid.c +2 -1
 			     (1 << KVM_FEATURE_POLL_CONTROL) |
-			     (1 << KVM_FEATURE_PV_SCHED_YIELD);
+			     (1 << KVM_FEATURE_PV_SCHED_YIELD) |
+			     (1 << KVM_FEATURE_SEV_LIVE_MIGRATION);

Do we want to enable this feature unconditionally ? Who will clear the feature flags for the non-SEV guest ?

The guest only enables/activates this feature if sev is active.

Quoted 70 lines
Quoted 70 lines
arch/x86/kvm/cpuid.c +0 -0
 		if (sched_info_on())
 			entry->eax |= (1 << KVM_FEATURE_STEAL_TIME);
arch/x86/kvm/svm.c +5 -0
index c99b0207a443..60ddc242a133 100644
@@ -7632,6 +7632,7 @@ static int svm_page_enc_status_hc(struct kvm *kvm, unsigned long gpa,
 				  unsigned long npages, unsigned long enc)
 {
 	struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
+	struct kvm_vcpu *vcpu = kvm->vcpus[0];
 	kvm_pfn_t pfn_start, pfn_end;
 	gfn_t gfn_start, gfn_end;
 	int ret;
@@ -7639,6 +7640,10 @@ static int svm_page_enc_status_hc(struct kvm *kvm, unsigned long gpa,
 	if (!sev_guest(kvm))
 		return -EINVAL;
 
+	if (!(vcpu->arch.msr_kvm_sev_live_migration_flag &
+		KVM_SEV_LIVE_MIGRATION_ENABLED))
+		return -ENOTTY;
+
 	if (!npages)
 		return 0;
 
arch/x86/kvm/x86.c +7 -0
index 2127ed937f53..82867b8798f8 100644
@@ -2880,6 +2880,10 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 		vcpu->arch.msr_kvm_poll_control = data;
 		break;
 
+	case MSR_KVM_SEV_LIVE_MIG_EN:
+		vcpu->arch.msr_kvm_sev_live_migration_flag = data;
+		break;
+
 	case MSR_IA32_MCG_CTL:
 	case MSR_IA32_MCG_STATUS:
 	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
@@ -3126,6 +3130,9 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 	case MSR_KVM_POLL_CONTROL:
 		msr_info->data = vcpu->arch.msr_kvm_poll_control;
 		break;
+	case MSR_KVM_SEV_LIVE_MIG_EN:
+		msr_info->data = vcpu->arch.msr_kvm_sev_live_migration_flag;
+		break;
 	case MSR_IA32_P5_MC_ADDR:
 	case MSR_IA32_P5_MC_TYPE:
 	case MSR_IA32_MCG_CAP:
arch/x86/mm/mem_encrypt.c +9 -1
index c9800fa811f6..f6a841494845 100644
@@ -502,8 +502,20 @@ void __init mem_encrypt_init(void)
 	 * With SEV, we need to make a hypercall when page encryption state is
 	 * changed.
 	 */
-	if (sev_active())
+	if (sev_active()) {
+		unsigned long nr_pages;
+
 		pv_ops.mmu.page_encryption_changed = set_memory_enc_dec_hypercall;
+
+		/*
+		 * Ensure that _bss_decrypted section is marked as decrypted in the
+		 * page encryption bitmap.
+		 */
+		nr_pages = DIV_ROUND_UP(__end_bss_decrypted - __start_bss_decrypted,
Quoted 10 lines

arch/x86/mm/mem_encrypt.c void __init mem_encrypt_init(void) · line 518

Quoted 4 lines
arch/x86/mm/mem_encrypt.c +4 -0
+			PAGE_SIZE);
+		set_memory_enc_dec_hypercall((unsigned long)__start_bss_decrypted,
+			nr_pages, 0);
+	}

Isn't this too late, should we be making hypercall at the same time we clear the encryption bit ?

Actually this is being done somewhat lazily, after the guest enables/activates the live migration feature, it should be fine to do it here or it can be moved into sev_map_percpu_data() where the first hypercalls are done, in both cases the __bss_decrypted section will be marked before the live migration process is initiated.

Quoted 3 lines

arch/x86/mm/mem_encrypt.c void __init mem_encrypt_init(void) · line 521

Quoted 3 lines
arch/x86/mm/mem_encrypt.c +0 -0
 #endif
 
 	pr_info("AMD %s active\n",

Thanks,
Ashish

A message that is not in the archive (sent off-list or not yet received): 20200330162730.GA21567@ashkalra_ubuntu_server

A message that is not in the archive (sent off-list or not yet received): [email protected]

A message that is not in the archive (sent off-list or not yet received): 20200331171336.GA24050@ashkalra_ubuntu_server

A message that is not in the archive (sent off-list or not yet received): 20200401070931.GA8562@ashkalra_ubuntu_server

#95 lore raw

Re: [PATCH v6 13/14] KVM: x86: Introduce new KVM_FEATURE_SEV_LIVE_MIGRATION feature & Custom MSR.

Hello Brijesh,

Quoted 23 lines · Ashish Kalra
Quoted 21 lines

Hello Brijesh,

Quoted 7 lines
Quoted 5 lines

Actually this is being done somewhat lazily, after the guest enables/activates the live migration feature, it should be fine to do it here or it can be moved into sev_map_percpu_data() where the first hypercalls are done, in both cases the __bss_decrypted section will be marked before the live migration process is initiated.

Quoted 7 lines

IMO, its not okay to do it here or inside sev_map_percpu_data(). So far, as soon as C-bit state is changed in page table we make a hypercall. It will be good idea to stick to that approach. I don't see any reason why we need to make an exception for the __bss_decrypted unless I am missing something. What will happen if VMM initiate the migration while guest BIOS is booting?  Are you saying its not supported ?

The one thing this will require is checking for KVM para capability KVM_FEATURE_SEV_LIVE_MIGRATION as part of this code in startup_64(), i need to verify if i can check for this feature so early in startup code.

Quoted 5 lines
Quoted 5 lines

I need to check for this capability and do the wrmsrl() here as this will be the 1st hypercall in the guest kernel and i will need to enable live migration feature and hypercall support on the host before making the hypercall.

I added the KVM para feature capability check here in startup_64(), and as i thought this does "not" work and also as a side effect disables the KVM paravirtualization check and so KVM paravirtualization is not detected later during kernel boot and all KVM paravirt features remain disabled.

Digged deeper into this and here's what happens ...

kvm_para_has_feature() calls kvm_arch_para_feature() which in turn calls kvm_cpuid_base() and this invokes __kvm_cpuid_base(). As the "boot_cpu_data" is still not populated/setup, therefore, __kvm_cpuid_base() does not detect X86_FEATURE_HYPERVISOR and also as a side effect sets the variable kvm_cpuid_base == 0.

So as the kvm_para_feature() is not detected in startup_64(), therefore the hypercall does not get invoked and also as the side effect of calling kvm_para_feature() in startup_64(), the static variable "kvm_cpuid_base" gets set to 0, and later during hypervisor detection (kvm_detect), this variable's setting causes kvm_detect() to return failure and hence KVM paravirtualization features don't get enabled for the guest kernel.

So, calling kvm_para_has_feature() so early in startup_64() code is not going to work, hence, it is probably best to do the hypercall to mark __bss_decrypted section as decrypted (lazily) as part of sev_map_percpu_data() as per my original thought.

Thanks,
Ashish

Reviewed-by on v6 13/14
#96 lore raw

Re: [PATCH v6 13/14] KVM: x86: Introduce new KVM_FEATURE_SEV_LIVE_MIGRATION feature & Custom MSR.

In reply to Ashish Kalra
Quoted 191 lines · Ashish Kalra

From: Ashish Kalra <[email protected]>

Add new KVM_FEATURE_SEV_LIVE_MIGRATION feature for guest to check for host-side support for SEV live migration. Also add a new custom MSR_KVM_SEV_LIVE_MIG_EN for guest to enable the SEV live migration feature.

Also, ensure that _bss_decrypted section is marked as decrypted in the page encryption bitmap.

---
  Documentation/virt/kvm/cpuid.rst     |  4 ++++
  Documentation/virt/kvm/msr.rst       | 10 ++++++++++
  arch/x86/include/asm/kvm_host.h      |  3 +++
  arch/x86/include/uapi/asm/kvm_para.h |  5 +++++
  arch/x86/kernel/kvm.c                |  4 ++++
  arch/x86/kvm/cpuid.c                 |  3 ++-
  arch/x86/kvm/svm.c                   |  5 +++++
  arch/x86/kvm/x86.c                   |  7 +++++++
  arch/x86/mm/mem_encrypt.c            | 14 +++++++++++++-
  9 files changed, 53 insertions(+), 2 deletions(-)
Documentation/virt/kvm/cpuid.rst +4 -0
index 01b081f6e7ea..fcb191bb3016 100644
@@ -86,6 +86,10 @@ KVM_FEATURE_PV_SCHED_YIELD        13          guest checks this feature bit
                                                before using paravirtualized
                                                sched yield.
 
+KVM_FEATURE_SEV_LIVE_MIGRATION    14          guest checks this feature bit
+                                              before enabling SEV live
+                                              migration feature.
+
  KVM_FEATURE_CLOCSOURCE_STABLE_BIT 24          host will warn if no guest-side
                                                per-cpu warps are expeced in
                                                kvmclock
Documentation/virt/kvm/msr.rst +10 -0
index 33892036672d..7cd7786bbb03 100644
@@ -319,3 +319,13 @@ data:
 
  	KVM guests can request the host not to poll on HLT, for example if
  	they are performing polling themselves.
+
+MSR_KVM_SEV_LIVE_MIG_EN:
+        0x4b564d06
+
+	Control SEV Live Migration features.
+
+data:
+        Bit 0 enables (1) or disables (0) host-side SEV Live Migration feature.
+        Bit 1 enables (1) or disables (0) support for SEV Live Migration extensions.
+        All other bits are reserved.
arch/x86/include/asm/kvm_host.h +3 -0
index a96ef6338cd2..ad5faaed43c0 100644
@@ -780,6 +780,9 @@ struct kvm_vcpu_arch {
 
  	u64 msr_kvm_poll_control;
 
+	/* SEV Live Migration MSR (AMD only) */
+	u64 msr_kvm_sev_live_migration_flag;
+
  	/*
  	 * Indicates the guest is trying to write a gfn that contains one or
  	 * more of the PTEs used to translate the write itself, i.e. the access
arch/x86/include/uapi/asm/kvm_para.h +5 -0
index 2a8e0b6b9805..d9d4953b42ad 100644
@@ -31,6 +31,7 @@
  #define KVM_FEATURE_PV_SEND_IPI	11
  #define KVM_FEATURE_POLL_CONTROL	12
  #define KVM_FEATURE_PV_SCHED_YIELD	13
+#define KVM_FEATURE_SEV_LIVE_MIGRATION	14
 
  #define KVM_HINTS_REALTIME      0
 
@@ -50,6 +51,7 @@
  #define MSR_KVM_STEAL_TIME  0x4b564d03
  #define MSR_KVM_PV_EOI_EN      0x4b564d04
  #define MSR_KVM_POLL_CONTROL	0x4b564d05
+#define MSR_KVM_SEV_LIVE_MIG_EN	0x4b564d06
 
  struct kvm_steal_time {
  	__u64 steal;
@@ -122,4 +124,7 @@ struct kvm_vcpu_pv_apf_data {
  #define KVM_PV_EOI_ENABLED KVM_PV_EOI_MASK
  #define KVM_PV_EOI_DISABLED 0x0
 
+#define KVM_SEV_LIVE_MIGRATION_ENABLED			(1 << 0)
+#define KVM_SEV_LIVE_MIGRATION_EXTENSIONS_SUPPORTED	(1 << 1)
+
  #endif /* _UAPI_ASM_X86_KVM_PARA_H */
arch/x86/kernel/kvm.c +4 -0
index 6efe0410fb72..8fcee0b45231 100644
@@ -418,6 +418,10 @@ static void __init sev_map_percpu_data(void)
  	if (!sev_active())
  		return;
 
+	if (kvm_para_has_feature(KVM_FEATURE_SEV_LIVE_MIGRATION)) {
+		wrmsrl(MSR_KVM_SEV_LIVE_MIG_EN, KVM_SEV_LIVE_MIGRATION_ENABLED);
+	}
+
  	for_each_possible_cpu(cpu) {
  		__set_percpu_decrypted(&per_cpu(apf_reason, cpu), sizeof(apf_reason));
  		__set_percpu_decrypted(&per_cpu(steal_time, cpu), sizeof(steal_time));
arch/x86/kvm/cpuid.c +2 -1
index b1c469446b07..74c8b2a7270c 100644
@@ -716,7 +716,8 @@ static inline int __do_cpuid_func(struct kvm_cpuid_entry2 *entry, u32 function,
  			     (1 << KVM_FEATURE_ASYNC_PF_VMEXIT) |
  			     (1 << KVM_FEATURE_PV_SEND_IPI) |
  			     (1 << KVM_FEATURE_POLL_CONTROL) |
-			     (1 << KVM_FEATURE_PV_SCHED_YIELD);
+			     (1 << KVM_FEATURE_PV_SCHED_YIELD) |
+			     (1 << KVM_FEATURE_SEV_LIVE_MIGRATION);
 
  		if (sched_info_on())
  			entry->eax |= (1 << KVM_FEATURE_STEAL_TIME);
arch/x86/kvm/svm.c +5 -0
index c99b0207a443..60ddc242a133 100644
@@ -7632,6 +7632,7 @@ static int svm_page_enc_status_hc(struct kvm *kvm, unsigned long gpa,
  				  unsigned long npages, unsigned long enc)
  {
  	struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
+	struct kvm_vcpu *vcpu = kvm->vcpus[0];
  	kvm_pfn_t pfn_start, pfn_end;
  	gfn_t gfn_start, gfn_end;
  	int ret;
@@ -7639,6 +7640,10 @@ static int svm_page_enc_status_hc(struct kvm *kvm, unsigned long gpa,
  	if (!sev_guest(kvm))
  		return -EINVAL;
 
+	if (!(vcpu->arch.msr_kvm_sev_live_migration_flag &
+		KVM_SEV_LIVE_MIGRATION_ENABLED))
+		return -ENOTTY;
+
  	if (!npages)
  		return 0;
 
arch/x86/kvm/x86.c +7 -0
index 2127ed937f53..82867b8798f8 100644
@@ -2880,6 +2880,10 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
  		vcpu->arch.msr_kvm_poll_control = data;
  		break;
 
+	case MSR_KVM_SEV_LIVE_MIG_EN:
+		vcpu->arch.msr_kvm_sev_live_migration_flag = data;
+		break;
+
  	case MSR_IA32_MCG_CTL:
  	case MSR_IA32_MCG_STATUS:
  	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
@@ -3126,6 +3130,9 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
  	case MSR_KVM_POLL_CONTROL:
  		msr_info->data = vcpu->arch.msr_kvm_poll_control;
  		break;
+	case MSR_KVM_SEV_LIVE_MIG_EN:
+		msr_info->data = vcpu->arch.msr_kvm_sev_live_migration_flag;
+		break;
  	case MSR_IA32_P5_MC_ADDR:
  	case MSR_IA32_P5_MC_TYPE:
  	case MSR_IA32_MCG_CAP:
arch/x86/mm/mem_encrypt.c +6 -1
index c9800fa811f6..f6a841494845 100644
@@ -502,8 +502,20 @@ void __init mem_encrypt_init(void)
  	 * With SEV, we need to make a hypercall when page encryption state is
  	 * changed.
  	 */
-	if (sev_active())
+	if (sev_active()) {
+		unsigned long nr_pages;
+
  		pv_ops.mmu.page_encryption_changed = set_memory_enc_dec_hypercall;
+
+		/*
+		 * Ensure that _bss_decrypted section is marked as decrypted in the

arch/x86/mm/mem_encrypt.c void __init mem_encrypt_init(void) · line 521

Quoted 10 lines
arch/x86/mm/mem_encrypt.c +7 -0
+		 * page encryption bitmap.
+		 */
+		nr_pages = DIV_ROUND_UP(__end_bss_decrypted - __start_bss_decrypted,
+			PAGE_SIZE);
+		set_memory_enc_dec_hypercall((unsigned long)__start_bss_decrypted,
+			nr_pages, 0);
+	}
  #endif
 
  	pr_info("AMD %s active\n",
#97 lore raw

PATCH v6 14/14 KVM: x86: Add kexec support for SEV Live Migration.

In reply to Ashish Kalra

From: Ashish Kalra <[email protected]>

Reset the host's page encryption bitmap related to kernel specific page encryption status settings before we load a new kernel by kexec. We cannot reset the complete page encryption bitmap here as we need to retain the UEFI/OVMF firmware specific settings.

---
 arch/x86/kernel/kvm.c | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)
arch/x86/kernel/kvm.c +28 -0
index 8fcee0b45231..ba6cce3c84af 100644
@@ -34,6 +34,7 @@
 #include <asm/hypervisor.h>
 #include <asm/tlb.h>
 #include <asm/cpuidle_haltpoll.h>
+#include <asm/e820/api.h>
 
 static int kvmapf = 1;
 
@@ -357,6 +358,33 @@ static void kvm_pv_guest_cpu_reboot(void *unused)
 	 */
 	if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))
 		wrmsrl(MSR_KVM_PV_EOI_EN, 0);
+	/*
+	 * Reset the host's page encryption bitmap related to kernel
+	 * specific page encryption status settings before we load a
+	 * new kernel by kexec. NOTE: We cannot reset the complete
+	 * page encryption bitmap here as we need to retain the
+	 * UEFI/OVMF firmware specific settings.
+	 */
+	if (kvm_para_has_feature(KVM_FEATURE_SEV_LIVE_MIGRATION) &&
+		(smp_processor_id() == 0)) {
+		unsigned long nr_pages;
+		int i;
+
+		for (i = 0; i < e820_table->nr_entries; i++) {
+			struct e820_entry *entry = &e820_table->entries[i];
+			unsigned long start_pfn, end_pfn;
+
+			if (entry->type != E820_TYPE_RAM)
+				continue;
+
+			start_pfn = entry->addr >> PAGE_SHIFT;
+			end_pfn = (entry->addr + entry->size) >> PAGE_SHIFT;
+			nr_pages = DIV_ROUND_UP(entry->size, PAGE_SIZE);
+
+			kvm_sev_hypercall3(KVM_HC_PAGE_ENC_STATUS,
+				entry->addr, nr_pages, 1);
+		}
+	}
 	kvm_pv_disable_apf();
 	kvm_disable_steal_time();
 }
Signature · 1 line
2.17.1
#98 lore raw

Re: [PATCH v6 14/14] KVM: x86: Add kexec support for SEV Live Migration.

Quoted 28 lines · Ashish Kalra

From: Ashish Kalra <[email protected]>

Reset the host's page encryption bitmap related to kernel specific page encryption status settings before we load a new kernel by kexec. We cannot reset the complete page encryption bitmap here as we need to retain the UEFI/OVMF firmware specific settings.

---
 arch/x86/kernel/kvm.c | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)
arch/x86/kernel/kvm.c +1 -0
index 8fcee0b45231..ba6cce3c84af 100644
@@ -34,6 +34,7 @@
 #include <asm/hypervisor.h>
 #include <asm/tlb.h>
 #include <asm/cpuidle_haltpoll.h>
+#include <asm/e820/api.h>
 
 static int kvmapf = 1;
 
@@ -357,6 +358,33 @@ static void kvm_pv_guest_cpu_reboot(void *unused)
 	 */
 	if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))

arch/x86/kernel/kvm.c static void kvm_pv_guest_cpu_reboot(void *unused) · line 369

Quoted 10 lines
arch/x86/kernel/kvm.c +9 -0
 		wrmsrl(MSR_KVM_PV_EOI_EN, 0);
+	/*
+	 * Reset the host's page encryption bitmap related to kernel
+	 * specific page encryption status settings before we load a
+	 * new kernel by kexec. NOTE: We cannot reset the complete
+	 * page encryption bitmap here as we need to retain the
+	 * UEFI/OVMF firmware specific settings.
+	 */
+	if (kvm_para_has_feature(KVM_FEATURE_SEV_LIVE_MIGRATION) &&
+		(smp_processor_id() == 0)) {

In patch 13/14, the KVM_FEATURE_SEV_LIVE_MIGRATION is set unconditionally and because of that now the below code will be executed on non-SEV guest. IMO, this feature must be cleared for non-SEV guest to avoid making unnecessary hypercall's.

Quoted 21 lines
arch/x86/kernel/kvm.c +18 -0
+		unsigned long nr_pages;
+		int i;
+
+		for (i = 0; i < e820_table->nr_entries; i++) {
+			struct e820_entry *entry = &e820_table->entries[i];
+			unsigned long start_pfn, end_pfn;
+
+			if (entry->type != E820_TYPE_RAM)
+				continue;
+
+			start_pfn = entry->addr >> PAGE_SHIFT;
+			end_pfn = (entry->addr + entry->size) >> PAGE_SHIFT;
+			nr_pages = DIV_ROUND_UP(entry->size, PAGE_SIZE);
+
+			kvm_sev_hypercall3(KVM_HC_PAGE_ENC_STATUS,
+				entry->addr, nr_pages, 1);
+		}
+	}
 	kvm_pv_disable_apf();
 	kvm_disable_steal_time();
 }
#99 lore raw

Re: [PATCH v6 14/14] KVM: x86: Add kexec support for SEV Live Migration.

Hello Brijesh,

Quoted 38 lines · Brijesh Singh
Quoted 36 lines · Ashish Kalra

From: Ashish Kalra <[email protected]>

Reset the host's page encryption bitmap related to kernel specific page encryption status settings before we load a new kernel by kexec. We cannot reset the complete page encryption bitmap here as we need to retain the UEFI/OVMF firmware specific settings.

---
 arch/x86/kernel/kvm.c | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)
arch/x86/kernel/kvm.c +8 -0
index 8fcee0b45231..ba6cce3c84af 100644
@@ -34,6 +34,7 @@
 #include <asm/hypervisor.h>
 #include <asm/tlb.h>
 #include <asm/cpuidle_haltpoll.h>
+#include <asm/e820/api.h>
 
 static int kvmapf = 1;
 
@@ -357,6 +358,33 @@ static void kvm_pv_guest_cpu_reboot(void *unused)
 	 */
 	if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))
 		wrmsrl(MSR_KVM_PV_EOI_EN, 0);
+	/*
+	 * Reset the host's page encryption bitmap related to kernel
+	 * specific page encryption status settings before we load a
+	 * new kernel by kexec. NOTE: We cannot reset the complete
+	 * page encryption bitmap here as we need to retain the
+	 * UEFI/OVMF firmware specific settings.
+	 */
Quoted 10 lines

arch/x86/kernel/kvm.c static void kvm_pv_guest_cpu_reboot(void *unused) · line 369

Quoted 2 lines
arch/x86/kernel/kvm.c +2 -0
+	if (kvm_para_has_feature(KVM_FEATURE_SEV_LIVE_MIGRATION) &&
+		(smp_processor_id() == 0)) {

In patch 13/14, the KVM_FEATURE_SEV_LIVE_MIGRATION is set unconditionally and because of that now the below code will be executed on non-SEV guest. IMO, this feature must be cleared for non-SEV guest to avoid making unnecessary hypercall's.

I will additionally add a sev_active() check here to ensure that we don't make the unnecassary hypercalls on non-SEV guests.

Quoted 13 lines
Quoted 13 lines
arch/x86/kernel/kvm.c +13 -0
+		unsigned long nr_pages;
+		int i;
+
+		for (i = 0; i < e820_table->nr_entries; i++) {
+			struct e820_entry *entry = &e820_table->entries[i];
+			unsigned long start_pfn, end_pfn;
+
+			if (entry->type != E820_TYPE_RAM)
+				continue;
+
+			start_pfn = entry->addr >> PAGE_SHIFT;
+			end_pfn = (entry->addr + entry->size) >> PAGE_SHIFT;
+			nr_pages = DIV_ROUND_UP(entry->size, PAGE_SIZE);
Quoted 8 lines

arch/x86/kernel/kvm.c static void kvm_pv_guest_cpu_reboot(void *unused) · line 390

Quoted 8 lines
arch/x86/kernel/kvm.c +5 -0
+
+			kvm_sev_hypercall3(KVM_HC_PAGE_ENC_STATUS,
+				entry->addr, nr_pages, 1);
+		}
+	}
 	kvm_pv_disable_apf();
 	kvm_disable_steal_time();
 }

Thanks,
Ashish

#100 lore raw

Re: [PATCH v6 14/14] KVM: x86: Add kexec support for SEV Live Migration.

Quoted 49 lines · Ashish Kalra

Hello Brijesh,

Quoted 36 lines · Brijesh Singh
Quoted 35 lines · Ashish Kalra

From: Ashish Kalra <[email protected]>

Reset the host's page encryption bitmap related to kernel specific page encryption status settings before we load a new kernel by kexec. We cannot reset the complete page encryption bitmap here as we need to retain the UEFI/OVMF firmware specific settings.

---
 arch/x86/kernel/kvm.c | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)
arch/x86/kernel/kvm.c +7 -0
index 8fcee0b45231..ba6cce3c84af 100644
@@ -34,6 +34,7 @@
 #include <asm/hypervisor.h>
 #include <asm/tlb.h>
 #include <asm/cpuidle_haltpoll.h>
+#include <asm/e820/api.h>
 
 static int kvmapf = 1;
 
@@ -357,6 +358,33 @@ static void kvm_pv_guest_cpu_reboot(void *unused)
 	 */
 	if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))
 		wrmsrl(MSR_KVM_PV_EOI_EN, 0);
+	/*
+	 * Reset the host's page encryption bitmap related to kernel
+	 * specific page encryption status settings before we load a
+	 * new kernel by kexec. NOTE: We cannot reset the complete
+	 * page encryption bitmap here as we need to retain the
+	 * UEFI/OVMF firmware specific settings.
Quoted 10 lines

arch/x86/kernel/kvm.c static void kvm_pv_guest_cpu_reboot(void *unused) · line 369

Quoted 3 lines
arch/x86/kernel/kvm.c +3 -0
+	 */
+	if (kvm_para_has_feature(KVM_FEATURE_SEV_LIVE_MIGRATION) &&
+		(smp_processor_id() == 0)) {

In patch 13/14, the KVM_FEATURE_SEV_LIVE_MIGRATION is set unconditionally and because of that now the below code will be executed on non-SEV guest. IMO, this feature must be cleared for non-SEV guest to avoid making unnecessary hypercall's.

Quoted 1 line

I will additionally add a sev_active() check here to ensure that we don't make the unnecassary hypercalls on non-SEV guests.

IMO, instead of using the sev_active() we should make sure that the feature is not enabled when SEV is not active.

Quoted 23 lines
Quoted 13 lines
Quoted 13 lines
arch/x86/kernel/kvm.c +13 -0
+		unsigned long nr_pages;
+		int i;
+
+		for (i = 0; i < e820_table->nr_entries; i++) {
+			struct e820_entry *entry = &e820_table->entries[i];
+			unsigned long start_pfn, end_pfn;
+
+			if (entry->type != E820_TYPE_RAM)
+				continue;
+
+			start_pfn = entry->addr >> PAGE_SHIFT;
+			end_pfn = (entry->addr + entry->size) >> PAGE_SHIFT;
+			nr_pages = DIV_ROUND_UP(entry->size, PAGE_SIZE);
Quoted 8 lines

arch/x86/kernel/kvm.c static void kvm_pv_guest_cpu_reboot(void *unused) · line 390

Quoted 8 lines
arch/x86/kernel/kvm.c +5 -0
+
+			kvm_sev_hypercall3(KVM_HC_PAGE_ENC_STATUS,
+				entry->addr, nr_pages, 1);
+		}
+	}
 	kvm_pv_disable_apf();
 	kvm_disable_steal_time();
 }

Thanks,
Ashish

#101 lore raw

Re: [PATCH v6 14/14] KVM: x86: Add kexec support for SEV Live Migration.

Hello Brijesh,

Quoted 51 lines · Brijesh Singh
Quoted 49 lines · Ashish Kalra

Hello Brijesh,

Quoted 46 lines · Brijesh Singh
Quoted 28 lines · Ashish Kalra

From: Ashish Kalra <[email protected]>

Reset the host's page encryption bitmap related to kernel specific page encryption status settings before we load a new kernel by kexec. We cannot reset the complete page encryption bitmap here as we need to retain the UEFI/OVMF firmware specific settings.

---
 arch/x86/kernel/kvm.c | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)
arch/x86/kernel/kvm.c +1 -0
index 8fcee0b45231..ba6cce3c84af 100644
@@ -34,6 +34,7 @@
 #include <asm/hypervisor.h>
 #include <asm/tlb.h>
 #include <asm/cpuidle_haltpoll.h>
+#include <asm/e820/api.h>
 
 static int kvmapf = 1;
 
@@ -357,6 +358,33 @@ static void kvm_pv_guest_cpu_reboot(void *unused)
 	 */
 	if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))

arch/x86/kernel/kvm.c static void kvm_pv_guest_cpu_reboot(void *unused) · line 369

Quoted 10 lines
arch/x86/kernel/kvm.c +9 -0
 		wrmsrl(MSR_KVM_PV_EOI_EN, 0);
+	/*
+	 * Reset the host's page encryption bitmap related to kernel
+	 * specific page encryption status settings before we load a
+	 * new kernel by kexec. NOTE: We cannot reset the complete
+	 * page encryption bitmap here as we need to retain the
+	 * UEFI/OVMF firmware specific settings.
+	 */
+	if (kvm_para_has_feature(KVM_FEATURE_SEV_LIVE_MIGRATION) &&
+		(smp_processor_id() == 0)) {

In patch 13/14, the KVM_FEATURE_SEV_LIVE_MIGRATION is set unconditionally and because of that now the below code will be executed on non-SEV guest. IMO, this feature must be cleared for non-SEV guest to avoid making unnecessary hypercall's.

Quoted 6 lines
Quoted 1 line

I will additionally add a sev_active() check here to ensure that we don't make the unnecassary hypercalls on non-SEV guests.

IMO, instead of using the sev_active() we should make sure that the feature is not enabled when SEV is not active.

Yes, now the KVM_FEATURE_SEV_LIVE_MIGRATION feature is enabled dynamically in svm_cpuid_update() after it gets called from svm_launch_finish(), which ensures that it only gets set when a SEV guest is active.

Thanks,
Ashish

Quoted 24 lines
Quoted 24 lines
Quoted 14 lines
Quoted 14 lines
arch/x86/kernel/kvm.c +13 -0
+		unsigned long nr_pages;
+		int i;
+
+		for (i = 0; i < e820_table->nr_entries; i++) {
+			struct e820_entry *entry = &e820_table->entries[i];
+			unsigned long start_pfn, end_pfn;
+
+			if (entry->type != E820_TYPE_RAM)
+				continue;
+
+			start_pfn = entry->addr >> PAGE_SHIFT;
+			end_pfn = (entry->addr + entry->size) >> PAGE_SHIFT;
+			nr_pages = DIV_ROUND_UP(entry->size, PAGE_SIZE);
Quoted 8 lines

arch/x86/kernel/kvm.c static void kvm_pv_guest_cpu_reboot(void *unused) · line 390

Quoted 8 lines
arch/x86/kernel/kvm.c +5 -0
+
+			kvm_sev_hypercall3(KVM_HC_PAGE_ENC_STATUS,
+				entry->addr, nr_pages, 1);
+		}
+	}
 	kvm_pv_disable_apf();
 	kvm_disable_steal_time();
 }

Thanks,
Ashish

#102 lore raw

Re: [PATCH v6 14/14] KVM: x86: Add kexec support for SEV Live Migration.

In reply to Ashish Kalra

Ccing kexec list.

Ashish, could you cc kexec list if you repost later?

Quoted 62 lines · Ashish Kalra

From: Ashish Kalra <[email protected]>

Reset the host's page encryption bitmap related to kernel specific page encryption status settings before we load a new kernel by kexec. We cannot reset the complete page encryption bitmap here as we need to retain the UEFI/OVMF firmware specific settings.

---
 arch/x86/kernel/kvm.c | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)
arch/x86/kernel/kvm.c +28 -0
index 8fcee0b45231..ba6cce3c84af 100644
@@ -34,6 +34,7 @@
 #include <asm/hypervisor.h>
 #include <asm/tlb.h>
 #include <asm/cpuidle_haltpoll.h>
+#include <asm/e820/api.h>
 
 static int kvmapf = 1;
 
@@ -357,6 +358,33 @@ static void kvm_pv_guest_cpu_reboot(void *unused)
 	 */
 	if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))
 		wrmsrl(MSR_KVM_PV_EOI_EN, 0);
+	/*
+	 * Reset the host's page encryption bitmap related to kernel
+	 * specific page encryption status settings before we load a
+	 * new kernel by kexec. NOTE: We cannot reset the complete
+	 * page encryption bitmap here as we need to retain the
+	 * UEFI/OVMF firmware specific settings.
+	 */
+	if (kvm_para_has_feature(KVM_FEATURE_SEV_LIVE_MIGRATION) &&
+		(smp_processor_id() == 0)) {
+		unsigned long nr_pages;
+		int i;
+
+		for (i = 0; i < e820_table->nr_entries; i++) {
+			struct e820_entry *entry = &e820_table->entries[i];
+			unsigned long start_pfn, end_pfn;
+
+			if (entry->type != E820_TYPE_RAM)
+				continue;
+
+			start_pfn = entry->addr >> PAGE_SHIFT;
+			end_pfn = (entry->addr + entry->size) >> PAGE_SHIFT;
+			nr_pages = DIV_ROUND_UP(entry->size, PAGE_SIZE);
+
+			kvm_sev_hypercall3(KVM_HC_PAGE_ENC_STATUS,
+				entry->addr, nr_pages, 1);
+		}
+	}
 	kvm_pv_disable_apf();
 	kvm_disable_steal_time();
 }
Signature · 1 line
2.17.1
#103 lore raw

Re: [PATCH v6 14/14] KVM: x86: Add kexec support for SEV Live Migration.

In reply to Ashish Kalra
Quoted 7 lines · Ashish Kalra

From: Ashish Kalra <[email protected]>

Reset the host's page encryption bitmap related to kernel specific page encryption status settings before we load a new kernel by kexec. We cannot reset the complete page encryption bitmap here as we need to retain the UEFI/OVMF firmware specific settings.

Can the commit message mention why host page encryption needs to be reset ? Since the theme of these patches is guest migration in-SEV context, it might be useful to mention why the host context comes in here.

Quoted 52 lines
---
  arch/x86/kernel/kvm.c | 28 ++++++++++++++++++++++++++++
  1 file changed, 28 insertions(+)
arch/x86/kernel/kvm.c +28 -0
index 8fcee0b45231..ba6cce3c84af 100644
@@ -34,6 +34,7 @@
  #include <asm/hypervisor.h>
  #include <asm/tlb.h>
  #include <asm/cpuidle_haltpoll.h>
+#include <asm/e820/api.h>
 
  static int kvmapf = 1;
 
@@ -357,6 +358,33 @@ static void kvm_pv_guest_cpu_reboot(void *unused)
  	 */
  	if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))
  		wrmsrl(MSR_KVM_PV_EOI_EN, 0);
+	/*
+	 * Reset the host's page encryption bitmap related to kernel
+	 * specific page encryption status settings before we load a
+	 * new kernel by kexec. NOTE: We cannot reset the complete
+	 * page encryption bitmap here as we need to retain the
+	 * UEFI/OVMF firmware specific settings.
+	 */
+	if (kvm_para_has_feature(KVM_FEATURE_SEV_LIVE_MIGRATION) &&
+		(smp_processor_id() == 0)) {
+		unsigned long nr_pages;
+		int i;
+
+		for (i = 0; i < e820_table->nr_entries; i++) {
+			struct e820_entry *entry = &e820_table->entries[i];
+			unsigned long start_pfn, end_pfn;
+
+			if (entry->type != E820_TYPE_RAM)
+				continue;
+
+			start_pfn = entry->addr >> PAGE_SHIFT;
+			end_pfn = (entry->addr + entry->size) >> PAGE_SHIFT;
+			nr_pages = DIV_ROUND_UP(entry->size, PAGE_SIZE);
+
+			kvm_sev_hypercall3(KVM_HC_PAGE_ENC_STATUS,
+				entry->addr, nr_pages, 1);
+		}
+	}
  	kvm_pv_disable_apf();
  	kvm_disable_steal_time();
  }
#104 lore raw

Re: [PATCH v6 14/14] KVM: x86: Add kexec support for SEV Live Migration.

The host's page encryption bitmap is maintained for the guest to keep the encrypted/decrypted state of the guest pages, therefore we need to explicitly mark all shared pages as encrypted again before rebooting into the new guest kernel.

Quoted 65 lines · Krish Sadhukhan
Quoted 7 lines · Ashish Kalra

From: Ashish Kalra <[email protected]>

Reset the host's page encryption bitmap related to kernel specific page encryption status settings before we load a new kernel by kexec. We cannot reset the complete page encryption bitmap here as we need to retain the UEFI/OVMF firmware specific settings.

Can the commit message mention why host page encryption needs to be reset ? Since the theme of these patches is guest migration in-SEV context, it might be useful to mention why the host context comes in here.

Quoted 50 lines
---
  arch/x86/kernel/kvm.c | 28 ++++++++++++++++++++++++++++
  1 file changed, 28 insertions(+)
arch/x86/kernel/kvm.c +28 -0
index 8fcee0b45231..ba6cce3c84af 100644
@@ -34,6 +34,7 @@
  #include <asm/hypervisor.h>
  #include <asm/tlb.h>
  #include <asm/cpuidle_haltpoll.h>
+#include <asm/e820/api.h>
  static int kvmapf = 1;
@@ -357,6 +358,33 @@ static void kvm_pv_guest_cpu_reboot(void *unused)
  	 */
  	if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))
  		wrmsrl(MSR_KVM_PV_EOI_EN, 0);
+	/*
+	 * Reset the host's page encryption bitmap related to kernel
+	 * specific page encryption status settings before we load a
+	 * new kernel by kexec. NOTE: We cannot reset the complete
+	 * page encryption bitmap here as we need to retain the
+	 * UEFI/OVMF firmware specific settings.
+	 */
+	if (kvm_para_has_feature(KVM_FEATURE_SEV_LIVE_MIGRATION) &&
+		(smp_processor_id() == 0)) {
+		unsigned long nr_pages;
+		int i;
+
+		for (i = 0; i < e820_table->nr_entries; i++) {
+			struct e820_entry *entry = &e820_table->entries[i];
+			unsigned long start_pfn, end_pfn;
+
+			if (entry->type != E820_TYPE_RAM)
+				continue;
+
+			start_pfn = entry->addr >> PAGE_SHIFT;
+			end_pfn = (entry->addr + entry->size) >> PAGE_SHIFT;
+			nr_pages = DIV_ROUND_UP(entry->size, PAGE_SIZE);
+
+			kvm_sev_hypercall3(KVM_HC_PAGE_ENC_STATUS,
+				entry->addr, nr_pages, 1);
+		}
+	}
  	kvm_pv_disable_apf();
  	kvm_disable_steal_time();
  }
Reviewed-by
#105 lore raw

Re: [PATCH v6 14/14] KVM: x86: Add kexec support for SEV Live Migration.

Quoted 59 lines · Ashish Kalra

The host's page encryption bitmap is maintained for the guest to keep the encrypted/decrypted state of the guest pages, therefore we need to explicitly mark all shared pages as encrypted again before rebooting into the new guest kernel.

Quoted 54 lines · Krish Sadhukhan
Quoted 7 lines · Ashish Kalra

From: Ashish Kalra <[email protected]>

Reset the host's page encryption bitmap related to kernel specific page encryption status settings before we load a new kernel by kexec. We cannot reset the complete page encryption bitmap here as we need to retain the UEFI/OVMF firmware specific settings.

Can the commit message mention why host page encryption needs to be reset ? Since the theme of these patches is guest migration in-SEV context, it might be useful to mention why the host context comes in here.

Quoted 41 lines
---
   arch/x86/kernel/kvm.c | 28 ++++++++++++++++++++++++++++
   1 file changed, 28 insertions(+)
arch/x86/kernel/kvm.c +23 -0
index 8fcee0b45231..ba6cce3c84af 100644
@@ -34,6 +34,7 @@
   #include <asm/hypervisor.h>
   #include <asm/tlb.h>
   #include <asm/cpuidle_haltpoll.h>
+#include <asm/e820/api.h>
   static int kvmapf = 1;
@@ -357,6 +358,33 @@ static void kvm_pv_guest_cpu_reboot(void *unused)
   	 */
   	if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))
   		wrmsrl(MSR_KVM_PV_EOI_EN, 0);
+	/*
+	 * Reset the host's page encryption bitmap related to kernel
+	 * specific page encryption status settings before we load a
+	 * new kernel by kexec. NOTE: We cannot reset the complete
+	 * page encryption bitmap here as we need to retain the
+	 * UEFI/OVMF firmware specific settings.
+	 */
+	if (kvm_para_has_feature(KVM_FEATURE_SEV_LIVE_MIGRATION) &&
+		(smp_processor_id() == 0)) {
+		unsigned long nr_pages;
+		int i;
+
+		for (i = 0; i < e820_table->nr_entries; i++) {
+			struct e820_entry *entry = &e820_table->entries[i];
+			unsigned long start_pfn, end_pfn;
+
+			if (entry->type != E820_TYPE_RAM)
+				continue;
+
+			start_pfn = entry->addr >> PAGE_SHIFT;
+			end_pfn = (entry->addr + entry->size) >> PAGE_SHIFT;
+			nr_pages = DIV_ROUND_UP(entry->size, PAGE_SIZE);
Quoted 8 lines
Quoted 8 lines

arch/x86/kernel/kvm.c static void kvm_pv_guest_cpu_reboot(void *unused) · line 390

Quoted 8 lines
arch/x86/kernel/kvm.c +5 -0
+
+			kvm_sev_hypercall3(KVM_HC_PAGE_ENC_STATUS,
+				entry->addr, nr_pages, 1);
+		}
+	}
   	kvm_pv_disable_apf();
   	kvm_disable_steal_time();
   }

Thanks for the explanation. It will certainly help one understand the context better if you add it to the commit message.

#106 lore raw
In reply to Ashish Kalra
Quoted 18 lines · Ashish Kalra

From: Ashish Kalra <[email protected]>

The series add support for AMD SEV guest live migration commands. To protect the confidentiality of an SEV protected guest memory while in transit we need to use the SEV commands defined in SEV API spec [1].

SEV guest VMs have the concept of private and shared memory. Private memory is encrypted with the guest-specific key, while shared memory may be encrypted with hypervisor key. The commands provided by the SEV FW are meant to be used for the private memory only. The patch series introduces a new hypercall. The guest OS can use this hypercall to notify the page encryption status. If the page is encrypted with guest specific-key then we use SEV command during the migration. If page is not encrypted then fallback to default.

The patch adds new ioctls KVM_{SET,GET}_PAGE_ENC_BITMAP. The ioctl can be used by the qemu to get the page encrypted bitmap. Qemu can consult this bitmap during the migration to know whether the page is encrypted.

Quoted 5 lines

[1] https://developer.amd.com/wp-content/resources/55766.PDF

Changes since v5:
- Fix build errors as

Which upstream tag should I use to apply this patch set? I tried the top of Linus's tree, and I get the following error when I apply this patch set.

$ git am PATCH-v6-01-14-KVM-SVM-Add-KVM_SEV-SEND_START-command.mbox
Applying: KVM: SVM: Add KVM_SEV SEND_START command
Applying: KVM: SVM: Add KVM_SEND_UPDATE_DATA command
Applying: KVM: SVM: Add KVM_SEV_SEND_FINISH command
Applying: KVM: SVM: Add KVM_SEV_RECEIVE_FINISH command
error: patch failed: Documentation/virt/kvm/amd-memory-encryption.rst:375
error: Documentation/virt/kvm/amd-memory-encryption.rst: patch does not apply
error: patch failed: arch/x86/kvm/svm.c:7632
error: arch/x86/kvm/svm.c: patch does not apply
Patch failed at 0004 KVM: SVM: Add KVM_SEV_RECEIVE_FINISH command

Thanks,

Venu

Quoted 85 lines

Changes since v4:
- Host support has been added to extend KVM capabilities/feature bits to include a new KVM_FEATURE_SEV_LIVE_MIGRATION, which the guest can query for host-side support for SEV live migration and a new custom MSR MSR_KVM_SEV_LIVE_MIG_EN is added for guest to enable the SEV live migration feature.
- Ensure that _bss_decrypted section is marked as decrypted in the page encryption bitmap.
- Fixing KVM_GET_PAGE_ENC_BITMAP ioctl to return the correct bitmap as per the number of pages being requested by the user. Ensure that we only copy bmap->num_pages bytes in the userspace buffer, if bmap->num_pages is not byte aligned we read the trailing bits from the userspace and copy those bits as is. This fixes guest page(s) corruption issues observed after migration completion.
- Add kexec support for SEV Live Migration to reset the host's page encryption bitmap related to kernel specific page encryption status settings before we load a new kernel by kexec. We cannot reset the complete page encryption bitmap here as we need to retain the UEFI/OVMF firmware specific settings.

Changes since v3:
- Rebasing to mainline and testing.
- Adding a new KVM_PAGE_ENC_BITMAP_RESET ioctl, which resets the page encryption bitmap on a guest reboot event.
- Adding a more reliable sanity check for GPA range being passed to the hypercall to ensure that guest MMIO ranges are also marked in the page encryption bitmap.

Changes since v2:
- reset the page encryption bitmap on vcpu reboot

Changes since v1:
- Add support to share the page encryption between the source and target machine.
- Fix review feedbacks from Tom Lendacky.
- Add check to limit the session blob length.
- Update KVM_GET_PAGE_ENC_BITMAP icotl to use the base_gfn instead of the memory slot when querying the bitmap.

Ashish Kalra (3):
KVM: x86: Introduce KVM_PAGE_ENC_BITMAP_RESET ioctl
KVM: x86: Introduce new KVM_FEATURE_SEV_LIVE_MIGRATION feature &
Custom MSR.
KVM: x86: Add kexec support for SEV Live Migration.

Brijesh Singh (11):
KVM: SVM: Add KVM_SEV SEND_START command
KVM: SVM: Add KVM_SEND_UPDATE_DATA command
KVM: SVM: Add KVM_SEV_SEND_FINISH command
KVM: SVM: Add support for KVM_SEV_RECEIVE_START command
KVM: SVM: Add KVM_SEV_RECEIVE_UPDATE_DATA command
KVM: SVM: Add KVM_SEV_RECEIVE_FINISH command
KVM: x86: Add AMD SEV specific Hypercall3
KVM: X86: Introduce KVM_HC_PAGE_ENC_STATUS hypercall
KVM: x86: Introduce KVM_GET_PAGE_ENC_BITMAP ioctl
mm: x86: Invoke hypercall when page encryption status is changed
KVM: x86: Introduce KVM_SET_PAGE_ENC_BITMAP ioctl

 .../virt/kvm/amd-memory-encryption.rst        | 120 +++
 Documentation/virt/kvm/api.rst                |  62 ++
 Documentation/virt/kvm/cpuid.rst              |   4 +
 Documentation/virt/kvm/hypercalls.rst         |  15 +
 Documentation/virt/kvm/msr.rst                |  10 +
 arch/x86/include/asm/kvm_host.h               |  10 +
 arch/x86/include/asm/kvm_para.h               |  12 +
 arch/x86/include/asm/paravirt.h               |  10 +
 arch/x86/include/asm/paravirt_types.h         |   2 +
 arch/x86/include/uapi/asm/kvm_para.h          |   5 +
 arch/x86/kernel/kvm.c                         |  32 +
 arch/x86/kernel/paravirt.c                    |   1 +
 arch/x86/kvm/cpuid.c                          |   3 +-
 arch/x86/kvm/svm.c                            | 699 +++++++++++++++++-
 arch/x86/kvm/vmx/vmx.c                        |   1 +
 arch/x86/kvm/x86.c                            |  43 ++
 arch/x86/mm/mem_encrypt.c                     |  69 +-
 arch/x86/mm/pat/set_memory.c                  |   7 +
 include/linux/psp-sev.h                       |   8 +-
 include/uapi/linux/kvm.h                      |  53 ++
 include/uapi/linux/kvm_para.h                 |   1 +
 21 files changed, 1157 insertions(+), 10 deletions(-)
Signature · 1 line
2.17.1
#107 lore raw

This is applied on top of Linux 5.6, as per commit below :

commit 7111951b8d4973bda27ff663f2cf18b663d15b48 (tag: v5.6, origin/master, origin/HEAD)

Author: Linus Torvalds <[email protected]>
Date:   Sun Mar 29 15:25:41 2020 -0700
    Linux 5.6
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

Thanks,
Ashish

Quoted 129 lines · Venu Busireddy
Quoted 18 lines · Ashish Kalra

From: Ashish Kalra <[email protected]>

The series add support for AMD SEV guest live migration commands. To protect the confidentiality of an SEV protected guest memory while in transit we need to use the SEV commands defined in SEV API spec [1].

SEV guest VMs have the concept of private and shared memory. Private memory is encrypted with the guest-specific key, while shared memory may be encrypted with hypervisor key. The commands provided by the SEV FW are meant to be used for the private memory only. The patch series introduces a new hypercall. The guest OS can use this hypercall to notify the page encryption status. If the page is encrypted with guest specific-key then we use SEV command during the migration. If page is not encrypted then fallback to default.

The patch adds new ioctls KVM_{SET,GET}_PAGE_ENC_BITMAP. The ioctl can be used by the qemu to get the page encrypted bitmap. Qemu can consult this bitmap during the migration to know whether the page is encrypted.

Quoted 5 lines

[1] https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fdeveloper.amd.com%2Fwp-content%2Fresources%2F55766.PDF&amp;data=02%7C01%7CAshish.Kalra%40amd.com%7Cb87828d7e1eb41fe401c08d7d4cf4937%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637211859771455979&amp;sdata=yN1OrvcuNb%2F8JAaLwlf2pIJtEvBRFOSvTKPYWz9ASUY%3D&amp;reserved=0

Changes since v5:
- Fix build errors as

Which upstream tag should I use to apply this patch set? I tried the top of Linus's tree, and I get the following error when I apply this patch set.

$ git am PATCH-v6-01-14-KVM-SVM-Add-KVM_SEV-SEND_START-command.mbox
Applying: KVM: SVM: Add KVM_SEV SEND_START command
Applying: KVM: SVM: Add KVM_SEND_UPDATE_DATA command
Applying: KVM: SVM: Add KVM_SEV_SEND_FINISH command
Applying: KVM: SVM: Add KVM_SEV_RECEIVE_FINISH command
error: patch failed: Documentation/virt/kvm/amd-memory-encryption.rst:375
error: Documentation/virt/kvm/amd-memory-encryption.rst: patch does not apply
error: patch failed: arch/x86/kvm/svm.c:7632
error: arch/x86/kvm/svm.c: patch does not apply
Patch failed at 0004 KVM: SVM: Add KVM_SEV_RECEIVE_FINISH command

Thanks,

Venu

Quoted 85 lines

Changes since v4:
- Host support has been added to extend KVM capabilities/feature bits to include a new KVM_FEATURE_SEV_LIVE_MIGRATION, which the guest can query for host-side support for SEV live migration and a new custom MSR MSR_KVM_SEV_LIVE_MIG_EN is added for guest to enable the SEV live migration feature.
- Ensure that _bss_decrypted section is marked as decrypted in the page encryption bitmap.
- Fixing KVM_GET_PAGE_ENC_BITMAP ioctl to return the correct bitmap as per the number of pages being requested by the user. Ensure that we only copy bmap->num_pages bytes in the userspace buffer, if bmap->num_pages is not byte aligned we read the trailing bits from the userspace and copy those bits as is. This fixes guest page(s) corruption issues observed after migration completion.
- Add kexec support for SEV Live Migration to reset the host's page encryption bitmap related to kernel specific page encryption status settings before we load a new kernel by kexec. We cannot reset the complete page encryption bitmap here as we need to retain the UEFI/OVMF firmware specific settings.

Changes since v3:
- Rebasing to mainline and testing.
- Adding a new KVM_PAGE_ENC_BITMAP_RESET ioctl, which resets the page encryption bitmap on a guest reboot event.
- Adding a more reliable sanity check for GPA range being passed to the hypercall to ensure that guest MMIO ranges are also marked in the page encryption bitmap.

Changes since v2:
- reset the page encryption bitmap on vcpu reboot

Changes since v1:
- Add support to share the page encryption between the source and target machine.
- Fix review feedbacks from Tom Lendacky.
- Add check to limit the session blob length.
- Update KVM_GET_PAGE_ENC_BITMAP icotl to use the base_gfn instead of the memory slot when querying the bitmap.

Ashish Kalra (3):
KVM: x86: Introduce KVM_PAGE_ENC_BITMAP_RESET ioctl
KVM: x86: Introduce new KVM_FEATURE_SEV_LIVE_MIGRATION feature &
Custom MSR.
KVM: x86: Add kexec support for SEV Live Migration.

Brijesh Singh (11):
KVM: SVM: Add KVM_SEV SEND_START command
KVM: SVM: Add KVM_SEND_UPDATE_DATA command
KVM: SVM: Add KVM_SEV_SEND_FINISH command
KVM: SVM: Add support for KVM_SEV_RECEIVE_START command
KVM: SVM: Add KVM_SEV_RECEIVE_UPDATE_DATA command
KVM: SVM: Add KVM_SEV_RECEIVE_FINISH command
KVM: x86: Add AMD SEV specific Hypercall3
KVM: X86: Introduce KVM_HC_PAGE_ENC_STATUS hypercall
KVM: x86: Introduce KVM_GET_PAGE_ENC_BITMAP ioctl
mm: x86: Invoke hypercall when page encryption status is changed
KVM: x86: Introduce KVM_SET_PAGE_ENC_BITMAP ioctl

 .../virt/kvm/amd-memory-encryption.rst        | 120 +++
 Documentation/virt/kvm/api.rst                |  62 ++
 Documentation/virt/kvm/cpuid.rst              |   4 +
 Documentation/virt/kvm/hypercalls.rst         |  15 +
 Documentation/virt/kvm/msr.rst                |  10 +
 arch/x86/include/asm/kvm_host.h               |  10 +
 arch/x86/include/asm/kvm_para.h               |  12 +
 arch/x86/include/asm/paravirt.h               |  10 +
 arch/x86/include/asm/paravirt_types.h         |   2 +
 arch/x86/include/uapi/asm/kvm_para.h          |   5 +
 arch/x86/kernel/kvm.c                         |  32 +
 arch/x86/kernel/paravirt.c                    |   1 +
 arch/x86/kvm/cpuid.c                          |   3 +-
 arch/x86/kvm/svm.c                            | 699 +++++++++++++++++-
 arch/x86/kvm/vmx/vmx.c                        |   1 +
 arch/x86/kvm/x86.c                            |  43 ++
 arch/x86/mm/mem_encrypt.c                     |  69 +-
 arch/x86/mm/pat/set_memory.c                  |   7 +
 include/linux/psp-sev.h                       |   8 +-
 include/uapi/linux/kvm.h                      |  53 ++
 include/uapi/linux/kvm_para.h                 |   1 +
 21 files changed, 1157 insertions(+), 10 deletions(-)
Signature · 1 line
2.17.1
#108 lore raw
Quoted 10 lines · Ashish Kalra

This is applied on top of Linux 5.6, as per commit below :

commit 7111951b8d4973bda27ff663f2cf18b663d15b48 (tag: v5.6, origin/master, origin/HEAD)

Author: Linus Torvalds <[email protected]>
Date:   Sun Mar 29 15:25:41 2020 -0700
    Linux 5.6
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

Not sure what I am missing here! This the current state of my sandbox:

$ git remote -v
origin  git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git (fetch)
origin  git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git (push)
$ git log --oneline
12acbbfef749 (HEAD -> master) KVM: SVM: Add KVM_SEV_SEND_FINISH command
e5f21e48bfff KVM: SVM: Add KVM_SEND_UPDATE_DATA command
6b2bcf682d08 KVM: SVM: Add KVM_SEV SEND_START command
7111951b8d49 (tag: v5.6, origin/master, origin/HEAD) Linux 5.6
$ git status
On branch master
Your branch is ahead of 'origin/master' by 3 commits.

As can be seen, I started with the commit (7111951b8d49) you mentioned. I could apply 3 of the patches, but 04/14 is failing.

Any suggestions?

Thanks,

Venu

Quoted 133 lines

Thanks,
Ashish

Quoted 129 lines · Venu Busireddy
Quoted 18 lines · Ashish Kalra

From: Ashish Kalra <[email protected]>

The series add support for AMD SEV guest live migration commands. To protect the confidentiality of an SEV protected guest memory while in transit we need to use the SEV commands defined in SEV API spec [1].

SEV guest VMs have the concept of private and shared memory. Private memory is encrypted with the guest-specific key, while shared memory may be encrypted with hypervisor key. The commands provided by the SEV FW are meant to be used for the private memory only. The patch series introduces a new hypercall. The guest OS can use this hypercall to notify the page encryption status. If the page is encrypted with guest specific-key then we use SEV command during the migration. If page is not encrypted then fallback to default.

The patch adds new ioctls KVM_{SET,GET}_PAGE_ENC_BITMAP. The ioctl can be used by the qemu to get the page encrypted bitmap. Qemu can consult this bitmap during the migration to know whether the page is encrypted.

Quoted 5 lines

[1] https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fdeveloper.amd.com%2Fwp-content%2Fresources%2F55766.PDF&amp;data=02%7C01%7CAshish.Kalra%40amd.com%7Cb87828d7e1eb41fe401c08d7d4cf4937%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637211859771455979&amp;sdata=yN1OrvcuNb%2F8JAaLwlf2pIJtEvBRFOSvTKPYWz9ASUY%3D&amp;reserved=0

Changes since v5:
- Fix build errors as

Which upstream tag should I use to apply this patch set? I tried the top of Linus's tree, and I get the following error when I apply this patch set.

$ git am PATCH-v6-01-14-KVM-SVM-Add-KVM_SEV-SEND_START-command.mbox
Applying: KVM: SVM: Add KVM_SEV SEND_START command
Applying: KVM: SVM: Add KVM_SEND_UPDATE_DATA command
Applying: KVM: SVM: Add KVM_SEV_SEND_FINISH command
Applying: KVM: SVM: Add KVM_SEV_RECEIVE_FINISH command
error: patch failed: Documentation/virt/kvm/amd-memory-encryption.rst:375
error: Documentation/virt/kvm/amd-memory-encryption.rst: patch does not apply
error: patch failed: arch/x86/kvm/svm.c:7632
error: arch/x86/kvm/svm.c: patch does not apply
Patch failed at 0004 KVM: SVM: Add KVM_SEV_RECEIVE_FINISH command

Thanks,

Venu

Quoted 85 lines

Changes since v4:
- Host support has been added to extend KVM capabilities/feature bits to include a new KVM_FEATURE_SEV_LIVE_MIGRATION, which the guest can query for host-side support for SEV live migration and a new custom MSR MSR_KVM_SEV_LIVE_MIG_EN is added for guest to enable the SEV live migration feature.
- Ensure that _bss_decrypted section is marked as decrypted in the page encryption bitmap.
- Fixing KVM_GET_PAGE_ENC_BITMAP ioctl to return the correct bitmap as per the number of pages being requested by the user. Ensure that we only copy bmap->num_pages bytes in the userspace buffer, if bmap->num_pages is not byte aligned we read the trailing bits from the userspace and copy those bits as is. This fixes guest page(s) corruption issues observed after migration completion.
- Add kexec support for SEV Live Migration to reset the host's page encryption bitmap related to kernel specific page encryption status settings before we load a new kernel by kexec. We cannot reset the complete page encryption bitmap here as we need to retain the UEFI/OVMF firmware specific settings.

Changes since v3:
- Rebasing to mainline and testing.
- Adding a new KVM_PAGE_ENC_BITMAP_RESET ioctl, which resets the page encryption bitmap on a guest reboot event.
- Adding a more reliable sanity check for GPA range being passed to the hypercall to ensure that guest MMIO ranges are also marked in the page encryption bitmap.

Changes since v2:
- reset the page encryption bitmap on vcpu reboot

Changes since v1:
- Add support to share the page encryption between the source and target machine.
- Fix review feedbacks from Tom Lendacky.
- Add check to limit the session blob length.
- Update KVM_GET_PAGE_ENC_BITMAP icotl to use the base_gfn instead of the memory slot when querying the bitmap.

Ashish Kalra (3):
KVM: x86: Introduce KVM_PAGE_ENC_BITMAP_RESET ioctl
KVM: x86: Introduce new KVM_FEATURE_SEV_LIVE_MIGRATION feature &
Custom MSR.
KVM: x86: Add kexec support for SEV Live Migration.

Brijesh Singh (11):
KVM: SVM: Add KVM_SEV SEND_START command
KVM: SVM: Add KVM_SEND_UPDATE_DATA command
KVM: SVM: Add KVM_SEV_SEND_FINISH command
KVM: SVM: Add support for KVM_SEV_RECEIVE_START command
KVM: SVM: Add KVM_SEV_RECEIVE_UPDATE_DATA command
KVM: SVM: Add KVM_SEV_RECEIVE_FINISH command
KVM: x86: Add AMD SEV specific Hypercall3
KVM: X86: Introduce KVM_HC_PAGE_ENC_STATUS hypercall
KVM: x86: Introduce KVM_GET_PAGE_ENC_BITMAP ioctl
mm: x86: Invoke hypercall when page encryption status is changed
KVM: x86: Introduce KVM_SET_PAGE_ENC_BITMAP ioctl

 .../virt/kvm/amd-memory-encryption.rst        | 120 +++
 Documentation/virt/kvm/api.rst                |  62 ++
 Documentation/virt/kvm/cpuid.rst              |   4 +
 Documentation/virt/kvm/hypercalls.rst         |  15 +
 Documentation/virt/kvm/msr.rst                |  10 +
 arch/x86/include/asm/kvm_host.h               |  10 +
 arch/x86/include/asm/kvm_para.h               |  12 +
 arch/x86/include/asm/paravirt.h               |  10 +
 arch/x86/include/asm/paravirt_types.h         |   2 +
 arch/x86/include/uapi/asm/kvm_para.h          |   5 +
 arch/x86/kernel/kvm.c                         |  32 +
 arch/x86/kernel/paravirt.c                    |   1 +
 arch/x86/kvm/cpuid.c                          |   3 +-
 arch/x86/kvm/svm.c                            | 699 +++++++++++++++++-
 arch/x86/kvm/vmx/vmx.c                        |   1 +
 arch/x86/kvm/x86.c                            |  43 ++
 arch/x86/mm/mem_encrypt.c                     |  69 +-
 arch/x86/mm/pat/set_memory.c                  |   7 +
 include/linux/psp-sev.h                       |   8 +-
 include/uapi/linux/kvm.h                      |  53 ++
 include/uapi/linux/kvm_para.h                 |   1 +
 21 files changed, 1157 insertions(+), 10 deletions(-)
Signature · 1 line
2.17.1
#109 lore raw

I just did a fresh install of Linus's tree and i can install these patches cleanly on top of the tree.

Thanks,
Ashish

Quoted 170 lines · Venu Busireddy
Quoted 10 lines · Ashish Kalra

This is applied on top of Linux 5.6, as per commit below :

commit 7111951b8d4973bda27ff663f2cf18b663d15b48 (tag: v5.6, origin/master, origin/HEAD)

Author: Linus Torvalds <[email protected]>
Date:   Sun Mar 29 15:25:41 2020 -0700
    Linux 5.6
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

Not sure what I am missing here! This the current state of my sandbox:

$ git remote -v
origin  git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git (fetch)
origin  git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git (push)
$ git log --oneline
12acbbfef749 (HEAD -> master) KVM: SVM: Add KVM_SEV_SEND_FINISH command
e5f21e48bfff KVM: SVM: Add KVM_SEND_UPDATE_DATA command
6b2bcf682d08 KVM: SVM: Add KVM_SEV SEND_START command
7111951b8d49 (tag: v5.6, origin/master, origin/HEAD) Linux 5.6
$ git status
On branch master
Your branch is ahead of 'origin/master' by 3 commits.

As can be seen, I started with the commit (7111951b8d49) you mentioned. I could apply 3 of the patches, but 04/14 is failing.

Any suggestions?

Thanks,

Venu

Quoted 133 lines

Thanks,
Ashish

Quoted 129 lines · Venu Busireddy
Quoted 18 lines · Ashish Kalra

From: Ashish Kalra <[email protected]>

The series add support for AMD SEV guest live migration commands. To protect the confidentiality of an SEV protected guest memory while in transit we need to use the SEV commands defined in SEV API spec [1].

SEV guest VMs have the concept of private and shared memory. Private memory is encrypted with the guest-specific key, while shared memory may be encrypted with hypervisor key. The commands provided by the SEV FW are meant to be used for the private memory only. The patch series introduces a new hypercall. The guest OS can use this hypercall to notify the page encryption status. If the page is encrypted with guest specific-key then we use SEV command during the migration. If page is not encrypted then fallback to default.

The patch adds new ioctls KVM_{SET,GET}_PAGE_ENC_BITMAP. The ioctl can be used by the qemu to get the page encrypted bitmap. Qemu can consult this bitmap during the migration to know whether the page is encrypted.

Quoted 5 lines

[1] https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fdeveloper.amd.com%2Fwp-content%2Fresources%2F55766.PDF&amp;data=02%7C01%7Cashish.kalra%40amd.com%7C2546be8861e3409b9d3408d7d4de683c%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637211924007781552&amp;sdata=lYAGaXWFveawb7Fre8Qo7iGyKcLREiodSgQswMBirHc%3D&amp;reserved=0

Changes since v5:
- Fix build errors as

Which upstream tag should I use to apply this patch set? I tried the top of Linus's tree, and I get the following error when I apply this patch set.

$ git am PATCH-v6-01-14-KVM-SVM-Add-KVM_SEV-SEND_START-command.mbox
Applying: KVM: SVM: Add KVM_SEV SEND_START command
Applying: KVM: SVM: Add KVM_SEND_UPDATE_DATA command
Applying: KVM: SVM: Add KVM_SEV_SEND_FINISH command
Applying: KVM: SVM: Add KVM_SEV_RECEIVE_FINISH command
error: patch failed: Documentation/virt/kvm/amd-memory-encryption.rst:375
error: Documentation/virt/kvm/amd-memory-encryption.rst: patch does not apply
error: patch failed: arch/x86/kvm/svm.c:7632
error: arch/x86/kvm/svm.c: patch does not apply
Patch failed at 0004 KVM: SVM: Add KVM_SEV_RECEIVE_FINISH command

Thanks,

Venu

Quoted 85 lines

Changes since v4:
- Host support has been added to extend KVM capabilities/feature bits to include a new KVM_FEATURE_SEV_LIVE_MIGRATION, which the guest can query for host-side support for SEV live migration and a new custom MSR MSR_KVM_SEV_LIVE_MIG_EN is added for guest to enable the SEV live migration feature.
- Ensure that _bss_decrypted section is marked as decrypted in the page encryption bitmap.
- Fixing KVM_GET_PAGE_ENC_BITMAP ioctl to return the correct bitmap as per the number of pages being requested by the user. Ensure that we only copy bmap->num_pages bytes in the userspace buffer, if bmap->num_pages is not byte aligned we read the trailing bits from the userspace and copy those bits as is. This fixes guest page(s) corruption issues observed after migration completion.
- Add kexec support for SEV Live Migration to reset the host's page encryption bitmap related to kernel specific page encryption status settings before we load a new kernel by kexec. We cannot reset the complete page encryption bitmap here as we need to retain the UEFI/OVMF firmware specific settings.

Changes since v3:
- Rebasing to mainline and testing.
- Adding a new KVM_PAGE_ENC_BITMAP_RESET ioctl, which resets the page encryption bitmap on a guest reboot event.
- Adding a more reliable sanity check for GPA range being passed to the hypercall to ensure that guest MMIO ranges are also marked in the page encryption bitmap.

Changes since v2:
- reset the page encryption bitmap on vcpu reboot

Changes since v1:
- Add support to share the page encryption between the source and target machine.
- Fix review feedbacks from Tom Lendacky.
- Add check to limit the session blob length.
- Update KVM_GET_PAGE_ENC_BITMAP icotl to use the base_gfn instead of the memory slot when querying the bitmap.

Ashish Kalra (3):
KVM: x86: Introduce KVM_PAGE_ENC_BITMAP_RESET ioctl
KVM: x86: Introduce new KVM_FEATURE_SEV_LIVE_MIGRATION feature &
Custom MSR.
KVM: x86: Add kexec support for SEV Live Migration.

Brijesh Singh (11):
KVM: SVM: Add KVM_SEV SEND_START command
KVM: SVM: Add KVM_SEND_UPDATE_DATA command
KVM: SVM: Add KVM_SEV_SEND_FINISH command
KVM: SVM: Add support for KVM_SEV_RECEIVE_START command
KVM: SVM: Add KVM_SEV_RECEIVE_UPDATE_DATA command
KVM: SVM: Add KVM_SEV_RECEIVE_FINISH command
KVM: x86: Add AMD SEV specific Hypercall3
KVM: X86: Introduce KVM_HC_PAGE_ENC_STATUS hypercall
KVM: x86: Introduce KVM_GET_PAGE_ENC_BITMAP ioctl
mm: x86: Invoke hypercall when page encryption status is changed
KVM: x86: Introduce KVM_SET_PAGE_ENC_BITMAP ioctl

 .../virt/kvm/amd-memory-encryption.rst        | 120 +++
 Documentation/virt/kvm/api.rst                |  62 ++
 Documentation/virt/kvm/cpuid.rst              |   4 +
 Documentation/virt/kvm/hypercalls.rst         |  15 +
 Documentation/virt/kvm/msr.rst                |  10 +
 arch/x86/include/asm/kvm_host.h               |  10 +
 arch/x86/include/asm/kvm_para.h               |  12 +
 arch/x86/include/asm/paravirt.h               |  10 +
 arch/x86/include/asm/paravirt_types.h         |   2 +
 arch/x86/include/uapi/asm/kvm_para.h          |   5 +
 arch/x86/kernel/kvm.c                         |  32 +
 arch/x86/kernel/paravirt.c                    |   1 +
 arch/x86/kvm/cpuid.c                          |   3 +-
 arch/x86/kvm/svm.c                            | 699 +++++++++++++++++-
 arch/x86/kvm/vmx/vmx.c                        |   1 +
 arch/x86/kvm/x86.c                            |  43 ++
 arch/x86/mm/mem_encrypt.c                     |  69 +-
 arch/x86/mm/pat/set_memory.c                  |   7 +
 include/linux/psp-sev.h                       |   8 +-
 include/uapi/linux/kvm.h                      |  53 ++
 include/uapi/linux/kvm_para.h                 |   1 +
 21 files changed, 1157 insertions(+), 10 deletions(-)
Signature · 1 line
2.17.1
#110 lore raw
Quoted 2 lines · Ashish Kalra

I just did a fresh install of Linus's tree and i can install these patches cleanly on top of the tree.

Figured out what the problem was. Though the patches are listed sorted at
https://lore.kernel.org/kvm/[email protected]/,
the patches inside
https://lore.kernel.org/kvm/[email protected]/t.mbox.gz
are not sorted (sequential). Hence, they were being applied out of order by 'git am ....mbox', which caused the error. I had to edit the mbox file by hand, or use a tool such as b4 (suggested by a colleague), to sort the patches in the mbox file. Once that is done, I was able to apply the entire patch set to my code base.

Thanks,

Venu

Quoted 175 lines

Thanks,
Ashish

Quoted 170 lines · Venu Busireddy
Quoted 10 lines · Ashish Kalra

This is applied on top of Linux 5.6, as per commit below :

commit 7111951b8d4973bda27ff663f2cf18b663d15b48 (tag: v5.6, origin/master, origin/HEAD)

Author: Linus Torvalds <[email protected]>
Date:   Sun Mar 29 15:25:41 2020 -0700
    Linux 5.6
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

Not sure what I am missing here! This the current state of my sandbox:

$ git remote -v
origin  git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git (fetch)
origin  git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git (push)
$ git log --oneline
12acbbfef749 (HEAD -> master) KVM: SVM: Add KVM_SEV_SEND_FINISH command
e5f21e48bfff KVM: SVM: Add KVM_SEND_UPDATE_DATA command
6b2bcf682d08 KVM: SVM: Add KVM_SEV SEND_START command
7111951b8d49 (tag: v5.6, origin/master, origin/HEAD) Linux 5.6
$ git status
On branch master
Your branch is ahead of 'origin/master' by 3 commits.

As can be seen, I started with the commit (7111951b8d49) you mentioned. I could apply 3 of the patches, but 04/14 is failing.

Any suggestions?

Thanks,

Venu

Quoted 133 lines

Thanks,
Ashish

Quoted 129 lines · Venu Busireddy
Quoted 18 lines · Ashish Kalra

From: Ashish Kalra <[email protected]>

The series add support for AMD SEV guest live migration commands. To protect the confidentiality of an SEV protected guest memory while in transit we need to use the SEV commands defined in SEV API spec [1].

SEV guest VMs have the concept of private and shared memory. Private memory is encrypted with the guest-specific key, while shared memory may be encrypted with hypervisor key. The commands provided by the SEV FW are meant to be used for the private memory only. The patch series introduces a new hypercall. The guest OS can use this hypercall to notify the page encryption status. If the page is encrypted with guest specific-key then we use SEV command during the migration. If page is not encrypted then fallback to default.

The patch adds new ioctls KVM_{SET,GET}_PAGE_ENC_BITMAP. The ioctl can be used by the qemu to get the page encrypted bitmap. Qemu can consult this bitmap during the migration to know whether the page is encrypted.

Quoted 5 lines

[1] https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fdeveloper.amd.com%2Fwp-content%2Fresources%2F55766.PDF&amp;data=02%7C01%7Cashish.kalra%40amd.com%7C2546be8861e3409b9d3408d7d4de683c%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637211924007781552&amp;sdata=lYAGaXWFveawb7Fre8Qo7iGyKcLREiodSgQswMBirHc%3D&amp;reserved=0

Changes since v5:
- Fix build errors as

Which upstream tag should I use to apply this patch set? I tried the top of Linus's tree, and I get the following error when I apply this patch set.

$ git am PATCH-v6-01-14-KVM-SVM-Add-KVM_SEV-SEND_START-command.mbox
Applying: KVM: SVM: Add KVM_SEV SEND_START command
Applying: KVM: SVM: Add KVM_SEND_UPDATE_DATA command
Applying: KVM: SVM: Add KVM_SEV_SEND_FINISH command
Applying: KVM: SVM: Add KVM_SEV_RECEIVE_FINISH command
error: patch failed: Documentation/virt/kvm/amd-memory-encryption.rst:375
error: Documentation/virt/kvm/amd-memory-encryption.rst: patch does not apply
error: patch failed: arch/x86/kvm/svm.c:7632
error: arch/x86/kvm/svm.c: patch does not apply
Patch failed at 0004 KVM: SVM: Add KVM_SEV_RECEIVE_FINISH command

Thanks,

Venu

Quoted 85 lines

Changes since v4:
- Host support has been added to extend KVM capabilities/feature bits to include a new KVM_FEATURE_SEV_LIVE_MIGRATION, which the guest can query for host-side support for SEV live migration and a new custom MSR MSR_KVM_SEV_LIVE_MIG_EN is added for guest to enable the SEV live migration feature.
- Ensure that _bss_decrypted section is marked as decrypted in the page encryption bitmap.
- Fixing KVM_GET_PAGE_ENC_BITMAP ioctl to return the correct bitmap as per the number of pages being requested by the user. Ensure that we only copy bmap->num_pages bytes in the userspace buffer, if bmap->num_pages is not byte aligned we read the trailing bits from the userspace and copy those bits as is. This fixes guest page(s) corruption issues observed after migration completion.
- Add kexec support for SEV Live Migration to reset the host's page encryption bitmap related to kernel specific page encryption status settings before we load a new kernel by kexec. We cannot reset the complete page encryption bitmap here as we need to retain the UEFI/OVMF firmware specific settings.

Changes since v3:
- Rebasing to mainline and testing.
- Adding a new KVM_PAGE_ENC_BITMAP_RESET ioctl, which resets the page encryption bitmap on a guest reboot event.
- Adding a more reliable sanity check for GPA range being passed to the hypercall to ensure that guest MMIO ranges are also marked in the page encryption bitmap.

Changes since v2:
- reset the page encryption bitmap on vcpu reboot

Changes since v1:
- Add support to share the page encryption between the source and target machine.
- Fix review feedbacks from Tom Lendacky.
- Add check to limit the session blob length.
- Update KVM_GET_PAGE_ENC_BITMAP icotl to use the base_gfn instead of the memory slot when querying the bitmap.

Ashish Kalra (3):
KVM: x86: Introduce KVM_PAGE_ENC_BITMAP_RESET ioctl
KVM: x86: Introduce new KVM_FEATURE_SEV_LIVE_MIGRATION feature &
Custom MSR.
KVM: x86: Add kexec support for SEV Live Migration.

Brijesh Singh (11):
KVM: SVM: Add KVM_SEV SEND_START command
KVM: SVM: Add KVM_SEND_UPDATE_DATA command
KVM: SVM: Add KVM_SEV_SEND_FINISH command
KVM: SVM: Add support for KVM_SEV_RECEIVE_START command
KVM: SVM: Add KVM_SEV_RECEIVE_UPDATE_DATA command
KVM: SVM: Add KVM_SEV_RECEIVE_FINISH command
KVM: x86: Add AMD SEV specific Hypercall3
KVM: X86: Introduce KVM_HC_PAGE_ENC_STATUS hypercall
KVM: x86: Introduce KVM_GET_PAGE_ENC_BITMAP ioctl
mm: x86: Invoke hypercall when page encryption status is changed
KVM: x86: Introduce KVM_SET_PAGE_ENC_BITMAP ioctl

 .../virt/kvm/amd-memory-encryption.rst        | 120 +++
 Documentation/virt/kvm/api.rst                |  62 ++
 Documentation/virt/kvm/cpuid.rst              |   4 +
 Documentation/virt/kvm/hypercalls.rst         |  15 +
 Documentation/virt/kvm/msr.rst                |  10 +
 arch/x86/include/asm/kvm_host.h               |  10 +
 arch/x86/include/asm/kvm_para.h               |  12 +
 arch/x86/include/asm/paravirt.h               |  10 +
 arch/x86/include/asm/paravirt_types.h         |   2 +
 arch/x86/include/uapi/asm/kvm_para.h          |   5 +
 arch/x86/kernel/kvm.c                         |  32 +
 arch/x86/kernel/paravirt.c                    |   1 +
 arch/x86/kvm/cpuid.c                          |   3 +-
 arch/x86/kvm/svm.c                            | 699 +++++++++++++++++-
 arch/x86/kvm/vmx/vmx.c                        |   1 +
 arch/x86/kvm/x86.c                            |  43 ++
 arch/x86/mm/mem_encrypt.c                     |  69 +-
 arch/x86/mm/pat/set_memory.c                  |   7 +
 include/linux/psp-sev.h                       |   8 +-
 include/uapi/linux/kvm.h                      |  53 ++
 include/uapi/linux/kvm_para.h                 |   1 +
 21 files changed, 1157 insertions(+), 10 deletions(-)
Signature · 1 line
2.17.1