KVM: s390: Add support for protected VMs

Introduction to Protected VMs.

Signed-off-by Janosch Frank <[email protected]>

---
 Documentation/virtual/kvm/s390-pv.txt | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 Documentation/virtual/kvm/s390-pv.txt

Documentation/virtual/kvm/s390-pv.txt new file +9 -0

new file mode 100644
index 000000000000..86ed95f36759
@@ -0,0 +1,23 @@
+Ultravisor and Protected VMs
+===========================
+
+Summary:
+
+Protected VMs (PVM) are KVM VMs, where KVM can't access the VM's state
+like guest memory and guest registers anymore. Instead the PVMs are
+mostly managed by a new entity called Ultravisor (UV), which provides
+an API, so KVM and the PVM can request management actions.

Documentation/virtual/kvm/s390-pv.txt new file +4 -0

+normally needed to be able to run a VM, some changes have been made in
+SIE behavior and fields have different meaning for a PVM. SIE exits
+are minimized as much as possible to improve speed and reduce exposed
+guest state.

Documentation/virtual/kvm/s390-pv.txt

Quoted 13 lines · On Thu, 24 Oct 2019 07:40:23 -0400 Janosch Frank <[email protected]> wrote:

Introduction to Protected VMs.

Signed-off-by Janosch Frank <[email protected]>

---
 Documentation/virtual/kvm/s390-pv.txt | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 Documentation/virtual/kvm/s390-pv.txt

Documentation/virtual/kvm/s390-pv.txt new file +0 -0

new file mode 100644
index 000000000000..86ed95f36759

This should be under /virt/, I think. Also, maybe start out with RST already for new files?

Documentation/virtual/kvm/s390-pv.txt line 7

Quoted 8 lines

Documentation/virtual/kvm/s390-pv.txt new file +7 -0

@@ -0,0 +1,23 @@
+Ultravisor and Protected VMs
+===========================
+
+Summary:
+
+Protected VMs (PVM) are KVM VMs, where KVM can't access the VM's state
+like guest memory and guest registers anymore. Instead the PVMs are

s/Instead/Instead,/

Documentation/virtual/kvm/s390-pv.txt line 9

Quoted 3 lines

Documentation/virtual/kvm/s390-pv.txt new file +2 -0

+mostly managed by a new entity called Ultravisor (UV), which provides
+an API, so KVM and the PVM can request management actions.

Hm...

"The UV provides an API (both for guests and hypervisors), where PVMs and KVM can request management actions." ?

Documentation/virtual/kvm/s390-pv.txt line 11

Quoted 3 lines

Documentation/virtual/kvm/s390-pv.txt new file +2 -0

+
+Each guest starts in the non-protected mode and then transitions into

"and then may make a request to transition into protected mode" ?

Documentation/virtual/kvm/s390-pv.txt line 19

Quoted 9 lines

Documentation/virtual/kvm/s390-pv.txt new file +8 -0

+protected mode. On transition KVM registers the guest and its VCPUs
+with the Ultravisor and prepares everything for running it.
+
+The Ultravisor will secure and decrypt the guest's boot memory
+(i.e. kernel/initrd). It will safeguard state changes like VCPU
+starts/stops and injected interrupts while the guest is running.
+
+As access to the guest's state, like the SIE state description is

"such as the SIE state description," ?

Quoted 4 lines

Documentation/virtual/kvm/s390-pv.txt new file +4 -0

+normally needed to be able to run a VM, some changes have been made in
+SIE behavior and fields have different meaning for a PVM. SIE exits
+are minimized as much as possible to improve speed and reduce exposed
+guest state.

From: Vasily Gorbik <[email protected]>

Introduce KVM_S390_PROTECTED_VIRTUALIZATION_HOST kbuild option for protected virtual machines hosting support code.

Add "prot_virt" command line option which controls if the kernel protected VMs support is enabled at runtime.

Extend ultravisor info definitions and expose it via uv_info struct filled in during startup.

Signed-off-by Vasily Gorbik <[email protected]>

---
  .../admin-guide/kernel-parameters.txt         |  5 ++
  arch/s390/boot/Makefile                       |  2 +-
  arch/s390/boot/uv.c                           | 20 +++++++-
  arch/s390/include/asm/uv.h                    | 46 ++++++++++++++++--
  arch/s390/kernel/Makefile                     |  1 +
  arch/s390/kernel/setup.c                      |  4 --
  arch/s390/kernel/uv.c                         | 48 +++++++++++++++++++
  arch/s390/kvm/Kconfig                         |  9 ++++
  8 files changed, 126 insertions(+), 9 deletions(-)

create mode 100644 arch/s390/kernel/uv.c

Documentation/admin-guide/kernel-parameters.txt +0 -0

index c7ac2f3ac99f..aa22e36b3105 100644

Quoted 32 lines · Janosch Frank

From: Vasily Gorbik <[email protected]>

Introduce KVM_S390_PROTECTED_VIRTUALIZATION_HOST kbuild option for protected virtual machines hosting support code.

Add "prot_virt" command line option which controls if the kernel protected VMs support is enabled at runtime.

Extend ultravisor info definitions and expose it via uv_info struct filled in during startup.

Signed-off-by Vasily Gorbik <[email protected]>

---
   .../admin-guide/kernel-parameters.txt         |  5 ++
   arch/s390/boot/Makefile                       |  2 +-
   arch/s390/boot/uv.c                           | 20 +++++++-
   arch/s390/include/asm/uv.h                    | 46 ++++++++++++++++--
   arch/s390/kernel/Makefile                     |  1 +
   arch/s390/kernel/setup.c                      |  4 --
   arch/s390/kernel/uv.c                         | 48 +++++++++++++++++++
   arch/s390/kvm/Kconfig                         |  9 ++++
   8 files changed, 126 insertions(+), 9 deletions(-)

create mode 100644 arch/s390/kernel/uv.c

Documentation/admin-guide/kernel-parameters.txt +0 -0

index c7ac2f3ac99f..aa22e36b3105 100644
@@ -3693,6 +3693,11 @@
   			before loading.
   			See Documentation/admin-guide/blockdev/ramdisk.rst.
 

Documentation/admin-guide/kernel-parameters.txt line 3699

Quoted 4 lines

Documentation/admin-guide/kernel-parameters.txt +4 -0

+	prot_virt=	[S390] enable hosting protected virtual machines
+			isolated from the hypervisor (if hardware supports
+			that).
+			Format: <bool>

Isn't that a virt driver detail that should come in via KVM module parameters? I don't see quite yet why this has to be a kernel parameter (that can be changed at runtime).

Quoted 25 lines · David Hildenbrand

Quoted 24 lines · Janosch Frank

From: Vasily Gorbik <[email protected]>

Introduce KVM_S390_PROTECTED_VIRTUALIZATION_HOST kbuild option for protected virtual machines hosting support code.

Add "prot_virt" command line option which controls if the kernel protected VMs support is enabled at runtime.

Extend ultravisor info definitions and expose it via uv_info struct filled in during startup.

Signed-off-by Vasily Gorbik <[email protected]>

---
   .../admin-guide/kernel-parameters.txt         |  5 ++ arch/s390/boot/Makefile                       |  2 +- arch/s390/boot/uv.c                           | 20 +++++++- arch/s390/include/asm/uv.h                    | 46 ++++++++++++++++-- arch/s390/kernel/Makefile                     |  1 + arch/s390/kernel/setup.c                      |  4 -- arch/s390/kernel/uv.c                         | 48 +++++++++++++++++++ arch/s390/kvm/Kconfig                         |  9 ++++

   8 files changed, 126 insertions(+), 9 deletions(-)

   create mode 100644 arch/s390/kernel/uv.c

Quoted 16 lines

Quoted 11 lines

Documentation/admin-guide/kernel-parameters.txt +0 -0

index c7ac2f3ac99f..aa22e36b3105 100644
@@ -3693,6 +3693,11 @@

               before loading.
               See Documentation/admin-guide/blockdev/ramdisk.rst. +    prot_virt=    [S390] enable hosting protected virtual machines +            isolated from the hypervisor (if hardware supports +            that).
+            Format: <bool>

Isn't that a virt driver detail that should come in via KVM module parameters? I don't see quite yet why this has to be a kernel parameter (that can be changed at runtime).

I was confused by "runtime" in "which controls if the kernel protected VMs support is enabled at runtime"

So this can't be changed at runtime. Can you clarify why kvm can't initialize that when loaded and why we need a kernel parameter?

Quoted 43 lines · David Hildenbrand

Quoted 41 lines · David Hildenbrand

Quoted 29 lines · Janosch Frank

From: Vasily Gorbik <[email protected]>

Introduce KVM_S390_PROTECTED_VIRTUALIZATION_HOST kbuild option for protected virtual machines hosting support code.

Add "prot_virt" command line option which controls if the kernel protected VMs support is enabled at runtime.

Extend ultravisor info definitions and expose it via uv_info struct filled in during startup.

Signed-off-by Vasily Gorbik <[email protected]>

---
    .../admin-guide/kernel-parameters.txt         |  5 ++
    arch/s390/boot/Makefile                       |  2 +-
    arch/s390/boot/uv.c                           | 20 +++++++-
    arch/s390/include/asm/uv.h                    | 46 ++++++++++++++++--
    arch/s390/kernel/Makefile                     |  1 +
    arch/s390/kernel/setup.c                      |  4 --
    arch/s390/kernel/uv.c                         | 48 +++++++++++++++++++
    arch/s390/kvm/Kconfig                         |  9 ++++
    8 files changed, 126 insertions(+), 9 deletions(-)

   create mode 100644 arch/s390/kernel/uv.c

Documentation/admin-guide/kernel-parameters.txt +0 -0

index c7ac2f3ac99f..aa22e36b3105 100644

Documentation/admin-guide/kernel-parameters.txt line 3698

Quoted 6 lines

Documentation/admin-guide/kernel-parameters.txt +3 -0

@@ -3693,6 +3693,11 @@
                before loading.
                See Documentation/admin-guide/blockdev/ramdisk.rst.
    +    prot_virt=    [S390] enable hosting protected virtual machines
+            isolated from the hypervisor (if hardware supports
+            that).
+            Format: <bool>

Isn't that a virt driver detail that should come in via KVM module parameters? I don't see quite yet why this has to be a kernel parameter (that can be changed at runtime).

Quoted 3 lines

I was confused by "runtime" in "which controls if the kernel protected VMs support is enabled at runtime"

So this can't be changed at runtime. Can you clarify why kvm can't initialize that when loaded and why we need a kernel parameter?

We have to do the opt-in very early for several reasons:
- we have to donate a potentially largish contiguous (in real) range of memory to the ultravisor

- The opt-in will also disable some features in the host that could affect guest integrity (e.g. time sync via STP to avoid the host messing with the guest time stepping). Linux is not happy when you remove features at a later point in time

Quoted 55 lines · Christian Borntraeger

Quoted 51 lines · David Hildenbrand

Quoted 35 lines · David Hildenbrand

Quoted 34 lines · Janosch Frank

From: Vasily Gorbik <[email protected]>

Introduce KVM_S390_PROTECTED_VIRTUALIZATION_HOST kbuild option for protected virtual machines hosting support code.

Add "prot_virt" command line option which controls if the kernel protected VMs support is enabled at runtime.

Extend ultravisor info definitions and expose it via uv_info struct filled in during startup.

Signed-off-by Vasily Gorbik <[email protected]>

---
    .../admin-guide/kernel-parameters.txt         |  5 ++
    arch/s390/boot/Makefile                       |  2 +-
    arch/s390/boot/uv.c                           | 20 +++++++-
    arch/s390/include/asm/uv.h                    | 46
++++++++++++++++-- arch/s390/kernel/Makefile
|  1 + arch/s390/kernel/setup.c                      |  4 --
    arch/s390/kernel/uv.c                         | 48

+++++++++++++++++++
arch/s390/kvm/Kconfig                         |  9 ++++ 8 files changed, 126 insertions(+), 9 deletions(-) create mode 100644 arch/s390/kernel/uv.c

(quoted diff) +0 -0

b/Documentation/admin-guide/kernel-parameters.txt index c7ac2f3ac99f..aa22e36b3105 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -3693,6 +3693,11 @@ before loading.
               See
Documentation/admin-guide/blockdev/ramdisk.rst.
   +    prot_virt=    [S390] enable hosting protected virtual

Quoted 9 lines

Quoted 4 lines

machines
+            isolated from the hypervisor (if hardware supports +            that).
+            Format: <bool>

Isn't that a virt driver detail that should come in via KVM module parameters? I don't see quite yet why this has to be a kernel parameter (that can be changed at runtime).

I was confused by "runtime" in "which controls if the kernel protected VMs support is enabled at runtime"

So this can't be changed at runtime. Can you clarify why kvm can't initialize that when loaded and why we need a kernel parameter?

Quoted 3 lines

We have to do the opt-in very early for several reasons:
- we have to donate a potentially largish contiguous (in real) range of memory to the ultravisor

If you'd be using CMA (or alloc_contig_pages() with less guarantees) you could be making good use of the memory until you actually start an encrypted guest.

I can see that using the memblock allocator early is easier. But you waste "largish ... range of memory" even if you never run VMs.

Maybe something to work on in the future.

Quoted 5 lines

- The opt-in will also disable some features in the host that could affect guest integrity (e.g. time sync via STP to avoid the host messing with the guest time stepping). Linux is not happy when you remove features at a later point in time

At least disabling STP shouldn't be a real issue if I'm not wrong (maybe I am). But there seem to be more features.

(when I saw "prot_virt" it felt like somebody is using a big hammer for small nails (e.g., compared to "stp=off").)

Can you guys add these details to the patch description?

Quoted 60 lines · On Thu, 24 Oct 2019 17:52:31 +0200 David Hildenbrand <[email protected]> wrote:

Quoted 58 lines · Christian Borntraeger

Quoted 46 lines · David Hildenbrand

Quoted 35 lines · David Hildenbrand

Quoted 34 lines · Janosch Frank

From: Vasily Gorbik <[email protected]>

Introduce KVM_S390_PROTECTED_VIRTUALIZATION_HOST kbuild option for protected virtual machines hosting support code.

Add "prot_virt" command line option which controls if the kernel protected VMs support is enabled at runtime.

Extend ultravisor info definitions and expose it via uv_info struct filled in during startup.

Signed-off-by Vasily Gorbik <[email protected]>

---
     .../admin-guide/kernel-parameters.txt         |  5 ++
     arch/s390/boot/Makefile                       |  2 +-
     arch/s390/boot/uv.c                           | 20 +++++++-
     arch/s390/include/asm/uv.h                    | 46
++++++++++++++++-- arch/s390/kernel/Makefile
|  1 + arch/s390/kernel/setup.c                      |  4 --
     arch/s390/kernel/uv.c                         | 48

+++++++++++++++++++
arch/s390/kvm/Kconfig                         |  9 ++++ 8 files changed, 126 insertions(+), 9 deletions(-) create mode 100644 arch/s390/kernel/uv.c

(quoted diff) +0 -0

b/Documentation/admin-guide/kernel-parameters.txt index c7ac2f3ac99f..aa22e36b3105 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -3693,6 +3693,11 @@ before loading.
               See
Documentation/admin-guide/blockdev/ramdisk.rst.
   +    prot_virt=    [S390] enable hosting protected virtual

Quoted 9 lines

Quoted 4 lines

machines
+            isolated from the hypervisor (if hardware supports +            that).
+            Format: <bool>

Isn't that a virt driver detail that should come in via KVM module parameters? I don't see quite yet why this has to be a kernel parameter (that can be changed at runtime).

Quoted 5 lines

I was confused by "runtime" in "which controls if the kernel protected VMs support is enabled at runtime"

So this can't be changed at runtime. Can you clarify why kvm can't initialize that when loaded and why we need a kernel parameter?

We have to do the opt-in very early for several reasons:
- we have to donate a potentially largish contiguous (in real) range of memory to the ultravisor

Quoted 3 lines

If you'd be using CMA (or alloc_contig_pages() with less guarantees) you could be making good use of the memory until you actually start an encrypted guest.

no, the memory needs to be allocated before any other interaction with the ultravisor is attempted, and the size depends on the size of the

_host_ memory. it can be a very substantial amount of memory, and thus it's very likely to fail unless it's done very early at boot time.

Quoted 10 lines

Quoted 6 lines

- The opt-in will also disable some features in the host that could affect guest integrity (e.g. time sync via STP to avoid the host messing with the guest time stepping). Linux is not happy when you remove features at a later point in time

At least disabling STP shouldn't be a real issue if I'm not wrong (maybe I am). But there seem to be more features.

Quoted 4 lines

(when I saw "prot_virt" it felt like somebody is using a big hammer for small nails (e.g., compared to "stp=off").)

Can you guys add these details to the patch description?

yeah, probably a good idea

From: Vasily Gorbik <[email protected]>

Introduce KVM_S390_PROTECTED_VIRTUALIZATION_HOST kbuild option for protected virtual machines hosting support code.

Add "prot_virt" command line option which controls if the kernel protected VMs support is enabled at runtime.

Extend ultravisor info definitions and expose it via uv_info struct filled in during startup.

Signed-off-by Vasily Gorbik <[email protected]>

---
 .../admin-guide/kernel-parameters.txt         |  5 ++
 arch/s390/boot/Makefile                       |  2 +-
 arch/s390/boot/uv.c                           | 20 +++++++-
 arch/s390/include/asm/uv.h                    | 46 ++++++++++++++++--
 arch/s390/kernel/Makefile                     |  1 +
 arch/s390/kernel/setup.c                      |  4 --
 arch/s390/kernel/uv.c                         | 48 +++++++++++++++++++
 arch/s390/kvm/Kconfig                         |  9 ++++
 8 files changed, 126 insertions(+), 9 deletions(-)
 create mode 100644 arch/s390/kernel/uv.c

arch/s390/kvm/Kconfig +0 -0

index d3db3d7ed077..652b36f0efca 100644

arch/s390/kvm/Kconfig +1 -0

+
 # OK, it's a little counter-intuitive to do this, but it puts it neatly under
 # the virtualization menu.
 source "drivers/vhost/Kconfig"

I think there's not enough information in here to allow someone configuring the kernel to decide what this is and if it would be useful to them. This should probably be at least point to some document giving some more details. Also, can you add a sentence where this feature is actually expected to be available?

arch/s390/kvm/Kconfig config KVM_S390_UCONTROL · line 65

Quoted 2 lines

arch/s390/kvm/Kconfig +2 -0

+
+	  If unsure, say Y.

Is 'Y' really the safe choice here? AFAICS, this is introducing new code and not only trying to call new interfaces, if available. Is there any drawback to enabling this on a kernel that won't run on a platform supporting this feature? Is this supposed to be a common setup?

From: Vasily Gorbik <[email protected]>

Introduce KVM_S390_PROTECTED_VIRTUALIZATION_HOST kbuild option for protected virtual machines hosting support code.

Add "prot_virt" command line option which controls if the kernel protected VMs support is enabled at runtime.

Extend ultravisor info definitions and expose it via uv_info struct filled in during startup.

Signed-off-by Vasily Gorbik <[email protected]>

---
 .../admin-guide/kernel-parameters.txt         |  5 ++
 arch/s390/boot/Makefile                       |  2 +-
 arch/s390/boot/uv.c                           | 20 +++++++-
 arch/s390/include/asm/uv.h                    | 46 ++++++++++++++++--
 arch/s390/kernel/Makefile                     |  1 +
 arch/s390/kernel/setup.c                      |  4 --
 arch/s390/kernel/uv.c                         | 48 +++++++++++++++++++
 arch/s390/kvm/Kconfig                         |  9 ++++
 8 files changed, 126 insertions(+), 9 deletions(-)
 create mode 100644 arch/s390/kernel/uv.c

Documentation/admin-guide/kernel-parameters.txt +5 -0

index c7ac2f3ac99f..aa22e36b3105 100644
@@ -3693,6 +3693,11 @@
 			before loading.
 			See Documentation/admin-guide/blockdev/ramdisk.rst.
 
+	prot_virt=	[S390] enable hosting protected virtual machines
+			isolated from the hypervisor (if hardware supports
+			that).
+			Format: <bool>
+
 	psi=		[KNL] Enable or disable pressure stall information
 			tracking.
 			Format: <bool>

arch/s390/boot/Makefile +1 -1

index e2c47d3a1c89..82247e71617a 100644
@@ -37,7 +37,7 @@ CFLAGS_sclp_early_core.o += -I$(srctree)/drivers/s390/char
 obj-y	:= head.o als.o startup.o mem_detect.o ipl_parm.o ipl_report.o
 obj-y	+= string.o ebcdic.o sclp_early_core.o mem.o ipl_vmparm.o cmdline.o
 obj-y	+= version.o pgm_check_info.o ctype.o text_dma.o
-obj-$(CONFIG_PROTECTED_VIRTUALIZATION_GUEST)	+= uv.o
+obj-$(findstring y, $(CONFIG_PROTECTED_VIRTUALIZATION_GUEST) $(CONFIG_KVM_S390_PROTECTED_VIRTUALIZATION_HOST))	+= uv.o
 obj-$(CONFIG_RELOCATABLE)	+= machine_kexec_reloc.o
 obj-$(CONFIG_RANDOMIZE_BASE)	+= kaslr.o
 targets	:= bzImage startup.a section_cmp.boot.data section_cmp.boot.preserved.data $(obj-y)

arch/s390/boot/uv.c +19 -1

index ed007f4a6444..88cf8825d169 100644
@@ -3,7 +3,12 @@
 #include <asm/facility.h>
 #include <asm/sections.h>
 
+#ifdef CONFIG_PROTECTED_VIRTUALIZATION_GUEST
 int __bootdata_preserved(prot_virt_guest);
+#endif
+#ifdef CONFIG_KVM_S390_PROTECTED_VIRTUALIZATION_HOST
+struct uv_info __bootdata_preserved(uv_info);
+#endif
 
 void uv_query_info(void)
 {

@@ -18,7 +23,20 @@ void uv_query_info(void)
 	if (uv_call(0, (uint64_t)&uvcb))
 		return;
 
-	if (test_bit_inv(BIT_UVC_CMD_SET_SHARED_ACCESS, (unsigned long *)uvcb.inst_calls_list) &&
+	if (IS_ENABLED(CONFIG_KVM_S390_PROTECTED_VIRTUALIZATION_HOST)) {
+		memcpy(uv_info.inst_calls_list, uvcb.inst_calls_list, sizeof(uv_info.inst_calls_list));
+		uv_info.uv_base_stor_len = uvcb.uv_base_stor_len;
+		uv_info.guest_base_stor_len = uvcb.conf_base_phys_stor_len;
+		uv_info.guest_virt_base_stor_len = uvcb.conf_base_virt_stor_len;
+		uv_info.guest_virt_var_stor_len = uvcb.conf_virt_var_stor_len;
+		uv_info.guest_cpu_stor_len = uvcb.cpu_stor_len;
+		uv_info.max_sec_stor_addr = ALIGN(uvcb.max_guest_stor_addr, PAGE_SIZE);
+		uv_info.max_num_sec_conf = uvcb.max_num_sec_conf;
+		uv_info.max_guest_cpus = uvcb.max_guest_cpus;
+	}
+
+	if (IS_ENABLED(CONFIG_PROTECTED_VIRTUALIZATION_GUEST) &&
+	    test_bit_inv(BIT_UVC_CMD_SET_SHARED_ACCESS, (unsigned long *)uvcb.inst_calls_list) &&
 	    test_bit_inv(BIT_UVC_CMD_REMOVE_SHARED_ACCESS, (unsigned long *)uvcb.inst_calls_list))
 		prot_virt_guest = 1;
 }

arch/s390/include/asm/uv.h +43 -3

index ef3c00b049ab..6db1bc495e67 100644
@@ -44,7 +44,19 @@ struct uv_cb_qui {
 	struct uv_cb_header header;
 	u64 reserved08;
 	u64 inst_calls_list[4];
-	u64 reserved30[15];
+	u64 reserved30[2];
+	u64 uv_base_stor_len;
+	u64 reserved48;
+	u64 conf_base_phys_stor_len;
+	u64 conf_base_virt_stor_len;
+	u64 conf_virt_var_stor_len;
+	u64 cpu_stor_len;
+	u32 reserved68[3];
+	u32 max_num_sec_conf;
+	u64 max_guest_stor_addr;
+	u8  reserved80[150-128];
+	u16 max_guest_cpus;
+	u64 reserved98;
 } __packed __aligned(8);
 
 struct uv_cb_share {

@@ -69,9 +81,21 @@ static inline int uv_call(unsigned long r1, unsigned long r2)
 	return cc;
 }
 
-#ifdef CONFIG_PROTECTED_VIRTUALIZATION_GUEST
+struct uv_info {
+	unsigned long inst_calls_list[4];
+	unsigned long uv_base_stor_len;
+	unsigned long guest_base_stor_len;
+	unsigned long guest_virt_base_stor_len;
+	unsigned long guest_virt_var_stor_len;
+	unsigned long guest_cpu_stor_len;
+	unsigned long max_sec_stor_addr;
+	unsigned int max_num_sec_conf;
+	unsigned short max_guest_cpus;
+};
+extern struct uv_info uv_info;
 extern int prot_virt_guest;
 
+#ifdef CONFIG_PROTECTED_VIRTUALIZATION_GUEST
 static inline int is_prot_virt_guest(void)
 {
 	return prot_virt_guest;

@@ -121,11 +145,27 @@ static inline int uv_remove_shared(unsigned long addr)
 	return share(addr, UVC_CMD_REMOVE_SHARED_ACCESS);
 }
 
-void uv_query_info(void);
 #else
 #define is_prot_virt_guest() 0
 static inline int uv_set_shared(unsigned long addr) { return 0; }
 static inline int uv_remove_shared(unsigned long addr) { return 0; }
+#endif
+
+#ifdef CONFIG_KVM_S390_PROTECTED_VIRTUALIZATION_HOST
+extern int prot_virt_host;
+
+static inline int is_prot_virt_host(void)
+{
+	return prot_virt_host;
+}
+#else
+#define is_prot_virt_host() 0
+#endif
+
+#if defined(CONFIG_PROTECTED_VIRTUALIZATION_GUEST) ||                          \
+	defined(CONFIG_KVM_S390_PROTECTED_VIRTUALIZATION_HOST)
+void uv_query_info(void);
+#else
 static inline void uv_query_info(void) {}
 #endif
 

arch/s390/kernel/Makefile +1 -0

index 7edbbcd8228a..fe4fe475f526 100644
@@ -78,6 +78,7 @@ obj-$(CONFIG_PERF_EVENTS)	+= perf_cpum_cf_events.o perf_regs.o
 obj-$(CONFIG_PERF_EVENTS)	+= perf_cpum_cf_diag.o
 
 obj-$(CONFIG_TRACEPOINTS)	+= trace.o
+obj-$(findstring y, $(CONFIG_PROTECTED_VIRTUALIZATION_GUEST) $(CONFIG_KVM_S390_PROTECTED_VIRTUALIZATION_HOST))	+= uv.o
 
 # vdso
 obj-y				+= vdso64/

arch/s390/kernel/setup.c +0 -4

index 3ff291bc63b7..f36370f8af38 100644
@@ -92,10 +92,6 @@ char elf_platform[ELF_PLATFORM_SIZE];
 
 unsigned long int_hwcap = 0;
 
-#ifdef CONFIG_PROTECTED_VIRTUALIZATION_GUEST
-int __bootdata_preserved(prot_virt_guest);
-#endif
-
 int __bootdata(noexec_disabled);
 int __bootdata(memory_end_set);
 unsigned long __bootdata(memory_end);

arch/s390/kernel/uv.c new file +48 -0

new file mode 100644
index 000000000000..35ce89695509
@@ -0,0 +1,48 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Common Ultravisor functions and initialization
+ *
+ * Copyright IBM Corp. 2019
+ */
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/sizes.h>
+#include <linux/bitmap.h>
+#include <linux/memblock.h>
+#include <asm/facility.h>
+#include <asm/sections.h>
+#include <asm/uv.h>
+
+#ifdef CONFIG_PROTECTED_VIRTUALIZATION_GUEST
+int __bootdata_preserved(prot_virt_guest);
+#endif
+
+#ifdef CONFIG_KVM_S390_PROTECTED_VIRTUALIZATION_HOST
+int prot_virt_host;
+EXPORT_SYMBOL(prot_virt_host);
+struct uv_info __bootdata_preserved(uv_info);
+EXPORT_SYMBOL(uv_info);
+
+static int __init prot_virt_setup(char *val)
+{
+	bool enabled;
+	int rc;
+
+	rc = kstrtobool(val, &enabled);
+	if (!rc && enabled)
+		prot_virt_host = 1;
+
+	if (is_prot_virt_guest() && prot_virt_host) {
+		prot_virt_host = 0;
+		pr_info("Running as protected virtualization guest.");
+	}
+
+	if (prot_virt_host && !test_facility(158)) {
+		prot_virt_host = 0;
+		pr_info("The ultravisor call facility is not available.");
+	}
+
+	return rc;
+}
+early_param("prot_virt", prot_virt_setup);
+#endif

arch/s390/kvm/Kconfig +2 -0

index d3db3d7ed077..652b36f0efca 100644
@@ -55,6 +55,15 @@ config KVM_S390_UCONTROL
 
 	  If unsure, say N.
 
+config KVM_S390_PROTECTED_VIRTUALIZATION_HOST
+	bool "Protected guests execution support"

Quoted 23 lines · Janosch Frank

From: Vasily Gorbik <[email protected]>

Introduce KVM_S390_PROTECTED_VIRTUALIZATION_HOST kbuild option for protected virtual machines hosting support code.

Add "prot_virt" command line option which controls if the kernel protected VMs support is enabled at runtime.

Extend ultravisor info definitions and expose it via uv_info struct filled in during startup.

Signed-off-by Vasily Gorbik <[email protected]>

---
 .../admin-guide/kernel-parameters.txt         |  5 ++
 arch/s390/boot/Makefile                       |  2 +-
 arch/s390/boot/uv.c                           | 20 +++++++-
 arch/s390/include/asm/uv.h                    | 46 ++++++++++++++++--
 arch/s390/kernel/Makefile                     |  1 +
 arch/s390/kernel/setup.c                      |  4 --
 arch/s390/kernel/uv.c                         | 48 +++++++++++++++++++
 arch/s390/kvm/Kconfig                         |  9 ++++
 8 files changed, 126 insertions(+), 9 deletions(-)
 create mode 100644 arch/s390/kernel/uv.c

Quoted 18 lines

arch/s390/kvm/Kconfig +9 -0

index d3db3d7ed077..652b36f0efca 100644
@@ -55,6 +55,15 @@ config KVM_S390_UCONTROL
 
 	  If unsure, say N.
 
+config KVM_S390_PROTECTED_VIRTUALIZATION_HOST
+	bool "Protected guests execution support"
+	depends on KVM
+	---help---
+	  Support hosting protected virtual machines isolated from the
+	  hypervisor.
+
+	  If unsure, say Y.
+
 # OK, it's a little counter-intuitive to do this, but it puts it neatly under

arch/s390/kvm/Kconfig config KVM_S390_UCONTROL · line 69

Quoted 3 lines

arch/s390/kvm/Kconfig +0 -0

 # the virtualization menu.
 source "drivers/vhost/Kconfig"

As we have the prot_virt kernel paramter there is a way to fence this during runtime Not sure if we really need a build time fence. We could get rid of CONFIG_KVM_S390_PROTECTED_VIRTUALIZATION_HOST and just use CONFIG_KVM instead, assuming that in the long run all distros will enable that anyway.

If other reviewers prefer to keep that extra option what about the following to the help section:

----
Support hosting protected virtual machines in KVM. The state of these machines like memory content or register content is protected from the host or host administrators.

Enabling this option will enable extra code that talks to a new firmware instance

called ultravisor that will take care of protecting the guest while also enabling KVM to run this guest.

This feature must be enable by the kernel command line option prot_virt.

If unsure, say Y.

Quoted 24 lines · On Fri, 1 Nov 2019 09:53:12 +0100 Christian Borntraeger…

Quoted 23 lines · Janosch Frank

From: Vasily Gorbik <[email protected]>

Introduce KVM_S390_PROTECTED_VIRTUALIZATION_HOST kbuild option for protected virtual machines hosting support code.

Add "prot_virt" command line option which controls if the kernel protected VMs support is enabled at runtime.

Extend ultravisor info definitions and expose it via uv_info struct filled in during startup.

Signed-off-by Vasily Gorbik <[email protected]>

---
 .../admin-guide/kernel-parameters.txt         |  5 ++
 arch/s390/boot/Makefile                       |  2 +-
 arch/s390/boot/uv.c                           | 20 +++++++-
 arch/s390/include/asm/uv.h                    | 46 ++++++++++++++++--
 arch/s390/kernel/Makefile                     |  1 +
 arch/s390/kernel/setup.c                      |  4 --
 arch/s390/kernel/uv.c                         | 48 +++++++++++++++++++
 arch/s390/kvm/Kconfig                         |  9 ++++
 8 files changed, 126 insertions(+), 9 deletions(-)
 create mode 100644 arch/s390/kernel/uv.c

(...)

Quoted 18 lines

Quoted 18 lines

arch/s390/kvm/Kconfig +9 -0

index d3db3d7ed077..652b36f0efca 100644
@@ -55,6 +55,15 @@ config KVM_S390_UCONTROL
 
 	  If unsure, say N.
 
+config KVM_S390_PROTECTED_VIRTUALIZATION_HOST
+	bool "Protected guests execution support"
+	depends on KVM
+	---help---
+	  Support hosting protected virtual machines isolated from the
+	  hypervisor.
+
+	  If unsure, say Y.
+
 # OK, it's a little counter-intuitive to do this, but it puts it neatly under

Quoted 8 lines

arch/s390/kvm/Kconfig config KVM_S390_UCONTROL · line 69

Quoted 3 lines

arch/s390/kvm/Kconfig +0 -0

 # the virtualization menu.
 source "drivers/vhost/Kconfig"

As we have the prot_virt kernel paramter there is a way to fence this during runtime Not sure if we really need a build time fence. We could get rid of CONFIG_KVM_S390_PROTECTED_VIRTUALIZATION_HOST and just use CONFIG_KVM instead, assuming that in the long run all distros will enable that anyway.

I still need to read through the rest of this patch set to have an informed opinion on that, which will probably take some more time.

Quoted 3 lines

If other reviewers prefer to keep that extra option what about the following to the help section:

Quoted 5 lines

----
Support hosting protected virtual machines in KVM. The state of these machines like memory content or register content is protected from the host or host administrators.

Enabling this option will enable extra code that talks to a new firmware instance

"...that allows the host kernel to talk..." ?

Quoted 5 lines

called ultravisor that will take care of protecting the guest while also enabling KVM to run this guest.

This feature must be enable by the kernel command line option prot_virt.

s/enable by/enabled via/

Quoted 2 lines

If unsure, say Y.

Looks better. I'm continuing to read the rest of this series before I say more, though :)

From: Vasily Gorbik <[email protected]>

Introduce KVM_S390_PROTECTED_VIRTUALIZATION_HOST kbuild option for protected virtual machines hosting support code.

Add "prot_virt" command line option which controls if the kernel protected VMs support is enabled at runtime.

Extend ultravisor info definitions and expose it via uv_info struct filled in during startup.

Signed-off-by Vasily Gorbik <[email protected]>

---
 .../admin-guide/kernel-parameters.txt         |  5 ++
 arch/s390/boot/Makefile                       |  2 +-
 arch/s390/boot/uv.c                           | 20 +++++++-
 arch/s390/include/asm/uv.h                    | 46 ++++++++++++++++--
 arch/s390/kernel/Makefile                     |  1 +
 arch/s390/kernel/setup.c                      |  4 --
 arch/s390/kernel/uv.c                         | 48 +++++++++++++++++++
 arch/s390/kvm/Kconfig                         |  9 ++++
 8 files changed, 126 insertions(+), 9 deletions(-)
 create mode 100644 arch/s390/kernel/uv.c

arch/s390/boot/uv.c +0 -0

index ed007f4a6444..88cf8825d169 100644

arch/s390/boot/uv.c +4 -0

+		memcpy(uv_info.inst_calls_list, uvcb.inst_calls_list, sizeof(uv_info.inst_calls_list));
+		uv_info.uv_base_stor_len = uvcb.uv_base_stor_len;
+		uv_info.guest_base_stor_len = uvcb.conf_base_phys_stor_len;
+		uv_info.guest_virt_base_stor_len = uvcb.conf_base_virt_stor_len;

arch/s390/boot/uv.c +0 -0

 		prot_virt_guest = 1;
 }

arch/s390/include/asm/uv.h +14 -2

index ef3c00b049ab..6db1bc495e67 100644
@@ -44,7 +44,19 @@ struct uv_cb_qui {
 	struct uv_cb_header header;
 	u64 reserved08;
 	u64 inst_calls_list[4];
-	u64 reserved30[15];
+	u64 reserved30[2];
+	u64 uv_base_stor_len;
+	u64 reserved48;
+	u64 conf_base_phys_stor_len;
+	u64 conf_base_virt_stor_len;
+	u64 conf_virt_var_stor_len;
+	u64 cpu_stor_len;
+	u32 reserved68[3];
+	u32 max_num_sec_conf;
+	u64 max_guest_stor_addr;
+	u8  reserved80[150-128];
+	u16 max_guest_cpus;
+	u64 reserved98;
 } __packed __aligned(8);
 
 struct uv_cb_share {

@@ -69,9 +81,21 @@ static inline int uv_call(unsigned long r1, unsigned long r2)
 	return cc;
 }
 
-#ifdef CONFIG_PROTECTED_VIRTUALIZATION_GUEST
+struct uv_info {

arch/s390/include/asm/uv.h +19 -1

+extern struct uv_info uv_info;
 extern int prot_virt_guest;
 
+#ifdef CONFIG_PROTECTED_VIRTUALIZATION_GUEST
 static inline int is_prot_virt_guest(void)
 {
 	return prot_virt_guest;

@@ -121,11 +145,27 @@ static inline int uv_remove_shared(unsigned long addr)
 	return share(addr, UVC_CMD_REMOVE_SHARED_ACCESS);
 }
 
-void uv_query_info(void);
 #else
 #define is_prot_virt_guest() 0
 static inline int uv_set_shared(unsigned long addr) { return 0; }
 static inline int uv_remove_shared(unsigned long addr) { return 0; }
+#endif
+
+#ifdef CONFIG_KVM_S390_PROTECTED_VIRTUALIZATION_HOST
+extern int prot_virt_host;
+
+static inline int is_prot_virt_host(void)
+{
+	return prot_virt_host;
+}
+#else
+#define is_prot_virt_host() 0
+#endif
+
+#if defined(CONFIG_PROTECTED_VIRTUALIZATION_GUEST) ||                          \
+	defined(CONFIG_KVM_S390_PROTECTED_VIRTUALIZATION_HOST)
+void uv_query_info(void);
+#else
 static inline void uv_query_info(void) {}
 #endif

arch/s390/kernel/uv.c new file +30 -0

new file mode 100644
index 000000000000..35ce89695509
@@ -0,0 +1,48 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Common Ultravisor functions and initialization
+ *
+ * Copyright IBM Corp. 2019
+ */
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/sizes.h>
+#include <linux/bitmap.h>
+#include <linux/memblock.h>
+#include <asm/facility.h>
+#include <asm/sections.h>
+#include <asm/uv.h>
+
+#ifdef CONFIG_PROTECTED_VIRTUALIZATION_GUEST
+int __bootdata_preserved(prot_virt_guest);
+#endif
+
+#ifdef CONFIG_KVM_S390_PROTECTED_VIRTUALIZATION_HOST
+int prot_virt_host;
+EXPORT_SYMBOL(prot_virt_host);
+struct uv_info __bootdata_preserved(uv_info);
+EXPORT_SYMBOL(uv_info);
+
+static int __init prot_virt_setup(char *val)
+{
+	bool enabled;
+	int rc;
+

arch/s390/kernel/uv.c new file +1 -0

+#endif

Quoted 23 lines · On Thu, 24 Oct 2019 07:40:24 -0400 Janosch Frank <[email protected]> wrote:

From: Vasily Gorbik <[email protected]>

Introduce KVM_S390_PROTECTED_VIRTUALIZATION_HOST kbuild option for protected virtual machines hosting support code.

Add "prot_virt" command line option which controls if the kernel protected VMs support is enabled at runtime.

Extend ultravisor info definitions and expose it via uv_info struct filled in during startup.

Signed-off-by Vasily Gorbik <[email protected]>

---
 .../admin-guide/kernel-parameters.txt         |  5 ++
 arch/s390/boot/Makefile                       |  2 +-
 arch/s390/boot/uv.c                           | 20 +++++++-
 arch/s390/include/asm/uv.h                    | 46 ++++++++++++++++--
 arch/s390/kernel/Makefile                     |  1 +
 arch/s390/kernel/setup.c                      |  4 --
 arch/s390/kernel/uv.c                         | 48 +++++++++++++++++++
 arch/s390/kvm/Kconfig                         |  9 ++++
 8 files changed, 126 insertions(+), 9 deletions(-)
 create mode 100644 arch/s390/kernel/uv.c

(...)

Quoted 5 lines

arch/s390/boot/uv.c +0 -0

index ed007f4a6444..88cf8825d169 100644

arch/s390/boot/uv.c line 11

Quoted 9 lines

arch/s390/boot/uv.c +5 -0

@@ -3,7 +3,12 @@
 #include <asm/facility.h>
 #include <asm/sections.h>
 
+#ifdef CONFIG_PROTECTED_VIRTUALIZATION_GUEST
 int __bootdata_preserved(prot_virt_guest);
+#endif
+#ifdef CONFIG_KVM_S390_PROTECTED_VIRTUALIZATION_HOST
+struct uv_info __bootdata_preserved(uv_info);
+#endif

Two functions with the same name, but different signatures look really ugly.

Also, what happens if I want to build just a single kernel image for both guest and host?

Quoted 8 lines

arch/s390/boot/uv.c +0 -0

 void uv_query_info(void)
 {

@@ -18,7 +23,20 @@ void uv_query_info(void)
 	if (uv_call(0, (uint64_t)&uvcb))
 		return;
 

arch/s390/boot/uv.c void uv_query_info(void) · line 26

Quoted 2 lines

arch/s390/boot/uv.c +1 -1

-	if (test_bit_inv(BIT_UVC_CMD_SET_SHARED_ACCESS, (unsigned long *)uvcb.inst_calls_list) &&
+	if (IS_ENABLED(CONFIG_KVM_S390_PROTECTED_VIRTUALIZATION_HOST)) {

Do we always have everything needed for a host if uv_call() is successful?

Quoted 5 lines

arch/s390/boot/uv.c +4 -0

+		memcpy(uv_info.inst_calls_list, uvcb.inst_calls_list, sizeof(uv_info.inst_calls_list));
+		uv_info.uv_base_stor_len = uvcb.uv_base_stor_len;
+		uv_info.guest_base_stor_len = uvcb.conf_base_phys_stor_len;
+		uv_info.guest_virt_base_stor_len = uvcb.conf_base_virt_stor_len;

arch/s390/boot/uv.c void uv_query_info(void) · line 40

Quoted 10 lines

arch/s390/boot/uv.c +9 -0

+		uv_info.guest_virt_var_stor_len = uvcb.conf_virt_var_stor_len;
+		uv_info.guest_cpu_stor_len = uvcb.cpu_stor_len;
+		uv_info.max_sec_stor_addr = ALIGN(uvcb.max_guest_stor_addr, PAGE_SIZE);
+		uv_info.max_num_sec_conf = uvcb.max_num_sec_conf;
+		uv_info.max_guest_cpus = uvcb.max_guest_cpus;
+	}
+
+	if (IS_ENABLED(CONFIG_PROTECTED_VIRTUALIZATION_GUEST) &&
+	    test_bit_inv(BIT_UVC_CMD_SET_SHARED_ACCESS, (unsigned long *)uvcb.inst_calls_list) &&
 	    test_bit_inv(BIT_UVC_CMD_REMOVE_SHARED_ACCESS, (unsigned long *)uvcb.inst_calls_list))

Especially as it looks like we need to test for those two commands to determine whether we have support for a guest.

Quoted 31 lines

arch/s390/boot/uv.c +0 -0

 		prot_virt_guest = 1;
 }

arch/s390/include/asm/uv.h +13 -1

index ef3c00b049ab..6db1bc495e67 100644
@@ -44,7 +44,19 @@ struct uv_cb_qui {
 	struct uv_cb_header header;
 	u64 reserved08;
 	u64 inst_calls_list[4];
-	u64 reserved30[15];
+	u64 reserved30[2];
+	u64 uv_base_stor_len;
+	u64 reserved48;
+	u64 conf_base_phys_stor_len;
+	u64 conf_base_virt_stor_len;
+	u64 conf_virt_var_stor_len;
+	u64 cpu_stor_len;
+	u32 reserved68[3];
+	u32 max_num_sec_conf;
+	u64 max_guest_stor_addr;
+	u8  reserved80[150-128];
+	u16 max_guest_cpus;
+	u64 reserved98;
 } __packed __aligned(8);
 
 struct uv_cb_share {

@@ -69,9 +81,21 @@ static inline int uv_call(unsigned long r1, unsigned long r2)
 	return cc;
 }
 

arch/s390/include/asm/uv.h static inline int uv_call(unsigned long r1, unsigned long r2) · line 94

Quoted 12 lines

arch/s390/include/asm/uv.h +11 -1

-#ifdef CONFIG_PROTECTED_VIRTUALIZATION_GUEST
+struct uv_info {
+	unsigned long inst_calls_list[4];
+	unsigned long uv_base_stor_len;
+	unsigned long guest_base_stor_len;
+	unsigned long guest_virt_base_stor_len;
+	unsigned long guest_virt_var_stor_len;
+	unsigned long guest_cpu_stor_len;
+	unsigned long max_sec_stor_addr;
+	unsigned int max_num_sec_conf;
+	unsigned short max_guest_cpus;
+};

What is the main difference between uv_info and uv_cb_qui? The alignment of max_sec_stor_addr?

Quoted 36 lines

arch/s390/include/asm/uv.h +19 -1

+extern struct uv_info uv_info;
 extern int prot_virt_guest;
 
+#ifdef CONFIG_PROTECTED_VIRTUALIZATION_GUEST
 static inline int is_prot_virt_guest(void)
 {
 	return prot_virt_guest;

@@ -121,11 +145,27 @@ static inline int uv_remove_shared(unsigned long addr)
 	return share(addr, UVC_CMD_REMOVE_SHARED_ACCESS);
 }
 
-void uv_query_info(void);
 #else
 #define is_prot_virt_guest() 0
 static inline int uv_set_shared(unsigned long addr) { return 0; }
 static inline int uv_remove_shared(unsigned long addr) { return 0; }
+#endif
+
+#ifdef CONFIG_KVM_S390_PROTECTED_VIRTUALIZATION_HOST
+extern int prot_virt_host;
+
+static inline int is_prot_virt_host(void)
+{
+	return prot_virt_host;
+}
+#else
+#define is_prot_virt_host() 0
+#endif
+
+#if defined(CONFIG_PROTECTED_VIRTUALIZATION_GUEST) ||                          \
+	defined(CONFIG_KVM_S390_PROTECTED_VIRTUALIZATION_HOST)
+void uv_query_info(void);
+#else
 static inline void uv_query_info(void) {}
 #endif

(...)

Quoted 21 lines · Cornelia Huck

Quoted 5 lines · On Thu, 24 Oct 2019 07:40:24 -0400 Janosch Frank <[email protected]> wrote:

arch/s390/boot/uv.c +0 -0

index ed007f4a6444..88cf8825d169 100644

arch/s390/boot/uv.c line 11

Quoted 9 lines

arch/s390/boot/uv.c +5 -0

@@ -3,7 +3,12 @@
 #include <asm/facility.h>
 #include <asm/sections.h>
 
+#ifdef CONFIG_PROTECTED_VIRTUALIZATION_GUEST
 int __bootdata_preserved(prot_virt_guest);
+#endif
+#ifdef CONFIG_KVM_S390_PROTECTED_VIRTUALIZATION_HOST
+struct uv_info __bootdata_preserved(uv_info);
+#endif  

Two functions with the same name, but different signatures look really ugly.

Quoted 2 lines

Also, what happens if I want to build just a single kernel image for both guest and host?

This is not two functions with the same name. It is 2 variable declarations with the __bootdata_preserved helper. We expect to have all distro kernels to enable both.

Quoted 9 lines

Quoted 9 lines

arch/s390/boot/uv.c +0 -0

 void uv_query_info(void)
 {

@@ -18,7 +23,20 @@ void uv_query_info(void)
 	if (uv_call(0, (uint64_t)&uvcb))
 		return;
 

Quoted 5 lines

arch/s390/boot/uv.c void uv_query_info(void) · line 26

Quoted 2 lines

arch/s390/boot/uv.c +1 -1

-	if (test_bit_inv(BIT_UVC_CMD_SET_SHARED_ACCESS, (unsigned long *)uvcb.inst_calls_list) &&
+	if (IS_ENABLED(CONFIG_KVM_S390_PROTECTED_VIRTUALIZATION_HOST)) {  

Do we always have everything needed for a host if uv_call() is successful?

The uv_call is the query call. It will provide the list of features. We check that later on.

Quoted 51 lines

Quoted 6 lines

arch/s390/boot/uv.c +4 -0

+		memcpy(uv_info.inst_calls_list, uvcb.inst_calls_list, sizeof(uv_info.inst_calls_list));
+		uv_info.uv_base_stor_len = uvcb.uv_base_stor_len;
+		uv_info.guest_base_stor_len = uvcb.conf_base_phys_stor_len;
+		uv_info.guest_virt_base_stor_len = uvcb.conf_base_virt_stor_len;

arch/s390/boot/uv.c void uv_query_info(void) · line 40

Quoted 10 lines

arch/s390/boot/uv.c +9 -0

+		uv_info.guest_virt_var_stor_len = uvcb.conf_virt_var_stor_len;
+		uv_info.guest_cpu_stor_len = uvcb.cpu_stor_len;
+		uv_info.max_sec_stor_addr = ALIGN(uvcb.max_guest_stor_addr, PAGE_SIZE);
+		uv_info.max_num_sec_conf = uvcb.max_num_sec_conf;
+		uv_info.max_guest_cpus = uvcb.max_guest_cpus;
+	}
+
+	if (IS_ENABLED(CONFIG_PROTECTED_VIRTUALIZATION_GUEST) &&
+	    test_bit_inv(BIT_UVC_CMD_SET_SHARED_ACCESS, (unsigned long *)uvcb.inst_calls_list) &&
 	    test_bit_inv(BIT_UVC_CMD_REMOVE_SHARED_ACCESS, (unsigned long *)uvcb.inst_calls_list))  

Especially as it looks like we need to test for those two commands to determine whether we have support for a guest.

Quoted 31 lines

arch/s390/boot/uv.c +0 -0

 		prot_virt_guest = 1;
 }

arch/s390/include/asm/uv.h +13 -1

index ef3c00b049ab..6db1bc495e67 100644
@@ -44,7 +44,19 @@ struct uv_cb_qui {
 	struct uv_cb_header header;
 	u64 reserved08;
 	u64 inst_calls_list[4];
-	u64 reserved30[15];
+	u64 reserved30[2];
+	u64 uv_base_stor_len;
+	u64 reserved48;
+	u64 conf_base_phys_stor_len;
+	u64 conf_base_virt_stor_len;
+	u64 conf_virt_var_stor_len;
+	u64 cpu_stor_len;
+	u32 reserved68[3];
+	u32 max_num_sec_conf;
+	u64 max_guest_stor_addr;
+	u8  reserved80[150-128];
+	u16 max_guest_cpus;
+	u64 reserved98;
 } __packed __aligned(8);
 
 struct uv_cb_share {

@@ -69,9 +81,21 @@ static inline int uv_call(unsigned long r1, unsigned long r2)
 	return cc;
 }
 

Quoted 15 lines

arch/s390/include/asm/uv.h static inline int uv_call(unsigned long r1, unsigned long r2) · line 94

Quoted 12 lines

arch/s390/include/asm/uv.h +11 -1

-#ifdef CONFIG_PROTECTED_VIRTUALIZATION_GUEST
+struct uv_info {
+	unsigned long inst_calls_list[4];
+	unsigned long uv_base_stor_len;
+	unsigned long guest_base_stor_len;
+	unsigned long guest_virt_base_stor_len;
+	unsigned long guest_virt_var_stor_len;
+	unsigned long guest_cpu_stor_len;
+	unsigned long max_sec_stor_addr;
+	unsigned int max_num_sec_conf;
+	unsigned short max_guest_cpus;
+};  

What is the main difference between uv_info and uv_cb_qui? The alignment of max_sec_stor_addr?

One is the hardware data structure for query, the other one is the Linux internal state.

Quoted 22 lines · On Mon, 4 Nov 2019 18:50:12 +0100 Christian Borntraeger…

Quoted 21 lines · Cornelia Huck

Quoted 5 lines · On Thu, 24 Oct 2019 07:40:24 -0400 Janosch Frank <[email protected]> wrote:

arch/s390/boot/uv.c +0 -0

index ed007f4a6444..88cf8825d169 100644

arch/s390/boot/uv.c line 11

Quoted 9 lines

arch/s390/boot/uv.c +5 -0

@@ -3,7 +3,12 @@
  #include <asm/facility.h>
  #include <asm/sections.h>
 
+#ifdef CONFIG_PROTECTED_VIRTUALIZATION_GUEST
  int __bootdata_preserved(prot_virt_guest);
+#endif
+#ifdef CONFIG_KVM_S390_PROTECTED_VIRTUALIZATION_HOST
+struct uv_info __bootdata_preserved(uv_info);
+#endif

Two functions with the same name, but different signatures look really ugly.

Quoted 6 lines

Quoted 2 lines

Also, what happens if I want to build just a single kernel image for both guest and host?

This is not two functions with the same name. It is 2 variable declarations with the __bootdata_preserved helper. We expect to have all distro kernels to enable both.

Ah ok, I misread that. (I'm blaming lack of sleep :/)

From: Vasily Gorbik <[email protected]>

Before being able to host protected virtual machines, donate some of the memory to the ultravisor. Besides that the ultravisor might impose addressing limitations for memory used to back protected VM storage. Treat that limit as protected virtualization host's virtual memory limit.

Signed-off-by Vasily Gorbik <[email protected]>

---
  arch/s390/include/asm/uv.h | 16 ++++++++++++
  arch/s390/kernel/setup.c   |  3 +++
  arch/s390/kernel/uv.c      | 53 ++++++++++++++++++++++++++++++++++++++
  3 files changed, 72 insertions(+)

arch/s390/include/asm/uv.h +16 -0

index 6db1bc495e67..82a46fb913e7 100644
@@ -23,12 +23,14 @@
  #define UVC_RC_NO_RESUME	0x0007
 
  #define UVC_CMD_QUI			0x0001
+#define UVC_CMD_INIT_UV			0x000f
  #define UVC_CMD_SET_SHARED_ACCESS	0x1000
  #define UVC_CMD_REMOVE_SHARED_ACCESS	0x1001
 
  /* Bits in installed uv calls */
  enum uv_cmds_inst {
  	BIT_UVC_CMD_QUI = 0,
+	BIT_UVC_CMD_INIT_UV = 1,
  	BIT_UVC_CMD_SET_SHARED_ACCESS = 8,
  	BIT_UVC_CMD_REMOVE_SHARED_ACCESS = 9,
  };

@@ -59,6 +61,15 @@ struct uv_cb_qui {
  	u64 reserved98;
  } __packed __aligned(8);
 
+struct uv_cb_init {
+	struct uv_cb_header header;
+	u64 reserved08[2];
+	u64 stor_origin;
+	u64 stor_len;
+	u64 reserved28[4];
+
+} __packed __aligned(8);
+
  struct uv_cb_share {
  	struct uv_cb_header header;
  	u64 reserved08[3];

@@ -158,8 +169,13 @@ static inline int is_prot_virt_host(void)
  {
  	return prot_virt_host;
  }
+
+void setup_uv(void);
+void adjust_to_uv_max(unsigned long *vmax);
  #else
  #define is_prot_virt_host() 0
+static inline void setup_uv(void) {}
+static inline void adjust_to_uv_max(unsigned long *vmax) {}
  #endif
 
  #if defined(CONFIG_PROTECTED_VIRTUALIZATION_GUEST) ||                          \

arch/s390/kernel/setup.c +0 -0

index f36370f8af38..d29d83c0b8df 100644

arch/s390/kernel/setup.c +2 -0

+
  	/* module area is at the end of the kernel address space. */
  	MODULES_END = vmax;
  	MODULES_VADDR = MODULES_END - MODULES_LEN;

@@ -1147,6 +1149,7 @@ void __init setup_arch(char **cmdline_p)
  	 */
  	memblock_trim_memory(1UL << (MAX_ORDER - 1 + PAGE_SHIFT));
 
+	setup_uv();
  	setup_memory_end();
  	setup_memory();
  	dma_contiguous_reserve(memory_end);

arch/s390/kernel/uv.c +25 -0

index 35ce89695509..f7778493e829 100644
@@ -45,4 +45,57 @@ static int __init prot_virt_setup(char *val)
  	return rc;
  }
  early_param("prot_virt", prot_virt_setup);
+
+static int __init uv_init(unsigned long stor_base, unsigned long stor_len)
+{
+	struct uv_cb_init uvcb = {
+		.header.cmd = UVC_CMD_INIT_UV,
+		.header.len = sizeof(uvcb),
+		.stor_origin = stor_base,
+		.stor_len = stor_len,
+	};
+	int cc;
+
+	cc = uv_call(0, (uint64_t)&uvcb);
+	if (cc || uvcb.header.rc != UVC_RC_EXECUTED) {
+		pr_err("Ultravisor init failed with cc: %d rc: 0x%hx\n", cc,
+		       uvcb.header.rc);
+		return -1;
+	}
+	return 0;
+}
+
+void __init setup_uv(void)
+{
+	unsigned long uv_stor_base;
+
+	if (!prot_virt_host)

arch/s390/kernel/uv.c +9 -0

+
+	if (uv_init(uv_stor_base, uv_info.uv_base_stor_len)) {
+		memblock_free(uv_stor_base, uv_info.uv_base_stor_len);
+		goto fail;
+	}
+
+	pr_info("Reserving %luMB as ultravisor base storage\n",
+		uv_info.uv_base_stor_len >> 20);
+	return;

Quoted 14 lines · Janosch Frank

From: Vasily Gorbik <[email protected]>

Before being able to host protected virtual machines, donate some of the memory to the ultravisor. Besides that the ultravisor might impose addressing limitations for memory used to back protected VM storage. Treat that limit as protected virtualization host's virtual memory limit.

Signed-off-by Vasily Gorbik <[email protected]>

---
  arch/s390/include/asm/uv.h | 16 ++++++++++++
  arch/s390/kernel/setup.c   |  3 +++
  arch/s390/kernel/uv.c      | 53 ++++++++++++++++++++++++++++++++++++++
  3 files changed, 72 insertions(+)

arch/s390/kernel/setup.c static void __init setup_memory_end(void) · line 569

Quoted 6 lines

arch/s390/kernel/setup.c +1 -0

@@ -567,6 +567,8 @@ static void __init setup_memory_end(void)
  			vmax = _REGION1_SIZE; /* 4-level kernel page table */
  	}
+	adjust_to_uv_max(&vmax);

I do wonder what would happen if vmax < max_physmem_end. Not sure if that is relevant at all.

Quoted 5 lines

arch/s390/kernel/setup.c +5 -0

+void __init setup_uv(void)
+{
+	unsigned long uv_stor_base;
+
+	if (!prot_virt_host)

arch/s390/kernel/setup.c static void __init setup_memory_end(void)

Quoted 10 lines

arch/s390/kernel/setup.c +10 -0

+		return;
+
+	uv_stor_base = (unsigned long)memblock_alloc_try_nid(
+		uv_info.uv_base_stor_len, SZ_1M, SZ_2G,
+		MEMBLOCK_ALLOC_ACCESSIBLE, NUMA_NO_NODE);
+	if (!uv_stor_base) {
+		pr_info("Failed to reserve %lu bytes for ultravisor base storage\n",
+			uv_info.uv_base_stor_len);
+		goto fail;
+	}

If I'm not wrong, we could setup/reserve a CMA area here and defer the actual allocation. Then, any MOVABLE data can end up on this CMA area until needed.

But I am neither an expert on CMA nor on UV, so most probably what I say is wrong ;)

Quoted 10 lines

arch/s390/kernel/setup.c +9 -0

+
+	if (uv_init(uv_stor_base, uv_info.uv_base_stor_len)) {
+		memblock_free(uv_stor_base, uv_info.uv_base_stor_len);
+		goto fail;
+	}
+
+	pr_info("Reserving %luMB as ultravisor base storage\n",
+		uv_info.uv_base_stor_len >> 20);
+	return;

arch/s390/kernel/setup.c

Quoted 10 lines

arch/s390/kernel/setup.c +9 -0

+fail:
+	prot_virt_host = 0;
+}
+
+void adjust_to_uv_max(unsigned long *vmax)
+{
+	if (prot_virt_host && *vmax > uv_info.max_sec_stor_addr)
+		*vmax = uv_info.max_sec_stor_addr;
+}
  #endif

Looks good to me from what I can tell.

Signature · 3 lines

Thanks,

David / dhildenb

Quoted 15 lines · David Hildenbrand

Quoted 14 lines · Janosch Frank

From: Vasily Gorbik <[email protected]>

Before being able to host protected virtual machines, donate some of the memory to the ultravisor. Besides that the ultravisor might impose addressing limitations for memory used to back protected VM storage. Treat that limit as protected virtualization host's virtual memory limit.

Signed-off-by Vasily Gorbik <[email protected]>

---
   arch/s390/include/asm/uv.h | 16 ++++++++++++
   arch/s390/kernel/setup.c   |  3 +++
   arch/s390/kernel/uv.c      | 53 ++++++++++++++++++++++++++++++++++++++
   3 files changed, 72 insertions(+)

Quoted 9 lines

arch/s390/kernel/setup.c static void __init setup_memory_end(void) · line 569

Quoted 6 lines

arch/s390/kernel/setup.c +1 -0

@@ -567,6 +567,8 @@ static void __init setup_memory_end(void)
   			vmax = _REGION1_SIZE; /* 4-level kernel page table */
   	}
+	adjust_to_uv_max(&vmax);

I do wonder what would happen if vmax < max_physmem_end. Not sure if that is relevant at all.

Then identity mapping would be shorter then actual physical memory available and everything above would be lost. But in reality "max_sec_stor_addr" is big enough to not worry about it in the foreseeable future at all.

Quoted 20 lines

Quoted 5 lines

arch/s390/kernel/setup.c +5 -0

+void __init setup_uv(void)
+{
+	unsigned long uv_stor_base;
+
+	if (!prot_virt_host)

arch/s390/kernel/setup.c static void __init setup_memory_end(void)

Quoted 10 lines

arch/s390/kernel/setup.c +10 -0

+		return;
+
+	uv_stor_base = (unsigned long)memblock_alloc_try_nid(
+		uv_info.uv_base_stor_len, SZ_1M, SZ_2G,
+		MEMBLOCK_ALLOC_ACCESSIBLE, NUMA_NO_NODE);
+	if (!uv_stor_base) {
+		pr_info("Failed to reserve %lu bytes for ultravisor base storage\n",
+			uv_info.uv_base_stor_len);
+		goto fail;
+	}

If I'm not wrong, we could setup/reserve a CMA area here and defer the actual allocation. Then, any MOVABLE data can end up on this CMA area until needed.

Quoted 2 lines

But I am neither an expert on CMA nor on UV, so most probably what I say is wrong ;)

From pure memory management this sounds like a good idea. And I tried
it and cma_declare_contiguous fulfills our needs, just had to export cma_alloc/cma_release symbols. Nevertheless, delaying ultravisor init means we would be potentially left with vmax == max_sec_stor_addr even if we wouldn't be able to run protected VMs after all (currently setup_uv() is called before kernel address space layout setup). Another much more fundamental reason is that ultravisor init has to be called with a single cpu running, which means it's easy to do before bringing other cpus up and we currently don't have api to stop cpus at a later point (stop_machine won't cut it).

From: Vasily Gorbik <[email protected]>

Before being able to host protected virtual machines, donate some of the memory to the ultravisor. Besides that the ultravisor might impose addressing limitations for memory used to back protected VM storage. Treat that limit as protected virtualization host's virtual memory limit.

Signed-off-by Vasily Gorbik <[email protected]>

---
 arch/s390/include/asm/uv.h | 16 ++++++++++++
 arch/s390/kernel/setup.c   |  3 +++
 arch/s390/kernel/uv.c      | 53 ++++++++++++++++++++++++++++++++++++++
 3 files changed, 72 insertions(+)

arch/s390/include/asm/uv.h +16 -0

index 6db1bc495e67..82a46fb913e7 100644
@@ -23,12 +23,14 @@
 #define UVC_RC_NO_RESUME	0x0007
 
 #define UVC_CMD_QUI			0x0001
+#define UVC_CMD_INIT_UV			0x000f
 #define UVC_CMD_SET_SHARED_ACCESS	0x1000
 #define UVC_CMD_REMOVE_SHARED_ACCESS	0x1001
 
 /* Bits in installed uv calls */
 enum uv_cmds_inst {
 	BIT_UVC_CMD_QUI = 0,
+	BIT_UVC_CMD_INIT_UV = 1,
 	BIT_UVC_CMD_SET_SHARED_ACCESS = 8,
 	BIT_UVC_CMD_REMOVE_SHARED_ACCESS = 9,
 };

@@ -59,6 +61,15 @@ struct uv_cb_qui {
 	u64 reserved98;
 } __packed __aligned(8);
 
+struct uv_cb_init {
+	struct uv_cb_header header;
+	u64 reserved08[2];
+	u64 stor_origin;
+	u64 stor_len;
+	u64 reserved28[4];
+
+} __packed __aligned(8);
+
 struct uv_cb_share {
 	struct uv_cb_header header;
 	u64 reserved08[3];

@@ -158,8 +169,13 @@ static inline int is_prot_virt_host(void)
 {
 	return prot_virt_host;
 }
+
+void setup_uv(void);
+void adjust_to_uv_max(unsigned long *vmax);
 #else
 #define is_prot_virt_host() 0
+static inline void setup_uv(void) {}
+static inline void adjust_to_uv_max(unsigned long *vmax) {}
 #endif
 
 #if defined(CONFIG_PROTECTED_VIRTUALIZATION_GUEST) ||                          \

arch/s390/kernel/setup.c +3 -0

index f36370f8af38..d29d83c0b8df 100644
@@ -567,6 +567,8 @@ static void __init setup_memory_end(void)
 			vmax = _REGION1_SIZE; /* 4-level kernel page table */
 	}
 
+	adjust_to_uv_max(&vmax);
+
 	/* module area is at the end of the kernel address space. */
 	MODULES_END = vmax;
 	MODULES_VADDR = MODULES_END - MODULES_LEN;

@@ -1147,6 +1149,7 @@ void __init setup_arch(char **cmdline_p)
 	 */
 	memblock_trim_memory(1UL << (MAX_ORDER - 1 + PAGE_SHIFT));
 
+	setup_uv();
 	setup_memory_end();
 	setup_memory();
 	dma_contiguous_reserve(memory_end);

arch/s390/kernel/uv.c +53 -0

index 35ce89695509..f7778493e829 100644
@@ -45,4 +45,57 @@ static int __init prot_virt_setup(char *val)
 	return rc;
 }
 early_param("prot_virt", prot_virt_setup);
+
+static int __init uv_init(unsigned long stor_base, unsigned long stor_len)
+{
+	struct uv_cb_init uvcb = {
+		.header.cmd = UVC_CMD_INIT_UV,
+		.header.len = sizeof(uvcb),
+		.stor_origin = stor_base,
+		.stor_len = stor_len,
+	};
+	int cc;
+
+	cc = uv_call(0, (uint64_t)&uvcb);
+	if (cc || uvcb.header.rc != UVC_RC_EXECUTED) {
+		pr_err("Ultravisor init failed with cc: %d rc: 0x%hx\n", cc,
+		       uvcb.header.rc);
+		return -1;
+	}
+	return 0;
+}
+
+void __init setup_uv(void)
+{
+	unsigned long uv_stor_base;
+
+	if (!prot_virt_host)
+		return;
+
+	uv_stor_base = (unsigned long)memblock_alloc_try_nid(
+		uv_info.uv_base_stor_len, SZ_1M, SZ_2G,
+		MEMBLOCK_ALLOC_ACCESSIBLE, NUMA_NO_NODE);
+	if (!uv_stor_base) {
+		pr_info("Failed to reserve %lu bytes for ultravisor base storage\n",
+			uv_info.uv_base_stor_len);
+		goto fail;
+	}
+
+	if (uv_init(uv_stor_base, uv_info.uv_base_stor_len)) {
+		memblock_free(uv_stor_base, uv_info.uv_base_stor_len);
+		goto fail;
+	}
+
+	pr_info("Reserving %luMB as ultravisor base storage\n",
+		uv_info.uv_base_stor_len >> 20);
+	return;
+fail:
+	prot_virt_host = 0;
+}
+
+void adjust_to_uv_max(unsigned long *vmax)
+{
+	if (prot_virt_host && *vmax > uv_info.max_sec_stor_addr)
+		*vmax = uv_info.max_sec_stor_addr;
+}
 #endif

From: Vasily Gorbik <[email protected]>

Before being able to host protected virtual machines, donate some of the memory to the ultravisor. Besides that the ultravisor might impose addressing limitations for memory used to back protected VM storage. Treat that limit as protected virtualization host's virtual memory limit.

Signed-off-by Vasily Gorbik <[email protected]>

---
 arch/s390/include/asm/uv.h | 16 ++++++++++++
 arch/s390/kernel/setup.c   |  3 +++
 arch/s390/kernel/uv.c      | 53 ++++++++++++++++++++++++++++++++++++++
 3 files changed, 72 insertions(+)

arch/s390/kernel/uv.c +6 -0

index 35ce89695509..f7778493e829 100644
@@ -45,4 +45,57 @@ static int __init prot_virt_setup(char *val)
 	return rc;
 }
 early_param("prot_virt", prot_virt_setup);
+
+static int __init uv_init(unsigned long stor_base, unsigned long stor_len)
+{
+	struct uv_cb_init uvcb = {
+		.header.cmd = UVC_CMD_INIT_UV,
+		.header.len = sizeof(uvcb),

arch/s390/kernel/uv.c +20 -0

+	}
+	return 0;
+}
+
+void __init setup_uv(void)
+{
+	unsigned long uv_stor_base;
+
+	if (!prot_virt_host)
+		return;
+
+	uv_stor_base = (unsigned long)memblock_alloc_try_nid(
+		uv_info.uv_base_stor_len, SZ_1M, SZ_2G,
+		MEMBLOCK_ALLOC_ACCESSIBLE, NUMA_NO_NODE);
+	if (!uv_stor_base) {
+		pr_info("Failed to reserve %lu bytes for ultravisor base storage\n",
+			uv_info.uv_base_stor_len);
+		goto fail;
+	}
+

arch/s390/kernel/uv.c +7 -0

+}
+
+void adjust_to_uv_max(unsigned long *vmax)
+{
+	if (prot_virt_host && *vmax > uv_info.max_sec_stor_addr)
+		*vmax = uv_info.max_sec_stor_addr;
+}
 #endif

Quoted 13 lines · On Thu, 24 Oct 2019 07:40:25 -0400 Janosch Frank <[email protected]> wrote:

From: Vasily Gorbik <[email protected]>

Before being able to host protected virtual machines, donate some of the memory to the ultravisor. Besides that the ultravisor might impose addressing limitations for memory used to back protected VM storage. Treat that limit as protected virtualization host's virtual memory limit.

Signed-off-by Vasily Gorbik <[email protected]>

---
 arch/s390/include/asm/uv.h | 16 ++++++++++++
 arch/s390/kernel/setup.c   |  3 +++
 arch/s390/kernel/uv.c      | 53 ++++++++++++++++++++++++++++++++++++++
 3 files changed, 72 insertions(+)

(...)

Quoted 19 lines

arch/s390/kernel/uv.c +11 -0

index 35ce89695509..f7778493e829 100644
@@ -45,4 +45,57 @@ static int __init prot_virt_setup(char *val)
 	return rc;
 }
 early_param("prot_virt", prot_virt_setup);
+
+static int __init uv_init(unsigned long stor_base, unsigned long stor_len)
+{
+	struct uv_cb_init uvcb = {
+		.header.cmd = UVC_CMD_INIT_UV,
+		.header.len = sizeof(uvcb),
+		.stor_origin = stor_base,
+		.stor_len = stor_len,
+	};
+	int cc;
+

arch/s390/kernel/uv.c static int __init prot_virt_setup(char *val) · line 63

Quoted 5 lines

arch/s390/kernel/uv.c +5 -0

+	cc = uv_call(0, (uint64_t)&uvcb);
+	if (cc || uvcb.header.rc != UVC_RC_EXECUTED) {
+		pr_err("Ultravisor init failed with cc: %d rc: 0x%hx\n", cc,
+		       uvcb.header.rc);
+		return -1;

Is there any reasonable case where that call might fail if we have the facility installed? Bad stor_base, maybe?

Quoted 28 lines

arch/s390/kernel/uv.c +27 -0

+	}
+	return 0;
+}
+
+void __init setup_uv(void)
+{
+	unsigned long uv_stor_base;
+
+	if (!prot_virt_host)
+		return;
+
+	uv_stor_base = (unsigned long)memblock_alloc_try_nid(
+		uv_info.uv_base_stor_len, SZ_1M, SZ_2G,
+		MEMBLOCK_ALLOC_ACCESSIBLE, NUMA_NO_NODE);
+	if (!uv_stor_base) {
+		pr_info("Failed to reserve %lu bytes for ultravisor base storage\n",
+			uv_info.uv_base_stor_len);
+		goto fail;
+	}
+
+	if (uv_init(uv_stor_base, uv_info.uv_base_stor_len)) {
+		memblock_free(uv_stor_base, uv_info.uv_base_stor_len);
+		goto fail;
+	}
+
+	pr_info("Reserving %luMB as ultravisor base storage\n",
+		uv_info.uv_base_stor_len >> 20);

arch/s390/kernel/uv.c static int __init prot_virt_setup(char *val) · line 93

Quoted 3 lines

arch/s390/kernel/uv.c +3 -0

+	return;
+fail:
+	prot_virt_host = 0;

So, what happens if the user requested protected virtualization and any of the above failed? We turn off host support, so any attempt to start a protected virtualization guest on that host will fail (hopefully with a meaningful error), I guess.

Is there any use case where we'd want to make failure to set this up fatal?

Quoted 9 lines

arch/s390/kernel/uv.c +7 -0

+}
+
+void adjust_to_uv_max(unsigned long *vmax)
+{
+	if (prot_virt_host && *vmax > uv_info.max_sec_stor_addr)
+		*vmax = uv_info.max_sec_stor_addr;
+}
 #endif

Let's add a KVM interface to create and destroy protected VMs.

arch/s390/kvm/kvm-s390.c +141 -1

+		mutex_lock(&kvm->lock);
+		kvm_s390_vcpu_block_all(kvm);
+		r = kvm_s390_pv_destroy_vm(kvm);
+		if (!r)
+			kvm_s390_pv_dealloc_vm(kvm);
+		kvm_s390_vcpu_unblock_all(kvm);
+		mutex_unlock(&kvm->lock);
+		break;
+	}
+	case KVM_PV_VM_SET_SEC_PARMS: {
+		struct kvm_s390_pv_sec_parm parms = {};
+		void *hdr;
+
+		r = -EFAULT;
+		if (copy_from_user(&parms, argp, sizeof(parms)))
+			break;
+
+		/* Currently restricted to 8KB */
+		r = -EINVAL;
+		if (parms.length > PAGE_SIZE * 2)
+			break;
+
+		r = -ENOMEM;
+		hdr = vmalloc(parms.length);
+		if (!hdr)
+			break;
+
+		r = -EFAULT;
+		if (!copy_from_user(hdr, (void __user *)parms.origin,
+				   parms.length))
+			r = kvm_s390_pv_set_sec_parms(kvm, hdr, parms.length);
+
+		vfree(hdr);
+		break;
+	}
+	case KVM_PV_VM_UNPACK: {
+		struct kvm_s390_pv_unp unp = {};
+
+		r = -EFAULT;
+		if (copy_from_user(&unp, argp, sizeof(unp)))
+			break;
+
+		r = kvm_s390_pv_unpack(kvm, unp.addr, unp.size, unp.tweak);
+		break;
+	}
+	case KVM_PV_VM_VERIFY: {
+		u32 ret;
+
+		r = -EINVAL;
+		if (!kvm_s390_pv_is_protected(kvm))
+			break;
+
+		r = uv_cmd_nodata(kvm_s390_pv_handle(kvm),
+				  UVC_CMD_VERIFY_IMG,
+				  &ret);
+		VM_EVENT(kvm, 3, "PROTVIRT VERIFY: rc %x rrc %x",
+			 ret >> 16, ret & 0x0000ffff);
+		break;
+	}
+	default:
+		return -ENOTTY;
+	}
+	return r;
+}
+#endif
+
  long kvm_arch_vm_ioctl(struct file *filp,
  		       unsigned int ioctl, unsigned long arg)
  {

@@ -2254,6 +2351,22 @@ long kvm_arch_vm_ioctl(struct file *filp,
  		mutex_unlock(&kvm->slots_lock);
  		break;
  	}
+#ifdef CONFIG_KVM_S390_PROTECTED_VIRTUALIZATION_HOST
+	case KVM_S390_PV_COMMAND: {
+		struct kvm_pv_cmd args;
+
+		r = -EINVAL;
+		if (!is_prot_virt_host())
+			break;
+
+		r = -EFAULT;
+		if (copy_from_user(&args, argp, sizeof(args)))
+			break;
+
+		r = kvm_s390_handle_pv(kvm, &args);
+		break;
+	}
+#endif
  	default:
  		r = -ENOTTY;
  	}

@@ -2529,6 +2642,9 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
 
  	if (vcpu->kvm->arch.use_cmma)
  		kvm_s390_vcpu_unsetup_cmma(vcpu);
+	if (IS_ENABLED(CONFIG_KVM_S390_PROTECTED_VIRTUALIZATION_HOST) &&
+	    kvm_s390_pv_handle_cpu(vcpu))
+		kvm_s390_pv_destroy_cpu(vcpu);
  	free_page((unsigned long)(vcpu->arch.sie_block));
 
  	kvm_vcpu_uninit(vcpu);

@@ -2555,8 +2671,13 @@ void kvm_arch_destroy_vm(struct kvm *kvm)
  {
  	kvm_free_vcpus(kvm);
  	sca_dispose(kvm);
-	debug_unregister(kvm->arch.dbf);
  	kvm_s390_gisa_destroy(kvm);
+	if (IS_ENABLED(CONFIG_KVM_S390_PROTECTED_VIRTUALIZATION_HOST) &&
+	    kvm_s390_pv_is_protected(kvm)) {
+		kvm_s390_pv_destroy_vm(kvm);
+		kvm_s390_pv_dealloc_vm(kvm);
+	}
+	debug_unregister(kvm->arch.dbf);
  	free_page((unsigned long)kvm->arch.sie_page2);
  	if (!kvm_is_ucontrol(kvm))
  		gmap_remove(kvm->arch.gmap);

@@ -2652,6 +2773,9 @@ static int sca_switch_to_extended(struct kvm *kvm)
  	unsigned int vcpu_idx;
  	u32 scaol, scaoh;
 
+	if (kvm->arch.use_esca)
+		return 0;
+
  	new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
  	if (!new_sca)
  		return -ENOMEM;

@@ -3073,6 +3197,15 @@ struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
  	rc = kvm_vcpu_init(vcpu, kvm, id);
  	if (rc)
  		goto out_free_sie_block;
+
+	if (kvm_s390_pv_is_protected(kvm)) {
+		rc = kvm_s390_pv_create_cpu(vcpu);
+		if (rc) {
+			kvm_vcpu_uninit(vcpu);
+			goto out_free_sie_block;
+		}
+	}
+
  	VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
  		 vcpu->arch.sie_block);
  	trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);

@@ -4338,6 +4471,28 @@ long kvm_arch_vcpu_async_ioctl(struct file *filp,
  	return -ENOIOCTLCMD;
  }
 
+#ifdef CONFIG_KVM_S390_PROTECTED_VIRTUALIZATION_HOST
+static int kvm_s390_handle_pv_vcpu(struct kvm_vcpu *vcpu,
+				   struct kvm_pv_cmd *cmd)
+{
+	int r = 0;
+
+	switch (cmd->cmd) {
+	case KVM_PV_VCPU_CREATE: {
+		r = kvm_s390_pv_create_cpu(vcpu);
+		break;
+	}
+	case KVM_PV_VCPU_DESTROY: {
+		r = kvm_s390_pv_destroy_cpu(vcpu);
+		break;
+	}
+	default:
+		r = -ENOTTY;
+	}
+	return r;
+}
+#endif
+
  long kvm_arch_vcpu_ioctl(struct file *filp,
  			 unsigned int ioctl, unsigned long arg)
  {

@@ -4470,6 +4625,22 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
  					   irq_state.len);
  		break;
  	}
+#ifdef CONFIG_KVM_S390_PROTECTED_VIRTUALIZATION_HOST
+	case KVM_S390_PV_COMMAND_VCPU: {
+		struct kvm_pv_cmd args;
+
+		r = -EINVAL;
+		if (!is_prot_virt_host())
+			break;
+
+		r = -EFAULT;
+		if (copy_from_user(&args, argp, sizeof(args)))
+			break;
+
+		r = kvm_s390_handle_pv_vcpu(vcpu, &args);
+		break;
+	}
+#endif
  	default:
  		r = -ENOTTY;
  	}

arch/s390/kvm/kvm-s390.h +47 -0

index 6d9448dbd052..0d61dcc51f0e 100644
@@ -196,6 +196,53 @@ static inline int kvm_s390_user_cpu_state_ctrl(struct kvm *kvm)
  	return kvm->arch.user_cpu_state_ctrl != 0;
  }
 
+#ifdef CONFIG_KVM_S390_PROTECTED_VIRTUALIZATION_HOST
+/* implemented in pv.c */
+void kvm_s390_pv_unpin(struct kvm *kvm);
+void kvm_s390_pv_dealloc_vm(struct kvm *kvm);
+int kvm_s390_pv_alloc_vm(struct kvm *kvm);
+int kvm_s390_pv_create_vm(struct kvm *kvm);
+int kvm_s390_pv_create_cpu(struct kvm_vcpu *vcpu);
+int kvm_s390_pv_destroy_vm(struct kvm *kvm);
+int kvm_s390_pv_destroy_cpu(struct kvm_vcpu *vcpu);
+int kvm_s390_pv_set_sec_parms(struct kvm *kvm, void *hdr, u64 length);
+int kvm_s390_pv_unpack(struct kvm *kvm, unsigned long addr, unsigned long size,
+		       unsigned long tweak);
+int kvm_s390_pv_verify(struct kvm *kvm);
+
+static inline bool kvm_s390_pv_is_protected(struct kvm *kvm)
+{
+	return !!kvm->arch.pv.handle;
+}
+
+static inline u64 kvm_s390_pv_handle(struct kvm *kvm)
+{
+	return kvm->arch.pv.handle;
+}
+
+static inline u64 kvm_s390_pv_handle_cpu(struct kvm_vcpu *vcpu)
+{
+	return vcpu->arch.pv.handle;
+}
+#else
+static inline void kvm_s390_pv_unpin(struct kvm *kvm) {}
+static inline void kvm_s390_pv_dealloc_vm(struct kvm *kvm) {}
+static inline int kvm_s390_pv_alloc_vm(struct kvm *kvm) { return 0; }
+static inline int kvm_s390_pv_create_vm(struct kvm *kvm) { return 0; }
+static inline int kvm_s390_pv_create_cpu(struct kvm_vcpu *vcpu) { return 0; }
+static inline int kvm_s390_pv_destroy_vm(struct kvm *kvm) { return 0; }
+static inline int kvm_s390_pv_destroy_cpu(struct kvm_vcpu *vcpu) { return 0; }
+static inline int kvm_s390_pv_set_sec_parms(struct kvm *kvm,
+					    u64 origin, u64 length) { return 0; }
+static inline int kvm_s390_pv_unpack(struct kvm *kvm, unsigned long addr,
+				     unsigned long size,  unsigned long tweak)
+{ return 0; }
+static inline int kvm_s390_pv_verify(struct kvm *kvm) { return 0; }
+static inline bool kvm_s390_pv_is_protected(struct kvm *kvm) { return 0; }
+static inline u64 kvm_s390_pv_handle(struct kvm *kvm) { return 0; }
+static inline u64 kvm_s390_pv_handle_cpu(struct kvm_vcpu *vcpu) { return 0; }
+#endif
+
  /* implemented in interrupt.c */
  int kvm_s390_handle_wait(struct kvm_vcpu *vcpu);
  void kvm_s390_vcpu_wakeup(struct kvm_vcpu *vcpu);

arch/s390/kvm/pv.c new file +38 -0

new file mode 100644
index 000000000000..94cf16f40f25
@@ -0,0 +1,237 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Hosting Secure Execution virtual machines
+ *
+ * Copyright IBM Corp. 2019
+ *    Author(s): Janosch Frank <[email protected]>
+ */
+#include <linux/kvm.h>
+#include <linux/kvm_host.h>
+#include <linux/pagemap.h>
+#include <asm/pgalloc.h>
+#include <asm/gmap.h>
+#include <asm/uv.h>
+#include <asm/gmap.h>
+#include <asm/mman.h>
+#include "kvm-s390.h"
+
+void kvm_s390_pv_dealloc_vm(struct kvm *kvm)
+{
+	vfree(kvm->arch.pv.stor_var);
+	free_pages(kvm->arch.pv.stor_base,
+		   get_order(uv_info.guest_base_stor_len));
+	memset(&kvm->arch.pv, 0, sizeof(kvm->arch.pv));
+}
+
+int kvm_s390_pv_alloc_vm(struct kvm *kvm)
+{
+	unsigned long base = uv_info.guest_base_stor_len;
+	unsigned long virt = uv_info.guest_virt_var_stor_len;
+	unsigned long npages = 0, vlen = 0;
+	struct kvm_memslots *slots;
+	struct kvm_memory_slot *memslot;
+
+	kvm->arch.pv.stor_var = NULL;
+	kvm->arch.pv.stor_base = __get_free_pages(GFP_KERNEL, get_order(base));
+	if (!kvm->arch.pv.stor_base)
+		return -ENOMEM;
+

arch/s390/kvm/pv.c new file +183 -0

+	vlen += uv_info.guest_virt_base_stor_len;
+	kvm->arch.pv.stor_var = vzalloc(vlen);
+	if (!kvm->arch.pv.stor_var) {
+		kvm_s390_pv_dealloc_vm(kvm);
+		return -ENOMEM;
+	}
+	return 0;
+}
+
+int kvm_s390_pv_destroy_vm(struct kvm *kvm)
+{
+	int rc;
+	u32 ret;
+
+	rc = uv_cmd_nodata(kvm_s390_pv_handle(kvm),
+			   UVC_CMD_DESTROY_SEC_CONF, &ret);
+	VM_EVENT(kvm, 3, "PROTVIRT DESTROY VM: rc %x rrc %x",
+		 ret >> 16, ret & 0x0000ffff);
+	return rc;
+}
+
+int kvm_s390_pv_destroy_cpu(struct kvm_vcpu *vcpu)
+{
+	int rc = 0;
+	u32 ret;
+
+	if (kvm_s390_pv_handle_cpu(vcpu)) {
+		rc = uv_cmd_nodata(kvm_s390_pv_handle_cpu(vcpu),
+				   UVC_CMD_DESTROY_SEC_CPU,
+				   &ret);
+
+		VCPU_EVENT(vcpu, 3, "PROTVIRT DESTROY VCPU: cpu %d rc %x rrc %x",
+			   vcpu->vcpu_id, ret >> 16, ret & 0x0000ffff);
+	}
+
+	free_pages(vcpu->arch.pv.stor_base,
+		   get_order(uv_info.guest_cpu_stor_len));
+	/* Clear cpu and vm handle */
+	memset(&vcpu->arch.sie_block->reserved10, 0,
+	       sizeof(vcpu->arch.sie_block->reserved10));
+	memset(&vcpu->arch.pv, 0, sizeof(vcpu->arch.pv));
+	vcpu->arch.sie_block->sdf = 0;
+	return rc;
+}
+
+int kvm_s390_pv_create_cpu(struct kvm_vcpu *vcpu)
+{
+	int rc;
+	struct uv_cb_csc uvcb = {
+		.header.cmd = UVC_CMD_CREATE_SEC_CPU,
+		.header.len = sizeof(uvcb),
+	};
+
+	/* EEXIST and ENOENT? */
+	if (kvm_s390_pv_handle_cpu(vcpu))
+		return -EINVAL;
+
+	vcpu->arch.pv.stor_base = __get_free_pages(GFP_KERNEL,
+						   get_order(uv_info.guest_cpu_stor_len));
+	if (!vcpu->arch.pv.stor_base)
+		return -ENOMEM;
+
+	/* Input */
+	uvcb.guest_handle = kvm_s390_pv_handle(vcpu->kvm);
+	uvcb.num = vcpu->arch.sie_block->icpua;
+	uvcb.state_origin = (u64)vcpu->arch.sie_block;
+	uvcb.stor_origin = (u64)vcpu->arch.pv.stor_base;
+
+	rc = uv_call(0, (u64)&uvcb);
+	VCPU_EVENT(vcpu, 3, "PROTVIRT CREATE VCPU: cpu %d handle %llx rc %x rrc %x",
+		   vcpu->vcpu_id, uvcb.cpu_handle, uvcb.header.rc,
+		   uvcb.header.rrc);
+
+	/* Output */
+	vcpu->arch.pv.handle = uvcb.cpu_handle;
+	vcpu->arch.sie_block->pv_handle_cpu = uvcb.cpu_handle;
+	vcpu->arch.sie_block->pv_handle_config = kvm_s390_pv_handle(vcpu->kvm);
+	vcpu->arch.sie_block->sdf = 2;
+	if (!rc)
+		return 0;
+
+	kvm_s390_pv_destroy_cpu(vcpu);
+	return -EINVAL;
+}
+
+int kvm_s390_pv_create_vm(struct kvm *kvm)
+{
+	int rc;
+
+	struct uv_cb_cgc uvcb = {
+		.header.cmd = UVC_CMD_CREATE_SEC_CONF,
+		.header.len = sizeof(uvcb)
+	};
+
+	if (kvm_s390_pv_handle(kvm))
+		return -EINVAL;
+
+	/* Inputs */
+	uvcb.guest_stor_origin = 0; /* MSO is 0 for KVM */
+	uvcb.guest_stor_len = kvm->arch.pv.guest_len;
+	uvcb.guest_asce = kvm->arch.gmap->asce;
+	uvcb.conf_base_stor_origin = (u64)kvm->arch.pv.stor_base;
+	uvcb.conf_var_stor_origin = (u64)kvm->arch.pv.stor_var;
+
+	rc = uv_call(0, (u64)&uvcb);
+	VM_EVENT(kvm, 3, "PROTVIRT CREATE VM: handle %llx len %llx rc %x rrc %x",
+		 uvcb.guest_handle, uvcb.guest_stor_len, uvcb.header.rc,
+		 uvcb.header.rrc);
+
+	/* Outputs */
+	kvm->arch.pv.handle = uvcb.guest_handle;
+
+	if (rc && (uvcb.header.rc & 0x8000)) {
+		kvm_s390_pv_destroy_vm(kvm);
+		kvm_s390_pv_dealloc_vm(kvm);
+		return -EINVAL;
+	}
+	return rc;
+}
+
+int kvm_s390_pv_set_sec_parms(struct kvm *kvm,
+			      void *hdr, u64 length)
+{
+	int rc;
+	struct uv_cb_ssc uvcb = {
+		.header.cmd = UVC_CMD_SET_SEC_CONF_PARAMS,
+		.header.len = sizeof(uvcb),
+		.sec_header_origin = (u64)hdr,
+		.sec_header_len = length,
+		.guest_handle = kvm_s390_pv_handle(kvm),
+	};
+
+	if (!kvm_s390_pv_handle(kvm))
+		return -EINVAL;
+
+	rc = uv_call(0, (u64)&uvcb);
+	VM_EVENT(kvm, 3, "PROTVIRT VM SET PARMS: rc %x rrc %x",
+		 uvcb.header.rc, uvcb.header.rrc);
+	if (rc)
+		return -EINVAL;
+	return 0;
+}
+
+int kvm_s390_pv_unpack(struct kvm *kvm, unsigned long addr, unsigned long size,
+		       unsigned long tweak)
+{
+	int i, rc = 0;
+	struct uv_cb_unp uvcb = {
+		.header.cmd = UVC_CMD_UNPACK_IMG,
+		.header.len = sizeof(uvcb),
+		.guest_handle = kvm_s390_pv_handle(kvm),
+		.tweak[0] = tweak
+	};
+
+	if (addr & ~PAGE_MASK || size & ~PAGE_MASK)
+		return -EINVAL;
+
+
+	VM_EVENT(kvm, 3, "PROTVIRT VM UNPACK: start addr %lx size %lx",
+		 addr, size);
+	for (i = 0; i < size / PAGE_SIZE; i++) {
+		uvcb.gaddr = addr + i * PAGE_SIZE;
+		uvcb.tweak[1] = i * PAGE_SIZE;
+retry:
+		rc = uv_call(0, (u64)&uvcb);
+		if (!rc)
+			continue;
+		/* If not yet mapped fault and retry */
+		if (uvcb.header.rc == 0x10a) {
+			rc = gmap_fault(kvm->arch.gmap, uvcb.gaddr,
+					FAULT_FLAG_WRITE);
+			if (rc)
+				return rc;
+			goto retry;
+		}
+		VM_EVENT(kvm, 3, "PROTVIRT VM UNPACK: failed addr %llx rc %x rrc %x",
+			 uvcb.gaddr, uvcb.header.rc, uvcb.header.rrc);
+		break;
+	}
+	VM_EVENT(kvm, 3, "PROTVIRT VM UNPACK: finished with rc %x rrc %x",
+		 uvcb.header.rc, uvcb.header.rrc);
+	return rc;
+}

include/uapi/linux/kvm.h +26 -0

index 52641d8ca9e8..bb37d5710c89 100644
@@ -1000,6 +1000,7 @@ struct kvm_ppc_resize_hpt {
  #define KVM_CAP_PMU_EVENT_FILTER 173
  #define KVM_CAP_ARM_IRQ_LINE_LAYOUT_2 174
  #define KVM_CAP_HYPERV_DIRECT_TLBFLUSH 175
+#define KVM_CAP_S390_PROTECTED 180
 
  #ifdef KVM_CAP_IRQ_ROUTING
 

@@ -1461,6 +1462,38 @@ struct kvm_enc_region {
  /* Available with KVM_CAP_ARM_SVE */
  #define KVM_ARM_VCPU_FINALIZE	  _IOW(KVMIO,  0xc2, int)
 
+struct kvm_s390_pv_sec_parm {
+	__u64	origin;
+	__u64	length;
+};
+
+struct kvm_s390_pv_unp {
+	__u64 addr;
+	__u64 size;
+	__u64 tweak;
+};
+
+enum pv_cmd_id {
+	KVM_PV_VM_CREATE,
+	KVM_PV_VM_DESTROY,
+	KVM_PV_VM_SET_SEC_PARMS,
+	KVM_PV_VM_UNPACK,
+	KVM_PV_VM_VERIFY,
+	KVM_PV_VCPU_CREATE,
+	KVM_PV_VCPU_DESTROY,
+};
+
+struct kvm_pv_cmd {
+	__u32	cmd;
+	__u16	rc;
+	__u16	rrc;

Quoted 1 line · Janosch Frank

Let's add a KVM interface to create and destroy protected VMs.

More details please.

[...]

arch/s390/kvm/kvm-s390.c static int kvm_s390_set_cmma_bits(struct kvm *kvm, · line 2177

Quoted 12 lines

arch/s390/kvm/kvm-s390.c +11 -0

+#ifdef CONFIG_KVM_S390_PROTECTED_VIRTUALIZATION_HOST
+static int kvm_s390_handle_pv(struct kvm *kvm, struct kvm_pv_cmd *cmd)
+{
+	int r = 0;
+	void __user *argp = (void __user *)cmd->data;
+
+	switch (cmd->cmd) {
+	case KVM_PV_VM_CREATE: {
+		r = kvm_s390_pv_alloc_vm(kvm);
+		if (r)
+			break;

So ... I can create multiple VMs?

Especially, I can call KVM_PV_VM_CREATE two times, setting "kvm->arch.pv.stor_var = NULL and leaking memory" on the second call. Not sure if that's desirable.

Shouldn't this be something like "KVM_PV_VM_INIT" and then make sure it can only be called once?

arch/s390/kvm/kvm-s390.c static int kvm_s390_set_cmma_bits(struct kvm *kvm, · line 2190

Quoted 13 lines

arch/s390/kvm/kvm-s390.c +13 -0

+
+		mutex_lock(&kvm->lock);
+		kvm_s390_vcpu_block_all(kvm);
+		/* FMT 4 SIE needs esca */
+		r = sca_switch_to_extended(kvm);
+		if (!r)
+			r = kvm_s390_pv_create_vm(kvm);
+		kvm_s390_vcpu_unblock_all(kvm);
+		mutex_unlock(&kvm->lock);
+		break;
+	}
+	case KVM_PV_VM_DESTROY: {
+		/* All VCPUs have to be destroyed before this call. */

Then please verify that? "KVM_PV_VM_DEINIT"

Also, who guarantees that user space calls this at all? Why is that needed? (IOW, when does user space call this?)

Quoted 297 lines

arch/s390/kvm/kvm-s390.c +141 -1

+		mutex_lock(&kvm->lock);
+		kvm_s390_vcpu_block_all(kvm);
+		r = kvm_s390_pv_destroy_vm(kvm);
+		if (!r)
+			kvm_s390_pv_dealloc_vm(kvm);
+		kvm_s390_vcpu_unblock_all(kvm);
+		mutex_unlock(&kvm->lock);
+		break;
+	}
+	case KVM_PV_VM_SET_SEC_PARMS: {
+		struct kvm_s390_pv_sec_parm parms = {};
+		void *hdr;
+
+		r = -EFAULT;
+		if (copy_from_user(&parms, argp, sizeof(parms)))
+			break;
+
+		/* Currently restricted to 8KB */
+		r = -EINVAL;
+		if (parms.length > PAGE_SIZE * 2)
+			break;
+
+		r = -ENOMEM;
+		hdr = vmalloc(parms.length);
+		if (!hdr)
+			break;
+
+		r = -EFAULT;
+		if (!copy_from_user(hdr, (void __user *)parms.origin,
+				   parms.length))
+			r = kvm_s390_pv_set_sec_parms(kvm, hdr, parms.length);
+
+		vfree(hdr);
+		break;
+	}
+	case KVM_PV_VM_UNPACK: {
+		struct kvm_s390_pv_unp unp = {};
+
+		r = -EFAULT;
+		if (copy_from_user(&unp, argp, sizeof(unp)))
+			break;
+
+		r = kvm_s390_pv_unpack(kvm, unp.addr, unp.size, unp.tweak);
+		break;
+	}
+	case KVM_PV_VM_VERIFY: {
+		u32 ret;
+
+		r = -EINVAL;
+		if (!kvm_s390_pv_is_protected(kvm))
+			break;
+
+		r = uv_cmd_nodata(kvm_s390_pv_handle(kvm),
+				  UVC_CMD_VERIFY_IMG,
+				  &ret);
+		VM_EVENT(kvm, 3, "PROTVIRT VERIFY: rc %x rrc %x",
+			 ret >> 16, ret & 0x0000ffff);
+		break;
+	}
+	default:
+		return -ENOTTY;
+	}
+	return r;
+}
+#endif
+
   long kvm_arch_vm_ioctl(struct file *filp,
   		       unsigned int ioctl, unsigned long arg)
   {

@@ -2254,6 +2351,22 @@ long kvm_arch_vm_ioctl(struct file *filp,
   		mutex_unlock(&kvm->slots_lock);
   		break;
   	}
+#ifdef CONFIG_KVM_S390_PROTECTED_VIRTUALIZATION_HOST
+	case KVM_S390_PV_COMMAND: {
+		struct kvm_pv_cmd args;
+
+		r = -EINVAL;
+		if (!is_prot_virt_host())
+			break;
+
+		r = -EFAULT;
+		if (copy_from_user(&args, argp, sizeof(args)))
+			break;
+
+		r = kvm_s390_handle_pv(kvm, &args);
+		break;
+	}
+#endif
   	default:
   		r = -ENOTTY;
   	}

@@ -2529,6 +2642,9 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
 
   	if (vcpu->kvm->arch.use_cmma)
   		kvm_s390_vcpu_unsetup_cmma(vcpu);
+	if (IS_ENABLED(CONFIG_KVM_S390_PROTECTED_VIRTUALIZATION_HOST) &&
+	    kvm_s390_pv_handle_cpu(vcpu))
+		kvm_s390_pv_destroy_cpu(vcpu);
   	free_page((unsigned long)(vcpu->arch.sie_block));
 
   	kvm_vcpu_uninit(vcpu);

@@ -2555,8 +2671,13 @@ void kvm_arch_destroy_vm(struct kvm *kvm)
   {
   	kvm_free_vcpus(kvm);
   	sca_dispose(kvm);
-	debug_unregister(kvm->arch.dbf);
   	kvm_s390_gisa_destroy(kvm);
+	if (IS_ENABLED(CONFIG_KVM_S390_PROTECTED_VIRTUALIZATION_HOST) &&
+	    kvm_s390_pv_is_protected(kvm)) {
+		kvm_s390_pv_destroy_vm(kvm);
+		kvm_s390_pv_dealloc_vm(kvm);
+	}
+	debug_unregister(kvm->arch.dbf);
   	free_page((unsigned long)kvm->arch.sie_page2);
   	if (!kvm_is_ucontrol(kvm))
   		gmap_remove(kvm->arch.gmap);

@@ -2652,6 +2773,9 @@ static int sca_switch_to_extended(struct kvm *kvm)
   	unsigned int vcpu_idx;
   	u32 scaol, scaoh;
 
+	if (kvm->arch.use_esca)
+		return 0;
+
   	new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
   	if (!new_sca)
   		return -ENOMEM;

@@ -3073,6 +3197,15 @@ struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
   	rc = kvm_vcpu_init(vcpu, kvm, id);
   	if (rc)
   		goto out_free_sie_block;
+
+	if (kvm_s390_pv_is_protected(kvm)) {
+		rc = kvm_s390_pv_create_cpu(vcpu);
+		if (rc) {
+			kvm_vcpu_uninit(vcpu);
+			goto out_free_sie_block;
+		}
+	}
+
   	VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
   		 vcpu->arch.sie_block);
   	trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);

@@ -4338,6 +4471,28 @@ long kvm_arch_vcpu_async_ioctl(struct file *filp,
   	return -ENOIOCTLCMD;
   }
 
+#ifdef CONFIG_KVM_S390_PROTECTED_VIRTUALIZATION_HOST
+static int kvm_s390_handle_pv_vcpu(struct kvm_vcpu *vcpu,
+				   struct kvm_pv_cmd *cmd)
+{
+	int r = 0;
+
+	switch (cmd->cmd) {
+	case KVM_PV_VCPU_CREATE: {
+		r = kvm_s390_pv_create_cpu(vcpu);
+		break;
+	}
+	case KVM_PV_VCPU_DESTROY: {
+		r = kvm_s390_pv_destroy_cpu(vcpu);
+		break;
+	}
+	default:
+		r = -ENOTTY;
+	}
+	return r;
+}
+#endif
+
   long kvm_arch_vcpu_ioctl(struct file *filp,
   			 unsigned int ioctl, unsigned long arg)
   {

@@ -4470,6 +4625,22 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
   					   irq_state.len);
   		break;
   	}
+#ifdef CONFIG_KVM_S390_PROTECTED_VIRTUALIZATION_HOST
+	case KVM_S390_PV_COMMAND_VCPU: {
+		struct kvm_pv_cmd args;
+
+		r = -EINVAL;
+		if (!is_prot_virt_host())
+			break;
+
+		r = -EFAULT;
+		if (copy_from_user(&args, argp, sizeof(args)))
+			break;
+
+		r = kvm_s390_handle_pv_vcpu(vcpu, &args);
+		break;
+	}
+#endif
   	default:
   		r = -ENOTTY;
   	}

arch/s390/kvm/kvm-s390.h +47 -0

index 6d9448dbd052..0d61dcc51f0e 100644
@@ -196,6 +196,53 @@ static inline int kvm_s390_user_cpu_state_ctrl(struct kvm *kvm)
   	return kvm->arch.user_cpu_state_ctrl != 0;
   }
 
+#ifdef CONFIG_KVM_S390_PROTECTED_VIRTUALIZATION_HOST
+/* implemented in pv.c */
+void kvm_s390_pv_unpin(struct kvm *kvm);
+void kvm_s390_pv_dealloc_vm(struct kvm *kvm);
+int kvm_s390_pv_alloc_vm(struct kvm *kvm);
+int kvm_s390_pv_create_vm(struct kvm *kvm);
+int kvm_s390_pv_create_cpu(struct kvm_vcpu *vcpu);
+int kvm_s390_pv_destroy_vm(struct kvm *kvm);
+int kvm_s390_pv_destroy_cpu(struct kvm_vcpu *vcpu);
+int kvm_s390_pv_set_sec_parms(struct kvm *kvm, void *hdr, u64 length);
+int kvm_s390_pv_unpack(struct kvm *kvm, unsigned long addr, unsigned long size,
+		       unsigned long tweak);
+int kvm_s390_pv_verify(struct kvm *kvm);
+
+static inline bool kvm_s390_pv_is_protected(struct kvm *kvm)
+{
+	return !!kvm->arch.pv.handle;
+}
+
+static inline u64 kvm_s390_pv_handle(struct kvm *kvm)
+{
+	return kvm->arch.pv.handle;
+}
+
+static inline u64 kvm_s390_pv_handle_cpu(struct kvm_vcpu *vcpu)
+{
+	return vcpu->arch.pv.handle;
+}
+#else
+static inline void kvm_s390_pv_unpin(struct kvm *kvm) {}
+static inline void kvm_s390_pv_dealloc_vm(struct kvm *kvm) {}
+static inline int kvm_s390_pv_alloc_vm(struct kvm *kvm) { return 0; }
+static inline int kvm_s390_pv_create_vm(struct kvm *kvm) { return 0; }
+static inline int kvm_s390_pv_create_cpu(struct kvm_vcpu *vcpu) { return 0; }
+static inline int kvm_s390_pv_destroy_vm(struct kvm *kvm) { return 0; }
+static inline int kvm_s390_pv_destroy_cpu(struct kvm_vcpu *vcpu) { return 0; }
+static inline int kvm_s390_pv_set_sec_parms(struct kvm *kvm,
+					    u64 origin, u64 length) { return 0; }
+static inline int kvm_s390_pv_unpack(struct kvm *kvm, unsigned long addr,
+				     unsigned long size,  unsigned long tweak)
+{ return 0; }
+static inline int kvm_s390_pv_verify(struct kvm *kvm) { return 0; }
+static inline bool kvm_s390_pv_is_protected(struct kvm *kvm) { return 0; }
+static inline u64 kvm_s390_pv_handle(struct kvm *kvm) { return 0; }
+static inline u64 kvm_s390_pv_handle_cpu(struct kvm_vcpu *vcpu) { return 0; }
+#endif
+
   /* implemented in interrupt.c */
   int kvm_s390_handle_wait(struct kvm_vcpu *vcpu);
   void kvm_s390_vcpu_wakeup(struct kvm_vcpu *vcpu);

arch/s390/kvm/pv.c new file +38 -0

new file mode 100644
index 000000000000..94cf16f40f25
@@ -0,0 +1,237 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Hosting Secure Execution virtual machines
+ *
+ * Copyright IBM Corp. 2019
+ *    Author(s): Janosch Frank <[email protected]>
+ */
+#include <linux/kvm.h>
+#include <linux/kvm_host.h>
+#include <linux/pagemap.h>
+#include <asm/pgalloc.h>
+#include <asm/gmap.h>
+#include <asm/uv.h>
+#include <asm/gmap.h>
+#include <asm/mman.h>
+#include "kvm-s390.h"
+
+void kvm_s390_pv_dealloc_vm(struct kvm *kvm)
+{
+	vfree(kvm->arch.pv.stor_var);
+	free_pages(kvm->arch.pv.stor_base,
+		   get_order(uv_info.guest_base_stor_len));
+	memset(&kvm->arch.pv, 0, sizeof(kvm->arch.pv));
+}
+
+int kvm_s390_pv_alloc_vm(struct kvm *kvm)
+{
+	unsigned long base = uv_info.guest_base_stor_len;
+	unsigned long virt = uv_info.guest_virt_var_stor_len;
+	unsigned long npages = 0, vlen = 0;
+	struct kvm_memslots *slots;
+	struct kvm_memory_slot *memslot;
+
+	kvm->arch.pv.stor_var = NULL;
+	kvm->arch.pv.stor_base = __get_free_pages(GFP_KERNEL, get_order(base));
+	if (!kvm->arch.pv.stor_base)
+		return -ENOMEM;
+

arch/s390/kvm/pv.c line 48

Quoted 10 lines

arch/s390/kvm/pv.c new file +10 -0

+	/*
+	 * Calculate current guest storage for allocation of the
+	 * variable storage, which is based on the length in MB.
+	 *
+	 * Slots are sorted by GFN
+	 */
+	mutex_lock(&kvm->slots_lock);
+	slots = kvm_memslots(kvm);
+	memslot = slots->memslots;
+	npages = memslot->base_gfn + memslot->npages;

What if

a) your guest has multiple memory slots
b) you hotplug memory and add memslots later

Do you dence that, and if so, how?

arch/s390/kvm/pv.c line 54

Quoted 6 lines

arch/s390/kvm/pv.c new file +6 -0

+
+	mutex_unlock(&kvm->slots_lock);
+	kvm->arch.pv.guest_len = npages * PAGE_SIZE;
+
+	/* Allocate variable storage */
+	vlen = ALIGN(virt * ((npages * PAGE_SIZE) / HPAGE_SIZE), PAGE_SIZE);

I get the feeling that prot virt mainly consumes memory ;)

Quoted 229 lines

arch/s390/kvm/pv.c new file +183 -0

+	vlen += uv_info.guest_virt_base_stor_len;
+	kvm->arch.pv.stor_var = vzalloc(vlen);
+	if (!kvm->arch.pv.stor_var) {
+		kvm_s390_pv_dealloc_vm(kvm);
+		return -ENOMEM;
+	}
+	return 0;
+}
+
+int kvm_s390_pv_destroy_vm(struct kvm *kvm)
+{
+	int rc;
+	u32 ret;
+
+	rc = uv_cmd_nodata(kvm_s390_pv_handle(kvm),
+			   UVC_CMD_DESTROY_SEC_CONF, &ret);
+	VM_EVENT(kvm, 3, "PROTVIRT DESTROY VM: rc %x rrc %x",
+		 ret >> 16, ret & 0x0000ffff);
+	return rc;
+}
+
+int kvm_s390_pv_destroy_cpu(struct kvm_vcpu *vcpu)
+{
+	int rc = 0;
+	u32 ret;
+
+	if (kvm_s390_pv_handle_cpu(vcpu)) {
+		rc = uv_cmd_nodata(kvm_s390_pv_handle_cpu(vcpu),
+				   UVC_CMD_DESTROY_SEC_CPU,
+				   &ret);
+
+		VCPU_EVENT(vcpu, 3, "PROTVIRT DESTROY VCPU: cpu %d rc %x rrc %x",
+			   vcpu->vcpu_id, ret >> 16, ret & 0x0000ffff);
+	}
+
+	free_pages(vcpu->arch.pv.stor_base,
+		   get_order(uv_info.guest_cpu_stor_len));
+	/* Clear cpu and vm handle */
+	memset(&vcpu->arch.sie_block->reserved10, 0,
+	       sizeof(vcpu->arch.sie_block->reserved10));
+	memset(&vcpu->arch.pv, 0, sizeof(vcpu->arch.pv));
+	vcpu->arch.sie_block->sdf = 0;
+	return rc;
+}
+
+int kvm_s390_pv_create_cpu(struct kvm_vcpu *vcpu)
+{
+	int rc;
+	struct uv_cb_csc uvcb = {
+		.header.cmd = UVC_CMD_CREATE_SEC_CPU,
+		.header.len = sizeof(uvcb),
+	};
+
+	/* EEXIST and ENOENT? */
+	if (kvm_s390_pv_handle_cpu(vcpu))
+		return -EINVAL;
+
+	vcpu->arch.pv.stor_base = __get_free_pages(GFP_KERNEL,
+						   get_order(uv_info.guest_cpu_stor_len));
+	if (!vcpu->arch.pv.stor_base)
+		return -ENOMEM;
+
+	/* Input */
+	uvcb.guest_handle = kvm_s390_pv_handle(vcpu->kvm);
+	uvcb.num = vcpu->arch.sie_block->icpua;
+	uvcb.state_origin = (u64)vcpu->arch.sie_block;
+	uvcb.stor_origin = (u64)vcpu->arch.pv.stor_base;
+
+	rc = uv_call(0, (u64)&uvcb);
+	VCPU_EVENT(vcpu, 3, "PROTVIRT CREATE VCPU: cpu %d handle %llx rc %x rrc %x",
+		   vcpu->vcpu_id, uvcb.cpu_handle, uvcb.header.rc,
+		   uvcb.header.rrc);
+
+	/* Output */
+	vcpu->arch.pv.handle = uvcb.cpu_handle;
+	vcpu->arch.sie_block->pv_handle_cpu = uvcb.cpu_handle;
+	vcpu->arch.sie_block->pv_handle_config = kvm_s390_pv_handle(vcpu->kvm);
+	vcpu->arch.sie_block->sdf = 2;
+	if (!rc)
+		return 0;
+
+	kvm_s390_pv_destroy_cpu(vcpu);
+	return -EINVAL;
+}
+
+int kvm_s390_pv_create_vm(struct kvm *kvm)
+{
+	int rc;
+
+	struct uv_cb_cgc uvcb = {
+		.header.cmd = UVC_CMD_CREATE_SEC_CONF,
+		.header.len = sizeof(uvcb)
+	};
+
+	if (kvm_s390_pv_handle(kvm))
+		return -EINVAL;
+
+	/* Inputs */
+	uvcb.guest_stor_origin = 0; /* MSO is 0 for KVM */
+	uvcb.guest_stor_len = kvm->arch.pv.guest_len;
+	uvcb.guest_asce = kvm->arch.gmap->asce;
+	uvcb.conf_base_stor_origin = (u64)kvm->arch.pv.stor_base;
+	uvcb.conf_var_stor_origin = (u64)kvm->arch.pv.stor_var;
+
+	rc = uv_call(0, (u64)&uvcb);
+	VM_EVENT(kvm, 3, "PROTVIRT CREATE VM: handle %llx len %llx rc %x rrc %x",
+		 uvcb.guest_handle, uvcb.guest_stor_len, uvcb.header.rc,
+		 uvcb.header.rrc);
+
+	/* Outputs */
+	kvm->arch.pv.handle = uvcb.guest_handle;
+
+	if (rc && (uvcb.header.rc & 0x8000)) {
+		kvm_s390_pv_destroy_vm(kvm);
+		kvm_s390_pv_dealloc_vm(kvm);
+		return -EINVAL;
+	}
+	return rc;
+}
+
+int kvm_s390_pv_set_sec_parms(struct kvm *kvm,
+			      void *hdr, u64 length)
+{
+	int rc;
+	struct uv_cb_ssc uvcb = {
+		.header.cmd = UVC_CMD_SET_SEC_CONF_PARAMS,
+		.header.len = sizeof(uvcb),
+		.sec_header_origin = (u64)hdr,
+		.sec_header_len = length,
+		.guest_handle = kvm_s390_pv_handle(kvm),
+	};
+
+	if (!kvm_s390_pv_handle(kvm))
+		return -EINVAL;
+
+	rc = uv_call(0, (u64)&uvcb);
+	VM_EVENT(kvm, 3, "PROTVIRT VM SET PARMS: rc %x rrc %x",
+		 uvcb.header.rc, uvcb.header.rrc);
+	if (rc)
+		return -EINVAL;
+	return 0;
+}
+
+int kvm_s390_pv_unpack(struct kvm *kvm, unsigned long addr, unsigned long size,
+		       unsigned long tweak)
+{
+	int i, rc = 0;
+	struct uv_cb_unp uvcb = {
+		.header.cmd = UVC_CMD_UNPACK_IMG,
+		.header.len = sizeof(uvcb),
+		.guest_handle = kvm_s390_pv_handle(kvm),
+		.tweak[0] = tweak
+	};
+
+	if (addr & ~PAGE_MASK || size & ~PAGE_MASK)
+		return -EINVAL;
+
+
+	VM_EVENT(kvm, 3, "PROTVIRT VM UNPACK: start addr %lx size %lx",
+		 addr, size);
+	for (i = 0; i < size / PAGE_SIZE; i++) {
+		uvcb.gaddr = addr + i * PAGE_SIZE;
+		uvcb.tweak[1] = i * PAGE_SIZE;
+retry:
+		rc = uv_call(0, (u64)&uvcb);
+		if (!rc)
+			continue;
+		/* If not yet mapped fault and retry */
+		if (uvcb.header.rc == 0x10a) {
+			rc = gmap_fault(kvm->arch.gmap, uvcb.gaddr,
+					FAULT_FLAG_WRITE);
+			if (rc)
+				return rc;
+			goto retry;
+		}
+		VM_EVENT(kvm, 3, "PROTVIRT VM UNPACK: failed addr %llx rc %x rrc %x",
+			 uvcb.gaddr, uvcb.header.rc, uvcb.header.rrc);
+		break;
+	}
+	VM_EVENT(kvm, 3, "PROTVIRT VM UNPACK: finished with rc %x rrc %x",
+		 uvcb.header.rc, uvcb.header.rrc);
+	return rc;
+}

include/uapi/linux/kvm.h +31 -0

index 52641d8ca9e8..bb37d5710c89 100644
@@ -1000,6 +1000,7 @@ struct kvm_ppc_resize_hpt {
   #define KVM_CAP_PMU_EVENT_FILTER 173
   #define KVM_CAP_ARM_IRQ_LINE_LAYOUT_2 174
   #define KVM_CAP_HYPERV_DIRECT_TLBFLUSH 175
+#define KVM_CAP_S390_PROTECTED 180
 
   #ifdef KVM_CAP_IRQ_ROUTING
 

@@ -1461,6 +1462,38 @@ struct kvm_enc_region {
   /* Available with KVM_CAP_ARM_SVE */
   #define KVM_ARM_VCPU_FINALIZE	  _IOW(KVMIO,  0xc2, int)
 
+struct kvm_s390_pv_sec_parm {
+	__u64	origin;
+	__u64	length;
+};
+
+struct kvm_s390_pv_unp {
+	__u64 addr;
+	__u64 size;
+	__u64 tweak;
+};
+
+enum pv_cmd_id {
+	KVM_PV_VM_CREATE,
+	KVM_PV_VM_DESTROY,
+	KVM_PV_VM_SET_SEC_PARMS,
+	KVM_PV_VM_UNPACK,
+	KVM_PV_VM_VERIFY,
+	KVM_PV_VCPU_CREATE,
+	KVM_PV_VCPU_DESTROY,
+};
+
+struct kvm_pv_cmd {
+	__u32	cmd;
+	__u16	rc;
+	__u16	rrc;
+	__u64	data;
+};
+
+/* Available with KVM_CAP_S390_SE */
+#define KVM_S390_PV_COMMAND		_IOW(KVMIO, 0xc3, struct kvm_pv_cmd)