Skip to contentSkip to conversation

s390x: PV fixups

PATCH In review Janosch Frank 3 participants 11 messages Oct 21, 2022 KVM

lore
Share & tools
lore
#0 lore raw

v3 cover s390x: PV fixups

A small set of patches that clean up the PV snippet handling.

v3:
* Dropped asm snippet linker script patch for now
* Replaced memalign_pages_flags() with memalign_pages()
* PV ASCEs will now recieve DT and TL fields from the main test ASCE

v2:
* Macro uses 64bit PSW mask
* SBLK reset on PV destroy and uv_init() early return have been split off

Janosch Frank (6):
s390x: snippets: asm: Add a macro to write an exception PSW
s390x: MAKEFILE: Use $< instead of pathsubst
lib: s390x: sie: Improve validity handling and make it vm specific
lib: s390x: Use a new asce for each PV guest
lib: s390x: Enable reusability of VMs that were in PV mode
lib: s390x: sie: Properly populate SCA

 lib/s390x/asm-offsets.c                  |  2 ++
 lib/s390x/sie.c                          | 37 +++++++++++++-------
 lib/s390x/sie.h                          | 43 ++++++++++++++++++++++--
 lib/s390x/uv.c                           | 35 +++++++++++++++++--
 lib/s390x/uv.h                           |  5 ++-
 s390x/Makefile                           |  2 +-
 s390x/cpu.S                              |  6 ++++
 s390x/snippets/asm/macros.S              | 28 +++++++++++++++
 s390x/snippets/asm/snippet-pv-diag-288.S |  4 +--
 s390x/snippets/asm/snippet-pv-diag-500.S |  6 ++--
 10 files changed, 140 insertions(+), 28 deletions(-)
 create mode 100644 s390x/snippets/asm/macros.S
Signature · 1 line
2.34.1
#1 lore raw

PATCH v3 1/6 s390x: snippets: asm: Add a macro to write an exception PSW

Setting exception new PSWs is commonly needed so let's add a macro for that.

---
 s390x/snippets/asm/macros.S              | 28 ++++++++++++++++++++++++
 s390x/snippets/asm/snippet-pv-diag-288.S |  4 ++--
 s390x/snippets/asm/snippet-pv-diag-500.S |  6 ++---
 3 files changed, 32 insertions(+), 6 deletions(-)
 create mode 100644 s390x/snippets/asm/macros.S
s390x/snippets/asm/macros.S new file +28 -0
new file mode 100644
index 00000000..667fb6dc
@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Commonly used assembly macros
+ *
+ * Copyright (c) 2022 IBM Corp
+ *
+ * Authors:
+ *  Janosch Frank <[email protected]>
+ */
+#include <asm/asm-offsets.h>
+
+/*
+ * Writes a PSW to addr_psw, useful for exception PSWs in lowcore
+ *
+ * reg is the scratch register used for temporary storage, it's NOT restored
+ * The psw address part is defined via psw_new_addr
+ * The psw mask part is always 64 bit
+ */
+.macro SET_PSW_NEW_ADDR reg, psw_new_addr, addr_psw
+larl	\reg, psw_mask_64
+stg	\reg, \addr_psw
+larl	\reg, \psw_new_addr
+stg	\reg, \addr_psw + 8
+.endm
+
+.section .rodata
+psw_mask_64:
+	.quad	0x0000000180000000
s390x/snippets/asm/snippet-pv-diag-288.S +2 -2
index aaee3cd1..63f2113b 100644
@@ -8,6 +8,7 @@
  *  Janosch Frank <[email protected]>
  */
 #include <asm/asm-offsets.h>
+#include "macros.S"
 .section .text
 
 /* Clean and pre-load registers that are used for diag 288 */
@@ -19,8 +20,7 @@ lghi	%r1, 2
 lghi	%r2, 3
 
 /* Let's jump to the pgm exit label on a PGM */
-larl	%r4, exit_pgm
-stg     %r4, GEN_LC_PGM_NEW_PSW + 8
+SET_PSW_NEW_ADDR 4, exit_pgm, GEN_LC_PGM_NEW_PSW
 
 /* Execute the diag288 */
 diag	%r0, %r2, 0x288
s390x/snippets/asm/snippet-pv-diag-500.S +2 -4
index 8dd66bd9..f4d75388 100644
@@ -8,6 +8,7 @@
  *  Janosch Frank <[email protected]>
  */
 #include <asm/asm-offsets.h>
+#include "macros.S"
 .section .text
 
 /* Clean and pre-load registers that are used for diag 500 */
@@ -21,10 +22,7 @@ lghi	%r3, 3
 lghi	%r4, 4
 
 /* Let's jump to the next label on a PGM */
-xgr	%r5, %r5
-stg	%r5, GEN_LC_PGM_NEW_PSW
-larl	%r5, next
-stg	%r5, GEN_LC_PGM_NEW_PSW + 8
+SET_PSW_NEW_ADDR 5, next, GEN_LC_PGM_NEW_PSW
 
 /* Execute the diag500 */
 diag	0, 0, 0x500
Signature · 1 line
2.34.1
#2 lore raw

PATCH v3 2/6 s390x: MAKEFILE: Use $< instead of pathsubst

In reply to Janosch Frank

No need to mangle strings if we already have the value at hand.

---
 s390x/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
s390x/Makefile +1 -1
index 649486f2..3b175015 100644
@@ -136,7 +136,7 @@ $(SNIPPET_DIR)/asm/%.gbin: $(SNIPPET_DIR)/asm/%.o
 	truncate -s '%4096' $@
 
 $(SNIPPET_DIR)/c/%.gbin: $(SNIPPET_DIR)/c/%.o $(snippet_lib) $(FLATLIBS)
-	$(CC) $(LDFLAGS) -o $@ -T $(SRCDIR)/s390x/snippets/c/flat.lds $(patsubst %.gbin,%.o,$@) $(snippet_lib) $(FLATLIBS)
+	$(CC) $(LDFLAGS) -o $@ -T $(SRCDIR)/s390x/snippets/c/flat.lds $< $(snippet_lib) $(FLATLIBS)
 	$(OBJCOPY) -O binary -j ".rodata" -j ".lowcore" -j ".text" -j ".data" -j ".bss" --set-section-flags .bss=alloc,load,contents $@ $@
 	truncate -s '%4096' $@
 
Signature · 1 line
2.34.1
#3 lore raw

PATCH v3 3/6 lib: s390x: sie: Improve validity handling and make it vm specific

In reply to Janosch Frank

The current library doesn't support running multiple vms at once as it stores the validity once and not per vm. Let's move the validity handling into the vm and introduce a new function to retrieve the vir.

---
 lib/s390x/sie.c | 27 ++++++++++++++-------------
 lib/s390x/sie.h |  6 ++++--
 2 files changed, 18 insertions(+), 15 deletions(-)
lib/s390x/sie.c +14 -13
index 00aff713..3fee3def 100644
@@ -15,19 +15,22 @@
 #include <libcflat.h>
 #include <alloc_page.h>
 
-static bool validity_expected;
-static uint16_t vir;		/* Validity interception reason */
-
-void sie_expect_validity(void)
+void sie_expect_validity(struct vm *vm)
 {
-	validity_expected = true;
-	vir = 0;
+	vm->validity_expected = true;
 }
 
-void sie_check_validity(uint16_t vir_exp)
+uint16_t sie_get_validity(struct vm *vm)
 {
+	assert(vm->sblk->icptcode == ICPT_VALIDITY);
+	return vm->sblk->ipb >> 16;
+}
+
+void sie_check_validity(struct vm *vm, uint16_t vir_exp)
+{
+	uint16_t vir = sie_get_validity(vm);
+
 	report(vir_exp == vir, "VALIDITY: %x", vir);
-	vir = 0;
 }
 
 void sie_handle_validity(struct vm *vm)
@@ -35,11 +38,9 @@ void sie_handle_validity(struct vm *vm)
 	if (vm->sblk->icptcode != ICPT_VALIDITY)
 		return;
 
-	vir = vm->sblk->ipb >> 16;
-
-	if (!validity_expected)
-		report_abort("VALIDITY: %x", vir);
-	validity_expected = false;
+	if (!vm->validity_expected)
+		report_abort("VALIDITY: %x", sie_get_validity(vm));
+	vm->validity_expected = false;
 }
 
 void sie(struct vm *vm)
lib/s390x/sie.h +4 -2
index de91ea5a..320c4218 100644
@@ -233,14 +233,16 @@ struct vm {
 	struct vm_uv uv;			/* PV UV information */
 	/* Ptr to first guest page */
 	uint8_t *guest_mem;
+	bool validity_expected;
 };
 
 extern void sie_entry(void);
 extern void sie_exit(void);
 extern void sie64a(struct kvm_s390_sie_block *sblk, struct vm_save_area *save_area);
 void sie(struct vm *vm);
-void sie_expect_validity(void);
-void sie_check_validity(uint16_t vir_exp);
+void sie_expect_validity(struct vm *vm);
+uint16_t sie_get_validity(struct vm *vm);
+void sie_check_validity(struct vm *vm, uint16_t vir_exp);
 void sie_handle_validity(struct vm *vm);
 void sie_guest_sca_create(struct vm *vm);
 void sie_guest_create(struct vm *vm, uint64_t guest_mem, uint64_t guest_mem_len);
Signature · 1 line
2.34.1
#4 lore raw

PATCH v3 4/6 lib: s390x: Use a new asce for each PV guest

In reply to Janosch Frank

Every PV guest needs its own ASCE so let's copy the topmost table designated by CR1 to create a new ASCE for the PV guest. Before and after SIE we now need to switch ASCEs to and from the PV guest / test ASCE. The SIE assembly function does that automatically.

---
 lib/s390x/asm-offsets.c |  2 ++
 lib/s390x/sie.c         |  2 ++
 lib/s390x/sie.h         |  2 ++
 lib/s390x/uv.c          | 24 +++++++++++++++++++++++-
 lib/s390x/uv.h          |  5 ++---
 s390x/cpu.S             |  6 ++++++
 6 files changed, 37 insertions(+), 4 deletions(-)
lib/s390x/asm-offsets.c +2 -0
index fbea3278..f612f327 100644
@@ -75,9 +75,11 @@ int main(void)
 	OFFSET(SIE_SAVEAREA_HOST_GRS, vm_save_area, host.grs[0]);
 	OFFSET(SIE_SAVEAREA_HOST_FPRS, vm_save_area, host.fprs[0]);
 	OFFSET(SIE_SAVEAREA_HOST_FPC, vm_save_area, host.fpc);
+	OFFSET(SIE_SAVEAREA_HOST_ASCE, vm_save_area, host.asce);
 	OFFSET(SIE_SAVEAREA_GUEST_GRS, vm_save_area, guest.grs[0]);
 	OFFSET(SIE_SAVEAREA_GUEST_FPRS, vm_save_area, guest.fprs[0]);
 	OFFSET(SIE_SAVEAREA_GUEST_FPC, vm_save_area, guest.fpc);
+	OFFSET(SIE_SAVEAREA_GUEST_ASCE, vm_save_area, guest.asce);
 	OFFSET(STACK_FRAME_INT_BACKCHAIN, stack_frame_int, back_chain);
 	OFFSET(STACK_FRAME_INT_FPC, stack_frame_int, fpc);
 	OFFSET(STACK_FRAME_INT_FPRS, stack_frame_int, fprs);
lib/s390x/sie.c +2 -0
index 3fee3def..6efad965 100644
@@ -85,6 +85,8 @@ void sie_guest_create(struct vm *vm, uint64_t guest_mem, uint64_t guest_mem_len)
 
 	/* Guest memory chunks are always 1MB */
 	assert(!(guest_mem_len & ~HPAGE_MASK));
+	/* For non-PV guests we re-use the host's ASCE for ease of use */
+	vm->save_area.guest.asce = stctg(1);
 	/* Currently MSO/MSL is the easiest option */
 	vm->sblk->mso = (uint64_t)guest_mem;
 	vm->sblk->msl = (uint64_t)guest_mem + ((guest_mem_len - 1) & HPAGE_MASK);
lib/s390x/sie.h +2 -0
index 320c4218..3e3605c9 100644
@@ -205,12 +205,14 @@ union {
 struct vm_uv {
 	uint64_t vm_handle;
 	uint64_t vcpu_handle;
+	uint64_t asce;
 	void *conf_base_stor;
 	void *conf_var_stor;
 	void *cpu_stor;
 };
 
 struct vm_save_regs {
+	uint64_t asce;
 	uint64_t grs[16];
 	uint64_t fprs[16];
 	uint32_t fpc;
lib/s390x/uv.c +23 -1
index 3b4cafa9..b2a43424 100644
@@ -90,6 +90,25 @@ void uv_init(void)
 	initialized = true;
 }
 
+/*
+ * Create a new ASCE for the UV config because they can't be shared
+ * for security reasons. We just simply copy the top most table into a
+ * fresh set of allocated pages and use those pages as the asce.
+ */
+static uint64_t create_asce(void)
+{
+	void *pgd_new, *pgd_old;
+	uint64_t asce = stctg(1);
+
+	pgd_new = memalign_pages(PAGE_SIZE, PAGE_SIZE * 4);
+	pgd_old = (void *)(asce & PAGE_MASK);
+
+	memcpy(pgd_new, pgd_old, PAGE_SIZE * 4);
+
+	asce = __pa(pgd_new) | ASCE_P | (asce & (ASCE_DT | ASCE_TL));
+	return asce;
+}
+
 void uv_create_guest(struct vm *vm)
 {
 	struct uv_cb_cgc uvcb_cgc = {
@@ -125,7 +144,8 @@ void uv_create_guest(struct vm *vm)
 	vm->uv.cpu_stor = memalign_pages_flags(PAGE_SIZE, uvcb_qui.cpu_stor_len, 0);
 	uvcb_csc.stor_origin = (uint64_t)vm->uv.cpu_stor;
 
-	uvcb_cgc.guest_asce = (uint64_t)stctg(1);
+	uvcb_cgc.guest_asce = create_asce();
+	vm->save_area.guest.asce = uvcb_cgc.guest_asce;
 	uvcb_cgc.guest_sca = (uint64_t)vm->sca;
 
 	cc = uv_call(0, (uint64_t)&uvcb_cgc);
@@ -166,6 +186,8 @@ void uv_destroy_guest(struct vm *vm)
 	assert(cc == 0);
 	free_pages(vm->uv.conf_base_stor);
 	free_pages(vm->uv.conf_var_stor);
+
+	free_pages((void *)(vm->uv.asce & PAGE_MASK));
 }
 
 int uv_unpack(struct vm *vm, uint64_t addr, uint64_t len, uint64_t tweak)
lib/s390x/uv.h +2 -3
index 44264861..5fe29bda 100644
@@ -28,9 +28,8 @@ static inline void uv_setup_asces(void)
 	/* We need to have a valid primary ASCE to run guests. */
 	setup_vm();
 
-	/* Set P bit in ASCE as it is required for PV guests */
-	asce = stctg(1) | ASCE_P;
-	lctlg(1, asce);
+	/* Grab the ASCE which setup_vm() just set up */
+	asce = stctg(1);
 
 	/* Copy ASCE into home space CR */
 	lctlg(13, asce);
s390x/cpu.S +6 -0
index 82b5e25d..45bd551a 100644
@@ -76,6 +76,9 @@ sie64a:
 	.endr
 	stfpc	SIE_SAVEAREA_HOST_FPC(%r3)
 
+	stctg	%c1, %c1, SIE_SAVEAREA_HOST_ASCE(%r3)
+	lctlg	%c1, %c1, SIE_SAVEAREA_GUEST_ASCE(%r3)
+
 	# Store scb and save_area pointer into stack frame
 	stg	%r2,__SF_SIE_CONTROL(%r15)	# save control block pointer
 	stg	%r3,__SF_SIE_SAVEAREA(%r15)	# save guest register save area
@@ -102,6 +105,9 @@ sie_exit:
 	# Load guest register save area
 	lg	%r14,__SF_SIE_SAVEAREA(%r15)
 
+	# Restore the host asce
+	lctlg	%c1, %c1, SIE_SAVEAREA_HOST_ASCE(%r14)
+
 	# Store guest's gprs, fprs and fpc
 	stmg	%r0,%r13,SIE_SAVEAREA_GUEST_GRS(%r14)	# save guest gprs 0-13
 	.irp i, 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
Signature · 1 line
2.34.1
Reviewed-by on v3 4/6
#5 lore raw

Re: [kvm-unit-tests PATCH v3 4/6] lib: s390x: Use a new asce for each PV guest

Quoted 6 lines · On Fri, 21 Oct 2022 06:39:00 +0000 Janosch Frank <[email protected]> wrote:

Every PV guest needs its own ASCE so let's copy the topmost table designated by CR1 to create a new ASCE for the PV guest. Before and after SIE we now need to switch ASCEs to and from the PV guest / test ASCE. The SIE assembly function does that automatically.

Quoted 146 lines
---
 lib/s390x/asm-offsets.c |  2 ++
 lib/s390x/sie.c         |  2 ++
 lib/s390x/sie.h         |  2 ++
 lib/s390x/uv.c          | 24 +++++++++++++++++++++++-
 lib/s390x/uv.h          |  5 ++---
 s390x/cpu.S             |  6 ++++++
 6 files changed, 37 insertions(+), 4 deletions(-)
lib/s390x/asm-offsets.c +2 -0
index fbea3278..f612f327 100644
@@ -75,9 +75,11 @@ int main(void)
 	OFFSET(SIE_SAVEAREA_HOST_GRS, vm_save_area, host.grs[0]);
 	OFFSET(SIE_SAVEAREA_HOST_FPRS, vm_save_area, host.fprs[0]);
 	OFFSET(SIE_SAVEAREA_HOST_FPC, vm_save_area, host.fpc);
+	OFFSET(SIE_SAVEAREA_HOST_ASCE, vm_save_area, host.asce);
 	OFFSET(SIE_SAVEAREA_GUEST_GRS, vm_save_area, guest.grs[0]);
 	OFFSET(SIE_SAVEAREA_GUEST_FPRS, vm_save_area, guest.fprs[0]);
 	OFFSET(SIE_SAVEAREA_GUEST_FPC, vm_save_area, guest.fpc);
+	OFFSET(SIE_SAVEAREA_GUEST_ASCE, vm_save_area, guest.asce);
 	OFFSET(STACK_FRAME_INT_BACKCHAIN, stack_frame_int, back_chain);
 	OFFSET(STACK_FRAME_INT_FPC, stack_frame_int, fpc);
 	OFFSET(STACK_FRAME_INT_FPRS, stack_frame_int, fprs);
lib/s390x/sie.c +2 -0
index 3fee3def..6efad965 100644
@@ -85,6 +85,8 @@ void sie_guest_create(struct vm *vm, uint64_t guest_mem, uint64_t guest_mem_len)
 
 	/* Guest memory chunks are always 1MB */
 	assert(!(guest_mem_len & ~HPAGE_MASK));
+	/* For non-PV guests we re-use the host's ASCE for ease of use */
+	vm->save_area.guest.asce = stctg(1);
 	/* Currently MSO/MSL is the easiest option */
 	vm->sblk->mso = (uint64_t)guest_mem;
 	vm->sblk->msl = (uint64_t)guest_mem + ((guest_mem_len - 1) & HPAGE_MASK);
lib/s390x/sie.h +2 -0
index 320c4218..3e3605c9 100644
@@ -205,12 +205,14 @@ union {
 struct vm_uv {
 	uint64_t vm_handle;
 	uint64_t vcpu_handle;
+	uint64_t asce;
 	void *conf_base_stor;
 	void *conf_var_stor;
 	void *cpu_stor;
 };
 
 struct vm_save_regs {
+	uint64_t asce;
 	uint64_t grs[16];
 	uint64_t fprs[16];
 	uint32_t fpc;
lib/s390x/uv.c +23 -1
index 3b4cafa9..b2a43424 100644
@@ -90,6 +90,25 @@ void uv_init(void)
 	initialized = true;
 }
 
+/*
+ * Create a new ASCE for the UV config because they can't be shared
+ * for security reasons. We just simply copy the top most table into a
+ * fresh set of allocated pages and use those pages as the asce.
+ */
+static uint64_t create_asce(void)
+{
+	void *pgd_new, *pgd_old;
+	uint64_t asce = stctg(1);
+
+	pgd_new = memalign_pages(PAGE_SIZE, PAGE_SIZE * 4);
+	pgd_old = (void *)(asce & PAGE_MASK);
+
+	memcpy(pgd_new, pgd_old, PAGE_SIZE * 4);
+
+	asce = __pa(pgd_new) | ASCE_P | (asce & (ASCE_DT | ASCE_TL));
+	return asce;
+}
+
 void uv_create_guest(struct vm *vm)
 {
 	struct uv_cb_cgc uvcb_cgc = {
@@ -125,7 +144,8 @@ void uv_create_guest(struct vm *vm)
 	vm->uv.cpu_stor = memalign_pages_flags(PAGE_SIZE, uvcb_qui.cpu_stor_len, 0);
 	uvcb_csc.stor_origin = (uint64_t)vm->uv.cpu_stor;
 
-	uvcb_cgc.guest_asce = (uint64_t)stctg(1);
+	uvcb_cgc.guest_asce = create_asce();
+	vm->save_area.guest.asce = uvcb_cgc.guest_asce;
 	uvcb_cgc.guest_sca = (uint64_t)vm->sca;
 
 	cc = uv_call(0, (uint64_t)&uvcb_cgc);
@@ -166,6 +186,8 @@ void uv_destroy_guest(struct vm *vm)
 	assert(cc == 0);
 	free_pages(vm->uv.conf_base_stor);
 	free_pages(vm->uv.conf_var_stor);
+
+	free_pages((void *)(vm->uv.asce & PAGE_MASK));
 }
 
 int uv_unpack(struct vm *vm, uint64_t addr, uint64_t len, uint64_t tweak)
lib/s390x/uv.h +2 -3
index 44264861..5fe29bda 100644
@@ -28,9 +28,8 @@ static inline void uv_setup_asces(void)
 	/* We need to have a valid primary ASCE to run guests. */
 	setup_vm();
 
-	/* Set P bit in ASCE as it is required for PV guests */
-	asce = stctg(1) | ASCE_P;
-	lctlg(1, asce);
+	/* Grab the ASCE which setup_vm() just set up */
+	asce = stctg(1);
 
 	/* Copy ASCE into home space CR */
 	lctlg(13, asce);
s390x/cpu.S +6 -0
index 82b5e25d..45bd551a 100644
@@ -76,6 +76,9 @@ sie64a:
 	.endr
 	stfpc	SIE_SAVEAREA_HOST_FPC(%r3)
 
+	stctg	%c1, %c1, SIE_SAVEAREA_HOST_ASCE(%r3)
+	lctlg	%c1, %c1, SIE_SAVEAREA_GUEST_ASCE(%r3)
+
 	# Store scb and save_area pointer into stack frame
 	stg	%r2,__SF_SIE_CONTROL(%r15)	# save control block pointer
 	stg	%r3,__SF_SIE_SAVEAREA(%r15)	# save guest register save area
@@ -102,6 +105,9 @@ sie_exit:
 	# Load guest register save area
 	lg	%r14,__SF_SIE_SAVEAREA(%r15)
 
+	# Restore the host asce
+	lctlg	%c1, %c1, SIE_SAVEAREA_HOST_ASCE(%r14)
+
 	# Store guest's gprs, fprs and fpc
 	stmg	%r0,%r13,SIE_SAVEAREA_GUEST_GRS(%r14)	# save guest gprs 0-13
 	.irp i, 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
#6 lore raw

PATCH v3 5/6 lib: s390x: Enable reusability of VMs that were in PV mode

In reply to Janosch Frank

Convert the sblk to non-PV when the PV guest is destroyed.

Early return in uv_init() instead of running into the assert. This is necessary since snippet_pv_init() will always call uv_init().

---
 lib/s390x/uv.c | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)
lib/s390x/uv.c +10 -1
index b2a43424..383271a5 100644
@@ -76,7 +76,8 @@ void uv_init(void)
 	int cc;
 
 	/* Let's not do this twice */
-	assert(!initialized);
+	if (initialized)
+		return;
 	/* Query is done on initialization but let's check anyway */
 	assert(uvcb_qui.header.rc == 1 || uvcb_qui.header.rc == 0x100);
 
@@ -188,6 +189,14 @@ void uv_destroy_guest(struct vm *vm)
 	free_pages(vm->uv.conf_var_stor);
 
 	free_pages((void *)(vm->uv.asce & PAGE_MASK));
+	memset(&vm->uv, 0, sizeof(vm->uv));
+
+	/* Convert the sblk back to non-PV */
+	vm->save_area.guest.asce = stctg(1);
+	vm->sblk->sdf = 0;
+	vm->sblk->sidad = 0;
+	vm->sblk->pv_handle_cpu = 0;
+	vm->sblk->pv_handle_config = 0;
 }
 
 int uv_unpack(struct vm *vm, uint64_t addr, uint64_t len, uint64_t tweak)
Signature · 1 line
2.34.1
Reviewed-by on v3 5/6
#7 lore raw

Re: [kvm-unit-tests PATCH v3 5/6] lib: s390x: Enable reusability of VMs that were in PV mode

Quoted 7 lines · Janosch Frank

Convert the sblk to non-PV when the PV guest is destroyed.

Early return in uv_init() instead of running into the assert. This is necessary since snippet_pv_init() will always call uv_init().

Quoted 33 lines
---
  lib/s390x/uv.c | 11 ++++++++++-
  1 file changed, 10 insertions(+), 1 deletion(-)
lib/s390x/uv.c +10 -1
index b2a43424..383271a5 100644
@@ -76,7 +76,8 @@ void uv_init(void)
  	int cc;
 
  	/* Let's not do this twice */
-	assert(!initialized);
+	if (initialized)
+		return;
  	/* Query is done on initialization but let's check anyway */
  	assert(uvcb_qui.header.rc == 1 || uvcb_qui.header.rc == 0x100);
 
@@ -188,6 +189,14 @@ void uv_destroy_guest(struct vm *vm)
  	free_pages(vm->uv.conf_var_stor);
 
  	free_pages((void *)(vm->uv.asce & PAGE_MASK));
+	memset(&vm->uv, 0, sizeof(vm->uv));
+
+	/* Convert the sblk back to non-PV */
+	vm->save_area.guest.asce = stctg(1);
+	vm->sblk->sdf = 0;
+	vm->sblk->sidad = 0;
+	vm->sblk->pv_handle_cpu = 0;
+	vm->sblk->pv_handle_config = 0;
  }
 
  int uv_unpack(struct vm *vm, uint64_t addr, uint64_t len, uint64_t tweak)
#8 lore raw

PATCH v3 6/6 lib: s390x: sie: Properly populate SCA

In reply to Janosch Frank

CPU0 is the only cpu that's being used but we should still mark it as online and set the SDA in the SCA.

---
 lib/s390x/sie.c |  8 ++++++++
 lib/s390x/sie.h | 35 ++++++++++++++++++++++++++++++++++-
 2 files changed, 42 insertions(+), 1 deletion(-)
lib/s390x/sie.c +8 -0
index 6efad965..a71985b6 100644
@@ -9,6 +9,7 @@
  */
 
 #include <asm/barrier.h>
+#include <bitops.h>
 #include <libcflat.h>
 #include <sie.h>
 #include <asm/page.h>
@@ -72,6 +73,13 @@ void sie_guest_sca_create(struct vm *vm)
 	vm->sblk->scaoh = ((uint64_t)vm->sca >> 32);
 	vm->sblk->scaol = (uint64_t)vm->sca & ~0x3fU;
 	vm->sblk->ecb2 |= ECB2_ESCA;
+
+	/* Enable SIGP sense running interpretation */
+	vm->sblk->ecb |= ECB_SRSI;
+
+	/* We assume that cpu 0 is always part of the vm */
+	vm->sca->mcn[0] = BIT(63);
+	vm->sca->cpu[0].sda = (uint64_t)vm->sblk;
 }
 
 /* Initializes the struct vm members like the SIE control block. */
lib/s390x/sie.h +34 -1
index 3e3605c9..a27a8401 100644
@@ -202,6 +202,39 @@ union {
 	uint64_t	pv_grregs[16];		/* 0x0380 */
 } __attribute__((packed));
 
+union esca_sigp_ctrl {
+	uint16_t value;
+	struct {
+		uint8_t c : 1;
+		uint8_t reserved: 7;
+		uint8_t scn;
+	};
+};
+
+struct esca_entry {
+	union esca_sigp_ctrl sigp_ctrl;
+	uint16_t   reserved1[3];
+	uint64_t   sda;
+	uint64_t   reserved2[6];
+};
+
+union ipte_control {
+	unsigned long val;
+	struct {
+		unsigned long k  : 1;
+		unsigned long kh : 31;
+		unsigned long kg : 32;
+	};
+};
+
+struct esca_block {
+	union ipte_control ipte_control;
+	uint64_t   reserved1[7];
+	uint64_t   mcn[4];
+	uint64_t   reserved2[20];
+	struct esca_entry cpu[256];
+};
+
 struct vm_uv {
 	uint64_t vm_handle;
 	uint64_t vcpu_handle;
@@ -230,7 +263,7 @@ struct vm_save_area {
 struct vm {
 	struct kvm_s390_sie_block *sblk;
 	struct vm_save_area save_area;
-	void *sca;				/* System Control Area */
+	struct esca_block *sca;			/* System Control Area */
 	uint8_t *crycb;				/* Crypto Control Block */
 	struct vm_uv uv;			/* PV UV information */
 	/* Ptr to first guest page */
Signature · 1 line
2.34.1
#9 lore raw
In reply to Janosch Frank
Quoted 4 lines · On Fri, 21 Oct 2022 06:38:56 +0000 Janosch Frank <[email protected]> wrote:

A small set of patches that clean up the PV snippet handling.

v3:
* Dropped asm snippet linker script patch for now

shame, I really liked that patch (modulo the nits)

Quoted 29 lines

* Replaced memalign_pages_flags() with memalign_pages()
* PV ASCEs will now recieve DT and TL fields from the main test ASCE

v2:
* Macro uses 64bit PSW mask
* SBLK reset on PV destroy and uv_init() early return have been split off

Janosch Frank (6):
s390x: snippets: asm: Add a macro to write an exception PSW
s390x: MAKEFILE: Use $< instead of pathsubst
lib: s390x: sie: Improve validity handling and make it vm specific
lib: s390x: Use a new asce for each PV guest
lib: s390x: Enable reusability of VMs that were in PV mode
lib: s390x: sie: Properly populate SCA

 lib/s390x/asm-offsets.c                  |  2 ++
 lib/s390x/sie.c                          | 37 +++++++++++++-------
 lib/s390x/sie.h                          | 43 ++++++++++++++++++++++--
 lib/s390x/uv.c                           | 35 +++++++++++++++++--
 lib/s390x/uv.h                           |  5 ++-
 s390x/Makefile                           |  2 +-
 s390x/cpu.S                              |  6 ++++
 s390x/snippets/asm/macros.S              | 28 +++++++++++++++
 s390x/snippets/asm/snippet-pv-diag-288.S |  4 +--
 s390x/snippets/asm/snippet-pv-diag-500.S |  6 ++--
 10 files changed, 140 insertions(+), 28 deletions(-)
 create mode 100644 s390x/snippets/asm/macros.S
#10 lore raw
Quoted 9 lines · Claudio Imbrenda
Quoted 4 lines · On Fri, 21 Oct 2022 06:38:56 +0000 Janosch Frank <[email protected]> wrote:

A small set of patches that clean up the PV snippet handling.

v3:
* Dropped asm snippet linker script patch for now

shame, I really liked that patch (modulo the nits)

You'll see it again soonish with another fix series. There are still a lot of patches that I need to upstream. :)

Quoted 31 lines
Quoted 30 lines

* Replaced memalign_pages_flags() with memalign_pages()
* PV ASCEs will now recieve DT and TL fields from the main test ASCE

v2:
* Macro uses 64bit PSW mask
* SBLK reset on PV destroy and uv_init() early return have been split off

Janosch Frank (6):
s390x: snippets: asm: Add a macro to write an exception PSW
s390x: MAKEFILE: Use $< instead of pathsubst
lib: s390x: sie: Improve validity handling and make it vm specific
lib: s390x: Use a new asce for each PV guest
lib: s390x: Enable reusability of VMs that were in PV mode
lib: s390x: sie: Properly populate SCA

  lib/s390x/asm-offsets.c                  |  2 ++
  lib/s390x/sie.c                          | 37 +++++++++++++-------
  lib/s390x/sie.h                          | 43 ++++++++++++++++++++++--
  lib/s390x/uv.c                           | 35 +++++++++++++++++--
  lib/s390x/uv.h                           |  5 ++-
  s390x/Makefile                           |  2 +-
  s390x/cpu.S                              |  6 ++++
  s390x/snippets/asm/macros.S              | 28 +++++++++++++++
  s390x/snippets/asm/snippet-pv-diag-288.S |  4 +--
  s390x/snippets/asm/snippet-pv-diag-500.S |  6 ++--
  10 files changed, 140 insertions(+), 28 deletions(-)

create mode 100644 s390x/snippets/asm/macros.S