s390x: Cleanup virtualization library
cover s390x: Cleanup virtualization library
In order to support sae virtualization we need to cleanup the library a bit and make it more flexible when it comes to guest creation and allocation.
Janosch Frank (5):
lib: s390x: Add function to get page root
lib: s390x: sie: Allocate physical guest memory via memalign
lib: s390x: sie: Free guest memory on destroy
lib: s390x: snippet: Add function to create a guest of specific length
lib: s390x: Remove kvm s390 prefix from sie control block
lib/s390x/asm/sie-arch.h | 2 +- lib/s390x/interrupt.c | 4 ++-- lib/s390x/mmu.h | 7 +++++++ lib/s390x/sie-icpt.c | 2 +- lib/s390x/sie-icpt.h | 2 +- lib/s390x/sie.c | 7 +++++-- lib/s390x/sie.h | 4 ++-- lib/s390x/snippet.h | 18 ++++++++++++++---- s390x/diag258.c | 4 ++-- s390x/edat.c | 5 +++-- s390x/mvpg-sie.c | 2 +- s390x/pv-edat1.c | 4 ++-- s390x/sie-dat.c | 2 +- s390x/skey.c | 7 +++---- 14 files changed, 45 insertions(+), 25 deletions(-)
Signature · 1 line
2.51.0
PATCH 1/5 lib: s390x: Add function to get page root
It's time to hide the access to cr1 behind a function and add typing to the tests that used void* instead of pgdt_t*.
--- lib/s390x/mmu.h | 7 +++++++ lib/s390x/sie.c | 3 ++- s390x/diag258.c | 4 ++-- s390x/edat.c | 5 +++-- s390x/mvpg-sie.c | 2 +- s390x/pv-edat1.c | 4 ++-- s390x/sie-dat.c | 2 +- s390x/skey.c | 7 +++---- 8 files changed, 21 insertions(+), 13 deletions(-)
lib/s390x/mmu.h +7 -0
@@ -10,6 +10,8 @@#ifndef _S390X_MMU_H_#define _S390X_MMU_H_ +#include <asm/arch_def.h> +enum pgt_level { pgtable_level_pgd =1 , pgtable_level_p4d,
@@ -95,4 +97,9 @@ static inline void unprotect_page(void *vaddr, unsigned long prot)void *get_dat_entry (pgd_t *pgtable,void *vaddr,enum pgt_level level); +static inline pgd_t *get_page_root (void ) +{ +return (pgd_t *)(stctg (1 ) &PAGE_MASK ); +} +#endif /* _ASMS390X_MMU_H_ */
lib/s390x/sie.c +2 -1
@@ -17,6 +17,7 @@#include <alloc_page.h> #include <vmalloc.h> #include <sclp.h> +#include <mmu.h> void sie_expect_validity (struct vm *vm) {
@@ -156,7 +157,7 @@ uint8_t *sie_guest_alloc(uint64_t guest_size)pgd_t *root;setup_vm (); - root = (pgd_t *)(stctg (1 ) &PAGE_MASK ); + root =get_page_root ();/* * Start of guest memory in host virtual space needs to be aligned to
s390x/diag258.c +2 -2
@@ -82,7 +82,7 @@ static void test_priv(void)static void *page_map_outside_real_space (phys_addr_t page_real) { -pgd_t *root = (pgd_t *)(stctg (1 ) &PAGE_MASK ); +pgd_t *root =get_page_root ();void *vaddr =alloc_vpage ();install_page (root, page_real, vaddr);
@@ -109,7 +109,7 @@ static void test_refbk_real(void) refbk_page =alloc_page ();/* Map refblk page outside of physical memory identity mapping */ - root = (pgd_t *)(stctg (1 ) &PAGE_MASK ); + root =get_page_root (); refbk =page_map_outside_real_space (virt_to_pte_phys (root, refbk_page));/* Assert the mapping really is outside identity mapping */
s390x/edat.c +3 -2
@@ -21,7 +21,8 @@ static uint8_t prefix_buf[LC_SIZE] __attribute__((aligned(LC_SIZE))); static unsigned int tmp[1024] __attribute__((aligned(PAGE_SIZE))); -static void *root, *mem, *m; +static void *mem, *m; +static pgd_t *root;volatile unsigned int *p;/*
@@ -237,7 +238,7 @@ static unsigned int setup(void)/* Setup DAT 1:1 mapping and memory management */ setup_vm (); - root = (void *)(stctg (1 ) &PAGE_MASK ); + root =get_page_root ();/* * Get a pgd worth of virtual memory, so we can test things later
s390x/mvpg-sie.c +1 -1
@@ -89,7 +89,7 @@ static void setup_guest(void)pgd_t *root;setup_vm (); - root = (pgd_t *)(stctg (1 ) &PAGE_MASK ); + root =get_page_root ();snippet_setup_guest (&vm, false); snippet_init(&vm, SNIPPET_NAME_START(c, mvpg_snippet),
s390x/pv-edat1.c +2 -2
@@ -34,7 +34,7 @@#define PARAM (n, step) (((unsigned long)(n) << 32) | (step))static struct vm vm; -static void *root; +static pgd_t *root;extern const char SNIPPET_NAME_START (c ,pv_memhog )[];extern const char SNIPPET_NAME_END (c ,pv_memhog )[];
@@ -441,7 +441,7 @@ static void init(void)setup_vm (); - root = (void *)(stctg (1 ) &PAGE_MASK ); + root =get_page_root ();ctl_set_bit (0 ,CTL0_EDAT ); guest_memory =alloc_pages (GUEST_ORDER -PAGE_SHIFT );
s390x/sie-dat.c +1 -1
@@ -69,7 +69,7 @@ static void setup_guest(void)pgd_t *root;setup_vm (); - root = (pgd_t *)(stctg (1 ) &PAGE_MASK ); + root =get_page_root ();snippet_setup_guest (&vm, false);
s390x/skey.c +3 -4
@@ -13,6 +13,7 @@#include <asm/interrupt.h> #include <vmalloc.h> #include <css.h> +#include <mmu.h> #include <asm/page.h> #include <asm/facility.h> #include <asm/mem.h>
@@ -465,10 +466,9 @@ static void test_set_prefix(void)uint32_t *prefix_ptr = (uint32_t *)pagebuf;uint32_t *no_override_prefix_ptr;uint32_t old_prefix; -pgd_t *root; +pgd_t *root =get_page_root ();report_prefix_push ("SET PREFIX" ); - root = (pgd_t *)(stctg (1 ) &PAGE_MASK ); old_prefix =get_prefix ();memcpy (lowcore_tmp,0 ,sizeof (lowcore_tmp));assert (((uint64_t )&lowcore_tmp >>31 ) ==0 );
@@ -583,11 +583,10 @@ static void test_msch(void)struct schib *schib = (struct schib *)pagebuf;struct schib *no_override_schib;int test_device_sid; -pgd_t *root; +pgd_t *root =get_page_root ();int cc;report_prefix_push ("MSCH" ); - root = (pgd_t *)(stctg (1 ) &PAGE_MASK ); test_device_sid =css_enumerate ();if (!(test_device_sid &SCHID_ONE )) {
Signature · 1 line
2.51.0
Re: [kvm-unit-tests 1/5] lib: s390x: Add function to get page root
On Wed Apr 15, 2026 at 10:45 AM CEST, Janosch Frank wrote:
[...]
Quoted 14 lines
lib/s390x/mmu.h +2
-0@@ -10,6 +10,8 @@#ifndef _S390X_MMU_H_#define _S390X_MMU_H_ +#include <asm/arch_def.h> +enum pgt_level { pgtable_level_pgd =1 , pgtable_level_p4d,
lib/s390x/mmu.h static inline void unprotect_page(void *vaddr, unsigned long prot) · line 100
Quoted 4 lines
lib/s390x/mmu.h +1
-0@@ -95,4 +97,9 @@ static inline void unprotect_page(void *vaddr, unsigned long prot)void *get_dat_entry (pgd_t *pgtable,void *vaddr,enum pgt_level level); +static inline pgd_t *get_page_root (void )
Only nit I have: naming is hard... do you think it makes sense to reflect that we're talking about the primary-space root here?
get_primary_page_root()?
It's not like we're using the other spaces much in kut, but maybe we will in the future.
Re: [kvm-unit-tests 1/5] lib: s390x: Add function to get page root
Quoted 24 lines · Nico Boehr
On Wed Apr 15, 2026 at 10:45 AM CEST, Janosch Frank wrote:
[...]Quoted 14 lines
lib/s390x/mmu.h +2
-0@@ -10,6 +10,8 @@#ifndef _S390X_MMU_H_#define _S390X_MMU_H_ +#include <asm/arch_def.h> +enum pgt_level { pgtable_level_pgd =1 , pgtable_level_p4d,lib/s390x/mmu.h static inline void unprotect_page(void *vaddr, unsigned long prot) · line 100
Quoted 4 lines
lib/s390x/mmu.h +1
-0@@ -95,4 +97,9 @@ static inline void unprotect_page(void *vaddr, unsigned long prot)void *get_dat_entry (pgd_t *pgtable,void *vaddr,enum pgt_level level); +static inline pgd_t *get_page_root (void )Only nit I have: naming is hard... do you think it makes sense to reflect that we're talking about the primary-space root here?
Quoted 4 lines
get_primary_page_root()?It's not like we're using the other spaces much in kut, but maybe we will in the future.
Sure, we can still make it take an enum in the future to switch the crs depending on which ASCE we want.
PATCH 2/5 lib: s390x: sie: Allocate physical guest memory via memalign
In reply to Janosch Frankalloc_pages_flags() alignes the allocation on the same order as its requested size. Since we use virtual memory for SIE we can instead align to 1MB by using memalign() which is less wasteful.
--- lib/s390x/sie.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
lib/s390x/sie.c +2 -1
@@ -171,7 +171,8 @@ uint8_t *sie_guest_alloc(uint64_t guest_size) guest_virt = (uint8_t *)ALIGN (get_ram_size () + guest_counter *4UL *SZ_1G ,SZ_2G ); guest_counter++; - guest_phys =alloc_pages (get_order (guest_size) -12 ); + guest_phys =memalign_pages_flags (SZ_1M , guest_size,0 ); +assert (guest_phys);/* * Establish a new mapping of the guest memory so it can be 2GB aligned * without actually requiring 2GB physical memory.
Signature · 1 line
2.51.0
PATCH 3/5 lib: s390x: sie: Free guest memory on destroy
In reply to Janosch FrankWe never freed the memory that the sie library allocates as the guest ram on destruction of the VM. Most tests reuse the VM or just leak the memory since the standard allocation is one megabyte and tests only use single digit numbers of VMs.
It's time to add automatic freeing to the sie library when a VM is destroyed.
--- lib/s390x/sie.c | 1 + 1 file changed, 1 insertion(+)
lib/s390x/sie.c +1 -0
@@ -192,4 +192,5 @@ void sie_guest_destroy(struct vm *vm)free_page (vm->sblk);if (vm->sblk->ecb2 &ECB2_ESCA )free_page (vm->sca); +free_pages ((void *)virt_to_pte_phys (get_page_root (), vm->guest_mem)); }
Signature · 1 line
2.51.0
Re: [kvm-unit-tests 3/5] lib: s390x: sie: Free guest memory on destroy
Quoted 18 lines · Janosch Frank
We never freed the memory that the sie library allocates as the guest ram on destruction of the VM. Most tests reuse the VM or just leak the memory since the standard allocation is one megabyte and tests only use single digit numbers of VMs.
It's time to add automatic freeing to the sie library when a VM is destroyed.
Signed-off-by Janosch Frank <[email protected]>--- lib/s390x/sie.c | 1 + 1 file changed, 1 insertion(+)lib/s390x/sie.c +0
-0
lib/s390x/sie.c void sie_guest_destroy(struct vm *vm) · line 196
Quoted 5 lines
lib/s390x/sie.c +1
-0@@ -192,4 +192,5 @@ void sie_guest_destroy(struct vm *vm)free_page (vm->sblk);if (vm->sblk->ecb2 &ECB2_ESCA )free_page (vm->sca); +free_pages ((void *)virt_to_pte_phys (get_page_root (), vm->guest_mem)); }
Is there a particular reason why we have a two-step process of calling sie_guest_alloc() for memory allocation
and thensie_guest_create() for initializing the sie_block etc
butsie_guest_destroy()
cleans up both?
Re: [kvm-unit-tests 3/5] lib: s390x: sie: Free guest memory on destroy
Quoted 30 lines · Nico Boehr
Quoted 18 lines · Janosch Frank
We never freed the memory that the sie library allocates as the guest ram on destruction of the VM. Most tests reuse the VM or just leak the memory since the standard allocation is one megabyte and tests only use single digit numbers of VMs.
It's time to add automatic freeing to the sie library when a VM is destroyed.
Signed-off-by Janosch Frank <[email protected]>--- lib/s390x/sie.c | 1 + 1 file changed, 1 insertion(+)lib/s390x/sie.c +0
-0lib/s390x/sie.c void sie_guest_destroy(struct vm *vm) · line 196
Quoted 5 lines
lib/s390x/sie.c +1
-0@@ -192,4 +192,5 @@ void sie_guest_destroy(struct vm *vm)free_page (vm->sblk);if (vm->sblk->ecb2 &ECB2_ESCA )free_page (vm->sca); +free_pages ((void *)virt_to_pte_phys (get_page_root (), vm->guest_mem)); }Is there a particular reason why we have a two-step process of calling
sie_guest_alloc()for memory allocation
and thensie_guest_create()for initializing thesie_blocketc
Quoted 4 lines
but
sie_guest_destroy()
cleans up both?
Historical reasons I guess.
There've been a lot of changes to the SIE lib over the years, snippet support itself isn't that consistent but it's currently good enough. s390x/sie.c is the only direct user of the allocation and even that usage looks like it could be converted.
Re: [kvm-unit-tests 3/5] lib: s390x: sie: Free guest memory on destroy
Quoted 36 lines · Janosch Frank
Quoted 30 lines · Nico Boehr
Quoted 18 lines · Janosch Frank
We never freed the memory that the sie library allocates as the guest ram on destruction of the VM. Most tests reuse the VM or just leak the memory since the standard allocation is one megabyte and tests only use single digit numbers of VMs.
It's time to add automatic freeing to the sie library when a VM is destroyed.
Signed-off-by Janosch Frank <[email protected]>--- lib/s390x/sie.c | 1 + 1 file changed, 1 insertion(+)lib/s390x/sie.c +0
-0lib/s390x/sie.c void sie_guest_destroy(struct vm *vm) · line 196
Quoted 5 lines
lib/s390x/sie.c +1
-0@@ -192,4 +192,5 @@ void sie_guest_destroy(struct vm *vm)free_page (vm->sblk);if (vm->sblk->ecb2 &ECB2_ESCA )free_page (vm->sca); +free_pages ((void *)virt_to_pte_phys (get_page_root (), vm->guest_mem)); }Is there a particular reason why we have a two-step process of calling
sie_guest_alloc()for memory allocation
and thensie_guest_create()for initializing thesie_blocketcQuoted 4 lines
but
sie_guest_destroy()
cleans up both?
Quoted 5 lines
Historical reasons I guess.
There've been a lot of changes to the SIE lib over the years, snippet support itself isn't that consistent but it's currently good enough.s390x/sie.cis the only direct user of the allocation and even that usage looks like it could be converted.
Yeah, I am a bit concerned about the intermixed dependencies and concerns between the two. Especially having free_pages() here even when the guest may (theoretically) have been allocated with something other than sie_guest_alloc(). It would be an error that's kind of hard to spot in a review since it's not obvious that sie_guest_destroy() pairs with sie_guest_alloc() AND sie_guest_create().
Would this be an option:
- add small function sie_guest_free() which frees only guest memory to sie lib this would be the counterpart to sie_guest_alloc()
- add function snippet_destroy_guest()
which calls sie_guest_destroy() and sie_guest_free()
this would be the counterpart to snippet_setup_guest()
Then this pairs nicely as follows for the SIE lib:sie_guest_alloc() - sie_guest_free() sie_guest_create() - sie_guest_destroy()
And the higher level snippet operations:snippet_setup_guest() - snippet_destroy_guest()
PATCH 4/5 lib: s390x: snippet: Add function to create a guest of specific length
In reply to Janosch FrankWhile 1MB is certainly enough to store the guest code, it's often not enough for memory tests. Let's add a separate function to allow arbitrary guest sizes.
--- lib/s390x/snippet.h | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-)
lib/s390x/snippet.h +14 -4
@@ -123,13 +123,17 @@ static inline void snippet_pv_init(struct vm *vm, const char *gbin, }/* Allocates and sets up a snippet based guest */ -static inline void snippet_setup_guest (struct vm *vm,bool is_pv) +static inline void snippet_setup_guest_len (struct vm *vm,bool is_pv, +unsigned long len) { -const unsigned long guest_size =SZ_1M ; -uint8_t *guest_start =sie_guest_alloc (guest_size); +uint8_t *guest_start; + +/* Guest sizes are specified in megabyte chunks */ +assert (!(len & ~HPAGE_MASK )); + guest_start =sie_guest_alloc (len);/* Initialize the vm struct and allocate control blocks */ -sie_guest_create (vm, (uint64_t )guest_start, guest_size); +sie_guest_create (vm, (uint64_t )guest_start, len);if (is_pv) {/* FMT4 needs a ESCA */
@@ -144,4 +148,10 @@ static inline void snippet_setup_guest(struct vm *vm, bool is_pv) } } +/* Allocates and sets up a snippet based guest */ +static inline void snippet_setup_guest (struct vm *vm,bool is_pv) +{ +snippet_setup_guest_len (vm, is_pv,SZ_1M ); +} +#endif
Signature · 1 line
2.51.0
Re: [kvm-unit-tests 4/5] lib: s390x: snippet: Add function to create a guest of specific length
Quoted 5 lines · Janosch Frank
While 1MB is certainly enough to store the guest code, it's often not enough for memory tests. Let's add a separate function to allow arbitrary guest sizes.
Signed-off-by Janosch Frank <[email protected]>
PATCH 5/5 lib: s390x: Remove kvm s390 prefix from sie control block
In reply to Janosch FrankWe're not kvm.
--- lib/s390x/asm/sie-arch.h | 2 +- lib/s390x/interrupt.c | 4 ++-- lib/s390x/sie-icpt.c | 2 +- lib/s390x/sie-icpt.h | 2 +- lib/s390x/sie.h | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-)
lib/s390x/asm/sie-arch.h +1 -1
@@ -29,7 +29,7 @@#define CPUSTAT_J 0x00000002#define CPUSTAT_P 0x00000001 -struct kvm_s390_sie_block { +struct sie_control_block {uint32_t cpuflags;/* 0x0000 */ uint32_t : 1; /* 0x0004 */uint32_t prefix :18 ;
lib/s390x/interrupt.c +2 -2
@@ -227,7 +227,7 @@ static void print_storage_exception_information(void)static void print_int_regs (struct stack_frame_int *stack,bool sie) { -struct kvm_s390_sie_block *sblk; +struct sie_control_block *sblk;printf ("\n" );printf ("%s\n" , sie ?"Guest registers:" :"Host registers:" );
@@ -240,7 +240,7 @@ static void print_int_regs(struct stack_frame_int *stack, bool sie) stack->grs0[6], stack->grs0[7], stack->grs0[8], stack->grs0[9]);if (sie) { - sblk = (struct kvm_s390_sie_block *)stack->grs0[12 ]; + sblk = (struct sie_control_block *)stack->grs0[12 ];printf ("%016lx %016lx %016lx %016lx\n" , stack->grs0[10 ], stack->grs0[11 ], sblk->gg14, sblk->gg15); }else {
lib/s390x/sie-icpt.c +1 -1
@@ -7,7 +7,7 @@#include <sie-icpt.h> -struct diag_itext sblk_ip_as_diag (struct kvm_s390_sie_block *sblk) +struct diag_itext sblk_ip_as_diag (struct sie_control_block *sblk) { union { struct {
lib/s390x/sie-icpt.h +1 -1
@@ -21,7 +21,7 @@ struct diag_itext { uint64_t : 16; }; -struct diag_itext sblk_ip_as_diag (struct kvm_s390_sie_block *sblk); +struct diag_itext sblk_ip_as_diag (struct sie_control_block *sblk);/** * sie_is_diag_icpt() - Check if intercept is due to diagnose instruction
lib/s390x/sie.h +2 -2
@@ -32,7 +32,7 @@ struct vm_save_area { };struct vm { -struct kvm_s390_sie_block *sblk; +struct sie_control_block *sblk;struct vm_save_area save_area;struct esca_block *sca;/* System Control Area */ uint8_t *crycb;/* Crypto Control Block */
@@ -46,7 +46,7 @@ extern void sie_entry(void);extern void sie_exit (void );extern void sie_entry_gregs (void );extern void sie_exit_gregs (void ); -extern void sie64a (struct kvm_s390_sie_block *sblk,struct vm_save_area *save_area); +extern void sie64a (struct sie_control_block *sblk,struct vm_save_area *save_area);void sie (struct vm *vm);void sie_expect_validity (struct vm *vm);uint16_t sie_get_validity (struct vm *vm);
Signature · 1 line
2.51.0
Re: [kvm-unit-tests 5/5] lib: s390x: Remove kvm s390 prefix from sie control block
Quoted 3 lines · Janosch Frank
We're not kvm.
Signed-off-by Janosch Frank <[email protected]>
Hm, is this just a cosmetic change?
On one hand this will make it harder to copy things from Linux to kut.
On the other hand, there's already a ton of divergence between Linux and kut. So it already isn't just copy&paste, this will make it just a bit harder.
Re: [kvm-unit-tests 5/5] lib: s390x: Remove kvm s390 prefix from sie control block
Quoted 6 lines · Nico Boehr
Quoted 3 lines · Janosch Frank
We're not kvm.
Signed-off-by Janosch Frank <[email protected]>Hm, is this just a cosmetic change?
Yes
Quoted 6 lines
On one hand this will make it harder to copy things from Linux to kut.
On the other hand, there's already a ton of divergence between Linux and kut. So it already isn't just copy&paste, this will make it just a bit harder.
For sae we already dropped the s390 part in the control block name in KVM. I aimed to have sie_control_block and sae_control_block once sae support is added.
But I'm not too hung up on this patch.