Skip to contentSkip to conversation

KVM: s390: gmap rewrite, the real deal

PATCH Superseded Claudio Imbrenda 9 participants 75 messages Sep 10, 2025 – Oct 27, 2025

Subsystems (7)
Memory management KVM s390 THP +3 more subsystems

v2 of 7 Superseded Latest is v7 →

lore
Share & tools
lore

Claudio Imbrenda

author maintainer
#0 lore raw

v2 cover KVM: s390: gmap rewrite, the real deal

This series is the last big series of the gmap rewrite. It introduces the new code and actually uses it. The old code is then removed.

The insertions/deletions balance is negative both for this series, and for the whole rewrite, also considering all the preparatory patches.

KVM on s390 will now use the mmu_notifier, like most other architectures. The gmap address space is now completely separate from userspace; no level of the page tables is shared between guest mapping and userspace.

One of the biggest advantages is that the page size of userspace is completely independent of the page size used by the guest. Userspace can mix normal pages, THPs, hugetlbfs, and more.

Patches 1 to 6 are mostly preparations; introducing some new bits and functions, and moving code around.

Patch 7 to 14 is the meat of the new gmap code; page table management functions and gmap management. This is the code that will be used to manage guest memory.

Patch 16 is unfortunately big; the existing code is converted to use the new gmap and all references to the old gmap are removed. This needs to be done all at once, unfortunately, hence the size of the patch.

Patch 17 and 18 remove all the now unused code.

Patch 19 and 20 allow for 1M pages to be used to back guests, and add some more functions that are useful for testing.

If you are not in the CC list, you probably have not received v1, as it was not sent out publicly; you did not miss it :) That's also where the various R-Bs come from.

Claudio Imbrenda (20):
KVM: s390: add P bit in table entry bitfields, move union vaddress
s390: Move sske_frame() to a header
KVM: s390: Add gmap_helper_set_unused()
KVM: s390: Enable KVM_GENERIC_MMU_NOTIFIER
KVM: s390: Add helper functions for fault handling
KVM: s390: Rename some functions in gaccess.c
KVM: s390: KVM-specific bitfields and helper functions
KVM: s390: KVM page table management functions: allocation
KVM: s390: KVM page table management functions: clear and replace
KVM: s390: KVM page table management functions: walks
KVM: s390: KVM page table management functions: storage keys
KVM: s390: KVM page table management functions: lifecycle management
KVM: s390: KVM page table management functions: CMMA
KVM: s390: New gmap code
KVM: s390: Stop using CONFIG_PGSTE
KVM: s390: Switch to new gmap
KVM: s390: Remove gmap from s390/mm
KVM: S390: Remove PGSTE code from linux/s390 mm
KVM: s390: Enable 1M pages for gmap
KVM: s390: Storage key manipulation IOCTL

 MAINTAINERS                          |    2 -
 arch/s390/Kconfig                    |    3 -
 arch/s390/include/asm/dat-bits.h     |   32 +-
 arch/s390/include/asm/gmap.h         |  174 --
 arch/s390/include/asm/gmap_helpers.h |    1 +
 arch/s390/include/asm/kvm_host.h     |    1 +
 arch/s390/include/asm/mmu.h          |   13 -
 arch/s390/include/asm/mmu_context.h  |    6 +-
 arch/s390/include/asm/page.h         |    4 -
 arch/s390/include/asm/pgalloc.h      |    2 -
 arch/s390/include/asm/pgtable.h      |  119 +-
 arch/s390/include/asm/tlb.h          |    3 -
 arch/s390/kvm/Kconfig                |    3 +-
 arch/s390/kvm/Makefile               |    3 +-
 arch/s390/kvm/dat.c                  | 1260 +++++++++++++
 arch/s390/kvm/dat.h                  |  843 +++++++++
 arch/s390/kvm/diag.c                 |    2 +-
 arch/s390/kvm/gaccess.c              |  599 ++++---
 arch/s390/kvm/gaccess.h              |   30 +-
 arch/s390/kvm/gmap-vsie.c            |  141 --
 arch/s390/kvm/gmap.c                 | 1066 +++++++++++
 arch/s390/kvm/gmap.h                 |  178 ++
 arch/s390/kvm/intercept.c            |    8 +-
 arch/s390/kvm/interrupt.c            |    2 +-
 arch/s390/kvm/kvm-s390.c             |  808 ++++-----
 arch/s390/kvm/kvm-s390.h             |   75 +-
 arch/s390/kvm/priv.c                 |  206 +--
 arch/s390/kvm/pv.c                   |   87 +-
 arch/s390/kvm/vsie.c                 |  116 +-
 arch/s390/mm/Makefile                |    1 -
 arch/s390/mm/fault.c                 |    4 +-
 arch/s390/mm/gmap.c                  | 2452 --------------------------
 arch/s390/mm/gmap_helpers.c          |   64 +
 arch/s390/mm/hugetlbpage.c           |   24 -
 arch/s390/mm/page-states.c           |    1 +
 arch/s390/mm/pageattr.c              |    7 -
 arch/s390/mm/pgalloc.c               |   29 -
 arch/s390/mm/pgtable.c               |  837 +--------
 include/uapi/linux/kvm.h             |   10 +
 mm/khugepaged.c                      |    9 -
 40 files changed, 4380 insertions(+), 4845 deletions(-)
 delete mode 100644 arch/s390/include/asm/gmap.h
 create mode 100644 arch/s390/kvm/dat.c
 create mode 100644 arch/s390/kvm/dat.h
 delete mode 100644 arch/s390/kvm/gmap-vsie.c
 create mode 100644 arch/s390/kvm/gmap.c
 create mode 100644 arch/s390/kvm/gmap.h
 delete mode 100644 arch/s390/mm/gmap.c
Signature · 1 line
2.51.0
author maintainer
#1 lore raw

PATCH v2 1/20 KVM: s390: add P bit in table entry bitfields, move union vaddress

Add P bit in hardware definition of region 3 and segment table entries.

Move union vaddress from kvm/gaccess.c to asm/dat_bits.h

---
 arch/s390/include/asm/dat-bits.h | 32 ++++++++++++++++++++++++++++++--
 arch/s390/kvm/gaccess.c          | 26 --------------------------
 2 files changed, 30 insertions(+), 28 deletions(-)
arch/s390/include/asm/dat-bits.h +30 -2
index 8d65eec2f124..c40874e0e426 100644
@@ -9,6 +9,32 @@
 #ifndef _S390_DAT_BITS_H
 #define _S390_DAT_BITS_H
 
+/*
+ * vaddress union in order to easily decode a virtual address into its
+ * region first index, region second index etc. parts.
+ */
+union vaddress {
+	unsigned long addr;
+	struct {
+		unsigned long rfx : 11;
+		unsigned long rsx : 11;
+		unsigned long rtx : 11;
+		unsigned long sx  : 11;
+		unsigned long px  : 8;
+		unsigned long bx  : 12;
+	};
+	struct {
+		unsigned long rfx01 : 2;
+		unsigned long	    : 9;
+		unsigned long rsx01 : 2;
+		unsigned long	    : 9;
+		unsigned long rtx01 : 2;
+		unsigned long	    : 9;
+		unsigned long sx01  : 2;
+		unsigned long	    : 29;
+	};
+};
+
 union asce {
 	unsigned long val;
 	struct {
@@ -98,7 +124,8 @@ union region3_table_entry {
 	struct {
 		unsigned long	: 53;
 		unsigned long fc: 1; /* Format-Control */
-		unsigned long	: 4;
+		unsigned long p : 1; /* DAT-Protection Bit */
+		unsigned long	: 3;
 		unsigned long i : 1; /* Region-Invalid Bit */
 		unsigned long cr: 1; /* Common-Region Bit */
 		unsigned long tt: 2; /* Table-Type Bits */
@@ -140,7 +167,8 @@ union segment_table_entry {
 	struct {
 		unsigned long	: 53;
 		unsigned long fc: 1; /* Format-Control */
-		unsigned long	: 4;
+		unsigned long p : 1; /* DAT-Protection Bit */
+		unsigned long	: 3;
 		unsigned long i : 1; /* Segment-Invalid Bit */
 		unsigned long cs: 1; /* Common-Segment Bit */
 		unsigned long tt: 2; /* Table-Type Bits */
arch/s390/kvm/gaccess.c +0 -26
index 21c2e61fece4..d691fac1cc12 100644
@@ -20,32 +20,6 @@
 
 #define GMAP_SHADOW_FAKE_TABLE 1ULL
 
-/*
- * vaddress union in order to easily decode a virtual address into its
- * region first index, region second index etc. parts.
- */
-union vaddress {
-	unsigned long addr;
-	struct {
-		unsigned long rfx : 11;
-		unsigned long rsx : 11;
-		unsigned long rtx : 11;
-		unsigned long sx  : 11;
-		unsigned long px  : 8;
-		unsigned long bx  : 12;
-	};
-	struct {
-		unsigned long rfx01 : 2;
-		unsigned long	    : 9;
-		unsigned long rsx01 : 2;
-		unsigned long	    : 9;
-		unsigned long rtx01 : 2;
-		unsigned long	    : 9;
-		unsigned long sx01  : 2;
-		unsigned long	    : 29;
-	};
-};
-
 /*
  * raddress union which will contain the result (real or absolute address)
  * after a page table walk. The rfaa, sfaa and pfra members are used to
Signature · 1 line
2.51.0
author maintainer
#2 lore raw

PATCH v2 2/20 s390: Move sske_frame() to a header

In reply to Claudio Imbrenda

Move the sske_frame() function to asm/pgtable.h, so it can be used in other modules too.

---
 arch/s390/include/asm/pgtable.h | 7 +++++++
 arch/s390/mm/pageattr.c         | 7 -------
 2 files changed, 7 insertions(+), 7 deletions(-)
arch/s390/include/asm/pgtable.h +7 -0
index 6d8bc27a366e..3c74b39bf669 100644
@@ -1142,6 +1142,13 @@ static inline pte_t pte_mkhuge(pte_t pte)
 }
 #endif
 
+static inline unsigned long sske_frame(unsigned long addr, unsigned char skey)
+{
+	asm volatile(".insn rrf,0xb22b0000,%[skey],%[addr],1,0"
+		     : [addr] "+a" (addr) : [skey] "d" (skey));
+	return addr;
+}
+
 #define IPTE_GLOBAL	0
 #define	IPTE_LOCAL	1
 
arch/s390/mm/pageattr.c +0 -7
index 348e759840e7..ceeb04136cec 100644
@@ -16,13 +16,6 @@
 #include <asm/asm.h>
 #include <asm/set_memory.h>
 
-static inline unsigned long sske_frame(unsigned long addr, unsigned char skey)
-{
-	asm volatile(".insn rrf,0xb22b0000,%[skey],%[addr],1,0"
-		     : [addr] "+a" (addr) : [skey] "d" (skey));
-	return addr;
-}
-
 void __storage_key_init_range(unsigned long start, unsigned long end)
 {
 	unsigned long boundary, size;
Signature · 1 line
2.51.0
author maintainer
#3 lore raw

PATCH v2 3/20 KVM: s390: Add gmap_helper_set_unused()

In reply to Claudio Imbrenda

Add gmap_helper_set_unused() to mark userspace ptes as unused.

Core mm code will use that information to discard unused pages instead of attempting to swap them.

---
 arch/s390/include/asm/gmap_helpers.h |  1 +
 arch/s390/mm/gmap_helpers.c          | 64 ++++++++++++++++++++++++++++
 2 files changed, 65 insertions(+)
arch/s390/include/asm/gmap_helpers.h +1 -0
index 5356446a61c4..459bd39d0887 100644
@@ -11,5 +11,6 @@
 void gmap_helper_zap_one_page(struct mm_struct *mm, unsigned long vmaddr);
 void gmap_helper_discard(struct mm_struct *mm, unsigned long vmaddr, unsigned long end);
 int gmap_helper_disable_cow_sharing(void);
+void gmap_helper_set_unused(struct mm_struct *mm, unsigned long vmaddr);
 
 #endif /* _ASM_S390_GMAP_HELPERS_H */
arch/s390/mm/gmap_helpers.c +64 -0

70 lines not on this page · raw message

Signature · 1 line
2.51.0
Reviewed-by Tested-by on v2 3/20
#4 lore raw

Re: [PATCH v2 03/20] KVM: s390: Add gmap_helper_set_unused()

Quoted 6 lines · Claudio Imbrenda

Add gmap_helper_set_unused() to mark userspace ptes as unused.

Core mm code will use that information to discard unused pages instead of attempting to swap them.

I observed 21965 calls to gmap_helper_set_unused() where we were successfully traversing the page tables to the PTE; of these 21868 made it to the part where _PAGE_UNUSED is set. So _rarely_ we seem to be unable to take the lock (or the PMD changed under us), but in the majority of cases we make it, so this seems fine to me.

#5 lore raw

Re: [PATCH v2 03/20] KVM: s390: Add gmap_helper_set_unused()

In reply to Claudio Imbrenda
Quoted 6 lines · Claudio Imbrenda

Add gmap_helper_set_unused() to mark userspace ptes as unused.

Core mm code will use that information to discard unused pages instead of attempting to swap them.

LGTM

Quoted 31 lines
---
 arch/s390/include/asm/gmap_helpers.h |  1 +
 arch/s390/mm/gmap_helpers.c          | 64 ++++++++++++++++++++++++++++
 2 files changed, 65 insertions(+)
arch/s390/include/asm/gmap_helpers.h +1 -0
index 5356446a61c4..459bd39d0887 100644
@@ -11,5 +11,6 @@
 void gmap_helper_zap_one_page(struct mm_struct *mm, unsigned long vmaddr);
 void gmap_helper_discard(struct mm_struct *mm, unsigned long vmaddr, unsigned long end);
 int gmap_helper_disable_cow_sharing(void);
+void gmap_helper_set_unused(struct mm_struct *mm, unsigned long vmaddr);
 
 #endif /* _ASM_S390_GMAP_HELPERS_H */
arch/s390/mm/gmap_helpers.c +7 -0
index a45d417ad951..69ffc0c6b654 100644
@@ -91,6 +91,70 @@ void gmap_helper_discard(struct mm_struct *mm, unsigned long vmaddr, unsigned lo
 }
 EXPORT_SYMBOL_GPL(gmap_helper_discard);
 
+/**
+ * gmap_helper_set_unused() - mark a pte entry as unused
+ * @mm: the mm
+ * @vmaddr: the userspace address whose pte is to be marked
+ *
+ * Mark the pte corresponding the given address as unused. This will cause
+ * core mm code to just drop this page instead of swapping it.

arch/s390/mm/gmap_helpers.c void gmap_helper_discard(struct mm_struct *mm, unsigned long vmaddr, unsigned lo · line 110

Quoted 10 lines
arch/s390/mm/gmap_helpers.c +10 -0
+ *
+ * This function needs to be called with interrupts disabled (for example
+ * while holding a spinlock), or while holding the mmap lock. Normally this
+ * function is called as a result of an unmap operation, and thus KVM common
+ * code will already hold kvm->mmu_lock in write mode.
+ *
+ * Context: Needs to be called while holding the mmap lock or with interrupts
+ *          disabled.
+ */
+void gmap_helper_set_unused(struct mm_struct *mm, unsigned long vmaddr)

Can you give this a better name? E.g. gmap_helper_try_set_pte_unused

Quoted 23 lines
arch/s390/mm/gmap_helpers.c +23 -0
+{
+	pmd_t *pmdp, pmd, pmdval;
+	pud_t *pudp, pud;
+	p4d_t *p4dp, p4d;
+	pgd_t *pgdp, pgd;
+	spinlock_t *ptl;
+	pte_t *ptep;
+
+	pgdp = pgd_offset(mm, vmaddr);
+	pgd = pgdp_get(pgdp);
+	if (pgd_none(pgd) || !pgd_present(pgd))
+		return;
+
+	p4dp = p4d_offset(pgdp, vmaddr);
+	p4d = p4dp_get(p4dp);
+	if (p4d_none(p4d) || !p4d_present(p4d))
+		return;
+
+	pudp = pud_offset(p4dp, vmaddr);
+	pud = pudp_get(pudp);
+	if (pud_none(pud) || pud_leaf(pud) || !pud_present(pud))
+		return;
+

arch/s390/mm/gmap_helpers.c void gmap_helper_discard(struct mm_struct *mm, unsigned long vmaddr, unsigned lo · line 143

Quoted 10 lines
arch/s390/mm/gmap_helpers.c +10 -0
+	pmdp = pmd_offset(pudp, vmaddr);
+	pmd = pmdp_get_lockless(pmdp);
+	if (pmd_none(pmd) || pmd_leaf(pmd) || !pmd_present(pmd))
+		return;
+
+	ptep = pte_offset_map_rw_nolock(mm, pmdp, vmaddr, &pmdval, &ptl);
+	if (!ptep)
+		return;
+
+	if (spin_trylock(ptl)) {

Missing the comment you promised :) about deadlock prevention.

arch/s390/mm/gmap_helpers.c void gmap_helper_discard(struct mm_struct *mm, unsigned long vmaddr, unsigned lo · line 146

Quoted 3 lines
arch/s390/mm/gmap_helpers.c +3 -0
+		/*
+		 * Make sure the pte we are touching is still the correct
+		 * one. In theory this check should not be needed, but

Why should it not be needed? I.e. why should we be protected against modification?

Quoted 14 lines
arch/s390/mm/gmap_helpers.c +11 -0
+		 * better safe than sorry.
+		 */
+		if (likely(pmd_same(pmdval, pmdp_get_lockless(pmdp))))
+			__atomic64_or(_PAGE_UNUSED, (long *)ptep);
+		spin_unlock(ptl);
+	}
+
+	pte_unmap(ptep);
+}
+EXPORT_SYMBOL_GPL(gmap_helper_set_unused);
+
 static int find_zeropage_pte_entry(pte_t *pte, unsigned long addr,
 				   unsigned long end, struct mm_walk *walk)
 {
Signature · 4 lines
IBM Deutschland Research & Development GmbH
Vorsitzender des Aufsichtsrats: Wolfgang Wendt
Geschäftsführung: David Faller
Sitz der Gesellschaft: Böblingen / Registergericht: Amtsgericht Stuttgart, HRB 243294
author maintainer
#6 lore raw

Re: [PATCH v2 03/20] KVM: s390: Add gmap_helper_set_unused()

Quoted 44 lines · On Fri, 12 Sep 2025 11:17:02 +0200 Nina Schoetterl-Glausch <[email protected]>…
Quoted 6 lines · Claudio Imbrenda

Add gmap_helper_set_unused() to mark userspace ptes as unused.

Core mm code will use that information to discard unused pages instead of attempting to swap them.

LGTM

Quoted 35 lines
---
 arch/s390/include/asm/gmap_helpers.h |  1 +
 arch/s390/mm/gmap_helpers.c          | 64 ++++++++++++++++++++++++++++
 2 files changed, 65 insertions(+)
arch/s390/include/asm/gmap_helpers.h +1 -0
index 5356446a61c4..459bd39d0887 100644
@@ -11,5 +11,6 @@
 void gmap_helper_zap_one_page(struct mm_struct *mm, unsigned long vmaddr);
 void gmap_helper_discard(struct mm_struct *mm, unsigned long vmaddr, unsigned long end);
 int gmap_helper_disable_cow_sharing(void);
+void gmap_helper_set_unused(struct mm_struct *mm, unsigned long vmaddr);
 
 #endif /* _ASM_S390_GMAP_HELPERS_H */
arch/s390/mm/gmap_helpers.c +11 -0
index a45d417ad951..69ffc0c6b654 100644
@@ -91,6 +91,70 @@ void gmap_helper_discard(struct mm_struct *mm, unsigned long vmaddr, unsigned lo
 }
 EXPORT_SYMBOL_GPL(gmap_helper_discard);
 
+/**
+ * gmap_helper_set_unused() - mark a pte entry as unused
+ * @mm: the mm
+ * @vmaddr: the userspace address whose pte is to be marked
+ *
+ * Mark the pte corresponding the given address as unused. This will cause
+ * core mm code to just drop this page instead of swapping it.
+ *
+ * This function needs to be called with interrupts disabled (for example
+ * while holding a spinlock), or while holding the mmap lock. Normally this
+ * function is called as a result of an unmap operation, and thus KVM common
Quoted 8 lines

arch/s390/mm/gmap_helpers.c void gmap_helper_discard(struct mm_struct *mm, unsigned long vmaddr, unsigned lo · line 110

Quoted 6 lines
arch/s390/mm/gmap_helpers.c +6 -0
+ * code will already hold kvm->mmu_lock in write mode.
+ *
+ * Context: Needs to be called while holding the mmap lock or with interrupts
+ *          disabled.
+ */
+void gmap_helper_set_unused(struct mm_struct *mm, unsigned long vmaddr)  

Can you give this a better name? E.g. gmap_helper_try_set_pte_unused

yes

Quoted 28 lines
Quoted 28 lines
arch/s390/mm/gmap_helpers.c +27 -0
+{
+	pmd_t *pmdp, pmd, pmdval;
+	pud_t *pudp, pud;
+	p4d_t *p4dp, p4d;
+	pgd_t *pgdp, pgd;
+	spinlock_t *ptl;
+	pte_t *ptep;
+
+	pgdp = pgd_offset(mm, vmaddr);
+	pgd = pgdp_get(pgdp);
+	if (pgd_none(pgd) || !pgd_present(pgd))
+		return;
+
+	p4dp = p4d_offset(pgdp, vmaddr);
+	p4d = p4dp_get(p4dp);
+	if (p4d_none(p4d) || !p4d_present(p4d))
+		return;
+
+	pudp = pud_offset(p4dp, vmaddr);
+	pud = pudp_get(pudp);
+	if (pud_none(pud) || pud_leaf(pud) || !pud_present(pud))
+		return;
+
+	pmdp = pmd_offset(pudp, vmaddr);
+	pmd = pmdp_get_lockless(pmdp);
+	if (pmd_none(pmd) || pmd_leaf(pmd) || !pmd_present(pmd))
+		return;
Quoted 8 lines

arch/s390/mm/gmap_helpers.c void gmap_helper_discard(struct mm_struct *mm, unsigned long vmaddr, unsigned lo · line 143

Quoted 6 lines
arch/s390/mm/gmap_helpers.c +6 -0
+
+	ptep = pte_offset_map_rw_nolock(mm, pmdp, vmaddr, &pmdval, &ptl);
+	if (!ptep)
+		return;
+
+	if (spin_trylock(ptl)) {  

Missing the comment you promised :) about deadlock prevention.

Ooops! will fix

Quoted 6 lines

arch/s390/mm/gmap_helpers.c void gmap_helper_discard(struct mm_struct *mm, unsigned long vmaddr, unsigned lo · line 146

Quoted 4 lines
arch/s390/mm/gmap_helpers.c +3 -0
+		/*
+		 * Make sure the pte we are touching is still the correct
+		 * one. In theory this check should not be needed, but  

Why should it not be needed? I.e. why should we be protected against modification?

I will add this in a comment:

a pmd pointing to a page table can change in only very few cases, and all cases will take the mm->mmap_lock in write mode and require IPC synchronization, which means that as long as interrupts are disabled or we are holding the mmap_lock, the pmd cannot change under our feet.

by keeping interrupts disabled, we are basically stalling any remote CPUs that might want to change the pmd, as the IPC will not complete until we re-enable them.

in our case, we call this function after calling pgste_get_lock(), which will disable interrupts until pgste_set_unlock() is called.

Quoted 15 lines
Quoted 14 lines
arch/s390/mm/gmap_helpers.c +11 -0
+		 * better safe than sorry.
+		 */
+		if (likely(pmd_same(pmdval, pmdp_get_lockless(pmdp))))
+			__atomic64_or(_PAGE_UNUSED, (long *)ptep);
+		spin_unlock(ptl);
+	}
+
+	pte_unmap(ptep);
+}
+EXPORT_SYMBOL_GPL(gmap_helper_set_unused);
+
 static int find_zeropage_pte_entry(pte_t *pte, unsigned long addr,
 				   unsigned long end, struct mm_walk *walk)
 {  
#7 lore raw

Re: [PATCH v2 03/20] KVM: s390: Add gmap_helper_set_unused()

Quoted 115 lines · Claudio Imbrenda
Quoted 44 lines · On Fri, 12 Sep 2025 11:17:02 +0200 Nina Schoetterl-Glausch <[email protected]>…
Quoted 6 lines · Claudio Imbrenda

Add gmap_helper_set_unused() to mark userspace ptes as unused.

Core mm code will use that information to discard unused pages instead of attempting to swap them.

LGTM

Quoted 35 lines
---
 arch/s390/include/asm/gmap_helpers.h |  1 +
 arch/s390/mm/gmap_helpers.c          | 64 ++++++++++++++++++++++++++++
 2 files changed, 65 insertions(+)
arch/s390/include/asm/gmap_helpers.h +1 -0
index 5356446a61c4..459bd39d0887 100644
@@ -11,5 +11,6 @@
 void gmap_helper_zap_one_page(struct mm_struct *mm, unsigned long vmaddr);
 void gmap_helper_discard(struct mm_struct *mm, unsigned long vmaddr, unsigned long end);
 int gmap_helper_disable_cow_sharing(void);
+void gmap_helper_set_unused(struct mm_struct *mm, unsigned long vmaddr);
 
 #endif /* _ASM_S390_GMAP_HELPERS_H */
arch/s390/mm/gmap_helpers.c +11 -0
index a45d417ad951..69ffc0c6b654 100644
@@ -91,6 +91,70 @@ void gmap_helper_discard(struct mm_struct *mm, unsigned long vmaddr, unsigned lo
 }
 EXPORT_SYMBOL_GPL(gmap_helper_discard);
 
+/**
+ * gmap_helper_set_unused() - mark a pte entry as unused
+ * @mm: the mm
+ * @vmaddr: the userspace address whose pte is to be marked
+ *
+ * Mark the pte corresponding the given address as unused. This will cause
+ * core mm code to just drop this page instead of swapping it.
+ *
+ * This function needs to be called with interrupts disabled (for example
+ * while holding a spinlock), or while holding the mmap lock. Normally this
+ * function is called as a result of an unmap operation, and thus KVM common
Quoted 8 lines

arch/s390/mm/gmap_helpers.c void gmap_helper_discard(struct mm_struct *mm, unsigned long vmaddr, unsigned lo · line 110

Quoted 6 lines
arch/s390/mm/gmap_helpers.c +6 -0
+ * code will already hold kvm->mmu_lock in write mode.
+ *
+ * Context: Needs to be called while holding the mmap lock or with interrupts
+ *          disabled.
+ */
+void gmap_helper_set_unused(struct mm_struct *mm, unsigned long vmaddr)  

Can you give this a better name? E.g. gmap_helper_try_set_pte_unused

yes

Quoted 28 lines
Quoted 28 lines
arch/s390/mm/gmap_helpers.c +27 -0
+{
+	pmd_t *pmdp, pmd, pmdval;
+	pud_t *pudp, pud;
+	p4d_t *p4dp, p4d;
+	pgd_t *pgdp, pgd;
+	spinlock_t *ptl;
+	pte_t *ptep;
+
+	pgdp = pgd_offset(mm, vmaddr);
+	pgd = pgdp_get(pgdp);
+	if (pgd_none(pgd) || !pgd_present(pgd))
+		return;
+
+	p4dp = p4d_offset(pgdp, vmaddr);
+	p4d = p4dp_get(p4dp);
+	if (p4d_none(p4d) || !p4d_present(p4d))
+		return;
+
+	pudp = pud_offset(p4dp, vmaddr);
+	pud = pudp_get(pudp);
+	if (pud_none(pud) || pud_leaf(pud) || !pud_present(pud))
+		return;
+
+	pmdp = pmd_offset(pudp, vmaddr);
+	pmd = pmdp_get_lockless(pmdp);
+	if (pmd_none(pmd) || pmd_leaf(pmd) || !pmd_present(pmd))
+		return;
Quoted 8 lines

arch/s390/mm/gmap_helpers.c void gmap_helper_discard(struct mm_struct *mm, unsigned long vmaddr, unsigned lo · line 143

Quoted 6 lines
arch/s390/mm/gmap_helpers.c +6 -0
+
+	ptep = pte_offset_map_rw_nolock(mm, pmdp, vmaddr, &pmdval, &ptl);
+	if (!ptep)
+		return;
+
+	if (spin_trylock(ptl)) {  

Missing the comment you promised :) about deadlock prevention.

Ooops! will fix

Quoted 6 lines

arch/s390/mm/gmap_helpers.c void gmap_helper_discard(struct mm_struct *mm, unsigned long vmaddr, unsigned lo · line 146

Quoted 4 lines
arch/s390/mm/gmap_helpers.c +3 -0
+		/*
+		 * Make sure the pte we are touching is still the correct
+		 * one. In theory this check should not be needed, but  

Why should it not be needed? I.e. why should we be protected against modification?

I will add this in a comment:

a pmd pointing to a page table can change in only very few cases, and all cases will take the mm->mmap_lock in write mode and require IPC synchronization, which means that as long as interrupts are disabled or we are holding the mmap_lock, the pmd cannot change under our feet.

by keeping interrupts disabled, we are basically stalling any remote CPUs that might want to change the pmd, as the IPC will not complete until we re-enable them.

Quoted 7 lines

in our case, we call this function after calling pgste_get_lock(), which will disable interrupts until pgste_set_unlock() is called.

Quoted 4 lines

arch/s390/mm/gmap_helpers.c void gmap_helper_discard(struct mm_struct *mm, unsigned long vmaddr, unsigned lo · line 150

Quoted 4 lines
arch/s390/mm/gmap_helpers.c +4 -0
+		 * better safe than sorry.
+		 */
+		if (likely(pmd_same(pmdval, pmdp_get_lockless(pmdp))))
+			__atomic64_or(_PAGE_UNUSED, (long *)ptep);

Should we do a WARN_ONCE? Would we be notified if it shows up in CI?
Is the "if" even meaningful? That is, if the pmd is not the same for what ever reason, are we guaranteed to see it here.
Does taking the page table lock also lock the pmd table? (I assume not)
What are the consequences of the or if the pmd is not the same, arbitrary memory corruption?

Quoted 11 lines
Quoted 11 lines
Quoted 10 lines
arch/s390/mm/gmap_helpers.c +7 -0
+		spin_unlock(ptl);
+	}
+
+	pte_unmap(ptep);
+}
+EXPORT_SYMBOL_GPL(gmap_helper_set_unused);
+
 static int find_zeropage_pte_entry(pte_t *pte, unsigned long addr,
 				   unsigned long end, struct mm_walk *walk)
 {  
Signature · 4 lines
IBM Deutschland Research & Development GmbH
Vorsitzender des Aufsichtsrats: Wolfgang Wendt
Geschäftsführung: David Faller
Sitz der Gesellschaft: Böblingen / Registergericht: Amtsgericht Stuttgart, HRB 243294
author maintainer
#8 lore raw

PATCH v2 4/20 KVM: s390: Enable KVM_GENERIC_MMU_NOTIFIER

In reply to Claudio Imbrenda

Enable KVM_GENERIC_MMU_NOTIFIER, for now with empty placeholder callbacks.

Also enable KVM_MMU_LOCKLESS_AGING and define KVM_HAVE_MMU_RWLOCK.

---
 arch/s390/include/asm/kvm_host.h |  1 +
 arch/s390/kvm/Kconfig            |  3 ++-
 arch/s390/kvm/kvm-s390.c         | 45 +++++++++++++++++++++++++++++++-
 3 files changed, 47 insertions(+), 2 deletions(-)
arch/s390/include/asm/kvm_host.h +1 -0
index cb89e54ada25..9cc61deaaf29 100644
@@ -27,6 +27,7 @@
 #include <asm/isc.h>
 #include <asm/guarded_storage.h>
 
+#define KVM_HAVE_MMU_RWLOCK
 #define KVM_MAX_VCPUS 255
 
 #define KVM_INTERNAL_MEM_SLOTS 1
arch/s390/kvm/Kconfig +2 -1
index cae908d64550..e86332b26511 100644
@@ -29,7 +29,8 @@ config KVM
 	select HAVE_KVM_INVALID_WAKEUPS
 	select HAVE_KVM_NO_POLL
 	select KVM_VFIO
-	select MMU_NOTIFIER
+	select KVM_GENERIC_MMU_NOTIFIER
+	select KVM_MMU_LOCKLESS_AGING
 	help
 	  Support hosting paravirtualized guest machines using the SIE
 	  virtualization capability on the mainframe. This should work
arch/s390/kvm/kvm-s390.c +44 -1

57 lines not on this page · raw message

Signature · 1 line
2.51.0
author maintainer
#9 lore raw

PATCH v2 5/20 KVM: s390: Add helper functions for fault handling

In reply to Claudio Imbrenda

Add some helper functions for handling multiple guest faults at the same time.

This will be needed for VSIE, where a nested guest access also needs to access all the page tables that map it.

---
 arch/s390/kvm/gaccess.h  | 14 ++++++++++
 arch/s390/kvm/kvm-s390.c | 44 +++++++++++++++++++++++++++++++
 arch/s390/kvm/kvm-s390.h | 56 ++++++++++++++++++++++++++++++++++++++++
 3 files changed, 114 insertions(+)
arch/s390/kvm/gaccess.h +14 -0
index 3fde45a151f2..9c82f7460821 100644
@@ -457,4 +457,18 @@ int kvm_s390_check_low_addr_prot_real(struct kvm_vcpu *vcpu, unsigned long gra);
 int kvm_s390_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *shadow,
 			  unsigned long saddr, unsigned long *datptr);
 
+static inline int __kvm_s390_faultin_read_gpa(struct kvm *kvm, struct guest_fault *f, gpa_t gaddr,
+					      unsigned long *val)
+{
+	phys_addr_t phys_addr;
+	int rc;
+
+	rc = __kvm_s390_faultin_gfn(kvm, f, gpa_to_gfn(gaddr), false);
+	if (!rc) {
+		phys_addr = PFN_PHYS(f->pfn) | offset_in_page(gaddr);
+		*val = *(unsigned long *)phys_to_virt(phys_addr);
+	}
+	return rc;
+}
+
 #endif /* __KVM_S390_GACCESS_H */
arch/s390/kvm/kvm-s390.c +44 -0

50 lines not on this page · raw message

arch/s390/kvm/kvm-s390.h +56 -0

74 lines not on this page · raw message

Signature · 1 line
2.51.0
#10 lore raw

Re: [PATCH v2 05/20] KVM: s390: Add helper functions for fault handling

Quoted 13 lines · Claudio Imbrenda

Add some helper functions for handling multiple guest faults at the same time.

This will be needed for VSIE, where a nested guest access also needs to access all the page tables that map it.

---
 arch/s390/kvm/gaccess.h  | 14 ++++++++++
 arch/s390/kvm/kvm-s390.c | 44 +++++++++++++++++++++++++++++++
 arch/s390/kvm/kvm-s390.h | 56 ++++++++++++++++++++++++++++++++++++++++
 3 files changed, 114 insertions(+)

[...]

Quoted 6 lines
arch/s390/kvm/kvm-s390.h +0 -0
index c44fe0c3a097..dabcf65f58ff 100644
@@ -22,6 +22,15 @@
 

[...]

Quoted 5 lines
arch/s390/kvm/kvm-s390.h +5 -0
+static inline void release_faultin_multiple(struct kvm *kvm, struct guest_fault *guest_faults,
+					    int n, bool ignore)
+{
+	int i;
+

arch/s390/kvm/kvm-s390.h int kvm_s390_cpus_from_pv(struct kvm *kvm, u16 *rc, u16 *rrc); · line 497

Quoted 10 lines
arch/s390/kvm/kvm-s390.h +10 -0
+	for (i = 0; i < n; i++) {
+		kvm_release_faultin_page(kvm, guest_faults[i].page, ignore,
+					 guest_faults[i].write_attempt);
+		guest_faults[i].page = NULL;
+	}
+}
+
+static inline bool __kvm_s390_multiple_faults_need_retry(struct kvm *kvm, unsigned long seq,
+							 struct guest_fault *guest_faults, int n,
+							 bool unsafe)

The name of the function does not at all suggest that it releases guest pages. Can you remove that and use

if (__kvm_s390_fault_array_needs_retry(...))

	release_faultin_array(...);

in the caller?
(I haven't yet looked at those)
"needs_retry" isn't telling me much right now, either. What is being retried and why?
Comments would not hurt :)

Quoted 34 lines
arch/s390/kvm/kvm-s390.h +31 -0
+{
+	int i;
+
+	for (i = 0; i < n; i++) {
+		if (!guest_faults[i].valid)
+			continue;
+		if ((unsafe && mmu_invalidate_retry_gfn_unsafe(kvm, seq, guest_faults[i].gfn)) ||
+		    (!unsafe && mmu_invalidate_retry_gfn(kvm, seq, guest_faults[i].gfn))) {
+			release_faultin_multiple(kvm, guest_faults, n, true);
+			return true;
+		}
+	}
+	return false;
+}
+
+static inline int __kvm_s390_faultin_gfn_range(struct kvm *kvm, struct guest_fault *guest_faults,
+					       gfn_t start, int n_pages, bool write_attempt)
+{
+	int i, rc = 0;
+
+	for (i = 0; !rc && i < n_pages; i++)
+		rc = __kvm_s390_faultin_gfn(kvm, guest_faults + i, start + i, write_attempt);
+	return rc;
+}
+
+#define release_faultin_array(kvm, array, ignore) \
+	release_faultin_multiple(kvm, array, ARRAY_SIZE(array), ignore)
+
+#define __kvm_s390_fault_array_needs_retry(kvm, seq, array, unsafe) \
+	__kvm_s390_multiple_faults_need_retry(kvm, seq, array, ARRAY_SIZE(array), unsafe)
+
 /* implemented in diag.c */
 int kvm_s390_handle_diag(struct kvm_vcpu *vcpu);
Signature · 4 lines
IBM Deutschland Research & Development GmbH
Vorsitzender des Aufsichtsrats: Wolfgang Wendt
Geschäftsführung: David Faller
Sitz der Gesellschaft: Böblingen / Registergericht: Amtsgericht Stuttgart, HRB 243294
author maintainer
#11 lore raw

Re: [PATCH v2 05/20] KVM: s390: Add helper functions for fault handling

Quoted 43 lines · On Fri, 12 Sep 2025 19:56:02 +0200 Nina Schoetterl-Glausch <[email protected]>…
Quoted 13 lines · Claudio Imbrenda

Add some helper functions for handling multiple guest faults at the same time.

This will be needed for VSIE, where a nested guest access also needs to access all the page tables that map it.

---
 arch/s390/kvm/gaccess.h  | 14 ++++++++++
 arch/s390/kvm/kvm-s390.c | 44 +++++++++++++++++++++++++++++++
 arch/s390/kvm/kvm-s390.h | 56 ++++++++++++++++++++++++++++++++++++++++
 3 files changed, 114 insertions(+)

[...]

Quoted 6 lines
arch/s390/kvm/kvm-s390.h +0 -0
index c44fe0c3a097..dabcf65f58ff 100644
@@ -22,6 +22,15 @@
 

[...]

Quoted 5 lines
arch/s390/kvm/kvm-s390.h +5 -0
+static inline void release_faultin_multiple(struct kvm *kvm, struct guest_fault *guest_faults,
+					    int n, bool ignore)
+{
+	int i;
+

arch/s390/kvm/kvm-s390.h int kvm_s390_cpus_from_pv(struct kvm *kvm, u16 *rc, u16 *rrc); · line 497

Quoted 10 lines
arch/s390/kvm/kvm-s390.h +10 -0
+	for (i = 0; i < n; i++) {
+		kvm_release_faultin_page(kvm, guest_faults[i].page, ignore,
+					 guest_faults[i].write_attempt);
+		guest_faults[i].page = NULL;
+	}
+}
+
+static inline bool __kvm_s390_multiple_faults_need_retry(struct kvm *kvm, unsigned long seq,
+							 struct guest_fault *guest_faults, int n,
+							 bool unsafe)  

The name of the function does not at all suggest that it releases guest pages. Can you remove that and use

Quoted 4 lines

if (__kvm_s390_fault_array_needs_retry(...))

	release_faultin_array(...);

in the caller?

yes and no, it would make the caller harder to read, because other things need to happen too, and it's easier to have a wrapper that takes care of cleanup.

I can surely give the function a better name, and add some comments; if you insist, I can also split it as you mentioned above -- if you think it's more readable after splitting, then so be it

Quoted 3 lines

(I haven't yet looked at those)
"needs_retry" isn't telling me much right now, either. What is being retried and why?

that's the name of the already existing common code function (actually, the existing function has an even more confusing name)

the whole fault-in needs to be retried; something happened while we were not holding locks, we don't know what, and the physical address we want to fault-in might now be incorrect. hence we need to retry

Quoted 1 line

Comments would not hurt :)

will add some

Quoted 36 lines
Quoted 35 lines
arch/s390/kvm/kvm-s390.h +31 -0
+{
+	int i;
+
+	for (i = 0; i < n; i++) {
+		if (!guest_faults[i].valid)
+			continue;
+		if ((unsafe && mmu_invalidate_retry_gfn_unsafe(kvm, seq, guest_faults[i].gfn)) ||
+		    (!unsafe && mmu_invalidate_retry_gfn(kvm, seq, guest_faults[i].gfn))) {
+			release_faultin_multiple(kvm, guest_faults, n, true);
+			return true;
+		}
+	}
+	return false;
+}
+
+static inline int __kvm_s390_faultin_gfn_range(struct kvm *kvm, struct guest_fault *guest_faults,
+					       gfn_t start, int n_pages, bool write_attempt)
+{
+	int i, rc = 0;
+
+	for (i = 0; !rc && i < n_pages; i++)
+		rc = __kvm_s390_faultin_gfn(kvm, guest_faults + i, start + i, write_attempt);
+	return rc;
+}
+
+#define release_faultin_array(kvm, array, ignore) \
+	release_faultin_multiple(kvm, array, ARRAY_SIZE(array), ignore)
+
+#define __kvm_s390_fault_array_needs_retry(kvm, seq, array, unsafe) \
+	__kvm_s390_multiple_faults_need_retry(kvm, seq, array, ARRAY_SIZE(array), unsafe)
+
 /* implemented in diag.c */
 int kvm_s390_handle_diag(struct kvm_vcpu *vcpu);
#12 lore raw

Re: [PATCH v2 05/20] KVM: s390: Add helper functions for fault handling

In reply to Claudio Imbrenda

arch/s390/kvm/gaccess.h int kvm_s390_check_low_addr_prot_real(struct kvm_vcpu *vcpu, unsigned long gra); · line 468

Quoted 9 lines · Claudio Imbrenda
arch/s390/kvm/gaccess.h +9 -0
+static inline int __kvm_s390_faultin_read_gpa(struct kvm *kvm, struct guest_fault *f, gpa_t gaddr,
+					      unsigned long *val)
+{
+	phys_addr_t phys_addr;
+	int rc;
+
+	rc = __kvm_s390_faultin_gfn(kvm, f, gpa_to_gfn(gaddr), false);
+	if (!rc) {
+		phys_addr = PFN_PHYS(f->pfn) | offset_in_page(gaddr);

pfn_to_phys() is more consistent with the phys_to_virt() below ;)

Quoted 6 lines
arch/s390/kvm/gaccess.h +5 -0
+		*val = *(unsigned long *)phys_to_virt(phys_addr);
+	}
+	return rc;
+}
+
 #endif /* __KVM_S390_GACCESS_H */

...

Quoted 14 lines
arch/s390/kvm/kvm-s390.h +14 -0
+static inline void release_faultin_multiple(struct kvm *kvm, struct guest_fault *guest_faults,
+					    int n, bool ignore)
+{
+	int i;
+
+	for (i = 0; i < n; i++) {
+		kvm_release_faultin_page(kvm, guest_faults[i].page, ignore,
+					 guest_faults[i].write_attempt);
+		guest_faults[i].page = NULL;
+	}
+}
+
+static inline bool __kvm_s390_multiple_faults_need_retry(struct kvm *kvm, unsigned long seq,
+							 struct guest_fault *guest_faults, int n,

arch/s390/kvm/kvm-s390.h int kvm_s390_cpus_from_pv(struct kvm *kvm, u16 *rc, u16 *rrc); · line 506

Quoted 10 lines
arch/s390/kvm/kvm-s390.h +10 -0
+							 bool unsafe)
+{
+	int i;
+
+	for (i = 0; i < n; i++) {
+		if (!guest_faults[i].valid)
+			continue;
+		if ((unsafe && mmu_invalidate_retry_gfn_unsafe(kvm, seq, guest_faults[i].gfn)) ||
+		    (!unsafe && mmu_invalidate_retry_gfn(kvm, seq, guest_faults[i].gfn))) {
+			release_faultin_multiple(kvm, guest_faults, n, true);

Calling release_faultin_multiple() on the whole range, before all gfns invalidated? Tolerate invalidation of entries that are (!guest_faults[i].valid)? But then why the continue above?

This function is a mistery to me, but that is - I am sure - due to my KVM ignorance...

arch/s390/kvm/kvm-s390.h int kvm_s390_cpus_from_pv(struct kvm *kvm, u16 *rc, u16 *rrc); · line 518

Quoted 12 lines
arch/s390/kvm/kvm-s390.h +12 -0
+			return true;
+		}
+	}
+	return false;
+}
+
+static inline int __kvm_s390_faultin_gfn_range(struct kvm *kvm, struct guest_fault *guest_faults,
+					       gfn_t start, int n_pages, bool write_attempt)
+{
+	int i, rc = 0;
+
+	for (i = 0; !rc && i < n_pages; i++)

Unless n_pages could ever be zero, the below reads better.

arch/s390/kvm/kvm-s390.h int kvm_s390_cpus_from_pv(struct kvm *kvm, u16 *rc, u16 *rrc); · line 519

Quoted 1 line
arch/s390/kvm/kvm-s390.h +1 -0
+		rc = __kvm_s390_faultin_gfn(kvm, guest_faults + i, start + i, write_attempt);
		if (rc)
			break;

Review

Quoted 2 lines
(quoted diff) +2 -0
+	return rc;
+}

...

Thanks!

author maintainer
#13 lore raw

Re: [PATCH v2 05/20] KVM: s390: Add helper functions for fault handling

Quoted 39 lines · On Thu, 18 Sep 2025 16:19:07 +0200 Alexander Gordeev <[email protected]>…

arch/s390/kvm/gaccess.h int kvm_s390_check_low_addr_prot_real(struct kvm_vcpu *vcpu, unsigned long gra); · line 468

Quoted 9 lines · Claudio Imbrenda
arch/s390/kvm/gaccess.h +9 -0
+static inline int __kvm_s390_faultin_read_gpa(struct kvm *kvm, struct guest_fault *f, gpa_t gaddr,
+					      unsigned long *val)
+{
+	phys_addr_t phys_addr;
+	int rc;
+
+	rc = __kvm_s390_faultin_gfn(kvm, f, gpa_to_gfn(gaddr), false);
+	if (!rc) {
+		phys_addr = PFN_PHYS(f->pfn) | offset_in_page(gaddr);  

pfn_to_phys() is more consistent with the phys_to_virt() below ;)

Quoted 6 lines
arch/s390/kvm/gaccess.h +5 -0
+		*val = *(unsigned long *)phys_to_virt(phys_addr);
+	}
+	return rc;
+}
+
 #endif /* __KVM_S390_GACCESS_H */  

...

Quoted 19 lines
arch/s390/kvm/kvm-s390.h +19 -0
+static inline void release_faultin_multiple(struct kvm *kvm, struct guest_fault *guest_faults,
+					    int n, bool ignore)
+{
+	int i;
+
+	for (i = 0; i < n; i++) {
+		kvm_release_faultin_page(kvm, guest_faults[i].page, ignore,
+					 guest_faults[i].write_attempt);
+		guest_faults[i].page = NULL;
+	}
+}
+
+static inline bool __kvm_s390_multiple_faults_need_retry(struct kvm *kvm, unsigned long seq,
+							 struct guest_fault *guest_faults, int n,
+							 bool unsafe)
+{
+	int i;
+
+	for (i = 0; i < n; i++) {
Quoted 8 lines

arch/s390/kvm/kvm-s390.h int kvm_s390_cpus_from_pv(struct kvm *kvm, u16 *rc, u16 *rrc); · line 506

Quoted 5 lines
arch/s390/kvm/kvm-s390.h +5 -0
+		if (!guest_faults[i].valid)
+			continue;
+		if ((unsafe && mmu_invalidate_retry_gfn_unsafe(kvm, seq, guest_faults[i].gfn)) ||
+		    (!unsafe && mmu_invalidate_retry_gfn(kvm, seq, guest_faults[i].gfn))) {
+			release_faultin_multiple(kvm, guest_faults, n, true);  

Calling release_faultin_multiple() on the whole range, before all gfns invalidated?

nothing is getting invalidated here, only a check whether the gfns (may) already been invalidated. in which case we give up and release the whole range

kvm_release_faultin_page() will return and do nothing if the struct page * is NULL

I guess the name of the function in common code is misleading, I will add some comments to explain what's going on here

Quoted 20 lines

Tolerate invalidation of entries that are (!guest_faults[i].valid)? But then why the continue above?

This function is a mistery to me, but that is - I am sure - due to my KVM ignorance...

arch/s390/kvm/kvm-s390.h int kvm_s390_cpus_from_pv(struct kvm *kvm, u16 *rc, u16 *rrc); · line 518

Quoted 12 lines
arch/s390/kvm/kvm-s390.h +12 -0
+			return true;
+		}
+	}
+	return false;
+}
+
+static inline int __kvm_s390_faultin_gfn_range(struct kvm *kvm, struct guest_fault *guest_faults,
+					       gfn_t start, int n_pages, bool write_attempt)
+{
+	int i, rc = 0;
+
+	for (i = 0; !rc && i < n_pages; i++)  

Unless n_pages could ever be zero, the below reads better.

Quoted 4 lines

arch/s390/kvm/kvm-s390.h int kvm_s390_cpus_from_pv(struct kvm *kvm, u16 *rc, u16 *rrc); · line 519

Quoted 1 line
arch/s390/kvm/kvm-s390.h +1 -0
+		rc = __kvm_s390_faultin_gfn(kvm, guest_faults + i, start + i, write_attempt);  
		if (rc)
			break;

yeah I guess it's more readable that way

Quoted 6 lines

Review

Quoted 3 lines
(quoted diff) +2 -0
+	return rc;
+}  

...

Thanks!

#14 lore raw

Re: [PATCH v2 05/20] KVM: s390: Add helper functions for fault handling

In reply to Claudio Imbrenda
Quoted 18 lines · Claudio Imbrenda

Add some helper functions for handling multiple guest faults at the same time.

This will be needed for VSIE, where a nested guest access also needs to access all the page tables that map it.

---
 arch/s390/kvm/gaccess.h  | 14 ++++++++++
 arch/s390/kvm/kvm-s390.c | 44 +++++++++++++++++++++++++++++++
 arch/s390/kvm/kvm-s390.h | 56 ++++++++++++++++++++++++++++++++++++++++
 3 files changed, 114 insertions(+)
arch/s390/kvm/gaccess.h +0 -0
index 3fde45a151f2..9c82f7460821 100644

arch/s390/kvm/gaccess.h int kvm_s390_check_low_addr_prot_real(struct kvm_vcpu *vcpu, unsigned long gra); · line 466

Quoted 10 lines
arch/s390/kvm/gaccess.h +7 -0
@@ -457,4 +457,18 @@ int kvm_s390_check_low_addr_prot_real(struct kvm_vcpu *vcpu, unsigned long gra);
 int kvm_s390_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *shadow,
 			  unsigned long saddr, unsigned long *datptr);
 
+static inline int __kvm_s390_faultin_read_gpa(struct kvm *kvm, struct guest_fault *f, gpa_t gaddr,
+					      unsigned long *val)
+{
+	phys_addr_t phys_addr;
+	int rc;
+
+	rc = __kvm_s390_faultin_gfn(kvm, f, gpa_to_gfn(gaddr), false);

Why not a "typical" flow (below)?

arch/s390/kvm/gaccess.h int kvm_s390_check_low_addr_prot_real(struct kvm_vcpu *vcpu, unsigned long gra); · line 471

Quoted 5 lines
arch/s390/kvm/gaccess.h +5 -0
+	if (!rc) {
+		phys_addr = PFN_PHYS(f->pfn) | offset_in_page(gaddr);
+		*val = *(unsigned long *)phys_to_virt(phys_addr);
+	}
+	return rc;
	if (rc)
		return rc;

	phys_addr = pfn_to_phys(f->pfn) | offset_in_page(gaddr);
	*val = *(unsigned long *)phys_to_virt(phys_addr);

	return 0;
Quoted 141 lines
arch/s390/kvm/gaccess.h +2 -0
+}
+
 #endif /* __KVM_S390_GACCESS_H */
arch/s390/kvm/kvm-s390.c +44 -0
index 61aa64886c36..af8a62abec48 100644
@@ -4858,6 +4858,50 @@ static void kvm_s390_assert_primary_as(struct kvm_vcpu *vcpu)
 		current->thread.gmap_int_code, current->thread.gmap_teid.val);
 }
 
+/**
+ * __kvm_s390_faultin_gfn() - fault in and pin a guest address
+ * @kvm: the guest
+ * @guest_fault: will be filled with information on the pin operation
+ * @gfn: guest frame
+ * @wr: if true indicates a write access
+ *
+ * Fault in and pin a guest address using absolute addressing, and without
+ * marking the page referenced.
+ *
+ * Context: Called with mm->mmap_lock in read mode.
+ *
+ * Return:
+ * * 0 in case of success,
+ * * -EFAULT if reading using the virtual address failed,
+ * * -EINTR if a signal is pending,
+ * * -EAGAIN if FOLL_NOWAIT was specified, but IO is needed
+ * * PGM_ADDRESSING if the guest address lies outside of guest memory.
+ */
+int __kvm_s390_faultin_gfn(struct kvm *kvm, struct guest_fault *guest_fault, gfn_t gfn, bool wr)
+{
+	struct kvm_memory_slot *slot;
+	kvm_pfn_t pfn;
+	int foll;
+
+	foll = wr ? FOLL_WRITE : 0;
+	slot = gfn_to_memslot(kvm, gfn);
+	pfn = __kvm_faultin_pfn(slot, gfn, foll, &guest_fault->writable, &guest_fault->page);
+	if (is_noslot_pfn(pfn))
+		return PGM_ADDRESSING;
+	if (is_sigpending_pfn(pfn))
+		return -EINTR;
+	if (pfn == KVM_PFN_ERR_NEEDS_IO)
+		return -EAGAIN;
+	if (is_error_pfn(pfn))
+		return -EFAULT;
+
+	guest_fault->pfn = pfn;
+	guest_fault->gfn = gfn;
+	guest_fault->write_attempt = wr;
+	guest_fault->valid = true;
+	return 0;
+}
+
 /*
  * __kvm_s390_handle_dat_fault() - handle a dat fault for the gmap of a vcpu
  * @vcpu: the vCPU whose gmap is to be fixed up
arch/s390/kvm/kvm-s390.h +56 -0
index c44fe0c3a097..dabcf65f58ff 100644
@@ -22,6 +22,15 @@
 
 #define KVM_S390_UCONTROL_MEMSLOT (KVM_USER_MEM_SLOTS + 0)
 
+struct guest_fault {
+	gfn_t gfn;		/* Guest frame */
+	kvm_pfn_t pfn;		/* Host PFN */
+	struct page *page;	/* Host page */
+	bool writable;		/* Mapping is writable */
+	bool write_attempt;	/* Write access attempted */
+	bool valid;		/* This entry contains valid data */
+};
+
 static inline void kvm_s390_fpu_store(struct kvm_run *run)
 {
 	fpu_stfpc(&run->s.regs.fpc);
@@ -464,12 +473,59 @@ int kvm_s390_cpus_from_pv(struct kvm *kvm, u16 *rc, u16 *rrc);
 int __kvm_s390_handle_dat_fault(struct kvm_vcpu *vcpu, gfn_t gfn, gpa_t gaddr, unsigned int flags);
 int __kvm_s390_mprotect_many(struct gmap *gmap, gpa_t gpa, u8 npages, unsigned int prot,
 			     unsigned long bits);
+int __kvm_s390_faultin_gfn(struct kvm *kvm, struct guest_fault *f, gfn_t gfn, bool wr);
 
 static inline int kvm_s390_handle_dat_fault(struct kvm_vcpu *vcpu, gpa_t gaddr, unsigned int flags)
 {
 	return __kvm_s390_handle_dat_fault(vcpu, gpa_to_gfn(gaddr), gaddr, flags);
 }
 
+static inline void release_faultin_multiple(struct kvm *kvm, struct guest_fault *guest_faults,
+					    int n, bool ignore)
+{
+	int i;
+
+	for (i = 0; i < n; i++) {
+		kvm_release_faultin_page(kvm, guest_faults[i].page, ignore,
+					 guest_faults[i].write_attempt);
+		guest_faults[i].page = NULL;
+	}
+}
+
+static inline bool __kvm_s390_multiple_faults_need_retry(struct kvm *kvm, unsigned long seq,
+							 struct guest_fault *guest_faults, int n,
+							 bool unsafe)
+{
+	int i;
+
+	for (i = 0; i < n; i++) {
+		if (!guest_faults[i].valid)
+			continue;
+		if ((unsafe && mmu_invalidate_retry_gfn_unsafe(kvm, seq, guest_faults[i].gfn)) ||
+		    (!unsafe && mmu_invalidate_retry_gfn(kvm, seq, guest_faults[i].gfn))) {
+			release_faultin_multiple(kvm, guest_faults, n, true);
+			return true;
+		}
+	}
+	return false;
+}
+
+static inline int __kvm_s390_faultin_gfn_range(struct kvm *kvm, struct guest_fault *guest_faults,
+					       gfn_t start, int n_pages, bool write_attempt)
+{
+	int i, rc = 0;
+
+	for (i = 0; !rc && i < n_pages; i++)
+		rc = __kvm_s390_faultin_gfn(kvm, guest_faults + i, start + i, write_attempt);
+	return rc;
+}
+
+#define release_faultin_array(kvm, array, ignore) \
+	release_faultin_multiple(kvm, array, ARRAY_SIZE(array), ignore)
+
+#define __kvm_s390_fault_array_needs_retry(kvm, seq, array, unsafe) \
+	__kvm_s390_multiple_faults_need_retry(kvm, seq, array, ARRAY_SIZE(array), unsafe)
+
 /* implemented in diag.c */
 int kvm_s390_handle_diag(struct kvm_vcpu *vcpu);
 
Signature · 1 line
2.51.0
author maintainer
#15 lore raw

Re: [PATCH v2 05/20] KVM: s390: Add helper functions for fault handling

Quoted 32 lines · On Thu, 18 Sep 2025 16:41:56 +0200 Alexander Gordeev <[email protected]>…
Quoted 18 lines · Claudio Imbrenda

Add some helper functions for handling multiple guest faults at the same time.

This will be needed for VSIE, where a nested guest access also needs to access all the page tables that map it.

---
 arch/s390/kvm/gaccess.h  | 14 ++++++++++
 arch/s390/kvm/kvm-s390.c | 44 +++++++++++++++++++++++++++++++
 arch/s390/kvm/kvm-s390.h | 56 ++++++++++++++++++++++++++++++++++++++++
 3 files changed, 114 insertions(+)
arch/s390/kvm/gaccess.h +0 -0
index 3fde45a151f2..9c82f7460821 100644

arch/s390/kvm/gaccess.h int kvm_s390_check_low_addr_prot_real(struct kvm_vcpu *vcpu, unsigned long gra); · line 466

Quoted 10 lines
arch/s390/kvm/gaccess.h +7 -0
@@ -457,4 +457,18 @@ int kvm_s390_check_low_addr_prot_real(struct kvm_vcpu *vcpu, unsigned long gra);
 int kvm_s390_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *shadow,
 			  unsigned long saddr, unsigned long *datptr);
 
+static inline int __kvm_s390_faultin_read_gpa(struct kvm *kvm, struct guest_fault *f, gpa_t gaddr,
+					      unsigned long *val)
+{
+	phys_addr_t phys_addr;
+	int rc;
+
+	rc = __kvm_s390_faultin_gfn(kvm, f, gpa_to_gfn(gaddr), false);  

Why not a "typical" flow (below)?

Quoted 14 lines

arch/s390/kvm/gaccess.h int kvm_s390_check_low_addr_prot_real(struct kvm_vcpu *vcpu, unsigned long gra); · line 471

Quoted 5 lines
arch/s390/kvm/gaccess.h +5 -0
+	if (!rc) {
+		phys_addr = PFN_PHYS(f->pfn) | offset_in_page(gaddr);
+		*val = *(unsigned long *)phys_to_virt(phys_addr);
+	}
+	return rc;  
	if (rc)
		return rc;

	phys_addr = pfn_to_phys(f->pfn) | offset_in_page(gaddr);
	*val = *(unsigned long *)phys_to_virt(phys_addr);

	return 0;

I swear, at some point it made more sense in the way I had written it :D

I'll fix it

Quoted 141 lines
Quoted 141 lines
arch/s390/kvm/gaccess.h +2 -0
+}
+
 #endif /* __KVM_S390_GACCESS_H */
arch/s390/kvm/kvm-s390.c +44 -0
index 61aa64886c36..af8a62abec48 100644
@@ -4858,6 +4858,50 @@ static void kvm_s390_assert_primary_as(struct kvm_vcpu *vcpu)
 		current->thread.gmap_int_code, current->thread.gmap_teid.val);
 }
 
+/**
+ * __kvm_s390_faultin_gfn() - fault in and pin a guest address
+ * @kvm: the guest
+ * @guest_fault: will be filled with information on the pin operation
+ * @gfn: guest frame
+ * @wr: if true indicates a write access
+ *
+ * Fault in and pin a guest address using absolute addressing, and without
+ * marking the page referenced.
+ *
+ * Context: Called with mm->mmap_lock in read mode.
+ *
+ * Return:
+ * * 0 in case of success,
+ * * -EFAULT if reading using the virtual address failed,
+ * * -EINTR if a signal is pending,
+ * * -EAGAIN if FOLL_NOWAIT was specified, but IO is needed
+ * * PGM_ADDRESSING if the guest address lies outside of guest memory.
+ */
+int __kvm_s390_faultin_gfn(struct kvm *kvm, struct guest_fault *guest_fault, gfn_t gfn, bool wr)
+{
+	struct kvm_memory_slot *slot;
+	kvm_pfn_t pfn;
+	int foll;
+
+	foll = wr ? FOLL_WRITE : 0;
+	slot = gfn_to_memslot(kvm, gfn);
+	pfn = __kvm_faultin_pfn(slot, gfn, foll, &guest_fault->writable, &guest_fault->page);
+	if (is_noslot_pfn(pfn))
+		return PGM_ADDRESSING;
+	if (is_sigpending_pfn(pfn))
+		return -EINTR;
+	if (pfn == KVM_PFN_ERR_NEEDS_IO)
+		return -EAGAIN;
+	if (is_error_pfn(pfn))
+		return -EFAULT;
+
+	guest_fault->pfn = pfn;
+	guest_fault->gfn = gfn;
+	guest_fault->write_attempt = wr;
+	guest_fault->valid = true;
+	return 0;
+}
+
 /*
  * __kvm_s390_handle_dat_fault() - handle a dat fault for the gmap of a vcpu
  * @vcpu: the vCPU whose gmap is to be fixed up
arch/s390/kvm/kvm-s390.h +56 -0
index c44fe0c3a097..dabcf65f58ff 100644
@@ -22,6 +22,15 @@
 
 #define KVM_S390_UCONTROL_MEMSLOT (KVM_USER_MEM_SLOTS + 0)
 
+struct guest_fault {
+	gfn_t gfn;		/* Guest frame */
+	kvm_pfn_t pfn;		/* Host PFN */
+	struct page *page;	/* Host page */
+	bool writable;		/* Mapping is writable */
+	bool write_attempt;	/* Write access attempted */
+	bool valid;		/* This entry contains valid data */
+};
+
 static inline void kvm_s390_fpu_store(struct kvm_run *run)
 {
 	fpu_stfpc(&run->s.regs.fpc);
@@ -464,12 +473,59 @@ int kvm_s390_cpus_from_pv(struct kvm *kvm, u16 *rc, u16 *rrc);
 int __kvm_s390_handle_dat_fault(struct kvm_vcpu *vcpu, gfn_t gfn, gpa_t gaddr, unsigned int flags);
 int __kvm_s390_mprotect_many(struct gmap *gmap, gpa_t gpa, u8 npages, unsigned int prot,
 			     unsigned long bits);
+int __kvm_s390_faultin_gfn(struct kvm *kvm, struct guest_fault *f, gfn_t gfn, bool wr);
 
 static inline int kvm_s390_handle_dat_fault(struct kvm_vcpu *vcpu, gpa_t gaddr, unsigned int flags)
 {
 	return __kvm_s390_handle_dat_fault(vcpu, gpa_to_gfn(gaddr), gaddr, flags);
 }
 
+static inline void release_faultin_multiple(struct kvm *kvm, struct guest_fault *guest_faults,
+					    int n, bool ignore)
+{
+	int i;
+
+	for (i = 0; i < n; i++) {
+		kvm_release_faultin_page(kvm, guest_faults[i].page, ignore,
+					 guest_faults[i].write_attempt);
+		guest_faults[i].page = NULL;
+	}
+}
+
+static inline bool __kvm_s390_multiple_faults_need_retry(struct kvm *kvm, unsigned long seq,
+							 struct guest_fault *guest_faults, int n,
+							 bool unsafe)
+{
+	int i;
+
+	for (i = 0; i < n; i++) {
+		if (!guest_faults[i].valid)
+			continue;
+		if ((unsafe && mmu_invalidate_retry_gfn_unsafe(kvm, seq, guest_faults[i].gfn)) ||
+		    (!unsafe && mmu_invalidate_retry_gfn(kvm, seq, guest_faults[i].gfn))) {
+			release_faultin_multiple(kvm, guest_faults, n, true);
+			return true;
+		}
+	}
+	return false;
+}
+
+static inline int __kvm_s390_faultin_gfn_range(struct kvm *kvm, struct guest_fault *guest_faults,
+					       gfn_t start, int n_pages, bool write_attempt)
+{
+	int i, rc = 0;
+
+	for (i = 0; !rc && i < n_pages; i++)
+		rc = __kvm_s390_faultin_gfn(kvm, guest_faults + i, start + i, write_attempt);
+	return rc;
+}
+
+#define release_faultin_array(kvm, array, ignore) \
+	release_faultin_multiple(kvm, array, ARRAY_SIZE(array), ignore)
+
+#define __kvm_s390_fault_array_needs_retry(kvm, seq, array, unsafe) \
+	__kvm_s390_multiple_faults_need_retry(kvm, seq, array, ARRAY_SIZE(array), unsafe)
+
 /* implemented in diag.c */
 int kvm_s390_handle_diag(struct kvm_vcpu *vcpu);
 
Signature · 1 line
2.51.0
author maintainer
#16 lore raw

PATCH v2 6/20 KVM: s390: Rename some functions in gaccess.c

In reply to Claudio Imbrenda

Rename some functions in gaccess.c to add a _gva or _gpa suffix to indicate whether the function accepts a virtual or a guest-absolute address.

This makes it easier to understand the code.

---
 arch/s390/kvm/gaccess.c | 51 +++++++++++++++++++----------------------
 1 file changed, 24 insertions(+), 27 deletions(-)
arch/s390/kvm/gaccess.c +24 -27
index d691fac1cc12..05fd3ee4b20d 100644
@@ -412,7 +412,7 @@ static int deref_table(struct kvm *kvm, unsigned long gpa, unsigned long *val)
 }
 
 /**
- * guest_translate - translate a guest virtual into a guest absolute address
+ * guest_translate_gva() - translate a guest virtual into a guest absolute address
  * @vcpu: virtual cpu
  * @gva: guest virtual address
  * @gpa: points to where guest physical (absolute) address should be stored
@@ -432,9 +432,9 @@ static int deref_table(struct kvm *kvm, unsigned long gpa, unsigned long *val)
  *	      the returned value is the program interruption code as defined
  *	      by the architecture
  */
-static unsigned long guest_translate(struct kvm_vcpu *vcpu, unsigned long gva,
-				     unsigned long *gpa, const union asce asce,
-				     enum gacc_mode mode, enum prot_type *prot)
+static unsigned long guest_translate_gva(struct kvm_vcpu *vcpu, unsigned long gva,
+					 unsigned long *gpa, const union asce asce,
+					 enum gacc_mode mode, enum prot_type *prot)
 {
 	union vaddress vaddr = {.addr = gva};
 	union raddress raddr = {.addr = gva};
@@ -615,8 +615,8 @@ static int low_address_protection_enabled(struct kvm_vcpu *vcpu,
 	return 1;
 }
 
-static int vm_check_access_key(struct kvm *kvm, u8 access_key,
-			       enum gacc_mode mode, gpa_t gpa)
+static int vm_check_access_key_gpa(struct kvm *kvm, u8 access_key,
+				   enum gacc_mode mode, gpa_t gpa)
 {
 	u8 storage_key, access_control;
 	bool fetch_protected;
@@ -678,9 +678,9 @@ static bool storage_prot_override_applies(u8 access_control)
 	return access_control == PAGE_SPO_ACC;
 }
 
-static int vcpu_check_access_key(struct kvm_vcpu *vcpu, u8 access_key,
-				 enum gacc_mode mode, union asce asce, gpa_t gpa,
-				 unsigned long ga, unsigned int len)
+static int vcpu_check_access_key_gpa(struct kvm_vcpu *vcpu, u8 access_key,
+				     enum gacc_mode mode, union asce asce, gpa_t gpa,
+				     unsigned long ga, unsigned int len)
 {
 	u8 storage_key, access_control;
 	unsigned long hva;
@@ -772,7 +772,7 @@ static int guest_range_to_gpas(struct kvm_vcpu *vcpu, unsigned long ga, u8 ar,
 			return trans_exc(vcpu, PGM_PROTECTION, ga, ar, mode,
 					 PROT_TYPE_LA);
 		if (psw_bits(*psw).dat) {
-			rc = guest_translate(vcpu, ga, &gpa, asce, mode, &prot);
+			rc = guest_translate_gva(vcpu, ga, &gpa, asce, mode, &prot);
 			if (rc < 0)
 				return rc;
 		} else {
@@ -784,8 +784,7 @@ static int guest_range_to_gpas(struct kvm_vcpu *vcpu, unsigned long ga, u8 ar,
 		}
 		if (rc)
 			return trans_exc(vcpu, rc, ga, ar, mode, prot);
-		rc = vcpu_check_access_key(vcpu, access_key, mode, asce, gpa, ga,
-					   fragment_len);
+		rc = vcpu_check_access_key_gpa(vcpu, access_key, mode, asce, gpa, ga, fragment_len);
 		if (rc)
 			return trans_exc(vcpu, rc, ga, ar, mode, PROT_TYPE_KEYC);
 		if (gpas)
@@ -797,8 +796,8 @@ static int guest_range_to_gpas(struct kvm_vcpu *vcpu, unsigned long ga, u8 ar,
 	return 0;
 }
 
-static int access_guest_page(struct kvm *kvm, enum gacc_mode mode, gpa_t gpa,
-			     void *data, unsigned int len)
+static int access_guest_page_gpa(struct kvm *kvm, enum gacc_mode mode, gpa_t gpa,
+				 void *data, unsigned int len)
 {
 	const unsigned int offset = offset_in_page(gpa);
 	const gfn_t gfn = gpa_to_gfn(gpa);
@@ -813,9 +812,8 @@ static int access_guest_page(struct kvm *kvm, enum gacc_mode mode, gpa_t gpa,
 	return rc;
 }
 
-static int
-access_guest_page_with_key(struct kvm *kvm, enum gacc_mode mode, gpa_t gpa,
-			   void *data, unsigned int len, u8 access_key)
+static int access_guest_page_with_key_gpa(struct kvm *kvm, enum gacc_mode mode, gpa_t gpa,
+					  void *data, unsigned int len, u8 access_key)
 {
 	struct kvm_memory_slot *slot;
 	bool writable;
@@ -823,7 +821,7 @@ access_guest_page_with_key(struct kvm *kvm, enum gacc_mode mode, gpa_t gpa,
 	hva_t hva;
 	int rc;
 
-	gfn = gpa >> PAGE_SHIFT;
+	gfn = gpa_to_gfn(gpa);
 	slot = gfn_to_memslot(kvm, gfn);
 	hva = gfn_to_hva_memslot_prot(slot, gfn, &writable);
 
@@ -856,7 +854,7 @@ int access_guest_abs_with_key(struct kvm *kvm, gpa_t gpa, void *data,
 
 	while (min(PAGE_SIZE - offset, len) > 0) {
 		fragment_len = min(PAGE_SIZE - offset, len);
-		rc = access_guest_page_with_key(kvm, mode, gpa, data, fragment_len, access_key);
+		rc = access_guest_page_with_key_gpa(kvm, mode, gpa, data, fragment_len, access_key);
 		if (rc)
 			return rc;
 		offset = 0;
@@ -916,15 +914,14 @@ int access_guest_with_key(struct kvm_vcpu *vcpu, unsigned long ga, u8 ar,
 	for (idx = 0; idx < nr_pages; idx++) {
 		fragment_len = min(PAGE_SIZE - offset_in_page(gpas[idx]), len);
 		if (try_fetch_prot_override && fetch_prot_override_applies(ga, fragment_len)) {
-			rc = access_guest_page(vcpu->kvm, mode, gpas[idx],
-					       data, fragment_len);
+			rc = access_guest_page_gpa(vcpu->kvm, mode, gpas[idx], data, fragment_len);
 		} else {
-			rc = access_guest_page_with_key(vcpu->kvm, mode, gpas[idx],
-							data, fragment_len, access_key);
+			rc = access_guest_page_with_key_gpa(vcpu->kvm, mode, gpas[idx],
+							    data, fragment_len, access_key);
 		}
 		if (rc == PGM_PROTECTION && try_storage_prot_override)
-			rc = access_guest_page_with_key(vcpu->kvm, mode, gpas[idx],
-							data, fragment_len, PAGE_SPO_ACC);
+			rc = access_guest_page_with_key_gpa(vcpu->kvm, mode, gpas[idx],
+							    data, fragment_len, PAGE_SPO_ACC);
 		if (rc)
 			break;
 		len -= fragment_len;
@@ -958,7 +955,7 @@ int access_guest_real(struct kvm_vcpu *vcpu, unsigned long gra,
 	while (len && !rc) {
 		gpa = kvm_s390_real_to_abs(vcpu, gra);
 		fragment_len = min(PAGE_SIZE - offset_in_page(gpa), len);
-		rc = access_guest_page(vcpu->kvm, mode, gpa, data, fragment_len);
+		rc = access_guest_page_gpa(vcpu->kvm, mode, gpa, data, fragment_len);
 		len -= fragment_len;
 		gra += fragment_len;
 		data += fragment_len;
@@ -1149,7 +1146,7 @@ int check_gpa_range(struct kvm *kvm, unsigned long gpa, unsigned long length,
 
 	while (length && !rc) {
 		fragment_len = min(PAGE_SIZE - offset_in_page(gpa), length);
-		rc = vm_check_access_key(kvm, access_key, mode, gpa);
+		rc = vm_check_access_key_gpa(kvm, access_key, mode, gpa);
 		length -= fragment_len;
 		gpa += fragment_len;
 	}
Signature · 1 line
2.51.0
author maintainer
#17 lore raw

PATCH v2 7/20 KVM: s390: KVM-specific bitfields and helper functions

In reply to Claudio Imbrenda

Add KVM-s390 specific bitfields and helper functions to manipulate DAT tables.

---
 arch/s390/kvm/dat.h | 693 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 693 insertions(+)
 create mode 100644 arch/s390/kvm/dat.h
arch/s390/kvm/dat.h new file +693 -0
new file mode 100644
index 000000000000..1e355239247b
@@ -0,0 +1,693 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ *  KVM guest address space mapping code
+ *
+ *    Copyright IBM Corp. 2007, 2024
+ *    Author(s): Claudio Imbrenda <[email protected]>
+ *               Martin Schwidefsky <[email protected]>
+ */
+
+#ifndef __KVM_S390_DAT_H
+#define __KVM_S390_DAT_H
+
+#include <linux/radix-tree.h>
+#include <linux/refcount.h>
+#include <linux/io.h>
+#include <linux/kvm_types.h>
+#include <asm/tlbflush.h>
+#include <asm/pgalloc.h>
+#include <asm/dat-bits.h>
+
+#define _ASCE(x) ((union asce) { .val = (x), })
+#define NULL_ASCE _ASCE(0)
+
+enum {
+	_DAT_TOKEN_NONE = 0,
+	_DAT_TOKEN_PIC,
+};
+
+#define _CRSTE_TOK(l, t, p) ((union crste) {	\
+		.tok.i = 1,			\
+		.tok.tt = (l),			\
+		.tok.type = (t),		\
+		.tok.par = (p)			\
+	})
+#define _CRSTE_PIC(l, p) _CRSTE_TOK(l, _DAT_TOKEN_PIC, p)
+
+#define _CRSTE_HOLE(l) _CRSTE_PIC(l, PGM_ADDRESSING)
+#define _CRSTE_EMPTY(l) _CRSTE_TOK(l, _DAT_TOKEN_NONE, 0)
+
+#define _PGD_HOLE _CRSTE_HOLE(TABLE_TYPE_REGION1)
+#define _P4D_HOLE _CRSTE_HOLE(TABLE_TYPE_REGION2)
+#define _PUD_HOLE _CRSTE_HOLE(TABLE_TYPE_REGION3)
+#define _PMD_HOLE _CRSTE_HOLE(TABLE_TYPE_SEGMENT)
+
+#define _PGD_EMPTY _CRSTE_EMPTY(TABLE_TYPE_REGION1)
+#define _P4D_EMPTY _CRSTE_EMPTY(TABLE_TYPE_REGION2)
+#define _PUD_EMPTY _CRSTE_EMPTY(TABLE_TYPE_REGION3)
+#define _PMD_EMPTY _CRSTE_EMPTY(TABLE_TYPE_SEGMENT)
+
+#define _PTE_TOK(t, p) ((union pte) { .tok.i = 1, .tok.type = (t), .tok.par = (p) })
+#define _PTE_PIC(p) _PTE_TOK(_DAT_TOKEN_PIC, p)
+#define _PTE_HOLE _PTE_PIC(PGM_ADDRESSING)
+#define _PTE_EMPTY _PTE_TOK(_DAT_TOKEN_NONE, 0)
+
+enum {
+	LEVEL_PTE = -1,
+	LEVEL_PMD = TABLE_TYPE_SEGMENT,
+	LEVEL_PUD,
+	LEVEL_P4D,
+	LEVEL_PGD
+};
+
+#define RADDR_LEVEL_MASK 0x7
+
+enum dat_walk_flags {
+	DAT_WALK_CONTINUE	= 0x20,
+	DAT_WALK_IGN_HOLES	= 0x10,
+	DAT_WALK_SPLIT		= 0x08,
+	DAT_WALK_ALLOC		= 0x04,
+	DAT_WALK_ANY		= 0x02,
+	DAT_WALK_LEAF		= 0x01,
+	DAT_WALK_DEFAULT	= 0
+};
+
+#define DAT_WALK_SPLIT_ALLOC (DAT_WALK_SPLIT | DAT_WALK_ALLOC)
+#define DAT_WALK_ALLOC_CONTINUE (DAT_WALK_CONTINUE | DAT_WALK_ALLOC)
+#define DAT_WALK_LEAF_ALLOC (DAT_WALK_LEAF | DAT_WALK_ALLOC)
+
+union pte {
+	unsigned long val;
+	union page_table_entry h;
+	struct {
+		unsigned long   :56; /* Hardware bits */
+		unsigned long u : 1; /* Page unused */
+		unsigned long s : 1; /* Special */
+		unsigned long w : 1; /* Writable */
+		unsigned long r : 1; /* Readable */
+		unsigned long d : 1; /* Dirty */
+		unsigned long y : 1; /* Young */
+		unsigned long sd: 1; /* Soft dirty */
+		unsigned long pr: 1; /* Present */
+	} s;
+	struct {
+		unsigned char hwbytes[7];
+		unsigned char swbyte;
+	};
+	union {
+		struct {
+			unsigned long type :16; /* Token type */
+			unsigned long par  :16; /* Token parameter */
+			unsigned long      :20;
+			unsigned long      : 1; /* Must be 0 */
+			unsigned long i    : 1; /* Must be 1 */
+			unsigned long      : 2;
+			unsigned long      : 7;
+			unsigned long pr   : 1; /* Must be 0 */
+		};
+		struct {
+			unsigned long token:32; /* Token and parameter */
+			unsigned long      :32;
+		};
+	} tok;
+};
+
+#define _PAGE_SD 0x002
+
+enum pgste_gps_usage {
+	PGSTE_GPS_USAGE_STABLE = 0,
+	PGSTE_GPS_USAGE_UNUSED,
+	PGSTE_GPS_USAGE_POT_VOLATILE,
+	PGSTE_GPS_USAGE_VOLATILE,
+};
+
+union pgste {
+	unsigned long val;
+	struct {
+		unsigned long acc          : 4;
+		unsigned long fp           : 1;
+		unsigned long              : 3;
+		unsigned long pcl          : 1;
+		unsigned long hr           : 1;
+		unsigned long hc           : 1;
+		unsigned long              : 2;
+		unsigned long gr           : 1;
+		unsigned long gc           : 1;
+		unsigned long              : 1;
+		unsigned long              :16; /* val16 */
+		unsigned long zero         : 1;
+		unsigned long nodat        : 1;
+		unsigned long              : 4;
+		unsigned long usage        : 2;
+		unsigned long              : 8;
+		unsigned long cmma_d       : 1; /* Dirty flag for CMMA bits */
+		unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */
+		unsigned long vsie_notif   : 1; /* Referenced in a shadow table */
+		unsigned long              : 5;
+		unsigned long              : 8;
+	};
+	struct {
+		unsigned short hwbytes0;
+		unsigned short val16;	/* used to store chunked values */
+		unsigned short hwbytes4;
+		unsigned char flags;	/* maps to the software bits */
+		unsigned char hwbyte7;
+	} __packed;
+};
+
+union pmd {
+	unsigned long val;
+	union segment_table_entry h;
+	struct {
+		struct {
+			unsigned long              :44; /* HW */
+			unsigned long              : 3; /* Unused */
+			unsigned long              : 1; /* HW */
+			unsigned long w            : 1; /* Writable soft-bit */
+			unsigned long r            : 1; /* Readable soft-bit */
+			unsigned long d            : 1; /* Dirty */
+			unsigned long y            : 1; /* Young */
+			unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */
+			unsigned long              : 3; /* HW */
+			unsigned long vsie_notif   : 1; /* Referenced in a shadow table */
+			unsigned long              : 1; /* Unused */
+			unsigned long              : 4; /* HW */
+			unsigned long sd           : 1; /* Soft-Dirty */
+			unsigned long pr           : 1; /* Present */
+		} fc1;
+	} s;
+};
+
+union pud {
+	unsigned long val;
+	union region3_table_entry h;
+	struct {
+		struct {
+			unsigned long              :33; /* HW */
+			unsigned long              :14; /* Unused */
+			unsigned long              : 1; /* HW */
+			unsigned long w            : 1; /* Writable soft-bit */
+			unsigned long r            : 1; /* Readable soft-bit */
+			unsigned long d            : 1; /* Dirty */
+			unsigned long y            : 1; /* Young */
+			unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */
+			unsigned long              : 3; /* HW */
+			unsigned long vsie_notif   : 1; /* Referenced in a shadow table */
+			unsigned long              : 1; /* Unused */
+			unsigned long              : 4; /* HW */
+			unsigned long sd           : 1; /* Soft-Dirty */
+			unsigned long pr           : 1; /* Present */
+		} fc1;
+	} s;
+};
+
+union p4d {
+	unsigned long val;
+	union region2_table_entry h;
+};
+
+union pgd {
+	unsigned long val;
+	union region1_table_entry h;
+};
+
+union crste {
+	unsigned long val;
+	union {
+		struct {
+			unsigned long   :52;
+			unsigned long   : 1;
+			unsigned long fc: 1;
+			unsigned long p : 1;
+			unsigned long   : 1;
+			unsigned long   : 2;
+			unsigned long i : 1;
+			unsigned long   : 1;
+			unsigned long tt: 2;
+			unsigned long   : 2;
+		};
+		struct {
+			unsigned long to:52;
+			unsigned long   : 1;
+			unsigned long fc: 1;
+			unsigned long p : 1;
+			unsigned long   : 1;
+			unsigned long tf: 2;
+			unsigned long i : 1;
+			unsigned long   : 1;
+			unsigned long tt: 2;
+			unsigned long tl: 2;
+		} fc0;
+		struct {
+			unsigned long    :47;
+			unsigned long av : 1; /* ACCF-Validity Control */
+			unsigned long acc: 4; /* Access-Control Bits */
+			unsigned long f  : 1; /* Fetch-Protection Bit */
+			unsigned long fc : 1; /* Format-Control */
+			unsigned long p  : 1; /* DAT-Protection Bit */
+			unsigned long iep: 1; /* Instruction-Execution-Protection */
+			unsigned long    : 2;
+			unsigned long i  : 1; /* Segment-Invalid Bit */
+			unsigned long cs : 1; /* Common-Segment Bit */
+			unsigned long tt : 2; /* Table-Type Bits */
+			unsigned long    : 2;
+		} fc1;
+	} h;
+	struct {
+		struct {
+			unsigned long              :47;
+			unsigned long              : 1; /* HW (should be 0) */
+			unsigned long w            : 1; /* Writable */
+			unsigned long r            : 1; /* Readable */
+			unsigned long d            : 1; /* Dirty */
+			unsigned long y            : 1; /* Young */
+			unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */
+			unsigned long              : 3; /* HW */
+			unsigned long vsie_notif   : 1; /* Referenced in a shadow table */
+			unsigned long              : 1;
+			unsigned long              : 4; /* HW */
+			unsigned long sd           : 1; /* Soft-Dirty */
+			unsigned long pr           : 1; /* Present */
+		} fc1;
+	} s;
+	union {
+		struct {
+			unsigned long type :16; /* Token type */
+			unsigned long par  :16; /* Token parameter */
+			unsigned long      :26;
+			unsigned long i    : 1; /* Must be 1 */
+			unsigned long      : 1;
+			unsigned long tt   : 2;
+			unsigned long      : 1;
+			unsigned long pr   : 1; /* Must be 0 */
+		};
+		struct {
+			unsigned long token:32; /* Token and parameter */
+			unsigned long      :32;
+		};
+	} tok;
+	union pmd pmd;
+	union pud pud;
+	union p4d p4d;
+	union pgd pgd;
+};
+
+union skey {
+	unsigned char skey;
+	struct {
+		unsigned char acc :4;
+		unsigned char fp  :1;
+		unsigned char r   :1;
+		unsigned char c   :1;
+		unsigned char zero:1;
+	};
+};
+
+static_assert(sizeof(union pgste) == sizeof(unsigned long));
+static_assert(sizeof(union pte) == sizeof(unsigned long));
+static_assert(sizeof(union pmd) == sizeof(unsigned long));
+static_assert(sizeof(union pud) == sizeof(unsigned long));
+static_assert(sizeof(union p4d) == sizeof(unsigned long));
+static_assert(sizeof(union pgd) == sizeof(unsigned long));
+static_assert(sizeof(union crste) == sizeof(unsigned long));
+static_assert(sizeof(union skey) == sizeof(char));
+
+struct segment_table {
+	union pmd pmds[_CRST_ENTRIES];
+};
+
+struct region3_table {
+	union pud puds[_CRST_ENTRIES];
+};
+
+struct region2_table {
+	union p4d p4ds[_CRST_ENTRIES];
+};
+
+struct region1_table {
+	union pgd pgds[_CRST_ENTRIES];
+};
+
+struct crst_table {
+	union {
+		union crste crstes[_CRST_ENTRIES];
+		struct segment_table segment;
+		struct region3_table region3;
+		struct region2_table region2;
+		struct region1_table region1;
+	};
+};
+
+struct page_table {
+	union pte ptes[_PAGE_ENTRIES];
+	union pgste pgstes[_PAGE_ENTRIES];
+};
+
+static_assert(sizeof(struct crst_table) == _CRST_TABLE_SIZE);
+static_assert(sizeof(struct page_table) == PAGE_SIZE);
+
+static inline union pte _pte(kvm_pfn_t pfn, bool w, bool d, bool s)
+{
+	union pte res = { .val = PFN_PHYS(pfn) };
+
+	res.h.p = !d;
+	res.s.y = 1;
+	res.s.pr = 1;
+	res.s.w = w;
+	res.s.d = d;
+	res.s.sd = d;
+	res.s.s = s;
+	return res;
+}
+
+static inline union crste _crste_fc0(kvm_pfn_t pfn, int tt)
+{
+	union crste res = { .val = PFN_PHYS(pfn) };
+
+	res.h.tt = tt;
+	res.h.fc0.tl = _REGION_ENTRY_LENGTH;
+	res.h.fc0.tf = 0;
+	return res;
+}
+
+static inline union crste _crste_fc1(kvm_pfn_t pfn, int tt, bool w, bool d)
+{
+	union crste res = { .val = PFN_PHYS(pfn) & _SEGMENT_MASK };
+
+	res.h.tt = tt;
+	res.h.p = !d;
+	res.h.fc = 1;
+	res.s.fc1.y = 1;
+	res.s.fc1.pr = 1;
+	res.s.fc1.w = w;
+	res.s.fc1.d = d;
+	res.s.fc1.sd = d;
+	return res;
+}
+
+static inline struct crst_table *crste_table_start(union crste *crstep)
+{
+	return (struct crst_table *)ALIGN_DOWN((unsigned long)crstep, _CRST_TABLE_SIZE);
+}
+
+static inline struct page_table *pte_table_start(union pte *ptep)
+{
+	return (struct page_table *)ALIGN_DOWN((unsigned long)ptep, _PAGE_TABLE_SIZE);
+}
+
+static inline bool crdte_crste(union crste *crstep, union crste old, union crste new, gfn_t gfn,
+			       union asce asce)
+{
+	void *table = crste_table_start(crstep);
+	unsigned long dtt = 0x10 | new.h.tt << 2;
+
+	return crdte(old.val, new.val, table, dtt, gfn_to_gpa(gfn), asce.val);
+}
+
+static __always_inline void idte_crste(union crste *crstep, gfn_t gfn, unsigned long opt,
+				       union asce asce, int local)
+{
+	unsigned long gaddr = gfn_to_gpa(gfn) & HPAGE_MASK;
+	unsigned long to = __pa(crste_table_start(crstep));
+
+	if (__builtin_constant_p(opt) && opt == 0) {
+		/* flush without guest asce */
+		asm volatile("idte	%[r1],0,%[r2],%[m4]"
+			: "+m" (*crstep)
+			: [r1] "a" (to), [r2] "a" (gaddr),
+			  [m4] "i" (local)
+			: "cc");
+	} else {
+		/* flush with guest asce */
+		asm volatile("idte	%[r1],%[r3],%[r2],%[m4]"
+			: "+m" (*crstep)
+			: [r1] "a" (to), [r2] "a" (gaddr | opt),
+			  [r3] "a" (asce.val), [m4] "i" (local)
+			: "cc");
+	}
+}
+
+static inline void dat_init_pgstes(struct page_table *pt, unsigned long val)
+{
+	memset64((void *)pt->pgstes, val, PTRS_PER_PTE);
+}
+
+static inline void dat_init_page_table(struct page_table *pt, unsigned long ptes,
+				       unsigned long pgstes)
+{
+	memset64((void *)pt->ptes, ptes, PTRS_PER_PTE);
+	dat_init_pgstes(pt, pgstes);
+}
+
+static inline gfn_t asce_end(union asce asce)
+{
+	return 1ULL << ((asce.dt + 1) * 11 + _SEGMENT_SHIFT - PAGE_SHIFT);
+}
+
+#define _CRSTE(x) ((union crste) { .val = _Generic((x),	\
+			union pgd : (x).val,		\
+			union p4d : (x).val,		\
+			union pud : (x).val,		\
+			union pmd : (x).val,		\
+			union crste : (x).val)})
+
+#define _CRSTEP(x) ((union crste *)_Generic((*(x)),	\
+				union pgd : (x),	\
+				union p4d : (x),	\
+				union pud : (x),	\
+				union pmd : (x),	\
+				union crste : (x)))
+
+#define _CRSTP(x) ((struct crst_table *)_Generic((*(x)),	\
+		struct crst_table : (x),			\
+		struct segment_table : (x),			\
+		struct region3_table : (x),			\
+		struct region2_table : (x),			\
+		struct region1_table : (x)))
+
+static inline bool asce_contains_gfn(union asce asce, gfn_t gfn)
+{
+	return gfn < asce_end(asce);
+}
+
+static inline bool is_pmd(union crste crste)
+{
+	return crste.h.tt == TABLE_TYPE_SEGMENT;
+}
+
+static inline bool is_pud(union crste crste)
+{
+	return crste.h.tt == TABLE_TYPE_REGION3;
+}
+
+static inline bool is_p4d(union crste crste)
+{
+	return crste.h.tt == TABLE_TYPE_REGION2;
+}
+
+static inline bool is_pgd(union crste crste)
+{
+	return crste.h.tt == TABLE_TYPE_REGION1;
+}
+
+static inline phys_addr_t pmd_origin_large(union pmd pmd)
+{
+	return pmd.val & _SEGMENT_ENTRY_ORIGIN_LARGE;
+}
+
+static inline phys_addr_t pud_origin_large(union pud pud)
+{
+	return pud.val & _REGION3_ENTRY_ORIGIN_LARGE;
+}
+
+static inline phys_addr_t crste_origin_large(union crste crste)
+{
+	if (is_pmd(crste))
+		return pmd_origin_large(crste.pmd);
+	return pud_origin_large(crste.pud);
+}
+
+#define crste_origin(x) (_Generic((x),				\
+		union pmd : (x).val & _SEGMENT_ENTRY_ORIGIN,	\
+		union pud : (x).val & _REGION_ENTRY_ORIGIN,	\
+		union p4d : (x).val & _REGION_ENTRY_ORIGIN,	\
+		union pgd : (x).val & _REGION_ENTRY_ORIGIN))
+
+static inline unsigned long pte_origin(union pte pte)
+{
+	return pte.val & PAGE_MASK;
+}
+
+static inline bool pmd_prefix(union pmd pmd)
+{
+	return pmd.h.fc && pmd.s.fc1.prefix_notif;
+}
+
+static inline bool pud_prefix(union pud pud)
+{
+	return pud.h.fc && pud.s.fc1.prefix_notif;
+}
+
+static inline bool crste_leaf(union crste crste)
+{
+	return (crste.h.tt <= LEVEL_PUD) && crste.h.fc;
+}
+
+static inline bool crste_prefix(union crste crste)
+{
+	return crste_leaf(crste) && crste.s.fc1.prefix_notif;
+}
+
+static inline bool crste_dirty(union crste crste)
+{
+	return crste_leaf(crste) && crste.s.fc1.d;
+}
+
+static inline union pgste *pgste_of(union pte *pte)
+{
+	return (union pgste *)(pte + _PAGE_ENTRIES);
+}
+
+static inline bool pte_hole(union pte pte)
+{
+	return pte.h.i && !pte.tok.pr && pte.tok.type != _DAT_TOKEN_NONE;
+}
+
+static inline bool _crste_hole(union crste crste)
+{
+	return crste.h.i && !crste.tok.pr && crste.tok.type != _DAT_TOKEN_NONE;
+}
+
+#define crste_hole(x) _crste_hole(_CRSTE(x))
+
+static inline bool _crste_none(union crste crste)
+{
+	return crste.h.i && !crste.tok.pr && crste.tok.type == _DAT_TOKEN_NONE;
+}
+
+#define crste_none(x) _crste_none(_CRSTE(x))
+
+static inline phys_addr_t large_pud_to_phys(union pud pud, gfn_t gfn)
+{
+	return pud_origin_large(pud) | (gfn_to_gpa(gfn) & ~_REGION3_MASK);
+}
+
+static inline phys_addr_t large_pmd_to_phys(union pmd pmd, gfn_t gfn)
+{
+	return pmd_origin_large(pmd) | (gfn_to_gpa(gfn) & ~_SEGMENT_MASK);
+}
+
+static inline phys_addr_t large_crste_to_phys(union crste crste, gfn_t gfn)
+{
+	if (unlikely(!crste.h.fc || crste.h.tt > TABLE_TYPE_REGION3))
+		return -1;
+	if (is_pmd(crste))
+		return large_pmd_to_phys(crste.pmd, gfn);
+	return large_pud_to_phys(crste.pud, gfn);
+}
+
+static inline void csp_invalidate_crste(union crste *crstep)
+{
+	csp((unsigned int *)crstep + 1, crstep->val, crstep->val | _REGION_ENTRY_INVALID);
+}
+
+static inline bool cspg_crste(union crste *crstep, union crste old, union crste new)
+{
+	return cspg(&crstep->val, old.val, new.val);
+}
+
+static inline struct page_table *dereference_pmd(union pmd pmd)
+{
+	return phys_to_virt(crste_origin(pmd));
+}
+
+static inline struct segment_table *dereference_pud(union pud pud)
+{
+	return phys_to_virt(crste_origin(pud));
+}
+
+static inline struct region3_table *dereference_p4d(union p4d p4d)
+{
+	return phys_to_virt(crste_origin(p4d));
+}
+
+static inline struct region2_table *dereference_pgd(union pgd pgd)
+{
+	return phys_to_virt(crste_origin(pgd));
+}
+
+static inline struct crst_table *_dereference_crste(union crste crste)
+{
+	if (unlikely(is_pmd(crste)))
+		return NULL;
+	return phys_to_virt(crste_origin(crste.pud));
+}
+
+#define dereference_crste(x) (_Generic((x),			\
+		union pud : _dereference_crste(_CRSTE(x)),	\
+		union p4d : _dereference_crste(_CRSTE(x)),	\
+		union pgd : _dereference_crste(_CRSTE(x)),	\
+		union crste : _dereference_crste(_CRSTE(x))))
+
+static inline struct crst_table *dereference_asce(union asce asce)
+{
+	return phys_to_virt(asce.val & _ASCE_ORIGIN);
+}
+
+static inline void asce_flush_tlb(union asce asce)
+{
+	if (cpu_has_idte())
+		__tlb_flush_idte(asce.val);
+	else
+		__tlb_flush_global();
+}
+
+static inline bool pgste_get_trylock(union pte *ptep, union pgste *res)
+{
+	union pgste *pgstep = pgste_of(ptep);
+	union pgste tmp_pgste;
+
+	if (READ_ONCE(pgstep->val) & PGSTE_PCL_BIT)
+		return false;
+	tmp_pgste.val = __atomic64_or_barrier(PGSTE_PCL_BIT, &pgstep->val);
+	if (tmp_pgste.pcl)
+		return false;
+	tmp_pgste.pcl = 1;
+	*res = tmp_pgste;
+	return true;
+}
+
+static inline union pgste pgste_get_lock(union pte *ptep)
+{
+	union pgste res;
+
+	while (!pgste_get_trylock(ptep, &res))
+		cpu_relax();
+	return res;
+}
+
+static inline void pgste_set_unlock(union pte *ptep, union pgste pgste)
+{
+	pgste.pcl = 0;
+	barrier();
+	WRITE_ONCE(*pgste_of(ptep), pgste);
+}
+
+static inline struct page_table *dat_alloc_empty_pt(void)
+{
+	return dat_alloc_pt(_PAGE_INVALID, 0);
+}
+
+static inline void dat_free_pt(struct page_table *pt)
+{
+	free_page((unsigned long)pt);
+}
+
+static inline void _dat_free_crst(struct crst_table *table)
+{
+	free_pages((unsigned long)table, CRST_ALLOC_ORDER);
+}
+
+#define dat_free_crst(x) _dat_free_crst(_CRSTP(x))
+
+#endif /* __KVM_S390_DAT_H */
Signature · 1 line
2.51.0
maintainer
#18 lore raw

Re: [PATCH v2 07/20] KVM: s390: KVM-specific bitfields and helper functions

Quoted 8 lines · Claudio Imbrenda

Add KVM-s390 specific bitfields and helper functions to manipulate DAT tables.

---
 arch/s390/kvm/dat.h | 693 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 693 insertions(+)
 create mode 100644 arch/s390/kvm/dat.h

...

arch/s390/kvm/dat.h line 679

Quoted 4 lines
arch/s390/kvm/dat.h new file +4 -0
+static inline struct page_table *dat_alloc_empty_pt(void)
+{
+	return dat_alloc_pt(_PAGE_INVALID, 0);
+}

This is calling a function which get's introduced with a later patch. It is not harmful, since all of this code is still unused, but still odd.

author maintainer
#19 lore raw

Re: [PATCH v2 07/20] KVM: s390: KVM-specific bitfields and helper functions

Quoted 12 lines · On Wed, 17 Sep 2025 14:18:22 +0200 Heiko Carstens <[email protected]> wrote:
Quoted 8 lines · Claudio Imbrenda

Add KVM-s390 specific bitfields and helper functions to manipulate DAT tables.

---
 arch/s390/kvm/dat.h | 693 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 693 insertions(+)
 create mode 100644 arch/s390/kvm/dat.h  

...

Quoted 8 lines

arch/s390/kvm/dat.h line 679

Quoted 4 lines
arch/s390/kvm/dat.h new file +4 -0
+static inline struct page_table *dat_alloc_empty_pt(void)
+{
+	return dat_alloc_pt(_PAGE_INVALID, 0);
+}  

This is calling a function which get's introduced with a later patch. It is not harmful, since all of this code is still unused, but still odd.

yeah but this should not be here, I'll move it to the appropriate patch

thanks for spotting this

author maintainer
#20 lore raw

PATCH v2 8/20 KVM: s390: KVM page table management functions: allocation

In reply to Claudio Imbrenda

Add page table management functions to be used for KVM guest (gmap) page tables.

This patch adds the boilerplate and functions for the allocation and deallocation of DAT tables.

---
 arch/s390/kvm/Makefile     |  1 +
 arch/s390/kvm/dat.c        | 91 ++++++++++++++++++++++++++++++++++++++
 arch/s390/kvm/dat.h        |  4 ++
 arch/s390/mm/page-states.c |  1 +
 4 files changed, 97 insertions(+)
 create mode 100644 arch/s390/kvm/dat.c
arch/s390/kvm/Makefile +1 -0
index 9a723c48b05a..84315d2f75fb 100644
@@ -9,6 +9,7 @@ ccflags-y := -Ivirt/kvm -Iarch/s390/kvm
 
 kvm-y += kvm-s390.o intercept.o interrupt.o priv.o sigp.o
 kvm-y += diag.o gaccess.o guestdbg.o vsie.o pv.o gmap-vsie.o
+kvm-y += dat.o
 
 kvm-$(CONFIG_VFIO_PCI_ZDEV_KVM) += pci.o
 obj-$(CONFIG_KVM) += kvm.o
arch/s390/kvm/dat.c new file +91 -0

91 lines not on this page · raw message

arch/s390/kvm/dat.h +4 -0
index 1e355239247b..5056cfa02619 100644
@@ -385,6 +385,10 @@ static inline union crste _crste_fc1(kvm_pfn_t pfn, int tt, bool w, bool d)
 	return res;
 }
 
+void dat_free_level(struct crst_table *table, bool owns_ptes);
+struct page_table *dat_alloc_pt(unsigned long pte_bits, unsigned long pgste_bits);
+struct crst_table *dat_alloc_crst(unsigned long init);
+
 static inline struct crst_table *crste_table_start(union crste *crstep)
 {
 	return (struct crst_table *)ALIGN_DOWN((unsigned long)crstep, _CRST_TABLE_SIZE);
arch/s390/mm/page-states.c +1 -0
index 01f9b39e65f5..5bee173db72e 100644
@@ -13,6 +13,7 @@
 #include <asm/page.h>
 
 int __bootdata_preserved(cmma_flag);
+EXPORT_SYMBOL(cmma_flag);
 
 void arch_free_page(struct page *page, int order)
 {
Signature · 1 line
2.51.0
maintainer
#21 lore raw

Re: [PATCH v2 08/20] KVM: s390: KVM page table management functions: allocation

Quoted 20 lines · Claudio Imbrenda

Add page table management functions to be used for KVM guest (gmap) page tables.

This patch adds the boilerplate and functions for the allocation and deallocation of DAT tables.

arch/s390/kvm/dat.h +4 -0
+void dat_free_level(struct crst_table *table, bool owns_ptes);
+struct page_table *dat_alloc_pt(unsigned long pte_bits, unsigned long pgste_bits);
+struct crst_table *dat_alloc_crst(unsigned long init);
+
  static inline struct crst_table *crste_table_start(union crste *crstep)
  {
  	return (struct crst_table *)ALIGN_DOWN((unsigned long)crstep, _CRST_TABLE_SIZE);
arch/s390/mm/page-states.c +0 -0
index 01f9b39e65f5..5bee173db72e 100644

arch/s390/mm/page-states.c line 16

Quoted 4 lines
arch/s390/mm/page-states.c +1 -0
@@ -13,6 +13,7 @@
  #include <asm/page.h>
 
  int __bootdata_preserved(cmma_flag);
+EXPORT_SYMBOL(cmma_flag);

Why does this need to be exported?

author maintainer
#22 lore raw

Re: [PATCH v2 08/20] KVM: s390: KVM page table management functions: allocation

Quoted 23 lines · On Thu, 11 Sep 2025 10:22:56 +0200 Janosch Frank <[email protected]> wrote:
Quoted 22 lines · Claudio Imbrenda

Add page table management functions to be used for KVM guest (gmap) page tables.

This patch adds the boilerplate and functions for the allocation and deallocation of DAT tables.

arch/s390/kvm/dat.h +4 -0
+void dat_free_level(struct crst_table *table, bool owns_ptes);
+struct page_table *dat_alloc_pt(unsigned long pte_bits, unsigned long pgste_bits);
+struct crst_table *dat_alloc_crst(unsigned long init);
+
  static inline struct crst_table *crste_table_start(union crste *crstep)
  {
  	return (struct crst_table *)ALIGN_DOWN((unsigned long)crstep, _CRST_TABLE_SIZE);
arch/s390/mm/page-states.c +0 -0
index 01f9b39e65f5..5bee173db72e 100644
@@ -13,6 +13,7 @@
  #include <asm/page.h>
 
Quoted 5 lines

arch/s390/mm/page-states.c line 16

Quoted 2 lines
arch/s390/mm/page-states.c +1 -0
  int __bootdata_preserved(cmma_flag);
+EXPORT_SYMBOL(cmma_flag);  

Why does this need to be exported?

because page-states.h uses it, and we include page-states.h in dat.c

maintainer
#23 lore raw

Re: [PATCH v2 08/20] KVM: s390: KVM page table management functions: allocation

In reply to Claudio Imbrenda
Quoted 14 lines · Claudio Imbrenda

Add page table management functions to be used for KVM guest (gmap) page tables.

This patch adds the boilerplate and functions for the allocation and deallocation of DAT tables.

---
 arch/s390/kvm/Makefile     |  1 +
 arch/s390/kvm/dat.c        | 91 ++++++++++++++++++++++++++++++++++++++
 arch/s390/kvm/dat.h        |  4 ++
 arch/s390/mm/page-states.c |  1 +
 4 files changed, 97 insertions(+)
 create mode 100644 arch/s390/kvm/dat.c

...

arch/s390/kvm/dat.c line 43

Quoted 13 lines
arch/s390/kvm/dat.c new file +13 -0
+static inline struct page_table *dat_alloc_pt_noinit(void)
+{
+	struct page *page;
+	void *virt;
+
+	page = alloc_pages(GFP_ATOMIC, 0);
+	if (!page)
+		return NULL;
+
+	virt = page_to_virt(page);
+	__arch_set_page_dat(virt, 1);
+	return virt;
+}

Is GFP_ATOMIC a typo, and this should have been GFP_KERNEL?

Otherwise I would guess this will cause problems in the future when under memory pressure allocating guest page tables fails easily, while before this change such allocations never failed.

author maintainer
#24 lore raw

Re: [PATCH v2 08/20] KVM: s390: KVM page table management functions: allocation

Quoted 34 lines · On Tue, 16 Sep 2025 18:26:53 +0200 Heiko Carstens <[email protected]> wrote:
Quoted 14 lines · Claudio Imbrenda

Add page table management functions to be used for KVM guest (gmap) page tables.

This patch adds the boilerplate and functions for the allocation and deallocation of DAT tables.

---
 arch/s390/kvm/Makefile     |  1 +
 arch/s390/kvm/dat.c        | 91 ++++++++++++++++++++++++++++++++++++++
 arch/s390/kvm/dat.h        |  4 ++
 arch/s390/mm/page-states.c |  1 +
 4 files changed, 97 insertions(+)
 create mode 100644 arch/s390/kvm/dat.c  

...

arch/s390/kvm/dat.c line 43

Quoted 13 lines
arch/s390/kvm/dat.c new file +13 -0
+static inline struct page_table *dat_alloc_pt_noinit(void)
+{
+	struct page *page;
+	void *virt;
+
+	page = alloc_pages(GFP_ATOMIC, 0);
+	if (!page)
+		return NULL;
+
+	virt = page_to_virt(page);
+	__arch_set_page_dat(virt, 1);
+	return virt;
+}  

Is GFP_ATOMIC a typo, and this should have been GFP_KERNEL?

Quoted 3 lines

Otherwise I would guess this will cause problems in the future when under memory pressure allocating guest page tables fails easily, while before this change such allocations never failed.

how so? the documentation in gfp_types.h says:

* %GFP_ATOMIC users can not sleep and need the allocation to succeed. A lower
* watermark is applied to allow access to "atomic reserves".
* The current implementation doesn't support NMI and few other strict
* non-preemptive contexts (e.g. raw_spin_lock). The same applies to %GFP_NOWAIT. *
* %GFP_KERNEL is typical for kernel-internal allocations. The caller requires
* %ZONE_NORMAL or a lower zone for direct access but can direct reclaim.

I think GFP_ATOMIC actually gives more guarantees?

#25 lore raw

Re: [PATCH v2 08/20] KVM: s390: KVM page table management functions: allocation

Am 16.09.25 um 18:47 schrieb Claudio Imbrenda:

Quoted 52 lines
Quoted 34 lines · On Tue, 16 Sep 2025 18:26:53 +0200 Heiko Carstens <[email protected]> wrote:
Quoted 14 lines · Claudio Imbrenda

Add page table management functions to be used for KVM guest (gmap) page tables.

This patch adds the boilerplate and functions for the allocation and deallocation of DAT tables.

---
  arch/s390/kvm/Makefile     |  1 +
  arch/s390/kvm/dat.c        | 91 ++++++++++++++++++++++++++++++++++++++
  arch/s390/kvm/dat.h        |  4 ++
  arch/s390/mm/page-states.c |  1 +
  4 files changed, 97 insertions(+)

create mode 100644 arch/s390/kvm/dat.c

...

arch/s390/kvm/dat.c line 43

Quoted 13 lines
arch/s390/kvm/dat.c new file +13 -0
+static inline struct page_table *dat_alloc_pt_noinit(void)
+{
+	struct page *page;
+	void *virt;
+
+	page = alloc_pages(GFP_ATOMIC, 0);
+	if (!page)
+		return NULL;
+
+	virt = page_to_virt(page);
+	__arch_set_page_dat(virt, 1);
+	return virt;
+}

Is GFP_ATOMIC a typo, and this should have been GFP_KERNEL?

Quoted 3 lines

Otherwise I would guess this will cause problems in the future when under memory pressure allocating guest page tables fails easily, while before this change such allocations never failed.

how so? the documentation in gfp_types.h says:

* %GFP_ATOMIC users can not sleep and need the allocation to succeed. A lower
* watermark is applied to allow access to "atomic reserves".
* The current implementation doesn't support NMI and few other strict
* non-preemptive contexts (e.g. raw_spin_lock). The same applies to %GFP_NOWAIT. *
* %GFP_KERNEL is typical for kernel-internal allocations. The caller requires
* %ZONE_NORMAL or a lower zone for direct access but can direct reclaim.

Quoted 1 line

I think GFP_ATOMIC actually gives more guarantees?

In real life GFP_ATOMIC can fail, GFP_KERNEL does not.All gfp allocation failures are usually the atomic ones.

author maintainer
#26 lore raw

Re: [PATCH v2 08/20] KVM: s390: KVM page table management functions: allocation

Quoted 53 lines · On Tue, 16 Sep 2025 19:01:11 +0200 Christian Borntraeger…

Am 16.09.25 um 18:47 schrieb Claudio Imbrenda:

Quoted 52 lines
Quoted 34 lines · On Tue, 16 Sep 2025 18:26:53 +0200 Heiko Carstens <[email protected]> wrote:
Quoted 14 lines · Claudio Imbrenda

Add page table management functions to be used for KVM guest (gmap) page tables.

This patch adds the boilerplate and functions for the allocation and deallocation of DAT tables.

---
  arch/s390/kvm/Makefile     |  1 +
  arch/s390/kvm/dat.c        | 91 ++++++++++++++++++++++++++++++++++++++
  arch/s390/kvm/dat.h        |  4 ++
  arch/s390/mm/page-states.c |  1 +
  4 files changed, 97 insertions(+)

create mode 100644 arch/s390/kvm/dat.c

...

arch/s390/kvm/dat.c line 43

Quoted 13 lines
arch/s390/kvm/dat.c new file +13 -0
+static inline struct page_table *dat_alloc_pt_noinit(void)
+{
+	struct page *page;
+	void *virt;
+
+	page = alloc_pages(GFP_ATOMIC, 0);
+	if (!page)
+		return NULL;
+
+	virt = page_to_virt(page);
+	__arch_set_page_dat(virt, 1);
+	return virt;
+}  

Is GFP_ATOMIC a typo, and this should have been GFP_KERNEL?

Quoted 3 lines

Otherwise I would guess this will cause problems in the future when under memory pressure allocating guest page tables fails easily, while before this change such allocations never failed.

how so? the documentation in gfp_types.h says:

* %GFP_ATOMIC users can not sleep and need the allocation to succeed. A lower
* watermark is applied to allow access to "atomic reserves".
* The current implementation doesn't support NMI and few other strict
* non-preemptive contexts (e.g. raw_spin_lock). The same applies to %GFP_NOWAIT. *
* %GFP_KERNEL is typical for kernel-internal allocations. The caller requires
* %ZONE_NORMAL or a lower zone for direct access but can direct reclaim.

Quoted 4 lines
Quoted 1 line

I think GFP_ATOMIC actually gives more guarantees?

In real life GFP_ATOMIC can fail, GFP_KERNEL does not.All gfp allocation failures are usually the atomic ones.

interesting... then I guess I need GFP_KERNEL | GFP_ATOMIC ?

#27 lore raw

Re: [PATCH v2 08/20] KVM: s390: KVM page table management functions: allocation

Am 16.09.25 um 19:05 schrieb Claudio Imbrenda:

Quoted 6 lines
Quoted 4 lines
Quoted 1 line

I think GFP_ATOMIC actually gives more guarantees?

In real life GFP_ATOMIC can fail, GFP_KERNEL does not.All gfp allocation failures are usually the atomic ones.

interesting... then I guess I need GFP_KERNEL | GFP_ATOMIC ?

No. ATOMIC always means: can fail.

maintainer
#28 lore raw

Re: [PATCH v2 08/20] KVM: s390: KVM page table management functions: allocation

Quoted 11 lines · Christian Borntraeger

Am 16.09.25 um 19:05 schrieb Claudio Imbrenda:

Quoted 6 lines
Quoted 4 lines
Quoted 1 line

I think GFP_ATOMIC actually gives more guarantees?

In real life GFP_ATOMIC can fail, GFP_KERNEL does not.All gfp allocation failures are usually the atomic ones.

interesting... then I guess I need GFP_KERNEL | GFP_ATOMIC ?

No. ATOMIC always means: can fail.

So GFP_KERNEL alone is what is needed here. It is undocumented that small GFP_KERNEL allocations will never fail (retried for ever):

https://lwn.net/Articles/627419/
https://lwn.net/Articles/723317/

While GFP_ATOMIC will fail e.g. if I/O is required to free up some memory.

maintainer
#29 lore raw

Re: [PATCH v2 08/20] KVM: s390: KVM page table management functions: allocation

Quoted 9 lines · Heiko Carstens
Quoted 8 lines · Christian Borntraeger

Am 16.09.25 um 19:05 schrieb Claudio Imbrenda:

Quoted 5 lines
Quoted 4 lines
Quoted 1 line

I think GFP_ATOMIC actually gives more guarantees?

In real life GFP_ATOMIC can fail, GFP_KERNEL does not.All gfp allocation failures are usually the atomic ones.

Quoted 6 lines
Quoted 3 lines
Quoted 1 line

interesting... then I guess I need GFP_KERNEL | GFP_ATOMIC ?

No. ATOMIC always means: can fail.

So GFP_KERNEL alone is what is needed here. It is undocumented that small GFP_KERNEL allocations will never fail (retried for ever):

Another correction: it should be GFP_KERNEL_ACCOUNT instead, like it used to be before in gmap_alloc_crst() since those page tables should be accounted to the process which is allocating them.

author maintainer
#30 lore raw

Re: [PATCH v2 08/20] KVM: s390: KVM page table management functions: allocation

Quoted 5 lines · On Wed, 17 Sep 2025 09:27:33 +0200 Heiko Carstens <[email protected]> wrote:
Quoted 4 lines · Heiko Carstens
Quoted 3 lines · Christian Borntraeger

Am 16.09.25 um 19:05 schrieb Claudio Imbrenda:

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 6 lines
Quoted 4 lines
Quoted 1 line

I think GFP_ATOMIC actually gives more guarantees?

In real life GFP_ATOMIC can fail, GFP_KERNEL does not.All gfp allocation failures are usually the atomic ones.

interesting... then I guess I need GFP_KERNEL | GFP_ATOMIC ?

No. ATOMIC always means: can fail.

my issue is that GFP_KERNEL can sleep, and this allocation is sometimes called from atomic contexts (e.g. while holding spinlocks)

the right way to do this would be with mempools, to allocate memory (and potentially sleep) when we are not in atomic context, and use it whenever needed. this is on my to-do list for the future, but right now I'd like to avoid having to refactor a ton of code.

Quoted 3 lines
Quoted 3 lines

So GFP_KERNEL alone is what is needed here. It is undocumented that small GFP_KERNEL allocations will never fail (retried for ever):

define "small" ?

Quoted 4 lines

Another correction: it should be GFP_KERNEL_ACCOUNT instead, like it used to be before in gmap_alloc_crst() since those page tables should be accounted to the process which is allocating them.

gfp_types.h says:

* %GFP_KERNEL_ACCOUNT is the same as GFP_KERNEL, except the allocation is
* accounted to kmemcg.

I thought that without GFP_KERNEL_ACCOUNT, it would be accounted toward the process?
The documentation is a little confusing here

maintainer
#31 lore raw

Re: [PATCH v2 08/20] KVM: s390: KVM page table management functions: allocation

Quoted 17 lines · Claudio Imbrenda
Quoted 5 lines · On Wed, 17 Sep 2025 09:27:33 +0200 Heiko Carstens <[email protected]> wrote:
Quoted 4 lines · Heiko Carstens
Quoted 3 lines · Christian Borntraeger

Am 16.09.25 um 19:05 schrieb Claudio Imbrenda:

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 6 lines
Quoted 4 lines
Quoted 1 line

I think GFP_ATOMIC actually gives more guarantees?

In real life GFP_ATOMIC can fail, GFP_KERNEL does not.All gfp allocation failures are usually the atomic ones.

interesting... then I guess I need GFP_KERNEL | GFP_ATOMIC ?

No. ATOMIC always means: can fail.

Quoted 7 lines

my issue is that GFP_KERNEL can sleep, and this allocation is sometimes called from atomic contexts (e.g. while holding spinlocks)

the right way to do this would be with mempools, to allocate memory (and potentially sleep) when we are not in atomic context, and use it whenever needed. this is on my to-do list for the future, but right now I'd like to avoid having to refactor a ton of code.

I doubt this is accetable even for an intermediate solution. As soon as the host is under memory pressure and starts doing I/O to free up memory, you will end up in -ENOMEM situations for simple guest page allocations.

What happens with a guest in such a situation? Is this gracefully handled without that the guest is terminated?

Quoted 4 lines
Quoted 3 lines

Another correction: it should be GFP_KERNEL_ACCOUNT instead, like it used to be before in gmap_alloc_crst() since those page tables should be accounted to the process which is allocating them.

Quoted 8 lines

gfp_types.h says:

* %GFP_KERNEL_ACCOUNT is the same as GFP_KERNEL, except the allocation is
* accounted to kmemcg.

I thought that without GFP_KERNEL_ACCOUNT, it would be accounted toward the process?
The documentation is a little confusing here

Documentation/core-api/memory-allocation.rst:

* Untrusted allocations triggered from userspace should be a subject of kmem accounting and must have ``__GFP_ACCOUNT`` bit set. There is the handy ``GFP_KERNEL_ACCOUNT`` shortcut for ``GFP_KERNEL`` allocations that should be accounted.

This is done for all page table allocations, except if mm == init_mm (and unfortunately also not for the s390 specific crste alloc functions - I'll send a patch for that).

author maintainer
#32 lore raw

Re: [PATCH v2 08/20] KVM: s390: KVM page table management functions: allocation

Quoted 26 lines · On Wed, 17 Sep 2025 14:30:06 +0200 Heiko Carstens <[email protected]> wrote:
Quoted 17 lines · Claudio Imbrenda
Quoted 5 lines · On Wed, 17 Sep 2025 09:27:33 +0200 Heiko Carstens <[email protected]> wrote:
Quoted 4 lines · Heiko Carstens
Quoted 3 lines · Christian Borntraeger

Am 16.09.25 um 19:05 schrieb Claudio Imbrenda:

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 6 lines
Quoted 4 lines
Quoted 1 line

I think GFP_ATOMIC actually gives more guarantees?

In real life GFP_ATOMIC can fail, GFP_KERNEL does not.All gfp allocation failures are usually the atomic ones.

interesting... then I guess I need GFP_KERNEL | GFP_ATOMIC ?

No. ATOMIC always means: can fail.

Quoted 7 lines

my issue is that GFP_KERNEL can sleep, and this allocation is sometimes called from atomic contexts (e.g. while holding spinlocks)

the right way to do this would be with mempools, to allocate memory (and potentially sleep) when we are not in atomic context, and use it whenever needed. this is on my to-do list for the future, but right now I'd like to avoid having to refactor a ton of code.

Quoted 7 lines

I doubt this is accetable even for an intermediate solution. As soon as the host is under memory pressure and starts doing I/O to free up memory, you will end up in -ENOMEM situations for simple guest page allocations.

What happens with a guest in such a situation? Is this gracefully handled without that the guest is terminated?

well, we return -ENOMEM to userspace (and qemu will probably kill the guest)

but if we can't even allocate 16kB, probably we're already in a pretty bad situation

if you think this is not acceptable, I guess I'll have to implement mempools

Quoted 16 lines
Quoted 5 lines
Quoted 4 lines

Another correction: it should be GFP_KERNEL_ACCOUNT instead, like it used to be before in gmap_alloc_crst() since those page tables should be accounted to the process which is allocating them.

Quoted 8 lines

gfp_types.h says:

* %GFP_KERNEL_ACCOUNT is the same as GFP_KERNEL, except the allocation is
* accounted to kmemcg.

I thought that without GFP_KERNEL_ACCOUNT, it would be accounted toward the process?
The documentation is a little confusing here

Documentation/core-api/memory-allocation.rst:

Quoted 8 lines

* Untrusted allocations triggered from userspace should be a subject of kmem accounting and must have ``__GFP_ACCOUNT`` bit set. There is the handy ``GFP_KERNEL_ACCOUNT`` shortcut for ``GFP_KERNEL`` allocations that should be accounted.

This is done for all page table allocations, except if mm == init_mm (and unfortunately also not for the s390 specific crste alloc functions - I'll send a patch for that).

alright, then GFP_KERNEL_ACCOUNT it is

#33 lore raw

Re: [PATCH v2 08/20] KVM: s390: KVM page table management functions: allocation

Am 17.09.25 um 15:11 schrieb Claudio Imbrenda:

Quoted 37 lines
Quoted 26 lines · On Wed, 17 Sep 2025 14:30:06 +0200 Heiko Carstens <[email protected]> wrote:
Quoted 17 lines · Claudio Imbrenda
Quoted 5 lines · On Wed, 17 Sep 2025 09:27:33 +0200 Heiko Carstens <[email protected]> wrote:
Quoted 4 lines · Heiko Carstens
Quoted 3 lines · Christian Borntraeger

Am 16.09.25 um 19:05 schrieb Claudio Imbrenda:

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 6 lines
Quoted 4 lines
Quoted 1 line

I think GFP_ATOMIC actually gives more guarantees?

In real life GFP_ATOMIC can fail, GFP_KERNEL does not.All gfp allocation failures are usually the atomic ones.

interesting... then I guess I need GFP_KERNEL | GFP_ATOMIC ?

No. ATOMIC always means: can fail.

Quoted 7 lines

my issue is that GFP_KERNEL can sleep, and this allocation is sometimes called from atomic contexts (e.g. while holding spinlocks)

the right way to do this would be with mempools, to allocate memory (and potentially sleep) when we are not in atomic context, and use it whenever needed. this is on my to-do list for the future, but right now I'd like to avoid having to refactor a ton of code.

Quoted 7 lines

I doubt this is accetable even for an intermediate solution. As soon as the host is under memory pressure and starts doing I/O to free up memory, you will end up in -ENOMEM situations for simple guest page allocations.

What happens with a guest in such a situation? Is this gracefully handled without that the guest is terminated?

Quoted 8 lines

well, we return -ENOMEM to userspace (and qemu will probably kill the guest)

but if we can't even allocate 16kB, probably we're already in a pretty bad situation

if you think this is not acceptable, I guess I'll have to implement mempools

This is not acceptable. 16k atomic allocations are pretty much guaranteed to fail after a while of high workload.
What are the callers of this allocation?

author maintainer
#34 lore raw

Re: [PATCH v2 08/20] KVM: s390: KVM page table management functions: allocation

Quoted 44 lines · On Wed, 17 Sep 2025 15:26:33 +0200 Christian Borntraeger…

Am 17.09.25 um 15:11 schrieb Claudio Imbrenda:

Quoted 43 lines
Quoted 26 lines · On Wed, 17 Sep 2025 14:30:06 +0200 Heiko Carstens <[email protected]> wrote:
Quoted 17 lines · Claudio Imbrenda
Quoted 5 lines · On Wed, 17 Sep 2025 09:27:33 +0200 Heiko Carstens <[email protected]> wrote:
Quoted 4 lines · Heiko Carstens
Quoted 3 lines · Christian Borntraeger

Am 16.09.25 um 19:05 schrieb Claudio Imbrenda:

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 6 lines
Quoted 4 lines
Quoted 1 line

I think GFP_ATOMIC actually gives more guarantees?

In real life GFP_ATOMIC can fail, GFP_KERNEL does not.All gfp allocation failures are usually the atomic ones.

interesting... then I guess I need GFP_KERNEL | GFP_ATOMIC ?

No. ATOMIC always means: can fail.

Quoted 7 lines

my issue is that GFP_KERNEL can sleep, and this allocation is sometimes called from atomic contexts (e.g. while holding spinlocks)

the right way to do this would be with mempools, to allocate memory (and potentially sleep) when we are not in atomic context, and use it whenever needed. this is on my to-do list for the future, but right now I'd like to avoid having to refactor a ton of code.

Quoted 7 lines

I doubt this is accetable even for an intermediate solution. As soon as the host is under memory pressure and starts doing I/O to free up memory, you will end up in -ENOMEM situations for simple guest page allocations.

What happens with a guest in such a situation? Is this gracefully handled without that the guest is terminated?

well, we return -ENOMEM to userspace (and qemu will probably kill the guest)

but if we can't even allocate 16kB, probably we're already in a pretty bad situation

Quoted 6 lines
Quoted 2 lines

if you think this is not acceptable, I guess I'll have to implement mempools

This is not acceptable. 16k atomic allocations are pretty much guaranteed to fail after a while of high workload.
What are the callers of this allocation?

literally anything that touches the gmap page tables, since we need to hold kvm->mmu_lock, which is an rw spinlock

I'll use mempools

#35 lore raw

Re: [PATCH v2 08/20] KVM: s390: KVM page table management functions: allocation

Am 17.09.25 um 16:00 schrieb Claudio Imbrenda:

Quoted 50 lines
Quoted 47 lines · On Wed, 17 Sep 2025 15:26:33 +0200 Christian Borntraeger…

Am 17.09.25 um 15:11 schrieb Claudio Imbrenda:

Quoted 45 lines
Quoted 26 lines · On Wed, 17 Sep 2025 14:30:06 +0200 Heiko Carstens <[email protected]> wrote:
Quoted 17 lines · Claudio Imbrenda
Quoted 5 lines · On Wed, 17 Sep 2025 09:27:33 +0200 Heiko Carstens <[email protected]> wrote:
Quoted 4 lines · Heiko Carstens
Quoted 3 lines · Christian Borntraeger

Am 16.09.25 um 19:05 schrieb Claudio Imbrenda:

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 6 lines
Quoted 4 lines
Quoted 1 line

I think GFP_ATOMIC actually gives more guarantees?

In real life GFP_ATOMIC can fail, GFP_KERNEL does not.All gfp allocation failures are usually the atomic ones.

interesting... then I guess I need GFP_KERNEL | GFP_ATOMIC ?

No. ATOMIC always means: can fail.

Quoted 7 lines

my issue is that GFP_KERNEL can sleep, and this allocation is sometimes called from atomic contexts (e.g. while holding spinlocks)

the right way to do this would be with mempools, to allocate memory (and potentially sleep) when we are not in atomic context, and use it whenever needed. this is on my to-do list for the future, but right now I'd like to avoid having to refactor a ton of code.

Quoted 7 lines

I doubt this is accetable even for an intermediate solution. As soon as the host is under memory pressure and starts doing I/O to free up memory, you will end up in -ENOMEM situations for simple guest page allocations.

What happens with a guest in such a situation? Is this gracefully handled without that the guest is terminated?

well, we return -ENOMEM to userspace (and qemu will probably kill the guest)

but if we can't even allocate 16kB, probably we're already in a pretty bad situation

if you think this is not acceptable, I guess I'll have to implement mempools

Quoted 6 lines
Quoted 3 lines

This is not acceptable. 16k atomic allocations are pretty much guaranteed to fail after a while of high workload.
What are the callers of this allocation?

literally anything that touches the gmap page tables, since we need to hold kvm->mmu_lock, which is an rw spinlock

So how is x86 allocating page table memory. I cant see GPF_ATOMIC or mempool over there.

author maintainer
#36 lore raw

Re: [PATCH v2 08/20] KVM: s390: KVM page table management functions: allocation

Quoted 51 lines · On Wed, 17 Sep 2025 16:05:44 +0200 Christian Borntraeger…

Am 17.09.25 um 16:00 schrieb Claudio Imbrenda:

Quoted 50 lines
Quoted 47 lines · On Wed, 17 Sep 2025 15:26:33 +0200 Christian Borntraeger…

Am 17.09.25 um 15:11 schrieb Claudio Imbrenda:

Quoted 45 lines
Quoted 26 lines · On Wed, 17 Sep 2025 14:30:06 +0200 Heiko Carstens <[email protected]> wrote:
Quoted 17 lines · Claudio Imbrenda
Quoted 5 lines · On Wed, 17 Sep 2025 09:27:33 +0200 Heiko Carstens <[email protected]> wrote:
Quoted 4 lines · Heiko Carstens
Quoted 3 lines · Christian Borntraeger

Am 16.09.25 um 19:05 schrieb Claudio Imbrenda:

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 6 lines
Quoted 4 lines
Quoted 1 line

I think GFP_ATOMIC actually gives more guarantees?

In real life GFP_ATOMIC can fail, GFP_KERNEL does not.All gfp allocation failures are usually the atomic ones.

interesting... then I guess I need GFP_KERNEL | GFP_ATOMIC ?

No. ATOMIC always means: can fail.

Quoted 7 lines

my issue is that GFP_KERNEL can sleep, and this allocation is sometimes called from atomic contexts (e.g. while holding spinlocks)

the right way to do this would be with mempools, to allocate memory (and potentially sleep) when we are not in atomic context, and use it whenever needed. this is on my to-do list for the future, but right now I'd like to avoid having to refactor a ton of code.

Quoted 7 lines

I doubt this is accetable even for an intermediate solution. As soon as the host is under memory pressure and starts doing I/O to free up memory, you will end up in -ENOMEM situations for simple guest page allocations.

What happens with a guest in such a situation? Is this gracefully handled without that the guest is terminated?

well, we return -ENOMEM to userspace (and qemu will probably kill the guest)

but if we can't even allocate 16kB, probably we're already in a pretty bad situation

if you think this is not acceptable, I guess I'll have to implement mempools

Quoted 8 lines
Quoted 6 lines
Quoted 3 lines

This is not acceptable. 16k atomic allocations are pretty much guaranteed to fail after a while of high workload.
What are the callers of this allocation?

literally anything that touches the gmap page tables, since we need to hold kvm->mmu_lock, which is an rw spinlock

So how is x86 allocating page table memory. I cant see GPF_ATOMIC or mempool over there.

I'll check, but I think they use an RCU?

author maintainer
#37 lore raw

Re: [PATCH v2 08/20] KVM: s390: KVM page table management functions: allocation

In reply to Christian Borntraeger
Quoted 51 lines · On Wed, 17 Sep 2025 16:05:44 +0200 Christian Borntraeger…

Am 17.09.25 um 16:00 schrieb Claudio Imbrenda:

Quoted 50 lines
Quoted 47 lines · On Wed, 17 Sep 2025 15:26:33 +0200 Christian Borntraeger…

Am 17.09.25 um 15:11 schrieb Claudio Imbrenda:

Quoted 45 lines
Quoted 26 lines · On Wed, 17 Sep 2025 14:30:06 +0200 Heiko Carstens <[email protected]> wrote:
Quoted 17 lines · Claudio Imbrenda
Quoted 5 lines · On Wed, 17 Sep 2025 09:27:33 +0200 Heiko Carstens <[email protected]> wrote:
Quoted 4 lines · Heiko Carstens
Quoted 3 lines · Christian Borntraeger

Am 16.09.25 um 19:05 schrieb Claudio Imbrenda:

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 6 lines
Quoted 4 lines
Quoted 1 line

I think GFP_ATOMIC actually gives more guarantees?

In real life GFP_ATOMIC can fail, GFP_KERNEL does not.All gfp allocation failures are usually the atomic ones.

interesting... then I guess I need GFP_KERNEL | GFP_ATOMIC ?

No. ATOMIC always means: can fail.

Quoted 7 lines

my issue is that GFP_KERNEL can sleep, and this allocation is sometimes called from atomic contexts (e.g. while holding spinlocks)

the right way to do this would be with mempools, to allocate memory (and potentially sleep) when we are not in atomic context, and use it whenever needed. this is on my to-do list for the future, but right now I'd like to avoid having to refactor a ton of code.

Quoted 7 lines

I doubt this is accetable even for an intermediate solution. As soon as the host is under memory pressure and starts doing I/O to free up memory, you will end up in -ENOMEM situations for simple guest page allocations.

What happens with a guest in such a situation? Is this gracefully handled without that the guest is terminated?

well, we return -ENOMEM to userspace (and qemu will probably kill the guest)

but if we can't even allocate 16kB, probably we're already in a pretty bad situation

if you think this is not acceptable, I guess I'll have to implement mempools

Quoted 8 lines
Quoted 6 lines
Quoted 3 lines

This is not acceptable. 16k atomic allocations are pretty much guaranteed to fail after a while of high workload.
What are the callers of this allocation?

literally anything that touches the gmap page tables, since we need to hold kvm->mmu_lock, which is an rw spinlock

So how is x86 allocating page table memory. I cant see GPF_ATOMIC or mempool over there.

after digging through the code: most other archs use kvm_mmu_memory_cache, which is similar in spirit to mempools

I don't know how I managed to never notice it until now

maintainer
#38 lore raw

Re: [PATCH v2 08/20] KVM: s390: KVM page table management functions: allocation

In reply to Claudio Imbrenda
Quoted 17 lines · Claudio Imbrenda
Quoted 15 lines · On Wed, 17 Sep 2025 14:30:06 +0200 Heiko Carstens <[email protected]> wrote:
Quoted 9 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line

No. ATOMIC always means: can fail.

my issue is that GFP_KERNEL can sleep, and this allocation is sometimes called from atomic contexts (e.g. while holding spinlocks)

the right way to do this would be with mempools, to allocate memory (and potentially sleep) when we are not in atomic context, and use it whenever needed. this is on my to-do list for the future, but right now I'd like to avoid having to refactor a ton of code.

I doubt this is accetable even for an intermediate solution. As soon as the host is under memory pressure and starts doing I/O to free up memory, you will end up in -ENOMEM situations for simple guest page allocations.

Quoted 8 lines
Quoted 2 lines

What happens with a guest in such a situation? Is this gracefully handled without that the guest is terminated?

well, we return -ENOMEM to userspace (and qemu will probably kill the guest)

but if we can't even allocate 16kB, probably we're already in a pretty bad situation

Not necessarily...

Quoted 2 lines

if you think this is not acceptable, I guess I'll have to implement mempools

...since the kernel _might_ be easily capable of freeing memory, e.g. by writing to swap or writing file contents to disk. But this wouldn't happen with GFP_ATOMIC allocations.

I'm just stating that with GFP_ATOMIC for guest page table allocations it is much more likely that guests might be killed compared to what we have now.

Also I'm wondering why such code paths where page tables in atomic context need to be allocated even exist. Is that s390 specific (aka the new code), or is this due to common code requirements?

author maintainer
#39 lore raw

Re: [PATCH v2 08/20] KVM: s390: KVM page table management functions: allocation

Quoted 36 lines · On Wed, 17 Sep 2025 15:31:42 +0200 Heiko Carstens <[email protected]> wrote:
Quoted 17 lines · Claudio Imbrenda
Quoted 15 lines · On Wed, 17 Sep 2025 14:30:06 +0200 Heiko Carstens <[email protected]> wrote:
Quoted 9 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line

No. ATOMIC always means: can fail.

my issue is that GFP_KERNEL can sleep, and this allocation is sometimes called from atomic contexts (e.g. while holding spinlocks)

the right way to do this would be with mempools, to allocate memory (and potentially sleep) when we are not in atomic context, and use it whenever needed. this is on my to-do list for the future, but right now I'd like to avoid having to refactor a ton of code.

I doubt this is accetable even for an intermediate solution. As soon as the host is under memory pressure and starts doing I/O to free up memory, you will end up in -ENOMEM situations for simple guest page allocations.

Quoted 8 lines
Quoted 2 lines

What happens with a guest in such a situation? Is this gracefully handled without that the guest is terminated?

well, we return -ENOMEM to userspace (and qemu will probably kill the guest)

but if we can't even allocate 16kB, probably we're already in a pretty bad situation

Not necessarily...

Quoted 2 lines

if you think this is not acceptable, I guess I'll have to implement mempools

...since the kernel _might_ be easily capable of freeing memory, e.g. by writing to swap or writing file contents to disk. But this wouldn't happen with GFP_ATOMIC allocations.

Quoted 6 lines

I'm just stating that with GFP_ATOMIC for guest page table allocations it is much more likely that guests might be killed compared to what we have now.

Also I'm wondering why such code paths where page tables in atomic context need to be allocated even exist. Is that s390 specific (aka the new code), or is this due to common code requirements?

this is in the gmap rewrite, so new code in this series

mempools it is, then

author maintainer
#40 lore raw

Re: [PATCH v2 08/20] KVM: s390: KVM page table management functions: allocation

In reply to Heiko Carstens
Quoted 13 lines · On Tue, 16 Sep 2025 19:36:44 +0200 Heiko Carstens <[email protected]> wrote:
Quoted 11 lines · Christian Borntraeger

Am 16.09.25 um 19:05 schrieb Claudio Imbrenda:

Quoted 6 lines
Quoted 4 lines
Quoted 1 line

I think GFP_ATOMIC actually gives more guarantees?

In real life GFP_ATOMIC can fail, GFP_KERNEL does not.All gfp allocation failures are usually the atomic ones.

interesting... then I guess I need GFP_KERNEL | GFP_ATOMIC ?

No. ATOMIC always means: can fail.

Quoted 8 lines

So GFP_KERNEL alone is what is needed here. It is undocumented that small GFP_KERNEL allocations will never fail (retried for ever):

https://lwn.net/Articles/627419/
https://lwn.net/Articles/723317/

While GFP_ATOMIC will fail e.g. if I/O is required to free up some memory.

ah, "how small is small" is answered in those links, ignore my question

author maintainer
#41 lore raw

PATCH v2 9/20 KVM: s390: KVM page table management functions: clear and replace

In reply to Claudio Imbrenda

Add page table management functions to be used for KVM guest (gmap) page tables.

This patch adds functions to clear, replace or exchange DAT table entries.

---
 arch/s390/kvm/dat.c | 120 ++++++++++++++++++++++++++++++++++++++++++++
 arch/s390/kvm/dat.h |  40 +++++++++++++++
 2 files changed, 160 insertions(+)
arch/s390/kvm/dat.c +120 -0
index 326be78adcda..f26e3579bd77 100644
@@ -89,3 +89,123 @@ void dat_free_level(struct crst_table *table, bool owns_ptes)
 	}
 	dat_free_crst(table);
 }
+
+/**
+ * dat_crstep_xchg - exchange a guest CRST with another
+ * @crstep: pointer to the CRST entry
+ * @new: replacement entry
+ * @gfn: the affected guest address
+ * @asce: the ASCE of the address space
+ *
+ * This function is assumed to be called with the guest_table_lock
+ * held.
+ */
+void dat_crstep_xchg(union crste *crstep, union crste new, gfn_t gfn, union asce asce)
+{
+	if (crstep->h.i) {
+		WRITE_ONCE(*crstep, new);
+		return;
+	} else if (cpu_has_edat2()) {
+		crdte_crste(crstep, *crstep, new, gfn, asce);
+		return;
+	}
+
+	if (machine_has_tlb_guest())
+		idte_crste(crstep, gfn, IDTE_GUEST_ASCE, asce, IDTE_GLOBAL);
+	else if (cpu_has_idte())
+		idte_crste(crstep, gfn, 0, NULL_ASCE, IDTE_GLOBAL);
+	else
+		csp_invalidate_crste(crstep);
+	WRITE_ONCE(*crstep, new);
+}
+
+/**
+ * dat_crstep_xchg_atomic - exchange a gmap pmd with another
+ * @crstep: pointer to the crste entry
+ * @old: expected old value
+ * @new: replacement entry
+ * @gfn: the affected guest address
+ * @asce: the asce of the address space
+ *
+ * This function should only be called on invalid crstes, or on crstes with
+ * FC = 1, as that guarantees the presence of CSPG.
+ *
+ * Return: true if the exchange was successful.
+ */
+bool dat_crstep_xchg_atomic(union crste *crstep, union crste old, union crste new, gfn_t gfn,
+			    union asce asce)
+{
+	if (old.h.i)
+		return arch_try_cmpxchg((long *)crstep, &old.val, new.val);
+	if (cpu_has_edat2())
+		return crdte_crste(crstep, old, new, gfn, asce);
+	if (cpu_has_idte())
+		return cspg_crste(crstep, old, new);
+
+	WARN_ONCE(1, "Machine does not have CSPG and DAT table was not invalid.");
+	return false;
+}
+
+static void dat_set_storage_key_from_pgste(union pte pte, union pgste pgste)
+{
+	union skey nkey = { .acc = pgste.acc, .fp = pgste.fp };
+
+	page_set_storage_key(pte_origin(pte), nkey.skey, 0);
+}
+
+static void dat_move_storage_key(union pte old, union pte new)
+{
+	page_set_storage_key(pte_origin(new), page_get_storage_key(pte_origin(old)), 1);
+}
+
+static union pgste dat_save_storage_key_into_pgste(union pte pte, union pgste pgste)
+{
+	union skey skey;
+
+	skey.skey = page_get_storage_key(pte_origin(pte));
+
+	pgste.acc = skey.acc;
+	pgste.fp = skey.fp;
+	pgste.gr |= skey.r;
+	pgste.gc |= skey.c;
+
+	return pgste;
+}
+
+union pgste __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new, gfn_t gfn,
+			    union asce asce, bool has_skeys)
+{
+	union pte old = READ_ONCE(*ptep);
+
+	/* Updating only the software bits while holding the pgste lock */
+	if (!((ptep->val ^ new.val) & ~_PAGE_SW_BITS)) {
+		WRITE_ONCE(ptep->swbyte, new.swbyte);
+		return pgste;
+	}
+
+	if (!old.h.i) {
+		unsigned long opts = IPTE_GUEST_ASCE | (pgste.nodat ? IPTE_NODAT : 0);
+
+		if (machine_has_tlb_guest())
+			__ptep_ipte(gfn_to_gpa(gfn), (void *)ptep, opts, asce.val, IPTE_GLOBAL);
+		else
+			__ptep_ipte(gfn_to_gpa(gfn), (void *)ptep, 0, 0, IPTE_GLOBAL);
+	}
+
+	if (has_skeys) {
+		if (old.h.i && !new.h.i)
+			/* Invalid to valid: restore storage keys from PGSTE */
+			dat_set_storage_key_from_pgste(new, pgste);
+		else if (!old.h.i && new.h.i)
+			/* Valid to invalid: save storage keys to PGSTE */
+			pgste = dat_save_storage_key_into_pgste(new, pgste);
+		else if (!old.h.i && !new.h.i)
+			/* Valid to valid: move storage keys */
+			if (old.h.pfra != new.h.pfra)
+				dat_move_storage_key(*ptep, new);
+		/* Invalid to invalid: nothing to do */
+	}
+
+	WRITE_ONCE(*ptep, new);
+	return pgste;
+}
arch/s390/kvm/dat.h +40 -0
index 5056cfa02619..9e23f6cdbf73 100644
@@ -385,6 +385,12 @@ static inline union crste _crste_fc1(kvm_pfn_t pfn, int tt, bool w, bool d)
 	return res;
 }
 
+union pgste __must_check __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new,
+					 gfn_t gfn, union asce asce, bool has_skeys);
+bool dat_crstep_xchg_atomic(union crste *crstep, union crste old, union crste new, gfn_t gfn,
+			    union asce asce);
+void dat_crstep_xchg(union crste *crstep, union crste new, gfn_t gfn, union asce asce);
+
 void dat_free_level(struct crst_table *table, bool owns_ptes);
 struct page_table *dat_alloc_pt(unsigned long pte_bits, unsigned long pgste_bits);
 struct crst_table *dat_alloc_crst(unsigned long init);
@@ -677,6 +683,21 @@ static inline void pgste_set_unlock(union pte *ptep, union pgste pgste)
 	WRITE_ONCE(*pgste_of(ptep), pgste);
 }
 
+static inline void dat_ptep_xchg(union pte *ptep, union pte new, gfn_t gfn, union asce asce,
+				 bool has_skeys)
+{
+	union pgste pgste;
+
+	pgste = pgste_get_lock(ptep);
+	pgste = __dat_ptep_xchg(ptep, pgste, new, gfn, asce, has_skeys);
+	pgste_set_unlock(ptep, pgste);
+}
+
+static inline void dat_ptep_clear(union pte *ptep, gfn_t gfn, union asce asce, bool has_skeys)
+{
+	dat_ptep_xchg(ptep, _PTE_EMPTY, gfn, asce, has_skeys);
+}
+
 static inline struct page_table *dat_alloc_empty_pt(void)
 {
 	return dat_alloc_pt(_PAGE_INVALID, 0);
@@ -694,4 +715,23 @@ static inline void _dat_free_crst(struct crst_table *table)
 
 #define dat_free_crst(x) _dat_free_crst(_CRSTP(x))
 
+static inline bool dat_pmdp_xchg_atomic(union pmd *pmdp, union pmd old, union pmd new,
+					gfn_t gfn, union asce asce)
+{
+	return dat_crstep_xchg_atomic(_CRSTEP(pmdp), _CRSTE(old), _CRSTE(new), gfn, asce);
+}
+
+static inline bool dat_pudp_xchg_atomic(union pud *pudp, union pud old, union pud new,
+					gfn_t gfn, union asce asce)
+{
+	return dat_crstep_xchg_atomic(_CRSTEP(pudp), _CRSTE(old), _CRSTE(new), gfn, asce);
+}
+
+static inline void dat_crstep_clear(union crste *crstep, gfn_t gfn, union asce asce)
+{
+	union crste newcrste = _CRSTE_EMPTY(crstep->h.tt);
+
+	dat_crstep_xchg(crstep, newcrste, gfn, asce);
+}
+
 #endif /* __KVM_S390_DAT_H */
Signature · 1 line
2.51.0
maintainer
#42 lore raw

Re: [PATCH v2 09/20] KVM: s390: KVM page table management functions: clear and replace

Quoted 37 lines · Claudio Imbrenda

Add page table management functions to be used for KVM guest (gmap) page tables.

This patch adds functions to clear, replace or exchange DAT table entries.

---
  arch/s390/kvm/dat.c | 120 ++++++++++++++++++++++++++++++++++++++++++++
  arch/s390/kvm/dat.h |  40 +++++++++++++++
  2 files changed, 160 insertions(+)
arch/s390/kvm/dat.c +17 -0
index 326be78adcda..f26e3579bd77 100644
@@ -89,3 +89,123 @@ void dat_free_level(struct crst_table *table, bool owns_ptes)
  	}
  	dat_free_crst(table);
  }
+
+/**
+ * dat_crstep_xchg - exchange a guest CRST with another
+ * @crstep: pointer to the CRST entry
+ * @new: replacement entry
+ * @gfn: the affected guest address
+ * @asce: the ASCE of the address space
+ *
+ * This function is assumed to be called with the guest_table_lock
+ * held.
+ */
+void dat_crstep_xchg(union crste *crstep, union crste new, gfn_t gfn, union asce asce)
+{
+	if (crstep->h.i) {
+		WRITE_ONCE(*crstep, new);
+		return;
+	} else if (cpu_has_edat2()) {

arch/s390/kvm/dat.c void dat_free_level(struct crst_table *table, bool owns_ptes) · line 118

Quoted 10 lines
arch/s390/kvm/dat.c +10 -0
+		crdte_crste(crstep, *crstep, new, gfn, asce);
+		return;
+	}
+
+	if (machine_has_tlb_guest())
+		idte_crste(crstep, gfn, IDTE_GUEST_ASCE, asce, IDTE_GLOBAL);
+	else if (cpu_has_idte())
+		idte_crste(crstep, gfn, 0, NULL_ASCE, IDTE_GLOBAL);
+	else
+		csp_invalidate_crste(crstep);

I'm wondering if we can make stfle 3 (DTE) a requirement for KVM or Linux as a whole since it was introduced with z990 AFAIK.

Quoted 29 lines
arch/s390/kvm/dat.c +29 -0
+	WRITE_ONCE(*crstep, new);
+}
+
+/**
+ * dat_crstep_xchg_atomic - exchange a gmap pmd with another
+ * @crstep: pointer to the crste entry
+ * @old: expected old value
+ * @new: replacement entry
+ * @gfn: the affected guest address
+ * @asce: the asce of the address space
+ *
+ * This function should only be called on invalid crstes, or on crstes with
+ * FC = 1, as that guarantees the presence of CSPG.
+ *
+ * Return: true if the exchange was successful.
+ */
+bool dat_crstep_xchg_atomic(union crste *crstep, union crste old, union crste new, gfn_t gfn,
+			    union asce asce)
+{
+	if (old.h.i)
+		return arch_try_cmpxchg((long *)crstep, &old.val, new.val);
+	if (cpu_has_edat2())
+		return crdte_crste(crstep, old, new, gfn, asce);
+	if (cpu_has_idte())
+		return cspg_crste(crstep, old, new);
+
+	WARN_ONCE(1, "Machine does not have CSPG and DAT table was not invalid.");
+	return false;
+}
author maintainer
#43 lore raw

Re: [PATCH v2 09/20] KVM: s390: KVM page table management functions: clear and replace

Quoted 43 lines · On Thu, 11 Sep 2025 14:57:40 +0200 Janosch Frank <[email protected]> wrote:
Quoted 42 lines · Claudio Imbrenda

Add page table management functions to be used for KVM guest (gmap) page tables.

This patch adds functions to clear, replace or exchange DAT table entries.

---
  arch/s390/kvm/dat.c | 120 ++++++++++++++++++++++++++++++++++++++++++++
  arch/s390/kvm/dat.h |  40 +++++++++++++++
  2 files changed, 160 insertions(+)
arch/s390/kvm/dat.c +22 -0
index 326be78adcda..f26e3579bd77 100644
@@ -89,3 +89,123 @@ void dat_free_level(struct crst_table *table, bool owns_ptes)
  	}
  	dat_free_crst(table);
  }
+
+/**
+ * dat_crstep_xchg - exchange a guest CRST with another
+ * @crstep: pointer to the CRST entry
+ * @new: replacement entry
+ * @gfn: the affected guest address
+ * @asce: the ASCE of the address space
+ *
+ * This function is assumed to be called with the guest_table_lock
+ * held.
+ */
+void dat_crstep_xchg(union crste *crstep, union crste new, gfn_t gfn, union asce asce)
+{
+	if (crstep->h.i) {
+		WRITE_ONCE(*crstep, new);
+		return;
+	} else if (cpu_has_edat2()) {
+		crdte_crste(crstep, *crstep, new, gfn, asce);
+		return;
+	}
+
+	if (machine_has_tlb_guest())
Quoted 8 lines

arch/s390/kvm/dat.c void dat_free_level(struct crst_table *table, bool owns_ptes) · line 118

Quoted 5 lines
arch/s390/kvm/dat.c +5 -0
+		idte_crste(crstep, gfn, IDTE_GUEST_ASCE, asce, IDTE_GLOBAL);
+	else if (cpu_has_idte())
+		idte_crste(crstep, gfn, 0, NULL_ASCE, IDTE_GLOBAL);
+	else
+		csp_invalidate_crste(crstep);  

I'm wondering if we can make stfle 3 (DTE) a requirement for KVM or Linux as a whole since it was introduced with z990 AFAIK.

AFAIK we don't support machines older than z10 anyway

but in that case we can only get rid of csp_invalidate_crste(), which is not much.

I can remove it, if you really think it's ugly

Quoted 30 lines
Quoted 30 lines
arch/s390/kvm/dat.c +29 -0
+	WRITE_ONCE(*crstep, new);
+}
+
+/**
+ * dat_crstep_xchg_atomic - exchange a gmap pmd with another
+ * @crstep: pointer to the crste entry
+ * @old: expected old value
+ * @new: replacement entry
+ * @gfn: the affected guest address
+ * @asce: the asce of the address space
+ *
+ * This function should only be called on invalid crstes, or on crstes with
+ * FC = 1, as that guarantees the presence of CSPG.
+ *
+ * Return: true if the exchange was successful.
+ */
+bool dat_crstep_xchg_atomic(union crste *crstep, union crste old, union crste new, gfn_t gfn,
+			    union asce asce)
+{
+	if (old.h.i)
+		return arch_try_cmpxchg((long *)crstep, &old.val, new.val);
+	if (cpu_has_edat2())
+		return crdte_crste(crstep, old, new, gfn, asce);
+	if (cpu_has_idte())
+		return cspg_crste(crstep, old, new);
+
+	WARN_ONCE(1, "Machine does not have CSPG and DAT table was not invalid.");
+	return false;
+}  
maintainer
#44 lore raw

Re: [PATCH v2 09/20] KVM: s390: KVM page table management functions: clear and replace

Quoted 55 lines · Claudio Imbrenda
Quoted 43 lines · On Thu, 11 Sep 2025 14:57:40 +0200 Janosch Frank <[email protected]> wrote:
Quoted 42 lines · Claudio Imbrenda

Add page table management functions to be used for KVM guest (gmap) page tables.

This patch adds functions to clear, replace or exchange DAT table entries.

---
   arch/s390/kvm/dat.c | 120 ++++++++++++++++++++++++++++++++++++++++++++
   arch/s390/kvm/dat.h |  40 +++++++++++++++
   2 files changed, 160 insertions(+)
arch/s390/kvm/dat.c +22 -0
index 326be78adcda..f26e3579bd77 100644
@@ -89,3 +89,123 @@ void dat_free_level(struct crst_table *table, bool owns_ptes)
   	}
   	dat_free_crst(table);
   }
+
+/**
+ * dat_crstep_xchg - exchange a guest CRST with another
+ * @crstep: pointer to the CRST entry
+ * @new: replacement entry
+ * @gfn: the affected guest address
+ * @asce: the ASCE of the address space
+ *
+ * This function is assumed to be called with the guest_table_lock
+ * held.
+ */
+void dat_crstep_xchg(union crste *crstep, union crste new, gfn_t gfn, union asce asce)
+{
+	if (crstep->h.i) {
+		WRITE_ONCE(*crstep, new);
+		return;
+	} else if (cpu_has_edat2()) {
+		crdte_crste(crstep, *crstep, new, gfn, asce);
+		return;
+	}
+
+	if (machine_has_tlb_guest())
Quoted 8 lines

arch/s390/kvm/dat.c void dat_free_level(struct crst_table *table, bool owns_ptes) · line 118

Quoted 5 lines
arch/s390/kvm/dat.c +5 -0
+		idte_crste(crstep, gfn, IDTE_GUEST_ASCE, asce, IDTE_GLOBAL);
+	else if (cpu_has_idte())
+		idte_crste(crstep, gfn, 0, NULL_ASCE, IDTE_GLOBAL);
+	else
+		csp_invalidate_crste(crstep);

I'm wondering if we can make stfle 3 (DTE) a requirement for KVM or Linux as a whole since it was introduced with z990 AFAIK.

Quoted 6 lines

AFAIK we don't support machines older than z10 anyway

but in that case we can only get rid of csp_invalidate_crste(), which is not much.

I can remove it, if you really think it's ugly

That was more of a general question, I think we have more of those checks in non-kvm code. Your code is fine as is.

Quoted 32 lines
Quoted 31 lines
Quoted 31 lines
arch/s390/kvm/dat.c +29 -0
+	WRITE_ONCE(*crstep, new);
+}
+
+/**
+ * dat_crstep_xchg_atomic - exchange a gmap pmd with another
+ * @crstep: pointer to the crste entry
+ * @old: expected old value
+ * @new: replacement entry
+ * @gfn: the affected guest address
+ * @asce: the asce of the address space
+ *
+ * This function should only be called on invalid crstes, or on crstes with
+ * FC = 1, as that guarantees the presence of CSPG.
+ *
+ * Return: true if the exchange was successful.
+ */
+bool dat_crstep_xchg_atomic(union crste *crstep, union crste old, union crste new, gfn_t gfn,
+			    union asce asce)
+{
+	if (old.h.i)
+		return arch_try_cmpxchg((long *)crstep, &old.val, new.val);
+	if (cpu_has_edat2())
+		return crdte_crste(crstep, old, new, gfn, asce);
+	if (cpu_has_idte())
+		return cspg_crste(crstep, old, new);
+
+	WARN_ONCE(1, "Machine does not have CSPG and DAT table was not invalid.");
+	return false;
+}
maintainer
#45 lore raw

Re: [PATCH v2 09/20] KVM: s390: KVM page table management functions: clear and replace

Quoted 23 lines · Janosch Frank
Quoted 22 lines · Claudio Imbrenda
Quoted 11 lines
Quoted 11 lines
arch/s390/kvm/dat.c +11 -0
+void dat_crstep_xchg(union crste *crstep, union crste new, gfn_t gfn, union asce asce)
+{
+	if (crstep->h.i) {
+		WRITE_ONCE(*crstep, new);
+		return;
+	} else if (cpu_has_edat2()) {
+		crdte_crste(crstep, *crstep, new, gfn, asce);
+		return;
+	}
+
+	if (machine_has_tlb_guest())
Quoted 8 lines

arch/s390/kvm/dat.c void dat_free_level(struct crst_table *table, bool owns_ptes) · line 118

Quoted 5 lines
arch/s390/kvm/dat.c +5 -0
+		idte_crste(crstep, gfn, IDTE_GUEST_ASCE, asce, IDTE_GLOBAL);
+	else if (cpu_has_idte())
+		idte_crste(crstep, gfn, 0, NULL_ASCE, IDTE_GLOBAL);
+	else
+		csp_invalidate_crste(crstep);

I'm wondering if we can make stfle 3 (DTE) a requirement for KVM or Linux as a whole since it was introduced with z990 AFAIK.

AFAIK we don't support machines older than z10 anyway

Quoted 7 lines
Quoted 4 lines

but in that case we can only get rid of csp_invalidate_crste(), which is not much.

I can remove it, if you really think it's ugly

That was more of a general question, I think we have more of those checks in non-kvm code. Your code is fine as is.

Until now we have a check for all privileged instructions before executing them, since at least in theory they can go away in future machines.

The only exception is the mvcos instruction, since it is used all over the place.

Unless there is a really good reason (e.g. large amounts of code could be removed, or performance) I'd like to keep it like that.

maintainer
#46 lore raw

Re: [PATCH v2 09/20] KVM: s390: KVM page table management functions: clear and replace

In reply to Claudio Imbrenda
Quoted 11 lines · Claudio Imbrenda

Add page table management functions to be used for KVM guest (gmap) page tables.

This patch adds functions to clear, replace or exchange DAT table entries.

---
 arch/s390/kvm/dat.c | 120 ++++++++++++++++++++++++++++++++++++++++++++
 arch/s390/kvm/dat.h |  40 +++++++++++++++
 2 files changed, 160 insertions(+)

...

arch/s390/kvm/dat.c void dat_free_level(struct crst_table *table, bool owns_ptes) · line 143

Quoted 9 lines
arch/s390/kvm/dat.c +9 -0
+bool dat_crstep_xchg_atomic(union crste *crstep, union crste old, union crste new, gfn_t gfn,
+			    union asce asce)
+{
+	if (old.h.i)
+		return arch_try_cmpxchg((long *)crstep, &old.val, new.val);
+	if (cpu_has_edat2())
+		return crdte_crste(crstep, old, new, gfn, asce);
+	if (cpu_has_idte())
+		return cspg_crste(crstep, old, new);

FWIW, CSPG is present if EDAT1 is installed. So this should be cpu_has_edat1() instead, I would guess.

author maintainer
#47 lore raw

Re: [PATCH v2 09/20] KVM: s390: KVM page table management functions: clear and replace

Quoted 15 lines · On Tue, 16 Sep 2025 18:47:31 +0200 Heiko Carstens <[email protected]> wrote:
Quoted 11 lines · Claudio Imbrenda

Add page table management functions to be used for KVM guest (gmap) page tables.

This patch adds functions to clear, replace or exchange DAT table entries.

---
 arch/s390/kvm/dat.c | 120 ++++++++++++++++++++++++++++++++++++++++++++
 arch/s390/kvm/dat.h |  40 +++++++++++++++
 2 files changed, 160 insertions(+)  

...

Quoted 12 lines

arch/s390/kvm/dat.c void dat_free_level(struct crst_table *table, bool owns_ptes) · line 143

Quoted 9 lines
arch/s390/kvm/dat.c +9 -0
+bool dat_crstep_xchg_atomic(union crste *crstep, union crste old, union crste new, gfn_t gfn,
+			    union asce asce)
+{
+	if (old.h.i)
+		return arch_try_cmpxchg((long *)crstep, &old.val, new.val);
+	if (cpu_has_edat2())
+		return crdte_crste(crstep, old, new, gfn, asce);
+	if (cpu_has_idte())
+		return cspg_crste(crstep, old, new);  

FWIW, CSPG is present if EDAT1 is installed. So this should be cpu_has_edat1() instead, I would guess.

no, CSPG is present with the DAT-Enhancement facility (z990), which also brings IDTE. EDAT1 is more modern (z10)

maintainer
#48 lore raw

Re: [PATCH v2 09/20] KVM: s390: KVM page table management functions: clear and replace

Quoted 15 lines · Claudio Imbrenda
Quoted 4 lines · On Tue, 16 Sep 2025 18:47:31 +0200 Heiko Carstens <[email protected]> wrote:
Quoted 4 lines
arch/s390/kvm/dat.c +4 -0
+bool dat_crstep_xchg_atomic(union crste *crstep, union crste old, union crste new, gfn_t gfn,
+			    union asce asce)
+{
+	if (old.h.i)
Quoted 8 lines

arch/s390/kvm/dat.c void dat_free_level(struct crst_table *table, bool owns_ptes) · line 143

Quoted 5 lines
arch/s390/kvm/dat.c +5 -0
+		return arch_try_cmpxchg((long *)crstep, &old.val, new.val);
+	if (cpu_has_edat2())
+		return crdte_crste(crstep, old, new, gfn, asce);
+	if (cpu_has_idte())
+		return cspg_crste(crstep, old, new);  

FWIW, CSPG is present if EDAT1 is installed. So this should be cpu_has_edat1() instead, I would guess.

Quoted 2 lines

no, CSPG is present with the DAT-Enhancement facility (z990), which also brings IDTE. EDAT1 is more modern (z10)

Right you are. The PoP says "DAT-Enhancement facility 1", which translatedto me to EDAT-1, but that is of course something completely different. Sorry for the noise.

author maintainer
#49 lore raw

PATCH v2 10/20 KVM: s390: KVM page table management functions: walks

In reply to Claudio Imbrenda

Add page table management functions to be used for KVM guest (gmap) page tables.

This patch adds functions to walk to specific table entries, or to perform actions on a range of entries.

---
 arch/s390/kvm/dat.c | 351 ++++++++++++++++++++++++++++++++++++++++++++
 arch/s390/kvm/dat.h |  38 +++++
 2 files changed, 389 insertions(+)
arch/s390/kvm/dat.c +351 -0
index f26e3579bd77..fe93e1c07158 100644
@@ -209,3 +209,354 @@ union pgste __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new, g
 	WRITE_ONCE(*ptep, new);
 	return pgste;
 }
+
+/*
+ * dat_split_pmd is assumed to be called with mmap_lock held in read or write mode
+ */
+static int dat_split_pmd(union pmd *pmdp, gfn_t gfn, union asce asce)
+{
+	struct page_table *pt;
+	union pmd new, old;
+	union pte init;
+	int i;
+
+	old = READ_ONCE(*pmdp);
+
+	/* Already split, nothing to do */
+	if (!old.h.i && !old.h.fc)
+		return 0;
+
+	pt = dat_alloc_pt_noinit();
+	if (!pt)
+		return -ENOMEM;
+	new.val = virt_to_phys(pt);
+
+	while (old.h.i || old.h.fc) {
+		init.val = pmd_origin_large(old);
+		init.h.p = old.h.p;
+		init.h.i = old.h.i;
+		init.s.d = old.s.fc1.d;
+		init.s.w = old.s.fc1.w;
+		init.s.y = old.s.fc1.y;
+		init.s.sd = old.s.fc1.sd;
+		init.s.pr = old.s.fc1.pr;
+		if (old.h.fc) {
+			for (i = 0; i < _PAGE_ENTRIES; i++)
+				pt->ptes[i].val = init.val | i * PAGE_SIZE;
+			/* no need to take locks as the page table is not installed yet */
+			dat_init_pgstes(pt, old.s.fc1.prefix_notif ? PGSTE_IN_BIT : 0);
+		} else {
+			dat_init_page_table(pt, init.val, 0);
+		}
+
+		if (dat_pmdp_xchg_atomic(pmdp, old, new, gfn, asce))
+			return 0;
+		old = READ_ONCE(*pmdp);
+	}
+
+	dat_free_pt(pt);
+	return 0;
+}
+
+static int dat_split_crste(union crste *crstep, gfn_t gfn, union asce asce)
+{
+	struct crst_table *table;
+	union crste old, new, init;
+	int i;
+
+	old = READ_ONCE(*crstep);
+	if (is_pmd(old))
+		return dat_split_pmd(&crstep->pmd, gfn, asce);
+
+	/* Already split, nothing to do */
+	if (!old.h.i && !old.h.fc)
+		return 0;
+
+	table = dat_alloc_crst_noinit();
+	if (!table)
+		return -ENOMEM;
+
+	new.val = virt_to_phys(table);
+	new.h.tt = old.h.tt;
+	new.h.fc0.tl = _REGION_ENTRY_LENGTH;
+
+	while (old.h.i || old.h.fc) {
+		init = old;
+		init.h.tt--;
+		if (old.h.fc) {
+			for (i = 0; i < _CRST_ENTRIES; i++)
+				table->crstes[i].val = init.val | i * HPAGE_SIZE;
+		} else {
+			crst_table_init((void *)table, init.val);
+		}
+		if (dat_crstep_xchg_atomic(crstep, old, new, gfn, asce))
+			return 0;
+		old = READ_ONCE(*crstep);
+	}
+
+	dat_free_crst(table);
+	return 0;
+}
+
+/**
+ * dat_entry_walk() - walk the gmap page tables
+ * @gfn: guest frame
+ * @asce: the ASCE of the address space
+ * @flags: flags from WALK_* macros
+ * @level: level to walk to, from LEVEL_* macros
+ * @last: will be filled the last visited non-pte DAT entry
+ * @ptepp: will be filled the last visited pte entry, if any, otherwise NULL
+ *
+ * Returns a table entry pointer for the given guest address and @level
+ *
+ * The @flags have the following meanings:
+ * * @DAT_WALK_IGN_HOLES: consider holes as normal table entries
+ * * @DAT_WALK_ALLOC: allocate new tables to reach the requested level, if needed
+ * * @DAT_WALK_SPLIT: split existing large pages to reach the requested level, if needed
+ * * @DAT_WALK_LEAF: return successfully whenever a large page is encountered
+ * * @DAT_WALK_ANY: return successfully even if the requested level could not be reached
+ * * @DAT_WALK_CONTINUE: walk to the requested level with the specified flags, and then try to
+ *                       continue walking to ptes with only DAT_WALK_ANY
+ *
+ * Context: called with kvm->mmu_lock held.
+ *
+ * Return:
+ * * PGM_ADDRESSING if the requested address lies outside memory
+ * * a PIC number if the requested address lies in a memory hole of type _DAT_TOKEN_PIC
+ * * -EFAULT if the requested address lies inside a memory hole of a different type
+ * * -EINVAL if the given ASCE is not compatible with the requested level
+ * * -EFBIG if the requested level could not be reached because a larger frame was found
+ * * -ENOENT if the requested level could not be reached for other reasons
+ * * -ENOMEM if running out of memory while allocating or splitting a table
+ */
+int dat_entry_walk(gfn_t gfn, union asce asce, int flags, int walk_level,
+		   union crste **last, union pte **ptepp)
+{
+	bool continue_anyway = flags & DAT_WALK_CONTINUE;
+	bool ign_holes = flags & DAT_WALK_IGN_HOLES;
+	bool allocate = flags & DAT_WALK_ALLOC;
+	bool split = flags & DAT_WALK_SPLIT;
+	bool leaf = flags & DAT_WALK_LEAF;
+	bool any = flags & DAT_WALK_ANY;
+	struct page_table *pgtable;
+	struct crst_table *table;
+	union crste entry;
+	int rc;
+
+	*last = NULL;
+	*ptepp = NULL;
+	if (WARN_ON_ONCE(unlikely(!asce.val)))
+		return -EINVAL;
+	if (WARN_ON_ONCE(unlikely(walk_level > asce.dt)))
+		return -EINVAL;
+	if (!asce_contains_gfn(asce, gfn))
+		return PGM_ADDRESSING;
+
+	table = dereference_asce(asce);
+	if (asce.dt >= ASCE_TYPE_REGION1) {
+		*last = table->crstes + pgd_index(gfn_to_gpa(gfn));
+		entry = READ_ONCE(**last);
+		if (WARN_ON_ONCE(unlikely(entry.h.tt != LEVEL_PGD)))
+			return -EINVAL;
+		if (crste_hole(entry) && !ign_holes)
+			return entry.tok.type == _DAT_TOKEN_PIC ? entry.tok.par : -EFAULT;
+		if (walk_level == LEVEL_PGD)
+			return 0;
+		if (entry.pgd.h.i) {
+			if (!allocate)
+				return any ? 0 : -ENOENT;
+			rc = dat_split_crste(*last, gfn, asce);
+			if (rc)
+				return rc;
+			entry = READ_ONCE(**last);
+		}
+		table = dereference_crste(entry.pgd);
+	}
+
+	if (asce.dt >= ASCE_TYPE_REGION2) {
+		*last = table->crstes + p4d_index(gfn_to_gpa(gfn));
+		entry = READ_ONCE(**last);
+		if (WARN_ON_ONCE(unlikely(entry.h.tt != LEVEL_P4D)))
+			return -EINVAL;
+		if (crste_hole(entry) && !ign_holes)
+			return entry.tok.type == _DAT_TOKEN_PIC ? entry.tok.par : -EFAULT;
+		if (walk_level == LEVEL_P4D)
+			return 0;
+		if (entry.p4d.h.i) {
+			if (!allocate)
+				return any ? 0 : -ENOENT;
+			rc = dat_split_crste(*last, gfn, asce);
+			if (rc)
+				return rc;
+			entry = READ_ONCE(**last);
+		}
+		table = dereference_crste(entry.p4d);
+	}
+
+	if (asce.dt >= ASCE_TYPE_REGION3) {
+		*last = table->crstes + pud_index(gfn_to_gpa(gfn));
+		entry = READ_ONCE(**last);
+		if (WARN_ON_ONCE(unlikely(entry.h.tt != LEVEL_PUD)))
+			return -EINVAL;
+		if (crste_hole(entry) && !ign_holes)
+			return entry.tok.type == _DAT_TOKEN_PIC ? entry.tok.par : -EFAULT;
+		if (walk_level == LEVEL_PUD && continue_anyway && !entry.pud.h.fc && !entry.h.i) {
+			walk_level = LEVEL_PTE;
+			allocate = false;
+		}
+		if (walk_level == LEVEL_PUD || ((leaf || any) && entry.pud.h.fc))
+			return 0;
+		if (entry.pud.h.i && !entry.pud.h.fc) {
+			if (!allocate)
+				return any ? 0 : -ENOENT;
+			rc = dat_split_crste(*last, gfn, asce);
+			if (rc)
+				return rc;
+			entry = READ_ONCE(**last);
+		}
+		if (walk_level <= LEVEL_PMD && entry.pud.h.fc) {
+			if (!split)
+				return -EFBIG;
+			rc = dat_split_crste(*last, gfn, asce);
+			if (rc)
+				return rc;
+			entry = READ_ONCE(**last);
+		}
+		table = dereference_crste(entry.pud);
+	}
+
+	*last = table->crstes + pmd_index(gfn_to_gpa(gfn));
+	entry = READ_ONCE(**last);
+	if (WARN_ON_ONCE(unlikely(entry.h.tt != LEVEL_PMD)))
+		return -EINVAL;
+	if (crste_hole(entry) && !ign_holes)
+		return entry.tok.type == _DAT_TOKEN_PIC ? entry.tok.par : -EFAULT;
+	if (continue_anyway && !entry.pmd.h.fc && !entry.h.i) {
+		walk_level = LEVEL_PTE;
+		allocate = false;
+	}
+	if (walk_level == LEVEL_PMD || ((leaf || any) && entry.pmd.h.fc))
+		return 0;
+
+	if (entry.pmd.h.i && !entry.pmd.h.fc) {
+		if (!allocate)
+			return any ? 0 : -ENOENT;
+		rc = dat_split_crste(*last, gfn, asce);
+		if (rc)
+			return rc;
+		entry = READ_ONCE(**last);
+	}
+	if (walk_level <= LEVEL_PTE && entry.pmd.h.fc) {
+		if (!split)
+			return -EFBIG;
+		rc = dat_split_crste(*last, gfn, asce);
+		if (rc)
+			return rc;
+		entry = READ_ONCE(**last);
+	}
+	pgtable = dereference_pmd(entry.pmd);
+	*ptepp = pgtable->ptes + pte_index(gfn_to_gpa(gfn));
+	if (pte_hole(**ptepp) && !ign_holes)
+		return (*ptepp)->tok.type == _DAT_TOKEN_PIC ? (*ptepp)->tok.par : -EFAULT;
+	return 0;
+}
+
+static long dat_pte_walk_range(gfn_t gfn, gfn_t end, struct page_table *table, struct dat_walk *w)
+{
+	unsigned int idx = gfn & (_PAGE_ENTRIES - 1);
+	long rc = 0;
+
+	for ( ; gfn < end; idx++, gfn++) {
+		if (pte_hole(READ_ONCE(table->ptes[idx]))) {
+			if (!(w->flags & DAT_WALK_IGN_HOLES))
+				return -EFAULT;
+			if (!(w->flags & DAT_WALK_ANY))
+				continue;
+		}
+
+		rc = w->ops->pte_entry(table->ptes + idx, gfn, gfn + 1, w);
+		if (rc)
+			break;
+	}
+	return rc;
+}
+
+static long dat_crste_walk_range(gfn_t start, gfn_t end, struct crst_table *table,
+				 struct dat_walk *walk)
+{
+	unsigned long idx, cur_shift, cur_size;
+	dat_walk_op the_op;
+	union crste crste;
+	gfn_t cur, next;
+	long rc = 0;
+
+	cur_shift = 8 + table->crstes[0].h.tt * 11;
+	idx = (start >> cur_shift) & (_CRST_ENTRIES - 1);
+	cur_size = 1UL << cur_shift;
+
+	for (cur = ALIGN_DOWN(start, cur_size); cur < end; idx++, cur = next) {
+		next = cur + cur_size;
+		walk->last = table->crstes + idx;
+		crste = READ_ONCE(*walk->last);
+
+		if (crste_hole(crste)) {
+			if (!(walk->flags & DAT_WALK_IGN_HOLES))
+				return -EFAULT;
+			if (!(walk->flags & DAT_WALK_ANY))
+				continue;
+		}
+
+		the_op = walk->ops->crste_ops[crste.h.tt];
+		if (the_op) {
+			rc = the_op(walk->last, cur, next, walk);
+			crste = READ_ONCE(*walk->last);
+		}
+		if (rc)
+			break;
+		if (!crste.h.i && !crste.h.fc) {
+			if (!is_pmd(crste))
+				rc = dat_crste_walk_range(max(start, cur), min(end, next),
+							  _dereference_crste(crste), walk);
+			else if (walk->ops->pte_entry)
+				rc = dat_pte_walk_range(max(start, cur), min(end, next),
+							dereference_pmd(crste.pmd), walk);
+		}
+	}
+	return rc;
+}
+
+/**
+ * _dat_walk_gfn_range() - walk DAT tables
+ * @start: the first guest page frame to walk
+ * @end: the guest page frame immediately after the last one to walk
+ * @asce: the ASCE of the guest mapping
+ * @ops: the gmap_walk_ops that will be used to perform the walk
+ * @flags: flags from WALK_* (currently only WALK_IGN_HOLES is supported)
+ * @priv: will be passed as-is to the callbacks
+ *
+ * Any callback returning non-zero causes the walk to stop immediately.
+ *
+ * Return: -EINVAL in case of error, -EFAULT if @start is too high for the given
+ *         asce unless the DAT_WALK_IGN_HOLES flag is specified, otherwise it
+ *         returns whatever the callbacks return.
+ */
+long _dat_walk_gfn_range(gfn_t start, gfn_t end, union asce asce,
+			 const struct dat_walk_ops *ops, int flags, void *priv)
+{
+	struct crst_table *table = dereference_asce(asce);
+	struct dat_walk walk = {
+		.ops	= ops,
+		.asce	= asce,
+		.priv	= priv,
+		.flags	= flags,
+		.start	= start,
+		.end	= end,
+	};
+
+	if (WARN_ON_ONCE(unlikely(!asce.val)))
+		return -EINVAL;
+	if (!asce_contains_gfn(asce, start))
+		return (flags & DAT_WALK_IGN_HOLES) ? 0 : -EFAULT;
+
+	return dat_crste_walk_range(start, min(end, asce_end(asce)), table, &walk);
+}
arch/s390/kvm/dat.h +38 -0
index 9e23f6cdbf73..de4bd2298945 100644
@@ -346,6 +346,34 @@ struct page_table {
 static_assert(sizeof(struct crst_table) == _CRST_TABLE_SIZE);
 static_assert(sizeof(struct page_table) == PAGE_SIZE);
 
+struct dat_walk;
+
+typedef long (*dat_walk_op)(union crste *crste, gfn_t gfn, gfn_t next, struct dat_walk *w);
+
+struct dat_walk_ops {
+	union {
+		dat_walk_op crste_ops[4];
+		struct {
+			dat_walk_op pmd_entry;
+			dat_walk_op pud_entry;
+			dat_walk_op p4d_entry;
+			dat_walk_op pgd_entry;
+		};
+	};
+	long (*pte_entry)(union pte *pte, gfn_t gfn, gfn_t next, struct dat_walk *w);
+};
+
+struct dat_walk {
+	const struct dat_walk_ops *ops;
+	union crste *last;
+	union pte *last_pte;
+	union asce asce;
+	gfn_t start;
+	gfn_t end;
+	int flags;
+	void *priv;
+};
+
 static inline union pte _pte(kvm_pfn_t pfn, bool w, bool d, bool s)
 {
 	union pte res = { .val = PFN_PHYS(pfn) };
@@ -391,6 +419,11 @@ bool dat_crstep_xchg_atomic(union crste *crstep, union crste old, union crste ne
 			    union asce asce);
 void dat_crstep_xchg(union crste *crstep, union crste new, gfn_t gfn, union asce asce);
 
+long _dat_walk_gfn_range(gfn_t start, gfn_t end, union asce asce,
+			 const struct dat_walk_ops *ops, int flags, void *priv);
+
+int dat_entry_walk(gfn_t gfn, union asce asce, int flags, int walk_level,
+		   union crste **last, union pte **ptepp);
 void dat_free_level(struct crst_table *table, bool owns_ptes);
 struct page_table *dat_alloc_pt(unsigned long pte_bits, unsigned long pgste_bits);
 struct crst_table *dat_alloc_crst(unsigned long init);
@@ -734,4 +767,9 @@ static inline void dat_crstep_clear(union crste *crstep, gfn_t gfn, union asce a
 	dat_crstep_xchg(crstep, newcrste, gfn, asce);
 }
 
+static inline int get_level(union crste *crstep, union pte *ptep)
+{
+	return ptep ? LEVEL_PTE : crstep->h.tt;
+}
+
 #endif /* __KVM_S390_DAT_H */
Signature · 1 line
2.51.0
maintainer
#50 lore raw

Re: [PATCH v2 10/20] KVM: s390: KVM page table management functions: walks

Quoted 41 lines · Claudio Imbrenda

Add page table management functions to be used for KVM guest (gmap) page tables.

This patch adds functions to walk to specific table entries, or to perform actions on a range of entries.

---
  arch/s390/kvm/dat.c | 351 ++++++++++++++++++++++++++++++++++++++++++++
  arch/s390/kvm/dat.h |  38 +++++
  2 files changed, 389 insertions(+)
arch/s390/kvm/dat.c +21 -0
index f26e3579bd77..fe93e1c07158 100644
@@ -209,3 +209,354 @@ union pgste __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new, g
  	WRITE_ONCE(*ptep, new);
  	return pgste;
  }
+
+/*
+ * dat_split_pmd is assumed to be called with mmap_lock held in read or write mode
+ */
+static int dat_split_pmd(union pmd *pmdp, gfn_t gfn, union asce asce)
+{
+	struct page_table *pt;
+	union pmd new, old;
+	union pte init;
+	int i;
+
+	old = READ_ONCE(*pmdp);
+
+	/* Already split, nothing to do */
+	if (!old.h.i && !old.h.fc)
+		return 0;
+
+	pt = dat_alloc_pt_noinit();
+	if (!pt)
+		return -ENOMEM;
+	new.val = virt_to_phys(pt);

arch/s390/kvm/dat.c union pgste __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new, g · line 242

Quoted 10 lines
arch/s390/kvm/dat.c +10 -0
+
+	while (old.h.i || old.h.fc) {
+		init.val = pmd_origin_large(old);
+		init.h.p = old.h.p;
+		init.h.i = old.h.i;
+		init.s.d = old.s.fc1.d;
+		init.s.w = old.s.fc1.w;
+		init.s.y = old.s.fc1.y;
+		init.s.sd = old.s.fc1.sd;
+		init.s.pr = old.s.fc1.pr;

This looks horrible but I haven't found a better solution.

arch/s390/kvm/dat.c union pgste __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new, g · line 247

Quoted 5 lines
arch/s390/kvm/dat.c +5 -0
+		if (old.h.fc) {
+			for (i = 0; i < _PAGE_ENTRIES; i++)
+				pt->ptes[i].val = init.val | i * PAGE_SIZE;
+			/* no need to take locks as the page table is not installed yet */
+			dat_init_pgstes(pt, old.s.fc1.prefix_notif ? PGSTE_IN_BIT : 0);

So, if the pmd had the IN bit, all PGSTEs will have it as well, right?
Why can't we notify and not copy this bit, so the notifier sets it for the ptes which actually need it? Or does it happen later?

Quoted 12 lines
arch/s390/kvm/dat.c +12 -0
+		} else {
+			dat_init_page_table(pt, init.val, 0);
+		}
+
+		if (dat_pmdp_xchg_atomic(pmdp, old, new, gfn, asce))
+			return 0;
+		old = READ_ONCE(*pmdp);
+	}
+
+	dat_free_pt(pt);
+	return 0;
+}
author maintainer
#51 lore raw

Re: [PATCH v2 10/20] KVM: s390: KVM page table management functions: walks

Quoted 46 lines · On Thu, 11 Sep 2025 14:56:59 +0200 Janosch Frank <[email protected]> wrote:
Quoted 45 lines · Claudio Imbrenda

Add page table management functions to be used for KVM guest (gmap) page tables.

This patch adds functions to walk to specific table entries, or to perform actions on a range of entries.

---
  arch/s390/kvm/dat.c | 351 ++++++++++++++++++++++++++++++++++++++++++++
  arch/s390/kvm/dat.h |  38 +++++
  2 files changed, 389 insertions(+)
arch/s390/kvm/dat.c +25 -0
index f26e3579bd77..fe93e1c07158 100644
@@ -209,3 +209,354 @@ union pgste __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new, g
  	WRITE_ONCE(*ptep, new);
  	return pgste;
  }
+
+/*
+ * dat_split_pmd is assumed to be called with mmap_lock held in read or write mode
+ */
+static int dat_split_pmd(union pmd *pmdp, gfn_t gfn, union asce asce)
+{
+	struct page_table *pt;
+	union pmd new, old;
+	union pte init;
+	int i;
+
+	old = READ_ONCE(*pmdp);
+
+	/* Already split, nothing to do */
+	if (!old.h.i && !old.h.fc)
+		return 0;
+
+	pt = dat_alloc_pt_noinit();
+	if (!pt)
+		return -ENOMEM;
+	new.val = virt_to_phys(pt);
+
+	while (old.h.i || old.h.fc) {
+		init.val = pmd_origin_large(old);
+		init.h.p = old.h.p;
Quoted 8 lines

arch/s390/kvm/dat.c union pgste __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new, g · line 242

Quoted 6 lines
arch/s390/kvm/dat.c +6 -0
+		init.h.i = old.h.i;
+		init.s.d = old.s.fc1.d;
+		init.s.w = old.s.fc1.w;
+		init.s.y = old.s.fc1.y;
+		init.s.sd = old.s.fc1.sd;
+		init.s.pr = old.s.fc1.pr;  

This looks horrible but I haven't found a better solution.

I know what you mean :)

Quoted 8 lines

arch/s390/kvm/dat.c union pgste __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new, g · line 247

Quoted 6 lines
arch/s390/kvm/dat.c +5 -0
+		if (old.h.fc) {
+			for (i = 0; i < _PAGE_ENTRIES; i++)
+				pt->ptes[i].val = init.val | i * PAGE_SIZE;
+			/* no need to take locks as the page table is not installed yet */
+			dat_init_pgstes(pt, old.s.fc1.prefix_notif ? PGSTE_IN_BIT : 0);  

So, if the pmd had the IN bit, all PGSTEs will have it as well, right?

yes

Quoted 2 lines

Why can't we notify and not copy this bit, so the notifier sets it for the ptes which actually need it? Or does it happen later?

because the notifier is a gmap/kvm thing, and this function (the whole dat.c actually) only deals with page tables.

Keeping things this way makes the code more structured and cleaner and less convoluted.

The unnecessary prefix_notif bits will be cleared once they get invalidated. This will happen very rarely, since most of the time a pmd or pud prefix will not need to be split. The only case I can think of right now is a weird interaction between huge pages and vsie.

In short: it only needs to be correct, does not need to be fast :)

Quoted 13 lines
Quoted 13 lines
arch/s390/kvm/dat.c +12 -0
+		} else {
+			dat_init_page_table(pt, init.val, 0);
+		}
+
+		if (dat_pmdp_xchg_atomic(pmdp, old, new, gfn, asce))
+			return 0;
+		old = READ_ONCE(*pmdp);
+	}
+
+	dat_free_pt(pt);
+	return 0;
+}  
#52 lore raw

Re: [PATCH v2 10/20] KVM: s390: KVM page table management functions: walks

Quoted 23 lines · Claudio Imbrenda
Quoted 20 lines · On Thu, 11 Sep 2025 14:56:59 +0200 Janosch Frank <[email protected]> wrote:
Quoted 19 lines · Claudio Imbrenda

Add page table management functions to be used for KVM guest (gmap) page tables.

This patch adds functions to walk to specific table entries, or to perform actions on a range of entries.

---
  arch/s390/kvm/dat.c | 351 ++++++++++++++++++++++++++++++++++++++++++++
  arch/s390/kvm/dat.h |  38 +++++
  2 files changed, 389 insertions(+)
arch/s390/kvm/dat.c +0 -0
index f26e3579bd77..fe93e1c07158 100644
@@ -209,3 +209,354 @@ union pgste __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new, g
  	WRITE_ONCE(*ptep, new);
  	return pgste;

[... snip ...]

Quoted 13 lines
Quoted 4 lines
Quoted 4 lines
arch/s390/kvm/dat.c +4 -0
+
+	while (old.h.i || old.h.fc) {
+		init.val = pmd_origin_large(old);
+		init.h.p = old.h.p;
Quoted 8 lines

arch/s390/kvm/dat.c union pgste __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new, g · line 242

Quoted 6 lines
arch/s390/kvm/dat.c +6 -0
+		init.h.i = old.h.i;
+		init.s.d = old.s.fc1.d;
+		init.s.w = old.s.fc1.w;
+		init.s.y = old.s.fc1.y;
+		init.s.sd = old.s.fc1.sd;
+		init.s.pr = old.s.fc1.pr;  

This looks horrible but I haven't found a better solution.

Quoted 1 line

I know what you mean :)

If nothing else, you may try and encapsulate this in a helper to concentrate the ugliness there. Then the while-body just calls some prepare_init() or similar?

maintainer
#53 lore raw

Re: [PATCH v2 10/20] KVM: s390: KVM page table management functions: walks

In reply to Claudio Imbrenda
Quoted 11 lines · Claudio Imbrenda

Add page table management functions to be used for KVM guest (gmap) page tables.

This patch adds functions to walk to specific table entries, or to perform actions on a range of entries.

---
 arch/s390/kvm/dat.c | 351 ++++++++++++++++++++++++++++++++++++++++++++
 arch/s390/kvm/dat.h |  38 +++++
 2 files changed, 389 insertions(+)

...

Quoted 3 lines
arch/s390/kvm/dat.c +3 -0
+int dat_entry_walk(gfn_t gfn, union asce asce, int flags, int walk_level,
+		   union crste **last, union pte **ptepp)
+{

...

arch/s390/kvm/dat.c union pgste __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new, g · line 360

Quoted 6 lines
arch/s390/kvm/dat.c +6 -0
+	table = dereference_asce(asce);
+	if (asce.dt >= ASCE_TYPE_REGION1) {
+		*last = table->crstes + pgd_index(gfn_to_gpa(gfn));
+		entry = READ_ONCE(**last);
+		if (WARN_ON_ONCE(unlikely(entry.h.tt != LEVEL_PGD)))
+			return -EINVAL;

Since I've seen this all over the place: this looks just wrong to me.

This is mixing some random software definition "LEVEL_PGD" with hardware bits. A "correct" table type compare would compare with TABLE_TYPE_REGION1 instead. Also using pgd_index() & friends here is semantically wrong.

For normal processes the pgd is always the top page table level (present and used), and lower levels like p4d may be folded. This is not the case with this code, where pgd, p4d, etc. seem to have a completely different meaning and are mapped to specific hardware page table levels / types, where the top levels (pgd, ...) are optional.

Why is this mixed? To me this looks like the potential source for a _lot_ of confusion and potential bugs.

author maintainer
#54 lore raw

Re: [PATCH v2 10/20] KVM: s390: KVM page table management functions: walks

Quoted 30 lines · On Tue, 16 Sep 2025 18:22:03 +0200 Heiko Carstens <[email protected]> wrote:
Quoted 11 lines · Claudio Imbrenda

Add page table management functions to be used for KVM guest (gmap) page tables.

This patch adds functions to walk to specific table entries, or to perform actions on a range of entries.

---
 arch/s390/kvm/dat.c | 351 ++++++++++++++++++++++++++++++++++++++++++++
 arch/s390/kvm/dat.h |  38 +++++
 2 files changed, 389 insertions(+)  

...

Quoted 3 lines
arch/s390/kvm/dat.c +3 -0
+int dat_entry_walk(gfn_t gfn, union asce asce, int flags, int walk_level,
+		   union crste **last, union pte **ptepp)
+{  

...

arch/s390/kvm/dat.c union pgste __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new, g · line 360

Quoted 6 lines
arch/s390/kvm/dat.c +6 -0
+	table = dereference_asce(asce);
+	if (asce.dt >= ASCE_TYPE_REGION1) {
+		*last = table->crstes + pgd_index(gfn_to_gpa(gfn));
+		entry = READ_ONCE(**last);
+		if (WARN_ON_ONCE(unlikely(entry.h.tt != LEVEL_PGD)))
+			return -EINVAL;  

Since I've seen this all over the place: this looks just wrong to me.

Quoted 2 lines

This is mixing some random software definition "LEVEL_PGD" with hardware bits. A "correct" table type compare would compare with TABLE_TYPE_REGION1

they are defined to be the same for this reason, but I understand what you mean, and I can use the TABLE_TYPE_* macros instead

Quoted 1 line

instead. Also using pgd_index() & friends here is semantically wrong.

how so? in the end all the various p?d_index() macros are just a shift-and-mask. I think it's more readable than open coding ((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1)) each time?

Quoted 6 lines

For normal processes the pgd is always the top page table level (present and used), and lower levels like p4d may be folded. This is not the case with this code, where pgd, p4d, etc. seem to have a completely different meaning and are mapped to specific hardware page table levels / types, where the top levels (pgd, ...) are optional.

yes, and we do not use pte_t or any of the p?d_t types in KVM for that reason

Quoted 3 lines

Why is this mixed? To me this looks like the potential source for a _lot_ of confusion and potential bugs.

maintainer
#55 lore raw

Re: [PATCH v2 10/20] KVM: s390: KVM page table management functions: walks

Quoted 11 lines · Claudio Imbrenda
Quoted 9 lines · On Tue, 16 Sep 2025 18:22:03 +0200 Heiko Carstens <[email protected]> wrote:

arch/s390/kvm/dat.c union pgste __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new, g · line 360

Quoted 6 lines
arch/s390/kvm/dat.c +6 -0
+	table = dereference_asce(asce);
+	if (asce.dt >= ASCE_TYPE_REGION1) {
+		*last = table->crstes + pgd_index(gfn_to_gpa(gfn));
+		entry = READ_ONCE(**last);
+		if (WARN_ON_ONCE(unlikely(entry.h.tt != LEVEL_PGD)))
+			return -EINVAL;  

Since I've seen this all over the place: this looks just wrong to me.

Quoted 5 lines
Quoted 2 lines

This is mixing some random software definition "LEVEL_PGD" with hardware bits. A "correct" table type compare would compare with TABLE_TYPE_REGION1

they are defined to be the same for this reason, but I understand what you mean, and I can use the TABLE_TYPE_* macros instead

Yes, please.

Quoted 6 lines
Quoted 2 lines

instead. Also using pgd_index() & friends here is semantically wrong.

how so? in the end all the various p?d_index() macros are just a shift-and-mask. I think it's more readable than open coding ((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1)) each time?

I'm not saying that this code is incorrect. What I'm complaining about is that this is mixing hardware and software definitions all over the place, which at least to me is very confusing. But ok, maybe I'm the only one.

So e.g. for the above, which operates with a region 1 table entry, I would expect to see similar code like in gaccess.c. E.g. _something_ like (and most likely incorrect):

	table = dereference_asce(asce);
	if (asce.dt >= ASCE_TYPE_REGION1) {
		union vaddress vaddr = {.addr = gfn_to_gpa(gfn)};

		*last = table->crstes + vaddr.rfx;
		entry = READ_ONCE(**last);
		if (WARN_ON_ONCE(entry.h.tt != TABLE_TYPE_REGION1))
			return -EINVAL;  

Or in other words, this doesn't mix hw vs sw functions. Anyway, just my 0.02 cents. If you disagree feel free to keep it.
You have to maintain this :)

Btw.: WARN_ON_ONCE() has a builtin unlikely(), please don't add unlikely again like above:

		if (WARN_ON_ONCE(unlikely(entry.h.tt != LEVEL_PGD)))
author maintainer
#56 lore raw

Re: [PATCH v2 10/20] KVM: s390: KVM page table management functions: walks

Quoted 32 lines · On Tue, 16 Sep 2025 19:24:33 +0200 Heiko Carstens <[email protected]> wrote:
Quoted 11 lines · Claudio Imbrenda
Quoted 9 lines · On Tue, 16 Sep 2025 18:22:03 +0200 Heiko Carstens <[email protected]> wrote:

arch/s390/kvm/dat.c union pgste __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new, g · line 360

Quoted 6 lines
arch/s390/kvm/dat.c +6 -0
+	table = dereference_asce(asce);
+	if (asce.dt >= ASCE_TYPE_REGION1) {
+		*last = table->crstes + pgd_index(gfn_to_gpa(gfn));
+		entry = READ_ONCE(**last);
+		if (WARN_ON_ONCE(unlikely(entry.h.tt != LEVEL_PGD)))
+			return -EINVAL;    

Since I've seen this all over the place: this looks just wrong to me.

Quoted 5 lines
Quoted 2 lines

This is mixing some random software definition "LEVEL_PGD" with hardware bits. A "correct" table type compare would compare with TABLE_TYPE_REGION1

they are defined to be the same for this reason, but I understand what you mean, and I can use the TABLE_TYPE_* macros instead

Yes, please.

Quoted 6 lines
Quoted 2 lines

instead. Also using pgd_index() & friends here is semantically wrong.

how so? in the end all the various p?d_index() macros are just a shift-and-mask. I think it's more readable than open coding ((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1)) each time?

I'm not saying that this code is incorrect. What I'm complaining about is that this is mixing hardware and software definitions all over the place, which at least to me is very confusing. But ok, maybe I'm the only one.

Quoted 7 lines

So e.g. for the above, which operates with a region 1 table entry, I would expect to see similar code like in gaccess.c. E.g. _something_ like (and most likely incorrect):

	table = dereference_asce(asce);
	if (asce.dt >= ASCE_TYPE_REGION1) {
		union vaddress vaddr = {.addr = gfn_to_gpa(gfn)};

hmm, that doesn't look ugly

maybe I can even add one more anonymous struct in vaddr, so that the gfn can be assigned directly: union vaddress vaddr = { .gfn = gfn };

Quoted 6 lines
		*last = table->crstes + vaddr.rfx;
		entry = READ_ONCE(**last);
		if (WARN_ON_ONCE(entry.h.tt != TABLE_TYPE_REGION1))
			return -EINVAL;  
Quoted 8 lines

Or in other words, this doesn't mix hw vs sw functions. Anyway, just my 0.02 cents. If you disagree feel free to keep it.
You have to maintain this :)

Btw.: WARN_ON_ONCE() has a builtin unlikely(), please don't add unlikely again like above:

		if (WARN_ON_ONCE(unlikely(entry.h.tt != LEVEL_PGD)))

oh, right, I'll fix that

maintainer
#57 lore raw

Re: [PATCH v2 10/20] KVM: s390: KVM page table management functions: walks

In reply to Claudio Imbrenda
Quoted 11 lines · Claudio Imbrenda

Add page table management functions to be used for KVM guest (gmap) page tables.

This patch adds functions to walk to specific table entries, or to perform actions on a range of entries.

---
 arch/s390/kvm/dat.c | 351 ++++++++++++++++++++++++++++++++++++++++++++
 arch/s390/kvm/dat.h |  38 +++++
 2 files changed, 389 insertions(+)

...

Quoted 37 lines
arch/s390/kvm/dat.c +37 -0
+/*
+ * dat_split_pmd is assumed to be called with mmap_lock held in read or write mode
+ */
+static int dat_split_pmd(union pmd *pmdp, gfn_t gfn, union asce asce)
+{
+	struct page_table *pt;
+	union pmd new, old;
+	union pte init;
+	int i;
+
+	old = READ_ONCE(*pmdp);
+
+	/* Already split, nothing to do */
+	if (!old.h.i && !old.h.fc)
+		return 0;
+
+	pt = dat_alloc_pt_noinit();
+	if (!pt)
+		return -ENOMEM;
+	new.val = virt_to_phys(pt);
+
+	while (old.h.i || old.h.fc) {
+		init.val = pmd_origin_large(old);
+		init.h.p = old.h.p;
+		init.h.i = old.h.i;
+		init.s.d = old.s.fc1.d;
+		init.s.w = old.s.fc1.w;
+		init.s.y = old.s.fc1.y;
+		init.s.sd = old.s.fc1.sd;
+		init.s.pr = old.s.fc1.pr;
+		if (old.h.fc) {
+			for (i = 0; i < _PAGE_ENTRIES; i++)
+				pt->ptes[i].val = init.val | i * PAGE_SIZE;
+			/* no need to take locks as the page table is not installed yet */
+			dat_init_pgstes(pt, old.s.fc1.prefix_notif ? PGSTE_IN_BIT : 0);
+		} else {
+			dat_init_page_table(pt, init.val, 0);

arch/s390/kvm/dat.c union pgste __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new, g · line 259

Quoted 10 lines
arch/s390/kvm/dat.c +10 -0
+		}
+
+		if (dat_pmdp_xchg_atomic(pmdp, old, new, gfn, asce))
+			return 0;
+		old = READ_ONCE(*pmdp);
+	}
+
+	dat_free_pt(pt);
+	return 0;
+}

Just to annoy you again: is there a particular reason why this function isn't called "dat_split_segment()" instead of "_pmd()"? I'm still trying to convince you to get rid of pmd(p) usage, whenever possible. IMHO this would make the code easier to understand and maintain as it doesn't mix things, as I already stated previously.

arch/s390/kvm/dat.c union pgste __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new, g · line 383

Quoted 8 lines
arch/s390/kvm/dat.c +8 -0
+	if (asce.dt >= ASCE_TYPE_REGION2) {
+		*last = table->crstes + p4d_index(gfn_to_gpa(gfn));
+		entry = READ_ONCE(**last);
+		if (WARN_ON_ONCE(unlikely(entry.h.tt != LEVEL_P4D)))
+			return -EINVAL;
+		if (crste_hole(entry) && !ign_holes)
+			return entry.tok.type == _DAT_TOKEN_PIC ? entry.tok.par : -EFAULT;
+		if (walk_level == LEVEL_P4D)

Also this (even though also as stated earlier): IMHO it would be good to get rid of the LEVEL defines and use the corresponding TABLE_TYPEs instead.

arch/s390/kvm/dat.c union pgste __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new, g · line 435

Quoted 2 lines
arch/s390/kvm/dat.c +2 -0
+	if (continue_anyway && !entry.pmd.h.fc && !entry.h.i) {
+		walk_level = LEVEL_PTE;

Hm, ok, there is no TABLE_TYPE for this level. Invention required :)

author maintainer
#58 lore raw

Re: [PATCH v2 10/20] KVM: s390: KVM page table management functions: walks

Quoted 58 lines · On Wed, 17 Sep 2025 14:55:29 +0200 Heiko Carstens <[email protected]> wrote:
Quoted 11 lines · Claudio Imbrenda

Add page table management functions to be used for KVM guest (gmap) page tables.

This patch adds functions to walk to specific table entries, or to perform actions on a range of entries.

---
 arch/s390/kvm/dat.c | 351 ++++++++++++++++++++++++++++++++++++++++++++
 arch/s390/kvm/dat.h |  38 +++++
 2 files changed, 389 insertions(+)  

...

Quoted 43 lines
arch/s390/kvm/dat.c +43 -0
+/*
+ * dat_split_pmd is assumed to be called with mmap_lock held in read or write mode
+ */
+static int dat_split_pmd(union pmd *pmdp, gfn_t gfn, union asce asce)
+{
+	struct page_table *pt;
+	union pmd new, old;
+	union pte init;
+	int i;
+
+	old = READ_ONCE(*pmdp);
+
+	/* Already split, nothing to do */
+	if (!old.h.i && !old.h.fc)
+		return 0;
+
+	pt = dat_alloc_pt_noinit();
+	if (!pt)
+		return -ENOMEM;
+	new.val = virt_to_phys(pt);
+
+	while (old.h.i || old.h.fc) {
+		init.val = pmd_origin_large(old);
+		init.h.p = old.h.p;
+		init.h.i = old.h.i;
+		init.s.d = old.s.fc1.d;
+		init.s.w = old.s.fc1.w;
+		init.s.y = old.s.fc1.y;
+		init.s.sd = old.s.fc1.sd;
+		init.s.pr = old.s.fc1.pr;
+		if (old.h.fc) {
+			for (i = 0; i < _PAGE_ENTRIES; i++)
+				pt->ptes[i].val = init.val | i * PAGE_SIZE;
+			/* no need to take locks as the page table is not installed yet */
+			dat_init_pgstes(pt, old.s.fc1.prefix_notif ? PGSTE_IN_BIT : 0);
+		} else {
+			dat_init_page_table(pt, init.val, 0);
+		}
+
+		if (dat_pmdp_xchg_atomic(pmdp, old, new, gfn, asce))
+			return 0;
+		old = READ_ONCE(*pmdp);
+	}
Quoted 8 lines

arch/s390/kvm/dat.c union pgste __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new, g · line 259

Quoted 4 lines
arch/s390/kvm/dat.c +4 -0
+
+	dat_free_pt(pt);
+	return 0;
+}  

Just to annoy you again: is there a particular reason why this function isn't called "dat_split_segment()" instead of "_pmd()"? I'm still trying to convince you to get rid of pmd(p) usage, whenever possible. IMHO this would make the

hmm I can rename it

Quoted 15 lines

code easier to understand and maintain as it doesn't mix things, as I already stated previously.

arch/s390/kvm/dat.c union pgste __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new, g · line 383

Quoted 8 lines
arch/s390/kvm/dat.c +8 -0
+	if (asce.dt >= ASCE_TYPE_REGION2) {
+		*last = table->crstes + p4d_index(gfn_to_gpa(gfn));
+		entry = READ_ONCE(**last);
+		if (WARN_ON_ONCE(unlikely(entry.h.tt != LEVEL_P4D)))
+			return -EINVAL;
+		if (crste_hole(entry) && !ign_holes)
+			return entry.tok.type == _DAT_TOKEN_PIC ? entry.tok.par : -EFAULT;
+		if (walk_level == LEVEL_P4D)  

Also this (even though also as stated earlier): IMHO it would be good to get rid of the LEVEL defines and use the corresponding TABLE_TYPEs instead.

Quoted 4 lines

arch/s390/kvm/dat.c union pgste __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new, g · line 435

Quoted 2 lines
arch/s390/kvm/dat.c +2 -0
+	if (continue_anyway && !entry.pmd.h.fc && !entry.h.i) {
+		walk_level = LEVEL_PTE;  

Hm, ok, there is no TABLE_TYPE for this level. Invention required :)

yes, this is why I have the LEVEL_* macros :)

do you think I should instead #define TABLE_TYPE_PAGE_TABLE -1 ?

maintainer
#59 lore raw

Re: [PATCH v2 10/20] KVM: s390: KVM page table management functions: walks

Quoted 9 lines · Claudio Imbrenda
Quoted 4 lines · Heiko Carstens

arch/s390/kvm/dat.c union pgste __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new, g · line 435

Quoted 2 lines
arch/s390/kvm/dat.c +2 -0
+	if (continue_anyway && !entry.pmd.h.fc && !entry.h.i) {
+		walk_level = LEVEL_PTE;  

Hm, ok, there is no TABLE_TYPE for this level. Invention required :)

yes, this is why I have the LEVEL_* macros :)

do you think I should instead #define TABLE_TYPE_PAGE_TABLE -1 ?

Yes, I think we can go with this, plus a comment what this (impossible) TABLE_TYPE is good for.

author maintainer
#60 lore raw

Re: [PATCH v2 10/20] KVM: s390: KVM page table management functions: walks

Quoted 7 lines · On Wed, 17 Sep 2025 15:24:19 +0200 Heiko Carstens <[email protected]> wrote:
Quoted 6 lines · Claudio Imbrenda
Quoted 4 lines · Heiko Carstens

arch/s390/kvm/dat.c union pgste __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new, g · line 435

Quoted 2 lines
arch/s390/kvm/dat.c +2 -0
+	if (continue_anyway && !entry.pmd.h.fc && !entry.h.i) {
+		walk_level = LEVEL_PTE;    

Hm, ok, there is no TABLE_TYPE for this level. Invention required :)

Quoted 6 lines
Quoted 3 lines

yes, this is why I have the LEVEL_* macros :)

do you think I should instead #define TABLE_TYPE_PAGE_TABLE -1 ?

Yes, I think we can go with this, plus a comment what this (impossible) TABLE_TYPE is good for.

ok

author maintainer
#61 lore raw

PATCH v2 11/20 KVM: s390: KVM page table management functions: storage keys

In reply to Claudio Imbrenda

Add page table management functions to be used for KVM guest (gmap) page tables.

This patch adds functions related to storage key handling.

---
 arch/s390/kvm/dat.c | 212 ++++++++++++++++++++++++++++++++++++++++++++
 arch/s390/kvm/dat.h |   6 ++
 2 files changed, 218 insertions(+)
arch/s390/kvm/dat.c +212 -0
index fe93e1c07158..f626e8c37770 100644
@@ -560,3 +560,215 @@ long _dat_walk_gfn_range(gfn_t start, gfn_t end, union asce asce,
 
 	return dat_crste_walk_range(start, min(end, asce_end(asce)), table, &walk);
 }
+
+int dat_get_storage_key(union asce asce, gfn_t gfn, union skey *skey)
+{
+	union crste *crstep;
+	union pgste pgste;
+	union pte *ptep;
+	int rc;
+
+	skey->skey = 0;
+	rc = dat_entry_walk(gfn, asce, DAT_WALK_ANY, LEVEL_PTE, &crstep, &ptep);
+	if (rc)
+		return rc;
+
+	if (!ptep) {
+		union crste crste;
+
+		crste = READ_ONCE(*crstep);
+		if (!crste.h.fc || !crste.s.fc1.pr)
+			return 0;
+		skey->skey = page_get_storage_key(large_crste_to_phys(crste, gfn));
+		return 0;
+	}
+	pgste = pgste_get_lock(ptep);
+	if (ptep->h.i) {
+		skey->acc = pgste.acc;
+		skey->fp = pgste.fp;
+	} else {
+		skey->skey = page_get_storage_key(pte_origin(*ptep));
+	}
+	skey->r |= pgste.gr;
+	skey->c |= pgste.gc;
+	pgste_set_unlock(ptep, pgste);
+	return 0;
+}
+
+static void dat_update_ptep_sd(union pgste old, union pgste pgste, union pte *ptep)
+{
+	if (pgste.acc != old.acc || pgste.fp != old.fp || pgste.gr != old.gr || pgste.gc != old.gc)
+		__atomic64_or(_PAGE_SD, &ptep->val);
+}
+
+int dat_set_storage_key(union asce asce, gfn_t gfn, union skey skey, bool nq)
+{
+	union pgste pgste, old;
+	union crste *crstep;
+	union pte *ptep;
+	int rc;
+
+	rc = dat_entry_walk(gfn, asce, DAT_WALK_LEAF_ALLOC, LEVEL_PTE, &crstep, &ptep);
+	if (rc)
+		return rc;
+
+	if (!ptep) {
+		page_set_storage_key(large_crste_to_phys(*crstep, gfn), skey.skey, !nq);
+		return 0;
+	}
+
+	old = pgste_get_lock(ptep);
+	pgste = old;
+
+	pgste.acc = skey.acc;
+	pgste.fp = skey.fp;
+	pgste.gc = skey.c;
+	pgste.gr = skey.r;
+
+	if (!ptep->h.i) {
+		union skey old_skey;
+
+		old_skey.skey = page_get_storage_key(pte_origin(*ptep));
+		pgste.hc |= old_skey.c;
+		pgste.hr |= old_skey.r;
+		skey.r = 0;
+		skey.c = 0;
+		page_set_storage_key(pte_origin(*ptep), skey.skey, !nq);
+	}
+
+	dat_update_ptep_sd(old, pgste, ptep);
+	pgste_set_unlock(ptep, pgste);
+	return 0;
+}
+
+static bool page_cond_set_storage_key(phys_addr_t paddr, union skey skey, union skey *oldkey,
+				      bool nq, bool mr, bool mc)
+{
+	oldkey->skey = page_get_storage_key(paddr);
+	if (oldkey->acc == skey.acc && oldkey->fp == skey.fp &&
+	    (oldkey->r == skey.r || mr) && (oldkey->c == skey.c || mc))
+		return false;
+	page_set_storage_key(paddr, skey.skey, !nq);
+	return true;
+}
+
+int dat_cond_set_storage_key(union asce asce, gfn_t gfn, union skey skey, union skey *oldkey,
+			     bool nq, bool mr, bool mc)
+{
+	union pgste pgste, old;
+	union crste *crstep;
+	union pte *ptep;
+	int rc;
+
+	rc = dat_entry_walk(gfn, asce, DAT_WALK_LEAF_ALLOC, LEVEL_PTE, &crstep, &ptep);
+	if (rc)
+		return rc;
+
+	if (!ptep)
+		return page_cond_set_storage_key(large_crste_to_phys(*crstep, gfn), skey, oldkey,
+						 nq, mr, mc);
+
+	old = pgste_get_lock(ptep);
+	pgste = old;
+
+	rc = 1;
+	pgste.acc = skey.acc;
+	pgste.fp = skey.fp;
+	pgste.gc = skey.c;
+	pgste.gr = skey.r;
+
+	if (!ptep->h.i) {
+		union skey prev;
+
+		rc = page_cond_set_storage_key(pte_origin(*ptep), skey, &prev, nq, mr, mc);
+		pgste.hc |= prev.c;
+		pgste.hr |= prev.r;
+		if (oldkey)
+			*oldkey = prev;
+	}
+
+	dat_update_ptep_sd(old, pgste, ptep);
+	pgste_set_unlock(ptep, pgste);
+	return rc;
+}
+
+int dat_reset_reference_bit(union asce asce, gfn_t gfn)
+{
+	union pgste pgste, old;
+	union crste *crstep;
+	union pte *ptep;
+	int rc;
+
+	rc = dat_entry_walk(gfn, asce, DAT_WALK_ANY, LEVEL_PTE, &crstep, &ptep);
+	if (rc)
+		return rc;
+
+	if (!ptep) {
+		union crste crste = READ_ONCE(*crstep);
+
+		if (!crste.h.fc || !crste.s.fc1.pr)
+			return 0;
+		return page_reset_referenced(large_crste_to_phys(*crstep, gfn));
+	}
+	old = pgste_get_lock(ptep);
+	pgste = old;
+
+	if (!ptep->h.i) {
+		rc = page_reset_referenced(pte_origin(*ptep));
+		pgste.hr = rc >> 1;
+	}
+	rc |= (pgste.gr << 1) | pgste.gc;
+	pgste.gr = 0;
+
+	dat_update_ptep_sd(old, pgste, ptep);
+	pgste_set_unlock(ptep, pgste);
+	return rc;
+}
+
+static long dat_reset_skeys_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk)
+{
+	union pgste pgste;
+
+	pgste = pgste_get_lock(ptep);
+	pgste.acc = 0;
+	pgste.fp = 0;
+	pgste.gr = 0;
+	pgste.gc = 0;
+	if (ptep->s.pr)
+		page_set_storage_key(pte_origin(*ptep), PAGE_DEFAULT_KEY, 1);
+	pgste_set_unlock(ptep, pgste);
+
+	if (need_resched())
+		return next;
+	return 0;
+}
+
+static long dat_reset_skeys_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk)
+{
+	phys_addr_t addr, end, origin = crste_origin_large(*crstep);
+
+	if (!crstep->h.fc || !crstep->s.fc1.pr)
+		return 0;
+
+	addr = ((max(gfn, walk->start) - gfn) << PAGE_SHIFT) + origin;
+	end = ((min(next, walk->end) - gfn) << PAGE_SHIFT) + origin;
+	while (ALIGN(addr + 1, _SEGMENT_SIZE) <= end)
+		addr = sske_frame(addr, PAGE_DEFAULT_KEY);
+	for ( ; addr < end; addr += PAGE_SIZE)
+		page_set_storage_key(addr, PAGE_DEFAULT_KEY, 1);
+
+	if (need_resched())
+		return next;
+	return 0;
+}
+
+long dat_reset_skeys(union asce asce, gfn_t start)
+{
+	const struct dat_walk_ops ops = {
+		.pte_entry = dat_reset_skeys_pte,
+		.pmd_entry = dat_reset_skeys_crste,
+		.pud_entry = dat_reset_skeys_crste,
+	};
+
+	return _dat_walk_gfn_range(start, asce_end(asce), asce, &ops, DAT_WALK_IGN_HOLES, NULL);
+}
arch/s390/kvm/dat.h +6 -0
index de4bd2298945..40f5c1371ef3 100644
@@ -427,6 +427,12 @@ int dat_entry_walk(gfn_t gfn, union asce asce, int flags, int walk_level,
 void dat_free_level(struct crst_table *table, bool owns_ptes);
 struct page_table *dat_alloc_pt(unsigned long pte_bits, unsigned long pgste_bits);
 struct crst_table *dat_alloc_crst(unsigned long init);
+int dat_get_storage_key(union asce asce, gfn_t gfn, union skey *skey);
+int dat_set_storage_key(union asce asce, gfn_t gfn, union skey skey, bool nq);
+int dat_cond_set_storage_key(union asce asce, gfn_t gfn, union skey skey, union skey *oldkey,
+			     bool nq, bool mr, bool mc);
+int dat_reset_reference_bit(union asce asce, gfn_t gfn);
+long dat_reset_skeys(union asce asce, gfn_t start);
 
 static inline struct crst_table *crste_table_start(union crste *crstep)
 {
Signature · 1 line
2.51.0
author maintainer
#62 lore raw

PATCH v2 12/20 KVM: s390: KVM page table management functions: lifecycle management

In reply to Claudio Imbrenda

Add page table management functions to be used for KVM guest (gmap) page tables.

This patch adds functions to handle memslot creation and destruction, additional per-pagetable data stored in the PGSTEs, mapping physical addresses into the gmap, and marking address ranges as prefix.

---
 arch/s390/kvm/dat.c | 227 ++++++++++++++++++++++++++++++++++++++++++++
 arch/s390/kvm/dat.h |  35 +++++++
 2 files changed, 262 insertions(+)
arch/s390/kvm/dat.c +227 -0
index f626e8c37770..4249400a9d21 100644
@@ -772,3 +772,230 @@ long dat_reset_skeys(union asce asce, gfn_t start)
 
 	return _dat_walk_gfn_range(start, asce_end(asce), asce, &ops, DAT_WALK_IGN_HOLES, NULL);
 }
+
+static long _dat_slot_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk)
+{
+	union crste dummy = { .val = (unsigned long)walk->priv };
+	union pte new_pte, pte = READ_ONCE(*ptep);
+
+	new_pte = _PTE_TOK(dummy.tok.type, dummy.tok.par);
+
+	/* Table entry already in the desired state */
+	if (pte.val == new_pte.val)
+		return 0;
+
+	dat_ptep_xchg(ptep, new_pte, gfn, walk->asce, false);
+	return 0;
+}
+
+static long _dat_slot_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk)
+{
+	union crste new_crste, crste = READ_ONCE(*crstep);
+
+	new_crste.val = (unsigned long)walk->priv;
+	new_crste.h.tt = crste.h.tt;
+
+	/* Table entry already in the desired state */
+	if (crste.val == new_crste.val)
+		return 0;
+
+	/* This table entry needs to be updated */
+	if (walk->start <= gfn && walk->end >= next) {
+		dat_crstep_xchg_atomic(crstep, crste, new_crste, gfn, walk->asce);
+		/* A lower level table was present, needs to be freed */
+		if (!crste.h.fc && !crste.h.i)
+			dat_free_level(dereference_crste(crste), true);
+		return 0;
+	}
+
+	/* A lower level table is present, things will handled there */
+	if (!crste.h.fc && !crste.h.i)
+		return 0;
+	/* Split (install a lower level table), and handle things there */
+	return dat_split_crste(crstep, gfn, walk->asce);
+}
+
+static const struct dat_walk_ops dat_slot_ops = {
+	.pte_entry = _dat_slot_pte,
+	.crste_ops = { _dat_slot_crste, _dat_slot_crste, _dat_slot_crste, _dat_slot_crste, },
+};
+
+int dat_set_slot(union asce asce, gfn_t start, gfn_t end, u16 type, u16 param)
+{
+	unsigned long token = _CRSTE_TOK(0, type, param).val;
+
+	return _dat_walk_gfn_range(start, end, asce, &dat_slot_ops,
+				   DAT_WALK_IGN_HOLES | DAT_WALK_ANY, (void *)token);
+}
+
+unsigned long dat_get_ptval(struct page_table *table, struct ptval_param param)
+{
+	union pgste *pgstes = table->pgstes + param.offset;
+	struct page *page = virt_to_page(table);
+	unsigned long res = 0;
+
+	lock_page(page);
+	switch (param.len) {
+	case 3:
+		res = pgstes->val16;
+		pgstes++;
+		fallthrough;
+	case 2:
+		res = res << 16 | pgstes->val16;
+		pgstes++;
+		fallthrough;
+	case 1:
+		res = res << 16 | pgstes->val16;
+		pgstes++;
+		fallthrough;
+	case 0:
+		res = res << 16 | pgstes->val16;
+		break;
+	}
+	unlock_page(page);
+
+	return res;
+}
+
+void dat_set_ptval(struct page_table *table, struct ptval_param param, unsigned long val)
+{
+	union pgste *pgstes = table->pgstes + param.offset;
+	struct page *page = virt_to_page(table);
+
+	lock_page(page);
+	switch (param.len) {
+	case 3:
+		pgstes->val16 = val >> 48;
+		pgstes++;
+		fallthrough;
+	case 2:
+		pgstes->val16 = val >> 32;
+		pgstes++;
+		fallthrough;
+	case 1:
+		pgstes->val16 = val >> 16;
+		pgstes++;
+		fallthrough;
+	case 0:
+		pgstes->val16 = val;
+		break;
+	}
+	unlock_page(page);
+}
+
+static long _dat_test_young_pte(union pte *ptep, gfn_t start, gfn_t end, struct dat_walk *walk)
+{
+	return ptep->s.y;
+}
+
+static long _dat_test_young_crste(union crste *crstep, gfn_t start, gfn_t end,
+				  struct dat_walk *walk)
+{
+	return crstep->h.fc && crstep->s.fc1.y;
+}
+
+static const struct dat_walk_ops test_age_ops = {
+	.pte_entry = _dat_test_young_pte,
+	.pmd_entry = _dat_test_young_crste,
+	.pud_entry = _dat_test_young_crste,
+};
+
+/**
+ * dat_test_age_gfn() - test young
+ * @kvm: the kvm instance
+ * @range: the range of guest addresses whose young status needs to be cleared
+ *
+ * Context: called by KVM common code with the kvm mmu write lock held
+ * Return: 1 if any page in the given range is young, otherwise 0.
+ */
+bool dat_test_age_gfn(union asce asce, gfn_t start, gfn_t end)
+{
+	return _dat_walk_gfn_range(start, end, asce, &test_age_ops, 0, NULL) > 0;
+}
+
+int dat_link(kvm_pfn_t pfn, gfn_t gfn, union asce asce, int level, bool w, bool d, bool s, bool sk)
+{
+	union crste oldval, newval;
+	union pte newpte, oldpte;
+	union crste *crstep;
+	union pgste pgste;
+	union pte *ptep;
+	int rc = 0;
+
+	rc = dat_entry_walk(gfn, asce, DAT_WALK_ALLOC_CONTINUE, level, &crstep, &ptep);
+	if (rc)
+		return rc == -EINVAL ? rc : -EAGAIN;
+
+	if (WARN_ON_ONCE(unlikely(get_level(crstep, ptep) > level)))
+		return -EINVAL;
+
+	if (ptep)  {
+		pgste = pgste_get_lock(ptep);
+		oldpte = *ptep;
+		newpte = _pte(pfn, w, d | oldpte.s.d, s);
+		newpte.s.sd = oldpte.s.sd;
+		oldpte.s.sd = 0;
+		if (oldpte.val == _PTE_EMPTY.val || oldpte.h.pfra == pfn)
+			pgste = __dat_ptep_xchg(ptep, pgste, newpte, gfn, asce, sk);
+		else
+			rc = -EAGAIN;
+		pgste_set_unlock(ptep, pgste);
+	} else {
+		oldval = READ_ONCE(*crstep);
+		newval = _crste_fc1(pfn, oldval.h.tt, w, d | oldval.s.fc1.d);
+		newval.s.fc1.sd = oldval.s.fc1.sd;
+		if (oldval.val != _CRSTE_EMPTY(oldval.h.tt).val &&
+		    crste_origin_large(oldval) != crste_origin_large(newval))
+			return -EAGAIN;
+		if (!dat_crstep_xchg_atomic(crstep, oldval, newval, gfn, asce))
+			return -EAGAIN;
+	}
+
+	return rc;
+}
+
+static long dat_set_pn_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk)
+{
+	union crste crste = READ_ONCE(*crstep);
+	int *n = walk->priv;
+
+	if (!crste.h.fc || crste.h.i || crste.h.p)
+		return 0;
+
+	*n = 2;
+	if (crste.s.fc1.prefix_notif)
+		return 0;
+	crste.s.fc1.prefix_notif = 1;
+	dat_crstep_xchg(crstep, crste, gfn, walk->asce);
+	return 0;
+}
+
+static long dat_set_pn_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk)
+{
+	int *n = walk->priv;
+	union pgste pgste;
+
+	pgste = pgste_get_lock(ptep);
+	if (!ptep->h.i && !ptep->h.p) {
+		pgste.prefix_notif = 1;
+		*n += 1;
+	}
+	pgste_set_unlock(ptep, pgste);
+	return 0;
+}
+
+int dat_set_prefix_notif_bit(union asce asce, gfn_t gfn)
+{
+	static const struct dat_walk_ops ops = {
+		.pte_entry = dat_set_pn_pte,
+		.pmd_entry = dat_set_pn_crste,
+		.pud_entry = dat_set_pn_crste,
+	};
+
+	int n = 0;
+
+	_dat_walk_gfn_range(gfn, gfn + 2, asce, &ops, DAT_WALK_IGN_HOLES, &n);
+	if (n != 2)
+		return -EAGAIN;
+	return 0;
+}
arch/s390/kvm/dat.h +35 -0
index 40f5c1371ef3..b695eae5d763 100644
@@ -374,6 +374,11 @@ struct dat_walk {
 	void *priv;
 };
 
+struct ptval_param {
+	unsigned char offset : 6;
+	unsigned char len : 2;
+};
+
 static inline union pte _pte(kvm_pfn_t pfn, bool w, bool d, bool s)
 {
 	union pte res = { .val = PFN_PHYS(pfn) };
@@ -413,6 +418,18 @@ static inline union crste _crste_fc1(kvm_pfn_t pfn, int tt, bool w, bool d)
 	return res;
 }
 
+/**
+ *	0	1	2	3	4	5	6	7
+ *	+-------+-------+-------+-------+-------+-------+-------+-------+
+ *  0	|				|	    PGT_ADDR		|
+ *  8	|	 VMADDR		|SPLTCNT|				|
+ * 16	|								|
+ * 24	|								|
+ */
+#define MKPTVAL(o, l) ((struct ptval_param) { .offset = (o), .len = ((l) + 1) / 2 - 1})
+#define PTVAL_PGT_ADDR	MKPTVAL(4, 8)
+#define PTVAL_VMADDR	MKPTVAL(8, 6)
+
 union pgste __must_check __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new,
 					 gfn_t gfn, union asce asce, bool has_skeys);
 bool dat_crstep_xchg_atomic(union crste *crstep, union crste old, union crste new, gfn_t gfn,
@@ -434,6 +451,14 @@ int dat_cond_set_storage_key(union asce asce, gfn_t gfn, union skey skey, union
 int dat_reset_reference_bit(union asce asce, gfn_t gfn);
 long dat_reset_skeys(union asce asce, gfn_t start);
 
+unsigned long dat_get_ptval(struct page_table *table, struct ptval_param param);
+void dat_set_ptval(struct page_table *table, struct ptval_param param, unsigned long val);
+
+int dat_set_slot(union asce asce, gfn_t start, gfn_t end, u16 type, u16 param);
+int dat_set_prefix_notif_bit(union asce asce, gfn_t gfn);
+bool dat_test_age_gfn(union asce asce, gfn_t start, gfn_t end);
+int dat_link(kvm_pfn_t pfn, gfn_t gfn, union asce asce, int level, bool w, bool d, bool s, bool sk);
+
 static inline struct crst_table *crste_table_start(union crste *crstep)
 {
 	return (struct crst_table *)ALIGN_DOWN((unsigned long)crstep, _CRST_TABLE_SIZE);
@@ -778,4 +803,14 @@ static inline int get_level(union crste *crstep, union pte *ptep)
 	return ptep ? LEVEL_PTE : crstep->h.tt;
 }
 
+static inline int dat_delete_slot(union asce asce, gfn_t start, unsigned long npages)
+{
+	return dat_set_slot(asce, start, start + npages, _DAT_TOKEN_PIC, PGM_ADDRESSING);
+}
+
+static inline int dat_create_slot(union asce asce, gfn_t start, unsigned long npages)
+{
+	return dat_set_slot(asce, start, start + npages, _DAT_TOKEN_NONE, 0);
+}
+
 #endif /* __KVM_S390_DAT_H */
Signature · 1 line
2.51.0
author maintainer
#63 lore raw

PATCH v2 13/20 KVM: s390: KVM page table management functions: CMMA

In reply to Claudio Imbrenda

Add page table management functions to be used for KVM guest (gmap) page tables.

This patch adds functions to handle CMMA and the ESSA instruction.

---
 arch/s390/kvm/dat.c | 259 ++++++++++++++++++++++++++++++++++++++++++++
 arch/s390/kvm/dat.h |  27 +++++
 2 files changed, 286 insertions(+)
arch/s390/kvm/dat.c +259 -0
index 4249400a9d21..bf9c8af1d74a 100644
@@ -999,3 +999,262 @@ int dat_set_prefix_notif_bit(union asce asce, gfn_t gfn)
 		return -EAGAIN;
 	return 0;
 }
+
+/**
+ * dat_perform_essa() - perform ESSA actions on the PGSTE.
+ * @asce: the asce to operate on.
+ * @gfn: the guest page frame to operate on.
+ * @orc: the specific action to perform, see the ESSA_SET_* macros.
+ * @state: the storage attributes to be returned to the guest.
+ * @dirty: returns whether the function dirtied a previously clean entry.
+ *
+ * Context: Called with kvm->mmu_lock held.
+ *
+ * Return:
+ * * 1 if the page state has been altered and the page is to be added to the CBRL
+ * * 0 if the page state has been altered, but the page is not to be added to the CBRL
+ * * -1 if the page state has not been altered and the page is not to be added to the CBRL
+ */
+int dat_perform_essa(union asce asce, gfn_t gfn, int orc, union essa_state *state, bool *dirty)
+{
+	union crste *crstep;
+	union pgste pgste;
+	union pte *ptep;
+	int res = 0;
+
+	if (dat_entry_walk(gfn, asce, 0, LEVEL_PTE, &crstep, &ptep)) {
+		*state = (union essa_state) { .exception = 1 };
+		return -1;
+	}
+
+	pgste = pgste_get_lock(ptep);
+
+	*state = (union essa_state) {
+		.content = (ptep->h.i << 1) + (ptep->h.i && pgste.zero),
+		.nodat = pgste.nodat,
+		.usage = pgste.usage,
+		};
+
+	switch (orc) {
+	case ESSA_GET_STATE:
+		res = -1;
+		break;
+	case ESSA_SET_STABLE:
+		pgste.usage = PGSTE_GPS_USAGE_STABLE;
+		pgste.nodat = 0;
+		break;
+	case ESSA_SET_UNUSED:
+		pgste.usage = PGSTE_GPS_USAGE_UNUSED;
+		if (ptep->h.i)
+			res = 1;
+		break;
+	case ESSA_SET_VOLATILE:
+		pgste.usage = PGSTE_GPS_USAGE_VOLATILE;
+		if (ptep->h.i)
+			res = 1;
+		break;
+	case ESSA_SET_POT_VOLATILE:
+		if (!ptep->h.i) {
+			pgste.usage = PGSTE_GPS_USAGE_POT_VOLATILE;
+		} else if (pgste.zero) {
+			pgste.usage = PGSTE_GPS_USAGE_VOLATILE;
+		} else if (!pgste.gc) {
+			pgste.usage = PGSTE_GPS_USAGE_VOLATILE;
+			res = 1;
+		}
+		break;
+	case ESSA_SET_STABLE_RESIDENT:
+		pgste.usage = PGSTE_GPS_USAGE_STABLE;
+		/*
+		 * Since the resident state can go away any time after this
+		 * call, we will not make this page resident. We can revisit
+		 * this decision if a guest will ever start using this.
+		 */
+		break;
+	case ESSA_SET_STABLE_IF_RESIDENT:
+		if (!ptep->h.i)
+			pgste.usage = PGSTE_GPS_USAGE_STABLE;
+		break;
+	case ESSA_SET_STABLE_NODAT:
+		pgste.usage = PGSTE_GPS_USAGE_STABLE;
+		pgste.nodat = 1;
+		break;
+	default:
+		WARN_ONCE(1, "Invalid ORC!");
+		res = -1;
+		break;
+	}
+	/* If we are discarding a page, set it to logical zero */
+	pgste.zero = res == 1;
+	if (orc > 0) {
+		*dirty = !pgste.cmma_d;
+		pgste.cmma_d = 1;
+	}
+
+	pgste_set_unlock(ptep, pgste);
+
+	return res;
+}
+
+static long dat_reset_cmma_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk)
+{
+	union pgste pgste;
+
+	pgste = pgste_get_lock(ptep);
+	pgste.usage = 0;
+	pgste.nodat = 0;
+	pgste.cmma_d = 0;
+	pgste_set_unlock(ptep, pgste);
+	if (need_resched())
+		return next;
+	return 0;
+}
+
+long dat_reset_cmma(union asce asce, gfn_t start)
+{
+	const struct dat_walk_ops dat_reset_cmma_ops = {
+		.pte_entry = dat_reset_cmma_pte,
+	};
+
+	return _dat_walk_gfn_range(start, asce_end(asce), asce, &dat_reset_cmma_ops,
+				   DAT_WALK_IGN_HOLES, NULL);
+}
+
+struct dat_get_cmma_state {
+	gfn_t start;
+	gfn_t end;
+	unsigned int count;
+	u8 *values;
+	atomic64_t *remaining;
+};
+
+static long __dat_peek_cmma_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk)
+{
+	struct dat_get_cmma_state *state = walk->priv;
+	union pgste pgste;
+
+	pgste = pgste_get_lock(ptep);
+	state->values[gfn - walk->start] = pgste.usage | (pgste.nodat << 6);
+	pgste_set_unlock(ptep, pgste);
+	state->end = next;
+
+	return 0;
+}
+
+static long __dat_peek_cmma_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk)
+{
+	struct dat_get_cmma_state *state = walk->priv;
+
+	if (crstep->h.i)
+		state->end = min(walk->end, next);
+	return 0;
+}
+
+int dat_peek_cmma(gfn_t start, union asce asce, unsigned int *count, u8 *values)
+{
+	const struct dat_walk_ops ops = {
+		.pte_entry = __dat_peek_cmma_pte,
+		.pmd_entry = __dat_peek_cmma_crste,
+		.pud_entry = __dat_peek_cmma_crste,
+		.p4d_entry = __dat_peek_cmma_crste,
+		.pgd_entry = __dat_peek_cmma_crste,
+	};
+	struct dat_get_cmma_state state = { .values = values, };
+	int rc;
+
+	rc = _dat_walk_gfn_range(start, start + *count, asce, &ops, DAT_WALK_DEFAULT, &state);
+	*count = state.end - start;
+	/* Return success if at least one value was saved, otherwise an error. */
+	return (rc == -EFAULT && *count > 0) ? 0 : rc;
+}
+
+static long __dat_get_cmma_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk)
+{
+	struct dat_get_cmma_state *state = walk->priv;
+	union pgste pgste;
+
+	if (state->start != -1) {
+		if ((gfn - state->end) > KVM_S390_MAX_BIT_DISTANCE)
+			return 1;
+		if (gfn - state->start >= state->count)
+			return 1;
+	}
+
+	if (!READ_ONCE(*pgste_of(ptep)).cmma_d)
+		return 0;
+
+	pgste = pgste_get_lock(ptep);
+	if (pgste.cmma_d) {
+		if (state->start == -1)
+			state->start = gfn;
+		pgste.cmma_d = 0;
+		atomic64_dec(state->remaining);
+		state->values[gfn - state->start] = pgste.usage | pgste.nodat << 6;
+		state->end = next;
+	}
+	pgste_set_unlock(ptep, pgste);
+	return 0;
+}
+
+int dat_get_cmma(union asce asce, gfn_t *start, unsigned int *count, u8 *values, atomic64_t *rem)
+{
+	const struct dat_walk_ops ops = { .pte_entry = __dat_get_cmma_pte, };
+	struct dat_get_cmma_state state = {
+		.remaining = rem,
+		.values = values,
+		.count = *count,
+		.start = -1,
+	};
+
+	_dat_walk_gfn_range(*start, asce_end(asce), asce, &ops, DAT_WALK_IGN_HOLES, &state);
+
+	if (state.start == -1) {
+		*count = 0;
+	} else {
+		*count = state.end - state.start;
+		*start = state.start;
+	}
+
+	return 0;
+}
+
+struct dat_set_cmma_state {
+	unsigned long mask;
+	const u8 *bits;
+};
+
+static long __dat_set_cmma_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk)
+{
+	struct dat_set_cmma_state *state = walk->priv;
+	union pgste pgste, tmp;
+
+	tmp.val = (state->bits[gfn - walk->start] << 24) & state->mask;
+
+	pgste = pgste_get_lock(ptep);
+	pgste.usage = tmp.usage;
+	pgste.nodat = tmp.nodat;
+	pgste_set_unlock(ptep, pgste);
+
+	return 0;
+}
+
+/*
+ * This function sets the CMMA attributes for the given pages. If the input
+ * buffer has zero length, no action is taken, otherwise the attributes are
+ * set and the mm->context.uses_cmm flag is set.
+ */
+int dat_set_cmma_bits(union asce asce, gfn_t gfn, unsigned long count,
+		      unsigned long mask, const uint8_t *bits)
+{
+	const struct dat_walk_ops ops = { .pte_entry = __dat_set_cmma_pte, };
+	struct dat_set_cmma_state state = { .mask = mask, .bits = bits, };
+	union crste *crstep;
+	union pte *ptep;
+	gfn_t cur;
+	int rc;
+
+	for (cur = ALIGN_DOWN(gfn, _PAGE_ENTRIES); cur < gfn + count; cur += _PAGE_ENTRIES)
+		dat_entry_walk(cur, asce, DAT_WALK_ALLOC, LEVEL_PTE, &crstep, &ptep);
+	rc = _dat_walk_gfn_range(gfn, gfn + count, asce, &ops, DAT_WALK_IGN_HOLES, &state);
+	return rc;
+}
arch/s390/kvm/dat.h +27 -0
index b695eae5d763..4d0ceeada40f 100644
@@ -18,6 +18,15 @@
 #include <asm/pgalloc.h>
 #include <asm/dat-bits.h>
 
+/*
+ * Base address and length must be sent at the start of each block, therefore
+ * it's cheaper to send some clean data, as long as it's less than the size of
+ * two longs.
+ */
+#define KVM_S390_MAX_BIT_DISTANCE (2 * sizeof(void *))
+/* for consistency */
+#define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX)
+
 #define _ASCE(x) ((union asce) { .val = (x), })
 #define NULL_ASCE _ASCE(0)
 
@@ -418,6 +427,17 @@ static inline union crste _crste_fc1(kvm_pfn_t pfn, int tt, bool w, bool d)
 	return res;
 }
 
+union essa_state {
+	unsigned char val;
+	struct {
+		unsigned char		: 2;
+		unsigned char nodat	: 1;
+		unsigned char exception	: 1;
+		unsigned char usage	: 2;
+		unsigned char content	: 2;
+	};
+};
+
 /**
  *	0	1	2	3	4	5	6	7
  *	+-------+-------+-------+-------+-------+-------+-------+-------+
@@ -459,6 +479,13 @@ int dat_set_prefix_notif_bit(union asce asce, gfn_t gfn);
 bool dat_test_age_gfn(union asce asce, gfn_t start, gfn_t end);
 int dat_link(kvm_pfn_t pfn, gfn_t gfn, union asce asce, int level, bool w, bool d, bool s, bool sk);
 
+int dat_perform_essa(union asce asce, gfn_t gfn, int orc, union essa_state *state, bool *dirty);
+long dat_reset_cmma(union asce asce, gfn_t start_gfn);
+int dat_peek_cmma(gfn_t start, union asce asce, unsigned int *count, u8 *values);
+int dat_get_cmma(union asce asce, gfn_t *start, unsigned int *count, u8 *values, atomic64_t *rem);
+int dat_set_cmma_bits(union asce asce, gfn_t gfn, unsigned long count, unsigned long mask,
+		      const uint8_t *bits);
+
 static inline struct crst_table *crste_table_start(union crste *crstep)
 {
 	return (struct crst_table *)ALIGN_DOWN((unsigned long)crstep, _CRST_TABLE_SIZE);
Signature · 1 line
2.51.0
author maintainer
#64 lore raw

PATCH v2 14/20 KVM: s390: New gmap code

In reply to Claudio Imbrenda

New gmap (guest map) code. This new gmap code will only be used by KVM.

This will replace the existing gmap.

---
 arch/s390/kvm/Makefile |    2 +-
 arch/s390/kvm/gmap.c   | 1066 ++++++++++++++++++++++++++++++++++++++++
 arch/s390/kvm/gmap.h   |  178 +++++++
 3 files changed, 1245 insertions(+), 1 deletion(-)
 create mode 100644 arch/s390/kvm/gmap.c
 create mode 100644 arch/s390/kvm/gmap.h
arch/s390/kvm/Makefile +1 -1
index 84315d2f75fb..21088265402c 100644
@@ -9,7 +9,7 @@ ccflags-y := -Ivirt/kvm -Iarch/s390/kvm
 
 kvm-y += kvm-s390.o intercept.o interrupt.o priv.o sigp.o
 kvm-y += diag.o gaccess.o guestdbg.o vsie.o pv.o gmap-vsie.o
-kvm-y += dat.o
+kvm-y += dat.o gmap.o
 
 kvm-$(CONFIG_VFIO_PCI_ZDEV_KVM) += pci.o
 obj-$(CONFIG_KVM) += kvm.o
arch/s390/kvm/gmap.c new file +1066 -0

1,066 lines not on this page · raw message

arch/s390/kvm/gmap.h new file +178 -0

178 lines not on this page · raw message

Signature · 1 line
2.51.0
author maintainer
#65 lore raw

PATCH v2 15/20 KVM: s390: Stop using CONFIG_PGSTE

In reply to Claudio Imbrenda

Switch to using IS_ENABLED(CONFIG_KVM) instead of CONFIG_PGSTE, since the latter will be removed soon.

Many CONFIG_PGSTE are left behind, because they will be removed completely in upcoming patches. The ones replaced here are mostly the ones that will stay.

---
 arch/s390/include/asm/mmu_context.h | 2 +-
 arch/s390/include/asm/pgtable.h     | 4 ++--
 arch/s390/mm/fault.c                | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)
arch/s390/include/asm/mmu_context.h +1 -1
index d9b8501bc93d..48e548c01daa 100644
@@ -29,7 +29,7 @@ static inline int init_new_context(struct task_struct *tsk,
 	atomic_set(&mm->context.protected_count, 0);
 	mm->context.gmap_asce = 0;
 	mm->context.flush_mm = 0;
-#ifdef CONFIG_PGSTE
+#if IS_ENABLED(CONFIG_KVM)
 	mm->context.has_pgste = 0;
 	mm->context.uses_skeys = 0;
 	mm->context.uses_cmm = 0;
arch/s390/include/asm/pgtable.h +2 -2
index 3c74b39bf669..fc6bb6d2d772 100644
@@ -577,7 +577,7 @@ static inline int mm_has_pgste(struct mm_struct *mm)
 
 static inline int mm_is_protected(struct mm_struct *mm)
 {
-#ifdef CONFIG_PGSTE
+#if IS_ENABLED(CONFIG_KVM)
 	if (unlikely(atomic_read(&mm->context.protected_count)))
 		return 1;
 #endif
@@ -632,7 +632,7 @@ static inline pud_t set_pud_bit(pud_t pud, pgprot_t prot)
 #define mm_forbids_zeropage mm_forbids_zeropage
 static inline int mm_forbids_zeropage(struct mm_struct *mm)
 {
-#ifdef CONFIG_PGSTE
+#if IS_ENABLED(CONFIG_KVM)
 	if (!mm->context.allow_cow_sharing)
 		return 1;
 #endif
arch/s390/mm/fault.c +2 -2
index e1ad05bfd28a..683afd496190 100644
@@ -398,7 +398,7 @@ void do_dat_exception(struct pt_regs *regs)
 }
 NOKPROBE_SYMBOL(do_dat_exception);
 
-#if IS_ENABLED(CONFIG_PGSTE)
+#if IS_ENABLED(CONFIG_KVM)
 
 void do_secure_storage_access(struct pt_regs *regs)
 {
@@ -465,4 +465,4 @@ void do_secure_storage_access(struct pt_regs *regs)
 }
 NOKPROBE_SYMBOL(do_secure_storage_access);
 
-#endif /* CONFIG_PGSTE */
+#endif /* CONFIG_KVM */
Signature · 1 line
2.51.0
Reviewed-by on v2 15/20
#66 lore raw

Re: [PATCH v2 15/20] KVM: s390: Stop using CONFIG_PGSTE

Quoted 7 lines · Claudio Imbrenda

Switch to using IS_ENABLED(CONFIG_KVM) instead of CONFIG_PGSTE, since the latter will be removed soon.

Many CONFIG_PGSTE are left behind, because they will be removed completely in upcoming patches. The ones replaced here are mostly the ones that will stay.

Quoted 1 line
author maintainer
#67 lore raw

PATCH v2 16/20 KVM: s390: Switch to new gmap

In reply to Claudio Imbrenda

Switch KVM/s390 to use the new gmap code.

Remove includes to <gmap.h> and include "gmap.h" instead; fix all the existing users of the old gmap functions to use the new ones instead.

---
 arch/s390/Kconfig                   |   2 +-
 arch/s390/include/asm/mmu_context.h |   4 -
 arch/s390/include/asm/tlb.h         |   3 -
 arch/s390/kvm/Makefile              |   2 +-
 arch/s390/kvm/diag.c                |   2 +-
 arch/s390/kvm/gaccess.c             | 526 ++++++++++++----------
 arch/s390/kvm/gaccess.h             |  16 +-
 arch/s390/kvm/gmap-vsie.c           | 141 ------
 arch/s390/kvm/gmap.c                |   6 +-
 arch/s390/kvm/intercept.c           |   8 +-
 arch/s390/kvm/interrupt.c           |   2 +-
 arch/s390/kvm/kvm-s390.c            | 671 ++++++++--------------------
 arch/s390/kvm/kvm-s390.h            |  19 +-
 arch/s390/kvm/priv.c                | 206 +++------
 arch/s390/kvm/pv.c                  |  84 ++--
 arch/s390/kvm/vsie.c                | 116 +++--
 16 files changed, 672 insertions(+), 1136 deletions(-)
 delete mode 100644 arch/s390/kvm/gmap-vsie.c
arch/s390/Kconfig +1 -1
index 0c16dc443e2f..d43f7a8a77fa 100644
@@ -33,7 +33,7 @@ config GENERIC_LOCKBREAK
 	def_bool y if PREEMPTION
 
 config PGSTE
-	def_bool y if KVM
+	def_bool n
 
 config AUDIT_ARCH
 	def_bool y
arch/s390/include/asm/mmu_context.h +0 -4
index 48e548c01daa..bd1ef5e2d2eb 100644
@@ -30,11 +30,7 @@ static inline int init_new_context(struct task_struct *tsk,
 	mm->context.gmap_asce = 0;
 	mm->context.flush_mm = 0;
 #if IS_ENABLED(CONFIG_KVM)
-	mm->context.has_pgste = 0;
-	mm->context.uses_skeys = 0;
-	mm->context.uses_cmm = 0;
 	mm->context.allow_cow_sharing = 1;
-	mm->context.allow_gmap_hpage_1m = 0;
 #endif
 	switch (mm->context.asce_limit) {
 	default:
arch/s390/include/asm/tlb.h +0 -3
index 1e50f6f1ad9d..7354b42ee994 100644
@@ -36,7 +36,6 @@ static inline bool __tlb_remove_folio_pages(struct mmu_gather *tlb,
 
 #include <asm/tlbflush.h>
 #include <asm-generic/tlb.h>
-#include <asm/gmap.h>
 
 /*
  * Release the page cache reference for a pte removed by
@@ -85,8 +84,6 @@ static inline void pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
 	tlb->mm->context.flush_mm = 1;
 	tlb->freed_tables = 1;
 	tlb->cleared_pmds = 1;
-	if (mm_has_pgste(tlb->mm))
-		gmap_unlink(tlb->mm, (unsigned long *)pte, address);
 	tlb_remove_ptdesc(tlb, virt_to_ptdesc(pte));
 }
 
arch/s390/kvm/Makefile +1 -1
index 21088265402c..1d092b75adc0 100644
@@ -8,7 +8,7 @@ include $(srctree)/virt/kvm/Makefile.kvm
 ccflags-y := -Ivirt/kvm -Iarch/s390/kvm
 
 kvm-y += kvm-s390.o intercept.o interrupt.o priv.o sigp.o
-kvm-y += diag.o gaccess.o guestdbg.o vsie.o pv.o gmap-vsie.o
+kvm-y += diag.o gaccess.o guestdbg.o vsie.o pv.o
 kvm-y += dat.o gmap.o
 
 kvm-$(CONFIG_VFIO_PCI_ZDEV_KVM) += pci.o
arch/s390/kvm/diag.c +1 -1
index 53233dec8cad..d89d1c381522 100644
@@ -10,13 +10,13 @@
 
 #include <linux/kvm.h>
 #include <linux/kvm_host.h>
-#include <asm/gmap.h>
 #include <asm/gmap_helpers.h>
 #include <asm/virtio-ccw.h>
 #include "kvm-s390.h"
 #include "trace.h"
 #include "trace-s390.h"
 #include "gaccess.h"
+#include "gmap.h"
 
 static void do_discard_gfn_range(struct kvm_vcpu *vcpu, gfn_t gfn_start, gfn_t gfn_end)
 {
arch/s390/kvm/gaccess.c +284 -242

665 lines not on this page · raw message

arch/s390/kvm/gaccess.h +11 -5
index 9c82f7460821..cd19a29d31d8 100644
@@ -450,12 +450,18 @@ void ipte_unlock(struct kvm *kvm);
 int ipte_lock_held(struct kvm *kvm);
 int kvm_s390_check_low_addr_prot_real(struct kvm_vcpu *vcpu, unsigned long gra);
 
-/* MVPG PEI indication bits */
-#define PEI_DAT_PROT 2
-#define PEI_NOT_PTE 4
+union mvpg_pei {
+	unsigned long val;
+	struct {
+		unsigned long addr    : 61;
+		unsigned long not_pte :  1;
+		unsigned long dat_prot:  1;
+		unsigned long real    :  1;
+	};
+};
 
-int kvm_s390_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *shadow,
-			  unsigned long saddr, unsigned long *datptr);
+int gaccess_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *sg, gpa_t saddr,
+			 union mvpg_pei *datptr, bool wr);
 
 static inline int __kvm_s390_faultin_read_gpa(struct kvm *kvm, struct guest_fault *f, gpa_t gaddr,
 					      unsigned long *val)
arch/s390/kvm/gmap-vsie.c deleted +0 -141

141 lines not on this page · raw message

arch/s390/kvm/gmap.c +3 -3
index 42c936362e3e..001b069a6c7b 100644
@@ -673,13 +673,13 @@ static int _gmap_enable_skeys(struct gmap *gmap)
 	gfn_t start = 0;
 	int rc;
 
-	if (mm_uses_skeys(gmap->kvm->mm))
+	if (gmap->uses_skeys)
 		return 0;
 
-	gmap->kvm->mm->context.uses_skeys = 1;
+	WRITE_ONCE(gmap->uses_skeys, 1);
 	rc = gmap_helper_disable_cow_sharing();
 	if (rc) {
-		gmap->kvm->mm->context.uses_skeys = 0;
+		WRITE_ONCE(gmap->uses_skeys, 0);
 		return rc;
 	}
 
arch/s390/kvm/intercept.c +6 -2
index c7908950c1f4..0481a30b2629 100644
@@ -367,7 +367,9 @@ static int handle_mvpg_pei(struct kvm_vcpu *vcpu)
 					      reg2, &srcaddr, GACC_FETCH, 0);
 	if (rc)
 		return kvm_s390_inject_prog_cond(vcpu, rc);
-	rc = kvm_s390_handle_dat_fault(vcpu, srcaddr, 0);
+	do {
+		rc = kvm_s390_handle_dat_fault(vcpu, srcaddr, 0);
+	} while (rc == -EAGAIN);
 	if (rc != 0)
 		return rc;
 
@@ -376,7 +378,9 @@ static int handle_mvpg_pei(struct kvm_vcpu *vcpu)
 					      reg1, &dstaddr, GACC_STORE, 0);
 	if (rc)
 		return kvm_s390_inject_prog_cond(vcpu, rc);
-	rc = kvm_s390_handle_dat_fault(vcpu, dstaddr, FOLL_WRITE);
+	do {
+		rc = kvm_s390_handle_dat_fault(vcpu, dstaddr, FOLL_WRITE);
+	} while (rc == -EAGAIN);
 	if (rc != 0)
 		return rc;
 
arch/s390/kvm/interrupt.c +1 -1
index 60c360c18690..7f07f6c0ab85 100644
@@ -26,7 +26,6 @@
 #include <linux/uaccess.h>
 #include <asm/sclp.h>
 #include <asm/isc.h>
-#include <asm/gmap.h>
 #include <asm/nmi.h>
 #include <asm/airq.h>
 #include <asm/tpi.h>
@@ -34,6 +33,7 @@
 #include "gaccess.h"
 #include "trace-s390.h"
 #include "pci.h"
+#include "gmap.h"
 
 #define PFAULT_INIT 0x0600
 #define PFAULT_DONE 0x0680
arch/s390/kvm/kvm-s390.c +187 -484

1,039 lines not on this page · raw message

arch/s390/kvm/kvm-s390.h +8 -11
index dabcf65f58ff..21d93eb9dee4 100644
@@ -19,6 +19,8 @@
 #include <asm/facility.h>
 #include <asm/processor.h>
 #include <asm/sclp.h>
+#include "dat.h"
+#include "gmap.h"
 
 #define KVM_S390_UCONTROL_MEMSLOT (KVM_USER_MEM_SLOTS + 0)
 
@@ -115,9 +117,7 @@ static inline int is_vcpu_idle(struct kvm_vcpu *vcpu)
 static inline int kvm_is_ucontrol(struct kvm *kvm)
 {
 #ifdef CONFIG_KVM_S390_UCONTROL
-	if (kvm->arch.gmap)
-		return 0;
-	return 1;
+	return kvm->arch.gmap->is_ucontrol;
 #else
 	return 0;
 #endif
@@ -441,14 +441,10 @@ int kvm_s390_skey_check_enable(struct kvm_vcpu *vcpu);
 /* implemented in vsie.c */
 int kvm_s390_handle_vsie(struct kvm_vcpu *vcpu);
 void kvm_s390_vsie_kick(struct kvm_vcpu *vcpu);
-void kvm_s390_vsie_gmap_notifier(struct gmap *gmap, unsigned long start,
-				 unsigned long end);
+void kvm_s390_vsie_gmap_notifier(struct gmap *gmap, gpa_t start, gpa_t end);
 void kvm_s390_vsie_init(struct kvm *kvm);
 void kvm_s390_vsie_destroy(struct kvm *kvm);
-int gmap_shadow_valid(struct gmap *sg, unsigned long asce, int edat_level);
-
-/* implemented in gmap-vsie.c */
-struct gmap *gmap_shadow(struct gmap *parent, unsigned long asce, int edat_level);
+int gmap_shadow_valid(struct gmap *sg, union asce asce, int edat_level);
 
 /* implemented in sigp.c */
 int kvm_s390_handle_sigp(struct kvm_vcpu *vcpu);
@@ -470,14 +466,15 @@ void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu);
 void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm);
 __u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu);
 int kvm_s390_cpus_from_pv(struct kvm *kvm, u16 *rc, u16 *rrc);
-int __kvm_s390_handle_dat_fault(struct kvm_vcpu *vcpu, gfn_t gfn, gpa_t gaddr, unsigned int flags);
+int __kvm_s390_handle_dat_fault(struct kvm_vcpu *vcpu, gpa_t gaddr,
+				unsigned int flags, bool pfault);
 int __kvm_s390_mprotect_many(struct gmap *gmap, gpa_t gpa, u8 npages, unsigned int prot,
 			     unsigned long bits);
 int __kvm_s390_faultin_gfn(struct kvm *kvm, struct guest_fault *f, gfn_t gfn, bool wr);
 
 static inline int kvm_s390_handle_dat_fault(struct kvm_vcpu *vcpu, gpa_t gaddr, unsigned int flags)
 {
-	return __kvm_s390_handle_dat_fault(vcpu, gpa_to_gfn(gaddr), gaddr, flags);
+	return __kvm_s390_handle_dat_fault(vcpu, gaddr, flags, false);
 }
 
 static inline void release_faultin_multiple(struct kvm *kvm, struct guest_fault *guest_faults,
arch/s390/kvm/priv.c +69 -137

346 lines not on this page · raw message

arch/s390/kvm/pv.c +30 -54

168 lines not on this page · raw message

arch/s390/kvm/vsie.c +70 -46

259 lines not on this page · raw message

Signature · 1 line
2.51.0
maintainer
#68 lore raw

Re: [PATCH v2 16/20] KVM: s390: Switch to new gmap

Quoted 6 lines · Claudio Imbrenda

Switch KVM/s390 to use the new gmap code.

Remove includes to <gmap.h> and include "gmap.h" instead; fix all the existing users of the old gmap functions to use the new ones instead.

...

Quoted 4 lines
arch/s390/kvm/gaccess.c +1 -3
-static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
-				  unsigned long *pgt, int *dat_protection,
-				  int *fake)
+static int walk_guest_tables(struct gmap *sg, unsigned long saddr, struct pgtwalk *w, bool wr)

...

arch/s390/kvm/gaccess.c int kvm_s390_check_low_addr_prot_real(struct kvm_vcpu *vcpu, unsigned long gra) · line 1257

Quoted 9 lines
arch/s390/kvm/gaccess.c +3 -3
-		if (rfte.i)
+		if (table.pgd.i)
 			return PGM_REGION_FIRST_TRANS;
-		if (rfte.tt != TABLE_TYPE_REGION1)
+		if (table.pgd.tt != TABLE_TYPE_REGION1)
 			return PGM_TRANSLATION_SPEC;
-		if (vaddr.rsx01 < rfte.tf || vaddr.rsx01 > rfte.tl)
+		if (vaddr.rsx01 < table.pgd.tf || vaddr.rsx01 > table.pgd.tl)
 			return PGM_REGION_SECOND_TRANS;

This (and other places): this was easy to map 1:1 to the architecture. But with the change to pgd & friends this is not the case anymore. From my point of view this is clearly worse compared to before; and I don't see a pressing need for such a change.

author maintainer
#69 lore raw

PATCH v2 17/20 KVM: s390: Remove gmap from s390/mm

In reply to Claudio Imbrenda

Remove the now unused include/asm/gmap.h and mm/gmap.c files.

---
 MAINTAINERS                     |    2 -
 arch/s390/include/asm/gmap.h    |  174 ---
 arch/s390/include/asm/pgtable.h |    9 -
 arch/s390/mm/Makefile           |    1 -
 arch/s390/mm/gmap.c             | 2452 -------------------------------
 arch/s390/mm/pgtable.c          |   10 -
 6 files changed, 2648 deletions(-)
 delete mode 100644 arch/s390/include/asm/gmap.h
 delete mode 100644 arch/s390/mm/gmap.c
MAINTAINERS +0 -2
index c0b444e5fd5a..261d74060d8d 100644
@@ -13295,14 +13295,12 @@ L:	[email protected]
 S:	Supported
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git
 F:	Documentation/virt/kvm/s390*
-F:	arch/s390/include/asm/gmap.h
 F:	arch/s390/include/asm/gmap_helpers.h
 F:	arch/s390/include/asm/kvm*
 F:	arch/s390/include/uapi/asm/kvm*
 F:	arch/s390/include/uapi/asm/uvdevice.h
 F:	arch/s390/kernel/uv.c
 F:	arch/s390/kvm/
-F:	arch/s390/mm/gmap.c
 F:	arch/s390/mm/gmap_helpers.c
 F:	drivers/s390/char/uvdevice.c
 F:	tools/testing/selftests/drivers/s390x/uvdevice/
arch/s390/include/asm/gmap.h deleted +0 -174

174 lines not on this page · raw message

arch/s390/include/asm/pgtable.h +0 -9
index fc6bb6d2d772..a012456140fd 100644
@@ -1376,8 +1376,6 @@ static inline int ptep_set_access_flags(struct vm_area_struct *vma,
 void ptep_set_pte_at(struct mm_struct *mm, unsigned long addr,
 		     pte_t *ptep, pte_t entry);
 void ptep_set_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep);
-void ptep_notify(struct mm_struct *mm, unsigned long addr,
-		 pte_t *ptep, unsigned long bits);
 int ptep_force_prot(struct mm_struct *mm, unsigned long gaddr,
 		    pte_t *ptep, int prot, unsigned long bit);
 void ptep_zap_unused(struct mm_struct *mm, unsigned long addr,
@@ -1403,10 +1401,6 @@ int set_pgste_bits(struct mm_struct *mm, unsigned long addr,
 int get_pgste(struct mm_struct *mm, unsigned long hva, unsigned long *pgstep);
 int pgste_perform_essa(struct mm_struct *mm, unsigned long hva, int orc,
 			unsigned long *oldpte, unsigned long *oldpgste);
-void gmap_pmdp_csp(struct mm_struct *mm, unsigned long vmaddr);
-void gmap_pmdp_invalidate(struct mm_struct *mm, unsigned long vmaddr);
-void gmap_pmdp_idte_local(struct mm_struct *mm, unsigned long vmaddr);
-void gmap_pmdp_idte_global(struct mm_struct *mm, unsigned long vmaddr);
 
 #define pgprot_writecombine	pgprot_writecombine
 pgprot_t pgprot_writecombine(pgprot_t prot);
@@ -1992,9 +1986,6 @@ extern int __vmem_map_4k_page(unsigned long addr, unsigned long phys, pgprot_t p
 extern int vmem_map_4k_page(unsigned long addr, unsigned long phys, pgprot_t prot);
 extern void vmem_unmap_4k_page(unsigned long addr);
 extern pte_t *vmem_get_alloc_pte(unsigned long addr, bool alloc);
-extern int s390_enable_sie(void);
-extern int s390_enable_skey(void);
-extern void s390_reset_cmma(struct mm_struct *mm);
 
 /* s390 has a private copy of get unmapped area to deal with cache synonyms */
 #define HAVE_ARCH_UNMAPPED_AREA
arch/s390/mm/Makefile +0 -1
index bd0401cc7ca5..193899c39ca7 100644
@@ -10,7 +10,6 @@ obj-$(CONFIG_CMM)		+= cmm.o
 obj-$(CONFIG_DEBUG_VIRTUAL)	+= physaddr.o
 obj-$(CONFIG_HUGETLB_PAGE)	+= hugetlbpage.o
 obj-$(CONFIG_PTDUMP)		+= dump_pagetables.o
-obj-$(CONFIG_PGSTE)		+= gmap.o
 obj-$(CONFIG_PFAULT)		+= pfault.o
 
 obj-$(subst m,y,$(CONFIG_KVM))	+= gmap_helpers.o
arch/s390/mm/gmap.c deleted +0 -2452

2,452 lines not on this page · raw message

arch/s390/mm/pgtable.c +0 -10
index 7df70cd8f739..ce2ba5e1f023 100644
@@ -370,8 +370,6 @@ static inline void pmdp_idte_local(struct mm_struct *mm,
 			    mm->context.asce, IDTE_LOCAL);
 	else
 		__pmdp_idte(addr, pmdp, 0, 0, IDTE_LOCAL);
-	if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m)
-		gmap_pmdp_idte_local(mm, addr);
 }
 
 static inline void pmdp_idte_global(struct mm_struct *mm,
@@ -380,16 +378,10 @@ static inline void pmdp_idte_global(struct mm_struct *mm,
 	if (machine_has_tlb_guest()) {
 		__pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE,
 			    mm->context.asce, IDTE_GLOBAL);
-		if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m)
-			gmap_pmdp_idte_global(mm, addr);
 	} else if (cpu_has_idte()) {
 		__pmdp_idte(addr, pmdp, 0, 0, IDTE_GLOBAL);
-		if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m)
-			gmap_pmdp_idte_global(mm, addr);
 	} else {
 		__pmdp_csp(pmdp);
-		if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m)
-			gmap_pmdp_csp(mm, addr);
 	}
 }
 
@@ -424,8 +416,6 @@ static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm,
 			  cpumask_of(smp_processor_id()))) {
 		set_pmd(pmdp, set_pmd_bit(*pmdp, __pgprot(_SEGMENT_ENTRY_INVALID)));
 		mm->context.flush_mm = 1;
-		if (mm_has_pgste(mm))
-			gmap_pmdp_invalidate(mm, addr);
 	} else {
 		pmdp_idte_global(mm, addr, pmdp);
 	}
Signature · 1 line
2.51.0
author maintainer
#70 lore raw

PATCH v2 18/20 KVM: S390: Remove PGSTE code from linux/s390 mm

In reply to Claudio Imbrenda

Remove the PGSTE config option.
Remove all code from linux/s390 mm that involves PGSTEs.

---
 arch/s390/Kconfig               |   3 -
 arch/s390/include/asm/mmu.h     |  13 -
 arch/s390/include/asm/page.h    |   4 -
 arch/s390/include/asm/pgalloc.h |   2 -
 arch/s390/include/asm/pgtable.h |  99 +---
 arch/s390/mm/hugetlbpage.c      |  24 -
 arch/s390/mm/pgalloc.c          |  29 --
 arch/s390/mm/pgtable.c          | 827 +-------------------------------
 mm/khugepaged.c                 |   9 -
 9 files changed, 14 insertions(+), 996 deletions(-)
arch/s390/Kconfig +0 -3
index d43f7a8a77fa..250cff17fa1a 100644
@@ -32,9 +32,6 @@ config GENERIC_BUG_RELATIVE_POINTERS
 config GENERIC_LOCKBREAK
 	def_bool y if PREEMPTION
 
-config PGSTE
-	def_bool n
-
 config AUDIT_ARCH
 	def_bool y
 
arch/s390/include/asm/mmu.h +0 -13
index f07e49b419ab..d4fd7bf3692e 100644
@@ -18,24 +18,11 @@ typedef struct {
 	unsigned long vdso_base;
 	/* The mmu context belongs to a secure guest. */
 	atomic_t protected_count;
-	/*
-	 * The following bitfields need a down_write on the mm
-	 * semaphore when they are written to. As they are only
-	 * written once, they can be read without a lock.
-	 */
-	/* The mmu context uses extended page tables. */
-	unsigned int has_pgste:1;
-	/* The mmu context uses storage keys. */
-	unsigned int uses_skeys:1;
-	/* The mmu context uses CMM. */
-	unsigned int uses_cmm:1;
 	/*
 	 * The mmu context allows COW-sharing of memory pages (KSM, zeropage).
 	 * Note that COW-sharing during fork() is currently always allowed.
 	 */
 	unsigned int allow_cow_sharing:1;
-	/* The gmaps associated with this context are allowed to use huge pages. */
-	unsigned int allow_gmap_hpage_1m:1;
 } mm_context_t;
 
 #define INIT_MM_CONTEXT(name)						   \
arch/s390/include/asm/page.h +0 -4
index 4e5dbabdf202..b4fb4d7adff4 100644
@@ -78,7 +78,6 @@ static inline void copy_page(void *to, void *from)
 #ifdef STRICT_MM_TYPECHECKS
 
 typedef struct { unsigned long pgprot; } pgprot_t;
-typedef struct { unsigned long pgste; } pgste_t;
 typedef struct { unsigned long pte; } pte_t;
 typedef struct { unsigned long pmd; } pmd_t;
 typedef struct { unsigned long pud; } pud_t;
@@ -94,7 +93,6 @@ static __always_inline unsigned long name ## _val(name ## _t name)	\
 #else /* STRICT_MM_TYPECHECKS */
 
 typedef unsigned long pgprot_t;
-typedef unsigned long pgste_t;
 typedef unsigned long pte_t;
 typedef unsigned long pmd_t;
 typedef unsigned long pud_t;
@@ -110,7 +108,6 @@ static __always_inline unsigned long name ## _val(name ## _t name)	\
 #endif /* STRICT_MM_TYPECHECKS */
 
 DEFINE_PGVAL_FUNC(pgprot)
-DEFINE_PGVAL_FUNC(pgste)
 DEFINE_PGVAL_FUNC(pte)
 DEFINE_PGVAL_FUNC(pmd)
 DEFINE_PGVAL_FUNC(pud)
@@ -120,7 +117,6 @@ DEFINE_PGVAL_FUNC(pgd)
 typedef pte_t *pgtable_t;
 
 #define __pgprot(x)	((pgprot_t) { (x) } )
-#define __pgste(x)	((pgste_t) { (x) } )
 #define __pte(x)        ((pte_t) { (x) } )
 #define __pmd(x)        ((pmd_t) { (x) } )
 #define __pud(x)	((pud_t) { (x) } )
arch/s390/include/asm/pgalloc.h +0 -2
index 5345398df653..d4067fe8b0f1 100644
@@ -23,9 +23,7 @@ unsigned long *crst_table_alloc(struct mm_struct *);
 void crst_table_free(struct mm_struct *, unsigned long *);
 
 unsigned long *page_table_alloc(struct mm_struct *);
-struct ptdesc *page_table_alloc_pgste(struct mm_struct *mm);
 void page_table_free(struct mm_struct *, unsigned long *);
-void page_table_free_pgste(struct ptdesc *ptdesc);
 
 static inline void crst_table_init(unsigned long *crst, unsigned long entry)
 {
arch/s390/include/asm/pgtable.h +7 -92

139 lines not on this page · raw message

arch/s390/mm/hugetlbpage.c +0 -24
index e88c02c9e642..2597c1766a62 100644
@@ -136,29 +136,6 @@ static inline pte_t __rste_to_pte(unsigned long rste)
 	return __pte(pteval);
 }
 
-static void clear_huge_pte_skeys(struct mm_struct *mm, unsigned long rste)
-{
-	struct folio *folio;
-	unsigned long size, paddr;
-
-	if (!mm_uses_skeys(mm) ||
-	    rste & _SEGMENT_ENTRY_INVALID)
-		return;
-
-	if ((rste & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) {
-		folio = page_folio(pud_page(__pud(rste)));
-		size = PUD_SIZE;
-		paddr = rste & PUD_MASK;
-	} else {
-		folio = page_folio(pmd_page(__pmd(rste)));
-		size = PMD_SIZE;
-		paddr = rste & PMD_MASK;
-	}
-
-	if (!test_and_set_bit(PG_arch_1, &folio->flags))
-		__storage_key_init_range(paddr, paddr + size);
-}
-
 void __set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
 		     pte_t *ptep, pte_t pte)
 {
@@ -174,7 +151,6 @@ void __set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
 	} else if (likely(pte_present(pte)))
 		rste |= _SEGMENT_ENTRY_LARGE;
 
-	clear_huge_pte_skeys(mm, rste);
 	set_pte(ptep, __pte(rste));
 }
 
arch/s390/mm/pgalloc.c +0 -29
index b449fd2605b0..c33f0d7cc540 100644
@@ -110,30 +110,6 @@ int crst_table_upgrade(struct mm_struct *mm, unsigned long end)
 	return -ENOMEM;
 }
 
-#ifdef CONFIG_PGSTE
-
-struct ptdesc *page_table_alloc_pgste(struct mm_struct *mm)
-{
-	struct ptdesc *ptdesc;
-	u64 *table;
-
-	ptdesc = pagetable_alloc(GFP_KERNEL, 0);
-	if (ptdesc) {
-		table = (u64 *)ptdesc_to_virt(ptdesc);
-		__arch_set_page_dat(table, 1);
-		memset64(table, _PAGE_INVALID, PTRS_PER_PTE);
-		memset64(table + PTRS_PER_PTE, 0, PTRS_PER_PTE);
-	}
-	return ptdesc;
-}
-
-void page_table_free_pgste(struct ptdesc *ptdesc)
-{
-	pagetable_free(ptdesc);
-}
-
-#endif /* CONFIG_PGSTE */
-
 unsigned long *page_table_alloc(struct mm_struct *mm)
 {
 	struct ptdesc *ptdesc;
@@ -173,11 +149,6 @@ void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable)
 	struct ptdesc *ptdesc = virt_to_ptdesc(pgtable);
 
 	call_rcu(&ptdesc->pt_rcu_head, pte_free_now);
-	/*
-	 * THPs are not allowed for KVM guests. Warn if pgste ever reaches here.
-	 * Turn to the generic pte_free_defer() version once gmap is removed.
-	 */
-	WARN_ON_ONCE(mm_has_pgste(mm));
 }
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 
arch/s390/mm/pgtable.c +7 -820

875 lines not on this page · raw message

mm/khugepaged.c +0 -9
index 15203ea7d007..ca73335347fd 100644
@@ -351,15 +351,6 @@ int hugepage_madvise(struct vm_area_struct *vma,
 {
 	switch (advice) {
 	case MADV_HUGEPAGE:
-#ifdef CONFIG_S390
-		/*
-		 * qemu blindly sets MADV_HUGEPAGE on all allocations, but s390
-		 * can't handle this properly after s390_enable_sie, so we simply
-		 * ignore the madvise to prevent qemu from causing a SIGSEGV.
-		 */
-		if (mm_has_pgste(vma->vm_mm))
-			return 0;
-#endif
 		*vm_flags &= ~VM_NOHUGEPAGE;
 		*vm_flags |= VM_HUGEPAGE;
 		/*
Signature · 1 line
2.51.0
#71 lore raw

Re: [PATCH v2 18/20] KVM: S390: Remove PGSTE code from linux/s390 mm

Quoted 16 lines · Claudio Imbrenda

Remove the PGSTE config option.
Remove all code from linux/s390 mm that involves PGSTEs.

---
 arch/s390/Kconfig               |   3 -
 arch/s390/include/asm/mmu.h     |  13 -
 arch/s390/include/asm/page.h    |   4 -
 arch/s390/include/asm/pgalloc.h |   2 -
 arch/s390/include/asm/pgtable.h |  99 +---
 arch/s390/mm/hugetlbpage.c      |  24 -
 arch/s390/mm/pgalloc.c          |  29 --
 arch/s390/mm/pgtable.c          | 827 +-------------------------------
 mm/khugepaged.c                 |   9 -
 9 files changed, 14 insertions(+), 996 deletions(-)

...

Quoted 30 lines

#define INIT_MM_CONTEXT(name) \

arch/s390/include/asm/page.h +0 -3
index 4e5dbabdf202..b4fb4d7adff4 100644
@@ -78,7 +78,6 @@ static inline void copy_page(void *to, void *from)
 #ifdef STRICT_MM_TYPECHECKS
 
 typedef struct { unsigned long pgprot; } pgprot_t;
-typedef struct { unsigned long pgste; } pgste_t;
 typedef struct { unsigned long pte; } pte_t;
 typedef struct { unsigned long pmd; } pmd_t;
 typedef struct { unsigned long pud; } pud_t;
@@ -94,7 +93,6 @@ static __always_inline unsigned long name ## _val(name ## _t name)	\
 #else /* STRICT_MM_TYPECHECKS */
 
 typedef unsigned long pgprot_t;
-typedef unsigned long pgste_t;
 typedef unsigned long pte_t;
 typedef unsigned long pmd_t;
 typedef unsigned long pud_t;
@@ -110,7 +108,6 @@ static __always_inline unsigned long name ## _val(name ## _t name)	\
 #endif /* STRICT_MM_TYPECHECKS */
 
 DEFINE_PGVAL_FUNC(pgprot)
-DEFINE_PGVAL_FUNC(pgste)
 DEFINE_PGVAL_FUNC(pte)
 DEFINE_PGVAL_FUNC(pmd)
 DEFINE_PGVAL_FUNC(pud)

arch/s390/include/asm/page.h DEFINE_PGVAL_FUNC(pgd) · line 122

Quoted 7 lines
arch/s390/include/asm/page.h +0 -1
@@ -120,7 +117,6 @@ DEFINE_PGVAL_FUNC(pgd)
 typedef pte_t *pgtable_t;
 
 #define __pgprot(x)	((pgprot_t) { (x) } )
-#define __pgste(x)	((pgste_t) { (x) } )
 #define __pte(x)        ((pte_t) { (x) } )
 #define __pmd(x)        ((pmd_t) { (x) } )
 #define __pud(x)	((pud_t) { (x) } )

You missed to remove the PGSTE_*_BIT{,S} and _PGSTE_GPS_* definitions in arch/s390/include/asm/pgtable.h
(I found them at line 417.. (based on 6.16 with your patches applied))

Or are they kept on purpose? I could not find any usage after your patches.

Steffen

author maintainer
#72 lore raw

Re: [PATCH v2 18/20] KVM: S390: Remove PGSTE code from linux/s390 mm

Quoted 63 lines · On Tue, 16 Sep 2025 09:30:38 +0200 Steffen Eiden <[email protected]> wrote:
Quoted 16 lines · Claudio Imbrenda

Remove the PGSTE config option.
Remove all code from linux/s390 mm that involves PGSTEs.

---
 arch/s390/Kconfig               |   3 -
 arch/s390/include/asm/mmu.h     |  13 -
 arch/s390/include/asm/page.h    |   4 -
 arch/s390/include/asm/pgalloc.h |   2 -
 arch/s390/include/asm/pgtable.h |  99 +---
 arch/s390/mm/hugetlbpage.c      |  24 -
 arch/s390/mm/pgalloc.c          |  29 --
 arch/s390/mm/pgtable.c          | 827 +-------------------------------
 mm/khugepaged.c                 |   9 -
 9 files changed, 14 insertions(+), 996 deletions(-)

...

Quoted 30 lines

#define INIT_MM_CONTEXT(name) \

arch/s390/include/asm/page.h +0 -3
index 4e5dbabdf202..b4fb4d7adff4 100644
@@ -78,7 +78,6 @@ static inline void copy_page(void *to, void *from)
 #ifdef STRICT_MM_TYPECHECKS
 
 typedef struct { unsigned long pgprot; } pgprot_t;
-typedef struct { unsigned long pgste; } pgste_t;
 typedef struct { unsigned long pte; } pte_t;
 typedef struct { unsigned long pmd; } pmd_t;
 typedef struct { unsigned long pud; } pud_t;
@@ -94,7 +93,6 @@ static __always_inline unsigned long name ## _val(name ## _t name)	\
 #else /* STRICT_MM_TYPECHECKS */
 
 typedef unsigned long pgprot_t;
-typedef unsigned long pgste_t;
 typedef unsigned long pte_t;
 typedef unsigned long pmd_t;
 typedef unsigned long pud_t;
@@ -110,7 +108,6 @@ static __always_inline unsigned long name ## _val(name ## _t name)	\
 #endif /* STRICT_MM_TYPECHECKS */
 
 DEFINE_PGVAL_FUNC(pgprot)
-DEFINE_PGVAL_FUNC(pgste)
 DEFINE_PGVAL_FUNC(pte)
 DEFINE_PGVAL_FUNC(pmd)
 DEFINE_PGVAL_FUNC(pud)

arch/s390/include/asm/page.h DEFINE_PGVAL_FUNC(pgd) · line 122

Quoted 7 lines
arch/s390/include/asm/page.h +0 -1
@@ -120,7 +117,6 @@ DEFINE_PGVAL_FUNC(pgd)
 typedef pte_t *pgtable_t;
 
 #define __pgprot(x)	((pgprot_t) { (x) } )
-#define __pgste(x)	((pgste_t) { (x) } )
 #define __pte(x)        ((pte_t) { (x) } )
 #define __pmd(x)        ((pmd_t) { (x) } )
 #define __pud(x)	((pud_t) { (x) } )  

You missed to remove the PGSTE_*_BIT{,S} and _PGSTE_GPS_* definitions in arch/s390/include/asm/pgtable.h
(I found them at line 417.. (based on 6.16 with your patches applied))

Quoted 3 lines

Or are they kept on purpose? I could not find any usage after your patches.

Steffen

I do use PGSTE_IN_BIT, PGSTE_PCL_BIT and PGSTE_UC_BIT, but I should probably move those to dat.h, and remove all the other unused PGSTE_*_BIT and _PGSTE_GPS_* macros from page.h

author maintainer
#73 lore raw

PATCH v2 19/20 KVM: s390: Enable 1M pages for gmap

In reply to Claudio Imbrenda

While userspace is allowed to have pages of any size, the new gmap would always use 4k pages to back the guest.

Enable 1M pages for gmap.

This allows 1M pages to be used to back a guest when userspace is using 1M pages for the corresponding addresses (e.g. THP or hugetlbfs).

Remove the limitation that disallowed having nested guests and hugepages at the same time.

---
 arch/s390/kvm/gmap.c     | 2 +-
 arch/s390/kvm/kvm-s390.c | 6 +-----
 arch/s390/kvm/pv.c       | 3 +++
 3 files changed, 5 insertions(+), 6 deletions(-)
arch/s390/kvm/gmap.c +1 -1
index 001b069a6c7b..42914de5c791 100644
@@ -599,7 +599,7 @@ static inline bool gmap_2g_allowed(struct gmap *gmap, gfn_t gfn)
 
 static inline bool gmap_1m_allowed(struct gmap *gmap, gfn_t gfn)
 {
-	return false;
+	return gmap->allow_hpage_1m;
 }
 
 int gmap_link(struct gmap *gmap, kvm_pfn_t pfn, struct page *page, gfn_t gfn, bool w, bool d)
arch/s390/kvm/kvm-s390.c +1 -5
index 1d6c15601a20..52b1709f0423 100644
@@ -847,6 +847,7 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
 			r = -EINVAL;
 		else {
 			r = 0;
+			WRITE_ONCE(kvm->arch.gmap->allow_hpage_1m, 1);
 			/*
 			 * We might have to create fake 4k page
 			 * tables. To avoid that the hardware works on
@@ -5903,11 +5904,6 @@ static int __init kvm_s390_init(void)
 		return -ENODEV;
 	}
 
-	if (nested && hpage) {
-		pr_info("A KVM host that supports nesting cannot back its KVM guests with huge pages\n");
-		return -EINVAL;
-	}
-
 	for (i = 0; i < 16; i++)
 		kvm_s390_fac_base[i] |=
 			stfle_fac_list[i] & nonhyp_mask(i);
arch/s390/kvm/pv.c +3 -0
index 7da8aa17458a..1083d18c5dfc 100644
@@ -618,6 +618,9 @@ int kvm_s390_pv_init_vm(struct kvm *kvm, u16 *rc, u16 *rrc)
 	uvcb.flags.ap_allow_instr = kvm->arch.model.uv_feat_guest.ap;
 	uvcb.flags.ap_instr_intr = kvm->arch.model.uv_feat_guest.ap_intr;
 
+	WRITE_ONCE(kvm->arch.gmap->allow_hpage_1m, 0);
+	gmap_split_huge_pages(kvm->arch.gmap);
+
 	cc = uv_call_sched(0, (u64)&uvcb);
 	*rc = uvcb.header.rc;
 	*rrc = uvcb.header.rrc;
Signature · 1 line
2.51.0
author maintainer
#74 lore raw

PATCH v2 20/20 KVM: s390: Storage key manipulation IOCTL

In reply to Claudio Imbrenda

Add a new IOCTL to allow userspace to manipulate storage keys directly.

This will make it easier to write selftests related to storage keys.

---
 arch/s390/kvm/kvm-s390.c | 48 ++++++++++++++++++++++++++++++++++++++++
 include/uapi/linux/kvm.h | 10 +++++++++
 2 files changed, 58 insertions(+)
arch/s390/kvm/kvm-s390.c +48 -0
index 52b1709f0423..daf34b668aa8 100644
@@ -553,6 +553,34 @@ static void __kvm_s390_exit(void)
 	debug_unregister(kvm_s390_dbf_uv);
 }
 
+static int kvm_s390_keyop(struct kvm *kvm, int op, unsigned long addr, union skey skey)
+{
+	union asce asce = kvm->arch.gmap->asce;
+	gfn_t gfn = gpa_to_gfn(addr);
+	int r;
+
+	switch (op) {
+	case KVM_S390_KEYOP_SSKE:
+		r = dat_cond_set_storage_key(asce, gfn, skey, &skey, 0, 0, 0);
+		if (r >= 0)
+			return skey.skey;
+		break;
+	case KVM_S390_KEYOP_ISKE:
+		r = dat_get_storage_key(asce, gfn, &skey);
+		if (!r)
+			return skey.skey;
+		break;
+	case KVM_S390_KEYOP_RRBE:
+		r = dat_reset_reference_bit(asce, gfn);
+		if (r > 0)
+			return r << 1;
+		break;
+	default:
+		return -EINVAL;
+	}
+	return r;
+}
+
 /* Section: device related */
 long kvm_arch_dev_ioctl(struct file *filp,
 			unsigned int ioctl, unsigned long arg)
@@ -2929,6 +2957,26 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
 			r = -EFAULT;
 		break;
 	}
+	case KVM_S390_KEYOP: {
+		struct kvm_s390_keyop kop;
+		union skey skey;
+
+		if (copy_from_user(&kop, argp, sizeof(kop))) {
+			r = -EFAULT;
+			break;
+		}
+		skey.skey = kop.key;
+
+		r = kvm_s390_keyop(kvm, kop.operation, kop.user_addr, skey);
+		if (r < 0)
+			break;
+
+		kop.key = r;
+		r = 0;
+		if (copy_to_user(argp, &kop, sizeof(kop)))
+			r = -EFAULT;
+		break;
+	}
 	case KVM_S390_ZPCI_OP: {
 		struct kvm_s390_zpci_op args;
 
include/uapi/linux/kvm.h +10 -0
index 7a4c35ff03fe..ac4b72924934 100644
@@ -1205,6 +1205,15 @@ struct kvm_vfio_spapr_tce {
 	__s32	tablefd;
 };
 
+#define KVM_S390_KEYOP_SSKE 0x01
+#define KVM_S390_KEYOP_ISKE 0x02
+#define KVM_S390_KEYOP_RRBE 0x03
+struct kvm_s390_keyop {
+	__u64 user_addr;
+	__u8  key;
+	__u8  operation;
+};
+
 /*
  * KVM_CREATE_VCPU receives as a parameter the vcpu slot, and returns
  * a vcpu fd.
@@ -1224,6 +1233,7 @@ struct kvm_vfio_spapr_tce {
 #define KVM_S390_UCAS_MAP        _IOW(KVMIO, 0x50, struct kvm_s390_ucas_mapping)
 #define KVM_S390_UCAS_UNMAP      _IOW(KVMIO, 0x51, struct kvm_s390_ucas_mapping)
 #define KVM_S390_VCPU_FAULT	 _IOW(KVMIO, 0x52, unsigned long)
+#define KVM_S390_KEYOP           _IOWR(KVMIO, 0x53, struct kvm_s390_keyop)
 
 /* Device model IOC */
 #define KVM_CREATE_IRQCHIP        _IO(KVMIO,   0x60)
Signature · 1 line
2.51.0