powerpc/book3s64/radix: Add support for memfd_secret

memfd_secret() gives a process a place to keep a secret that the kernel itself should not be able to read. The page is still mapped in that process but it is removed from the kernel linear map, so a kernel read or a speculative walk of that map cannot see it.

On PowerPC book3s64, if the same address is translated by two pages of different sizes at once, the TLB access can raise a machine check for a parity error or a multi-hit. So that means breaking a 2M or a 1G mapping of kernel linear map is not possible.

The way around that is to build the linear map out of base pages in the first place and only then allow a single page to be dropped and put back. So, that is what this series does by adding a linear_map_use_base_page cmdline option.

This is based out of mm-new branch for now (since few dependent function definitions have been modified there). Once v7.4 is merge window closes - I will rebase this to latest upstream powerpc tree.

Ritesh Harjani (IBM) (4):
powerpc/64s: Add linear_map_use_base_page command line option powerpc/64s/radix: Implement ARCH_HAS_SET_DIRECT_MAP
powerpc: Wire memfd_secret syscall
selftests: mm: Enable memfd_secret for powerpc

 .../admin-guide/kernel-parameters.txt         | 10 ++
 arch/powerpc/Kconfig                          |  1 +
 arch/powerpc/include/asm/book3s/64/mmu.h      |  2 +
 arch/powerpc/include/asm/set_memory.h         |  9 ++
 arch/powerpc/kernel/syscalls/syscall.tbl      |  2 +-
 arch/powerpc/mm/book3s64/pgtable.c            | 22 +++++
 arch/powerpc/mm/book3s64/radix_pgtable.c      |  2 +-
 arch/powerpc/mm/pageattr.c                    | 96 +++++++++++++++++++
 tools/testing/selftests/mm/Makefile           |  2 +-
 9 files changed, 143 insertions(+), 3 deletions(-)

--
2.39.5

The radix linear map is built with 1G and 2M mappings wherever it can. That is the right default for TLB reach, but it also means a single page cannot be pulled out of the linear map later. There is no support for splitting a live leaf, so memfd_secret() has nothing to work with.

debug_pagealloc=on already forces PAGE_SIZE mappings, but that turns on a lot of allocator debugging that nobody wants just for secretmem. Add linear_map_use_base_page so the linear map can be built out of base pages on its own. debug_pagealloc=on still implies it.

The extra page tables are roughly 3% of RAM (for 64K pagesize) and it can also cause extra TLB pressure, so leave the option off by default.

Only radix honours this for now (later we will implement this for Hash as well).

---
 .../admin-guide/kernel-parameters.txt         | 10 +++++++++
 arch/powerpc/include/asm/book3s/64/mmu.h      |  2 ++
 arch/powerpc/mm/book3s64/pgtable.c            | 22 +++++++++++++++++++
 arch/powerpc/mm/book3s64/radix_pgtable.c      |  2 +-
 4 files changed, 35 insertions(+), 1 deletion(-)

index 33cd30996e47..382d18c6e8e8 100644
@@ -3563,6 +3563,16 @@ Kernel parameters
 			If there are multiple matching configurations changing
 			the same attribute, the last one is used.
 
+	linear_map_use_base_page[=<bool>]
+			[PPC(Radix), EARLY]
+			Build the kernel linear mapping out of PAGE_SIZE
+			mappings only, instead of using 2M/1G mappings where
+			possible. This allows individual pages to be removed
+			from the linear mapping at runtime, which is required
+			by memfd_secret(2). It costs roughly 3% extra memory for
+			page tables and increases kernel TLB pressure, so it is
+			off by default. debug_pagealloc=on implies this.
+
 	liveupdate=	[KNL,EARLY]
 			Format: <bool>
 			Enable Live Update Orchestrator (LUO).

index 48631365b48c..3739e5677312 100644
@@ -73,6 +73,8 @@ extern unsigned int mmu_base_pid;
 
 extern unsigned long __ro_after_init memory_block_size;
 
+bool linear_map_uses_base_pagesize(void);
+
 #define PRTB_SIZE_SHIFT	(mmu_pid_bits + 4)
 #define PRTB_ENTRIES	(1ul << mmu_pid_bits)
 

index 85ab6723c8f2..1a007902f307 100644
@@ -48,6 +48,28 @@ static int __init parse_kfence_early_init(char *arg)
 early_param("kfence.sample_interval", parse_kfence_early_init);
 #endif
 
+static bool __ro_after_init linear_map_use_base_page;
+
+static int __init parse_linear_map_use_base_page(char *p)
+{
+	bool val;
+
+	if (!p)
+		val = true;
+	else if (kstrtobool(p, &val))
+		return -EINVAL;
+
+	linear_map_use_base_page = val;
+
+	return 0;
+}
+early_param("linear_map_use_base_page", parse_linear_map_use_base_page);
+
+bool linear_map_uses_base_pagesize(void)
+{
+	return debug_pagealloc_enabled() || linear_map_use_base_page;
+}
+
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 /*
  * This is called when relaxing access to a hugepage. It's also called in the page

index cf692b2b5f7b..2c4cb6d6ab6e 100644
@@ -308,7 +308,7 @@ static int __meminit create_physical_mapping(unsigned long start,
 	if (mapping_sz_limit < max_mapping_size)
 		max_mapping_size = mapping_sz_limit;
 
-	if (debug_pagealloc_enabled())
+	if (linear_map_uses_base_pagesize())
 		max_mapping_size = PAGE_SIZE;
 
 	start = ALIGN(start, PAGE_SIZE);

2.39.5

memfd_secret() keeps a secret by taking its page out of the kernel linear map. The process that owns the page can still read and write it through its own mapping, but the kernel no longer has a translation for that physical page, so a kernel read or a speculative walk of the linear map cannot see it.

Note that set_memory_np() and set_memory_p() already clear and restore the present bit for debug_pagealloc and KFENCE, so the same direct-map helpers are used here to implement ARCH_HAS_SET_DIRECT_MAP for radix.

Note that they can only run when the linear map is already made of base pages. For implementing this feature when the linear map is mapped with 2M (PMD) or 1G (PUD) mapping - a 2M or 1G leaf would have to be broken into PTEs first and doing that while the large translation is still in the TLB installs a second translation for the same address at a different page size.

On this, the processor architecture manual says that it is fatal:
If a given virtual address is mapped by two overlapping pages of different sizes (for example, a small page is mapped initially but is later re-mapped into a larger page without invalidating the original TLB entry), a machine check interrupt can result with an indication that either a parity error or a multi-hit occurred when the TLB was accessed to translate the address. The error condition can be corrected by invalidating the entire TLB and SLB.

The same multi-hit applies if the large page was installed first and is then replaced by PTEs without invalidating that TLB entry. Since today there is no support for doing that split safely, so can_set_direct_map() stays false unless the linear map is already page-sized.

---
 arch/powerpc/Kconfig                  |  1 +
 arch/powerpc/include/asm/set_memory.h |  9 +++
 arch/powerpc/mm/pageattr.c            | 96 +++++++++++++++++++++++++++
 3 files changed, 106 insertions(+)

index 2580e27e4328..3c5c07c67a5b 100644
@@ -150,6 +150,7 @@ config PPC
 	select ARCH_HAS_PTDUMP
 	select ARCH_HAS_PTE_SPECIAL
 	select ARCH_HAS_SCALED_CPUTIME		if VIRT_CPU_ACCOUNTING_NATIVE && PPC_BOOK3S_64
+	select ARCH_HAS_SET_DIRECT_MAP		if PPC_RADIX_MMU
 	select ARCH_HAS_SET_MEMORY
 	select ARCH_HAS_STRICT_KERNEL_RWX	if (PPC_BOOK3S || PPC_8xx) && !HIBERNATION
 	select ARCH_HAS_STRICT_KERNEL_RWX	if PPC_85xx && !HIBERNATION && !RANDOMIZE_BASE

index 9c8d5747755d..24779c00c1ca 100644
@@ -48,4 +48,13 @@ static inline int __must_check set_memory_rox(unsigned long addr, int numpages)
 }
 #define set_memory_rox set_memory_rox
 
+#ifdef CONFIG_ARCH_HAS_SET_DIRECT_MAP
+bool can_set_direct_map(void);
+#define can_set_direct_map can_set_direct_map
+
+int set_direct_map_invalid_noflush(struct page *page, unsigned int numpages);
+int set_direct_map_default_noflush(struct page *page, unsigned int numpages);
+bool kernel_page_present(struct page *page);
+#endif
+
 #endif

--
2.39.5

Wire the memfd_secret syscall. Currently this is only implemented for Book3S64 radix. On every other PowerPC configuration, either ARCH_HAS_SET_DIRECT_MAP is not enabled in Kconfig, or can_set_direct_map() returns false because the configuration is not supported. The syscall checks for that and returns -ENOSYS.

---
 arch/powerpc/kernel/syscalls/syscall.tbl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

index cfbb70039ff0..81677fdd0d05 100644
@@ -537,7 +537,7 @@
 444	common	landlock_create_ruleset		sys_landlock_create_ruleset
 445	common	landlock_add_rule		sys_landlock_add_rule
 446	common	landlock_restrict_self		sys_landlock_restrict_self
-# 447 reserved for memfd_secret
+447	common	memfd_secret			sys_memfd_secret
 448	common	process_mrelease		sys_process_mrelease
 449	common  futex_waitv                     sys_futex_waitv
 450 	nospu	set_mempolicy_home_node		sys_set_mempolicy_home_node

2.39.5

Now that we have the needed support on PowerPC Book3s64 for memfd_secret, enable the necessary selftests for the same.

---
 tools/testing/selftests/mm/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

index 2d5366196e30..3ada7f5f4151 100644
@@ -70,7 +70,7 @@ TEST_GEN_FILES += khugepaged
 TEST_GEN_FILES += madv_populate
 TEST_GEN_FILES += map_fixed_noreplace
 TEST_GEN_FILES += map_populate
-ifneq (,$(filter $(ARCH),arm64 riscv riscv64 x86 x86_64 loongarch32 loongarch64))
+ifneq (,$(filter $(ARCH),arm64 riscv riscv64 x86 x86_64 loongarch32 loongarch64 powerpc))
 TEST_GEN_FILES += memfd_secret
 endif
 TEST_GEN_FILES += memory-failure

--
2.39.5