Skip to contentSkip to conversation

mm: init_mlocked_on_free_v3

PATCH In review York Jasper Niebuhr 2 participants 2 messages Mar 29, 2024

Subsystems (7)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 mm: init_mlocked_on_free_v3

Implements the "init_mlocked_on_free" boot option. When this boot option is enabled, any mlock'ed pages are zeroed on free. If
the pages are munlock'ed beforehand, no initialization takes place. This boot option is meant to combat the performance hit of "init_on_free" as reported in commit 6471384af2a6 ("mm: security:
introduce init_on_alloc=1 and init_on_free=1 boot options"). With "init_mlocked_on_free=1" only relevant data is freed while everything else is left untouched by the kernel. Correspondingly, this patch introduces no performance hit for unmapping non-mlock'ed memory. The unmapping overhead for purely mlocked memory was measured to be approximately 13%. Realistically, most systems mlock only a fraction of the total memory so the real-world system overhead should be close to zero.

Optimally, userspace programs clear any key material or other confidential memory before exit and munlock the according memory regions. If a program crashes, userspace key managers fail to do this job. Accordingly, no munlock operations are performed so the data is caught and zeroed by the kernel. Should the program not crash, all memory will ideally be munlocked so no overhead is caused.

CONFIG_INIT_MLOCKED_ON_FREE_DEFAULT_ON can be set to enable "init_mlocked_on_free" by default.

---
 .../admin-guide/kernel-parameters.txt         |  6 +++
 include/linux/mm.h                            |  9 +++-
 mm/internal.h                                 |  1 +
 mm/memory.c                                   |  6 +++
 mm/mm_init.c                                  | 43 ++++++++++++++++---
 mm/page_alloc.c                               |  2 +-
 security/Kconfig.hardening                    | 15 +++++++
 7 files changed, 73 insertions(+), 9 deletions(-)
Documentation/admin-guide/kernel-parameters.txt +6 -0
index bb884c14b2f6..34bdbf29aaf9 100644
@@ -2148,6 +2148,12 @@
 			Format: 0 | 1
 			Default set by CONFIG_INIT_ON_FREE_DEFAULT_ON.
 
+	init_mlocked_on_free=	[MM] Fill freed userspace memory with zeroes if
+				it was mlock'ed and not explicitly munlock'ed
+				afterwards.
+				Format: 0 | 1
+				Default set by CONFIG_INIT_MLOCKED_ON_FREE_DEFAULT_ON
+
 	init_pkru=	[X86] Specify the default memory protection keys rights
 			register contents for all processes.  0x55555554 by
 			default (disallow access to all but pkey 0).  Can
include/linux/mm.h +8 -1
index 8eccaabe44ee..a216489aef2b 100644
@@ -3751,7 +3751,14 @@ DECLARE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free);
 static inline bool want_init_on_free(void)
 {
 	return static_branch_maybe(CONFIG_INIT_ON_FREE_DEFAULT_ON,
-				   &init_on_free);
+				&init_on_free);
+}
+
+DECLARE_STATIC_KEY_MAYBE(CONFIG_INIT_MLOCKED_ON_FREE_DEFAULT_ON, init_mlocked_on_free);
+static inline bool want_init_mlocked_on_free(void)
+{
+	return static_branch_maybe(CONFIG_INIT_MLOCKED_ON_FREE_DEFAULT_ON,
+				&init_mlocked_on_free);
 }
 
 extern bool _debug_pagealloc_enabled_early;
mm/internal.h +1 -0
index 8e11f7b2da21..8ec85b5e62e6 100644
@@ -506,6 +506,7 @@ extern void __putback_isolated_page(struct page *page, unsigned int order,
 extern void memblock_free_pages(struct page *page, unsigned long pfn,
 					unsigned int order);
 extern void __free_pages_core(struct page *page, unsigned int order);
+extern void kernel_init_pages(struct page *page, int numpages);
 
 /*
  * This will have no effect, other than possibly generating a warning, if the
mm/memory.c +6 -0
index 36191a9c799c..861d93c3e444 100644
@@ -1506,6 +1506,12 @@ static __always_inline void zap_present_folio_ptes(struct mmu_gather *tlb,
 		if (unlikely(page_mapcount(page) < 0))
 			print_bad_pte(vma, addr, ptent, page);
 	}
+
+	if (want_init_mlocked_on_free() && folio_test_mlocked(folio)
+		&& !delay_rmap && folio_test_anon(folio)) {
+		kernel_init_pages(page, 1);
+	}
+
 	if (unlikely(__tlb_remove_folio_pages(tlb, page, nr, delay_rmap))) {
 		*force_flush = true;
 		*force_break = true;
mm/mm_init.c +36 -7

78 lines not on this page · raw message

mm/page_alloc.c +1 -1
index 5b8ab1bfb9d5..8a8ca555b9d6 100644
@@ -1038,7 +1038,7 @@ static inline bool should_skip_kasan_poison(struct page *page)
 	return page_kasan_tag(page) == KASAN_TAG_KERNEL;
 }
 
-static void kernel_init_pages(struct page *page, int numpages)
+void kernel_init_pages(struct page *page, int numpages)
 {
 	int i;
 
security/Kconfig.hardening +15 -0
index 2cff851ebfd7..effbf5982be1 100644
@@ -255,6 +255,21 @@ config INIT_ON_FREE_DEFAULT_ON
 	  touching "cold" memory areas. Most cases see 3-5% impact. Some
 	  synthetic workloads have measured as high as 8%.
 
+config INIT_MLOCKED_ON_FREE_DEFAULT_ON
+	bool "Enable mlocked memory zeroing on free"
+	depends on !KMSAN
+	help
+	  This config has the effect of setting "init_mlocked_on_free=1"
+	  on the kernel command line. If it is enabled, all mlocked process
+	  memory is zeroed when freed. This restriction to mlocked memory
+	  improves performance over "init_on_free" but can still be used to
+	  protect confidential data like key material from content exposures
+	  to other processes, as well as live forensics and cold boot attacks.
+	  Any non-mlocked memory is not cleared before it is reassigned. This
+	  configuration can be overwritten by setting "init_mlocked_on_free=0"
+	  on the command line. The "init_on_free" boot option takes
+	  precedence over "init_mlocked_on_free".
+
 config CC_HAS_ZERO_CALL_USED_REGS
 	def_bool $(cc-option,-fzero-call-used-regs=used-gpr)
 	# https://github.com/ClangBuiltLinux/linux/issues/1766

base-commit: a6bd6c9333397f5a0e2667d4d82fef8c970108f2

Signature · 1 line
2.34.1
#1 lore raw

mm/memory.c static __always_inline void zap_present_folio_ptes(struct mmu_gather *tlb, · line 1512

Quoted 3 lines · York Jasper Niebuhr
mm/memory.c +3 -0
+	if (want_init_mlocked_on_free() && folio_test_mlocked(folio)
+		&& !delay_rmap && folio_test_anon(folio)) {
+		kernel_init_pages(page, 1);

Confusing indentation. One of these two options:

	if (want_init_mlocked_on_free() && folio_test_mlocked(folio) &&
	    !delay_rmap && folio_test_anon(folio)) {
		kernel_init_pages(page, 1);

	if (want_init_mlocked_on_free() && folio_test_mlocked(folio) &&
			!delay_rmap && folio_test_anon(folio)) {
		kernel_init_pages(page, 1);

Also, '1' is incorrect. Should be folio_nr_pages(folio).

mm/mm_init.c static void __init mem_debugging_and_hardening_init(void) · line 2575

Quoted 8 lines
mm/mm_init.c +2 -1
@@ -2559,12 +2570,21 @@ static void __init mem_debugging_and_hardening_init(void)
 	}
 #endif
 
-	if ((_init_on_alloc_enabled_early || _init_on_free_enabled_early) &&
+	if ((_init_on_alloc_enabled_early || _init_on_free_enabled_early
+		|| _init_mlocked_on_free_enabled_early) &&
 	    page_poisoning_requested) {

Wrong indentation again. I'm not going to point these out every time, just fix it throughout.