arm64: mm: Fix CFI failure due to kpti_ng_pgd_alloc function signature

PATCH Superseded Kees Cook 5 participants 6 messages Aug 29, 2025 – Sep 1, 2025 arm64

lore

Share & tools

lore

Kees Cook

author

2025-08-29 15:49 UTC #0 lore raw

PATCH 1/1 arm64: mm: Fix CFI failure due to kpti_ng_pgd_alloc function signature

Seen during KPTI initialization:

CFI failure at create_kpti_ng_temp_pgd+0x124/0xce8 (target: kpti_ng_pgd_alloc+0x0/0x14; expected type: 0xd61b88b6)

The call site is alloc_init_pud() at arch/arm64/mm/mmu.c:

  pud_phys = pgtable_alloc(TABLE_PUD);

alloc_init_pud() has the prototype:

static void alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end,
phys_addr_t phys, pgprot_t prot, phys_addr_t (*pgtable_alloc)(enum pgtable_type), int flags)

where the pgtable_alloc() prototype is declared.

The target (kpti_ng_pgd_alloc) is used in arch/arm64/kernel/cpufeature.c:

create_kpti_ng_temp_pgd(kpti_ng_temp_pgd, __pa(alloc), KPTI_NG_TEMP_VA,

                          PAGE_SIZE, PAGE_KERNEL, kpti_ng_pgd_alloc, 0);

which is an alias for __create_pgd_mapping_locked() with prototype:

extern __alias(__create_pgd_mapping_locked)
void create_kpti_ng_temp_pgd(pgd_t *pgdir, phys_addr_t phys,
unsigned long virt,
phys_addr_t size, pgprot_t prot, phys_addr_t (*pgtable_alloc)(enum pgtable_type), int flags);

__create_pgd_mapping_locked() passes the function pointer down:

__create_pgd_mapping_locked() -> alloc_init_p4d() -> alloc_init_pud()

But the target function (kpti_ng_pgd_alloc) has the wrong signature:

  static phys_addr_t __init kpti_ng_pgd_alloc(int shift);

The "int" should be "enum pgtable_type".

To make "enum pgtable_type" available to cpufeature.c, move enum pgtable_type definition from arch/arm64/mm/mmu.c to arch/arm64/include/asm/mmu.h.

Adjust kpti_ng_pgd_alloc to use "enum pgtable_type" instead of "int". The function behavior remains identical (parameter is unused).

Fixes 47546a1912fc ("arm64: mm: install KPTI nG mappings with MMU enabled")

Signed-off-by Kees Cook <[email protected]>

---
Cc: Ard Biesheuvel <[email protected]>
Cc: Catalin Marinas <[email protected]>
Cc: Will Deacon <[email protected]>
Cc: Oliver Upton <[email protected]>
Cc: Anshuman Khandual <[email protected]>
Cc: Yue Haibing <[email protected]>
Cc: Mark Rutland <[email protected]>
Cc: Marc Zyngier <[email protected]>
Cc: Mark Brown <[email protected]>
Cc: <[email protected]>

---
 arch/arm64/include/asm/mmu.h   | 7 +++++++
 arch/arm64/kernel/cpufeature.c | 5 +++--
 arch/arm64/mm/mmu.c            | 7 -------
 3 files changed, 10 insertions(+), 9 deletions(-)

arch/arm64/include/asm/mmu.h +7 -0

index 6e8aa8e72601..49f1a810df16 100644
@@ -17,6 +17,13 @@
 #include <linux/refcount.h>
 #include <asm/cpufeature.h>
 
+enum pgtable_type {
+	TABLE_PTE,
+	TABLE_PMD,
+	TABLE_PUD,
+	TABLE_P4D,
+};
+
 typedef struct {
 	atomic64_t	id;
 #ifdef CONFIG_COMPAT

arch/arm64/kernel/cpufeature.c +3 -2

index 9ad065f15f1d..e49d142a281f 100644
@@ -84,6 +84,7 @@
 #include <asm/hwcap.h>
 #include <asm/insn.h>
 #include <asm/kvm_host.h>
+#include <asm/mmu.h>
 #include <asm/mmu_context.h>
 #include <asm/mte.h>
 #include <asm/hypervisor.h>

@@ -1945,11 +1946,11 @@ static bool has_pmuv3(const struct arm64_cpu_capabilities *entry, int scope)
 extern
 void create_kpti_ng_temp_pgd(pgd_t *pgdir, phys_addr_t phys, unsigned long virt,
 			     phys_addr_t size, pgprot_t prot,
-			     phys_addr_t (*pgtable_alloc)(int), int flags);
+			     phys_addr_t (*pgtable_alloc)(enum pgtable_type), int flags);
 
 static phys_addr_t __initdata kpti_ng_temp_alloc;
 
-static phys_addr_t __init kpti_ng_pgd_alloc(int shift)
+static phys_addr_t __init kpti_ng_pgd_alloc(enum pgtable_type type)
 {
 	kpti_ng_temp_alloc -= PAGE_SIZE;
 	return kpti_ng_temp_alloc;

arch/arm64/mm/mmu.c +0 -7

index 34e5d78af076..183801520740 100644
@@ -47,13 +47,6 @@
 #define NO_CONT_MAPPINGS	BIT(1)
 #define NO_EXEC_MAPPINGS	BIT(2)	/* assumes FEAT_HPDS is not used */
 
-enum pgtable_type {
-	TABLE_PTE,
-	TABLE_PMD,
-	TABLE_PUD,
-	TABLE_P4D,
-};
-
 u64 kimage_voffset __ro_after_init;
 EXPORT_SYMBOL(kimage_voffset);

Signature · 1 line

2.34.1

Ard Biesheuvel

Acked-by on v1 1/1

2025-08-29 15:59 UTC #1 lore raw

Quoted 42 lines · Kees Cook

Seen during KPTI initialization:
CFI failure at create_kpti_ng_temp_pgd+0x124/0xce8 (target: kpti_ng_pgd_alloc+0x0/0x14; expected type: 0xd61b88b6)
The call site is alloc_init_pud() at arch/arm64/mm/mmu.c:
  pud_phys = pgtable_alloc(TABLE_PUD);
alloc_init_pud() has the prototype:
static void alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end,
phys_addr_t phys, pgprot_t prot, phys_addr_t (*pgtable_alloc)(enum pgtable_type), int flags)
where the pgtable_alloc() prototype is declared.
The target (kpti_ng_pgd_alloc) is used in arch/arm64/kernel/cpufeature.c:
create_kpti_ng_temp_pgd(kpti_ng_temp_pgd, __pa(alloc), KPTI_NG_TEMP_VA,
                          PAGE_SIZE, PAGE_KERNEL, kpti_ng_pgd_alloc, 0);
which is an alias for __create_pgd_mapping_locked() with prototype:
extern __alias(__create_pgd_mapping_locked)
void create_kpti_ng_temp_pgd(pgd_t *pgdir, phys_addr_t phys,
unsigned long virt,
phys_addr_t size, pgprot_t prot, phys_addr_t (*pgtable_alloc)(enum pgtable_type), int flags);
__create_pgd_mapping_locked() passes the function pointer down:
__create_pgd_mapping_locked() -> alloc_init_p4d() -> alloc_init_pud()
But the target function (kpti_ng_pgd_alloc) has the wrong signature:
  static phys_addr_t __init kpti_ng_pgd_alloc(int shift);
The "int" should be "enum pgtable_type".

Quoted 8 lines

To make "enum pgtable_type" available to cpufeature.c, move enum pgtable_type definition from arch/arm64/mm/mmu.c to arch/arm64/include/asm/mmu.h.
Adjust kpti_ng_pgd_alloc to use "enum pgtable_type" instead of "int". The function behavior remains identical (parameter is unused).
Fixes 47546a1912fc ("arm64: mm: install KPTI nG mappings with MMU enabled")

Shouldn't this be

Fixes c64f46ee1377 ("arm64: mm: use enum to identify pgtable level

instead of *_SHIFT")

Quoted 1 line

Signed-off-by Kees Cook <[email protected]>

Acked-by Ard Biesheuvel <[email protected]>

Quoted 82 lines

---
 arch/arm64/include/asm/mmu.h   | 7 +++++++
 arch/arm64/kernel/cpufeature.c | 5 +++--
 arch/arm64/mm/mmu.c            | 7 -------
 3 files changed, 10 insertions(+), 9 deletions(-)

arch/arm64/include/asm/mmu.h +7 -0

index 6e8aa8e72601..49f1a810df16 100644
@@ -17,6 +17,13 @@
 #include <linux/refcount.h>
 #include <asm/cpufeature.h>
 
+enum pgtable_type {
+       TABLE_PTE,
+       TABLE_PMD,
+       TABLE_PUD,
+       TABLE_P4D,
+};
+
 typedef struct {
        atomic64_t      id;
 #ifdef CONFIG_COMPAT

arch/arm64/kernel/cpufeature.c +3 -2

index 9ad065f15f1d..e49d142a281f 100644
@@ -84,6 +84,7 @@
 #include <asm/hwcap.h>
 #include <asm/insn.h>
 #include <asm/kvm_host.h>
+#include <asm/mmu.h>
 #include <asm/mmu_context.h>
 #include <asm/mte.h>
 #include <asm/hypervisor.h>

@@ -1945,11 +1946,11 @@ static bool has_pmuv3(const struct arm64_cpu_capabilities *entry, int scope)
 extern
 void create_kpti_ng_temp_pgd(pgd_t *pgdir, phys_addr_t phys, unsigned long virt,
                             phys_addr_t size, pgprot_t prot,
-                            phys_addr_t (*pgtable_alloc)(int), int flags);
+                            phys_addr_t (*pgtable_alloc)(enum pgtable_type), int flags);
 
 static phys_addr_t __initdata kpti_ng_temp_alloc;
 
-static phys_addr_t __init kpti_ng_pgd_alloc(int shift)
+static phys_addr_t __init kpti_ng_pgd_alloc(enum pgtable_type type)
 {
        kpti_ng_temp_alloc -= PAGE_SIZE;
        return kpti_ng_temp_alloc;

arch/arm64/mm/mmu.c +0 -8

index 34e5d78af076..183801520740 100644
@@ -47,13 +47,6 @@
 #define NO_CONT_MAPPINGS       BIT(1)
 #define NO_EXEC_MAPPINGS       BIT(2)  /* assumes FEAT_HPDS is not used */
 
-enum pgtable_type {
-       TABLE_PTE,
-       TABLE_PMD,
-       TABLE_PUD,
-       TABLE_P4D,
-};
-
 u64 kimage_voffset __ro_after_init;
 EXPORT_SYMBOL(kimage_voffset);
 
--

2.34.1

Ryan Roberts

Reviewed-by

2025-08-29 16:06 UTC #2 lore raw

+ Kevin Broadsky.

Quoted 50 lines · Ard Biesheuvel

Quoted 49 lines · Kees Cook
Seen during KPTI initialization:
CFI failure at create_kpti_ng_temp_pgd+0x124/0xce8 (target: kpti_ng_pgd_alloc+0x0/0x14; expected type: 0xd61b88b6)
The call site is alloc_init_pud() at arch/arm64/mm/mmu.c:
  pud_phys = pgtable_alloc(TABLE_PUD);
alloc_init_pud() has the prototype:
static void alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end,
phys_addr_t phys, pgprot_t prot, phys_addr_t (*pgtable_alloc)(enum pgtable_type), int flags)
where the pgtable_alloc() prototype is declared.
The target (kpti_ng_pgd_alloc) is used in arch/arm64/kernel/cpufeature.c:
create_kpti_ng_temp_pgd(kpti_ng_temp_pgd, __pa(alloc), KPTI_NG_TEMP_VA,
                          PAGE_SIZE, PAGE_KERNEL, kpti_ng_pgd_alloc, 0);
which is an alias for __create_pgd_mapping_locked() with prototype:
extern __alias(__create_pgd_mapping_locked)
void create_kpti_ng_temp_pgd(pgd_t *pgdir, phys_addr_t phys,
unsigned long virt,
phys_addr_t size, pgprot_t prot, phys_addr_t (*pgtable_alloc)(enum pgtable_type), int flags);
__create_pgd_mapping_locked() passes the function pointer down:
__create_pgd_mapping_locked() -> alloc_init_p4d() -> alloc_init_pud()
But the target function (kpti_ng_pgd_alloc) has the wrong signature:
  static phys_addr_t __init kpti_ng_pgd_alloc(int shift);
The "int" should be "enum pgtable_type".
To make "enum pgtable_type" available to cpufeature.c, move enum pgtable_type definition from arch/arm64/mm/mmu.c to arch/arm64/include/asm/mmu.h.
Adjust kpti_ng_pgd_alloc to use "enum pgtable_type" instead of "int". The function behavior remains identical (parameter is unused).

Quoted 6 lines

Quoted 1 line
Fixes 47546a1912fc ("arm64: mm: install KPTI nG mappings with MMU enabled")
Shouldn't this be
Fixes c64f46ee1377 ("arm64: mm: use enum to identify pgtable level
instead of *_SHIFT")

You beat me to it; agreed. I've added Kevin, who made that change incase there are any other subtlety.

Quoted 5 lines

Quoted 3 lines
Signed-off-by Kees Cook <[email protected]>
Acked-by Ard Biesheuvel <[email protected]>

Reviewed-by Ryan Roberts <[email protected]>

Quoted 84 lines

---
 arch/arm64/include/asm/mmu.h   | 7 +++++++
 arch/arm64/kernel/cpufeature.c | 5 +++--
 arch/arm64/mm/mmu.c            | 7 -------
 3 files changed, 10 insertions(+), 9 deletions(-)

arch/arm64/include/asm/mmu.h +7 -0

index 6e8aa8e72601..49f1a810df16 100644
@@ -17,6 +17,13 @@
 #include <linux/refcount.h>
 #include <asm/cpufeature.h>
 
+enum pgtable_type {
+       TABLE_PTE,
+       TABLE_PMD,
+       TABLE_PUD,
+       TABLE_P4D,
+};
+
 typedef struct {
        atomic64_t      id;
 #ifdef CONFIG_COMPAT

arch/arm64/kernel/cpufeature.c +3 -2

index 9ad065f15f1d..e49d142a281f 100644
@@ -84,6 +84,7 @@
 #include <asm/hwcap.h>
 #include <asm/insn.h>
 #include <asm/kvm_host.h>
+#include <asm/mmu.h>
 #include <asm/mmu_context.h>
 #include <asm/mte.h>
 #include <asm/hypervisor.h>

@@ -1945,11 +1946,11 @@ static bool has_pmuv3(const struct arm64_cpu_capabilities *entry, int scope)
 extern
 void create_kpti_ng_temp_pgd(pgd_t *pgdir, phys_addr_t phys, unsigned long virt,
                             phys_addr_t size, pgprot_t prot,
-                            phys_addr_t (*pgtable_alloc)(int), int flags);
+                            phys_addr_t (*pgtable_alloc)(enum pgtable_type), int flags);
 
 static phys_addr_t __initdata kpti_ng_temp_alloc;
 
-static phys_addr_t __init kpti_ng_pgd_alloc(int shift)
+static phys_addr_t __init kpti_ng_pgd_alloc(enum pgtable_type type)
 {
        kpti_ng_temp_alloc -= PAGE_SIZE;
        return kpti_ng_temp_alloc;

arch/arm64/mm/mmu.c +0 -8

index 34e5d78af076..183801520740 100644
@@ -47,13 +47,6 @@
 #define NO_CONT_MAPPINGS       BIT(1)
 #define NO_EXEC_MAPPINGS       BIT(2)  /* assumes FEAT_HPDS is not used */
 
-enum pgtable_type {
-       TABLE_PTE,
-       TABLE_PMD,
-       TABLE_PUD,
-       TABLE_P4D,
-};
-
 u64 kimage_voffset __ro_after_init;
 EXPORT_SYMBOL(kimage_voffset);
 
--

2.34.1

Mark Rutland

reviewer

2025-08-29 16:06 UTC #3 lore raw

In reply to Kees Cook

Quoted 41 lines · Kees Cook

Seen during KPTI initialization:
CFI failure at create_kpti_ng_temp_pgd+0x124/0xce8 (target: kpti_ng_pgd_alloc+0x0/0x14; expected type: 0xd61b88b6)
The call site is alloc_init_pud() at arch/arm64/mm/mmu.c:
  pud_phys = pgtable_alloc(TABLE_PUD);
alloc_init_pud() has the prototype:
static void alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end,
phys_addr_t phys, pgprot_t prot, phys_addr_t (*pgtable_alloc)(enum pgtable_type), int flags)
where the pgtable_alloc() prototype is declared.
The target (kpti_ng_pgd_alloc) is used in arch/arm64/kernel/cpufeature.c:
create_kpti_ng_temp_pgd(kpti_ng_temp_pgd, __pa(alloc), KPTI_NG_TEMP_VA,
                          PAGE_SIZE, PAGE_KERNEL, kpti_ng_pgd_alloc, 0);
which is an alias for __create_pgd_mapping_locked() with prototype:
extern __alias(__create_pgd_mapping_locked)
void create_kpti_ng_temp_pgd(pgd_t *pgdir, phys_addr_t phys,
unsigned long virt,
phys_addr_t size, pgprot_t prot, phys_addr_t (*pgtable_alloc)(enum pgtable_type), int flags);
__create_pgd_mapping_locked() passes the function pointer down:
__create_pgd_mapping_locked() -> alloc_init_p4d() -> alloc_init_pud()
But the target function (kpti_ng_pgd_alloc) has the wrong signature:
  static phys_addr_t __init kpti_ng_pgd_alloc(int shift);
The "int" should be "enum pgtable_type".

Quoted 8 lines

To make "enum pgtable_type" available to cpufeature.c, move enum pgtable_type definition from arch/arm64/mm/mmu.c to arch/arm64/include/asm/mmu.h.
Adjust kpti_ng_pgd_alloc to use "enum pgtable_type" instead of "int". The function behavior remains identical (parameter is unused).
Fixes 47546a1912fc ("arm64: mm: install KPTI nG mappings with MMU enabled")

That doesn't look right; that commit is from June 2022, and we only introduced enum pgtable_type in May 2025 in commit:

c64f46ee13779616 ("arm64: mm: use enum to identify pgtable level instead of *_SHIFT")

... which landed in v6.16.

AFAICT, that's the commit which broke things.

The actual fix looks fine, though I suspect we should move more of this logic into mmu.c.

Mark.

Quoted 83 lines

Signed-off-by Kees Cook <[email protected]>

---
 arch/arm64/include/asm/mmu.h   | 7 +++++++
 arch/arm64/kernel/cpufeature.c | 5 +++--
 arch/arm64/mm/mmu.c            | 7 -------
 3 files changed, 10 insertions(+), 9 deletions(-)

arch/arm64/include/asm/mmu.h +7 -0

index 6e8aa8e72601..49f1a810df16 100644
@@ -17,6 +17,13 @@
 #include <linux/refcount.h>
 #include <asm/cpufeature.h>
 
+enum pgtable_type {
+	TABLE_PTE,
+	TABLE_PMD,
+	TABLE_PUD,
+	TABLE_P4D,
+};
+
 typedef struct {
 	atomic64_t	id;
 #ifdef CONFIG_COMPAT

arch/arm64/kernel/cpufeature.c +3 -2

index 9ad065f15f1d..e49d142a281f 100644
@@ -84,6 +84,7 @@
 #include <asm/hwcap.h>
 #include <asm/insn.h>
 #include <asm/kvm_host.h>
+#include <asm/mmu.h>
 #include <asm/mmu_context.h>
 #include <asm/mte.h>
 #include <asm/hypervisor.h>

@@ -1945,11 +1946,11 @@ static bool has_pmuv3(const struct arm64_cpu_capabilities *entry, int scope)
 extern
 void create_kpti_ng_temp_pgd(pgd_t *pgdir, phys_addr_t phys, unsigned long virt,
 			     phys_addr_t size, pgprot_t prot,
-			     phys_addr_t (*pgtable_alloc)(int), int flags);
+			     phys_addr_t (*pgtable_alloc)(enum pgtable_type), int flags);
 
 static phys_addr_t __initdata kpti_ng_temp_alloc;
 
-static phys_addr_t __init kpti_ng_pgd_alloc(int shift)
+static phys_addr_t __init kpti_ng_pgd_alloc(enum pgtable_type type)
 {
 	kpti_ng_temp_alloc -= PAGE_SIZE;
 	return kpti_ng_temp_alloc;

arch/arm64/mm/mmu.c +0 -7

index 34e5d78af076..183801520740 100644
@@ -47,13 +47,6 @@
 #define NO_CONT_MAPPINGS	BIT(1)
 #define NO_EXEC_MAPPINGS	BIT(2)	/* assumes FEAT_HPDS is not used */
 
-enum pgtable_type {
-	TABLE_PTE,
-	TABLE_PMD,
-	TABLE_PUD,
-	TABLE_P4D,
-};
-
 u64 kimage_voffset __ro_after_init;
 EXPORT_SYMBOL(kimage_voffset);

Signature · 1 line

2.34.1

Kees Cook

author

2025-08-29 16:41 UTC #4 lore raw

Quoted 11 lines · Mark Rutland

Quoted 1 line · Kees Cook
Fixes 47546a1912fc ("arm64: mm: install KPTI nG mappings with MMU enabled")
That doesn't look right; that commit is from June 2022, and we only introduced enum pgtable_type in May 2025 in commit:
c64f46ee13779616 ("arm64: mm: use enum to identify pgtable level instead of *_SHIFT")
... which landed in v6.16.
AFAICT, that's the commit which broke things.

Oops, yes, I didn't look close enough. Want a v2 with that fixed up?

Signature · 1 line

Kees Cook

Kevin Brodsky

2025-09-01 07:33 UTC #5 lore raw

In reply to Mark Rutland

Quoted 2 lines · Mark Rutland

The actual fix looks fine, though I suspect we should move more of this logic into mmu.c.

Agreed, I'll look into that. Having two separate declarations in cpufeature.c and mmu.c is ugly and leads to the kind of mistake that I made.

The fix looks perfectly fine FWIW - not adding my Reviewed-by as I saw Catalin took v2 already :)

- Kevin