arm64: implement read-only page tables

index 977bea16cf1b..65f31f615d99 100644
@@ -34,6 +34,7 @@ static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm)
 }
 #endif
 
+#ifndef __HAVE_ARCH_PTE_FREE_KERNEL
 /**
  * pte_free_kernel - free PTE-level kernel page table page
  * @mm: the mm_struct of the current context

@@ -43,6 +44,7 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
 {
 	free_page((unsigned long)pte);
 }
+#endif
 
 /**
  * __pte_alloc_one - allocate a page for PTE-level user page table

@@ -91,6 +93,7 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm)
  * done with a reference count in struct page.
  */
 
+#ifndef __HAVE_ARCH_PTE_FREE
 /**
  * pte_free - free PTE-level user page table page
  * @mm: the mm_struct of the current context

@@ -101,11 +104,11 @@ static inline void pte_free(struct mm_struct *mm, struct page *pte_page)
 	pgtable_pte_page_dtor(pte_page);
 	__free_page(pte_page);
 }
+#endif
 
 
 #if CONFIG_PGTABLE_LEVELS > 2
 
-#ifndef __HAVE_ARCH_PMD_ALLOC_ONE
 /**
  * pmd_alloc_one - allocate a page for PMD-level page table
  * @mm: the mm_struct of the current context

@@ -116,7 +119,7 @@ static inline void pte_free(struct mm_struct *mm, struct page *pte_page)
  *
  * Return: pointer to the allocated memory or %NULL on error
  */
-static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
+static inline pmd_t *__pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
 {
 	struct page *page;
 	gfp_t gfp = GFP_PGTABLE_USER;

@@ -132,6 +135,12 @@ static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
 	}
 	return (pmd_t *)page_address(page);
 }
+
+#ifndef __HAVE_ARCH_PMD_ALLOC_ONE
+static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
+{
+	return __pmd_alloc_one(mm, addr);
+}
 #endif
 
 #ifndef __HAVE_ARCH_PMD_FREE

index c4ba047a82d2..8d3806c68687 100644
@@ -34,6 +34,9 @@
 #include <linux/mm_types.h>
 #include <linux/sched.h>
 
+int set_pgtable_ro(void *addr);
+int set_pgtable_rw(void *addr);
+
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 #define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE
 

index a3bacd79507a..61f4aca08b95 100644
@@ -153,6 +153,20 @@ int set_memory_valid(unsigned long addr, int numpages, int enable)
 					__pgprot(PTE_VALID));
 }
 
+int set_pgtable_ro(void *addr)
+{
+	return __change_memory_common((u64)addr, PAGE_SIZE,
+				      __pgprot(PTE_RDONLY),
+				      __pgprot(PTE_WRITE));
+}
+
+int set_pgtable_rw(void *addr)
+{
+	return __change_memory_common((u64)addr, PAGE_SIZE,
+				      __pgprot(PTE_WRITE),
+				      __pgprot(PTE_RDONLY));
+}
+
 int set_direct_map_invalid_noflush(struct page *page)
 {
 	struct page_change_data data = {

index 4335800201c9..71dfbe0452bb 100644
@@ -50,6 +50,7 @@ enum fixed_addresses {
 
 	FIX_EARLYCON_MEM_BASE,
 	FIX_TEXT_POKE0,
+	FIX_TEXT_POKE_PTE,
 
 #ifdef CONFIG_ACPI_APEI_GHES
 	/* Used for GHES mapping from assorted contexts */

index 237224484d0f..d54ac9f8d6c7 100644
@@ -30,7 +30,11 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmdp)
 	pudval_t pudval = PUD_TYPE_TABLE;
 
 	pudval |= (mm == &init_mm) ? PUD_TABLE_UXN : PUD_TABLE_PXN;
-	__pud_populate(pudp, __pa(pmdp), pudval);
+	if (page_tables_are_ro())
+		xchg_ro_pte(mm, (pte_t *)pudp,
+			    __pte(__phys_to_pud_val(__pa(pmdp) | pudval)));
+	else
+		__pud_populate(pudp, __pa(pmdp), pudval);
 }
 #else
 static inline void __pud_populate(pud_t *pudp, phys_addr_t pmdp, pudval_t prot)

@@ -51,7 +55,11 @@ static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4dp, pud_t *pudp)
 	p4dval_t p4dval = P4D_TYPE_TABLE;
 
 	p4dval |= (mm == &init_mm) ? P4D_TABLE_UXN : P4D_TABLE_PXN;
-	__p4d_populate(p4dp, __pa(pudp), p4dval);
+	if (page_tables_are_ro())
+		xchg_ro_pte(mm, (pte_t *)p4dp,
+			    __pte(__phys_to_p4d_val(__pa(pudp) | p4dval)));
+	else
+		__p4d_populate(p4dp, __pa(pudp), p4dval);
 }
 #else
 static inline void __p4d_populate(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot)

@@ -76,15 +84,27 @@ static inline void __pmd_populate(pmd_t *pmdp, phys_addr_t ptep,
 static inline void
 pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep)
 {
+	pmdval_t pmdval = PMD_TYPE_TABLE | PMD_TABLE_UXN;
+
 	VM_BUG_ON(mm && mm != &init_mm);
-	__pmd_populate(pmdp, __pa(ptep), PMD_TYPE_TABLE | PMD_TABLE_UXN);
+	if (page_tables_are_ro())
+		xchg_ro_pte(mm, (pte_t *)pmdp,
+			    __pte(__phys_to_pmd_val(__pa(ptep) | pmdval)));
+	else
+		__pmd_populate(pmdp, __pa(ptep), pmdval);
 }
 
 static inline void
 pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep)
 {
+	pmdval_t pmdval = PMD_TYPE_TABLE | PMD_TABLE_PXN;
+
 	VM_BUG_ON(mm == &init_mm);
-	__pmd_populate(pmdp, page_to_phys(ptep), PMD_TYPE_TABLE | PMD_TABLE_PXN);
+	if (page_tables_are_ro())
+		xchg_ro_pte(mm, (pte_t *)pmdp,
+			    __pte(__phys_to_pmd_val(page_to_phys(ptep) | pmdval)));
+	else
+		__pmd_populate(pmdp, page_to_phys(ptep), pmdval);
 }
 
 #endif

181 lines not on this page · raw message

index 77341b160aca..5a5055c3e1c2 100644
@@ -220,7 +220,13 @@ int ptep_set_access_flags(struct vm_area_struct *vma,
 		pteval ^= PTE_RDONLY;
 		pteval |= pte_val(entry);
 		pteval ^= PTE_RDONLY;
-		pteval = cmpxchg_relaxed(&pte_val(*ptep), old_pteval, pteval);
+		if (page_tables_are_ro())
+			pteval = pte_val(cmpxchg_ro_pte(vma->vm_mm, ptep,
+							__pte(old_pteval),
+							__pte(pteval)));
+		else
+			pteval = cmpxchg_relaxed(&pte_val(*ptep), old_pteval,
+						 pteval);
 	} while (pteval != old_pteval);
 
 	/* Invalidate a stale read-only entry */

100 lines not on this page · raw message

index acfae9b41cc8..a52c3162beae 100644
@@ -394,8 +394,11 @@ static phys_addr_t __pgd_pgtable_alloc(int shift)
 	void *ptr = (void *)__get_free_page(GFP_PGTABLE_KERNEL);
 	BUG_ON(!ptr);
 
-	/* Ensure the zeroed page is visible to the page table walker */
-	dsb(ishst);
+	if (page_tables_are_ro())
+		set_pgtable_ro(ptr);
+	else
+		/* Ensure the zeroed page is visible to the page table walker */
+		dsb(ishst);
 	return __pa(ptr);
 }
 

index 4a64089e5771..637d6eceeada 100644
@@ -9,8 +9,10 @@
 #include <linux/mm.h>
 #include <linux/gfp.h>
 #include <linux/highmem.h>
+#include <linux/set_memory.h>
 #include <linux/slab.h>
 
+#include <asm/mmu_context.h>
 #include <asm/pgalloc.h>
 #include <asm/page.h>
 #include <asm/tlbflush.h>

@@ -20,24 +22,33 @@ static struct kmem_cache *pgd_cache __ro_after_init;
 pgd_t *pgd_alloc(struct mm_struct *mm)
 {
 	gfp_t gfp = GFP_PGTABLE_USER;
+	pgd_t *pgd;
 
-	if (PGD_SIZE == PAGE_SIZE)
-		return (pgd_t *)__get_free_page(gfp);
-	else
+	if (PGD_SIZE < PAGE_SIZE && !page_tables_are_ro())
 		return kmem_cache_alloc(pgd_cache, gfp);
+
+	pgd = (pgd_t *)__get_free_page(gfp);
+	if (!pgd)
+		return NULL;
+	if (page_tables_are_ro())
+		set_pgtable_ro(pgd);
+	return pgd;
 }
 
 void pgd_free(struct mm_struct *mm, pgd_t *pgd)
 {
-	if (PGD_SIZE == PAGE_SIZE)
-		free_page((unsigned long)pgd);
-	else
+	if (PGD_SIZE < PAGE_SIZE && !page_tables_are_ro()) {
 		kmem_cache_free(pgd_cache, pgd);
+	} else {
+		if (page_tables_are_ro())
+			set_pgtable_rw(pgd);
+		free_page((unsigned long)pgd);
+	}
 }
 
 void __init pgtable_cache_init(void)
 {
-	if (PGD_SIZE == PAGE_SIZE)
+	if (PGD_SIZE == PAGE_SIZE || page_tables_are_ro())
 		return;
 
 #ifdef CONFIG_ARM64_PA_BITS_52

index d54ac9f8d6c7..737e9f32b199 100644
@@ -14,6 +14,8 @@
 #include <asm/tlbflush.h>
 
 #define __HAVE_ARCH_PGD_FREE
+#define __HAVE_ARCH_PUD_ALLOC_ONE
+#define __HAVE_ARCH_PUD_FREE
 #include <asm-generic/pgalloc.h>
 
 #define PGD_SIZE	(PTRS_PER_PGD * sizeof(pgd_t))

@@ -45,6 +47,9 @@ static inline void __pud_populate(pud_t *pudp, phys_addr_t pmdp, pudval_t prot)
 
 #if CONFIG_PGTABLE_LEVELS > 3
 
+pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr);
+void pud_free(struct mm_struct *mm, pud_t *pud);
+
 static inline void __p4d_populate(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot)
 {
 	set_p4d(p4dp, __p4d(__phys_to_p4d_val(pudp) | prot));

index c995d1f4594f..6557626752fc 100644
@@ -94,6 +94,8 @@ static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp,
 static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pudp,
 				  unsigned long addr)
 {
+	if (page_tables_are_ro())
+		set_pgtable_rw(pudp);
 	tlb_remove_table(tlb, virt_to_page(pudp));
 }
 #endif

index a52c3162beae..03d77c4c3570 100644
@@ -1645,3 +1645,23 @@ static int __init prevent_bootmem_remove_init(void)
 }
 early_initcall(prevent_bootmem_remove_init);
 #endif
+
+#ifndef __PAGETABLE_PUD_FOLDED
+pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
+{
+	pud_t *pud = __pud_alloc_one(mm, addr);
+
+	if (!pud)
+		return NULL;
+	if (page_tables_are_ro())
+		set_pgtable_ro(pud);
+	return pud;
+}
+
+void pud_free(struct mm_struct *mm, pud_t *pud)
+{
+	if (page_tables_are_ro())
+		set_pgtable_rw(pud);
+	free_page((u64)pud);
+}
+#endif

index 737e9f32b199..63f9ae9e96fe 100644
@@ -16,12 +16,17 @@
 #define __HAVE_ARCH_PGD_FREE
 #define __HAVE_ARCH_PUD_ALLOC_ONE
 #define __HAVE_ARCH_PUD_FREE
+#define __HAVE_ARCH_PMD_ALLOC_ONE
+#define __HAVE_ARCH_PMD_FREE
 #include <asm-generic/pgalloc.h>
 
 #define PGD_SIZE	(PTRS_PER_PGD * sizeof(pgd_t))
 
 #if CONFIG_PGTABLE_LEVELS > 2
 
+pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr);
+void pmd_free(struct mm_struct *mm, pmd_t *pmd);
+
 static inline void __pud_populate(pud_t *pudp, phys_addr_t pmdp, pudval_t prot)
 {
 	set_pud(pudp, __pud(__phys_to_pud_val(pmdp) | prot));

index 6557626752fc..0f54fbb59bba 100644
@@ -85,6 +85,8 @@ static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp,
 {
 	struct page *page = virt_to_page(pmdp);
 
+	if (page_tables_are_ro())
+		set_pgtable_rw(pmdp);
 	pgtable_pmd_page_dtor(page);
 	tlb_remove_table(tlb, page);
 }

index 03d77c4c3570..e55d91a5f1ed 100644
@@ -1665,3 +1665,24 @@ void pud_free(struct mm_struct *mm, pud_t *pud)
 	free_page((u64)pud);
 }
 #endif
+
+#ifndef __PAGETABLE_PMD_FOLDED
+pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
+{
+	pmd_t *pmd = __pmd_alloc_one(mm, addr);
+
+	if (!pmd)
+		return NULL;
+	if (page_tables_are_ro())
+		set_pgtable_ro(pmd);
+	return pmd;
+}
+
+void pmd_free(struct mm_struct *mm, pmd_t *pmd)
+{
+	if (page_tables_are_ro())
+		set_pgtable_rw(pmd);
+	pgtable_pmd_page_dtor(virt_to_page(pmd));
+	free_page((u64)pmd);
+}
+#endif

index 63f9ae9e96fe..18a5bb0c9ee4 100644
@@ -18,10 +18,15 @@
 #define __HAVE_ARCH_PUD_FREE
 #define __HAVE_ARCH_PMD_ALLOC_ONE
 #define __HAVE_ARCH_PMD_FREE
+#define __HAVE_ARCH_PTE_ALLOC_ONE
+#define __HAVE_ARCH_PTE_FREE
 #include <asm-generic/pgalloc.h>
 
 #define PGD_SIZE	(PTRS_PER_PGD * sizeof(pgd_t))
 
+pgtable_t pte_alloc_one(struct mm_struct *mm);
+void pte_free(struct mm_struct *mm, struct page *pte_page);
+
 #if CONFIG_PGTABLE_LEVELS > 2
 
 pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr);

index 0f54fbb59bba..e69a44160cce 100644
@@ -75,6 +75,8 @@ static inline void tlb_flush(struct mmu_gather *tlb)
 static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
 				  unsigned long addr)
 {
+	if (page_tables_are_ro())
+		set_pgtable_rw(page_address(pte));
 	pgtable_pte_page_dtor(pte);
 	tlb_remove_table(tlb, pte);
 }

index e55d91a5f1ed..949846654797 100644
@@ -1686,3 +1686,26 @@ void pmd_free(struct mm_struct *mm, pmd_t *pmd)
 	free_page((u64)pmd);
 }
 #endif
+
+pgtable_t pte_alloc_one(struct mm_struct *mm)
+{
+	pgtable_t pgt = __pte_alloc_one(mm, GFP_PGTABLE_USER);
+
+	VM_BUG_ON(mm == &init_mm);
+
+	if (!pgt)
+		return NULL;
+	if (page_tables_are_ro())
+		set_pgtable_ro(page_address(pgt));
+	return pgt;
+}
+
+void pte_free(struct mm_struct *mm, struct page *pte_page)
+{
+	VM_BUG_ON(mm == &init_mm);
+
+	if (page_tables_are_ro())
+		set_pgtable_rw(page_address(pte_page));
+	pgtable_pte_page_dtor(pte_page);
+	__free_page(pte_page);
+}

index 18a5bb0c9ee4..073482634e74 100644
@@ -20,6 +20,9 @@
 #define __HAVE_ARCH_PMD_FREE
 #define __HAVE_ARCH_PTE_ALLOC_ONE
 #define __HAVE_ARCH_PTE_FREE
+#define __HAVE_ARCH_PTE_ALLOC_ONE_KERNEL
+#define __HAVE_ARCH_PTE_FREE_KERNEL
+
 #include <asm-generic/pgalloc.h>
 
 #define PGD_SIZE	(PTRS_PER_PGD * sizeof(pgd_t))

@@ -27,6 +30,9 @@
 pgtable_t pte_alloc_one(struct mm_struct *mm);
 void pte_free(struct mm_struct *mm, struct page *pte_page);
 
+pte_t *pte_alloc_one_kernel(struct mm_struct *mm);
+void pte_free_kernel(struct mm_struct *mm, pte_t *pte);
+
 #if CONFIG_PGTABLE_LEVELS > 2
 
 pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr);

index 949846654797..971501535757 100644
@@ -1402,7 +1402,7 @@ int pmd_free_pte_page(pmd_t *pmdp, unsigned long addr)
 	table = pte_offset_kernel(pmdp, addr);
 	pmd_clear(pmdp);
 	__flush_tlb_kernel_pgtable(addr);
-	pte_free_kernel(NULL, table);
+	pte_free_kernel(&init_mm, table);
 	return 1;
 }
 

@@ -1709,3 +1709,25 @@ void pte_free(struct mm_struct *mm, struct page *pte_page)
 	pgtable_pte_page_dtor(pte_page);
 	__free_page(pte_page);
 }
+
+pte_t *pte_alloc_one_kernel(struct mm_struct *mm)
+{
+	pte_t *pte = __pte_alloc_one_kernel(mm);
+
+	VM_BUG_ON(mm != &init_mm);
+
+	if (!pte)
+		return NULL;
+	if (page_tables_are_ro())
+		set_pgtable_ro(pte);
+	return pte;
+}
+
+void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
+{
+	VM_BUG_ON(mm != &init_mm);
+
+	if (page_tables_are_ro())
+		set_pgtable_rw(pte);
+	free_page((u64)pte);
+}

index 971501535757..b1212f6d48f2 100644
@@ -559,8 +559,23 @@ static void __init map_mem(pgd_t *pgdp)
 	memblock_clear_nomap(kernel_start, kernel_end - kernel_start);
 }
 
+static void mark_pgtables_ro(const pmd_t *pmdp, int level, int num_entries)
+{
+	while (num_entries--) {
+		if (pmd_valid(*pmdp) && pmd_table(*pmdp)) {
+			pmd_t *next = __va(__pmd_to_phys(*pmdp));
+
+			if (level < 2)
+				mark_pgtables_ro(next, level + 1, PTRS_PER_PMD);
+			set_pgtable_ro(next);
+		}
+		pmdp++;
+	}
+}
+
 void mark_rodata_ro(void)
 {
+	int pgd_level = 4 - CONFIG_PGTABLE_LEVELS;
 	unsigned long section_size;
 
 	/*

@@ -571,6 +586,11 @@ void mark_rodata_ro(void)
 	update_mapping_prot(__pa_symbol(__start_rodata), (unsigned long)__start_rodata,
 			    section_size, PAGE_KERNEL_RO);
 
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+	mark_pgtables_ro((pmd_t *)&tramp_pg_dir, pgd_level, PTRS_PER_PGD);
+#endif
+	mark_pgtables_ro((pmd_t *)&swapper_pg_dir, pgd_level, PTRS_PER_PGD);
+
 	debug_checkwx();
 }
 

index bc8713a76e03..e8aacf6ea207 100644
@@ -79,6 +79,14 @@ static inline unsigned long pud_index(unsigned long address)
 #define pud_index pud_index
 #endif
 
+#ifndef p4d_index
+static inline unsigned long p4d_index(unsigned long address)
+{
+	return (address >> P4D_SHIFT) & (PTRS_PER_P4D - 1);
+}
+#define p4d_index p4d_index
+#endif
+
 #ifndef pgd_index
 /* Must be a compile-time constant, so implement it as a macro */
 #define pgd_index(a)  (((a) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))

index e1be6c429810..e3e50adfae18 100644
@@ -92,7 +92,7 @@ static int __init set_permissions(pte_t *ptep, unsigned long addr, void *data)
 		pte = set_pte_bit(pte, __pgprot(PTE_RDONLY));
 	if (md->attribute & EFI_MEMORY_XP)
 		pte = set_pte_bit(pte, __pgprot(PTE_PXN));
-	set_pte(ptep, pte);
+	set_pte_at(&efi_mm, addr, ptep, pte);
 	return 0;
 }
 

index ffb9c229610a..099b28b00f4c 100644
@@ -252,8 +252,8 @@ void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr,
 
 	ncontig = num_contig_ptes(sz, &pgsize);
 
-	for (i = 0; i < ncontig; i++, ptep++)
-		set_pte(ptep, pte);
+	for (i = 0; i < ncontig; i++, ptep++, addr += pgsize)
+		set_pte_at(mm, addr, ptep, pte);
 }
 
 pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,