Skip to contentSkip to conversation

4level core patch

PATCH In review Andreas Kleen 1 participant 1 message Oct 25, 2004

Subsystems (10)
lore
Share & tools
lore
#0 lore raw

PATCH 3/17 4level core patch

4level core patch

This patch extends the Linux MM to 4level page tables.

The conversion is quite straight forward. The additional level is called pml4. The biggest change is that I rewrote copy_page_range(), because the old version just wasn't scalable anymore.

This is the core patch for mm/*, fs/*, include/linux/*

It breaks all architectures, which will be fixed in separate patches.

pgd_offset() and pgd_offset_k() have been renamed to make sure all unconverted code causes an compile error. They are replaced with pml4_offset()/pml4_offset_k() and pml4_pgd_offset()/pml4_pgd_offset_k() Please note than when you use pml4_offset_k() you also have
to use pml4_pgd_offset_k() because IA64 requires a special case here.

fs/exec.c +5 -1
diff -urpN -X ../KDIFX linux-2.6.10rc1/fs/exec.c linux-2.6.10rc1-4level/fs/exec.c
@@ -299,6 +299,7 @@ void install_arg_page(struct vm_area_str
 			struct page *page, unsigned long address)
 {
 	struct mm_struct *mm = vma->vm_mm;
+	pml4_t *pml4;
 	pgd_t * pgd;
 	pmd_t * pmd;
 	pte_t * pte;
@@ -307,9 +308,12 @@ void install_arg_page(struct vm_area_str
 		goto out_sig;
 
 	flush_dcache_page(page);
-	pgd = pgd_offset(mm, address);
+	pml4 = pml4_offset(mm, address);
 
 	spin_lock(&mm->page_table_lock);
+	pgd = pgd_alloc(mm, pml4, address);
+	if (!pgd) 
+		goto out;
 	pmd = pmd_alloc(mm, pgd, address);
 	if (!pmd)
 		goto out;
include/linux/init_task.h +1 -1
diff -urpN -X ../KDIFX linux-2.6.10rc1/include/linux/init_task.h linux-2.6.10rc1-4level/include/linux/init_task.h
@@ -34,7 +34,7 @@
 #define INIT_MM(name) \
 {			 					\
 	.mm_rb		= RB_ROOT,				\
-	.pgd		= swapper_pg_dir, 			\
+	.pml4		= swapper_pml4, 			\
 	.mm_users	= ATOMIC_INIT(2), 			\
 	.mm_count	= ATOMIC_INIT(1), 			\
 	.mmap_sem	= __RWSEM_INITIALIZER(name.mmap_sem),	\
include/linux/mm.h +16 -1
diff -urpN -X ../KDIFX linux-2.6.10rc1/include/linux/mm.h linux-2.6.10rc1-4level/include/linux/mm.h
@@ -564,7 +564,7 @@ int unmap_vmas(struct mmu_gather **tlbp,
 		struct vm_area_struct *start_vma, unsigned long start_addr,
 		unsigned long end_addr, unsigned long *nr_accounted,
 		struct zap_details *);
-void clear_page_tables(struct mmu_gather *tlb, unsigned long first, int nr);
+void clear_page_range(struct mmu_gather *tlb, unsigned long addr, unsigned long end);
 int copy_page_range(struct mm_struct *dst, struct mm_struct *src,
 			struct vm_area_struct *vma);
 int zeromap_page_range(struct vm_area_struct *vma, unsigned long from,
@@ -580,6 +580,7 @@ static inline void unmap_shared_mapping_
 
 extern int vmtruncate(struct inode * inode, loff_t offset);
 extern pmd_t *FASTCALL(__pmd_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address));
+extern pgd_t fastcall *__pgd_alloc(struct mm_struct *mm, pml4_t *pgd, unsigned long address);
 extern pte_t *FASTCALL(pte_alloc_kernel(struct mm_struct *mm, pmd_t *pmd, unsigned long address));
 extern pte_t *FASTCALL(pte_alloc_map(struct mm_struct *mm, pmd_t *pmd, unsigned long address));
 extern int install_page(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, struct page *page, pgprot_t prot);
@@ -635,6 +636,20 @@ static inline pmd_t *pmd_alloc(struct mm
 	return pmd_offset(pgd, address);
 }
 
+static inline pgd_t *pgd_alloc(struct mm_struct *mm, pml4_t *pml4, unsigned long address)
+{
+	if (pml4_none(*pml4))
+		return __pgd_alloc(mm, pml4, address); 
+	return pml4_pgd_offset(pml4, address); 
+}
+
+static inline pgd_t *pgd_alloc_k(struct mm_struct *mm, pml4_t *pml4, unsigned long address)
+{
+	if (pml4_none(*pml4))
+		return __pgd_alloc(mm, pml4, address); 
+	return pml4_pgd_offset_k(pml4, address); 
+}
+
 extern void free_area_init(unsigned long * zones_size);
 extern void free_area_init_node(int nid, pg_data_t *pgdat,
 	unsigned long * zones_size, unsigned long zone_start_pfn, 
include/linux/sched.h +1 -1
diff -urpN -X ../KDIFX linux-2.6.10rc1/include/linux/sched.h linux-2.6.10rc1-4level/include/linux/sched.h
@@ -211,7 +211,7 @@ struct mm_struct {
 	void (*unmap_area) (struct vm_area_struct *area);
 	unsigned long mmap_base;		/* base of mmap area */
 	unsigned long free_area_cache;		/* first hole */
-	pgd_t * pgd;
+	pml4_t * pml4;
 	atomic_t mm_users;			/* How many users with user space? */
 	atomic_t mm_count;			/* How many references to "struct mm_struct" (users count as 1) */
 	int map_count;				/* number of VMAs */
kernel/fork.c +3 -3
diff -urpN -X ../KDIFX linux-2.6.10rc1/kernel/fork.c linux-2.6.10rc1-4level/kernel/fork.c
@@ -261,15 +261,15 @@ fail_nomem:
 
 static inline int mm_alloc_pgd(struct mm_struct * mm)
 {
-	mm->pgd = pgd_alloc(mm);
-	if (unlikely(!mm->pgd))
+	mm->pml4 = pml4_alloc(mm);
+	if (unlikely(!mm->pml4))
 		return -ENOMEM;
 	return 0;
 }
 
 static inline void mm_free_pgd(struct mm_struct * mm)
 {
-	pgd_free(mm->pgd);
+	pml4_free(mm->pml4);
 }
 #else
 #define dup_mmap(mm, oldmm)	(0)
mm/fremap.c +11 -4
diff -urpN -X ../KDIFX linux-2.6.10rc1/mm/fremap.c linux-2.6.10rc1-4level/mm/fremap.c
@@ -64,9 +64,11 @@ int install_page(struct mm_struct *mm, s
 	pmd_t *pmd;
 	pte_t pte_val;
 
-	pgd = pgd_offset(mm, addr);
 	spin_lock(&mm->page_table_lock);
-
+	pgd = pgd_alloc(mm, pml4_offset(mm, addr), addr); 
+	if (!pgd) 
+	    goto err_unlock;
+	
 	pmd = pmd_alloc(mm, pgd, addr);
 	if (!pmd)
 		goto err_unlock;
@@ -111,13 +113,18 @@ int install_file_pte(struct mm_struct *m
 		unsigned long addr, unsigned long pgoff, pgprot_t prot)
 {
 	int err = -ENOMEM;
+	pml4_t *pml4;
 	pte_t *pte;
 	pgd_t *pgd;
 	pmd_t *pmd;
 	pte_t pte_val;
 
-	pgd = pgd_offset(mm, addr);
-	spin_lock(&mm->page_table_lock);
+	pml4 = pml4_offset(mm, addr); 
+
+	spin_lock(&mm->page_table_lock);  
+	pgd = pgd_alloc(mm, pml4, addr); 
+	if (!pgd) 
+		goto err_unlock;
 
 	pmd = pmd_alloc(mm, pgd, addr);
 	if (!pmd)
mm/memory.c +428 -192

830 lines not on this page · raw message

mm/mempolicy.c +15 -3
diff -urpN -X ../KDIFX linux-2.6.10rc1/mm/mempolicy.c linux-2.6.10rc1-4level/mm/mempolicy.c
@@ -233,13 +233,24 @@ static struct mempolicy *mpol_new(int mo
 
 /* Ensure all existing pages follow the policy. */
 static int
-verify_pages(unsigned long addr, unsigned long end, unsigned long *nodes)
+verify_pages(struct mm_struct *mm,
+	     unsigned long addr, unsigned long end, unsigned long *nodes)
 {
 	while (addr < end) {
 		struct page *p;
 		pte_t *pte;
 		pmd_t *pmd;
-		pgd_t *pgd = pgd_offset_k(addr);
+		pgd_t *pgd;
+		pml4_t *pml4;
+		pml4 = pml4_offset(mm, addr); 
+		if (pml4_none(*pml4)) { 
+			unsigned long next = (addr + PML4_SIZE) & PML4_MASK; 
+			if (next > addr)
+				break;
+			addr = next;
+			continue;
+		} 
+		pgd = pml4_pgd_offset(pml4, addr);
 		if (pgd_none(*pgd)) {
 			addr = (addr + PGDIR_SIZE) & PGDIR_MASK;
 			continue;
@@ -282,7 +293,8 @@ check_range(struct mm_struct *mm, unsign
 		if (prev && prev->vm_end < vma->vm_start)
 			return ERR_PTR(-EFAULT);
 		if ((flags & MPOL_MF_STRICT) && !is_vm_hugetlb_page(vma)) {
-			err = verify_pages(vma->vm_start, vma->vm_end, nodes);
+			err = verify_pages(vma->vm_mm, 
+					   vma->vm_start, vma->vm_end, nodes);
 			if (err) {
 				first = ERR_PTR(err);
 				break;
mm/mmap.c +20 -17
diff -urpN -X ../KDIFX linux-2.6.10rc1/mm/mmap.c linux-2.6.10rc1-4level/mm/mmap.c
@@ -1459,7 +1459,7 @@ find_extend_vma(struct mm_struct * mm, u
  * the page tables themselves.
  *
  * Right now we try to free page tables if we have a nice
- * PGDIR-aligned area that got free'd up. We could be more
+ * PGD/PML4-aligned area that got free'd up. We could be more
  * granular if we want to, but this is fast and simple,
  * and covers the bad cases.
  *
@@ -1471,7 +1471,7 @@ static void free_pgtables(struct mmu_gat
 {
 	unsigned long first = start & PGDIR_MASK;
 	unsigned long last = end + PGDIR_SIZE - 1;
-	unsigned long start_index, end_index;
+	unsigned long start_pml4_index, start_pgd_index;
 	struct mm_struct *mm = tlb->mm;
 
 	if (!prev) {
@@ -1495,24 +1495,26 @@ static void free_pgtables(struct mmu_gat
 			if (last > next->vm_start)
 				last = next->vm_start;
 		}
-		if (prev->vm_end > first)
-			first = prev->vm_end + PGDIR_SIZE - 1;
+		if (prev->vm_end > first) {
+			first = prev->vm_end + PML4_SIZE - 1;
+			if (first < prev->vm_end) 
+				first = TASK_SIZE;
+		}
 		break;
 	}
 no_mmaps:
-	if (last < first)	/* for arches with discontiguous pgd indices */
+	if (last < first)	/* for arches with discontiguous indices */
 		return;
-	/*
-	 * If the PGD bits are not consecutive in the virtual address, the
-	 * old method of shifting the VA >> by PGDIR_SHIFT doesn't work.
-	 */
-	start_index = pgd_index(first);
-	if (start_index < FIRST_USER_PGD_NR)
-		start_index = FIRST_USER_PGD_NR;
-	end_index = pgd_index(last);
-	if (end_index > start_index) {
-		clear_page_tables(tlb, start_index, end_index - start_index);
-		flush_tlb_pgtables(mm, first & PGDIR_MASK, last & PGDIR_MASK);
+	start_pml4_index = pml4_index(first);
+	start_pgd_index = pgd_index(first);
+	if (start_pml4_index == 0 && start_pgd_index < FIRST_USER_PGD_NR) { 
+		start_pgd_index = FIRST_USER_PGD_NR;
+		first = start_pgd_index * PGDIR_SIZE;
+	}
+	if (pml4_index(last) > start_pml4_index || 
+	    pgd_index(last) > start_pgd_index) {
+		clear_page_range(tlb, first, last);
+		flush_tlb_pgtables(mm, first & PML4_MASK, last & PML4_MASK);
 	}
 }
 
@@ -1841,7 +1843,8 @@ void exit_mmap(struct mm_struct *mm)
 					~0UL, &nr_accounted, NULL);
 	vm_unacct_memory(nr_accounted);
 	BUG_ON(mm->map_count);	/* This is just debugging */
-	clear_page_tables(tlb, FIRST_USER_PGD_NR, USER_PTRS_PER_PGD);
+	clear_page_range(tlb, FIRST_USER_PGD_NR * PGDIR_SIZE, 
+			 (TASK_SIZE + PGDIR_SIZE - 1) & ~(PGDIR_SIZE - 1));
 	tlb_finish_mmu(tlb, 0, MM_VM_SIZE(mm));
 
 	vma = mm->mmap;
mm/mprotect.c +36 -8

59 lines not on this page · raw message

mm/mremap.c +18 -3
diff -urpN -X ../KDIFX linux-2.6.10rc1/mm/mremap.c linux-2.6.10rc1-4level/mm/mremap.c
@@ -24,11 +24,16 @@
 
 static pte_t *get_one_pte_map_nested(struct mm_struct *mm, unsigned long addr)
 {
+	pml4_t *pml4;
 	pgd_t *pgd;
 	pmd_t *pmd;
 	pte_t *pte = NULL;
 
-	pgd = pgd_offset(mm, addr);
+	pml4 = pml4_offset(mm, addr);
+	if (pml4_none(*pml4))
+		goto end;	
+
+	pgd = pml4_pgd_offset(pml4, addr);
 	if (pgd_none(*pgd))
 		goto end;
 	if (pgd_bad(*pgd)) {
@@ -57,10 +62,15 @@ end:
 
 static pte_t *get_one_pte_map(struct mm_struct *mm, unsigned long addr)
 {
+	pml4_t *pml4;
 	pgd_t *pgd;
 	pmd_t *pmd;
 
-	pgd = pgd_offset(mm, addr);
+	pml4 = pml4_offset(mm, addr);
+	if (pml4_none(*pml4))
+		return NULL;
+	
+	pgd = pml4_pgd_offset(pml4, addr);
 	if (pgd_none(*pgd))
 		return NULL;
 	pmd = pmd_offset(pgd, addr);
@@ -71,10 +81,15 @@ static pte_t *get_one_pte_map(struct mm_
 
 static inline pte_t *alloc_one_pte_map(struct mm_struct *mm, unsigned long addr)
 {
+	pgd_t *pgd; 
 	pmd_t *pmd;
 	pte_t *pte = NULL;
 
-	pmd = pmd_alloc(mm, pgd_offset(mm, addr), addr);
+	pgd = pgd_alloc(mm, pml4_offset(mm,addr), addr);
+	if (!pgd) 
+		return NULL;
+
+	pmd = pmd_alloc(mm, pgd, addr);
 	if (pmd)
 		pte = pte_alloc_map(mm, pmd, addr);
 	return pte;
mm/msync.c +36 -7

65 lines not on this page · raw message

mm/rmap.c +18 -3
diff -urpN -X ../KDIFX linux-2.6.10rc1/mm/rmap.c linux-2.6.10rc1-4level/mm/rmap.c
@@ -258,6 +258,7 @@ static int page_referenced_one(struct pa
 {
 	struct mm_struct *mm = vma->vm_mm;
 	unsigned long address;
+	pml4_t *pml4;
 	pgd_t *pgd;
 	pmd_t *pmd;
 	pte_t *pte;
@@ -271,7 +272,11 @@ static int page_referenced_one(struct pa
 
 	spin_lock(&mm->page_table_lock);
 
-	pgd = pgd_offset(mm, address);
+	pml4 = pml4_offset(mm, address);
+	if (!pml4_present(*pml4))
+		goto out_unlock; 
+
+	pgd = pml4_pgd_offset(pml4, address);
 	if (!pgd_present(*pgd))
 		goto out_unlock;
 
@@ -496,6 +501,7 @@ static int try_to_unmap_one(struct page 
 {
 	struct mm_struct *mm = vma->vm_mm;
 	unsigned long address;
+	pml4_t *pml4;
 	pgd_t *pgd;
 	pmd_t *pmd;
 	pte_t *pte;
@@ -514,7 +520,11 @@ static int try_to_unmap_one(struct page 
 	 */
 	spin_lock(&mm->page_table_lock);
 
-	pgd = pgd_offset(mm, address);
+	pml4 = pml4_offset(mm, address);
+	if (!pml4_present(*pml4))
+		goto out_unlock; 
+
+	pgd = pml4_pgd_offset(pml4, address);
 	if (!pgd_present(*pgd))
 		goto out_unlock;
 
@@ -624,6 +634,7 @@ static void try_to_unmap_cluster(unsigne
 	unsigned int *mapcount, struct vm_area_struct *vma)
 {
 	struct mm_struct *mm = vma->vm_mm;
+	pml4_t *pml4;
 	pgd_t *pgd;
 	pmd_t *pmd;
 	pte_t *pte;
@@ -646,7 +657,11 @@ static void try_to_unmap_cluster(unsigne
 	if (end > vma->vm_end)
 		end = vma->vm_end;
 
-	pgd = pgd_offset(mm, address);
+	pml4 = pml4_offset(mm, address);
+	if (!pml4_present(*pml4))
+		goto out_unlock;
+
+	pgd = pml4_pgd_offset(pml4, address);
 	if (!pgd_present(*pgd))
 		goto out_unlock;
 
mm/swapfile.c +49 -12

94 lines not on this page · raw message

mm/vmalloc.c +68 -17

125 lines not on this page · raw message