Skip to contentSkip to conversation

Kernel bug fix: hugepage_ free_pgtables()

PATCH Chen, Kenneth W 1 participant 1 message Dec 24, 2003

lore
Share & tools
lore
#0 lore raw

We recently discovered a bug on ia64 when unmapping an address space that
belongs to huge page region. The generic code unmap_region() calls free_pgtables() to free any possible pages that are used for page tables.
However, it does no differentiation whether that region is mapped for normal
page or huge tlb page. Problem arises when free_pgtables() calculates PGDIR
aligned area based on the default page size, where in the huge page case, it
should really be using huge tlb page size instead. The pgd_index calculation
should also be adjusted accordingly.

So we need an architecture specific code to handle huge tlb cases. It also
requires changes in generic part of kernel. The generic kernel changes has
made into Andrew Morton's 2.6.0-mm1 already. Here is the ia64 part of patch
to take the new free page table semantics.

A bit more details on the kernel bug: When there are two huge page mappings,
like the two in the example, first one at the end of PGDIR_SIZE, and second
one starts at next PGDIR_SIZE (64GB with 16K page size):
8000000ff0000000-8000001000000000 rw-s 8000001000000000-8000001010000000 rw-s
Unmapping the first vma would trick free_pgtable to think it can remove one
set of pgd indexed at 0x400, and it went ahead purge the entire pmd/pte that
are still in use by the second mapping. Now any subsequent access to pmd/pte
for the second active mapping will trigger the bug. We've seen hard kernel
hang on some platform, some other platform will generate MCA, plus all kinds
of unpleasant result.

David, please apply for 2.6.

- Ken

arch/ia64/mm/hugetlbpage.c +53 -11
diff -Nur linux-2.6.0/arch/ia64/mm/hugetlbpage.c linux-2.6.0-ken/arch/ia64/mm/hugetlbpage.c
@@ -144,17 +144,6 @@
 
 	return 0;
 }
-/* This function checks if the address and address+len falls out of HugeTLB region.  It
- * return -EINVAL if any part of address range falls in HugeTLB region.
- */
-int  check_valid_hugepage_range(unsigned long addr, unsigned long len)
-{
-	if (REGION_NUMBER(addr) == REGION_HPAGE)
-		return -EINVAL;
-	if (REGION_NUMBER(addr+len) == REGION_HPAGE)
-		return -EINVAL;
-	return 0;
-}
 
 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
 			struct vm_area_struct *vma)
@@ -272,6 +261,59 @@
 	free_huge_page(page);
 }
 
+/*
+ * Same as generic free_pgtables(), except constant PGDIR_* and pgd_offset
+ * are hugetlb region specific.
+ */
+void hugetlb_free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *prev,
+	unsigned long start, unsigned long end)
+{
+	unsigned long first = start & HUGETLB_PGDIR_MASK;
+	unsigned long last = end + HUGETLB_PGDIR_SIZE - 1;
+	unsigned long start_index, end_index;
+	struct mm_struct *mm = tlb->mm;
+
+	if (!prev) {
+		prev = mm->mmap;
+		if (!prev)
+			goto no_mmaps;
+		if (prev->vm_end > start) {
+			if (last > prev->vm_start)
+				last = prev->vm_start;
+			goto no_mmaps;
+		}
+	}
+	for (;;) {
+		struct vm_area_struct *next = prev->vm_next;
+
+		if (next) {
+			if (next->vm_start < start) {
+				prev = next;
+				continue;
+			}
+			if (last > next->vm_start)
+				last = next->vm_start;
+		}
+		if (prev->vm_end > first)
+			first = prev->vm_end + HUGETLB_PGDIR_SIZE - 1;
+		break;
+	}
+no_mmaps:
+	if (last < first)	/* for arches with discontiguous pgd indices */
+		return;
+	/*
+	 * If the PGD bits are not consecutive in the virtual address, the
+	 * old method of shifting the VA >> by PGDIR_SHIFT doesn't work.
+	 */
+
+	start_index = pgd_index(htlbpage_to_page(first));
+	end_index = pgd_index(htlbpage_to_page(last));
+
+	if (end_index > start_index) {
+		clear_page_tables(tlb, start_index, end_index - start_index);
+	}
+}
+
 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
 {
 	struct mm_struct *mm = vma->vm_mm;
include/asm-ia64/page.h +4 -2
diff -Nur linux-2.6.0/include/asm-ia64/page.h linux-2.6.0-ken/include/asm-ia64/page.h
@@ -63,7 +63,7 @@
 # define HPAGE_SIZE	(__IA64_UL_CONST(1) << HPAGE_SHIFT)
 # define HPAGE_MASK	(~(HPAGE_SIZE - 1))
 # define HAVE_ARCH_HUGETLB_UNMAPPED_AREA
-# define ARCH_HAS_VALID_HUGEPAGE_RANGE
+# define ARCH_HAS_HUGEPAGE_ONLY_RANGE
 #endif /* CONFIG_HUGETLB_PAGE */
 
 #ifdef __ASSEMBLY__
@@ -137,7 +137,9 @@
 # define htlbpage_to_page(x)	((REGION_NUMBER(x) << 61)				\
 				 | (REGION_OFFSET(x) >> (HPAGE_SHIFT-PAGE_SHIFT)))
 # define HUGETLB_PAGE_ORDER	(HPAGE_SHIFT - PAGE_SHIFT)
-extern int  check_valid_hugepage_range(unsigned long addr, unsigned long len);
+# define is_hugepage_only_range(addr, len)		\
+	 (REGION_NUMBER(addr) == REGION_HPAGE &&	\
+	  REGION_NUMBER((addr)+(len)) == REGION_HPAGE)
 #endif
 
 static __inline__ int
include/asm-ia64/pgtable.h +9 -0
diff -Nur linux-2.6.0/include/asm-ia64/pgtable.h linux-2.6.0-ken/include/asm-ia64/pgtable.h
@@ -455,6 +455,15 @@
 /* We provide our own get_unmapped_area to cope with VA holes for userland */
 #define HAVE_ARCH_UNMAPPED_AREA
 
+#ifdef CONFIG_HUGETLB_PAGE
+#define HUGETLB_PGDIR_SHIFT	(HPAGE_SHIFT + 2*(PAGE_SHIFT-3))
+#define HUGETLB_PGDIR_SIZE	(__IA64_UL(1) << HUGETLB_PGDIR_SHIFT)
+#define HUGETLB_PGDIR_MASK	(~(HUGETLB_PGDIR_SIZE-1))
+struct mmu_gather;
+extern void hugetlb_free_pgtables(struct mmu_gather *tlb,
+	struct vm_area_struct * prev, unsigned long start, unsigned long end);
+#endif
+
 typedef pte_t *pte_addr_t;
 
 /*