kanoj-mm15-2.3.10 Fix ia32 SMP/clone pte races

@@ -2157,3 +2157,45 @@
 
 #undef APIC_DIVISOR
 
+#define PMD_TABLE_MASK ((PTRS_PER_PMD-1) * sizeof(pmd_t))
+
+void pte_range(struct mm_struct *mm, unsigned long start, unsigned long end,
+						int op)
+{
+	pgd_t * dir = pgd_offset(mm, start) - 1;
+
+	for(;;) {
+		pmd_t *pmd;
+
+		dir++;
+		if ((pgd_none(*dir)) || (pgd_bad(*dir))) {
+			start = (start + PGDIR_SIZE) & PGDIR_MASK;
+			if (start >= end)
+				return;
+			continue;
+		}
+
+		pmd = pmd_offset(dir, start);
+		do {
+			if ((pmd_none(*pmd)) || (pmd_bad(*pmd))) {
+				start = (start + PMD_SIZE) & PMD_MASK;
+				if (start >= end)
+					return;
+				pmd++;
+				continue;
+			}
+
+			switch(op) {
+				case 1:
+					pmd_mkabsent(pmd);
+					break;
+				case 0:
+					pmd_mkpresent(pmd);
+					break;
+			}
+			start = (start + PMD_SIZE) & PMD_MASK;
+			if (start >= end)
+				return;
+		} while ((unsigned long)pmd & PMD_TABLE_MASK);
+	}
+}

@@ -636,6 +636,11 @@
 #define module_map	vmalloc
 #define module_unmap	vfree
 
+#define pte_freeze_mm_range(mm, s, e)  0
+#define pte_unfreeze_mm_range(mm, s, e)
+#define pte_freeze_dirty(vma, page_table, pte, wr)
+#define pte_unfreeze_dirty(page_table, wr)
+
 /* Needs to be defined here and not in linux/mm.h, as it is arch dependent */
 #define PageSkip(page)		(0)
 #define kern_addr_valid(addr)	(1)

@@ -7,6 +7,11 @@
 #define module_map	vmalloc
 #define module_unmap	vfree
 
+#define pte_freeze_mm_range(mm, s, e)  0
+#define pte_unfreeze_mm_range(mm, s, e)
+#define pte_freeze_dirty(vma, page_table, pte, wr)
+#define pte_unfreeze_dirty(page_table, wr)
+
 extern int do_check_pgt_cache(int, int);
 
 /* Needs to be defined here and not in linux/mm.h, as it is arch dependent */

68 lines not on this page · raw message

@@ -824,6 +824,11 @@
 #define module_map      vmalloc
 #define module_unmap    vfree
 
+#define pte_freeze_mm_range(mm, s, e)  0
+#define pte_unfreeze_mm_range(mm, s, e)
+#define pte_freeze_dirty(vma, page_table, pte, wr)
+#define pte_unfreeze_dirty(page_table, wr)
+
 /* Needs to be defined here and not in linux/mm.h, as it is arch dependent */
 #define PageSkip(page)		(0)
 #define kern_addr_valid(addr)	(1)

@@ -587,6 +587,11 @@
 #define module_map      vmalloc
 #define module_unmap    vfree
 
+#define pte_freeze_mm_range(mm, s, e)  0
+#define pte_unfreeze_mm_range(mm, s, e)
+#define pte_freeze_dirty(vma, page_table, pte, wr)
+#define pte_unfreeze_dirty(page_table, wr)
+
 /* Needs to be defined here and not in linux/mm.h, as it is arch dependent */
 #define PageSkip(page)		(0)
 #define kern_addr_valid(addr)	(1)

@@ -615,6 +615,11 @@
 #define module_map      vmalloc
 #define module_unmap    vfree
 
+#define pte_freeze_mm_range(mm, s, e)  0
+#define pte_unfreeze_mm_range(mm, s, e)
+#define pte_freeze_dirty(vma, page_table, pte, wr)
+#define pte_unfreeze_dirty(page_table, wr)
+
 /* CONFIG_APUS */
 /* For virtual address to physical address conversion */
 extern void cache_clear(__u32 addr, int length);

@@ -576,6 +576,11 @@
 #define module_unmap    vfree
 extern unsigned long *sparc_valid_addr_bitmap;
 
+#define pte_freeze_mm_range(mm, s, e)  0
+#define pte_unfreeze_mm_range(mm, s, e)
+#define pte_freeze_dirty(vma, page_table, pte, wr)
+#define pte_unfreeze_dirty(page_table, wr)
+
 /* Needs to be defined here and not in linux/mm.h, as it is arch dependent */
 #define PageSkip(page)		(test_bit(PG_skip, &(page)->flags))
 #define kern_addr_valid(addr)	(test_bit(__pa((unsigned long)(addr))>>20, sparc_valid_addr_bitmap))

@@ -648,6 +648,11 @@
 extern void module_unmap (void *addr);
 extern unsigned long *sparc64_valid_addr_bitmap;
 
+#define pte_freeze_mm_range(mm, s, e)  0
+#define pte_unfreeze_mm_range(mm, s, e)
+#define pte_freeze_dirty(vma, page_table, pte, wr)
+#define pte_unfreeze_dirty(page_table, wr)
+
 /* Needs to be defined here and not in linux/mm.h, as it is arch dependent */
 #define PageSkip(page)		(test_bit(PG_skip, &(page)->flags))
 #define kern_addr_valid(addr)	(test_bit(__pa((unsigned long)(addr))>>22, sparc64_valid_addr_bitmap))

@@ -231,9 +231,10 @@
 static inline int dup_mmap(struct mm_struct * mm)
 {
 	struct vm_area_struct * mpnt, *tmp, **pprev;
-	int retval;
+	int retval, ret;
 
 	flush_cache_mm(current->mm);
+	ret = pte_freeze_mm_range(mm, 0, TASK_SIZE);
 	pprev = &mm->mmap;
 	for (mpnt = current->mm->mmap ; mpnt ; mpnt = mpnt->vm_next) {
 		struct file *file;

@@ -284,7 +285,9 @@
 		build_mmap_avl(mm);
 
 fail_nomem:
-	flush_tlb_mm(current->mm);
+	if (ret == 0)
+		flush_tlb_mm(current->mm);
+	pte_unfreeze_mm_range(mm, 0, TASK_SIZE);
 	return retval;
 }
 

@@ -1604,13 +1604,11 @@
 	int error = 0;
 
 	dir = pgd_offset(vma->vm_mm, address);
-	flush_cache_range(vma->vm_mm, end - size, end);
 	while (address < end) {
 		error |= filemap_sync_pmd_range(dir, address, end - address, vma, flags);
 		address = (address + PGDIR_SIZE) & PGDIR_MASK;
 		dir++;
 	}
-	flush_tlb_range(vma->vm_mm, end - size, end);
 	return error;
 }
 

@@ -1688,7 +1686,10 @@
 	unsigned long start, unsigned long end, int flags)
 {
 	if (vma->vm_file && vma->vm_ops && vma->vm_ops->sync) {
-		int error;
+		int error, ret;
+
+		flush_cache_range(vma->vm_mm, start, end);
+		ret = pte_freeze_mm_range(vma->vm_mm, start, end);
 		error = vma->vm_ops->sync(vma, start, end-start, flags);
 		if (!error && (flags & MS_SYNC)) {
 			struct file * file = vma->vm_file;

@@ -1697,6 +1698,9 @@
 				error = file_fsync(file, dentry);
 			}
 		}
+		if (ret == 0)
+			flush_tlb_range(vma->vm_mm, start, end);
+		pte_unfreeze_mm_range(vma->vm_mm, start, end);
 		return error;
 	}
 	return 0;

@@ -671,6 +671,7 @@
 	 */
 	while ((mpnt = free) != NULL) {
 		unsigned long st, end, size;
+		int ret;
 
 		free = free->vm_next;
 

@@ -679,6 +680,8 @@
 		end = end > mpnt->vm_end ? mpnt->vm_end : end;
 		size = end - st;
 
+		flush_cache_range(mm, st, end);
+		ret = pte_freeze_mm_range(mm, st, end);
 		lock_kernel();
 		if (mpnt->vm_ops && mpnt->vm_ops->unmap)
 			mpnt->vm_ops->unmap(mpnt, st, size);

@@ -687,9 +690,10 @@
 		remove_shared_vm_struct(mpnt);
 		mm->map_count--;
 
-		flush_cache_range(mm, st, end);
 		zap_page_range(mm, st, size);
-		flush_tlb_range(mm, st, end);
+		if (ret == 0)
+			flush_tlb_range(mm, st, end);
+		pte_unfreeze_mm_range(mm, st, end);
 
 		/*
 		 * Fix the mapping, and free the old area if it wasn't reused.

@@ -67,15 +67,19 @@
 {
 	pgd_t *dir;
 	unsigned long beg = start;
+	int ret;
 
 	dir = pgd_offset(current->mm, start);
 	flush_cache_range(current->mm, beg, end);
+	ret = pte_freeze_mm_range(current->mm, beg, end);
 	while (start < end) {
 		change_pmd_range(dir, start, end - start, newprot);
 		start = (start + PGDIR_SIZE) & PGDIR_MASK;
 		dir++;
 	}
-	flush_tlb_range(current->mm, beg, end);
+	if (ret == 0)
+		flush_tlb_range(current->mm, beg, end);
+	pte_unfreeze_mm_range(current->mm, beg, end);
 	return;
 }
 

@@ -91,9 +91,10 @@
 	unsigned long new_addr, unsigned long old_addr, unsigned long len)
 {
 	unsigned long offset = len;
+	int ret;
 
 	flush_cache_range(mm, old_addr, old_addr + len);
-	flush_tlb_range(mm, old_addr, old_addr + len);
+	ret = pte_freeze_mm_range(mm, old_addr, old_addr + len);
 
 	/*
 	 * This is not the clever way to do this, but we're taking the

@@ -105,6 +106,9 @@
 		if (move_one_page(mm, old_addr + offset, new_addr + offset))
 			goto oops_we_failed;
 	}
+	if (ret == 0)
+		flush_tlb_range(mm, old_addr, old_addr + len);
+	pte_unfreeze_mm_range(mm, old_addr, old_addr + len);
 	return 0;
 
 	/*

@@ -115,10 +119,10 @@
 	 * the old page tables)
 	 */
 oops_we_failed:
+	pte_unfreeze_mm_range(mm, old_addr, old_addr + len);
 	flush_cache_range(mm, new_addr, new_addr + len);
 	while ((offset += PAGE_SIZE) < len)
 		move_one_page(mm, new_addr + offset, old_addr + offset);
-	zap_page_range(mm, new_addr, new_addr + len);
 	flush_tlb_range(mm, new_addr, new_addr + len);
 	return -1;
 }

@@ -38,6 +38,7 @@
 	unsigned long entry;
 	unsigned long page_addr;
 	struct page * page;
+	int writable;
 
 	pte = *page_table;
 	if (!pte_present(pte))

@@ -49,8 +50,9 @@
 	page = mem_map + MAP_NR(page_addr);
 	spin_lock(&tsk->mm->page_table_lock);
 	if (pte_val(pte) != pte_val(*page_table))
-		goto out_failed_unlock;
+		goto out_failed_unlock_nofreeze;
 
+	pte_freeze_dirty(vma, page_table, pte, writable);
 	/*
 	 * Dont be too eager to get aging right if
 	 * memory is dangerously low.

@@ -155,7 +157,7 @@
 	 */
 	entry = get_swap_page();
 	if (!entry)
-		goto out_failed; /* No swap space left */
+		goto out_failed_unlock; /* No swap space left */
 		
 	vma->vm_mm->rss--;
 	tsk->nswap++;

@@ -175,6 +177,8 @@
 	__free_page(page);
 	return 1;
 out_failed_unlock:
+	pte_unfreeze_dirty(page_table, writable);
+out_failed_unlock_nofreeze:
 	spin_unlock(&tsk->mm->page_table_lock);
 out_failed:
 	return 0;