Another cache flush & fork/exec speedup

This one still has problems. It's self hosting, but it falls over under heavy stress (with segfaults that seem to be related to COW).

So, treat it with caution.

The benchmarks show that the fork/exec time has basically been cut in half in this incarnation.

My next task will be to instrument the kernel page flushing like the user one has been done.

After this it's the hard tasks:

1. Alter the page table layout to recover an extra flag for separate I/D cache flushing
2. work out a scheme for doing last man out flushes of congruently mapped shared pages.

James

# This is a BitKeeper generated patch for the following project:
# Project Name: Linux kernel tree
# This patch format is intended for GNU patch command version 2.5 or higher. # This patch includes the following deltas:
# ChangeSet 1.1792 -> 1.1796
# mm/fremap.c 1.16 -> 1.18
# include/asm-parisc/cache.h 1.6 -> 1.7
# arch/parisc/kernel/pacache.S 1.5 -> 1.6
# include/asm-parisc/page.h 1.5 -> 1.8
# arch/parisc/kernel/entry.S 1.14 -> 1.17
# arch/parisc/kernel/cache.c 1.12 -> 1.14
# mm/memory.c 1.154 -> 1.156
# include/asm-parisc/tlb.h 1.1 -> 1.2
# arch/parisc/kernel/sys_parisc.c 1.17 -> 1.18
# include/asm-parisc/pgtable.h 1.16 -> 1.17
# include/asm-parisc/cacheflush.h 1.9 -> 1.12
# arch/parisc/kernel/signal.c 1.14 -> 1.15
#
# The following is the BitKeeper ChangeSet Log
# --------------------------------------------
# 04/04/14 [email protected] 1.1790.1.1
# ICI
# --------------------------------------------
# 04/04/14 [email protected] 1.1793

# hand
# --------------------------------------------
# 04/04/15	[email protected]	1.1794
# First draft of working stingy cache flushing
# --------------------------------------------
# 04/04/16	[email protected]	1.1795
# More clean ups
# --------------------------------------------
# 04/04/16	[email protected]	1.1796
# Fix flushing
# 
# - eliminate flush_icache_page()
# - make copy_user_page do correct flushing
# --------------------------------------------
#
diff -Nru a/arch/parisc/kernel/cache.c b/arch/parisc/kernel/cache.c
--- a/arch/parisc/kernel/cache.c	Sat Apr 17 19:16:12 2004
+++ b/arch/parisc/kernel/cache.c	Sat Apr 17 19:16:12 2004
@@ -230,6 +230,7 @@
 void __flush_dcache_page(struct page *page)
 {
 	struct list_head *l;
+	pte_t *pte;

flush_kernel_dcache_page(page_address(page));

@@ -260,9 +261,10 @@
* have to find a congruent address with an existing
* translation */

- if (!translation_exists(mpnt, addr))
+ if (!(pte = translation_exists(mpnt, addr)))
continue;

+ pte_clear_flush(pte);
__flush_cache_page(mpnt, addr);

/* If we find an address to flush, that will also
@@ -293,15 +295,205 @@
/* This is just for speed. If the page translation isn't
* there there's no point exciting the nadtlb handler into
* a nullification frenzy */
- if(!translation_exists(mpnt, addr))
+ if(!(pte = translation_exists(mpnt, addr)))
continue;

+ pte_clear_flush(pte);
__flush_cache_page(mpnt, addr);

return;
}
}
EXPORT_SYMBOL(__flush_dcache_page);
+
+/* set to max pages to flush before a full flush. Zero means no limit */ +#define MAX_FLUSH_PAGES 0
+#undef DEBUG_PAGE_FLUSHING
+
+#ifdef DEBUG_PAGE_FLUSHING
+#define DBG(a...) printk(a)
+#else
+#define DBG(...)
+#endif
+
+#if (MAX_FLUSH_PAGES != 0)
+
+/* we get to use the bottom 12 bits of the addr for flags since the
+ * address must be page aligned */
+#define ICACHE_FLUSH_FLAG 0x1
+
+void flush_cache_mm(struct mm_struct *mm)
+{
+ struct vm_area_struct *vma;
+ unsigned long count = 0, actual_count = 0;
+ unsigned long sr3 = mfsp(3), cr25 = mfctl(25);
+ unsigned long *pages;
+
+ preempt_disable();
+ if(mm != current->active_mm) {
+ DBG("flush_tlb_mm: current MM is not active ");
+ /* FIXME: awful hack: move the process the mm belongs
+ * to temporarily to being the active one. This only
+ * works because we can never get back into user
+ * context from here. */
+ mtctl(__pa(mm->pgd), 25);
+ mtsp(mm->context, 3);
+ }
+
+ pages = kmalloc(MAX_FLUSH_PAGES * sizeof(unsigned long), GFP_ATOMIC);
+ if(!pages) {
+ printk(KERN_ERR "flush_tlb_mm: allocation failed: full flush\n");
+ goto full_flush;
+ }
+
+ for (vma = mm->mmap; vma != NULL; vma = vma->vm_next) {
+ unsigned long start;
+
+ pmd_t *pmd;
+ pgd_t *pgd;
+ pte_t *pte;
+
+ count += (vma->vm_end - vma->vm_start)/PAGE_SIZE;
+
+ for (start = vma->vm_start; start < vma->vm_end;
+ start += PAGE_SIZE) {
+ pgd = pgd_offset(mm, start);
+
+ if (pgd_none(*pgd)) {
+ start = (start & PGDIR_MASK) + PGDIR_SIZE - PAGE_SIZE;
+ continue;
+ }
+
+ pmd = pmd_offset(pgd, start);
+ if (pmd_none(*pmd)) {
+ start = (start & PMD_MASK) + PMD_SIZE - PAGE_SIZE;
+ continue;
+ }
+ pte = pte_offset_map(pmd, start);
+ if(pte_val(*pte)==0 || !pte_present(*pte))
+ continue;
+
+ /* FIXME: Here we could also skip over any
+ * shared mapping page (i.e. equivalently
+ * aliased) with at least one other user */
+
+ pages[actual_count] = start;
+
+ if (vma->vm_flags & VM_EXEC)
+ pages[actual_count] |= ICACHE_FLUSH_FLAG;
+ if(++actual_count >= MAX_FLUSH_PAGES)
+ goto full_flush_free;
+
+ }
+ }
+
+ DBG("FLUSHED %lu (actual %lu)\n", count, actual_count);
+ for(count = 0; count < actual_count; count++) {
+ unsigned long addr = pages[count] & PAGE_MASK;
+ flush_user_dcache_page(addr);
+ if(pages[count] & ICACHE_FLUSH_FLAG) {
+ flush_user_icache_page(addr);
+ pitlb_user(addr);
+ }
+ pdtlb_user(addr);
+ }
+ out_free:
+ kfree(pages);
+ out:
+ mtsp(sr3, 3);
+ mtctl(cr25, 25);
+ preempt_enable();
+ return;
+
+ full_flush_free:
+ DBG("flush_cache_mm: over max pages %ld (count %ld), flushing everything\n", actual_count, count);
+ flush_cache_all();
+ goto out_free;
+
+ full_flush:
+ flush_cache_all();
+ goto out;
+}
+
+#else
+
+void flush_cache_mm(struct mm_struct *mm)
+{
+ struct vm_area_struct *vma;
+ unsigned long count = 0, actual_count = 0;
+ unsigned long sr3 = mfsp(3), cr25 = mfctl(25);
+#if 1
+ static int flushed = 0;
+
+ if(unlikely(!flushed)) {
+ printk("flush_cache_mm: INIT FLUSH ALL\n");
+ flushed = 1;
+ flush_cache_all();
+ return;
+ }
+#endif
+ preempt_disable();
+ if(mm != current->active_mm) {
+ DBG("flush_tlb_mm: current MM is not active ");
+ /* FIXME: awful hack: move the process the mm belongs
+ * to temporarily to being the active one. This only
+ * works because we can never get back into user
+ * context from here. */
+ mtctl(__pa(mm->pgd), 25);
+ mtsp(mm->context, 3);
+ }
+
+ for (vma = mm->mmap; vma != NULL; vma = vma->vm_next) {
+ unsigned long start;
+
+ pmd_t *pmd;
+ pgd_t *pgd;
+ pte_t *pte;
+
+ count += (vma->vm_end - vma->vm_start)/PAGE_SIZE;
+
+ for (start = vma->vm_start; start < vma->vm_end;
+ start += PAGE_SIZE) {
+ pgd = pgd_offset(mm, start);
+
+ if (pgd_none(*pgd)) {
+ start = (start & PGDIR_MASK) + PGDIR_SIZE - PAGE_SIZE;
+ continue;
+ }
+
+ pmd = pmd_offset(pgd, start);
+ if (pmd_none(*pmd)) {
+ start = (start & PMD_MASK) + PMD_SIZE - PAGE_SIZE;
+ continue;
+ }
+ pte = pte_offset_map(pmd, start);
+ if(pte_val(*pte)==0 || !pte_present(*pte))
+ continue;
+
+ if(!pte_flush(*pte))
+ continue;
+
+ /* FIXME: Here we could also skip over any
+ * shared mapping page (i.e. equivalently
+ * aliased) with at least one other user */
+ pte_clear_flush(pte);
+ flush_user_dcache_page(start);
+ if (vma->vm_flags & VM_EXEC) {
+ flush_user_icache_page(start);
+ pitlb_user(start);
+ }
+ pdtlb_user(start);
+ actual_count++;
+ }
+ }
+ mtsp(sr3, 3);
+ mtctl(cr25, 25);
+ preempt_enable();
+ DBG("FLUSHED %lu (actual %lu)\n", count, actual_count);
+}
+#endif
+
+EXPORT_SYMBOL(flush_cache_mm);

/* Defined in arch/parisc/kernel/pacache.S */
EXPORT_SYMBOL(flush_kernel_dcache_range_asm);

diff -Nru a/arch/parisc/kernel/entry.S b/arch/parisc/kernel/entry.S
@@ -49,7 +49,6 @@
 	.level 2.0
 #endif
 
-	.import         pa_dbit_lock,data
 
 	/* space_to_prot macro creates a prot id from a space id */
 

@@ -1017,6 +1016,10 @@
 	or		t1,pte,t0	/* t0 has R bit set */
 	and,*<>         t1,pte,%r0      /* test and nullify if already set */
 	std             t0,0(ptp)       /* write back pte */
+	ldi		_PAGE_FLUSH,t1
+	or		t1,t0,t0
+	and,*<>		t1,pte,%r0
+	std		t0,0(ptp)
 
 	space_to_prot   spc prot        /* create prot id from space */
 	depd            pte,8,7,prot    /* add in prot bits from pte */

@@ -1093,7 +1096,7 @@
 	shladd           t0,3,ptp,ptp
 	ldi		_PAGE_ACCESSED,t1
 	ldd              0(ptp),pte
-	bb,>=,n          pte,_PAGE_PRESENT_BIT,nadtlb_check_flush_20w
+	bb,>=,n          pte,_PAGE_PRESENT_BIT,nadtlb_emulate
 
 	space_to_prot   spc prot        /* create prot id from space */
 	depd            pte,8,7,prot    /* add in prot bits from pte */

@@ -1112,23 +1115,6 @@
 	rfir
 	nop
 
-nadtlb_check_flush_20w:
-	bb,>=,n          pte,_PAGE_FLUSH_BIT,nadtlb_emulate
-
-	/* Insert a "flush only" translation */
-
-	depdi,z         7,7,3,prot
-	depdi           1,10,1,prot
-
-	/* Get rid of prot bits and convert to page addr for idtlbt */
-
-	depdi		0,63,12,pte
-	extrd,u         pte,56,52,pte
-	idtlbt          pte,prot
-
-	rfir
-	nop
-
 #else
 
 dtlb_miss_11:

@@ -1514,6 +1500,10 @@
 	or		t1,pte,t0	/* t0 has R bit set */
 	and,*<>         t1,pte,%r0      /* test and nullify if already set */
 	std             t0,0(ptp)       /* write back pte */
+	ldi		_PAGE_FLUSH,t1
+	or		t1,t0,t0
+	and,*<>		t1,pte,%r0
+	std		t0,0(ptp)
 
 	space_to_prot   spc prot        /* create prot id from space */
 	depd            pte,8,7,prot    /* add in prot bits from pte */

diff -Nru a/arch/parisc/kernel/signal.c b/arch/parisc/kernel/signal.c
@@ -375,10 +375,9 @@
 	}
 #endif
 
-	flush_user_dcache_range((unsigned long) &frame->tramp[0],
-			   (unsigned long) &frame->tramp[TRAMP_SIZE]);
-	flush_user_icache_range((unsigned long) &frame->tramp[0],
-			   (unsigned long) &frame->tramp[TRAMP_SIZE]);
+	__flush_cache_range(current->active_mm,
+			    (unsigned long) &frame->tramp[0],
+			    (unsigned long) &frame->tramp[TRAMP_SIZE]);
 
 	/* TRAMP Words 0-4, Lenght 5 = SIGRESTARTBLOCK_TRAMP
 	 * TRAMP Words 5-9, Length 4 = SIGRETURN_TRAMP

diff -Nru a/include/asm-parisc/cache.h b/include/asm-parisc/cache.h
@@ -63,9 +63,11 @@
 extern int icache_stride;
 extern struct pdc_cache_info cache_info;
 
-#define pdtlb(addr)         asm volatile("pdtlb 0(%%sr1,%0)" : : "r" (addr));
-#define pitlb(addr)         asm volatile("pitlb 0(%%sr1,%0)" : : "r" (addr));
-#define pdtlb_kernel(addr)  asm volatile("pdtlb 0(%0)" : : "r" (addr));
+#define pdtlb(addr)         asm volatile("pdtlb 0(%%sr1,%0)" : : "r" (addr))
+#define pitlb(addr)         asm volatile("pitlb 0(%%sr1,%0)" : : "r" (addr))
+#define pdtlb_kernel(addr)  asm volatile("pdtlb 0(%0)" : : "r" (addr))
+#define pdtlb_user(addr)    asm volatile("pdtlb 0(%%sr3,%0)" : : "r" (addr))
+#define pitlb_user(addr)    asm volatile("pitlb 0(%%sr3,%0)" : : "r" (addr))
 
 #endif /* ! __ASSEMBLY__ */
 

diff -Nru a/include/asm-parisc/page.h b/include/asm-parisc/page.h
@@ -19,15 +19,10 @@
 
 extern void purge_kernel_dcache_page(unsigned long);
 extern void copy_user_page_asm(void *to, void *from);
+extern void copy_user_page_vaddr_asm(void *to, void *from, unsigned long vaddr);
 extern void clear_user_page_asm(void *page, unsigned long vaddr);
 
-static inline void
-copy_user_page(void *vto, void *vfrom, unsigned long vaddr, struct page *pg)
-{
-	copy_user_page_asm(vto, vfrom);
-	flush_kernel_dcache_page(vto);
-	/* XXX: ppc flushes icache too, should we? */
-}
+/* see cacheflush.h for copy_user_page */
 
 static inline void
 clear_user_page(void *page, unsigned long vaddr, struct page *pg)

diff -Nru a/include/asm-parisc/tlb.h b/include/asm-parisc/tlb.h
@@ -7,7 +7,7 @@
 } while (0)
 
 #define tlb_start_vma(tlb, vma) \
-do {	if (!(tlb)->fullmm)	\
+do {	\
 		flush_cache_range(vma, vma->vm_start, vma->vm_end); \
 } while (0)
 

diff -Nru a/mm/memory.c b/mm/memory.c
@@ -1035,11 +1035,13 @@
 		int reuse = can_share_swap_page(old_page);
 		unlock_page(old_page);
 		if (reuse) {
-			flush_cache_page(vma, address);
+			//flush_cache_page(vma, address);
+			flush_dcache_page(old_page);
 			entry = maybe_mkwrite(pte_mkyoung(pte_mkdirty(pte)),
 					      vma);
 			ptep_establish(vma, address, page_table, entry);
 			update_mmu_cache(vma, address, entry);
+			//flush_kernel_dcache_page(page_address(pte_page(entry)));
 			pte_unmap(page_table);
 			spin_unlock(&mm->page_table_lock);
 			return VM_FAULT_MINOR;