Possibly working for/exec speedup patch

This one has degraded slightly from the previous (still showing a 50% decrease in fork/exec, though).

This one, I can't actually seem to break under stress (although I'm sure someone else will manage...).

James

# This is a BitKeeper generated patch for the following project:
# Project Name: Linux kernel tree
# This patch format is intended for GNU patch command version 2.5 or higher. # This patch includes the following deltas:
# ChangeSet 1.1792 -> 1.1799
# mm/fremap.c 1.16 -> 1.18
# include/asm-parisc/cache.h 1.6 -> 1.8
# arch/parisc/kernel/pacache.S 1.5 -> 1.8
# include/asm-parisc/page.h 1.5 -> 1.8
# arch/parisc/kernel/entry.S 1.14 -> 1.17
# arch/parisc/kernel/cache.c 1.12 -> 1.15
# mm/memory.c 1.154 -> 1.157
# include/asm-parisc/tlb.h 1.1 -> 1.2
# arch/parisc/kernel/sys_parisc.c 1.17 -> 1.18
# include/asm-parisc/pgtable.h 1.16 -> 1.18
# include/asm-parisc/cacheflush.h 1.9 -> 1.14
# arch/parisc/kernel/signal.c 1.14 -> 1.15
#
# The following is the BitKeeper ChangeSet Log
# --------------------------------------------
# 04/04/14 [email protected] 1.1790.1.1
# ICI
# --------------------------------------------
# 04/04/14 [email protected] 1.1793

# hand
# --------------------------------------------
# 04/04/15	[email protected]	1.1794
# First draft of working stingy cache flushing
# --------------------------------------------
# 04/04/16	[email protected]	1.1795
# More clean ups
# --------------------------------------------
# 04/04/16	[email protected]	1.1796
# Fix flushing
# 
# - eliminate flush_icache_page()
# - make copy_user_page do correct flushing
# --------------------------------------------
# 04/04/18	[email protected]	1.1797
# Add missing page alignment to cache flush arguments
# --------------------------------------------
# 04/04/18	[email protected]	1.1798
# Add kernel dcache flush limitation
# --------------------------------------------
# 04/04/18	[email protected]	1.1799
# Eliminate the PG_dcache_dirty bit
# 
# We now rely on the FLUSH_PAGE bit telling us if
# the page needs flushing or not
# --------------------------------------------
#
diff -Nru a/arch/parisc/kernel/cache.c b/arch/parisc/kernel/cache.c
--- a/arch/parisc/kernel/cache.c	Sun Apr 18 12:07:31 2004
+++ b/arch/parisc/kernel/cache.c	Sun Apr 18 12:07:31 2004
@@ -68,12 +68,14 @@
 {
 	struct page *page = pte_page(pte);

- if (VALID_PAGE(page) && page->mapping &&
- test_bit(PG_dcache_dirty, &page->flags)) {
-
+ if (VALID_PAGE(page))
+ /* We used to check to see if the page needed flushing here.
+ * Now this API automatically detects whether the page
+ * needs flushing or not. This is used to defer calls to
+ * kernel page flushing until mmu update time.
+ *
+ * DO NOT REMOVE OR CONDITIONALISE THIS */
flush_kernel_dcache_page(page_address(page));
- clear_bit(PG_dcache_dirty, &page->flags);
- }
}

void
@@ -230,6 +232,7 @@
void __flush_dcache_page(struct page *page)
{
struct list_head *l;
+ pte_t *pte;

flush_kernel_dcache_page(page_address(page));

@@ -260,9 +263,10 @@
* have to find a congruent address with an existing
* translation */

- if (!translation_exists(mpnt, addr))
+ if (!(pte = translation_exists(mpnt, addr)))
continue;

+ pte_clear_flush(pte);
__flush_cache_page(mpnt, addr);

/* If we find an address to flush, that will also
@@ -293,15 +297,205 @@
/* This is just for speed. If the page translation isn't
* there there's no point exciting the nadtlb handler into
* a nullification frenzy */
- if(!translation_exists(mpnt, addr))
+ if(!(pte = translation_exists(mpnt, addr)))
continue;

+ pte_clear_flush(pte);
__flush_cache_page(mpnt, addr);

return;
}
}
EXPORT_SYMBOL(__flush_dcache_page);
+
+/* set to max pages to flush before a full flush. Zero means no limit */ +#define MAX_FLUSH_PAGES 0
+#undef DEBUG_PAGE_FLUSHING
+
+#ifdef DEBUG_PAGE_FLUSHING
+#define DBG(a...) printk(a)
+#else
+#define DBG(...)
+#endif
+
+#if (MAX_FLUSH_PAGES != 0)
+
+/* we get to use the bottom 12 bits of the addr for flags since the
+ * address must be page aligned */
+#define ICACHE_FLUSH_FLAG 0x1
+
+void flush_cache_mm(struct mm_struct *mm)
+{
+ struct vm_area_struct *vma;
+ unsigned long count = 0, actual_count = 0;
+ unsigned long sr3 = mfsp(3), cr25 = mfctl(25);
+ unsigned long *pages;
+
+ preempt_disable();
+ if(mm != current->active_mm) {
+ DBG("flush_tlb_mm: current MM is not active ");
+ /* FIXME: awful hack: move the process the mm belongs
+ * to temporarily to being the active one. This only
+ * works because we can never get back into user
+ * context from here. */
+ mtctl(__pa(mm->pgd), 25);
+ mtsp(mm->context, 3);
+ }
+
+ pages = kmalloc(MAX_FLUSH_PAGES * sizeof(unsigned long), GFP_ATOMIC);
+ if(!pages) {
+ printk(KERN_ERR "flush_tlb_mm: allocation failed: full flush\n");
+ goto full_flush;
+ }
+
+ for (vma = mm->mmap; vma != NULL; vma = vma->vm_next) {
+ unsigned long start;
+
+ pmd_t *pmd;
+ pgd_t *pgd;
+ pte_t *pte;
+
+ count += (vma->vm_end - vma->vm_start)/PAGE_SIZE;
+
+ for (start = vma->vm_start; start < vma->vm_end;
+ start += PAGE_SIZE) {
+ pgd = pgd_offset(mm, start);
+
+ if (pgd_none(*pgd)) {
+ start = (start & PGDIR_MASK) + PGDIR_SIZE - PAGE_SIZE;
+ continue;
+ }
+
+ pmd = pmd_offset(pgd, start);
+ if (pmd_none(*pmd)) {
+ start = (start & PMD_MASK) + PMD_SIZE - PAGE_SIZE;
+ continue;
+ }
+ pte = pte_offset_map(pmd, start);
+ if(pte_val(*pte)==0 || !pte_present(*pte))
+ continue;
+
+ /* FIXME: Here we could also skip over any
+ * shared mapping page (i.e. equivalently
+ * aliased) with at least one other user */
+
+ pages[actual_count] = start;
+
+ if (vma->vm_flags & VM_EXEC)
+ pages[actual_count] |= ICACHE_FLUSH_FLAG;
+ if(++actual_count >= MAX_FLUSH_PAGES)
+ goto full_flush_free;
+
+ }
+ }
+
+ DBG("FLUSHED %lu (actual %lu)\n", count, actual_count);
+ for(count = 0; count < actual_count; count++) {
+ unsigned long addr = pages[count] & PAGE_MASK;
+ flush_user_dcache_page(addr);
+ if(pages[count] & ICACHE_FLUSH_FLAG) {
+ flush_user_icache_page(addr);
+ pitlb_user(addr);
+ }
+ pdtlb_user(addr);
+ }
+ out_free:
+ kfree(pages);
+ out:
+ mtsp(sr3, 3);
+ mtctl(cr25, 25);
+ preempt_enable();
+ return;
+
+ full_flush_free:
+ DBG("flush_cache_mm: over max pages %ld (count %ld), flushing everything\n", actual_count, count);
+ flush_cache_all();
+ goto out_free;
+
+ full_flush:
+ flush_cache_all();
+ goto out;
+}
+
+#else
+
+void flush_cache_mm(struct mm_struct *mm)
+{
+ struct vm_area_struct *vma;
+ unsigned long count = 0, actual_count = 0;
+ unsigned long sr3 = mfsp(3), cr25 = mfctl(25);
+#if 1
+ static int flushed = 0;
+
+ if(unlikely(!flushed)) {
+ printk("flush_cache_mm: INIT FLUSH ALL\n");
+ flushed = 1;
+ flush_cache_all();
+ return;
+ }
+#endif
+ preempt_disable();
+ if(mm != current->active_mm) {
+ DBG("flush_tlb_mm: current MM is not active ");
+ /* FIXME: awful hack: move the process the mm belongs
+ * to temporarily to being the active one. This only
+ * works because we can never get back into user
+ * context from here. */
+ mtctl(__pa(mm->pgd), 25);
+ mtsp(mm->context, 3);
+ }
+
+ for (vma = mm->mmap; vma != NULL; vma = vma->vm_next) {
+ unsigned long start;
+
+ pmd_t *pmd;
+ pgd_t *pgd;
+ pte_t *pte;
+
+ count += (vma->vm_end - vma->vm_start)/PAGE_SIZE;
+
+ for (start = vma->vm_start; start < vma->vm_end;
+ start += PAGE_SIZE) {
+ pgd = pgd_offset(mm, start);
+
+ if (pgd_none(*pgd)) {
+ start = (start & PGDIR_MASK) + PGDIR_SIZE - PAGE_SIZE;
+ continue;
+ }
+
+ pmd = pmd_offset(pgd, start);
+ if (pmd_none(*pmd)) {
+ start = (start & PMD_MASK) + PMD_SIZE - PAGE_SIZE;
+ continue;
+ }
+ pte = pte_offset_map(pmd, start);
+ if(pte_val(*pte)==0 || !pte_present(*pte))
+ continue;
+
+ if(!pte_flush(*pte))
+ continue;
+
+ /* FIXME: Here we could also skip over any
+ * shared mapping page (i.e. equivalently
+ * aliased) with at least one other user */
+ pte_clear_flush(pte);
+ flush_user_dcache_page(start);
+ if (vma->vm_flags & VM_EXEC) {
+ flush_user_icache_page(start);
+ pitlb_user(start);
+ }
+ pdtlb_user(start);
+ actual_count++;
+ }
+ }
+ mtsp(sr3, 3);
+ mtctl(cr25, 25);
+ preempt_enable();
+ DBG("FLUSHED %lu (actual %lu)\n", count, actual_count);
+}
+#endif
+
+EXPORT_SYMBOL(flush_cache_mm);

/* Defined in arch/parisc/kernel/pacache.S */
EXPORT_SYMBOL(flush_kernel_dcache_range_asm);

diff -Nru a/arch/parisc/kernel/entry.S b/arch/parisc/kernel/entry.S
@@ -49,7 +49,6 @@
 	.level 2.0
 #endif
 
-	.import         pa_dbit_lock,data
 
 	/* space_to_prot macro creates a prot id from a space id */
 

@@ -1017,6 +1016,10 @@
 	or		t1,pte,t0	/* t0 has R bit set */
 	and,*<>         t1,pte,%r0      /* test and nullify if already set */
 	std             t0,0(ptp)       /* write back pte */
+	ldi		_PAGE_FLUSH,t1
+	or		t1,t0,t0
+	and,*<>		t1,pte,%r0
+	std		t0,0(ptp)
 
 	space_to_prot   spc prot        /* create prot id from space */
 	depd            pte,8,7,prot    /* add in prot bits from pte */

@@ -1093,7 +1096,7 @@
 	shladd           t0,3,ptp,ptp
 	ldi		_PAGE_ACCESSED,t1
 	ldd              0(ptp),pte
-	bb,>=,n          pte,_PAGE_PRESENT_BIT,nadtlb_check_flush_20w
+	bb,>=,n          pte,_PAGE_PRESENT_BIT,nadtlb_emulate
 
 	space_to_prot   spc prot        /* create prot id from space */
 	depd            pte,8,7,prot    /* add in prot bits from pte */

@@ -1112,23 +1115,6 @@
 	rfir
 	nop
 
-nadtlb_check_flush_20w:
-	bb,>=,n          pte,_PAGE_FLUSH_BIT,nadtlb_emulate
-
-	/* Insert a "flush only" translation */
-
-	depdi,z         7,7,3,prot
-	depdi           1,10,1,prot
-
-	/* Get rid of prot bits and convert to page addr for idtlbt */
-
-	depdi		0,63,12,pte
-	extrd,u         pte,56,52,pte
-	idtlbt          pte,prot
-
-	rfir
-	nop
-
 #else
 
 dtlb_miss_11:

@@ -1514,6 +1500,10 @@
 	or		t1,pte,t0	/* t0 has R bit set */
 	and,*<>         t1,pte,%r0      /* test and nullify if already set */
 	std             t0,0(ptp)       /* write back pte */
+	ldi		_PAGE_FLUSH,t1
+	or		t1,t0,t0
+	and,*<>		t1,pte,%r0
+	std		t0,0(ptp)
 
 	space_to_prot   spc prot        /* create prot id from space */
 	depd            pte,8,7,prot    /* add in prot bits from pte */

diff -Nru a/arch/parisc/kernel/signal.c b/arch/parisc/kernel/signal.c
@@ -375,10 +375,9 @@
 	}
 #endif
 
-	flush_user_dcache_range((unsigned long) &frame->tramp[0],
-			   (unsigned long) &frame->tramp[TRAMP_SIZE]);
-	flush_user_icache_range((unsigned long) &frame->tramp[0],
-			   (unsigned long) &frame->tramp[TRAMP_SIZE]);
+	__flush_cache_range(current->active_mm,
+			    (unsigned long) &frame->tramp[0],
+			    (unsigned long) &frame->tramp[TRAMP_SIZE]);
 
 	/* TRAMP Words 0-4, Lenght 5 = SIGRESTARTBLOCK_TRAMP
 	 * TRAMP Words 5-9, Length 4 = SIGRETURN_TRAMP

diff -Nru a/include/asm-parisc/cache.h b/include/asm-parisc/cache.h
@@ -46,7 +46,7 @@
 extern void flush_kernel_icache_range_asm(unsigned long, unsigned long);
 extern void flush_user_dcache_range_asm(unsigned long, unsigned long);
 extern void flush_kernel_dcache_range_asm(unsigned long, unsigned long);
-extern void flush_kernel_dcache_page(void *);
+extern void __flush_kernel_dcache_page(void *);
 extern void flush_kernel_icache_page(void *);
 extern void disable_sr_hashing(void);   /* turns off space register hashing */
 extern void disable_sr_hashing_asm(int); /* low level support for above */

@@ -63,9 +63,11 @@
 extern int icache_stride;
 extern struct pdc_cache_info cache_info;
 
-#define pdtlb(addr)         asm volatile("pdtlb 0(%%sr1,%0)" : : "r" (addr));
-#define pitlb(addr)         asm volatile("pitlb 0(%%sr1,%0)" : : "r" (addr));
-#define pdtlb_kernel(addr)  asm volatile("pdtlb 0(%0)" : : "r" (addr));
+#define pdtlb(addr)         asm volatile("pdtlb 0(%%sr1,%0)" : : "r" (addr))
+#define pitlb(addr)         asm volatile("pitlb 0(%%sr1,%0)" : : "r" (addr))
+#define pdtlb_kernel(addr)  asm volatile("pdtlb 0(%0)" : : "r" (addr))
+#define pdtlb_user(addr)    asm volatile("pdtlb 0(%%sr3,%0)" : : "r" (addr))
+#define pitlb_user(addr)    asm volatile("pitlb 0(%%sr3,%0)" : : "r" (addr))
 
 #endif /* ! __ASSEMBLY__ */
 

diff -Nru a/include/asm-parisc/page.h b/include/asm-parisc/page.h
@@ -19,15 +19,10 @@
 
 extern void purge_kernel_dcache_page(unsigned long);
 extern void copy_user_page_asm(void *to, void *from);
+extern void copy_user_page_vaddr_asm(void *to, void *from, unsigned long vaddr);
 extern void clear_user_page_asm(void *page, unsigned long vaddr);
 
-static inline void
-copy_user_page(void *vto, void *vfrom, unsigned long vaddr, struct page *pg)
-{
-	copy_user_page_asm(vto, vfrom);
-	flush_kernel_dcache_page(vto);
-	/* XXX: ppc flushes icache too, should we? */
-}
+/* see cacheflush.h for copy_user_page */
 
 static inline void
 clear_user_page(void *page, unsigned long vaddr, struct page *pg)

diff -Nru a/include/asm-parisc/tlb.h b/include/asm-parisc/tlb.h
@@ -7,7 +7,7 @@
 } while (0)
 
 #define tlb_start_vma(tlb, vma) \
-do {	if (!(tlb)->fullmm)	\
+do {	\
 		flush_cache_range(vma, vma->vm_start, vma->vm_end); \
 } while (0)