Skip to contentSkip to conversation

Cache flushing update

parisc-linux James Bottomley 1 participant 1 message Apr 29, 2004

Subsystems (4)
lore
Share & tools
lore
#0 lore raw

This one is against 2.6.6-rc3-pa1 and should work on every pa system type.

It seems safe: I can't actually find a stress test that breaks it.

The fork speedup varies between 50-60% depending on the machine you try it on (the smallest speedup is on pa1.1 systems with small caches).

James

# This is a BitKeeper generated patch for the following project:
# Project Name: Linux kernel tree
# This patch format is intended for GNU patch command version 2.5 or higher. # This patch includes the following deltas:
# ChangeSet 1.1599 -> 1.1601
# mm/fremap.c 1.17 -> 1.18
# include/asm-parisc/io.h 1.7 -> 1.8
# include/asm-parisc/cache.h 1.6 -> 1.8
# arch/parisc/kernel/pacache.S 1.5.1.1 -> 1.10
# include/asm-parisc/page.h 1.5.1.2 -> 1.9
# arch/parisc/kernel/entry.S 1.14.1.3 -> 1.19
# drivers/video/console/sticore.c 1.16 -> 1.17
# drivers/scsi/ncr53c8xx.c 1.39 -> 1.40
# include/asm-parisc/dma-mapping.h 1.7 -> 1.8
# arch/parisc/kernel/cache.c 1.12.1.4 -> 1.19
# mm/memory.c 1.161 -> 1.162
# include/asm-parisc/tlb.h 1.1 -> 1.2
# arch/parisc/kernel/sys_parisc.c 1.17 -> 1.18
# include/asm-parisc/pgtable.h 1.16.1.1 -> 1.20
# arch/parisc/kernel/pci-dma.c 1.10 -> 1.11
# include/asm-parisc/cacheflush.h 1.9.1.2 -> 1.18
# arch/parisc/kernel/signal.c 1.14 -> 1.15
#
# The following is the BitKeeper ChangeSet Log
# --------------------------------------------
# 04/04/28 [email protected] 1.1600
# Merge akpm changes
# --------------------------------------------
# 04/04/28 [email protected] 1.1601
# Fix BK botched merge
# --------------------------------------------
#

arch/parisc/kernel/cache.c +205 -10
diff -Nru a/arch/parisc/kernel/cache.c b/arch/parisc/kernel/cache.c
@@ -68,12 +68,14 @@
 {
 	struct page *page = pte_page(pte);
 
-	if (VALID_PAGE(page) && page_mapping(page) &&
-	    test_bit(PG_dcache_dirty, &page->flags)) {
-
+	if (VALID_PAGE(page))
+		/* We used to check to see if the page needed flushing here.
+		 * Now this API automatically detects whether the page
+		 * needs flushing or not.  This is used to defer calls to
+		 * kernel page flushing until mmu update time.
+		 *
+		 * DO NOT REMOVE OR CONDITIONALISE THIS */
 		flush_kernel_dcache_page(page_address(page));
-		clear_bit(PG_dcache_dirty, &page->flags);
-	}
 }
 
 void
@@ -231,6 +233,7 @@
 {
 	struct address_space *mapping = page_mapping(page);
 	struct list_head *l;
+	pte_t *pte;
 
 	flush_kernel_dcache_page(page_address(page));
 
@@ -261,10 +264,11 @@
 		 * have to find a congruent address with an existing
 		 * translation */
 
-		if (!translation_exists(mpnt, addr))
+		if (!(pte = translation_exists(mpnt, addr)))
 			continue;
 
-		__flush_cache_page(mpnt, addr);
+
+		__flush_cache_page(pte, mpnt, addr);
 
 		/* If we find an address to flush, that will also
 		 * bring all the private mappings up to date (see
@@ -294,18 +298,209 @@
 		/* This is just for speed.  If the page translation isn't
 		 * there there's no point exciting the nadtlb handler into
 		 * a nullification frenzy */
-		if(!translation_exists(mpnt, addr))
+		if(!(pte = translation_exists(mpnt, addr)))
 			continue;
 
-		__flush_cache_page(mpnt, addr);
+		__flush_cache_page(pte, mpnt, addr);
 
 		return;
 	}
 }
 EXPORT_SYMBOL(__flush_dcache_page);
 
+/* set to max pages to flush before a full flush.  Zero means no limit */
+#define MAX_FLUSH_PAGES 0
+#undef	DEBUG_PAGE_FLUSHING
+
+#ifdef DEBUG_PAGE_FLUSHING
+#define DBG(a...)	printk(a)
+#else
+#define DBG(...)
+#endif
+
+#if (MAX_FLUSH_PAGES != 0)
+
+/* we get to use the bottom 12 bits of the addr for flags since the
+ * address must be page aligned */
+#define ICACHE_FLUSH_FLAG	0x1
+
+void flush_cache_mm(struct mm_struct *mm)
+{
+	struct vm_area_struct *vma;
+	unsigned long count = 0, actual_count = 0;
+	unsigned long sr3 = mfsp(3), cr25 = mfctl(25);
+	unsigned long *pages;
+
+	preempt_disable();
+	if(mm != current->active_mm) {
+		DBG("flush_tlb_mm: current MM is not active ");
+		/* FIXME: awful hack: move the process the mm belongs
+		 * to temporarily to being the active one.  This only
+		 * works because we can never get back into user
+		 * context from here. */
+		mtctl(__pa(mm->pgd), 25);
+		mtsp(mm->context, 3);
+	}
+
+	pages = kmalloc(MAX_FLUSH_PAGES * sizeof(unsigned long), GFP_ATOMIC);
+	if(!pages) {
+		printk(KERN_ERR "flush_tlb_mm: allocation failed: full flush\n");
+		goto full_flush;
+	}
+
+	for (vma = mm->mmap; vma != NULL; vma = vma->vm_next) {
+		unsigned long start;
+
+		pmd_t *pmd;
+		pgd_t *pgd;
+		pte_t *pte;
+
+		count += (vma->vm_end - vma->vm_start)/PAGE_SIZE;
+
+		for (start = vma->vm_start; start < vma->vm_end;
+		     start += PAGE_SIZE) {
+			pgd = pgd_offset(mm, start);
+
+			if (pgd_none(*pgd)) {
+				start = (start & PGDIR_MASK) + PGDIR_SIZE - PAGE_SIZE;
+				continue;
+			}
+
+			pmd = pmd_offset(pgd, start);
+			if (pmd_none(*pmd)) {
+				start = (start & PMD_MASK) + PMD_SIZE - PAGE_SIZE;
+				continue;
+			}
+			pte = pte_offset_map(pmd, start);
+			if(pte_val(*pte)==0 || !pte_present(*pte))
+				continue;
+
+			/* FIXME: Here we could also skip over any
+			 * shared mapping page (i.e. equivalently
+			 * aliased) with at least one other user */
+			
+			pages[actual_count] = start;
+		 
+			if (vma->vm_flags & VM_EXEC)
+				pages[actual_count] |= ICACHE_FLUSH_FLAG;
+			if(++actual_count >= MAX_FLUSH_PAGES)
+				goto full_flush_free;
+
+		}
+	}
+			
+	DBG("FLUSHED %lu (actual %lu)\n", count, actual_count);
+	for(count = 0; count < actual_count; count++) {
+		unsigned long addr = pages[count] & PAGE_MASK;
+		flush_user_dcache_page(addr);
+		if(pages[count] & ICACHE_FLUSH_FLAG) {
+			flush_user_icache_page(addr);
+			pitlb_user(addr);
+		}
+		pdtlb_user(addr);
+	}
+ out_free:
+	kfree(pages);
+ out:
+	mtsp(sr3, 3);
+	mtctl(cr25, 25);
+	preempt_enable();
+	return;
+
+ full_flush_free:
+	DBG("flush_cache_mm: over max pages %ld (count %ld), flushing everything\n", actual_count, count);
+	flush_cache_all();
+	goto out_free;
+
+ full_flush:
+	flush_cache_all();
+	goto out;
+}
+
+#else
+
+void flush_cache_mm(struct mm_struct *mm)
+{
+	struct vm_area_struct *vma;
+	unsigned long count = 0, actual_count = 0;
+	unsigned long sr3 = mfsp(3), cr25 = mfctl(25);
+#if 1
+	static int flushed = 0;
+
+	if(unlikely(!flushed)) {
+		printk("flush_cache_mm: INIT FLUSH ALL\n");
+		flushed = 1;
+		flush_cache_all();
+		return;
+	}
+#endif
+	preempt_disable();
+	if(mm != current->active_mm) {
+		DBG("flush_tlb_mm: current MM is not active ");
+		/* FIXME: awful hack: move the process the mm belongs
+		 * to temporarily to being the active one.  This only
+		 * works because we can never get back into user
+		 * context from here. */
+		mtctl(__pa(mm->pgd), 25);
+		mtsp(mm->context, 3);
+	}
+
+	for (vma = mm->mmap; vma != NULL; vma = vma->vm_next) {
+		unsigned long start;
+
+		pmd_t *pmd;
+		pgd_t *pgd;
+		pte_t *pte;
+
+		count += (vma->vm_end - vma->vm_start)/PAGE_SIZE;
+
+		for (start = vma->vm_start; start < vma->vm_end;
+		     start += PAGE_SIZE) {
+			pgd = pgd_offset(mm, start);
+
+			if (pgd_none(*pgd)) {
+				start = (start & PGDIR_MASK) + PGDIR_SIZE - PAGE_SIZE;
+				continue;
+			}
+
+			pmd = pmd_offset(pgd, start);
+			if (pmd_none(*pmd)) {
+				start = (start & PMD_MASK) + PMD_SIZE - PAGE_SIZE;
+				continue;
+			}
+			pte = pte_offset_map(pmd, start);
+			if(!pte_present(*pte))
+				continue;
+
+			if(!pte_flush(*pte))
+				continue;
+
+			/* FIXME: Here we could also skip over any
+			 * shared mapping page (i.e. equivalently
+			 * aliased) with at least one other user */
+			page_begin_flush(pte);
+			pdtlb_user(start);
+			flush_user_dcache_page(start);
+			if (vma->vm_flags & VM_EXEC) {
+				flush_user_icache_page(start);
+				pitlb_user(start);
+			}
+			page_end_flush(pte);
+			pdtlb_user(start);
+			actual_count++;
+		}
+	}
+	mtsp(sr3, 3);
+	mtctl(cr25, 25);
+	preempt_enable();
+	DBG("FLUSHED %lu (actual %lu)\n", count, actual_count);
+}
+#endif
+
+EXPORT_SYMBOL(flush_cache_mm);
+
 /* Defined in arch/parisc/kernel/pacache.S */
 EXPORT_SYMBOL(flush_kernel_dcache_range_asm);
-EXPORT_SYMBOL(flush_kernel_dcache_page);
+EXPORT_SYMBOL(__flush_kernel_dcache_page);
 EXPORT_SYMBOL(flush_data_cache_local);
 EXPORT_SYMBOL(flush_kernel_icache_range_asm);
arch/parisc/kernel/entry.S +11 -65

139 lines not on this page · raw message

arch/parisc/kernel/pacache.S +63 -10

157 lines not on this page · raw message

arch/parisc/kernel/pci-dma.c +21 -12
diff -Nru a/arch/parisc/kernel/pci-dma.c b/arch/parisc/kernel/pci-dma.c
@@ -362,7 +362,7 @@
 	size = 1 << (order + PAGE_SHIFT);
 	vaddr = pcxl_alloc_range(size);
 	paddr = __get_free_pages(flag, order);
-	flush_kernel_dcache_range(paddr, size);
+	flush_kernel_dcache_range_asm(paddr, paddr + size);
 	paddr = __pa(paddr);
 	map_uncached_pages(vaddr, size, paddr);
 	*dma_handle = (dma_addr_t) paddr;
@@ -396,12 +396,13 @@
 		BUG();
 	}
 
-	flush_kernel_dcache_range((unsigned long) addr, size);
+	flush_kernel_dcache_range_asm((unsigned long) addr, (unsigned long)addr + size);
 	return virt_to_phys(addr);
 }
 
 static void pa11_dma_unmap_single(struct device *dev, dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
 {
+	unsigned long addr = (unsigned long)phys_to_virt(dma_handle);
 	if (direction == DMA_NONE) {
 		printk(KERN_ERR "pa11_dma_unmap_single(PCI_DMA_NONE) called by %p\n", __builtin_return_address(0));
 		BUG();
@@ -416,7 +417,7 @@
 	 * pci_dma_sync_single_* has been called and the buffer reused.
 	 */
 
-	flush_kernel_dcache_range((unsigned long) phys_to_virt(dma_handle), size);
+	flush_kernel_dcache_range_asm(addr, addr + size);
 	return;
 }
 
@@ -431,7 +432,7 @@
 		unsigned long vaddr = sg_virt_addr(sglist);
 		sg_dma_address(sglist) = (dma_addr_t) virt_to_phys(vaddr);
 		sg_dma_len(sglist) = sglist->length;
-		flush_kernel_dcache_range(vaddr, sglist->length);
+		flush_kernel_dcache_range_asm(vaddr, vaddr + sglist->length);
 	}
 	return nents;
 }
@@ -448,25 +449,29 @@
 
 	/* once we do combining we'll need to use phys_to_virt(sg_dma_address(sglist)) */
 
-	for (i = 0; i < nents; i++, sglist++ )
-		flush_kernel_dcache_range(sg_virt_addr(sglist), sglist->length);
+	for (i = 0; i < nents; i++, sglist++ ) {
+		unsigned long vaddr = sg_virt_addr(sglist);
+		flush_kernel_dcache_range_asm(vaddr, vaddr + sglist->length);
+	}
 	return;
 }
 
 static void pa11_dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle, unsigned long offset, size_t size, enum dma_data_direction direction)
 {
+	unsigned long addr = (unsigned long)phys_to_virt(dma_handle) + offset;
 	if (direction == DMA_NONE)
 	    BUG();
 
-	flush_kernel_dcache_range((unsigned long) phys_to_virt(dma_handle) + offset, size);
+	flush_kernel_dcache_range_asm(addr, addr + size);
 }
 
 static void pa11_dma_sync_single_for_device(struct device *dev, dma_addr_t dma_handle, unsigned long offset, size_t size, enum dma_data_direction direction)
 {
+	unsigned long addr = (unsigned long)phys_to_virt(dma_handle) + offset;
 	if (direction == DMA_NONE)
 	    BUG();
 
-	flush_kernel_dcache_range((unsigned long) phys_to_virt(dma_handle) + offset, size);
+	flush_kernel_dcache_range_asm(addr, addr + size);
 }
 
 static void pa11_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sglist, int nents, enum dma_data_direction direction)
@@ -475,8 +480,10 @@
 
 	/* once we do combining we'll need to use phys_to_virt(sg_dma_address(sglist)) */
 
-	for (i = 0; i < nents; i++, sglist++ )
-		flush_kernel_dcache_range(sg_virt_addr(sglist), sglist->length);
+	for (i = 0; i < nents; i++, sglist++ ) {
+		unsigned long vaddr = sg_virt_addr(sglist);
+		flush_kernel_dcache_range_asm(vaddr, vaddr + sglist->length);
+	}
 }
 
 static void pa11_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sglist, int nents, enum dma_data_direction direction)
@@ -485,8 +492,10 @@
 
 	/* once we do combining we'll need to use phys_to_virt(sg_dma_address(sglist)) */
 
-	for (i = 0; i < nents; i++, sglist++ )
-		flush_kernel_dcache_range(sg_virt_addr(sglist), sglist->length);
+	for (i = 0; i < nents; i++, sglist++ ) {
+		unsigned long vaddr = sg_virt_addr(sglist);
+		flush_kernel_dcache_range_asm(vaddr, vaddr + sglist->length);
+	}
 }
 
 struct hppa_dma_ops pcxl_dma_ops = {
arch/parisc/kernel/signal.c +3 -4
diff -Nru a/arch/parisc/kernel/signal.c b/arch/parisc/kernel/signal.c
@@ -375,10 +375,9 @@
 	}
 #endif
 
-	flush_user_dcache_range((unsigned long) &frame->tramp[0],
-			   (unsigned long) &frame->tramp[TRAMP_SIZE]);
-	flush_user_icache_range((unsigned long) &frame->tramp[0],
-			   (unsigned long) &frame->tramp[TRAMP_SIZE]);
+	__flush_cache_range(current->active_mm,
+			    (unsigned long) &frame->tramp[0],
+			    (unsigned long) &frame->tramp[TRAMP_SIZE]);
 
 	/* TRAMP Words 0-4, Lenght 5 = SIGRESTARTBLOCK_TRAMP
 	 * TRAMP Words 5-9, Length 4 = SIGRETURN_TRAMP
drivers/scsi/ncr53c8xx.c +1 -0
diff -Nru a/drivers/scsi/ncr53c8xx.c b/drivers/scsi/ncr53c8xx.c
@@ -8621,6 +8621,7 @@
 	tpnt->sg_tablesize	= SCSI_NCR_SG_TABLESIZE;
 	tpnt->cmd_per_lun	= SCSI_NCR_CMD_PER_LUN;
 	tpnt->use_clustering	= DISABLE_CLUSTERING;
+	tpnt->max_sectors	= 8;
 
 	if (device->differential)
 		driver_setup.diff_support = device->differential;
drivers/video/console/sticore.c +1 -1
diff -Nru a/drivers/video/console/sticore.c b/drivers/video/console/sticore.c
@@ -237,7 +237,7 @@
 static void sti_flush(unsigned long from, unsigned long len)
 {
 	flush_data_cache();
-	flush_kernel_dcache_range(from, len);
+	flush_kernel_dcache_range(from, from+len);
 	flush_icache_range(from, from+len);
 }
 
include/asm-parisc/cache.h +6 -4
diff -Nru a/include/asm-parisc/cache.h b/include/asm-parisc/cache.h
@@ -46,7 +46,7 @@
 extern void flush_kernel_icache_range_asm(unsigned long, unsigned long);
 extern void flush_user_dcache_range_asm(unsigned long, unsigned long);
 extern void flush_kernel_dcache_range_asm(unsigned long, unsigned long);
-extern void flush_kernel_dcache_page(void *);
+extern void __flush_kernel_dcache_page(void *);
 extern void flush_kernel_icache_page(void *);
 extern void disable_sr_hashing(void);   /* turns off space register hashing */
 extern void disable_sr_hashing_asm(int); /* low level support for above */
@@ -63,9 +63,11 @@
 extern int icache_stride;
 extern struct pdc_cache_info cache_info;
 
-#define pdtlb(addr)         asm volatile("pdtlb 0(%%sr1,%0)" : : "r" (addr));
-#define pitlb(addr)         asm volatile("pitlb 0(%%sr1,%0)" : : "r" (addr));
-#define pdtlb_kernel(addr)  asm volatile("pdtlb 0(%0)" : : "r" (addr));
+#define pdtlb(addr)         asm volatile("pdtlb 0(%%sr1,%0)" : : "r" (addr))
+#define pitlb(addr)         asm volatile("pitlb 0(%%sr1,%0)" : : "r" (addr))
+#define pdtlb_kernel(addr)  asm volatile("pdtlb 0(%0)" : : "r" (addr))
+#define pdtlb_user(addr)    asm volatile("pdtlb 0(%%sr3,%0)" : : "r" (addr))
+#define pitlb_user(addr)    asm volatile("pitlb 0(%%sr3,%0)" : : "r" (addr))
 
 #endif /* ! __ASSEMBLY__ */
 
include/asm-parisc/cacheflush.h +162 -88

319 lines not on this page · raw message

include/asm-parisc/dma-mapping.h +2 -1
diff -Nru a/include/asm-parisc/dma-mapping.h b/include/asm-parisc/dma-mapping.h
@@ -200,8 +200,9 @@
 dma_cache_sync(void *vaddr, size_t size,
 	       enum dma_data_direction direction)
 {
+	unsigned long start = (unsigned long)vaddr, end = start + size;
 	if(hppa_dma_ops->dma_sync_single_for_cpu)
-		flush_kernel_dcache_range((unsigned long)vaddr, size);
+		flush_kernel_dcache_range_asm(start, end);
 }
 
 static inline void *
include/asm-parisc/io.h +3 -3
diff -Nru a/include/asm-parisc/io.h b/include/asm-parisc/io.h
@@ -298,9 +298,9 @@
 #define IO_SPACE_LIMIT 0x00ffffff
 
 
-#define dma_cache_inv(_start,_size)		do { flush_kernel_dcache_range(_start,_size); } while (0)
-#define dma_cache_wback(_start,_size)		do { flush_kernel_dcache_range(_start,_size); } while (0)
-#define dma_cache_wback_inv(_start,_size)	do { flush_kernel_dcache_range(_start,_size); } while (0)
+#define dma_cache_inv(_start,_size)		do { flush_kernel_dcache_range_asm(_start,(_start) + (_size)); } while (0)
+#define dma_cache_wback(_start,_size)		do { flush_kernel_dcache_range_asm(_start,(_start) + (_size)); } while (0)
+#define dma_cache_wback_inv(_start,_size)	do { flush_kernel_dcache_range_asm(_start,(_start) + (_size)); } while (0)
 
 /* PA machines have an MM I/O space from 0xf0000000-0xffffffff in 32
  * bit mode and from 0xfffffffff0000000-0xfffffffffffffff in 64 bit
include/asm-parisc/page.h +2 -7
diff -Nru a/include/asm-parisc/page.h b/include/asm-parisc/page.h
@@ -19,15 +19,10 @@
 
 extern void purge_kernel_dcache_page(unsigned long);
 extern void copy_user_page_asm(void *to, void *from);
+extern void copy_user_page_vaddr_asm(void *to, void *from, unsigned long vaddr);
 extern void clear_user_page_asm(void *page, unsigned long vaddr);
 
-static inline void
-copy_user_page(void *vto, void *vfrom, unsigned long vaddr, struct page *pg)
-{
-	copy_user_page_asm(vto, vfrom);
-	flush_kernel_dcache_page(vto);
-	/* XXX: ppc flushes icache too, should we? */
-}
+/* see cacheflush.h for copy_user_page */
 
 static inline void
 clear_user_page(void *page, unsigned long vaddr, struct page *pg)
include/asm-parisc/pgtable.h +38 -30

121 lines not on this page · raw message

include/asm-parisc/tlb.h +1 -1
diff -Nru a/include/asm-parisc/tlb.h b/include/asm-parisc/tlb.h
@@ -7,7 +7,7 @@
 } while (0)
 
 #define tlb_start_vma(tlb, vma) \
-do {	if (!(tlb)->fullmm)	\
+do {	\
 		flush_cache_range(vma, vma->vm_start, vma->vm_end); \
 } while (0)