Cache flushing update
Subsystems (4)
This one is against 2.6.6-rc3-pa1 and should work on every pa system type.
It seems safe: I can't actually find a stress test that breaks it.
The fork speedup varies between 50-60% depending on the machine you try it on (the smallest speedup is on pa1.1 systems with small caches).
James
# This is a BitKeeper generated patch for the following project:
# Project Name: Linux kernel tree
# This patch format is intended for GNU patch command version 2.5 or higher. # This patch includes the following deltas:
# ChangeSet 1.1599 -> 1.1601
# mm/fremap.c 1.17 -> 1.18
# include/asm-parisc/io.h 1.7 -> 1.8
# include/asm-parisc/cache.h 1.6 -> 1.8
# arch/parisc/kernel/pacache.S 1.5.1.1 -> 1.10
# include/asm-parisc/page.h 1.5.1.2 -> 1.9
# arch/parisc/kernel/entry.S 1.14.1.3 -> 1.19
# drivers/video/console/sticore.c 1.16 -> 1.17
# drivers/scsi/ncr53c8xx.c 1.39 -> 1.40
# include/asm-parisc/dma-mapping.h 1.7 -> 1.8
# arch/parisc/kernel/cache.c 1.12.1.4 -> 1.19
# mm/memory.c 1.161 -> 1.162
# include/asm-parisc/tlb.h 1.1 -> 1.2
# arch/parisc/kernel/sys_parisc.c 1.17 -> 1.18
# include/asm-parisc/pgtable.h 1.16.1.1 -> 1.20
# arch/parisc/kernel/pci-dma.c 1.10 -> 1.11
# include/asm-parisc/cacheflush.h 1.9.1.2 -> 1.18
# arch/parisc/kernel/signal.c 1.14 -> 1.15
#
# The following is the BitKeeper ChangeSet Log
# --------------------------------------------
# 04/04/28 [email protected] 1.1600
# Merge akpm changes
# --------------------------------------------
# 04/04/28 [email protected] 1.1601
# Fix BK botched merge
# --------------------------------------------
#
arch/parisc/kernel/cache.c +205 -10
@@ -68,12 +68,14 @@ {struct page *page =pte_page (pte); -if (VALID_PAGE (page) &&page_mapping (page) && -test_bit (PG_dcache_dirty, &page->flags)) { - +if (VALID_PAGE (page)) +/* We used to check to see if the page needed flushing here. +* Now this API automatically detects whether the page +* needs flushing or not. This is used to defer calls to +* kernel page flushing until mmu update time. +* +* DO NOT REMOVE OR CONDITIONALISE THIS */ flush_kernel_dcache_page (page_address (page)); -clear_bit (PG_dcache_dirty, &page->flags); - } }void
@@ -231,6 +233,7 @@ {struct address_space *mapping =page_mapping (page);struct list_head *l; +pte_t *pte;flush_kernel_dcache_page (page_address (page));
@@ -261,10 +264,11 @@* have to find a congruent address with an existing * translation */ -if (!translation_exists (mpnt, addr)) +if (!(pte =translation_exists (mpnt, addr)))continue ; -__flush_cache_page (mpnt, addr); + +__flush_cache_page (pte, mpnt, addr);/* If we find an address to flush, that will also * bring all the private mappings up to date (see
@@ -294,18 +298,209 @@/* This is just for speed. If the page translation isn't * there there's no point exciting the nadtlb handler into * a nullification frenzy */ -if (!translation_exists (mpnt, addr)) +if (!(pte =translation_exists (mpnt, addr)))continue ; -__flush_cache_page (mpnt, addr); +__flush_cache_page (pte, mpnt, addr);return ; } }EXPORT_SYMBOL (__flush_dcache_page); +/* set to max pages to flush before a full flush. Zero means no limit */ +#define MAX_FLUSH_PAGES 0 +#undef DEBUG_PAGE_FLUSHING + +#ifdef DEBUG_PAGE_FLUSHING +#define DBG(a...) printk(a) +#else +#define DBG (...) +#endif + +#if (MAX_FLUSH_PAGES !=0 ) + +/* we get to use the bottom 12 bits of the addr for flags since the +* address must be page aligned */ +#define ICACHE_FLUSH_FLAG 0x1 + +void flush_cache_mm (struct mm_struct *mm) +{ +struct vm_area_struct *vma; +unsigned long count =0 , actual_count =0 ; +unsigned long sr3 =mfsp (3 ), cr25 =mfctl (25 ); +unsigned long *pages; + +preempt_disable (); +if (mm != current->active_mm) { +DBG ("flush_tlb_mm: current MM is not active " ); +/* FIXME: awful hack: move the process the mm belongs +* to temporarily to being the active one. This only +* works because we can never get back into user +* context from here. */ +mtctl (__pa (mm->pgd),25 ); +mtsp (mm->context,3 ); + } + + pages =kmalloc (MAX_FLUSH_PAGES *sizeof (unsigned long ),GFP_ATOMIC ); +if (!pages) { +printk (KERN_ERR "flush_tlb_mm: allocation failed: full flush\n" ); + goto full_flush; + } + +for (vma = mm->mmap; vma !=NULL ; vma = vma->vm_next) { +unsigned long start; + +pmd_t *pmd; +pgd_t *pgd; +pte_t *pte; + + count += (vma->vm_end - vma->vm_start)/PAGE_SIZE ; + +for (start = vma->vm_start; start < vma->vm_end; + start +=PAGE_SIZE ) { + pgd =pgd_offset (mm, start); + +if (pgd_none (*pgd)) { + start = (start &PGDIR_MASK ) +PGDIR_SIZE -PAGE_SIZE ; +continue ; + } + + pmd =pmd_offset (pgd, start); +if (pmd_none (*pmd)) { + start = (start &PMD_MASK ) +PMD_SIZE -PAGE_SIZE ; +continue ; + } + pte =pte_offset_map (pmd, start); +if (pte_val (*pte)==0 || !pte_present (*pte)) +continue ; + +/* FIXME: Here we could also skip over any +* shared mapping page (i.e. equivalently +* aliased) with at least one other user */ + + pages[actual_count] = start; + +if (vma->vm_flags &VM_EXEC ) + pages[actual_count] |=ICACHE_FLUSH_FLAG ; +if (++actual_count >=MAX_FLUSH_PAGES ) + goto full_flush_free; + + } + } + +DBG ("FLUSHED %lu (actual %lu)\n" , count, actual_count); +for (count =0 ; count < actual_count; count++) { +unsigned long addr = pages[count] &PAGE_MASK ; +flush_user_dcache_page (addr); +if (pages[count] &ICACHE_FLUSH_FLAG ) { +flush_user_icache_page (addr); +pitlb_user (addr); + } +pdtlb_user (addr); + } + out_free: +kfree (pages); + out: +mtsp (sr3,3 ); +mtctl (cr25,25 ); +preempt_enable (); +return ; + + full_flush_free: +DBG ("flush_cache_mm: over max pages %ld (count %ld), flushing everything\n" , actual_count, count); +flush_cache_all (); + goto out_free; + + full_flush: +flush_cache_all (); + goto out; +} + +#else + +void flush_cache_mm (struct mm_struct *mm) +{ +struct vm_area_struct *vma; +unsigned long count =0 , actual_count =0 ; +unsigned long sr3 =mfsp (3 ), cr25 =mfctl (25 ); +#if 1 +static int flushed =0 ; + +if (unlikely (!flushed)) { +printk ("flush_cache_mm: INIT FLUSH ALL\n" ); + flushed =1 ; +flush_cache_all (); +return ; + } +#endif +preempt_disable (); +if (mm != current->active_mm) { +DBG ("flush_tlb_mm: current MM is not active " ); +/* FIXME: awful hack: move the process the mm belongs +* to temporarily to being the active one. This only +* works because we can never get back into user +* context from here. */ +mtctl (__pa (mm->pgd),25 ); +mtsp (mm->context,3 ); + } + +for (vma = mm->mmap; vma !=NULL ; vma = vma->vm_next) { +unsigned long start; + +pmd_t *pmd; +pgd_t *pgd; +pte_t *pte; + + count += (vma->vm_end - vma->vm_start)/PAGE_SIZE ; + +for (start = vma->vm_start; start < vma->vm_end; + start +=PAGE_SIZE ) { + pgd =pgd_offset (mm, start); + +if (pgd_none (*pgd)) { + start = (start &PGDIR_MASK ) +PGDIR_SIZE -PAGE_SIZE ; +continue ; + } + + pmd =pmd_offset (pgd, start); +if (pmd_none (*pmd)) { + start = (start &PMD_MASK ) +PMD_SIZE -PAGE_SIZE ; +continue ; + } + pte =pte_offset_map (pmd, start); +if (!pte_present (*pte)) +continue ; + +if (!pte_flush (*pte)) +continue ; + +/* FIXME: Here we could also skip over any +* shared mapping page (i.e. equivalently +* aliased) with at least one other user */ +page_begin_flush (pte); +pdtlb_user (start); +flush_user_dcache_page (start); +if (vma->vm_flags &VM_EXEC ) { +flush_user_icache_page (start); +pitlb_user (start); + } +page_end_flush (pte); +pdtlb_user (start); + actual_count++; + } + } +mtsp (sr3,3 ); +mtctl (cr25,25 ); +preempt_enable (); +DBG ("FLUSHED %lu (actual %lu)\n" , count, actual_count); +} +#endif + +EXPORT_SYMBOL (flush_cache_mm); +/* Defined in arch/parisc/kernel/pacache.S */ EXPORT_SYMBOL (flush_kernel_dcache_range_asm); -EXPORT_SYMBOL (flush_kernel_dcache_page); +EXPORT_SYMBOL (__flush_kernel_dcache_page);EXPORT_SYMBOL (flush_data_cache_local);EXPORT_SYMBOL (flush_kernel_icache_range_asm);
arch/parisc/kernel/entry.S +11 -65
139 lines not on this page · raw message
arch/parisc/kernel/pacache.S +63 -10
157 lines not on this page · raw message
arch/parisc/kernel/pci-dma.c +21 -12
@@ -362,7 +362,7 @@ size =1 << (order +PAGE_SHIFT ); vaddr =pcxl_alloc_range (size); paddr =__get_free_pages (flag, order); -flush_kernel_dcache_range (paddr, size); +flush_kernel_dcache_range_asm (paddr, paddr + size); paddr =__pa (paddr);map_uncached_pages (vaddr, size, paddr); *dma_handle = (dma_addr_t ) paddr;
@@ -396,12 +396,13 @@BUG (); } -flush_kernel_dcache_range ((unsigned long ) addr, size); +flush_kernel_dcache_range_asm ((unsigned long ) addr, (unsigned long )addr + size);return virt_to_phys (addr); }static void pa11_dma_unmap_single (struct device *dev,dma_addr_t dma_handle,size_t size,enum dma_data_direction direction) { +unsigned long addr = (unsigned long )phys_to_virt (dma_handle);if (direction ==DMA_NONE ) {printk (KERN_ERR "pa11_dma_unmap_single(PCI_DMA_NONE) called by %p\n" ,__builtin_return_address (0 ));BUG ();
@@ -416,7 +417,7 @@* pci_dma_sync_single_* has been called and the buffer reused. */ -flush_kernel_dcache_range ((unsigned long )phys_to_virt (dma_handle), size); +flush_kernel_dcache_range_asm (addr, addr + size);return ; }
@@ -431,7 +432,7 @@unsigned long vaddr =sg_virt_addr (sglist);sg_dma_address (sglist) = (dma_addr_t )virt_to_phys (vaddr);sg_dma_len (sglist) = sglist->length; -flush_kernel_dcache_range (vaddr, sglist->length); +flush_kernel_dcache_range_asm (vaddr, vaddr + sglist->length); }return nents; }
@@ -448,25 +449,29 @@/* once we do combining we'll need to use phys_to_virt(sg_dma_address(sglist)) */ -for (i =0 ; i < nents; i++, sglist++ ) -flush_kernel_dcache_range (sg_virt_addr (sglist), sglist->length); +for (i =0 ; i < nents; i++, sglist++ ) { +unsigned long vaddr =sg_virt_addr (sglist); +flush_kernel_dcache_range_asm (vaddr, vaddr + sglist->length); + }return ; }static void pa11_dma_sync_single_for_cpu (struct device *dev,dma_addr_t dma_handle,unsigned long offset,size_t size,enum dma_data_direction direction) { +unsigned long addr = (unsigned long )phys_to_virt (dma_handle) + offset;if (direction ==DMA_NONE )BUG (); -flush_kernel_dcache_range ((unsigned long )phys_to_virt (dma_handle) + offset, size); +flush_kernel_dcache_range_asm (addr, addr + size); }static void pa11_dma_sync_single_for_device (struct device *dev,dma_addr_t dma_handle,unsigned long offset,size_t size,enum dma_data_direction direction) { +unsigned long addr = (unsigned long )phys_to_virt (dma_handle) + offset;if (direction ==DMA_NONE )BUG (); -flush_kernel_dcache_range ((unsigned long )phys_to_virt (dma_handle) + offset, size); +flush_kernel_dcache_range_asm (addr, addr + size); }static void pa11_dma_sync_sg_for_cpu (struct device *dev,struct scatterlist *sglist,int nents,enum dma_data_direction direction)
@@ -475,8 +480,10 @@/* once we do combining we'll need to use phys_to_virt(sg_dma_address(sglist)) */ -for (i =0 ; i < nents; i++, sglist++ ) -flush_kernel_dcache_range (sg_virt_addr (sglist), sglist->length); +for (i =0 ; i < nents; i++, sglist++ ) { +unsigned long vaddr =sg_virt_addr (sglist); +flush_kernel_dcache_range_asm (vaddr, vaddr + sglist->length); + } }static void pa11_dma_sync_sg_for_device (struct device *dev,struct scatterlist *sglist,int nents,enum dma_data_direction direction)
@@ -485,8 +492,10 @@/* once we do combining we'll need to use phys_to_virt(sg_dma_address(sglist)) */ -for (i =0 ; i < nents; i++, sglist++ ) -flush_kernel_dcache_range (sg_virt_addr (sglist), sglist->length); +for (i =0 ; i < nents; i++, sglist++ ) { +unsigned long vaddr =sg_virt_addr (sglist); +flush_kernel_dcache_range_asm (vaddr, vaddr + sglist->length); + } }struct hppa_dma_ops pcxl_dma_ops = {
arch/parisc/kernel/signal.c +3 -4
@@ -375,10 +375,9 @@ }#endif -flush_user_dcache_range ((unsigned long ) &frame->tramp[0 ], - (unsigned long ) &frame->tramp[TRAMP_SIZE ]); -flush_user_icache_range ((unsigned long ) &frame->tramp[0 ], - (unsigned long ) &frame->tramp[TRAMP_SIZE ]); +__flush_cache_range (current->active_mm, + (unsigned long ) &frame->tramp[0 ], + (unsigned long ) &frame->tramp[TRAMP_SIZE ]);/* TRAMP Words 0-4, Lenght 5 = SIGRESTARTBLOCK_TRAMP * TRAMP Words 5-9, Length 4 = SIGRETURN_TRAMP
drivers/scsi/ncr53c8xx.c +1 -0
@@ -8621,6 +8621,7 @@ tpnt->sg_tablesize =SCSI_NCR_SG_TABLESIZE ; tpnt->cmd_per_lun =SCSI_NCR_CMD_PER_LUN ; tpnt->use_clustering =DISABLE_CLUSTERING ; + tpnt->max_sectors =8 ;if (device->differential) driver_setup.diff_support = device->differential;
drivers/video/console/sticore.c +1 -1
@@ -237,7 +237,7 @@static void sti_flush (unsigned long from,unsigned long len) {flush_data_cache (); -flush_kernel_dcache_range (from, len); +flush_kernel_dcache_range (from, from+len);flush_icache_range (from, from+len); }
include/asm-parisc/cache.h +6 -4
@@ -46,7 +46,7 @@extern void flush_kernel_icache_range_asm (unsigned long ,unsigned long );extern void flush_user_dcache_range_asm (unsigned long ,unsigned long );extern void flush_kernel_dcache_range_asm (unsigned long ,unsigned long ); -extern void flush_kernel_dcache_page (void *); +extern void __flush_kernel_dcache_page (void *);extern void flush_kernel_icache_page (void *);extern void disable_sr_hashing (void );/* turns off space register hashing */ extern void disable_sr_hashing_asm (int );/* low level support for above */
@@ -63,9 +63,11 @@extern int icache_stride;extern struct pdc_cache_info cache_info; -#define pdtlb (addr) asm volatile("pdtlb 0(%%sr1,%0)" : : "r" (addr)); -#define pitlb (addr) asm volatile("pitlb 0(%%sr1,%0)" : : "r" (addr)); -#define pdtlb_kernel (addr) asm volatile("pdtlb 0(%0)" : : "r" (addr)); +#define pdtlb (addr) asm volatile("pdtlb 0(%%sr1,%0)" : : "r" (addr)) +#define pitlb (addr) asm volatile("pitlb 0(%%sr1,%0)" : : "r" (addr)) +#define pdtlb_kernel (addr) asm volatile("pdtlb 0(%0)" : : "r" (addr)) +#define pdtlb_user (addr) asm volatile("pdtlb 0(%%sr3,%0)" : : "r" (addr)) +#define pitlb_user (addr) asm volatile("pitlb 0(%%sr3,%0)" : : "r" (addr))#endif /* ! __ASSEMBLY__ */
include/asm-parisc/cacheflush.h +162 -88
319 lines not on this page · raw message
include/asm-parisc/dma-mapping.h +2 -1
@@ -200,8 +200,9 @@ dma_cache_sync(void *vaddr, size_t size, enum dma_data_direction direction) { +unsigned long start = (unsigned long )vaddr, end = start + size;if (hppa_dma_ops->dma_sync_single_for_cpu) -flush_kernel_dcache_range ((unsigned long )vaddr, size); +flush_kernel_dcache_range_asm (start, end); } static inline void *
include/asm-parisc/io.h +3 -3
@@ -298,9 +298,9 @@#define IO_SPACE_LIMIT 0x00ffffff -#define dma_cache_inv (_start,_size) do { flush_kernel_dcache_range(_start,_size); } while (0) -#define dma_cache_wback (_start,_size) do { flush_kernel_dcache_range(_start,_size); } while (0) -#define dma_cache_wback_inv (_start,_size) do { flush_kernel_dcache_range(_start,_size); } while (0) +#define dma_cache_inv (_start,_size) do { flush_kernel_dcache_range_asm(_start,(_start) + (_size)); } while (0) +#define dma_cache_wback (_start,_size) do { flush_kernel_dcache_range_asm(_start,(_start) + (_size)); } while (0) +#define dma_cache_wback_inv (_start,_size) do { flush_kernel_dcache_range_asm(_start,(_start) + (_size)); } while (0)/* PA machines have an MM I/O space from 0xf0000000-0xffffffff in 32 * bit mode and from 0xfffffffff0000000-0xfffffffffffffff in 64 bit
include/asm-parisc/page.h +2 -7
@@ -19,15 +19,10 @@extern void purge_kernel_dcache_page (unsigned long );extern void copy_user_page_asm (void *to,void *from); +extern void copy_user_page_vaddr_asm (void *to,void *from,unsigned long vaddr);extern void clear_user_page_asm (void *page,unsigned long vaddr); -static inline void -copy_user_page (void *vto,void *vfrom,unsigned long vaddr,struct page *pg) -{ -copy_user_page_asm (vto, vfrom); -flush_kernel_dcache_page (vto); -/* XXX: ppc flushes icache too, should we? */ -} +/* see cacheflush.h for copy_user_page */ static inline void clear_user_page (void *page,unsigned long vaddr,struct page *pg)
include/asm-parisc/pgtable.h +38 -30
121 lines not on this page · raw message
include/asm-parisc/tlb.h +1 -1
@@ -7,7 +7,7 @@ } while (0)#define tlb_start_vma (tlb, vma) \ -do { if (!(tlb)->fullmm) \ +do { \ flush_cache_range(vma, vma->vm_start, vma->vm_end); \ } while (0)