WIP: PowerPC cache cleanup
A message that is not in the archive (sent off-list or not yet received): 1320986410.21206.camel@pasglop
RFC PATCH 2/2 WIP: PowerPC cache cleanup
[My apologies for the resend, it does not seem to have hit the MLs. I think my git send-email "cc-cmd" may have broken somehow, oops.]
This badly needs breaking up, and a better changelog... oh well...
The big changes:
* The "ppc64_caches" structure is now "powerpc_caches" and is used on both PPC32 and PPC64. I hated staring at the pages and pages of assembly code, so nearly all of the functions are now C with tiny snippets of inline ASM in the loops.
* Lots of ugly assembly functions in arch/powerpc/kernel/misc_*.S were rewritten as cleaner inline ASM in arch/powerpc/mm/cache.c
* I'm not sure that the physical address functions from those files actually came out cleaner, but they are now more correct.
* I'm not 100% sure I like the new FOR_EACH_CACHE_LINE() macro, but it sure does make a lot of the other code much cleaner.
* I have a bit of a temptation to try to merge the 32/64-bit variants of copy_page() into a single C function. A quick test seems to show that I can get nearly identical output to the 64-bit ASM with very little work.
--- arch/powerpc/include/asm/cache.h | 155 ++++++++++++--- arch/powerpc/include/asm/cacheflush.h | 3 - arch/powerpc/include/asm/page.h | 6 + arch/powerpc/include/asm/page_32.h | 4 +- arch/powerpc/include/asm/page_64.h | 17 -- arch/powerpc/kernel/align.c | 7 +- arch/powerpc/kernel/asm-offsets.c | 13 +- arch/powerpc/kernel/head_32.S | 9 +- arch/powerpc/kernel/head_64.S | 2 +- arch/powerpc/kernel/misc_32.S | 193 ------------------ arch/powerpc/kernel/misc_64.S | 182 ----------------- arch/powerpc/kernel/ppc_ksyms.c | 3 - arch/powerpc/kernel/setup-common.c | 103 ++++++++++ arch/powerpc/kernel/setup.h | 1 + arch/powerpc/kernel/setup_32.c | 11 +- arch/powerpc/kernel/setup_64.c | 118 +---------- arch/powerpc/kernel/vdso.c | 27 +-- arch/powerpc/lib/copypage_64.S | 10 +- arch/powerpc/mm/Makefile | 2 +- arch/powerpc/mm/cache.c | 279 ++++++++++++++++++++++++++ arch/powerpc/mm/dma-noncoherent.c | 2 +- arch/powerpc/platforms/52xx/lite5200_sleep.S | 9 +- arch/powerpc/platforms/powermac/pci.c | 2 +- arch/powerpc/xmon/xmon.c | 53 +++--- drivers/macintosh/smu.c | 8 +- 25 files changed, 599 insertions(+), 620 deletions(-) create mode 100644 arch/powerpc/mm/cache.c
arch/powerpc/include/asm/cache.h +125 -30
@@ -3,47 +3,142 @@#ifdef __KERNEL__ - -/* bytes per L1 cache line */ -#if defined(CONFIG_8xx) || defined(CONFIG_403GCX ) -#define L1_CACHE_SHIFT 4 -#define MAX_COPY_PREFETCH 1 +/* +* Various PowerPC CPUs which are otherwise compatible have different L1 +* cache line sizes. +* +* Unfortunately, lots of kernel code assumes that L1_CACHE_BYTES and +* L1_CACHE_SHIFT are compile-time constants that can be used to align +* data-structures to avoid false cacheline sharing, so we can't just +* compute them at runtime from the cputable values. +* +* So for alignment purposes, we will compute these values as safe maximums +* of all the CPU support compiled into the kernel. +*/ +#if defined(CONFIG_PPC64 ) || defined(CONFIG_PPC_47x) +# define L1_CACHE_SHIFT_MAX 7/* 128-byte cache blocks */ #elif defined(CONFIG_PPC_E500MC ) -#define L1_CACHE_SHIFT 6 -#define MAX_COPY_PREFETCH 4 -#elif defined(CONFIG_PPC32 ) -#define MAX_COPY_PREFETCH 4 -#if defined(CONFIG_PPC_47x) -#define L1_CACHE_SHIFT 7 +# define L1_CACHE_SHIFT_MAX 6/* 64-byte cache blocks */ #else -#define L1_CACHE_SHIFT 5 +# define L1_CACHE_SHIFT_MAX 5/* 32-byte cache blocks */ #endif +#define L1_CACHE_BYTES_MAX (1 << L1_CACHE_SHIFT_MAX) + +#define L1_CACHE_SHIFT L1_CACHE_SHIFT_MAX +#define L1_CACHE_BYTES L1_CACHE_BYTES_MAX +#define SMP_CACHE_BYTES L1_CACHE_BYTES_MAX + +/* +* Unfortunately, for other purposes, we can't just use a safe maximum value +* because it gets used in loops when invalidating or clearing cachelines and +* it would be very bad to only flush/invalidate/zero/etc every 4th one. +* +* During early initialization we load these values from the device-tree and +* the cputable into the powerpc_caches structure, but we need to be able to +* clear pages before that occurs, so these need sane default values. +* +* As explained in the powerpc_caches structure definition, the defaults +* should be safe minimums, so that's what we compute here. +*/ +#if defined(CONFIG_8xx) || defined(CONFIG_403GCX ) +# define L1_CACHE_SHIFT_MIN 4/* 16-byte cache blocks */ +#elif defined(CONFIG_PPC32 ) +# define L1_CACHE_SHIFT_MIN 5/* 32-byte cache blocks */ #else /* CONFIG_PPC64 */ -#define L1_CACHE_SHIFT 7 +# define L1_CACHE_SHIFT_MIN 6/* 64-byte cache blocks */ #endif +#define L1_CACHE_BYTES_MIN (1 << L1_CACHE_SHIFT_MIN) -#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT) +/* +* Apparently the 8xx and the 403GCX have tiny caches, so they never prefetch +* more than a single cacheline in the ASM memory copy functions. +* +* All other 32-bit CPUs prefetch 4 cachelines, and the 64-bit CPUs have +* their own copy routines which prefetch the entire page. +*/ +#ifdef PPC32 +# if defined(CONFIG_8xx) || defined(CONFIG_403GCX ) +# define MAX_COPY_PREFETCH 1 +# else +# define MAX_COPY_PREFETCH 4 +# endif +#endif -#define SMP_CACHE_BYTES L1_CACHE_BYTES +#ifndef __ASSEMBLY__ -#if defined(__powerpc64__) && !defined(__ASSEMBLY__) -struct ppc64_caches { -u32 dsize;/* L1 d-cache size */ -u32 dline_size;/* L1 d-cache line size */ -u32 log_dline_size; -u32 dlines_per_page; -u32 isize;/* L1 i-cache size */ -u32 iline_size;/* L1 i-cache line size */ -u32 log_iline_size; -u32 ilines_per_page; -}; +/* +* A handy macro to iterate over all the cachelines referring to memory from +* "START" through "STOP - 1", inclusive. +*/ +#define FOR_EACH_CACHELINE (LINE ,START ,STOP ,CACHE ) \ + for (u32 linesize__ = powerpc_caches.CACHE##_block_bytes, \ + (LINE) = (START) & ~(linesize__ - 1); \ + (LINE) < (STOP); (LINE) += linesize__) + +/* Write out a data cache block if it is dirty */ +static inline void dcbst (unsigned long addr) +{ + asmvolatile ("dcbst %y0" ::"Z" (addr) :"memory" ); +} -extern struct ppc64_caches ppc64_caches; -#endif /* __powerpc64__ && ! __ASSEMBLY__ */ +/* Invalidate a data cache block (will lose data if dirty!) */ +static inline void dcbi (unsigned long addr) +{ + asmvolatile ("dcbi %y0" ::"Z" (addr) :"memory" ); +} + +/* Write out (if dirty) and invalidate a data cache block */ +static inline void dcbf (unsigned long addr) +{ + asmvolatile ("dcbf %y0" ::"Z" (addr) :"memory" ); +} + +/* Populate a data cache block with zeros */ +static inline void dcbz (unsigned long addr) +{ + asmvolatile ("dcbz %y0" ::"Z" (addr) :"memory" ); +} + +/* Invalidate an instruction cache block */ +static inline void icbi (unsigned long addr) +{ + asmvolatile ("icbi %y0" ::"Z" (addr) :"memory" ); +} + +/* +* This structure contains the various PowerPC cache parameters computed +* shortly after the device-tree has been unflattened during boot. +* +* Prior to that they have statically initialized values from L1_CACHE_*_MIN +* computed above. +* +* NOTE: If the dcache/icache are separate then ucache_* should be zeroed, +* otherwise dcache == icache == ucache. +*/ +struct powerpc_caches { +/* Data cache parameters */ +u32 dcache_total_bytes; +u32 dcache_block_bytes; +u32 dcache_block_shift; +u32 dcache_blocks_per_page; + +/* Instruction cache parameters */ +u32 icache_total_bytes; +u32 icache_block_bytes; +u32 icache_block_shift; +u32 icache_blocks_per_page; + +/* Unified cache parameters (If != 0, all 3 caches must be equal) */ +u32 ucache_total_bytes; +u32 ucache_block_bytes; +u32 ucache_block_shift; +u32 ucache_blocks_per_page; +}; +extern struct powerpc_caches powerpc_caches; -#if !defined(__ASSEMBLY__)#define __read_mostly __attribute__((__section__(".data..read_mostly"))) -#endif + +#endif /* not __ASSEMBLY__ */ #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_CACHE_H */
arch/powerpc/include/asm/cacheflush.h +0 -3
@@ -47,12 +47,9 @@ extern void __flush_dcache_icache_phys(unsigned long physaddr);#endif /* CONFIG_PPC32 && !CONFIG_BOOKE */ extern void flush_dcache_range (unsigned long start,unsigned long stop); -#ifdef CONFIG_PPC32 extern void clean_dcache_range (unsigned long start,unsigned long stop);extern void invalidate_dcache_range (unsigned long start,unsigned long stop); -#endif /* CONFIG_PPC32 */ #ifdef CONFIG_PPC64 -extern void flush_inval_dcache_range (unsigned long start,unsigned long stop);extern void flush_dcache_phys_range (unsigned long start,unsigned long stop);#endif
arch/powerpc/include/asm/page.h +6 -0
@@ -286,11 +286,17 @@ static inline int hugepd_ok(hugepd_t hpd)#endif /* CONFIG_HUGETLB_PAGE */ struct page ; +extern void clear_pages (void *page,int order);extern void clear_user_page (void *page,unsigned long vaddr,struct page *pg);extern void copy_user_page (void *to,void *from,unsigned long vaddr,struct page *p);extern int page_is_ram (unsigned long pfn); +static inline void clear_page (void *page) +{ +clear_pages (page,0 ); +} +#ifdef CONFIG_PPC_SMLPAR void arch_free_page (struct page *page,int order);#define HAVE_ARCH_FREE_PAGE
arch/powerpc/include/asm/page_32.h +1 -3
@@ -10,7 +10,7 @@#define VM_DATA_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS32#ifdef CONFIG_NOT_COHERENT_CACHE -#define ARCH_DMA_MINALIGN L1_CACHE_BYTES +#define ARCH_DMA_MINALIGN L1_CACHE_BYTES_MAX#endif #ifdef CONFIG_PTE_64BIT
@@ -37,8 +37,6 @@ typedef unsigned long pte_basic_t;#endif struct page ; -extern void clear_pages (void *page,int order); -static inline void clear_page (void *page) {clear_pages (page,0 ); }extern void copy_page (void *to,void *from);#include <asm-generic/getorder.h>
arch/powerpc/include/asm/page_64.h +0 -17
@@ -42,23 +42,6 @@typedef unsigned long pte_basic_t ; -static __inline__void clear_page (void *addr) -{ -unsigned long lines, line_size; - - line_size = ppc64_caches.dline_size; - lines = ppc64_caches.dlines_per_page; - - __asm__ __volatile__( -"mtctr %1 # clear_page\n\ -1: dcbz 0,%0\n\ -add %0,%0,%3\n\ -bdnz+ 1b" - :"=r" (addr) - :"r" (lines),"0" (addr),"r" (line_size) - :"ctr" ,"memory" ); -} -extern void copy_page (void *to,void *from);/* Log 2 of page table size */
arch/powerpc/kernel/align.c +1 -6
@@ -233,14 +233,9 @@ static inline unsigned make_dsisr(unsigned instr)*/ static int emulate_dcbz (struct pt_regs *regs,unsigned char __user *addr) { +int i, size = powerpc_caches.dcache_block_bytes;long __user *p; -int i, size; -#ifdef __powerpc64__ - size = ppc64_caches.dline_size; -#else - size =L1_CACHE_BYTES ; -#endif p = (long __user *) (regs->dar & -size);if (user_mode (regs) && !access_ok (VERIFY_WRITE , p, size))return -EFAULT ;
arch/powerpc/kernel/asm-offsets.c +7 -6
@@ -126,13 +126,14 @@ int main(void)DEFINE (TI_TASK , offsetof(struct thread_info , task));DEFINE (TI_CPU , offsetof(struct thread_info , cpu)); +DEFINE (DCACHE_BLOCK_SHIFT , offsetof(struct powerpc_caches , dcache_block_shift)); +DEFINE (DCACHE_BLOCK_BYTES , offsetof(struct powerpc_caches , dcache_block_bytes)); +DEFINE (DCACHE_BLOCKS_PER_PAGE , offsetof(struct powerpc_caches , dcache_blocks_per_page)); +DEFINE (ICACHE_BLOCK_SHIFT , offsetof(struct powerpc_caches , icache_block_shift)); +DEFINE (ICACHE_BLOCK_BYTES , offsetof(struct powerpc_caches , icache_block_bytes)); +DEFINE (ICACHE_BLOCKS_PER_PAGE , offsetof(struct powerpc_caches , icache_blocks_per_page)); +#ifdef CONFIG_PPC64 -DEFINE (DCACHEL1LINESIZE , offsetof(struct ppc64_caches , dline_size)); -DEFINE (DCACHEL1LOGLINESIZE , offsetof(struct ppc64_caches , log_dline_size)); -DEFINE (DCACHEL1LINESPERPAGE , offsetof(struct ppc64_caches , dlines_per_page)); -DEFINE (ICACHEL1LINESIZE , offsetof(struct ppc64_caches , iline_size)); -DEFINE (ICACHEL1LOGLINESIZE , offsetof(struct ppc64_caches , log_iline_size)); -DEFINE (ICACHEL1LINESPERPAGE , offsetof(struct ppc64_caches , ilines_per_page));/* paca */ DEFINE (PACA_SIZE ,sizeof (struct paca_struct ));DEFINE (PACA_LOCK_TOKEN , offsetof(struct paca_struct , lock_token));
arch/powerpc/kernel/head_32.S +8 -1
@@ -786,7 +786,14 @@ relocate_kernel: _ENTRY(copy_and_flush) addi r5,r5,-4 addi r6,r6,-4 -4: li r0,L1_CACHE_BYTES/4 +4: li r0,L1_CACHE_BYTES_MIN/4 /* Use the smallest common */ + /* denominator cache line */ + /* size. This results in */ + /* extra cache line flushes */ + /* but operation is correct. */ + /* Can't get cache line size */ + /* from device-tree yet */ + mtctr r0 3: addi r6,r6,4 /* copy a cache line */ lwzx r0,r6,r4
arch/powerpc/kernel/head_64.S +1 -1
@@ -480,7 +480,7 @@ p_end: .llong _end - _stext _GLOBAL(copy_and_flush) addi r5,r5,-8 addi r6,r6,-8 -4: li r0,8 /* Use the smallest common */ +4: li r0,L1_CACHE_BYTES_MIN/8 /* Use the smallest common */ /* denominator cache line */ /* size. This results in */ /* extra cache line flushes */
arch/powerpc/kernel/misc_32.S +0 -193
199 lines not on this page · raw message
arch/powerpc/kernel/misc_64.S +0 -182
188 lines not on this page · raw message
arch/powerpc/kernel/ppc_ksyms.c +0 -3
@@ -53,7 +53,6 @@ extern void program_check_exception(struct pt_regs *regs);extern void single_step_exception (struct pt_regs *regs);extern int sys_sigreturn (struct pt_regs *regs); -EXPORT_SYMBOL (clear_pages);EXPORT_SYMBOL (ISA_DMA_THRESHOLD );EXPORT_SYMBOL (DMA_MODE_READ );EXPORT_SYMBOL (DMA_MODE_WRITE );
@@ -113,8 +112,6 @@ EXPORT_SYMBOL(giveup_spe);#ifndef CONFIG_PPC64 EXPORT_SYMBOL (flush_instruction_cache);#endif -EXPORT_SYMBOL (__flush_icache_range); -EXPORT_SYMBOL (flush_dcache_range);#ifdef CONFIG_SMP #ifdef CONFIG_PPC32
arch/powerpc/kernel/setup-common.c +103 -0
115 lines not on this page · raw message
arch/powerpc/kernel/setup.h +1 -0
@@ -1,6 +1,7 @@#ifndef _POWERPC_KERNEL_SETUP_H#define _POWERPC_KERNEL_SETUP_H +void initialize_cache_info (void );void check_for_initrd (void );void do_init_bootmem (void );void setup_panic (void );
arch/powerpc/kernel/setup_32.c +3 -8
@@ -63,14 +63,6 @@ EXPORT_SYMBOL(vgacon_remap_base);#endif /* -* These are used in binfmt_elf.c to put aux entries on the stack -* for each elf executable being started. -*/ -int dcache_bsize; -int icache_bsize; -int ucache_bsize; - -/* * We're called here very early in the boot. We determine the machine * type and call the appropriate low-level setup functions. * -- Cort <[email protected]>
@@ -286,10 +278,13 @@ void __init setup_arch(char **cmdline_p) { *cmdline_p = cmd_line; +initialize_early_cache_info (); +/* so udelay does something sensible, assume <= 1000 bogomips */ loops_per_jiffy =500000000 /HZ ;unflatten_device_tree (); +initialize_cache_info ();check_for_initrd ();if (ppc_md.init_early)
arch/powerpc/kernel/setup_64.c +6 -112
148 lines not on this page · raw message
arch/powerpc/kernel/vdso.c +12 -15
@@ -726,6 +726,7 @@ static int __init vdso_init(void) vdso_data->version.major =SYSTEMCFG_MAJOR ; vdso_data->version.minor =SYSTEMCFG_MINOR ; vdso_data->processor =mfspr (SPRN_PVR ); +/* * Fake the old platform number for pSeries and iSeries and add * in LPAR bit if necessary
@@ -734,29 +735,25 @@ static int __init vdso_init(void)if (firmware_has_feature (FW_FEATURE_LPAR )) vdso_data->platform |=1 ; vdso_data->physicalMemorySize =memblock_phys_mem_size (); - vdso_data->dcache_size = ppc64_caches.dsize; - vdso_data->dcache_line_size = ppc64_caches.dline_size; - vdso_data->icache_size = ppc64_caches.isize; - vdso_data->icache_line_size = ppc64_caches.iline_size; -/* XXXOJN: Blocks should be added to ppc64_caches and used instead */ - vdso_data->dcache_block_size = ppc64_caches.dline_size; - vdso_data->icache_block_size = ppc64_caches.iline_size; - vdso_data->dcache_log_block_size = ppc64_caches.log_dline_size; - vdso_data->icache_log_block_size = ppc64_caches.log_iline_size; +/* There are more cache parameters saved for 64-bit than 32-bit */ + vdso_data->dcache_size = powerpc_caches.dcache_total_size; + vdso_data->icache_size = powerpc_caches.icache_total_size; + vdso_data->dcache_line_size = powerpc_caches.dcache_block_bytes; + vdso_data->icache_line_size = powerpc_caches.icache_block_bytes;/* * Calculate the size of the 64 bits vDSO */ vdso64_pages = (&vdso64_end - &vdso64_start) >>PAGE_SHIFT ;DBG ("vdso64_kbase: %p, 0x%x pages\n" , vdso64_kbase, vdso64_pages); -#else - vdso_data->dcache_block_size =L1_CACHE_BYTES ; - vdso_data->dcache_log_block_size =L1_CACHE_SHIFT ; - vdso_data->icache_block_size =L1_CACHE_BYTES ; - vdso_data->icache_log_block_size =L1_CACHE_SHIFT ; -#endif /* CONFIG_PPC64 */ +#endif +/* Save the cache-block sizes for the VDSO */ + vdso_data->dcache_block_size = powerpc_caches.dcache_block_bytes; + vdso_data->icache_block_size = powerpc_caches.icache_block_bytes; + vdso_data->dcache_log_block_size = powerpc_caches.dcache_block_shift; + vdso_data->icache_log_block_size = powerpc_caches.icache_block_shift;/* * Calculate the size of the 32 bits vDSO
arch/powerpc/lib/copypage_64.S +5 -5
@@ -12,17 +12,17 @@ #include <asm/asm-offsets.h> .section ".toc","aw" -PPC64_CACHES: - .tc ppc64_caches[TC],ppc64_caches +POWERPC_CACHES: + .tc powerpc_caches[TC],powerpc_caches .section ".text" _GLOBAL(copy_page) lis r5,PAGE_SIZE@h ori r5,r5,PAGE_SIZE@l BEGIN_FTR_SECTION - ld r10,PPC64_CACHES@toc(r2) - lwz r11,DCACHEL1LOGLINESIZE(r10) /* log2 of cache line size */ - lwz r12,DCACHEL1LINESIZE(r10) /* get cache line size */ + ld r10,POWERPC_CACHES@toc(r2) + lwz r11,DCACHE_BLOCK_SHIFT(r10) /* log2 of cache line size */ + lwz r12,DCACHE_BLOCK_BYTES(r10) /* get cache line size */ li r9,0 srd r8,r5,r11
arch/powerpc/mm/Makefile +1 -1
@@ -6,7 +6,7 @@ subdir-ccflags-$(CONFIG_PPC_WERROR) := -Werror ccflags-$(CONFIG_PPC64) := -mno-minimal-toc -obj-y := fault.o mem.o pgtable.o gup.o \ +obj-y := cache.o fault.o mem.o pgtable.o gup.o \ init_$(CONFIG_WORD_SIZE).o \ pgtable_$(CONFIG_WORD_SIZE).o obj-$(CONFIG_PPC_MMU_NOHASH) += mmu_context_nohash.o tlb_nohash.o \
arch/powerpc/mm/cache.c new file +279 -0
279 lines not on this page · raw message
arch/powerpc/mm/dma-noncoherent.c +1 -1
@@ -328,7 +328,7 @@ void __dma_sync(void *vaddr, size_t size, int direction)* invalidate only when cache-line aligned otherwise there is * the potential for discarding uncommitted data from the cache */ -if ((start & (L1_CACHE_BYTES -1 )) || (size & (L1_CACHE_BYTES -1 ))) +if ((start | size) & (powerpc_caches.dcache_block_bytes -1 ))flush_dcache_range (start, end);else invalidate_dcache_range (start, end);
arch/powerpc/platforms/52xx/lite5200_sleep.S +7 -2
@@ -394,11 +394,16 @@ restore_regs: /* cache flushing code. copied from arch/ppc/boot/util.S */ -#define NUM_CACHE_LINES (128*8) +#define NUM_CACHE_LINES ((128 * 8) << (L1_CACHE_SHIFT_MAX - L1_CACHE_SHIFT_MIN)) /* * Flush data cache * Do this by just reading lots of stuff into the cache. + * + * NOTE: This does not handle variable-sized cachelines properly, but since + * we are just trying to flush the data cache by reading lots of data, + * this works anyways. We just make sure we read as many cachelines + * as we could possibly need to overflow the cache on any hardware. */ flush_data_cache: lis r3,CONFIG_KERNEL_START@h
@@ -407,6 +412,6 @@ flush_data_cache: mtctr r4 1: lwz r4,0(r3) - addi r3,r3,L1_CACHE_BYTES /* Next line, please */ + addi r3,r3,L1_CACHE_BYTES_MIN /* Next line, please */ bdnz 1b blr
arch/powerpc/platforms/powermac/pci.c +1 -1
@@ -1135,7 +1135,7 @@ int pmac_pci_enable_device_hook(struct pci_dev *dev)pci_write_config_byte (dev,PCI_LATENCY_TIMER ,16 );pci_write_config_byte (dev,PCI_CACHE_LINE_SIZE , -L1_CACHE_BYTES >>2 ); + powerpc_caches.dcache_block_bytes >>2 ); }return 0 ;
arch/powerpc/xmon/xmon.c +25 -28
80 lines not on this page · raw message
drivers/macintosh/smu.c +6 -2
@@ -136,7 +136,9 @@ static void smu_start_cmd(void)/* Flush command and data to RAM */ faddr = (unsigned long )smu->cmd_buf; fend = faddr + smu->cmd_buf->length +2 ; -flush_inval_dcache_range (faddr, fend); +flush_dcache_range (faddr, fend); +mb (); +isync ();/* We also disable NAP mode for the duration of the command
@@ -198,7 +200,9 @@ static irqreturn_t smu_db_intr(int irq, void *arg)* reply length (it's only 2 cache lines anyway) */ faddr = (unsigned long )smu->cmd_buf; -flush_inval_dcache_range (faddr, faddr +256 ); +flush_dcache_range (faddr, faddr +256 ); +mb (); +isync ();/* Now check ack */ ack = (~cmd->cmd) &0xff ;
Signature · 1 line
1.7.2.5
Quoted 2 lines · Kyle Moffett
[My apologies for the resend, it does not seem to have hit the MLs. I think my git send-email "cc-cmd" may have broken somehow, oops.]
Or the ML took a while because it's big :-) I got both.
I'll try to review this week. Probably wont get to it today tho.
Thanks for looking at this !
Cheers,
Ben.
Quoted 1,686 lines
This badly needs breaking up, and a better changelog... oh well...
The big changes:
* The "
ppc64_caches" structure is now "powerpc_caches" and is used on both PPC32 and PPC64. I hated staring at the pages and pages of assembly code, so nearly all of the functions are now C with tiny snippets of inline ASM in the loops.* Lots of ugly assembly functions in
arch/powerpc/kernel/misc_*.Swere rewritten as cleaner inline ASM inarch/powerpc/mm/cache.c* I'm not sure that the physical address functions from those files actually came out cleaner, but they are now more correct.
* I'm not 100% sure I like the new
FOR_EACH_CACHE_LINE()macro, but it sure does make a lot of the other code much cleaner.* I have a bit of a temptation to try to merge the 32/64-bit variants of
copy_page()into a single C function. A quick test seems to show that I can get nearly identical output to the 64-bit ASM with very little work.--- arch/powerpc/include/asm/cache.h | 155 ++++++++++++--- arch/powerpc/include/asm/cacheflush.h | 3 - arch/powerpc/include/asm/page.h | 6 + arch/powerpc/include/asm/page_32.h | 4 +- arch/powerpc/include/asm/page_64.h | 17 -- arch/powerpc/kernel/align.c | 7 +- arch/powerpc/kernel/asm-offsets.c | 13 +- arch/powerpc/kernel/head_32.S | 9 +- arch/powerpc/kernel/head_64.S | 2 +- arch/powerpc/kernel/misc_32.S | 193 ------------------ arch/powerpc/kernel/misc_64.S | 182 ----------------- arch/powerpc/kernel/ppc_ksyms.c | 3 - arch/powerpc/kernel/setup-common.c | 103 ++++++++++ arch/powerpc/kernel/setup.h | 1 + arch/powerpc/kernel/setup_32.c | 11 +- arch/powerpc/kernel/setup_64.c | 118 +---------- arch/powerpc/kernel/vdso.c | 27 +-- arch/powerpc/lib/copypage_64.S | 10 +- arch/powerpc/mm/Makefile | 2 +- arch/powerpc/mm/cache.c | 279 ++++++++++++++++++++++++++ arch/powerpc/mm/dma-noncoherent.c | 2 +- arch/powerpc/platforms/52xx/lite5200_sleep.S | 9 +- arch/powerpc/platforms/powermac/pci.c | 2 +- arch/powerpc/xmon/xmon.c | 53 +++--- drivers/macintosh/smu.c | 8 +- 25 files changed, 599 insertions(+), 620 deletions(-) create mode 100644 arch/powerpc/mm/cache.carch/powerpc/include/asm/cache.h +125
-30@@ -3,47 +3,142 @@#ifdef __KERNEL__ - -/* bytes per L1 cache line */ -#if defined(CONFIG_8xx) || defined(CONFIG_403GCX ) -#define L1_CACHE_SHIFT 4 -#define MAX_COPY_PREFETCH 1 +/* +* Various PowerPC CPUs which are otherwise compatible have different L1 +* cache line sizes. +* +* Unfortunately, lots of kernel code assumes that L1_CACHE_BYTES and +* L1_CACHE_SHIFT are compile-time constants that can be used to align +* data-structures to avoid false cacheline sharing, so we can't just +* compute them at runtime from the cputable values. +* +* So for alignment purposes, we will compute these values as safe maximums +* of all the CPU support compiled into the kernel. +*/ +#if defined(CONFIG_PPC64 ) || defined(CONFIG_PPC_47x) +# define L1_CACHE_SHIFT_MAX 7/* 128-byte cache blocks */ #elif defined(CONFIG_PPC_E500MC ) -#define L1_CACHE_SHIFT 6 -#define MAX_COPY_PREFETCH 4 -#elif defined(CONFIG_PPC32 ) -#define MAX_COPY_PREFETCH 4 -#if defined(CONFIG_PPC_47x) -#define L1_CACHE_SHIFT 7 +# define L1_CACHE_SHIFT_MAX 6/* 64-byte cache blocks */ #else -#define L1_CACHE_SHIFT 5 +# define L1_CACHE_SHIFT_MAX 5/* 32-byte cache blocks */ #endif +#define L1_CACHE_BYTES_MAX (1 << L1_CACHE_SHIFT_MAX) + +#define L1_CACHE_SHIFT L1_CACHE_SHIFT_MAX +#define L1_CACHE_BYTES L1_CACHE_BYTES_MAX +#define SMP_CACHE_BYTES L1_CACHE_BYTES_MAX + +/* +* Unfortunately, for other purposes, we can't just use a safe maximum value +* because it gets used in loops when invalidating or clearing cachelines and +* it would be very bad to only flush/invalidate/zero/etc every 4th one. +* +* During early initialization we load these values from the device-tree and +* the cputable into the powerpc_caches structure, but we need to be able to +* clear pages before that occurs, so these need sane default values. +* +* As explained in the powerpc_caches structure definition, the defaults +* should be safe minimums, so that's what we compute here. +*/ +#if defined(CONFIG_8xx) || defined(CONFIG_403GCX ) +# define L1_CACHE_SHIFT_MIN 4/* 16-byte cache blocks */ +#elif defined(CONFIG_PPC32 ) +# define L1_CACHE_SHIFT_MIN 5/* 32-byte cache blocks */ #else /* CONFIG_PPC64 */ -#define L1_CACHE_SHIFT 7 +# define L1_CACHE_SHIFT_MIN 6/* 64-byte cache blocks */ #endif +#define L1_CACHE_BYTES_MIN (1 << L1_CACHE_SHIFT_MIN) -#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT) +/* +* Apparently the 8xx and the 403GCX have tiny caches, so they never prefetch +* more than a single cacheline in the ASM memory copy functions. +* +* All other 32-bit CPUs prefetch 4 cachelines, and the 64-bit CPUs have +* their own copy routines which prefetch the entire page. +*/ +#ifdef PPC32 +# if defined(CONFIG_8xx) || defined(CONFIG_403GCX ) +# define MAX_COPY_PREFETCH 1 +# else +# define MAX_COPY_PREFETCH 4 +# endif +#endif -#define SMP_CACHE_BYTES L1_CACHE_BYTES +#ifndef __ASSEMBLY__ -#if defined(__powerpc64__) && !defined(__ASSEMBLY__) -struct ppc64_caches { -u32 dsize;/* L1 d-cache size */ -u32 dline_size;/* L1 d-cache line size */ -u32 log_dline_size; -u32 dlines_per_page; -u32 isize;/* L1 i-cache size */ -u32 iline_size;/* L1 i-cache line size */ -u32 log_iline_size; -u32 ilines_per_page; -}; +/* +* A handy macro to iterate over all the cachelines referring to memory from +* "START" through "STOP - 1", inclusive. +*/ +#define FOR_EACH_CACHELINE (LINE ,START ,STOP ,CACHE ) \ + for (u32 linesize__ = powerpc_caches.CACHE##_block_bytes, \ + (LINE) = (START) & ~(linesize__ - 1); \ + (LINE) < (STOP); (LINE) += linesize__) + +/* Write out a data cache block if it is dirty */ +static inline void dcbst (unsigned long addr) +{ + asmvolatile ("dcbst %y0" ::"Z" (addr) :"memory" ); +} -extern struct ppc64_caches ppc64_caches; -#endif /* __powerpc64__ && ! __ASSEMBLY__ */ +/* Invalidate a data cache block (will lose data if dirty!) */ +static inline void dcbi (unsigned long addr) +{ + asmvolatile ("dcbi %y0" ::"Z" (addr) :"memory" ); +} + +/* Write out (if dirty) and invalidate a data cache block */ +static inline void dcbf (unsigned long addr) +{ + asmvolatile ("dcbf %y0" ::"Z" (addr) :"memory" ); +} + +/* Populate a data cache block with zeros */ +static inline void dcbz (unsigned long addr) +{ + asmvolatile ("dcbz %y0" ::"Z" (addr) :"memory" ); +} + +/* Invalidate an instruction cache block */ +static inline void icbi (unsigned long addr) +{ + asmvolatile ("icbi %y0" ::"Z" (addr) :"memory" ); +} + +/* +* This structure contains the various PowerPC cache parameters computed +* shortly after the device-tree has been unflattened during boot. +* +* Prior to that they have statically initialized values from L1_CACHE_*_MIN +* computed above. +* +* NOTE: If the dcache/icache are separate then ucache_* should be zeroed, +* otherwise dcache == icache == ucache. +*/ +struct powerpc_caches { +/* Data cache parameters */ +u32 dcache_total_bytes; +u32 dcache_block_bytes; +u32 dcache_block_shift; +u32 dcache_blocks_per_page; + +/* Instruction cache parameters */ +u32 icache_total_bytes; +u32 icache_block_bytes; +u32 icache_block_shift; +u32 icache_blocks_per_page; + +/* Unified cache parameters (If != 0, all 3 caches must be equal) */ +u32 ucache_total_bytes; +u32 ucache_block_bytes; +u32 ucache_block_shift; +u32 ucache_blocks_per_page; +}; +extern struct powerpc_caches powerpc_caches; -#if !defined(__ASSEMBLY__)#define __read_mostly __attribute__((__section__(".data..read_mostly"))) -#endif + +#endif /* not __ASSEMBLY__ */ #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_CACHE_H */ arch/powerpc/include/asm/cacheflush.h +0
-3@@ -47,12 +47,9 @@ extern void __flush_dcache_icache_phys(unsigned long physaddr);#endif /* CONFIG_PPC32 && !CONFIG_BOOKE */ extern void flush_dcache_range (unsigned long start,unsigned long stop); -#ifdef CONFIG_PPC32 extern void clean_dcache_range (unsigned long start,unsigned long stop);extern void invalidate_dcache_range (unsigned long start,unsigned long stop); -#endif /* CONFIG_PPC32 */ #ifdef CONFIG_PPC64 -extern void flush_inval_dcache_range (unsigned long start,unsigned long stop);extern void flush_dcache_phys_range (unsigned long start,unsigned long stop);#endif arch/powerpc/include/asm/page.h +6
-0@@ -286,11 +286,17 @@ static inline int hugepd_ok(hugepd_t hpd)#endif /* CONFIG_HUGETLB_PAGE */ struct page ; +extern void clear_pages (void *page,int order);extern void clear_user_page (void *page,unsigned long vaddr,struct page *pg);extern void copy_user_page (void *to,void *from,unsigned long vaddr,struct page *p);extern int page_is_ram (unsigned long pfn); +static inline void clear_page (void *page) +{ +clear_pages (page,0 ); +} +#ifdef CONFIG_PPC_SMLPAR void arch_free_page (struct page *page,int order);#define HAVE_ARCH_FREE_PAGE arch/powerpc/include/asm/page_32.h +1
-3@@ -10,7 +10,7 @@#define VM_DATA_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS32#ifdef CONFIG_NOT_COHERENT_CACHE -#define ARCH_DMA_MINALIGN L1_CACHE_BYTES +#define ARCH_DMA_MINALIGN L1_CACHE_BYTES_MAX#endif #ifdef CONFIG_PTE_64BIT @@ -37,8 +37,6 @@ typedef unsigned long pte_basic_t;#endif struct page ; -extern void clear_pages (void *page,int order); -static inline void clear_page (void *page) {clear_pages (page,0 ); }extern void copy_page (void *to,void *from);#include <asm-generic/getorder.h> arch/powerpc/include/asm/page_64.h +0
-17@@ -42,23 +42,6 @@typedef unsigned long pte_basic_t ; -static __inline__void clear_page (void *addr) -{ -unsigned long lines, line_size; - - line_size = ppc64_caches.dline_size; - lines = ppc64_caches.dlines_per_page; - - __asm__ __volatile__( -"mtctr %1 # clear_page\n\ -1: dcbz 0,%0\n\ -add %0,%0,%3\n\ -bdnz+ 1b" - :"=r" (addr) - :"r" (lines),"0" (addr),"r" (line_size) - :"ctr" ,"memory" ); -} -extern void copy_page (void *to,void *from);/* Log 2 of page table size */ arch/powerpc/kernel/align.c +1
-6@@ -233,14 +233,9 @@ static inline unsigned make_dsisr(unsigned instr)*/ static int emulate_dcbz (struct pt_regs *regs,unsigned char __user *addr) { +int i, size = powerpc_caches.dcache_block_bytes;long __user *p; -int i, size; -#ifdef __powerpc64__ - size = ppc64_caches.dline_size; -#else - size =L1_CACHE_BYTES ; -#endif p = (long __user *) (regs->dar & -size);if (user_mode (regs) && !access_ok (VERIFY_WRITE , p, size))return -EFAULT ;arch/powerpc/kernel/asm-offsets.c +7
-6@@ -126,13 +126,14 @@ int main(void)DEFINE (TI_TASK , offsetof(struct thread_info , task));DEFINE (TI_CPU , offsetof(struct thread_info , cpu)); +DEFINE (DCACHE_BLOCK_SHIFT , offsetof(struct powerpc_caches , dcache_block_shift)); +DEFINE (DCACHE_BLOCK_BYTES , offsetof(struct powerpc_caches , dcache_block_bytes)); +DEFINE (DCACHE_BLOCKS_PER_PAGE , offsetof(struct powerpc_caches , dcache_blocks_per_page)); +DEFINE (ICACHE_BLOCK_SHIFT , offsetof(struct powerpc_caches , icache_block_shift)); +DEFINE (ICACHE_BLOCK_BYTES , offsetof(struct powerpc_caches , icache_block_bytes)); +DEFINE (ICACHE_BLOCKS_PER_PAGE , offsetof(struct powerpc_caches , icache_blocks_per_page)); +#ifdef CONFIG_PPC64 -DEFINE (DCACHEL1LINESIZE , offsetof(struct ppc64_caches , dline_size)); -DEFINE (DCACHEL1LOGLINESIZE , offsetof(struct ppc64_caches , log_dline_size)); -DEFINE (DCACHEL1LINESPERPAGE , offsetof(struct ppc64_caches , dlines_per_page)); -DEFINE (ICACHEL1LINESIZE , offsetof(struct ppc64_caches , iline_size)); -DEFINE (ICACHEL1LOGLINESIZE , offsetof(struct ppc64_caches , log_iline_size)); -DEFINE (ICACHEL1LINESPERPAGE , offsetof(struct ppc64_caches , ilines_per_page));/* paca */ DEFINE (PACA_SIZE ,sizeof (struct paca_struct ));DEFINE (PACA_LOCK_TOKEN , offsetof(struct paca_struct , lock_token));arch/powerpc/kernel/head_32.S +8
-1@@ -786,7 +786,14 @@ relocate_kernel: _ENTRY(copy_and_flush) addi r5,r5,-4 addi r6,r6,-4 -4: li r0,L1_CACHE_BYTES/4 +4: li r0,L1_CACHE_BYTES_MIN/4 /* Use the smallest common */ + /* denominator cache line */ + /* size. This results in */ + /* extra cache line flushes */ + /* but operation is correct. */ + /* Can't get cache line size */ + /* from device-tree yet */ + mtctr r0 3: addi r6,r6,4 /* copy a cache line */ lwzx r0,r6,r4arch/powerpc/kernel/head_64.S +1
-1@@ -480,7 +480,7 @@ p_end: .llong _end - _stext _GLOBAL(copy_and_flush) addi r5,r5,-8 addi r6,r6,-8 -4: li r0,8 /* Use the smallest common */ +4: li r0,L1_CACHE_BYTES_MIN/8 /* Use the smallest common */ /* denominator cache line */ /* size. This results in */ /* extra cache line flushes */arch/powerpc/kernel/misc_32.S +0
-193@@ -321,199 +321,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_UNIFIED_ID_CACHE) blr /* - * Write any modified data cache blocks out to memory - * and invalidate the corresponding instruction cache blocks. - * This is a no-op on the 601. - * - * flush_icache_range(unsigned long start, unsigned long stop) - */ -_KPROBE(__flush_icache_range) -BEGIN_FTR_SECTION - blr /* for 601, do nothing */ -END_FTR_SECTION_IFSET(CPU_FTR_COHERENT_ICACHE) - li r5,L1_CACHE_BYTES-1 - andc r3,r3,r5 - subf r4,r3,r4 - add r4,r4,r5 - srwi. r4,r4,L1_CACHE_SHIFT - beqlr - mtctr r4 - mr r6,r3 -1: dcbst 0,r3 - addi r3,r3,L1_CACHE_BYTES - bdnz 1b - sync /* wait for dcbst's to get to ram */ -#ifndef CONFIG_44x - mtctr r4 -2: icbi 0,r6 - addi r6,r6,L1_CACHE_BYTES - bdnz 2b -#else - /* Flash invalidate on 44x because we are passed kmapped addresses and - this doesn't work for userspace pages due to the virtually tagged - icache. Sigh. */ - iccci 0, r0 -#endif - sync /* additional sync needed on g4 */ - isync - blr -/* - * Write any modified data cache blocks out to memory. - * Does not invalidate the corresponding cache lines (especially for - * any corresponding instruction cache). - * - * clean_dcache_range(unsigned long start, unsigned long stop) - */ -_GLOBAL(clean_dcache_range) - li r5,L1_CACHE_BYTES-1 - andc r3,r3,r5 - subf r4,r3,r4 - add r4,r4,r5 - srwi. r4,r4,L1_CACHE_SHIFT - beqlr - mtctr r4 - -1: dcbst 0,r3 - addi r3,r3,L1_CACHE_BYTES - bdnz 1b - sync /* wait for dcbst's to get to ram */ - blr - -/* - * Write any modified data cache blocks out to memory and invalidate them. - * Does not invalidate the corresponding instruction cache blocks. - * - * flush_dcache_range(unsigned long start, unsigned long stop) - */ -_GLOBAL(flush_dcache_range) - li r5,L1_CACHE_BYTES-1 - andc r3,r3,r5 - subf r4,r3,r4 - add r4,r4,r5 - srwi. r4,r4,L1_CACHE_SHIFT - beqlr - mtctr r4 - -1: dcbf 0,r3 - addi r3,r3,L1_CACHE_BYTES - bdnz 1b - sync /* wait for dcbst's to get to ram */ - blr - -/* - * Like above, but invalidate the D-cache. This is used by the 8xx - * to invalidate the cache so the PPC core doesn't get stale data - * from the CPM (no cache snooping here :-). - * - * invalidate_dcache_range(unsigned long start, unsigned long stop) - */ -_GLOBAL(invalidate_dcache_range) - li r5,L1_CACHE_BYTES-1 - andc r3,r3,r5 - subf r4,r3,r4 - add r4,r4,r5 - srwi. r4,r4,L1_CACHE_SHIFT - beqlr - mtctr r4 - -1: dcbi 0,r3 - addi r3,r3,L1_CACHE_BYTES - bdnz 1b - sync /* wait for dcbi's to get to ram */ - blr - -/* - * Flush a particular page from the data cache to RAM. - * Note: this is necessary because the instruction cache does *not* - * snoop from the data cache. - * This is a no-op on the 601 which has a unified cache. - * - * void __flush_dcache_icache(void *page) - */ -_GLOBAL(__flush_dcache_icache) -BEGIN_FTR_SECTION - blr -END_FTR_SECTION_IFSET(CPU_FTR_COHERENT_ICACHE) - rlwinm r3,r3,0,0,31-PAGE_SHIFT /* Get page base address */ - li r4,PAGE_SIZE/L1_CACHE_BYTES /* Number of lines in a page */ - mtctr r4 - mr r6,r3 -0: dcbst 0,r3 /* Write line to ram */ - addi r3,r3,L1_CACHE_BYTES - bdnz 0b - sync -#ifdef CONFIG_44x - /* We don't flush the icache on 44x. Those have a virtual icache - * and we don't have access to the virtual address here (it's - * not the page vaddr but where it's mapped in user space). The - * flushing of the icache on these is handled elsewhere, when - * a change in the address space occurs, before returning to - * user space - */ -BEGIN_MMU_FTR_SECTION - blr -END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_44x) -#endif /* CONFIG_44x */ - mtctr r4 -1: icbi 0,r6 - addi r6,r6,L1_CACHE_BYTES - bdnz 1b - sync - isync - blr - -#ifndef CONFIG_BOOKE -/* - * Flush a particular page from the data cache to RAM, identified - * by its physical address. We turn off the MMU so we can just use - * the physical address (this may be a highmem page without a kernel - * mapping). - * - * void __flush_dcache_icache_phys(unsigned long physaddr) - */ -_GLOBAL(__flush_dcache_icache_phys) -BEGIN_FTR_SECTION - blr /* for 601, do nothing */ -END_FTR_SECTION_IFSET(CPU_FTR_COHERENT_ICACHE) - mfmsr r10 - rlwinm r0,r10,0,28,26 /* clear DR */ - mtmsr r0 - isync - rlwinm r3,r3,0,0,31-PAGE_SHIFT /* Get page base address */ - li r4,PAGE_SIZE/L1_CACHE_BYTES /* Number of lines in a page */ - mtctr r4 - mr r6,r3 -0: dcbst 0,r3 /* Write line to ram */ - addi r3,r3,L1_CACHE_BYTES - bdnz 0b - sync - mtctr r4 -1: icbi 0,r6 - addi r6,r6,L1_CACHE_BYTES - bdnz 1b - sync - mtmsr r10 /* restore DR */ - isync - blr -#endif /* CONFIG_BOOKE */ - -/* - * Clear pages using the dcbz instruction, which doesn't cause any - * memory traffic (except to write out any cache lines which get - * displaced). This only works on cacheable memory. - * - * void clear_pages(void *page, int order) ; - */ -_GLOBAL(clear_pages) - li r0,PAGE_SIZE/L1_CACHE_BYTES - slw r0,r0,r4 - mtctr r0 -1: dcbz 0,r3 - addi r3,r3,L1_CACHE_BYTES - bdnz 1b - blr - -/* * Copy a whole page. We use the dcbz instruction on the destination * to reduce memory traffic (it eliminates the unnecessary reads of * the destination into cache). This requires that the destinationarch/powerpc/kernel/misc_64.S +0
-182@@ -53,188 +53,6 @@ _GLOBAL(call_handle_irq) mtlr r0 blr - .section ".toc","aw" -PPC64_CACHES: - .tc ppc64_caches[TC],ppc64_caches - .section ".text" - -/* - * Write any modified data cache blocks out to memory - * and invalidate the corresponding instruction cache blocks. - * - * flush_icache_range(unsigned long start, unsigned long stop) - * - * flush all bytes from start through stop-1 inclusive - */ - -_KPROBE(__flush_icache_range) - -/* - * Flush the data cache to memory - * - * Different systems have different cache line sizes - * and in some cases i-cache and d-cache line sizes differ from - * each other. - */ - ld r10,PPC64_CACHES@toc(r2) - lwz r7,DCACHEL1LINESIZE(r10)/* Get cache line size */ - addi r5,r7,-1 - andc r6,r3,r5 /* round low to line bdy */ - subf r8,r6,r4 /* compute length */ - add r8,r8,r5 /* ensure we get enough */ - lwz r9,DCACHEL1LOGLINESIZE(r10) /* Get log-2 of cache line size */ - srw. r8,r8,r9 /* compute line count */ - beqlr /* nothing to do? */ - mtctr r8 -1: dcbst 0,r6 - add r6,r6,r7 - bdnz 1b - sync - -/* Now invalidate the instruction cache */ - - lwz r7,ICACHEL1LINESIZE(r10) /* Get Icache line size */ - addi r5,r7,-1 - andc r6,r3,r5 /* round low to line bdy */ - subf r8,r6,r4 /* compute length */ - add r8,r8,r5 - lwz r9,ICACHEL1LOGLINESIZE(r10) /* Get log-2 of Icache line size */ - srw. r8,r8,r9 /* compute line count */ - beqlr /* nothing to do? */ - mtctr r8 -2: icbi 0,r6 - add r6,r6,r7 - bdnz 2b - isync - blr - .previous .text -/* - * Like above, but only do the D-cache. - * - * flush_dcache_range(unsigned long start, unsigned long stop) - * - * flush all bytes from start to stop-1 inclusive - */ -_GLOBAL(flush_dcache_range) - -/* - * Flush the data cache to memory - * - * Different systems have different cache line sizes - */ - ld r10,PPC64_CACHES@toc(r2) - lwz r7,DCACHEL1LINESIZE(r10) /* Get dcache line size */ - addi r5,r7,-1 - andc r6,r3,r5 /* round low to line bdy */ - subf r8,r6,r4 /* compute length */ - add r8,r8,r5 /* ensure we get enough */ - lwz r9,DCACHEL1LOGLINESIZE(r10) /* Get log-2 of dcache line size */ - srw. r8,r8,r9 /* compute line count */ - beqlr /* nothing to do? */ - mtctr r8 -0: dcbst 0,r6 - add r6,r6,r7 - bdnz 0b - sync - blr - -/* - * Like above, but works on non-mapped physical addresses. - * Use only for non-LPAR setups ! It also assumes real mode - * is cacheable. Used for flushing out the DART before using - * it as uncacheable memory - * - * flush_dcache_phys_range(unsigned long start, unsigned long stop) - * - * flush all bytes from start to stop-1 inclusive - */ -_GLOBAL(flush_dcache_phys_range) - ld r10,PPC64_CACHES@toc(r2) - lwz r7,DCACHEL1LINESIZE(r10) /* Get dcache line size */ - addi r5,r7,-1 - andc r6,r3,r5 /* round low to line bdy */ - subf r8,r6,r4 /* compute length */ - add r8,r8,r5 /* ensure we get enough */ - lwz r9,DCACHEL1LOGLINESIZE(r10) /* Get log-2 of dcache line size */ - srw. r8,r8,r9 /* compute line count */ - beqlr /* nothing to do? */ - mfmsr r5 /* Disable MMU Data Relocation */ - ori r0,r5,MSR_DR - xori r0,r0,MSR_DR - sync - mtmsr r0 - sync - isync - mtctr r8 -0: dcbst 0,r6 - add r6,r6,r7 - bdnz 0b - sync - isync - mtmsr r5 /* Re-enable MMU Data Relocation */ - sync - isync - blr - -_GLOBAL(flush_inval_dcache_range) - ld r10,PPC64_CACHES@toc(r2) - lwz r7,DCACHEL1LINESIZE(r10) /* Get dcache line size */ - addi r5,r7,-1 - andc r6,r3,r5 /* round low to line bdy */ - subf r8,r6,r4 /* compute length */ - add r8,r8,r5 /* ensure we get enough */ - lwz r9,DCACHEL1LOGLINESIZE(r10)/* Get log-2 of dcache line size */ - srw. r8,r8,r9 /* compute line count */ - beqlr /* nothing to do? */ - sync - isync - mtctr r8 -0: dcbf 0,r6 - add r6,r6,r7 - bdnz 0b - sync - isync - blr - - -/* - * Flush a particular page from the data cache to RAM. - * Note: this is necessary because the instruction cache does *not* - * snoop from the data cache. - * - * void __flush_dcache_icache(void *page) - */ -_GLOBAL(__flush_dcache_icache) -/* - * Flush the data cache to memory - * - * Different systems have different cache line sizes - */ - -/* Flush the dcache */ - ld r7,PPC64_CACHES@toc(r2) - clrrdi r3,r3,PAGE_SHIFT /* Page align */ - lwz r4,DCACHEL1LINESPERPAGE(r7) /* Get # dcache lines per page */ - lwz r5,DCACHEL1LINESIZE(r7) /* Get dcache line size */ - mr r6,r3 - mtctr r4 -0: dcbst 0,r6 - add r6,r6,r5 - bdnz 0b - sync - -/* Now invalidate the icache */ - - lwz r4,ICACHEL1LINESPERPAGE(r7) /* Get # icache lines per page */ - lwz r5,ICACHEL1LINESIZE(r7) /* Get icache line size */ - mtctr r4 -1: icbi 0,r3 - add r3,r3,r5 - bdnz 1b - isync - blr - - #if defined(CONFIG_PPC_PMAC) || defined(CONFIG_PPC_MAPLE) /* * Do an IO access in real modearch/powerpc/kernel/ppc_ksyms.c +0
-3@@ -53,7 +53,6 @@ extern void program_check_exception(struct pt_regs *regs);extern void single_step_exception (struct pt_regs *regs);extern int sys_sigreturn (struct pt_regs *regs); -EXPORT_SYMBOL (clear_pages);EXPORT_SYMBOL (ISA_DMA_THRESHOLD );EXPORT_SYMBOL (DMA_MODE_READ );EXPORT_SYMBOL (DMA_MODE_WRITE );@@ -113,8 +112,6 @@ EXPORT_SYMBOL(giveup_spe);#ifndef CONFIG_PPC64 EXPORT_SYMBOL (flush_instruction_cache);#endif -EXPORT_SYMBOL (__flush_icache_range); -EXPORT_SYMBOL (flush_dcache_range);#ifdef CONFIG_SMP #ifdef CONFIG_PPC32 arch/powerpc/kernel/setup-common.c +103
-0@@ -83,6 +83,54 @@ unsigned long klimit = (unsigned long) _end;char cmd_line[COMMAND_LINE_SIZE ];/* +* Initialize these values to minimum safe defaults in case they need to be +* used early during the boot process. While this may not seem safe, it is +* actually safe in practice, because all of the kernel loops that use this +* data operate on whole pages. +* +* The PowerPC Book III-E spec documents that the pagesize is an even +* multiple of the cache block size and the cache blocks are always +* page-aligned. +* +* So, for example, when clearing a whole page there are only two things that +* can be done wrong with "dcbz": +* +* (1) Call "dcbz" with an address outside the page you want to zero. +* +* (2) Call "dcbz" too few times to actually hit all of the cachelines, +* IE: Use a too-large cacheline stride. +* +* So as long as we ensure that this number is small enough for the current +* CPU everything will operate correctly, albeit with a slight performance +* hit, until we get a chance to parse the device-tree for the right value. +* +* NOTE: Userspace expects an exact value, so none of the above applies after +* the device tree has been unflattened and actual values computed. +* +* See arch/powerpc/asm/caches.h for more information. +*/ +struct powerpc_caches powerpc_caches = { +/* Data cache sizes */ + .dcache_total_bytes =0 ,/* Unknown */ + .dcache_block_bytes =L1_CACHE_BYTES_MIN , + .dcache_block_shift =L1_CACHE_SHIFT_MIN , + .dcache_blocks_per_page = (PAGE_SIZE >>L1_CACHE_SHIFT_MIN ), + +/* Instruction cache sizes */ + .icache_total_bytes =0 , + .icache_block_bytes =L1_CACHE_BYTES_MIN , + .icache_block_shift =L1_CACHE_SHIFT_MIN , + .icache_blocks_per_page = (PAGE_SIZE >>L1_CACHE_SHIFT_MIN ), + +/* Unified cache (assume cache is split by default) */ + .ucache_total_bytes =0 , + .ucache_block_bytes =0 , + .ucache_block_shift =0 , + .ucache_blocks_per_page =0 , +}; +EXPORT_SYMBOL_GPL (powerpc_caches); + +/* * This still seems to be needed... -- paulus */ struct screen_info screen_info = {@@ -349,6 +397,61 @@ const struct seq_operations cpuinfo_op = { .show = show_cpuinfo, }; +/* Helper functions to compute various values from a cache block size */ +static void __init set_dcache_block_data (u32 bytes) +{ +u32 shift =__ilog2 (bytes); + powerpc_caches.dcache_block_bytes = bytes; + powerpc_caches.dcache_block_shift = shift; + powerpc_caches.dcache_blocks_per_page = (PAGE_SIZE >> shift); +} +static void __init set_icache_block_data (u32 bytes) +{ +u32 shift =__ilog2 (bytes); + powerpc_caches.icache_block_bytes = bytes; + powerpc_caches.icache_block_shift = shift; + powerpc_caches.icache_blocks_per_page = (PAGE_SIZE >> shift); +} + +/* +* Preinitialize the powerpc_caches structure from the cputable. We will +* later scan the device-tree for this information, which may be more +* accurate. +*/ +void __init initialize_early_cache_info (void ) +{ +set_dcache_block_data (cur_cpu_spec->dcache_bsize); +set_icache_block_data (cur_cpu_spec->icache_bsize); +} + +/* +* Initialize the powerpc_caches structure from the device-tree for use by +* copy_page(), cache flush routines, and AT_DCACHEBSIZE elf headers. +* +* In the unlikely event that the device-tree doesn't have this information, +* the defaults loaded by initialize_early_cache_info() from the cputable +* will be used. +*/ +void __init initialize_cache_info (void ) +{ +/* Assume that the cache properties are the same across all nodes */ +struct device_node *np =of_find_node_by_type (NULL ,"cpu" ); +u32 value =0 ; + +/* First check data/instruction cache block sizes */ +if ( !of_property_read_u32 (np,"d-cache-block-size" , &value) || + !of_property_read_u32 (np,"d-cache-line-size" , &value)) +set_dcache_block_data (value); + +if ( !of_property_read_u32 (np,"i-cache-block-size" , &value) || + !of_property_read_u32 (np,"i-cache-line-size" , &value)) +set_icache_block_data (value); + +/* Also read total cache sizes (no defaults here) */ +of_property_read_u32 (np,"d-cache-size" , &powerpc_caches.dcache_total_bytes); +of_property_read_u32 (np,"i-cache-size" , &powerpc_caches.icache_total_bytes); +} +void __init check_for_initrd (void ) {#ifdef CONFIG_BLK_DEV_INITRD arch/powerpc/kernel/setup.h +1
-0@@ -1,6 +1,7 @@#ifndef _POWERPC_KERNEL_SETUP_H#define _POWERPC_KERNEL_SETUP_H +void initialize_cache_info (void );void check_for_initrd (void );void do_init_bootmem (void );void setup_panic (void );arch/powerpc/kernel/setup_32.c +3
-8@@ -63,14 +63,6 @@ EXPORT_SYMBOL(vgacon_remap_base);#endif /* -* These are used in binfmt_elf.c to put aux entries on the stack -* for each elf executable being started. -*/ -int dcache_bsize; -int icache_bsize; -int ucache_bsize; - -/* * We're called here very early in the boot. We determine the machine * type and call the appropriate low-level setup functions. * -- Cort <[email protected]> @@ -286,10 +278,13 @@ void __init setup_arch(char **cmdline_p) { *cmdline_p = cmd_line; +initialize_early_cache_info (); +/* so udelay does something sensible, assume <= 1000 bogomips */ loops_per_jiffy =500000000 /HZ ;unflatten_device_tree (); +initialize_cache_info ();check_for_initrd ();if (ppc_md.init_early)arch/powerpc/kernel/setup_64.c +6
-112@@ -77,25 +77,6 @@ int boot_cpuid = 0;int __initdata spinning_secondaries;u64 ppc64_pft_size; -/* Pick defaults since we might want to patch instructions -* before we've read this from the device tree. -*/ -struct ppc64_caches ppc64_caches = { - .dline_size =0x40 , - .log_dline_size =6 , - .iline_size =0x40 , - .log_iline_size =6 -}; -EXPORT_SYMBOL_GPL (ppc64_caches); - -/* -* These are used in binfmt_elf.c to put aux entries on the stack -* for each elf executable being started. -*/ -int dcache_bsize; -int icache_bsize; -int ucache_bsize; -#ifdef CONFIG_SMP static char *smt_enabled_cmdline;@@ -265,82 +246,6 @@ void smp_release_cpus(void)#endif /* CONFIG_SMP || CONFIG_KEXEC */ /* -* Initialize some remaining members of the ppc64_caches and systemcfg -* structures -* (at least until we get rid of them completely). This is mostly some -* cache informations about the CPU that will be used by cache flush -* routines and/or provided to userland -*/ -static void __init initialize_cache_info (void ) -{ -struct device_node *np; -unsigned long num_cpus =0 ; - -DBG (" -> initialize_cache_info()\n" ); - -for_each_node_by_type (np,"cpu" ) { - num_cpus +=1 ; - -/* -* We're assuming *all* of the CPUs have the same -* d-cache and i-cache sizes... -Peter -*/ -if (num_cpus ==1 ) { -const u32 *sizep, *lsizep; -u32 size, lsize; - - size =0 ; - lsize = cur_cpu_spec->dcache_bsize; - sizep =of_get_property (np,"d-cache-size" ,NULL ); -if (sizep !=NULL ) - size = *sizep; - lsizep =of_get_property (np,"d-cache-block-size" , -NULL ); -/* fallback if block size missing */ -if (lsizep ==NULL ) - lsizep =of_get_property (np, -"d-cache-line-size" , -NULL ); -if (lsizep !=NULL ) - lsize = *lsizep; -if (sizep ==0 || lsizep ==0 ) -DBG ("Argh, can't find dcache properties ! " -"sizep: %p, lsizep: %p\n" , sizep, lsizep); - - ppc64_caches.dsize = size; - ppc64_caches.dline_size = lsize; - ppc64_caches.log_dline_size =__ilog2 (lsize); - ppc64_caches.dlines_per_page =PAGE_SIZE / lsize; - - size =0 ; - lsize = cur_cpu_spec->icache_bsize; - sizep =of_get_property (np,"i-cache-size" ,NULL ); -if (sizep !=NULL ) - size = *sizep; - lsizep =of_get_property (np,"i-cache-block-size" , -NULL ); -if (lsizep ==NULL ) - lsizep =of_get_property (np, -"i-cache-line-size" , -NULL ); -if (lsizep !=NULL ) - lsize = *lsizep; -if (sizep ==0 || lsizep ==0 ) -DBG ("Argh, can't find icache properties ! " -"sizep: %p, lsizep: %p\n" , sizep, lsizep); - - ppc64_caches.isize = size; - ppc64_caches.iline_size = lsize; - ppc64_caches.log_iline_size =__ilog2 (lsize); - ppc64_caches.ilines_per_page =PAGE_SIZE / lsize; - } - } - -DBG (" <- initialize_cache_info()\n" ); -} - - -/* * Do some initial setup of the system. The parameters are those which * were passed in from the bootloader. */ @@ -365,10 +270,7 @@ void __init setup_system(void)*/ unflatten_device_tree (); -/* -* Fill the ppc64_caches & systemcfg structures with informations -* retrieved from the device-tree. -*/ +/* Fill the powerpc_caches structure with device-tree data */ initialize_cache_info ();#ifdef CONFIG_PPC_RTAS @@ -423,12 +325,10 @@ void __init setup_system(void)printk ("-----------------------------------------------------\n" );printk ("ppc64_pft_size = 0x%llx\n" , ppc64_pft_size);printk ("physicalMemorySize = 0x%llx\n" ,memblock_phys_mem_size ()); -if (ppc64_caches.dline_size !=0x80 ) -printk ("ppc64_caches.dcache_line_size = 0x%x\n" , - ppc64_caches.dline_size); -if (ppc64_caches.iline_size !=0x80 ) -printk ("ppc64_caches.icache_line_size = 0x%x\n" , - ppc64_caches.iline_size); +if (powerpc_caches.dcache_block_bytes !=0x80 ) +printk ("dcache_block_bytes = 0x%x\n" , powerpc_caches.dcache_block_bytes); +if (powerpc_caches.icache_block_bytes !=0x80 ) +printk ("icache_block_bytes = 0x%x\n" , powerpc_caches.icache_block_bytes);#ifdef CONFIG_PPC_STD_MMU_64 if (htab_address)printk ("htab_address = 0x%p\n" , htab_address);@@ -545,13 +445,7 @@ void __init setup_arch(char **cmdline_p) *cmdline_p = cmd_line; -/* -* Set cache line size based on type of cpu as a default. -* Systems with OF can look in the properties on the cpu node(s) -* for a possibly more accurate value. -*/ - dcache_bsize = ppc64_caches.dline_size; - icache_bsize = ppc64_caches.iline_size; +initialize_early_cache_info ();/* reboot on panic */ panic_timeout =180 ;arch/powerpc/kernel/vdso.c +12
-15@@ -726,6 +726,7 @@ static int __init vdso_init(void) vdso_data->version.major =SYSTEMCFG_MAJOR ; vdso_data->version.minor =SYSTEMCFG_MINOR ; vdso_data->processor =mfspr (SPRN_PVR ); +/* * Fake the old platform number for pSeries and iSeries and add * in LPAR bit if necessary @@ -734,29 +735,25 @@ static int __init vdso_init(void)if (firmware_has_feature (FW_FEATURE_LPAR )) vdso_data->platform |=1 ; vdso_data->physicalMemorySize =memblock_phys_mem_size (); - vdso_data->dcache_size = ppc64_caches.dsize; - vdso_data->dcache_line_size = ppc64_caches.dline_size; - vdso_data->icache_size = ppc64_caches.isize; - vdso_data->icache_line_size = ppc64_caches.iline_size; -/* XXXOJN: Blocks should be added to ppc64_caches and used instead */ - vdso_data->dcache_block_size = ppc64_caches.dline_size; - vdso_data->icache_block_size = ppc64_caches.iline_size; - vdso_data->dcache_log_block_size = ppc64_caches.log_dline_size; - vdso_data->icache_log_block_size = ppc64_caches.log_iline_size; +/* There are more cache parameters saved for 64-bit than 32-bit */ + vdso_data->dcache_size = powerpc_caches.dcache_total_size; + vdso_data->icache_size = powerpc_caches.icache_total_size; + vdso_data->dcache_line_size = powerpc_caches.dcache_block_bytes; + vdso_data->icache_line_size = powerpc_caches.icache_block_bytes;/* * Calculate the size of the 64 bits vDSO */ vdso64_pages = (&vdso64_end - &vdso64_start) >>PAGE_SHIFT ;DBG ("vdso64_kbase: %p, 0x%x pages\n" , vdso64_kbase, vdso64_pages); -#else - vdso_data->dcache_block_size =L1_CACHE_BYTES ; - vdso_data->dcache_log_block_size =L1_CACHE_SHIFT ; - vdso_data->icache_block_size =L1_CACHE_BYTES ; - vdso_data->icache_log_block_size =L1_CACHE_SHIFT ; -#endif /* CONFIG_PPC64 */ +#endif +/* Save the cache-block sizes for the VDSO */ + vdso_data->dcache_block_size = powerpc_caches.dcache_block_bytes; + vdso_data->icache_block_size = powerpc_caches.icache_block_bytes; + vdso_data->dcache_log_block_size = powerpc_caches.dcache_block_shift; + vdso_data->icache_log_block_size = powerpc_caches.icache_block_shift;/* * Calculate the size of the 32 bits vDSO arch/powerpc/lib/copypage_64.S +5
-5@@ -12,17 +12,17 @@ #include <asm/asm-offsets.h> .section ".toc","aw" -PPC64_CACHES: - .tc ppc64_caches[TC],ppc64_caches +POWERPC_CACHES: + .tc powerpc_caches[TC],powerpc_caches .section ".text" _GLOBAL(copy_page) lis r5,PAGE_SIZE@h ori r5,r5,PAGE_SIZE@l BEGIN_FTR_SECTION - ld r10,PPC64_CACHES@toc(r2) - lwz r11,DCACHEL1LOGLINESIZE(r10) /* log2 of cache line size */ - lwz r12,DCACHEL1LINESIZE(r10) /* get cache line size */ + ld r10,POWERPC_CACHES@toc(r2) + lwz r11,DCACHE_BLOCK_SHIFT(r10) /* log2 of cache line size */ + lwz r12,DCACHE_BLOCK_BYTES(r10) /* get cache line size */ li r9,0 srd r8,r5,r11arch/powerpc/mm/Makefile +1
-1@@ -6,7 +6,7 @@ subdir-ccflags-$(CONFIG_PPC_WERROR) := -Werror ccflags-$(CONFIG_PPC64) := -mno-minimal-toc -obj-y := fault.o mem.o pgtable.o gup.o \ +obj-y := cache.o fault.o mem.o pgtable.o gup.o \ init_$(CONFIG_WORD_SIZE).o \ pgtable_$(CONFIG_WORD_SIZE).o obj-$(CONFIG_PPC_MMU_NOHASH) += mmu_context_nohash.o tlb_nohash.o \arch/powerpc/mm/cache.c new file +279
-0@@ -0,0 +1,279 @@ +#include <linux/kprobes.h> +#include <linux/export.h> +#include <linux/types.h> + +#include <asm/cputable.h> +#include <asm/system.h> +#include <asm/cache.h> +#include <asm/page.h> +#include <asm/mmu.h> + +/* +* Write any modified data cache blocks out to memory. +* Does not invalidate the corresponding cache lines (especially for +* any corresponding instruction cache). +*/ +void clean_dcache_range (unsigned long start,unsigned long stop) +{ +unsigned long addr; + FOR_EACH_CACHELINE(addr, start, stop, dcache) +dcbst (addr); +mb (); +} + +/* +* Write any modified data cache blocks out to memory and invalidate them. +* Does not invalidate the corresponding instruction cache blocks. +*/ +void flush_dcache_range (unsigned long start,unsigned long stop) +{ +unsigned long addr; + FOR_EACH_CACHELINE(addr, start, stop, dcache) +dcbf (addr); +mb (); +} +EXPORT_SYMBOL (flush_dcache_range); + +/* +* Like above, but invalidate the D-cache. This is used by the 8xx +* to invalidate the cache so the PPC core doesn't get stale data +* from the CPM (no cache snooping here :-). +* +* invalidate_dcache_range(unsigned long start, unsigned long stop) +*/ +void invalidate_dcache_range (unsigned long start,unsigned long stop) +{ +unsigned long addr; + FOR_EACH_CACHELINE(addr, start, stop, dcache) +dcbi (addr); +mb (); +} + +/* +* Unfortunately, we cannot flush individual chunks of the icache on 44x as +* we are passed kmapped addresses and we have a virtually-tagged icache. +* +* The only workaround is to invalidate the whole icache. +* +* NOTE: The CPU does not use the operands for this instruction, so +* they are passed as dummies. +*/ +__kprobes void __flush_icache_range (unsigned long start,unsigned long stop) +{ +unsigned long addr; + +if (cpu_has_feature (CPU_FTR_COHERENT_ICACHE )) +return ; + +/* First ensure that data has been written to memory */ + FOR_EACH_CACHELINE(addr, start, stop, dcache) +dcbst (addr); +mb (); + +#ifdef CONFIG_44x +if (mmu_has_feature (MMU_FTR_TYPE_44x)) { + asmvolatile ("iccci 0, r0" :::"memory" ); +return ; + } +#endif + +/* Now discard the corresponding icache */ + FOR_EACH_CACHELINE(addr, start, stop, icache) +icbi (addr); +mb (); +isync (); +} +EXPORT_SYMBOL (__flush_icache_range); + +/* +* Flush a particular page from the data cache to RAM. +* Note: this is necessary because the instruction cache does *not* +* snoop from the data cache. +* This is a no-op on the 601 which has a unified cache. +* +* void __flush_dcache_icache(void *page) +*/ +void __flush_dcache_icache (void *page) +{ +unsigned long base = ((unsigned long )page) & ~(PAGE_SIZE -1 ); +unsigned long addr; + +if (cpu_has_feature (CPU_FTR_COHERENT_ICACHE )) +return ; + +/* First ensure that data has been written to memory */ + FOR_EACH_CACHELINE(addr, base, base + PAGE_SIZE, dcache) +dcbst (addr); + +#ifdef CONFIG_44x +/* +* We don't flush the icache on 44x. Those have a virtual icache and +* we don't have access to the virtual address here (it's not the +* page vaddr but where it's mapped in user space). The flushing of +* the icache on these is handled elsewhere, when a change in the +* address space occurs, before returning to user space. +*/ +if (mmu_has_feature (MMU_FTR_TYPE_44x)) +return ; +#endif + + FOR_EACH_CACHELINE(addr, base, base + PAGE_SIZE, icache) +icbi (addr); + +mb (); +isync (); +} + +/* +* Clear pages using the dcbz instruction, which doesn't cause any +* memory traffic (except to write out any cache lines which get +* displaced). This only works on cacheable memory. +* +*/ +void clear_pages (void *page,int order) +{ +unsigned long addr, base = (unsigned long )page; + FOR_EACH_CACHELINE(addr, base, base + (PAGE_SIZE << order), dcache) +dcbz (addr); +} +EXPORT_SYMBOL (clear_pages); + +#if defined(CONFIG_PPC32 ) && !defined(CONFIG_BOOKE ) +/* +* Flush a particular page from the data cache to RAM, identified +* by its physical address. We turn off the MMU so we can just use +* the physical address (this may be a highmem page without a kernel +* mapping). +*/ +void __flush_dcache_icache_phys (unsigned long phys_page) +{ +u32 d_size = powerpc_caches.dcache_block_bytes; +u32 i_size = powerpc_caches.icache_block_bytes; +u32 d_per_page = powerpc_caches.dcache_blocks_per_page; +u32 i_per_page = powerpc_caches.icache_blocks_per_page; + +/* Temporary registers for the ASM to use */ +unsigned long old_msr, tmp_msr, d_phys_page, i_phys_page; + +if (cpu_has_feature (CPU_FTR_COHERENT_ICACHE )) +return ; + +/* Page base address (used in 2 different loops) */ + d_phys_page = i_phys_page = phys_page & ~(PAGE_SIZE -1 ); + +/* +* This part needs to be 100% ASM because we disable the MMU, and we +* can't accidentally let some C code go poking at memory while the +* MMU isn't enabled. +* +* NOTE: This looks blatantly unsafe with respect to interrupts. +* Hopefully all the callers provide sufficient protection? +*/ + asmvolatile ( +/* First disable the MMU */ +"mfmsr %[old_msr]\n\t" +"rlwinm %[tmp_msr], %[old_msr], 0, 28, 26\n\t" +"mtmsr %[tmp_msr]\n\t" +"isync\n\t" + +/* Clean the data cache */ +"mtctr %[d_per_page]\n" +"0: dcbst 0, %[d_phys_page]\n\t" +"add %[d_phys_page], %[d_phys_page], %[d_size]\n\t" +"bdnz 0b\n\t" +"sync\n\t" + +/* Invalidate the instruction cache */ +"mtctr %[i_per_page]\n" +"0: icbi 0, %[i_phys_page]\n\t" +"add %[i_phys_page], %[i_phys_page], %[i_size]\n\t" +"bdnz 0b\n\t" + +/* Finally, re-enable the MMU */ +"sync\n\t" +"mtmsr %[old_msr]\n\t" +"isync\n\t" + +/* Temporary variables and inputs */ + : [old_msr]"=&r" (old_msr), + [tmp_msr]"=&r" (tmp_msr), + [d_phys_page]"=b" (d_phys_page), + [i_phys_page]"=b" (i_phys_page) + +/* Inputs */ + : [d_size]"b" (d_size), + [i_size]"b" (i_size), + [d_per_page]"b" (d_per_page), + [i_per_page]"b" (i_per_page), +"[d_phys_page]" (d_phys_page), +"[i_phys_page]" (i_phys_page) + +/* Clobbers */ + :"memory" ,"c" + ); +} +#endif /* CONFIG_PPC32 && !CONFIG_BOOKE */ + +#ifdef CONFIG_PPC64 +/* +* Data cache flush that works on non-mapped physical addresses. +* Use only for non-LPAR setups ! It also assumes real mode +* is cacheable. Used for flushing out the DART before using +* it as uncacheable memory +*/ +void flush_dcache_phys_range (unsigned long start,unsigned long stop) +{ +/* System data cache block size */ +unsigned long bytes = powerpc_caches.dcache_block_bytes; +unsigned long shift = powerpc_caches.dcache_block_shift; + +/* Temporary registers for the ASM to use */ +unsigned long old_msr, tmp_msr; + +/* Compute a start address and number of cachelines */ +unsigned long phys_addr = start & ~(bytes -1 ); +unsigned long nr_lines = ((stop - phys_addr) + (bytes -1 )) >> shift; + +/* +* This part needs to be 100% ASM because we disable the MMU, and we +* can't accidentally let some C code go poking at memory while the +* MMU isn't enabled. +* +* NOTE: This looks blatantly unsafe with respect to interrupts. +* Hopefully all the callers provide sufficient protection? +*/ + asmvolatile ( +/* First disable the MMU */ +"mfmsr %[old_msr]\n\t" +"rlwinm %[tmp_msr], %[old_msr], 0, 28, 26\n\t" +"mtmsr %[tmp_msr]\n\t" +"isync\n\t" + +/* Clean the data cache */ +"mtctr %[nr_lines]\n" +"0: dcbst 0, %[phys_addr]\n\t" +"add %[phys_addr], %[phys_addr], %[bytes]\n\t" +"bdnz 0b\n\t" +"sync\n\t" +"isync\n\t" + +/* Finally, re-enable the MMU */ +"mtmsr %[old_msr]\n\t" +"sync\n\t" +"isync\n\t" + +/* Temporary variables and inputs */ + : [old_msr]"=&r" (old_msr), + [tmp_msr]"=&r" (tmp_msr), + [phys_addr]"=b" (phys_addr) + +/* Inputs */ + : [bytes]"b" (bytes), + [nr_lines]"b" (nr_lines), +"[phys_addr]" (phys_addr) + +/* Clobbers */ + :"memory" ,"c" + ); +} +#endif /* CONFIG_PPC64 */ arch/powerpc/mm/dma-noncoherent.c +1
-1@@ -328,7 +328,7 @@ void __dma_sync(void *vaddr, size_t size, int direction)* invalidate only when cache-line aligned otherwise there is * the potential for discarding uncommitted data from the cache */ -if ((start & (L1_CACHE_BYTES -1 )) || (size & (L1_CACHE_BYTES -1 ))) +if ((start | size) & (powerpc_caches.dcache_block_bytes -1 ))flush_dcache_range (start, end);else invalidate_dcache_range (start, end);arch/powerpc/platforms/52xx/lite5200_sleep.S +7
-2@@ -394,11 +394,16 @@ restore_regs: /* cache flushing code. copied from arch/ppc/boot/util.S */ -#define NUM_CACHE_LINES (128*8) +#define NUM_CACHE_LINES ((128 * 8) << (L1_CACHE_SHIFT_MAX - L1_CACHE_SHIFT_MIN)) /* * Flush data cache * Do this by just reading lots of stuff into the cache. + * + * NOTE: This does not handle variable-sized cachelines properly, but since + * we are just trying to flush the data cache by reading lots of data, + * this works anyways. We just make sure we read as many cachelines + * as we could possibly need to overflow the cache on any hardware. */ flush_data_cache: lis r3,CONFIG_KERNEL_START@h@@ -407,6 +412,6 @@ flush_data_cache: mtctr r4 1: lwz r4,0(r3) - addi r3,r3,L1_CACHE_BYTES /* Next line, please */ + addi r3,r3,L1_CACHE_BYTES_MIN /* Next line, please */ bdnz 1b blrarch/powerpc/platforms/powermac/pci.c +1
-1@@ -1135,7 +1135,7 @@ int pmac_pci_enable_device_hook(struct pci_dev *dev)pci_write_config_byte (dev,PCI_LATENCY_TIMER ,16 );pci_write_config_byte (dev,PCI_CACHE_LINE_SIZE , -L1_CACHE_BYTES >>2 ); + powerpc_caches.dcache_block_bytes >>2 ); }return 0 ;arch/powerpc/xmon/xmon.c +25
-28@@ -26,6 +26,7 @@#include <asm/ptrace.h> #include <asm/string.h> +#include <asm/cache.h> #include <asm/prom.h> #include <asm/machdep.h> #include <asm/xmon.h> @@ -254,16 +255,6 @@ static inline void store_inst(void *p) asmvolatile ("dcbst 0,%0; sync; icbi 0,%0; isync" : :"r" (p)); } -static inline void cflush (void *p) -{ - asmvolatile ("dcbf 0,%0; icbi 0,%0" : :"r" (p)); -} - -static inline void cinval (void *p) -{ - asmvolatile ("dcbi 0,%0; icbi 0,%0" : :"r" (p)); -} -/* * Disable surveillance (the service processor watchdog function) * while we are in xmon. @@ -1513,10 +1504,9 @@ static void prregs(struct pt_regs *fp)static void cacheflush (void ) { -int cmd; -unsigned long nflush; +unsigned long nflush, i; - cmd =inchar (); +int cmd =inchar ();if (cmd !='i' ) termch = cmd;scanhex ((void *)&adrs);@@ -1524,23 +1514,30 @@ static void cacheflush(void) termch =0 ; nflush =1 ;scanhex (&nflush); - nflush = (nflush +L1_CACHE_BYTES -1 ) /L1_CACHE_BYTES ; -if (setjmp (bus_error_jmp) ==0 ) { - catch_memory_errors =1 ; -sync (); -if (cmd !='i' ) { -for (; nflush >0 ; --nflush, adrs +=L1_CACHE_BYTES ) -cflush ((void *) adrs); - }else { -for (; nflush >0 ; --nflush, adrs +=L1_CACHE_BYTES ) -cinval ((void *) adrs); - } -sync (); -/* wait a little while to see if we get a machine check */ -__delay (200 ); +if (setjmp (bus_error_jmp) !=0 ) { + catch_memory_errors =0 ; +return ; } - catch_memory_errors =0 ; + catch_memory_errors =1 ; +sync (); + +/* First flush/invalidate data caches */ +if (cmd !='i' ) { + FOR_EACH_CACHELINE(i, adrs, adrs + nflush, dcache) +dcbf (i); + }else { + FOR_EACH_CACHELINE(i, adrs, adrs + nflush, dcache) +dcbi (i); + } + +/* Now invalidate instruction caches */ + FOR_EACH_CACHELINE(i, adrs, adrs + nflush, icache) +icbi (i); + +sync (); +/* wait a little while to see if we get a machine check */ +__delay (200 ); } static unsigned longdrivers/macintosh/smu.c +6
-2@@ -136,7 +136,9 @@ static void smu_start_cmd(void)/* Flush command and data to RAM */ faddr = (unsigned long )smu->cmd_buf; fend = faddr + smu->cmd_buf->length +2 ; -flush_inval_dcache_range (faddr, fend); +flush_dcache_range (faddr, fend); +mb (); +isync ();/* We also disable NAP mode for the duration of the command @@ -198,7 +200,9 @@ static irqreturn_t smu_db_intr(int irq, void *arg)* reply length (it's only 2 cache lines anyway) */ faddr = (unsigned long )smu->cmd_buf; -flush_inval_dcache_range (faddr, faddr +256 ); +flush_dcache_range (faddr, faddr +256 ); +mb (); +isync ();/* Now check ack */ ack = (~cmd->cmd) &0xff ;