parisc: Optimize per-pagetable spinlocks
Subsystems (2)
Helge Deller
maintainerPATCH 1/1 parisc: Optimize per-pagetable spinlocks
On parisc a spinlock is stored in the next page behind the pgd which protects against parallel accesses to the pgd. That's why one additional page (PGD_ALLOC_ORDER) is allocated for the pgd.
Matthew Wilcox suggested that we instead should use a pointer in the struct page table for this spinlock and noted, that the comments for the PGD_ORDER and PMD_ORDER defines were wrong.
This patch fixes the comments and implements a parisc_pgd_lock spinlock in the struct page table.
---
arch/parisc/include/asm/pgalloc.h +40 -19
@@ -26,34 +26,55 @@* kernel for machines with under 4GB of memory) */ static inline pgd_t *pgd_alloc (struct mm_struct *mm) { -pgd_t *pgd = (pgd_t *)__get_free_pages (GFP_KERNEL , -PGD_ALLOC_ORDER ); -pgd_t *actual_pgd = pgd; +pgd_t *pgd; +pmd_t *pmd; +spinlock_t *pgd_lock; +struct page *page; + + pmd = (pmd_t *)__get_free_pages (GFP_KERNEL ,PGD_ALLOC_ORDER ); +if (unlikely (pmd ==NULL )) +return NULL ; + +memset (pmd,0 ,PAGE_SIZE <<PGD_ALLOC_ORDER ); -if (likely (pgd !=NULL )) { -memset (pgd,0 ,PAGE_SIZE <<PGD_ALLOC_ORDER );#if CONFIG_PGTABLE_LEVELS ==3 - actual_pgd +=PTRS_PER_PGD ; -/* Populate first pmd with allocated memory. We mark it -* with PxD_FLAG_ATTACHED as a signal to the system that this -* pmd entry may not be cleared. */ -set_pgd (actual_pgd,__pgd ((PxD_FLAG_PRESENT | - PxD_FLAG_VALID | - PxD_FLAG_ATTACHED) - + (__u32)(__pa ((unsigned long )pgd) >> PxD_VALUE_SHIFT))); -/* The first pmd entry also is marked with PxD_FLAG_ATTACHED as -* a signal that this pmd may not be freed */ -set_pgd (pgd,__pgd (PxD_FLAG_ATTACHED)); + pgd = (pgd_t *) (pmd +PTRS_PER_PMD ); +/* +* Mark first pmd and pgd with PxD_FLAG_ATTACHED as a signal to the +* system that this pmd entry may not be cleared. +*/ +set_pgd (pgd,__pgd ((PxD_FLAG_PRESENT | + PxD_FLAG_VALID | + PxD_FLAG_ATTACHED) + + (__u32)(__pa ((unsigned long )pmd) >> PxD_VALUE_SHIFT))); +set_pmd (pmd,__pmd (PxD_FLAG_ATTACHED)); +#else + pgd = (pgd_t *) pmd;#endif + +/* allocate & initialize pgd_spinlock() for this PGD */ + pgd_lock =kmalloc (sizeof (*pgd_lock),GFP_KERNEL ); +if (unlikely (pgd_lock ==NULL )) { +free_pages ((unsigned long )pmd,PGD_ALLOC_ORDER ); +return NULL ; } -spin_lock_init (pgd_spinlock (actual_pgd)); -return actual_pgd; + + page =virt_to_page ((unsigned long ) pgd); + page->parisc_pgd_lock = pgd_lock; +spin_lock_init (pgd_lock); + +return pgd; }static inline void pgd_free (struct mm_struct *mm,pgd_t *pgd) { +spinlock_t *lock; + + lock =pgd_spinlock (pgd); +kfree (lock); +#if CONFIG_PGTABLE_LEVELS ==3 - pgd -=PTRS_PER_PGD ; + pgd -=PTRS_PER_PMD ;#endif free_pages ((unsigned long )pgd,PGD_ALLOC_ORDER ); }
arch/parisc/include/asm/pgtable.h +22 -25
106 lines not on this page · raw message
arch/parisc/kernel/asm-offsets.c +2 -0
@@ -263,6 +263,8 @@ int main(void)DEFINE (TIF_BLOCKSTEP_PA_BIT ,31 -TIF_BLOCKSTEP );DEFINE (TIF_SINGLESTEP_PA_BIT ,31 -TIF_SINGLESTEP );BLANK (); +DEFINE (ASM_PGD_LOCK_OFFSET , offsetof(struct page , parisc_pgd_lock)); +BLANK ();DEFINE (ASM_PMD_SHIFT ,PMD_SHIFT );DEFINE (ASM_PGDIR_SHIFT ,PGDIR_SHIFT );DEFINE (ASM_BITS_PER_PGD ,BITS_PER_PGD );
arch/parisc/kernel/entry.S +19 -19
@@ -35,10 +35,10 @@ .level 2.0 #endif - .import pa_tlb_lock,data - .macro load_pa_tlb_lock reg - mfctl %cr25,\reg - addil L%(PAGE_SIZE << (PGD_ALLOC_ORDER - 1)),\reg + .macro load_pgd_spinlock reg, tmp + mfctl %cr25,\tmp + LDREG ASM_PGD_LOCK_OFFSET(\tmp),\tmp + tophys \tmp,\reg .endm /* space_to_prot macro creates a prot id from a space id */
@@ -448,7 +448,7 @@ .macro tlb_lock spc,ptp,pte,tmp,tmp1,fault #ifdef CONFIG_SMP 98: cmpib,COND(=),n 0,\spc,2f - load_pa_tlb_lock \tmp + load_pgd_spinlock \tmp,\tmp1 1: LDCW 0(\tmp),\tmp1 cmpib,COND(=) 0,\tmp1,1b nop
@@ -478,9 +478,9 @@ .endm /* Release pa_tlb_lock lock. */ - .macro tlb_unlock1 spc,tmp + .macro tlb_unlock1 spc,tmp,tmp1 #ifdef CONFIG_SMP -98: load_pa_tlb_lock \tmp +98: load_pgd_spinlock \tmp,\tmp1 99: ALTERNATIVE(98b, 99b, ALT_COND_NO_SMP, INSN_NOP) tlb_unlock0 \spc,\tmp #endif
@@ -1165,7 +1165,7 @@ dtlb_miss_20w: idtlbt pte,prot - tlb_unlock1 spc,t0 + tlb_unlock1 spc,t0,t1 rfir nop
@@ -1191,7 +1191,7 @@ nadtlb_miss_20w: idtlbt pte,prot - tlb_unlock1 spc,t0 + tlb_unlock1 spc,t0,t1 rfir nop
@@ -1225,7 +1225,7 @@ dtlb_miss_11: mtsp t1, %sr1 /* Restore sr1 */ - tlb_unlock1 spc,t0 + tlb_unlock1 spc,t0,t1 rfir nop
@@ -1258,7 +1258,7 @@ nadtlb_miss_11: mtsp t1, %sr1 /* Restore sr1 */ - tlb_unlock1 spc,t0 + tlb_unlock1 spc,t0,t1 rfir nop
@@ -1287,7 +1287,7 @@ dtlb_miss_20: idtlbt pte,prot - tlb_unlock1 spc,t0 + tlb_unlock1 spc,t0,t1 rfir nop
@@ -1315,7 +1315,7 @@ nadtlb_miss_20: idtlbt pte,prot - tlb_unlock1 spc,t0 + tlb_unlock1 spc,t0,t1 rfir nop
@@ -1422,7 +1422,7 @@ itlb_miss_20w: iitlbt pte,prot - tlb_unlock1 spc,t0 + tlb_unlock1 spc,t0,t1 rfir nop
@@ -1446,7 +1446,7 @@ naitlb_miss_20w: iitlbt pte,prot - tlb_unlock1 spc,t0 + tlb_unlock1 spc,t0,t1 rfir nop
@@ -1480,7 +1480,7 @@ itlb_miss_11: mtsp t1, %sr1 /* Restore sr1 */ - tlb_unlock1 spc,t0 + tlb_unlock1 spc,t0,t1 rfir nop
@@ -1504,7 +1504,7 @@ naitlb_miss_11: mtsp t1, %sr1 /* Restore sr1 */ - tlb_unlock1 spc,t0 + tlb_unlock1 spc,t0,t1 rfir nop
@@ -1534,7 +1534,7 @@ itlb_miss_20: iitlbt pte,prot - tlb_unlock1 spc,t0 + tlb_unlock1 spc,t0,t1 rfir nop
@@ -1554,7 +1554,7 @@ naitlb_miss_20: iitlbt pte,prot - tlb_unlock1 spc,t0 + tlb_unlock1 spc,t0,t1 rfir nop
arch/parisc/mm/init.c +19 -0
@@ -681,6 +681,24 @@ static void __init parisc_bootmem_free(void)free_area_init (max_zone_pfn); } +static void __init parisc_init_pgd_lock (void ) +{ +struct page *page; + + page =virt_to_page ((unsigned long ) &swapper_pg_dir); + page->parisc_pgd_lock = &pa_swapper_pg_lock; +} + +spinlock_t *pgd_spinlock (pgd_t *pgd) +{ +struct page *page; + + page =virt_to_page ((unsigned long ) pgd); +WARN_ON (IS_ENABLED (CONFIG_DEBUG_VM ) && (pgd == swapper_pg_dir) && + (page->parisc_pgd_lock != &pa_swapper_pg_lock)); +return page->parisc_pgd_lock; +} +void __init paging_init (void ) {setup_bootmem ();
@@ -691,6 +709,7 @@ void __init paging_init(void)sparse_init ();parisc_bootmem_free (); +parisc_init_pgd_lock (); }#ifdef CONFIG_PA20
include/linux/mm_types.h +1 -0
@@ -150,6 +150,7 @@ struct page {union {struct mm_struct *pt_mm;/* x86 pgds only */ atomic_t pt_frag_refcount;/* powerpc */ +spinlock_t *parisc_pgd_lock;/* parisc pgds only */ };#if ALLOC_SPLIT_PTLOCKS spinlock_t *ptl;
Helge Deller
maintainerRFC PATCH v2 1/1 parisc: Optimize per-pagetable spinlocks
* Helge Deller <[email protected]>:
Quoted 8 lines
On parisc a spinlock is stored in the next page behind the pgd which protects against parallel accesses to the pgd. That's why one additional page (
PGD_ALLOC_ORDER) is allocated for the pgd.Matthew Wilcox suggested that we instead should use a pointer in the struct page table for this spinlock and noted, that the comments for the
PGD_ORDERandPMD_ORDERdefines were wrong.
Quoted 6 lines
This patch fixes the comments and implements a
parisc_pgd_lockspinlock in the struct page table.Suggested-by Matthew Wilcox <[email protected]>Fixes b37d1c1898b2 ("parisc: Use per-pagetable spinlock")Signed-off-by Helge Deller <[email protected]>
This is an updated v2 patch, still work-in-progress.
Instead of old patch it now loads cr28 (tr4) with pointer to tlb_lock pointer when loading the pgd into cr25 for fault handlers.
Idea to use cr28 is from Dave Anglin.
---
arch/parisc/include/asm/mmu_context.h +8 -0
@@ -50,6 +50,14 @@ static inline void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next, struct task_struct *tsk) {if (prev != next) { +#ifdef CONFIG_SMP +/* phys address of tlb lock in cr28 (tr4) for TLB faults */ +struct page *page; + + page =virt_to_page ((unsigned long ) next->pgd); +/* BUG_ON(!page->parisc_pgd_lock); */ +mtctl (__pa (page->parisc_pgd_lock),28 ); +#endif mtctl (__pa (next->pgd),25 );load_context (next->context); }
arch/parisc/include/asm/pgalloc.h +40 -19
74 lines not on this page · raw message
arch/parisc/include/asm/pgtable.h +22 -25
106 lines not on this page · raw message
arch/parisc/kernel/entry.S +6 -7
@@ -35,10 +35,9 @@ .level 2.0 #endif - .import pa_tlb_lock,data - .macro load_pa_tlb_lock reg - mfctl %cr25,\reg - addil L%(PAGE_SIZE << (PGD_ALLOC_ORDER - 1)),\reg + /* Load offset to PGD Lock - needs to be 16-byte aligned! */ + .macro load_pgd_spinlock reg + mfctl %cr28,\reg .endm /* space_to_prot macro creates a prot id from a space id */
@@ -448,7 +447,7 @@ .macro tlb_lock spc,ptp,pte,tmp,tmp1,fault #ifdef CONFIG_SMP 98: cmpib,COND(=),n 0,\spc,2f - load_pa_tlb_lock \tmp + load_pgd_spinlock \tmp 1: LDCW 0(\tmp),\tmp1 cmpib,COND(=) 0,\tmp1,1b nop
@@ -480,9 +479,9 @@ /* Release pa_tlb_lock lock. */ .macro tlb_unlock1 spc,tmp #ifdef CONFIG_SMP -98: load_pa_tlb_lock \tmp -99: ALTERNATIVE(98b, 99b, ALT_COND_NO_SMP, INSN_NOP) +98: load_pgd_spinlock \tmp tlb_unlock0 \spc,\tmp +99: ALTERNATIVE(98b, 99b, ALT_COND_NO_SMP, INSN_NOP) #endif .endm
arch/parisc/mm/init.c +17 -0
@@ -681,6 +681,22 @@ static void __init parisc_bootmem_free(void)free_area_init (max_zone_pfn); } +static void __init parisc_init_pgd_lock (void ) +{ +struct page *page; + + page =virt_to_page ((unsigned long ) &swapper_pg_dir); + page->parisc_pgd_lock = &pa_swapper_pg_lock; +} + +spinlock_t *pgd_spinlock (pgd_t *pgd) +{ +struct page *page; + + page =virt_to_page ((unsigned long ) pgd); +return page->parisc_pgd_lock; +} +void __init paging_init (void ) {setup_bootmem ();
@@ -691,6 +707,7 @@ void __init paging_init(void)sparse_init ();parisc_bootmem_free (); +parisc_init_pgd_lock (); }#ifdef CONFIG_PA20
include/linux/mm_types.h +1 -0
@@ -150,6 +150,7 @@ struct page {union {struct mm_struct *pt_mm;/* x86 pgds only */ atomic_t pt_frag_refcount;/* powerpc */ +spinlock_t *parisc_pgd_lock;/* parisc pgds only */ };#if ALLOC_SPLIT_PTLOCKS spinlock_t *ptl;