parisc: Optimize per-pagetable spinlocks
Helge Deller
maintainerPATCH 1/1 parisc: Optimize per-pagetable spinlocks
On parisc a spinlock is stored in the next page behind the pgd which protects against parallel accesses to the pgd. That's why one additional page (PGD_ALLOC_ORDER) is allocated for the pgd.
Matthew Wilcox suggested that we instead should use a pointer in the struct page table for this spinlock and noted, that the comments for the PGD_ORDER and PMD_ORDER defines were wrong.
Both suggestions are addressed with this patch. Instead of having an own spinlock to protect the pgd, we now switch to use the existing page_table_lock. Additionally, beside loading the pgd into cr25 in switch_mm_irqs_off(), the physical address of this lock is loaded into cr28 (tr4), so that we can avoid implementing a complicated lookup in assembly for this lock in the TLB fault handlers.
The existing Hybrid L2/L3 page table scheme (where the pmd is adjacent to the pgd) has been dropped with this patch.
Remove the locking in set_pte() and the huge-page pte functions too. They trigger a spinlock recursion on 32bit machines and seem unnecessary.
--- arch/parisc/Kconfig | 10 +++ arch/parisc/include/asm/mmu_context.h | 7 ++ arch/parisc/include/asm/page.h | 2 +- arch/parisc/include/asm/pgalloc.h | 76 ++++------------- arch/parisc/include/asm/pgtable.h | 89 ++++---------------- arch/parisc/kernel/asm-offsets.c | 1 - arch/parisc/kernel/entry.S | 116 +++++++++++--------------- arch/parisc/mm/hugetlbpage.c | 13 --- arch/parisc/mm/init.c | 10 +-- 9 files changed, 110 insertions(+), 214 deletions(-)
arch/parisc/Kconfig +10 -0
@@ -311,6 +311,16 @@ config IRQSTACKS for handling hard and soft interrupts. This can help avoid overflowing the process kernel stacks. +config TLB_PTLOCK + bool "Use page table locks in TLB fault handler" + depends on SMP + default n + help + Select this option to enable page table locking in the TLB + fault handler. This ensures that page table entries are + updated consistently on SMP machines at the expense of some + loss in performance. + config HOTPLUG_CPU bool default y if SMP
arch/parisc/include/asm/mmu_context.h +7 -0
@@ -5,6 +5,7 @@#include <linux/mm.h> #include <linux/sched.h> #include <linux/atomic.h> +#include <linux/spinlock.h> #include <asm-generic/mm_hooks.h> /* on PA-RISC, we actually have enough contexts to justify an allocator
@@ -50,6 +51,12 @@ static inline void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next, struct task_struct *tsk) {if (prev != next) { +#ifdef CONFIG_TLB_PTLOCK +/* put physical address of page_table_lock in cr28 (tr4) +for TLB faults */ +spinlock_t *pgd_lock = &next->page_table_lock; +mtctl (__pa (__ldcw_align (&pgd_lock->rlock.raw_lock)),28 ); +#endif mtctl (__pa (next->pgd),25 );load_context (next->context); }
arch/parisc/include/asm/page.h +1 -1
@@ -112,7 +112,7 @@ extern int npmem_ranges;#else #define BITS_PER_PTE_ENTRY 2#define BITS_PER_PMD_ENTRY 2 -#define BITS_PER_PGD_ENTRY BITS_PER_PMD_ENTRY +#define BITS_PER_PGD_ENTRY 2#endif #define PGD_ENTRY_SIZE (1UL << BITS_PER_PGD_ENTRY)#define PMD_ENTRY_SIZE (1UL << BITS_PER_PMD_ENTRY)
arch/parisc/include/asm/pgalloc.h +18 -58
106 lines not on this page · raw message
arch/parisc/include/asm/pgtable.h +18 -71
170 lines not on this page · raw message
arch/parisc/kernel/asm-offsets.c +0 -1
@@ -268,7 +268,6 @@ int main(void)DEFINE (ASM_BITS_PER_PGD ,BITS_PER_PGD );DEFINE (ASM_BITS_PER_PMD ,BITS_PER_PMD );DEFINE (ASM_BITS_PER_PTE ,BITS_PER_PTE ); -DEFINE (ASM_PGD_PMD_OFFSET , -(PAGE_SIZE <<PGD_ORDER ));DEFINE (ASM_PMD_ENTRY , ((PAGE_OFFSET &PMD_MASK ) >>PMD_SHIFT ));DEFINE (ASM_PGD_ENTRY ,PAGE_OFFSET >>PGDIR_SHIFT );DEFINE (ASM_PGD_ENTRY_SIZE ,PGD_ENTRY_SIZE );
arch/parisc/kernel/entry.S +51 -65
342 lines not on this page · raw message
arch/parisc/mm/hugetlbpage.c +0 -13
@@ -142,24 +142,17 @@ static void __set_huge_pte_at(struct mm_struct *mm, unsigned long addr,void set_huge_pte_at (struct mm_struct *mm,unsigned long addr,pte_t *ptep,pte_t entry) { -unsigned long flags; - -spin_lock_irqsave (pgd_spinlock ((mm)->pgd), flags);__set_huge_pte_at (mm, addr, ptep, entry); -spin_unlock_irqrestore (pgd_spinlock ((mm)->pgd), flags); }pte_t huge_ptep_get_and_clear (struct mm_struct *mm,unsigned long addr,pte_t *ptep) { -unsigned long flags;pte_t entry; -spin_lock_irqsave (pgd_spinlock ((mm)->pgd), flags); entry = *ptep;__set_huge_pte_at (mm, addr, ptep,__pte (0 )); -spin_unlock_irqrestore (pgd_spinlock ((mm)->pgd), flags);return entry; }
@@ -168,29 +161,23 @@ pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,void huge_ptep_set_wrprotect (struct mm_struct *mm,unsigned long addr,pte_t *ptep) { -unsigned long flags;pte_t old_pte; -spin_lock_irqsave (pgd_spinlock ((mm)->pgd), flags); old_pte = *ptep;__set_huge_pte_at (mm, addr, ptep,pte_wrprotect (old_pte)); -spin_unlock_irqrestore (pgd_spinlock ((mm)->pgd), flags); }int huge_ptep_set_access_flags (struct vm_area_struct *vma,unsigned long addr,pte_t *ptep,pte_t pte,int dirty) { -unsigned long flags;int changed;struct mm_struct *mm = vma->vm_mm; -spin_lock_irqsave (pgd_spinlock ((mm)->pgd), flags); changed = !pte_same (*ptep, pte);if (changed) {__set_huge_pte_at (mm, addr, ptep, pte); } -spin_unlock_irqrestore (pgd_spinlock ((mm)->pgd), flags);return changed; }
arch/parisc/mm/init.c +5 -5
@@ -37,11 +37,6 @@ extern int data_start;extern void parisc_kernel_start (void );/* Kernel entry point in head.S */ #if CONFIG_PGTABLE_LEVELS ==3 -/* NOTE: This layout exactly conforms to the hybrid L2/L3 page table layout -* with the first pmd adjacent to the pgd and below it. gcc doesn't actually -* guarantee that global objects will be laid out in memory in the same order -* as the order of declaration, so put these in different sections and use -* the linker script to order them. */ pmd_t pmd0[PTRS_PER_PMD] __section(".data..vm0.pmd") __attribute__ ((aligned(PAGE_SIZE)));#endif
@@ -559,6 +554,11 @@ void __init mem_init(void)BUILD_BUG_ON (PGD_ENTRY_SIZE !=sizeof (pgd_t));BUILD_BUG_ON (PAGE_SHIFT +BITS_PER_PTE +BITS_PER_PMD +BITS_PER_PGD >BITS_PER_LONG ); +#if CONFIG_PGTABLE_LEVELS ==3 +BUILD_BUG_ON (PT_INITIAL >PTRS_PER_PMD ); +#else +BUILD_BUG_ON (PT_INITIAL >PTRS_PER_PGD ); +#endif high_memory =__va ((max_pfn <<PAGE_SHIFT ));set_max_mapnr (max_low_pfn);
--
2.29.2