Align VM locks, new spinlock patch
Subsystems (6)
Quoted 3 lines · Andrew Morton
I think the coding can be simplified somewhat. How does this look?
...
Looks fine, it makes the code more common between UP and SMP. I consolidated the changes and appended the new version below.
The peak throughput results (MB/sec) for dbench on 2.4.10:
CPU 2.4.10 2.4.10 + improvement
spinlock patch by patch
U 102,9 103,0 0,0% 1 95,1 96,9 1,9% 2 155,5 156,0 0,3% 4 206,9 222,1 7,3%
8 194,9 233,7 19,9%
Throughput (MB/sec) for dbench with 8 clients on 8 CPUs:
CPU 2.4.10 2.4.10 + improvement
spinlock patch by patch
8 173,9 223,0 28,2%
Juergen
______________________________________________________________
Juergen Doelle
IBM Linux Technology Center - kernel performance
[email protected]
= = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
include/linux/spinlock.h renamed from include/linux/spinlock.h.orig +16 -0
@@ -133,4 +133,20 @@extern int atomic_dec_and_lock (atomic_t *atomic,spinlock_t *lock);#endif +#ifdef CONFIG_SMP +#include <linux/cache.h> + +typedef union { +spinlock_t lock; +char fill_up[(SMP_CACHE_BYTES )]; +}spinlock_cacheline_t __attribute__ ((aligned (SMP_CACHE_BYTES ))); + +#else /* SMP */ + +typedef struct { +spinlock_t lock; +}spinlock_cacheline_t ; + + +#endif #endif /* __LINUX_SPINLOCK_H */
include/linux/swap.h renamed from include/linux/swap.h.orig +6 -2
@@ -86,7 +86,10 @@extern atomic_t nr_async_pages;extern atomic_t page_cache_size;extern atomic_t buffermem_pages; -extern spinlock_t pagecache_lock; + +extern spinlock_cacheline_t pagecache_lock_cacheline; +#define pagecache_lock (pagecache_lock_cacheline.lock) +extern void __remove_inode_page (struct page *);/* Incomplete types for prototype declarations: */
@@ -159,7 +162,8 @@extern unsigned long swap_cache_find_success;#endif -extern spinlock_t pagemap_lru_lock; +extern spinlock_cacheline_t pagemap_lru_lock_cacheline; +#define pagemap_lru_lock pagemap_lru_lock_cacheline.lockextern void FASTCALL (mark_page_accessed (struct page *));
include/asm-i386/smplock.h renamed from include/asm-i386/smplock.h.orig +2 -1
@@ -8,7 +8,8 @@#include <linux/sched.h> #include <asm/current.h> -extern spinlock_t kernel_flag; +extern spinlock_cacheline_t kernel_flag_cacheline; +#define kernel_flag kernel_flag_cacheline.lock#define kernel_locked () spin_is_locked(&kernel_flag)
arch/i386/kernel/i386_ksyms.c renamed from arch/i386/kernel/i386_ksyms.c.orig +1 -1
@@ -120,7 +120,7 @@#ifdef CONFIG_SMP EXPORT_SYMBOL (cpu_data); -EXPORT_SYMBOL (kernel_flag); +EXPORT_SYMBOL (kernel_flag_cacheline);EXPORT_SYMBOL (smp_num_cpus);EXPORT_SYMBOL (cpu_online_map);EXPORT_SYMBOL_NOVERS (__write_lock_failed);
arch/i386/kernel/smp.c renamed from arch/i386/kernel/smp.c.orig +1 -1
@@ -101,7 +101,7 @@*/ /* The 'big kernel lock' */ -spinlock_t kernel_flag =SPIN_LOCK_UNLOCKED ; +spinlock_cacheline_t kernel_flag_cacheline = {SPIN_LOCK_UNLOCKED };struct tlb_state cpu_tlbstate[NR_CPUS ] = {[0 ...NR_CPUS -1 ] = { &init_mm,0 }};
fs/buffer.c renamed from fs/buffer.c.orig +4 -1
@@ -81,7 +81,10 @@static rwlock_t hash_table_lock =RW_LOCK_UNLOCKED ;static struct buffer_head *lru_list[NR_LIST ]; -static spinlock_t lru_list_lock =SPIN_LOCK_UNLOCKED ; + +static spinlock_cacheline_t lru_list_lock_cacheline = {SPIN_LOCK_UNLOCKED }; +#define lru_list_lock lru_list_lock_cacheline.lock +static int nr_buffers_type[NR_LIST ];static unsigned long size_buffers_type[NR_LIST ];
mm/filemap.c renamed from mm/filemap.c.orig +3 -2
@@ -46,12 +46,13 @@unsigned int page_hash_bits;struct page **page_hash_table; -spinlock_t pagecache_lock____cacheline_aligned_in_smp =SPIN_LOCK_UNLOCKED ; +spinlock_cacheline_t pagecache_lock_cacheline = {SPIN_LOCK_UNLOCKED }; +/* * NOTE: to avoid deadlocking you must never acquire the pagecache_lock with * the pagemap_lru_lock held. */ -spinlock_t pagemap_lru_lock____cacheline_aligned_in_smp =SPIN_LOCK_UNLOCKED ; +spinlock_cacheline_t pagemap_lru_lock_cacheline = {SPIN_LOCK_UNLOCKED };#define CLUSTER_PAGES (1 << page_cluster)#define CLUSTER_OFFSET (x) (((x) >> page_cluster) << page_cluster)
mm/highmem.c renamed from mm/highmem.c.orig +2 -1
@@ -32,7 +32,8 @@*/ static int pkmap_count[LAST_PKMAP ];static unsigned int last_pkmap_nr; -static spinlock_t kmap_lock =SPIN_LOCK_UNLOCKED ; +static spinlock_cacheline_t kmap_lock_cacheline = {SPIN_LOCK_UNLOCKED }; +#define kmap_lock kmap_lock_cacheline.lockpte_t * pkmap_page_table;