introducing __cacheline_aligned_in_smp

PATCH New Anton Blanchard 1 participant 1 message Nov 28, 2001

Subsystems (6)

Memory management VFS MM core Scheduler +2 more subsystems

lore

Share & tools

lore

Anton Blanchard

author

2001-11-28 06:18 UTC #0 lore raw

PATCH 1/1 introducing __cacheline_aligned_in_smp

Hi,

Here is a very simple patch that offers pretty substantial improvements in scalability. Without the patch dbench shows inverse scalability from 5 to 12 cpus on this 12 way ppc64 machine. With the patch we are still positively scaling at 12 cpus. Before the patch we were doing 212 MB/s at 12 clients, after the patch we do 320 MB/s.

Basically it creates __cacheline_aligned_in_smp which does the same thing as __cacheline_aligned - ensures a spinlock gets an entire cacheline to itself.

The results can be found here:

http://samba.org/~anton/linux/cacheline_aligned/

I think this is a candidate for 2.4, but it changes the alignment of some things on UP (like softirq) so some testing would be a good idea :)

Anton

fs/block_dev.c +1 -1

diff -urN linuxppc_2_4_devel/fs/block_dev.c linuxppc_2_4_devel_work/fs/block_dev.c
@@ -234,7 +237,7 @@
 #define HASH_SIZE	(1UL << HASH_BITS)
 #define HASH_MASK	(HASH_SIZE-1)
 static struct list_head bdev_hashtable[HASH_SIZE];
-static spinlock_t bdev_lock = SPIN_LOCK_UNLOCKED;
+static spinlock_t bdev_lock __cacheline_aligned_in_smp = SPIN_LOCK_UNLOCKED;
 static kmem_cache_t * bdev_cachep;
 
 #define alloc_bdev() \

fs/buffer.c +1 -1

diff -urN linuxppc_2_4_devel/fs/buffer.c linuxppc_2_4_devel_work/fs/buffer.c
@@ -73,7 +73,7 @@
 static rwlock_t hash_table_lock = RW_LOCK_UNLOCKED;
 
 static struct buffer_head *lru_list[NR_LIST];
-static spinlock_t lru_list_lock = SPIN_LOCK_UNLOCKED;
+static spinlock_t lru_list_lock __cacheline_aligned_in_smp = SPIN_LOCK_UNLOCKED;
 static int nr_buffers_type[NR_LIST];
 static unsigned long size_buffers_type[NR_LIST];

include/linux/cache.h +8 -0

diff -urN linuxppc_2_4_devel/include/linux/cache.h linuxppc_2_4_devel_work/include/linux/cache.h
@@ -34,4 +34,12 @@
 #endif
 #endif /* __cacheline_aligned */
 
+#ifndef __cacheline_aligned_in_smp
+#ifdef CONFIG_SMP
+#define __cacheline_aligned_in_smp __cacheline_aligned
+#else
+#define __cacheline_aligned_in_smp
+#endif /* CONFIG_SMP */
+#endif
+
 #endif /* __LINUX_CACHE_H */

kernel/fork.c +1 -1

diff -urN linuxppc_2_4_devel/kernel/fork.c linuxppc_2_4_devel_work/kernel/fork.c
@@ -206,7 +206,7 @@
 	return retval;
 }
 
-spinlock_t mmlist_lock __cacheline_aligned = SPIN_LOCK_UNLOCKED;
+spinlock_t mmlist_lock __cacheline_aligned_in_smp = SPIN_LOCK_UNLOCKED;
 int mmlist_nr;
 
 #define allocate_mm()	(kmem_cache_alloc(mm_cachep, SLAB_KERNEL))

kernel/softirq.c +3 -3

diff -urN linuxppc_2_4_devel/kernel/softirq.c linuxppc_2_4_devel_work/kernel/softirq.c
@@ -42,7 +42,7 @@
 
 irq_cpustat_t irq_stat[NR_CPUS];
 
-static struct softirq_action softirq_vec[32] __cacheline_aligned;
+static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;
 
 /*
  * we cannot loop indefinitely here to avoid userspace starvation,

@@ -146,8 +146,8 @@
 
 /* Tasklets */
 
-struct tasklet_head tasklet_vec[NR_CPUS] __cacheline_aligned;
-struct tasklet_head tasklet_hi_vec[NR_CPUS] __cacheline_aligned;
+struct tasklet_head tasklet_vec[NR_CPUS] __cacheline_aligned_in_smp;
+struct tasklet_head tasklet_hi_vec[NR_CPUS] __cacheline_aligned_in_smp;
 
 void __tasklet_schedule(struct tasklet_struct *t)
 {

mm/filemap.c +2 -2

diff -urN linuxppc_2_4_devel/mm/filemap.c linuxppc_2_4_devel_work/mm/filemap.c
@@ -53,7 +54,7 @@
 EXPORT_SYMBOL(vm_min_readahead);
 
 
-spinlock_t pagecache_lock ____cacheline_aligned_in_smp = SPIN_LOCK_UNLOCKED;
+spinlock_t pagecache_lock __cacheline_aligned_in_smp = SPIN_LOCK_UNLOCKED;
 /*
  * NOTE: to avoid deadlocking you must never acquire the pagemap_lru_lock 
  *	with the pagecache_lock held.

@@ -63,7 +64,7 @@
  *		pagemap_lru_lock ->
  *			pagecache_lock
  */
-spinlock_t pagemap_lru_lock ____cacheline_aligned_in_smp = SPIN_LOCK_UNLOCKED;
+spinlock_t pagemap_lru_lock __cacheline_aligned_in_smp = SPIN_LOCK_UNLOCKED;
 
 #define CLUSTER_PAGES		(1 << page_cluster)
 #define CLUSTER_OFFSET(x)	(((x) >> page_cluster) << page_cluster)

fs/dcache.c renamed from fs/dcace.c +1 -1

diff -urN linuxppc_2_4_devel/fs/dcache.c linuxppc_2_4_devel_work/fs/dcache.c
@@ -29,7 +29,7 @@
 #define DCACHE_PARANOIA 1
 /* #define DCACHE_DEBUG 1 */
 
-spinlock_t dcache_lock = SPIN_LOCK_UNLOCKED;
+spinlock_t dcache_lock __cacheline_aligned_in_smp = SPIN_LOCK_UNLOCKED;
 
 /* Right now the dcache depends on the kernel lock */
 #define check_lock()	if (!kernel_locked()) BUG()