Skip to contentSkip to conversation

Per cpu atomics in core allocators, cleanup and more this_cpu_ops

PATCH In review Christoph Lameter 5 participants 38 messages Dec 18, 2009 – Jan 5, 2010

Subsystems (14)
Memory management x86 PowerPC MIPS +10 more subsystems
lore
Share & tools
lore
#0 lore raw

v8 cover Per cpu atomics in core allocators, cleanup and more this_cpu_ops

Leftovers from the earlier patchset rediffed to 2.6.33-rc1. Mostly applications of per cpu counters to core components.

After this patchset there will be only one user of local_t left: Mathieu's trace ringbuffer.

I added some patches that define additional this_cpu ops in order to help Mathieu with making the trace ringbuffer use this_cpu ops. These have barely been tested (boots fine on 32 and 64 bit but there is no user of these operations). RFC state.

V7->V8
- Fix issue in slub patch
- Fix issue in modules patch
- Rediff page allocator patch
- Provide new this_cpu ops needed for ringbuffer [RFC state]

V6->V7
- Drop patches merged in 2.6.33 merge cycle
- Drop risky slub patches

V5->V6:
- Drop patches merged by Tejun.
- Drop irqless slub fastpath for now.
- Patches against Tejun percpu for-next branch.

V4->V5:
- Avoid setup_per_cpu_area() modifications and fold the remainder of the patch into the page allocator patch.
- Irq disable / per cpu ptr fixes for page allocator patch.

V3->V4:
- Fix various macro definitions.
- Provide experimental percpu based fastpath that does not disable interrupts for SLUB.

V2->V3:
- Available via git tree against latest upstream from
git://git.kernel.org/pub/scm/linux/kernel/git/christoph/percpu.git linus
- Rework SLUB per cpu operations. Get rid of dynamic DMA slab creation for CONFIG_ZONE_DMA
- Create fallback framework so that 64 bit ops on 32 bit platforms can fallback to the use of preempt or interrupt disable. 64 bit platforms can use 64 bit atomic per cpu ops.

V1->V2:
- Various minor fixes
- Add SLUB conversion
- Add Page allocator conversion
- Patch against the git tree of today

#1 lore raw

PATCH v8 1/16 this_cpu_ops: page allocator conversion

Use the per cpu allocator functionality to avoid per cpu arrays in struct zone.

This drastically reduces the size of struct zone for systems with large amounts of processors and allows placement of critical variables of struct zone in one cacheline even on very large systems.

Another effect is that the pagesets of one processor are placed near one another. If multiple pagesets from different zones fit into one cacheline then additional cacheline fetches can be avoided on the hot paths when allocating memory from multiple zones.

Bootstrap becomes simpler if we use the same scheme for UP, SMP, NUMA. #ifdefs are reduced and we can drop the zone_pcp macro.

Hotplug handling is also simplified since cpu alloc can bring up and shut down cpu areas for a specific cpu as a whole. So there is no need to allocate or free individual pagesets.

V7-V8:
- Explain chicken egg dilemmna with percpu allocator.

V4-V5:
- Fix up cases where per_cpu_ptr is called before irq disable
- Integrate the bootstrap logic that was separate before.

---
 include/linux/mm.h     |    4 
 include/linux/mmzone.h |   12 --
 mm/page_alloc.c        |  197 ++++++++++++++++++++-----------------------------
 mm/vmstat.c            |   14 +--
 4 files changed, 92 insertions(+), 135 deletions(-)
include/linux/mm.h +0 -4
Index: linux-2.6/include/linux/mm.h
===================================================================
@@ -1076,11 +1076,7 @@ extern void si_meminfo(struct sysinfo * 
 extern void si_meminfo_node(struct sysinfo *val, int nid);
 extern int after_bootmem;
 
-#ifdef CONFIG_NUMA
 extern void setup_per_cpu_pageset(void);
-#else
-static inline void setup_per_cpu_pageset(void) {}
-#endif
 
 extern void zone_pcp_update(struct zone *zone);
 
include/linux/mmzone.h +2 -10
Index: linux-2.6/include/linux/mmzone.h
===================================================================
@@ -184,13 +184,7 @@ struct per_cpu_pageset {
 	s8 stat_threshold;
 	s8 vm_stat_diff[NR_VM_ZONE_STAT_ITEMS];
 #endif
-} ____cacheline_aligned_in_smp;
-
-#ifdef CONFIG_NUMA
-#define zone_pcp(__z, __cpu) ((__z)->pageset[(__cpu)])
-#else
-#define zone_pcp(__z, __cpu) (&(__z)->pageset[(__cpu)])
-#endif
+};
 
 #endif /* !__GENERATING_BOUNDS.H */
 
@@ -306,10 +300,8 @@ struct zone {
 	 */
 	unsigned long		min_unmapped_pages;
 	unsigned long		min_slab_pages;
-	struct per_cpu_pageset	*pageset[NR_CPUS];
-#else
-	struct per_cpu_pageset	pageset[NR_CPUS];
 #endif
+	struct per_cpu_pageset	*pageset;
 	/*
 	 * free areas of different sizes
 	 */
mm/page_alloc.c +82 -115

320 lines not on this page · raw message

mm/vmstat.c +8 -6
Index: linux-2.6/mm/vmstat.c
===================================================================
@@ -139,7 +139,8 @@ static void refresh_zone_stat_thresholds
 		threshold = calculate_threshold(zone);
 
 		for_each_online_cpu(cpu)
-			zone_pcp(zone, cpu)->stat_threshold = threshold;
+			per_cpu_ptr(zone->pageset, cpu)->stat_threshold
+							= threshold;
 	}
 }
 
@@ -149,7 +150,8 @@ static void refresh_zone_stat_thresholds
 void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
 				int delta)
 {
-	struct per_cpu_pageset *pcp = zone_pcp(zone, smp_processor_id());
+	struct per_cpu_pageset *pcp = this_cpu_ptr(zone->pageset);
+
 	s8 *p = pcp->vm_stat_diff + item;
 	long x;
 
@@ -202,7 +204,7 @@ EXPORT_SYMBOL(mod_zone_page_state);
  */
 void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
 {
-	struct per_cpu_pageset *pcp = zone_pcp(zone, smp_processor_id());
+	struct per_cpu_pageset *pcp = this_cpu_ptr(zone->pageset);
 	s8 *p = pcp->vm_stat_diff + item;
 
 	(*p)++;
@@ -223,7 +225,7 @@ EXPORT_SYMBOL(__inc_zone_page_state);
 
 void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
 {
-	struct per_cpu_pageset *pcp = zone_pcp(zone, smp_processor_id());
+	struct per_cpu_pageset *pcp = this_cpu_ptr(zone->pageset);
 	s8 *p = pcp->vm_stat_diff + item;
 
 	(*p)--;
@@ -300,7 +302,7 @@ void refresh_cpu_vm_stats(int cpu)
 	for_each_populated_zone(zone) {
 		struct per_cpu_pageset *p;
 
-		p = zone_pcp(zone, cpu);
+		p = per_cpu_ptr(zone->pageset, cpu);
 
 		for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
 			if (p->vm_stat_diff[i]) {
@@ -741,7 +743,7 @@ static void zoneinfo_show_print(struct s
 	for_each_online_cpu(i) {
 		struct per_cpu_pageset *pageset;
 
-		pageset = zone_pcp(zone, i);
+		pageset = per_cpu_ptr(zone->pageset, i);
 		seq_printf(m,
 			   "\n    cpu: %i"
 			   "\n              count: %i"
#2 lore raw

PATCH v8 2/16 this_cpu ops: Remove pageset_notifier

In reply to Christoph Lameter

Remove the pageset notifier since it only marks that a processor exists on a specific node. Move that code into the vmstat notifier.

---
 mm/page_alloc.c |   27 ---------------------------
 mm/vmstat.c     |    1 +
 2 files changed, 1 insertion(+), 27 deletions(-)
mm/vmstat.c +1 -0
Index: linux-2.6/mm/vmstat.c
===================================================================
@@ -908,6 +908,7 @@ static int __cpuinit vmstat_cpuup_callba
 	case CPU_ONLINE:
 	case CPU_ONLINE_FROZEN:
 		start_cpu_timer(cpu);
+		node_set_state(cpu_to_node(cpu), N_CPU);
 		break;
 	case CPU_DOWN_PREPARE:
 	case CPU_DOWN_PREPARE_FROZEN:
mm/page_alloc.c +0 -27
Index: linux-2.6/mm/page_alloc.c
===================================================================
@@ -3122,27 +3122,6 @@ static void setup_pagelist_highmark(stru
 		pcp->batch = PAGE_SHIFT * 8;
 }
 
-
-static int __cpuinit pageset_cpuup_callback(struct notifier_block *nfb,
-		unsigned long action,
-		void *hcpu)
-{
-	int cpu = (long)hcpu;
-
-	switch (action) {
-	case CPU_UP_PREPARE:
-	case CPU_UP_PREPARE_FROZEN:
-		node_set_state(cpu_to_node(cpu), N_CPU);
-		break;
-	default:
-		break;
-	}
-	return ret;
-}
-
-static struct notifier_block __cpuinitdata pageset_notifier =
-	{ &pageset_cpuup_callback, NULL, 0 };
-
 /*
  * Allocate per cpu pagesets and initialize them.
  * Before this call only boot pagesets were available.
@@ -3168,12 +3147,6 @@ void __init setup_per_cpu_pageset(void)
 						percpu_pagelist_fraction));
 		}
 	}
-	/*
-	 * The boot cpu is always the first active.
-	 * The boot node has a processor.
-	 */
-	node_set_state(cpu_to_node(smp_processor_id()), N_CPU);
-	register_cpu_notifier(&pageset_notifier);
 }
 
 static noinline __init_refok
#3 lore raw

PATCH v8 3/16 Use this_cpu operations in slub

In reply to Christoph Lameter

Using per cpu allocations removes the needs for the per cpu arrays in the kmem_cache struct. These could get quite big if we have to support systems with thousands of cpus. The use of this_cpu_xx operations results in:

1. The size of kmem_cache for SMP configuration shrinks since we will only need 1 pointer instead of NR_CPUS. The same pointer can be used by all processors. Reduces cache footprint of the allocator.

2. We can dynamically size kmem_cache according to the actual nodes in the system meaning less memory overhead for configurations that may potentially support up to 1k NUMA nodes / 4k cpus.

3. We can remove the diddle widdle with allocating and releasing of kmem_cache_cpu structures when bringing up and shutting down cpus. The cpu alloc logic will do it all for us. Removes some portions of the cpu hotplug functionality.

4. Fastpath performance increases since per cpu pointer lookups and address calculations are avoided.

V7-V8
- Convert missed get_cpu_slab() under CONFIG_SLUB_STATS

---
 include/linux/slub_def.h |    6 -
 mm/slub.c                |  202 +++++++++++------------------------------------
 2 files changed, 49 insertions(+), 159 deletions(-)
include/linux/slub_def.h +1 -5
Index: linux-2.6/include/linux/slub_def.h
===================================================================
@@ -69,6 +69,7 @@ struct kmem_cache_order_objects {
  * Slab cache management.
  */
 struct kmem_cache {
+	struct kmem_cache_cpu *cpu_slab;
 	/* Used for retriving partial slabs etc */
 	unsigned long flags;
 	int size;		/* The size of an object including meta data */
@@ -104,11 +105,6 @@ struct kmem_cache {
 	int remote_node_defrag_ratio;
 	struct kmem_cache_node *node[MAX_NUMNODES];
 #endif
-#ifdef CONFIG_SMP
-	struct kmem_cache_cpu *cpu_slab[NR_CPUS];
-#else
-	struct kmem_cache_cpu cpu_slab;
-#endif
 };
 
 /*
mm/slub.c +48 -154

347 lines not on this page · raw message

#4 lore raw

Re: [this_cpu_xx V8 03/16] Use this_cpu operations in slub

Quoted 17 lines · Christoph Lameter

Using per cpu allocations removes the needs for the per cpu arrays in the kmem_cache struct. These could get quite big if we have to support systems with thousands of cpus. The use of this_cpu_xx operations results in:

1. The size of kmem_cache for SMP configuration shrinks since we will only need 1 pointer instead of NR_CPUS. The same pointer can be used by all processors. Reduces cache footprint of the allocator.

2. We can dynamically size kmem_cache according to the actual nodes in the system meaning less memory overhead for configurations that may potentially support up to 1k NUMA nodes / 4k cpus.

3. We can remove the diddle widdle with allocating and releasing of kmem_cache_cpu structures when bringing up and shutting down cpus. The cpu alloc logic will do it all for us. Removes some portions of the cpu hotplug functionality.

Quoted 8 lines

4. Fastpath performance increases since per cpu pointer lookups and address calculations are avoided.

V7-V8
- Convert missed get_cpu_slab() under CONFIG_SLUB_STATS

Patches 3-6 applied.

#5 lore raw

PATCH v8 4/16 SLUB: Get rid of dynamic DMA kmalloc cache allocation

In reply to Christoph Lameter

Dynamic DMA kmalloc cache allocation is troublesome since the new percpu allocator does not support allocations in atomic contexts. Reserve some statically allocated kmalloc_cpu structures instead.

---
 include/linux/slub_def.h |   19 +++++++++++--------
 mm/slub.c                |   24 ++++++++++--------------
 2 files changed, 21 insertions(+), 22 deletions(-)
include/linux/slub_def.h +11 -8
Index: linux-2.6/include/linux/slub_def.h
===================================================================
@@ -131,11 +131,21 @@ struct kmem_cache {
 
 #define SLUB_PAGE_SHIFT (PAGE_SHIFT + 2)
 
+#ifdef CONFIG_ZONE_DMA
+#define SLUB_DMA __GFP_DMA
+/* Reserve extra caches for potential DMA use */
+#define KMALLOC_CACHES (2 * SLUB_PAGE_SHIFT - 6)
+#else
+/* Disable DMA functionality */
+#define SLUB_DMA (__force gfp_t)0
+#define KMALLOC_CACHES SLUB_PAGE_SHIFT
+#endif
+
 /*
  * We keep the general caches in an array of slab caches that are used for
  * 2^x bytes of allocations.
  */
-extern struct kmem_cache kmalloc_caches[SLUB_PAGE_SHIFT];
+extern struct kmem_cache kmalloc_caches[KMALLOC_CACHES];
 
 /*
  * Sorry that the following has to be that ugly but some versions of GCC
@@ -203,13 +213,6 @@ static __always_inline struct kmem_cache
 	return &kmalloc_caches[index];
 }
 
-#ifdef CONFIG_ZONE_DMA
-#define SLUB_DMA __GFP_DMA
-#else
-/* Disable DMA functionality */
-#define SLUB_DMA (__force gfp_t)0
-#endif
-
 void *kmem_cache_alloc(struct kmem_cache *, gfp_t);
 void *__kmalloc(size_t size, gfp_t flags);
 
mm/slub.c +10 -14
Index: linux-2.6/mm/slub.c
===================================================================
@@ -2092,7 +2092,7 @@ static inline int alloc_kmem_cache_cpus(
 {
 	int cpu;
 
-	if (s < kmalloc_caches + SLUB_PAGE_SHIFT && s >= kmalloc_caches)
+	if (s < kmalloc_caches + KMALLOC_CACHES && s >= kmalloc_caches)
 		/*
 		 * Boot time creation of the kmalloc array. Use static per cpu data
 		 * since the per cpu allocator is not available yet.
@@ -2539,7 +2539,7 @@ EXPORT_SYMBOL(kmem_cache_destroy);
  *		Kmalloc subsystem
  *******************************************************************/
 
-struct kmem_cache kmalloc_caches[SLUB_PAGE_SHIFT] __cacheline_aligned;
+struct kmem_cache kmalloc_caches[KMALLOC_CACHES] __cacheline_aligned;
 EXPORT_SYMBOL(kmalloc_caches);
 
 static int __init setup_slub_min_order(char *str)
@@ -2629,6 +2629,7 @@ static noinline struct kmem_cache *dma_k
 	char *text;
 	size_t realsize;
 	unsigned long slabflags;
+	int i;
 
 	s = kmalloc_caches_dma[index];
 	if (s)
@@ -2649,18 +2650,13 @@ static noinline struct kmem_cache *dma_k
 	text = kasprintf(flags & ~SLUB_DMA, "kmalloc_dma-%d",
 			 (unsigned int)realsize);
 
-	if (flags & __GFP_WAIT)
-		s = kmalloc(kmem_size, flags & ~SLUB_DMA);
-	else {
-		int i;
+	s = NULL;
+	for (i = 0; i < KMALLOC_CACHES; i++)
+		if (!kmalloc_caches[i].size)
+			break;
 
-		s = NULL;
-		for (i = 0; i < SLUB_PAGE_SHIFT; i++)
-			if (kmalloc_caches[i].size) {
-				s = kmalloc_caches + i;
-				break;
-			}
-	}
+	BUG_ON(i >= KMALLOC_CACHES);
+	s = kmalloc_caches + i;
 
 	/*
 	 * Must defer sysfs creation to a workqueue because we don't know
@@ -2674,7 +2670,7 @@ static noinline struct kmem_cache *dma_k
 
 	if (!s || !text || !kmem_cache_open(s, flags, text,
 			realsize, ARCH_KMALLOC_MINALIGN, slabflags, NULL)) {
-		kfree(s);
+		s->size = 0;
 		kfree(text);
 		goto unlock_out;
 	}
#6 lore raw

PATCH v8 5/16 this_cpu: Remove slub kmem_cache fields

In reply to Christoph Lameter

Remove the fields in struct kmem_cache_cpu that were used to cache data from struct kmem_cache when they were in different cachelines. The cacheline that holds the per cpu array pointer now also holds these values. We can cut down the struct kmem_cache_cpu size to almost half.

The get_freepointer() and set_freepointer() functions that used to be only intended for the slow path now are also useful for the hot path since access to the size field does not require accessing an additional cacheline anymore. This results in consistent use of functions for setting the freepointer of objects throughout SLUB.

Also we initialize all possible kmem_cache_cpu structures when a slab is created. No need to initialize them when a processor or node comes online.

---
 include/linux/slub_def.h |    2 -
 mm/slub.c                |   76 ++++++++++-------------------------------------
 2 files changed, 17 insertions(+), 61 deletions(-)
include/linux/slub_def.h +0 -2
Index: linux-2.6/include/linux/slub_def.h
===================================================================
@@ -38,8 +38,6 @@ struct kmem_cache_cpu {
 	void **freelist;	/* Pointer to first free per cpu object */
 	struct page *page;	/* The slab from which we are allocating */
 	int node;		/* The node of the page (or -1 for debug) */
-	unsigned int offset;	/* Freepointer offset (in word units) */
-	unsigned int objsize;	/* Size of an object (from kmem_cache) */
 #ifdef CONFIG_SLUB_STATS
 	unsigned stat[NR_SLUB_STAT_ITEMS];
 #endif
mm/slub.c +17 -59

190 lines not on this page · raw message

#7 lore raw

PATCH v8 6/16 Make slub statistics use this_cpu_inc

In reply to Christoph Lameter

this_cpu_inc() translates into a single instruction on x86 and does not need any register. So use it in stat(). We also want to avoid the calculation of the per cpu kmem_cache_cpu structure pointer. So pass a kmem_cache pointer instead of a kmem_cache_cpu pointer.

---
 mm/slub.c |   43 ++++++++++++++++++++-----------------------
 1 file changed, 20 insertions(+), 23 deletions(-)
mm/slub.c +20 -23
Index: linux-2.6/mm/slub.c
===================================================================
@@ -217,10 +217,10 @@ static inline void sysfs_slab_remove(str
 
 #endif
 
-static inline void stat(struct kmem_cache_cpu *c, enum stat_item si)
+static inline void stat(struct kmem_cache *s, enum stat_item si)
 {
 #ifdef CONFIG_SLUB_STATS
-	c->stat[si]++;
+	__this_cpu_inc(s->cpu_slab->stat[si]);
 #endif
 }
 
@@ -1108,7 +1108,7 @@ static struct page *allocate_slab(struct
 		if (!page)
 			return NULL;
 
-		stat(this_cpu_ptr(s->cpu_slab), ORDER_FALLBACK);
+		stat(s, ORDER_FALLBACK);
 	}
 
 	if (kmemcheck_enabled
@@ -1406,23 +1406,22 @@ static struct page *get_partial(struct k
 static void unfreeze_slab(struct kmem_cache *s, struct page *page, int tail)
 {
 	struct kmem_cache_node *n = get_node(s, page_to_nid(page));
-	struct kmem_cache_cpu *c = this_cpu_ptr(s->cpu_slab);
 
 	__ClearPageSlubFrozen(page);
 	if (page->inuse) {
 
 		if (page->freelist) {
 			add_partial(n, page, tail);
-			stat(c, tail ? DEACTIVATE_TO_TAIL : DEACTIVATE_TO_HEAD);
+			stat(s, tail ? DEACTIVATE_TO_TAIL : DEACTIVATE_TO_HEAD);
 		} else {
-			stat(c, DEACTIVATE_FULL);
+			stat(s, DEACTIVATE_FULL);
 			if (SLABDEBUG && PageSlubDebug(page) &&
 						(s->flags & SLAB_STORE_USER))
 				add_full(n, page);
 		}
 		slab_unlock(page);
 	} else {
-		stat(c, DEACTIVATE_EMPTY);
+		stat(s, DEACTIVATE_EMPTY);
 		if (n->nr_partial < s->min_partial) {
 			/*
 			 * Adding an empty slab to the partial slabs in order
@@ -1438,7 +1437,7 @@ static void unfreeze_slab(struct kmem_ca
 			slab_unlock(page);
 		} else {
 			slab_unlock(page);
-			stat(__this_cpu_ptr(s->cpu_slab), FREE_SLAB);
+			stat(s, FREE_SLAB);
 			discard_slab(s, page);
 		}
 	}
@@ -1453,7 +1452,7 @@ static void deactivate_slab(struct kmem_
 	int tail = 1;
 
 	if (page->freelist)
-		stat(c, DEACTIVATE_REMOTE_FREES);
+		stat(s, DEACTIVATE_REMOTE_FREES);
 	/*
 	 * Merge cpu freelist into slab freelist. Typically we get here
 	 * because both freelists are empty. So this is unlikely
@@ -1479,7 +1478,7 @@ static void deactivate_slab(struct kmem_
 
 static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
 {
-	stat(c, CPUSLAB_FLUSH);
+	stat(s, CPUSLAB_FLUSH);
 	slab_lock(c->page);
 	deactivate_slab(s, c);
 }
@@ -1619,7 +1618,7 @@ static void *__slab_alloc(struct kmem_ca
 	if (unlikely(!node_match(c, node)))
 		goto another_slab;
 
-	stat(c, ALLOC_REFILL);
+	stat(s, ALLOC_REFILL);
 
 load_freelist:
 	object = c->page->freelist;
@@ -1634,7 +1633,7 @@ load_freelist:
 	c->node = page_to_nid(c->page);
 unlock_out:
 	slab_unlock(c->page);
-	stat(c, ALLOC_SLOWPATH);
+	stat(s, ALLOC_SLOWPATH);
 	return object;
 
 another_slab:
@@ -1644,7 +1643,7 @@ new_slab:
 	new = get_partial(s, gfpflags, node);
 	if (new) {
 		c->page = new;
-		stat(c, ALLOC_FROM_PARTIAL);
+		stat(s, ALLOC_FROM_PARTIAL);
 		goto load_freelist;
 	}
 
@@ -1658,7 +1657,7 @@ new_slab:
 
 	if (new) {
 		c = __this_cpu_ptr(s->cpu_slab);
-		stat(c, ALLOC_SLAB);
+		stat(s, ALLOC_SLAB);
 		if (c->page)
 			flush_slab(s, c);
 		slab_lock(new);
@@ -1713,7 +1712,7 @@ static __always_inline void *slab_alloc(
 
 	else {
 		c->freelist = get_freepointer(s, object);
-		stat(c, ALLOC_FASTPATH);
+		stat(s, ALLOC_FASTPATH);
 	}
 	local_irq_restore(flags);
 
@@ -1780,10 +1779,8 @@ static void __slab_free(struct kmem_cach
 {
 	void *prior;
 	void **object = (void *)x;
-	struct kmem_cache_cpu *c;
 
-	c = __this_cpu_ptr(s->cpu_slab);
-	stat(c, FREE_SLOWPATH);
+	stat(s, FREE_SLOWPATH);
 	slab_lock(page);
 
 	if (unlikely(SLABDEBUG && PageSlubDebug(page)))
@@ -1796,7 +1793,7 @@ checks_ok:
 	page->inuse--;
 
 	if (unlikely(PageSlubFrozen(page))) {
-		stat(c, FREE_FROZEN);
+		stat(s, FREE_FROZEN);
 		goto out_unlock;
 	}
 
@@ -1809,7 +1806,7 @@ checks_ok:
 	 */
 	if (unlikely(!prior)) {
 		add_partial(get_node(s, page_to_nid(page)), page, 1);
-		stat(c, FREE_ADD_PARTIAL);
+		stat(s, FREE_ADD_PARTIAL);
 	}
 
 out_unlock:
@@ -1822,10 +1819,10 @@ slab_empty:
 		 * Slab still on the partial list.
 		 */
 		remove_partial(s, page);
-		stat(c, FREE_REMOVE_PARTIAL);
+		stat(s, FREE_REMOVE_PARTIAL);
 	}
 	slab_unlock(page);
-	stat(c, FREE_SLAB);
+	stat(s, FREE_SLAB);
 	discard_slab(s, page);
 	return;
 
@@ -1863,7 +1860,7 @@ static __always_inline void slab_free(st
 	if (likely(page == c->page && c->node >= 0)) {
 		set_freepointer(s, object, c->freelist);
 		c->freelist = object;
-		stat(c, FREE_FASTPATH);
+		stat(s, FREE_FASTPATH);
 	} else
 		__slab_free(s, page, x, addr);
 
#8 lore raw

PATCH v8 7/16 Module handling: Use this_cpu_xx to dynamically allocate counters

In reply to Christoph Lameter

Use cpu ops to deal with the per cpu data instead of a local_t. Reduces memory requirements, cache footprint and decreases cycle counts.

The this_cpu_xx operations are also used for !SMP mode. Otherwise we could not drop the use of __module_ref_addr() which would make per cpu data handling complicated. this_cpu_xx operations have their own fallback for !SMP.

The last hold out of users of local_t is the tracing ringbuffer after this patch has been applied.

---
 include/linux/module.h     |   38 ++++++++++++++------------------------
 kernel/module.c            |   30 ++++++++++++++++--------------
 kernel/trace/ring_buffer.c |    1 +
 3 files changed, 31 insertions(+), 38 deletions(-)
include/linux/module.h +14 -24
Index: linux-2.6/include/linux/module.h
===================================================================
@@ -16,8 +16,7 @@
 #include <linux/kobject.h>
 #include <linux/moduleparam.h>
 #include <linux/tracepoint.h>
-
-#include <asm/local.h>
+#include <linux/percpu.h>
 #include <asm/module.h>
 
 #include <trace/events/module.h>
@@ -363,11 +362,9 @@ struct module
 	/* Destruction function. */
 	void (*exit)(void);
 
-#ifdef CONFIG_SMP
-	char *refptr;
-#else
-	local_t ref;
-#endif
+	struct module_ref {
+		int count;
+	} *refptr;
 #endif
 
 #ifdef CONFIG_CONSTRUCTORS
@@ -454,25 +451,16 @@ void __symbol_put(const char *symbol);
 #define symbol_put(x) __symbol_put(MODULE_SYMBOL_PREFIX #x)
 void symbol_put_addr(void *addr);
 
-static inline local_t *__module_ref_addr(struct module *mod, int cpu)
-{
-#ifdef CONFIG_SMP
-	return (local_t *) (mod->refptr + per_cpu_offset(cpu));
-#else
-	return &mod->ref;
-#endif
-}
-
 /* Sometimes we know we already have a refcount, and it's easier not
    to handle the error case (which only happens with rmmod --wait). */
 static inline void __module_get(struct module *module)
 {
 	if (module) {
-		unsigned int cpu = get_cpu();
-		local_inc(__module_ref_addr(module, cpu));
+		preempt_disable();
+		__this_cpu_inc(module->refptr->count);
 		trace_module_get(module, _THIS_IP_,
-				 local_read(__module_ref_addr(module, cpu)));
-		put_cpu();
+				 __this_cpu_read(module->refptr->count));
+		preempt_enable();
 	}
 }
 
@@ -481,15 +469,17 @@ static inline int try_module_get(struct 
 	int ret = 1;
 
 	if (module) {
-		unsigned int cpu = get_cpu();
+		preempt_disable();
+
 		if (likely(module_is_live(module))) {
-			local_inc(__module_ref_addr(module, cpu));
+			__this_cpu_inc(module->refptr->count);
 			trace_module_get(module, _THIS_IP_,
-				local_read(__module_ref_addr(module, cpu)));
+				__this_cpu_read(module->refptr->count));
 		}
 		else
 			ret = 0;
-		put_cpu();
+
+		preempt_enable();
 	}
 	return ret;
 }
kernel/module.c +16 -14
Index: linux-2.6/kernel/module.c
===================================================================
@@ -474,9 +474,10 @@ static void module_unload_init(struct mo
 
 	INIT_LIST_HEAD(&mod->modules_which_use_me);
 	for_each_possible_cpu(cpu)
-		local_set(__module_ref_addr(mod, cpu), 0);
+		per_cpu_ptr(mod->refptr, cpu)->count = 0;
+
 	/* Hold reference count during initialization. */
-	local_set(__module_ref_addr(mod, raw_smp_processor_id()), 1);
+	__this_cpu_write(mod->refptr->count, 1);
 	/* Backwards compatibility macros put refcount during init. */
 	mod->waiter = current;
 }
@@ -555,6 +556,7 @@ static void module_unload_free(struct mo
 				kfree(use);
 				sysfs_remove_link(i->holders_dir, mod->name);
 				/* There can be at most one match. */
+				free_percpu(i->refptr);
 				break;
 			}
 		}
@@ -619,7 +621,7 @@ unsigned int module_refcount(struct modu
 	int cpu;
 
 	for_each_possible_cpu(cpu)
-		total += local_read(__module_ref_addr(mod, cpu));
+		total += per_cpu_ptr(mod->refptr, cpu)->count;
 	return total;
 }
 EXPORT_SYMBOL(module_refcount);
@@ -796,14 +798,15 @@ static struct module_attribute refcnt = 
 void module_put(struct module *module)
 {
 	if (module) {
-		unsigned int cpu = get_cpu();
-		local_dec(__module_ref_addr(module, cpu));
+		preempt_disable();
+		__this_cpu_dec(module->refptr->count);
+
 		trace_module_put(module, _RET_IP_,
-				 local_read(__module_ref_addr(module, cpu)));
+				 __this_cpu_read(module->refptr->count));
 		/* Maybe they're waiting for us to drop reference? */
 		if (unlikely(!module_is_live(module)))
 			wake_up_process(module->waiter);
-		put_cpu();
+		preempt_enable();
 	}
 }
 EXPORT_SYMBOL(module_put);
@@ -1394,9 +1397,9 @@ static void free_module(struct module *m
 	kfree(mod->args);
 	if (mod->percpu)
 		percpu_modfree(mod->percpu);
-#if defined(CONFIG_MODULE_UNLOAD) && defined(CONFIG_SMP)
+#if defined(CONFIG_MODULE_UNLOAD)
 	if (mod->refptr)
-		percpu_modfree(mod->refptr);
+		free_percpu(mod->refptr);
 #endif
 	/* Free lock-classes: */
 	lockdep_free_key_range(mod->module_core, mod->core_size);
@@ -2159,9 +2162,8 @@ static noinline struct module *load_modu
 	mod = (void *)sechdrs[modindex].sh_addr;
 	kmemleak_load_module(mod, hdr, sechdrs, secstrings);
 
-#if defined(CONFIG_MODULE_UNLOAD) && defined(CONFIG_SMP)
-	mod->refptr = percpu_modalloc(sizeof(local_t), __alignof__(local_t),
-				      mod->name);
+#if defined(CONFIG_MODULE_UNLOAD)
+	mod->refptr = alloc_percpu(struct module_ref);
 	if (!mod->refptr) {
 		err = -ENOMEM;
 		goto free_init;
@@ -2393,8 +2395,8 @@ static noinline struct module *load_modu
 	kobject_put(&mod->mkobj.kobj);
  free_unload:
 	module_unload_free(mod);
-#if defined(CONFIG_MODULE_UNLOAD) && defined(CONFIG_SMP)
-	percpu_modfree(mod->refptr);
+#if defined(CONFIG_MODULE_UNLOAD)
+	free_percpu(mod->refptr);
  free_init:
 #endif
 	module_free(mod, mod->module_init);
kernel/trace/ring_buffer.c +1 -0
Index: linux-2.6/kernel/trace/ring_buffer.c
===================================================================
@@ -12,6 +12,7 @@
 #include <linux/hardirq.h>
 #include <linux/kmemcheck.h>
 #include <linux/module.h>
+#include <asm/local.h>
 #include <linux/percpu.h>
 #include <linux/mutex.h>
 #include <linux/init.h>
maintainer
#9 lore raw

Re: [this_cpu_xx V8 07/16] Module handling: Use this_cpu_xx to dynamically allocate counters

Hello,

Quoted 11 lines · Christoph Lameter

Use cpu ops to deal with the per cpu data instead of a local_t. Reduces memory requirements, cache footprint and decreases cycle counts.

The this_cpu_xx operations are also used for !SMP mode. Otherwise we could not drop the use of __module_ref_addr() which would make per cpu data handling complicated. this_cpu_xx operations have their own fallback for !SMP.

The last hold out of users of local_t is the tracing ringbuffer after this patch has been applied.

Please keep Rusty Russell cc'd on module changes.

kernel/trace/ring_buffer.c line 15

Quoted 9 lines

Index: linux-2.6/kernel/trace/ring_buffer.c
===================================================================

kernel/trace/ring_buffer.c +1 -0
@@ -12,6 +12,7 @@
 #include <linux/hardirq.h>
 #include <linux/kmemcheck.h>
 #include <linux/module.h>
+#include <asm/local.h>

This doesn't belong to this patch, right? I stripped this part out, added Cc: to Rusty and applied 1, 2, 7 and 8 to percpu branch. I'll post the updated patch here.

Thanks.

Signature · 1 line
tejun
maintainer
#10 lore raw

Re: [this_cpu_xx V8 07/16] Module handling: Use this_cpu_xx to dynamically allocate counters

Here's the version I committed to percpu branch.

Thanks.

Quoted 1 line

From 8af47ddd01364ae3c663c0bc92415c06fe887ba1 Mon Sep 17 00:00:00 2001

From: Christoph Lameter <[email protected]>
Date: Fri, 18 Dec 2009 16:26:24 -0600
Subject: [PATCH] module: Use this_cpu_xx to dynamically allocate counters

Use cpu ops to deal with the per cpu data instead of a local_t. Reduces memory requirements, cache footprint and decreases cycle counts.

The this_cpu_xx operations are also used for !SMP mode. Otherwise we could not drop the use of __module_ref_addr() which would make per cpu data handling complicated. this_cpu_xx operations have their own fallback for !SMP.

The last hold out of users of local_t is the tracing ringbuffer after this patch has been applied.

---
 include/linux/module.h |   38 ++++++++++++++------------------------
 kernel/module.c        |   30 ++++++++++++++++--------------
 2 files changed, 30 insertions(+), 38 deletions(-)
include/linux/module.h +14 -24
index 482efc8..9350577 100644
@@ -16,8 +16,7 @@
 #include <linux/kobject.h>
 #include <linux/moduleparam.h>
 #include <linux/tracepoint.h>
-
-#include <asm/local.h>
+#include <linux/percpu.h>
 #include <asm/module.h>
 
 #include <trace/events/module.h>
@@ -361,11 +360,9 @@ struct module
 	/* Destruction function. */
 	void (*exit)(void);
 
-#ifdef CONFIG_SMP
-	char *refptr;
-#else
-	local_t ref;
-#endif
+	struct module_ref {
+		int count;
+	} *refptr;
 #endif
 
 #ifdef CONFIG_CONSTRUCTORS
@@ -452,25 +449,16 @@ void __symbol_put(const char *symbol);
 #define symbol_put(x) __symbol_put(MODULE_SYMBOL_PREFIX #x)
 void symbol_put_addr(void *addr);
 
-static inline local_t *__module_ref_addr(struct module *mod, int cpu)
-{
-#ifdef CONFIG_SMP
-	return (local_t *) (mod->refptr + per_cpu_offset(cpu));
-#else
-	return &mod->ref;
-#endif
-}
-
 /* Sometimes we know we already have a refcount, and it's easier not
    to handle the error case (which only happens with rmmod --wait). */
 static inline void __module_get(struct module *module)
 {
 	if (module) {
-		unsigned int cpu = get_cpu();
-		local_inc(__module_ref_addr(module, cpu));
+		preempt_disable();
+		__this_cpu_inc(module->refptr->count);
 		trace_module_get(module, _THIS_IP_,
-				 local_read(__module_ref_addr(module, cpu)));
-		put_cpu();
+				 __this_cpu_read(module->refptr->count));
+		preempt_enable();
 	}
 }
 
@@ -479,15 +467,17 @@ static inline int try_module_get(struct module *module)
 	int ret = 1;
 
 	if (module) {
-		unsigned int cpu = get_cpu();
+		preempt_disable();
+
 		if (likely(module_is_live(module))) {
-			local_inc(__module_ref_addr(module, cpu));
+			__this_cpu_inc(module->refptr->count);
 			trace_module_get(module, _THIS_IP_,
-				local_read(__module_ref_addr(module, cpu)));
+				__this_cpu_read(module->refptr->count));
 		}
 		else
 			ret = 0;
-		put_cpu();
+
+		preempt_enable();
 	}
 	return ret;
 }
kernel/module.c +16 -14
index 64787cd..9bc04de 100644
@@ -474,9 +474,10 @@ static void module_unload_init(struct module *mod)
 
 	INIT_LIST_HEAD(&mod->modules_which_use_me);
 	for_each_possible_cpu(cpu)
-		local_set(__module_ref_addr(mod, cpu), 0);
+		per_cpu_ptr(mod->refptr, cpu)->count = 0;
+
 	/* Hold reference count during initialization. */
-	local_set(__module_ref_addr(mod, raw_smp_processor_id()), 1);
+	__this_cpu_write(mod->refptr->count, 1);
 	/* Backwards compatibility macros put refcount during init. */
 	mod->waiter = current;
 }
@@ -555,6 +556,7 @@ static void module_unload_free(struct module *mod)
 				kfree(use);
 				sysfs_remove_link(i->holders_dir, mod->name);
 				/* There can be at most one match. */
+				free_percpu(i->refptr);
 				break;
 			}
 		}
@@ -619,7 +621,7 @@ unsigned int module_refcount(struct module *mod)
 	int cpu;
 
 	for_each_possible_cpu(cpu)
-		total += local_read(__module_ref_addr(mod, cpu));
+		total += per_cpu_ptr(mod->refptr, cpu)->count;
 	return total;
 }
 EXPORT_SYMBOL(module_refcount);
@@ -796,14 +798,15 @@ static struct module_attribute refcnt = {
 void module_put(struct module *module)
 {
 	if (module) {
-		unsigned int cpu = get_cpu();
-		local_dec(__module_ref_addr(module, cpu));
+		preempt_disable();
+		__this_cpu_dec(module->refptr->count);
+
 		trace_module_put(module, _RET_IP_,
-				 local_read(__module_ref_addr(module, cpu)));
+				 __this_cpu_read(module->refptr->count));
 		/* Maybe they're waiting for us to drop reference? */
 		if (unlikely(!module_is_live(module)))
 			wake_up_process(module->waiter);
-		put_cpu();
+		preempt_enable();
 	}
 }
 EXPORT_SYMBOL(module_put);
@@ -1377,9 +1380,9 @@ static void free_module(struct module *mod)
 	kfree(mod->args);
 	if (mod->percpu)
 		percpu_modfree(mod->percpu);
-#if defined(CONFIG_MODULE_UNLOAD) && defined(CONFIG_SMP)
+#if defined(CONFIG_MODULE_UNLOAD)
 	if (mod->refptr)
-		percpu_modfree(mod->refptr);
+		free_percpu(mod->refptr);
 #endif
 	/* Free lock-classes: */
 	lockdep_free_key_range(mod->module_core, mod->core_size);
@@ -2145,9 +2148,8 @@ static noinline struct module *load_module(void __user *umod,
 	mod = (void *)sechdrs[modindex].sh_addr;
 	kmemleak_load_module(mod, hdr, sechdrs, secstrings);
 
-#if defined(CONFIG_MODULE_UNLOAD) && defined(CONFIG_SMP)
-	mod->refptr = percpu_modalloc(sizeof(local_t), __alignof__(local_t),
-				      mod->name);
+#if defined(CONFIG_MODULE_UNLOAD)
+	mod->refptr = alloc_percpu(struct module_ref);
 	if (!mod->refptr) {
 		err = -ENOMEM;
 		goto free_init;
@@ -2373,8 +2375,8 @@ static noinline struct module *load_module(void __user *umod,
 	kobject_put(&mod->mkobj.kobj);
  free_unload:
 	module_unload_free(mod);
-#if defined(CONFIG_MODULE_UNLOAD) && defined(CONFIG_SMP)
-	percpu_modfree(mod->refptr);
+#if defined(CONFIG_MODULE_UNLOAD)
+	free_percpu(mod->refptr);
  free_init:
 #endif
 	module_free(mod, mod->module_init);
Signature · 1 line
1.6.4.2
maintainer
#11 lore raw

Re: [this_cpu_xx V8 07/16] Module handling: Use this_cpu_xx to dynamically allocate counters

Quoted 16 lines · Tejun Heo

Here's the version I committed to percpu branch.

Thanks.

From 8af47ddd01364ae3c663c0bc92415c06fe887ba1 Mon Sep 17 00:00:00 2001
From: Christoph Lameter <[email protected]>
Date: Fri, 18 Dec 2009 16:26:24 -0600
Subject: [PATCH] module: Use this_cpu_xx to dynamically allocate counters

Use cpu ops to deal with the per cpu data instead of a local_t. Reduces memory requirements, cache footprint and decreases cycle counts.

The this_cpu_xx operations are also used for !SMP mode. Otherwise we could not drop the use of __module_ref_addr() which would make per cpu data handling complicated. this_cpu_xx operations have their own fallback for !SMP.

Quoted 5 lines

The last hold out of users of local_t is the tracing ringbuffer after this patch has been applied.

This was changed to Acked-by as Rusty acked on the previous thread.

Thanks.

Signature · 1 line
tejun
#12 lore raw

Re: [this_cpu_xx V8 07/16] Module handling: Use this_cpu_xx to dynamically allocate counters

In reply to Tejun Heo

kernel/module.c static void module_unload_free(struct module *mod) · line 562

Quoted 8 lines · Tejun Heo
kernel/module.c +1 -0
@@ -555,6 +556,7 @@ static void module_unload_free(struct module *mod)
 				kfree(use);
 				sysfs_remove_link(i->holders_dir, mod->name);
 				/* There can be at most one match. */
+				free_percpu(i->refptr);
 				break;
 			}
 		}

This looks very wrong.

Rusty

maintainer
#13 lore raw

Re: [this_cpu_xx V8 07/16] Module handling: Use this_cpu_xx to dynamically allocate counters

Quoted 11 lines · Rusty Russell

kernel/module.c static void module_unload_free(struct module *mod) · line 562

Quoted 8 lines · Tejun Heo
kernel/module.c +1 -0
@@ -555,6 +556,7 @@ static void module_unload_free(struct module *mod)
 				kfree(use);
 				sysfs_remove_link(i->holders_dir, mod->name);
 				/* There can be at most one match. */
+				free_percpu(i->refptr);
 				break;
 			}
 		}

This looks very wrong.

Indeed, thanks for spotting it. Christoph, I'm rolling back all patches from this series. Please re-post with updates.

Thanks.

Signature · 1 line
tejun
#14 lore raw

Re: [this_cpu_xx V8 07/16] Module handling: Use this_cpu_xx to dynamically allocate counters

Quoted 13 lines · Tejun Heo
Quoted 11 lines · Rusty Russell

kernel/module.c static void module_unload_free(struct module *mod) · line 562

Quoted 8 lines · Tejun Heo
kernel/module.c +1 -0
@@ -555,6 +556,7 @@ static void module_unload_free(struct module *mod)
 				kfree(use);
 				sysfs_remove_link(i->holders_dir, mod->name);
 				/* There can be at most one match. */
+				free_percpu(i->refptr);
 				break;
 			}
 		}

This looks very wrong.

Quoted 2 lines

Indeed, thanks for spotting it. Christoph, I'm rolling back all patches from this series. Please re-post with updates.

Simply drop the chunk?

#15 lore raw

Re: [this_cpu_xx V8 07/16] Module handling: Use this_cpu_xx to dynamically allocate counters

In reply to Tejun Heo

include/linux/module.h line 19

Quoted 10 lines · Tejun Heo

index 482efc8..9350577 100644

include/linux/module.h +1 -2
@@ -16,8 +16,7 @@
 #include <linux/kobject.h>
 #include <linux/moduleparam.h>
 #include <linux/tracepoint.h>
-
-#include <asm/local.h>
+#include <linux/percpu.h>

This is going to break ringbuffer.c.

#16 lore raw

Re: [this_cpu_xx V8 07/16] Module handling: Use this_cpu_xx to dynamically allocate counters

In reply to Tejun Heo
Quoted 14 lines · Tejun Heo

kernel/trace/ring_buffer.c line 15

Quoted 10 lines

Index: linux-2.6/kernel/trace/ring_buffer.c
===================================================================

kernel/trace/ring_buffer.c +1 -0
@@ -12,6 +12,7 @@
 #include <linux/hardirq.h>
 #include <linux/kmemcheck.h>
 #include <linux/module.h>
+#include <asm/local.h>

This doesn't belong to this patch, right? I stripped this part out, added Cc: to Rusty and applied 1, 2, 7 and 8 to percpu branch. I'll post the updated patch here. Thanks.

If you strip this part out then ringbuffer.c will no longer compile since it relies on the "#include <local.h>" that is removed by this patch from the module.h file.

maintainer
#17 lore raw

Re: [this_cpu_xx V8 07/16] Module handling: Use this_cpu_xx to dynamically allocate counters

Quoted 17 lines · Christoph Lameter
Quoted 14 lines · Tejun Heo

kernel/trace/ring_buffer.c line 15

Quoted 10 lines

Index: linux-2.6/kernel/trace/ring_buffer.c
===================================================================

kernel/trace/ring_buffer.c +1 -0
@@ -12,6 +12,7 @@
 #include <linux/hardirq.h>
 #include <linux/kmemcheck.h>
 #include <linux/module.h>
+#include <asm/local.h>

This doesn't belong to this patch, right? I stripped this part out, added Cc: to Rusty and applied 1, 2, 7 and 8 to percpu branch. I'll post the updated patch here. Thanks.

Quoted 3 lines

If you strip this part out then ringbuffer.c will no longer compile since it relies on the "#include <local.h>" that is removed by this patch from the module.h file.

Oh... alright. I'll add that comment and drop the offending chunk and recommit.

Thanks.

Signature · 1 line
tejun
#18 lore raw

Re: [this_cpu_xx V8 07/16] Module handling: Use this_cpu_xx to dynamically allocate counters

Quoted 18 lines · Tejun Heo
Quoted 17 lines · Christoph Lameter
Quoted 14 lines · Tejun Heo

kernel/trace/ring_buffer.c line 15

Quoted 10 lines

Index: linux-2.6/kernel/trace/ring_buffer.c
===================================================================

kernel/trace/ring_buffer.c +1 -0
@@ -12,6 +12,7 @@
 #include <linux/hardirq.h>
 #include <linux/kmemcheck.h>
 #include <linux/module.h>
+#include <asm/local.h>

This doesn't belong to this patch, right? I stripped this part out, added Cc: to Rusty and applied 1, 2, 7 and 8 to percpu branch. I'll post the updated patch here. Thanks.

Quoted 6 lines
Quoted 3 lines

If you strip this part out then ringbuffer.c will no longer compile since it relies on the "#include <local.h>" that is removed by this patch from the module.h file.

Oh... alright. I'll add that comment and drop the offending chunk and recommit.

So we need a separate patch to deal with remval of the #include <asm/local.h> from modules.h?

#19 lore raw

Re: [this_cpu_xx V8 07/16] Module handling: Use this_cpu_xx to dynamically allocate counters

Quoted 24 lines · * Christoph Lameter ([email protected]) wrote:
Quoted 22 lines · Tejun Heo
Quoted 20 lines · Christoph Lameter
Quoted 14 lines · Tejun Heo

kernel/trace/ring_buffer.c line 15

Quoted 10 lines

Index: linux-2.6/kernel/trace/ring_buffer.c
===================================================================

kernel/trace/ring_buffer.c +1 -0
@@ -12,6 +12,7 @@
 #include <linux/hardirq.h>
 #include <linux/kmemcheck.h>
 #include <linux/module.h>
+#include <asm/local.h>

This doesn't belong to this patch, right? I stripped this part out, added Cc: to Rusty and applied 1, 2, 7 and 8 to percpu branch. I'll post the updated patch here. Thanks.

If you strip this part out then ringbuffer.c will no longer compile since it relies on the "#include <local.h>" that is removed by this patch from the module.h file.

Quoted 6 lines
Quoted 2 lines

Oh... alright. I'll add that comment and drop the offending chunk and recommit.

So we need a separate patch to deal with remval of the #include <asm/local.h> from modules.h?

I think this would make sense, yes. This would be a patch specific to the ring-buffer code that would go through (or be acked by) Steven.

Mathieu

Signature · 2 lines
Mathieu Desnoyers
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68
#20 lore raw

PATCH v8 8/16 Remove cpu_local_xx macros

In reply to Christoph Lameter

These macros have not been used for awhile now and are not used by the last local_t user in the tree (trace ringbuffer logic).

---
 arch/alpha/include/asm/local.h   |   17 -----------------
 arch/m32r/include/asm/local.h    |   25 -------------------------
 arch/mips/include/asm/local.h    |   25 -------------------------
 arch/powerpc/include/asm/local.h |   25 -------------------------
 arch/x86/include/asm/local.h     |   37 -------------------------------------
 include/asm-generic/local.h      |   19 -------------------
 6 files changed, 148 deletions(-)
arch/alpha/include/asm/local.h +0 -17
Index: linux-2.6/arch/alpha/include/asm/local.h
===================================================================
@@ -98,21 +98,4 @@ static __inline__ long local_sub_return(
 #define __local_add(i,l)	((l)->a.counter+=(i))
 #define __local_sub(i,l)	((l)->a.counter-=(i))
 
-/* Use these for per-cpu local_t variables: on some archs they are
- * much more efficient than these naive implementations.  Note they take
- * a variable, not an address.
- */
-#define cpu_local_read(l)	local_read(&__get_cpu_var(l))
-#define cpu_local_set(l, i)	local_set(&__get_cpu_var(l), (i))
-
-#define cpu_local_inc(l)	local_inc(&__get_cpu_var(l))
-#define cpu_local_dec(l)	local_dec(&__get_cpu_var(l))
-#define cpu_local_add(i, l)	local_add((i), &__get_cpu_var(l))
-#define cpu_local_sub(i, l)	local_sub((i), &__get_cpu_var(l))
-
-#define __cpu_local_inc(l)	__local_inc(&__get_cpu_var(l))
-#define __cpu_local_dec(l)	__local_dec(&__get_cpu_var(l))
-#define __cpu_local_add(i, l)	__local_add((i), &__get_cpu_var(l))
-#define __cpu_local_sub(i, l)	__local_sub((i), &__get_cpu_var(l))
-
 #endif /* _ALPHA_LOCAL_H */
arch/m32r/include/asm/local.h +0 -25
Index: linux-2.6/arch/m32r/include/asm/local.h
===================================================================
@@ -338,29 +338,4 @@ static inline void local_set_mask(unsign
  * a variable, not an address.
  */
 
-/* Need to disable preemption for the cpu local counters otherwise we could
-   still access a variable of a previous CPU in a non local way. */
-#define cpu_local_wrap_v(l)	 	\
-	({ local_t res__;		\
-	   preempt_disable(); 		\
-	   res__ = (l);			\
-	   preempt_enable();		\
-	   res__; })
-#define cpu_local_wrap(l)		\
-	({ preempt_disable();		\
-	   l;				\
-	   preempt_enable(); })		\
-
-#define cpu_local_read(l)    cpu_local_wrap_v(local_read(&__get_cpu_var(l)))
-#define cpu_local_set(l, i)  cpu_local_wrap(local_set(&__get_cpu_var(l), (i)))
-#define cpu_local_inc(l)     cpu_local_wrap(local_inc(&__get_cpu_var(l)))
-#define cpu_local_dec(l)     cpu_local_wrap(local_dec(&__get_cpu_var(l)))
-#define cpu_local_add(i, l)  cpu_local_wrap(local_add((i), &__get_cpu_var(l)))
-#define cpu_local_sub(i, l)  cpu_local_wrap(local_sub((i), &__get_cpu_var(l)))
-
-#define __cpu_local_inc(l)	cpu_local_inc(l)
-#define __cpu_local_dec(l)	cpu_local_dec(l)
-#define __cpu_local_add(i, l)	cpu_local_add((i), (l))
-#define __cpu_local_sub(i, l)	cpu_local_sub((i), (l))
-
 #endif /* __M32R_LOCAL_H */
arch/mips/include/asm/local.h +0 -25
Index: linux-2.6/arch/mips/include/asm/local.h
===================================================================
@@ -193,29 +193,4 @@ static __inline__ long local_sub_return(
 #define __local_add(i, l)	((l)->a.counter+=(i))
 #define __local_sub(i, l)	((l)->a.counter-=(i))
 
-/* Need to disable preemption for the cpu local counters otherwise we could
-   still access a variable of a previous CPU in a non atomic way. */
-#define cpu_local_wrap_v(l)	 	\
-	({ local_t res__;		\
-	   preempt_disable(); 		\
-	   res__ = (l);			\
-	   preempt_enable();		\
-	   res__; })
-#define cpu_local_wrap(l)		\
-	({ preempt_disable();		\
-	   l;				\
-	   preempt_enable(); })		\
-
-#define cpu_local_read(l)    cpu_local_wrap_v(local_read(&__get_cpu_var(l)))
-#define cpu_local_set(l, i)  cpu_local_wrap(local_set(&__get_cpu_var(l), (i)))
-#define cpu_local_inc(l)     cpu_local_wrap(local_inc(&__get_cpu_var(l)))
-#define cpu_local_dec(l)     cpu_local_wrap(local_dec(&__get_cpu_var(l)))
-#define cpu_local_add(i, l)  cpu_local_wrap(local_add((i), &__get_cpu_var(l)))
-#define cpu_local_sub(i, l)  cpu_local_wrap(local_sub((i), &__get_cpu_var(l)))
-
-#define __cpu_local_inc(l)	cpu_local_inc(l)
-#define __cpu_local_dec(l)	cpu_local_dec(l)
-#define __cpu_local_add(i, l)	cpu_local_add((i), (l))
-#define __cpu_local_sub(i, l)	cpu_local_sub((i), (l))
-
 #endif /* _ARCH_MIPS_LOCAL_H */
arch/powerpc/include/asm/local.h +0 -25
Index: linux-2.6/arch/powerpc/include/asm/local.h
===================================================================
@@ -172,29 +172,4 @@ static __inline__ long local_dec_if_posi
 #define __local_add(i,l)	((l)->a.counter+=(i))
 #define __local_sub(i,l)	((l)->a.counter-=(i))
 
-/* Need to disable preemption for the cpu local counters otherwise we could
-   still access a variable of a previous CPU in a non atomic way. */
-#define cpu_local_wrap_v(l)	 	\
-	({ local_t res__;		\
-	   preempt_disable(); 		\
-	   res__ = (l);			\
-	   preempt_enable();		\
-	   res__; })
-#define cpu_local_wrap(l)		\
-	({ preempt_disable();		\
-	   l;				\
-	   preempt_enable(); })		\
-
-#define cpu_local_read(l)    cpu_local_wrap_v(local_read(&__get_cpu_var(l)))
-#define cpu_local_set(l, i)  cpu_local_wrap(local_set(&__get_cpu_var(l), (i)))
-#define cpu_local_inc(l)     cpu_local_wrap(local_inc(&__get_cpu_var(l)))
-#define cpu_local_dec(l)     cpu_local_wrap(local_dec(&__get_cpu_var(l)))
-#define cpu_local_add(i, l)  cpu_local_wrap(local_add((i), &__get_cpu_var(l)))
-#define cpu_local_sub(i, l)  cpu_local_wrap(local_sub((i), &__get_cpu_var(l)))
-
-#define __cpu_local_inc(l)	cpu_local_inc(l)
-#define __cpu_local_dec(l)	cpu_local_dec(l)
-#define __cpu_local_add(i, l)	cpu_local_add((i), (l))
-#define __cpu_local_sub(i, l)	cpu_local_sub((i), (l))
-
 #endif /* _ARCH_POWERPC_LOCAL_H */
arch/x86/include/asm/local.h +0 -37
Index: linux-2.6/arch/x86/include/asm/local.h
===================================================================
@@ -195,41 +195,4 @@ static inline long local_sub_return(long
 #define __local_add(i, l)	local_add((i), (l))
 #define __local_sub(i, l)	local_sub((i), (l))
 
-/* Use these for per-cpu local_t variables: on some archs they are
- * much more efficient than these naive implementations.  Note they take
- * a variable, not an address.
- *
- * X86_64: This could be done better if we moved the per cpu data directly
- * after GS.
- */
-
-/* Need to disable preemption for the cpu local counters otherwise we could
-   still access a variable of a previous CPU in a non atomic way. */
-#define cpu_local_wrap_v(l)		\
-({					\
-	local_t res__;			\
-	preempt_disable(); 		\
-	res__ = (l);			\
-	preempt_enable();		\
-	res__;				\
-})
-#define cpu_local_wrap(l)		\
-({					\
-	preempt_disable();		\
-	(l);				\
-	preempt_enable();		\
-})					\
-
-#define cpu_local_read(l)    cpu_local_wrap_v(local_read(&__get_cpu_var((l))))
-#define cpu_local_set(l, i)  cpu_local_wrap(local_set(&__get_cpu_var((l)), (i)))
-#define cpu_local_inc(l)     cpu_local_wrap(local_inc(&__get_cpu_var((l))))
-#define cpu_local_dec(l)     cpu_local_wrap(local_dec(&__get_cpu_var((l))))
-#define cpu_local_add(i, l)  cpu_local_wrap(local_add((i), &__get_cpu_var((l))))
-#define cpu_local_sub(i, l)  cpu_local_wrap(local_sub((i), &__get_cpu_var((l))))
-
-#define __cpu_local_inc(l)	cpu_local_inc((l))
-#define __cpu_local_dec(l)	cpu_local_dec((l))
-#define __cpu_local_add(i, l)	cpu_local_add((i), (l))
-#define __cpu_local_sub(i, l)	cpu_local_sub((i), (l))
-
 #endif /* _ASM_X86_LOCAL_H */
include/asm-generic/local.h +0 -19
Index: linux-2.6/include/asm-generic/local.h
===================================================================
@@ -52,23 +52,4 @@ typedef struct
 #define __local_add(i,l)	local_set((l), local_read(l) + (i))
 #define __local_sub(i,l)	local_set((l), local_read(l) - (i))
 
-/* Use these for per-cpu local_t variables: on some archs they are
- * much more efficient than these naive implementations.  Note they take
- * a variable (eg. mystruct.foo), not an address.
- */
-#define cpu_local_read(l)	local_read(&__get_cpu_var(l))
-#define cpu_local_set(l, i)	local_set(&__get_cpu_var(l), (i))
-#define cpu_local_inc(l)	local_inc(&__get_cpu_var(l))
-#define cpu_local_dec(l)	local_dec(&__get_cpu_var(l))
-#define cpu_local_add(i, l)	local_add((i), &__get_cpu_var(l))
-#define cpu_local_sub(i, l)	local_sub((i), &__get_cpu_var(l))
-
-/* Non-atomic increments, ie. preemption disabled and won't be touched
- * in interrupt, etc.  Some archs can optimize this case well.
- */
-#define __cpu_local_inc(l)	__local_inc(&__get_cpu_var(l))
-#define __cpu_local_dec(l)	__local_dec(&__get_cpu_var(l))
-#define __cpu_local_add(i, l)	__local_add((i), &__get_cpu_var(l))
-#define __cpu_local_sub(i, l)	__local_sub((i), &__get_cpu_var(l))
-
 #endif /* _ASM_GENERIC_LOCAL_H */
#21 lore raw

PATCH v8 9/16 Allow arch to provide inc/dec functionality for each size separately

In reply to Christoph Lameter

Current this_cpu ops only allow an arch to specify add RMW operations or inc and dec for all sizes. Some arches can do more efficient inc and dec operations. Allow size specific override of fallback functions like with the other operations.

---
 include/linux/percpu.h |   96 +++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 90 insertions(+), 6 deletions(-)
include/linux/percpu.h +90 -6
Index: linux-2.6/include/linux/percpu.h
===================================================================
@@ -257,6 +257,18 @@ do {									\
 	}								\
 } while (0)
 
+#define __pcpu_size_call_noparam(stem, variable)			\
+do {									\
+	switch(sizeof(variable)) {					\
+		case 1: stem##1(variable);break;			\
+		case 2: stem##2(variable);break;			\
+		case 4: stem##4(variable);break;			\
+		case 8: stem##8(variable);break;			\
+		default: 						\
+			__bad_size_call_parameter();break;		\
+	}								\
+} while (0)
+
 /*
  * Optimized manipulation for memory allocated through the per cpu
  * allocator or for addresses of per cpu variables (can be determined
@@ -352,11 +364,35 @@ do {									\
 #endif
 
 #ifndef this_cpu_inc
-# define this_cpu_inc(pcp)		this_cpu_add((pcp), 1)
+# ifndef this_cpu_inc_1
+#  define this_cpu_inc_1(pcp)	__this_cpu_add((pcp), 1)
+# endif
+# ifndef this_cpu_inc_2
+#  define this_cpu_inc_2(pcp)	__this_cpu_add((pcp), 1)
+# endif
+# ifndef this_cpu_inc_4
+#  define this_cpu_inc_4(pcp)	__this_cpu_add((pcp), 1)
+# endif
+# ifndef this_cpu_inc_8
+#  define this_cpu_inc_8(pcp)	__this_cpu_add((pcp), 1)
+# endif
+# define this_cpu_inc(pcp)		__pcpu_size_call_noparam(this_cpu_inc_, (pcp))
 #endif
 
 #ifndef this_cpu_dec
-# define this_cpu_dec(pcp)		this_cpu_sub((pcp), 1)
+# ifndef this_cpu_dec_1
+#  define this_cpu_dec_1(pcp)	__this_cpu_sub((pcp), 1)
+# endif
+# ifndef this_cpu_dec_2
+#  define this_cpu_dec_2(pcp)	__this_cpu_sub((pcp), 1)
+# endif
+# ifndef this_cpu_dec_4
+#  define this_cpu_dec_4(pcp)	__this_cpu_sub((pcp), 1)
+# endif
+# ifndef this_cpu_dec_8
+#  define this_cpu_dec_8(pcp)	__this_cpu_sub((pcp), 1)
+# endif
+# define this_cpu_dec(pcp)		__pcpu_size_call_noparam(this_cpu_dec_, (pcp))
 #endif
 
 #ifndef this_cpu_and
@@ -479,11 +515,35 @@ do {									\
 #endif
 
 #ifndef __this_cpu_inc
-# define __this_cpu_inc(pcp)		__this_cpu_add((pcp), 1)
+# ifndef __this_cpu_inc_1
+#  define __this_cpu_inc_1(pcp)	__this_cpu_add_1((pcp), 1)
+# endif
+# ifndef __this_cpu_inc_2
+#  define __this_cpu_inc_2(pcp)	__this_cpu_add_2((pcp), 1)
+# endif
+# ifndef __this_cpu_inc_4
+#  define __this_cpu_inc_4(pcp)	__this_cpu_add_4((pcp), 1)
+# endif
+# ifndef __this_cpu_inc_8
+#  define __this_cpu_inc_8(pcp)	__this_cpu_add_8((pcp), 1)
+# endif
+# define __this_cpu_inc(pcp)	__pcpu_size_call_noparam(__this_cpu_inc_, (pcp))
 #endif
 
 #ifndef __this_cpu_dec
-# define __this_cpu_dec(pcp)		__this_cpu_sub((pcp), 1)
+# ifndef __this_cpu_dec_1
+#  define __this_cpu_dec_1(pcp)	__this_cpu_sub((pcp), 1)
+# endif
+# ifndef __this_cpu_dec_2
+#  define __this_cpu_dec_2(pcp)	__this_cpu_sub((pcp), 1)
+# endif
+# ifndef __this_cpu_dec_4
+#  define __this_cpu_dec_4(pcp)	__this_cpu_sub((pcp), 1)
+# endif
+# ifndef __this_cpu_dec_8
+#  define __this_cpu_dec_8(pcp)	__this_cpu_sub((pcp), 1)
+# endif
+# define __this_cpu_dec(pcp)	__pcpu_size_call_noparam(__this_cpu_dec_, (pcp))
 #endif
 
 #ifndef __this_cpu_and
@@ -570,11 +630,35 @@ do {									\
 #endif
 
 #ifndef irqsafe_cpu_inc
-# define irqsafe_cpu_inc(pcp)	irqsafe_cpu_add((pcp), 1)
+# ifndef irqsafe_cpu_inc_1
+#  define irqsafe_cpu_inc_1(pcp)	irqsafe_cpu_add_1((pcp), 1)
+# endif
+# ifndef irqsafe_cpu_inc_2
+#  define irqsafe_cpu_inc_2(pcp)	irqsafe_cpu_add_2((pcp), 1)
+# endif
+# ifndef irqsafe_cpu_inc_4
+#  define irqsafe_cpu_inc_4(pcp)	irqsafe_cpu_add_4((pcp), 1)
+# endif
+# ifndef irqsafe_cpu_inc_8
+#  define irqsafe_cpu_inc_8(pcp)	irqsafe_cpu_add_8((pcp), 1)
+# endif
+# define irqsafe_cpu_inc(pcp)	__pcpu_size_call_noparam(irqsafe_cpu_inc_, (pcp))
 #endif
 
 #ifndef irqsafe_cpu_dec
-# define irqsafe_cpu_dec(pcp)	irqsafe_cpu_sub((pcp), 1)
+# ifndef irqsafe_cpu_dec_1
+#  define irqsafe_cpu_dec_1(pcp)	irqsafe_cpu_sub_1((pcp), 1)
+# endif
+# ifndef irqsafe_cpu_dec_2
+#  define irqsafe_cpu_dec_2(pcp)	irqsafe_cpu_sub_2((pcp), 1)
+# endif
+# ifndef irqsafe_cpu_dec_4
+#  define irqsafe_cpu_dec_4(pcp)	irqsafe_cpu_sub_4((pcp), 1)
+# endif
+# ifndef irqsafe_cpu_dec_8
+#  define irqsafe_cpu_dec_8(pcp)	irqsafe_cpu_sub_8((pcp), 1)
+# endif
+# define irqsafe_cpu_dec(pcp)	__pcpu_size_call_noparam(irqsafe_cpu_dec_, (pcp))
 #endif
 
 #ifndef irqsafe_cpu_and
maintainer
#22 lore raw

Re: [this_cpu_xx V8 09/16] Allow arch to provide inc/dec functionality for each size separately

Quoted 4 lines · Christoph Lameter

Current this_cpu ops only allow an arch to specify add RMW operations or inc and dec for all sizes. Some arches can do more efficient inc and dec operations. Allow size specific override of fallback functions like with the other operations.

Wouldn't it be better to use __builtin_constant_p() and switching in arch add/dec macros? It just seems a bit extreme to define all those different variants from generic header. I'm quite unsure whether providing overrides for all the different size variants from generic header is necessary at all. If an arch is gonna override the operation, it's probably best to just let it override the whole thing.

Thanks.

Signature · 1 line
tejun
#23 lore raw

Re: [this_cpu_xx V8 09/16] Allow arch to provide inc/dec functionality for each size separately

Quoted 8 lines · Tejun Heo
Quoted 4 lines · Christoph Lameter

Current this_cpu ops only allow an arch to specify add RMW operations or inc and dec for all sizes. Some arches can do more efficient inc and dec operations. Allow size specific override of fallback functions like with the other operations.

Wouldn't it be better to use __builtin_constant_p() and switching in arch add/dec macros? It just seems a bit extreme to define all those

Yes that could be done but I am on vacation till next year ;-)...

maintainer
#24 lore raw

Re: [this_cpu_xx V8 09/16] Allow arch to provide inc/dec functionality for each size separately

Hello, Christoph.

Quoted 8 lines · Christoph Lameter
Quoted 6 lines · Tejun Heo
Quoted 4 lines · Christoph Lameter

Current this_cpu ops only allow an arch to specify add RMW operations or inc and dec for all sizes. Some arches can do more efficient inc and dec operations. Allow size specific override of fallback functions like with the other operations.

Quoted 4 lines
Quoted 2 lines

Wouldn't it be better to use __builtin_constant_p() and switching in arch add/dec macros? It just seems a bit extreme to define all those

Yes that could be done but I am on vacation till next year ;-)...

I'm not sure how much I would be working from now till the end of the year but if I end up doing some work I'll give it a shot.

Happy holidays and enjoy your vacation!

Signature · 1 line
tejun
#25 lore raw

PATCH v8 10/16 Support generating inc/dec for this_cpu_inc/dec

In reply to Christoph Lameter

Support generating inc/dec instruction. Currently we create an add 1 instruction. Saves one byte per use of this_cpu_xx.

---
 arch/x86/include/asm/percpu.h |   47 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 47 insertions(+)
arch/x86/include/asm/percpu.h +47 -0
Index: linux-2.6/arch/x86/include/asm/percpu.h
===================================================================
@@ -133,6 +133,29 @@ do {							\
 	pfo_ret__;					\
 })
 
+#define percpu_var_op(op, var)				\
+({							\
+	switch (sizeof(var)) {				\
+	case 1:						\
+		asm(op "b "__percpu_arg(0)		\
+		    : "+m" (var) :);			\
+		break;					\
+	case 2:						\
+		asm(op "w "__percpu_arg(0)		\
+		    : "+m" (var) :);			\
+		break;					\
+	case 4:						\
+		asm(op "l "__percpu_arg(0)		\
+		    : "+m" (var) :);			\
+		break;					\
+	case 8:						\
+		asm(op "q "__percpu_arg(0)		\
+		    : "+m" (var) :);			\
+		break;					\
+	default: __bad_percpu_size();			\
+	}						\
+})
+
 /*
  * percpu_read() makes gcc load the percpu variable every time it is
  * accessed while percpu_read_stable() allows the value to be cached.
@@ -163,6 +186,12 @@ do {							\
 #define __this_cpu_add_1(pcp, val)	percpu_to_op("add", (pcp), val)
 #define __this_cpu_add_2(pcp, val)	percpu_to_op("add", (pcp), val)
 #define __this_cpu_add_4(pcp, val)	percpu_to_op("add", (pcp), val)
+#define __this_cpu_inc_1(pcp)		percpu_var_op("inc", (pcp))
+#define __this_cpu_inc_2(pcp)		percpu_var_op("inc", (pcp))
+#define __this_cpu_inc_4(pcp)		percpu_var_op("inc", (pcp))
+#define __this_cpu_dec_1(pcp)		percpu_var_op("dec", (pcp))
+#define __this_cpu_dec_2(pcp)		percpu_var_op("dec", (pcp))
+#define __this_cpu_dec_4(pcp)		percpu_var_op("dec", (pcp))
 #define __this_cpu_and_1(pcp, val)	percpu_to_op("and", (pcp), val)
 #define __this_cpu_and_2(pcp, val)	percpu_to_op("and", (pcp), val)
 #define __this_cpu_and_4(pcp, val)	percpu_to_op("and", (pcp), val)
@@ -182,6 +211,12 @@ do {							\
 #define this_cpu_add_1(pcp, val)	percpu_to_op("add", (pcp), val)
 #define this_cpu_add_2(pcp, val)	percpu_to_op("add", (pcp), val)
 #define this_cpu_add_4(pcp, val)	percpu_to_op("add", (pcp), val)
+#define this_cpu_inc_1(pcp)		percpu_var_op("inc", (pcp))
+#define this_cpu_inc_2(pcp)		percpu_var_op("inc", (pcp))
+#define this_cpu_inc_4(pcp)		percpu_var_op("inc", (pcp))
+#define this_cpu_dec_1(pcp)		percpu_var_op("dec", (pcp))
+#define this_cpu_dec_2(pcp)		percpu_var_op("dec", (pcp))
+#define this_cpu_dec_4(pcp)		percpu_var_op("dec", (pcp))
 #define this_cpu_and_1(pcp, val)	percpu_to_op("and", (pcp), val)
 #define this_cpu_and_2(pcp, val)	percpu_to_op("and", (pcp), val)
 #define this_cpu_and_4(pcp, val)	percpu_to_op("and", (pcp), val)
@@ -195,6 +230,12 @@ do {							\
 #define irqsafe_cpu_add_1(pcp, val)	percpu_to_op("add", (pcp), val)
 #define irqsafe_cpu_add_2(pcp, val)	percpu_to_op("add", (pcp), val)
 #define irqsafe_cpu_add_4(pcp, val)	percpu_to_op("add", (pcp), val)
+#define irqsafe_cpu_inc_1(pcp)		percpu_var_op("inc", (pcp))
+#define irqsafe_cpu_inc_2(pcp)		percpu_var_op("inc", (pcp))
+#define irqsafe_cpu_inc_4(pcp)		percpu_var_op("inc", (pcp))
+#define irqsafe_cpu_dec_1(pcp)		percpu_var_op("dec", (pcp))
+#define irqsafe_cpu_dec_2(pcp)		percpu_var_op("dec", (pcp))
+#define irqsafe_cpu_dec_4(pcp)		percpu_var_op("dec", (pcp))
 #define irqsafe_cpu_and_1(pcp, val)	percpu_to_op("and", (pcp), val)
 #define irqsafe_cpu_and_2(pcp, val)	percpu_to_op("and", (pcp), val)
 #define irqsafe_cpu_and_4(pcp, val)	percpu_to_op("and", (pcp), val)
@@ -213,6 +254,8 @@ do {							\
 #define __this_cpu_read_8(pcp)		percpu_from_op("mov", (pcp), "m"(pcp))
 #define __this_cpu_write_8(pcp, val)	percpu_to_op("mov", (pcp), val)
 #define __this_cpu_add_8(pcp, val)	percpu_to_op("add", (pcp), val)
+#define __this_cpu_inc_8(pcp)		percpu_var_op("inc", (pcp))
+#define __this_cpu_dec_8(pcp)		percpu_var_op("dec", (pcp))
 #define __this_cpu_and_8(pcp, val)	percpu_to_op("and", (pcp), val)
 #define __this_cpu_or_8(pcp, val)	percpu_to_op("or", (pcp), val)
 #define __this_cpu_xor_8(pcp, val)	percpu_to_op("xor", (pcp), val)
@@ -220,11 +263,15 @@ do {							\
 #define this_cpu_read_8(pcp)		percpu_from_op("mov", (pcp), "m"(pcp))
 #define this_cpu_write_8(pcp, val)	percpu_to_op("mov", (pcp), val)
 #define this_cpu_add_8(pcp, val)	percpu_to_op("add", (pcp), val)
+#define this_cpu_inc_8(pcp)		percpu_var_op("inc", (pcp))
+#define this_cpu_dec_8(pcp)		percpu_var_op("dec", (pcp))
 #define this_cpu_and_8(pcp, val)	percpu_to_op("and", (pcp), val)
 #define this_cpu_or_8(pcp, val)		percpu_to_op("or", (pcp), val)
 #define this_cpu_xor_8(pcp, val)	percpu_to_op("xor", (pcp), val)
 
 #define irqsafe_cpu_add_8(pcp, val)	percpu_to_op("add", (pcp), val)
+#define irqsafe_cpu_inc_8(pcp)		percpu_var_op("inc", (pcp))
+#define irqsafe_cpu_dec_8(pcp)		percpu_var_op("dec", (pcp))
 #define irqsafe_cpu_and_8(pcp, val)	percpu_to_op("and", (pcp), val)
 #define irqsafe_cpu_or_8(pcp, val)	percpu_to_op("or", (pcp), val)
 #define irqsafe_cpu_xor_8(pcp, val)	percpu_to_op("xor", (pcp), val)
#26 lore raw

PATCH v8 11/16 Generic support for this_cpu_cmpxchg

In reply to Christoph Lameter

Provide support for this_cpu_cmpxchg.

---
 include/linux/percpu.h |   99 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 99 insertions(+)
include/linux/percpu.h +99 -0
Index: linux-2.6/include/linux/percpu.h
===================================================================
@@ -443,6 +443,62 @@ do {									\
 # define this_cpu_xor(pcp, val)		__pcpu_size_call(this_cpu_or_, (pcp), (val))
 #endif
 
+static inline unsigned long __this_cpu_cmpxchg_generic(volatile void *pptr,
+		unsigned long old, unsigned long new, int size)
+{
+	unsigned long prev;
+	volatile void * ptr = __this_cpu_ptr(pptr);
+
+	switch (size) {
+	case 1: prev = *(u8 *)ptr;
+		if (prev == old)
+			*(u8 *)ptr = (u8)new;
+		break;
+	case 2: prev = *(u16 *)ptr;
+		if (prev == old)
+			*(u16 *)ptr = (u16)new;
+		break;
+	case 4: prev = *(u32 *)ptr;
+		if (prev == old)
+			*(u32 *)ptr = (u32)new;
+		break;
+	case 8: prev = *(u64 *)ptr;
+		if (prev == old)
+			*(u64 *)ptr = (u64)new;
+		break;
+	default:
+		__bad_size_call_parameter();
+	}
+	return prev;
+}
+
+static inline unsigned long this_cpu_cmpxchg_generic(volatile void *ptr,
+		unsigned long old, unsigned long new, int size)
+{
+	unsigned long prev;
+
+	preempt_disable();
+	prev = __this_cpu_cmpxchg_generic(ptr, old, new, size);
+	preempt_enable();
+	return prev;
+}
+
+#ifndef this_cpu_cmpxchg
+# ifndef this_cpu_cmpxchg_1
+#  define this_cpu_cmpxchg_1(pcp, old, new) this_cpu_cmpxchg_generic((pcp), (old), (new), 1)
+# endif
+# ifndef this_cpu_cmpxchg_2
+#  define this_cpu_cmpxchg_2(pcp, old, new) this_cpu_cmpxchg_generic((pcp), (old), (new), 2)
+# endif
+# ifndef this_cpu_cmpxchg_4
+#  define this_cpu_cmpxchg_4(pcp, old, new) this_cpu_cmpxchg_generic((pcp), (old), (new), 4)
+# endif
+# ifndef this_cpu_cmpxchg_8
+#  define this_cpu_cmpxchg_8(pcp, old, new) this_cpu_cmpxchg_generic((pcp), (old), (new), 8)
+# endif
+# define this_cpu_cmpxchg(pcp, old, new) __pcpu_size_call_return(__this_cpu_cmpxchg_, (old), (new))
+#endif
+
 /*
  * Generic percpu operations that do not require preemption handling.
  * Either we do not care about races or the caller has the
@@ -594,6 +650,22 @@ do {									\
 # define __this_cpu_xor(pcp, val)	__pcpu_size_call(__this_cpu_xor_, (pcp), (val))
 #endif
 
+#ifndef __this_cpu_cmpxchg
+# ifndef __this_cpu_cmpxchg_1
+#  define __this_cpu_cmpxchg_1(pcp, old, new) __this_cpu_cmpxchg_generic((pcp), (old), (new), 1)
+# endif
+# ifndef __this_cpu_cmpxchg_2
+#  define __this_cpu_cmpxchg_2(pcp, old, new) __this_cpu_cmpxchg_generic((pcp), (old), (new), 2)
+# endif
+# ifndef __this_cpu_cmpxchg_4
+#  define __this_cpu_cmpxchg_4(pcp, old, new) __this_cpu_cmpxchg_generic((pcp), (old), (new), 4)
+# endif
+# ifndef __this_cpu_cmpxchg_8
+#  define __this_cpu_cmpxchg_8(pcp, old, new) __this_cpu_cmpxchg_generic((pcp), (old), (new), 8)
+# endif
+# define __this_cpu_cmpxchg(pcp, old, new) __pcpu_size_call_return(__this_cpu_cmpxchg_, (old), (new))
+#endif
+
 /*
  * IRQ safe versions of the per cpu RMW operations. Note that these operations
  * are *not* safe against modification of the same variable from another
@@ -709,4 +781,31 @@ do {									\
 # define irqsafe_cpu_xor(pcp, val) __pcpu_size_call(irqsafe_cpu_xor_, (val))
 #endif
 
+static inline unsigned long irqsafe_cpu_cmpxchg_generic(volatile void *ptr,
+		unsigned long old, unsigned long new, int size)
+{
+	unsigned long flags, prev;
+
+	local_irq_save(flags);
+	prev = __this_cpu_cmpxchg_generic(ptr, old, new, size);
+	local_irq_restore(flags);
+	return prev;
+}
+
+#ifndef irqsafe_cpu_cmpxchg
+# ifndef irqsafe_cpu_cmpxchg_1
+#  define irqsafe_cpu_cmpxchg_1(pcp, old, new) irqsafe_cpu_cmpxchg_generic(((pcp), (old), (new), 1)
+# endif
+# ifndef irqsafe_cpu_cmpxchg_2
+#  define irqsafe_cpu_cmpxchg_2(pcp, old, new) irqsafe_cpu_cmpxchg_generic(((pcp), (old), (new), 2)
+# endif
+# ifndef irqsafe_cpu_cmpxchg_4
+#  define irqsafe_cpu_cmpxchg_4(pcp, old, new) irqsafe_cpu_cmpxchg_generic(((pcp), (old), (new), 4)
+# endif
+# ifndef irqsafe_cpu_cmpxchg_8
+#  define irqsafe_cpu_cmpxchg_8(pcp, old, new) irqsafe_cpu_cmpxchg_generic(((pcp), (old), (new), 8)
+# endif
+# define irqsafe_cpu_cmpxchg(pcp, old, new) __pcpu_size_call_return(irqsafe_cpu_cmpxchg_, (old), (new))
+#endif
+
 #endif /* __LINUX_PERCPU_H */
#27 lore raw

Re: [this_cpu_xx V8 11/16] Generic support for this_cpu_cmpxchg

Quoted 34 lines · * Christoph Lameter ([email protected]) wrote:

Provide support for this_cpu_cmpxchg.

---
 include/linux/percpu.h |   99 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 99 insertions(+)

Index: linux-2.6/include/linux/percpu.h
===================================================================

include/linux/percpu.h +18 -0
@@ -443,6 +443,62 @@ do {									\
 # define this_cpu_xor(pcp, val)		__pcpu_size_call(this_cpu_or_, (pcp), (val))
 #endif
 
+static inline unsigned long __this_cpu_cmpxchg_generic(volatile void *pptr,
+		unsigned long old, unsigned long new, int size)
+{
+	unsigned long prev;
+	volatile void * ptr = __this_cpu_ptr(pptr);
+
+	switch (size) {
+	case 1: prev = *(u8 *)ptr;
+		if (prev == old)
+			*(u8 *)ptr = (u8)new;
+		break;
+	case 2: prev = *(u16 *)ptr;
+		if (prev == old)
+			*(u16 *)ptr = (u16)new;
+		break;
+	case 4: prev = *(u32 *)ptr;
+		if (prev == old)
+			*(u32 *)ptr = (u32)new;

include/linux/percpu.h do { \ · line 473

Quoted 10 lines
include/linux/percpu.h +10 -0
+		break;
+	case 8: prev = *(u64 *)ptr;
+		if (prev == old)
+			*(u64 *)ptr = (u64)new;
+		break;
+	default:
+		__bad_size_call_parameter();
+	}
+	return prev;
+}

Hi Christoph,

I am a bit concerned about the "generic" version of this_cpu_cmpxchg. Given that what LTTng needs is basically an atomic, nmi-safe version of the primitive (on all architectures that have something close to a NMI), this means that it could not switch over to your primitives until we add the equivalent support we currently have with local_t to all architectures. The transition would be faster if we create an atomic_cpu_*() variant which would map to local_t operations in the initial version.

Or maybe have I missed something in your patchset that address this ?

Thanks,

Mathieu

Quoted 88 lines
include/linux/percpu.h +71 -0
+
+static inline unsigned long this_cpu_cmpxchg_generic(volatile void *ptr,
+		unsigned long old, unsigned long new, int size)
+{
+	unsigned long prev;
+
+	preempt_disable();
+	prev = __this_cpu_cmpxchg_generic(ptr, old, new, size);
+	preempt_enable();
+	return prev;
+}
+
+#ifndef this_cpu_cmpxchg
+# ifndef this_cpu_cmpxchg_1
+#  define this_cpu_cmpxchg_1(pcp, old, new) this_cpu_cmpxchg_generic((pcp), (old), (new), 1)
+# endif
+# ifndef this_cpu_cmpxchg_2
+#  define this_cpu_cmpxchg_2(pcp, old, new) this_cpu_cmpxchg_generic((pcp), (old), (new), 2)
+# endif
+# ifndef this_cpu_cmpxchg_4
+#  define this_cpu_cmpxchg_4(pcp, old, new) this_cpu_cmpxchg_generic((pcp), (old), (new), 4)
+# endif
+# ifndef this_cpu_cmpxchg_8
+#  define this_cpu_cmpxchg_8(pcp, old, new) this_cpu_cmpxchg_generic((pcp), (old), (new), 8)
+# endif
+# define this_cpu_cmpxchg(pcp, old, new) __pcpu_size_call_return(__this_cpu_cmpxchg_, (old), (new))
+#endif
+
 /*
  * Generic percpu operations that do not require preemption handling.
  * Either we do not care about races or the caller has the
@@ -594,6 +650,22 @@ do {									\
 # define __this_cpu_xor(pcp, val)	__pcpu_size_call(__this_cpu_xor_, (pcp), (val))
 #endif
 
+#ifndef __this_cpu_cmpxchg
+# ifndef __this_cpu_cmpxchg_1
+#  define __this_cpu_cmpxchg_1(pcp, old, new) __this_cpu_cmpxchg_generic((pcp), (old), (new), 1)
+# endif
+# ifndef __this_cpu_cmpxchg_2
+#  define __this_cpu_cmpxchg_2(pcp, old, new) __this_cpu_cmpxchg_generic((pcp), (old), (new), 2)
+# endif
+# ifndef __this_cpu_cmpxchg_4
+#  define __this_cpu_cmpxchg_4(pcp, old, new) __this_cpu_cmpxchg_generic((pcp), (old), (new), 4)
+# endif
+# ifndef __this_cpu_cmpxchg_8
+#  define __this_cpu_cmpxchg_8(pcp, old, new) __this_cpu_cmpxchg_generic((pcp), (old), (new), 8)
+# endif
+# define __this_cpu_cmpxchg(pcp, old, new) __pcpu_size_call_return(__this_cpu_cmpxchg_, (old), (new))
+#endif
+
 /*
  * IRQ safe versions of the per cpu RMW operations. Note that these operations
  * are *not* safe against modification of the same variable from another
@@ -709,4 +781,31 @@ do {									\
 # define irqsafe_cpu_xor(pcp, val) __pcpu_size_call(irqsafe_cpu_xor_, (val))
 #endif
 
+static inline unsigned long irqsafe_cpu_cmpxchg_generic(volatile void *ptr,
+		unsigned long old, unsigned long new, int size)
+{
+	unsigned long flags, prev;
+
+	local_irq_save(flags);
+	prev = __this_cpu_cmpxchg_generic(ptr, old, new, size);
+	local_irq_restore(flags);
+	return prev;
+}
+
+#ifndef irqsafe_cpu_cmpxchg
+# ifndef irqsafe_cpu_cmpxchg_1
+#  define irqsafe_cpu_cmpxchg_1(pcp, old, new) irqsafe_cpu_cmpxchg_generic(((pcp), (old), (new), 1)
+# endif
+# ifndef irqsafe_cpu_cmpxchg_2
+#  define irqsafe_cpu_cmpxchg_2(pcp, old, new) irqsafe_cpu_cmpxchg_generic(((pcp), (old), (new), 2)
+# endif
+# ifndef irqsafe_cpu_cmpxchg_4
+#  define irqsafe_cpu_cmpxchg_4(pcp, old, new) irqsafe_cpu_cmpxchg_generic(((pcp), (old), (new), 4)
+# endif
+# ifndef irqsafe_cpu_cmpxchg_8
+#  define irqsafe_cpu_cmpxchg_8(pcp, old, new) irqsafe_cpu_cmpxchg_generic(((pcp), (old), (new), 8)
+# endif
+# define irqsafe_cpu_cmpxchg(pcp, old, new) __pcpu_size_call_return(irqsafe_cpu_cmpxchg_, (old), (new))
+#endif
+
 #endif /* __LINUX_PERCPU_H */
Signature · 1 line
-- 
Signature · 2 lines
Mathieu Desnoyers
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68
#28 lore raw

Re: [this_cpu_xx V8 11/16] Generic support for this_cpu_cmpxchg

Quoted 10 lines · Mathieu Desnoyers

I am a bit concerned about the "generic" version of this_cpu_cmpxchg. Given that what LTTng needs is basically an atomic, nmi-safe version of the primitive (on all architectures that have something close to a NMI), this means that it could not switch over to your primitives until we add the equivalent support we currently have with local_t to all architectures. The transition would be faster if we create an atomic_cpu_*() variant which would map to local_t operations in the initial version.

Or maybe have I missed something in your patchset that address this ?

NMI safeness is not covered by this_cpu operations.

We could add nmi_safe_.... ops?

The atomic_cpu reference make me think that you want full (LOCK) semantics? Then use the regular atomic ops?

#29 lore raw

Re: [this_cpu_xx V8 11/16] Generic support for this_cpu_cmpxchg

Quoted 11 lines · * Christoph Lameter ([email protected]) wrote:
Quoted 9 lines · Mathieu Desnoyers

I am a bit concerned about the "generic" version of this_cpu_cmpxchg. Given that what LTTng needs is basically an atomic, nmi-safe version of the primitive (on all architectures that have something close to a NMI), this means that it could not switch over to your primitives until we add the equivalent support we currently have with local_t to all architectures. The transition would be faster if we create an atomic_cpu_*() variant which would map to local_t operations in the initial version.

Quoted 8 lines
Quoted 1 line

Or maybe have I missed something in your patchset that address this ?

NMI safeness is not covered by this_cpu operations.

We could add nmi_safe_.... ops?

The atomic_cpu reference make me think that you want full (LOCK) semantics? Then use the regular atomic ops?

nmi_safe would probably make sense here.

But given that we have to disable preemption to add precision in terms of trace clock timestamp, I wonder if we would really gain something considerable performance-wise.

I also thought about the design change this requires for the per-cpu buffer commit count pointer which would have to become a per-cpu pointer independent of the buffer structure, and I foresee a problem with Steven's irq off tracing which need to perform buffer exchanges while tracing is active. Basically, having only one top-level pointer for the buffer makes it possible to exchange it atomically, but if we have to have two separate pointers (one for per-cpu buffer, one for per-cpu commit count array), then we are stucked.

So given that per-cpu ops limits us in terms of data structure layout, I am less and less sure it's the best fit for ring buffers, especially if we don't gain much performance-wise.

Thanks,

Mathieu

Signature · 2 lines
Mathieu Desnoyers
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68
#30 lore raw

Re: [this_cpu_xx V8 11/16] Generic support for this_cpu_cmpxchg

Quoted 11 lines · Mathieu Desnoyers
Quoted 11 lines
Quoted 10 lines

I am a bit concerned about the "generic" version of this_cpu_cmpxchg. Given that what LTTng needs is basically an atomic, nmi-safe version of the primitive (on all architectures that have something close to a NMI), this means that it could not switch over to your primitives until we add the equivalent support we currently have with local_t to all architectures. The transition would be faster if we create an atomic_cpu_*() variant which would map to local_t operations in the initial version.

Or maybe have I missed something in your patchset that address this ?

Quoted 8 lines
Quoted 6 lines

NMI safeness is not covered by this_cpu operations.

We could add nmi_safe_.... ops?

The atomic_cpu reference make me think that you want full (LOCK) semantics? Then use the regular atomic ops?

nmi_safe would probably make sense here.

I am not sure how to implement fallback for nmi_safe operations though since there is no way of disabling NMIs.

Quoted 3 lines

But given that we have to disable preemption to add precision in terms of trace clock timestamp, I wonder if we would really gain something considerable performance-wise.

Not sure what exactly you attempt to do there.

Quoted 8 lines

I also thought about the design change this requires for the per-cpu buffer commit count pointer which would have to become a per-cpu pointer independent of the buffer structure, and I foresee a problem with Steven's irq off tracing which need to perform buffer exchanges while tracing is active. Basically, having only one top-level pointer for the buffer makes it possible to exchange it atomically, but if we have to have two separate pointers (one for per-cpu buffer, one for per-cpu commit count array), then we are stucked.

You just need to keep percpu pointers that are offsets into the percpu area. They can be relocated as needed to the processor specific addresses using the cpu ops.

Quoted 3 lines

So given that per-cpu ops limits us in terms of data structure layout, I am less and less sure it's the best fit for ring buffers, especially if we don't gain much performance-wise.

I dont understand how exactly the ring buffer logic works and what you are trying to do here.

The ringbuffers are per cpu structures right and you do not change cpus while performing operations on them? If not then the per cpu ops are not useful to you.

If you dont: How can you safely use the local_t operations for the ringbuffer logic?

#31 lore raw

Re: [this_cpu_xx V8 11/16] Generic support for this_cpu_cmpxchg

Quoted 48 lines · * Christoph Lameter ([email protected]) wrote:
Quoted 11 lines · Mathieu Desnoyers
Quoted 11 lines
Quoted 10 lines

I am a bit concerned about the "generic" version of this_cpu_cmpxchg. Given that what LTTng needs is basically an atomic, nmi-safe version of the primitive (on all architectures that have something close to a NMI), this means that it could not switch over to your primitives until we add the equivalent support we currently have with local_t to all architectures. The transition would be faster if we create an atomic_cpu_*() variant which would map to local_t operations in the initial version.

Or maybe have I missed something in your patchset that address this ?

Quoted 8 lines
Quoted 6 lines

NMI safeness is not covered by this_cpu operations.

We could add nmi_safe_.... ops?

The atomic_cpu reference make me think that you want full (LOCK) semantics? Then use the regular atomic ops?

nmi_safe would probably make sense here.

I am not sure how to implement fallback for nmi_safe operations though since there is no way of disabling NMIs.

Quoted 3 lines

But given that we have to disable preemption to add precision in terms of trace clock timestamp, I wonder if we would really gain something considerable performance-wise.

Not sure what exactly you attempt to do there.

Quoted 8 lines

I also thought about the design change this requires for the per-cpu buffer commit count pointer which would have to become a per-cpu pointer independent of the buffer structure, and I foresee a problem with Steven's irq off tracing which need to perform buffer exchanges while tracing is active. Basically, having only one top-level pointer for the buffer makes it possible to exchange it atomically, but if we have to have two separate pointers (one for per-cpu buffer, one for per-cpu commit count array), then we are stucked.

You just need to keep percpu pointers that are offsets into the percpu area. They can be relocated as needed to the processor specific addresses using the cpu ops.

Quoted 3 lines

So given that per-cpu ops limits us in terms of data structure layout, I am less and less sure it's the best fit for ring buffers, especially if we don't gain much performance-wise.

Quoted 6 lines

I dont understand how exactly the ring buffer logic works and what you are trying to do here.

The ringbuffers are per cpu structures right and you do not change cpus while performing operations on them? If not then the per cpu ops are not useful to you.

Trying to make a long story short:

This scheme where Steven moves buffers from one CPU to another, he only performs this operation when some other exclusion mechanism ensures that the buffer is not used for writing by the CPU when this move operation is done.

It is therefore correct, and needs local_t type to deal with the data structure hierarchy vs atomic exchange as I pointed in my email.

Mathieu

Quoted 4 lines

If you dont: How can you safely use the local_t operations for the ringbuffer logic?

Signature · 2 lines
Mathieu Desnoyers
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68
#32 lore raw

Re: [this_cpu_xx V8 11/16] Generic support for this_cpu_cmpxchg

Quoted 7 lines · Mathieu Desnoyers

This scheme where Steven moves buffers from one CPU to another, he only performs this operation when some other exclusion mechanism ensures that the buffer is not used for writing by the CPU when this move operation is done.

It is therefore correct, and needs local_t type to deal with the data structure hierarchy vs atomic exchange as I pointed in my email.

Yes this_cpu_xx does not seem to work for your scheme. Please review the patchset that I sent you instead.

#33 lore raw

PATCH v8 12/16 Add percpu cmpxchg operations

In reply to Christoph Lameter

These are needed for the ringbuffer logic.

---
 arch/x86/include/asm/percpu.h |   12 ++++++++++++
 1 file changed, 12 insertions(+)
arch/x86/include/asm/percpu.h +12 -0
Index: linux-2.6/arch/x86/include/asm/percpu.h
===================================================================
@@ -201,6 +201,9 @@ do {							\
 #define __this_cpu_xor_1(pcp, val)	percpu_to_op("xor", (pcp), val)
 #define __this_cpu_xor_2(pcp, val)	percpu_to_op("xor", (pcp), val)
 #define __this_cpu_xor_4(pcp, val)	percpu_to_op("xor", (pcp), val)
+#define __this_cpu_cmpxchg_1(pcp, old,new)	cmpxchg_local(__this_cpu_ptr(pcp), old, new)
+#define __this_cpu_cmpxchg_2(pcp, old,new)	cmpxchg_local(__this_cpu_ptr(pcp), old, new)
+#define __this_cpu_cmpxchg_4(pcp, old,new)	cmpxchg_local(__this_cpu_ptr(pcp), old, new)
 
 #define this_cpu_read_1(pcp)		percpu_from_op("mov", (pcp), "m"(pcp))
 #define this_cpu_read_2(pcp)		percpu_from_op("mov", (pcp), "m"(pcp))
@@ -226,6 +229,9 @@ do {							\
 #define this_cpu_xor_1(pcp, val)	percpu_to_op("xor", (pcp), val)
 #define this_cpu_xor_2(pcp, val)	percpu_to_op("xor", (pcp), val)
 #define this_cpu_xor_4(pcp, val)	percpu_to_op("xor", (pcp), val)
+#define this_cpu_cmpxchg_1(pcp, old,new)	cmpxchg_local(this_cpu_ptr(pcp), old, new)
+#define this_cpu_cmpxchg_2(pcp, old,new)	cmpxchg_local(this_cpu_ptr(pcp), old, new)
+#define this_cpu_cmpxchg_4(pcp, old,new)	cmpxchg_local(this_cpu_ptr(pcp), old, new)
 
 #define irqsafe_cpu_add_1(pcp, val)	percpu_to_op("add", (pcp), val)
 #define irqsafe_cpu_add_2(pcp, val)	percpu_to_op("add", (pcp), val)
@@ -245,6 +251,9 @@ do {							\
 #define irqsafe_cpu_xor_1(pcp, val)	percpu_to_op("xor", (pcp), val)
 #define irqsafe_cpu_xor_2(pcp, val)	percpu_to_op("xor", (pcp), val)
 #define irqsafe_cpu_xor_4(pcp, val)	percpu_to_op("xor", (pcp), val)
+#define irqsafe_cpu_cmpxchg_1(pcp, old,new)	cmpxchg_local(this_cpu_ptr(pcp), old, new)
+#define irqsafe_cpu_cmpxchg_2(pcp, old,new)	cmpxchg_local(this_cpu_ptr(pcp), old, new)
+#define irqsafe_cpu_cmpxchg_4(pcp, old,new)	cmpxchg_local(this_cpu_ptr(pcp), old, new)
 
 /*
  * Per cpu atomic 64 bit operations are only available under 64 bit.
@@ -259,6 +268,7 @@ do {							\
 #define __this_cpu_and_8(pcp, val)	percpu_to_op("and", (pcp), val)
 #define __this_cpu_or_8(pcp, val)	percpu_to_op("or", (pcp), val)
 #define __this_cpu_xor_8(pcp, val)	percpu_to_op("xor", (pcp), val)
+#define __this_cpu_cmpxchg_8(pcp, old,new)	cmpxchg_local(__this_cpu_ptr(pcp), old, new)
 
 #define this_cpu_read_8(pcp)		percpu_from_op("mov", (pcp), "m"(pcp))
 #define this_cpu_write_8(pcp, val)	percpu_to_op("mov", (pcp), val)
@@ -268,6 +278,7 @@ do {							\
 #define this_cpu_and_8(pcp, val)	percpu_to_op("and", (pcp), val)
 #define this_cpu_or_8(pcp, val)		percpu_to_op("or", (pcp), val)
 #define this_cpu_xor_8(pcp, val)	percpu_to_op("xor", (pcp), val)
+#define this_cpu_cmpxchg_8(pcp, old,new)	cmpxchg_local(this_cpu_ptr(pcp), old, new)
 
 #define irqsafe_cpu_add_8(pcp, val)	percpu_to_op("add", (pcp), val)
 #define irqsafe_cpu_inc_8(pcp)		percpu_var_op("inc", (pcp))
@@ -275,6 +286,7 @@ do {							\
 #define irqsafe_cpu_and_8(pcp, val)	percpu_to_op("and", (pcp), val)
 #define irqsafe_cpu_or_8(pcp, val)	percpu_to_op("or", (pcp), val)
 #define irqsafe_cpu_xor_8(pcp, val)	percpu_to_op("xor", (pcp), val)
+#define irqsafe_cpu_cmpxchg_8(pcp, old,new)	cmpxchg_local(this_cpu_ptr(pcp), old, new)
 
 #endif
 
#34 lore raw

PATCH v8 13/16 Generic support for this_cpu_xchg

In reply to Christoph Lameter

Provide generic this_cpu_xchg() and all fallbacks.

---
 include/linux/percpu.h |   95 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 95 insertions(+)
include/linux/percpu.h +95 -0
Index: linux-2.6/include/linux/percpu.h
===================================================================
@@ -499,6 +499,58 @@ static inline unsigned long this_cpu_cmp
 # define this_cpu_cmpxchg(pcp, old, new) __pcpu_size_call_return(__this_cpu_cmpxchg_, (old), (new))
 #endif
 
+static inline unsigned long __this_cpu_xchg_generic(volatile void *pptr,
+		unsigned long new, int size)
+{
+	unsigned long prev;
+	volatile void *ptr = __this_cpu_ptr(pptr);
+
+	switch (size) {
+	case 1: prev = *(u8 *)ptr;
+		*(u8 *)ptr = (u8)new;
+		break;
+	case 2: prev = *(u16 *)ptr;
+		*(u16 *)ptr = (u16)new;
+		break;
+	case 4: prev = *(u32 *)ptr;
+		*(u32 *)ptr = (u32)new;
+		break;
+	case 8: prev = *(u64 *)ptr;
+		*(u64 *)ptr = (u64)new;
+		break;
+	default:
+		__bad_size_call_parameter();
+	}
+	return prev;
+}
+
+static inline unsigned long this_cpu_xchg_generic(volatile void *ptr,
+		unsigned long new, int size)
+{
+	unsigned long prev;
+
+	preempt_disable();
+	prev = __this_cpu_xchg_generic(ptr, new, size);
+	preempt_enable();
+	return prev;
+}
+
+#ifndef this_cpu_xchg
+# ifndef this_cpu_xchg_1
+#  define this_cpu_xchg_1(pcp, new) this_cpu_xchg_generic((pcp), (new), 1)
+# endif
+# ifndef this_cpu_xchg_2
+#  define this_cpu_xchg_2(pcp, new) this_cpu_xchg_generic((pcp), (new), 2)
+# endif
+# ifndef this_cpu_xchg_4
+#  define this_cpu_xchg_4(pcp, new) this_cpu_xchg_generic((pcp), (new), 4)
+# endif
+# ifndef this_cpu_xchg_8
+#  define this_cpu_xchg_8(pcp, new) this_cpu_xchg_generic((pcp), (new), 8)
+# endif
+# define this_cpu_xchg(pcp, new) __pcpu_size_call_return(__this_cpu_xchg_, (new))
+#endif
+
 /*
  * Generic percpu operations that do not require preemption handling.
  * Either we do not care about races or the caller has the
@@ -666,6 +718,22 @@ do {									\
 # define __this_cpu_cmpxchg(pcp, old, new) __pcpu_size_call_return(__this_cpu_cmpxchg_, (old), (new))
 #endif
 
+#ifndef __this_cpu_xchg
+# ifndef __this_cpu_xchg_1
+#  define __this_cpu_xchg_1(pcp, new) __this_cpu_xchg_generic((pcp), (new), 1)
+# endif
+# ifndef __this_cpu_xchg_2
+#  define __this_cpu_xchg_2(pcp, new) __this_cpu_xchg_generic((pcp), (new), 2)
+# endif
+# ifndef __this_cpu_xchg_4
+#  define __this_cpu_xchg_4(pcp, new) __this_cpu_xchg_generic((pcp), (new), 4)
+# endif
+# ifndef __this_cpu_xchg_8
+#  define __this_cpu_xchg_8(pcp, new) __this_cpu_xchg_generic((pcp), (new), 8)
+# endif
+# define __this_cpu_xchg(pcp, new) __pcpu_size_call_return(__this_cpu_xchg_, (new))
+#endif
+
 /*
  * IRQ safe versions of the per cpu RMW operations. Note that these operations
  * are *not* safe against modification of the same variable from another
@@ -808,4 +876,31 @@ static inline unsigned long irqsafe_cpu_
 # define irqsafe_cpu_cmpxchg(pcp, old, new) __pcpu_size_call_return(irqsafe_cpu_cmpxchg_, (old), (new))
 #endif
 
+static inline unsigned long irqsafe_cpu_xchg_generic(volatile void *ptr,
+		unsigned long new, int size)
+{
+	unsigned long flags, prev;
+
+	local_irq_save(flags);
+	prev = __this_cpu_xchg_generic(ptr, new, size);
+	local_irq_restore(flags);
+	return prev;
+}
+
+#ifndef irqsafe_cpu_xchg
+# ifndef irqsafe_cpu_xchg_1
+#  define irqsafe_cpu_xchg_1(pcp, new) irqsafe_cpu_xchg_generic(((pcp), (new), 1)
+# endif
+# ifndef irqsafe_cpu_xchg_2
+#  define irqsafe_cpu_xchg_2(pcp, new) irqsafe_cpu_xchg_generic(((pcp), (new), 2)
+# endif
+# ifndef irqsafe_cpu_xchg_4
+#  define irqsafe_cpu_xchg_4(pcp, new) irqsafe_cpu_xchg_generic(((pcp), (new), 4)
+# endif
+# ifndef irqsafe_cpu_xchg_8
+#  define irqsafe_cpu_xchg_8(pcp, new) irqsafe_cpu_xchg_generic(((pcp), (new), 8)
+# endif
+# define irqsafe_cpu_xchg(pcp, new) __pcpu_size_call_return(irqsafe_cpu_xchg_, (new))
+#endif
+
 #endif /* __LINUX_PERCPU_H */
#35 lore raw

PATCH v8 14/16 x86 percpu xchg operation

In reply to Christoph Lameter

Xchg on x86 implies LOCK semantics and therefore is slow. Approximate it using this_cpu_cmpxchg().

---
 arch/x86/include/asm/percpu.h |   21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)
arch/x86/include/asm/percpu.h +21 -0
Index: linux-2.6/arch/x86/include/asm/percpu.h
===================================================================
@@ -156,6 +156,15 @@ do {							\
 	}						\
 })
 
+#define this_cpu_xchg_x86(var, new)						\
+({	__typeof(*var) __tmp_var,__tmp_old;				\
+	do {								\
+		__tmp_old = __this_cpu_read(var);			\
+		__tmp_var = __this_cpu_cmpxchg(var, __tmp_old, new);	\
+	} while (__tmp_var != __tmp_old);				\
+	__tmp_old;							\
+})
+
 /*
  * percpu_read() makes gcc load the percpu variable every time it is
  * accessed while percpu_read_stable() allows the value to be cached.
@@ -204,6 +213,9 @@ do {							\
 #define __this_cpu_cmpxchg_1(pcp, old,new)	cmpxchg_local(__this_cpu_ptr(pcp), old, new)
 #define __this_cpu_cmpxchg_2(pcp, old,new)	cmpxchg_local(__this_cpu_ptr(pcp), old, new)
 #define __this_cpu_cmpxchg_4(pcp, old,new)	cmpxchg_local(__this_cpu_ptr(pcp), old, new)
+#define __this_cpu_xchg_1(pcp, new)	this_cpu_xchg_x86((pcp), new)
+#define __this_cpu_xchg_2(pcp, new)	this_cpu_xchg_x86((pcp), new)
+#define __this_cpu_xchg_4(pcp, new)	this_cpu_xchg_x86((pcp), new)
 
 #define this_cpu_read_1(pcp)		percpu_from_op("mov", (pcp), "m"(pcp))
 #define this_cpu_read_2(pcp)		percpu_from_op("mov", (pcp), "m"(pcp))
@@ -232,6 +244,9 @@ do {							\
 #define this_cpu_cmpxchg_1(pcp, old,new)	cmpxchg_local(this_cpu_ptr(pcp), old, new)
 #define this_cpu_cmpxchg_2(pcp, old,new)	cmpxchg_local(this_cpu_ptr(pcp), old, new)
 #define this_cpu_cmpxchg_4(pcp, old,new)	cmpxchg_local(this_cpu_ptr(pcp), old, new)
+#define this_cpu_xchg_1(pcp, new)	this_cpu_xchg_x86((pcp), new)
+#define this_cpu_xchg_2(pcp, new)	this_cpu_xchg_x86((pcp), new)
+#define this_cpu_xchg_4(pcp, new)	this_cpu_xchg_x86((pcp), new)
 
 #define irqsafe_cpu_add_1(pcp, val)	percpu_to_op("add", (pcp), val)
 #define irqsafe_cpu_add_2(pcp, val)	percpu_to_op("add", (pcp), val)
@@ -254,6 +269,9 @@ do {							\
 #define irqsafe_cpu_cmpxchg_1(pcp, old,new)	cmpxchg_local(this_cpu_ptr(pcp), old, new)
 #define irqsafe_cpu_cmpxchg_2(pcp, old,new)	cmpxchg_local(this_cpu_ptr(pcp), old, new)
 #define irqsafe_cpu_cmpxchg_4(pcp, old,new)	cmpxchg_local(this_cpu_ptr(pcp), old, new)
+#define irqsafe_cpu_xchg_1(pcp, new)	this_cpu_xchg_x86((pcp), new)
+#define irqsafe_cpu_xchg_2(pcp, new)	this_cpu_xchg_x86((pcp), new)
+#define irqsafe_cpu_xchg_4(pcp, new)	this_cpu_xchg_x86((pcp), new)
 
 /*
  * Per cpu atomic 64 bit operations are only available under 64 bit.
@@ -269,6 +287,7 @@ do {							\
 #define __this_cpu_or_8(pcp, val)	percpu_to_op("or", (pcp), val)
 #define __this_cpu_xor_8(pcp, val)	percpu_to_op("xor", (pcp), val)
 #define __this_cpu_cmpxchg_8(pcp, old,new)	cmpxchg_local(__this_cpu_ptr(pcp), old, new)
+#define __this_cpu_xchg_8(pcp, new)	this_cpu_xchg_x86((pcp), new)
 
 #define this_cpu_read_8(pcp)		percpu_from_op("mov", (pcp), "m"(pcp))
 #define this_cpu_write_8(pcp, val)	percpu_to_op("mov", (pcp), val)
@@ -279,6 +298,7 @@ do {							\
 #define this_cpu_or_8(pcp, val)		percpu_to_op("or", (pcp), val)
 #define this_cpu_xor_8(pcp, val)	percpu_to_op("xor", (pcp), val)
 #define this_cpu_cmpxchg_8(pcp, old,new)	cmpxchg_local(this_cpu_ptr(pcp), old, new)
+#define this_cpu_xchg_8(pcp, new)	this_cpu_xchg_x86((pcp), new)
 
 #define irqsafe_cpu_add_8(pcp, val)	percpu_to_op("add", (pcp), val)
 #define irqsafe_cpu_inc_8(pcp)		percpu_var_op("inc", (pcp))
@@ -287,6 +307,7 @@ do {							\
 #define irqsafe_cpu_or_8(pcp, val)	percpu_to_op("or", (pcp), val)
 #define irqsafe_cpu_xor_8(pcp, val)	percpu_to_op("xor", (pcp), val)
 #define irqsafe_cpu_cmpxchg_8(pcp, old,new)	cmpxchg_local(this_cpu_ptr(pcp), old, new)
+#define irqsafe_cpu_xchg_8(pcp, new)	this_cpu_xchg_x86((pcp), new)
 
 #endif
 
#36 lore raw

PATCH v8 15/16 Generic support for this_cpu_add_return()

In reply to Christoph Lameter

This allows core code to use this_cpu_add_return() Provides all fallback scenarios.

---
 include/linux/percpu.h |   90 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 90 insertions(+)
include/linux/percpu.h +90 -0
Index: linux-2.6/include/linux/percpu.h
===================================================================
@@ -551,6 +551,53 @@ static inline unsigned long this_cpu_xch
 # define this_cpu_xchg(pcp, new) __pcpu_size_call_return(__this_cpu_xchg_, (new))
 #endif
 
+static inline unsigned long __this_cpu_add_return_generic(volatile void *ptr,
+		unsigned long val, int size)
+{
+	unsigned long result;
+
+	switch (size) {
+	case 1: result = (*__this_cpu_ptr((u8 *)ptr) += val);
+		break;
+	case 2: result = (*__this_cpu_ptr((u16 *)ptr) += val);
+		break;
+	case 4: result = (*__this_cpu_ptr((u32 *)ptr) += val);
+		break;
+	case 8: result = (*__this_cpu_ptr((u64 *)ptr) += val);
+		break;
+	default:
+		__bad_size_call_parameter();
+	}
+	return result;
+}
+
+static inline unsigned long this_cpu_add_return_generic(volatile void *ptr,
+		unsigned long val, int size)
+{
+	unsigned long result;
+
+	preempt_disable();
+	result = __this_cpu_add_return_generic(ptr, val, size);
+	preempt_enable();
+	return result;
+}
+
+#ifndef this_cpu_add_return
+# ifndef this_cpu_add_return_1
+#  define this_cpu_add_return_1(pcp, val) this_cpu_add_return_generic((pcp), (val), 1)
+# endif
+# ifndef this_cpu_add_return_2
+#  define this_cpu_add_return_2(pcp, val) this_cpu_add_return_generic((pcp), (val), 2)
+# endif
+# ifndef this_cpu_add_return_4
+#  define this_cpu_add_return_4(pcp, val) this_cpu_add_return_generic((pcp), (val), 4)
+# endif
+# ifndef this_cpu_add_return_8
+#  define this_cpu_add_return_8(pcp, val) this_cpu_add_return_generic((pcp), (val), 8)
+# endif
+# define this_cpu_add_return(pcp, val) __pcpu_size_call_return(__this_cpu_add_return_, (val))
+#endif
+
 /*
  * Generic percpu operations that do not require preemption handling.
  * Either we do not care about races or the caller has the
@@ -734,6 +781,22 @@ do {									\
 # define __this_cpu_xchg(pcp, new) __pcpu_size_call_return(__this_cpu_xchg_, (new))
 #endif
 
+#ifndef __this_cpu_add_return
+# ifndef __this_cpu_add_return_1
+#  define __this_cpu_add_return_1(pcp, val) __this_cpu_add_return_generic((pcp), (val), 1)
+# endif
+# ifndef __this_cpu_add_return_2
+#  define __this_cpu_add_return_2(pcp, val) __this_cpu_add_return_generic((pcp), (val), 2)
+# endif
+# ifndef __this_cpu_add_return_4
+#  define __this_cpu_add_return_4(pcp, val) __this_cpu_add_return_generic((pcp), (val), 4)
+# endif
+# ifndef __this_cpu_add_return_8
+#  define __this_cpu_add_return_8(pcp, val) __this_cpu_add_return_generic((pcp), (val), 8)
+# endif
+# define __this_cpu_add_return(pcp, val) __pcpu_size_call_return(__this_cpu_add_return_, (val))
+#endif
+
 /*
  * IRQ safe versions of the per cpu RMW operations. Note that these operations
  * are *not* safe against modification of the same variable from another
@@ -903,4 +966,31 @@ static inline unsigned long irqsafe_cpu_
 # define irqsafe_cpu_xchg(pcp, new) __pcpu_size_call_return(irqsafe_cpu_xchg_, (new))
 #endif
 
+static inline unsigned long irqsafe_cpu_add_return_generic(volatile void *ptr,
+		unsigned long val, int size)
+{
+	unsigned long flags, result;
+
+	local_irq_save(flags);
+	result = __this_cpu_add_return_generic(ptr, val, size);
+	local_irq_restore(flags);
+	return result;
+}
+
+#ifndef irqsafe_cpu_add_return
+# ifndef irqsafe_cpu_add_return_1
+#  define irqsafe_cpu_add_return_1(pcp, val) irqsafe_cpu_add_return_generic(((pcp), (val), 1)
+# endif
+# ifndef irqsafe_cpu_add_return_2
+#  define irqsafe_cpu_add_return_2(pcp, val) irqsafe_cpu_add_return_generic(((pcp), (val), 2)
+# endif
+# ifndef irqsafe_cpu_add_return_4
+#  define irqsafe_cpu_add_return_4(pcp, val) irqsafe_cpu_add_return_generic(((pcp), (val), 4)
+# endif
+# ifndef irqsafe_cpu_add_return_8
+#  define irqsafe_cpu_add_return_8(pcp, val) irqsafe_cpu_add_return_generic(((pcp), (val), 8)
+# endif
+# define irqsafe_cpu_add_return(pcp, val) __pcpu_size_call_return(irqsafe_cpu_add_return_, (val))
+#endif
+
 #endif /* __LINUX_PERCPU_H */
#37 lore raw

PATCH v8 16/16 x86 support for this_cpu_add_return

In reply to Christoph Lameter

x86 had the xadd instruction that can be use to have a percpu atomic increment and return instruction.

---
 arch/x86/include/asm/percpu.h |   35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)
arch/x86/include/asm/percpu.h +35 -0
Index: linux-2.6/arch/x86/include/asm/percpu.h
===================================================================
@@ -165,6 +165,35 @@ do {							\
 	__tmp_old;							\
 })
 
+#define this_cpu_add_return_x86(var, val)		\
+({							\
+	typeof(var) pfo_ret__;				\
+	switch (sizeof(var)) {				\
+	case 1:						\
+		asm("xaddb %0,"__percpu_arg(1)		\
+		    : "+r" (val), "+m" (var)		\
+		    : "qi" ((pto_T__)(val)));		\
+		break;					\
+	case 2:						\
+		asm("xaddw %0,"__percpu_arg(1)		\
+		    : "+r" (val), "+m" (var)		\
+		    : "=r" (pfo_ret__)			\
+		break;					\
+	case 4:						\
+		asm("xaddl %0,"__percpu_arg(1)		\
+		    : "+r" (val), "+m" (var)		\
+		    : "=r" (pfo_ret__)			\
+		break;					\
+	case 8:						\
+		asm("xaddq %0,"__percpu_arg(1)		\
+		    : "+r" (val), "+m" (var)		\
+		    : "re" ((pto_T__)(val)));		\
+		break;					\
+	default: __bad_percpu_size();			\
+	}						\
+	pfo_ret__;					\
+})
+
 /*
  * percpu_read() makes gcc load the percpu variable every time it is
  * accessed while percpu_read_stable() allows the value to be cached.
@@ -216,6 +245,9 @@ do {							\
 #define __this_cpu_xchg_1(pcp, new)	this_cpu_xchg_x86((pcp), new)
 #define __this_cpu_xchg_2(pcp, new)	this_cpu_xchg_x86((pcp), new)
 #define __this_cpu_xchg_4(pcp, new)	this_cpu_xchg_x86((pcp), new)
+#define __this_cpu_add_return_1(pcp, val)	this_cpu_add_return_x86((pcp), val)
+#define __this_cpu_add_return_2(pcp, val)	this_cpu_add_return_x86((pcp), val)
+#define __this_cpu_add_return_4(pcp, val)	this_cpu_add_return_x86((pcp), val)
 
 #define this_cpu_read_1(pcp)		percpu_from_op("mov", (pcp), "m"(pcp))
 #define this_cpu_read_2(pcp)		percpu_from_op("mov", (pcp), "m"(pcp))
@@ -272,6 +304,9 @@ do {							\
 #define irqsafe_cpu_xchg_1(pcp, new)	this_cpu_xchg_x86((pcp), new)
 #define irqsafe_cpu_xchg_2(pcp, new)	this_cpu_xchg_x86((pcp), new)
 #define irqsafe_cpu_xchg_4(pcp, new)	this_cpu_xchg_x86((pcp), new)
+#define irqsafe_cpu_add_return_1(pcp, val)	this_cpu_add_return_x86((pcp), val)
+#define irqsafe_cpu_add_return_2(pcp, val)	this_cpu_add_return_x86((pcp), val)
+#define irqsafe_cpu_add_return_4(pcp, val)	this_cpu_add_return_x86((pcp), val)
 
 /*
  * Per cpu atomic 64 bit operations are only available under 64 bit.