Per cpu atomics in core allocators, cleanup and more this_cpu_ops
Subsystems (14)
v8 cover Per cpu atomics in core allocators, cleanup and more this_cpu_ops
Leftovers from the earlier patchset rediffed to 2.6.33-rc1. Mostly applications of per cpu counters to core components.
After this patchset there will be only one user of local_t left: Mathieu's trace ringbuffer.
I added some patches that define additional this_cpu ops in order to help Mathieu with making the trace ringbuffer use this_cpu ops. These have barely been tested (boots fine on 32 and 64 bit but there is no user of these operations). RFC state.
V7->V8
- Fix issue in slub patch
- Fix issue in modules patch
- Rediff page allocator patch
- Provide new this_cpu ops needed for ringbuffer [RFC state]
V6->V7
- Drop patches merged in 2.6.33 merge cycle
- Drop risky slub patches
V5->V6:
- Drop patches merged by Tejun.
- Drop irqless slub fastpath for now.
- Patches against Tejun percpu for-next branch.
V4->V5:
- Avoid setup_per_cpu_area() modifications and fold the remainder of the patch into the page allocator patch.
- Irq disable / per cpu ptr fixes for page allocator patch.
V3->V4:
- Fix various macro definitions.
- Provide experimental percpu based fastpath that does not disable interrupts for SLUB.
V2->V3:
- Available via git tree against latest upstream from
git://git.kernel.org/pub/scm/linux/kernel/git/christoph/percpu.git linus
- Rework SLUB per cpu operations. Get rid of dynamic DMA slab creation for CONFIG_ZONE_DMA
- Create fallback framework so that 64 bit ops on 32 bit platforms can fallback to the use of preempt or interrupt disable. 64 bit platforms can use 64 bit atomic per cpu ops.
V1->V2:
- Various minor fixes
- Add SLUB conversion
- Add Page allocator conversion
- Patch against the git tree of today
PATCH v8 1/16 this_cpu_ops: page allocator conversion
Use the per cpu allocator functionality to avoid per cpu arrays in struct zone.
This drastically reduces the size of struct zone for systems with large amounts of processors and allows placement of critical variables of struct zone in one cacheline even on very large systems.
Another effect is that the pagesets of one processor are placed near one another. If multiple pagesets from different zones fit into one cacheline then additional cacheline fetches can be avoided on the hot paths when allocating memory from multiple zones.
Bootstrap becomes simpler if we use the same scheme for UP, SMP, NUMA. #ifdefs are reduced and we can drop the zone_pcp macro.
Hotplug handling is also simplified since cpu alloc can bring up and shut down cpu areas for a specific cpu as a whole. So there is no need to allocate or free individual pagesets.
V7-V8:
- Explain chicken egg dilemmna with percpu allocator.
V4-V5:
- Fix up cases where per_cpu_ptr is called before irq disable
- Integrate the bootstrap logic that was separate before.
--- include/linux/mm.h | 4 include/linux/mmzone.h | 12 -- mm/page_alloc.c | 197 ++++++++++++++++++++----------------------------- mm/vmstat.c | 14 +-- 4 files changed, 92 insertions(+), 135 deletions(-)
include/linux/mm.h +0 -4
@@ -1076,11 +1076,7 @@ extern void si_meminfo(struct sysinfo *extern void si_meminfo_node (struct sysinfo *val,int nid);extern int after_bootmem; -#ifdef CONFIG_NUMA extern void setup_per_cpu_pageset (void ); -#else -static inline void setup_per_cpu_pageset (void ) {} -#endif extern void zone_pcp_update (struct zone *zone);
include/linux/mmzone.h +2 -10
@@ -184,13 +184,7 @@ struct per_cpu_pageset {s8 stat_threshold;s8 vm_stat_diff[NR_VM_ZONE_STAT_ITEMS ]; #endif -}____cacheline_aligned_in_smp ; - -#ifdef CONFIG_NUMA -#define zone_pcp (__z, __cpu) ((__z)->pageset[(__cpu)]) -#else -#define zone_pcp (__z, __cpu) (&(__z)->pageset[(__cpu)]) -#endif +};#endif /* !__GENERATING_BOUNDS.H */
@@ -306,10 +300,8 @@ struct zone {*/ unsigned long min_unmapped_pages;unsigned long min_slab_pages; -struct per_cpu_pageset *pageset[NR_CPUS ]; -#else -struct per_cpu_pageset pageset[NR_CPUS ];#endif +struct per_cpu_pageset *pageset;/* * free areas of different sizes */
mm/page_alloc.c +82 -115
320 lines not on this page · raw message
mm/vmstat.c +8 -6
@@ -139,7 +139,8 @@ static void refresh_zone_stat_thresholds threshold =calculate_threshold (zone);for_each_online_cpu (cpu) -zone_pcp (zone, cpu)->stat_threshold = threshold; +per_cpu_ptr (zone->pageset, cpu)->stat_threshold + = threshold; } }
@@ -149,7 +150,8 @@ static void refresh_zone_stat_thresholdsvoid __mod_zone_page_state (struct zone *zone,enum zone_stat_item item,int delta) { -struct per_cpu_pageset *pcp =zone_pcp (zone,smp_processor_id ()); +struct per_cpu_pageset *pcp =this_cpu_ptr (zone->pageset); +s8 *p = pcp->vm_stat_diff + item;long x;
@@ -202,7 +204,7 @@ EXPORT_SYMBOL(mod_zone_page_state);*/ void __inc_zone_state (struct zone *zone,enum zone_stat_item item) { -struct per_cpu_pageset *pcp =zone_pcp (zone,smp_processor_id ()); +struct per_cpu_pageset *pcp =this_cpu_ptr (zone->pageset);s8 *p = pcp->vm_stat_diff + item; (*p)++;
@@ -223,7 +225,7 @@ EXPORT_SYMBOL(__inc_zone_page_state);void __dec_zone_state (struct zone *zone,enum zone_stat_item item) { -struct per_cpu_pageset *pcp =zone_pcp (zone,smp_processor_id ()); +struct per_cpu_pageset *pcp =this_cpu_ptr (zone->pageset);s8 *p = pcp->vm_stat_diff + item; (*p)--;
@@ -300,7 +302,7 @@ void refresh_cpu_vm_stats(int cpu)for_each_populated_zone (zone) {struct per_cpu_pageset *p; - p =zone_pcp (zone, cpu); + p =per_cpu_ptr (zone->pageset, cpu);for (i =0 ; i <NR_VM_ZONE_STAT_ITEMS ; i++)if (p->vm_stat_diff[i]) {
@@ -741,7 +743,7 @@ static void zoneinfo_show_print(struct sfor_each_online_cpu (i) {struct per_cpu_pageset *pageset; - pageset =zone_pcp (zone, i); + pageset =per_cpu_ptr (zone->pageset, i);seq_printf (m,"\n cpu: %i" "\n count: %i"
PATCH v8 2/16 this_cpu ops: Remove pageset_notifier
In reply to Christoph LameterRemove the pageset notifier since it only marks that a processor exists on a specific node. Move that code into the vmstat notifier.
--- mm/page_alloc.c | 27 --------------------------- mm/vmstat.c | 1 + 2 files changed, 1 insertion(+), 27 deletions(-)
mm/vmstat.c +1 -0
@@ -908,6 +908,7 @@ static int __cpuinit vmstat_cpuup_callbacase CPU_ONLINE :case CPU_ONLINE_FROZEN :start_cpu_timer (cpu); +node_set_state (cpu_to_node (cpu),N_CPU );break ;case CPU_DOWN_PREPARE :case CPU_DOWN_PREPARE_FROZEN :
mm/page_alloc.c +0 -27
@@ -3122,27 +3122,6 @@ static void setup_pagelist_highmark(stru pcp->batch =PAGE_SHIFT *8 ; } - -static int __cpuinit pageset_cpuup_callback (struct notifier_block *nfb, -unsigned long action, -void *hcpu) -{ -int cpu = (long )hcpu; - -switch (action) { -case CPU_UP_PREPARE : -case CPU_UP_PREPARE_FROZEN : -node_set_state (cpu_to_node (cpu),N_CPU ); -break ; -default : -break ; - } -return ret; -} - -static struct notifier_block __cpuinitdata pageset_notifier = - { &pageset_cpuup_callback,NULL ,0 }; -/* * Allocate per cpu pagesets and initialize them. * Before this call only boot pagesets were available.
@@ -3168,12 +3147,6 @@ void __init setup_per_cpu_pageset(void) percpu_pagelist_fraction)); } } -/* -* The boot cpu is always the first active. -* The boot node has a processor. -*/ -node_set_state (cpu_to_node (smp_processor_id ()),N_CPU ); -register_cpu_notifier (&pageset_notifier); }static noinline __init_refok
PATCH v8 3/16 Use this_cpu operations in slub
In reply to Christoph LameterUsing per cpu allocations removes the needs for the per cpu arrays in the kmem_cache struct. These could get quite big if we have to support systems with thousands of cpus. The use of this_cpu_xx operations results in:
1. The size of kmem_cache for SMP configuration shrinks since we will only need 1 pointer instead of NR_CPUS. The same pointer can be used by all processors. Reduces cache footprint of the allocator.
2. We can dynamically size kmem_cache according to the actual nodes in the system meaning less memory overhead for configurations that may potentially support up to 1k NUMA nodes / 4k cpus.
3. We can remove the diddle widdle with allocating and releasing of kmem_cache_cpu structures when bringing up and shutting down cpus. The cpu alloc logic will do it all for us. Removes some portions of the cpu hotplug functionality.
4. Fastpath performance increases since per cpu pointer lookups and address calculations are avoided.
V7-V8
- Convert missed get_cpu_slab() under CONFIG_SLUB_STATS
--- include/linux/slub_def.h | 6 - mm/slub.c | 202 +++++++++++------------------------------------ 2 files changed, 49 insertions(+), 159 deletions(-)
include/linux/slub_def.h +1 -5
@@ -69,6 +69,7 @@ struct kmem_cache_order_objects {* Slab cache management. */ struct kmem_cache { +struct kmem_cache_cpu *cpu_slab;/* Used for retriving partial slabs etc */ unsigned long flags;int size;/* The size of an object including meta data */
@@ -104,11 +105,6 @@ struct kmem_cache {int remote_node_defrag_ratio;struct kmem_cache_node *node[MAX_NUMNODES ];#endif -#ifdef CONFIG_SMP -struct kmem_cache_cpu *cpu_slab[NR_CPUS ]; -#else -struct kmem_cache_cpu cpu_slab; -#endif };/*
mm/slub.c +48 -154
347 lines not on this page · raw message
Re: [this_cpu_xx V8 03/16] Use this_cpu operations in slub
Quoted 17 lines · Christoph Lameter
Using per cpu allocations removes the needs for the per cpu arrays in the
kmem_cachestruct. These could get quite big if we have to support systems with thousands of cpus. The use ofthis_cpu_xxoperations results in:1. The size of
kmem_cachefor SMP configuration shrinks since we will only need 1 pointer instead ofNR_CPUS. The same pointer can be used by all processors. Reduces cache footprint of the allocator.2. We can dynamically size
kmem_cacheaccording to the actual nodes in the system meaning less memory overhead for configurations that may potentially support up to 1k NUMA nodes / 4k cpus.3. We can remove the diddle widdle with allocating and releasing of
kmem_cache_cpustructures when bringing up and shutting down cpus. The cpu alloc logic will do it all for us. Removes some portions of the cpu hotplug functionality.
Quoted 8 lines
4. Fastpath performance increases since per cpu pointer lookups and address calculations are avoided.
V7-V8
- Convert missedget_cpu_slab()underCONFIG_SLUB_STATSCc Pekka Enberg <[email protected]>Signed-off-by Christoph Lameter <[email protected]>
Patches 3-6 applied.
PATCH v8 4/16 SLUB: Get rid of dynamic DMA kmalloc cache allocation
In reply to Christoph LameterDynamic DMA kmalloc cache allocation is troublesome since the new percpu allocator does not support allocations in atomic contexts. Reserve some statically allocated kmalloc_cpu structures instead.
--- include/linux/slub_def.h | 19 +++++++++++-------- mm/slub.c | 24 ++++++++++-------------- 2 files changed, 21 insertions(+), 22 deletions(-)
include/linux/slub_def.h +11 -8
@@ -131,11 +131,21 @@ struct kmem_cache {#define SLUB_PAGE_SHIFT (PAGE_SHIFT + 2) +#ifdef CONFIG_ZONE_DMA +#define SLUB_DMA __GFP_DMA +/* Reserve extra caches for potential DMA use */ +#define KMALLOC_CACHES (2 * SLUB_PAGE_SHIFT - 6) +#else +/* Disable DMA functionality */ +#define SLUB_DMA (__force gfp_t)0 +#define KMALLOC_CACHES SLUB_PAGE_SHIFT +#endif +/* * We keep the general caches in an array of slab caches that are used for * 2^x bytes of allocations. */ -extern struct kmem_cache kmalloc_caches[SLUB_PAGE_SHIFT ]; +extern struct kmem_cache kmalloc_caches[KMALLOC_CACHES ];/* * Sorry that the following has to be that ugly but some versions of GCC
@@ -203,13 +213,6 @@ static __always_inline struct kmem_cachereturn &kmalloc_caches[index]; } -#ifdef CONFIG_ZONE_DMA -#define SLUB_DMA __GFP_DMA -#else -/* Disable DMA functionality */ -#define SLUB_DMA (__force gfp_t)0 -#endif -void *kmem_cache_alloc (struct kmem_cache *,gfp_t );void *__kmalloc (size_t size,gfp_t flags);
mm/slub.c +10 -14
@@ -2092,7 +2092,7 @@ static inline int alloc_kmem_cache_cpus( {int cpu; -if (s < kmalloc_caches +SLUB_PAGE_SHIFT && s >= kmalloc_caches) +if (s < kmalloc_caches +KMALLOC_CACHES && s >= kmalloc_caches)/* * Boot time creation of the kmalloc array. Use static per cpu data * since the per cpu allocator is not available yet.
@@ -2539,7 +2539,7 @@ EXPORT_SYMBOL(kmem_cache_destroy);* Kmalloc subsystem *******************************************************************/ -struct kmem_cache kmalloc_caches[SLUB_PAGE_SHIFT] __cacheline_aligned; +struct kmem_cache kmalloc_caches[KMALLOC_CACHES] __cacheline_aligned;EXPORT_SYMBOL (kmalloc_caches);static int __init setup_slub_min_order (char *str)
@@ -2629,6 +2629,7 @@ static noinline struct kmem_cache *dma_kchar *text;size_t realsize;unsigned long slabflags; +int i; s = kmalloc_caches_dma[index];if (s)
@@ -2649,18 +2650,13 @@ static noinline struct kmem_cache *dma_k text =kasprintf (flags & ~SLUB_DMA ,"kmalloc_dma-%d" , (unsigned int )realsize); -if (flags & __GFP_WAIT) - s =kmalloc (kmem_size, flags & ~SLUB_DMA ); -else { -int i; + s =NULL ; +for (i =0 ; i <KMALLOC_CACHES ; i++) +if (!kmalloc_caches[i].size) +break ; - s =NULL ; -for (i =0 ; i <SLUB_PAGE_SHIFT ; i++) -if (kmalloc_caches[i].size) { - s = kmalloc_caches + i; -break ; - } - } +BUG_ON (i >=KMALLOC_CACHES ); + s = kmalloc_caches + i;/* * Must defer sysfs creation to a workqueue because we don't know
@@ -2674,7 +2670,7 @@ static noinline struct kmem_cache *dma_kif (!s || !text || !kmem_cache_open (s, flags, text, realsize,ARCH_KMALLOC_MINALIGN , slabflags,NULL )) { -kfree (s); + s->size =0 ;kfree (text); goto unlock_out; }
PATCH v8 5/16 this_cpu: Remove slub kmem_cache fields
In reply to Christoph LameterRemove the fields in struct kmem_cache_cpu that were used to cache data from struct kmem_cache when they were in different cachelines. The cacheline that holds the per cpu array pointer now also holds these values. We can cut down the struct kmem_cache_cpu size to almost half.
The get_freepointer() and set_freepointer() functions that used to be only intended for the slow path now are also useful for the hot path since access to the size field does not require accessing an additional cacheline anymore. This results in consistent use of functions for setting the freepointer of objects throughout SLUB.
Also we initialize all possible kmem_cache_cpu structures when a slab is created. No need to initialize them when a processor or node comes online.
--- include/linux/slub_def.h | 2 - mm/slub.c | 76 ++++++++++------------------------------------- 2 files changed, 17 insertions(+), 61 deletions(-)
include/linux/slub_def.h +0 -2
@@ -38,8 +38,6 @@ struct kmem_cache_cpu {void **freelist;/* Pointer to first free per cpu object */ struct page *page;/* The slab from which we are allocating */ int node;/* The node of the page (or -1 for debug) */ -unsigned int offset;/* Freepointer offset (in word units) */ -unsigned int objsize;/* Size of an object (from kmem_cache) */ #ifdef CONFIG_SLUB_STATS unsigned stat[NR_SLUB_STAT_ITEMS ];#endif
mm/slub.c +17 -59
190 lines not on this page · raw message
PATCH v8 6/16 Make slub statistics use this_cpu_inc
In reply to Christoph Lameterthis_cpu_inc() translates into a single instruction on x86 and does not need any register. So use it in stat(). We also want to avoid the calculation of the per cpu kmem_cache_cpu structure pointer. So pass a kmem_cache pointer instead of a kmem_cache_cpu pointer.
--- mm/slub.c | 43 ++++++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 23 deletions(-)
mm/slub.c +20 -23
@@ -217,10 +217,10 @@ static inline void sysfs_slab_remove(str#endif -static inline void stat (struct kmem_cache_cpu *c,enum stat_item si) +static inline void stat (struct kmem_cache *s,enum stat_item si) {#ifdef CONFIG_SLUB_STATS - c->stat[si]++; +__this_cpu_inc (s->cpu_slab->stat[si]);#endif }
@@ -1108,7 +1108,7 @@ static struct page *allocate_slab(structif (!page)return NULL ; -stat (this_cpu_ptr (s->cpu_slab),ORDER_FALLBACK ); +stat (s,ORDER_FALLBACK ); } if (kmemcheck_enabled
@@ -1406,23 +1406,22 @@ static struct page *get_partial(struct kstatic void unfreeze_slab (struct kmem_cache *s,struct page *page,int tail) {struct kmem_cache_node *n =get_node (s,page_to_nid (page)); -struct kmem_cache_cpu *c =this_cpu_ptr (s->cpu_slab);__ClearPageSlubFrozen (page);if (page->inuse) {if (page->freelist) {add_partial (n, page, tail); -stat (c, tail ?DEACTIVATE_TO_TAIL :DEACTIVATE_TO_HEAD ); +stat (s, tail ?DEACTIVATE_TO_TAIL :DEACTIVATE_TO_HEAD ); }else { -stat (c,DEACTIVATE_FULL ); +stat (s,DEACTIVATE_FULL );if (SLABDEBUG &&PageSlubDebug (page) && (s->flags &SLAB_STORE_USER ))add_full (n, page); }slab_unlock (page); }else { -stat (c,DEACTIVATE_EMPTY ); +stat (s,DEACTIVATE_EMPTY );if (n->nr_partial < s->min_partial) {/* * Adding an empty slab to the partial slabs in order
@@ -1438,7 +1437,7 @@ static void unfreeze_slab(struct kmem_caslab_unlock (page); }else {slab_unlock (page); -stat (__this_cpu_ptr (s->cpu_slab),FREE_SLAB ); +stat (s,FREE_SLAB );discard_slab (s, page); } }
@@ -1453,7 +1452,7 @@ static void deactivate_slab(struct kmem_int tail =1 ;if (page->freelist) -stat (c,DEACTIVATE_REMOTE_FREES ); +stat (s,DEACTIVATE_REMOTE_FREES );/* * Merge cpu freelist into slab freelist. Typically we get here * because both freelists are empty. So this is unlikely
@@ -1479,7 +1478,7 @@ static void deactivate_slab(struct kmem_static inline void flush_slab (struct kmem_cache *s,struct kmem_cache_cpu *c) { -stat (c,CPUSLAB_FLUSH ); +stat (s,CPUSLAB_FLUSH );slab_lock (c->page);deactivate_slab (s, c); }
@@ -1619,7 +1618,7 @@ static void *__slab_alloc(struct kmem_caif (unlikely (!node_match (c, node))) goto another_slab; -stat (c,ALLOC_REFILL ); +stat (s,ALLOC_REFILL ); load_freelist: object = c->page->freelist;
@@ -1634,7 +1633,7 @@ load_freelist: c->node =page_to_nid (c->page); unlock_out:slab_unlock (c->page); -stat (c,ALLOC_SLOWPATH ); +stat (s,ALLOC_SLOWPATH );return object; another_slab:
@@ -1644,7 +1643,7 @@ new_slab: new =get_partial (s, gfpflags, node);if (new) { c->page = new; -stat (c,ALLOC_FROM_PARTIAL ); +stat (s,ALLOC_FROM_PARTIAL ); goto load_freelist; }
@@ -1658,7 +1657,7 @@ new_slab:if (new) { c =__this_cpu_ptr (s->cpu_slab); -stat (c,ALLOC_SLAB ); +stat (s,ALLOC_SLAB );if (c->page)flush_slab (s, c);slab_lock (new);
@@ -1713,7 +1712,7 @@ static __always_inline void *slab_alloc(else { c->freelist =get_freepointer (s, object); -stat (c,ALLOC_FASTPATH ); +stat (s,ALLOC_FASTPATH ); }local_irq_restore (flags);
@@ -1780,10 +1779,8 @@ static void __slab_free(struct kmem_cach {void *prior;void **object = (void *)x; -struct kmem_cache_cpu *c; - c =__this_cpu_ptr (s->cpu_slab); -stat (c,FREE_SLOWPATH ); +stat (s,FREE_SLOWPATH );slab_lock (page);if (unlikely (SLABDEBUG &&PageSlubDebug (page)))
@@ -1796,7 +1793,7 @@ checks_ok: page->inuse--;if (unlikely (PageSlubFrozen (page))) { -stat (c,FREE_FROZEN ); +stat (s,FREE_FROZEN ); goto out_unlock; }
@@ -1809,7 +1806,7 @@ checks_ok:*/ if (unlikely (!prior)) {add_partial (get_node (s,page_to_nid (page)), page,1 ); -stat (c,FREE_ADD_PARTIAL ); +stat (s,FREE_ADD_PARTIAL ); } out_unlock:
@@ -1822,10 +1819,10 @@ slab_empty:* Slab still on the partial list. */ remove_partial (s, page); -stat (c,FREE_REMOVE_PARTIAL ); +stat (s,FREE_REMOVE_PARTIAL ); }slab_unlock (page); -stat (c,FREE_SLAB ); +stat (s,FREE_SLAB );discard_slab (s, page);return ;
@@ -1863,7 +1860,7 @@ static __always_inline void slab_free(stif (likely (page == c->page && c->node >=0 )) {set_freepointer (s, object, c->freelist); c->freelist = object; -stat (c,FREE_FASTPATH ); +stat (s,FREE_FASTPATH ); }else __slab_free (s, page, x, addr);
PATCH v8 7/16 Module handling: Use this_cpu_xx to dynamically allocate counters
In reply to Christoph LameterUse cpu ops to deal with the per cpu data instead of a local_t. Reduces memory requirements, cache footprint and decreases cycle counts.
The this_cpu_xx operations are also used for !SMP mode. Otherwise we could not drop the use of __module_ref_addr() which would make per cpu data handling complicated. this_cpu_xx operations have their own fallback for !SMP.
The last hold out of users of local_t is the tracing ringbuffer after this patch has been applied.
--- include/linux/module.h | 38 ++++++++++++++------------------------ kernel/module.c | 30 ++++++++++++++++-------------- kernel/trace/ring_buffer.c | 1 + 3 files changed, 31 insertions(+), 38 deletions(-)
include/linux/module.h +14 -24
@@ -16,8 +16,7 @@#include <linux/kobject.h> #include <linux/moduleparam.h> #include <linux/tracepoint.h> - -#include <asm/local.h> +#include <linux/percpu.h> #include <asm/module.h> #include <trace/events/module.h>
@@ -363,11 +362,9 @@ struct module/* Destruction function. */ void (*exit)(void ); -#ifdef CONFIG_SMP -char *refptr; -#else -local_t ref; -#endif +struct module_ref { +int count; + } *refptr;#endif #ifdef CONFIG_CONSTRUCTORS
@@ -454,25 +451,16 @@ void __symbol_put(const char *symbol);#define symbol_put (x) __symbol_put(MODULE_SYMBOL_PREFIX #x)void symbol_put_addr (void *addr); -static inline local_t *__module_ref_addr (struct module *mod,int cpu) -{ -#ifdef CONFIG_SMP -return (local_t *) (mod->refptr +per_cpu_offset (cpu)); -#else -return &mod->ref; -#endif -} -/* Sometimes we know we already have a refcount, and it's easier not to handle the error case (which only happens with rmmod --wait). */ static inline void __module_get (struct module *module) {if (module) { -unsigned int cpu =get_cpu (); -local_inc (__module_ref_addr (module, cpu)); +preempt_disable (); +__this_cpu_inc (module->refptr->count);trace_module_get (module, _THIS_IP_, -local_read (__module_ref_addr (module, cpu))); -put_cpu (); +__this_cpu_read (module->refptr->count)); +preempt_enable (); } }
@@ -481,15 +469,17 @@ static inline int try_module_get(structint ret =1 ;if (module) { -unsigned int cpu =get_cpu (); +preempt_disable (); +if (likely (module_is_live (module))) { -local_inc (__module_ref_addr (module, cpu)); +__this_cpu_inc (module->refptr->count);trace_module_get (module, _THIS_IP_, -local_read (__module_ref_addr (module, cpu))); +__this_cpu_read (module->refptr->count)); }else ret =0 ; -put_cpu (); + +preempt_enable (); }return ret; }
kernel/module.c +16 -14
@@ -474,9 +474,10 @@ static void module_unload_init(struct moINIT_LIST_HEAD (&mod->modules_which_use_me);for_each_possible_cpu (cpu) -local_set (__module_ref_addr (mod, cpu),0 ); +per_cpu_ptr (mod->refptr, cpu)->count =0 ; +/* Hold reference count during initialization. */ -local_set (__module_ref_addr (mod,raw_smp_processor_id ()),1 ); +__this_cpu_write (mod->refptr->count,1 );/* Backwards compatibility macros put refcount during init. */ mod->waiter = current; }
@@ -555,6 +556,7 @@ static void module_unload_free(struct mokfree (use);sysfs_remove_link (i->holders_dir, mod->name);/* There can be at most one match. */ +free_percpu (i->refptr);break ; } }
@@ -619,7 +621,7 @@ unsigned int module_refcount(struct moduint cpu;for_each_possible_cpu (cpu) - total +=local_read (__module_ref_addr (mod, cpu)); + total +=per_cpu_ptr (mod->refptr, cpu)->count;return total; }EXPORT_SYMBOL (module_refcount);
@@ -796,14 +798,15 @@ static struct module_attribute refcnt =void module_put (struct module *module) {if (module) { -unsigned int cpu =get_cpu (); -local_dec (__module_ref_addr (module, cpu)); +preempt_disable (); +__this_cpu_dec (module->refptr->count); +trace_module_put (module, _RET_IP_, -local_read (__module_ref_addr (module, cpu))); +__this_cpu_read (module->refptr->count));/* Maybe they're waiting for us to drop reference? */ if (unlikely (!module_is_live (module)))wake_up_process (module->waiter); -put_cpu (); +preempt_enable (); } }EXPORT_SYMBOL (module_put);
@@ -1394,9 +1397,9 @@ static void free_module(struct module *mkfree (mod->args);if (mod->percpu)percpu_modfree (mod->percpu); -#if defined(CONFIG_MODULE_UNLOAD ) && defined(CONFIG_SMP ) +#if defined(CONFIG_MODULE_UNLOAD )if (mod->refptr) -percpu_modfree (mod->refptr); +free_percpu (mod->refptr);#endif /* Free lock-classes: */ lockdep_free_key_range (mod->module_core, mod->core_size);
@@ -2159,9 +2162,8 @@ static noinline struct module *load_modu mod = (void *)sechdrs[modindex].sh_addr;kmemleak_load_module (mod, hdr, sechdrs, secstrings); -#if defined(CONFIG_MODULE_UNLOAD ) && defined(CONFIG_SMP ) - mod->refptr =percpu_modalloc (sizeof (local_t), __alignof__(local_t ), - mod->name); +#if defined(CONFIG_MODULE_UNLOAD ) + mod->refptr = alloc_percpu(struct module_ref);if (!mod->refptr) { err = -ENOMEM ; goto free_init;
@@ -2393,8 +2395,8 @@ static noinline struct module *load_modukobject_put (&mod->mkobj.kobj); free_unload:module_unload_free (mod); -#if defined(CONFIG_MODULE_UNLOAD ) && defined(CONFIG_SMP ) -percpu_modfree (mod->refptr); +#if defined(CONFIG_MODULE_UNLOAD ) +free_percpu (mod->refptr); free_init:#endif module_free (mod, mod->module_init);
kernel/trace/ring_buffer.c +1 -0
@@ -12,6 +12,7 @@#include <linux/hardirq.h> #include <linux/kmemcheck.h> #include <linux/module.h> +#include <asm/local.h> #include <linux/percpu.h> #include <linux/mutex.h> #include <linux/init.h>
Tejun Heo
maintainerRe: [this_cpu_xx V8 07/16] Module handling: Use this_cpu_xx to dynamically allocate counters
Hello,
Quoted 11 lines · Christoph Lameter
Use cpu ops to deal with the per cpu data instead of a
local_t. Reduces memory requirements, cache footprint and decreases cycle counts.The
this_cpu_xxoperations are also used for !SMP mode. Otherwise we could not drop the use of__module_ref_addr()which would make per cpu data handling complicated.this_cpu_xxoperations have their own fallback for !SMP.The last hold out of users of
local_tis the tracing ringbuffer after this patch has been applied.Signed-off-by Christoph Lameter <[email protected]>
Please keep Rusty Russell cc'd on module changes.
kernel/trace/ring_buffer.c line 15
Quoted 9 lines
Index:
linux-2.6/kernel/trace/ring_buffer.c
===================================================================kernel/trace/ring_buffer.c +1
-0@@ -12,6 +12,7 @@#include <linux/hardirq.h> #include <linux/kmemcheck.h> #include <linux/module.h> +#include <asm/local.h>
This doesn't belong to this patch, right? I stripped this part out, added Cc: to Rusty and applied 1, 2, 7 and 8 to percpu branch. I'll post the updated patch here.
Thanks.
Signature · 1 line
tejun
Tejun Heo
maintainerRe: [this_cpu_xx V8 07/16] Module handling: Use this_cpu_xx to dynamically allocate counters
Here's the version I committed to percpu branch.
Thanks.
Quoted 1 line
From
8af47ddd01364ae3c663c0bc92415c06fe887ba1Mon Sep 17 00:00:00 2001
From: Christoph Lameter <[email protected]>
Date: Fri, 18 Dec 2009 16:26:24 -0600
Subject: [PATCH] module: Use this_cpu_xx to dynamically allocate counters
Use cpu ops to deal with the per cpu data instead of a local_t. Reduces memory requirements, cache footprint and decreases cycle counts.
The this_cpu_xx operations are also used for !SMP mode. Otherwise we could not drop the use of __module_ref_addr() which would make per cpu data handling complicated. this_cpu_xx operations have their own fallback for !SMP.
The last hold out of users of local_t is the tracing ringbuffer after this patch has been applied.
--- include/linux/module.h | 38 ++++++++++++++------------------------ kernel/module.c | 30 ++++++++++++++++-------------- 2 files changed, 30 insertions(+), 38 deletions(-)
include/linux/module.h +14 -24
@@ -16,8 +16,7 @@#include <linux/kobject.h> #include <linux/moduleparam.h> #include <linux/tracepoint.h> - -#include <asm/local.h> +#include <linux/percpu.h> #include <asm/module.h> #include <trace/events/module.h>
@@ -361,11 +360,9 @@ struct module/* Destruction function. */ void (*exit)(void ); -#ifdef CONFIG_SMP -char *refptr; -#else -local_t ref; -#endif +struct module_ref { +int count; + } *refptr;#endif #ifdef CONFIG_CONSTRUCTORS
@@ -452,25 +449,16 @@ void __symbol_put(const char *symbol);#define symbol_put (x) __symbol_put(MODULE_SYMBOL_PREFIX #x)void symbol_put_addr (void *addr); -static inline local_t *__module_ref_addr (struct module *mod,int cpu) -{ -#ifdef CONFIG_SMP -return (local_t *) (mod->refptr +per_cpu_offset (cpu)); -#else -return &mod->ref; -#endif -} -/* Sometimes we know we already have a refcount, and it's easier not to handle the error case (which only happens with rmmod --wait). */ static inline void __module_get (struct module *module) {if (module) { -unsigned int cpu =get_cpu (); -local_inc (__module_ref_addr (module, cpu)); +preempt_disable (); +__this_cpu_inc (module->refptr->count);trace_module_get (module, _THIS_IP_, -local_read (__module_ref_addr (module, cpu))); -put_cpu (); +__this_cpu_read (module->refptr->count)); +preempt_enable (); } }
@@ -479,15 +467,17 @@ static inline int try_module_get(struct module *module)int ret =1 ;if (module) { -unsigned int cpu =get_cpu (); +preempt_disable (); +if (likely (module_is_live (module))) { -local_inc (__module_ref_addr (module, cpu)); +__this_cpu_inc (module->refptr->count);trace_module_get (module, _THIS_IP_, -local_read (__module_ref_addr (module, cpu))); +__this_cpu_read (module->refptr->count)); }else ret =0 ; -put_cpu (); + +preempt_enable (); }return ret; }
kernel/module.c +16 -14
@@ -474,9 +474,10 @@ static void module_unload_init(struct module *mod)INIT_LIST_HEAD (&mod->modules_which_use_me);for_each_possible_cpu (cpu) -local_set (__module_ref_addr (mod, cpu),0 ); +per_cpu_ptr (mod->refptr, cpu)->count =0 ; +/* Hold reference count during initialization. */ -local_set (__module_ref_addr (mod,raw_smp_processor_id ()),1 ); +__this_cpu_write (mod->refptr->count,1 );/* Backwards compatibility macros put refcount during init. */ mod->waiter = current; }
@@ -555,6 +556,7 @@ static void module_unload_free(struct module *mod)kfree (use);sysfs_remove_link (i->holders_dir, mod->name);/* There can be at most one match. */ +free_percpu (i->refptr);break ; } }
@@ -619,7 +621,7 @@ unsigned int module_refcount(struct module *mod)int cpu;for_each_possible_cpu (cpu) - total +=local_read (__module_ref_addr (mod, cpu)); + total +=per_cpu_ptr (mod->refptr, cpu)->count;return total; }EXPORT_SYMBOL (module_refcount);
@@ -796,14 +798,15 @@ static struct module_attribute refcnt = {void module_put (struct module *module) {if (module) { -unsigned int cpu =get_cpu (); -local_dec (__module_ref_addr (module, cpu)); +preempt_disable (); +__this_cpu_dec (module->refptr->count); +trace_module_put (module, _RET_IP_, -local_read (__module_ref_addr (module, cpu))); +__this_cpu_read (module->refptr->count));/* Maybe they're waiting for us to drop reference? */ if (unlikely (!module_is_live (module)))wake_up_process (module->waiter); -put_cpu (); +preempt_enable (); } }EXPORT_SYMBOL (module_put);
@@ -1377,9 +1380,9 @@ static void free_module(struct module *mod)kfree (mod->args);if (mod->percpu)percpu_modfree (mod->percpu); -#if defined(CONFIG_MODULE_UNLOAD ) && defined(CONFIG_SMP ) +#if defined(CONFIG_MODULE_UNLOAD )if (mod->refptr) -percpu_modfree (mod->refptr); +free_percpu (mod->refptr);#endif /* Free lock-classes: */ lockdep_free_key_range (mod->module_core, mod->core_size);
@@ -2145,9 +2148,8 @@ static noinline struct module *load_module(void __user *umod, mod = (void *)sechdrs[modindex].sh_addr;kmemleak_load_module (mod, hdr, sechdrs, secstrings); -#if defined(CONFIG_MODULE_UNLOAD ) && defined(CONFIG_SMP ) - mod->refptr =percpu_modalloc (sizeof (local_t), __alignof__(local_t ), - mod->name); +#if defined(CONFIG_MODULE_UNLOAD ) + mod->refptr = alloc_percpu(struct module_ref);if (!mod->refptr) { err = -ENOMEM ; goto free_init;
@@ -2373,8 +2375,8 @@ static noinline struct module *load_module(void __user *umod,kobject_put (&mod->mkobj.kobj); free_unload:module_unload_free (mod); -#if defined(CONFIG_MODULE_UNLOAD ) && defined(CONFIG_SMP ) -percpu_modfree (mod->refptr); +#if defined(CONFIG_MODULE_UNLOAD ) +free_percpu (mod->refptr); free_init:#endif module_free (mod, mod->module_init);
Signature · 1 line
1.6.4.2
Tejun Heo
maintainerRe: [this_cpu_xx V8 07/16] Module handling: Use this_cpu_xx to dynamically allocate counters
Quoted 16 lines · Tejun Heo
Here's the version I committed to percpu branch.
Thanks.
From
8af47ddd01364ae3c663c0bc92415c06fe887ba1Mon Sep 17 00:00:00 2001
From: Christoph Lameter <[email protected]>
Date: Fri, 18 Dec 2009 16:26:24 -0600
Subject: [PATCH] module: Usethis_cpu_xxto dynamically allocate countersUse cpu ops to deal with the per cpu data instead of a
local_t. Reduces memory requirements, cache footprint and decreases cycle counts.The
this_cpu_xxoperations are also used for !SMP mode. Otherwise we could not drop the use of__module_ref_addr()which would make per cpu data handling complicated.this_cpu_xxoperations have their own fallback for !SMP.
Quoted 5 lines
The last hold out of users of
local_tis the tracing ringbuffer after this patch has been applied.Signed-off-by Christoph Lameter <[email protected]>Cc Rusty Russell <[email protected]>
This was changed to Acked-by as Rusty acked on the previous thread.
Thanks.
Signature · 1 line
tejun
Re: [this_cpu_xx V8 07/16] Module handling: Use this_cpu_xx to dynamically allocate counters
In reply to Tejun Heokernel/module.c static void module_unload_free(struct module *mod) · line 562
Quoted 8 lines · Tejun Heo
kernel/module.c +1
-0@@ -555,6 +556,7 @@ static void module_unload_free(struct module *mod)kfree (use);sysfs_remove_link (i->holders_dir, mod->name);/* There can be at most one match. */ +free_percpu (i->refptr);break ; } }
This looks very wrong.
Rusty
Tejun Heo
maintainerRe: [this_cpu_xx V8 07/16] Module handling: Use this_cpu_xx to dynamically allocate counters
Quoted 11 lines · Rusty Russell
kernel/module.c static void module_unload_free(struct module *mod) · line 562
Quoted 8 lines · Tejun Heo
kernel/module.c +1
-0@@ -555,6 +556,7 @@ static void module_unload_free(struct module *mod)kfree (use);sysfs_remove_link (i->holders_dir, mod->name);/* There can be at most one match. */ +free_percpu (i->refptr);break ; } }This looks very wrong.
Indeed, thanks for spotting it. Christoph, I'm rolling back all patches from this series. Please re-post with updates.
Thanks.
Signature · 1 line
tejun
Re: [this_cpu_xx V8 07/16] Module handling: Use this_cpu_xx to dynamically allocate counters
Quoted 13 lines · Tejun Heo
Quoted 11 lines · Rusty Russell
kernel/module.c static void module_unload_free(struct module *mod) · line 562
Quoted 8 lines · Tejun Heo
kernel/module.c +1
-0@@ -555,6 +556,7 @@ static void module_unload_free(struct module *mod)kfree (use);sysfs_remove_link (i->holders_dir, mod->name);/* There can be at most one match. */ +free_percpu (i->refptr);break ; } }This looks very wrong.
Quoted 2 lines
Indeed, thanks for spotting it. Christoph, I'm rolling back all patches from this series. Please re-post with updates.
Simply drop the chunk?
Re: [this_cpu_xx V8 07/16] Module handling: Use this_cpu_xx to dynamically allocate counters
In reply to Tejun Heoinclude/linux/module.h line 19
Quoted 10 lines · Tejun Heo
index 482efc8..9350577 100644
include/linux/module.h +1
-2@@ -16,8 +16,7 @@#include <linux/kobject.h> #include <linux/moduleparam.h> #include <linux/tracepoint.h> - -#include <asm/local.h> +#include <linux/percpu.h>
This is going to break ringbuffer.c.
Re: [this_cpu_xx V8 07/16] Module handling: Use this_cpu_xx to dynamically allocate counters
In reply to Tejun HeoQuoted 14 lines · Tejun Heo
kernel/trace/ring_buffer.c line 15
Quoted 10 lines
Index:
linux-2.6/kernel/trace/ring_buffer.c
===================================================================kernel/trace/ring_buffer.c +1
-0@@ -12,6 +12,7 @@#include <linux/hardirq.h> #include <linux/kmemcheck.h> #include <linux/module.h> +#include <asm/local.h> This doesn't belong to this patch, right? I stripped this part out, added Cc: to Rusty and applied 1, 2, 7 and 8 to percpu branch. I'll post the updated patch here. Thanks.
If you strip this part out then ringbuffer.c will no longer compile since it relies on the "#include <local.h>" that is removed by this patch from the module.h file.
Tejun Heo
maintainerRe: [this_cpu_xx V8 07/16] Module handling: Use this_cpu_xx to dynamically allocate counters
Quoted 17 lines · Christoph Lameter
Quoted 14 lines · Tejun Heo
kernel/trace/ring_buffer.c line 15
Quoted 10 lines
Index:
linux-2.6/kernel/trace/ring_buffer.c
===================================================================kernel/trace/ring_buffer.c +1
-0@@ -12,6 +12,7 @@#include <linux/hardirq.h> #include <linux/kmemcheck.h> #include <linux/module.h> +#include <asm/local.h> This doesn't belong to this patch, right? I stripped this part out, added Cc: to Rusty and applied 1, 2, 7 and 8 to percpu branch. I'll post the updated patch here. Thanks.
Quoted 3 lines
If you strip this part out then
ringbuffer.cwill no longer compile since it relies on the "#include <local.h>" that is removed by this patch from themodule.hfile.
Oh... alright. I'll add that comment and drop the offending chunk and recommit.
Thanks.
Signature · 1 line
tejun
Re: [this_cpu_xx V8 07/16] Module handling: Use this_cpu_xx to dynamically allocate counters
Quoted 18 lines · Tejun Heo
Quoted 17 lines · Christoph Lameter
Quoted 14 lines · Tejun Heo
kernel/trace/ring_buffer.c line 15
Quoted 10 lines
Index:
linux-2.6/kernel/trace/ring_buffer.c
===================================================================kernel/trace/ring_buffer.c +1
-0@@ -12,6 +12,7 @@#include <linux/hardirq.h> #include <linux/kmemcheck.h> #include <linux/module.h> +#include <asm/local.h> This doesn't belong to this patch, right? I stripped this part out, added Cc: to Rusty and applied 1, 2, 7 and 8 to percpu branch. I'll post the updated patch here. Thanks.
Quoted 6 lines
Quoted 3 lines
If you strip this part out then
ringbuffer.cwill no longer compile since it relies on the "#include <local.h>" that is removed by this patch from themodule.hfile.Oh... alright. I'll add that comment and drop the offending chunk and recommit.
So we need a separate patch to deal with remval of the #include <asm/local.h> from modules.h?
Re: [this_cpu_xx V8 07/16] Module handling: Use this_cpu_xx to dynamically allocate counters
Quoted 24 lines · * Christoph Lameter ([email protected]) wrote:
Quoted 22 lines · Tejun Heo
Quoted 20 lines · Christoph Lameter
Quoted 14 lines · Tejun Heo
kernel/trace/ring_buffer.c line 15
Quoted 10 lines
Index:
linux-2.6/kernel/trace/ring_buffer.c
===================================================================kernel/trace/ring_buffer.c +1
-0@@ -12,6 +12,7 @@#include <linux/hardirq.h> #include <linux/kmemcheck.h> #include <linux/module.h> +#include <asm/local.h> This doesn't belong to this patch, right? I stripped this part out, added Cc: to Rusty and applied 1, 2, 7 and 8 to percpu branch. I'll post the updated patch here. Thanks.
If you strip this part out then
ringbuffer.cwill no longer compile since it relies on the "#include <local.h>" that is removed by this patch from themodule.hfile.
Quoted 6 lines
Quoted 2 lines
Oh... alright. I'll add that comment and drop the offending chunk and recommit.
So we need a separate patch to deal with remval of the #include <
asm/local.h> frommodules.h?
I think this would make sense, yes. This would be a patch specific to the ring-buffer code that would go through (or be acked by) Steven.
Mathieu
Signature · 2 lines
Mathieu Desnoyers OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F BA06 3F25 A8FE 3BAE 9A68
PATCH v8 8/16 Remove cpu_local_xx macros
In reply to Christoph LameterThese macros have not been used for awhile now and are not used by the last local_t user in the tree (trace ringbuffer logic).
--- arch/alpha/include/asm/local.h | 17 ----------------- arch/m32r/include/asm/local.h | 25 ------------------------- arch/mips/include/asm/local.h | 25 ------------------------- arch/powerpc/include/asm/local.h | 25 ------------------------- arch/x86/include/asm/local.h | 37 ------------------------------------- include/asm-generic/local.h | 19 ------------------- 6 files changed, 148 deletions(-)
arch/alpha/include/asm/local.h +0 -17
@@ -98,21 +98,4 @@ static __inline__ long local_sub_return(#define __local_add (i,l) ((l)->a.counter+=(i))#define __local_sub (i,l) ((l)->a.counter-=(i)) -/* Use these for per-cpu local_t variables: on some archs they are -* much more efficient than these naive implementations. Note they take -* a variable, not an address. -*/ -#define cpu_local_read (l) local_read(&__get_cpu_var(l)) -#define cpu_local_set (l, i) local_set(&__get_cpu_var(l), (i)) - -#define cpu_local_inc (l) local_inc(&__get_cpu_var(l)) -#define cpu_local_dec (l) local_dec(&__get_cpu_var(l)) -#define cpu_local_add (i, l) local_add((i), &__get_cpu_var(l)) -#define cpu_local_sub (i, l) local_sub((i), &__get_cpu_var(l)) - -#define __cpu_local_inc (l) __local_inc(&__get_cpu_var(l)) -#define __cpu_local_dec (l) __local_dec(&__get_cpu_var(l)) -#define __cpu_local_add (i, l) __local_add((i), &__get_cpu_var(l)) -#define __cpu_local_sub (i, l) __local_sub((i), &__get_cpu_var(l)) -#endif /* _ALPHA_LOCAL_H */
arch/m32r/include/asm/local.h +0 -25
@@ -338,29 +338,4 @@ static inline void local_set_mask(unsign* a variable, not an address. */ -/* Need to disable preemption for the cpu local counters otherwise we could -still access a variable of a previous CPU in a non local way. */ -#define cpu_local_wrap_v (l) \ - ({ local_t res__; \ - preempt_disable(); \ - res__ = (l); \ - preempt_enable(); \ - res__; }) -#define cpu_local_wrap (l) \ - ({ preempt_disable(); \ - l; \ - preempt_enable(); }) \ - -#define cpu_local_read (l) cpu_local_wrap_v(local_read(&__get_cpu_var(l))) -#define cpu_local_set (l, i) cpu_local_wrap(local_set(&__get_cpu_var(l), (i))) -#define cpu_local_inc (l) cpu_local_wrap(local_inc(&__get_cpu_var(l))) -#define cpu_local_dec (l) cpu_local_wrap(local_dec(&__get_cpu_var(l))) -#define cpu_local_add (i, l) cpu_local_wrap(local_add((i), &__get_cpu_var(l))) -#define cpu_local_sub (i, l) cpu_local_wrap(local_sub((i), &__get_cpu_var(l))) - -#define __cpu_local_inc (l) cpu_local_inc(l) -#define __cpu_local_dec (l) cpu_local_dec(l) -#define __cpu_local_add (i, l) cpu_local_add((i), (l)) -#define __cpu_local_sub (i, l) cpu_local_sub((i), (l)) -#endif /* __M32R_LOCAL_H */
arch/mips/include/asm/local.h +0 -25
@@ -193,29 +193,4 @@ static __inline__ long local_sub_return(#define __local_add (i, l) ((l)->a.counter+=(i))#define __local_sub (i, l) ((l)->a.counter-=(i)) -/* Need to disable preemption for the cpu local counters otherwise we could -still access a variable of a previous CPU in a non atomic way. */ -#define cpu_local_wrap_v (l) \ - ({ local_t res__; \ - preempt_disable(); \ - res__ = (l); \ - preempt_enable(); \ - res__; }) -#define cpu_local_wrap (l) \ - ({ preempt_disable(); \ - l; \ - preempt_enable(); }) \ - -#define cpu_local_read (l) cpu_local_wrap_v(local_read(&__get_cpu_var(l))) -#define cpu_local_set (l, i) cpu_local_wrap(local_set(&__get_cpu_var(l), (i))) -#define cpu_local_inc (l) cpu_local_wrap(local_inc(&__get_cpu_var(l))) -#define cpu_local_dec (l) cpu_local_wrap(local_dec(&__get_cpu_var(l))) -#define cpu_local_add (i, l) cpu_local_wrap(local_add((i), &__get_cpu_var(l))) -#define cpu_local_sub (i, l) cpu_local_wrap(local_sub((i), &__get_cpu_var(l))) - -#define __cpu_local_inc (l) cpu_local_inc(l) -#define __cpu_local_dec (l) cpu_local_dec(l) -#define __cpu_local_add (i, l) cpu_local_add((i), (l)) -#define __cpu_local_sub (i, l) cpu_local_sub((i), (l)) -#endif /* _ARCH_MIPS_LOCAL_H */
arch/powerpc/include/asm/local.h +0 -25
@@ -172,29 +172,4 @@ static __inline__ long local_dec_if_posi#define __local_add (i,l) ((l)->a.counter+=(i))#define __local_sub (i,l) ((l)->a.counter-=(i)) -/* Need to disable preemption for the cpu local counters otherwise we could -still access a variable of a previous CPU in a non atomic way. */ -#define cpu_local_wrap_v (l) \ - ({ local_t res__; \ - preempt_disable(); \ - res__ = (l); \ - preempt_enable(); \ - res__; }) -#define cpu_local_wrap (l) \ - ({ preempt_disable(); \ - l; \ - preempt_enable(); }) \ - -#define cpu_local_read (l) cpu_local_wrap_v(local_read(&__get_cpu_var(l))) -#define cpu_local_set (l, i) cpu_local_wrap(local_set(&__get_cpu_var(l), (i))) -#define cpu_local_inc (l) cpu_local_wrap(local_inc(&__get_cpu_var(l))) -#define cpu_local_dec (l) cpu_local_wrap(local_dec(&__get_cpu_var(l))) -#define cpu_local_add (i, l) cpu_local_wrap(local_add((i), &__get_cpu_var(l))) -#define cpu_local_sub (i, l) cpu_local_wrap(local_sub((i), &__get_cpu_var(l))) - -#define __cpu_local_inc (l) cpu_local_inc(l) -#define __cpu_local_dec (l) cpu_local_dec(l) -#define __cpu_local_add (i, l) cpu_local_add((i), (l)) -#define __cpu_local_sub (i, l) cpu_local_sub((i), (l)) -#endif /* _ARCH_POWERPC_LOCAL_H */
arch/x86/include/asm/local.h +0 -37
@@ -195,41 +195,4 @@ static inline long local_sub_return(long#define __local_add (i, l) local_add((i), (l))#define __local_sub (i, l) local_sub((i), (l)) -/* Use these for per-cpu local_t variables: on some archs they are -* much more efficient than these naive implementations. Note they take -* a variable, not an address. -* -* X86_64: This could be done better if we moved the per cpu data directly -* after GS. -*/ - -/* Need to disable preemption for the cpu local counters otherwise we could -still access a variable of a previous CPU in a non atomic way. */ -#define cpu_local_wrap_v (l) \ -({ \ - local_t res__; \ - preempt_disable(); \ - res__ = (l); \ - preempt_enable(); \ - res__; \ -}) -#define cpu_local_wrap (l) \ -({ \ - preempt_disable(); \ - (l); \ - preempt_enable(); \ -}) \ - -#define cpu_local_read (l) cpu_local_wrap_v(local_read(&__get_cpu_var((l)))) -#define cpu_local_set (l, i) cpu_local_wrap(local_set(&__get_cpu_var((l)), (i))) -#define cpu_local_inc (l) cpu_local_wrap(local_inc(&__get_cpu_var((l)))) -#define cpu_local_dec (l) cpu_local_wrap(local_dec(&__get_cpu_var((l)))) -#define cpu_local_add (i, l) cpu_local_wrap(local_add((i), &__get_cpu_var((l)))) -#define cpu_local_sub (i, l) cpu_local_wrap(local_sub((i), &__get_cpu_var((l)))) - -#define __cpu_local_inc (l) cpu_local_inc((l)) -#define __cpu_local_dec (l) cpu_local_dec((l)) -#define __cpu_local_add (i, l) cpu_local_add((i), (l)) -#define __cpu_local_sub (i, l) cpu_local_sub((i), (l)) -#endif /* _ASM_X86_LOCAL_H */
include/asm-generic/local.h +0 -19
@@ -52,23 +52,4 @@ typedef struct#define __local_add (i,l) local_set((l), local_read(l) + (i))#define __local_sub (i,l) local_set((l), local_read(l) - (i)) -/* Use these for per-cpu local_t variables: on some archs they are -* much more efficient than these naive implementations. Note they take -* a variable (eg. mystruct.foo), not an address. -*/ -#define cpu_local_read (l) local_read(&__get_cpu_var(l)) -#define cpu_local_set (l, i) local_set(&__get_cpu_var(l), (i)) -#define cpu_local_inc (l) local_inc(&__get_cpu_var(l)) -#define cpu_local_dec (l) local_dec(&__get_cpu_var(l)) -#define cpu_local_add (i, l) local_add((i), &__get_cpu_var(l)) -#define cpu_local_sub (i, l) local_sub((i), &__get_cpu_var(l)) - -/* Non-atomic increments, ie. preemption disabled and won't be touched -* in interrupt, etc. Some archs can optimize this case well. -*/ -#define __cpu_local_inc (l) __local_inc(&__get_cpu_var(l)) -#define __cpu_local_dec (l) __local_dec(&__get_cpu_var(l)) -#define __cpu_local_add (i, l) __local_add((i), &__get_cpu_var(l)) -#define __cpu_local_sub (i, l) __local_sub((i), &__get_cpu_var(l)) -#endif /* _ASM_GENERIC_LOCAL_H */
PATCH v8 9/16 Allow arch to provide inc/dec functionality for each size separately
In reply to Christoph LameterCurrent this_cpu ops only allow an arch to specify add RMW operations or inc and dec for all sizes. Some arches can do more efficient inc and dec operations. Allow size specific override of fallback functions like with the other operations.
--- include/linux/percpu.h | 96 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 90 insertions(+), 6 deletions(-)
include/linux/percpu.h +90 -6
@@ -257,6 +257,18 @@ do { \ } \ } while (0) +#define __pcpu_size_call_noparam (stem, variable) \ +do { \ + switch(sizeof(variable)) { \ + case 1: stem##1(variable);break; \ + case 2: stem##2(variable);break; \ + case 4: stem##4(variable);break; \ + case 8: stem##8(variable);break; \ + default: \ + __bad_size_call_parameter();break; \ + } \ +} while (0) +/* * Optimized manipulation for memory allocated through the per cpu * allocator or for addresses of per cpu variables (can be determined
@@ -352,11 +364,35 @@ do { \#endif #ifndef this_cpu_inc -# define this_cpu_inc (pcp) this_cpu_add((pcp), 1) +# ifndef this_cpu_inc_1 +# define this_cpu_inc_1 (pcp) __this_cpu_add((pcp), 1) +# endif +# ifndef this_cpu_inc_2 +# define this_cpu_inc_2 (pcp) __this_cpu_add((pcp), 1) +# endif +# ifndef this_cpu_inc_4 +# define this_cpu_inc_4 (pcp) __this_cpu_add((pcp), 1) +# endif +# ifndef this_cpu_inc_8 +# define this_cpu_inc_8 (pcp) __this_cpu_add((pcp), 1) +# endif +# define this_cpu_inc (pcp) __pcpu_size_call_noparam(this_cpu_inc_, (pcp))#endif #ifndef this_cpu_dec -# define this_cpu_dec (pcp) this_cpu_sub((pcp), 1) +# ifndef this_cpu_dec_1 +# define this_cpu_dec_1 (pcp) __this_cpu_sub((pcp), 1) +# endif +# ifndef this_cpu_dec_2 +# define this_cpu_dec_2 (pcp) __this_cpu_sub((pcp), 1) +# endif +# ifndef this_cpu_dec_4 +# define this_cpu_dec_4 (pcp) __this_cpu_sub((pcp), 1) +# endif +# ifndef this_cpu_dec_8 +# define this_cpu_dec_8 (pcp) __this_cpu_sub((pcp), 1) +# endif +# define this_cpu_dec (pcp) __pcpu_size_call_noparam(this_cpu_dec_, (pcp))#endif #ifndef this_cpu_and
@@ -479,11 +515,35 @@ do { \#endif #ifndef __this_cpu_inc -# define __this_cpu_inc (pcp) __this_cpu_add((pcp), 1) +# ifndef __this_cpu_inc_1 +# define __this_cpu_inc_1 (pcp) __this_cpu_add_1((pcp), 1) +# endif +# ifndef __this_cpu_inc_2 +# define __this_cpu_inc_2 (pcp) __this_cpu_add_2((pcp), 1) +# endif +# ifndef __this_cpu_inc_4 +# define __this_cpu_inc_4 (pcp) __this_cpu_add_4((pcp), 1) +# endif +# ifndef __this_cpu_inc_8 +# define __this_cpu_inc_8 (pcp) __this_cpu_add_8((pcp), 1) +# endif +# define __this_cpu_inc (pcp) __pcpu_size_call_noparam(__this_cpu_inc_, (pcp))#endif #ifndef __this_cpu_dec -# define __this_cpu_dec (pcp) __this_cpu_sub((pcp), 1) +# ifndef __this_cpu_dec_1 +# define __this_cpu_dec_1 (pcp) __this_cpu_sub((pcp), 1) +# endif +# ifndef __this_cpu_dec_2 +# define __this_cpu_dec_2 (pcp) __this_cpu_sub((pcp), 1) +# endif +# ifndef __this_cpu_dec_4 +# define __this_cpu_dec_4 (pcp) __this_cpu_sub((pcp), 1) +# endif +# ifndef __this_cpu_dec_8 +# define __this_cpu_dec_8 (pcp) __this_cpu_sub((pcp), 1) +# endif +# define __this_cpu_dec (pcp) __pcpu_size_call_noparam(__this_cpu_dec_, (pcp))#endif #ifndef __this_cpu_and
@@ -570,11 +630,35 @@ do { \#endif #ifndef irqsafe_cpu_inc -# define irqsafe_cpu_inc (pcp) irqsafe_cpu_add((pcp), 1) +# ifndef irqsafe_cpu_inc_1 +# define irqsafe_cpu_inc_1 (pcp) irqsafe_cpu_add_1((pcp), 1) +# endif +# ifndef irqsafe_cpu_inc_2 +# define irqsafe_cpu_inc_2 (pcp) irqsafe_cpu_add_2((pcp), 1) +# endif +# ifndef irqsafe_cpu_inc_4 +# define irqsafe_cpu_inc_4 (pcp) irqsafe_cpu_add_4((pcp), 1) +# endif +# ifndef irqsafe_cpu_inc_8 +# define irqsafe_cpu_inc_8 (pcp) irqsafe_cpu_add_8((pcp), 1) +# endif +# define irqsafe_cpu_inc (pcp) __pcpu_size_call_noparam(irqsafe_cpu_inc_, (pcp))#endif #ifndef irqsafe_cpu_dec -# define irqsafe_cpu_dec (pcp) irqsafe_cpu_sub((pcp), 1) +# ifndef irqsafe_cpu_dec_1 +# define irqsafe_cpu_dec_1 (pcp) irqsafe_cpu_sub_1((pcp), 1) +# endif +# ifndef irqsafe_cpu_dec_2 +# define irqsafe_cpu_dec_2 (pcp) irqsafe_cpu_sub_2((pcp), 1) +# endif +# ifndef irqsafe_cpu_dec_4 +# define irqsafe_cpu_dec_4 (pcp) irqsafe_cpu_sub_4((pcp), 1) +# endif +# ifndef irqsafe_cpu_dec_8 +# define irqsafe_cpu_dec_8 (pcp) irqsafe_cpu_sub_8((pcp), 1) +# endif +# define irqsafe_cpu_dec (pcp) __pcpu_size_call_noparam(irqsafe_cpu_dec_, (pcp))#endif #ifndef irqsafe_cpu_and
Tejun Heo
maintainerRe: [this_cpu_xx V8 09/16] Allow arch to provide inc/dec functionality for each size separately
Quoted 4 lines · Christoph Lameter
Current
this_cpuops only allow an arch to specify add RMW operations or inc and dec for all sizes. Some arches can do more efficient inc and dec operations. Allow size specific override of fallback functions like with the other operations.
Wouldn't it be better to use __builtin_constant_p() and switching in arch add/dec macros? It just seems a bit extreme to define all those different variants from generic header. I'm quite unsure whether providing overrides for all the different size variants from generic header is necessary at all. If an arch is gonna override the operation, it's probably best to just let it override the whole thing.
Thanks.
Signature · 1 line
tejun
Re: [this_cpu_xx V8 09/16] Allow arch to provide inc/dec functionality for each size separately
Quoted 8 lines · Tejun Heo
Quoted 4 lines · Christoph Lameter
Current
this_cpuops only allow an arch to specify add RMW operations or inc and dec for all sizes. Some arches can do more efficient inc and dec operations. Allow size specific override of fallback functions like with the other operations.Wouldn't it be better to use
__builtin_constant_p()and switching in arch add/dec macros? It just seems a bit extreme to define all those
Yes that could be done but I am on vacation till next year ;-)...
Tejun Heo
maintainerRe: [this_cpu_xx V8 09/16] Allow arch to provide inc/dec functionality for each size separately
Hello, Christoph.
Quoted 8 lines · Christoph Lameter
Quoted 6 lines · Tejun Heo
Quoted 4 lines · Christoph Lameter
Current
this_cpuops only allow an arch to specify add RMW operations or inc and dec for all sizes. Some arches can do more efficient inc and dec operations. Allow size specific override of fallback functions like with the other operations.
Quoted 4 lines
Quoted 2 lines
Wouldn't it be better to use
__builtin_constant_p()and switching in arch add/dec macros? It just seems a bit extreme to define all thoseYes that could be done but I am on vacation till next year ;-)...
I'm not sure how much I would be working from now till the end of the year but if I end up doing some work I'll give it a shot.
Happy holidays and enjoy your vacation!
Signature · 1 line
tejun
PATCH v8 10/16 Support generating inc/dec for this_cpu_inc/dec
In reply to Christoph LameterSupport generating inc/dec instruction. Currently we create an add 1 instruction. Saves one byte per use of this_cpu_xx.
--- arch/x86/include/asm/percpu.h | 47 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+)
arch/x86/include/asm/percpu.h +47 -0
@@ -133,6 +133,29 @@ do { \ pfo_ret__; \ }) +#define percpu_var_op (op, var) \ +({ \ + switch (sizeof(var)) { \ + case 1: \ + asm(op "b "__percpu_arg(0) \ + : "+m" (var) :); \ + break; \ + case 2: \ + asm(op "w "__percpu_arg(0) \ + : "+m" (var) :); \ + break; \ + case 4: \ + asm(op "l "__percpu_arg(0) \ + : "+m" (var) :); \ + break; \ + case 8: \ + asm(op "q "__percpu_arg(0) \ + : "+m" (var) :); \ + break; \ + default: __bad_percpu_size(); \ + } \ +}) +/* * percpu_read() makes gcc load the percpu variable every time it is * accessed while percpu_read_stable() allows the value to be cached.
@@ -163,6 +186,12 @@ do { \#define __this_cpu_add_1 (pcp, val) percpu_to_op("add", (pcp), val)#define __this_cpu_add_2 (pcp, val) percpu_to_op("add", (pcp), val)#define __this_cpu_add_4 (pcp, val) percpu_to_op("add", (pcp), val) +#define __this_cpu_inc_1 (pcp) percpu_var_op("inc", (pcp)) +#define __this_cpu_inc_2 (pcp) percpu_var_op("inc", (pcp)) +#define __this_cpu_inc_4 (pcp) percpu_var_op("inc", (pcp)) +#define __this_cpu_dec_1 (pcp) percpu_var_op("dec", (pcp)) +#define __this_cpu_dec_2 (pcp) percpu_var_op("dec", (pcp)) +#define __this_cpu_dec_4 (pcp) percpu_var_op("dec", (pcp))#define __this_cpu_and_1 (pcp, val) percpu_to_op("and", (pcp), val)#define __this_cpu_and_2 (pcp, val) percpu_to_op("and", (pcp), val)#define __this_cpu_and_4 (pcp, val) percpu_to_op("and", (pcp), val)
@@ -182,6 +211,12 @@ do { \#define this_cpu_add_1 (pcp, val) percpu_to_op("add", (pcp), val)#define this_cpu_add_2 (pcp, val) percpu_to_op("add", (pcp), val)#define this_cpu_add_4 (pcp, val) percpu_to_op("add", (pcp), val) +#define this_cpu_inc_1 (pcp) percpu_var_op("inc", (pcp)) +#define this_cpu_inc_2 (pcp) percpu_var_op("inc", (pcp)) +#define this_cpu_inc_4 (pcp) percpu_var_op("inc", (pcp)) +#define this_cpu_dec_1 (pcp) percpu_var_op("dec", (pcp)) +#define this_cpu_dec_2 (pcp) percpu_var_op("dec", (pcp)) +#define this_cpu_dec_4 (pcp) percpu_var_op("dec", (pcp))#define this_cpu_and_1 (pcp, val) percpu_to_op("and", (pcp), val)#define this_cpu_and_2 (pcp, val) percpu_to_op("and", (pcp), val)#define this_cpu_and_4 (pcp, val) percpu_to_op("and", (pcp), val)
@@ -195,6 +230,12 @@ do { \#define irqsafe_cpu_add_1 (pcp, val) percpu_to_op("add", (pcp), val)#define irqsafe_cpu_add_2 (pcp, val) percpu_to_op("add", (pcp), val)#define irqsafe_cpu_add_4 (pcp, val) percpu_to_op("add", (pcp), val) +#define irqsafe_cpu_inc_1 (pcp) percpu_var_op("inc", (pcp)) +#define irqsafe_cpu_inc_2 (pcp) percpu_var_op("inc", (pcp)) +#define irqsafe_cpu_inc_4 (pcp) percpu_var_op("inc", (pcp)) +#define irqsafe_cpu_dec_1 (pcp) percpu_var_op("dec", (pcp)) +#define irqsafe_cpu_dec_2 (pcp) percpu_var_op("dec", (pcp)) +#define irqsafe_cpu_dec_4 (pcp) percpu_var_op("dec", (pcp))#define irqsafe_cpu_and_1 (pcp, val) percpu_to_op("and", (pcp), val)#define irqsafe_cpu_and_2 (pcp, val) percpu_to_op("and", (pcp), val)#define irqsafe_cpu_and_4 (pcp, val) percpu_to_op("and", (pcp), val)
@@ -213,6 +254,8 @@ do { \#define __this_cpu_read_8 (pcp) percpu_from_op("mov", (pcp), "m"(pcp))#define __this_cpu_write_8 (pcp, val) percpu_to_op("mov", (pcp), val)#define __this_cpu_add_8 (pcp, val) percpu_to_op("add", (pcp), val) +#define __this_cpu_inc_8 (pcp) percpu_var_op("inc", (pcp)) +#define __this_cpu_dec_8 (pcp) percpu_var_op("dec", (pcp))#define __this_cpu_and_8 (pcp, val) percpu_to_op("and", (pcp), val)#define __this_cpu_or_8 (pcp, val) percpu_to_op("or", (pcp), val)#define __this_cpu_xor_8 (pcp, val) percpu_to_op("xor", (pcp), val)
@@ -220,11 +263,15 @@ do { \#define this_cpu_read_8 (pcp) percpu_from_op("mov", (pcp), "m"(pcp))#define this_cpu_write_8 (pcp, val) percpu_to_op("mov", (pcp), val)#define this_cpu_add_8 (pcp, val) percpu_to_op("add", (pcp), val) +#define this_cpu_inc_8 (pcp) percpu_var_op("inc", (pcp)) +#define this_cpu_dec_8 (pcp) percpu_var_op("dec", (pcp))#define this_cpu_and_8 (pcp, val) percpu_to_op("and", (pcp), val)#define this_cpu_or_8 (pcp, val) percpu_to_op("or", (pcp), val)#define this_cpu_xor_8 (pcp, val) percpu_to_op("xor", (pcp), val)#define irqsafe_cpu_add_8 (pcp, val) percpu_to_op("add", (pcp), val) +#define irqsafe_cpu_inc_8 (pcp) percpu_var_op("inc", (pcp)) +#define irqsafe_cpu_dec_8 (pcp) percpu_var_op("dec", (pcp))#define irqsafe_cpu_and_8 (pcp, val) percpu_to_op("and", (pcp), val)#define irqsafe_cpu_or_8 (pcp, val) percpu_to_op("or", (pcp), val)#define irqsafe_cpu_xor_8 (pcp, val) percpu_to_op("xor", (pcp), val)
PATCH v8 11/16 Generic support for this_cpu_cmpxchg
In reply to Christoph LameterProvide support for this_cpu_cmpxchg.
--- include/linux/percpu.h | 99 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+)
include/linux/percpu.h +99 -0
@@ -443,6 +443,62 @@ do { \# define this_cpu_xor (pcp, val) __pcpu_size_call(this_cpu_or_, (pcp), (val))#endif +static inline unsigned long __this_cpu_cmpxchg_generic (volatile void *pptr, +unsigned long old,unsigned long new,int size) +{ +unsigned long prev; +volatile void * ptr =__this_cpu_ptr (pptr); + +switch (size) { +case 1 : prev = *(u8 *)ptr; +if (prev == old) + *(u8 *)ptr = (u8 )new; +break ; +case 2 : prev = *(u16 *)ptr; +if (prev == old) + *(u16 *)ptr = (u16 )new; +break ; +case 4 : prev = *(u32 *)ptr; +if (prev == old) + *(u32 *)ptr = (u32 )new; +break ; +case 8 : prev = *(u64 *)ptr; +if (prev == old) + *(u64 *)ptr = (u64 )new; +break ; +default : +__bad_size_call_parameter (); + } +return prev; +} + +static inline unsigned long this_cpu_cmpxchg_generic (volatile void *ptr, +unsigned long old,unsigned long new,int size) +{ +unsigned long prev; + +preempt_disable (); + prev =__this_cpu_cmpxchg_generic (ptr, old, new, size); +preempt_enable (); +return prev; +} + +#ifndef this_cpu_cmpxchg +# ifndef this_cpu_cmpxchg_1 +# define this_cpu_cmpxchg_1 (pcp, old, new) this_cpu_cmpxchg_generic((pcp), (old), (new), 1) +# endif +# ifndef this_cpu_cmpxchg_2 +# define this_cpu_cmpxchg_2 (pcp, old, new) this_cpu_cmpxchg_generic((pcp), (old), (new), 2) +# endif +# ifndef this_cpu_cmpxchg_4 +# define this_cpu_cmpxchg_4 (pcp, old, new) this_cpu_cmpxchg_generic((pcp), (old), (new), 4) +# endif +# ifndef this_cpu_cmpxchg_8 +# define this_cpu_cmpxchg_8 (pcp, old, new) this_cpu_cmpxchg_generic((pcp), (old), (new), 8) +# endif +# define this_cpu_cmpxchg (pcp, old, new) __pcpu_size_call_return(__this_cpu_cmpxchg_, (old), (new)) +#endif +/* * Generic percpu operations that do not require preemption handling. * Either we do not care about races or the caller has the
@@ -594,6 +650,22 @@ do { \# define __this_cpu_xor (pcp, val) __pcpu_size_call(__this_cpu_xor_, (pcp), (val))#endif +#ifndef __this_cpu_cmpxchg +# ifndef __this_cpu_cmpxchg_1 +# define __this_cpu_cmpxchg_1 (pcp, old, new) __this_cpu_cmpxchg_generic((pcp), (old), (new), 1) +# endif +# ifndef __this_cpu_cmpxchg_2 +# define __this_cpu_cmpxchg_2 (pcp, old, new) __this_cpu_cmpxchg_generic((pcp), (old), (new), 2) +# endif +# ifndef __this_cpu_cmpxchg_4 +# define __this_cpu_cmpxchg_4 (pcp, old, new) __this_cpu_cmpxchg_generic((pcp), (old), (new), 4) +# endif +# ifndef __this_cpu_cmpxchg_8 +# define __this_cpu_cmpxchg_8 (pcp, old, new) __this_cpu_cmpxchg_generic((pcp), (old), (new), 8) +# endif +# define __this_cpu_cmpxchg (pcp, old, new) __pcpu_size_call_return(__this_cpu_cmpxchg_, (old), (new)) +#endif +/* * IRQ safe versions of the per cpu RMW operations. Note that these operations * are *not* safe against modification of the same variable from another
@@ -709,4 +781,31 @@ do { \# define irqsafe_cpu_xor (pcp, val) __pcpu_size_call(irqsafe_cpu_xor_, (val))#endif +static inline unsigned long irqsafe_cpu_cmpxchg_generic (volatile void *ptr, +unsigned long old,unsigned long new,int size) +{ +unsigned long flags, prev; + +local_irq_save (flags); + prev =__this_cpu_cmpxchg_generic (ptr, old, new, size); +local_irq_restore (flags); +return prev; +} + +#ifndef irqsafe_cpu_cmpxchg +# ifndef irqsafe_cpu_cmpxchg_1 +# define irqsafe_cpu_cmpxchg_1 (pcp, old, new) irqsafe_cpu_cmpxchg_generic(((pcp), (old), (new), 1) +# endif +# ifndef irqsafe_cpu_cmpxchg_2 +# define irqsafe_cpu_cmpxchg_2 (pcp, old, new) irqsafe_cpu_cmpxchg_generic(((pcp), (old), (new), 2) +# endif +# ifndef irqsafe_cpu_cmpxchg_4 +# define irqsafe_cpu_cmpxchg_4 (pcp, old, new) irqsafe_cpu_cmpxchg_generic(((pcp), (old), (new), 4) +# endif +# ifndef irqsafe_cpu_cmpxchg_8 +# define irqsafe_cpu_cmpxchg_8 (pcp, old, new) irqsafe_cpu_cmpxchg_generic(((pcp), (old), (new), 8) +# endif +# define irqsafe_cpu_cmpxchg (pcp, old, new) __pcpu_size_call_return(irqsafe_cpu_cmpxchg_, (old), (new)) +#endif +#endif /* __LINUX_PERCPU_H */
Re: [this_cpu_xx V8 11/16] Generic support for this_cpu_cmpxchg
Quoted 34 lines · * Christoph Lameter ([email protected]) wrote:
Provide support for
this_cpu_cmpxchg.Signed-off-by Christoph Lameter <[email protected]>--- include/linux/percpu.h | 99 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+)Index:
linux-2.6/include/linux/percpu.h
===================================================================include/linux/percpu.h +18
-0@@ -443,6 +443,62 @@ do { \# define this_cpu_xor (pcp, val) __pcpu_size_call(this_cpu_or_, (pcp), (val))#endif +static inline unsigned long __this_cpu_cmpxchg_generic (volatile void *pptr, +unsigned long old,unsigned long new,int size) +{ +unsigned long prev; +volatile void * ptr =__this_cpu_ptr (pptr); + +switch (size) { +case 1 : prev = *(u8 *)ptr; +if (prev == old) + *(u8 *)ptr = (u8 )new; +break ; +case 2 : prev = *(u16 *)ptr; +if (prev == old) + *(u16 *)ptr = (u16 )new; +break ; +case 4 : prev = *(u32 *)ptr; +if (prev == old) + *(u32 *)ptr = (u32 )new;
include/linux/percpu.h do { \ · line 473
Quoted 10 lines
include/linux/percpu.h +10
-0+break ; +case 8 : prev = *(u64 *)ptr; +if (prev == old) + *(u64 *)ptr = (u64 )new; +break ; +default : +__bad_size_call_parameter (); + } +return prev; +}
Hi Christoph,
I am a bit concerned about the "generic" version of this_cpu_cmpxchg. Given that what LTTng needs is basically an atomic, nmi-safe version of the primitive (on all architectures that have something close to a NMI), this means that it could not switch over to your primitives until we add the equivalent support we currently have with local_t to all architectures. The transition would be faster if we create an atomic_cpu_*() variant which would map to local_t operations in the initial version.
Or maybe have I missed something in your patchset that address this ?
Thanks,
Mathieu
Quoted 88 lines
include/linux/percpu.h +71
-0+ +static inline unsigned long this_cpu_cmpxchg_generic (volatile void *ptr, +unsigned long old,unsigned long new,int size) +{ +unsigned long prev; + +preempt_disable (); + prev =__this_cpu_cmpxchg_generic (ptr, old, new, size); +preempt_enable (); +return prev; +} + +#ifndef this_cpu_cmpxchg +# ifndef this_cpu_cmpxchg_1 +# define this_cpu_cmpxchg_1 (pcp, old, new) this_cpu_cmpxchg_generic((pcp), (old), (new), 1) +# endif +# ifndef this_cpu_cmpxchg_2 +# define this_cpu_cmpxchg_2 (pcp, old, new) this_cpu_cmpxchg_generic((pcp), (old), (new), 2) +# endif +# ifndef this_cpu_cmpxchg_4 +# define this_cpu_cmpxchg_4 (pcp, old, new) this_cpu_cmpxchg_generic((pcp), (old), (new), 4) +# endif +# ifndef this_cpu_cmpxchg_8 +# define this_cpu_cmpxchg_8 (pcp, old, new) this_cpu_cmpxchg_generic((pcp), (old), (new), 8) +# endif +# define this_cpu_cmpxchg (pcp, old, new) __pcpu_size_call_return(__this_cpu_cmpxchg_, (old), (new)) +#endif +/* * Generic percpu operations that do not require preemption handling. * Either we do not care about races or the caller has the @@ -594,6 +650,22 @@ do { \# define __this_cpu_xor (pcp, val) __pcpu_size_call(__this_cpu_xor_, (pcp), (val))#endif +#ifndef __this_cpu_cmpxchg +# ifndef __this_cpu_cmpxchg_1 +# define __this_cpu_cmpxchg_1 (pcp, old, new) __this_cpu_cmpxchg_generic((pcp), (old), (new), 1) +# endif +# ifndef __this_cpu_cmpxchg_2 +# define __this_cpu_cmpxchg_2 (pcp, old, new) __this_cpu_cmpxchg_generic((pcp), (old), (new), 2) +# endif +# ifndef __this_cpu_cmpxchg_4 +# define __this_cpu_cmpxchg_4 (pcp, old, new) __this_cpu_cmpxchg_generic((pcp), (old), (new), 4) +# endif +# ifndef __this_cpu_cmpxchg_8 +# define __this_cpu_cmpxchg_8 (pcp, old, new) __this_cpu_cmpxchg_generic((pcp), (old), (new), 8) +# endif +# define __this_cpu_cmpxchg (pcp, old, new) __pcpu_size_call_return(__this_cpu_cmpxchg_, (old), (new)) +#endif +/* * IRQ safe versions of the per cpu RMW operations. Note that these operations * are *not* safe against modification of the same variable from another @@ -709,4 +781,31 @@ do { \# define irqsafe_cpu_xor (pcp, val) __pcpu_size_call(irqsafe_cpu_xor_, (val))#endif +static inline unsigned long irqsafe_cpu_cmpxchg_generic (volatile void *ptr, +unsigned long old,unsigned long new,int size) +{ +unsigned long flags, prev; + +local_irq_save (flags); + prev =__this_cpu_cmpxchg_generic (ptr, old, new, size); +local_irq_restore (flags); +return prev; +} + +#ifndef irqsafe_cpu_cmpxchg +# ifndef irqsafe_cpu_cmpxchg_1 +# define irqsafe_cpu_cmpxchg_1 (pcp, old, new) irqsafe_cpu_cmpxchg_generic(((pcp), (old), (new), 1) +# endif +# ifndef irqsafe_cpu_cmpxchg_2 +# define irqsafe_cpu_cmpxchg_2 (pcp, old, new) irqsafe_cpu_cmpxchg_generic(((pcp), (old), (new), 2) +# endif +# ifndef irqsafe_cpu_cmpxchg_4 +# define irqsafe_cpu_cmpxchg_4 (pcp, old, new) irqsafe_cpu_cmpxchg_generic(((pcp), (old), (new), 4) +# endif +# ifndef irqsafe_cpu_cmpxchg_8 +# define irqsafe_cpu_cmpxchg_8 (pcp, old, new) irqsafe_cpu_cmpxchg_generic(((pcp), (old), (new), 8) +# endif +# define irqsafe_cpu_cmpxchg (pcp, old, new) __pcpu_size_call_return(irqsafe_cpu_cmpxchg_, (old), (new)) +#endif +#endif /* __LINUX_PERCPU_H */ Signature · 1 line
--
Signature · 2 lines
Mathieu Desnoyers OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F BA06 3F25 A8FE 3BAE 9A68
Re: [this_cpu_xx V8 11/16] Generic support for this_cpu_cmpxchg
Quoted 10 lines · Mathieu Desnoyers
I am a bit concerned about the "generic" version of
this_cpu_cmpxchg. Given that what LTTng needs is basically an atomic, nmi-safe version of the primitive (on all architectures that have something close to a NMI), this means that it could not switch over to your primitives until we add the equivalent support we currently have withlocal_tto all architectures. The transition would be faster if we create an atomic_cpu_*() variant which would map tolocal_toperations in the initial version.Or maybe have I missed something in your patchset that address this ?
NMI safeness is not covered by this_cpu operations.
We could add nmi_safe_.... ops?
The atomic_cpu reference make me think that you want full (LOCK) semantics? Then use the regular atomic ops?
Re: [this_cpu_xx V8 11/16] Generic support for this_cpu_cmpxchg
Quoted 11 lines · * Christoph Lameter ([email protected]) wrote:
Quoted 9 lines · Mathieu Desnoyers
I am a bit concerned about the "generic" version of
this_cpu_cmpxchg. Given that what LTTng needs is basically an atomic, nmi-safe version of the primitive (on all architectures that have something close to a NMI), this means that it could not switch over to your primitives until we add the equivalent support we currently have withlocal_tto all architectures. The transition would be faster if we create an atomic_cpu_*() variant which would map tolocal_toperations in the initial version.
Quoted 8 lines
Quoted 1 line
Or maybe have I missed something in your patchset that address this ?
NMI safeness is not covered by
this_cpuoperations.We could add
nmi_safe_.... ops?The
atomic_cpureference make me think that you want full (LOCK) semantics? Then use the regular atomic ops?
nmi_safe would probably make sense here.
But given that we have to disable preemption to add precision in terms of trace clock timestamp, I wonder if we would really gain something considerable performance-wise.
I also thought about the design change this requires for the per-cpu buffer commit count pointer which would have to become a per-cpu pointer independent of the buffer structure, and I foresee a problem with Steven's irq off tracing which need to perform buffer exchanges while tracing is active. Basically, having only one top-level pointer for the buffer makes it possible to exchange it atomically, but if we have to have two separate pointers (one for per-cpu buffer, one for per-cpu commit count array), then we are stucked.
So given that per-cpu ops limits us in terms of data structure layout, I am less and less sure it's the best fit for ring buffers, especially if we don't gain much performance-wise.
Thanks,
Mathieu
Signature · 2 lines
Mathieu Desnoyers OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F BA06 3F25 A8FE 3BAE 9A68
Re: [this_cpu_xx V8 11/16] Generic support for this_cpu_cmpxchg
Quoted 11 lines · Mathieu Desnoyers
Quoted 11 lines
Quoted 10 lines
I am a bit concerned about the "generic" version of
this_cpu_cmpxchg. Given that what LTTng needs is basically an atomic, nmi-safe version of the primitive (on all architectures that have something close to a NMI), this means that it could not switch over to your primitives until we add the equivalent support we currently have withlocal_tto all architectures. The transition would be faster if we create an atomic_cpu_*() variant which would map tolocal_toperations in the initial version.Or maybe have I missed something in your patchset that address this ?
Quoted 8 lines
Quoted 6 lines
NMI safeness is not covered by
this_cpuoperations.We could add
nmi_safe_.... ops?The
atomic_cpureference make me think that you want full (LOCK) semantics? Then use the regular atomic ops?
nmi_safewould probably make sense here.
I am not sure how to implement fallback for nmi_safe operations though since there is no way of disabling NMIs.
Quoted 3 lines
But given that we have to disable preemption to add precision in terms of trace clock timestamp, I wonder if we would really gain something considerable performance-wise.
Not sure what exactly you attempt to do there.
Quoted 8 lines
I also thought about the design change this requires for the per-cpu buffer commit count pointer which would have to become a per-cpu pointer independent of the buffer structure, and I foresee a problem with Steven's irq off tracing which need to perform buffer exchanges while tracing is active. Basically, having only one top-level pointer for the buffer makes it possible to exchange it atomically, but if we have to have two separate pointers (one for per-cpu buffer, one for per-cpu commit count array), then we are stucked.
You just need to keep percpu pointers that are offsets into the percpu area. They can be relocated as needed to the processor specific addresses using the cpu ops.
Quoted 3 lines
So given that per-cpu ops limits us in terms of data structure layout, I am less and less sure it's the best fit for ring buffers, especially if we don't gain much performance-wise.
I dont understand how exactly the ring buffer logic works and what you are trying to do here.
The ringbuffers are per cpu structures right and you do not change cpus while performing operations on them? If not then the per cpu ops are not useful to you.
If you dont: How can you safely use the local_t operations for the ringbuffer logic?
Re: [this_cpu_xx V8 11/16] Generic support for this_cpu_cmpxchg
Quoted 48 lines · * Christoph Lameter ([email protected]) wrote:
Quoted 11 lines · Mathieu Desnoyers
Quoted 11 lines
Quoted 10 lines
I am a bit concerned about the "generic" version of
this_cpu_cmpxchg. Given that what LTTng needs is basically an atomic, nmi-safe version of the primitive (on all architectures that have something close to a NMI), this means that it could not switch over to your primitives until we add the equivalent support we currently have withlocal_tto all architectures. The transition would be faster if we create an atomic_cpu_*() variant which would map tolocal_toperations in the initial version.Or maybe have I missed something in your patchset that address this ?
Quoted 8 lines
Quoted 6 lines
NMI safeness is not covered by
this_cpuoperations.We could add
nmi_safe_.... ops?The
atomic_cpureference make me think that you want full (LOCK) semantics? Then use the regular atomic ops?
nmi_safewould probably make sense here.I am not sure how to implement fallback for
nmi_safeoperations though since there is no way of disabling NMIs.Quoted 3 lines
But given that we have to disable preemption to add precision in terms of trace clock timestamp, I wonder if we would really gain something considerable performance-wise.
Not sure what exactly you attempt to do there.
Quoted 8 lines
I also thought about the design change this requires for the per-cpu buffer commit count pointer which would have to become a per-cpu pointer independent of the buffer structure, and I foresee a problem with Steven's irq off tracing which need to perform buffer exchanges while tracing is active. Basically, having only one top-level pointer for the buffer makes it possible to exchange it atomically, but if we have to have two separate pointers (one for per-cpu buffer, one for per-cpu commit count array), then we are stucked.
You just need to keep percpu pointers that are offsets into the percpu area. They can be relocated as needed to the processor specific addresses using the cpu ops.
Quoted 3 lines
So given that per-cpu ops limits us in terms of data structure layout, I am less and less sure it's the best fit for ring buffers, especially if we don't gain much performance-wise.
Quoted 6 lines
I dont understand how exactly the ring buffer logic works and what you are trying to do here.
The ringbuffers are per cpu structures right and you do not change cpus while performing operations on them? If not then the per cpu ops are not useful to you.
Trying to make a long story short:
This scheme where Steven moves buffers from one CPU to another, he only performs this operation when some other exclusion mechanism ensures that the buffer is not used for writing by the CPU when this move operation is done.
It is therefore correct, and needs local_t type to deal with the data structure hierarchy vs atomic exchange as I pointed in my email.
Mathieu
Quoted 4 lines
If you dont: How can you safely use the
local_toperations for the ringbuffer logic?
Signature · 2 lines
Mathieu Desnoyers OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F BA06 3F25 A8FE 3BAE 9A68
Re: [this_cpu_xx V8 11/16] Generic support for this_cpu_cmpxchg
Quoted 7 lines · Mathieu Desnoyers
This scheme where Steven moves buffers from one CPU to another, he only performs this operation when some other exclusion mechanism ensures that the buffer is not used for writing by the CPU when this move operation is done.
It is therefore correct, and needs
local_ttype to deal with the data structure hierarchy vs atomic exchange as I pointed in my email.
Yes this_cpu_xx does not seem to work for your scheme. Please review the patchset that I sent you instead.
PATCH v8 12/16 Add percpu cmpxchg operations
In reply to Christoph LameterThese are needed for the ringbuffer logic.
--- arch/x86/include/asm/percpu.h | 12 ++++++++++++ 1 file changed, 12 insertions(+)
arch/x86/include/asm/percpu.h +12 -0
@@ -201,6 +201,9 @@ do { \#define __this_cpu_xor_1 (pcp, val) percpu_to_op("xor", (pcp), val)#define __this_cpu_xor_2 (pcp, val) percpu_to_op("xor", (pcp), val)#define __this_cpu_xor_4 (pcp, val) percpu_to_op("xor", (pcp), val) +#define __this_cpu_cmpxchg_1 (pcp, old,new) cmpxchg_local(__this_cpu_ptr(pcp), old, new) +#define __this_cpu_cmpxchg_2 (pcp, old,new) cmpxchg_local(__this_cpu_ptr(pcp), old, new) +#define __this_cpu_cmpxchg_4 (pcp, old,new) cmpxchg_local(__this_cpu_ptr(pcp), old, new)#define this_cpu_read_1 (pcp) percpu_from_op("mov", (pcp), "m"(pcp))#define this_cpu_read_2 (pcp) percpu_from_op("mov", (pcp), "m"(pcp))
@@ -226,6 +229,9 @@ do { \#define this_cpu_xor_1 (pcp, val) percpu_to_op("xor", (pcp), val)#define this_cpu_xor_2 (pcp, val) percpu_to_op("xor", (pcp), val)#define this_cpu_xor_4 (pcp, val) percpu_to_op("xor", (pcp), val) +#define this_cpu_cmpxchg_1 (pcp, old,new) cmpxchg_local(this_cpu_ptr(pcp), old, new) +#define this_cpu_cmpxchg_2 (pcp, old,new) cmpxchg_local(this_cpu_ptr(pcp), old, new) +#define this_cpu_cmpxchg_4 (pcp, old,new) cmpxchg_local(this_cpu_ptr(pcp), old, new)#define irqsafe_cpu_add_1 (pcp, val) percpu_to_op("add", (pcp), val)#define irqsafe_cpu_add_2 (pcp, val) percpu_to_op("add", (pcp), val)
@@ -245,6 +251,9 @@ do { \#define irqsafe_cpu_xor_1 (pcp, val) percpu_to_op("xor", (pcp), val)#define irqsafe_cpu_xor_2 (pcp, val) percpu_to_op("xor", (pcp), val)#define irqsafe_cpu_xor_4 (pcp, val) percpu_to_op("xor", (pcp), val) +#define irqsafe_cpu_cmpxchg_1 (pcp, old,new) cmpxchg_local(this_cpu_ptr(pcp), old, new) +#define irqsafe_cpu_cmpxchg_2 (pcp, old,new) cmpxchg_local(this_cpu_ptr(pcp), old, new) +#define irqsafe_cpu_cmpxchg_4 (pcp, old,new) cmpxchg_local(this_cpu_ptr(pcp), old, new)/* * Per cpu atomic 64 bit operations are only available under 64 bit.
@@ -259,6 +268,7 @@ do { \#define __this_cpu_and_8 (pcp, val) percpu_to_op("and", (pcp), val)#define __this_cpu_or_8 (pcp, val) percpu_to_op("or", (pcp), val)#define __this_cpu_xor_8 (pcp, val) percpu_to_op("xor", (pcp), val) +#define __this_cpu_cmpxchg_8 (pcp, old,new) cmpxchg_local(__this_cpu_ptr(pcp), old, new)#define this_cpu_read_8 (pcp) percpu_from_op("mov", (pcp), "m"(pcp))#define this_cpu_write_8 (pcp, val) percpu_to_op("mov", (pcp), val)
@@ -268,6 +278,7 @@ do { \#define this_cpu_and_8 (pcp, val) percpu_to_op("and", (pcp), val)#define this_cpu_or_8 (pcp, val) percpu_to_op("or", (pcp), val)#define this_cpu_xor_8 (pcp, val) percpu_to_op("xor", (pcp), val) +#define this_cpu_cmpxchg_8 (pcp, old,new) cmpxchg_local(this_cpu_ptr(pcp), old, new)#define irqsafe_cpu_add_8 (pcp, val) percpu_to_op("add", (pcp), val)#define irqsafe_cpu_inc_8 (pcp) percpu_var_op("inc", (pcp))
@@ -275,6 +286,7 @@ do { \#define irqsafe_cpu_and_8 (pcp, val) percpu_to_op("and", (pcp), val)#define irqsafe_cpu_or_8 (pcp, val) percpu_to_op("or", (pcp), val)#define irqsafe_cpu_xor_8 (pcp, val) percpu_to_op("xor", (pcp), val) +#define irqsafe_cpu_cmpxchg_8 (pcp, old,new) cmpxchg_local(this_cpu_ptr(pcp), old, new)#endif
PATCH v8 13/16 Generic support for this_cpu_xchg
In reply to Christoph LameterProvide generic this_cpu_xchg() and all fallbacks.
--- include/linux/percpu.h | 95 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+)
include/linux/percpu.h +95 -0
@@ -499,6 +499,58 @@ static inline unsigned long this_cpu_cmp# define this_cpu_cmpxchg (pcp, old, new) __pcpu_size_call_return(__this_cpu_cmpxchg_, (old), (new))#endif +static inline unsigned long __this_cpu_xchg_generic (volatile void *pptr, +unsigned long new,int size) +{ +unsigned long prev; +volatile void *ptr =__this_cpu_ptr (pptr); + +switch (size) { +case 1 : prev = *(u8 *)ptr; + *(u8 *)ptr = (u8 )new; +break ; +case 2 : prev = *(u16 *)ptr; + *(u16 *)ptr = (u16 )new; +break ; +case 4 : prev = *(u32 *)ptr; + *(u32 *)ptr = (u32 )new; +break ; +case 8 : prev = *(u64 *)ptr; + *(u64 *)ptr = (u64 )new; +break ; +default : +__bad_size_call_parameter (); + } +return prev; +} + +static inline unsigned long this_cpu_xchg_generic (volatile void *ptr, +unsigned long new,int size) +{ +unsigned long prev; + +preempt_disable (); + prev =__this_cpu_xchg_generic (ptr, new, size); +preempt_enable (); +return prev; +} + +#ifndef this_cpu_xchg +# ifndef this_cpu_xchg_1 +# define this_cpu_xchg_1 (pcp, new) this_cpu_xchg_generic((pcp), (new), 1) +# endif +# ifndef this_cpu_xchg_2 +# define this_cpu_xchg_2 (pcp, new) this_cpu_xchg_generic((pcp), (new), 2) +# endif +# ifndef this_cpu_xchg_4 +# define this_cpu_xchg_4 (pcp, new) this_cpu_xchg_generic((pcp), (new), 4) +# endif +# ifndef this_cpu_xchg_8 +# define this_cpu_xchg_8 (pcp, new) this_cpu_xchg_generic((pcp), (new), 8) +# endif +# define this_cpu_xchg (pcp, new) __pcpu_size_call_return(__this_cpu_xchg_, (new)) +#endif +/* * Generic percpu operations that do not require preemption handling. * Either we do not care about races or the caller has the
@@ -666,6 +718,22 @@ do { \# define __this_cpu_cmpxchg (pcp, old, new) __pcpu_size_call_return(__this_cpu_cmpxchg_, (old), (new))#endif +#ifndef __this_cpu_xchg +# ifndef __this_cpu_xchg_1 +# define __this_cpu_xchg_1 (pcp, new) __this_cpu_xchg_generic((pcp), (new), 1) +# endif +# ifndef __this_cpu_xchg_2 +# define __this_cpu_xchg_2 (pcp, new) __this_cpu_xchg_generic((pcp), (new), 2) +# endif +# ifndef __this_cpu_xchg_4 +# define __this_cpu_xchg_4 (pcp, new) __this_cpu_xchg_generic((pcp), (new), 4) +# endif +# ifndef __this_cpu_xchg_8 +# define __this_cpu_xchg_8 (pcp, new) __this_cpu_xchg_generic((pcp), (new), 8) +# endif +# define __this_cpu_xchg (pcp, new) __pcpu_size_call_return(__this_cpu_xchg_, (new)) +#endif +/* * IRQ safe versions of the per cpu RMW operations. Note that these operations * are *not* safe against modification of the same variable from another
@@ -808,4 +876,31 @@ static inline unsigned long irqsafe_cpu_# define irqsafe_cpu_cmpxchg (pcp, old, new) __pcpu_size_call_return(irqsafe_cpu_cmpxchg_, (old), (new))#endif +static inline unsigned long irqsafe_cpu_xchg_generic (volatile void *ptr, +unsigned long new,int size) +{ +unsigned long flags, prev; + +local_irq_save (flags); + prev =__this_cpu_xchg_generic (ptr, new, size); +local_irq_restore (flags); +return prev; +} + +#ifndef irqsafe_cpu_xchg +# ifndef irqsafe_cpu_xchg_1 +# define irqsafe_cpu_xchg_1 (pcp, new) irqsafe_cpu_xchg_generic(((pcp), (new), 1) +# endif +# ifndef irqsafe_cpu_xchg_2 +# define irqsafe_cpu_xchg_2 (pcp, new) irqsafe_cpu_xchg_generic(((pcp), (new), 2) +# endif +# ifndef irqsafe_cpu_xchg_4 +# define irqsafe_cpu_xchg_4 (pcp, new) irqsafe_cpu_xchg_generic(((pcp), (new), 4) +# endif +# ifndef irqsafe_cpu_xchg_8 +# define irqsafe_cpu_xchg_8 (pcp, new) irqsafe_cpu_xchg_generic(((pcp), (new), 8) +# endif +# define irqsafe_cpu_xchg (pcp, new) __pcpu_size_call_return(irqsafe_cpu_xchg_, (new)) +#endif +#endif /* __LINUX_PERCPU_H */
PATCH v8 14/16 x86 percpu xchg operation
In reply to Christoph LameterXchg on x86 implies LOCK semantics and therefore is slow. Approximate it using this_cpu_cmpxchg().
--- arch/x86/include/asm/percpu.h | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+)
arch/x86/include/asm/percpu.h +21 -0
@@ -156,6 +156,15 @@ do { \ } \ }) +#define this_cpu_xchg_x86 (var, new) \ +({ __typeof(*var) __tmp_var,__tmp_old; \ + do { \ + __tmp_old = __this_cpu_read(var); \ + __tmp_var = __this_cpu_cmpxchg(var, __tmp_old, new); \ + } while (__tmp_var != __tmp_old); \ + __tmp_old; \ +}) +/* * percpu_read() makes gcc load the percpu variable every time it is * accessed while percpu_read_stable() allows the value to be cached.
@@ -204,6 +213,9 @@ do { \#define __this_cpu_cmpxchg_1 (pcp, old,new) cmpxchg_local(__this_cpu_ptr(pcp), old, new)#define __this_cpu_cmpxchg_2 (pcp, old,new) cmpxchg_local(__this_cpu_ptr(pcp), old, new)#define __this_cpu_cmpxchg_4 (pcp, old,new) cmpxchg_local(__this_cpu_ptr(pcp), old, new) +#define __this_cpu_xchg_1 (pcp, new) this_cpu_xchg_x86((pcp), new) +#define __this_cpu_xchg_2 (pcp, new) this_cpu_xchg_x86((pcp), new) +#define __this_cpu_xchg_4 (pcp, new) this_cpu_xchg_x86((pcp), new)#define this_cpu_read_1 (pcp) percpu_from_op("mov", (pcp), "m"(pcp))#define this_cpu_read_2 (pcp) percpu_from_op("mov", (pcp), "m"(pcp))
@@ -232,6 +244,9 @@ do { \#define this_cpu_cmpxchg_1 (pcp, old,new) cmpxchg_local(this_cpu_ptr(pcp), old, new)#define this_cpu_cmpxchg_2 (pcp, old,new) cmpxchg_local(this_cpu_ptr(pcp), old, new)#define this_cpu_cmpxchg_4 (pcp, old,new) cmpxchg_local(this_cpu_ptr(pcp), old, new) +#define this_cpu_xchg_1 (pcp, new) this_cpu_xchg_x86((pcp), new) +#define this_cpu_xchg_2 (pcp, new) this_cpu_xchg_x86((pcp), new) +#define this_cpu_xchg_4 (pcp, new) this_cpu_xchg_x86((pcp), new)#define irqsafe_cpu_add_1 (pcp, val) percpu_to_op("add", (pcp), val)#define irqsafe_cpu_add_2 (pcp, val) percpu_to_op("add", (pcp), val)
@@ -254,6 +269,9 @@ do { \#define irqsafe_cpu_cmpxchg_1 (pcp, old,new) cmpxchg_local(this_cpu_ptr(pcp), old, new)#define irqsafe_cpu_cmpxchg_2 (pcp, old,new) cmpxchg_local(this_cpu_ptr(pcp), old, new)#define irqsafe_cpu_cmpxchg_4 (pcp, old,new) cmpxchg_local(this_cpu_ptr(pcp), old, new) +#define irqsafe_cpu_xchg_1 (pcp, new) this_cpu_xchg_x86((pcp), new) +#define irqsafe_cpu_xchg_2 (pcp, new) this_cpu_xchg_x86((pcp), new) +#define irqsafe_cpu_xchg_4 (pcp, new) this_cpu_xchg_x86((pcp), new)/* * Per cpu atomic 64 bit operations are only available under 64 bit.
@@ -269,6 +287,7 @@ do { \#define __this_cpu_or_8 (pcp, val) percpu_to_op("or", (pcp), val)#define __this_cpu_xor_8 (pcp, val) percpu_to_op("xor", (pcp), val)#define __this_cpu_cmpxchg_8 (pcp, old,new) cmpxchg_local(__this_cpu_ptr(pcp), old, new) +#define __this_cpu_xchg_8 (pcp, new) this_cpu_xchg_x86((pcp), new)#define this_cpu_read_8 (pcp) percpu_from_op("mov", (pcp), "m"(pcp))#define this_cpu_write_8 (pcp, val) percpu_to_op("mov", (pcp), val)
@@ -279,6 +298,7 @@ do { \#define this_cpu_or_8 (pcp, val) percpu_to_op("or", (pcp), val)#define this_cpu_xor_8 (pcp, val) percpu_to_op("xor", (pcp), val)#define this_cpu_cmpxchg_8 (pcp, old,new) cmpxchg_local(this_cpu_ptr(pcp), old, new) +#define this_cpu_xchg_8 (pcp, new) this_cpu_xchg_x86((pcp), new)#define irqsafe_cpu_add_8 (pcp, val) percpu_to_op("add", (pcp), val)#define irqsafe_cpu_inc_8 (pcp) percpu_var_op("inc", (pcp))
@@ -287,6 +307,7 @@ do { \#define irqsafe_cpu_or_8 (pcp, val) percpu_to_op("or", (pcp), val)#define irqsafe_cpu_xor_8 (pcp, val) percpu_to_op("xor", (pcp), val)#define irqsafe_cpu_cmpxchg_8 (pcp, old,new) cmpxchg_local(this_cpu_ptr(pcp), old, new) +#define irqsafe_cpu_xchg_8 (pcp, new) this_cpu_xchg_x86((pcp), new)#endif
PATCH v8 15/16 Generic support for this_cpu_add_return()
In reply to Christoph LameterThis allows core code to use this_cpu_add_return() Provides all fallback scenarios.
--- include/linux/percpu.h | 90 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+)
include/linux/percpu.h +90 -0
@@ -551,6 +551,53 @@ static inline unsigned long this_cpu_xch# define this_cpu_xchg (pcp, new) __pcpu_size_call_return(__this_cpu_xchg_, (new))#endif +static inline unsigned long __this_cpu_add_return_generic (volatile void *ptr, +unsigned long val,int size) +{ +unsigned long result; + +switch (size) { +case 1 : result = (*__this_cpu_ptr ((u8 *)ptr) += val); +break ; +case 2 : result = (*__this_cpu_ptr ((u16 *)ptr) += val); +break ; +case 4 : result = (*__this_cpu_ptr ((u32 *)ptr) += val); +break ; +case 8 : result = (*__this_cpu_ptr ((u64 *)ptr) += val); +break ; +default : +__bad_size_call_parameter (); + } +return result; +} + +static inline unsigned long this_cpu_add_return_generic (volatile void *ptr, +unsigned long val,int size) +{ +unsigned long result; + +preempt_disable (); + result =__this_cpu_add_return_generic (ptr, val, size); +preempt_enable (); +return result; +} + +#ifndef this_cpu_add_return +# ifndef this_cpu_add_return_1 +# define this_cpu_add_return_1 (pcp, val) this_cpu_add_return_generic((pcp), (val), 1) +# endif +# ifndef this_cpu_add_return_2 +# define this_cpu_add_return_2 (pcp, val) this_cpu_add_return_generic((pcp), (val), 2) +# endif +# ifndef this_cpu_add_return_4 +# define this_cpu_add_return_4 (pcp, val) this_cpu_add_return_generic((pcp), (val), 4) +# endif +# ifndef this_cpu_add_return_8 +# define this_cpu_add_return_8 (pcp, val) this_cpu_add_return_generic((pcp), (val), 8) +# endif +# define this_cpu_add_return (pcp, val) __pcpu_size_call_return(__this_cpu_add_return_, (val)) +#endif +/* * Generic percpu operations that do not require preemption handling. * Either we do not care about races or the caller has the
@@ -734,6 +781,22 @@ do { \# define __this_cpu_xchg (pcp, new) __pcpu_size_call_return(__this_cpu_xchg_, (new))#endif +#ifndef __this_cpu_add_return +# ifndef __this_cpu_add_return_1 +# define __this_cpu_add_return_1 (pcp, val) __this_cpu_add_return_generic((pcp), (val), 1) +# endif +# ifndef __this_cpu_add_return_2 +# define __this_cpu_add_return_2 (pcp, val) __this_cpu_add_return_generic((pcp), (val), 2) +# endif +# ifndef __this_cpu_add_return_4 +# define __this_cpu_add_return_4 (pcp, val) __this_cpu_add_return_generic((pcp), (val), 4) +# endif +# ifndef __this_cpu_add_return_8 +# define __this_cpu_add_return_8 (pcp, val) __this_cpu_add_return_generic((pcp), (val), 8) +# endif +# define __this_cpu_add_return (pcp, val) __pcpu_size_call_return(__this_cpu_add_return_, (val)) +#endif +/* * IRQ safe versions of the per cpu RMW operations. Note that these operations * are *not* safe against modification of the same variable from another
@@ -903,4 +966,31 @@ static inline unsigned long irqsafe_cpu_# define irqsafe_cpu_xchg (pcp, new) __pcpu_size_call_return(irqsafe_cpu_xchg_, (new))#endif +static inline unsigned long irqsafe_cpu_add_return_generic (volatile void *ptr, +unsigned long val,int size) +{ +unsigned long flags, result; + +local_irq_save (flags); + result =__this_cpu_add_return_generic (ptr, val, size); +local_irq_restore (flags); +return result; +} + +#ifndef irqsafe_cpu_add_return +# ifndef irqsafe_cpu_add_return_1 +# define irqsafe_cpu_add_return_1 (pcp, val) irqsafe_cpu_add_return_generic(((pcp), (val), 1) +# endif +# ifndef irqsafe_cpu_add_return_2 +# define irqsafe_cpu_add_return_2 (pcp, val) irqsafe_cpu_add_return_generic(((pcp), (val), 2) +# endif +# ifndef irqsafe_cpu_add_return_4 +# define irqsafe_cpu_add_return_4 (pcp, val) irqsafe_cpu_add_return_generic(((pcp), (val), 4) +# endif +# ifndef irqsafe_cpu_add_return_8 +# define irqsafe_cpu_add_return_8 (pcp, val) irqsafe_cpu_add_return_generic(((pcp), (val), 8) +# endif +# define irqsafe_cpu_add_return (pcp, val) __pcpu_size_call_return(irqsafe_cpu_add_return_, (val)) +#endif +#endif /* __LINUX_PERCPU_H */
PATCH v8 16/16 x86 support for this_cpu_add_return
In reply to Christoph Lameterx86 had the xadd instruction that can be use to have a percpu atomic increment and return instruction.
--- arch/x86/include/asm/percpu.h | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+)
arch/x86/include/asm/percpu.h +35 -0
@@ -165,6 +165,35 @@ do { \ __tmp_old; \ }) +#define this_cpu_add_return_x86 (var, val) \ +({ \ + typeof(var) pfo_ret__; \ + switch (sizeof(var)) { \ + case 1: \ + asm("xaddb %0,"__percpu_arg(1) \ + : "+r" (val), "+m" (var) \ + : "qi" ((pto_T__)(val))); \ + break; \ + case 2: \ + asm("xaddw %0,"__percpu_arg(1) \ + : "+r" (val), "+m" (var) \ + : "=r" (pfo_ret__) \ + break; \ + case 4: \ + asm("xaddl %0,"__percpu_arg(1) \ + : "+r" (val), "+m" (var) \ + : "=r" (pfo_ret__) \ + break; \ + case 8: \ + asm("xaddq %0,"__percpu_arg(1) \ + : "+r" (val), "+m" (var) \ + : "re" ((pto_T__)(val))); \ + break; \ + default: __bad_percpu_size(); \ + } \ + pfo_ret__; \ +}) +/* * percpu_read() makes gcc load the percpu variable every time it is * accessed while percpu_read_stable() allows the value to be cached.
@@ -216,6 +245,9 @@ do { \#define __this_cpu_xchg_1 (pcp, new) this_cpu_xchg_x86((pcp), new)#define __this_cpu_xchg_2 (pcp, new) this_cpu_xchg_x86((pcp), new)#define __this_cpu_xchg_4 (pcp, new) this_cpu_xchg_x86((pcp), new) +#define __this_cpu_add_return_1 (pcp, val) this_cpu_add_return_x86((pcp), val) +#define __this_cpu_add_return_2 (pcp, val) this_cpu_add_return_x86((pcp), val) +#define __this_cpu_add_return_4 (pcp, val) this_cpu_add_return_x86((pcp), val)#define this_cpu_read_1 (pcp) percpu_from_op("mov", (pcp), "m"(pcp))#define this_cpu_read_2 (pcp) percpu_from_op("mov", (pcp), "m"(pcp))
@@ -272,6 +304,9 @@ do { \#define irqsafe_cpu_xchg_1 (pcp, new) this_cpu_xchg_x86((pcp), new)#define irqsafe_cpu_xchg_2 (pcp, new) this_cpu_xchg_x86((pcp), new)#define irqsafe_cpu_xchg_4 (pcp, new) this_cpu_xchg_x86((pcp), new) +#define irqsafe_cpu_add_return_1 (pcp, val) this_cpu_add_return_x86((pcp), val) +#define irqsafe_cpu_add_return_2 (pcp, val) this_cpu_add_return_x86((pcp), val) +#define irqsafe_cpu_add_return_4 (pcp, val) this_cpu_add_return_x86((pcp), val)/* * Per cpu atomic 64 bit operations are only available under 64 bit.