Per cpu atomics in core allocators and cleanup
Subsystems (14)
v7 cover Per cpu atomics in core allocators and cleanup
Leftovers from the earlier patchset. Mostly applications of per cpu counters to core components.
After this patchset there will be only one user of local_t left: Mathieu's trace ringbuffer. Does it really need these ops?
V6->V7
- Drop patches merged in 2.6.33 merge cycle
- Drop risky slub patches
V5->V6:
- Drop patches merged by Tejun.
- Drop irqless slub fastpath for now.
- Patches against Tejun percpu for-next branch.
V4->V5:
- Avoid setup_per_cpu_area() modifications and fold the remainder of the patch into the page allocator patch.
- Irq disable / per cpu ptr fixes for page allocator patch.
V3->V4:
- Fix various macro definitions.
- Provide experimental percpu based fastpath that does not disable interrupts for SLUB.
V2->V3:
- Available via git tree against latest upstream from
git://git.kernel.org/pub/scm/linux/kernel/git/christoph/percpu.git linus
- Rework SLUB per cpu operations. Get rid of dynamic DMA slab creation for CONFIG_ZONE_DMA
- Create fallback framework so that 64 bit ops on 32 bit platforms can fallback to the use of preempt or interrupt disable. 64 bit platforms can use 64 bit atomic per cpu ops.
V1->V2:
- Various minor fixes
- Add SLUB conversion
- Add Page allocator conversion
- Patch against the git tree of today
PATCH v7 1/8 this_cpu_ops: page allocator conversion
Use the per cpu allocator functionality to avoid per cpu arrays in struct zone.
This drastically reduces the size of struct zone for systems with large amounts of processors and allows placement of critical variables of struct zone in one cacheline even on very large systems.
Another effect is that the pagesets of one processor are placed near one another. If multiple pagesets from different zones fit into one cacheline then additional cacheline fetches can be avoided on the hot paths when allocating memory from multiple zones.
Bootstrap becomes simpler if we use the same scheme for UP, SMP, NUMA. #ifdefs are reduced and we can drop the zone_pcp macro.
Hotplug handling is also simplified since cpu alloc can bring up and shut down cpu areas for a specific cpu as a whole. So there is no need to allocate or free individual pagesets.
V4-V5:
- Fix up cases where per_cpu_ptr is called before irq disable
- Integrate the bootstrap logic that was separate before.
--- include/linux/mm.h | 4 - include/linux/mmzone.h | 12 --- mm/page_alloc.c | 187 ++++++++++++++++++------------------------------- mm/vmstat.c | 14 ++- 4 files changed, 81 insertions(+), 136 deletions(-)
include/linux/mm.h +0 -4
@@ -1061,11 +1061,7 @@ extern void si_meminfo(struct sysinfo *extern void si_meminfo_node (struct sysinfo *val,int nid);extern int after_bootmem; -#ifdef CONFIG_NUMA extern void setup_per_cpu_pageset (void ); -#else -static inline void setup_per_cpu_pageset (void ) {} -#endif extern void zone_pcp_update (struct zone *zone);
include/linux/mmzone.h +2 -10
@@ -184,13 +184,7 @@ struct per_cpu_pageset {s8 stat_threshold;s8 vm_stat_diff[NR_VM_ZONE_STAT_ITEMS ]; #endif -}____cacheline_aligned_in_smp ; - -#ifdef CONFIG_NUMA -#define zone_pcp (__z, __cpu) ((__z)->pageset[(__cpu)]) -#else -#define zone_pcp (__z, __cpu) (&(__z)->pageset[(__cpu)]) -#endif +};#endif /* !__GENERATING_BOUNDS.H */
@@ -306,10 +300,8 @@ struct zone {*/ unsigned long min_unmapped_pages;unsigned long min_slab_pages; -struct per_cpu_pageset *pageset[NR_CPUS ]; -#else -struct per_cpu_pageset pageset[NR_CPUS ];#endif +struct per_cpu_pageset *pageset;/* * free areas of different sizes */
mm/page_alloc.c +71 -116
313 lines not on this page · raw message
mm/vmstat.c +8 -6
@@ -139,7 +139,8 @@ static void refresh_zone_stat_thresholds threshold =calculate_threshold (zone);for_each_online_cpu (cpu) -zone_pcp (zone, cpu)->stat_threshold = threshold; +per_cpu_ptr (zone->pageset, cpu)->stat_threshold + = threshold; } }
@@ -149,7 +150,8 @@ static void refresh_zone_stat_thresholdsvoid __mod_zone_page_state (struct zone *zone,enum zone_stat_item item,int delta) { -struct per_cpu_pageset *pcp =zone_pcp (zone,smp_processor_id ()); +struct per_cpu_pageset *pcp =this_cpu_ptr (zone->pageset); +s8 *p = pcp->vm_stat_diff + item;long x;
@@ -202,7 +204,7 @@ EXPORT_SYMBOL(mod_zone_page_state);*/ void __inc_zone_state (struct zone *zone,enum zone_stat_item item) { -struct per_cpu_pageset *pcp =zone_pcp (zone,smp_processor_id ()); +struct per_cpu_pageset *pcp =this_cpu_ptr (zone->pageset);s8 *p = pcp->vm_stat_diff + item; (*p)++;
@@ -223,7 +225,7 @@ EXPORT_SYMBOL(__inc_zone_page_state);void __dec_zone_state (struct zone *zone,enum zone_stat_item item) { -struct per_cpu_pageset *pcp =zone_pcp (zone,smp_processor_id ()); +struct per_cpu_pageset *pcp =this_cpu_ptr (zone->pageset);s8 *p = pcp->vm_stat_diff + item; (*p)--;
@@ -300,7 +302,7 @@ void refresh_cpu_vm_stats(int cpu)for_each_populated_zone (zone) {struct per_cpu_pageset *p; - p =zone_pcp (zone, cpu); + p =per_cpu_ptr (zone->pageset, cpu);for (i =0 ; i <NR_VM_ZONE_STAT_ITEMS ; i++)if (p->vm_stat_diff[i]) {
@@ -738,7 +740,7 @@ static void zoneinfo_show_print(struct sfor_each_online_cpu (i) {struct per_cpu_pageset *pageset; - pageset =zone_pcp (zone, i); + pageset =per_cpu_ptr (zone->pageset, i);seq_printf (m,"\n cpu: %i" "\n count: %i"
Tejun Heo
maintainerRe: [this_cpu_xx V7 1/8] this_cpu_ops: page allocator conversion
Hello,
mm/page_alloc.c void zone_pcp_update(struct zone *zone) · line 3237
Quoted 6 lines · Christoph Lameter
static
__meminitvoidzone_pcp_init(struct zone *zone){mm/page_alloc.c +2
-2-int cpu; -unsigned long batch =zone_batchsize (zone); +/* Use boot pagesets until we have the per cpu allocator up */ + zone->pageset = &per_cpu_var (boot_pageset);
Ummm... this scares me a little bit. Before it was a pointer to statically allocated area which can be used from basically anywhere. Now, it's being initialized to a percpu pointer which won't be available before setup_per_cpu_areas() is complete and the above initialization takes place from setup_arch() which is before percpu initialization. I don't think there's anything which would access page allocator between the two places, but it still seems a bit risky. Maybe it's better to keep the boot_pageset a static array? Or am I misunderstanding something?
Thanks.
Signature · 1 line
tejun
Re: [this_cpu_xx V7 1/8] this_cpu_ops: page allocator conversion
Quoted 19 lines · Tejun Heo
Hello,
mm/page_alloc.c void zone_pcp_update(struct zone *zone) · line 3237
Quoted 6 lines · Christoph Lameter
static
__meminitvoidzone_pcp_init(struct zone *zone){mm/page_alloc.c +2
-2-int cpu; -unsigned long batch =zone_batchsize (zone); +/* Use boot pagesets until we have the per cpu allocator up */ + zone->pageset = &per_cpu_var (boot_pageset);Ummm... this scares me a little bit. Before it was a pointer to statically allocated area which can be used from basically anywhere. Now, it's being initialized to a percpu pointer which won't be available before
setup_per_cpu_areas()is complete and the above initialization takes place fromsetup_arch()which is before percpu initialization. I don't think there's anything which would access page allocator between the two places, but it still seems a bit risky. Maybe it's better to keep theboot_pageseta static array? Or am I misunderstanding something?
A static array would have to be dimensioned to NR_CPUS. That is one thing we are trying to avoid.
The assignment of the pageset "percpu" pointer does not mean that the pcp is usable. It must first be properly initialized through setup_pageset().
setup_pageset() is run for each cpu. zone->pageset is the same for all cpus that is why it is in zone_pcp_init() and not in setup_pageset().
The boot pageset initialization was moved into __build_all_zonelists(). We could move the zone->pageset initialization there too?
Tejun Heo
maintainerRe: [this_cpu_xx V7 1/8] this_cpu_ops: page allocator conversion
Hello, Christoph.
Quoted 11 lines · Christoph Lameter
A static array would have to be dimensioned to
NR_CPUS. That is one thing we are trying to avoid.The assignment of the pageset "percpu" pointer does not mean that the pcp is usable. It must first be properly initialized through
setup_pageset().
setup_pageset()is run for each cpu.zone->pagesetis the same for all cpus that is why it is inzone_pcp_init()and not insetup_pageset().The boot pageset initialization was moved into
__build_all_zonelists(). We could move thezone->pagesetinitialization there too?
Maybe that is a bit less scary. (at least for me :-) The reason why I'm a bit worried is that different architectures handle percpu pointers differently before setup_per_cpu_areas(). x86 sets up the offsets and stuff such that cpu0 can access the original percpu section in the kernel image. ia64 sets up everything properly way before setup_per_cpu_areas() and in some archs percpu pointers are completely invalid before setup_per_cpu_areas(). So, percpu pointer being handled in generic code which is being called before percpu setup is a bit worrying.
Another thing is that there were attempts to simplify memory initialization stages such that bootmem is removed and page / k* allocators can be used earlier which kind of puts percpu allocator in the dependency loop but I don't think it's something we need to worry about at this point.
Thanks.
Signature · 1 line
tejun
Re: [this_cpu_xx V7 1/8] this_cpu_ops: page allocator conversion
Quoted 4 lines · Tejun Heo
Quoted 2 lines
The boot pageset initialization was moved into
__build_all_zonelists(). We could move thezone->pagesetinitialization there too?Maybe that is a bit less scary. (at least for me :-) The reason why
Wel sorry moving the ->pageset assign to __build_all_zonelists does not work since __build_all_zonelists does not scan through
zones. zone_pcp_init is called reliable first for all zones. I think just leave it at is.
Quoted 8 lines
I'm a bit worried is that different architectures handle percpu pointers differently before
setup_per_cpu_areas(). x86 sets up the offsets and stuff such that cpu0 can access the original percpu section in the kernel image. ia64 sets up everything properly way beforesetup_per_cpu_areas()and in some archs percpu pointers are completely invalid beforesetup_per_cpu_areas(). So, percpu pointer being handled in generic code which is being called before percpu setup is a bit worrying.
True but we are not dereferencing a per cpu pointer here. It is simply the assignment of the unreferenced native per cpu address generated by the linker. This address is unaffected by allocator bootstrap.
Quoted 5 lines
Another thing is that there were attempts to simplify memory initialization stages such that bootmem is removed and page / k* allocators can be used earlier which kind of puts percpu allocator in the dependency loop but I don't think it's something we need to worry about at this point.
We already worried about this earlier. The initialization we are talking about does not require the per cpu allocator to be up. It just requires the static per cpu areas to function with a single pcp while the zonelists are being built.
The assignment of the final per cpu areas (dynamically allocated) occurs after all the other memory allocators have been bootstrapped.
Tejun Heo
maintainerRe: [this_cpu_xx V7 1/8] this_cpu_ops: page allocator conversion
Hello,
Quoted 9 lines · Christoph Lameter
Quoted 4 lines
Quoted 2 lines
The boot pageset initialization was moved into
__build_all_zonelists(). We could move thezone->pagesetinitialization there too?Maybe that is a bit less scary. (at least for me :-) The reason why
Wel sorry moving the ->pageset assign to
__build_all_zonelistsdoes not work since__build_all_zonelistsdoes not scan through
zones.zone_pcp_initis called reliable first for all zones. I think just leave it at is.
Ah, well, alright.
Quoted 9 lines
Quoted 8 lines
I'm a bit worried is that different architectures handle percpu pointers differently before
setup_per_cpu_areas(). x86 sets up the offsets and stuff such that cpu0 can access the original percpu section in the kernel image. ia64 sets up everything properly way beforesetup_per_cpu_areas()and in some archs percpu pointers are completely invalid beforesetup_per_cpu_areas(). So, percpu pointer being handled in generic code which is being called before percpu setup is a bit worrying.
Quoted 3 lines
True but we are not dereferencing a per cpu pointer here. It is simply the assignment of the unreferenced native per cpu address generated by the linker. This address is unaffected by allocator bootstrap.
Yeap, the code is fine. Probably I'm just being a bit paranoid. If you don't mind, can you please add a comment pointing out that the percpu pointer isn't dereferenced before later initialization is done which happens after percpu initialization?
Quoted 2 lines
The assignment of the final per cpu areas (dynamically allocated) occurs after all the other memory allocators have been bootstrapped.
Great, thanks for the explanation.
Signature · 1 line
tejun
PATCH v7 2/8 this_cpu ops: Remove pageset_notifier
In reply to Christoph LameterRemove the pageset notifier since it only marks that a processor exists on a specific node. Move that code into the vmstat notifier.
--- mm/page_alloc.c | 28 ---------------------------- mm/vmstat.c | 1 + 2 files changed, 1 insertion(+), 28 deletions(-)
mm/vmstat.c +1 -0
@@ -905,6 +905,7 @@ static int __cpuinit vmstat_cpuup_callbacase CPU_ONLINE :case CPU_ONLINE_FROZEN :start_cpu_timer (cpu); +node_set_state (cpu_to_node (cpu),N_CPU );break ;case CPU_DOWN_PREPARE :case CPU_DOWN_PREPARE_FROZEN :
mm/page_alloc.c +0 -28
@@ -3107,27 +3107,6 @@ static void setup_pagelist_highmark(stru pcp->batch =PAGE_SHIFT *8 ; } - -static int __cpuinit pageset_cpuup_callback (struct notifier_block *nfb, -unsigned long action, -void *hcpu) -{ -int cpu = (long )hcpu; - -switch (action) { -case CPU_UP_PREPARE : -case CPU_UP_PREPARE_FROZEN : -node_set_state (cpu_to_node (cpu),N_CPU ); -break ; -default : -break ; - } -return NOTIFY_OK ; -} - -static struct notifier_block __cpuinitdata pageset_notifier = - { &pageset_cpuup_callback,NULL ,0 }; -/* * Allocate per cpu pagesets and initialize them. * Before this call only boot pagesets were available.
@@ -3153,13 +3132,6 @@ void __init setup_per_cpu_pageset(void) percpu_pagelist_fraction)); } } - -/* -* The boot cpu is always the first active. -* The boot node has a processor -*/ -node_set_state (cpu_to_node (smp_processor_id ()),N_CPU ); -register_cpu_notifier (&pageset_notifier); }static noinline __init_refok
PATCH v7 3/8 Use this_cpu operations in slub
In reply to Christoph LameterUsing per cpu allocations removes the needs for the per cpu arrays in the kmem_cache struct. These could get quite big if we have to support systems with thousands of cpus. The use of this_cpu_xx operations results in:
1. The size of kmem_cache for SMP configuration shrinks since we will only need 1 pointer instead of NR_CPUS. The same pointer can be used by all processors. Reduces cache footprint of the allocator.
2. We can dynamically size kmem_cache according to the actual nodes in the system meaning less memory overhead for configurations that may potentially support up to 1k NUMA nodes / 4k cpus.
3. We can remove the diddle widdle with allocating and releasing of kmem_cache_cpu structures when bringing up and shutting down cpus. The cpu alloc logic will do it all for us. Removes some portions of the cpu hotplug functionality.
4. Fastpath performance increases since per cpu pointer lookups and address calculations are avoided.
--- include/linux/slub_def.h | 6 - mm/slub.c | 200 +++++++++++------------------------------------ 2 files changed, 48 insertions(+), 158 deletions(-)
include/linux/slub_def.h +1 -5
@@ -69,6 +69,7 @@ struct kmem_cache_order_objects {* Slab cache management. */ struct kmem_cache { +struct kmem_cache_cpu *cpu_slab;/* Used for retriving partial slabs etc */ unsigned long flags;int size;/* The size of an object including meta data */
@@ -104,11 +105,6 @@ struct kmem_cache {int remote_node_defrag_ratio;struct kmem_cache_node *node[MAX_NUMNODES ];#endif -#ifdef CONFIG_SMP -struct kmem_cache_cpu *cpu_slab[NR_CPUS ]; -#else -struct kmem_cache_cpu cpu_slab; -#endif };/*
mm/slub.c +47 -153
339 lines not on this page · raw message
PATCH v7 4/8 SLUB: Get rid of dynamic DMA kmalloc cache allocation
In reply to Christoph LameterDynamic DMA kmalloc cache allocation is troublesome since the new percpu allocator does not support allocations in atomic contexts. Reserve some statically allocated kmalloc_cpu structures instead.
--- include/linux/slub_def.h | 19 +++++++++++-------- mm/slub.c | 24 ++++++++++-------------- 2 files changed, 21 insertions(+), 22 deletions(-)
include/linux/slub_def.h +11 -8
@@ -131,11 +131,21 @@ struct kmem_cache {#define SLUB_PAGE_SHIFT (PAGE_SHIFT + 2) +#ifdef CONFIG_ZONE_DMA +#define SLUB_DMA __GFP_DMA +/* Reserve extra caches for potential DMA use */ +#define KMALLOC_CACHES (2 * SLUB_PAGE_SHIFT - 6) +#else +/* Disable DMA functionality */ +#define SLUB_DMA (__force gfp_t)0 +#define KMALLOC_CACHES SLUB_PAGE_SHIFT +#endif +/* * We keep the general caches in an array of slab caches that are used for * 2^x bytes of allocations. */ -extern struct kmem_cache kmalloc_caches[SLUB_PAGE_SHIFT ]; +extern struct kmem_cache kmalloc_caches[KMALLOC_CACHES ];/* * Sorry that the following has to be that ugly but some versions of GCC
@@ -203,13 +213,6 @@ static __always_inline struct kmem_cachereturn &kmalloc_caches[index]; } -#ifdef CONFIG_ZONE_DMA -#define SLUB_DMA __GFP_DMA -#else -/* Disable DMA functionality */ -#define SLUB_DMA (__force gfp_t)0 -#endif -void *kmem_cache_alloc (struct kmem_cache *,gfp_t );void *__kmalloc (size_t size,gfp_t flags);
mm/slub.c +10 -14
@@ -2092,7 +2092,7 @@ static inline int alloc_kmem_cache_cpus( {int cpu; -if (s < kmalloc_caches +SLUB_PAGE_SHIFT && s >= kmalloc_caches) +if (s < kmalloc_caches +KMALLOC_CACHES && s >= kmalloc_caches)/* * Boot time creation of the kmalloc array. Use static per cpu data * since the per cpu allocator is not available yet.
@@ -2539,7 +2539,7 @@ EXPORT_SYMBOL(kmem_cache_destroy);* Kmalloc subsystem *******************************************************************/ -struct kmem_cache kmalloc_caches[SLUB_PAGE_SHIFT] __cacheline_aligned; +struct kmem_cache kmalloc_caches[KMALLOC_CACHES] __cacheline_aligned;EXPORT_SYMBOL (kmalloc_caches);static int __init setup_slub_min_order (char *str)
@@ -2629,6 +2629,7 @@ static noinline struct kmem_cache *dma_kchar *text;size_t realsize;unsigned long slabflags; +int i; s = kmalloc_caches_dma[index];if (s)
@@ -2649,18 +2650,13 @@ static noinline struct kmem_cache *dma_k text =kasprintf (flags & ~SLUB_DMA ,"kmalloc_dma-%d" , (unsigned int )realsize); -if (flags & __GFP_WAIT) - s =kmalloc (kmem_size, flags & ~SLUB_DMA ); -else { -int i; + s =NULL ; +for (i =0 ; i <KMALLOC_CACHES ; i++) +if (!kmalloc_caches[i].size) +break ; - s =NULL ; -for (i =0 ; i <SLUB_PAGE_SHIFT ; i++) -if (kmalloc_caches[i].size) { - s = kmalloc_caches + i; -break ; - } - } +BUG_ON (i >=KMALLOC_CACHES ); + s = kmalloc_caches + i;/* * Must defer sysfs creation to a workqueue because we don't know
@@ -2674,7 +2670,7 @@ static noinline struct kmem_cache *dma_kif (!s || !text || !kmem_cache_open (s, flags, text, realsize,ARCH_KMALLOC_MINALIGN , slabflags,NULL )) { -kfree (s); + s->size =0 ;kfree (text); goto unlock_out; }
PATCH v7 5/8 this_cpu: Remove slub kmem_cache fields
In reply to Christoph LameterRemove the fields in struct kmem_cache_cpu that were used to cache data from struct kmem_cache when they were in different cachelines. The cacheline that holds the per cpu array pointer now also holds these values. We can cut down the struct kmem_cache_cpu size to almost half.
The get_freepointer() and set_freepointer() functions that used to be only intended for the slow path now are also useful for the hot path since access to the size field does not require accessing an additional cacheline anymore. This results in consistent use of functions for setting the freepointer of objects throughout SLUB.
Also we initialize all possible kmem_cache_cpu structures when a slab is created. No need to initialize them when a processor or node comes online.
--- include/linux/slub_def.h | 2 - mm/slub.c | 76 ++++++++++------------------------------------- 2 files changed, 17 insertions(+), 61 deletions(-)
include/linux/slub_def.h +0 -2
@@ -38,8 +38,6 @@ struct kmem_cache_cpu {void **freelist;/* Pointer to first free per cpu object */ struct page *page;/* The slab from which we are allocating */ int node;/* The node of the page (or -1 for debug) */ -unsigned int offset;/* Freepointer offset (in word units) */ -unsigned int objsize;/* Size of an object (from kmem_cache) */ #ifdef CONFIG_SLUB_STATS unsigned stat[NR_SLUB_STAT_ITEMS ];#endif
mm/slub.c +17 -59
190 lines not on this page · raw message
PATCH v7 6/8 Make slub statistics use this_cpu_inc
In reply to Christoph Lameterthis_cpu_inc() translates into a single instruction on x86 and does not need any register. So use it in stat(). We also want to avoid the calculation of the per cpu kmem_cache_cpu structure pointer. So pass a kmem_cache pointer instead of a kmem_cache_cpu pointer.
--- mm/slub.c | 43 ++++++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 23 deletions(-)
mm/slub.c +20 -23
@@ -217,10 +217,10 @@ static inline void sysfs_slab_remove(str#endif -static inline void stat (struct kmem_cache_cpu *c,enum stat_item si) +static inline void stat (struct kmem_cache *s,enum stat_item si) {#ifdef CONFIG_SLUB_STATS - c->stat[si]++; +__this_cpu_inc (s->cpu_slab->stat[si]);#endif }
@@ -1108,7 +1108,7 @@ static struct page *allocate_slab(structif (!page)return NULL ; -stat (this_cpu_ptr (s->cpu_slab),ORDER_FALLBACK ); +stat (s,ORDER_FALLBACK ); } if (kmemcheck_enabled
@@ -1406,23 +1406,22 @@ static struct page *get_partial(struct kstatic void unfreeze_slab (struct kmem_cache *s,struct page *page,int tail) {struct kmem_cache_node *n =get_node (s,page_to_nid (page)); -struct kmem_cache_cpu *c =this_cpu_ptr (s->cpu_slab);__ClearPageSlubFrozen (page);if (page->inuse) {if (page->freelist) {add_partial (n, page, tail); -stat (c, tail ?DEACTIVATE_TO_TAIL :DEACTIVATE_TO_HEAD ); +stat (s, tail ?DEACTIVATE_TO_TAIL :DEACTIVATE_TO_HEAD ); }else { -stat (c,DEACTIVATE_FULL ); +stat (s,DEACTIVATE_FULL );if (SLABDEBUG &&PageSlubDebug (page) && (s->flags &SLAB_STORE_USER ))add_full (n, page); }slab_unlock (page); }else { -stat (c,DEACTIVATE_EMPTY ); +stat (s,DEACTIVATE_EMPTY );if (n->nr_partial < s->min_partial) {/* * Adding an empty slab to the partial slabs in order
@@ -1438,7 +1437,7 @@ static void unfreeze_slab(struct kmem_caslab_unlock (page); }else {slab_unlock (page); -stat (__this_cpu_ptr (s->cpu_slab),FREE_SLAB ); +stat (s,FREE_SLAB );discard_slab (s, page); } }
@@ -1453,7 +1452,7 @@ static void deactivate_slab(struct kmem_int tail =1 ;if (page->freelist) -stat (c,DEACTIVATE_REMOTE_FREES ); +stat (s,DEACTIVATE_REMOTE_FREES );/* * Merge cpu freelist into slab freelist. Typically we get here * because both freelists are empty. So this is unlikely
@@ -1479,7 +1478,7 @@ static void deactivate_slab(struct kmem_static inline void flush_slab (struct kmem_cache *s,struct kmem_cache_cpu *c) { -stat (c,CPUSLAB_FLUSH ); +stat (s,CPUSLAB_FLUSH );slab_lock (c->page);deactivate_slab (s, c); }
@@ -1619,7 +1618,7 @@ static void *__slab_alloc(struct kmem_caif (unlikely (!node_match (c, node))) goto another_slab; -stat (c,ALLOC_REFILL ); +stat (s,ALLOC_REFILL ); load_freelist: object = c->page->freelist;
@@ -1634,7 +1633,7 @@ load_freelist: c->node =page_to_nid (c->page); unlock_out:slab_unlock (c->page); -stat (c,ALLOC_SLOWPATH ); +stat (s,ALLOC_SLOWPATH );return object; another_slab:
@@ -1644,7 +1643,7 @@ new_slab: new =get_partial (s, gfpflags, node);if (new) { c->page = new; -stat (c,ALLOC_FROM_PARTIAL ); +stat (s,ALLOC_FROM_PARTIAL ); goto load_freelist; }
@@ -1658,7 +1657,7 @@ new_slab:if (new) { c =__this_cpu_ptr (s->cpu_slab); -stat (c,ALLOC_SLAB ); +stat (s,ALLOC_SLAB );if (c->page)flush_slab (s, c);slab_lock (new);
@@ -1713,7 +1712,7 @@ static __always_inline void *slab_alloc(else { c->freelist =get_freepointer (s, object); -stat (c,ALLOC_FASTPATH ); +stat (s,ALLOC_FASTPATH ); }local_irq_restore (flags);
@@ -1780,10 +1779,8 @@ static void __slab_free(struct kmem_cach {void *prior;void **object = (void *)x; -struct kmem_cache_cpu *c; - c =__this_cpu_ptr (s->cpu_slab); -stat (c,FREE_SLOWPATH ); +stat (s,FREE_SLOWPATH );slab_lock (page);if (unlikely (SLABDEBUG &&PageSlubDebug (page)))
@@ -1796,7 +1793,7 @@ checks_ok: page->inuse--;if (unlikely (PageSlubFrozen (page))) { -stat (c,FREE_FROZEN ); +stat (s,FREE_FROZEN ); goto out_unlock; }
@@ -1809,7 +1806,7 @@ checks_ok:*/ if (unlikely (!prior)) {add_partial (get_node (s,page_to_nid (page)), page,1 ); -stat (c,FREE_ADD_PARTIAL ); +stat (s,FREE_ADD_PARTIAL ); } out_unlock:
@@ -1822,10 +1819,10 @@ slab_empty:* Slab still on the partial list. */ remove_partial (s, page); -stat (c,FREE_REMOVE_PARTIAL ); +stat (s,FREE_REMOVE_PARTIAL ); }slab_unlock (page); -stat (c,FREE_SLAB ); +stat (s,FREE_SLAB );discard_slab (s, page);return ;
@@ -1863,7 +1860,7 @@ static __always_inline void slab_free(stif (likely (page == c->page && c->node >=0 )) {set_freepointer (s, object, c->freelist); c->freelist = object; -stat (c,FREE_FASTPATH ); +stat (s,FREE_FASTPATH ); }else __slab_free (s, page, x, addr);
Re: [this_cpu_xx V7 6/8] Make slub statistics use this_cpu_inc
Quoted 5 lines · Christoph Lameter
this_cpu_inc()translates into a single instruction on x86 and does not need any register. So use it instat(). We also want to avoid the calculation of the per cpukmem_cache_cpustructure pointer. So pass akmem_cachepointer instead of akmem_cache_cpupointer.
Hmm, last time I checked, [__]this_cpu_inc were not using "inc" on x86, but the default [__]this_cpu_add((pcp), 1)
Re: [this_cpu_xx V7 6/8] Make slub statistics use this_cpu_inc
Quoted 9 lines · Eric Dumazet
Quoted 5 lines · Christoph Lameter
this_cpu_inc()translates into a single instruction on x86 and does not need any register. So use it instat(). We also want to avoid the calculation of the per cpukmem_cache_cpustructure pointer. So pass akmem_cachepointer instead of akmem_cache_cpupointer.Hmm, last time I checked, [__]this_cpu_inc were not using "inc" on x86, but the default [__]this_cpu_add((pcp), 1)
Well the add 1 is still a single instruction so its okay. We can easily add support for inc on x86 if makes a difference?
Re: [this_cpu_xx V7 6/8] Make slub statistics use this_cpu_inc
Quoted 9 lines · Christoph Lameter
Quoted 7 lines · Eric Dumazet
Quoted 5 lines · Christoph Lameter
this_cpu_inc()translates into a single instruction on x86 and does not need any register. So use it instat(). We also want to avoid the calculation of the per cpukmem_cache_cpustructure pointer. So pass akmem_cachepointer instead of akmem_cache_cpupointer.
Quoted 6 lines
Quoted 2 lines
Hmm, last time I checked, [__]this_cpu_inc were not using "inc" on x86, but the default [__]this_cpu_add((pcp), 1)
Well the add 1 is still a single instruction so its okay. We can easily add support for inc on x86 if makes a difference?
Probably not, only one byte of text per use can be saved.
Sorry, rereading your changelog I dont know why I thought about "inc" instruction.
PATCH v7 7/8 Module handling: Use this_cpu_xx to dynamically allocate counters
In reply to Christoph LameterUse cpu ops to deal with the per cpu data instead of a local_t. Reduces memory requirements, cache footprint and decreases cycle counts.
The this_cpu_xx operations are also used for !SMP mode. Otherwise we could not drop the use of __module_ref_addr() which would make per cpu data handling complicated. this_cpu_xx operations have their own fallback for !SMP.
The last hold out of users of local_t is the tracing ringbuffer after this patch has been applied.
--- include/linux/module.h | 36 ++++++++++++------------------------ kernel/module.c | 30 ++++++++++++++++-------------- kernel/trace/ring_buffer.c | 1 + 3 files changed, 29 insertions(+), 38 deletions(-)
include/linux/module.h +12 -24
@@ -16,8 +16,7 @@#include <linux/kobject.h> #include <linux/moduleparam.h> #include <linux/tracepoint.h> - -#include <asm/local.h> +#include <linux/percpu.h> #include <asm/module.h> #include <trace/events/module.h>
@@ -361,11 +360,9 @@ struct module/* Destruction function. */ void (*exit)(void ); -#ifdef CONFIG_SMP -char *refptr; -#else -local_t ref; -#endif +struct module_ref { +int count; + } *refptr;#endif #ifdef CONFIG_CONSTRUCTORS
@@ -452,25 +449,16 @@ void __symbol_put(const char *symbol);#define symbol_put (x) __symbol_put(MODULE_SYMBOL_PREFIX #x)void symbol_put_addr (void *addr); -static inline local_t *__module_ref_addr (struct module *mod,int cpu) -{ -#ifdef CONFIG_SMP -return (local_t *) (mod->refptr +per_cpu_offset (cpu)); -#else -return &mod->ref; -#endif -} -/* Sometimes we know we already have a refcount, and it's easier not to handle the error case (which only happens with rmmod --wait). */ static inline void __module_get (struct module *module) {if (module) { -unsigned int cpu =get_cpu (); -local_inc (__module_ref_addr (module, cpu)); +preempt_disable (); +__this_cpu_inc (module->refptr->count);trace_module_get (module, _THIS_IP_, -local_read (__module_ref_addr (module, cpu))); -put_cpu (); +__this_cpu_read (module->refptr->count)); +preempt_enable (); } }
@@ -479,15 +467,15 @@ static inline int try_module_get(structint ret =1 ;if (module) { -unsigned int cpu =get_cpu ();if (likely (module_is_live (module))) { -local_inc (__module_ref_addr (module, cpu)); +preempt_disable (); +__this_cpu_inc (module->refptr->count);trace_module_get (module, _THIS_IP_, -local_read (__module_ref_addr (module, cpu))); +__this_cpu_read (module->refptr->count)); +preempt_enable (); }else ret =0 ; -put_cpu (); }return ret; }
kernel/module.c +16 -14
@@ -474,9 +474,10 @@ static void module_unload_init(struct moINIT_LIST_HEAD (&mod->modules_which_use_me);for_each_possible_cpu (cpu) -local_set (__module_ref_addr (mod, cpu),0 ); +per_cpu_ptr (mod->refptr, cpu)->count =0 ; +/* Hold reference count during initialization. */ -local_set (__module_ref_addr (mod,raw_smp_processor_id ()),1 ); +__this_cpu_write (mod->refptr->count,1 );/* Backwards compatibility macros put refcount during init. */ mod->waiter = current; }
@@ -555,6 +556,7 @@ static void module_unload_free(struct mokfree (use);sysfs_remove_link (i->holders_dir, mod->name);/* There can be at most one match. */ +free_percpu (i->refptr);break ; } }
@@ -619,7 +621,7 @@ unsigned int module_refcount(struct moduint cpu;for_each_possible_cpu (cpu) - total +=local_read (__module_ref_addr (mod, cpu)); + total +=per_cpu_ptr (mod->refptr, cpu)->count;return total; }EXPORT_SYMBOL (module_refcount);
@@ -796,14 +798,15 @@ static struct module_attribute refcnt =void module_put (struct module *module) {if (module) { -unsigned int cpu =get_cpu (); -local_dec (__module_ref_addr (module, cpu)); +preempt_disable (); +__this_cpu_dec (module->refptr->count); +trace_module_put (module, _RET_IP_, -local_read (__module_ref_addr (module, cpu))); +__this_cpu_read (module->refptr->count));/* Maybe they're waiting for us to drop reference? */ if (unlikely (!module_is_live (module)))wake_up_process (module->waiter); -put_cpu (); +preempt_enable (); } }EXPORT_SYMBOL (module_put);
@@ -1380,9 +1383,9 @@ static void free_module(struct module *mkfree (mod->args);if (mod->percpu)percpu_modfree (mod->percpu); -#if defined(CONFIG_MODULE_UNLOAD ) && defined(CONFIG_SMP ) +#if defined(CONFIG_MODULE_UNLOAD )if (mod->refptr) -percpu_modfree (mod->refptr); +free_percpu (mod->refptr);#endif /* Free lock-classes: */ lockdep_free_key_range (mod->module_core, mod->core_size);
@@ -2148,9 +2151,8 @@ static noinline struct module *load_modu mod = (void *)sechdrs[modindex].sh_addr;kmemleak_load_module (mod, hdr, sechdrs, secstrings); -#if defined(CONFIG_MODULE_UNLOAD ) && defined(CONFIG_SMP ) - mod->refptr =percpu_modalloc (sizeof (local_t), __alignof__(local_t ), - mod->name); +#if defined(CONFIG_MODULE_UNLOAD ) + mod->refptr = alloc_percpu(struct module_ref);if (!mod->refptr) { err = -ENOMEM ; goto free_init;
@@ -2376,8 +2378,8 @@ static noinline struct module *load_modukobject_put (&mod->mkobj.kobj); free_unload:module_unload_free (mod); -#if defined(CONFIG_MODULE_UNLOAD ) && defined(CONFIG_SMP ) -percpu_modfree (mod->refptr); +#if defined(CONFIG_MODULE_UNLOAD ) +free_percpu (mod->refptr); free_init:#endif module_free (mod, mod->module_init);
kernel/trace/ring_buffer.c +1 -0
@@ -12,6 +12,7 @@#include <linux/hardirq.h> #include <linux/kmemcheck.h> #include <linux/module.h> +#include <asm/local.h> #include <linux/percpu.h> #include <linux/mutex.h> #include <linux/init.h>
Tejun Heo
maintainerRe: [this_cpu_xx V7 7/8] Module handling: Use this_cpu_xx to dynamically allocate counters
(Rusty Russell cc'd.)
Quoted 7 lines · Christoph Lameter
include/linux/module.h +0
-2@@ -479,15 +467,15 @@ static inline int try_module_get(structint ret =1 ;if (module) { -unsigned int cpu =get_cpu ();if (likely (module_is_live (module))) { -local_inc (__module_ref_addr (module, cpu));
include/linux/module.h static inline int try_module_get(struct · line 490
Quoted 10 lines
include/linux/module.h +4
-2+preempt_disable (); +__this_cpu_inc (module->refptr->count);trace_module_get (module, _THIS_IP_, -local_read (__module_ref_addr (module, cpu))); +__this_cpu_read (module->refptr->count)); +preempt_enable (); }else ret =0 ; -put_cpu ();
I think you need preemption disabled while checking whether module_is_live(). The state is protected by stop_machine or synchronize_sched().
Thanks.
Signature · 1 line
tejun
Re: [this_cpu_xx V7 7/8] Module handling: Use this_cpu_xx to dynamically allocate counters
Quoted 16 lines · Tejun Heo
(Rusty Russell cc'd.)
Quoted 13 lines · Christoph Lameter
include/linux/module.h +4
-3@@ -479,15 +467,15 @@ static inline int try_module_get(structint ret =1 ;if (module) { -unsigned int cpu =get_cpu ();if (likely (module_is_live (module))) { -local_inc (__module_ref_addr (module, cpu)); +preempt_disable (); +__this_cpu_inc (module->refptr->count);trace_module_get (module, _THIS_IP_, -local_read (__module_ref_addr (module, cpu))); +__this_cpu_read (module->refptr->count)); +preempt_enable ();
Quoted 8 lines
include/linux/module.h static inline int try_module_get(struct · line 490
Quoted 4 lines
include/linux/module.h +0
-1}else ret =0 ; -put_cpu ();I think you need preemption disabled while checking whether
module_is_live(). The state is protected bystop_machineorsynchronize_sched().
Yes.
Thanks,
Rusty.
Re: [this_cpu_xx V7 7/8] Module handling: Use this_cpu_xx to dynamically allocate counters
Fixup patch:
Subject: Extend preempt_disable section around module_is_live()
--- include/linux/module.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-)
include/linux/module.h +4 -2
@@ -467,15 +467,17 @@ static inline int try_module_get(structint ret =1 ;if (module) { +preempt_disable (); +if (likely (module_is_live (module))) { -preempt_disable ();__this_cpu_inc (module->refptr->count);trace_module_get (module, _THIS_IP_,__this_cpu_read (module->refptr->count)); -preempt_enable (); }else ret =0 ; + +preempt_enable (); }return ret; }
Tejun Heo
maintainerRe: [this_cpu_xx V7 7/8] Module handling: Use this_cpu_xx to dynamically allocate counters
Hello,
Quoted 5 lines · Christoph Lameter
Fixup patch:
Subject: Extend
preempt_disablesection aroundmodule_is_live()Signed-off-by Christoph Lameter <[email protected]>
Looks good to me but as the patch currently isn't any tree, it'll probably be better to merge it into the original patch. Rusty, do you want to take this one? If not, I can route it through percpu tree.
Thanks.
Signature · 1 line
tejun
Rusty Russell
Acked-byRe: [this_cpu_xx V7 7/8] Module handling: Use this_cpu_xx to dynamically allocate counters
Quoted 5 lines · Tejun Heo
Hello,
Quoted 2 lines · Christoph Lameter
Fixup patch:
Quoted 7 lines
Quoted 3 lines
Subject: Extend
preempt_disablesection aroundmodule_is_live()Signed-off-by Christoph Lameter <[email protected]>Looks good to me but as the patch currently isn't any tree, it'll probably be better to merge it into the original patch. Rusty, do you want to take this one? If not, I can route it through percpu tree.
Easiest to go with your stuff I think:
Thanks,
Rusty.
PATCH v7 8/8 Remove cpu_local_xx macros
In reply to Christoph LameterThese macros have not been used for awhile now and are not used by the last local_t user in the tree (trace ringbuffer logic).
arch/alpha/include/asm/local.h +0 -17
@@ -98,21 +98,4 @@ static __inline__ long local_sub_return(#define __local_add (i,l) ((l)->a.counter+=(i))#define __local_sub (i,l) ((l)->a.counter-=(i)) -/* Use these for per-cpu local_t variables: on some archs they are -* much more efficient than these naive implementations. Note they take -* a variable, not an address. -*/ -#define cpu_local_read (l) local_read(&__get_cpu_var(l)) -#define cpu_local_set (l, i) local_set(&__get_cpu_var(l), (i)) - -#define cpu_local_inc (l) local_inc(&__get_cpu_var(l)) -#define cpu_local_dec (l) local_dec(&__get_cpu_var(l)) -#define cpu_local_add (i, l) local_add((i), &__get_cpu_var(l)) -#define cpu_local_sub (i, l) local_sub((i), &__get_cpu_var(l)) - -#define __cpu_local_inc (l) __local_inc(&__get_cpu_var(l)) -#define __cpu_local_dec (l) __local_dec(&__get_cpu_var(l)) -#define __cpu_local_add (i, l) __local_add((i), &__get_cpu_var(l)) -#define __cpu_local_sub (i, l) __local_sub((i), &__get_cpu_var(l)) -#endif /* _ALPHA_LOCAL_H */
arch/m32r/include/asm/local.h +0 -25
@@ -338,29 +338,4 @@ static inline void local_set_mask(unsign* a variable, not an address. */ -/* Need to disable preemption for the cpu local counters otherwise we could -still access a variable of a previous CPU in a non local way. */ -#define cpu_local_wrap_v (l) \ - ({ local_t res__; \ - preempt_disable(); \ - res__ = (l); \ - preempt_enable(); \ - res__; }) -#define cpu_local_wrap (l) \ - ({ preempt_disable(); \ - l; \ - preempt_enable(); }) \ - -#define cpu_local_read (l) cpu_local_wrap_v(local_read(&__get_cpu_var(l))) -#define cpu_local_set (l, i) cpu_local_wrap(local_set(&__get_cpu_var(l), (i))) -#define cpu_local_inc (l) cpu_local_wrap(local_inc(&__get_cpu_var(l))) -#define cpu_local_dec (l) cpu_local_wrap(local_dec(&__get_cpu_var(l))) -#define cpu_local_add (i, l) cpu_local_wrap(local_add((i), &__get_cpu_var(l))) -#define cpu_local_sub (i, l) cpu_local_wrap(local_sub((i), &__get_cpu_var(l))) - -#define __cpu_local_inc (l) cpu_local_inc(l) -#define __cpu_local_dec (l) cpu_local_dec(l) -#define __cpu_local_add (i, l) cpu_local_add((i), (l)) -#define __cpu_local_sub (i, l) cpu_local_sub((i), (l)) -#endif /* __M32R_LOCAL_H */
arch/mips/include/asm/local.h +0 -25
@@ -193,29 +193,4 @@ static __inline__ long local_sub_return(#define __local_add (i, l) ((l)->a.counter+=(i))#define __local_sub (i, l) ((l)->a.counter-=(i)) -/* Need to disable preemption for the cpu local counters otherwise we could -still access a variable of a previous CPU in a non atomic way. */ -#define cpu_local_wrap_v (l) \ - ({ local_t res__; \ - preempt_disable(); \ - res__ = (l); \ - preempt_enable(); \ - res__; }) -#define cpu_local_wrap (l) \ - ({ preempt_disable(); \ - l; \ - preempt_enable(); }) \ - -#define cpu_local_read (l) cpu_local_wrap_v(local_read(&__get_cpu_var(l))) -#define cpu_local_set (l, i) cpu_local_wrap(local_set(&__get_cpu_var(l), (i))) -#define cpu_local_inc (l) cpu_local_wrap(local_inc(&__get_cpu_var(l))) -#define cpu_local_dec (l) cpu_local_wrap(local_dec(&__get_cpu_var(l))) -#define cpu_local_add (i, l) cpu_local_wrap(local_add((i), &__get_cpu_var(l))) -#define cpu_local_sub (i, l) cpu_local_wrap(local_sub((i), &__get_cpu_var(l))) - -#define __cpu_local_inc (l) cpu_local_inc(l) -#define __cpu_local_dec (l) cpu_local_dec(l) -#define __cpu_local_add (i, l) cpu_local_add((i), (l)) -#define __cpu_local_sub (i, l) cpu_local_sub((i), (l)) -#endif /* _ARCH_MIPS_LOCAL_H */
arch/powerpc/include/asm/local.h +0 -25
@@ -172,29 +172,4 @@ static __inline__ long local_dec_if_posi#define __local_add (i,l) ((l)->a.counter+=(i))#define __local_sub (i,l) ((l)->a.counter-=(i)) -/* Need to disable preemption for the cpu local counters otherwise we could -still access a variable of a previous CPU in a non atomic way. */ -#define cpu_local_wrap_v (l) \ - ({ local_t res__; \ - preempt_disable(); \ - res__ = (l); \ - preempt_enable(); \ - res__; }) -#define cpu_local_wrap (l) \ - ({ preempt_disable(); \ - l; \ - preempt_enable(); }) \ - -#define cpu_local_read (l) cpu_local_wrap_v(local_read(&__get_cpu_var(l))) -#define cpu_local_set (l, i) cpu_local_wrap(local_set(&__get_cpu_var(l), (i))) -#define cpu_local_inc (l) cpu_local_wrap(local_inc(&__get_cpu_var(l))) -#define cpu_local_dec (l) cpu_local_wrap(local_dec(&__get_cpu_var(l))) -#define cpu_local_add (i, l) cpu_local_wrap(local_add((i), &__get_cpu_var(l))) -#define cpu_local_sub (i, l) cpu_local_wrap(local_sub((i), &__get_cpu_var(l))) - -#define __cpu_local_inc (l) cpu_local_inc(l) -#define __cpu_local_dec (l) cpu_local_dec(l) -#define __cpu_local_add (i, l) cpu_local_add((i), (l)) -#define __cpu_local_sub (i, l) cpu_local_sub((i), (l)) -#endif /* _ARCH_POWERPC_LOCAL_H */
arch/x86/include/asm/local.h +0 -37
@@ -195,41 +195,4 @@ static inline long local_sub_return(long#define __local_add (i, l) local_add((i), (l))#define __local_sub (i, l) local_sub((i), (l)) -/* Use these for per-cpu local_t variables: on some archs they are -* much more efficient than these naive implementations. Note they take -* a variable, not an address. -* -* X86_64: This could be done better if we moved the per cpu data directly -* after GS. -*/ - -/* Need to disable preemption for the cpu local counters otherwise we could -still access a variable of a previous CPU in a non atomic way. */ -#define cpu_local_wrap_v (l) \ -({ \ - local_t res__; \ - preempt_disable(); \ - res__ = (l); \ - preempt_enable(); \ - res__; \ -}) -#define cpu_local_wrap (l) \ -({ \ - preempt_disable(); \ - (l); \ - preempt_enable(); \ -}) \ - -#define cpu_local_read (l) cpu_local_wrap_v(local_read(&__get_cpu_var((l)))) -#define cpu_local_set (l, i) cpu_local_wrap(local_set(&__get_cpu_var((l)), (i))) -#define cpu_local_inc (l) cpu_local_wrap(local_inc(&__get_cpu_var((l)))) -#define cpu_local_dec (l) cpu_local_wrap(local_dec(&__get_cpu_var((l)))) -#define cpu_local_add (i, l) cpu_local_wrap(local_add((i), &__get_cpu_var((l)))) -#define cpu_local_sub (i, l) cpu_local_wrap(local_sub((i), &__get_cpu_var((l)))) - -#define __cpu_local_inc (l) cpu_local_inc((l)) -#define __cpu_local_dec (l) cpu_local_dec((l)) -#define __cpu_local_add (i, l) cpu_local_add((i), (l)) -#define __cpu_local_sub (i, l) cpu_local_sub((i), (l)) -#endif /* _ASM_X86_LOCAL_H */
include/asm-generic/local.h +0 -19
@@ -52,23 +52,4 @@ typedef struct#define __local_add (i,l) local_set((l), local_read(l) + (i))#define __local_sub (i,l) local_set((l), local_read(l) - (i)) -/* Use these for per-cpu local_t variables: on some archs they are -* much more efficient than these naive implementations. Note they take -* a variable (eg. mystruct.foo), not an address. -*/ -#define cpu_local_read (l) local_read(&__get_cpu_var(l)) -#define cpu_local_set (l, i) local_set(&__get_cpu_var(l), (i)) -#define cpu_local_inc (l) local_inc(&__get_cpu_var(l)) -#define cpu_local_dec (l) local_dec(&__get_cpu_var(l)) -#define cpu_local_add (i, l) local_add((i), &__get_cpu_var(l)) -#define cpu_local_sub (i, l) local_sub((i), &__get_cpu_var(l)) - -/* Non-atomic increments, ie. preemption disabled and won't be touched -* in interrupt, etc. Some archs can optimize this case well. -*/ -#define __cpu_local_inc (l) __local_inc(&__get_cpu_var(l)) -#define __cpu_local_dec (l) __local_dec(&__get_cpu_var(l)) -#define __cpu_local_add (i, l) __local_add((i), &__get_cpu_var(l)) -#define __cpu_local_sub (i, l) __local_sub((i), &__get_cpu_var(l)) -#endif /* _ASM_GENERIC_LOCAL_H */
Tejun Heo
maintainerRe: [this_cpu_xx V7 8/8] Remove cpu_local_xx macros
Quoted 4 lines · Christoph Lameter
These macros have not been used for awhile now and are not used by the last
local_tuser in the tree (trace ringbuffer logic).Signed-off-by Christoph Lameter <[email protected]>
Cool. Other than the things I commented before, the series looks great to me.
Thanks for doing this.
Signature · 1 line
tejun
Quoted 2 lines · Christoph Lameter
Leftovers from the earlier patchset. Mostly applications of per cpu counters to core components.
I can pick up the SLUB patches. Is that OK with you Tejun?
Tejun Heo
maintainerQuoted 5 lines · Pekka Enberg
Quoted 2 lines · Christoph Lameter
Leftovers from the earlier patchset. Mostly applications of per cpu counters to core components.
I can pick up the SLUB patches. Is that OK with you Tejun?
Yeap, now that this_cpu stuff is upstream, no reason to route them through percpu tree. I'll be happy to pick up whatever is left.
Thanks.
Signature · 1 line
tejun
Small fixup patch to the slub patches:
Subject: Update remaining reference to get_cpu_slab
If CONFIG_SLUB_STATS is set then some additional code is being compiled that was not updated since it was a recent addition to slub.
--- mm/slub.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
mm/slub.c +1 -1
@@ -4221,7 +4221,7 @@ static void clear_stat(struct kmem_cacheint cpu;for_each_online_cpu (cpu) -get_cpu_slab (s, cpu)->stat[si] =0 ; +per_cpu_ptr (s->cpu_slab, cpu)->stat[si] =0 ; }#define STAT_ATTR (si, text) \
Quoted 6 lines · Christoph Lameter
Leftovers from the earlier patchset. Mostly applications of per cpu counters to core components.
After this patchset there will be only one user of
local_tleft: Mathieu's trace ringbuffer. Does it really need these ops?
What kernel are these patches based on? They do not cleanly apply and when fixed up, they do not build against 2.6.32.
Signature · 3 lines
Mel Gorman Part-time Phd Student Linux Technology Center University of Limerick IBM Dublin Software Lab
Quoted 2 lines · Mel Gorman
What kernel are these patches based on? They do not cleanly apply and when fixed up, they do not build against 2.6.32.
Upstream. Linus tree.
With todays git tree I get a reject. Will rediff when rc1 is out.
Quoted 6 lines · * Christoph Lameter ([email protected]) wrote:
Leftovers from the earlier patchset. Mostly applications of per cpu counters to core components.
After this patchset there will be only one user of
local_tleft: Mathieu's trace ringbuffer. Does it really need these ops?
Besides my own ring buffer implementation in LTTng, at least Steven's kernel/trace/ring_buffer.c (in mainline) use this too. We would need a way to directly map to the same resulting behavior with per-cpu variables.
In LTTng, I use local_cmpxchg, local_read, local_add and, in some setups, local_add_return to manage the write counter and commit counters. These per-cpu counters are kept in per-cpu buffer management data allocated for each data collection "channel".
The current way I allocate this structure for all cpus is:
chan->buf = alloc_percpu(struct ltt_chanbuf);
But note that each struct ltt_chanbuf contains a pointer to an array containing each sub-buffer commit counters for the given buffer:
struct commit_counters {
local_t cc;
local_t cc_sb; /* Incremented _once_ at sb switch */
local_t events; /* Event count */
};
struct ltt_chanbuf {
struct ltt_chanbuf_alloc a; /* Parent. First field. */
/* First 32 bytes cache-hot cacheline */
local_t offset; /* Current offset in the buffer */
struct commit_counters *commit_count;
/* Commit count per sub-buffer */
atomic_long_t consumed; /*
* Current offset in the buffer
* standard atomic access (shared) */
....
So I think accessing the "local_t offset" through percpu pointers should be fine if I allocate struct ltt_chanbuf through the per cpu API. However, I wonder how to deal with the commit_count counters, because there is an indirection level.
Thanks,
Mathieu
Signature · 2 lines
Mathieu Desnoyers OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F BA06 3F25 A8FE 3BAE 9A68
Tejun Heo
maintainerHello, Mathieu.
Quoted 4 lines · Mathieu Desnoyers
So I think accessing the "
local_toffset" through percpu pointers should be fine if I allocate structltt_chanbufthrough the per cpu API. However, I wonder how to deal with thecommit_countcounters, because there is an indirection level.
Are they different in numbers for different cpus?
Thanks.
Signature · 1 line
tejun
Quoted 9 lines · * Tejun Heo ([email protected]) wrote:
Hello, Mathieu.
Quoted 4 lines · Mathieu Desnoyers
So I think accessing the "
local_toffset" through percpu pointers should be fine if I allocate structltt_chanbufthrough the per cpu API. However, I wonder how to deal with thecommit_countcounters, because there is an indirection level.Are they different in numbers for different cpus?
Nope, there is the same number of sub-buffers for each per-cpu buffer. I just want to see if supplementary indirections are allowed after dereferencing the per-cpu pointer ?
Thanks,
Mathieu
Quoted 5 lines
Thanks.
Signature · 1 line
tejun
Signature · 2 lines
Mathieu Desnoyers OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F BA06 3F25 A8FE 3BAE 9A68
Tejun Heo
maintainerHello,
Quoted 3 lines · Mathieu Desnoyers
Nope, there is the same number of sub-buffers for each per-cpu buffer. I just want to see if supplementary indirections are allowed after dereferencing the per-cpu pointer ?
Hmmm... you can store percpu pointer to a variable. If there are the same number of commit_count for each cpu, they can be allocated using percpu allocator and their pointers can be stored, offset and dereferenced. Would that be enough?
Thanks.
Signature · 1 line
tejun
Quoted 11 lines · * Tejun Heo ([email protected]) wrote:
Hello,
Quoted 3 lines · Mathieu Desnoyers
Nope, there is the same number of sub-buffers for each per-cpu buffer. I just want to see if supplementary indirections are allowed after dereferencing the per-cpu pointer ?
Hmmm... you can store percpu pointer to a variable. If there are the same number of
commit_countfor each cpu, they can be allocated using percpu allocator and their pointers can be stored, offset and dereferenced. Would that be enough?
Yes, I think I could allocate, from the channel structure perspective:
- A percpu pointer to the per-cpu buffer structures
- A percpu pointer to the per-cpu commit counters.
This should fix my problem. The main change here is that the pointer to the commit counters would not be located in the per-cpu buffer structures anymore.
However, I would need:
this_cpu_cmpxchg(scalar, oldv, newv)
(maps to x86 cmpxchg)
this_cpu_add_return(scalar, value)
(maps to x86 xadd)
too. Is that a planned addition ?
(while we are at it, we might as will add the xchg instruction, althrough it has an implied LOCK prefix on x86).
Thanks,
Mathieu
Quoted 5 lines
Thanks.
Signature · 1 line
tejun
Signature · 2 lines
Mathieu Desnoyers OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F BA06 3F25 A8FE 3BAE 9A68
Quoted 9 lines
However, I would need:
this_cpu_cmpxchg(scalar, oldv, newv)
(maps to x86 cmpxchg)
this_cpu_add_return(scalar, value)
(maps to x86 xadd)too. Is that a planned addition ?
It was not necessary. Its easy to add though.
Quoted 2 lines
(while we are at it, we might as will add the xchg instruction, althrough it has an implied LOCK prefix on x86).
Well yeah thats a thorny one. One could use the cmpxchg instead?
Quoted 8 lines · * Christoph Lameter ([email protected]) wrote:
Quoted 8 lines
However, I would need:
this_cpu_cmpxchg(scalar, oldv, newv)
(maps to x86 cmpxchg)
this_cpu_add_return(scalar, value)
(maps to x86 xadd)
Quoted 8 lines
Quoted 1 line
too. Is that a planned addition ?
It was not necessary. Its easy to add though.
Quoted 2 lines
(while we are at it, we might as will add the xchg instruction, althrough it has an implied LOCK prefix on x86).
Well yeah thats a thorny one. One could use the cmpxchg instead?
Yes, although maybe it would make sense to encapsulate it in a xchg primitive anyway, in case some architecture has a better xchg than x86. For instance, powerpc, with its linked load/store conditional, can skip a comparison for xchg that's otherwise required for cmpxchg.
Some quick test on my Intel Xeon E5405:
local cmpxchg: 14 cycles xchg: 18 cycles
So yes, indeed, the non-LOCK prefixed local cmpxchg seems a bit faster than the xchg, given the latter has an implied LOCK prefix.
Code used for local cmpxchg:
old = var;
do {
ret = cmpxchg_local(&var, old, 4);
if (likely(ret == old))
break;
old = ret;
} while (1);Thanks,
Mathieu
Signature · 2 lines
Mathieu Desnoyers OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F BA06 3F25 A8FE 3BAE 9A68
Quoted 8 lines · Mathieu Desnoyers
Some quick test on my Intel Xeon E5405:
local cmpxchg: 14 cycles xchg: 18 cyclesSo yes, indeed, the non-LOCK prefixed local cmpxchg seems a bit faster than the xchg, given the latter has an implied LOCK prefix.
Quoted 9 lines
Code used for local cmpxchg:
old = var; do { ret = cmpxchg_local(&var, old, 4); if (likely(ret == old)) break; old = ret; } while (1);
Great. Could you also put that into "patch-format"?
Quoted 20 lines · * Christoph Lameter ([email protected]) wrote:
Quoted 8 lines · Mathieu Desnoyers
Some quick test on my Intel Xeon E5405:
local cmpxchg: 14 cycles xchg: 18 cyclesSo yes, indeed, the non-LOCK prefixed local cmpxchg seems a bit faster than the xchg, given the latter has an implied LOCK prefix.
Quoted 9 lines
Code used for local cmpxchg:
old = var; do { ret = cmpxchg_local(&var, old, 4); if (likely(ret == old)) break; old = ret; } while (1);
Quoted 2 lines
Great. Could you also put that into "patch-format"?
Sure, can you point me to a git tree I should work on top of which includes the per cpu infrastructure to extend ?
Mathieu
Signature · 2 lines
Mathieu Desnoyers OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F BA06 3F25 A8FE 3BAE 9A68
Quoted 2 lines · Mathieu Desnoyers
Sure, can you point me to a git tree I should work on top of which includes the per cpu infrastructure to extend ?
Linus' git tree contains what yoou need. I have a early draft here of a patch to implement the generic portions. Unfinished. I hope I have time to complete this. Feel free to complete it but keep me posted so that I wont repeat anything you do.
The modifications to asm-generic/cmpxchg-local wont work since we need to do this_cpu_ptr() pointer calculations within the protected sections. I was in the middle of getting rid of it when I found it was time to go home...
--- include/asm-generic/cmpxchg-local.h | 24 ++++- include/linux/percpu.h | 151 ++++++++++++++++++++++++++++++++++++ 2 files changed, 169 insertions(+), 6 deletions(-)
include/asm-generic/cmpxchg-local.h +18 -6
@@ -6,13 +6,12 @@extern unsigned long wrong_size_cmpxchg (volatile void *ptr);/* -* Generic version of __cmpxchg_local (disables interrupts). Takes an unsigned -* long parameter, supporting various types of architectures. +* Generic version of __cmpxchg_local. */ -static inline unsigned long __cmpxchg_local_generic (volatile void *ptr, +static inline unsigned long ____cmpxchg_local_generic (volatile void *ptr,unsigned long old,unsigned long new,int size) { -unsigned long flags, prev; +unsigned long prev;/* * Sanity checking, compile-time.
@@ -20,7 +19,6 @@ static inline unsigned long __cmpxchg_loif (size ==8 &&sizeof (unsigned long ) !=8 )wrong_size_cmpxchg (ptr); -local_irq_save (flags);switch (size) {case 1 : prev = *(u8 *)ptr;if (prev == old)
@@ -41,11 +39,25 @@ static inline unsigned long __cmpxchg_lodefault :wrong_size_cmpxchg (ptr); } -local_irq_restore (flags);return prev; }/* +* Generic version of __cmpxchg_local (disables interrupts). Takes an unsigned +* long parameter, supporting various types of architectures. +*/ +static inline unsigned long __cmpxchg_local_generic (volatile void *ptr, +unsigned long old,unsigned long new,int size) +{ +unsigned long flags, r; + +local_irq_save (flags); + r =____cmpxchg_local_generic (ptr, old, new ,size); +local_irq_restore (flags); +return r; +} + +/* * Generic version of __cmpxchg64_local. Takes an u64 parameter. */ static inline u64 __cmpxchg64_local_generic(volatile void *ptr,
include/linux/percpu.h +151 -0
167 lines not on this page · raw message