Implement this_cpu_inc_return (and friends) and use it in various places
Subsystems (9)
cover Implement this_cpu_inc_return (and friends) and use it in various places
Extend the this_cpu_ops infrastructure to support this_cpu_inc_return and then use the new functionality in various subsystems. Provide an optimize x86 implementation for these operations.
All these patches depends on the initial patch being applied first.
Patches have been reviewed a couple of times and I would suggest that they go through Tejun's percpu tree for merging.
PATCH 1/9 percpu: Generic support for this_cpu_add,sub,dec,inc_return
Introduce generic support for this_cpu_add_return etc.
The fallback is to realize these operations with simpler __this_cpu_ops.
--- include/linux/percpu.h | 77 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+)
include/linux/percpu.h +77 -0
@@ -240,6 +240,25 @@ extern void __bad_size_call_parameter(vo pscr_ret__; \ }) +#define __pcpu_size_call_return2 (stem, pcp, ...) \ +({ \ + typeof(pcp) ret__; \ + __verify_pcpu_ptr(&(pcp)); \ + switch(sizeof(pcp)) { \ + case 1: ret__ = stem##1(pcp, __VA_ARGS__); \ + break; \ + case 2: ret__ = stem##2(pcp, __VA_ARGS__); \ + break; \ + case 4: ret__ = stem##4(pcp, __VA_ARGS__); \ + break; \ + case 8: ret__ = stem##8(pcp, __VA_ARGS__); \ + break; \ + default: \ + __bad_size_call_parameter();break; \ + } \ + ret__; \ +}) +#define __pcpu_size_call (stem, variable, ...) \ do { \ __verify_pcpu_ptr(&(variable)); \
@@ -529,6 +548,64 @@ do { \# define __this_cpu_xor (pcp, val) __pcpu_size_call(__this_cpu_xor_, (pcp), (val))#endif +#define _this_cpu_generic_add_return (pcp, val) \ +({ \ + typeof(pcp) ret__; \ + preempt_disable(); \ + __this_cpu_add(pcp, val); \ + ret__ = __this_cpu_read(pcp); \ + preempt_enable(); \ + ret__; \ +}) + +#ifndef this_cpu_add_return +# ifndef this_cpu_add_return_1 +# define this_cpu_add_return_1 (pcp, val) _this_cpu_generic_add_return(pcp, val) +# endif +# ifndef this_cpu_add_return_2 +# define this_cpu_add_return_2 (pcp, val) _this_cpu_generic_add_return(pcp, val) +# endif +# ifndef this_cpu_add_return_4 +# define this_cpu_add_return_4 (pcp, val) _this_cpu_generic_add_return(pcp, val) +# endif +# ifndef this_cpu_add_return_8 +# define this_cpu_add_return_8 (pcp, val) _this_cpu_generic_add_return(pcp, val) +# endif +# define this_cpu_add_return (pcp, val) __pcpu_size_call_return2(this_cpu_add_return_, pcp, val) +#endif + +#define this_cpu_sub_return (pcp, val) this_cpu_add_return(pcp, -(val)) +#define this_cpu_inc_return (pcp) this_cpu_add_return(pcp, 1) +#define this_cpu_dec_return (pcp) this_cpu_add_return(pcp, -1) + +#define __this_cpu_generic_add_return (pcp, val) \ +({ \ + typeof(pcp) ret__; \ + __this_cpu_add(pcp, val); \ + ret__ = __this_cpu_read(pcp); \ + ret__; \ +}) + +#ifndef __this_cpu_add_return +# ifndef __this_cpu_add_return_1 +# define __this_cpu_add_return_1 (pcp, val) __this_cpu_generic_add_return(pcp, val) +# endif +# ifndef __this_cpu_add_return_2 +# define __this_cpu_add_return_2 (pcp, val) __this_cpu_generic_add_return(pcp, val) +# endif +# ifndef __this_cpu_add_return_4 +# define __this_cpu_add_return_4 (pcp, val) __this_cpu_generic_add_return(pcp, val) +# endif +# ifndef __this_cpu_add_return_8 +# define __this_cpu_add_return_8 (pcp, val) __this_cpu_generic_add_return(pcp, val) +# endif +# define __this_cpu_add_return (pcp, val) __pcpu_size_call_return2(this_cpu_add_return_, pcp, val) +#endif + +#define __this_cpu_sub_return (pcp, val) this_cpu_add_return(pcp, -(val)) +#define __this_cpu_inc_return (pcp) this_cpu_add_return(pcp, 1) +#define __this_cpu_dec_return (pcp) this_cpu_add_return(pcp, -1) +/* * IRQ safe versions of the per cpu RMW operations. Note that these operations * are *not* safe against modification of the same variable from another
PATCH 6/9 Taskstats: Use this_cpu_ops
In reply to Christoph LameterUse this_cpu_inc_return in one place and avoid ugly __raw_get_cpu in another.
V3->V4:
- Fix off by one.
--- kernel/taskstats.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-)
kernel/taskstats.c +2 -3
@@ -89,8 +89,7 @@ static int prepare_reply(struct genl_infreturn -ENOMEM ;if (!info) { -int seq =get_cpu_var (taskstats_seqnum)++; -put_cpu_var (taskstats_seqnum); +int seq =this_cpu_inc_return (taskstats_seqnum) -1 ; reply =genlmsg_put (skb,0 , seq, &family,0 , cmd); }else
@@ -581,7 +580,7 @@ void taskstats_exit(struct task_struct *fill_tgid_exit (tsk); } - listeners = &__raw_get_cpu_var (listener_array); + listeners =__this_cpu_ptr (listener_array);if (list_empty (&listeners->list))return ;
PATCH 5/9 highmem: Use this_cpu_xx_return() operations
In reply to Christoph LameterUse this_cpu operations to optimize access primitives for highmem.
The main effect is the avoidance of address calculations through the use of a segment prefix.
V3->V4
- kmap_atomic_idx: Do not return a value.
- Use __this_cpu_dec without HIGHMEM_DEBUG
--- include/linux/highmem.h | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-)
include/linux/highmem.h +8 -5
@@ -81,7 +81,8 @@ DECLARE_PER_CPU(int, __kmap_atomic_idx);static inline int kmap_atomic_idx_push (void ) { -int idx =__get_cpu_var (__kmap_atomic_idx)++; +int idx =__this_cpu_inc_return (__kmap_atomic_idx) -1 ; +#ifdef CONFIG_DEBUG_HIGHMEM WARN_ON_ONCE (in_irq () && !irqs_disabled ());BUG_ON (idx >KM_TYPE_NR );
@@ -91,16 +92,18 @@ static inline int kmap_atomic_idx_push(vstatic inline int kmap_atomic_idx (void ) { -return __get_cpu_var (__kmap_atomic_idx) -1 ; +return __this_cpu_read (__kmap_atomic_idx) -1 ; } -static inline int kmap_atomic_idx_pop (void ) +static inline void kmap_atomic_idx_pop (void ) { -int idx = --__get_cpu_var (__kmap_atomic_idx);#ifdef CONFIG_DEBUG_HIGHMEM +int idx =__this_cpu_dec_return (__kmap_atomic_idx); +BUG_ON (idx <0 ); +#else +__this_cpu_dec (__kmap_atomic_idx);#endif -return idx; }#endif
PATCH 7/9 fs: Use this_cpu_inc_return in buffer.c
In reply to Christoph Lameter__this_cpu_inc can create a single instruction with the same effect as the _get_cpu_var(..)++ construct in buffer.c.
--- fs/buffer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
fs/buffer.c +1 -1
@@ -3201,7 +3201,7 @@ static void recalc_bh_state(void)int i;int tot =0 ; -if (__get_cpu_var (bh_accounting).ratelimit++ <4096 ) +if (__this_cpu_inc_return (bh_accounting.ratelimit) -1 <4096 )return ;__this_cpu_write (bh_accounting.ratelimit,0 );for_each_online_cpu (i)
PATCH 8/9 random: Use this_cpu_inc_return
In reply to Christoph Lameter__this_cpu_inc can create a single instruction to do the same as __get_cpu_var()++.
--- drivers/char/random.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/char/random.c +1 -1
@@ -626,7 +626,7 @@ static void add_timer_randomness(structpreempt_disable ();/* if over the trickle threshold, use only 1 in 4096 samples */ if (input_pool.entropy_count > trickle_thresh && - (__get_cpu_var (trickle_count)++ &0xfff )) + ((__this_cpu_inc_return (trickle_count) -1 ) &0xfff )) goto out; sample.jiffies = jiffies;
PATCH 9/9 Xen: Use this_cpu_inc_return
In reply to Christoph Lameter__this_cpu_inc_return reduces code and simplifies code.
--- drivers/xen/events.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/xen/events.c +1 -1
@@ -1109,7 +1109,7 @@ static void __xen_evtchn_do_upcall(void) vcpu_info->evtchn_upcall_pending =0 ; -if (__get_cpu_var (xed_nesting_count)++) +if (__this_cpu_inc_return (xed_nesting_count) -1 ) goto out;#ifndef CONFIG_X86 /* No need for a barrier -- XCHG is a barrier on x86. */
PATCH 4/9 vmstat: Use this_cpu_inc_return for vm statistics
In reply to Christoph Lameterthis_cpu_inc_return() saves us a memory access there. Code size does not change.
V1->V2:
- Fixed the location of the __per_cpu pointer attributes
- Sparse checkedV2->V3:
- Move fixes to __percpu attribute usage to earlier patch
--- mm/vmstat.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-)
mm/vmstat.c +2 -6
@@ -227,9 +227,7 @@ void __inc_zone_state(struct zone *zone,s8 __percpu *p = pcp->vm_stat_diff + item;s8 v, t; -__this_cpu_inc (*p); - - v =__this_cpu_read (*p); + v =__this_cpu_inc_return (*p); t =__this_cpu_read (pcp->stat_threshold);if (unlikely (v > t)) {s8 overstep = t >>1 ;
@@ -251,9 +249,7 @@ void __dec_zone_state(struct zone *zone,s8 __percpu *p = pcp->vm_stat_diff + item;s8 v, t; -__this_cpu_dec (*p); - - v =__this_cpu_read (*p); + v =__this_cpu_dec_return (*p); t =__this_cpu_read (pcp->stat_threshold);if (unlikely (v < - t)) {s8 overstep = t >>1 ;
PATCH 3/9 x86: Use this_cpu_inc_return for nmi counter
In reply to Christoph Lameterthis_cpu_inc_return() saves us a memory access there.
--- arch/x86/kernel/apic/nmi.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
arch/x86/kernel/apic/nmi.c +1 -2
@@ -432,8 +432,7 @@ nmi_watchdog_tick(struct pt_regs *regs,* Ayiee, looks like this CPU is stuck ... * wait a few IRQs (5 seconds) before doing the oops ... */ -__this_cpu_inc (alert_counter); -if (__this_cpu_read (alert_counter) ==5 * nmi_hz) +if (__this_cpu_inc_return (alert_counter) ==5 * nmi_hz)/* * die_nmi will return ONLY if NOTIFY_STOP happens.. */
PATCH 2/9 x86: Support for this_cpu_add,sub,dec,inc_return
In reply to Christoph LameterSupply an implementation for x86 in order to generate more efficient code.
V2->V3:
- Cleanup
- Remove strange type checking from percpu_add_return_op.
--- arch/x86/include/asm/percpu.h | 46 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+)
arch/x86/include/asm/percpu.h +46 -0
@@ -177,6 +177,41 @@ do { \ } \ }while (0 ) + +/* +* Add return operation + */ +#define percpu_add_return_op (var, val) \ +({ \ + typedef typeof(var) pao_T__; \ + typeof(var) ret__ = val; \ + switch (sizeof(var)) { \ + case 1: \ + asm("xaddb %0, "__percpu_arg(1) \ + : "+q" (ret__), "+m" (var) \ + : : "memory"); \ + break; \ + case 2: \ + asm("xaddw %0, "__percpu_arg(1) \ + : "+r" (ret__), "+m" (var) \ + : : "memory"); \ + break; \ + case 4: \ + asm("xaddl %0, "__percpu_arg(1) \ + : "+r"(ret__), "+m" (var) \ + : : "memory"); \ + break; \ + case 8: \ + asm("xaddq %0, "__percpu_arg(1) \ + : "+re" (ret__), "+m" (var) \ + : : "memory"); \ + break; \ + default: __bad_percpu_size(); \ + } \ + ret__ += val; \ + ret__; \ +}) +#define percpu_from_op (op, var, constraint) \ ({ \ typeof(var) pfo_ret__; \
@@ -300,6 +335,14 @@ do { \#define irqsafe_cpu_xor_2 (pcp, val) percpu_to_op("xor", (pcp), val)#define irqsafe_cpu_xor_4 (pcp, val) percpu_to_op("xor", (pcp), val) +#ifndef CONFIG_M386 +#define __this_cpu_add_return_1 (pcp, val) percpu_add_return_op(pcp, val) +#define __this_cpu_add_return_2 (pcp, val) percpu_add_return_op(pcp, val) +#define __this_cpu_add_return_4 (pcp, val) percpu_add_return_op(pcp, val) +#define this_cpu_add_return_1 (pcp, val) percpu_add_return_op(pcp, val) +#define this_cpu_add_return_2 (pcp, val) percpu_add_return_op(pcp, val) +#define this_cpu_add_return_4 (pcp, val) percpu_add_return_op(pcp, val) +#endif /* * Per cpu atomic 64 bit operations are only available under 64 bit. * 32 bit must fall back to generic operations.
@@ -324,6 +367,9 @@ do { \#define irqsafe_cpu_or_8 (pcp, val) percpu_to_op("or", (pcp), val)#define irqsafe_cpu_xor_8 (pcp, val) percpu_to_op("xor", (pcp), val) +#define __this_cpu_add_return_8 (pcp, val) percpu_add_return_op(pcp, val) +#define this_cpu_add_return_8 (pcp, val) percpu_add_return_op(pcp, val) +#endif /* This is not atomic against other CPUs -- CPU preemption needs to be off */
Re: [cpuops inc_return V1 8/9] random: Use this_cpu_inc_return
In reply to Christoph Lameterdrivers/char/random.c static void add_timer_randomness(struct · line 629
Quoted 6 lines · Christoph Lameter
plain text document attachment (
cpuops_inc_return_random)__this_cpu_inccan create a single instruction to do the same as __get_cpu_var()++.drivers/char/random.c +1
-1- (__get_cpu_var(trickle_count)++ & 0xfff)) + ((__this_cpu_inc_return(trickle_count) - 1) & 0xfff))
I see you've added a "- 1" to mimic the post-increment ++ semantic.
The goal of this code to add only 1 sample in every 4096, but we don't really care which, so this semantic isn't actually important here.
Signature · 1 line
Mathematics is the supreme nostalgia of our time.
Re: [cpuops inc_return V1 8/9] random: Use this_cpu_inc_return
Quoted 5 lines · Matt Mackall
Quoted 3 lines · Christoph Lameter
plain text document attachment (
cpuops_inc_return_random)__this_cpu_inccan create a single instruction to do the same as __get_cpu_var()++.
Quoted 7 lines
drivers/char/random.c static void add_timer_randomness(struct · line 629
Quoted 2 lines
drivers/char/random.c +1
-1- (__get_cpu_var(trickle_count)++ & 0xfff)) + ((__this_cpu_inc_return(trickle_count) - 1) & 0xfff))I see you've added a "- 1" to mimic the post-increment ++ semantic.
The goal of this code to add only 1 sample in every 4096, but we don't really care which, so this semantic isn't actually important here.
-1 also removes the + 1 that the inc_return() semantics require because we use xadd on x86. -1 will generate more compact code.
Re: [cpuops inc_return V1 6/9] Taskstats: Use this_cpu_ops
In reply to Christoph LameterHello Christoph,
Quoted 18 lines · Christoph Lameter
plain text document attachment (
cpuops_inc_return_taskstats)
Usethis_cpu_inc_returnin one place and avoid ugly__raw_get_cpuin another.
V3->V4:
- Fix off by one.Cc Michael Holzheu <[email protected]>Signed-off-by Christoph Lameter <[email protected]>--- kernel/taskstats.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-)Index:
linux-2.6/kernel/taskstats.c
===================================================================
kernel/taskstats.c static int prepare_reply(struct genl_inf · line 92
Quoted 6 lines
kernel/taskstats.c +1
-2@@ -89,8 +89,7 @@ static int prepare_reply(struct genl_infreturn -ENOMEM ;if (!info) { -int seq =get_cpu_var (taskstats_seqnum)++; -put_cpu_var (taskstats_seqnum); +int seq =this_cpu_inc_return (taskstats_seqnum) -1 ;
This looks good now.
kernel/taskstats.c void taskstats_exit(struct task_struct * · line 583
Quoted 8 lines
kernel/taskstats.c +1
-1reply =genlmsg_put (skb,0 , seq, &family,0 , cmd); }else @@ -581,7 +580,7 @@ void taskstats_exit(struct task_struct *fill_tgid_exit (tsk); } - listeners = &__raw_get_cpu_var (listener_array); + listeners =__this_cpu_ptr (listener_array);
I think this is wrong and should be:
- listeners = &__raw_get_cpu_var(listener_array);
+ listeners = __this_cpu_ptr(>>>> & <<< listener_array);
Michael
Re: [cpuops inc_return V1 6/9] Taskstats: Use this_cpu_ops
Review
Quoted 5 lines · Michael Holzheu
I think this is wrong and should be:
(quoted diff) +1
-1- listeners = &__raw_get_cpu_var(listener_array); + listeners = __this_cpu_ptr(>>>> & <<< listener_array);
Correct. CONFIG_TASKSTATS was not set. Thus
Subject: Taskstats: Use this_cpu_ops
Use this_cpu_inc_return in one place and avoid ugly __raw_get_cpu in another.
V3->V4:
- Fix off by one.
V4-V4f:
- Use &listener_array
--- kernel/taskstats.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-)
kernel/taskstats.c +2 -3
@@ -89,8 +89,7 @@ static int prepare_reply(struct genl_infreturn -ENOMEM ;if (!info) { -int seq =get_cpu_var (taskstats_seqnum)++; -put_cpu_var (taskstats_seqnum); +int seq =this_cpu_inc_return (taskstats_seqnum) -1 ; reply =genlmsg_put (skb,0 , seq, &family,0 , cmd); }else
@@ -581,7 +580,7 @@ void taskstats_exit(struct task_struct *fill_tgid_exit (tsk); } - listeners = &__raw_get_cpu_var (listener_array); + listeners =__this_cpu_ptr (&listener_array);if (list_empty (&listeners->list))return ;
Straggler: Connector: Use this_cpu_operations
In reply to Christoph LameterThe patch was originally in the use cpuops patchset but it needs an inc_return and is therefore dependant on an extension of the cpu ops. Fixed up and verified that it compiles.
Subject: Connector: Use this_cpu operations
get_seq can benefit from this_cpu_operations. Address calculation is avoided and the increment is done using an xadd.
--- drivers/connector/cn_proc.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)
drivers/connector/cn_proc.c +3 -2
@@ -43,9 +43,10 @@ static DEFINE_PER_CPU(__u32, proc_event_static inline void get_seq (__u32 *ts,int *cpu) { - *ts =get_cpu_var (proc_event_counts)++; +preempt_disable (); + *ts =__this_cpu_inc_return (proc_event_counts) -1 ; *cpu =smp_processor_id (); -put_cpu_var (proc_event_counts); +preempt_enable (); }void proc_fork_connector (struct task_struct *task)
Tejun Heo
maintainerPATCH 1/9 percpu: Generic support for this_cpu_add,sub,dec,inc_return
In reply to Christoph LameterFrom: Christoph Lameter <[email protected]>
Subject: percpu: Generic support for this_cpu_add, sub, dec, inc_return
Introduce generic support for this_cpu_add_return etc.
The fallback is to realize these operations with simpler __this_cpu_ops.
tj: - Reformatted __cpu_size_call_return2() to make it more consistent
with its neighbors.
- Dropped unnecessary temp variable ret__ from __this_cpu_generic_add_return().
---
I updated it a bit. If you don't like it, please scream.
Thanks.
include/linux/percpu.h | 71 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+)
include/linux/percpu.h +71 -0
@@ -240,6 +240,21 @@ extern void __bad_size_call_parameter(vo pscr_ret__; \ }) +#define __pcpu_size_call_return2 (stem, variable, ...) \ +({ \ + typeof(variable) pscr2_ret__; \ + __verify_pcpu_ptr(&(variable)); \ + switch(sizeof(variable)) { \ + case 1: pscr2_ret__ = stem##1(variable, __VA_ARGS__); break; \ + case 2: pscr2_ret__ = stem##2(variable, __VA_ARGS__); break; \ + case 4: pscr2_ret__ = stem##4(variable, __VA_ARGS__); break; \ + case 8: pscr2_ret__ = stem##8(variable, __VA_ARGS__); break; \ + default: \ + __bad_size_call_parameter(); break; \ + } \ + pscr2_ret__; \ +}) +#define __pcpu_size_call (stem, variable, ...) \ do { \ __verify_pcpu_ptr(&(variable)); \
@@ -529,6 +544,62 @@ do { \# define __this_cpu_xor (pcp, val) __pcpu_size_call(__this_cpu_xor_, (pcp), (val))#endif +#define _this_cpu_generic_add_return (pcp, val) \ +({ \ + typeof(pcp) ret__; \ + preempt_disable(); \ + __this_cpu_add(pcp, val); \ + ret__ = __this_cpu_read(pcp); \ + preempt_enable(); \ + ret__; \ +}) + +#ifndef this_cpu_add_return +# ifndef this_cpu_add_return_1 +# define this_cpu_add_return_1 (pcp, val) _this_cpu_generic_add_return(pcp, val) +# endif +# ifndef this_cpu_add_return_2 +# define this_cpu_add_return_2 (pcp, val) _this_cpu_generic_add_return(pcp, val) +# endif +# ifndef this_cpu_add_return_4 +# define this_cpu_add_return_4 (pcp, val) _this_cpu_generic_add_return(pcp, val) +# endif +# ifndef this_cpu_add_return_8 +# define this_cpu_add_return_8 (pcp, val) _this_cpu_generic_add_return(pcp, val) +# endif +# define this_cpu_add_return (pcp, val) __pcpu_size_call_return2(this_cpu_add_return_, pcp, val) +#endif + +#define this_cpu_sub_return (pcp, val) this_cpu_add_return(pcp, -(val)) +#define this_cpu_inc_return (pcp) this_cpu_add_return(pcp, 1) +#define this_cpu_dec_return (pcp) this_cpu_add_return(pcp, -1) + +#define __this_cpu_generic_add_return (pcp, val) \ +({ \ + __this_cpu_add(pcp, val); \ + __this_cpu_read(pcp); \ +}) + +#ifndef __this_cpu_add_return +# ifndef __this_cpu_add_return_1 +# define __this_cpu_add_return_1 (pcp, val) __this_cpu_generic_add_return(pcp, val) +# endif +# ifndef __this_cpu_add_return_2 +# define __this_cpu_add_return_2 (pcp, val) __this_cpu_generic_add_return(pcp, val) +# endif +# ifndef __this_cpu_add_return_4 +# define __this_cpu_add_return_4 (pcp, val) __this_cpu_generic_add_return(pcp, val) +# endif +# ifndef __this_cpu_add_return_8 +# define __this_cpu_add_return_8 (pcp, val) __this_cpu_generic_add_return(pcp, val) +# endif +# define __this_cpu_add_return (pcp, val) __pcpu_size_call_return2(this_cpu_add_return_, pcp, val) +#endif + +#define __this_cpu_sub_return (pcp, val) this_cpu_add_return(pcp, -(val)) +#define __this_cpu_inc_return (pcp) this_cpu_add_return(pcp, 1) +#define __this_cpu_dec_return (pcp) this_cpu_add_return(pcp, -1) +/* * IRQ safe versions of the per cpu RMW operations. Note that these operations * are *not* safe against modification of the same variable from another
Tejun Heo
maintainerPATCH 2/9 x86: Support for this_cpu_add, sub, dec, inc_return
In reply to Christoph LameterFrom: Christoph Lameter <[email protected]>
Supply an implementation for x86 in order to generate more efficient code.
V2->V3:
- Cleanup
- Remove strange type checking from percpu_add_return_op.
tj: - Dropped unused typedef from percpu_add_return_op().
- Renamed ret__ to paro_ret__ in percpu_add_return_op().
- Minor indentation adjustments.
---
I got a bit fascistic with this one too. Here's the updated version.
Thank you.
arch/x86/include/asm/percpu.h | 43 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+)
arch/x86/include/asm/percpu.h +43 -0
@@ -177,6 +177,39 @@ do { \ } \ }while (0 ) +/* +* Add return operation + */ +#define percpu_add_return_op (var, val) \ +({ \ + typeof(var) paro_ret__ = val; \ + switch (sizeof(var)) { \ + case 1: \ + asm("xaddb %0, "__percpu_arg(1) \ + : "+q" (paro_ret__), "+m" (var) \ + : : "memory"); \ + break; \ + case 2: \ + asm("xaddw %0, "__percpu_arg(1) \ + : "+r" (paro_ret__), "+m" (var) \ + : : "memory"); \ + break; \ + case 4: \ + asm("xaddl %0, "__percpu_arg(1) \ + : "+r" (paro_ret__), "+m" (var) \ + : : "memory"); \ + break; \ + case 8: \ + asm("xaddq %0, "__percpu_arg(1) \ + : "+re" (paro_ret__), "+m" (var) \ + : : "memory"); \ + break; \ + default: __bad_percpu_size(); \ + } \ + paro_ret__ += val; \ + paro_ret__; \ +}) +#define percpu_from_op (op, var, constraint) \ ({ \ typeof(var) pfo_ret__; \
@@ -300,6 +333,14 @@ do { \#define irqsafe_cpu_xor_2 (pcp, val) percpu_to_op("xor", (pcp), val)#define irqsafe_cpu_xor_4 (pcp, val) percpu_to_op("xor", (pcp), val) +#ifndef CONFIG_M386 +#define __this_cpu_add_return_1 (pcp, val) percpu_add_return_op(pcp, val) +#define __this_cpu_add_return_2 (pcp, val) percpu_add_return_op(pcp, val) +#define __this_cpu_add_return_4 (pcp, val) percpu_add_return_op(pcp, val) +#define this_cpu_add_return_1 (pcp, val) percpu_add_return_op(pcp, val) +#define this_cpu_add_return_2 (pcp, val) percpu_add_return_op(pcp, val) +#define this_cpu_add_return_4 (pcp, val) percpu_add_return_op(pcp, val) +#endif /* * Per cpu atomic 64 bit operations are only available under 64 bit. * 32 bit must fall back to generic operations.
@@ -324,6 +365,8 @@ do { \#define irqsafe_cpu_or_8 (pcp, val) percpu_to_op("or", (pcp), val)#define irqsafe_cpu_xor_8 (pcp, val) percpu_to_op("xor", (pcp), val) +#define __this_cpu_add_return_8 (pcp, val) percpu_add_return_op(pcp, val) +#define this_cpu_add_return_8 (pcp, val) percpu_add_return_op(pcp, val)#endif /* This is not atomic against other CPUs -- CPU preemption needs to be off */
Tejun Heo
maintainerRe: [cpuops inc_return V1 3/9] x86: Use this_cpu_inc_return for nmi counter
In reply to Christoph LameterQuoted 6 lines · Christoph Lameter
this_cpu_inc_return()saves us a memory access there.Reviewed-by Tejun Heo <[email protected]>Reviewed-by Pekka Enberg <[email protected]>Reviewed-by Mathieu Desnoyers <[email protected]>Signed-off-by Christoph Lameter <[email protected]>
03-05, the updated 06, 07-09 and the connector patch applied and pushed out to percpu#for-next.
Thanks.
Signature · 1 line
tejun
Re: [cpuops inc_return V1 3/9] x86: Use this_cpu_inc_return for nmi counter
Quoted 10 lines · Tejun Heo
Quoted 6 lines · Christoph Lameter
this_cpu_inc_return()saves us a memory access there.Reviewed-by Tejun Heo <[email protected]>Reviewed-by Pekka Enberg <[email protected]>Reviewed-by Mathieu Desnoyers <[email protected]>Signed-off-by Christoph Lameter <[email protected]>03-05, the updated 06, 07-09 and the connector patch applied and pushed out to percpu#for-next.
Great. Then you are ready for the next set with the this_cpu_cmpxchg stuff?
Tejun Heo
maintainerRe: [cpuops inc_return V1 3/9] x86: Use this_cpu_inc_return for nmi counter
Hello, Christoph.
Quoted 10 lines · Christoph Lameter
Quoted 8 lines · Tejun Heo
Quoted 6 lines · Christoph Lameter
this_cpu_inc_return()saves us a memory access there.Reviewed-by Tejun Heo <[email protected]>Reviewed-by Pekka Enberg <[email protected]>Reviewed-by Mathieu Desnoyers <[email protected]>Signed-off-by Christoph Lameter <[email protected]>
Quoted 5 lines
Quoted 2 lines
03-05, the updated 06, 07-09 and the connector patch applied and pushed out to percpu#for-next.
Great. Then you are ready for the next set with the
this_cpu_cmpxchgstuff?
Heh, sure, shoot. :-)
Signature · 1 line
tejun