percpu: Consistent per cpu operations V1
Subsystems (56)
PATCH 7/41 drivers/oprofile: Replace __get_cpu_var uses for address calculation
In reply to Christoph LameterReplace the uses of __get_cpu_var for address calculation with this_cpu_ptr.
drivers/oprofile/cpu_buffer.c +5 -5
@@ -45,7 +45,7 @@ unsigned long oprofile_get_cpu_buffer_sivoid oprofile_cpu_buffer_inc_smpl_lost (void ) { -struct oprofile_cpu_buffer *cpu_buf = &__get_cpu_var (op_cpu_buffer); +struct oprofile_cpu_buffer *cpu_buf =this_cpu_ptr (&op_cpu_buffer); cpu_buf->sample_lost_overflow++; }
@@ -297,7 +297,7 @@ __oprofile_add_ext_sample(unsigned long unsigned long event, int is_kernel, struct task_struct *task) { - struct oprofile_cpu_buffer *cpu_buf = &__get_cpu_var(op_cpu_buffer); + struct oprofile_cpu_buffer *cpu_buf = this_cpu_ptr(&op_cpu_buffer);unsigned long backtrace = oprofile_backtrace_depth;/*
@@ -357,7 +357,7 @@ oprofile_write_reserve(struct op_entry * {struct op_sample *sample;int is_kernel = !user_mode (regs); -struct oprofile_cpu_buffer *cpu_buf = &__get_cpu_var (op_cpu_buffer); +struct oprofile_cpu_buffer *cpu_buf =this_cpu_ptr (&op_cpu_buffer); cpu_buf->sample_received++;
@@ -412,13 +412,13 @@ int oprofile_write_commit(struct op_entrvoid oprofile_add_pc (unsigned long pc,int is_kernel,unsigned long event) { -struct oprofile_cpu_buffer *cpu_buf = &__get_cpu_var (op_cpu_buffer); +struct oprofile_cpu_buffer *cpu_buf =this_cpu_ptr (&op_cpu_buffer);log_sample (cpu_buf, pc,0 , is_kernel, event,NULL ); }void oprofile_add_trace (unsigned long pc) { -struct oprofile_cpu_buffer *cpu_buf = &__get_cpu_var (op_cpu_buffer); +struct oprofile_cpu_buffer *cpu_buf =this_cpu_ptr (&op_cpu_buffer);if (!cpu_buf->tracing)return ;
drivers/oprofile/timer_int.c +1 -1
@@ -32,7 +32,7 @@ static enum hrtimer_restart oprofile_hrtstatic void __oprofile_hrtimer_start (void *unused) { -struct hrtimer *hrtimer = &__get_cpu_var (oprofile_hrtimer); +struct hrtimer *hrtimer =this_cpu_ptr (&oprofile_hrtimer);if (!ctr_running)return ;
PATCH 31/41 s390: Replace __get_cpu_var uses
In reply to Christoph Lameter__get_cpu_var() is used for multiple purposes in the kernel source. One of them is address calculation via the form &__get_cpu_var(x). This calculates the address for the instance of the percpu variable of the current processor based on an offset.
Other use cases are for storing and retrieving data from the current processors percpu area. __get_cpu_var() can be used as an lvalue when writing data or on the right side of an assignment.
__get_cpu_var() is defined as :
#define __get_cpu_var(var) (*this_cpu_ptr(&(var)))
__get_cpu_var() always only does an address determination. However, store and retrieve operations could use a segment prefix (or global register on other platforms) to avoid the address calculation.
this_cpu_write() and this_cpu_read() can directly take an offset into a percpu area and use optimized assembly code to read and write per cpu variables.
This patch converts __get_cpu_var into either an explicit address calculation using this_cpu_ptr() or into a use of this_cpu operations that use the offset. Thereby address calculations are avoided and less registers are used when code is generated.
At the end of the patch set all uses of __get_cpu_var have been removed so the macro is removed too.
The patch set includes passes over all arches as well. Once these operations are used throughout then specialized macros can be defined in non -x86 arches as well in order to optimize per cpu access by f.e. using a global register that may be set to the per cpu base.
Transformations done to __get_cpu_var()
1. Determine the address of the percpu instance of the current processor.
DEFINE_PER_CPU(int, y);
int *x = &__get_cpu_var(y);
Converts to
int *x = this_cpu_ptr(&y);2. Same as #1 but this time an array structure is involved.
DEFINE_PER_CPU(int, y[20]);
int *x = __get_cpu_var(y);
Converts to
int *x = this_cpu_ptr(y);3. Retrieve the content of the current processors instance of a per cpu variable.
DEFINE_PER_CPU(int, y); int x = __get_cpu_var(y)
Converts to
int x = __this_cpu_read(y);
4. Retrieve the content of a percpu struct
DEFINE_PER_CPU(struct mystruct, y); struct mystruct x = __get_cpu_var(y);
Converts to
memcpy(&x, this_cpu_ptr(&y), sizeof(x));
5. Assignment to a per cpu variable
DEFINE_PER_CPU(int, y) __get_cpu_var(y) = x;
Converts to
this_cpu_write(y, x);
6. Increment/Decrement etc of a per cpu variable
DEFINE_PER_CPU(int, y); __get_cpu_var(y)++
Converts to
this_cpu_inc(y)
arch/s390/include/asm/irq.h +1 -1
@@ -66,7 +66,7 @@ DECLARE_PER_CPU_SHARED_ALIGNED(struct ir static __always_inline void inc_irq_stat(enum interruption_class irq) { -__get_cpu_var (irq_stat).irqs[irq]++; +__this_cpu_inc (irq_stat.irqs[irq]); }struct ext_code {
arch/s390/include/asm/cputime.h +1 -1
@@ -184,7 +184,7 @@ cputime64_t s390_get_idle_time(int cpu);static inline int s390_nohz_delay (int cpu) { -return __get_cpu_var (s390_idle).nohz_delay !=0 ; +return __this_cpu_read (s390_idle.nohz_delay) !=0 ; }#define arch_needs_cpu (cpu) s390_nohz_delay(cpu)
arch/s390/kernel/kprobes.c +4 -4
@@ -366,9 +366,9 @@ static void __kprobes disable_singlestep*/ static void __kprobes push_kprobe (struct kprobe_ctlblk *kcb,struct kprobe *p) { - kcb->prev_kprobe.kp =__get_cpu_var (current_kprobe); + kcb->prev_kprobe.kp =__this_cpu_read (current_kprobe); kcb->prev_kprobe.status = kcb->kprobe_status; -__get_cpu_var (current_kprobe) = p; +__this_cpu_write (current_kprobe, p); }/*
@@ -378,7 +378,7 @@ static void __kprobes push_kprobe(struct*/ static void __kprobes pop_kprobe (struct kprobe_ctlblk *kcb) { -__get_cpu_var (current_kprobe) = kcb->prev_kprobe.kp; +__this_cpu_write (current_kprobe, kcb->prev_kprobe.kp); kcb->kprobe_status = kcb->prev_kprobe.status; }
@@ -459,7 +459,7 @@ static int __kprobes kprobe_handler(struenable_singlestep (kcb, regs, (unsigned long ) p->ainsn.insn);return 1 ; }else if (kprobe_running ()) { - p =__get_cpu_var (current_kprobe); + p =__this_cpu_read (current_kprobe);if (p->break_handler && p->break_handler (p, regs)) {/* * Continuation after the jprobe completed and
arch/s390/kernel/nmi.c +3 -3
@@ -53,8 +53,8 @@ void s390_handle_mcck(void)*/ local_irq_save (flags);local_mcck_disable (); - mcck =__get_cpu_var (cpu_mcck); -memset (&__get_cpu_var (cpu_mcck),0 ,sizeof (struct mcck_struct )); + mcck =__this_cpu_read (cpu_mcck); +memset (this_cpu_ptr (&cpu_mcck),0 ,sizeof (struct mcck_struct ));clear_thread_flag (TIF_MCCK_PENDING );local_mcck_enable ();local_irq_restore (flags);
@@ -253,7 +253,7 @@ void notrace s390_do_machine_check(strucnmi_enter ();inc_irq_stat (NMI_NMI ); mci = (struct mci *) &S390_lowcore.mcck_interruption_code; - mcck = &__get_cpu_var (cpu_mcck); + mcck =this_cpu_ptr (&cpu_mcck); umode =user_mode (regs);if (mci->sd) {
arch/s390/kernel/perf_cpum_cf.c +11 -11
@@ -173,7 +173,7 @@ static int validate_ctr_auth(const struc*/ static void cpumf_pmu_enable (struct pmu *pmu) { -struct cpu_hw_events *cpuhw = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuhw =this_cpu_ptr (&cpu_hw_events);int err;if (cpuhw->flags &PMU_F_ENABLED )
@@ -196,7 +196,7 @@ static void cpumf_pmu_enable(struct pmu*/ static void cpumf_pmu_disable (struct pmu *pmu) { -struct cpu_hw_events *cpuhw = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuhw =this_cpu_ptr (&cpu_hw_events);int err;u64 inactive;
@@ -230,7 +230,7 @@ static void cpumf_measurement_alert(strureturn ;inc_irq_stat (IRQEXT_CMC ); - cpuhw = &__get_cpu_var (cpu_hw_events); + cpuhw =this_cpu_ptr (&cpu_hw_events);/* Measurement alerts are shared and might happen when the PMU * is not reserved. Ignore these alerts in this case. */
@@ -250,7 +250,7 @@ static void cpumf_measurement_alert(stru#define PMC_RELEASE 1static void setup_pmc_cpu (void *flags) { -struct cpu_hw_events *cpuhw = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuhw =this_cpu_ptr (&cpu_hw_events);switch (*((int *) flags)) {case PMC_INIT :
@@ -481,7 +481,7 @@ static void cpumf_pmu_read(struct perf_estatic void cpumf_pmu_start (struct perf_event *event,int flags) { -struct cpu_hw_events *cpuhw = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuhw =this_cpu_ptr (&cpu_hw_events);struct hw_perf_event *hwc = &event->hw;if (WARN_ON_ONCE (!(hwc->state &PERF_HES_STOPPED )))
@@ -512,7 +512,7 @@ static void cpumf_pmu_start(struct perf_static void cpumf_pmu_stop (struct perf_event *event,int flags) { -struct cpu_hw_events *cpuhw = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuhw =this_cpu_ptr (&cpu_hw_events);struct hw_perf_event *hwc = &event->hw;if (!(hwc->state &PERF_HES_STOPPED )) {
@@ -533,7 +533,7 @@ static void cpumf_pmu_stop(struct perf_estatic int cpumf_pmu_add (struct perf_event *event,int flags) { -struct cpu_hw_events *cpuhw = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuhw =this_cpu_ptr (&cpu_hw_events);/* Check authorization for the counter set to which this * counter belongs.
@@ -557,7 +557,7 @@ static int cpumf_pmu_add(struct perf_evestatic void cpumf_pmu_del (struct perf_event *event,int flags) { -struct cpu_hw_events *cpuhw = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuhw =this_cpu_ptr (&cpu_hw_events);cpumf_pmu_stop (event,PERF_EF_UPDATE );
@@ -581,7 +581,7 @@ static void cpumf_pmu_del(struct perf_ev*/ static void cpumf_pmu_start_txn (struct pmu *pmu) { -struct cpu_hw_events *cpuhw = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuhw =this_cpu_ptr (&cpu_hw_events);perf_pmu_disable (pmu); cpuhw->flags |=PERF_EVENT_TXN ;
@@ -595,7 +595,7 @@ static void cpumf_pmu_start_txn(struct p*/ static void cpumf_pmu_cancel_txn (struct pmu *pmu) { -struct cpu_hw_events *cpuhw = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuhw =this_cpu_ptr (&cpu_hw_events);WARN_ON (cpuhw->tx_state != cpuhw->state);
@@ -610,7 +610,7 @@ static void cpumf_pmu_cancel_txn(struct*/ static int cpumf_pmu_commit_txn (struct pmu *pmu) { -struct cpu_hw_events *cpuhw = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuhw =this_cpu_ptr (&cpu_hw_events);u64 state;/* check if the updated state can be scheduled */
arch/s390/kernel/processor.c +2 -2
@@ -23,8 +23,8 @@ static DEFINE_PER_CPU(struct cpuid, cpu_*/ void cpu_init (void ) { -struct s390_idle_data *idle = &__get_cpu_var (s390_idle); -struct cpuid *id = &__get_cpu_var (cpu_id); +struct s390_idle_data *idle =this_cpu_ptr (&s390_idle); +struct cpuid *id =this_cpu_ptr (&cpu_id);get_cpu_id (id);atomic_inc (&init_mm.mm_count);
arch/s390/kernel/time.c +3 -3
@@ -92,7 +92,7 @@ void clock_comparator_work(void)struct clock_event_device *cd; S390_lowcore.clock_comparator =-1ULL ; - cd = &__get_cpu_var (comparators); + cd =this_cpu_ptr (&comparators); cd->event_handler (cd); }
@@ -360,7 +360,7 @@ EXPORT_SYMBOL(get_sync_clock);*/ static void disable_sync_clock (void *dummy) { -atomic_t *sw_ptr = &__get_cpu_var (clock_sync_word); +atomic_t *sw_ptr =this_cpu_ptr (&clock_sync_word);/* * Clear the in-sync bit 2^31. All get_sync_clock calls will * fail until the sync bit is turned back on. In addition
@@ -377,7 +377,7 @@ static void disable_sync_clock(void *dum*/ static void enable_sync_clock (void ) { -atomic_t *sw_ptr = &__get_cpu_var (clock_sync_word); +atomic_t *sw_ptr =this_cpu_ptr (&clock_sync_word);atomic_set_mask (0x80000000 , sw_ptr); }
arch/s390/kernel/vtime.c +1 -1
@@ -154,7 +154,7 @@ EXPORT_SYMBOL_GPL(vtime_account_system);void __kprobes vtime_stop_cpu (void ) { -struct s390_idle_data *idle = &__get_cpu_var (s390_idle); +struct s390_idle_data *idle =this_cpu_ptr (&s390_idle);unsigned long long idle_time;unsigned long psw_mask;
arch/s390/oprofile/hwsampler.c +1 -1
@@ -228,7 +228,7 @@ static inline unsigned long *trailer_entstatic void hws_ext_handler (struct ext_code ext_code,unsigned int param32,unsigned long param64) { -struct hws_cpu_buffer *cb = &__get_cpu_var (sampler_cpu_buffer); +struct hws_cpu_buffer *cb =this_cpu_ptr (&sampler_cpu_buffer);if (!(param32 &CPU_MF_INT_SF_MASK ))return ;
arch/s390/kernel/irq.c +1 -1
@@ -252,7 +252,7 @@ static irqreturn_t do_ext_interrupt(int ext_code = *(struct ext_code *) ®s->int_code;if (ext_code.code !=0x1004 ) -__get_cpu_var (s390_idle).nohz_delay =1 ; +__this_cpu_write (s390_idle.nohz_delay,1 ); index =ext_hash (ext_code.code);rcu_read_lock ();
arch/s390/include/asm/percpu.h +8 -8
@@ -31,7 +31,7 @@pcp_op_T__ old__, new__, prev__; \pcp_op_T__ *ptr__; \preempt_disable (); \ - ptr__ =__this_cpu_ptr (&(pcp)); \ + ptr__ =raw_cpu_ptr (&(pcp)); \ prev__ = *ptr__; \ do { \ old__ = prev__; \
@@ -70,7 +70,7 @@pcp_op_T__ val__ = (val); \pcp_op_T__ old__, *ptr__; \preempt_disable (); \ - ptr__ =__this_cpu_ptr (&(pcp)); \ + ptr__ =raw_cpu_ptr (&(pcp)); \if (__builtin_constant_p (val__) && \ ((szcast )val__ >-129 ) && ((szcast )val__ <128 )) { \ asm volatile( \
@@ -97,7 +97,7 @@pcp_op_T__ val__ = (val); \pcp_op_T__ old__, *ptr__; \preempt_disable (); \ - ptr__ =__this_cpu_ptr (&(pcp)); \ + ptr__ =raw_cpu_ptr (&(pcp)); \ asmvolatile ( \ op" %[old__],%[val__],%[ptr__]\n" \ : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \
@@ -116,7 +116,7 @@pcp_op_T__ val__ = (val); \pcp_op_T__ old__, *ptr__; \preempt_disable (); \ - ptr__ =__this_cpu_ptr (&(pcp)); \ + ptr__ =raw_cpu_ptr (&(pcp)); \ asmvolatile ( \ op" %[old__],%[val__],%[ptr__]\n" \ : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \
@@ -138,7 +138,7 @@pcp_op_T__ ret__; \pcp_op_T__ *ptr__; \preempt_disable (); \ - ptr__ =__this_cpu_ptr (&(pcp)); \ + ptr__ =raw_cpu_ptr (&(pcp)); \ ret__ =cmpxchg (ptr__, oval, nval); \preempt_enable (); \ ret__; \
@@ -154,7 +154,7 @@typeof (pcp) *ptr__; \typeof (pcp) ret__; \preempt_disable (); \ - ptr__ =__this_cpu_ptr (&(pcp)); \ + ptr__ =raw_cpu_ptr (&(pcp)); \ ret__ =xchg (ptr__, nval); \preempt_enable (); \ ret__; \
@@ -173,8 +173,8 @@typeof (pcp2) *p2__; \int ret__; \preempt_disable (); \ - p1__ =__this_cpu_ptr (&(pcp1)); \ - p2__ =__this_cpu_ptr (&(pcp2)); \ + p1__ =raw_cpu_ptr (&(pcp1)); \ + p2__ =raw_cpu_ptr (&(pcp2)); \ ret__ =__cmpxchg_double (p1__, p2__, o1__, o2__, n1__, n2__); \preempt_enable (); \ ret__; \
PATCH 14/41 x86: Rename __this_cpu_xxx_# operations to raw_cpu_xxx_#
In reply to Christoph LameterThis is required now since the core portion expects a different naming.
Change the names from __this_cpu_xxxx_# to raw_cpu_xxxx_#.
Also changes the uses of the x86 percpu primitives in preempt.h.
arch/x86/include/asm/percpu.h +49 -49
@@ -362,25 +362,25 @@ do { \*/ #define this_cpu_read_stable (var) percpu_from_op("mov", var, "p" (&(var))) -#define __this_cpu_read_1 (pcp) percpu_from_op("mov", (pcp), "m"(pcp)) -#define __this_cpu_read_2 (pcp) percpu_from_op("mov", (pcp), "m"(pcp)) -#define __this_cpu_read_4 (pcp) percpu_from_op("mov", (pcp), "m"(pcp)) - -#define __this_cpu_write_1 (pcp, val) percpu_to_op("mov", (pcp), val) -#define __this_cpu_write_2 (pcp, val) percpu_to_op("mov", (pcp), val) -#define __this_cpu_write_4 (pcp, val) percpu_to_op("mov", (pcp), val) -#define __this_cpu_add_1 (pcp, val) percpu_add_op((pcp), val) -#define __this_cpu_add_2 (pcp, val) percpu_add_op((pcp), val) -#define __this_cpu_add_4 (pcp, val) percpu_add_op((pcp), val) -#define __this_cpu_and_1 (pcp, val) percpu_to_op("and", (pcp), val) -#define __this_cpu_and_2 (pcp, val) percpu_to_op("and", (pcp), val) -#define __this_cpu_and_4 (pcp, val) percpu_to_op("and", (pcp), val) -#define __this_cpu_or_1 (pcp, val) percpu_to_op("or", (pcp), val) -#define __this_cpu_or_2 (pcp, val) percpu_to_op("or", (pcp), val) -#define __this_cpu_or_4 (pcp, val) percpu_to_op("or", (pcp), val) -#define __this_cpu_xchg_1 (pcp, val) percpu_xchg_op(pcp, val) -#define __this_cpu_xchg_2 (pcp, val) percpu_xchg_op(pcp, val) -#define __this_cpu_xchg_4 (pcp, val) percpu_xchg_op(pcp, val) +#define raw_cpu_read_1 (pcp) percpu_from_op("mov", (pcp), "m"(pcp)) +#define raw_cpu_read_2 (pcp) percpu_from_op("mov", (pcp), "m"(pcp)) +#define raw_cpu_read_4 (pcp) percpu_from_op("mov", (pcp), "m"(pcp)) + +#define raw_cpu_write_1 (pcp, val) percpu_to_op("mov", (pcp), val) +#define raw_cpu_write_2 (pcp, val) percpu_to_op("mov", (pcp), val) +#define raw_cpu_write_4 (pcp, val) percpu_to_op("mov", (pcp), val) +#define raw_cpu_add_1 (pcp, val) percpu_add_op((pcp), val) +#define raw_cpu_add_2 (pcp, val) percpu_add_op((pcp), val) +#define raw_cpu_add_4 (pcp, val) percpu_add_op((pcp), val) +#define raw_cpu_and_1 (pcp, val) percpu_to_op("and", (pcp), val) +#define raw_cpu_and_2 (pcp, val) percpu_to_op("and", (pcp), val) +#define raw_cpu_and_4 (pcp, val) percpu_to_op("and", (pcp), val) +#define raw_cpu_or_1 (pcp, val) percpu_to_op("or", (pcp), val) +#define raw_cpu_or_2 (pcp, val) percpu_to_op("or", (pcp), val) +#define raw_cpu_or_4 (pcp, val) percpu_to_op("or", (pcp), val) +#define raw_cpu_xchg_1 (pcp, val) percpu_xchg_op(pcp, val) +#define raw_cpu_xchg_2 (pcp, val) percpu_xchg_op(pcp, val) +#define raw_cpu_xchg_4 (pcp, val) percpu_xchg_op(pcp, val)#define this_cpu_read_1 (pcp) percpu_from_op("mov", (pcp), "m"(pcp))#define this_cpu_read_2 (pcp) percpu_from_op("mov", (pcp), "m"(pcp))
@@ -401,16 +401,16 @@ do { \#define this_cpu_xchg_2 (pcp, nval) percpu_xchg_op(pcp, nval)#define this_cpu_xchg_4 (pcp, nval) percpu_xchg_op(pcp, nval) -#define __this_cpu_add_return_1 (pcp, val) percpu_add_return_op(pcp, val) -#define __this_cpu_add_return_2 (pcp, val) percpu_add_return_op(pcp, val) -#define __this_cpu_add_return_4 (pcp, val) percpu_add_return_op(pcp, val) -#define __this_cpu_cmpxchg_1 (pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) -#define __this_cpu_cmpxchg_2 (pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) -#define __this_cpu_cmpxchg_4 (pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) - -#define this_cpu_add_return_1 (pcp, val) percpu_add_return_op(pcp, val) -#define this_cpu_add_return_2 (pcp, val) percpu_add_return_op(pcp, val) -#define this_cpu_add_return_4 (pcp, val) percpu_add_return_op(pcp, val) +#define raw_cpu_add_return_1 (pcp, val) percpu_add_return_op(pcp, val) +#define raw_cpu_add_return_2 (pcp, val) percpu_add_return_op(pcp, val) +#define raw_cpu_add_return_4 (pcp, val) percpu_add_return_op(pcp, val) +#define raw_cpu_cmpxchg_1 (pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) +#define raw_cpu_cmpxchg_2 (pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) +#define raw_cpu_cmpxchg_4 (pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) + +#define this_cpu_add_return_1 (pcp, val) percpu_add_return_op(pcp, val) +#define this_cpu_add_return_2 (pcp, val) percpu_add_return_op(pcp, val) +#define this_cpu_add_return_4 (pcp, val) percpu_add_return_op(pcp, val)#define this_cpu_cmpxchg_1 (pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)#define this_cpu_cmpxchg_2 (pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)#define this_cpu_cmpxchg_4 (pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)
@@ -427,7 +427,7 @@ do { \ __ret; \ }) -#define __this_cpu_cmpxchg_double_4 percpu_cmpxchg8b_double +#define raw_cpu_cmpxchg_double_4 percpu_cmpxchg8b_double#define this_cpu_cmpxchg_double_4 percpu_cmpxchg8b_double#endif /* CONFIG_X86_CMPXCHG64 */
@@ -436,22 +436,22 @@ do { \* 32 bit must fall back to generic operations. */ #ifdef CONFIG_X86_64 -#define __this_cpu_read_8 (pcp) percpu_from_op("mov", (pcp), "m"(pcp)) -#define __this_cpu_write_8 (pcp, val) percpu_to_op("mov", (pcp), val) -#define __this_cpu_add_8 (pcp, val) percpu_add_op((pcp), val) -#define __this_cpu_and_8 (pcp, val) percpu_to_op("and", (pcp), val) -#define __this_cpu_or_8 (pcp, val) percpu_to_op("or", (pcp), val) -#define __this_cpu_add_return_8 (pcp, val) percpu_add_return_op(pcp, val) -#define __this_cpu_xchg_8 (pcp, nval) percpu_xchg_op(pcp, nval) -#define __this_cpu_cmpxchg_8 (pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) - -#define this_cpu_read_8 (pcp) percpu_from_op("mov", (pcp), "m"(pcp)) -#define this_cpu_write_8 (pcp, val) percpu_to_op("mov", (pcp), val) -#define this_cpu_add_8 (pcp, val) percpu_add_op((pcp), val) -#define this_cpu_and_8 (pcp, val) percpu_to_op("and", (pcp), val) -#define this_cpu_or_8 (pcp, val) percpu_to_op("or", (pcp), val) -#define this_cpu_add_return_8 (pcp, val) percpu_add_return_op(pcp, val) -#define this_cpu_xchg_8 (pcp, nval) percpu_xchg_op(pcp, nval) +#define raw_cpu_read_8 (pcp) percpu_from_op("mov", (pcp), "m"(pcp)) +#define raw_cpu_write_8 (pcp, val) percpu_to_op("mov", (pcp), val) +#define raw_cpu_add_8 (pcp, val) percpu_add_op((pcp), val) +#define raw_cpu_and_8 (pcp, val) percpu_to_op("and", (pcp), val) +#define raw_cpu_or_8 (pcp, val) percpu_to_op("or", (pcp), val) +#define raw_cpu_add_return_8 (pcp, val) percpu_add_return_op(pcp, val) +#define raw_cpu_xchg_8 (pcp, nval) percpu_xchg_op(pcp, nval) +#define raw_cpu_cmpxchg_8 (pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) + +#define this_cpu_read_8 (pcp) percpu_from_op("mov", (pcp), "m"(pcp)) +#define this_cpu_write_8 (pcp, val) percpu_to_op("mov", (pcp), val) +#define this_cpu_add_8 (pcp, val) percpu_add_op((pcp), val) +#define this_cpu_and_8 (pcp, val) percpu_to_op("and", (pcp), val) +#define this_cpu_or_8 (pcp, val) percpu_to_op("or", (pcp), val) +#define this_cpu_add_return_8 (pcp, val) percpu_add_return_op(pcp, val) +#define this_cpu_xchg_8 (pcp, nval) percpu_xchg_op(pcp, nval)#define this_cpu_cmpxchg_8 (pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)/*
@@ -474,7 +474,7 @@ do { \ __ret; \ }) -#define __this_cpu_cmpxchg_double_8 percpu_cmpxchg16b_double +#define raw_cpu_cmpxchg_double_8 percpu_cmpxchg16b_double#define this_cpu_cmpxchg_double_8 percpu_cmpxchg16b_double#endif
@@ -495,9 +495,9 @@ static __always_inline int x86_this_cpu_unsigned long __percpu *a = (unsigned long *)addr + nr /BITS_PER_LONG ;#ifdef CONFIG_X86_64 -return ((1UL << (nr %BITS_PER_LONG )) &__this_cpu_read_8 (*a)) !=0 ; +return ((1UL << (nr %BITS_PER_LONG )) &raw_cpu_read_8 (*a)) !=0 ;#else -return ((1UL << (nr %BITS_PER_LONG )) &__this_cpu_read_4 (*a)) !=0 ; +return ((1UL << (nr %BITS_PER_LONG )) &raw_cpu_read_4 (*a)) !=0 ;#endif }
arch/x86/include/asm/preempt.h +8 -8
@@ -13,12 +13,12 @@ DECLARE_PER_CPU(int, __preempt_count);*/ static __always_inline int preempt_count(void) { -return __this_cpu_read_4 (__preempt_count) & ~PREEMPT_NEED_RESCHED ; +return raw_cpu_read_4 (__preempt_count) & ~PREEMPT_NEED_RESCHED ; } static __always_inline void preempt_count_set(int pc) { -__this_cpu_write_4 (__preempt_count, pc); +raw_cpu_write_4 (__preempt_count, pc); }/*
@@ -47,17 +47,17 @@ static __always_inline void preempt_coun static __always_inline void set_preempt_need_resched(void) { -__this_cpu_and_4 (__preempt_count, ~PREEMPT_NEED_RESCHED ); +raw_cpu_and_4 (__preempt_count, ~PREEMPT_NEED_RESCHED ); } static __always_inline void clear_preempt_need_resched(void) { -__this_cpu_or_4 (__preempt_count,PREEMPT_NEED_RESCHED ); +raw_cpu_or_4 (__preempt_count,PREEMPT_NEED_RESCHED ); } static __always_inline bool test_preempt_need_resched(void) { -return !(__this_cpu_read_4 (__preempt_count) &PREEMPT_NEED_RESCHED ); +return !(raw_cpu_read_4 (__preempt_count) &PREEMPT_NEED_RESCHED ); }/*
@@ -66,12 +66,12 @@ static __always_inline bool test_preempt static __always_inline void __preempt_count_add(int val) { -__this_cpu_add_4 (__preempt_count, val); +raw_cpu_add_4 (__preempt_count, val); } static __always_inline void __preempt_count_sub(int val) { -__this_cpu_add_4 (__preempt_count, -val); +raw_cpu_add_4 (__preempt_count, -val); } static __always_inline bool __preempt_count_dec_and_test(void)
@@ -84,7 +84,7 @@ static __always_inline bool __preempt_co*/ static __always_inline bool should_resched(void) { -return unlikely (!__this_cpu_read_4 (__preempt_count)); +return unlikely (!raw_cpu_read_4 (__preempt_count)); }#ifdef CONFIG_PREEMPT
PATCH 9/41 drivers/clocksource: Replace __get_cpu_var used for address calculation
In reply to Christoph LameterReplace __get_cpu_var used for address calculation with this_cpu_ptr.
drivers/clocksource/metag_generic.c +1 -1
@@ -90,7 +90,7 @@ static struct clocksource clocksource_mestatic irqreturn_t metag_timer_interrupt (int irq,void *dummy) { -struct clock_event_device *evt = &__get_cpu_var (local_clockevent); +struct clock_event_device *evt =this_cpu_ptr (&local_clockevent); evt->event_handler (evt);
PATCH 35/41 tile: Replace __get_cpu_var uses
In reply to Christoph Lameter__get_cpu_var() is used for multiple purposes in the kernel source. One of them is address calculation via the form &__get_cpu_var(x). This calculates the address for the instance of the percpu variable of the current processor based on an offset.
Other use cases are for storing and retrieving data from the current processors percpu area. __get_cpu_var() can be used as an lvalue when writing data or on the right side of an assignment.
__get_cpu_var() is defined as :
#define __get_cpu_var(var) (*this_cpu_ptr(&(var)))
__get_cpu_var() always only does an address determination. However, store and retrieve operations could use a segment prefix (or global register on other platforms) to avoid the address calculation.
this_cpu_write() and this_cpu_read() can directly take an offset into a percpu area and use optimized assembly code to read and write per cpu variables.
This patch converts __get_cpu_var into either an explicit address calculation using this_cpu_ptr() or into a use of this_cpu operations that use the offset. Thereby address calculations are avoided and less registers are used when code is generated.
At the end of the patch set all uses of __get_cpu_var have been removed so the macro is removed too.
The patch set includes passes over all arches as well. Once these operations are used throughout then specialized macros can be defined in non -x86 arches as well in order to optimize per cpu access by f.e. using a global register that may be set to the per cpu base.
Transformations done to __get_cpu_var()
1. Determine the address of the percpu instance of the current processor.
DEFINE_PER_CPU(int, y);
int *x = &__get_cpu_var(y);
Converts to
int *x = this_cpu_ptr(&y);2. Same as #1 but this time an array structure is involved.
DEFINE_PER_CPU(int, y[20]);
int *x = __get_cpu_var(y);
Converts to
int *x = this_cpu_ptr(y);3. Retrieve the content of the current processors instance of a per cpu variable.
DEFINE_PER_CPU(int, y); int x = __get_cpu_var(y)
Converts to
int x = __this_cpu_read(y);
4. Retrieve the content of a percpu struct
DEFINE_PER_CPU(struct mystruct, y); struct mystruct x = __get_cpu_var(y);
Converts to
memcpy(&x, this_cpu_ptr(&y), sizeof(x));
5. Assignment to a per cpu variable
DEFINE_PER_CPU(int, y) __get_cpu_var(y) = x;
Converts to
this_cpu_write(y, x);
6. Increment/Decrement etc of a per cpu variable
DEFINE_PER_CPU(int, y); __get_cpu_var(y)++
Converts to
this_cpu_inc(y)
arch/tile/include/asm/mmu_context.h +3 -3
@@ -84,7 +84,7 @@ static inline void enter_lazy_tlb(struct* clear any pending DMA interrupts. */ if (current->thread.tile_dma_state.enabled) -install_page_table (mm->pgd,__get_cpu_var (current_asid)); +install_page_table (mm->pgd,__this_cpu_read (current_asid));#endif }
@@ -96,12 +96,12 @@ static inline void switch_mm(struct mm_sint cpu =smp_processor_id ();/* Pick new ASID. */ -int asid =__get_cpu_var (current_asid) +1 ; +int asid =__this_cpu_read (current_asid) +1 ;if (asid > max_asid) { asid = min_asid;local_flush_tlb (); } -__get_cpu_var (current_asid) = asid; +__this_cpu_write (current_asid, asid);/* Clear cpu from the old mm, and set it in the new one. */ cpumask_clear_cpu (cpu,mm_cpumask (prev));
arch/tile/kernel/irq.c +7 -7
@@ -79,7 +79,7 @@ static DEFINE_SPINLOCK(available_irqs_lo*/ void tile_dev_intr (struct pt_regs *regs,int intnum) { -int depth =__get_cpu_var (irq_depth)++; +int depth =__this_cpu_inc_return (irq_depth);unsigned long original_irqs;unsigned long remaining_irqs;struct pt_regs *old_regs;
@@ -126,7 +126,7 @@ void tile_dev_intr(struct pt_regs *regs,/* Count device irqs; Linux IPIs are counted elsewhere. */ if (irq !=IRQ_RESCHEDULE ) -__get_cpu_var (irq_stat).irq_dev_intr_count++; +__this_cpu_inc (irq_stat.irq_dev_intr_count);generic_handle_irq (irq); }
@@ -136,10 +136,10 @@ void tile_dev_intr(struct pt_regs *regs,* including any that were reenabled during interrupt * handling. */ -if (depth ==0 ) -unmask_irqs (~__get_cpu_var (irq_disable_mask)); +if (depth ==1 ) +unmask_irqs (~__this_cpu_read (irq_disable_mask)); -__get_cpu_var (irq_depth)--; +__this_cpu_dec (irq_depth);/* * Track time spent against the current process again and
@@ -157,7 +157,7 @@ void tile_dev_intr(struct pt_regs *regs,static void tile_irq_chip_enable (struct irq_data *d) {get_cpu_var (irq_disable_mask) &= ~(1UL << d->irq); -if (__get_cpu_var (irq_depth) ==0 ) +if (__this_cpu_read (irq_depth) ==0 )unmask_irqs (1UL << d->irq);put_cpu_var (irq_disable_mask); }
@@ -203,7 +203,7 @@ static void tile_irq_chip_ack(struct irq*/ static void tile_irq_chip_eoi (struct irq_data *d) { -if (!(__get_cpu_var (irq_disable_mask) & (1UL << d->irq))) +if (!(__this_cpu_read (irq_disable_mask) & (1UL << d->irq)))unmask_irqs (1UL << d->irq); }
arch/tile/kernel/messaging.c +3 -3
@@ -28,7 +28,7 @@ static DEFINE_PER_CPU(HV_MsgState, msg_svoid init_messaging (void ) {/* Allocate storage for messages in kernel space */ -HV_MsgState *state = &__get_cpu_var (msg_state); +HV_MsgState *state =this_cpu_ptr (&msg_state);int rc =hv_register_message_state (state);if (rc !=HV_OK )panic ("hv_register_message_state: error %d" , rc);
@@ -68,7 +68,7 @@ void hv_message_intr(struct pt_regs *reg#endif while (1 ) { - rmi =hv_receive_message (__get_cpu_var (msg_state), + rmi =hv_receive_message (__this_cpu_read (msg_state), (HV_VirtAddr ) message,sizeof (message));if (rmi.msglen ==0 )
@@ -96,7 +96,7 @@ void hv_message_intr(struct pt_regs *regstruct hv_driver_cb *cb = (struct hv_driver_cb *)him->intarg; cb->callback (cb, him->intdata); -__get_cpu_var (irq_stat).irq_hv_msg_count++; +__this_cpu_inc (irq_stat.irq_hv_msg_count); } }
arch/tile/kernel/process.c +1 -1
@@ -64,7 +64,7 @@ early_param("idle", idle_setup);void arch_cpu_idle (void ) { -__get_cpu_var (irq_stat).idle_timestamp = jiffies; +__this_cpu_write (irq_stat.idle_timestamp, jiffies);_cpu_idle (); }
arch/tile/kernel/setup.c +2 -1
@@ -1220,7 +1220,8 @@ static void __init validate_hv(void)* various asid variables to their appropriate initial states. */ asid_range =hv_inquire_asid (0 ); -__get_cpu_var (current_asid) = min_asid = asid_range.start; + min_asid = asid_range.start; +__this_cpu_write (current_asid, min_asid); max_asid = asid_range.start + asid_range.size -1 ; if (hv_confstr(HV_CONFSTR_CHIP_MODEL, (HV_VirtAddr)chip_model,
arch/tile/kernel/single_step.c +2 -2
@@ -740,7 +740,7 @@ static DEFINE_PER_CPU(unsigned long, ss_void gx_singlestep_handle (struct pt_regs *regs,int fault_num) { -unsigned long *ss_pc = &__get_cpu_var (ss_saved_pc); +unsigned long *ss_pc =this_cpu_ptr (&ss_saved_pc);struct thread_info *info = (void *)current_thread_info ();int is_single_step =test_ti_thread_flag (info,TIF_SINGLESTEP );unsigned long control =__insn_mfspr (SPR_SINGLE_STEP_CONTROL_K );
@@ -766,7 +766,7 @@ void gx_singlestep_handle(struct pt_regsvoid single_step_once (struct pt_regs *regs) { -unsigned long *ss_pc = &__get_cpu_var (ss_saved_pc); +unsigned long *ss_pc =this_cpu_ptr (&ss_saved_pc);unsigned long control =__insn_mfspr (SPR_SINGLE_STEP_CONTROL_K ); *ss_pc = regs->pc;
arch/tile/kernel/smp.c +1 -1
@@ -188,7 +188,7 @@ void flush_icache_range(unsigned long st/* Called when smp_send_reschedule() triggers IRQ_RESCHEDULE. */ static irqreturn_t handle_reschedule_ipi (int irq,void *token) { -__get_cpu_var (irq_stat).irq_resched_count++; +__this_cpu_inc (irq_stat.irq_resched_count);scheduler_ipi ();return IRQ_HANDLED ;
arch/tile/kernel/smpboot.c +3 -3
@@ -41,7 +41,7 @@ void __init smp_prepare_boot_cpu(void)int cpu =smp_processor_id ();set_cpu_online (cpu,1 );set_cpu_present (cpu,1 ); -__get_cpu_var (cpu_state) =CPU_ONLINE ; +__this_cpu_write (cpu_state,CPU_ONLINE );init_messaging (); }
@@ -158,7 +158,7 @@ static void start_secondary(void)/* printk(KERN_DEBUG "Initializing CPU#%d\n", cpuid); */ /* Initialize the current asid for our first page table. */ -__get_cpu_var (current_asid) = min_asid; +__this_cpu_write (current_asid, min_asid);/* Set up this thread as another owner of the init_mm */ atomic_inc (&init_mm.mm_count);
@@ -201,7 +201,7 @@ void online_secondary(void)notify_cpu_starting (smp_processor_id ());set_cpu_online (smp_processor_id (),1 ); -__get_cpu_var (cpu_state) =CPU_ONLINE ; +__this_cpu_write (cpu_state,CPU_ONLINE );/* Set up tile-specific state for this cpu. */ setup_cpu (0 );
arch/tile/kernel/time.c +4 -4
@@ -162,7 +162,7 @@ static DEFINE_PER_CPU(struct clock_eventvoid setup_tile_timer (void ) { -struct clock_event_device *evt = &__get_cpu_var (tile_timer); +struct clock_event_device *evt =this_cpu_ptr (&tile_timer);/* Fill in fields that are speed-specific. */ clockevents_calc_mult_shift (evt, cycles_per_sec,TILE_MINSEC );
@@ -182,7 +182,7 @@ void setup_tile_timer(void)void do_timer_interrupt (struct pt_regs *regs,int fault_num) {struct pt_regs *old_regs =set_irq_regs (regs); -struct clock_event_device *evt = &__get_cpu_var (tile_timer); +struct clock_event_device *evt =this_cpu_ptr (&tile_timer);/* * Mask the timer interrupt here, since we are a oneshot timer
@@ -194,7 +194,7 @@ void do_timer_interrupt(struct pt_regs *irq_enter ();/* Track interrupt count. */ -__get_cpu_var (irq_stat).irq_timer_count++; +__this_cpu_inc (irq_stat.irq_timer_count);/* Call the generic timer handler */ evt->event_handler (evt);
@@ -235,7 +235,7 @@ cycles_t ns2cycles(unsigned long nsecs)* We do not have to disable preemption here as each core has the same * clock frequency. */ -struct clock_event_device *dev = &__raw_get_cpu_var (tile_timer); +struct clock_event_device *dev =raw_cpu_ptr (&tile_timer);return ((u64 )nsecs * dev->mult) >> dev->shift; }
arch/tile/mm/highmem.c +1 -1
@@ -103,7 +103,7 @@ static void kmap_atomic_register(structspin_lock (&_lock);/* With interrupts disabled, now fill in the per-cpu info. */ - amp = &__get_cpu_var (amps).per_type[type]; + amp =this_cpu_ptr (&s.per_type[type]); amp->page = page; amp->cpu =smp_processor_id (); amp->va = va;
arch/tile/mm/init.c +2 -2
@@ -593,14 +593,14 @@ static void __init kernel_physical_mappiinterrupt_mask_set_mask (-1ULL ); rc =flush_and_install_context (__pa (pgtables),init_pgprot ((unsigned long )pgtables), -__get_cpu_var (current_asid), +__this_cpu_read (current_asid),cpumask_bits (my_cpu_mask));interrupt_mask_restore_mask (irqmask);BUG_ON (rc !=0 );/* Copy the page table back to the normal swapper_pg_dir. */ memcpy (pgd_base, pgtables,sizeof (pgtables)); -__install_page_table (pgd_base,__get_cpu_var (current_asid), +__install_page_table (pgd_base,__this_cpu_read (current_asid), swapper_pgprot);/*
arch/tile/include/asm/irqflags.h +2 -2
@@ -140,12 +140,12 @@ extern unsigned int debug_smp_processor_/* * Read the set of maskable interrupts. -* We avoid the preemption warning here via __this_cpu_ptr since even +* We avoid the preemption warning here via raw_cpu_ptr since even * if irqs are already enabled, it's harmless to read the wrong cpu's * enabled mask. */ #define arch_local_irqs_enabled () \ - (*__this_cpu_ptr(&interrupts_enabled_mask)) + raw_cpu_read(interrupts_enabled_mask)/* Re-enable all maskable interrupts. */ #define arch_local_irq_enable () \
PATCH 27/41 irqchips: Replace __this_cpu_ptr uses
In reply to Christoph LameterThese are generally replaced with raw_cpu_ptr. However, in gic_get_percpu_base() we immediately dereference the pointer. This is equivalent to a raw_cpu_read. So use that operation there.
drivers/irqchip/irq-gic.c +5 -5
@@ -102,7 +102,7 @@ static struct gic_chip_data gic_data[MAX#ifdef CONFIG_GIC_NON_BANKED static void __iomem *gic_get_percpu_base (union gic_base *base) { -return *__this_cpu_ptr (base->percpu_base); +return raw_cpu_read (base->percpu_base); }static void __iomem *gic_get_common_base (union gic_base *base)
@@ -552,11 +552,11 @@ static void gic_cpu_save(unsigned int giif (!dist_base || !cpu_base)return ; - ptr =__this_cpu_ptr (gic_data[gic_nr].saved_ppi_enable); + ptr =raw_cpu_ptr (gic_data[gic_nr].saved_ppi_enable);for (i =0 ; i <DIV_ROUND_UP (32 ,32 ); i++) ptr[i] =readl_relaxed (dist_base +GIC_DIST_ENABLE_SET + i *4 ); - ptr =__this_cpu_ptr (gic_data[gic_nr].saved_ppi_conf); + ptr =raw_cpu_ptr (gic_data[gic_nr].saved_ppi_conf);for (i =0 ; i <DIV_ROUND_UP (32 ,16 ); i++) ptr[i] =readl_relaxed (dist_base +GIC_DIST_CONFIG + i *4 );
@@ -578,11 +578,11 @@ static void gic_cpu_restore(unsigned intif (!dist_base || !cpu_base)return ; - ptr =__this_cpu_ptr (gic_data[gic_nr].saved_ppi_enable); + ptr =raw_cpu_ptr (gic_data[gic_nr].saved_ppi_enable);for (i =0 ; i <DIV_ROUND_UP (32 ,32 ); i++)writel_relaxed (ptr[i], dist_base +GIC_DIST_ENABLE_SET + i *4 ); - ptr =__this_cpu_ptr (gic_data[gic_nr].saved_ppi_conf); + ptr =raw_cpu_ptr (gic_data[gic_nr].saved_ppi_conf);for (i =0 ; i <DIV_ROUND_UP (32 ,16 ); i++)writel_relaxed (ptr[i], dist_base +GIC_DIST_CONFIG + i *4 );
kernel/irq/chip.c +1 -1
@@ -638,7 +638,7 @@ void handle_percpu_devid_irq(unsigned in {struct irq_chip *chip =irq_desc_get_chip (desc);struct irqaction *action = desc->action; -void *dev_id =__this_cpu_ptr (action->percpu_dev_id); +void *dev_id =raw_cpu_ptr (action->percpu_dev_id);irqreturn_t res;kstat_incr_irqs_this_cpu (irq, desc);
cover percpu: Consistent per cpu operations V1
The kernel has never been audited to ensure that this_cpu operations are consistently used throughout the kernel. The code generated in many places can be improved through the use of this_cpu operations (which uses a segment register for relocation of per cpu offsets instead of performing address calculations).
The patchset also addresses various consistency issues in general with the per cpu macros.
A. The semantics of __this_cpu_ptr() differs from this_cpu_ptr only
because checks are skipped. This is typically shown through a raw_ prefix. So this patchset changes the places where __this_cpu_ptr() is used to raw_cpu_ptr().
B. There has been the long term wish by some that __this_cpu operations
would check for preemption. However, there are cases where preemption checks need to be skipped. This patchset adds raw_cpu operations that do not check for preemption and then adds preemption checks to the __this_cpu operations.
C. The use of __get_cpu_var is always a reference to a percpu variable
that can also be handled via a this_cpu operation. This patchset replaces all uses of __get_cpu_var with this_cpu operations.
D. We can then use this_cpu RMW operations in various places replacing
sequences of instructions by a single one.
E. The use of this_cpu operations throughout will allow other arches than
x86 to implement optimized references and RMV operations to work with per cpu local data.
F. The use of this_cpu operations opens up the possibility to
further optimize code that relies on synchronization through per cpu data.
The patchset works in a couple of stages:
I. Patches 1-12 are patches that simply replace uses of __get_cpu_var
with this_cpu_ptr. They do not depend on any changes to the percpu code.
II. Patch 13 adds the additional raw_cpu operations and raw_cpu_ptr().
Patch 14 converts the existing __this_cpu_xx_# primitive in the x86 code to raw_cpu_xx_#.
III. Patch 15-17 use the raw_cpu operations in places that would give
us false positives once they are enabled.
IV. Patch 18 adds preemption checks to __this_cpu operations to allow
checking if preemption is properly disabled when these functions are used.
V. Patches 19-39 are conversion patches that use this_cpu operations
in various kernel subsystems/drivers or arch code.
VI. Patches 41/42 remove no longer used functions (__this_cpu_ptr
and __get_cpu_var). These should only be applied after all the conversion patches have made it and after we have done additonal passes through the kernel to ensure that none of the uses of these functions remain.
PATCH 3/41 percpu: Replace __get_cpu_var with this_cpu_ptr
One case of using __get_cpu_var in the get_cpu_var macro for address calculation.
include/linux/percpu.h +1 -1
@@ -28,7 +28,7 @@*/ #define get_cpu_var (var) (*({ \ preempt_disable(); \ - &__get_cpu_var(var); })) + this_cpu_ptr(&var); }))/* * The weird & is necessary because sparse considers (void)(var) to be
PATCH 1/41 mm: Replace __get_cpu_var uses with this_cpu_ptr
In reply to Christoph LameterReplace places where __get_cpu_var() is used for an address calculation with this_cpu_ptr().
lib/radix-tree.c +3 -3
@@ -221,7 +221,7 @@ radix_tree_node_alloc(struct radix_tree_* succeed in getting a node here (and never reach * kmem_cache_alloc) */ - rtp = &__get_cpu_var (radix_tree_preloads); + rtp =this_cpu_ptr (&radix_tree_preloads);if (rtp->nr) { ret = rtp->nodes[rtp->nr -1 ]; rtp->nodes[rtp->nr -1 ] =NULL ;
@@ -277,14 +277,14 @@ static int __radix_tree_preload(gfp_t gfint ret = -ENOMEM ;preempt_disable (); - rtp = &__get_cpu_var (radix_tree_preloads); + rtp =this_cpu_ptr (&radix_tree_preloads);while (rtp->nr <ARRAY_SIZE (rtp->nodes)) {preempt_enable (); node =kmem_cache_alloc (radix_tree_node_cachep, gfp_mask);if (node ==NULL ) goto out;preempt_disable (); - rtp = &__get_cpu_var (radix_tree_preloads); + rtp =this_cpu_ptr (&radix_tree_preloads);if (rtp->nr <ARRAY_SIZE (rtp->nodes)) rtp->nodes[rtp->nr++] = node;else
mm/memcontrol.c +1 -1
@@ -2432,7 +2432,7 @@ static void drain_stock(struct memcg_sto*/ static void drain_local_stock (struct work_struct *dummy) { -struct memcg_stock_pcp *stock = &__get_cpu_var (memcg_stock); +struct memcg_stock_pcp *stock =this_cpu_ptr (&memcg_stock);drain_stock (stock);clear_bit (FLUSHING_CACHED_CHARGE , &stock->flags); }
mm/memory-failure.c +1 -1
@@ -1286,7 +1286,7 @@ static void memory_failure_work_func(strunsigned long proc_flags;int gotten; - mf_cpu = &__get_cpu_var (memory_failure_cpu); + mf_cpu =this_cpu_ptr (&memory_failure_cpu);for (;;) {spin_lock_irqsave (&mf_cpu->lock, proc_flags); gotten =kfifo_get (&mf_cpu->fifo, &entry);
mm/page-writeback.c +2 -2
@@ -1628,7 +1628,7 @@ void balance_dirty_pages_ratelimited(str* 1000+ tasks, all of them start dirtying pages at exactly the same * time, hence all honoured too large initial task->nr_dirtied_pause. */ - p = &__get_cpu_var (bdp_ratelimits); + p =this_cpu_ptr (&bdp_ratelimits);if (unlikely (current->nr_dirtied >= ratelimit)) *p =0 ;else if (unlikely (*p >= ratelimit_pages)) {
@@ -1640,7 +1640,7 @@ void balance_dirty_pages_ratelimited(str* short-lived tasks (eg. gcc invocations in a kernel build) escaping * the dirty throttling and livelock other long-run dirtiers. */ - p = &__get_cpu_var (dirty_throttle_leaks); + p =this_cpu_ptr (&dirty_throttle_leaks);if (*p >0 && current->nr_dirtied < ratelimit) {unsigned long nr_pages_dirtied; nr_pages_dirtied =min (*p, ratelimit - current->nr_dirtied);
mm/swap.c +1 -1
@@ -409,7 +409,7 @@ void rotate_reclaimable_page(struct pagepage_cache_get (page);local_irq_save (flags); - pvec = &__get_cpu_var (lru_rotate_pvecs); + pvec =this_cpu_ptr (&lru_rotate_pvecs);if (!pagevec_add (pvec, page))pagevec_move_tail (pvec);local_irq_restore (flags);
mm/vmalloc.c +1 -1
@@ -1488,7 +1488,7 @@ void vfree(const void *addr)if (!addr)return ;if (unlikely (in_interrupt ())) { -struct vfree_deferred *p = &__get_cpu_var (vfree_deferred); +struct vfree_deferred *p =this_cpu_ptr (&vfree_deferred);if (llist_add ((struct llist_node *)addr, &p->list))schedule_work (&p->wq); }else
mm/slub.c +3 -3
@@ -2175,7 +2175,7 @@ static inline void *new_slab_objects(str page =new_slab (s, flags, node);if (page) { - c =__this_cpu_ptr (s->cpu_slab); + c =raw_cpu_ptr (s->cpu_slab);if (c->page)flush_slab (s, c);
@@ -2395,7 +2395,7 @@ redo:* and the retrieval of the tid. */ preempt_disable (); - c =__this_cpu_ptr (s->cpu_slab); + c =this_cpu_ptr (s->cpu_slab);/* * The transaction ids are globally unique per cpu and per operation on
@@ -2650,7 +2650,7 @@ redo:* during the cmpxchg then the free will succedd. */ preempt_disable (); - c =__this_cpu_ptr (s->cpu_slab); + c =this_cpu_ptr (s->cpu_slab); tid = c->tid;preempt_enable ();
mm/vmstat.c +1 -1
@@ -502,7 +502,7 @@ static int refresh_cpu_vm_stats(void)continue ;if (__this_cpu_read (p->pcp.count)) { -drain_zone_pages (zone,__this_cpu_ptr (&p->pcp)); +drain_zone_pages (zone,this_cpu_ptr (&p->pcp)); changes++; }#endif
PATCH 4/41 kernel misc: Replace __get_cpu_var uses
In reply to Christoph LameterReplace uses of __get_cpu_var for address calculation with this_cpu_ptr.
kernel/printk/printk.c +4 -4
@@ -2447,7 +2447,7 @@ static void wake_up_klogd_work_func(struint pending =__this_cpu_xchg (printk_pending,0 );if (pending &PRINTK_PENDING_SCHED ) { -char *buf =__get_cpu_var (printk_sched_buf); +char *buf =this_cpu_ptr (printk_sched_buf);pr_warn ("[sched_delayed] %s" , buf); }
@@ -2465,7 +2465,7 @@ void wake_up_klogd(void)preempt_disable ();if (waitqueue_active (&log_wait)) {this_cpu_or (printk_pending,PRINTK_PENDING_WAKEUP ); -irq_work_queue (&__get_cpu_var (wake_up_klogd_work)); +irq_work_queue (this_cpu_ptr (&wake_up_klogd_work)); }preempt_enable (); }
@@ -2478,14 +2478,14 @@ int printk_sched(const char *fmt, ...)int r;local_irq_save (flags); - buf =__get_cpu_var (printk_sched_buf); + buf =this_cpu_ptr (printk_sched_buf);va_start (args, fmt); r =vsnprintf (buf,PRINTK_BUF_SIZE , fmt, args);va_end (args);__this_cpu_or (printk_pending,PRINTK_PENDING_SCHED ); -irq_work_queue (&__get_cpu_var (wake_up_klogd_work)); +irq_work_queue (this_cpu_ptr (&wake_up_klogd_work));local_irq_restore (flags);return r;
kernel/smp.c +3 -3
@@ -177,7 +177,7 @@ static void generic_exec_single(int cpu,*/ void generic_smp_call_function_single_interrupt (void ) { -struct call_single_queue *q = &__get_cpu_var (call_single_queue); +struct call_single_queue *q =this_cpu_ptr (&call_single_queue);LIST_HEAD (list);/*
@@ -245,7 +245,7 @@ int smp_call_function_single(int cpu, smstruct call_single_data *csd = &d;if (!wait) - csd = &__get_cpu_var (csd_data); + csd =this_cpu_ptr (&csd_data);csd_lock (csd);
@@ -393,7 +393,7 @@ void smp_call_function_many(const structreturn ; } - cfd = &__get_cpu_var (cfd_data); + cfd =this_cpu_ptr (&cfd_data);cpumask_and (cfd->cpumask, mask, cpu_online_mask);cpumask_clear_cpu (this_cpu, cfd->cpumask);
PATCH 6/41 drivers/cpuidle: Replace __get_cpu_var uses for address calculation
In reply to Christoph LameterAll of these are for address calculation. Replace with this_cpu_ptr().
drivers/cpuidle/governors/ladder.c +2 -2
@@ -66,7 +66,7 @@ static inline void ladder_do_selection(sstatic int ladder_select_state (struct cpuidle_driver *drv,struct cpuidle_device *dev) { -struct ladder_device *ldev = &__get_cpu_var (ladder_devices); +struct ladder_device *ldev =this_cpu_ptr (&ladder_devices);struct ladder_device_state *last_state;int last_residency, last_idx = ldev->last_state_idx;int latency_req =pm_qos_request (PM_QOS_CPU_DMA_LATENCY );
@@ -170,7 +170,7 @@ static int ladder_enable_device(struct c*/ static void ladder_reflect (struct cpuidle_device *dev,int index) { -struct ladder_device *ldev = &__get_cpu_var (ladder_devices); +struct ladder_device *ldev =this_cpu_ptr (&ladder_devices);if (index >0 ) ldev->last_state_idx = index; }
drivers/cpuidle/governors/menu.c +3 -3
@@ -286,7 +286,7 @@ again:*/ static int menu_select (struct cpuidle_driver *drv,struct cpuidle_device *dev) { -struct menu_device *data = &__get_cpu_var (menu_devices); +struct menu_device *data =this_cpu_ptr (&menu_devices);int latency_req =pm_qos_request (PM_QOS_CPU_DMA_LATENCY );int i;int multiplier;
@@ -375,7 +375,7 @@ static int menu_select(struct cpuidle_dr*/ static void menu_reflect (struct cpuidle_device *dev,int index) { -struct menu_device *data = &__get_cpu_var (menu_devices); +struct menu_device *data =this_cpu_ptr (&menu_devices); data->last_state_idx = index;if (index >=0 ) data->needs_update =1 ;
@@ -388,7 +388,7 @@ static void menu_reflect(struct cpuidle_*/ static void menu_update (struct cpuidle_driver *drv,struct cpuidle_device *dev) { -struct menu_device *data = &__get_cpu_var (menu_devices); +struct menu_device *data =this_cpu_ptr (&menu_devices);int last_idx = data->last_state_idx;unsigned int last_idle_us =cpuidle_get_last_residency (dev);struct cpuidle_state *target = &drv->states[last_idx];
PATCH 12/41 metag: Replace __get_cpu_var uses for address calculation
In reply to Christoph LameterReplace __get_cpu_var uses for address calculation with this_cpu_ptr().
arch/metag/kernel/perf/perf_event.c +7 -7
@@ -258,7 +258,7 @@ int metag_pmu_event_set_period(struct pestatic void metag_pmu_start (struct perf_event *event,int flags) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);struct hw_perf_event *hwc = &event->hw;int idx = hwc->idx;
@@ -306,7 +306,7 @@ static void metag_pmu_stop(struct perf_estatic int metag_pmu_add (struct perf_event *event,int flags) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);struct hw_perf_event *hwc = &event->hw;int idx =0 , ret =0 ;
@@ -348,7 +348,7 @@ out:static void metag_pmu_del (struct perf_event *event,int flags) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);struct hw_perf_event *hwc = &event->hw;int idx = hwc->idx;
@@ -607,7 +607,7 @@ static int _hw_perf_event_init(struct pestatic void metag_pmu_enable_counter (struct hw_perf_event *event,int idx) { -struct cpu_hw_events *events = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *events =this_cpu_ptr (&cpu_hw_events);unsigned int config = event->config;unsigned int tmp = config &0xf0 ;unsigned long flags;
@@ -680,7 +680,7 @@ unlock:static void metag_pmu_disable_counter (struct hw_perf_event *event,int idx) { -struct cpu_hw_events *events = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *events =this_cpu_ptr (&cpu_hw_events);unsigned int tmp =0 ;unsigned long flags;
@@ -728,7 +728,7 @@ out:static void metag_pmu_write_counter (int idx,u32 val) { -struct cpu_hw_events *events = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *events =this_cpu_ptr (&cpu_hw_events);u32 tmp =0 ;unsigned long flags;
@@ -761,7 +761,7 @@ static int metag_pmu_event_map(int idx)static irqreturn_t metag_pmu_counter_overflow (int irq,void *dev) {int idx = (int )dev; -struct cpu_hw_events *cpuhw = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuhw =this_cpu_ptr (&cpu_hw_events);struct perf_event *event = cpuhw->events[idx];struct hw_perf_event *hwc = &event->hw;struct pt_regs *regs =get_irq_regs ();
PATCH 41/41 percpu: Remove __this_cpu_ptr
In reply to Christoph LameterThe __this_cpu_ptr macro is no longer in use so drop it.
include/linux/percpu.h +0 -3
@@ -139,9 +139,6 @@ extern int __init pcpu_page_first_chunk( pcpu_fc_populate_pte_fn_t populate_pte_fn);#endif -/* Keep until we have removed all uses of __this_cpu_ptr */ -#define __this_cpu_ptr raw_cpu_ptr -/* * Use this to get to a cpu's version of the per-cpu object * dynamically allocated. Non-atomic access to the current CPU's
PATCH 38/41 alpha: Replace __get_cpu_var
In reply to Christoph Lameter__get_cpu_var() is used for multiple purposes in the kernel source. One of them is address calculation via the form &__get_cpu_var(x). This calculates the address for the instance of the percpu variable of the current processor based on an offset.
Other use cases are for storing and retrieving data from the current processors percpu area. __get_cpu_var() can be used as an lvalue when writing data or on the right side of an assignment.
__get_cpu_var() is defined as :
#define __get_cpu_var(var) (*this_cpu_ptr(&(var)))
__get_cpu_var() always only does an address determination. However, store and retrieve operations could use a segment prefix (or global register on other platforms) to avoid the address calculation.
this_cpu_write() and this_cpu_read() can directly take an offset into a percpu area and use optimized assembly code to read and write per cpu variables.
This patch converts __get_cpu_var into either an explicit address calculation using this_cpu_ptr() or into a use of this_cpu operations that use the offset. Thereby address calculations are avoided and less registers are used when code is generated.
At the end of the patch set all uses of __get_cpu_var have been removed so the macro is removed too.
The patch set includes passes over all arches as well. Once these operations are used throughout then specialized macros can be defined in non -x86 arches as well in order to optimize per cpu access by f.e. using a global register that may be set to the per cpu base.
Transformations done to __get_cpu_var()
1. Determine the address of the percpu instance of the current processor.
DEFINE_PER_CPU(int, y);
int *x = &__get_cpu_var(y);
Converts to
int *x = this_cpu_ptr(&y);2. Same as #1 but this time an array structure is involved.
DEFINE_PER_CPU(int, y[20]);
int *x = __get_cpu_var(y);
Converts to
int *x = this_cpu_ptr(y);3. Retrieve the content of the current processors instance of a per cpu variable.
DEFINE_PER_CPU(int, y); int x = __get_cpu_var(y)
Converts to
int x = __this_cpu_read(y);
4. Retrieve the content of a percpu struct
DEFINE_PER_CPU(struct mystruct, y); struct mystruct x = __get_cpu_var(y);
Converts to
memcpy(&x, this_cpu_ptr(&y), sizeof(x));
5. Assignment to a per cpu variable
DEFINE_PER_CPU(int, y) __get_cpu_var(y) = x;
Converts to
this_cpu_write(y, x);
6. Increment/Decrement etc of a per cpu variable
DEFINE_PER_CPU(int, y); __get_cpu_var(y)++
Converts to
this_cpu_inc(y)
arch/alpha/kernel/perf_event.c +7 -7
@@ -431,7 +431,7 @@ static void maybe_change_configuration(s*/ static int alpha_pmu_add (struct perf_event *event,int flags) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);struct hw_perf_event *hwc = &event->hw;int n0;int ret;
@@ -483,7 +483,7 @@ static int alpha_pmu_add(struct perf_eve*/ static void alpha_pmu_del (struct perf_event *event,int flags) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);struct hw_perf_event *hwc = &event->hw;unsigned long irq_flags;int j;
@@ -531,7 +531,7 @@ static void alpha_pmu_read(struct perf_estatic void alpha_pmu_stop (struct perf_event *event,int flags) {struct hw_perf_event *hwc = &event->hw; -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);if (!(hwc->state &PERF_HES_STOPPED )) { cpuc->idx_mask &= ~(1UL <<hwc->idx);
@@ -551,7 +551,7 @@ static void alpha_pmu_stop(struct perf_estatic void alpha_pmu_start (struct perf_event *event,int flags) {struct hw_perf_event *hwc = &event->hw; -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);if (WARN_ON_ONCE (!(hwc->state &PERF_HES_STOPPED )))return ;
@@ -724,7 +724,7 @@ static int alpha_pmu_event_init(struct p*/ static void alpha_pmu_enable (struct pmu *pmu) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);if (cpuc->enabled)return ;
@@ -750,7 +750,7 @@ static void alpha_pmu_enable(struct pmustatic void alpha_pmu_disable (struct pmu *pmu) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);if (!cpuc->enabled)return ;
@@ -815,7 +815,7 @@ static void alpha_perf_event_irq_handlerint idx, j;__this_cpu_inc (irq_pmi_count); - cpuc = &__get_cpu_var (cpu_hw_events); + cpuc =this_cpu_ptr (&cpu_hw_events);/* Completely counting through the PMC's period to trigger a new PMC * overflow interrupt while in this interrupt routine is utterly
arch/alpha/kernel/time.c +3 -3
@@ -56,9 +56,9 @@ unsigned long est_cycle_freq;DEFINE_PER_CPU (u8, irq_work_pending); -#define set_irq_work_pending_flag () __get_cpu_var(irq_work_pending) = 1 -#define test_irq_work_pending () __get_cpu_var(irq_work_pending) -#define clear_irq_work_pending () __get_cpu_var(irq_work_pending) = 0 +#define set_irq_work_pending_flag () __this_cpu_write(irq_work_pending, 1) +#define test_irq_work_pending () __this_cpu_read(irq_work_pending) +#define clear_irq_work_pending () __this_cpu_writeirq_work_pending, 0)void arch_irq_work_raise (void ) {
PATCH 39/41 sh: Replace __get_cpu_var uses
In reply to Christoph Lameter__get_cpu_var() is used for multiple purposes in the kernel source. One of them is address calculation via the form &__get_cpu_var(x). This calculates the address for the instance of the percpu variable of the current processor based on an offset.
Other use cases are for storing and retrieving data from the current processors percpu area. __get_cpu_var() can be used as an lvalue when writing data or on the right side of an assignment.
__get_cpu_var() is defined as :
#define __get_cpu_var(var) (*this_cpu_ptr(&(var)))
__get_cpu_var() always only does an address determination. However, store and retrieve operations could use a segment prefix (or global register on other platforms) to avoid the address calculation.
this_cpu_write() and this_cpu_read() can directly take an offset into a percpu area and use optimized assembly code to read and write per cpu variables.
This patch converts __get_cpu_var into either an explicit address calculation using this_cpu_ptr() or into a use of this_cpu operations that use the offset. Thereby address calculations are avoided and less registers are used when code is generated.
At the end of the patch set all uses of __get_cpu_var have been removed so the macro is removed too.
The patch set includes passes over all arches as well. Once these operations are used throughout then specialized macros can be defined in non -x86 arches as well in order to optimize per cpu access by f.e. using a global register that may be set to the per cpu base.
Transformations done to __get_cpu_var()
1. Determine the address of the percpu instance of the current processor.
DEFINE_PER_CPU(int, y);
int *x = &__get_cpu_var(y);
Converts to
int *x = this_cpu_ptr(&y);2. Same as #1 but this time an array structure is involved.
DEFINE_PER_CPU(int, y[20]);
int *x = __get_cpu_var(y);
Converts to
int *x = this_cpu_ptr(y);3. Retrieve the content of the current processors instance of a per cpu variable.
DEFINE_PER_CPU(int, y); int x = __get_cpu_var(y)
Converts to
int x = __this_cpu_read(y);
4. Retrieve the content of a percpu struct
DEFINE_PER_CPU(struct mystruct, y); struct mystruct x = __get_cpu_var(y);
Converts to
memcpy(&x, this_cpu_ptr(&y), sizeof(x));
5. Assignment to a per cpu variable
DEFINE_PER_CPU(int, y) __get_cpu_var(y) = x;
Converts to
this_cpu_write(y, x);
6. Increment/Decrement etc of a per cpu variable
DEFINE_PER_CPU(int, y); __get_cpu_var(y)++
Converts to
this_cpu_inc(y)
arch/sh/kernel/hw_breakpoint.c +2 -2
@@ -52,7 +52,7 @@ int arch_install_hw_breakpoint(struct peint i;for (i =0 ; i < sh_ubc->num_events; i++) { -struct perf_event **slot = &__get_cpu_var (bp_per_reg[i]); +struct perf_event **slot =this_cpu_ptr (&bp_per_reg[i]);if (!*slot) { *slot = bp;
@@ -84,7 +84,7 @@ void arch_uninstall_hw_breakpoint(structint i;for (i =0 ; i < sh_ubc->num_events; i++) { -struct perf_event **slot = &__get_cpu_var (bp_per_reg[i]); +struct perf_event **slot =this_cpu_ptr (&bp_per_reg[i]);if (*slot == bp) { *slot =NULL ;
arch/sh/kernel/kprobes.c +15 -15
@@ -102,7 +102,7 @@ int __kprobes kprobe_handle_illslot(unsivoid __kprobes arch_remove_kprobe (struct kprobe *p) { -struct kprobe *saved = &__get_cpu_var (saved_next_opcode); +struct kprobe *saved =this_cpu_ptr (&saved_next_opcode);if (saved->addr) {arch_disarm_kprobe (p);
@@ -111,7 +111,7 @@ void __kprobes arch_remove_kprobe(struct saved->addr =NULL ; saved->opcode =0 ; - saved = &__get_cpu_var (saved_next_opcode2); + saved =this_cpu_ptr (&saved_next_opcode2);if (saved->addr) {arch_disarm_kprobe (saved);
@@ -129,14 +129,14 @@ static void __kprobes save_previous_kprostatic void __kprobes restore_previous_kprobe (struct kprobe_ctlblk *kcb) { -__get_cpu_var (current_kprobe) = kcb->prev_kprobe.kp; +__this_cpu_write (current_kprobe, kcb->prev_kprobe.kp); kcb->kprobe_status = kcb->prev_kprobe.status; }static void __kprobes set_current_kprobe (struct kprobe *p,struct pt_regs *regs,struct kprobe_ctlblk *kcb) { -__get_cpu_var (current_kprobe) = p; +__this_cpu_write (current_kprobe, p); }/*
@@ -146,15 +146,15 @@ static void __kprobes set_current_kprobe*/ static void __kprobes prepare_singlestep (struct kprobe *p,struct pt_regs *regs) { -__get_cpu_var (saved_current_opcode).addr = (kprobe_opcode_t *)regs->pc; +__this_cpu_write (saved_current_opcode.addr, (kprobe_opcode_t *)regs->pc);if (p !=NULL ) {struct kprobe *op1, *op2;arch_disarm_kprobe (p); - op1 = &__get_cpu_var (saved_next_opcode); - op2 = &__get_cpu_var (saved_next_opcode2); + op1 =this_cpu_ptr (&saved_next_opcode); + op2 =this_cpu_ptr (&saved_next_opcode2);if (OPCODE_JSR (p->opcode) ||OPCODE_JMP (p->opcode)) {unsigned int reg_nr = ((p->opcode >>8 ) &0x000F );
@@ -249,7 +249,7 @@ static int __kprobes kprobe_handler(stru kcb->kprobe_status =KPROBE_REENTER ;return 1 ; }else { - p =__get_cpu_var (current_kprobe); + p =__this_cpu_read (current_kprobe);if (p->break_handler && p->break_handler (p, regs)) { goto ss_probe; }
@@ -336,9 +336,9 @@ int __kprobes trampoline_probe_handler(scontinue ;if (ri->rp && ri->rp->handler) { -__get_cpu_var (current_kprobe) = &ri->rp->kp; +__this_cpu_write (current_kprobe, &ri->rp->kp); ri->rp->handler (ri, regs); -__get_cpu_var (current_kprobe) =NULL ; +__this_cpu_write (current_kprobe,NULL ); } orig_ret_address = (unsigned long )ri->ret_addr;
@@ -383,19 +383,19 @@ static int __kprobes post_kprobe_handler cur->post_handler (cur, regs,0 ); } - p = &__get_cpu_var (saved_next_opcode); + p =this_cpu_ptr (&saved_next_opcode);if (p->addr) {arch_disarm_kprobe (p); p->addr =NULL ; p->opcode =0 ; - addr =__get_cpu_var (saved_current_opcode).addr; -__get_cpu_var (saved_current_opcode).addr =NULL ; + addr =__this_cpu_read (saved_current_opcode).addr; +__this_cpu_write (saved_current_opcode.addr,NULL ); p =get_kprobe (addr);arch_arm_kprobe (p); - p = &__get_cpu_var (saved_next_opcode2); + p =this_cpu_ptr (&saved_next_opcode2);if (p->addr) {arch_disarm_kprobe (p); p->addr =NULL ;
@@ -511,7 +511,7 @@ int __kprobes kprobe_exceptions_notify(sif (kprobe_handler (args->regs)) { ret =NOTIFY_STOP ; }else { - p =__get_cpu_var (current_kprobe); + p =__this_cpu_read (current_kprobe);if (p->break_handler && p->break_handler (p, args->regs)) ret =NOTIFY_STOP ;
arch/sh/kernel/localtimer.c +1 -1
@@ -32,7 +32,7 @@ static DEFINE_PER_CPU(struct clock_event*/ void local_timer_interrupt (void ) { -struct clock_event_device *clk = &__get_cpu_var (local_clockevent); +struct clock_event_device *clk =this_cpu_ptr (&local_clockevent);irq_enter (); clk->event_handler (clk);
arch/sh/kernel/perf_event.c +4 -4
@@ -227,7 +227,7 @@ again:static void sh_pmu_stop (struct perf_event *event,int flags) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);struct hw_perf_event *hwc = &event->hw;int idx = hwc->idx;
@@ -245,7 +245,7 @@ static void sh_pmu_stop(struct perf_evenstatic void sh_pmu_start (struct perf_event *event,int flags) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);struct hw_perf_event *hwc = &event->hw;int idx = hwc->idx;
@@ -262,7 +262,7 @@ static void sh_pmu_start(struct perf_evestatic void sh_pmu_del (struct perf_event *event,int flags) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);sh_pmu_stop (event,PERF_EF_UPDATE );__clear_bit (event->hw.idx, cpuc->used_mask);
@@ -272,7 +272,7 @@ static void sh_pmu_del(struct perf_eventstatic int sh_pmu_add (struct perf_event *event,int flags) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);struct hw_perf_event *hwc = &event->hw;int idx = hwc->idx;int ret = -EAGAIN ;
arch/sh/kernel/smp.c +1 -1
@@ -111,7 +111,7 @@ void play_dead_common(void)irq_ctx_exit (raw_smp_processor_id ());mb (); -__get_cpu_var (cpu_state) =CPU_DEAD ; +__this_cpu_write (cpu_state,CPU_DEAD );local_irq_disable (); }
PATCH 40/41 Remove __get_cpu_var and __raw_get_cpu_var macros [only in 3.15]
In reply to Christoph LameterNo user is left in the kernel source tree. Therefore we can drop the definitions.
[Patch should not be merged until all the replacement patches have been merged. Probably this means hold until the 3.15 merge window]
include/asm-generic/percpu.h +0 -5
@@ -65,9 +65,6 @@ extern unsigned long __per_cpu_offset[NR#define this_cpu_ptr (ptr) raw_cpu_ptr(ptr)#endif -#define __get_cpu_var (var) (*this_cpu_ptr(&(var))) -#define __raw_get_cpu_var (var) (*raw_cpu_ptr(&(var))) -#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA extern void setup_per_cpu_areas (void );#endif
@@ -80,8 +77,6 @@ extern void setup_per_cpu_areas(void); })#define per_cpu (var, cpu) (*((void)(cpu), VERIFY_PERCPU_PTR(&(var)))) -#define __get_cpu_var (var) (*VERIFY_PERCPU_PTR(&(var))) -#define __raw_get_cpu_var (var) (*VERIFY_PERCPU_PTR(&(var)))#define this_cpu_ptr (ptr) per_cpu_ptr(ptr, 0)#define raw_cpu_ptr (ptr) this_cpu_ptr(ptr)
PATCH 37/41 avr32: Replace __get_cpu_var with __this_cpu_write
In reply to Christoph LameterReplace the single use of __get_cpu_var in avr32 with __this_cpu_write.
arch/avr32/kernel/kprobes.c +1 -1
@@ -104,7 +104,7 @@ static void __kprobes resume_execution(sstatic void __kprobes set_current_kprobe (struct kprobe *p) { -__get_cpu_var (current_kprobe) = p; +__this_cpu_write (current_kprobe, p); }static int __kprobes kprobe_handler (struct pt_regs *regs)
PATCH 34/41 sparc: Replace __get_cpu_var uses
In reply to Christoph Lameter__get_cpu_var() is used for multiple purposes in the kernel source. One of them is address calculation via the form &__get_cpu_var(x). This calculates the address for the instance of the percpu variable of the current processor based on an offset.
Other use cases are for storing and retrieving data from the current processors percpu area. __get_cpu_var() can be used as an lvalue when writing data or on the right side of an assignment.
__get_cpu_var() is defined as :
#define __get_cpu_var(var) (*this_cpu_ptr(&(var)))
__get_cpu_var() always only does an address determination. However, store and retrieve operations could use a segment prefix (or global register on other platforms) to avoid the address calculation.
this_cpu_write() and this_cpu_read() can directly take an offset into a percpu area and use optimized assembly code to read and write per cpu variables.
This patch converts __get_cpu_var into either an explicit address calculation using this_cpu_ptr() or into a use of this_cpu operations that use the offset. Thereby address calculations are avoided and less registers are used when code is generated.
At the end of the patch set all uses of __get_cpu_var have been removed so the macro is removed too.
The patch set includes passes over all arches as well. Once these operations are used throughout then specialized macros can be defined in non -x86 arches as well in order to optimize per cpu access by f.e. using a global register that may be set to the per cpu base.
Transformations done to __get_cpu_var()
1. Determine the address of the percpu instance of the current processor.
DEFINE_PER_CPU(int, y);
int *x = &__get_cpu_var(y);
Converts to
int *x = this_cpu_ptr(&y);2. Same as #1 but this time an array structure is involved.
DEFINE_PER_CPU(int, y[20]);
int *x = __get_cpu_var(y);
Converts to
int *x = this_cpu_ptr(y);3. Retrieve the content of the current processors instance of a per cpu variable.
DEFINE_PER_CPU(int, y); int x = __get_cpu_var(y)
Converts to
int x = __this_cpu_read(y);
4. Retrieve the content of a percpu struct
DEFINE_PER_CPU(struct mystruct, y); struct mystruct x = __get_cpu_var(y);
Converts to
memcpy(&x, this_cpu_ptr(&y), sizeof(x));
5. Assignment to a per cpu variable
DEFINE_PER_CPU(int, y) __get_cpu_var(y) = x;
Converts to
this_cpu_write(y, x);
6. Increment/Decrement etc of a per cpu variable
DEFINE_PER_CPU(int, y); __get_cpu_var(y)++
Converts to
this_cpu_inc(y)
arch/sparc/include/asm/cpudata_32.h +1 -1
@@ -26,6 +26,6 @@ typedef struct {DECLARE_PER_CPU (cpuinfo_sparc, __cpu_data);#define cpu_data (__cpu) per_cpu(__cpu_data, (__cpu)) -#define local_cpu_data () __get_cpu_var(__cpu_data) +#define local_cpu_data () __this_cpu_read(__cpu_data)#endif /* _SPARC_CPUDATA_H */
arch/sparc/include/asm/cpudata_64.h +1 -1
@@ -33,7 +33,7 @@ typedef struct {DECLARE_PER_CPU (cpuinfo_sparc, __cpu_data);#define cpu_data (__cpu) per_cpu(__cpu_data, (__cpu)) -#define local_cpu_data () __get_cpu_var(__cpu_data) +#define local_cpu_data () __this_cpu_read(__cpu_data)extern const struct seq_operations cpuinfo_op;
arch/sparc/kernel/kprobes.c +3 -3
@@ -83,7 +83,7 @@ static void __kprobes save_previous_kprostatic void __kprobes restore_previous_kprobe (struct kprobe_ctlblk *kcb) { -__get_cpu_var (current_kprobe) = kcb->prev_kprobe.kp; +__this_cpu_write (current_kprobe, kcb->prev_kprobe.kp); kcb->kprobe_status = kcb->prev_kprobe.status; kcb->kprobe_orig_tnpc = kcb->prev_kprobe.orig_tnpc; kcb->kprobe_orig_tstate_pil = kcb->prev_kprobe.orig_tstate_pil;
@@ -92,7 +92,7 @@ static void __kprobes restore_previous_kstatic void __kprobes set_current_kprobe (struct kprobe *p,struct pt_regs *regs,struct kprobe_ctlblk *kcb) { -__get_cpu_var (current_kprobe) = p; +__this_cpu_write (current_kprobe, p); kcb->kprobe_orig_tnpc = regs->tnpc; kcb->kprobe_orig_tstate_pil = (regs->tstate &TSTATE_PIL ); }
@@ -155,7 +155,7 @@ static int __kprobes kprobe_handler(stru ret =1 ; goto no_kprobe; } - p =__get_cpu_var (current_kprobe); + p =__this_cpu_read (current_kprobe);if (p->break_handler && p->break_handler (p, regs)) goto ss_probe; }
arch/sparc/kernel/leon_smp.c +1 -1
@@ -354,7 +354,7 @@ static void leon_ipi_resched(int cpu)void leonsmp_ipi_interrupt (void ) { -struct leon_ipi_work *work = &__get_cpu_var (leon_ipi_work); +struct leon_ipi_work *work =this_cpu_ptr (&leon_ipi_work);if (work->single) { work->single =0 ;
arch/sparc/kernel/nmi.c +8 -8
@@ -111,20 +111,20 @@ notrace __kprobes void perfctr_irq(int i pcr_ops->write_pcr (0 , pcr_ops->pcr_nmi_disable); sum =local_cpu_data ().irq0_irqs; -if (__get_cpu_var (nmi_touch)) { -__get_cpu_var (nmi_touch) =0 ; +if (__this_cpu_read (nmi_touch)) { +__this_cpu_write (nmi_touch,0 ); touched =1 ; } -if (!touched &&__get_cpu_var (last_irq_sum) == sum) { +if (!touched &&__this_cpu_read (last_irq_sum) == sum) {__this_cpu_inc (alert_counter);if (__this_cpu_read (alert_counter) ==30 * nmi_hz)die_nmi ("BUG: NMI Watchdog detected LOCKUP" , regs, panic_on_timeout); }else { -__get_cpu_var (last_irq_sum) = sum; +__this_cpu_write (last_irq_sum, sum);__this_cpu_write (alert_counter,0 ); } -if (__get_cpu_var (wd_enabled)) { +if (__this_cpu_read (wd_enabled)) { pcr_ops->write_pic (0 , pcr_ops->nmi_picl_value (nmi_hz)); pcr_ops->write_pcr (0 , pcr_ops->pcr_nmi_enable); }
@@ -166,7 +166,7 @@ static void report_broken_nmi(int cpu, ivoid stop_nmi_watchdog (void *unused) { pcr_ops->write_pcr (0 , pcr_ops->pcr_nmi_disable); -__get_cpu_var (wd_enabled) =0 ; +__this_cpu_write (wd_enabled,0 );atomic_dec (&nmi_active); }
@@ -219,7 +219,7 @@ error:void start_nmi_watchdog (void *unused) { -__get_cpu_var (wd_enabled) =1 ; +__this_cpu_write (wd_enabled,1 );atomic_inc (&nmi_active); pcr_ops->write_pcr (0 , pcr_ops->pcr_nmi_disable);
@@ -230,7 +230,7 @@ void start_nmi_watchdog(void *unused)static void nmi_adjust_hz_one (void *unused) { -if (!__get_cpu_var (wd_enabled)) +if (!__this_cpu_read (wd_enabled))return ; pcr_ops->write_pcr (0 , pcr_ops->pcr_nmi_disable);
arch/sparc/kernel/pci_sun4v.c +4 -4
@@ -48,7 +48,7 @@ static int iommu_batch_initialized;/* Interrupts must be disabled. */ static inline void iommu_batch_start (struct device *dev,unsigned long prot,unsigned long entry) { -struct iommu_batch *p = &__get_cpu_var (iommu_batch); +struct iommu_batch *p =this_cpu_ptr (&iommu_batch); p->dev = dev; p->prot = prot;
@@ -94,7 +94,7 @@ static long iommu_batch_flush(struct iomstatic inline void iommu_batch_new_entry (unsigned long entry) { -struct iommu_batch *p = &__get_cpu_var (iommu_batch); +struct iommu_batch *p =this_cpu_ptr (&iommu_batch);if (p->entry + p->npages == entry)return ;
@@ -106,7 +106,7 @@ static inline void iommu_batch_new_entry/* Interrupts must be disabled. */ static inline long iommu_batch_add (u64 phys_page) { -struct iommu_batch *p = &__get_cpu_var (iommu_batch); +struct iommu_batch *p =this_cpu_ptr (&iommu_batch);BUG_ON (p->npages >=PGLIST_NENTS );
@@ -120,7 +120,7 @@ static inline long iommu_batch_add(u64 p/* Interrupts must be disabled. */ static inline long iommu_batch_end (void ) { -struct iommu_batch *p = &__get_cpu_var (iommu_batch); +struct iommu_batch *p =this_cpu_ptr (&iommu_batch);BUG_ON (p->npages >=PGLIST_NENTS );
arch/sparc/kernel/perf_event.c +13 -13
@@ -1013,7 +1013,7 @@ static void update_pcrs_for_enable(strucstatic void sparc_pmu_enable (struct pmu *pmu) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);int i;if (cpuc->enabled)
@@ -1031,7 +1031,7 @@ static void sparc_pmu_enable(struct pmustatic void sparc_pmu_disable (struct pmu *pmu) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);int i;if (!cpuc->enabled)
@@ -1065,7 +1065,7 @@ static int active_event_index(struct cpustatic void sparc_pmu_start (struct perf_event *event,int flags) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);int idx =active_event_index (cpuc, event);if (flags &PERF_EF_RELOAD ) {
@@ -1080,7 +1080,7 @@ static void sparc_pmu_start(struct perf_static void sparc_pmu_stop (struct perf_event *event,int flags) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);int idx =active_event_index (cpuc, event);if (!(event->hw.state &PERF_HES_STOPPED )) {
@@ -1096,7 +1096,7 @@ static void sparc_pmu_stop(struct perf_estatic void sparc_pmu_del (struct perf_event *event,int _flags) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);unsigned long flags;int i;
@@ -1133,7 +1133,7 @@ static void sparc_pmu_del(struct perf_evstatic void sparc_pmu_read (struct perf_event *event) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);int idx =active_event_index (cpuc, event);struct hw_perf_event *hwc = &event->hw;
@@ -1145,7 +1145,7 @@ static DEFINE_MUTEX(pmc_grab_mutex);static void perf_stop_nmi_watchdog (void *unused) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);int i;stop_nmi_watchdog (NULL );
@@ -1356,7 +1356,7 @@ static int collect_events(struct perf_evstatic int sparc_pmu_add (struct perf_event *event,int ef_flags) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);int n0, ret = -EAGAIN ;unsigned long flags;
@@ -1498,7 +1498,7 @@ static int sparc_pmu_event_init(struct p*/ static void sparc_pmu_start_txn (struct pmu *pmu) { -struct cpu_hw_events *cpuhw = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuhw =this_cpu_ptr (&cpu_hw_events);perf_pmu_disable (pmu); cpuhw->group_flag |=PERF_EVENT_TXN ;
@@ -1511,7 +1511,7 @@ static void sparc_pmu_start_txn(struct p*/ static void sparc_pmu_cancel_txn (struct pmu *pmu) { -struct cpu_hw_events *cpuhw = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuhw =this_cpu_ptr (&cpu_hw_events); cpuhw->group_flag &= ~PERF_EVENT_TXN ;perf_pmu_enable (pmu);
@@ -1524,13 +1524,13 @@ static void sparc_pmu_cancel_txn(struct*/ static int sparc_pmu_commit_txn (struct pmu *pmu) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);int n;if (!sparc_pmu)return -EINVAL ; - cpuc = &__get_cpu_var (cpu_hw_events); + cpuc =this_cpu_ptr (&cpu_hw_events); n = cpuc->n_events;if (check_excludes (cpuc->event,0 , n))return -EINVAL ;
@@ -1601,7 +1601,7 @@ static int __kprobes perf_event_nmi_hand regs = args->regs; - cpuc = &__get_cpu_var (cpu_hw_events); + cpuc =this_cpu_ptr (&cpu_hw_events);/* If the PMU has the TOE IRQ enable bits, we need to do a * dummy write to the %pcr to clear the overflow bits and thus
arch/sparc/kernel/sun4d_smp.c +1 -1
@@ -204,7 +204,7 @@ static void __init smp4d_ipi_init(void)void sun4d_ipi_interrupt (void ) { -struct sun4d_ipi_work *work = &__get_cpu_var (sun4d_ipi_work); +struct sun4d_ipi_work *work =this_cpu_ptr (&sun4d_ipi_work);if (work->single) { work->single =0 ;
arch/sparc/kernel/time_64.c +1 -1
@@ -766,7 +766,7 @@ void setup_sparc64_timer(void) : /* no outputs */ : "r" (pstate)); - sevt = &__get_cpu_var (sparc64_events); + sevt =this_cpu_ptr (&sparc64_events);memcpy (sevt, &sparc64_clockevent,sizeof (*sevt)); sevt->cpumask =cpumask_of (smp_processor_id ());
arch/sparc/mm/tlb.c +2 -2
@@ -53,14 +53,14 @@ out:void arch_enter_lazy_mmu_mode (void ) { -struct tlb_batch *tb = &__get_cpu_var (tlb_batch); +struct tlb_batch *tb =this_cpu_ptr (&tlb_batch); tb->active =1 ; }void arch_leave_lazy_mmu_mode (void ) { -struct tlb_batch *tb = &__get_cpu_var (tlb_batch); +struct tlb_batch *tb =this_cpu_ptr (&tlb_batch);if (tb->tlb_nr)flush_tlb_pending ();
PATCH 33/41 powerpc: Replace __get_cpu_var uses
In reply to Christoph Lameter__get_cpu_var() is used for multiple purposes in the kernel source. One of them is address calculation via the form &__get_cpu_var(x). This calculates the address for the instance of the percpu variable of the current processor based on an offset.
Other use cases are for storing and retrieving data from the current processors percpu area. __get_cpu_var() can be used as an lvalue when writing data or on the right side of an assignment.
__get_cpu_var() is defined as :
#define __get_cpu_var(var) (*this_cpu_ptr(&(var)))
__get_cpu_var() always only does an address determination. However, store and retrieve operations could use a segment prefix (or global register on other platforms) to avoid the address calculation.
this_cpu_write() and this_cpu_read() can directly take an offset into a percpu area and use optimized assembly code to read and write per cpu variables.
This patch converts __get_cpu_var into either an explicit address calculation using this_cpu_ptr() or into a use of this_cpu operations that use the offset. Thereby address calculations are avoided and less registers are used when code is generated.
At the end of the patch set all uses of __get_cpu_var have been removed so the macro is removed too.
The patch set includes passes over all arches as well. Once these operations are used throughout then specialized macros can be defined in non -x86 arches as well in order to optimize per cpu access by f.e. using a global register that may be set to the per cpu base.
Transformations done to __get_cpu_var()
1. Determine the address of the percpu instance of the current processor.
DEFINE_PER_CPU(int, y);
int *x = &__get_cpu_var(y);
Converts to
int *x = this_cpu_ptr(&y);2. Same as #1 but this time an array structure is involved.
DEFINE_PER_CPU(int, y[20]);
int *x = __get_cpu_var(y);
Converts to
int *x = this_cpu_ptr(y);3. Retrieve the content of the current processors instance of a per cpu variable.
DEFINE_PER_CPU(int, y); int x = __get_cpu_var(y)
Converts to
int x = __this_cpu_read(y);
4. Retrieve the content of a percpu struct
DEFINE_PER_CPU(struct mystruct, y); struct mystruct x = __get_cpu_var(y);
Converts to
memcpy(&x, this_cpu_ptr(&y), sizeof(x));
5. Assignment to a per cpu variable
DEFINE_PER_CPU(int, y) __get_cpu_var(y) = x;
Converts to
this_cpu_write(y, x);
6. Increment/Decrement etc of a per cpu variable
DEFINE_PER_CPU(int, y); __get_cpu_var(y)++
Converts to
this_cpu_inc(y)
arch/powerpc/include/asm/cputime.h +3 -3
@@ -56,10 +56,10 @@ static inline unsigned long cputime_to_jstatic inline cputime_t cputime_to_scaled (const cputime_t ct) {if (cpu_has_feature (CPU_FTR_SPURR ) && -__get_cpu_var (cputime_last_delta)) +__this_cpu_read (cputime_last_delta))return (__force u64 ) ct * -__get_cpu_var (cputime_scaled_last_delta) / -__get_cpu_var (cputime_last_delta); +__this_cpu_read (cputime_scaled_last_delta) / +__this_cpu_read (cputime_last_delta);return ct; }
arch/powerpc/include/asm/hardirq.h +1 -1
@@ -19,7 +19,7 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpust#define __ARCH_IRQ_STAT -#define local_softirq_pending () __get_cpu_var(irq_stat).__softirq_pending +#define local_softirq_pending () __this_cpu_read(irq_stat.__softirq_pending)static inline void ack_bad_irq (unsigned int irq) {
arch/powerpc/include/asm/tlbflush.h +2 -2
@@ -107,14 +107,14 @@ extern void __flush_tlb_pending(struct pstatic inline void arch_enter_lazy_mmu_mode (void ) { -struct ppc64_tlb_batch *batch = &__get_cpu_var (ppc64_tlb_batch); +struct ppc64_tlb_batch *batch =this_cpu_ptr (&ppc64_tlb_batch); batch->active =1 ; }static inline void arch_leave_lazy_mmu_mode (void ) { -struct ppc64_tlb_batch *batch = &__get_cpu_var (ppc64_tlb_batch); +struct ppc64_tlb_batch *batch =this_cpu_ptr (&ppc64_tlb_batch);if (batch->index)__flush_tlb_pending (batch);
arch/powerpc/include/asm/xics.h +4 -4
@@ -97,7 +97,7 @@ DECLARE_PER_CPU(struct xics_cppr, xics_cstatic inline void xics_push_cppr (unsigned int vec) { -struct xics_cppr *os_cppr = &__get_cpu_var (xics_cppr); +struct xics_cppr *os_cppr =this_cpu_ptr (&xics_cppr);if (WARN_ON (os_cppr->index >=MAX_NUM_PRIORITIES -1 ))return ;
@@ -110,7 +110,7 @@ static inline void xics_push_cppr(unsignstatic inline unsigned char xics_pop_cppr (void ) { -struct xics_cppr *os_cppr = &__get_cpu_var (xics_cppr); +struct xics_cppr *os_cppr =this_cpu_ptr (&xics_cppr);if (WARN_ON (os_cppr->index <1 ))return LOWEST_PRIORITY ;
@@ -120,7 +120,7 @@ static inline unsigned char xics_pop_cppstatic inline void xics_set_base_cppr (unsigned char cppr) { -struct xics_cppr *os_cppr = &__get_cpu_var (xics_cppr); +struct xics_cppr *os_cppr =this_cpu_ptr (&xics_cppr);/* we only really want to set the priority when there's * just one cppr value on the stack
@@ -132,7 +132,7 @@ static inline void xics_set_base_cppr(unstatic inline unsigned char xics_cppr_top (void ) { -struct xics_cppr *os_cppr = &__get_cpu_var (xics_cppr); +struct xics_cppr *os_cppr =this_cpu_ptr (&xics_cppr);return os_cppr->stack[os_cppr->index]; }
arch/powerpc/kernel/dbell.c +1 -1
@@ -41,7 +41,7 @@ void doorbell_exception(struct pt_regs *may_hard_irq_enable (); -__get_cpu_var (irq_stat).doorbell_irqs++; +__this_cpu_inc (irq_stat.doorbell_irqs);smp_ipi_demux ();
arch/powerpc/kernel/hw_breakpoint.c +3 -3
@@ -64,7 +64,7 @@ int hw_breakpoint_slots(int type)int arch_install_hw_breakpoint (struct perf_event *bp) {struct arch_hw_breakpoint *info =counter_arch_bp (bp); -struct perf_event **slot = &__get_cpu_var (bp_per_reg); +struct perf_event **slot =this_cpu_ptr (&bp_per_reg); *slot = bp;
@@ -89,7 +89,7 @@ int arch_install_hw_breakpoint(struct pe*/ void arch_uninstall_hw_breakpoint (struct perf_event *bp) { -struct perf_event **slot = &__get_cpu_var (bp_per_reg); +struct perf_event **slot =this_cpu_ptr (&bp_per_reg);if (*slot != bp) {WARN_ONCE (1 ,"Can't find the breakpoint" );
@@ -227,7 +227,7 @@ int __kprobes hw_breakpoint_handler(stru*/ rcu_read_lock (); - bp =__get_cpu_var (bp_per_reg); + bp =__this_cpu_read (bp_per_reg);if (!bp) goto out; info =counter_arch_bp (bp);
arch/powerpc/kernel/irq.c +1 -1
@@ -114,7 +114,7 @@ static inline notrace void set_soft_enab static inline notrace int decrementer_check_overflow(void) {u64 now =get_tb_or_rtc (); -u64 *next_tb = &__get_cpu_var (decrementers_next_tb); +u64 *next_tb =this_cpu_ptr (&decrementers_next_tb);return now >= *next_tb; }
arch/powerpc/kernel/kprobes.c +3 -3
@@ -118,7 +118,7 @@ static void __kprobes save_previous_kprostatic void __kprobes restore_previous_kprobe (struct kprobe_ctlblk *kcb) { -__get_cpu_var (current_kprobe) = kcb->prev_kprobe.kp; +__this_cpu_write (current_kprobe, kcb->prev_kprobe.kp); kcb->kprobe_status = kcb->prev_kprobe.status; kcb->kprobe_saved_msr = kcb->prev_kprobe.saved_msr; }
@@ -126,7 +126,7 @@ static void __kprobes restore_previous_kstatic void __kprobes set_current_kprobe (struct kprobe *p,struct pt_regs *regs,struct kprobe_ctlblk *kcb) { -__get_cpu_var (current_kprobe) = p; +__this_cpu_write (current_kprobe, p); kcb->kprobe_saved_msr = regs->msr; }
@@ -191,7 +191,7 @@ static int __kprobes kprobe_handler(stru ret =1 ; goto no_kprobe; } - p =__get_cpu_var (current_kprobe); + p =__this_cpu_read (current_kprobe);if (p->break_handler && p->break_handler (p, regs)) { goto ss_probe; }
arch/powerpc/kernel/process.c +5 -5
@@ -456,7 +456,7 @@ static inline int set_dawr(struct arch_hint set_breakpoint (struct arch_hw_breakpoint *brk) { -__get_cpu_var (current_brk) = *brk; +__this_cpu_write (current_brk, *brk);if (cpu_has_feature (CPU_FTR_DAWR ))return set_dawr (brk);
@@ -690,7 +690,7 @@ struct task_struct *__switch_to(struct t* schedule DABR */ #ifndef CONFIG_HAVE_HW_BREAKPOINT -if (unlikely (hw_brk_match (&__get_cpu_var (current_brk), &new->thread.hw_brk))) +if (unlikely (hw_brk_match (this_cpu_ptr (¤t_brk), &new->thread.hw_brk)))set_breakpoint (&new->thread.hw_brk);#endif /* CONFIG_HAVE_HW_BREAKPOINT */ #endif
@@ -704,7 +704,7 @@ struct task_struct *__switch_to(struct t* Collect processor utilization data per process */ if (firmware_has_feature (FW_FEATURE_SPLPAR )) { -struct cpu_usage *cu = &__get_cpu_var (cpu_usage_array); +struct cpu_usage *cu =this_cpu_ptr (&cpu_usage_array);long unsigned start_tb, current_tb; start_tb = old_thread->start_tb; cu->current_tb = current_tb =mfspr (SPRN_PURR );
@@ -714,7 +714,7 @@ struct task_struct *__switch_to(struct t#endif /* CONFIG_PPC64 */ #ifdef CONFIG_PPC_BOOK3S_64 - batch = &__get_cpu_var (ppc64_tlb_batch); + batch =this_cpu_ptr (&ppc64_tlb_batch);if (batch->active) {current_thread_info ()->local_flags |= _TLF_LAZY_MMU;if (batch->index)
@@ -737,7 +737,7 @@ struct task_struct *__switch_to(struct t#ifdef CONFIG_PPC_BOOK3S_64 if (current_thread_info ()->local_flags & _TLF_LAZY_MMU) {current_thread_info ()->local_flags &= ~_TLF_LAZY_MMU; - batch = &__get_cpu_var (ppc64_tlb_batch); + batch =this_cpu_ptr (&ppc64_tlb_batch); batch->active =1 ; }#endif /* CONFIG_PPC_BOOK3S_64 */
arch/powerpc/kernel/smp.c +3 -3
@@ -240,7 +240,7 @@ void smp_muxed_ipi_message_pass(int cpu,irqreturn_t smp_ipi_demux (void ) { -struct cpu_messages *info = &__get_cpu_var (ipi_message); +struct cpu_messages *info =this_cpu_ptr (&ipi_message);unsigned int all;mb ();/* order any irq clear */
@@ -425,9 +425,9 @@ void generic_mach_cpu_die(void)idle_task_exit (); cpu =smp_processor_id ();printk (KERN_DEBUG "CPU%d offline\n" , cpu); -__get_cpu_var (cpu_state) =CPU_DEAD ; +__this_cpu_write (cpu_state,CPU_DEAD );smp_wmb (); -while (__get_cpu_var (cpu_state) !=CPU_UP_PREPARE ) +while (__this_cpu_read (cpu_state) !=CPU_UP_PREPARE )cpu_relax (); }
arch/powerpc/kernel/sysfs.c +2 -2
@@ -98,10 +98,10 @@ void ppc_enable_pmcs(void)ppc_set_pmu_inuse (1 );/* Only need to enable them once */ -if (__get_cpu_var (pmcs_enabled)) +if (__this_cpu_read (pmcs_enabled))return ; -__get_cpu_var (pmcs_enabled) =1 ; +__this_cpu_write (pmcs_enabled,1 );if (ppc_md.enable_pmcs) ppc_md.enable_pmcs ();
arch/powerpc/kernel/time.c +8 -8
@@ -457,9 +457,9 @@ static inline void clear_irq_work_pendinDEFINE_PER_CPU (u8, irq_work_pending); -#define set_irq_work_pending_flag () __get_cpu_var(irq_work_pending) = 1 -#define test_irq_work_pending () __get_cpu_var(irq_work_pending) -#define clear_irq_work_pending () __get_cpu_var(irq_work_pending) = 0 +#define set_irq_work_pending_flag () __this_cpu_write(irq_work_pending, 1) +#define test_irq_work_pending () __this_cpu_read(irq_work_pending) +#define clear_irq_work_pending () __this_cpu_write(irq_work_pending, 0)#endif /* 32 vs 64 bit */
@@ -485,8 +485,8 @@ void arch_irq_work_raise(void)void timer_interrupt (struct pt_regs * regs) {struct pt_regs *old_regs; -u64 *next_tb = &__get_cpu_var (decrementers_next_tb); -struct clock_event_device *evt = &__get_cpu_var (decrementers); +u64 *next_tb =this_cpu_ptr (&decrementers_next_tb); +struct clock_event_device *evt =this_cpu_ptr (&decrementers);u64 now;/* Ensure a positive value is written to the decrementer, or else
@@ -510,7 +510,7 @@ void timer_interrupt(struct pt_regs * re*/ may_hard_irq_enable (); -__get_cpu_var (irq_stat).timer_irqs++; +__this_cpu_inc (irq_stat.timer_irqs);#if defined(CONFIG_PPC32 ) && defined(CONFIG_PMAC )if (atomic_read (&ppc_n_lost_interrupts) !=0 )
@@ -541,7 +541,7 @@ void timer_interrupt(struct pt_regs * re#ifdef CONFIG_PPC64 /* collect purr register values often, for accurate calculations */ if (firmware_has_feature (FW_FEATURE_SPLPAR )) { -struct cpu_usage *cu = &__get_cpu_var (cpu_usage_array); +struct cpu_usage *cu =this_cpu_ptr (&cpu_usage_array); cu->current_tb =mfspr (SPRN_PURR ); }#endif
@@ -801,7 +801,7 @@ static void __init clocksource_init(voidstatic int decrementer_set_next_event (unsigned long evt,struct clock_event_device *dev) { -__get_cpu_var (decrementers_next_tb) =get_tb_or_rtc () + evt; +__this_cpu_write (decrementers_next_tb,get_tb_or_rtc () + evt);set_dec (evt);return 0 ; }
arch/powerpc/kernel/traps.c +2 -2
@@ -673,7 +673,7 @@ void machine_check_exception(struct pt_renum ctx_state prev_state =exception_enter ();int recover =0 ; -__get_cpu_var (irq_stat).mce_exceptions++; +__this_cpu_inc (irq_stat.mce_exceptions);/* See if any machine dependent calls. In theory, we would want * to call the CPU first, and call the ppc_md. one if the CPU
@@ -1456,7 +1456,7 @@ void vsx_unavailable_tm(struct pt_regs *void performance_monitor_exception (struct pt_regs *regs) { -__get_cpu_var (irq_stat).pmu_irqs++; +__this_cpu_inc (irq_stat.pmu_irqs);perf_irq (regs); }
arch/powerpc/kvm/e500.c +7 -7
@@ -74,11 +74,11 @@ static inline int local_sid_setup_one(stunsigned long sid;int ret =-1 ; - sid = ++(__get_cpu_var (pcpu_last_used_sid)); + sid =__this_cpu_inc_return (pcpu_last_used_sid);if (sid <NUM_TIDS ) { -__get_cpu_var (pcpu_sids).entry[sid] = entry; + __this_cpu_write(pcpu_sids)entry[sid], entry); entry->val = sid; - entry->pentry = &__get_cpu_var (pcpu_sids).entry[sid]; + entry->pentry =this_cpu_ptr (&pcpu_sids.entry[sid]); ret = sid; }
@@ -106,8 +106,8 @@ static inline int local_sid_setup_one(ststatic inline int local_sid_lookup (struct id *entry) {if (entry && entry->val !=0 && -__get_cpu_var (pcpu_sids).entry[entry->val] == entry && - entry->pentry == &__get_cpu_var (pcpu_sids).entry[entry->val]) +__this_cpu_read (pcpu_sids.entry[entry->val]) == entry && + entry->pentry ==this_cpu_ptr (&pcpu_sids.entry[entry->val]))return entry->val;return -1 ; }
@@ -115,8 +115,8 @@ static inline int local_sid_lookup(struc/* Invalidate all id mappings on local core -- call with preempt disabled */ static inline void local_sid_destroy_all (void ) { -__get_cpu_var (pcpu_last_used_sid) =0 ; -memset (&__get_cpu_var (pcpu_sids),0 ,sizeof (__get_cpu_var (pcpu_sids))); +__this_cpu_write (pcpu_last_used_sid,0 ); +memset (this_cpu_ptr (&pcpu_sids),0 ,sizeof (pcpu_sids)); }static void *kvmppc_e500_id_table_alloc (struct kvmppc_vcpu_e500 *vcpu_e500)
arch/powerpc/kvm/e500mc.c +2 -2
@@ -139,9 +139,9 @@ static void kvmppc_core_vcpu_load_e500mcmtspr (SPRN_GESR , vcpu->arch.shared->esr);if (vcpu->arch.oldpir !=mfspr (SPRN_PIR ) || -__get_cpu_var (last_vcpu_on_cpu) != vcpu) { +__this_cpu_read (last_vcpu_on_cpu) != vcpu) {kvmppc_e500_tlbil_all (vcpu_e500); -__get_cpu_var (last_vcpu_on_cpu) = vcpu; +__this_cpu_read (last_vcpu_on_cpu) = vcpu; }kvmppc_load_guest_fp (vcpu);
arch/powerpc/mm/hash_native_64.c +1 -1
@@ -649,7 +649,7 @@ static void native_flush_hash_range(unsiunsigned long want_v;unsigned long flags;real_pte_t pte; -struct ppc64_tlb_batch *batch = &__get_cpu_var (ppc64_tlb_batch); +struct ppc64_tlb_batch *batch =this_cpu_ptr (&ppc64_tlb_batch);unsigned long psize = batch->psize;int ssize = batch->ssize;int i;
arch/powerpc/mm/hash_utils_64.c +1 -1
@@ -1283,7 +1283,7 @@ void flush_hash_range(unsigned long numbelse {int i;struct ppc64_tlb_batch *batch = - &__get_cpu_var (ppc64_tlb_batch); +this_cpu_ptr (&ppc64_tlb_batch);for (i =0 ; i < number; i++) flush_hash_page(batch->vpn[i], batch->pte[i],
arch/powerpc/mm/hugetlbpage-book3e.c +3 -3
@@ -80,14 +80,14 @@ void book3e_hugetlb_preload(struct vm_ar ncams =mfspr (SPRN_TLB1CFG ) & TLBnCFG_N_ENTRY;/* We have to use the CAM(TLB1) on FSL parts for hugepages */ - index =__get_cpu_var (next_tlbcam_idx); + index =__this_cpu_read (next_tlbcam_idx);mtspr (SPRN_MAS0 ,MAS0_ESEL (index) |MAS0_TLBSEL (1 ));/* Just round-robin the entries and wrap when we hit the end */ if (unlikely (index == ncams -1 )) -__get_cpu_var (next_tlbcam_idx) = tlbcam_index; +__this_cpu_write (next_tlbcam_idx, tlbcam_index);else -__get_cpu_var (next_tlbcam_idx)++; +__this_cpu_inc (next_tlbcam_idx);#endif mas1 =MAS1_VALID |MAS1_TID (mm->context.id) |MAS1_TSIZE (tsize); mas2 = ea & ~((1UL << shift) -1 );
arch/powerpc/mm/hugetlbpage.c +1 -1
@@ -472,7 +472,7 @@ static void hugepd_free(struct mmu_gathe { struct hugepd_freelist **batchp; - batchp = &__get_cpu_var(hugepd_freelist_cur); + batchp = this_cpu_ptr(&hugepd_freelist_cur); if (atomic_read(&tlb->mm->mm_users) < 2 || cpumask_equal(mm_cpumask(tlb->mm),
arch/powerpc/mm/stab.c +6 -6
@@ -133,12 +133,12 @@ static int __ste_allocate(unsigned long stab_entry =make_ste (get_paca ()->stab_addr,GET_ESID (ea), vsid);if (!is_kernel_addr (ea)) { - offset =__get_cpu_var (stab_cache_ptr); + offset =__this_cpu_read (stab_cache_ptr);if (offset <NR_STAB_CACHE_ENTRIES ) -__get_cpu_var (stab_cache[offset++]) = stab_entry; +__this_cpu_read (stab_cache[offset++]) = stab_entry;else offset =NR_STAB_CACHE_ENTRIES +1 ; -__get_cpu_var (stab_cache_ptr) = offset; +__this_cpu_write (stab_cache_ptr, offset);/* Order update */ asmvolatile ("sync" :::"memory" );
@@ -177,12 +177,12 @@ void switch_stab(struct task_struct *tsk*/ hard_irq_disable (); - offset =__get_cpu_var (stab_cache_ptr); + offset =__this_cpu_read (stab_cache_ptr);if (offset <=NR_STAB_CACHE_ENTRIES ) {int i;for (i =0 ; i < offset; i++) { - ste = stab +__get_cpu_var (stab_cache[i]); + ste = stab +__this_cpu_read (stab_cache[i]); ste->esid_data =0 ;/* invalidate entry */ } }else {
@@ -206,7 +206,7 @@ void switch_stab(struct task_struct *tsk asmvolatile ("sync; slbia; sync" :::"memory" ); -__get_cpu_var (stab_cache_ptr) =0 ; +__this_cpu_write (stab_cache_ptr,0 );/* Now preload some entries for the new task */ if (test_tsk_thread_flag (tsk,TIF_32BIT ))
arch/powerpc/perf/core-book3s.c +11 -11
@@ -332,7 +332,7 @@ static void power_pmu_bhrb_reset(void)static void power_pmu_bhrb_enable (struct perf_event *event) { -struct cpu_hw_events *cpuhw = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuhw =this_cpu_ptr (&cpu_hw_events);if (!ppmu->bhrb_nr)return ;
@@ -347,7 +347,7 @@ static void power_pmu_bhrb_enable(structstatic void power_pmu_bhrb_disable (struct perf_event *event) { -struct cpu_hw_events *cpuhw = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuhw =this_cpu_ptr (&cpu_hw_events);if (!ppmu->bhrb_nr)return ;
@@ -961,7 +961,7 @@ static void power_pmu_disable(struct pmuif (!ppmu)return ;local_irq_save (flags); - cpuhw = &__get_cpu_var (cpu_hw_events); + cpuhw =this_cpu_ptr (&cpu_hw_events);if (!cpuhw->disabled) {/*
@@ -1027,7 +1027,7 @@ static void power_pmu_enable(struct pmureturn ;local_irq_save (flags); - cpuhw = &__get_cpu_var (cpu_hw_events); + cpuhw =this_cpu_ptr (&cpu_hw_events);if (!cpuhw->disabled) goto out;
@@ -1211,7 +1211,7 @@ static int power_pmu_add(struct perf_eve* Add the event to the list (if there is room) * and check whether the total set is still feasible. */ - cpuhw = &__get_cpu_var (cpu_hw_events); + cpuhw =this_cpu_ptr (&cpu_hw_events); n0 = cpuhw->n_events;if (n0 >= ppmu->n_counter) goto out;
@@ -1277,7 +1277,7 @@ static void power_pmu_del(struct perf_evpower_pmu_read (event); - cpuhw = &__get_cpu_var (cpu_hw_events); + cpuhw =this_cpu_ptr (&cpu_hw_events);for (i =0 ; i < cpuhw->n_events; ++i) {if (event == cpuhw->event[i]) {while (++i < cpuhw->n_events) {
@@ -1383,7 +1383,7 @@ static void power_pmu_stop(struct perf_e*/ void power_pmu_start_txn (struct pmu *pmu) { -struct cpu_hw_events *cpuhw = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuhw =this_cpu_ptr (&cpu_hw_events);perf_pmu_disable (pmu); cpuhw->group_flag |=PERF_EVENT_TXN ;
@@ -1397,7 +1397,7 @@ void power_pmu_start_txn(struct pmu *pmu*/ void power_pmu_cancel_txn (struct pmu *pmu) { -struct cpu_hw_events *cpuhw = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuhw =this_cpu_ptr (&cpu_hw_events); cpuhw->group_flag &= ~PERF_EVENT_TXN ;perf_pmu_enable (pmu);
@@ -1415,7 +1415,7 @@ int power_pmu_commit_txn(struct pmu *pmuif (!ppmu)return -EAGAIN ; - cpuhw = &__get_cpu_var (cpu_hw_events); + cpuhw =this_cpu_ptr (&cpu_hw_events); n = cpuhw->n_events;if (check_excludes (cpuhw->event, cpuhw->flags,0 , n))return -EAGAIN ;
@@ -1772,7 +1772,7 @@ static void record_and_restart(struct peif (event->attr.sample_type &PERF_SAMPLE_BRANCH_STACK ) {struct cpu_hw_events *cpuhw; - cpuhw = &__get_cpu_var (cpu_hw_events); + cpuhw =this_cpu_ptr (&cpu_hw_events);power_pmu_bhrb_read (cpuhw); data.br_stack = &cpuhw->bhrb_stack; }
@@ -1845,7 +1845,7 @@ static bool pmc_overflow(unsigned long vstatic void perf_event_interrupt (struct pt_regs *regs) {int i, j; -struct cpu_hw_events *cpuhw = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuhw =this_cpu_ptr (&cpu_hw_events);struct perf_event *event;unsigned long val[8 ];int found, active;
arch/powerpc/perf/core-fsl-emb.c +3 -3
@@ -210,7 +210,7 @@ static void fsl_emb_pmu_disable(struct punsigned long flags;local_irq_save (flags); - cpuhw = &__get_cpu_var (cpu_hw_events); + cpuhw =this_cpu_ptr (&cpu_hw_events);if (!cpuhw->disabled) { cpuhw->disabled =1 ;
@@ -249,7 +249,7 @@ static void fsl_emb_pmu_enable(struct pmunsigned long flags;local_irq_save (flags); - cpuhw = &__get_cpu_var (cpu_hw_events); + cpuhw =this_cpu_ptr (&cpu_hw_events);if (!cpuhw->disabled) goto out;
@@ -653,7 +653,7 @@ static void record_and_restart(struct pestatic void perf_event_interrupt (struct pt_regs *regs) {int i; -struct cpu_hw_events *cpuhw = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuhw =this_cpu_ptr (&cpu_hw_events);struct perf_event *event;unsigned long val;int found =0 ;
arch/powerpc/platforms/cell/interrupt.c +3 -3
@@ -82,7 +82,7 @@ static void iic_unmask(struct irq_data *static void iic_eoi (struct irq_data *d) { -struct iic *iic = &__get_cpu_var (cpu_iic); +struct iic *iic =this_cpu_ptr (&cpu_iic);out_be64 (&iic->regs->prio, iic->eoi_stack[--iic->eoi_ptr]);BUG_ON (iic->eoi_ptr <0 ); }
@@ -148,7 +148,7 @@ static unsigned int iic_get_irq(void)struct iic *iic;unsigned int virq; - iic = &__get_cpu_var (cpu_iic); + iic =this_cpu_ptr (&cpu_iic); *(unsigned long *) &pending =in_be64 ((u64 __iomem *) &iic->regs->pending_destr);if (!(pending.flags &CBE_IIC_IRQ_VALID ))
@@ -163,7 +163,7 @@ static unsigned int iic_get_irq(void)void iic_setup_cpu (void ) { -out_be64 (&__get_cpu_var (cpu_iic).regs->prio,0xff ); +out_be64 (this_cpu_ptr (&cpu_iic.regs->prio),0xff ); }u8 iic_get_target_id (int cpu)
arch/powerpc/platforms/ps3/interrupt.c +1 -1
@@ -711,7 +711,7 @@ void __init ps3_register_ipi_irq(unsignestatic unsigned int ps3_get_irq (void ) { -struct ps3_private *pd = &__get_cpu_var (ps3_private); +struct ps3_private *pd =this_cpu_ptr (&ps3_private);u64 x = (pd->bmp.status & pd->bmp.mask);unsigned int plug;
arch/powerpc/platforms/pseries/dtl.c +1 -1
@@ -75,7 +75,7 @@ static atomic_t dtl_count;*/ static void consume_dtle (struct dtl_entry *dtle,u64 index) { -struct dtl_ring *dtlr = &__get_cpu_var (dtl_rings); +struct dtl_ring *dtlr =this_cpu_ptr (&dtl_rings);struct dtl_entry *wp = dtlr->write_ptr;struct lppaca *vpa = local_paca->lppaca_ptr;
arch/powerpc/platforms/pseries/hvCall_inst.c +2 -2
@@ -109,7 +109,7 @@ static void probe_hcall_entry(void *ignoif (opcode >MAX_HCALL_OPCODE )return ; - h = &__get_cpu_var (hcall_stats)[opcode /4 ]; + h =this_cpu_ptr (&hcall_stats[opcode /4 ]); h->tb_start =mftb (); h->purr_start =mfspr (SPRN_PURR ); }
@@ -122,7 +122,7 @@ static void probe_hcall_exit(void *ignorif (opcode >MAX_HCALL_OPCODE )return ; - h = &__get_cpu_var (hcall_stats)[opcode /4 ]; + h =this_cpu_ptr (&hcall_stats[opcode /4 ]); h->num_calls++; h->tb_total +=mftb () - h->tb_start; h->purr_total +=mfspr (SPRN_PURR ) - h->purr_start;
arch/powerpc/platforms/pseries/iommu.c +4 -4
@@ -200,7 +200,7 @@ static int tce_buildmulti_pSeriesLP(strulocal_irq_save (flags);/* to protect tcep and the page behind it */ - tcep =__get_cpu_var (tce_page); + tcep =__this_cpu_read (tce_page);/* This is safe to do since interrupts are off when we're called * from iommu_alloc{,_sg}()
@@ -213,7 +213,7 @@ static int tce_buildmulti_pSeriesLP(strureturn tce_build_pSeriesLP (tbl, tcenum, npages, uaddr, direction, attrs); } -__get_cpu_var (tce_page) = tcep; +__this_cpu_write (tce_page, tcep); } rpn =__pa (uaddr) >>TCE_SHIFT ;
@@ -399,7 +399,7 @@ static int tce_setrange_multi_pSeriesLP(long l, limit;local_irq_disable ();/* to protect tcep and the page behind it */ - tcep =__get_cpu_var (tce_page); + tcep =__this_cpu_read (tce_page);if (!tcep) { tcep = (__be64 *)__get_free_page (GFP_ATOMIC );
@@ -407,7 +407,7 @@ static int tce_setrange_multi_pSeriesLP(local_irq_enable ();return -ENOMEM ; } -__get_cpu_var (tce_page) = tcep; +__this_cpu_write (tce_page, tcep); } proto_tce =TCE_PCI_READ |TCE_PCI_WRITE ;
arch/powerpc/platforms/pseries/lpar.c +3 -3
@@ -513,7 +513,7 @@ static void pSeries_lpar_flush_hash_rangunsigned long vpn;unsigned long i, pix, rc;unsigned long flags =0 ; -struct ppc64_tlb_batch *batch = &__get_cpu_var (ppc64_tlb_batch); +struct ppc64_tlb_batch *batch =this_cpu_ptr (&ppc64_tlb_batch);int lock_tlbie = !mmu_has_feature (MMU_FTR_LOCKLESS_TLBIE );unsigned long param[9 ];unsigned long hash, index, shift, hidx, slot;
@@ -688,7 +688,7 @@ void __trace_hcall_entry(unsigned long olocal_irq_save (flags); - depth = &__get_cpu_var (hcall_trace_depth); + depth =this_cpu_ptr (&hcall_trace_depth);if (*depth) goto out;
@@ -713,7 +713,7 @@ void __trace_hcall_exit(long opcode, unslocal_irq_save (flags); - depth = &__get_cpu_var (hcall_trace_depth); + depth =this_cpu_ptr (&hcall_trace_depth);if (*depth) goto out;
arch/powerpc/platforms/pseries/ras.c +2 -2
@@ -301,8 +301,8 @@ static struct rtas_error_log *fwnmi_get_/* If it isn't an extended log we can use the per cpu 64bit buffer */ h = (struct rtas_error_log *)&savep[1 ];if (!h->extended) { -memcpy (&__get_cpu_var (mce_data_buf), h,sizeof (__u64)); - errhdr = (struct rtas_error_log *)&__get_cpu_var (mce_data_buf); +memcpy (this_cpu_ptr (&mce_data_buf), h,sizeof (__u64)); + errhdr = (struct rtas_error_log *)this_cpu_ptr (&mce_data_buf); }else {int len;
arch/powerpc/sysdev/xics/xics-common.c +1 -1
@@ -155,7 +155,7 @@ int __init xics_smp_probe(void)void xics_teardown_cpu (void ) { -struct xics_cppr *os_cppr = &__get_cpu_var (xics_cppr); +struct xics_cppr *os_cppr =this_cpu_ptr (&xics_cppr);/* * we have to reset the cppr index to 0 because we're
arch/powerpc/kernel/iommu.c +1 -1
@@ -208,7 +208,7 @@ static unsigned long iommu_range_alloc(s* We don't need to disable preemption here because any CPU can * safely use any IOMMU pool. */ - pool_nr =__raw_get_cpu_var (iommu_pool_hash) & (tbl->nr_pools -1 ); + pool_nr =__this_cpu_read (iommu_pool_hash) & (tbl->nr_pools -1 );if (largealloc) pool = &(tbl->large_pool);
PATCH 36/41 blackfin: Replace __get_cpu_var uses
In reply to Christoph Lameter__get_cpu_var() is used for multiple purposes in the kernel source. One of them is address calculation via the form &__get_cpu_var(x). This calculates the address for the instance of the percpu variable of the current processor based on an offset.
Other use cases are for storing and retrieving data from the current processors percpu area. __get_cpu_var() can be used as an lvalue when writing data or on the right side of an assignment.
__get_cpu_var() is defined as :
#define __get_cpu_var(var) (*this_cpu_ptr(&(var)))
__get_cpu_var() always only does an address determination. However, store and retrieve operations could use a segment prefix (or global register on other platforms) to avoid the address calculation.
this_cpu_write() and this_cpu_read() can directly take an offset into a percpu area and use optimized assembly code to read and write per cpu variables.
This patch converts __get_cpu_var into either an explicit address calculation using this_cpu_ptr() or into a use of this_cpu operations that use the offset. Thereby address calculations are avoided and less registers are used when code is generated.
At the end of the patch set all uses of __get_cpu_var have been removed so the macro is removed too.
The patch set includes passes over all arches as well. Once these operations are used throughout then specialized macros can be defined in non -x86 arches as well in order to optimize per cpu access by f.e. using a global register that may be set to the per cpu base.
Transformations done to __get_cpu_var()
1. Determine the address of the percpu instance of the current processor.
DEFINE_PER_CPU(int, y);
int *x = &__get_cpu_var(y);
Converts to
int *x = this_cpu_ptr(&y);2. Same as #1 but this time an array structure is involved.
DEFINE_PER_CPU(int, y[20]);
int *x = __get_cpu_var(y);
Converts to
int *x = this_cpu_ptr(y);3. Retrieve the content of the current processors instance of a per cpu variable.
DEFINE_PER_CPU(int, y); int x = __get_cpu_var(y)
Converts to
int x = __this_cpu_read(y);
4. Retrieve the content of a percpu struct
DEFINE_PER_CPU(struct mystruct, y); struct mystruct x = __get_cpu_var(y);
Converts to
memcpy(&x, this_cpu_ptr(&y), sizeof(x));
5. Assignment to a per cpu variable
DEFINE_PER_CPU(int, y) __get_cpu_var(y) = x;
Converts to
this_cpu_write(y, x);
6. Increment/Decrement etc of a per cpu variable
DEFINE_PER_CPU(int, y); __get_cpu_var(y)++
Converts to
this_cpu_inc(y)
arch/blackfin/include/asm/ipipe.h +1 -1
@@ -157,7 +157,7 @@ static inline unsigned long __ipipe_ffnz }#define __ipipe_do_root_xirq (ipd, irq) \ - ((ipd)->irqs[irq].handler(irq, &__raw_get_cpu_var(__ipipe_tick_regs))) + ((ipd)->irqs[irq].handler(irq, raw_cpu_ptr(&__ipipe_tick_regs)))#define __ipipe_run_irqtail (irq)/* Must be a macro */ \ do { \
arch/blackfin/kernel/perf_event.c +5 -5
@@ -300,7 +300,7 @@ again:static void bfin_pmu_stop (struct perf_event *event,int flags) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);struct hw_perf_event *hwc = &event->hw;int idx = hwc->idx;
@@ -318,7 +318,7 @@ static void bfin_pmu_stop(struct perf_evstatic void bfin_pmu_start (struct perf_event *event,int flags) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);struct hw_perf_event *hwc = &event->hw;int idx = hwc->idx;
@@ -335,7 +335,7 @@ static void bfin_pmu_start(struct perf_estatic void bfin_pmu_del (struct perf_event *event,int flags) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);bfin_pmu_stop (event,PERF_EF_UPDATE );__clear_bit (event->hw.idx, cpuc->used_mask);
@@ -345,7 +345,7 @@ static void bfin_pmu_del(struct perf_evestatic int bfin_pmu_add (struct perf_event *event,int flags) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);struct hw_perf_event *hwc = &event->hw;int idx = hwc->idx;int ret = -EAGAIN ;
@@ -429,7 +429,7 @@ static int bfin_pmu_event_init(struct pestatic void bfin_pmu_enable (struct pmu *pmu) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);struct perf_event *event;struct hw_perf_event *hwc;int i;
arch/blackfin/mach-common/ints-priority.c +4 -4
@@ -1312,12 +1312,12 @@ asmlinkage int __ipipe_grab_irq(int vec,bfin_write_TIMER_STATUS (1 );/* Latch TIMIL0 */ #endif /* This is basically what we need from the register frame. */ -__raw_get_cpu_var (__ipipe_tick_regs).ipend = regs->ipend; -__raw_get_cpu_var (__ipipe_tick_regs).pc = regs->pc; +__this_cpu_write (__ipipe_tick_regs.ipend, regs->ipend); +__this_cpu_write (__ipipe_tick_regs.pc, regs->pc);if (this_domain != ipipe_root_domain) -__raw_get_cpu_var (__ipipe_tick_regs).ipend &= ~0x10 ; +__this_cpu_and (__ipipe_tick_regs.ipend, ~0x10 );else -__raw_get_cpu_var (__ipipe_tick_regs).ipend |=0x10 ; +__this_cpu_or (__ipipe_tick_regs.ipend,0x10 ); }/*
arch/blackfin/mach-common/smp.c +1 -1
@@ -147,7 +147,7 @@ static irqreturn_t ipi_handler_int1(intplatform_clear_ipi (cpu,IRQ_SUPPLE_1 );smp_rmb (); - bfin_ipi_data = &__get_cpu_var (bfin_ipi); + bfin_ipi_data =this_cpu_ptr (&bfin_ipi);while ((pending =atomic_xchg (&bfin_ipi_data->bits,0 )) !=0 ) { msg =0 ; do {
PATCH 32/41 ia64: Replace __get_cpu_var uses
In reply to Christoph Lameter__get_cpu_var() is used for multiple purposes in the kernel source. One of them is address calculation via the form &__get_cpu_var(x). This calculates the address for the instance of the percpu variable of the current processor based on an offset.
Other use cases are for storing and retrieving data from the current processors percpu area. __get_cpu_var() can be used as an lvalue when writing data or on the right side of an assignment.
__get_cpu_var() is defined as :
#define __get_cpu_var(var) (*this_cpu_ptr(&(var)))
__get_cpu_var() always only does an address determination. However, store and retrieve operations could use a segment prefix (or global register on other platforms) to avoid the address calculation.
this_cpu_write() and this_cpu_read() can directly take an offset into a percpu area and use optimized assembly code to read and write per cpu variables.
This patch converts __get_cpu_var into either an explicit address calculation using this_cpu_ptr() or into a use of this_cpu operations that use the offset. Thereby address calculations are avoided and less registers are used when code is generated.
At the end of the patch set all uses of __get_cpu_var have been removed so the macro is removed too.
The patch set includes passes over all arches as well. Once these operations are used throughout then specialized macros can be defined in non -x86 arches as well in order to optimize per cpu access by f.e. using a global register that may be set to the per cpu base.
Transformations done to __get_cpu_var()
1. Determine the address of the percpu instance of the current processor.
DEFINE_PER_CPU(int, y);
int *x = &__get_cpu_var(y);
Converts to
int *x = this_cpu_ptr(&y);2. Same as #1 but this time an array structure is involved.
DEFINE_PER_CPU(int, y[20]);
int *x = __get_cpu_var(y);
Converts to
int *x = this_cpu_ptr(y);3. Retrieve the content of the current processors instance of a per cpu variable.
DEFINE_PER_CPU(int, y); int x = __get_cpu_var(y)
Converts to
int x = __this_cpu_read(y);
4. Retrieve the content of a percpu struct
DEFINE_PER_CPU(struct mystruct, y); struct mystruct x = __get_cpu_var(y);
Converts to
memcpy(&x, this_cpu_ptr(&y), sizeof(x));
5. Assignment to a per cpu variable
DEFINE_PER_CPU(int, y) __get_cpu_var(y) = x;
Converts to
this_cpu_write(y, x);
6. Increment/Decrement etc of a per cpu variable
DEFINE_PER_CPU(int, y); __get_cpu_var(y)++
Converts to
this_cpu_inc(y)
arch/alpha/kernel/perf_event.c +1 -1
@@ -814,7 +814,7 @@ static void alpha_perf_event_irq_handlerstruct hw_perf_event *hwc;int idx, j; -__get_cpu_var (irq_pmi_count)++; +__this_cpu_inc (irq_pmi_count); cpuc = &__get_cpu_var (cpu_hw_events);/* Completely counting through the PMC's period to trigger a new PMC
arch/ia64/kernel/irq.c +1 -1
@@ -42,7 +42,7 @@ ia64_vector __ia64_irq_to_vector(int irqunsigned int __ia64_local_vector_to_irq (ia64_vector vec) { -return __get_cpu_var (vector_irq)[vec]; +return __this_cpu_read (vector_irq[vec]); }#endif
arch/ia64/kernel/irq_ia64.c +2 -2
@@ -338,7 +338,7 @@ static irqreturn_t smp_irq_move_cleanup_int irq;struct irq_desc *desc;struct irq_cfg *cfg; - irq =__get_cpu_var (vector_irq)[vector]; + irq =__this_cpu_read (vector_irq[vector]);if (irq <0 )continue ;
@@ -352,7 +352,7 @@ static irqreturn_t smp_irq_move_cleanup_ goto unlock;spin_lock_irqsave (&vector_lock, flags); -__get_cpu_var (vector_irq)[vector] =-1 ; +__this_cpu_write (vector_irq[vector],-1 );cpu_clear (me, vector_table[vector]);spin_unlock_irqrestore (&vector_lock, flags); cfg->move_cleanup_count--;
arch/ia64/kernel/kprobes.c +3 -3
@@ -396,7 +396,7 @@ static void __kprobes restore_previous_k {unsigned int i; i =atomic_read (&kcb->prev_kprobe_index); -__get_cpu_var (current_kprobe) = kcb->prev_kprobe[i-1 ].kp; +__this_cpu_write (current_kprobe, kcb->prev_kprobe[i-1 ].kp); kcb->kprobe_status = kcb->prev_kprobe[i-1 ].status;atomic_sub (1 , &kcb->prev_kprobe_index); }
@@ -404,7 +404,7 @@ static void __kprobes restore_previous_kstatic void __kprobes set_current_kprobe (struct kprobe *p,struct kprobe_ctlblk *kcb) { -__get_cpu_var (current_kprobe) = p; +__this_cpu_write (current_kprobe, p); }static void kretprobe_trampoline (void )
@@ -823,7 +823,7 @@ static int __kprobes pre_kprobes_handler/* * jprobe instrumented function just completed */ - p =__get_cpu_var (current_kprobe); + p =__this_cpu_read (current_kprobe);if (p->break_handler && p->break_handler (p, regs)) { goto ss_probe; }
arch/ia64/kernel/mca.c +8 -8
@@ -1341,7 +1341,7 @@ ia64_mca_handler(struct pt_regs *regs, sia64_mlogbuf_finish (1 ); } -if (__get_cpu_var (ia64_mca_tr_reload)) { +if (__this_cpu_read (ia64_mca_tr_reload)) {mca_insert_tr (0x1 );/*Reload dynamic itrs*/ mca_insert_tr (0x2 );/*Reload dynamic itrs*/ }
@@ -1874,14 +1874,14 @@ ia64_mca_cpu_init(void *cpu_data) "MCA", cpu);format_mca_init_stack (data, offsetof(struct ia64_mca_cpu , init_stack),"INIT" , cpu); -__get_cpu_var (ia64_mca_data) = __per_cpu_mca[cpu] =__pa (data); +__this_cpu_write (ia64_mca_data, (__per_cpu_mca[cpu] =__pa (data)));/* * Stash away a copy of the PTE needed to map the per-CPU page. * We may need it during MCA recovery. */ -__get_cpu_var (ia64_mca_per_cpu_pte) = -pte_val (mk_pte_phys (__pa (cpu_data),PAGE_KERNEL )); +__this_cpu_write (ia64_mca_per_cpu_pte, +pte_val (mk_pte_phys (__pa (cpu_data),PAGE_KERNEL )));/* * Also, stash away a copy of the PAL address and the PTE
@@ -1890,10 +1890,10 @@ ia64_mca_cpu_init(void *cpu_data) pal_vaddr =efi_get_pal_addr ();if (!pal_vaddr)return ; -__get_cpu_var (ia64_mca_pal_base) = -GRANULEROUNDDOWN ((unsigned long ) pal_vaddr); -__get_cpu_var (ia64_mca_pal_pte) =pte_val (mk_pte_phys (__pa (pal_vaddr), -PAGE_KERNEL )); +__this_cpu_write (ia64_mca_pal_base, +GRANULEROUNDDOWN ((unsigned long ) pal_vaddr)); +__this_cpu_write (ia64_mca_pal_pte,pte_val (mk_pte_phys (__pa (pal_vaddr), +PAGE_KERNEL ))); }static void ia64_mca_cmc_vector_adjust (void *dummy)
arch/ia64/kernel/process.c +3 -3
@@ -215,7 +215,7 @@ static inline void play_dead(void)unsigned int this_cpu =smp_processor_id ();/* Ack it */ -__get_cpu_var (cpu_state) =CPU_DEAD ; +__this_cpu_write (cpu_state,CPU_DEAD );max_xtp ();local_irq_disable ();
@@ -273,7 +273,7 @@ ia64_save_extra (struct task_struct *tasif ((task->thread.flags &IA64_THREAD_PM_VALID ) !=0 )pfm_save_regs (task); - info =__get_cpu_var (pfm_syst_info); + info =__this_cpu_read (pfm_syst_info);if (info &PFM_CPUINFO_SYST_WIDE )pfm_syst_wide_update_task (task, info,0 );#endif
@@ -293,7 +293,7 @@ ia64_load_extra (struct task_struct *tasif ((task->thread.flags &IA64_THREAD_PM_VALID ) !=0 )pfm_load_regs (task); - info =__get_cpu_var (pfm_syst_info); + info =__this_cpu_read (pfm_syst_info);if (info &PFM_CPUINFO_SYST_WIDE )pfm_syst_wide_update_task (task, info,1 );#endif
arch/ia64/sn/kernel/sn2/sn2_smp.c +14 -14
@@ -134,8 +134,8 @@ sn2_ipi_flush_all_tlb(struct mm_struct * itc =ia64_get_itc ();smp_flush_tlb_cpumask (*mm_cpumask (mm)); itc =ia64_get_itc () - itc; -__get_cpu_var (ptcstats).shub_ipi_flushes_itc_clocks += itc; -__get_cpu_var (ptcstats).shub_ipi_flushes++; +__this_cpu_add (ptcstats.shub_ipi_flushes_itc_clocks, itc); +__this_cpu_inc (ptcstats.shub_ipi_flushes); }/**
@@ -199,14 +199,14 @@ sn2_global_tlb_purge(struct mm_struct *m start += (1UL << nbits); }while (start < end);ia64_srlz_i (); -__get_cpu_var (ptcstats).ptc_l++; +__this_cpu_inc (ptcstats.ptc_l);preempt_enable ();return ; }if (atomic_read (&mm->mm_users) ==1 && mymm) {flush_tlb_mm (mm); -__get_cpu_var (ptcstats).change_rid++; +__this_cpu_inc (ptcstats.change_rid);preempt_enable ();return ; }
@@ -250,11 +250,11 @@ sn2_global_tlb_purge(struct mm_struct *mspin_lock_irqsave (PTC_LOCK (shub1), flags); itc2 =ia64_get_itc (); -__get_cpu_var (ptcstats).lock_itc_clocks += itc2 - itc; -__get_cpu_var (ptcstats).shub_ptc_flushes++; -__get_cpu_var (ptcstats).nodes_flushed += nix; +__this_cpu_add (ptcstats.lock_itc_clocks, itc2 - itc); +__this_cpu_inc (ptcstats.shub_ptc_flushes); +__this_cpu_add (ptcstats.nodes_flushed, nix);if (!mymm) -__get_cpu_var (ptcstats).shub_ptc_flushes_not_my_mm++; +__this_cpu_inc (ptcstats.shub_ptc_flushes_not_my_mm);if (use_cpu_ptcga && !mymm) { old_rr =ia64_get_rr (start);
@@ -299,9 +299,9 @@ sn2_global_tlb_purge(struct mm_struct *m done: itc2 =ia64_get_itc () - itc2; -__get_cpu_var (ptcstats).shub_itc_clocks += itc2; -if (itc2 >__get_cpu_var (ptcstats).shub_itc_clocks_max) -__get_cpu_var (ptcstats).shub_itc_clocks_max = itc2; + __this_cpu_add(ptcstats)shub_itc_clocks, itc2); +if (itc2 >__this_cpu_read (ptcstats.shub_itc_clocks_max)) + __this_cpu_write(ptcstats).shub_itc_clocks_max, itc2);if (old_rr) {ia64_set_rr (start, old_rr);
@@ -311,7 +311,7 @@ done:spin_unlock_irqrestore (PTC_LOCK (shub1), flags);if (flush_opt ==1 && deadlock) { -__get_cpu_var (ptcstats).deadlocks++; +__this_cpu_inc (ptcstats.deadlocks);sn2_ipi_flush_all_tlb (mm); }
@@ -334,7 +334,7 @@ sn2_ptc_deadlock_recovery(short *nasids,short nasid, i;unsigned long *piows, zeroval, n; -__get_cpu_var (ptcstats).deadlocks++; +__this_cpu_inc (ptcstats.deadlocks); piows = (unsigned long *) pda->pio_write_status_addr; zeroval = pda->pio_write_status_val;
@@ -349,7 +349,7 @@ sn2_ptc_deadlock_recovery(short *nasids, ptc1 =CHANGE_NASID (nasid, ptc1); n =sn2_ptc_deadlock_recovery_core (ptc0, data0, ptc1, data1, piows, zeroval); -__get_cpu_var (ptcstats).deadlocks2 += n; +__this_cpu_add (ptcstats.deadlocks2, n); } }
arch/ia64/include/asm/hw_irq.h +1 -1
@@ -160,7 +160,7 @@ static inline ia64_vector __ia64_irq_to_static inline unsigned int __ia64_local_vector_to_irq (ia64_vector vec) { -return __get_cpu_var (vector_irq)[vec]; +return __this_cpu_read (vector_irq[vec]); }#endif
arch/ia64/include/asm/sn/nodepda.h +1 -1
@@ -70,7 +70,7 @@ typedef struct nodepda_s nodepda_t;*/ DECLARE_PER_CPU(struct nodepda_s *, __sn_nodepda); -#define sn_nodepda (__get_cpu_var(__sn_nodepda)) +#define sn_nodepda __this_cpu_read(__sn_nodepda)#define NODEPDA (cnodeid) (sn_nodepda->pernode_pdaindr[cnodeid])/*
arch/ia64/include/asm/switch_to.h +1 -1
@@ -32,7 +32,7 @@ extern void ia64_load_extra (struct task#ifdef CONFIG_PERFMON DECLARE_PER_CPU(unsigned long, pfm_syst_info); -# define PERFMON_IS_SYSWIDE () (__get_cpu_var(pfm_syst_info) & 0x1) +# define PERFMON_IS_SYSWIDE () (__this_cpu_read(pfm_syst_info) & 0x1)#else # define PERFMON_IS_SYSWIDE () (0)#endif
arch/ia64/include/asm/sn/arch.h +2 -2
@@ -57,7 +57,7 @@ struct sn_hub_info_s {u16 nasid_bitmask; }; DECLARE_PER_CPU(struct sn_hub_info_s, __sn_hub_info); -#define sn_hub_info (&__get_cpu_var(__sn_hub_info)) +#define sn_hub_info this_cpu_ptr(&__sn_hub_info)#define is_shub2 () (sn_hub_info->shub2)#define is_shub1 () (sn_hub_info->shub2 == 0)
@@ -72,7 +72,7 @@ DECLARE_PER_CPU(struct sn_hub_info_s, __* cpu. */ DECLARE_PER_CPU (short, __sn_cnodeid_to_nasid[MAX_COMPACT_NODES ]); -#define sn_cnodeid_to_nasid (&__get_cpu_var(__sn_cnodeid_to_nasid[0])) +#define sn_cnodeid_to_nasid this_cpu_ptr(&__sn_cnodeid_to_nasid[0])extern u8 sn_partition_id;
arch/ia64/include/asm/uv/uv_hub.h +1 -1
@@ -108,7 +108,7 @@ struct uv_hub_info_s {unsigned char n_val; }; DECLARE_PER_CPU(struct uv_hub_info_s, __uv_hub_info); -#define uv_hub_info (&__get_cpu_var(__uv_hub_info)) +#define uv_hub_info this_cpu_ptr(&__uv_hub_info)#define uv_cpu_hub_info (cpu) (&per_cpu(__uv_hub_info, cpu))/*
arch/ia64/kernel/traps.c +1 -1
@@ -299,7 +299,7 @@ handle_fpu_swa (int fp_fault, struct pt_if (!(current->thread.flags &IA64_THREAD_FPEMU_NOPRINT )) {unsigned long count, current_jiffies = jiffies; -struct fpu_swa_msg *cp = &__get_cpu_var (cpulast); +struct fpu_swa_msg *cp =this_cpu_ptr (&cpulast);if (unlikely (current_jiffies > cp->time)) cp->count =0 ;
arch/ia64/xen/time.c +1 -1
@@ -68,7 +68,7 @@ static void get_runstate_snapshot(structBUG_ON (preemptible ()); - state = &__get_cpu_var (xen_runstate); + state =this_cpu_ptr (&xen_runstate);/* * The runstate info is always updated by the hypervisor on
PATCH 30/41 mips: Replace __get_cpu_var uses
In reply to Christoph Lameter__get_cpu_var() is used for multiple purposes in the kernel source. One of them is address calculation via the form &__get_cpu_var(x). This calculates the address for the instance of the percpu variable of the current processor based on an offset.
Other use cases are for storing and retrieving data from the current processors percpu area. __get_cpu_var() can be used as an lvalue when writing data or on the right side of an assignment.
__get_cpu_var() is defined as :
#define __get_cpu_var(var) (*this_cpu_ptr(&(var)))
__get_cpu_var() always only does an address determination. However, store and retrieve operations could use a segment prefix (or global register on other platforms) to avoid the address calculation.
this_cpu_write() and this_cpu_read() can directly take an offset into a percpu area and use optimized assembly code to read and write per cpu variables.
This patch converts __get_cpu_var into either an explicit address calculation using this_cpu_ptr() or into a use of this_cpu operations that use the offset. Thereby address calculations are avoided and less registers are used when code is generated.
At the end of the patch set all uses of __get_cpu_var have been removed so the macro is removed too.
The patch set includes passes over all arches as well. Once these operations are used throughout then specialized macros can be defined in non -x86 arches as well in order to optimize per cpu access by f.e. using a global register that may be set to the per cpu base.
Transformations done to __get_cpu_var()
1. Determine the address of the percpu instance of the current processor.
DEFINE_PER_CPU(int, y);
int *x = &__get_cpu_var(y);
Converts to
int *x = this_cpu_ptr(&y);2. Same as #1 but this time an array structure is involved.
DEFINE_PER_CPU(int, y[20]);
int *x = __get_cpu_var(y);
Converts to
int *x = this_cpu_ptr(y);3. Retrieve the content of the current processors instance of a per cpu variable.
DEFINE_PER_CPU(int, y); int x = __get_cpu_var(y)
Converts to
int x = __this_cpu_read(y);
4. Retrieve the content of a percpu struct
DEFINE_PER_CPU(struct mystruct, y); struct mystruct x = __get_cpu_var(y);
Converts to
memcpy(&x, this_cpu_ptr(&y), sizeof(x));
5. Assignment to a per cpu variable
DEFINE_PER_CPU(int, y) __get_cpu_var(y) = x;
Converts to
this_cpu_write(y, x);
6. Increment/Decrement etc of a per cpu variable
DEFINE_PER_CPU(int, y); __get_cpu_var(y)++
Converts to
this_cpu_inc(y)
arch/mips/include/asm/fpu_emulator.h +1 -1
@@ -43,7 +43,7 @@ DECLARE_PER_CPU(struct mips_fpu_emulator#define MIPS_FPU_EMU_INC_STATS (M ) \ do { \ preempt_disable(); \ - __local_inc(&__get_cpu_var(fpuemustats).M); \ + __this_cpu_inc(fpuemustats.M); \ preempt_enable(); \ } while (0)
arch/mips/cavium-octeon/octeon-irq.c +15 -15
@@ -264,13 +264,13 @@ static void octeon_irq_ciu_enable_local(unsigned long *pen;unsigned long flags;union octeon_ciu_chip_data cd; -raw_spinlock_t *lock = &__get_cpu_var (octeon_irq_ciu_spinlock); +raw_spinlock_t *lock =this_cpu_ptr (&octeon_irq_ciu_spinlock); cd.p =irq_data_get_irq_chip_data (data);raw_spin_lock_irqsave (lock, flags);if (cd.s.line ==0 ) { - pen = &__get_cpu_var (octeon_irq_ciu0_en_mirror); + pen =this_cpu_ptr (&octeon_irq_ciu0_en_mirror);__set_bit (cd.s.bit, pen);/* * Must be visible to octeon_irq_ip{2,3}_ciu() before
@@ -279,7 +279,7 @@ static void octeon_irq_ciu_enable_local(wmb ();cvmx_write_csr (CVMX_CIU_INTX_EN0 (cvmx_get_core_num () *2 ), *pen); }else { - pen = &__get_cpu_var (octeon_irq_ciu1_en_mirror); + pen =this_cpu_ptr (&octeon_irq_ciu1_en_mirror);__set_bit (cd.s.bit, pen);/* * Must be visible to octeon_irq_ip{2,3}_ciu() before
@@ -296,13 +296,13 @@ static void octeon_irq_ciu_disable_localunsigned long *pen;unsigned long flags;union octeon_ciu_chip_data cd; -raw_spinlock_t *lock = &__get_cpu_var (octeon_irq_ciu_spinlock); +raw_spinlock_t *lock =this_cpu_ptr (&octeon_irq_ciu_spinlock); cd.p =irq_data_get_irq_chip_data (data);raw_spin_lock_irqsave (lock, flags);if (cd.s.line ==0 ) { - pen = &__get_cpu_var (octeon_irq_ciu0_en_mirror); + pen =this_cpu_ptr (&octeon_irq_ciu0_en_mirror);__clear_bit (cd.s.bit, pen);/* * Must be visible to octeon_irq_ip{2,3}_ciu() before
@@ -311,7 +311,7 @@ static void octeon_irq_ciu_disable_localwmb ();cvmx_write_csr (CVMX_CIU_INTX_EN0 (cvmx_get_core_num () *2 ), *pen); }else { - pen = &__get_cpu_var (octeon_irq_ciu1_en_mirror); + pen =this_cpu_ptr (&octeon_irq_ciu1_en_mirror);__clear_bit (cd.s.bit, pen);/* * Must be visible to octeon_irq_ip{2,3}_ciu() before
@@ -431,11 +431,11 @@ static void octeon_irq_ciu_enable_local_if (cd.s.line ==0 ) {int index =cvmx_get_core_num () *2 ; -set_bit (cd.s.bit, &__get_cpu_var (octeon_irq_ciu0_en_mirror)); +set_bit (cd.s.bit,this_cpu_ptr (&octeon_irq_ciu0_en_mirror));cvmx_write_csr (CVMX_CIU_INTX_EN0_W1S (index), mask); }else {int index =cvmx_get_core_num () *2 +1 ; -set_bit (cd.s.bit, &__get_cpu_var (octeon_irq_ciu1_en_mirror)); +set_bit (cd.s.bit,this_cpu_ptr (&octeon_irq_ciu1_en_mirror));cvmx_write_csr (CVMX_CIU_INTX_EN1_W1S (index), mask); } }
@@ -450,11 +450,11 @@ static void octeon_irq_ciu_disable_localif (cd.s.line ==0 ) {int index =cvmx_get_core_num () *2 ; -clear_bit (cd.s.bit, &__get_cpu_var (octeon_irq_ciu0_en_mirror)); +clear_bit (cd.s.bit,this_cpu_ptr (&octeon_irq_ciu0_en_mirror));cvmx_write_csr (CVMX_CIU_INTX_EN0_W1C (index), mask); }else {int index =cvmx_get_core_num () *2 +1 ; -clear_bit (cd.s.bit, &__get_cpu_var (octeon_irq_ciu1_en_mirror)); +clear_bit (cd.s.bit,this_cpu_ptr (&octeon_irq_ciu1_en_mirror));cvmx_write_csr (CVMX_CIU_INTX_EN1_W1C (index), mask); } }
@@ -1063,7 +1063,7 @@ static void octeon_irq_ip2_ciu(void)const unsigned long core_id =cvmx_get_core_num ();u64 ciu_sum =cvmx_read_csr (CVMX_CIU_INTX_SUM0 (core_id *2 )); - ciu_sum &=__get_cpu_var (octeon_irq_ciu0_en_mirror); + ciu_sum &=__this_cpu_read (octeon_irq_ciu0_en_mirror);if (likely (ciu_sum)) {int bit =fls64 (ciu_sum) -1 ;int irq = octeon_irq_ciu_to_irq[0 ][bit];
@@ -1080,7 +1080,7 @@ static void octeon_irq_ip3_ciu(void) {u64 ciu_sum =cvmx_read_csr (CVMX_CIU_INT_SUM1 ); - ciu_sum &=__get_cpu_var (octeon_irq_ciu1_en_mirror); + ciu_sum &=__this_cpu_read (octeon_irq_ciu1_en_mirror);if (likely (ciu_sum)) {int bit =fls64 (ciu_sum) -1 ;int irq = octeon_irq_ciu_to_irq[1 ][bit];
@@ -1129,10 +1129,10 @@ static void octeon_irq_init_ciu_percpu(vint coreid =cvmx_get_core_num (); -__get_cpu_var (octeon_irq_ciu0_en_mirror) =0 ; -__get_cpu_var (octeon_irq_ciu1_en_mirror) =0 ; +__this_cpu_write (octeon_irq_ciu0_en_mirror,0 ); +__this_cpu_write (octeon_irq_ciu1_en_mirror,0 );wmb (); -raw_spin_lock_init (&__get_cpu_var (octeon_irq_ciu_spinlock)); +raw_spin_lock_init (this_cpu_ptr (&octeon_irq_ciu_spinlock));/* * Disable All CIU Interrupts. The ones we need will be * enabled later. Read the SUM register so we know the write
arch/mips/kernel/kprobes.c +3 -3
@@ -224,7 +224,7 @@ static void save_previous_kprobe(structstatic void restore_previous_kprobe (struct kprobe_ctlblk *kcb) { -__get_cpu_var (current_kprobe) = kcb->prev_kprobe.kp; +__this_cpu_write (current_kprobe, kcb->prev_kprobe.kp); kcb->kprobe_status = kcb->prev_kprobe.status; kcb->kprobe_old_SR = kcb->prev_kprobe.old_SR; kcb->kprobe_saved_SR = kcb->prev_kprobe.saved_SR;
@@ -234,7 +234,7 @@ static void restore_previous_kprobe(strustatic void set_current_kprobe (struct kprobe *p,struct pt_regs *regs,struct kprobe_ctlblk *kcb) { -__get_cpu_var (current_kprobe) = p; +__this_cpu_write (current_kprobe, p); kcb->kprobe_saved_SR = kcb->kprobe_old_SR = (regs->cp0_status &ST0_IE ); kcb->kprobe_saved_epc = regs->cp0_epc; }
@@ -385,7 +385,7 @@ static int __kprobes kprobe_handler(stru ret =1 ; goto no_kprobe; } - p =__get_cpu_var (current_kprobe); + p =__this_cpu_read (current_kprobe);if (p->break_handler && p->break_handler (p, regs)) goto ss_probe; }
arch/mips/kernel/perf_event_mipsxx.c +7 -7
@@ -340,7 +340,7 @@ static int mipsxx_pmu_alloc_counter(strustatic void mipsxx_pmu_enable_event (struct hw_perf_event *evt,int idx) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);WARN_ON (idx <0 || idx >= mipspmu.num_counters);
@@ -360,7 +360,7 @@ static void mipsxx_pmu_enable_event(strustatic void mipsxx_pmu_disable_event (int idx) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);unsigned long flags;WARN_ON (idx <0 || idx >= mipspmu.num_counters);
@@ -460,7 +460,7 @@ static void mipspmu_stop(struct perf_evestatic int mipspmu_add (struct perf_event *event,int flags) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);struct hw_perf_event *hwc = &event->hw;int idx;int err =0 ;
@@ -496,7 +496,7 @@ out:static void mipspmu_del (struct perf_event *event,int flags) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);struct hw_perf_event *hwc = &event->hw;int idx = hwc->idx;
@@ -1270,7 +1270,7 @@ static int __hw_perf_event_init(struct pstatic void pause_local_counters (void ) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);int ctr = mipspmu.num_counters;unsigned long flags;
@@ -1286,7 +1286,7 @@ static void pause_local_counters(void)static void resume_local_counters (void ) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);int ctr = mipspmu.num_counters; do {
@@ -1297,7 +1297,7 @@ static void resume_local_counters(void)static int mipsxx_pmu_handle_shared_irq (void ) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);struct perf_sample_data data;unsigned int counters = mipspmu.num_counters;u64 counter;
arch/mips/kernel/smp-bmips.c +1 -1
@@ -304,7 +304,7 @@ static irqreturn_t bmips_ipi_interrupt(iint action, cpu = irq -IPI0_IRQ ;spin_lock_irqsave (&ipi_lock, flags); - action =__get_cpu_var (ipi_action_mask); + action =__this_cpu_read (ipi_action_mask);per_cpu (ipi_action_mask, cpu) =0 ;clear_c0_cause (cpu ?C_SW1 :C_SW0 );spin_unlock_irqrestore (&ipi_lock, flags);
PATCH 29/41 arm: Replace __this_cpu_ptr with raw_cpu_ptr
In reply to Christoph Lameter__this_cpu_ptr is being phased out. So replace with raw_cpu_ptr.
arch/arm/kernel/smp_twd.c +6 -6
@@ -92,7 +92,7 @@ static int twd_timer_ack(void)static void twd_timer_stop (void ) { -struct clock_event_device *clk =__this_cpu_ptr (twd_evt); +struct clock_event_device *clk =raw_cpu_ptr (twd_evt);twd_set_mode (CLOCK_EVT_MODE_UNUSED , clk);disable_percpu_irq (clk->irq);
@@ -108,7 +108,7 @@ static void twd_update_frequency(void *n { twd_timer_rate = *((unsigned long *) new_rate); -clockevents_update_freq (__this_cpu_ptr (twd_evt), twd_timer_rate); +clockevents_update_freq (raw_cpu_ptr (twd_evt), twd_timer_rate); } static int twd_rate_change(struct notifier_block *nb,
@@ -134,7 +134,7 @@ static struct notifier_block twd_clk_nbstatic int twd_clk_init (void ) { -if (twd_evt &&__this_cpu_ptr (twd_evt) && !IS_ERR (twd_clk)) +if (twd_evt &&raw_cpu_ptr (twd_evt) && !IS_ERR (twd_clk))return clk_notifier_register (twd_clk, &twd_clk_nb);return 0 ;
@@ -153,7 +153,7 @@ static void twd_update_frequency(void *d { twd_timer_rate =clk_get_rate (twd_clk); -clockevents_update_freq (__this_cpu_ptr (twd_evt), twd_timer_rate); +clockevents_update_freq (raw_cpu_ptr (twd_evt), twd_timer_rate); } static int twd_cpufreq_transition(struct notifier_block *nb,
@@ -179,7 +179,7 @@ static struct notifier_block twd_cpufreqstatic int twd_cpufreq_init (void ) { -if (twd_evt &&__this_cpu_ptr (twd_evt) && !IS_ERR (twd_clk)) +if (twd_evt &&raw_cpu_ptr (twd_evt) && !IS_ERR (twd_clk))return cpufreq_register_notifier (&twd_cpufreq_nb,CPUFREQ_TRANSITION_NOTIFIER );
@@ -269,7 +269,7 @@ static void twd_get_clock(struct device_*/ static void twd_timer_setup (void ) { -struct clock_event_device *clk =__this_cpu_ptr (twd_evt); +struct clock_event_device *clk =raw_cpu_ptr (twd_evt);int cpu =smp_processor_id ();/*
arch/arm/mach-msm/timer.c +1 -1
@@ -221,7 +221,7 @@ static void __init msm_timer_init(u32 dg }/* Immediately configure the timer on the boot CPU */ -msm_local_timer_setup (__this_cpu_ptr (msm_evt)); +msm_local_timer_setup (raw_cpu_ptr (msm_evt)); } err:
PATCH 28/41 x86: Replace __get_cpu_var uses
In reply to Christoph Lameter__get_cpu_var() is used for multiple purposes in the kernel source. One of them is address calculation via the form &__get_cpu_var(x). This calculates the address for the instance of the percpu variable of the current processor based on an offset.
Other use cases are for storing and retrieving data from the current processors percpu area. __get_cpu_var() can be used as an lvalue when writing data or on the right side of an assignment.
__get_cpu_var() is defined as :
#define __get_cpu_var(var) (*this_cpu_ptr(&(var)))
__get_cpu_var() always only does an address determination. However, store and retrieve operations could use a segment prefix (or global register on other platforms) to avoid the address calculation.
this_cpu_write() and this_cpu_read() can directly take an offset into a percpu area and use optimized assembly code to read and write per cpu variables.
This patch converts __get_cpu_var into either an explicit address calculation using this_cpu_ptr() or into a use of this_cpu operations that use the offset. Thereby address calculations are avoided and less registers are used when code is generated.
Transformations done to __get_cpu_var()
1. Determine the address of the percpu instance of the current processor.
DEFINE_PER_CPU(int, y);
int *x = &__get_cpu_var(y);
Converts to
int *x = this_cpu_ptr(&y);2. Same as #1 but this time an array structure is involved.
DEFINE_PER_CPU(int, y[20]);
int *x = __get_cpu_var(y);
Converts to
int *x = this_cpu_ptr(y);3. Retrieve the content of the current processors instance of a per cpu variable.
DEFINE_PER_CPU(int, y); int x = __get_cpu_var(y)
Converts to
int x = __this_cpu_read(y);
4. Retrieve the content of a percpu struct
DEFINE_PER_CPU(struct mystruct, y); struct mystruct x = __get_cpu_var(y);
Converts to
memcpy(&x, this_cpu_ptr(&y), sizeof(x));
5. Assignment to a per cpu variable
DEFINE_PER_CPU(int, y) __get_cpu_var(y) = x;
Converts to
this_cpu_write(y, x);
6. Increment/Decrement etc of a per cpu variable
DEFINE_PER_CPU(int, y); __get_cpu_var(y)++
Converts to
this_cpu_inc(y)
arch/x86/kernel/cpu/mcheck/mce_intel.c +10 -10
@@ -86,7 +86,7 @@ void mce_intel_cmci_poll(void) {if (__this_cpu_read (cmci_storm_state) ==CMCI_STORM_NONE )return ; -machine_check_poll (MCP_TIMESTAMP , &__get_cpu_var (mce_banks_owned)); +machine_check_poll (MCP_TIMESTAMP ,this_cpu_ptr (&mce_banks_owned)); }void mce_intel_hcpu_update (unsigned long cpu)
@@ -179,7 +179,7 @@ static void intel_threshold_interrupt(vo {if (cmci_storm_detect ())return ; -machine_check_poll (MCP_TIMESTAMP , &__get_cpu_var (mce_banks_owned)); +machine_check_poll (MCP_TIMESTAMP ,this_cpu_ptr (&mce_banks_owned));mce_notify_irq (); }
@@ -190,7 +190,7 @@ static void intel_threshold_interrupt(vo*/ static void cmci_discover (int banks) { -unsigned long *owned = (void *)&__get_cpu_var (mce_banks_owned); +unsigned long *owned = (void *)this_cpu_ptr (&mce_banks_owned);unsigned long flags;int i;int bios_wrong_thresh =0 ;
@@ -212,7 +212,7 @@ static void cmci_discover(int banks)/* Already owned by someone else? */ if (val &MCI_CTL2_CMCI_EN ) {clear_bit (i, owned); -__clear_bit (i,__get_cpu_var (mce_poll_banks)); +__clear_bit (i,this_cpu_ptr (mce_poll_banks));continue ; }
@@ -236,7 +236,7 @@ static void cmci_discover(int banks)/* Did the enable bit stick? -- the bank supports CMCI */ if (val &MCI_CTL2_CMCI_EN ) {set_bit (i, owned); -__clear_bit (i,__get_cpu_var (mce_poll_banks)); +__clear_bit (i,this_cpu_ptr (mce_poll_banks));/* * We are able to set thresholds for some banks that * had a threshold of 0. This means the BIOS has not
@@ -247,7 +247,7 @@ static void cmci_discover(int banks) (val & MCI_CTL2_CMCI_THRESHOLD_MASK)) bios_wrong_thresh =1 ; }else { -WARN_ON (!test_bit (i,__get_cpu_var (mce_poll_banks))); +WARN_ON (!test_bit (i,this_cpu_ptr (mce_poll_banks))); } }raw_spin_unlock_irqrestore (&cmci_discover_lock, flags);
@@ -268,10 +268,10 @@ void cmci_recheck(void)unsigned long flags;int banks; -if (!mce_available (__this_cpu_ptr (&cpu_info)) || !cmci_supported (&banks)) +if (!mce_available (raw_cpu_ptr (&cpu_info)) || !cmci_supported (&banks))return ;local_irq_save (flags); -machine_check_poll (MCP_TIMESTAMP , &__get_cpu_var (mce_banks_owned)); +machine_check_poll (MCP_TIMESTAMP ,this_cpu_ptr (&mce_banks_owned));local_irq_restore (flags); }
@@ -280,12 +280,12 @@ static void __cmci_disable_bank(int bank {u64 val; -if (!test_bit (bank,__get_cpu_var (mce_banks_owned))) +if (!test_bit (bank,this_cpu_ptr (mce_banks_owned)))return ;rdmsrl (MSR_IA32_MCx_CTL2 (bank), val); val &= ~MCI_CTL2_CMCI_EN ;wrmsrl (MSR_IA32_MCx_CTL2 (bank), val); -__clear_bit (bank,__get_cpu_var (mce_banks_owned)); +__clear_bit (bank,this_cpu_ptr (mce_banks_owned)); }/*
arch/x86/kernel/irq_64.c +3 -3
@@ -52,13 +52,13 @@ static inline void stack_overflow_check( regs->sp <= curbase + THREAD_SIZE)return ; - irq_stack_top = (u64 )__get_cpu_var (irq_stack_union.irq_stack) + + irq_stack_top = (u64 )this_cpu_ptr (irq_stack_union.irq_stack) +STACK_TOP_MARGIN ; - irq_stack_bottom = (u64 )__get_cpu_var (irq_stack_ptr); + irq_stack_bottom = (u64 )__this_cpu_read (irq_stack_ptr);if (regs->sp >= irq_stack_top && regs->sp <= irq_stack_bottom)return ; - oist = &__get_cpu_var (orig_ist); + oist =this_cpu_ptr (&orig_ist); estack_top = (u64 )oist->ist[0 ] -EXCEPTION_STKSZ +STACK_TOP_MARGIN ; estack_bottom = (u64 )oist->ist[N_EXCEPTION_STACKS -1 ];if (regs->sp >= estack_top && regs->sp <= estack_bottom)
arch/x86/kernel/kvm.c +11 -11
@@ -243,9 +243,9 @@ u32 kvm_read_and_reset_pf_reason(void) {u32 reason =0 ; -if (__get_cpu_var (apf_reason).enabled) { - reason =__get_cpu_var (apf_reason).reason; -__get_cpu_var (apf_reason).reason =0 ; +if (__this_cpu_read (apf_reason.enabled)) { + reason =__this_cpu_read (apf_reason.reason); +__this_cpu_write (apf_reason.reason,0 ); }return reason;
@@ -316,7 +316,7 @@ static void kvm_guest_apic_eoi_write(u32* there's no need for lock or memory barriers. * An optimization barrier is implied in apic write. */ -if (__test_and_clear_bit (KVM_PV_EOI_BIT , &__get_cpu_var (kvm_apic_eoi))) +if (__test_and_clear_bit (KVM_PV_EOI_BIT ,this_cpu_ptr (&kvm_apic_eoi)))return ;apic_write (APIC_EOI ,APIC_EOI_ACK ); }
@@ -327,13 +327,13 @@ void kvm_guest_cpu_init(void)return ;if (kvm_para_has_feature (KVM_FEATURE_ASYNC_PF ) && kvmapf) { -u64 pa =slow_virt_to_phys (&__get_cpu_var (apf_reason)); +u64 pa =slow_virt_to_phys (this_cpu_ptr (&apf_reason));#ifdef CONFIG_PREEMPT pa |=KVM_ASYNC_PF_SEND_ALWAYS ;#endif wrmsrl (MSR_KVM_ASYNC_PF_EN , pa |KVM_ASYNC_PF_ENABLED ); -__get_cpu_var (apf_reason).enabled =1 ; +__this_cpu_write (apf_reason.enabled,1 );printk (KERN_INFO "KVM setup async PF for cpu %d\n" ,smp_processor_id ()); }
@@ -342,8 +342,8 @@ void kvm_guest_cpu_init(void)unsigned long pa;/* Size alignment is implied but just to make it explicit. */ BUILD_BUG_ON (__alignof__(kvm_apic_eoi ) <4 ); -__get_cpu_var (kvm_apic_eoi) =0 ; - pa =slow_virt_to_phys (&__get_cpu_var (kvm_apic_eoi)) +__this_cpu_write (kvm_apic_eoi,0 ); + pa =slow_virt_to_phys (this_cpu_ptr (&kvm_apic_eoi)) |KVM_MSR_ENABLED ;wrmsrl (MSR_KVM_PV_EOI_EN , pa); }
@@ -354,11 +354,11 @@ void kvm_guest_cpu_init(void)static void kvm_pv_disable_apf (void ) { -if (!__get_cpu_var (apf_reason).enabled) +if (!__this_cpu_read (apf_reason.enabled))return ;wrmsrl (MSR_KVM_ASYNC_PF_EN ,0 ); -__get_cpu_var (apf_reason).enabled =0 ; +__this_cpu_write (apf_reason.enabled,0 );printk (KERN_INFO "Unregister pv shared memory for cpu %d\n" ,smp_processor_id ());
@@ -683,7 +683,7 @@ static void kvm_lock_spinning(struct arcif (in_nmi ())return ; - w = &__get_cpu_var (klock_waiting); + w =this_cpu_ptr (&klock_waiting); cpu =smp_processor_id (); start =spin_time_start ();
arch/x86/kvm/svm.c +3 -3
@@ -654,7 +654,7 @@ static int svm_hardware_enable(void *garif (static_cpu_has (X86_FEATURE_TSCRATEMSR )) {wrmsrl (MSR_AMD64_TSC_RATIO ,TSC_RATIO_DEFAULT ); -__get_cpu_var (current_tsc_ratio) =TSC_RATIO_DEFAULT ; +__this_cpu_write (current_tsc_ratio,TSC_RATIO_DEFAULT ); }
@@ -1312,8 +1312,8 @@ static void svm_vcpu_load(struct kvm_vcprdmsrl (host_save_user_msrs[i], svm->host_user_msrs[i]);if (static_cpu_has (X86_FEATURE_TSCRATEMSR ) && - svm->tsc_ratio !=__get_cpu_var (current_tsc_ratio)) { -__get_cpu_var (current_tsc_ratio) = svm->tsc_ratio; + svm->tsc_ratio !=__this_cpu_read (current_tsc_ratio)) { +__this_cpu_write (current_tsc_ratio, svm->tsc_ratio);wrmsrl (MSR_AMD64_TSC_RATIO , svm->tsc_ratio); } }
arch/x86/kvm/x86.c +1 -1
@@ -1507,7 +1507,7 @@ static int kvm_guest_time_update(struct/* Keep irq disabled to prevent changes to the clock */ local_irq_save (flags); - this_tsc_khz =__get_cpu_var (cpu_tsc_khz); + this_tsc_khz =__this_cpu_read (cpu_tsc_khz);if (unlikely (this_tsc_khz ==0 )) {local_irq_restore (flags);kvm_make_request (KVM_REQ_CLOCK_UPDATE , v);
arch/x86/oprofile/op_model_p4.c +1 -1
@@ -372,7 +372,7 @@ static unsigned int get_stagger(void) {#ifdef CONFIG_SMP int cpu =smp_processor_id (); -return cpu !=cpumask_first (__get_cpu_var (cpu_sibling_map)); +return cpu !=cpumask_first (this_cpu_ptr (cpu_sibling_map));#endif return 0 ; }
arch/x86/xen/enlighten.c +2 -2
@@ -820,7 +820,7 @@ static void xen_convert_trap_info(constvoid xen_copy_trap_info (struct trap_info *traps) { -const struct desc_ptr *desc = &__get_cpu_var (idt_desc); +const struct desc_ptr *desc =this_cpu_ptr (&idt_desc);xen_convert_trap_info (desc, traps); }
@@ -837,7 +837,7 @@ static void xen_load_idt(const struct despin_lock (&lock); -__get_cpu_var (idt_desc) = *desc; +memcpy (this_cpu_ptr (&idt_desc), desc,sizeof (idt_desc));xen_convert_trap_info (desc, traps);
arch/x86/kernel/apb_timer.c +2 -2
@@ -146,7 +146,7 @@ static inline int is_apbt_capable(void)static int __init apbt_clockevent_register (void ) {struct sfi_timer_table_entry *mtmr; -struct apbt_dev *adev = &__get_cpu_var (cpu_apbt_dev); +struct apbt_dev *adev =this_cpu_ptr (&cpu_apbt_dev); mtmr =sfi_get_mtmr (APBT_CLOCKEVENT0_NUM );if (mtmr ==NULL ) {
@@ -200,7 +200,7 @@ void apbt_setup_secondary_clock(void)if (!cpu)return ; - adev = &__get_cpu_var (cpu_apbt_dev); + adev =this_cpu_ptr (&cpu_apbt_dev);if (!adev->timer) { adev->timer = dw_apb_clockevent_init(cpu, adev->name, APBT_CLOCKEVENT_RATING, adev_virt_addr(adev),
arch/x86/kernel/apic/apic.c +2 -2
@@ -547,7 +547,7 @@ static DEFINE_PER_CPU(struct clock_event*/ static void setup_APIC_timer (void ) { -struct clock_event_device *levt = &__get_cpu_var (lapic_events); +struct clock_event_device *levt =this_cpu_ptr (&lapic_events);if (this_cpu_has (X86_FEATURE_ARAT )) { lapic_clockevent.features &= ~CLOCK_EVT_FEAT_C3STOP ;
@@ -682,7 +682,7 @@ calibrate_by_pmtimer(long deltapm, longstatic int __init calibrate_APIC_clock (void ) { -struct clock_event_device *levt = &__get_cpu_var (lapic_events); +struct clock_event_device *levt =this_cpu_ptr (&lapic_events);void (*real_handler)(struct clock_event_device *dev);unsigned long deltaj;long delta, deltatsc;
arch/x86/kernel/cpu/mcheck/mce-inject.c +3 -3
@@ -83,7 +83,7 @@ static DEFINE_MUTEX(mce_inject_mutex);static int mce_raise_notify (unsigned int cmd,struct pt_regs *regs) {int cpu =smp_processor_id (); -struct mce *m = &__get_cpu_var (injectm); +struct mce *m =this_cpu_ptr (&injectm);if (!cpumask_test_cpu (cpu, mce_inject_cpumask))return NMI_DONE ;cpumask_clear_cpu (cpu, mce_inject_cpumask);
@@ -97,7 +97,7 @@ static int mce_raise_notify(unsigned intstatic void mce_irq_ipi (void *info) {int cpu =smp_processor_id (); -struct mce *m = &__get_cpu_var (injectm); +struct mce *m =this_cpu_ptr (&injectm);if (cpumask_test_cpu (cpu, mce_inject_cpumask) && m->inject_flags &MCJ_EXCEPTION ) {
@@ -109,7 +109,7 @@ static void mce_irq_ipi(void *info)/* Inject mce on current CPU */ static int raise_local (void ) { -struct mce *m = &__get_cpu_var (injectm); +struct mce *m =this_cpu_ptr (&injectm);int context =MCJ_CTX (m->inject_flags);int ret =0 ;int cpu = m->extcpu;
arch/x86/kernel/cpu/mcheck/mce.c +22 -22
163 lines not on this page · raw message
arch/x86/kernel/cpu/mcheck/mce_amd.c +1 -1
@@ -310,7 +310,7 @@ static void amd_threshold_interrupt(void* event. */ machine_check_poll (MCP_TIMESTAMP , - &__get_cpu_var (mce_poll_banks)); +this_cpu_ptr (&mce_poll_banks));if (high &MASK_OVERFLOW_HI ) {rdmsrl (address, m.misc);
arch/x86/kernel/cpu/perf_event.c +11 -11
@@ -493,7 +493,7 @@ static int __x86_pmu_event_init(struct pvoid x86_pmu_disable_all (void ) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);int idx;for (idx =0 ; idx < x86_pmu.num_counters; idx++) {
@@ -511,7 +511,7 @@ void x86_pmu_disable_all(void)static void x86_pmu_disable (struct pmu *pmu) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);if (!x86_pmu_initialized ())return ;
@@ -528,7 +528,7 @@ static void x86_pmu_disable(struct pmu *void x86_pmu_enable_all (int added) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);int idx;for (idx =0 ; idx < x86_pmu.num_counters; idx++) {
@@ -874,7 +874,7 @@ static void x86_pmu_start(struct perf_evstatic void x86_pmu_enable (struct pmu *pmu) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);struct perf_event *event;struct hw_perf_event *hwc;int i, added = cpuc->n_added;
@@ -1023,7 +1023,7 @@ void x86_pmu_enable_event(struct perf_ev*/ static int x86_pmu_add (struct perf_event *event,int flags) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);struct hw_perf_event *hwc;int assign[X86_PMC_IDX_MAX ];int n, n0, ret;
@@ -1070,7 +1070,7 @@ out:static void x86_pmu_start (struct perf_event *event,int flags) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);int idx = event->hw.idx;if (WARN_ON_ONCE (!(event->hw.state &PERF_HES_STOPPED )))
@@ -1149,7 +1149,7 @@ void perf_event_print_debug(void)void x86_pmu_stop (struct perf_event *event,int flags) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);struct hw_perf_event *hwc = &event->hw;if (__test_and_clear_bit (hwc->idx, cpuc->active_mask)) {
@@ -1171,7 +1171,7 @@ void x86_pmu_stop(struct perf_event *evestatic void x86_pmu_del (struct perf_event *event,int flags) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);int i;/*
@@ -1213,7 +1213,7 @@ int x86_pmu_handle_irq(struct pt_regs *rint idx, handled =0 ;u64 val; - cpuc = &__get_cpu_var (cpu_hw_events); + cpuc =this_cpu_ptr (&cpu_hw_events);/* * Some chipsets need to unmask the LVTPC in a particular spot
@@ -1608,7 +1608,7 @@ static void x86_pmu_cancel_txn(struct pm*/ static int x86_pmu_commit_txn (struct pmu *pmu) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);int assign[X86_PMC_IDX_MAX ];int n, ret;
@@ -1958,7 +1958,7 @@ static unsigned long get_segment_base(unif (idx >GDT_ENTRIES )return 0 ; - desc =__this_cpu_ptr (&gdt_page.gdt[0 ]); + desc =raw_cpu_ptr (gdt_page.gdt); }return get_desc_base (desc + idx);
arch/x86/kernel/cpu/perf_event_amd.c +2 -2
@@ -699,7 +699,7 @@ __init int amd_pmu_init(void)void amd_pmu_enable_virt (void ) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events); cpuc->perf_ctr_virt_mask =0 ;
@@ -711,7 +711,7 @@ EXPORT_SYMBOL_GPL(amd_pmu_enable_virt);void amd_pmu_disable_virt (void ) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);/* * We only mask out the Host-only bit so that host-only counting works
arch/x86/kernel/cpu/perf_event_intel.c +9 -9
@@ -1046,7 +1046,7 @@ static inline bool intel_pmu_needs_lbr_sstatic void intel_pmu_disable_all (void ) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);wrmsrl (MSR_CORE_PERF_GLOBAL_CTRL ,0 );
@@ -1059,7 +1059,7 @@ static void intel_pmu_disable_all(void)static void intel_pmu_enable_all (int added) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);intel_pmu_pebs_enable_all ();intel_pmu_lbr_enable_all ();
@@ -1093,7 +1093,7 @@ static void intel_pmu_enable_all(int add*/ static void intel_pmu_nhm_workaround (void ) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);static const unsigned long nhm_magic[4 ] = {0x4300B5 ,0x4300D2 ,
@@ -1192,7 +1192,7 @@ static inline bool event_is_checkpointedstatic void intel_pmu_disable_event (struct perf_event *event) {struct hw_perf_event *hwc = &event->hw; -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);if (unlikely (hwc->idx ==INTEL_PMC_IDX_FIXED_BTS )) {intel_pmu_disable_bts ();
@@ -1256,7 +1256,7 @@ static void intel_pmu_enable_fixed(strucstatic void intel_pmu_enable_event (struct perf_event *event) {struct hw_perf_event *hwc = &event->hw; -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);if (unlikely (hwc->idx ==INTEL_PMC_IDX_FIXED_BTS )) {if (!__this_cpu_read (cpu_hw_events.enabled))
@@ -1350,7 +1350,7 @@ static int intel_pmu_handle_irq(struct pu64 status;int handled; - cpuc = &__get_cpu_var (cpu_hw_events); + cpuc =this_cpu_ptr (&cpu_hw_events);/* * No known reason to not always do late ACK,
@@ -1775,7 +1775,7 @@ EXPORT_SYMBOL_GPL(perf_guest_get_msrs);static struct perf_guest_switch_msr *intel_guest_get_msrs (int *nr) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);struct perf_guest_switch_msr *arr = cpuc->guest_switch_msrs; arr[0 ].msr =MSR_CORE_PERF_GLOBAL_CTRL ;
@@ -1796,7 +1796,7 @@ static struct perf_guest_switch_msr *intstatic struct perf_guest_switch_msr *core_guest_get_msrs (int *nr) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);struct perf_guest_switch_msr *arr = cpuc->guest_switch_msrs;int idx;
@@ -1830,7 +1830,7 @@ static void core_pmu_enable_event(structstatic void core_pmu_enable_all (int added) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);int idx;for (idx =0 ; idx < x86_pmu.num_counters; idx++) {
arch/x86/kernel/cpu/perf_event_intel_ds.c +10 -10
@@ -457,7 +457,7 @@ void intel_pmu_enable_bts(u64 config)void intel_pmu_disable_bts (void ) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);unsigned long debugctlmsr;if (!cpuc->ds)
@@ -474,7 +474,7 @@ void intel_pmu_disable_bts(void)int intel_pmu_drain_bts_buffer (void ) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);struct debug_store *ds = cpuc->ds;struct bts_record {u64 from;
@@ -694,7 +694,7 @@ struct event_constraint *intel_pebs_consvoid intel_pmu_pebs_enable (struct perf_event *event) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);struct hw_perf_event *hwc = &event->hw; hwc->config &= ~ARCH_PERFMON_EVENTSEL_INT ;
@@ -709,7 +709,7 @@ void intel_pmu_pebs_enable(struct perf_evoid intel_pmu_pebs_disable (struct perf_event *event) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);struct hw_perf_event *hwc = &event->hw; cpuc->pebs_enabled &= ~(1ULL << hwc->idx);
@@ -727,7 +727,7 @@ void intel_pmu_pebs_disable(struct perf_void intel_pmu_pebs_enable_all (void ) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);if (cpuc->pebs_enabled)wrmsrl (MSR_IA32_PEBS_ENABLE , cpuc->pebs_enabled);
@@ -735,7 +735,7 @@ void intel_pmu_pebs_enable_all(void)void intel_pmu_pebs_disable_all (void ) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);if (cpuc->pebs_enabled)wrmsrl (MSR_IA32_PEBS_ENABLE ,0 );
@@ -743,7 +743,7 @@ void intel_pmu_pebs_disable_all(void)static int intel_pmu_pebs_fixup_ip (struct pt_regs *regs) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);unsigned long from = cpuc->lbr_entries[0 ].from;unsigned long old_to, to = cpuc->lbr_entries[0 ].to;unsigned long ip = regs->ip;
@@ -850,7 +850,7 @@ static void __intel_pmu_pebs_event(struc* We cast to the biggest pebs_record but are careful not to * unconditionally access the 'extra' entries. */ -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);struct pebs_record_hsw *pebs = __pebs;struct perf_sample_data data;struct pt_regs regs;
@@ -939,7 +939,7 @@ static void __intel_pmu_pebs_event(strucstatic void intel_pmu_drain_pebs_core (struct pt_regs *iregs) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);struct debug_store *ds = cpuc->ds;struct perf_event *event = cpuc->events[0 ];/* PMC0 only */ struct pebs_record_core *at, *top;
@@ -980,7 +980,7 @@ static void intel_pmu_drain_pebs_core(ststatic void intel_pmu_drain_pebs_nhm (struct pt_regs *iregs) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);struct debug_store *ds = cpuc->ds;struct perf_event *event =NULL ;void *at, *top;
arch/x86/kernel/cpu/perf_event_intel_lbr.c +6 -6
@@ -133,7 +133,7 @@ static void intel_pmu_lbr_filter(structstatic void __intel_pmu_lbr_enable (void ) {u64 debugctl; -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);if (cpuc->lbr_sel)wrmsrl (MSR_LBR_SELECT , cpuc->lbr_sel->config);
@@ -183,7 +183,7 @@ void intel_pmu_lbr_reset(void)void intel_pmu_lbr_enable (struct perf_event *event) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);if (!x86_pmu.lbr_nr)return ;
@@ -203,7 +203,7 @@ void intel_pmu_lbr_enable(struct perf_evvoid intel_pmu_lbr_disable (struct perf_event *event) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);if (!x86_pmu.lbr_nr)return ;
@@ -220,7 +220,7 @@ void intel_pmu_lbr_disable(struct perf_evoid intel_pmu_lbr_enable_all (void ) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);if (cpuc->lbr_users)__intel_pmu_lbr_enable ();
@@ -228,7 +228,7 @@ void intel_pmu_lbr_enable_all(void)void intel_pmu_lbr_disable_all (void ) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);if (cpuc->lbr_users)__intel_pmu_lbr_disable ();
@@ -332,7 +332,7 @@ static void intel_pmu_lbr_read_64(structvoid intel_pmu_lbr_read (void ) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);if (!cpuc->lbr_users)return ;
arch/x86/kernel/cpu/perf_event_knc.c +1 -1
@@ -217,7 +217,7 @@ static int knc_pmu_handle_irq(struct pt_int bit, loops;u64 status; - cpuc = &__get_cpu_var (cpu_hw_events); + cpuc =this_cpu_ptr (&cpu_hw_events);knc_pmu_disable_all ();
arch/x86/kernel/cpu/perf_event_p4.c +3 -3
@@ -915,7 +915,7 @@ static inline void p4_pmu_disable_event(static void p4_pmu_disable_all (void ) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);int idx;for (idx =0 ; idx < x86_pmu.num_counters; idx++) {
@@ -984,7 +984,7 @@ static void p4_pmu_enable_event(struct pstatic void p4_pmu_enable_all (int added) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);int idx;for (idx =0 ; idx < x86_pmu.num_counters; idx++) {
@@ -1004,7 +1004,7 @@ static int p4_pmu_handle_irq(struct pt_rint idx, handled =0 ;u64 val; - cpuc = &__get_cpu_var (cpu_hw_events); + cpuc =this_cpu_ptr (&cpu_hw_events);for (idx =0 ; idx < x86_pmu.num_counters; idx++) {int overflow;
arch/x86/kernel/hw_breakpoint.c +4 -4
@@ -110,7 +110,7 @@ int arch_install_hw_breakpoint(struct peint i;for (i =0 ; i <HBP_NUM ; i++) { -struct perf_event **slot = &__get_cpu_var (bp_per_reg[i]); +struct perf_event **slot =this_cpu_ptr (&bp_per_reg[i]);if (!*slot) { *slot = bp;
@@ -124,7 +124,7 @@ int arch_install_hw_breakpoint(struct peset_debugreg (info->address, i);__this_cpu_write (cpu_debugreg[i], info->address); - dr7 = &__get_cpu_var (cpu_dr7); + dr7 =this_cpu_ptr (&cpu_dr7); *dr7 |=encode_dr7 (i, info->len, info->type);set_debugreg (*dr7,7 );
@@ -148,7 +148,7 @@ void arch_uninstall_hw_breakpoint(structint i;for (i =0 ; i <HBP_NUM ; i++) { -struct perf_event **slot = &__get_cpu_var (bp_per_reg[i]); +struct perf_event **slot =this_cpu_ptr (&bp_per_reg[i]);if (*slot == bp) { *slot =NULL ;
@@ -159,7 +159,7 @@ void arch_uninstall_hw_breakpoint(structif (WARN_ONCE (i ==HBP_NUM ,"Can't find any breakpoint slot" ))return ; - dr7 = &__get_cpu_var (cpu_dr7); + dr7 =this_cpu_ptr (&cpu_dr7); *dr7 &= ~__encode_dr7 (i, info->len, info->type);set_debugreg (*dr7,7 );
arch/x86/kvm/vmx.c +5 -5
@@ -1518,7 +1518,7 @@ static void reload_tss(void)/* * VT restores TR but not its size. Useless. */ -struct desc_ptr *gdt = &__get_cpu_var (host_gdt); +struct desc_ptr *gdt =this_cpu_ptr (&host_gdt);struct desc_struct *descs; descs = (void *)gdt->address;
@@ -1564,7 +1564,7 @@ static bool update_transition_efer(strucstatic unsigned long segment_base (u16 selector) { -struct desc_ptr *gdt = &__get_cpu_var (host_gdt); +struct desc_ptr *gdt =this_cpu_ptr (&host_gdt);struct desc_struct *d;unsigned long table_base;unsigned long v;
@@ -1690,7 +1690,7 @@ static void __vmx_load_host_state(struct*/ if (!user_has_fpu () && !vmx->vcpu.guest_fpu_loaded)stts (); -load_gdt (&__get_cpu_var (host_gdt)); +load_gdt (this_cpu_ptr (&host_gdt)); }static void vmx_load_host_state (struct vcpu_vmx *vmx)
@@ -1720,7 +1720,7 @@ static void vmx_vcpu_load(struct kvm_vcp }if (vmx->loaded_vmcs->cpu != cpu) { -struct desc_ptr *gdt = &__get_cpu_var (host_gdt); +struct desc_ptr *gdt =this_cpu_ptr (&host_gdt);unsigned long sysenter_esp;kvm_make_request (KVM_REQ_TLB_FLUSH , vcpu);
@@ -2658,7 +2658,7 @@ static int hardware_enable(void *garbageept_sync_global (); } -native_store_gdt (&__get_cpu_var (host_gdt)); +native_store_gdt (this_cpu_ptr (&host_gdt));return 0 ; }
arch/x86/mm/kmemcheck/kmemcheck.c +7 -7
@@ -134,7 +134,7 @@ static DEFINE_PER_CPU(struct kmemcheck_cbool kmemcheck_active (struct pt_regs *regs) { -struct kmemcheck_context *data = &__get_cpu_var (kmemcheck_context); +struct kmemcheck_context *data =this_cpu_ptr (&kmemcheck_context);return data->balance >0 ; }
@@ -142,7 +142,7 @@ bool kmemcheck_active(struct pt_regs *re/* Save an address that needs to be shown/hidden */ static void kmemcheck_save_addr (unsigned long addr) { -struct kmemcheck_context *data = &__get_cpu_var (kmemcheck_context); +struct kmemcheck_context *data =this_cpu_ptr (&kmemcheck_context);BUG_ON (data->n_addrs >=ARRAY_SIZE (data->addr)); data->addr[data->n_addrs++] = addr;
@@ -150,7 +150,7 @@ static void kmemcheck_save_addr(unsignedstatic unsigned int kmemcheck_show_all (void ) { -struct kmemcheck_context *data = &__get_cpu_var (kmemcheck_context); +struct kmemcheck_context *data =this_cpu_ptr (&kmemcheck_context);unsigned int i;unsigned int n;
@@ -163,7 +163,7 @@ static unsigned int kmemcheck_show_all(vstatic unsigned int kmemcheck_hide_all (void ) { -struct kmemcheck_context *data = &__get_cpu_var (kmemcheck_context); +struct kmemcheck_context *data =this_cpu_ptr (&kmemcheck_context);unsigned int i;unsigned int n;
@@ -179,7 +179,7 @@ static unsigned int kmemcheck_hide_all(v*/ void kmemcheck_show (struct pt_regs *regs) { -struct kmemcheck_context *data = &__get_cpu_var (kmemcheck_context); +struct kmemcheck_context *data =this_cpu_ptr (&kmemcheck_context);BUG_ON (!irqs_disabled ());
@@ -220,7 +220,7 @@ void kmemcheck_show(struct pt_regs *regs*/ void kmemcheck_hide (struct pt_regs *regs) { -struct kmemcheck_context *data = &__get_cpu_var (kmemcheck_context); +struct kmemcheck_context *data =this_cpu_ptr (&kmemcheck_context);int n;BUG_ON (!irqs_disabled ());
@@ -522,7 +522,7 @@ static void kmemcheck_access(struct pt_rconst uint8_t *insn_primary;unsigned int size; -struct kmemcheck_context *data = &__get_cpu_var (kmemcheck_context); +struct kmemcheck_context *data =this_cpu_ptr (&kmemcheck_context);/* Recursive fault -- ouch. */ if (data->busy) {
arch/x86/oprofile/nmi_int.c +4 -4
@@ -64,11 +64,11 @@ u64 op_x86_get_ctrl(struct op_x86_model_static int profile_exceptions_notify (unsigned int val,struct pt_regs *regs) {if (ctr_running) - model->check_ctrs (regs, &__get_cpu_var (cpu_msrs)); + model->check_ctrs (regs,this_cpu_ptr (&cpu_msrs));else if (!nmi_enabled)return NMI_DONE ;else - model->stop (&__get_cpu_var (cpu_msrs)); + model->stop (this_cpu_ptr (&cpu_msrs));return NMI_HANDLED ; }
@@ -91,7 +91,7 @@ static void nmi_cpu_save_registers(strucstatic void nmi_cpu_start (void *dummy) { -struct op_msrs const *msrs = &__get_cpu_var (cpu_msrs); +struct op_msrs const *msrs =this_cpu_ptr (&cpu_msrs);if (!msrs->controls)WARN_ON_ONCE (1 );else
@@ -111,7 +111,7 @@ static int nmi_start(void)static void nmi_cpu_stop (void *dummy) { -struct op_msrs const *msrs = &__get_cpu_var (cpu_msrs); +struct op_msrs const *msrs =this_cpu_ptr (&cpu_msrs);if (!msrs->controls)WARN_ON_ONCE (1 );else
arch/x86/platform/uv/uv_time.c +1 -1
@@ -365,7 +365,7 @@ __setup("uvrtcevt", uv_enable_evt_rtc);static __init void uv_rtc_register_clockevents (struct work_struct *dummy) { -struct clock_event_device *ced = &__get_cpu_var (cpu_ced); +struct clock_event_device *ced =this_cpu_ptr (&cpu_ced); *ced = clock_event_device_uv; ced->cpumask =cpumask_of (smp_processor_id ());
arch/x86/xen/multicalls.c +4 -4
@@ -54,7 +54,7 @@ DEFINE_PER_CPU(unsigned long, xen_mc_irqvoid xen_mc_flush (void ) { -struct mc_buffer *b = &__get_cpu_var (mc_buffer); +struct mc_buffer *b =this_cpu_ptr (&mc_buffer);struct multicall_entry *mc;int ret =0 ;unsigned long flags;
@@ -131,7 +131,7 @@ void xen_mc_flush(void)struct multicall_space __xen_mc_entry (size_t args) { -struct mc_buffer *b = &__get_cpu_var (mc_buffer); +struct mc_buffer *b =this_cpu_ptr (&mc_buffer);struct multicall_space ret;unsigned argidx =roundup (b->argidx,sizeof (u64));
@@ -162,7 +162,7 @@ struct multicall_space __xen_mc_entry(sistruct multicall_space xen_mc_extend_args (unsigned long op,size_t size) { -struct mc_buffer *b = &__get_cpu_var (mc_buffer); +struct mc_buffer *b =this_cpu_ptr (&mc_buffer);struct multicall_space ret = {NULL ,NULL };BUG_ON (preemptible ());
@@ -192,7 +192,7 @@ out:void xen_mc_callback (void (*fn)(void *),void *data) { -struct mc_buffer *b = &__get_cpu_var (mc_buffer); +struct mc_buffer *b =this_cpu_ptr (&mc_buffer);struct callback *cb;if (b->cbidx ==MC_BATCH ) {
arch/x86/xen/time.c +5 -5
@@ -80,7 +80,7 @@ static void get_runstate_snapshot(structBUG_ON (preemptible ()); - state = &__get_cpu_var (xen_runstate); + state =this_cpu_ptr (&xen_runstate);/* * The runstate info is always updated by the hypervisor on
@@ -123,7 +123,7 @@ static void do_stolen_accounting(void)WARN_ON (state.state != RUNSTATE_running); - snap = &__get_cpu_var (xen_runstate_snapshot); + snap =this_cpu_ptr (&xen_runstate_snapshot);/* work out how much time the VCPU has not been runn*ing* */ runnable = state.time[RUNSTATE_runnable] - snap->time[RUNSTATE_runnable];
@@ -158,7 +158,7 @@ cycle_t xen_clocksource_read(void)cycle_t ret;preempt_disable_notrace (); - src = &__get_cpu_var (xen_vcpu)->time; + src =this_cpu_ptr (&xen_vcpu->time); ret =pvclock_clocksource_read (src);preempt_enable_notrace ();return ret;
@@ -397,7 +397,7 @@ static DEFINE_PER_CPU(struct xen_clock_estatic irqreturn_t xen_timer_interrupt (int irq,void *dev_id) { -struct clock_event_device *evt = &__get_cpu_var (xen_clock_events).evt; +struct clock_event_device *evt =this_cpu_ptr (&xen_clock_events.evt);irqreturn_t ret; ret =IRQ_NONE ;
@@ -459,7 +459,7 @@ void xen_setup_cpu_clockevents(void) {BUG_ON (preemptible ()); -clockevents_register_device (&__get_cpu_var (xen_clock_events).evt); +clockevents_register_device (this_cpu_ptr (&xen_clock_events.evt)); }void xen_timer_resume (void )
arch/x86/include/asm/uv/uv_hub.h +1 -1
@@ -164,7 +164,7 @@ struct uv_hub_info_s { }; DECLARE_PER_CPU(struct uv_hub_info_s, __uv_hub_info); -#define uv_hub_info (&__get_cpu_var(__uv_hub_info)) +#define uv_hub_info this_cpu_ptr(&__uv_hub_info)#define uv_cpu_hub_info (cpu) (&per_cpu(__uv_hub_info, cpu))/*
arch/x86/include/asm/perf_event_p4.h +1 -1
@@ -189,7 +189,7 @@ static inline int p4_ht_thread(int cpu) {#ifdef CONFIG_SMP if (smp_num_siblings ==2 ) -return cpu !=cpumask_first (__get_cpu_var (cpu_sibling_map)); +return cpu !=cpumask_first (this_cpu_ptr (cpu_sibling_map));#endif return 0 ; }
arch/x86/kernel/apic/x2apic_cluster.c +1 -1
@@ -43,7 +43,7 @@ __x2apic_send_IPI_mask(const struct cpum* We are to modify mask, so we need an own copy * and be sure it's manipulated with irq off. */ - ipi_mask_ptr =__raw_get_cpu_var (ipi_mask); + ipi_mask_ptr =this_cpu_ptr (ipi_mask);cpumask_copy (ipi_mask_ptr, mask);/*
arch/x86/include/asm/debugreg.h +2 -2
@@ -97,11 +97,11 @@ extern void hw_breakpoint_restore(void);DECLARE_PER_CPU (int, debug_stack_usage);static inline void debug_stack_usage_inc (void ) { -__get_cpu_var (debug_stack_usage)++; +__this_cpu_inc (debug_stack_usage); }static inline void debug_stack_usage_dec (void ) { -__get_cpu_var (debug_stack_usage)--; +__this_cpu_dec (debug_stack_usage); }int is_debug_stack (unsigned long addr);void debug_stack_set_zero (void );
arch/x86/kernel/cpu/common.c +3 -3
@@ -1149,9 +1149,9 @@ DEFINE_PER_CPU(int, debug_stack_usage);int is_debug_stack (unsigned long addr) { -return __get_cpu_var (debug_stack_usage) || - (addr <=__get_cpu_var (debug_stack_addr) && - addr > (__get_cpu_var (debug_stack_addr) -DEBUG_STKSZ )); +return __this_cpu_read (debug_stack_usage) || + (addr <=__this_cpu_read (debug_stack_addr) && + addr > (__this_cpu_read (debug_stack_addr) -DEBUG_STKSZ )); }DEFINE_PER_CPU (u32, debug_idt_ctr);
arch/x86/xen/spinlock.c +1 -1
@@ -109,7 +109,7 @@ static bool xen_pvspin = true;static void xen_lock_spinning (struct arch_spinlock *lock,__ticket_t want) {int irq =__this_cpu_read (lock_kicker_irq); -struct xen_lock_waiting *w = &__get_cpu_var (lock_waiting); +struct xen_lock_waiting *w =this_cpu_ptr (&lock_waiting);int cpu =smp_processor_id ();u64 start;unsigned long flags;
PATCH 25/41 drivers/net/ethernet/tile: Replace __get_cpu_var uses for address calculation
In reply to Christoph LameterReplace with this_cpu_ptr.
drivers/net/ethernet/tile/tilegx.c +9 -9
@@ -427,7 +427,7 @@ static void tile_net_pop_all_buffers(int/* Provide linux buffers to mPIPE. */ static void tile_net_provide_needed_buffers (void ) { -struct tile_net_info *info = &__get_cpu_var (per_cpu_info); +struct tile_net_info *info =this_cpu_ptr (&per_cpu_info);int instance, kind;for (instance =0 ; instance <NR_MPIPE_MAX && info->mpipe[instance].has_iqueue; instance++) {
@@ -653,7 +653,7 @@ drop:*/ static int tile_net_poll (struct napi_struct *napi,int budget) { -struct tile_net_info *info = &__get_cpu_var (per_cpu_info); +struct tile_net_info *info =this_cpu_ptr (&per_cpu_info);unsigned int work =0 ;gxio_mpipe_idesc_t *idesc;int instance, i, n;
@@ -699,7 +699,7 @@ done:/* Handle an ingress interrupt from an instance on the current cpu. */ static irqreturn_t tile_net_handle_ingress_irq (int irq,void *id) { -struct tile_net_info *info = &__get_cpu_var (per_cpu_info); +struct tile_net_info *info =this_cpu_ptr (&per_cpu_info);napi_schedule (&info->mpipe[(uint64_t )id].napi);return IRQ_HANDLED ; }
@@ -762,7 +762,7 @@ static enum hrtimer_restart tile_net_han/* Make sure the egress timer is scheduled. */ static void tile_net_schedule_egress_timer (void ) { -struct tile_net_info *info = &__get_cpu_var (per_cpu_info); +struct tile_net_info *info =this_cpu_ptr (&per_cpu_info);if (!info->egress_timer_scheduled) { hrtimer_start(&info->egress_timer,
@@ -779,7 +779,7 @@ static void tile_net_schedule_egress_tim*/ static enum hrtimer_restart tile_net_handle_egress_timer (struct hrtimer *t) { -struct tile_net_info *info = &__get_cpu_var (per_cpu_info); +struct tile_net_info *info =this_cpu_ptr (&per_cpu_info);unsigned long irqflags;bool pending = false;int i, instance;
@@ -1933,7 +1933,7 @@ static void tso_egress(struct net_device*/ static int tile_net_tx_tso (struct sk_buff *skb,struct net_device *dev) { -struct tile_net_info *info = &__get_cpu_var (per_cpu_info); +struct tile_net_info *info =this_cpu_ptr (&per_cpu_info);struct tile_net_priv *priv =netdev_priv (dev);int channel = priv->echannel;int instance = priv->instance;
@@ -2002,7 +2002,7 @@ static unsigned int tile_net_tx_frags(st/* Help the kernel transmit a packet. */ static int tile_net_tx (struct sk_buff *skb,struct net_device *dev) { -struct tile_net_info *info = &__get_cpu_var (per_cpu_info); +struct tile_net_info *info =this_cpu_ptr (&per_cpu_info);struct tile_net_priv *priv =netdev_priv (dev);int instance = priv->instance;struct mpipe_data *md = &mpipe_data[instance];
@@ -2141,7 +2141,7 @@ static int tile_net_set_mac_address(strustatic void tile_net_netpoll (struct net_device *dev) {int instance =mpipe_instance (dev); -struct tile_net_info *info = &__get_cpu_var (per_cpu_info); +struct tile_net_info *info =this_cpu_ptr (&per_cpu_info);struct mpipe_data *md = &mpipe_data[instance];disable_percpu_irq (md->ingress_irq);
@@ -2248,7 +2248,7 @@ static void tile_net_dev_init(const char/* Per-cpu module initialization. */ static void tile_net_init_module_percpu (void *unused) { -struct tile_net_info *info = &__get_cpu_var (per_cpu_info); +struct tile_net_info *info =this_cpu_ptr (&per_cpu_info);int my_cpu =smp_processor_id ();int instance;
drivers/net/ethernet/tile/tilepro.c +4 -4
@@ -993,13 +993,13 @@ static void tile_net_register(void *dev_PDEBUG ("tile_net_register(queue_id %d)\n" , queue_id);if (!strcmp (dev->name,"xgbe0" )) - info = &__get_cpu_var (hv_xgbe0); + info =this_cpu_ptr (&hv_xgbe0);else if (!strcmp (dev->name,"xgbe1" )) - info = &__get_cpu_var (hv_xgbe1); + info =this_cpu_ptr (&hv_xgbe1);else if (!strcmp (dev->name,"gbe0" )) - info = &__get_cpu_var (hv_gbe0); + info =this_cpu_ptr (&hv_gbe0);else if (!strcmp (dev->name,"gbe1" )) - info = &__get_cpu_var (hv_gbe1); + info =this_cpu_ptr (&hv_gbe1);else BUG ();
PATCH 26/41 md: Replace __this_cpu_ptr with raw_cpu_ptr
In reply to Christoph Lameter__this_cpu_ptr is being phased out.
drivers/md/dm-stats.c +1 -1
@@ -548,7 +548,7 @@ void dm_stats_account_io(struct dm_stats* A race condition can at worst result in the merged flag being * misrepresented, so we don't have to disable preemption here. */ - last =__this_cpu_ptr (stats->last); + last =raw_cpu_ptr (stats->last); stats_aux->merged = (bi_sector == (ACCESS_ONCE(last->last_sector) && ((bi_rw & (REQ_WRITE | REQ_DISCARD)) ==
PATCH 24/41 net: Replace get_cpu_var through this_cpu_ptr
In reply to Christoph LameterReplace uses of get_cpu_var for address calculation through this_cpu_ptr.
net/core/dev.c +8 -8
@@ -2130,7 +2130,7 @@ static inline void __netif_reschedule(stunsigned long flags;local_irq_save (flags); - sd = &__get_cpu_var (softnet_data); + sd =this_cpu_ptr (&softnet_data); q->next_sched =NULL ; *sd->output_queue_tailp = q; sd->output_queue_tailp = &q->next_sched;
@@ -2152,7 +2152,7 @@ void dev_kfree_skb_irq(struct sk_buff *sunsigned long flags;local_irq_save (flags); - sd = &__get_cpu_var (softnet_data); + sd =this_cpu_ptr (&softnet_data); skb->next = sd->completion_queue; sd->completion_queue = skb;raise_softirq_irqoff (NET_TX_SOFTIRQ );
@@ -3122,7 +3122,7 @@ static void rps_trigger_softirq(void *dastatic int rps_ipi_queued (struct softnet_data *sd) {#ifdef CONFIG_RPS -struct softnet_data *mysd = &__get_cpu_var (softnet_data); +struct softnet_data *mysd =this_cpu_ptr (&softnet_data);if (sd != mysd) { sd->rps_ipi_next = mysd->rps_ipi_list;
@@ -3149,7 +3149,7 @@ static bool skb_flow_limit(struct sk_bufif (qlen < (netdev_max_backlog >>1 ))return false; - sd = &__get_cpu_var (softnet_data); + sd =this_cpu_ptr (&softnet_data);rcu_read_lock (); fl =rcu_dereference (sd->flow_limit);
@@ -3291,7 +3291,7 @@ EXPORT_SYMBOL(netif_rx_ni);static void net_tx_action (struct softirq_action *h) { -struct softnet_data *sd = &__get_cpu_var (softnet_data); +struct softnet_data *sd =this_cpu_ptr (&softnet_data);if (sd->completion_queue) {struct sk_buff *clist;
@@ -3711,7 +3711,7 @@ EXPORT_SYMBOL(netif_receive_skb);static void flush_backlog (void *arg) {struct net_device *dev = arg; -struct softnet_data *sd = &__get_cpu_var (softnet_data); +struct softnet_data *sd =this_cpu_ptr (&softnet_data);struct sk_buff *skb, *tmp;rps_lock (sd);
@@ -4157,7 +4157,7 @@ void __napi_schedule(struct napi_structunsigned long flags;local_irq_save (flags); -____napi_schedule (&__get_cpu_var (softnet_data), n); +____napi_schedule (this_cpu_ptr (&softnet_data), n);local_irq_restore (flags); }EXPORT_SYMBOL (__napi_schedule);
@@ -4285,7 +4285,7 @@ EXPORT_SYMBOL(netif_napi_del);static void net_rx_action (struct softirq_action *h) { -struct softnet_data *sd = &__get_cpu_var (softnet_data); +struct softnet_data *sd =this_cpu_ptr (&softnet_data);unsigned long time_limit = jiffies +2 ;int budget = netdev_budget;void *have;
net/core/drop_monitor.c +1 -1
@@ -147,7 +147,7 @@ static void trace_drop_common(struct sk_unsigned long flags;local_irq_save (flags); - data = &__get_cpu_var (dm_cpu_data); + data =this_cpu_ptr (&dm_cpu_data);spin_lock (&data->lock); dskb = data->skb;
net/core/skbuff.c +1 -1
@@ -371,7 +371,7 @@ static void *__netdev_alloc_frag(unsigneunsigned long flags;local_irq_save (flags); - nc = &__get_cpu_var (netdev_alloc_cache); + nc =this_cpu_ptr (&netdev_alloc_cache);if (unlikely (!nc->frag.page)) { refill:for (order =NETDEV_FRAG_PAGE_MAX_ORDER ; ;) {
net/ipv4/tcp_output.c +1 -1
@@ -815,7 +815,7 @@ void tcp_wfree(struct sk_buff *skb)/* queue this socket to tasklet queue */ local_irq_save (flags); - tsq = &__get_cpu_var (tsq_tasklet); + tsq =this_cpu_ptr (&tsq_tasklet);list_add (&tp->tsq_node, &tsq->head);tasklet_schedule (&tsq->tasklet);local_irq_restore (flags);
net/ipv6/syncookies.c +1 -1
@@ -67,7 +67,7 @@ static u32 cookie_hash(const struct in6_net_get_random_once (syncookie6_secret,sizeof (syncookie6_secret)); - tmp =__get_cpu_var (ipv6_cookie_scratch); + tmp =this_cpu_ptr (ipv6_cookie_scratch);/* * we have 320 bits of information to hash, copy in the remaining
net/rds/ib_rdma.c +1 -1
@@ -267,7 +267,7 @@ static inline struct rds_ib_mr *rds_ib_runsigned long *flag;preempt_disable (); - flag = &__get_cpu_var (clean_list_grace); + flag =this_cpu_ptr (&clean_list_grace);set_bit (CLEAN_LIST_BUSY_BIT , flag); ret =llist_del_first (&pool->clean_list);if (ret)
include/net/netfilter/nf_conntrack.h +1 -1
@@ -235,7 +235,7 @@ extern s32 (*nf_ct_nat_offset)(const str DECLARE_PER_CPU(struct nf_conn, nf_conntrack_untracked);static inline struct nf_conn *nf_ct_untracked_get (void ) { -return &__raw_get_cpu_var (nf_conntrack_untracked); +return raw_cpu_ptr (&nf_conntrack_untracked); }void nf_ct_untracked_status_or (unsigned long bits);
include/net/snmp.h +2 -2
@@ -170,7 +170,7 @@ struct linux_xfrm_mib {#define SNMP_ADD_STATS64_BH (mib, field, addend) \ do { \ - __typeof__(*mib[0]) *ptr = __this_cpu_ptr((mib)[0]); \ + __typeof__(*mib[0]) *ptr = raw_cpu_ptr((mib)[0]); \ u64_stats_update_begin(&ptr->syncp); \ ptr->mibs[field] += addend; \ u64_stats_update_end(&ptr->syncp); \
@@ -192,7 +192,7 @@ struct linux_xfrm_mib {#define SNMP_UPD_PO_STATS64_BH (mib, basefield, addend) \ do { \ __typeof__(*mib[0]) *ptr; \ - ptr = __this_cpu_ptr((mib)[0]); \ + ptr = raw_cpu_ptr((mib)[0]); \ u64_stats_update_begin(&ptr->syncp); \ ptr->mibs[basefield##PKTS]++; \ ptr->mibs[basefield##OCTETS] += addend; \
net/ipv4/route.c +2 -2
@@ -1306,7 +1306,7 @@ static bool rt_cache_route(struct fib_nhif (rt_is_input_route (rt)) { p = (struct rtable **)&nh->nh_rth_input; }else { - p = (struct rtable **)__this_cpu_ptr (nh->nh_pcpu_rth_output); + p = (struct rtable **)raw_cpu_ptr (nh->nh_pcpu_rth_output); } orig = *p;
@@ -1932,7 +1932,7 @@ static struct rtable *__mkroute_output(c do_cache = false; goto add; } - prth =__this_cpu_ptr (nh->nh_pcpu_rth_output); + prth =raw_cpu_ptr (nh->nh_pcpu_rth_output); } rth =rcu_dereference (*prth);if (rt_cache_valid (rth)) {
net/ipv4/tcp.c +1 -1
@@ -2981,7 +2981,7 @@ struct tcp_md5sig_pool *tcp_get_md5sig_plocal_bh_disable (); p =ACCESS_ONCE (tcp_md5sig_pool);if (p) -return __this_cpu_ptr (p); +return raw_cpu_ptr (p);local_bh_enable ();return NULL ;
net/ipv4/syncookies.c +1 -1
@@ -40,7 +40,7 @@ static u32 cookie_hash(__be32 saddr, __bnet_get_random_once (syncookie_secret,sizeof (syncookie_secret)); - tmp =__get_cpu_var (ipv4_cookie_scratch); + tmp =this_cpu_ptr (ipv4_cookie_scratch);memcpy (tmp +4 , syncookie_secret[c],sizeof (syncookie_secret[c])); tmp[0 ] = (__force u32 )saddr; tmp[1 ] = (__force u32 )daddr;
PATCH 23/41 watchdog: Replace __raw_get_cpu_var uses
In reply to Christoph LameterMost of these are the uses of &__raw_get_cpu_var for address calculation.
touch_softlockup_watchdog_sync() uses __raw_get_cpu_var to write to per cpu variables. Use __this_cpu_write instead.
kernel/watchdog.c +5 -5
@@ -174,8 +174,8 @@ EXPORT_SYMBOL(touch_nmi_watchdog);void touch_softlockup_watchdog_sync (void ) { -__raw_get_cpu_var (softlockup_touch_sync) = true; -__raw_get_cpu_var (watchdog_touch_ts) =0 ; +__this_cpu_write (softlockup_touch_sync, true); +__this_cpu_write (watchdog_touch_ts,0 ); }#ifdef CONFIG_HARDLOCKUP_DETECTOR
@@ -341,7 +341,7 @@ static void watchdog_set_prio(unsigned istatic void watchdog_enable (unsigned int cpu) { -struct hrtimer *hrtimer = &__raw_get_cpu_var (watchdog_hrtimer); +struct hrtimer *hrtimer =raw_cpu_ptr (&watchdog_hrtimer);/* kick off the timer for the hardlockup detector */ hrtimer_init (hrtimer,CLOCK_MONOTONIC ,HRTIMER_MODE_REL );
@@ -361,7 +361,7 @@ static void watchdog_enable(unsigned intstatic void watchdog_disable (unsigned int cpu) { -struct hrtimer *hrtimer = &__raw_get_cpu_var (watchdog_hrtimer); +struct hrtimer *hrtimer =raw_cpu_ptr (&watchdog_hrtimer);watchdog_set_prio (SCHED_NORMAL ,0 );hrtimer_cancel (hrtimer);
@@ -488,7 +488,7 @@ static struct smp_hotplug_thread watchdostatic void restart_watchdog_hrtimer (void *info) { -struct hrtimer *hrtimer = &__raw_get_cpu_var (watchdog_hrtimer); +struct hrtimer *hrtimer =raw_cpu_ptr (&watchdog_hrtimer);int ret;/*
PATCH 22/41 rcu: Replace __this_cpu_ptr uses with raw_cpu_ptr
In reply to Christoph Lameter__this_cpu_ptr is being phased out.
One special case is increment_cpu_stall_ticks(). A per cpu variable is incremented so use raw_cpu_inc().
kernel/rcu/tree.c +3 -3
@@ -1904,7 +1904,7 @@ rcu_send_cbs_to_orphanage(int cpu, strucstatic void rcu_adopt_orphan_cbs (struct rcu_state *rsp) {int i; -struct rcu_data *rdp =__this_cpu_ptr (rsp->rda); +struct rcu_data *rdp =raw_cpu_ptr (rsp->rda);/* No-CBs CPUs are handled specially. */ if (rcu_nocb_adopt_orphan_cbs (rsp, rdp))
@@ -2283,7 +2283,7 @@ static void __rcu_process_callbacks(struct rcu_state *rsp) {unsigned long flags; -struct rcu_data *rdp =__this_cpu_ptr (rsp->rda); +struct rcu_data *rdp =raw_cpu_ptr (rsp->rda);WARN_ON_ONCE (rdp->beenonline ==0 );
@@ -2871,7 +2871,7 @@ static void rcu_barrier_callback(structstatic void rcu_barrier_func (void *type) {struct rcu_state *rsp = type; -struct rcu_data *rdp =__this_cpu_ptr (rsp->rda); +struct rcu_data *rdp =raw_cpu_ptr (rsp->rda);_rcu_barrier_trace (rsp,"IRQ" ,-1 , rsp->n_barrier_done);atomic_inc (&rsp->barrier_cpu_count);
kernel/rcu/tree_plugin.h +2 -2
@@ -1830,7 +1830,7 @@ static void rcu_oom_notify_cpu(void *unustruct rcu_data *rdp;for_each_rcu_flavor (rsp) { - rdp =__this_cpu_ptr (rsp->rda); + rdp =raw_cpu_ptr (rsp->rda);if (rdp->qlen_lazy !=0 ) {atomic_inc (&oom_callback_count); rsp->call (&rdp->oom_head, rcu_oom_callback);
@@ -1971,7 +1971,7 @@ static void increment_cpu_stall_ticks(vostruct rcu_state *rsp;for_each_rcu_flavor (rsp) - __this_cpu_ptr(rsp->rda)->ticks_this_gp++; + raw_cpu_inc(rsp->rda->ticks_this_gp); }#else /* #ifdef CONFIG_RCU_CPU_STALL_INFO */
PATCH 17/41 net:i Replace __this_cpu_inc in route.c with raw_cpu_inc
In reply to Christoph LameterThe RT_CACHE_STAT_INC macro triggers the new preemption checks for __this_cpu ops.
I do not see any other synchronization that would allow the use of a __this_cpu operation here however in commit dbd2915ce87e811165da0717f8e159276ebb803e Andrew justifies the use of raw_smp_processor_id() here because "we do not care" about races. In the past we agreed that the price of disabling interrupts here to get consistent counters would be too high. These counters may be inaccurate due to race conditions.
The use of __this_cpu op improves the situation already from what commit dbd2915ce87e811165da0717f8e159276ebb803e did since the single instruction emitted on x86 does not allow the race to occur anymore. However, non x86 platforms could still experience a race here.
Trace:
[ 1277.189084] __this_cpu_add operation in preemptible [00000000] code: avahi-daemon/1193 [ 1277.189085] caller is __this_cpu_preempt_check+0x38/0x60 [ 1277.189086] CPU: 1 PID: 1193 Comm: avahi-daemon Tainted: GF 3.12.0-rc4+ #187 [ 1277.189087] Hardware name: FUJITSU CELSIUS W530 Power/D3227-A1, BIOS V4.6.5.4 R1.10.0 for D3227-A1x 09/16/2013 [ 1277.189088] 0000000000000001 ffff8807ef78fa00 ffffffff816d5a57 ffff8807ef78ffd8 [ 1277.189089] ffff8807ef78fa30 ffffffff8137359c ffff8807ef78fba0 ffff88079f822b40 [ 1277.189091] 0000000020000000 ffff8807ee32c800 ffff8807ef78fa70 ffffffff813735f8 [ 1277.189093] Call Trace: [ 1277.189094] [<ffffffff816d5a57>] dump_stack+0x4e/0x82 [ 1277.189096] [<ffffffff8137359c>] check_preemption_disabled+0xec/0x110 [ 1277.189097] [<ffffffff813735f8>] __this_cpu_preempt_check+0x38/0x60 [ 1277.189098] [<ffffffff81610d65>] __ip_route_output_key+0x575/0x8c0 [ 1277.189100] [<ffffffff816110d7>] ip_route_output_flow+0x27/0x70 [ 1277.189101] [<ffffffff81616c80>] ? ip_copy_metadata+0x1a0/0x1a0 [ 1277.189102] [<ffffffff81640b15>] udp_sendmsg+0x825/0xa20 [ 1277.189104] [<ffffffff811b4aa9>] ? do_sys_poll+0x449/0x5d0 [ 1277.189105] [<ffffffff8164c695>] inet_sendmsg+0x85/0xc0 [ 1277.189106] [<ffffffff815c6e3c>] sock_sendmsg+0x9c/0xd0 [ 1277.189108] [<ffffffff813735f8>] ? __this_cpu_preempt_check+0x38/0x60 [ 1277.189109] [<ffffffff815c7550>] ? move_addr_to_kernel+0x40/0xa0 [ 1277.189111] [<ffffffff815c71ec>] ___sys_sendmsg+0x37c/0x390 [ 1277.189112] [<ffffffff8136613a>] ? string.isra.3+0x3a/0xd0 [ 1277.189113] [<ffffffff8136613a>] ? string.isra.3+0x3a/0xd0 [ 1277.189115] [<ffffffff81367b54>] ? vsnprintf+0x364/0x650 [ 1277.189116] [<ffffffff81367ee9>] ? snprintf+0x39/0x40 [ 1277.189118] [<ffffffff813735f8>] ? __this_cpu_preempt_check+0x38/0x60 [ 1277.189119] [<ffffffff815c7ff9>] __sys_sendmsg+0x49/0x90 [ 1277.189121] [<ffffffff815c8052>] SyS_sendmsg+0x12/0x20 [ 1277.189122] [<ffffffff816e4fd3>] tracesys+0xe1/0xe6
net/ipv4/route.c +1 -1
@@ -197,7 +197,7 @@ const __u8 ip_tos2prio[16] = {EXPORT_SYMBOL (ip_tos2prio); static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat); -#define RT_CACHE_STAT_INC (field) __this_cpu_inc(rt_cache_stat.field) +#define RT_CACHE_STAT_INC (field) raw_cpu_inc(rt_cache_stat.field)#ifdef CONFIG_PROC_FS static void *rt_cache_seq_start (struct seq_file *seq,loff_t *pos)
PATCH 21/41 block: Replace __this_cpu_ptr with raw_cpu_ptr
In reply to Christoph Lameter__this_cpu_ptr is being phased out.
fs/ext4/mballoc.c +1 -1
@@ -4085,7 +4085,7 @@ static void ext4_mb_group_or_file(struct* per cpu locality group is to reduce the contention between block * request from multiple CPUs. */ - ac->ac_lg =__this_cpu_ptr (sbi->s_locality_groups); + ac->ac_lg =raw_cpu_ptr (sbi->s_locality_groups);/* we're going to use group allocation */ ac->ac_flags |=EXT4_MB_HINT_GROUP_ALLOC ;
fs/buffer.c +1 -1
@@ -1312,7 +1312,7 @@ static void bh_lru_install(struct buffer }while (out <BH_LRU_SIZE ) bhs[out++] =NULL ; -memcpy (__this_cpu_ptr (&bh_lrus.bhs), bhs,sizeof (bhs)); +memcpy (this_cpu_ptr (&bh_lrus.bhs), bhs,sizeof (bhs)); }bh_lru_unlock ();
PATCH 20/41 scheduler: Replace __get_cpu_var with this_cpu_ptr
In reply to Christoph LameterConvert all uses of __get_cpu_var for address calculation to use this_cpu_ptr instead.
include/linux/kernel_stat.h +2 -2
@@ -44,8 +44,8 @@ DECLARE_PER_CPU(struct kernel_stat, ksta DECLARE_PER_CPU(struct kernel_cpustat, kernel_cpustat);/* Must have preemption disabled for this to be meaningful. */ -#define kstat_this_cpu (&__get_cpu_var(kstat)) -#define kcpustat_this_cpu (&__get_cpu_var(kernel_cpustat)) +#define kstat_this_cpu this_cpu_ptr(&kstat) +#define kcpustat_this_cpu this_cpu_ptr(&kernel_cpustat)#define kstat_cpu (cpu) per_cpu(kstat, cpu)#define kcpustat_cpu (cpu) per_cpu(kernel_cpustat, cpu)
kernel/events/callchain.c +2 -2
@@ -137,7 +137,7 @@ static struct perf_callchain_entry *get_int cpu;struct callchain_cpus_entries *entries; - *rctx =get_recursion_context (__get_cpu_var (callchain_recursion)); + *rctx =get_recursion_context (this_cpu_ptr (callchain_recursion));if (*rctx ==-1 )return NULL ;
@@ -153,7 +153,7 @@ static struct perf_callchain_entry *get_static void put_callchain_entry (int rctx) { -put_recursion_context (__get_cpu_var (callchain_recursion), rctx); +put_recursion_context (this_cpu_ptr (callchain_recursion), rctx); } struct perf_callchain_entry *
kernel/events/core.c +11 -11
@@ -240,10 +240,10 @@ void perf_sample_event_took(u64 sample_lreturn ;/* decay the counter by 1 average sample */ - local_samples_len =__get_cpu_var (running_sample_length); + local_samples_len =__this_cpu_read (running_sample_length); local_samples_len -= local_samples_len/NR_ACCUMULATED_SAMPLES ; local_samples_len += sample_len_ns; -__get_cpu_var (running_sample_length) = local_samples_len; +__this_cpu_write (running_sample_length, local_samples_len);/* * note: this will be biased artifically low until we have
@@ -869,7 +869,7 @@ static DEFINE_PER_CPU(struct list_head,static void perf_pmu_rotate_start (struct pmu *pmu) {struct perf_cpu_context *cpuctx =this_cpu_ptr (pmu->pmu_cpu_context); -struct list_head *head = &__get_cpu_var (rotation_list); +struct list_head *head =this_cpu_ptr (&rotation_list);WARN_ON (!irqs_disabled ());
@@ -2354,7 +2354,7 @@ void __perf_event_task_sched_out(struct* to check if we have to switch out PMU state. * cgroup event are system-wide mode only */ -if (atomic_read (&__get_cpu_var (perf_cgroup_events))) +if (atomic_read (this_cpu_ptr (&perf_cgroup_events)))perf_cgroup_sched_out (task, next); }
@@ -2599,11 +2599,11 @@ void __perf_event_task_sched_in(struct t* to check if we have to switch in PMU state. * cgroup event are system-wide mode only */ -if (atomic_read (&__get_cpu_var (perf_cgroup_events))) +if (atomic_read (this_cpu_ptr (&perf_cgroup_events)))perf_cgroup_sched_in (prev, task);/* check for system-wide branch_stack events */ -if (atomic_read (&__get_cpu_var (perf_branch_stack_events))) +if (atomic_read (this_cpu_ptr (&perf_branch_stack_events)))perf_branch_stack_sched_in (prev, task); }
@@ -2854,7 +2854,7 @@ bool perf_event_can_stop_tick(void)void perf_event_task_tick (void ) { -struct list_head *head = &__get_cpu_var (rotation_list); +struct list_head *head =this_cpu_ptr (&rotation_list);struct perf_cpu_context *cpuctx, *tmp;struct perf_event_context *ctx;int throttled;
@@ -5554,7 +5554,7 @@ static void do_perf_sw_event(enum perf_tstruct perf_sample_data *data, struct pt_regs *regs) { - struct swevent_htable *swhash = &__get_cpu_var(swevent_htable); + struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);struct perf_event *event;struct hlist_head *head;
@@ -5573,7 +5573,7 @@ end:int perf_swevent_get_recursion_context (void ) { -struct swevent_htable *swhash = &__get_cpu_var (swevent_htable); +struct swevent_htable *swhash =this_cpu_ptr (&swevent_htable);return get_recursion_context (swhash->recursion); }
@@ -5581,7 +5581,7 @@ EXPORT_SYMBOL_GPL(perf_swevent_get_recurinline void perf_swevent_put_recursion_context (int rctx) { -struct swevent_htable *swhash = &__get_cpu_var (swevent_htable); +struct swevent_htable *swhash =this_cpu_ptr (&swevent_htable);put_recursion_context (swhash->recursion, rctx); }
@@ -5610,7 +5610,7 @@ static void perf_swevent_read(struct perstatic int perf_swevent_add (struct perf_event *event,int flags) { -struct swevent_htable *swhash = &__get_cpu_var (swevent_htable); +struct swevent_htable *swhash =this_cpu_ptr (&swevent_htable);struct hw_perf_event *hwc = &event->hw;struct hlist_head *head;
kernel/sched/fair.c +1 -1
@@ -6146,7 +6146,7 @@ static int load_balance(int this_cpu, ststruct sched_group *group;struct rq *busiest;unsigned long flags; -struct cpumask *cpus =__get_cpu_var (load_balance_mask); +struct cpumask *cpus =this_cpu_ptr (load_balance_mask);struct lb_env env = { .sd = sd,
kernel/sched/rt.c +1 -1
@@ -1387,7 +1387,7 @@ static DEFINE_PER_CPU(cpumask_var_t, locstatic int find_lowest_rq (struct task_struct *task) {struct sched_domain *sd; -struct cpumask *lowest_mask =__get_cpu_var (local_cpu_mask); +struct cpumask *lowest_mask =this_cpu_ptr (local_cpu_mask);int this_cpu =smp_processor_id ();int cpu =task_cpu (task);
kernel/sched/sched.h +2 -2
@@ -545,10 +545,10 @@ static inline int cpu_of(struct rq *rq) DECLARE_PER_CPU(struct rq, runqueues);#define cpu_rq (cpu) (&per_cpu(runqueues, (cpu))) -#define this_rq () (&__get_cpu_var(runqueues)) +#define this_rq () this_cpu_ptr(&runqueues)#define task_rq (p) cpu_rq(task_cpu(p))#define cpu_curr (cpu) (cpu_rq(cpu)->curr) -#define raw_rq () (&__raw_get_cpu_var(runqueues)) +#define raw_rq () raw_cpu_ptr(&runqueues)static inline u64 rq_clock (struct rq *rq) {
kernel/user-return-notifier.c +2 -2
@@ -14,7 +14,7 @@ static DEFINE_PER_CPU(struct hlist_head,void user_return_notifier_register (struct user_return_notifier *urn) {set_tsk_thread_flag (current,TIF_USER_RETURN_NOTIFY ); -hlist_add_head (&urn->link, &__get_cpu_var (return_notifier_list)); +hlist_add_head (&urn->link,this_cpu_ptr (&return_notifier_list)); }EXPORT_SYMBOL_GPL (user_return_notifier_register);
@@ -25,7 +25,7 @@ EXPORT_SYMBOL_GPL(user_return_notifier_rvoid user_return_notifier_unregister (struct user_return_notifier *urn) {hlist_del (&urn->link); -if (hlist_empty (&__get_cpu_var (return_notifier_list))) +if (hlist_empty (this_cpu_ptr (&return_notifier_list)))clear_tsk_thread_flag (current,TIF_USER_RETURN_NOTIFY ); }EXPORT_SYMBOL_GPL (user_return_notifier_unregister);
kernel/taskstats.c +1 -1
@@ -638,7 +638,7 @@ void taskstats_exit(struct task_struct *fill_tgid_exit (tsk); } - listeners =__this_cpu_ptr (&listener_array); + listeners =raw_cpu_ptr (&listener_array);if (list_empty (&listeners->list))return ;
PATCH 19/41 time: Replace __get_cpu_var uses
In reply to Christoph LameterConvert uses of __get_cpu_var for creating a address from a percpu offset to this_cpu_ptr.
The two cases where get_cpu_var is used to actually access a percpu variable are changed to use this_cpu_read/raw_cpu_read.
kernel/hrtimer.c +11 -11
@@ -597,7 +597,7 @@ hrtimer_force_reprogram(struct hrtimer_cstatic int hrtimer_reprogram (struct hrtimer *timer,struct hrtimer_clock_base *base) { -struct hrtimer_cpu_base *cpu_base = &__get_cpu_var (hrtimer_bases); +struct hrtimer_cpu_base *cpu_base =this_cpu_ptr (&hrtimer_bases);ktime_t expires =ktime_sub (hrtimer_get_expires (timer), base->offset);int res;
@@ -680,7 +680,7 @@ static inline ktime_t hrtimer_update_bas*/ static void retrigger_next_event (void *arg) { -struct hrtimer_cpu_base *base = &__get_cpu_var (hrtimer_bases); +struct hrtimer_cpu_base *base =this_cpu_ptr (&hrtimer_bases);if (!hrtimer_hres_active ())return ;
@@ -954,7 +954,7 @@ remove_hrtimer(struct hrtimer *timer, st*/ debug_deactivate (timer);timer_stats_hrtimer_clear_start_info (timer); - reprogram = base->cpu_base == &__get_cpu_var (hrtimer_bases); + reprogram = base->cpu_base ==this_cpu_ptr (&hrtimer_bases);/* * We must preserve the CALLBACK state flag here, * otherwise we could move the timer base in
@@ -1009,7 +1009,7 @@ int __hrtimer_start_range_ns(struct hrti* * XXX send_remote_softirq() ? */ -if (leftmost && new_base->cpu_base == &__get_cpu_var (hrtimer_bases) +if (leftmost && new_base->cpu_base ==this_cpu_ptr (&hrtimer_bases) &&hrtimer_enqueue_reprogram (timer, new_base)) {if (wakeup) {/*
@@ -1142,7 +1142,7 @@ EXPORT_SYMBOL_GPL(hrtimer_get_remaining)*/ ktime_t hrtimer_get_next_event (void ) { -struct hrtimer_cpu_base *cpu_base = &__get_cpu_var (hrtimer_bases); +struct hrtimer_cpu_base *cpu_base =this_cpu_ptr (&hrtimer_bases);struct hrtimer_clock_base *base = cpu_base->clock_base;ktime_t delta, mindelta = { .tv64 =KTIME_MAX };unsigned long flags;
@@ -1183,7 +1183,7 @@ static void __hrtimer_init(struct hrtimememset (timer,0 ,sizeof (struct hrtimer )); - cpu_base = &__raw_get_cpu_var (hrtimer_bases); + cpu_base =raw_cpu_ptr (&hrtimer_bases);if (clock_id ==CLOCK_REALTIME && mode !=HRTIMER_MODE_ABS ) clock_id =CLOCK_MONOTONIC ;
@@ -1226,7 +1226,7 @@ int hrtimer_get_res(const clockid_t whicstruct hrtimer_cpu_base *cpu_base;int base =hrtimer_clockid_to_base (which_clock); - cpu_base = &__raw_get_cpu_var (hrtimer_bases); + cpu_base =raw_cpu_ptr (&hrtimer_bases); *tp =ktime_to_timespec (cpu_base->clock_base[base].resolution);return 0 ;
@@ -1281,7 +1281,7 @@ static void __run_hrtimer(struct hrtimer*/ void hrtimer_interrupt (struct clock_event_device *dev) { -struct hrtimer_cpu_base *cpu_base = &__get_cpu_var (hrtimer_bases); +struct hrtimer_cpu_base *cpu_base =this_cpu_ptr (&hrtimer_bases);ktime_t expires_next, now, entry_time, delta;int i, retries =0 ;
@@ -1415,7 +1415,7 @@ static void __hrtimer_peek_ahead_timers(if (!hrtimer_hres_active ())return ; - td = &__get_cpu_var (tick_cpu_device); + td =this_cpu_ptr (&tick_cpu_device);if (td && td->evtdev)hrtimer_interrupt (td->evtdev); }
@@ -1479,7 +1479,7 @@ void hrtimer_run_pending(void)void hrtimer_run_queues (void ) {struct timerqueue_node *node; -struct hrtimer_cpu_base *cpu_base = &__get_cpu_var (hrtimer_bases); +struct hrtimer_cpu_base *cpu_base =this_cpu_ptr (&hrtimer_bases);struct hrtimer_clock_base *base;int index, gettime =1 ;
@@ -1717,7 +1717,7 @@ static void migrate_hrtimers(int scpu)local_irq_disable (); old_base = &per_cpu (hrtimer_bases, scpu); - new_base = &__get_cpu_var (hrtimer_bases); + new_base =this_cpu_ptr (&hrtimer_bases);/* * The caller is globally serialized and nobody else * takes two locks at once, deadlock is not possible.
kernel/irq_work.c +3 -3
@@ -70,7 +70,7 @@ void irq_work_queue(struct irq_work *wor/* Queue the entry and raise the IPI if needed. */ preempt_disable (); -llist_add (&work->llnode, &__get_cpu_var (irq_work_list)); +llist_add (&work->llnode,this_cpu_ptr (&irq_work_list));/* * If the work is not "lazy" or the tick is stopped, raise the irq
@@ -90,7 +90,7 @@ bool irq_work_needs_cpu(void) {struct llist_head *this_list; - this_list = &__get_cpu_var (irq_work_list); + this_list =this_cpu_ptr (&irq_work_list);if (llist_empty (this_list))return false;
@@ -115,7 +115,7 @@ static void __irq_work_run(void)__this_cpu_write (irq_work_raised,0 );barrier (); - this_list = &__get_cpu_var (irq_work_list); + this_list =this_cpu_ptr (&irq_work_list);if (llist_empty (this_list))return ;
kernel/sched/clock.c +1 -1
@@ -94,7 +94,7 @@ static DEFINE_PER_CPU_SHARED_ALIGNED(strstatic inline struct sched_clock_data *this_scd (void ) { -return &__get_cpu_var (sched_clock_data); +return this_cpu_ptr (&sched_clock_data); }static inline struct sched_clock_data *cpu_sdc (int cpu)
kernel/softirq.c +2 -2
@@ -482,7 +482,7 @@ static void tasklet_action(struct softirlocal_irq_disable (); list =__this_cpu_read (tasklet_vec.head);__this_cpu_write (tasklet_vec.head,NULL ); -__this_cpu_write (tasklet_vec.tail, &__get_cpu_var (tasklet_vec).head); +__this_cpu_write (tasklet_vec.tail,this_cpu_ptr (&tasklet_vec.head));local_irq_enable ();while (list) {
@@ -517,7 +517,7 @@ static void tasklet_hi_action(struct soflocal_irq_disable (); list =__this_cpu_read (tasklet_hi_vec.head);__this_cpu_write (tasklet_hi_vec.head,NULL ); -__this_cpu_write (tasklet_hi_vec.tail, &__get_cpu_var (tasklet_hi_vec).head); +__this_cpu_write (tasklet_hi_vec.tail,this_cpu_ptr (&tasklet_hi_vec.head));local_irq_enable ();while (list) {
kernel/time/tick-common.c +3 -3
@@ -226,7 +226,7 @@ int tick_get_housekeeping_cpu(void)void tick_install_replacement (struct clock_event_device *newdev) { -struct tick_device *td = &__get_cpu_var (tick_cpu_device); +struct tick_device *td =this_cpu_ptr (&tick_cpu_device);int cpu =smp_processor_id ();clockevents_exchange_device (td->evtdev, newdev);
@@ -376,14 +376,14 @@ void tick_shutdown(unsigned int *cpup)void tick_suspend (void ) { -struct tick_device *td = &__get_cpu_var (tick_cpu_device); +struct tick_device *td =this_cpu_ptr (&tick_cpu_device);clockevents_shutdown (td->evtdev); }void tick_resume (void ) { -struct tick_device *td = &__get_cpu_var (tick_cpu_device); +struct tick_device *td =this_cpu_ptr (&tick_cpu_device);int broadcast =tick_resume_broadcast ();clockevents_set_mode (td->evtdev,CLOCK_EVT_MODE_RESUME );
kernel/time/tick-oneshot.c +1 -1
@@ -59,7 +59,7 @@ void tick_setup_oneshot(struct clock_eve*/ int tick_switch_to_oneshot (void (*handler)(struct clock_event_device *)) { -struct tick_device *td = &__get_cpu_var (tick_cpu_device); +struct tick_device *td =this_cpu_ptr (&tick_cpu_device);struct clock_event_device *dev = td->evtdev; if (!dev || !(dev->features & CLOCK_EVT_FEAT_ONESHOT) ||
kernel/time/tick-sched.c +11 -11
@@ -200,7 +200,7 @@ static void tick_nohz_restart_sched_tick*/ void __tick_nohz_full_check (void ) { -struct tick_sched *ts = &__get_cpu_var (tick_cpu_sched); +struct tick_sched *ts =this_cpu_ptr (&tick_cpu_sched);if (tick_nohz_full_cpu (smp_processor_id ())) {if (ts->tick_stopped && !is_idle_task (current)) {
@@ -226,7 +226,7 @@ static DEFINE_PER_CPU(struct irq_work, nvoid tick_nohz_full_kick (void ) {if (tick_nohz_full_cpu (smp_processor_id ())) -irq_work_queue (&__get_cpu_var (nohz_full_kick_work)); +irq_work_queue (this_cpu_ptr (&nohz_full_kick_work)); }static void nohz_full_kick_ipi (void *info)
@@ -533,7 +533,7 @@ static ktime_t tick_nohz_stop_sched_tickunsigned long seq, last_jiffies, next_jiffies, delta_jiffies;ktime_t last_update, expires, ret = { .tv64 =0 };unsigned long rcu_delta_jiffies; -struct clock_event_device *dev =__get_cpu_var (tick_cpu_device).evtdev; +struct clock_event_device *dev =__this_cpu_read (tick_cpu_device.evtdev);u64 time_delta;/* Read jiffies and the time when jiffies were updated last */
@@ -798,7 +798,7 @@ void tick_nohz_idle_enter(void)local_irq_disable (); - ts = &__get_cpu_var (tick_cpu_sched); + ts =this_cpu_ptr (&tick_cpu_sched);/* * set ts->inidle unconditionally. even if the system did not * switch to nohz mode the cpu frequency governers rely on the
@@ -821,7 +821,7 @@ EXPORT_SYMBOL_GPL(tick_nohz_idle_enter);*/ void tick_nohz_irq_exit (void ) { -struct tick_sched *ts = &__get_cpu_var (tick_cpu_sched); +struct tick_sched *ts =this_cpu_ptr (&tick_cpu_sched);if (ts->inidle)__tick_nohz_idle_enter (ts);
@@ -836,7 +836,7 @@ void tick_nohz_irq_exit(void)*/ ktime_t tick_nohz_get_sleep_length (void ) { -struct tick_sched *ts = &__get_cpu_var (tick_cpu_sched); +struct tick_sched *ts =this_cpu_ptr (&tick_cpu_sched);return ts->sleep_length; }
@@ -950,7 +950,7 @@ static int tick_nohz_reprogram(struct ti*/ static void tick_nohz_handler (struct clock_event_device *dev) { -struct tick_sched *ts = &__get_cpu_var (tick_cpu_sched); +struct tick_sched *ts =this_cpu_ptr (&tick_cpu_sched);struct pt_regs *regs =get_irq_regs ();ktime_t now =ktime_get ();
@@ -970,7 +970,7 @@ static void tick_nohz_handler(struct clo*/ static void tick_nohz_switch_to_nohz (void ) { -struct tick_sched *ts = &__get_cpu_var (tick_cpu_sched); +struct tick_sched *ts =this_cpu_ptr (&tick_cpu_sched);ktime_t next;if (!tick_nohz_enabled)
@@ -1108,7 +1108,7 @@ early_param("skew_tick", skew_tick);*/ void tick_setup_sched_timer (void ) { -struct tick_sched *ts = &__get_cpu_var (tick_cpu_sched); +struct tick_sched *ts =this_cpu_ptr (&tick_cpu_sched);ktime_t now =ktime_get ();/*
@@ -1175,7 +1175,7 @@ void tick_clock_notify(void)*/ void tick_oneshot_notify (void ) { -struct tick_sched *ts = &__get_cpu_var (tick_cpu_sched); +struct tick_sched *ts =this_cpu_ptr (&tick_cpu_sched);set_bit (0 , &ts->check_clocks); }
@@ -1190,7 +1190,7 @@ void tick_oneshot_notify(void)*/ int tick_check_oneshot_change (int allow_nohz) { -struct tick_sched *ts = &__get_cpu_var (tick_cpu_sched); +struct tick_sched *ts =this_cpu_ptr (&tick_cpu_sched);if (!test_and_clear_bit (0 , &ts->check_clocks))return 0 ;
kernel/timer.c +1 -1
@@ -621,7 +621,7 @@ static inline void debug_assert_init(strstatic void do_init_timer (struct timer_list *timer,unsigned int flags,const char *name,struct lock_class_key *key) { -struct tvec_base *base =__raw_get_cpu_var (tvec_bases); +struct tvec_base *base =raw_cpu_read (tvec_bases); timer->entry.next =NULL ; timer->base = (void *)((unsigned long )base | flags);
drivers/clocksource/dummy_timer.c +1 -1
@@ -28,7 +28,7 @@ static void dummy_timer_set_mode(enum clstatic void dummy_timer_setup (void ) {int cpu =smp_processor_id (); -struct clock_event_device *evt =__this_cpu_ptr (&dummy_timer_evt); +struct clock_event_device *evt =raw_cpu_ptr (&dummy_timer_evt); evt->name ="dummy_timer" ; evt->features = CLOCK_EVT_FEAT_PERIODIC |
PATCH 18/41 percpu: Add preemption checks to __this_cpu ops
In reply to Christoph LameterWe define a check function in order to avoid trouble with the include files. Then the higher level __this_cpu macros are modified to invoke the preemption check.
include/linux/percpu.h +29 -10
@@ -175,6 +175,12 @@ extern phys_addr_t per_cpu_ptr_to_phys(vextern void __bad_size_call_parameter (void ); +#ifdef CONFIG_DEBUG_PREEMPT +extern void __this_cpu_preempt_check (const char *op); +#else +static inline void __this_cpu_preempt_check (const char *op) { } +#endif +#define __pcpu_size_call_return (stem, variable) \ ({ typeof(variable) pscr_ret__; \ __verify_pcpu_ptr(&(variable)); \
@@ -727,18 +733,24 @@ do { \/* * Generic percpu operations for context that are safe from preemption/interrupts. -* Checks will be added here soon. */ #ifndef __this_cpu_read -# define __this_cpu_read (pcp) __pcpu_size_call_return(raw_cpu_read_, (pcp)) +# define __this_cpu_read (pcp) \ + (__this_cpu_preempt_check("read"),__pcpu_size_call_return(raw_cpu_read_, (pcp)))#endif #ifndef __this_cpu_write -# define __this_cpu_write (pcp, val) __pcpu_size_call(raw_cpu_write_, (pcp), (val)) +# define __this_cpu_write (pcp, val) \ +do { __this_cpu_preempt_check("write"); \ + __pcpu_size_call(raw_cpu_write_, (pcp), (val)); \ +} while (0)#endif #ifndef __this_cpu_add -# define __this_cpu_add (pcp, val) __pcpu_size_call(raw_cpu_add_, (pcp), (val)) +# define __this_cpu_add (pcp, val) \ +do { __this_cpu_preempt_check("add"); \ + __pcpu_size_call(raw_cpu_add_, (pcp), (val)); \ +} while (0)#endif #ifndef __this_cpu_sub
@@ -754,16 +766,23 @@ do { \#endif #ifndef __this_cpu_and -# define __this_cpu_and (pcp, val) __pcpu_size_call(raw_cpu_and_, (pcp), (val)) +# define __this_cpu_and (pcp, val) \ +do { __this_cpu_preempt_check("and"); \ + __pcpu_size_call(raw_cpu_and_, (pcp), (val)); \ +} while (0) +#endif #ifndef __this_cpu_or -# define __this_cpu_or (pcp, val) __pcpu_size_call(raw_cpu_or_, (pcp), (val)) +# define __this_cpu_or (pcp, val) \ +do { __this_cpu_preempt_check("or"); \ + __pcpu_size_call(raw_cpu_or_, (pcp), (val)); \ +} while (0)#endif #ifndef __this_cpu_add_return# define __this_cpu_add_return (pcp, val) \ - __pcpu_size_call_return2(raw_cpu_add_return_, pcp, val) + (__this_cpu_preempt_check("add_return"),__pcpu_size_call_return2(raw_cpu_add_return_, pcp, val))#endif #define __this_cpu_sub_return (pcp, val) __this_cpu_add_return(pcp, -(typeof(pcp))(val))
@@ -772,17 +791,17 @@ do { \#ifndef __this_cpu_xchg# define __this_cpu_xchg (pcp, nval) \ - __pcpu_size_call_return2(raw_cpu_xchg_, (pcp), nval) + (__this_cpu_preempt_check("xchg"),__pcpu_size_call_return2(raw_cpu_xchg_, (pcp), nval))#endif #ifndef __this_cpu_cmpxchg# define __this_cpu_cmpxchg (pcp, oval, nval) \ - __pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval) + (__this_cpu_preempt_check("cmpxchg"),__pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval))#endif #ifndef __this_cpu_cmpxchg_double# define __this_cpu_cmpxchg_double (pcp1, pcp2, oval1, oval2, nval1, nval2) \ - __pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2)) + (__this_cpu_preempt_check("cmpxchg_double"),__pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2)))#endif #endif /* __LINUX_PERCPU_H */
lib/smp_processor_id.c +17 -4
@@ -7,7 +7,7 @@#include <linux/kallsyms.h> #include <linux/sched.h> -notrace unsigned int debug_smp_processor_id(void) +notrace static unsigned int check_preemption_disabled(char *what) {int this_cpu =raw_smp_processor_id ();
@@ -38,9 +38,9 @@ notrace unsigned int debug_smp_processorif (!printk_ratelimit ()) goto out_enable; -printk (KERN_ERR "BUG: using smp_processor_id() in preemptible [%08x] " -"code: %s/%d\n" , -preempt_count () -1 , current->comm, current->pid); +printk (KERN_ERR "BUG: using %s in preemptible [%08x] code: %s/%d\n" , + what,preempt_count () -1 , current->comm, current->pid); +print_symbol ("caller is %s\n" , (long )__builtin_return_address (0 ));dump_stack ();
@@ -50,5 +50,18 @@ out:return this_cpu; } +notrace unsigned int debug_smp_processor_id(void) +{ +return check_preemption_disabled ("smp_processor_id()" ); +}EXPORT_SYMBOL (debug_smp_processor_id); +notrace void __this_cpu_preempt_check(const char *op) +{ +char text[40 ]; + +snprintf (text,sizeof (text),"__this_cpu_%s()" , op); +check_preemption_disabled (text); +} +EXPORT_SYMBOL (__this_cpu_preempt_check); +
PATCH 15/41 mm: Use raw_cpu ops for determining current NUMA node
In reply to Christoph LameterWith the preempt checking logic for __this_cpu_ops we will get false positives from locations in the code that use numa_node_id.
Before the __this_cpu ops where introduced there were no checks for preemption present either. smp_raw_processor_id() was used. See http://www.spinics.net/lists/linux-numa/msg00641.html
Therefore we need to use raw_cpu_read here to avoid false postives.
Note that this issue has been discussed in prior years.
If the process changes nodes after retrieving the current numa node then that is acceptable since most uses of numa_node etc are for optimization and not for correctness.
There were suggestions to implement a raw_numa_node_id in order to do preempt checks for numa_node_id as well. But I think we better defer that to another patch since that would mean investigating how numa_node_id() is used throughout the kernel which would increase the scope of this patchset significantly. After all preemption was never checked before when numa_node_id() was used.
Some sample traces:
__this_cpu_read operation in preemptible [00000000] code: login/1456 caller is __this_cpu_preempt_check+0x2b/0x2d
CPU: 0 PID: 1456 Comm: login Not tainted 3.12.0-rc4-cl-00062-g2fe80d3-dirty #185 Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011 000000000000013c ffff88001f31ba58 ffffffff8147cf5e ffff88001f31bfd8 ffff88001f31ba88 ffffffff8127eea9 0000000000000000 ffff88001f3975c0 00000000f7707000 ffff88001f3975c0 ffff88001f31bac0 ffffffff8127eeef Call Trace: [<ffffffff8147cf5e>] dump_stack+0x4e/0x82 [<ffffffff8127eea9>] check_preemption_disabled+0xc5/0xe0 [<ffffffff8127eeef>] __this_cpu_preempt_check+0x2b/0x2d [<ffffffff81030ff5>] ? show_stack+0x3b/0x3d [<ffffffff810ebee3>] get_task_policy+0x1d/0x49 [<ffffffff810ed705>] get_vma_policy+0x14/0x76 [<ffffffff810ed8ff>] alloc_pages_vma+0x35/0xff [<ffffffff810dad97>] handle_mm_fault+0x290/0x73b [<ffffffff810503da>] __do_page_fault+0x3fe/0x44d [<ffffffff8109b360>] ? trace_hardirqs_on_caller+0x142/0x19e [<ffffffff8109b3c9>] ? trace_hardirqs_on+0xd/0xf [<ffffffff81278bed>] ? trace_hardirqs_off_thunk+0x3a/0x3c [<ffffffff810be97f>] ? find_get_pages_contig+0x18e/0x18e [<ffffffff810be97f>] ? find_get_pages_contig+0x18e/0x18e [<ffffffff81050451>] do_page_fault+0x9/0xc [<ffffffff81483602>] page_fault+0x22/0x30 [<ffffffff810be97f>] ? find_get_pages_contig+0x18e/0x18e [<ffffffff810be97f>] ? find_get_pages_contig+0x18e/0x18e [<ffffffff810be4c3>] ? file_read_actor+0x3a/0x15a [<ffffffff810be97f>] ? find_get_pages_contig+0x18e/0x18e [<ffffffff810bffab>] generic_file_aio_read+0x38e/0x624 [<ffffffff810f6d69>] do_sync_read+0x54/0x73 [<ffffffff810f7890>] vfs_read+0x9d/0x12a [<ffffffff810f7a59>] SyS_read+0x47/0x7e [<ffffffff81484f21>] cstar_dispatch+0x7/0x23
caller is __this_cpu_preempt_check+0x2b/0x2d
CPU: 0 PID: 1456 Comm: login Not tainted 3.12.0-rc4-cl-00062-g2fe80d3-dirty #185 Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011 00000000000000e8 ffff88001f31bbf8 ffffffff8147cf5e ffff88001f31bfd8 ffff88001f31bc28 ffffffff8127eea9 ffffffff823c5c40 00000000000213da 0000000000000000 0000000000000000 ffff88001f31bc60 ffffffff8127eeef Call Trace: [<ffffffff8147cf5e>] dump_stack+0x4e/0x82 [<ffffffff8127eea9>] check_preemption_disabled+0xc5/0xe0 [<ffffffff8127eeef>] __this_cpu_preempt_check+0x2b/0x2d [<ffffffff810e006e>] ? install_special_mapping+0x11/0xe4 [<ffffffff810ec8a8>] alloc_pages_current+0x8f/0xbc [<ffffffff810bec6b>] __page_cache_alloc+0xb/0xd [<ffffffff810c7e90>] __do_page_cache_readahead+0xf4/0x219 [<ffffffff810c7e0e>] ? __do_page_cache_readahead+0x72/0x219 [<ffffffff810c827c>] ra_submit+0x1c/0x20 [<ffffffff810c850c>] ondemand_readahead+0x28c/0x2b4 [<ffffffff810c85e9>] page_cache_sync_readahead+0x38/0x3a [<ffffffff810bfe7e>] generic_file_aio_read+0x261/0x624 [<ffffffff810f6d69>] do_sync_read+0x54/0x73 [<ffffffff810f7890>] vfs_read+0x9d/0x12a [<ffffffff810f7a59>] SyS_read+0x47/0x7e [<ffffffff81484f21>] cstar_dispatch+0x7/0x23
include/linux/topology.h +2 -2
@@ -188,7 +188,7 @@ DECLARE_PER_CPU(int, numa_node);/* Returns the number of the current Node. */ static inline int numa_node_id (void ) { -return __this_cpu_read (numa_node); +return raw_cpu_read (numa_node); }#endif
@@ -245,7 +245,7 @@ static inline void set_numa_mem(int node/* Returns the number of the nearest Node with memory */ static inline int numa_mem_id (void ) { -return __this_cpu_read (_numa_mem_); +return raw_cpu_read (_numa_mem_); }#endif
PATCH 16/41 modules: Use raw_cpu_write for initialization of per cpu refcount.
In reply to Christoph LameterThe initialization of a structure is not subject to synchronization. The use of __this_cpu would trigger a false positive with the additional preemption checks for __this_cpu ops.
So simply disable the check through the use of raw_cpu ops.
Trace:
[ 0.668066] __this_cpu_write operation in preemptible [00000000] code: modprobe/286 [ 0.668108] caller is __this_cpu_preempt_check+0x38/0x60 [ 0.668111] CPU: 3 PID: 286 Comm: modprobe Tainted: GF 3.12.0-rc4+ #187 [ 0.668112] Hardware name: FUJITSU CELSIUS W530 Power/D3227-A1, BIOS V4.6.5.4 R1.10.0 for D3227-A1x 09/16/2013 [ 0.668113] 0000000000000003 ffff8807edda1d18 ffffffff816d5a57 ffff8807edda1fd8 [ 0.668117] ffff8807edda1d48 ffffffff8137359c ffff8807edda1ef8 ffffffffa0002178 [ 0.668121] ffffc90000067730 ffff8807edda1e48 ffff8807edda1d88 ffffffff813735f8 [ 0.668124] Call Trace: [ 0.668129] [<ffffffff816d5a57>] dump_stack+0x4e/0x82 [ 0.668132] [<ffffffff8137359c>] check_preemption_disabled+0xec/0x110 [ 0.668135] [<ffffffff813735f8>] __this_cpu_preempt_check+0x38/0x60 [ 0.668139] [<ffffffff810c24fd>] load_module+0xcfd/0x2650 [ 0.668143] [<ffffffff816dd922>] ? page_fault+0x22/0x30 [ 0.668146] [<ffffffff810c3ef6>] SyS_init_module+0xa6/0xd0 [ 0.668150] [<ffffffff816e4fd3>] tracesys+0xe1/0xe6
kernel/module.c +1 -1
@@ -640,7 +640,7 @@ static int module_unload_init(struct modINIT_LIST_HEAD (&mod->target_list);/* Hold reference count during initialization. */ -__this_cpu_write (mod->refptr->incs,1 ); +raw_cpu_write (mod->refptr->incs,1 );return 0 ; }
PATCH 13/41 percpu: Add raw_cpu_ops
In reply to Christoph LameterThe patches following this one will add preemption checks to __this_cpu ops so we need to have an alternative way to use this_cpu operations without preemption checks.
raw_cpu_ops will be the basis for all other ops since these will be the operations that do not implement any checks.
Primitive operations are renamed by this patch from __this_cpu_xxx to raw_cpu_xxxx.
include/linux/percpu.h +201 -139
@@ -139,6 +139,9 @@ extern int __init pcpu_page_first_chunk( pcpu_fc_populate_pte_fn_t populate_pte_fn);#endif +/* Keep until we have removed all uses of __this_cpu_ptr */ +#define __this_cpu_ptr raw_cpu_ptr +/* * Use this to get to a cpu's version of the per-cpu object * dynamically allocated. Non-atomic access to the current CPU's
@@ -242,6 +245,8 @@ do { \ }while (0 )/* +* this_cpu operations (C) 2008-2013 Christoph Lameter <[email protected]> +* * Optimized manipulation for memory allocated through the per cpu * allocator or for addresses of per cpu variables. *
@@ -295,7 +300,7 @@ do { \do { \unsigned long flags; \raw_local_irq_save (flags); \ - *__this_cpu_ptr(&(pcp)) op val; \ + *raw_cpu_ptr(&(pcp)) op val; \raw_local_irq_restore (flags); \ }while (0 )
@@ -380,8 +385,8 @@ do { \typeof (pcp) ret__; \unsigned long flags; \raw_local_irq_save (flags); \ -__this_cpu_add (pcp, val); \ - ret__ =__this_cpu_read (pcp); \ +raw_cpu_add (pcp, val); \ + ret__ =raw_cpu_read (pcp); \raw_local_irq_restore (flags); \ ret__; \ })
@@ -410,8 +415,8 @@ do { \ ({typeof (pcp) ret__; \unsigned long flags; \raw_local_irq_save (flags); \ - ret__ =__this_cpu_read (pcp); \ -__this_cpu_write (pcp, nval); \ + ret__ =raw_cpu_read (pcp); \ +raw_cpu_write (pcp, nval); \raw_local_irq_restore (flags); \ ret__; \ })
@@ -438,9 +443,9 @@ do { \typeof (pcp) ret__; \unsigned long flags; \raw_local_irq_save (flags); \ - ret__ =__this_cpu_read (pcp); \ + ret__ =raw_cpu_read (pcp); \if (ret__ == (oval)) \ -__this_cpu_write (pcp, nval); \ +raw_cpu_write (pcp, nval); \raw_local_irq_restore (flags); \ ret__; \ })
@@ -475,7 +480,7 @@ do { \int ret__; \unsigned long flags; \raw_local_irq_save (flags); \ - ret__ =__this_cpu_generic_cmpxchg_double (pcp1, pcp2, \ + ret__ =raw_cpu_generic_cmpxchg_double (pcp1, pcp2, \ oval1, oval2, nval1, nval2); \raw_local_irq_restore (flags); \ ret__; \
@@ -503,12 +508,8 @@ do { \#endif /* -* Generic percpu operations for context that are safe from preemption/interrupts. -* Either we do not care about races or the caller has the -* responsibility of handling preemption/interrupt issues. Arch code can still -* override these instructions since the arch per cpu code may be more -* efficient and may actually get race freeness for free (that is the -* case for x86 for example). +* Generic percpu operations for contexts where we do not want to do +* any checks for preemptiosn. * * If there is no other protection through preempt disable and/or * disabling interupts then one of these RMW operations can show unexpected
@@ -516,211 +517,272 @@ do { \* or an interrupt occurred and the same percpu variable was modified from * the interrupt context. */ -#ifndef __this_cpu_read -# ifndef __this_cpu_read_1 -# define __this_cpu_read_1 (pcp) (*__this_cpu_ptr(&(pcp))) +#ifndef raw_cpu_read +# ifndef raw_cpu_read_1 +# define raw_cpu_read_1 (pcp) (*raw_cpu_ptr(&(pcp)))# endif -# ifndef __this_cpu_read_2 -# define __this_cpu_read_2 (pcp) (*__this_cpu_ptr(&(pcp))) +# ifndef raw_cpu_read_2 +# define raw_cpu_read_2 (pcp) (*raw_cpu_ptr(&(pcp)))# endif -# ifndef __this_cpu_read_4 -# define __this_cpu_read_4 (pcp) (*__this_cpu_ptr(&(pcp))) +# ifndef raw_cpu_read_4 +# define raw_cpu_read_4 (pcp) (*raw_cpu_ptr(&(pcp)))# endif -# ifndef __this_cpu_read_8 -# define __this_cpu_read_8 (pcp) (*__this_cpu_ptr(&(pcp))) +# ifndef raw_cpu_read_8 +# define raw_cpu_read_8 (pcp) (*raw_cpu_ptr(&(pcp)))# endif -# define __this_cpu_read (pcp) __pcpu_size_call_return(__this_cpu_read_, (pcp)) +# define raw_cpu_read (pcp) __pcpu_size_call_return(raw_cpu_read_, (pcp))#endif -#define __this_cpu_generic_to_op (pcp, val, op) \ +#define raw_cpu_generic_to_op (pcp, val, op) \ do { \ - *__this_cpu_ptr(&(pcp)) op val; \ + *raw_cpu_ptr(&(pcp)) op val; \ } while (0) -#ifndef __this_cpu_write -# ifndef __this_cpu_write_1 -# define __this_cpu_write_1 (pcp, val) __this_cpu_generic_to_op((pcp), (val), =) + +#ifndef raw_cpu_write +# ifndef raw_cpu_write_1 +# define raw_cpu_write_1 (pcp, val) raw_cpu_generic_to_op((pcp), (val), =)# endif -# ifndef __this_cpu_write_2 -# define __this_cpu_write_2 (pcp, val) __this_cpu_generic_to_op((pcp), (val), =) +# ifndef raw_cpu_write_2 +# define raw_cpu_write_2 (pcp, val) raw_cpu_generic_to_op((pcp), (val), =)# endif -# ifndef __this_cpu_write_4 -# define __this_cpu_write_4 (pcp, val) __this_cpu_generic_to_op((pcp), (val), =) +# ifndef raw_cpu_write_4 +# define raw_cpu_write_4 (pcp, val) raw_cpu_generic_to_op((pcp), (val), =)# endif -# ifndef __this_cpu_write_8 -# define __this_cpu_write_8 (pcp, val) __this_cpu_generic_to_op((pcp), (val), =) +# ifndef raw_cpu_write_8 +# define raw_cpu_write_8 (pcp, val) raw_cpu_generic_to_op((pcp), (val), =)# endif -# define __this_cpu_write (pcp, val) __pcpu_size_call(__this_cpu_write_, (pcp), (val)) +# define raw_cpu_write (pcp, val) __pcpu_size_call(raw_cpu_write_, (pcp), (val))#endif -#ifndef __this_cpu_add -# ifndef __this_cpu_add_1 -# define __this_cpu_add_1 (pcp, val) __this_cpu_generic_to_op((pcp), (val), +=) +#ifndef raw_cpu_add +# ifndef raw_cpu_add_1 +# define raw_cpu_add_1 (pcp, val) raw_cpu_generic_to_op((pcp), (val), +=)# endif -# ifndef __this_cpu_add_2 -# define __this_cpu_add_2 (pcp, val) __this_cpu_generic_to_op((pcp), (val), +=) +# ifndef raw_cpu_add_2 +# define raw_cpu_add_2 (pcp, val) raw_cpu_generic_to_op((pcp), (val), +=)# endif -# ifndef __this_cpu_add_4 -# define __this_cpu_add_4 (pcp, val) __this_cpu_generic_to_op((pcp), (val), +=) +# ifndef raw_cpu_add_4 +# define raw_cpu_add_4 (pcp, val) raw_cpu_generic_to_op((pcp), (val), +=)# endif -# ifndef __this_cpu_add_8 -# define __this_cpu_add_8 (pcp, val) __this_cpu_generic_to_op((pcp), (val), +=) +# ifndef raw_cpu_add_8 +# define raw_cpu_add_8 (pcp, val) raw_cpu_generic_to_op((pcp), (val), +=)# endif -# define __this_cpu_add (pcp, val) __pcpu_size_call(__this_cpu_add_, (pcp), (val)) +# define raw_cpu_add (pcp, val) __pcpu_size_call(raw_cpu_add_, (pcp), (val))#endif -#ifndef __this_cpu_sub -# define __this_cpu_sub (pcp, val) __this_cpu_add((pcp), -(typeof(pcp))(val)) +#ifndef raw_cpu_sub +# define raw_cpu_sub (pcp, val) raw_cpu_add((pcp), -(val))#endif -#ifndef __this_cpu_inc -# define __this_cpu_inc (pcp) __this_cpu_add((pcp), 1) +#ifndef raw_cpu_inc +# define raw_cpu_inc (pcp) raw_cpu_add((pcp), 1)#endif -#ifndef __this_cpu_dec -# define __this_cpu_dec (pcp) __this_cpu_sub((pcp), 1) +#ifndef raw_cpu_dec +# define raw_cpu_dec (pcp) raw_cpu_sub((pcp), 1)#endif -#ifndef __this_cpu_and -# ifndef __this_cpu_and_1 -# define __this_cpu_and_1 (pcp, val) __this_cpu_generic_to_op((pcp), (val), &=) +#ifndef raw_cpu_and +# ifndef raw_cpu_and_1 +# define raw_cpu_and_1 (pcp, val) raw_cpu_generic_to_op((pcp), (val), &=)# endif -# ifndef __this_cpu_and_2 -# define __this_cpu_and_2 (pcp, val) __this_cpu_generic_to_op((pcp), (val), &=) +# ifndef raw_cpu_and_2 +# define raw_cpu_and_2 (pcp, val) raw_cpu_generic_to_op((pcp), (val), &=)# endif -# ifndef __this_cpu_and_4 -# define __this_cpu_and_4 (pcp, val) __this_cpu_generic_to_op((pcp), (val), &=) +# ifndef raw_cpu_and_4 +# define raw_cpu_and_4 (pcp, val) raw_cpu_generic_to_op((pcp), (val), &=)# endif -# ifndef __this_cpu_and_8 -# define __this_cpu_and_8 (pcp, val) __this_cpu_generic_to_op((pcp), (val), &=) +# ifndef raw_cpu_and_8 +# define raw_cpu_and_8 (pcp, val) raw_cpu_generic_to_op((pcp), (val), &=)# endif -# define __this_cpu_and (pcp, val) __pcpu_size_call(__this_cpu_and_, (pcp), (val)) +# define raw_cpu_and (pcp, val) __pcpu_size_call(raw_cpu_and_, (pcp), (val))#endif -#ifndef __this_cpu_or -# ifndef __this_cpu_or_1 -# define __this_cpu_or_1 (pcp, val) __this_cpu_generic_to_op((pcp), (val), |=) +#ifndef raw_cpu_or +# ifndef raw_cpu_or_1 +# define raw_cpu_or_1 (pcp, val) raw_cpu_generic_to_op((pcp), (val), |=)# endif -# ifndef __this_cpu_or_2 -# define __this_cpu_or_2 (pcp, val) __this_cpu_generic_to_op((pcp), (val), |=) +# ifndef raw_cpu_or_2 +# define raw_cpu_or_2 (pcp, val) raw_cpu_generic_to_op((pcp), (val), |=)# endif -# ifndef __this_cpu_or_4 -# define __this_cpu_or_4 (pcp, val) __this_cpu_generic_to_op((pcp), (val), |=) +# ifndef raw_cpu_or_4 +# define raw_cpu_or_4 (pcp, val) raw_cpu_generic_to_op((pcp), (val), |=)# endif -# ifndef __this_cpu_or_8 -# define __this_cpu_or_8 (pcp, val) __this_cpu_generic_to_op((pcp), (val), |=) +# ifndef raw_cpu_or_8 +# define raw_cpu_or_8 (pcp, val) raw_cpu_generic_to_op((pcp), (val), |=)# endif -# define __this_cpu_or (pcp, val) __pcpu_size_call(__this_cpu_or_, (pcp), (val)) +# define raw_cpu_or (pcp, val) __pcpu_size_call(raw_cpu_or_, (pcp), (val))#endif -#define __this_cpu_generic_add_return (pcp, val) \ +#define raw_cpu_generic_add_return (pcp, val) \ ({ \ - __this_cpu_add(pcp, val); \ - __this_cpu_read(pcp); \ + raw_cpu_add(pcp, val); \ + raw_cpu_read(pcp); \ }) -#ifndef __this_cpu_add_return -# ifndef __this_cpu_add_return_1 -# define __this_cpu_add_return_1 (pcp, val) __this_cpu_generic_add_return(pcp, val) +#ifndef raw_cpu_add_return +# ifndef raw_cpu_add_return_1 +# define raw_cpu_add_return_1 (pcp, val) raw_cpu_generic_add_return(pcp, val)# endif -# ifndef __this_cpu_add_return_2 -# define __this_cpu_add_return_2 (pcp, val) __this_cpu_generic_add_return(pcp, val) +# ifndef raw_cpu_add_return_2 +# define raw_cpu_add_return_2 (pcp, val) raw_cpu_generic_add_return(pcp, val)# endif -# ifndef __this_cpu_add_return_4 -# define __this_cpu_add_return_4 (pcp, val) __this_cpu_generic_add_return(pcp, val) +# ifndef raw_cpu_add_return_4 +# define raw_cpu_add_return_4 (pcp, val) raw_cpu_generic_add_return(pcp, val)# endif -# ifndef __this_cpu_add_return_8 -# define __this_cpu_add_return_8 (pcp, val) __this_cpu_generic_add_return(pcp, val) +# ifndef raw_cpu_add_return_8 +# define raw_cpu_add_return_8 (pcp, val) raw_cpu_generic_add_return(pcp, val)# endif -# define __this_cpu_add_return (pcp, val) \ - __pcpu_size_call_return2(__this_cpu_add_return_, pcp, val) +# define raw_cpu_add_return (pcp, val) \ + __pcpu_size_call_return2(raw_add_return_, pcp, val)#endif -#define __this_cpu_sub_return (pcp, val) __this_cpu_add_return(pcp, -(typeof(pcp))(val)) -#define __this_cpu_inc_return (pcp) __this_cpu_add_return(pcp, 1) -#define __this_cpu_dec_return (pcp) __this_cpu_add_return(pcp, -1) +#define raw_cpu_sub_return (pcp, val) raw_cpu_add_return(pcp, -(typeof(pcp))(val)) +#define raw_cpu_inc_return (pcp) raw_cpu_add_return(pcp, 1) +#define raw_cpu_dec_return (pcp) raw_cpu_add_return(pcp, -1) -#define __this_cpu_generic_xchg (pcp, nval) \ +#define raw_cpu_generic_xchg (pcp, nval) \ ({ typeof(pcp) ret__; \ - ret__ = __this_cpu_read(pcp); \ - __this_cpu_write(pcp, nval); \ + ret__ = raw_cpu_read(pcp); \ + raw_cpu_write(pcp, nval); \ ret__; \ }) -#ifndef __this_cpu_xchg -# ifndef __this_cpu_xchg_1 -# define __this_cpu_xchg_1 (pcp, nval) __this_cpu_generic_xchg(pcp, nval) +#ifndef raw_cpu_xchg +# ifndef raw_cpu_xchg_1 +# define raw_cpu_xchg_1 (pcp, nval) raw_cpu_generic_xchg(pcp, nval)# endif -# ifndef __this_cpu_xchg_2 -# define __this_cpu_xchg_2 (pcp, nval) __this_cpu_generic_xchg(pcp, nval) +# ifndef raw_cpu_xchg_2 +# define raw_cpu_xchg_2 (pcp, nval) raw_cpu_generic_xchg(pcp, nval)# endif -# ifndef __this_cpu_xchg_4 -# define __this_cpu_xchg_4 (pcp, nval) __this_cpu_generic_xchg(pcp, nval) +# ifndef raw_cpu_xchg_4 +# define raw_cpu_xchg_4 (pcp, nval) raw_cpu_generic_xchg(pcp, nval)# endif -# ifndef __this_cpu_xchg_8 -# define __this_cpu_xchg_8 (pcp, nval) __this_cpu_generic_xchg(pcp, nval) +# ifndef raw_cpu_xchg_8 +# define raw_cpu_xchg_8 (pcp, nval) raw_cpu_generic_xchg(pcp, nval)# endif -# define __this_cpu_xchg (pcp, nval) \ - __pcpu_size_call_return2(__this_cpu_xchg_, (pcp), nval) +# define raw_cpu_xchg (pcp, nval) \ + __pcpu_size_call_return2(raw_cpu_xchg_, (pcp), nval)#endif -#define __this_cpu_generic_cmpxchg (pcp, oval, nval) \ +#define raw_cpu_generic_cmpxchg (pcp, oval, nval) \ ({ \ typeof(pcp) ret__; \ - ret__ = __this_cpu_read(pcp); \ + ret__ = raw_cpu_read(pcp); \ if (ret__ == (oval)) \ - __this_cpu_write(pcp, nval); \ + raw_cpu_write(pcp, nval); \ ret__; \ }) -#ifndef __this_cpu_cmpxchg -# ifndef __this_cpu_cmpxchg_1 -# define __this_cpu_cmpxchg_1 (pcp, oval, nval) __this_cpu_generic_cmpxchg(pcp, oval, nval) +#ifndef raw_cpu_cmpxchg +# ifndef raw_cpu_cmpxchg_1 +# define raw_cpu_cmpxchg_1 (pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval)# endif -# ifndef __this_cpu_cmpxchg_2 -# define __this_cpu_cmpxchg_2 (pcp, oval, nval) __this_cpu_generic_cmpxchg(pcp, oval, nval) +# ifndef raw_cpu_cmpxchg_2 +# define raw_cpu_cmpxchg_2 (pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval)# endif -# ifndef __this_cpu_cmpxchg_4 -# define __this_cpu_cmpxchg_4 (pcp, oval, nval) __this_cpu_generic_cmpxchg(pcp, oval, nval) +# ifndef raw_cpu_cmpxchg_4 +# define raw_cpu_cmpxchg_4 (pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval)# endif -# ifndef __this_cpu_cmpxchg_8 -# define __this_cpu_cmpxchg_8 (pcp, oval, nval) __this_cpu_generic_cmpxchg(pcp, oval, nval) +# ifndef raw_cpu_cmpxchg_8 +# define raw_cpu_cmpxchg_8 (pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval)# endif -# define __this_cpu_cmpxchg (pcp, oval, nval) \ - __pcpu_size_call_return2(__this_cpu_cmpxchg_, pcp, oval, nval) +# define raw_cpu_cmpxchg (pcp, oval, nval) \ + __pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval)#endif -#define __this_cpu_generic_cmpxchg_double (pcp1, pcp2, oval1, oval2, nval1, nval2) \ +#define raw_cpu_generic_cmpxchg_double (pcp1, pcp2, oval1, oval2, nval1, nval2) \ ({ \ int __ret = 0; \ - if (__this_cpu_read(pcp1) == (oval1) && \ - __this_cpu_read(pcp2) == (oval2)) { \ - __this_cpu_write(pcp1, (nval1)); \ - __this_cpu_write(pcp2, (nval2)); \ + if (raw_cpu_read(pcp1) == (oval1) && \ + raw_cpu_read(pcp2) == (oval2)) { \ + raw_cpu_write(pcp1, (nval1)); \ + raw_cpu_write(pcp2, (nval2)); \ __ret = 1; \ } \ (__ret); \ }) -#ifndef __this_cpu_cmpxchg_double -# ifndef __this_cpu_cmpxchg_double_1 -# define __this_cpu_cmpxchg_double_1 (pcp1, pcp2, oval1, oval2, nval1, nval2) \ - __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) -# endif -# ifndef __this_cpu_cmpxchg_double_2 -# define __this_cpu_cmpxchg_double_2 (pcp1, pcp2, oval1, oval2, nval1, nval2) \ - __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) -# endif -# ifndef __this_cpu_cmpxchg_double_4 -# define __this_cpu_cmpxchg_double_4 (pcp1, pcp2, oval1, oval2, nval1, nval2) \ - __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) -# endif -# ifndef __this_cpu_cmpxchg_double_8 -# define __this_cpu_cmpxchg_double_8 (pcp1, pcp2, oval1, oval2, nval1, nval2) \ - __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) +#ifndef raw_cpu_cmpxchg_double +# ifndef raw_cpu_cmpxchg_double_1 +# define raw_cpu_cmpxchg_double_1 (pcp1, pcp2, oval1, oval2, nval1, nval2) \ + raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) +# endif +# ifndef raw_cpu_cmpxchg_double_2 +# define raw_cpu_cmpxchg_double_2 (pcp1, pcp2, oval1, oval2, nval1, nval2) \ + raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) +# endif +# ifndef raw_cpu_cmpxchg_double_4 +# define raw_cpu_cmpxchg_double_4 (pcp1, pcp2, oval1, oval2, nval1, nval2) \ + raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) +# endif +# ifndef raw_cpu_cmpxchg_double_8 +# define raw_cpu_cmpxchg_double_8 (pcp1, pcp2, oval1, oval2, nval1, nval2) \ + raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)# endif +# define raw_cpu_cmpxchg_double (pcp1, pcp2, oval1, oval2, nval1, nval2) \ + __pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2)) +#endif + +/* +* Generic percpu operations for context that are safe from preemption/interrupts. +* Checks will be added here soon. +*/ +#ifndef __this_cpu_read +# define __this_cpu_read (pcp) __pcpu_size_call_return(raw_cpu_read_, (pcp)) +#endif + +#ifndef __this_cpu_write +# define __this_cpu_write (pcp, val) __pcpu_size_call(raw_cpu_write_, (pcp), (val)) +#endif + +#ifndef __this_cpu_add +# define __this_cpu_add (pcp, val) __pcpu_size_call(raw_cpu_add_, (pcp), (val)) +#endif + +#ifndef __this_cpu_sub +# define __this_cpu_sub (pcp, val) __this_cpu_add((pcp), -(typeof(pcp))(val)) +#endif + +#ifndef __this_cpu_inc +# define __this_cpu_inc (pcp) __this_cpu_add((pcp), 1) +#endif + +#ifndef __this_cpu_dec +# define __this_cpu_dec (pcp) __this_cpu_sub((pcp), 1) +#endif + +#ifndef __this_cpu_and +# define __this_cpu_and (pcp, val) __pcpu_size_call(raw_cpu_and_, (pcp), (val)) +#endif + +#ifndef __this_cpu_or +# define __this_cpu_or (pcp, val) __pcpu_size_call(raw_cpu_or_, (pcp), (val)) +#endif + +#ifndef __this_cpu_add_return +# define __this_cpu_add_return (pcp, val) \ + __pcpu_size_call_return2(raw_cpu_add_return_, pcp, val) +#endif + +#define __this_cpu_sub_return (pcp, val) __this_cpu_add_return(pcp, -(typeof(pcp))(val)) +#define __this_cpu_inc_return (pcp) __this_cpu_add_return(pcp, 1) +#define __this_cpu_dec_return (pcp) __this_cpu_add_return(pcp, -1) + +#ifndef __this_cpu_xchg +# define __this_cpu_xchg (pcp, nval) \ + __pcpu_size_call_return2(raw_cpu_xchg_, (pcp), nval) +#endif + +#ifndef __this_cpu_cmpxchg +# define __this_cpu_cmpxchg (pcp, oval, nval) \ + __pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval) +#endif + +#ifndef __this_cpu_cmpxchg_double# define __this_cpu_cmpxchg_double (pcp1, pcp2, oval1, oval2, nval1, nval2) \ - __pcpu_double_call_return_bool(__this_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2)) + __pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2))#endif #endif /* __LINUX_PERCPU_H */
arch/x86/include/asm/percpu.h +1 -1
@@ -52,7 +52,7 @@* Compared to the generic __my_cpu_offset version, the following * saves one instruction and avoids clobbering a temp register. */ -#define __this_cpu_ptr (ptr) \ +#define raw_cpu_ptr (ptr) \ ({ \ unsigned long tcp_ptr__; \ __verify_pcpu_ptr(ptr); \
include/asm-generic/percpu.h +5 -5
@@ -56,17 +56,17 @@ extern unsigned long __per_cpu_offset[NR#define per_cpu (var, cpu) \ (*SHIFT_PERCPU_PTR(&(var), per_cpu_offset(cpu))) -#ifndef __this_cpu_ptr -#define __this_cpu_ptr (ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset) +#ifndef raw_cpu_ptr +#define raw_cpu_ptr (ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)#endif #ifdef CONFIG_DEBUG_PREEMPT #define this_cpu_ptr (ptr) SHIFT_PERCPU_PTR(ptr, my_cpu_offset)#else -#define this_cpu_ptr (ptr) __this_cpu_ptr(ptr) +#define this_cpu_ptr (ptr) raw_cpu_ptr(ptr)#endif #define __get_cpu_var (var) (*this_cpu_ptr(&(var))) -#define __raw_get_cpu_var (var) (*__this_cpu_ptr(&(var))) +#define __raw_get_cpu_var (var) (*raw_cpu_ptr(&(var)))#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA extern void setup_per_cpu_areas (void );
@@ -83,7 +83,7 @@ extern void setup_per_cpu_areas(void);#define __get_cpu_var (var) (*VERIFY_PERCPU_PTR(&(var)))#define __raw_get_cpu_var (var) (*VERIFY_PERCPU_PTR(&(var)))#define this_cpu_ptr (ptr) per_cpu_ptr(ptr, 0) -#define __this_cpu_ptr (ptr) this_cpu_ptr(ptr) +#define raw_cpu_ptr (ptr) this_cpu_ptr(ptr)#endif /* SMP */
PATCH 11/41 parisc: Replace __get_cpu_var uses for address calculation
In reply to Christoph LameterConvert to the use of this_cpu_ptr().
arch/parisc/lib/memcpy.c +1 -1
@@ -470,7 +470,7 @@ static unsigned long pa_memcpy(void *dstreturn 0 ;/* if a load or store fault occured we can get the faulty addr */ - d = &__get_cpu_var (exception_data); + d =this_cpu_ptr (&exception_data); fault_addr = d->fault_addr;/* error in load or store? */
arch/parisc/mm/fault.c +1 -1
@@ -151,7 +151,7 @@ int fixup_exception(struct pt_regs *regs fix =search_exception_tables (regs->iaoq[0 ]);if (fix) {struct exception_data *d; - d = &__get_cpu_var (exception_data); + d =this_cpu_ptr (&exception_data); d->fault_ip = regs->iaoq[0 ]; d->fault_space = regs->isr; d->fault_addr = regs->ior;
PATCH 10/41 staging/zsmalloc: Replace instances of using __get_cpu_var for address calculation
In reply to Christoph LameterReplace the use of __get_cpu_var for address calculation with this_cpu_ptr().
drivers/staging/zsmalloc/zsmalloc-main.c +2 -2
@@ -1003,7 +1003,7 @@ void *zs_map_object(struct zs_pool *pool class = &pool->size_class[class_idx]; off =obj_idx_to_offset (page, obj_idx, class->size); - area = &get_cpu_var (zs_map_area); + area =this_cpu_ptr (&zs_map_area); area->vm_mm = mm;if (off + class->size <=PAGE_SIZE ) {/* this object is contained entirely within a page */
@@ -1037,7 +1037,7 @@ void zs_unmap_object(struct zs_pool *poo class = &pool->size_class[class_idx]; off =obj_idx_to_offset (page, obj_idx, class->size); - area = &__get_cpu_var (zs_map_area); + area =this_cpu_ptr (&zs_map_area);if (off + class->size <=PAGE_SIZE )kunmap_atomic (area->vm_addr);else {
PATCH 8/41 drivers/leds: Replace __get_cpu_var use through this_cpu_ptr
In reply to Christoph LameterUse this_cpu_ptr for the address calculation instead of __get_cpu_var.
drivers/leds/trigger/ledtrig-cpu.c +1 -1
@@ -46,7 +46,7 @@ static DEFINE_PER_CPU(struct led_trigger*/ void ledtrig_cpu (enum cpu_led_event ledevt) { -struct led_trigger_cpu *trig = &__get_cpu_var (cpu_trig); +struct led_trigger_cpu *trig =this_cpu_ptr (&cpu_trig);/* Locate the correct CPU LED */ switch (ledevt) {
PATCH 5/41 drivers/char/random: Replace __get_cpu_var uses
In reply to Christoph LameterA single case of using __get_cpu_var for address calculation.
drivers/char/random.c +1 -1
@@ -838,7 +838,7 @@ static DEFINE_PER_CPU(struct fast_pool,void add_interrupt_randomness (int irq,int irq_flags) {struct entropy_store *r; -struct fast_pool *fast_pool = &__get_cpu_var (irq_randomness); +struct fast_pool *fast_pool =this_cpu_ptr (&irq_randomness);struct pt_regs *regs =get_irq_regs ();unsigned long now = jiffies;cycles_t cycles =random_get_entropy ();
PATCH 2/41 tracing: Replace __get_cpu_var uses with this_cpu_ptr
In reply to Christoph LameterReplace uses of &__get_cpu_var for address calculation with this_cpu_ptr.
include/linux/kprobes.h +1 -1
@@ -355,7 +355,7 @@ static inline void reset_current_kprobe(static inline struct kprobe_ctlblk *get_kprobe_ctlblk (void ) { -return (&__get_cpu_var (kprobe_ctlblk)); +return this_cpu_ptr (&kprobe_ctlblk); }int register_kprobe (struct kprobe *p);
kernel/trace/ftrace.c +2 -2
@@ -864,7 +864,7 @@ function_profile_call(unsigned long ip,local_irq_save (flags); - stat = &__get_cpu_var (ftrace_profile_stats); + stat =this_cpu_ptr (&ftrace_profile_stats);if (!stat->hash || !ftrace_profile_enabled) goto out;
@@ -895,7 +895,7 @@ static void profile_graph_return(structunsigned long flags;local_irq_save (flags); - stat = &__get_cpu_var (ftrace_profile_stats); + stat =this_cpu_ptr (&ftrace_profile_stats);if (!stat->hash || !ftrace_profile_enabled) goto out;
kernel/trace/trace.c +1 -1
@@ -1685,7 +1685,7 @@ static void __ftrace_trace_stack(struct*/ barrier ();if (use_stack ==1 ) { - trace.entries = &__get_cpu_var (ftrace_stack).calls[0 ]; + trace.entries =this_cpu_ptr (ftrace_stack.calls); trace.max_entries =FTRACE_STACK_MAX_ENTRIES ;if (regs)
James Bottomley
maintainerRe: [PATCH 11/41] parisc: Replace __get_cpu_var uses for address calculation
In reply to Christoph LameterQuoted 5 lines · Christoph Lameter
Convert to the use of
this_cpu_ptr().Cc "James E.J. Bottomley" <[email protected]>Cc Helge Deller <[email protected]>
You say this in the commit, but the linux-parisc list isn't cc'd on the email ...
James
Re: [PATCH 22/41] rcu: Replace __this_cpu_ptr uses with raw_cpu_ptr
In reply to Christoph LameterQuoted 8 lines · Christoph Lameter
__this_cpu_ptris being phased out.One special case is
increment_cpu_stall_ticks(). A per cpu variable is incremented so useraw_cpu_inc().Cc Dipankar Sarma <[email protected]>Cc "Paul E. McKenney" <[email protected]>Signed-off-by Christoph Lameter <[email protected]>
This does not build on 3.13-rc2. Ah, need a couple of patches earlier in the series.
But... But... Didn't we recently change -to- __this_cpu_ptr() from the older __get_cpu_var()?
Thanx, Paul
------------------------------------------------------------------------
/media/homes/git/linux-2.6-tip/kernel/rcu/tree.c: In function ‘rcu_adopt_orphan_cbs’:
/media/homes/git/linux-2.6-tip/kernel/rcu/tree.c:1943:9: error: implicit declaration of function ‘raw_cpu_ptr’ [-Werror=implicit-function-declaration] /media/homes/git/linux-2.6-tip/kernel/rcu/tree.c:1943:25: warning: initialization makes pointer from integer without a cast [enabled by default] /media/homes/git/linux-2.6-tip/kernel/rcu/tree.c: In function ‘__rcu_process_callbacks’:
/media/homes/git/linux-2.6-tip/kernel/rcu/tree.c:2322:25: warning: initialization makes pointer from integer without a cast [enabled by default] /media/homes/git/linux-2.6-tip/kernel/rcu/tree.c: In function ‘rcu_barrier_func’:
/media/homes/git/linux-2.6-tip/kernel/rcu/tree.c:2923:25: warning: initialization makes pointer from integer without a cast [enabled by default] In file included from /media/homes/git/linux-2.6-tip/kernel/rcu/tree.c:3467:0:
/media/homes/git/linux-2.6-tip/kernel/rcu/tree_plugin.h: In function ‘rcu_oom_notify_cpu’:
/media/homes/git/linux-2.6-tip/kernel/rcu/tree_plugin.h:1846:7: warning: assignment makes pointer from integer without a cast [enabled by default] /media/homes/git/linux-2.6-tip/kernel/rcu/tree_plugin.h: In function ‘increment_cpu_stall_ticks’:
/media/homes/git/linux-2.6-tip/kernel/rcu/tree_plugin.h:1988:3: error: implicit declaration of function ‘raw_cpu_inc’ [-Werror=implicit-function-declaration]
cc1: some warnings being treated as errors
make[2]: *** [kernel/rcu/tree.o] Error 1
make[1]: *** [kernel/rcu/tree.o] Error 2
make: *** [sub-make] Error 2
Re: [PATCH 21/41] block: Replace __this_cpu_ptr with raw_cpu_ptr
In reply to Christoph LameterQuoted 23 lines · Christoph Lameter
__this_cpu_ptris being phased out.Cc Jens Axboe <[email protected]>Signed-off-by Christoph Lameter <[email protected]>Index:
linux/fs/ext4/mballoc.c
===================================================================fs/ext4/mballoc.c +1
-1@@ -4085,7 +4085,7 @@ static void ext4_mb_group_or_file(struct* per cpu locality group is to reduce the contention between block * request from multiple CPUs. */ - ac->ac_lg =__this_cpu_ptr (sbi->s_locality_groups); + ac->ac_lg =raw_cpu_ptr (sbi->s_locality_groups);/* we're going to use group allocation */ ac->ac_flags |=EXT4_MB_HINT_GROUP_ALLOC ;Index:
linux/fs/buffer.c
===================================================================
fs/buffer.c static void bh_lru_install(struct buffer · line 1317
Quoted 7 lines
fs/buffer.c +1
-1@@ -1312,7 +1312,7 @@ static void bh_lru_install(struct buffer }while (out <BH_LRU_SIZE ) bhs[out++] =NULL ; -memcpy (__this_cpu_ptr (&bh_lrus.bhs), bhs,sizeof (bhs)); +memcpy (this_cpu_ptr (&bh_lrus.bhs), bhs,sizeof (bhs)); }bh_lru_unlock ();
Christoph, I applied the fs/buffer.c bit, the ext4 one should probably go through Ted.
Signature · 1 line
Jens Axboe
Ingo Molnar
maintainerQuoted 5 lines · Christoph Lameter
The kernel has never been audited to ensure that
this_cpuoperations are consistently used throughout the kernel. The code generated in many places can be improved through the use ofthis_cpuoperations (which uses a segment register for relocation of per cpu offsets instead of performing address calculations).
Your series lacks the this_cpu ops preempt debugging patches that you worked on before, what happened to them?
Those debug facilities caught kernel real bugs and they are thus essential before any percpu extension can be done.
Thanks,
Ingo
Re: [PATCH 21/41] block: Replace __this_cpu_ptr with raw_cpu_ptr
In reply to Jens AxboeQuoted 2 lines · Jens Axboe
Christoph, I applied the
fs/buffer.cbit, the ext4 one should probably go through Ted.
This is not going to work unless earlier patchses have been applied since raw_cpu_ptr is otherwise not availab.e
Re: [PATCH 22/41] rcu: Replace __this_cpu_ptr uses with raw_cpu_ptr
In reply to Paul E. McKenneyQuoted 5 lines · Paul E. McKenney
This does not build on 3.13-rc2. Ah, need a couple of patches earlier in the series.
But... But... Didn't we recently change -to-
__this_cpu_ptr()from the older__get_cpu_var()?
We moved from __get_cpu_var to this_cpu_ptr in a earlier patch proposal. The __this_cpu_ptrs were then decided to have to be called raw_cpu_ptr. You merged patch from the proposal. The patch was merged a bit too early into your tree.
Quoted 11 lines · Ingo Molnar
Quoted 5 lines · Christoph Lameter
The kernel has never been audited to ensure that
this_cpuoperations are consistently used throughout the kernel. The code generated in many places can be improved through the use ofthis_cpuoperations (which uses a segment register for relocation of per cpu offsets instead of performing address calculations).Your series lacks the
this_cpuops preempt debugging patches that you worked on before, what happened to them?
They are included. Please read the complete description. Patches 13 14 and 18 implement the preemption checks.
Quoted 2 lines
Those debug facilities caught kernel real bugs and they are thus essential before any percpu extension can be done.
Yes please apply the preemptiopn debug patches included in this series ASAP.
Re: [PATCH 21/41] block: Replace __this_cpu_ptr with raw_cpu_ptr
In reply to Christoph LameterQuoted 7 lines · Christoph Lameter
Quoted 2 lines · Jens Axboe
Christoph, I applied the
fs/buffer.cbit, the ext4 one should probably go through Ted.This is not going to work unless earlier patchses have been applied since
raw_cpu_ptris otherwise not availab.e
Overnigh caught that too. You might want to make such requirements explicit in the patch, when sending it out...
Signature · 1 line
Jens Axboe
Ingo Molnar
maintainerQuoted 11 lines · Christoph Lameter
Quoted 9 lines · Ingo Molnar
Quoted 5 lines · Christoph Lameter
The kernel has never been audited to ensure that
this_cpuoperations are consistently used throughout the kernel. The code generated in many places can be improved through the use ofthis_cpuoperations (which uses a segment register for relocation of per cpu offsets instead of performing address calculations).
Quoted 5 lines
Quoted 2 lines
Your series lacks the
this_cpuops preempt debugging patches that you worked on before, what happened to them?They are included. Please read the complete description. Patches 13 14 and 18 implement the preemption checks.
Indeed I missed them. They should probably go first in the series.
Other than that they are looking fine now.
Thanks,
Ingo
Re: [PATCH 11/41] parisc: Replace __get_cpu_var uses for address calculation
In reply to James BottomleyQuoted 9 lines · James Bottomley
Quoted 5 lines · Christoph Lameter
Convert to the use of
this_cpu_ptr().Cc "James E.J. Bottomley" <[email protected]>Cc Helge Deller <[email protected]>You say this in the commit, but the linux-parisc list isn't cc'd on the email ...
Weird my testoutput before the patchset was sent out shows that it was included then :
Quoted 1 line
From [email protected] Tue Dec 3 16:11:39 2013
Message-Id: <[email protected]>
User-Agent: quilt/0.50-1
Date: Tue, 03 Dec 2013 16:11:37 -0600
From: Christoph Lameter <[email protected]>
To: Tejun Heo <[email protected]>
Cc: [email protected],
[email protected],
[email protected],
Ingo Molnar <[email protected]>,
Peter Zijlstra <[email protected]>,
Thomas Gleixner <[email protected]>,
"James E.J. Bottomley" <[email protected]>,
Helge Deller <[email protected]>,
[email protected]
Subject: [PATCH 11/41] parisc: Replace __get_cpu_var uses for address calculation
References: <[email protected]>
Content-Disposition: inline; filename=this_parisc
Convert to the use of this_cpu_ptr().
Quoted 4 lines · Ingo Molnar
Quoted 2 lines
They are included. Please read the complete description. Patches 13 14 and 18 implement the preemption checks.
Indeed I missed them. They should probably go first in the series.
The first patches in the series do not affect __this_cpu ops but only use this_cpu ops. this_cpu_ptr already checks for preemption. The first patches are therefore indepedent of new preemptin checking and I hope that they can be immediately merged even if there are further issues with the other patches.
Quoted 1 line
Quoted 1 line
Other than that they are looking fine now. >
Great. Thanks.
quilt 0.60 does not include CCed email without name
In reply to a message not in the archiveQuoted 5 lines
Quoted 2 lines
You say this in the commit, but the linux-parisc list isn't cc'd on the email ...
Weird my testoutput before the patchset was sent out shows that it was included then :
Ok while the test output was done with quilt 0.50 the patch was actually send with quilt 0.60 from another system. quilt 0.60 skips all Cc:s that do not have a full name.
So Helge was included since it had a name
Quoted 1 line
Cc Helge Deller <[email protected]>
but
Quoted 1 line
did not because no name was available.
Martin?
Re: quilt 0.60 does not include CCed email without name
Quoted 14 lines · Christoph Lameter
Quoted 5 lines
Quoted 2 lines
You say this in the commit, but the linux-parisc list isn't cc'd on the email ...
Weird my testoutput before the patchset was sent out shows that it was included then :
Ok while the test output was done with quilt 0.50 the patch was actually send with quilt 0.60 from another system. quilt 0.60 skips all Cc:s that do not have a full name.
So Helge was included since it had a name
Quoted 1 line
Cc Helge Deller <[email protected]>
Quoted 7 lines
but
Quoted 1 line
did not because no name was available.
Martin?
I'm sorry, I never use the quilt mail command myself. After a quick investiguation of the git history, this bug seems to be caused by:
http://git.savannah.gnu.org/cgit/quilt.git/commit/?id=02f15444278eb3a854acae0192124153e0c6d9d4
I think that the last lines should be changed as follows (and same for the one before):
- -e 's/^\('"$*"'\):\(.*<.*@.*>\)/Recipient-Cc:\2/Ip' \
+ -e 's/^\('"$*"'\):\(.*@.*\)/Recipient-Cc:\2/Ip' \
I'm CCing Andreas so that he can comment, and if nobody complains I'll commit it to our git.
Many thanks for the detailed bug report,
Mt.
Signature · 2 lines
Simplicity does not precede complexity, but follows it. -- "Epigrams in Programming", by Alan J. Perlis of Yale University.
Re: quilt 0.60 does not include CCed email without name
Martin,
Quoted 7 lines
[...] bug seems to be caused by:
http://git.savannah.gnu.org/cgit/quilt.git/commit/?id=02f15444278eb3a854acae0192124153e0c6d9d4I think that the last lines should be changed as follows (and same for the one before):
- -e 's/^\('"$*"'\):\(.*<.*@.*>\)/Recipient-Cc:\2/Ip' \
+ -e 's/^\('"$*"'\):\(.*@.*\)/Recipient-Cc:\2/Ip' \
or something similar like .\+@.\+ for the email address part, yes.
Thanks,
Andreas
Re: quilt 0.60 does not include CCed email without name
Quoted 11 lines · Andreas Gruenbacher
Martin,
Quoted 7 lines
[...] bug seems to be caused by:
http://git.savannah.gnu.org/cgit/quilt.git/commit/?id=02f15444278eb3a854acae0192124153e0c6d9d4I think that the last lines should be changed as follows (and same for the one before):
- -e 's/^\('"$*"'\):\(.*<.*@.*>\)/Recipient-Cc:\2/Ip' \
+ -e 's/^\('"$*"'\):\(.*@.*\)/Recipient-Cc:\2/Ip' \or something similar like .\+@.\+ for the email address part, yes.
Done. This will be part of quilt 0.61.
Thanks, Mt.
Re: [PATCH 25/41] drivers/net/ethernet/tile: Replace __get_cpu_var uses for address calculation
In reply to Christoph LameterQuoted 4 lines · Christoph Lameter
Replace with
this_cpu_ptr.Cc Chris Metcalf <[email protected]>Signed-off-by Christoph Lameter <[email protected]>
Signature · 2 lines
Chris Metcalf, Tilera Corp. http://www.tilera.com
Re: [PATCH 35/41] tile: Replace __get_cpu_var uses
In reply to Christoph LameterQuoted 9 lines · Christoph Lameter
__get_cpu_var()is used for multiple purposes in the kernel source. One of them is address calculation via the form&__get_cpu_var(x). This calculates the address for the instance of the percpu variable of the current processor based on an offset.[...]
Cc Chris Metcalf <[email protected]>Signed-off-by Christoph Lameter <[email protected]>
Except for a comment typo:
arch/tile/include/asm/irqflags.h extern unsigned int debug_smp_processor_ · line 148
Quoted 10 lines
/*
* Read the set of maskable interrupts.
- * We avoid the preemption warning here via__this_cpu_ptrsince even
+ * We avoid the preemption warning here viaraw_cpu_ptrsince even
* if irqs are already enabled, it's harmless to read the wrong cpu's
* enabled mask.
*/
#definearch_local_irqs_enabled()\arch/tile/include/asm/irqflags.h +1
-1- (*__this_cpu_ptr (&interrupts_enabled_mask)) +raw_cpu_read (interrupts_enabled_mask)
Should be raw_cpu_read() in the comment.
I assume you are planning to submit these through your own tree, not the arch trees?
Signature · 2 lines
Chris Metcalf, Tilera Corp. http://www.tilera.com
Re: [PATCH 35/41] tile: Replace __get_cpu_var uses
Quoted 14 lines · Chris Metcalf
Except for a comment typo:
arch/tile/include/asm/irqflags.h extern unsigned int debug_smp_processor_ · line 148
Quoted 10 lines
/*
* Read the set of maskable interrupts.
- * We avoid the preemption warning here via__this_cpu_ptrsince even
+ * We avoid the preemption warning here viaraw_cpu_ptrsince even
* if irqs are already enabled, it's harmless to read the wrong cpu's
* enabled mask.
*/
#definearch_local_irqs_enabled()\arch/tile/include/asm/irqflags.h +1
-1- (*__this_cpu_ptr (&interrupts_enabled_mask)) +raw_cpu_read (interrupts_enabled_mask)Should be
raw_cpu_read()in the comment.
Ok fixed.
Quoted 1 line
I assume you are planning to submit these through your own tree, not the arch trees?
Well once the core pieces are in then they can go through the arch trees.
Re: [PATCH 13/41] percpu: Add raw_cpu_ops
In reply to Christoph LameterQuoted 11 lines · Christoph Lameter
The patches following this one will add preemption checks to
__this_cpuops so we need to have an alternative way to usethis_cpuoperations without preemption checks.
raw_cpu_opswill be the basis for all other ops since these will be the operations that do not implement any checks.Primitive operations are renamed by this patch from
__this_cpu_xxxtoraw_cpu_xxxx.Signed-off-by Christoph Lameter <[email protected]>
Applied to percpu/for-3.14.
Thanks.
Signature · 1 line
tejun
Ingo Molnar
maintainerRe: [PATCH 13/41] percpu: Add raw_cpu_ops
Quoted 8 lines · Tejun Heo
Quoted 7 lines · Christoph Lameter
The patches following this one will add preemption checks to
__this_cpuops so we need to have an alternative way to usethis_cpuoperations without preemption checks.
raw_cpu_opswill be the basis for all other ops since these will be the operations that do not implement any checks.
Quoted 6 lines
Quoted 4 lines
Primitive operations are renamed by this patch from
__this_cpu_xxxtoraw_cpu_xxxx.Signed-off-by Christoph Lameter <[email protected]>Applied to percpu/for-3.14.
Please also pick up the related debugging bits:
[PATCH 18/41] percpu: Add preemption checks to __this_cpu ops
as we don't want to expand percpu ops without doing proper debugging.
Thanks,
Ingo
Tejun Heo
maintainerRe: [PATCH 13/41] percpu: Add raw_cpu_ops
Quoted 16 lines · Ingo Molnar
Quoted 13 lines · Tejun Heo
Quoted 11 lines · Christoph Lameter
The patches following this one will add preemption checks to
__this_cpuops so we need to have an alternative way to usethis_cpuoperations without preemption checks.
raw_cpu_opswill be the basis for all other ops since these will be the operations that do not implement any checks.Primitive operations are renamed by this patch from
__this_cpu_xxxtoraw_cpu_xxxx.Signed-off-by Christoph Lameter <[email protected]>
Quoted 7 lines
Quoted 1 line
Applied to percpu/for-3.14.
Please also pick up the related debugging bits:
[PATCH 18/41] percpu: Add preemption checks to
__this_cpuopsas we don't want to expand percpu ops without doing proper debugging.
Yeah, trying to figure out which should go through which tree. Christoph, should I also pickup 14? Or can that go through x86?
Thanks.
Signature · 1 line
tejun
Re: [PATCH 13/41] percpu: Add raw_cpu_ops
Quoted 19 lines · Tejun Heo
Quoted 18 lines · Ingo Molnar
Quoted 14 lines · Tejun Heo
Quoted 11 lines · Christoph Lameter
The patches following this one will add preemption checks to
__this_cpuops so we need to have an alternative way to usethis_cpuoperations without preemption checks.
raw_cpu_opswill be the basis for all other ops since these will be the operations that do not implement any checks.Primitive operations are renamed by this patch from
__this_cpu_xxxtoraw_cpu_xxxx.Signed-off-by Christoph Lameter <[email protected]>Applied to percpu/for-3.14.
Quoted 8 lines
Quoted 5 lines
Please also pick up the related debugging bits:
[PATCH 18/41] percpu: Add preemption checks to
__this_cpuopsas we don't want to expand percpu ops without doing proper debugging.
Yeah, trying to figure out which should go through which tree. Christoph, should I also pickup 14? Or can that go through x86?
Well, it appears to have dependencies so I doubt it can be kept separate. In any case, provided all debugging is properly productized:
Thanks,
Ingo
Re: [PATCH 13/41] percpu: Add raw_cpu_ops
In reply to Tejun HeoQuoted 4 lines · Tejun Heo
Quoted 1 line
as we don't want to expand percpu ops without doing proper debugging.
Yeah, trying to figure out which should go through which tree. Christoph, should I also pickup 14? Or can that go through x86?
If you do that then you need to also pick up the patches in between otherwise the debug functions will get quite loud.
Re: [PATCH 13/41] percpu: Add raw_cpu_ops
In reply to Ingo MolnarQuoted 5 lines · Ingo Molnar
Quoted 2 lines
Yeah, trying to figure out which should go through which tree. Christoph, should I also pickup 14? Or can that go through x86?
Well, it appears to have dependencies so I doubt it can be kept separate. In any case, provided all debugging is properly productized:
13/14 can be just merged. There are no dependencies on earlier patches.
If you want to merge 18 then you would need to pick up also 15-17.
Ingo Molnar
maintainerRe: [PATCH 13/41] percpu: Add raw_cpu_ops
Quoted 5 lines · Christoph Lameter
Quoted 3 lines · Ingo Molnar
Quoted 2 lines
Yeah, trying to figure out which should go through which tree. Christoph, should I also pickup 14? Or can that go through x86?
Quoted 7 lines
Quoted 2 lines
Well, it appears to have dependencies so I doubt it can be kept separate. In any case, provided all debugging is properly productized:
13/14 can be just merged. There are no dependencies on earlier patches.
If you want to merge 18 then you would need to pick up also 15-17.
So I'm referring to the 'This is required' wording here:
| Subject: [PATCH 14/41] x86: Rename __this_cpu_xxx_# operations to raw_cpu_xxx_# | | This is required now since the core portion expects a different | naming. | | Change the names from __this_cpu_xxxx_# to raw_cpu_xxxx_#. | | Also changes the uses of the x86 percpu primitives in preempt.h. | | This is required now since the core portion expects a different naming.
By what is this required - by the previous patch #13?
Also, the changelog indicates it's two changes - please split the patch in two if so.
(Tejun, please wait for this review to be complete before committing anything.)
Thanks,
Ingo
Tejun Heo
maintainerRe: [PATCH 13/41] percpu: Add raw_cpu_ops
Hello,
Quoted 2 lines · Ingo Molnar
(Tejun, please wait for this review to be complete before committing anything.)
Eh... alright, backing out the patch. Please let me know when things are ready.
Thanks.
Signature · 1 line
tejun
Re: [PATCH 13/41] percpu: Add raw_cpu_ops
In reply to Ingo MolnarQuoted 1 line · Ingo Molnar
By what is this required - by the previous patch #13?
Yes its required in order to keep the __this_cpu ops working.
Quoted 2 lines
Also, the changelog indicates it's two changes - please split the patch in two if so.
Well this is how the x86 portions came from Peter. Patch can be split according to the file modified in x86.
Ingo Molnar
maintainerRe: [PATCH 13/41] percpu: Add raw_cpu_ops
Quoted 10 lines · Christoph Lameter
Quoted 1 line · Ingo Molnar
By what is this required - by the previous patch #13?
Yes its required in order to keep the
__this_cpuops working.Quoted 2 lines
Also, the changelog indicates it's two changes - please split the patch in two if so.
Well this is how the x86 portions came from Peter. [...]
That's an interesting piece of information, in particular because you lost From: PeterZ authorship there, don't do that!
In any case, your reply does not address my review feedback.
Thanks,
Ingo
Re: [PATCH 13/41] percpu: Add raw_cpu_ops
Quoted 4 lines · Ingo Molnar
Quoted 1 line
Well this is how the x86 portions came from Peter. [...]
That's an interesting piece of information, in particular because you lost From: PeterZ authorship there, don't do that!
Ok will add that. There is a signoff there. Is that ok with you Peter?
Thinking about this some more: Separating these two issues would make it impossible to build x86 since the hackery in x86/include/asm/preempt.h depends on having the right names in x86/include/asm/percpu.h. Changing the names first would result in preempt.h not being usable.
Ingo Molnar
maintainerRe: [PATCH 13/41] percpu: Add raw_cpu_ops
Quoted 9 lines · Christoph Lameter
Quoted 4 lines · Ingo Molnar
Quoted 1 line
Well this is how the x86 portions came from Peter. [...]
That's an interesting piece of information, in particular because you lost From: PeterZ authorship there, don't do that!
Ok will add that. There is a signoff there. Is that ok with you Peter?
Signoff alone does not signal authorship, only a "From: " line does.
Quoted 5 lines
Thinking about this some more: Separating these two issues would make it impossible to build x86 since the hackery in
x86/include/asm/preempt.hdepends on having the right names inx86/include/asm/percpu.h. Changing the names first would result inpreempt.hnot being usable.
The series must be bisectable at every commit, of course.
Thanks,
Ingo
Re: [PATCH 13/41] percpu: Add raw_cpu_ops
Quoted 7 lines · Ingo Molnar
Quoted 5 lines
Thinking about this some more: Separating these two issues would make it impossible to build x86 since the hackery in
x86/include/asm/preempt.hdepends on having the right names inx86/include/asm/percpu.h. Changing the names first would result inpreempt.hnot being usable.The series must be bisectable at every commit, of course.
Ok patch with revised header:
From: Peter Zijlstra <[email protected]>
Subject: x86: Rename __this_cpu_xxx_# operations to raw_cpu_xxx_#
This is required after the raw_cpu changes in the core since x86 needs to follow a different naming for __this_cpu primitives now.
Change the names from __this_cpu_xxxx_# to raw_cpu_xxxx_#.
Also changes the uses of the x86 percpu primitives in preempt.h. These depend directly on asm/percpu.h (header #include nesting issue).
arch/x86/include/asm/percpu.h +49 -49
@@ -362,25 +362,25 @@ do { \*/ #define this_cpu_read_stable (var) percpu_from_op("mov", var, "p" (&(var))) -#define __this_cpu_read_1 (pcp) percpu_from_op("mov", (pcp), "m"(pcp)) -#define __this_cpu_read_2 (pcp) percpu_from_op("mov", (pcp), "m"(pcp)) -#define __this_cpu_read_4 (pcp) percpu_from_op("mov", (pcp), "m"(pcp)) - -#define __this_cpu_write_1 (pcp, val) percpu_to_op("mov", (pcp), val) -#define __this_cpu_write_2 (pcp, val) percpu_to_op("mov", (pcp), val) -#define __this_cpu_write_4 (pcp, val) percpu_to_op("mov", (pcp), val) -#define __this_cpu_add_1 (pcp, val) percpu_add_op((pcp), val) -#define __this_cpu_add_2 (pcp, val) percpu_add_op((pcp), val) -#define __this_cpu_add_4 (pcp, val) percpu_add_op((pcp), val) -#define __this_cpu_and_1 (pcp, val) percpu_to_op("and", (pcp), val) -#define __this_cpu_and_2 (pcp, val) percpu_to_op("and", (pcp), val) -#define __this_cpu_and_4 (pcp, val) percpu_to_op("and", (pcp), val) -#define __this_cpu_or_1 (pcp, val) percpu_to_op("or", (pcp), val) -#define __this_cpu_or_2 (pcp, val) percpu_to_op("or", (pcp), val) -#define __this_cpu_or_4 (pcp, val) percpu_to_op("or", (pcp), val) -#define __this_cpu_xchg_1 (pcp, val) percpu_xchg_op(pcp, val) -#define __this_cpu_xchg_2 (pcp, val) percpu_xchg_op(pcp, val) -#define __this_cpu_xchg_4 (pcp, val) percpu_xchg_op(pcp, val) +#define raw_cpu_read_1 (pcp) percpu_from_op("mov", (pcp), "m"(pcp)) +#define raw_cpu_read_2 (pcp) percpu_from_op("mov", (pcp), "m"(pcp)) +#define raw_cpu_read_4 (pcp) percpu_from_op("mov", (pcp), "m"(pcp)) + +#define raw_cpu_write_1 (pcp, val) percpu_to_op("mov", (pcp), val) +#define raw_cpu_write_2 (pcp, val) percpu_to_op("mov", (pcp), val) +#define raw_cpu_write_4 (pcp, val) percpu_to_op("mov", (pcp), val) +#define raw_cpu_add_1 (pcp, val) percpu_add_op((pcp), val) +#define raw_cpu_add_2 (pcp, val) percpu_add_op((pcp), val) +#define raw_cpu_add_4 (pcp, val) percpu_add_op((pcp), val) +#define raw_cpu_and_1 (pcp, val) percpu_to_op("and", (pcp), val) +#define raw_cpu_and_2 (pcp, val) percpu_to_op("and", (pcp), val) +#define raw_cpu_and_4 (pcp, val) percpu_to_op("and", (pcp), val) +#define raw_cpu_or_1 (pcp, val) percpu_to_op("or", (pcp), val) +#define raw_cpu_or_2 (pcp, val) percpu_to_op("or", (pcp), val) +#define raw_cpu_or_4 (pcp, val) percpu_to_op("or", (pcp), val) +#define raw_cpu_xchg_1 (pcp, val) percpu_xchg_op(pcp, val) +#define raw_cpu_xchg_2 (pcp, val) percpu_xchg_op(pcp, val) +#define raw_cpu_xchg_4 (pcp, val) percpu_xchg_op(pcp, val)#define this_cpu_read_1 (pcp) percpu_from_op("mov", (pcp), "m"(pcp))#define this_cpu_read_2 (pcp) percpu_from_op("mov", (pcp), "m"(pcp))
@@ -401,16 +401,16 @@ do { \#define this_cpu_xchg_2 (pcp, nval) percpu_xchg_op(pcp, nval)#define this_cpu_xchg_4 (pcp, nval) percpu_xchg_op(pcp, nval) -#define __this_cpu_add_return_1 (pcp, val) percpu_add_return_op(pcp, val) -#define __this_cpu_add_return_2 (pcp, val) percpu_add_return_op(pcp, val) -#define __this_cpu_add_return_4 (pcp, val) percpu_add_return_op(pcp, val) -#define __this_cpu_cmpxchg_1 (pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) -#define __this_cpu_cmpxchg_2 (pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) -#define __this_cpu_cmpxchg_4 (pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) - -#define this_cpu_add_return_1 (pcp, val) percpu_add_return_op(pcp, val) -#define this_cpu_add_return_2 (pcp, val) percpu_add_return_op(pcp, val) -#define this_cpu_add_return_4 (pcp, val) percpu_add_return_op(pcp, val) +#define raw_cpu_add_return_1 (pcp, val) percpu_add_return_op(pcp, val) +#define raw_cpu_add_return_2 (pcp, val) percpu_add_return_op(pcp, val) +#define raw_cpu_add_return_4 (pcp, val) percpu_add_return_op(pcp, val) +#define raw_cpu_cmpxchg_1 (pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) +#define raw_cpu_cmpxchg_2 (pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) +#define raw_cpu_cmpxchg_4 (pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) + +#define this_cpu_add_return_1 (pcp, val) percpu_add_return_op(pcp, val) +#define this_cpu_add_return_2 (pcp, val) percpu_add_return_op(pcp, val) +#define this_cpu_add_return_4 (pcp, val) percpu_add_return_op(pcp, val)#define this_cpu_cmpxchg_1 (pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)#define this_cpu_cmpxchg_2 (pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)#define this_cpu_cmpxchg_4 (pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)
@@ -427,7 +427,7 @@ do { \ __ret; \ }) -#define __this_cpu_cmpxchg_double_4 percpu_cmpxchg8b_double +#define raw_cpu_cmpxchg_double_4 percpu_cmpxchg8b_double#define this_cpu_cmpxchg_double_4 percpu_cmpxchg8b_double#endif /* CONFIG_X86_CMPXCHG64 */
@@ -436,22 +436,22 @@ do { \* 32 bit must fall back to generic operations. */ #ifdef CONFIG_X86_64 -#define __this_cpu_read_8 (pcp) percpu_from_op("mov", (pcp), "m"(pcp)) -#define __this_cpu_write_8 (pcp, val) percpu_to_op("mov", (pcp), val) -#define __this_cpu_add_8 (pcp, val) percpu_add_op((pcp), val) -#define __this_cpu_and_8 (pcp, val) percpu_to_op("and", (pcp), val) -#define __this_cpu_or_8 (pcp, val) percpu_to_op("or", (pcp), val) -#define __this_cpu_add_return_8 (pcp, val) percpu_add_return_op(pcp, val) -#define __this_cpu_xchg_8 (pcp, nval) percpu_xchg_op(pcp, nval) -#define __this_cpu_cmpxchg_8 (pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) - -#define this_cpu_read_8 (pcp) percpu_from_op("mov", (pcp), "m"(pcp)) -#define this_cpu_write_8 (pcp, val) percpu_to_op("mov", (pcp), val) -#define this_cpu_add_8 (pcp, val) percpu_add_op((pcp), val) -#define this_cpu_and_8 (pcp, val) percpu_to_op("and", (pcp), val) -#define this_cpu_or_8 (pcp, val) percpu_to_op("or", (pcp), val) -#define this_cpu_add_return_8 (pcp, val) percpu_add_return_op(pcp, val) -#define this_cpu_xchg_8 (pcp, nval) percpu_xchg_op(pcp, nval) +#define raw_cpu_read_8 (pcp) percpu_from_op("mov", (pcp), "m"(pcp)) +#define raw_cpu_write_8 (pcp, val) percpu_to_op("mov", (pcp), val) +#define raw_cpu_add_8 (pcp, val) percpu_add_op((pcp), val) +#define raw_cpu_and_8 (pcp, val) percpu_to_op("and", (pcp), val) +#define raw_cpu_or_8 (pcp, val) percpu_to_op("or", (pcp), val) +#define raw_cpu_add_return_8 (pcp, val) percpu_add_return_op(pcp, val) +#define raw_cpu_xchg_8 (pcp, nval) percpu_xchg_op(pcp, nval) +#define raw_cpu_cmpxchg_8 (pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) + +#define this_cpu_read_8 (pcp) percpu_from_op("mov", (pcp), "m"(pcp)) +#define this_cpu_write_8 (pcp, val) percpu_to_op("mov", (pcp), val) +#define this_cpu_add_8 (pcp, val) percpu_add_op((pcp), val) +#define this_cpu_and_8 (pcp, val) percpu_to_op("and", (pcp), val) +#define this_cpu_or_8 (pcp, val) percpu_to_op("or", (pcp), val) +#define this_cpu_add_return_8 (pcp, val) percpu_add_return_op(pcp, val) +#define this_cpu_xchg_8 (pcp, nval) percpu_xchg_op(pcp, nval)#define this_cpu_cmpxchg_8 (pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)/*
@@ -474,7 +474,7 @@ do { \ __ret; \ }) -#define __this_cpu_cmpxchg_double_8 percpu_cmpxchg16b_double +#define raw_cpu_cmpxchg_double_8 percpu_cmpxchg16b_double#define this_cpu_cmpxchg_double_8 percpu_cmpxchg16b_double#endif
@@ -495,9 +495,9 @@ static __always_inline int x86_this_cpu_unsigned long __percpu *a = (unsigned long *)addr + nr /BITS_PER_LONG ;#ifdef CONFIG_X86_64 -return ((1UL << (nr %BITS_PER_LONG )) &__this_cpu_read_8 (*a)) !=0 ; +return ((1UL << (nr %BITS_PER_LONG )) &raw_cpu_read_8 (*a)) !=0 ;#else -return ((1UL << (nr %BITS_PER_LONG )) &__this_cpu_read_4 (*a)) !=0 ; +return ((1UL << (nr %BITS_PER_LONG )) &raw_cpu_read_4 (*a)) !=0 ;#endif }
arch/x86/include/asm/preempt.h +8 -8
@@ -13,12 +13,12 @@ DECLARE_PER_CPU(int, __preempt_count);*/ static __always_inline int preempt_count(void) { -return __this_cpu_read_4 (__preempt_count) & ~PREEMPT_NEED_RESCHED ; +return raw_cpu_read_4 (__preempt_count) & ~PREEMPT_NEED_RESCHED ; } static __always_inline void preempt_count_set(int pc) { -__this_cpu_write_4 (__preempt_count, pc); +raw_cpu_write_4 (__preempt_count, pc); }/*
@@ -47,17 +47,17 @@ static __always_inline void preempt_coun static __always_inline void set_preempt_need_resched(void) { -__this_cpu_and_4 (__preempt_count, ~PREEMPT_NEED_RESCHED ); +raw_cpu_and_4 (__preempt_count, ~PREEMPT_NEED_RESCHED ); } static __always_inline void clear_preempt_need_resched(void) { -__this_cpu_or_4 (__preempt_count,PREEMPT_NEED_RESCHED ); +raw_cpu_or_4 (__preempt_count,PREEMPT_NEED_RESCHED ); } static __always_inline bool test_preempt_need_resched(void) { -return !(__this_cpu_read_4 (__preempt_count) &PREEMPT_NEED_RESCHED ); +return !(raw_cpu_read_4 (__preempt_count) &PREEMPT_NEED_RESCHED ); }/*
@@ -66,12 +66,12 @@ static __always_inline bool test_preempt static __always_inline void __preempt_count_add(int val) { -__this_cpu_add_4 (__preempt_count, val); +raw_cpu_add_4 (__preempt_count, val); } static __always_inline void __preempt_count_sub(int val) { -__this_cpu_add_4 (__preempt_count, -val); +raw_cpu_add_4 (__preempt_count, -val); } static __always_inline bool __preempt_count_dec_and_test(void)
@@ -84,7 +84,7 @@ static __always_inline bool __preempt_co*/ static __always_inline bool should_resched(void) { -return unlikely (!__this_cpu_read_4 (__preempt_count)); +return unlikely (!raw_cpu_read_4 (__preempt_count)); }#ifdef CONFIG_PREEMPT
Tejun Heo
maintainerRe: [PATCH 01/41] mm: Replace __get_cpu_var uses with this_cpu_ptr
In reply to Christoph LameterQuoted 6 lines · Christoph Lameter
Replace places where
__get_cpu_var()is used for an address calculation withthis_cpu_ptr().
Applied to percpu/for-3.18-consistent-ops. Please let me know if this patch should be routed differently. Note that this patch was to be applied to percpu/for-3.17 but delayed due to build issues caused by cpumask_var_t.
Thanks.
Signature · 1 line
tejun
Tejun Heo
maintainerRe: [PATCH 02/41] tracing: Replace __get_cpu_var uses with this_cpu_ptr
In reply to Christoph LameterQuoted 7 lines · Christoph Lameter
Replace uses of
&__get_cpu_varfor address calculation withthis_cpu_ptr.CC Steven Rostedt <[email protected]>CC Frederic Weisbecker <[email protected]>CC Ingo Molnar <[email protected]>Acked-by Masami Hiramatsu <[email protected]>Signed-off-by Christoph Lameter <[email protected]>
Applied to percpu/for-3.18-consistent-ops. Please let me know if this patch should be routed differently. Note that this patch was to be applied to percpu/for-3.17 but delayed due to build issues caused by cpumask_var_t.
Thanks.
Signature · 1 line
tejun
Tejun Heo
maintainerRe: [PATCH 03/41] percpu: Replace __get_cpu_var with this_cpu_ptr
In reply to Christoph LameterQuoted 4 lines · Christoph Lameter
One case of using
__get_cpu_varin theget_cpu_varmacro for address calculation.Signed-off-by Christoph Lameter <[email protected]>
Applied to percpu/for-3.18-consistent-ops. Please let me know if this patch should be routed differently. Note that this patch was to be applied to percpu/for-3.17 but delayed due to build issues caused by cpumask_var_t.
Thanks.
Signature · 1 line
tejun
Tejun Heo
maintainerRe: [PATCH 04/41] kernel misc: Replace __get_cpu_var uses
In reply to Christoph LameterQuoted 4 lines · Christoph Lameter
Replace uses of
__get_cpu_varfor address calculation withthis_cpu_ptr.Signed-off-by Christoph Lameter <[email protected]>
Applied to percpu/for-3.18-consistent-ops. Please let me know if this patch should be routed differently. Note that this patch was to be applied to percpu/for-3.17 but delayed due to build issues caused by cpumask_var_t.
Thanks.
Signature · 1 line
tejun
Tejun Heo
maintainerRe: [PATCH 05/41] drivers/char/random: Replace __get_cpu_var uses
In reply to Christoph LameterQuoted 5 lines · Christoph Lameter
A single case of using
__get_cpu_varfor address calculation.Cc Arnd Bergmann <[email protected]>Cc Greg Kroah-Hartman <[email protected]>Signed-off-by Christoph Lameter <[email protected]>
Applied to percpu/for-3.18-consistent-ops. Please let me know if this patch should be routed differently. Note that this patch was to be applied to percpu/for-3.17 but delayed due to build issues caused by cpumask_var_t.
Thanks.
Signature · 1 line
tejun
Tejun Heo
maintainerRe: [PATCH 06/41] drivers/cpuidle: Replace __get_cpu_var uses for address calculation
In reply to Christoph LameterQuoted 8 lines · Christoph Lameter
All of these are for address calculation. Replace with
this_cpu_ptr().Cc Daniel Lezcano <[email protected]>[cpufreq changes]Signed-off-by Christoph Lameter <[email protected]>
Applied to percpu/for-3.18-consistent-ops. Please let me know if this patch should be routed differently. Note that this patch was to be applied to percpu/for-3.17 but delayed due to build issues caused by cpumask_var_t.
Thanks.
Signature · 1 line
tejun
Tejun Heo
maintainerRe: [PATCH 08/41] drivers/leds: Replace __get_cpu_var use through this_cpu_ptr
In reply to Christoph LameterQuoted 4 lines · Christoph Lameter
Use
this_cpu_ptrfor the address calculation instead of__get_cpu_var.Acked-by Bryan Wu <[email protected]>Signed-off-by Christoph Lameter <[email protected]>
Applied to percpu/for-3.18-consistent-ops. Please let me know if this patch should be routed differently. Note that this patch was to be applied to percpu/for-3.17 but delayed due to build issues caused by cpumask_var_t.
Thanks.
Signature · 1 line
tejun
Tejun Heo
maintainerRe: [PATCH 09/41] drivers/clocksource: Replace __get_cpu_var used for address calculation
In reply to Christoph LameterQuoted 4 lines · Christoph Lameter
Replace
__get_cpu_varused for address calculation withthis_cpu_ptr.Acked-by James Hogan <[email protected]>Signed-off-by Christoph Lameter <[email protected]>
Applied to percpu/for-3.18-consistent-ops. Please let me know if this patch should be routed differently. Note that this patch was to be applied to percpu/for-3.17 but delayed due to build issues caused by cpumask_var_t.
Thanks.
Signature · 1 line
tejun
Tejun Heo
maintainerRe: [PATCH 10/41] staging/zsmalloc: Replace instances of using __get_cpu_var for address calculation
In reply to Christoph LameterQuoted 4 lines · Christoph Lameter
Replace the use of
__get_cpu_varfor address calculation withthis_cpu_ptr().Signed-off-by Christoph Lameter <[email protected]>
Applied to percpu/for-3.18-consistent-ops. Please let me know if this patch should be routed differently. Note that this patch was to be applied to percpu/for-3.17 but delayed due to build issues caused by cpumask_var_t.
Thanks.
Signature · 1 line
tejun
Re: [PATCH 02/41] tracing: Replace __get_cpu_var uses with this_cpu_ptr
In reply to Tejun HeoQuoted 9 lines · On Tue, 26 Aug 2014 13:59:55 -0400 Tejun Heo <[email protected]> wrote:
Quoted 7 lines · Christoph Lameter
Replace uses of
&__get_cpu_varfor address calculation withthis_cpu_ptr.CC Steven Rostedt <[email protected]>CC Frederic Weisbecker <[email protected]>CC Ingo Molnar <[email protected]>Acked-by Masami Hiramatsu <[email protected]>Signed-off-by Christoph Lameter <[email protected]>
Quoted 5 lines
Applied to percpu/for-3.18-consistent-ops. Please let me know if this patch should be routed differently. Note that this patch was to be applied to percpu/for-3.17 but delayed due to build issues caused by
cpumask_var_t.
Sorry for the late reply, had other issues to attend to.
-- Steve