Use this_cpu_ops
Subsystems (23)
cover Use this_cpu_ops
Use the this_cpu_ops in various parts of the kernel.
This patchset does not require any upgrade of the percpu infrastructure. The patches included here are simple replacements of operations on percpu variables with the more efficient and compact code provide by this_cpu_ops.
These patches can be individually applied and are not depending on any other patches. I would suggest they go through Tejun's percpu tree if they are not picked up by the maintainers of the subsystems affected.
PATCH 1/11 percpucounter: Optimize __percpu_counter_add a bit through the use of this_cpu() options.
The this_cpu_* options can be used to optimize __percpu_counter_add a bit. Avoids some address arithmetic and saves 12 bytes.
Before:
00000000000001d3 <__percpu_counter_add>:
1d3: 55 push %rbp 1d4: 48 89 e5 mov %rsp,%rbp 1d7: 41 55 push %r13 1d9: 41 54 push %r12 1db: 53 push %rbx 1dc: 48 89 fb mov %rdi,%rbx 1df: 48 83 ec 08 sub $0x8,%rsp 1e3: 4c 8b 67 30 mov 0x30(%rdi),%r12 1e7: 65 4c 03 24 25 00 00 add %gs:0x0,%r12 1ee: 00 00 1f0: 4d 63 2c 24 movslq (%r12),%r13 1f4: 48 63 c2 movslq %edx,%rax 1f7: 49 01 f5 add %rsi,%r13 1fa: 49 39 c5 cmp %rax,%r13 1fd: 7d 0a jge 209 <__percpu_counter_add+0x36> 1ff: f7 da neg %edx 201: 48 63 d2 movslq %edx,%rdx 204: 49 39 d5 cmp %rdx,%r13 207: 7f 1e jg 227 <__percpu_counter_add+0x54> 209: 48 89 df mov %rbx,%rdi 20c: e8 00 00 00 00 callq 211 <__percpu_counter_add+0x3e> 211: 4c 01 6b 18 add %r13,0x18(%rbx) 215: 48 89 df mov %rbx,%rdi 218: 41 c7 04 24 00 00 00 movl $0x0,(%r12) 21f: 00 220: e8 00 00 00 00 callq 225 <__percpu_counter_add+0x52> 225: eb 04 jmp 22b <__percpu_counter_add+0x58> 227: 45 89 2c 24 mov %r13d,(%r12) 22b: 5b pop %rbx 22c: 5b pop %rbx 22d: 41 5c pop %r12 22f: 41 5d pop %r13 231: c9 leaveq 232: c3 retq
After:
00000000000001d3 <__percpu_counter_add>:
1d3: 55 push %rbp 1d4: 48 63 ca movslq %edx,%rcx 1d7: 48 89 e5 mov %rsp,%rbp 1da: 41 54 push %r12 1dc: 53 push %rbx 1dd: 48 89 fb mov %rdi,%rbx 1e0: 48 8b 47 30 mov 0x30(%rdi),%rax 1e4: 65 44 8b 20 mov %gs:(%rax),%r12d 1e8: 4d 63 e4 movslq %r12d,%r12 1eb: 49 01 f4 add %rsi,%r12 1ee: 49 39 cc cmp %rcx,%r12 1f1: 7d 0a jge 1fd <__percpu_counter_add+0x2a> 1f3: f7 da neg %edx 1f5: 48 63 d2 movslq %edx,%rdx 1f8: 49 39 d4 cmp %rdx,%r12 1fb: 7f 21 jg 21e <__percpu_counter_add+0x4b> 1fd: 48 89 df mov %rbx,%rdi 200: e8 00 00 00 00 callq 205 <__percpu_counter_add+0x32> 205: 4c 01 63 18 add %r12,0x18(%rbx) 209: 48 8b 43 30 mov 0x30(%rbx),%rax 20d: 48 89 df mov %rbx,%rdi 210: 65 c7 00 00 00 00 00 movl $0x0,%gs:(%rax) 217: e8 00 00 00 00 callq 21c <__percpu_counter_add+0x49> 21c: eb 04 jmp 222 <__percpu_counter_add+0x4f> 21e: 65 44 89 20 mov %r12d,%gs:(%rax) 222: 5b pop %rbx 223: 41 5c pop %r12
225: c9 leaveq
226: c3 retq
--- lib/percpu_counter.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-)
lib/percpu_counter.c +3 -5
@@ -72,18 +72,16 @@ EXPORT_SYMBOL(percpu_counter_set);void __percpu_counter_add (struct percpu_counter *fbc,s64 amount,s32 batch) {s64 count; -s32 *pcount;preempt_disable (); - pcount =this_cpu_ptr (fbc->counters); - count = *pcount + amount; + count =__this_cpu_read (*fbc->counters) + amount;if (count >= batch || count <= -batch) {spin_lock (&fbc->lock); fbc->count += count; - *pcount =0 ; +__this_cpu_write (*fbc->counters,0 );spin_unlock (&fbc->lock); }else { - *pcount = count; +__this_cpu_write (*fbc->counters, count); }preempt_enable (); }
PATCH 5/11 core: Replace __get_cpu_var with __this_cpu_read if not used for an address.
In reply to Christoph Lameter__get_cpu_var() can be replaced with this_cpu_read and will then use a single read instruction with implied address calculation to access the correct per cpu instance.
However, the address of a per cpu variable passed to __this_cpu_read() cannot be determed (since its an implied address conversion through segment prefixes). Therefore apply this only to uses of __get_cpu_var where the addres of the variable is not used.
--- include/asm-generic/irq_regs.h | 8 +++---- include/linux/elevator.h | 12 ++--------- include/linux/kernel_stat.h | 2 - kernel/exit.c | 2 - kernel/fork.c | 2 - kernel/hrtimer.c | 2 - kernel/printk.c | 4 +-- kernel/rcutree.c | 4 +-- kernel/softirq.c | 44 ++++++++++++++++++++--------------------- kernel/time/tick-common.c | 2 - kernel/time/tick-oneshot.c | 4 +-- kernel/watchdog.c | 36 ++++++++++++++++----------------- mm/slab.c | 6 ++--- 13 files changed, 61 insertions(+), 67 deletions(-)
mm/slab.c +3 -3
@@ -829,12 +829,12 @@ static void init_reap_node(int cpu)static void next_reap_node (void ) { -int node =__get_cpu_var (slab_reap_node); +int node =__this_cpu_read (slab_reap_node); node =next_node (node, node_online_map);if (unlikely (node >=MAX_NUMNODES )) node =first_node (node_online_map); -__get_cpu_var (slab_reap_node) = node; +__this_cpu_write (slab_reap_node, node); }#else
@@ -1012,7 +1012,7 @@ static void __drain_alien_cache(struct k*/ static void reap_alien (struct kmem_cache *cachep,struct kmem_list3 *l3) { -int node =__get_cpu_var (slab_reap_node); +int node =__this_cpu_read (slab_reap_node);if (l3->alien) {struct array_cache *ac = l3->alien[node];
kernel/rcutree.c +2 -2
@@ -367,8 +367,8 @@ void rcu_irq_exit(void)WARN_ON_ONCE (rdtp->dynticks &0x1 );/* If the interrupt queued a callback, get out of dyntick mode. */ -if (__get_cpu_var (rcu_sched_data).nxtlist || -__get_cpu_var (rcu_bh_data).nxtlist) +if (__this_cpu_read (rcu_sched_data.nxtlist) || +__this_cpu_read (rcu_bh_data.nxtlist))set_need_resched (); }
kernel/softirq.c +22 -22
105 lines not on this page · raw message
kernel/time/tick-common.c +1 -1
@@ -49,7 +49,7 @@ struct tick_device *tick_get_device(int*/ int tick_is_oneshot_available (void ) { -struct clock_event_device *dev =__get_cpu_var (tick_cpu_device).evtdev; +struct clock_event_device *dev =__this_cpu_read (tick_cpu_device.evtdev);return dev && (dev->features &CLOCK_EVT_FEAT_ONESHOT ); }
kernel/time/tick-oneshot.c +2 -2
@@ -95,7 +95,7 @@ int tick_dev_program_event(struct clock_*/ int tick_program_event (ktime_t expires,int force) { -struct clock_event_device *dev =__get_cpu_var (tick_cpu_device).evtdev; +struct clock_event_device *dev =__this_cpu_read (tick_cpu_device.evtdev);return tick_dev_program_event (dev, expires, force); }
@@ -167,7 +167,7 @@ int tick_oneshot_mode_active(void)int ret;local_irq_save (flags); - ret =__get_cpu_var (tick_cpu_device).mode ==TICKDEV_MODE_ONESHOT ; + ret =__this_cpu_read (tick_cpu_device.mode) ==TICKDEV_MODE_ONESHOT ;local_irq_restore (flags);return ret;
kernel/watchdog.c +18 -18
@@ -116,12 +116,12 @@ static void __touch_watchdog(void) {int this_cpu =smp_processor_id (); -__get_cpu_var (watchdog_touch_ts) =get_timestamp (this_cpu); +__this_cpu_write (watchdog_touch_ts,get_timestamp (this_cpu)); }void touch_softlockup_watchdog (void ) { -__raw_get_cpu_var (watchdog_touch_ts) =0 ; +__this_cpu_write (watchdog_touch_ts,0 ); }EXPORT_SYMBOL (touch_softlockup_watchdog);
@@ -165,12 +165,12 @@ void touch_softlockup_watchdog_sync(void/* watchdog detector functions */ static int is_hardlockup (void ) { -unsigned long hrint =__get_cpu_var (hrtimer_interrupts); +unsigned long hrint =__this_cpu_read (hrtimer_interrupts); -if (__get_cpu_var (hrtimer_interrupts_saved) == hrint) +if (__this_cpu_read (hrtimer_interrupts_saved) == hrint)return 1 ; -__get_cpu_var (hrtimer_interrupts_saved) = hrint; +__this_cpu_write (hrtimer_interrupts_saved, hrint);return 0 ; }#endif
@@ -203,8 +203,8 @@ static void watchdog_overflow_callback(s/* Ensure the watchdog never gets throttled */ event->hw.interrupts =0 ; -if (__get_cpu_var (watchdog_nmi_touch) == true) { -__get_cpu_var (watchdog_nmi_touch) = false; +if (__this_cpu_read (watchdog_nmi_touch) == true) { +__this_cpu_write (watchdog_nmi_touch, false);return ; }
@@ -218,7 +218,7 @@ static void watchdog_overflow_callback(sint this_cpu =smp_processor_id ();/* only print hardlockups once */ -if (__get_cpu_var (hard_watchdog_warn) == true) +if (__this_cpu_read (hard_watchdog_warn) == true)return ;if (hardlockup_panic)
@@ -226,16 +226,16 @@ static void watchdog_overflow_callback(s else WARN(1, "Watchdog detected hard LOCKUP on cpu %d", this_cpu); -__get_cpu_var (hard_watchdog_warn) = true; +__this_cpu_write (hard_watchdog_warn, true); return; } -__get_cpu_var (hard_watchdog_warn) = false; +__this_cpu_write (hard_watchdog_warn, false);return ; }static void watchdog_interrupt_count (void ) { -__get_cpu_var (hrtimer_interrupts)++; +__this_cpu_inc (hrtimer_interrupts); }#else static inline void watchdog_interrupt_count (void ) {return ; }
@@ -244,7 +244,7 @@ static inline void watchdog_interrupt_co/* watchdog kicker functions */ static enum hrtimer_restart watchdog_timer_fn (struct hrtimer *hrtimer) { -unsigned long touch_ts =__get_cpu_var (watchdog_touch_ts); +unsigned long touch_ts =__this_cpu_read (watchdog_touch_ts);struct pt_regs *regs =get_irq_regs ();int duration;
@@ -252,18 +252,18 @@ static enum hrtimer_restart watchdog_timwatchdog_interrupt_count ();/* kick the softlockup detector */ -wake_up_process (__get_cpu_var (softlockup_watchdog)); +wake_up_process (__this_cpu_read (softlockup_watchdog));/* .. and repeat */ hrtimer_forward_now (hrtimer,ns_to_ktime (get_sample_period ()));if (touch_ts ==0 ) { -if (unlikely (__get_cpu_var (softlockup_touch_sync))) { +if (unlikely (__this_cpu_read (softlockup_touch_sync))) {/* * If the time stamp was touched atomically * make sure the scheduler tick is up to date. */ -__get_cpu_var (softlockup_touch_sync) = false; +__this_cpu_write (softlockup_touch_sync, false);sched_clock_tick (); }__touch_watchdog ();
@@ -279,7 +279,7 @@ static enum hrtimer_restart watchdog_tim duration =is_softlockup (touch_ts);if (unlikely (duration)) {/* only warn once */ -if (__get_cpu_var (soft_watchdog_warn) == true) +if (__this_cpu_read (soft_watchdog_warn) == true)return HRTIMER_RESTART ; printk(KERN_ERR "BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]\n",
@@ -294,9 +294,9 @@ static enum hrtimer_restart watchdog_timif (softlockup_panic)panic ("softlockup: hung tasks" ); -__get_cpu_var (soft_watchdog_warn) = true; +__this_cpu_write (soft_watchdog_warn, true); }else -__get_cpu_var (soft_watchdog_warn) = false; +__this_cpu_write (soft_watchdog_warn, false);return HRTIMER_RESTART ; }
kernel/printk.c +2 -2
@@ -1074,8 +1074,8 @@ static DEFINE_PER_CPU(int, printk_pendinvoid printk_tick (void ) { -if (__get_cpu_var (printk_pending)) { -__get_cpu_var (printk_pending) =0 ; +if (__this_cpu_read (printk_pending)) { +__this_cpu_write (printk_pending,0 );wake_up_interruptible (&log_wait); } }
include/asm-generic/irq_regs.h +4 -4
@@ -22,15 +22,15 @@ DECLARE_PER_CPU(struct pt_regs *, __irq_static inline struct pt_regs *get_irq_regs (void ) { -return __get_cpu_var (__irq_regs); +return __this_cpu_read (__irq_regs); }static inline struct pt_regs *set_irq_regs (struct pt_regs *new_regs) { -struct pt_regs *old_regs, **pp_regs = &__get_cpu_var (__irq_regs); +struct pt_regs *old_regs; - old_regs = *pp_regs; - *pp_regs = new_regs; + old_regs =__this_cpu_read (__irq_regs); +__this_cpu_write (__irq_regs, new_regs);return old_regs; }
include/linux/elevator.h +3 -9
@@ -195,15 +195,9 @@ enum {/* * io context count accounting */ -#define elv_ioc_count_mod (name, __val) \ - do { \ - preempt_disable(); \ - __get_cpu_var(name) += (__val); \ - preempt_enable(); \ - } while (0) - -#define elv_ioc_count_inc (name) elv_ioc_count_mod(name, 1) -#define elv_ioc_count_dec (name) elv_ioc_count_mod(name, -1) +#define elv_ioc_count_mod (name, __val) this_cpu_add(name, __val) +#define elv_ioc_count_inc (name) this_cpu_inc(name) +#define elv_ioc_count_dec (name) this_cpu_dec(name)#define elv_ioc_count_read (name) \ ({ \
include/linux/kernel_stat.h +1 -1
@@ -47,7 +47,7 @@ extern unsigned long long nr_context_swi#ifndef CONFIG_GENERIC_HARDIRQS #define kstat_irqs_this_cpu (irq) \ - (kstat_this_cpu.irqs[irq]) + (this_cpu_read(kstat.irqs[irq])struct irq_desc ;
kernel/exit.c +1 -1
@@ -69,7 +69,7 @@ static void __unhash_process(struct tasklist_del_rcu (&p->tasks);list_del_init (&p->sibling); -__get_cpu_var (process_counts)--; +__this_cpu_dec (process_counts); }list_del_rcu (&p->thread_group); }
kernel/fork.c +1 -1
@@ -1282,7 +1282,7 @@ static struct task_struct *copy_process(attach_pid (p,PIDTYPE_SID ,task_session (current));list_add_tail (&p->sibling, &p->real_parent->children);list_add_tail_rcu (&p->tasks, &init_task.tasks); -__get_cpu_var (process_counts)++; +__this_cpu_inc (process_counts); }attach_pid (p,PIDTYPE_PID , pid); nr_threads++;
kernel/hrtimer.c +1 -1
@@ -497,7 +497,7 @@ static inline int hrtimer_is_hres_enable*/ static inline int hrtimer_hres_active (void ) { -return __get_cpu_var (hrtimer_bases).hres_active; +return __this_cpu_read (hrtimer_bases.hres_active); }/*
PATCH 6/11 drivers: Replace __get_cpu_var with __this_cpu_read if not used for an address.
In reply to Christoph Lameter__get_cpu_var() can be replaced with this_cpu_read and will then use a single read instruction with implied address calculation to access the correct per cpu instance.
However, the address of a per cpu variable passed to __this_cpu_read() cannot be determed (since its an implied address conversion through segment prefixes). Therefore apply this only to uses of __get_cpu_var where the addres of the variable is not used.
V3->V4:
- Move one instance of this_cpu_inc_return to a later patch so that this one can go in without percpu infrastructrure changes.
--- drivers/acpi/processor_idle.c | 6 +++--- drivers/cpuidle/cpuidle.c | 2 +- drivers/s390/cio/cio.c | 2 +- drivers/staging/speakup/fakekey.c | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-)
drivers/acpi/processor_idle.c +3 -3
@@ -746,7 +746,7 @@ static int acpi_idle_enter_c1(struct cpustruct acpi_processor *pr;struct acpi_processor_cx *cx =cpuidle_get_statedata (state); - pr =__get_cpu_var (processors); + pr =__this_cpu_read (processors);if (unlikely (!pr))return 0 ;
@@ -787,7 +787,7 @@ static int acpi_idle_enter_simple(structs64 idle_time_ns;s64 idle_time; - pr =__get_cpu_var (processors); + pr =__this_cpu_read (processors);if (unlikely (!pr))return 0 ;
@@ -864,7 +864,7 @@ static int acpi_idle_enter_bm(struct cpus64 idle_time; - pr =__get_cpu_var (processors); + pr =__this_cpu_read (processors);if (unlikely (!pr))return 0 ;
drivers/cpuidle/cpuidle.c +1 -1
@@ -49,7 +49,7 @@ static int __cpuidle_register_device(str*/ static void cpuidle_idle_call (void ) { -struct cpuidle_device *dev =__get_cpu_var (cpuidle_devices); +struct cpuidle_device *dev =__this_cpu_read (cpuidle_devices);struct cpuidle_state *target_state;int next_state;
drivers/s390/cio/cio.c +1 -1
@@ -619,7 +619,7 @@ void __irq_entry do_IRQ(struct pt_regs *s390_idle_check (regs, S390_lowcore.int_clock, S390_lowcore.async_enter_timer);irq_enter (); -__get_cpu_var (s390_idle).nohz_delay =1 ; +__this_cpu_write (s390_idle.nohz_delay,1 );if (S390_lowcore.int_clock >= S390_lowcore.clock_comparator)/* Serve timer interrupts first. */ clock_comparator_work ();
drivers/staging/speakup/fakekey.c +2 -2
@@ -78,10 +78,10 @@ void speakup_fake_down_arrow(void)/* don't change CPU */ preempt_disable (); -__get_cpu_var (reporting_keystroke) = true; + __this_cpu_write(reporting_keystroke), true);input_report_key (virt_keyboard,KEY_DOWN ,PRESSED );input_report_key (virt_keyboard,KEY_DOWN ,RELEASED ); -__get_cpu_var (reporting_keystroke) = false; +__this_cpu_write (reporting_keystroke, false);/* reenable preemption */ preempt_enable ();
PATCH 11/11 Xen: Use this_cpu_ops
In reply to Christoph LameterUse this_cpu_ops to reduce code size and simplify things in various places.
V3->V4:
Move instance of this_cpu_inc_return to a later patchset so that this patch can be applied without infrastructure changes.
--- arch/x86/xen/enlighten.c | 4 ++-- arch/x86/xen/multicalls.h | 2 +- arch/x86/xen/spinlock.c | 8 ++++---- arch/x86/xen/time.c | 8 ++++---- drivers/xen/events.c | 10 +++++----- 5 files changed, 16 insertions(+), 16 deletions(-)
arch/x86/xen/enlighten.c +2 -2
@@ -574,8 +574,8 @@ static void xen_write_idt_entry(gate_despreempt_disable (); - start =__get_cpu_var (idt_desc).address; - end = start +__get_cpu_var (idt_desc).size +1 ; + start =__this_cpu_read (idt_desc.address); + end = start +__this_cpu_read (idt_desc.size) +1 ;xen_mc_flush ();
arch/x86/xen/multicalls.h +1 -1
@@ -22,7 +22,7 @@ static inline void xen_mc_batch(void)unsigned long flags;/* need to disable interrupts until this entry is complete */ local_irq_save (flags); -__get_cpu_var (xen_mc_irq_flags) = flags; +__this_cpu_write (xen_mc_irq_flags, flags); }static inline struct multicall_space xen_mc_entry (size_t args)
arch/x86/xen/spinlock.c +4 -4
@@ -159,8 +159,8 @@ static inline struct xen_spinlock *spinn {struct xen_spinlock *prev; - prev =__get_cpu_var (lock_spinners); -__get_cpu_var (lock_spinners) = xl; + prev =__this_cpu_read (lock_spinners); +__this_cpu_write (lock_spinners, xl);wmb ();/* set lock of interest before count */
@@ -179,14 +179,14 @@ static inline void unspinning_lock(struc asm(LOCK_PREFIX " decw %0" :"+m" (xl->spinners) : :"memory" );wmb ();/* decrement count before restoring lock */ -__get_cpu_var (lock_spinners) = prev; +__this_cpu_write (lock_spinners, prev); } static noinline int xen_spin_lock_slow(struct arch_spinlock *lock, bool irq_enable) {struct xen_spinlock *xl = (struct xen_spinlock *)lock;struct xen_spinlock *prev; -int irq =__get_cpu_var (lock_kicker_irq); +int irq =__this_cpu_read (lock_kicker_irq);int ret;u64 start;
arch/x86/xen/time.c +4 -4
@@ -135,24 +135,24 @@ static void do_stolen_accounting(void)/* Add the appropriate number of ticks of stolen time, including any left-overs from last time. */ - stolen = runnable + offline +__get_cpu_var (xen_residual_stolen); + stolen = runnable + offline +__this_cpu_read (xen_residual_stolen);if (stolen <0 ) stolen =0 ; ticks =iter_div_u64_rem (stolen,NS_PER_TICK , &stolen); -__get_cpu_var (xen_residual_stolen) = stolen; +__this_cpu_write (xen_residual_stolen, stolen);account_steal_ticks (ticks);/* Add the appropriate number of ticks of blocked time, including any left-overs from last time. */ - blocked +=__get_cpu_var (xen_residual_blocked); + blocked +=__this_cpu_read (xen_residual_blocked);if (blocked <0 ) blocked =0 ; ticks =iter_div_u64_rem (blocked,NS_PER_TICK , &blocked); -__get_cpu_var (xen_residual_blocked) = blocked; +__this_cpu_write (xen_residual_blocked, blocked);account_idle_ticks (ticks); }
drivers/xen/events.c +4 -4
@@ -355,7 +355,7 @@ static void unmask_evtchn(int port)struct evtchn_unmask unmask = { .port = port }; (void )HYPERVISOR_event_channel_op (EVTCHNOP_unmask, &unmask); }else { -struct vcpu_info *vcpu_info =__get_cpu_var (xen_vcpu); +struct vcpu_info *vcpu_info =__this_cpu_read (xen_vcpu);sync_clear_bit (port, &s->evtchn_mask[0 ]);
@@ -1101,7 +1101,7 @@ static void __xen_evtchn_do_upcall(void) {int cpu =get_cpu ();struct shared_info *s = HYPERVISOR_shared_info; -struct vcpu_info *vcpu_info =__get_cpu_var (xen_vcpu); +struct vcpu_info *vcpu_info =__this_cpu_read (xen_vcpu);unsigned count; do {
@@ -1141,8 +1141,8 @@ static void __xen_evtchn_do_upcall(void)BUG_ON (!irqs_disabled ()); - count =__get_cpu_var (xed_nesting_count); -__get_cpu_var (xed_nesting_count) =0 ; + count =__this_cpu_read (xed_nesting_count); +__this_cpu_write (xed_nesting_count,0 ); }while (count !=1 || vcpu_info->evtchn_upcall_pending); out:
PATCH 9/11 Connector: Use this_cpu operations
In reply to Christoph Lameterget_seq can benefit from this_cpu_operations. Address calculation is avoided and the increment is done using an xadd.
--- drivers/connector/cn_proc.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)
drivers/connector/cn_proc.c +3 -2
@@ -43,9 +43,10 @@ static DEFINE_PER_CPU(__u32, proc_event_static inline void get_seq (__u32 *ts,int *cpu) { - *ts =get_cpu_var (proc_event_counts)++; +preempt_disable (); + *ts =__this_cpu_inc (proc_event_counts); *cpu =smp_processor_id (); -put_cpu_var (proc_event_counts); +preempt_enable (); }void proc_fork_connector (struct task_struct *task)
PATCH 10/11 fs: Use this_cpu_xx operations in buffer.c
In reply to Christoph LameterOptimize various per cpu area operations through the new this cpu operations. These operations avoid address calculations through the use of segment prefixes avoid multiple memory references through RMW instructions etc.
Reduces code size:
Before:
[email protected]$ size fs/buffer.o text data bss dec hex filename 19169 80 28 19277 4b4d fs/buffer.o
After:
[email protected]$ size fs/buffer.o text data bss dec hex filename 19138 80 28 19246 4b2e fs/buffer.o
V3->V4:
- Move the use of this_cpu_inc_return into a later patch so that this one can go in without percpu infrastructure changes.
--- fs/buffer.c | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-)
fs/buffer.c +18 -19
@@ -1270,12 +1270,10 @@ static inline void check_irqs_on(void)static void bh_lru_install (struct buffer_head *bh) {struct buffer_head *evictee =NULL ; -struct bh_lru *lru;check_irqs_on ();bh_lru_lock (); - lru = &__get_cpu_var (bh_lrus); -if (lru->bhs[0 ] != bh) { +if (__this_cpu_read (bh_lrus.bhs[0 ]) != bh) {struct buffer_head *bhs[BH_LRU_SIZE ];int in;int out =0 ;
@@ -1283,7 +1281,8 @@ static void bh_lru_install(struct bufferget_bh (bh); bhs[out++] = bh;for (in =0 ; in <BH_LRU_SIZE ; in++) { -struct buffer_head *bh2 = lru->bhs[in]; +struct buffer_head *bh2 = +__this_cpu_read (bh_lrus.bhs[in]);if (bh2 == bh) {__brelse (bh2);
@@ -1298,7 +1297,7 @@ static void bh_lru_install(struct buffer }while (out <BH_LRU_SIZE ) bhs[out++] =NULL ; -memcpy (lru->bhs, bhs,sizeof (bhs)); +memcpy (__this_cpu_ptr (&bh_lrus.bhs), bhs,sizeof (bhs)); }bh_lru_unlock ();
@@ -1313,23 +1312,22 @@ static struct buffer_head * lookup_bh_lru(struct block_device *bdev, sector_t block, unsigned size) { struct buffer_head *ret = NULL; -struct bh_lru *lru;unsigned int i;check_irqs_on ();bh_lru_lock (); - lru = &__get_cpu_var (bh_lrus);for (i =0 ; i <BH_LRU_SIZE ; i++) { -struct buffer_head *bh = lru->bhs[i]; +struct buffer_head *bh =__this_cpu_read (bh_lrus.bhs[i]);if (bh && bh->b_bdev == bdev && bh->b_blocknr == block && bh->b_size == size) {if (i) {while (i) { - lru->bhs[i] = lru->bhs[i -1 ]; +__this_cpu_write (bh_lrus.bhs[i], +__this_cpu_read (bh_lrus.bhs[i -1 ])); i--; } - lru->bhs[0 ] = bh; +__this_cpu_write (bh_lrus.bhs[0 ], bh); }get_bh (bh); ret = bh;
@@ -3203,22 +3201,23 @@ static void recalc_bh_state(void)int i;int tot =0 ; -if (__get_cpu_var (bh_accounting).ratelimit++ <4096 ) +if (__get_cpu_var (bh_accounting).ratelimit++ <4096 )return ; -__get_cpu_var (bh_accounting).ratelimit =0 ; +__this_cpu_write (bh_accounting.ratelimit,0 );for_each_online_cpu (i) tot +=per_cpu (bh_accounting, i).nr; buffer_heads_over_limit = (tot > max_buffer_heads); } - +struct buffer_head *alloc_buffer_head (gfp_t gfp_flags) {struct buffer_head *ret =kmem_cache_zalloc (bh_cachep, gfp_flags);if (ret) {INIT_LIST_HEAD (&ret->b_assoc_buffers); -get_cpu_var (bh_accounting).nr++; +preempt_disable (); +__this_cpu_inc (bh_accounting.nr);recalc_bh_state (); -put_cpu_var (bh_accounting); +preempt_enable (); }return ret; }
@@ -3228,9 +3227,10 @@ void free_buffer_head(struct buffer_head {BUG_ON (!list_empty (&bh->b_assoc_buffers));kmem_cache_free (bh_cachep, bh); -get_cpu_var (bh_accounting).nr--; +preempt_disable (); +__this_cpu_dec (bh_accounting.nr);recalc_bh_state (); -put_cpu_var (bh_accounting); +preempt_enable (); }EXPORT_SYMBOL (free_buffer_head);
@@ -3243,9 +3243,8 @@ static void buffer_exit_cpu(int cpu)brelse (b->bhs[i]); b->bhs[i] =NULL ; } -get_cpu_var (bh_accounting).nr +=per_cpu (bh_accounting, cpu).nr; +this_cpu_add (bh_accounting.nr,per_cpu (bh_accounting, cpu).nr);per_cpu (bh_accounting, cpu).nr =0 ; -put_cpu_var (bh_accounting); } static int buffer_cpu_notify(struct notifier_block *self,
PATCH 8/11 Fakekey: Simplify speakup_fake_key_pressed through this_cpu_ops
In reply to Christoph LameterThe whole function can be expressed as a simple this_cpu_read() operation. The function overhead is now likely multiple times that of the single instruction that is executed in it.
--- drivers/staging/speakup/fakekey.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-)
drivers/staging/speakup/fakekey.c +1 -6
@@ -96,10 +96,5 @@ void speakup_fake_down_arrow(void)*/ bool speakup_fake_key_pressed (void ) { -bool is_pressed; - - is_pressed =get_cpu_var (reporting_keystroke); -put_cpu_var (reporting_keystroke); - -return is_pressed; +return this_cpu_read (reporting_keystroke); }
PATCH 7/11 kprobes: Use this_cpu_ops
In reply to Christoph LameterUse this_cpu ops in various places to optimize per cpu data access.
--- arch/x86/kernel/kprobes.c | 14 +++++++------- include/linux/kprobes.h | 4 ++-- kernel/kprobes.c | 8 ++++---- 3 files changed, 13 insertions(+), 13 deletions(-)
arch/x86/kernel/kprobes.c +7 -7
@@ -403,7 +403,7 @@ static void __kprobes save_previous_kprostatic void __kprobes restore_previous_kprobe (struct kprobe_ctlblk *kcb) { -__get_cpu_var (current_kprobe) = kcb->prev_kprobe.kp; +__this_cpu_write (current_kprobe, kcb->prev_kprobe.kp); kcb->kprobe_status = kcb->prev_kprobe.status; kcb->kprobe_old_flags = kcb->prev_kprobe.old_flags; kcb->kprobe_saved_flags = kcb->prev_kprobe.saved_flags;
@@ -412,7 +412,7 @@ static void __kprobes restore_previous_kstatic void __kprobes set_current_kprobe (struct kprobe *p,struct pt_regs *regs,struct kprobe_ctlblk *kcb) { -__get_cpu_var (current_kprobe) = p; +__this_cpu_write (current_kprobe, p); kcb->kprobe_saved_flags = kcb->kprobe_old_flags = (regs->flags & (X86_EFLAGS_TF |X86_EFLAGS_IF ));if (is_IF_modifier (p->ainsn.insn))
@@ -586,7 +586,7 @@ static int __kprobes kprobe_handler(strupreempt_enable_no_resched ();return 1 ; }else if (kprobe_running ()) { - p =__get_cpu_var (current_kprobe); + p =__this_cpu_read (current_kprobe);if (p->break_handler && p->break_handler (p, regs)) {setup_singlestep (p, regs, kcb,0 );return 1 ;
@@ -759,11 +759,11 @@ static __used __kprobes void *trampoline orig_ret_address = (unsigned long )ri->ret_addr;if (ri->rp && ri->rp->handler) { -__get_cpu_var (current_kprobe) = &ri->rp->kp; +__this_cpu_write (current_kprobe, &ri->rp->kp);get_kprobe_ctlblk ()->kprobe_status =KPROBE_HIT_ACTIVE ; ri->ret_addr = correct_ret_addr; ri->rp->handler (ri, regs); -__get_cpu_var (current_kprobe) =NULL ; +__this_cpu_write (current_kprobe,NULL ); }recycle_rp_inst (ri, &empty_rp);
@@ -1198,10 +1198,10 @@ static void __kprobes optimized_callback regs->ip = (unsigned long )op->kp.addr +INT3_SIZE ; regs->orig_ax = ~0UL ; -__get_cpu_var (current_kprobe) = &op->kp; +__this_cpu_write (current_kprobe, &op->kp); kcb->kprobe_status =KPROBE_HIT_ACTIVE ;opt_pre_handler (&op->kp, regs); -__get_cpu_var (current_kprobe) =NULL ; +__this_cpu_write (current_kprobe,NULL ); }preempt_enable_no_resched (); }
include/linux/kprobes.h +2 -2
@@ -303,12 +303,12 @@ struct hlist_head * kretprobe_inst_table/* kprobe_running() will just return the current_kprobe on this CPU */ static inline struct kprobe *kprobe_running (void ) { -return (__get_cpu_var (current_kprobe)); +return (__this_cpu_read (current_kprobe)); }static inline void reset_current_kprobe (void ) { -__get_cpu_var (current_kprobe) =NULL ; +__this_cpu_write (current_kprobe,NULL ); }static inline struct kprobe_ctlblk *get_kprobe_ctlblk (void )
kernel/kprobes.c +4 -4
@@ -317,12 +317,12 @@ void __kprobes free_optinsn_slot(kprobe_/* We have preemption disabled.. so it is safe to use __ versions */ static inline void set_kprobe_instance (struct kprobe *kp) { -__get_cpu_var (kprobe_instance) = kp; +__this_cpu_write (kprobe_instance, kp); }static inline void reset_kprobe_instance (void ) { -__get_cpu_var (kprobe_instance) =NULL ; +__this_cpu_write (kprobe_instance,NULL ); }/*
@@ -775,7 +775,7 @@ static void __kprobes aggr_post_handler(static int __kprobes aggr_fault_handler (struct kprobe *p,struct pt_regs *regs,int trapnr) { -struct kprobe *cur =__get_cpu_var (kprobe_instance); +struct kprobe *cur =__this_cpu_read (kprobe_instance);/* * if we faulted "during" the execution of a user specified
@@ -790,7 +790,7 @@ static int __kprobes aggr_fault_handler(static int __kprobes aggr_break_handler (struct kprobe *p,struct pt_regs *regs) { -struct kprobe *cur =__get_cpu_var (kprobe_instance); +struct kprobe *cur =__this_cpu_read (kprobe_instance);int ret =0 ;if (cur && cur->break_handler) {
PATCH 4/11 x86: Use this_cpu_ops for current_cpu_data accesses
In reply to Christoph LameterCurrent_cpu_data accesses are per cpu accesses. We can also use this_cpu_ops if a scalar is retrieved.
--- arch/x86/kernel/cpu/cpufreq/powernow-k8.c | 2 +- arch/x86/kernel/cpu/intel_cacheinfo.c | 4 ++-- arch/x86/kernel/smpboot.c | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-)
arch/x86/kernel/smpboot.c +4 -4
@@ -430,7 +430,7 @@ void __cpuinit set_cpu_sibling_map(int ccpumask_set_cpu (cpu, c->llc_shared_map); -if (current_cpu_data.x86_max_cores ==1 ) { +if (__this_cpu_read (cpu_info.x86_max_cores) ==1 ) {cpumask_copy (cpu_core_mask (cpu),cpu_sibling_mask (cpu)); c->booted_cores =1 ;return ;
@@ -1377,7 +1377,7 @@ void play_dead_common(void)mb ();/* Ack it */ -__get_cpu_var (cpu_state) =CPU_DEAD ; +__this_cpu_write (cpu_state,CPU_DEAD );/* * With physical CPU hotplug, we should halt the cpu
@@ -1401,7 +1401,7 @@ static inline void mwait_play_dead(void)return ;if (!cpu_has (¤t_cpu_data,X86_FEATURE_CLFLSH ))return ; -if (current_cpu_data.cpuid_level <CPUID_MWAIT_LEAF ) +if (__this_cpu_read (cpu_info.cpuid_level) <CPUID_MWAIT_LEAF )return ; eax =CPUID_MWAIT_LEAF ;
@@ -1452,7 +1452,7 @@ static inline void mwait_play_dead(void)static inline void hlt_play_dead (void ) { -if (current_cpu_data.x86 >=4 ) +if (__this_cpu_read (cpu_info.x86) >=4 )wbinvd ();while (1 ) {
arch/x86/kernel/cpu/cpufreq/powernow-k8.c +1 -1
@@ -521,7 +521,7 @@ static void check_supported_cpu(void *_r *rc = -ENODEV ; -if (current_cpu_data.x86_vendor !=X86_VENDOR_AMD ) +if (__this_cpu_read (cpu_info.x86_vendor) !=X86_VENDOR_AMD )return ; eax =cpuid_eax (CPUID_PROCESSOR_SIGNATURE );
arch/x86/kernel/cpu/intel_cacheinfo.c +2 -2
@@ -266,7 +266,7 @@ amd_cpuid4(int leaf, union _cpuid4_leaf_ line_size = l2.line_size; lines_per_tag = l2.lines_per_tag;/* cpu_data has errata corrections for K7 applied */ - size_in_kb = current_cpu_data.x86_cache_size; + size_in_kb =__this_cpu_read (cpu_info.x86_cache_size);break ;case 3 :if (!l3.val)
@@ -288,7 +288,7 @@ amd_cpuid4(int leaf, union _cpuid4_leaf_ eax->split.type = types[leaf]; eax->split.level = levels[leaf]; eax->split.num_threads_sharing =0 ; - eax->split.num_cores_on_die = current_cpu_data.x86_max_cores -1 ; + eax->split.num_cores_on_die =__this_cpu_read (cpu_info.x86_max_cores) -1 ;if (assoc ==0xffff )
PATCH 2/11 vmstat: Optimize zone counter modifications through the use of this cpu operations
In reply to Christoph Lameterthis cpu operations can be used to slightly optimize the function. The changes will avoid some address calculations and replace them with the use of the percpu segment register.
If one would have this_cpu_inc_return and this_cpu_dec_return then it would be possible to optimize inc_zone_page_state and dec_zone_page_state even more.
V1->V2:
- Fix __dec_zone_state overflow handling
- Use s8 variables for temporary storage.
V2->V3:
- Put __percpu annotations in correct places.
--- mm/vmstat.c | 56 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 32 insertions(+), 24 deletions(-)
mm/vmstat.c +32 -24
@@ -167,18 +167,20 @@ static void refresh_zone_stat_thresholdsvoid __mod_zone_page_state (struct zone *zone,enum zone_stat_item item,int delta) { -struct per_cpu_pageset *pcp =this_cpu_ptr (zone->pageset); - -s8 *p = pcp->vm_stat_diff + item; +struct per_cpu_pageset __percpu *pcp = zone->pageset; +s8 __percpu *p = pcp->vm_stat_diff + item;long x; +long t; + + x = delta +__this_cpu_read (*p); - x = delta + *p; + t =__this_cpu_read (pcp->stat_threshold); -if (unlikely (x > pcp->stat_threshold || x < -pcp->stat_threshold)) { +if (unlikely (x > t || x < -t)) {zone_page_state_add (x, zone, item); x =0 ; } - *p = x; +__this_cpu_write (*p, x); }EXPORT_SYMBOL (__mod_zone_page_state);
@@ -221,16 +223,19 @@ EXPORT_SYMBOL(mod_zone_page_state);*/ void __inc_zone_state (struct zone *zone,enum zone_stat_item item) { -struct per_cpu_pageset *pcp =this_cpu_ptr (zone->pageset); -s8 *p = pcp->vm_stat_diff + item; - - (*p)++; +struct per_cpu_pageset __percpu *pcp = zone->pageset; +s8 __percpu *p = pcp->vm_stat_diff + item; +s8 v, t; + +__this_cpu_inc (*p); + + v =__this_cpu_read (*p); + t =__this_cpu_read (pcp->stat_threshold); +if (unlikely (v > t)) { +s8 overstep = t >>1 ; -if (unlikely (*p > pcp->stat_threshold)) { -int overstep = pcp->stat_threshold /2 ; - -zone_page_state_add (*p + overstep, zone, item); - *p = -overstep; +zone_page_state_add (v + overstep, zone, item); +__this_cpu_write (*p, - overstep); } }
@@ -242,16 +247,19 @@ EXPORT_SYMBOL(__inc_zone_page_state);void __dec_zone_state (struct zone *zone,enum zone_stat_item item) { -struct per_cpu_pageset *pcp =this_cpu_ptr (zone->pageset); -s8 *p = pcp->vm_stat_diff + item; - - (*p)--; - -if (unlikely (*p < - pcp->stat_threshold)) { -int overstep = pcp->stat_threshold /2 ; +struct per_cpu_pageset __percpu *pcp = zone->pageset; +s8 __percpu *p = pcp->vm_stat_diff + item; +s8 v, t; + +__this_cpu_dec (*p); + + v =__this_cpu_read (*p); + t =__this_cpu_read (pcp->stat_threshold); +if (unlikely (v < - t)) { +s8 overstep = t >>1 ; -zone_page_state_add (*p - overstep, zone, item); - *p = overstep; +zone_page_state_add (v - overstep, zone, item); +__this_cpu_write (*p, overstep); } }
PATCH 3/11 x86: Use this_cpu_ops to optimize code
In reply to Christoph LameterGo through x86 code and replace __get_cpu_var and get_cpu_var instances that refer to a scalar and are not used for address determinations.
--- arch/x86/include/asm/debugreg.h | 2 +- arch/x86/kernel/apic/io_apic.c | 4 ++-- arch/x86/kernel/apic/nmi.c | 24 ++++++++++++------------ arch/x86/kernel/apic/x2apic_uv_x.c | 8 ++++---- arch/x86/kernel/cpu/cpufreq/powernow-k8.c | 2 +- arch/x86/kernel/cpu/mcheck/mce.c | 6 +++--- arch/x86/kernel/cpu/perf_event.c | 27 +++++++++++---------------- arch/x86/kernel/cpu/perf_event_intel.c | 4 ++-- arch/x86/kernel/ftrace.c | 6 +++--- arch/x86/kernel/hw_breakpoint.c | 12 ++++++------ arch/x86/kernel/irq.c | 6 +++--- arch/x86/kernel/irq_32.c | 4 ++-- arch/x86/kernel/tsc.c | 2 +- arch/x86/kvm/x86.c | 8 ++++---- arch/x86/oprofile/nmi_int.c | 2 +- 15 files changed, 56 insertions(+), 61 deletions(-)
arch/x86/include/asm/debugreg.h +1 -1
@@ -94,7 +94,7 @@ static inline void hw_breakpoint_disablestatic inline int hw_breakpoint_active (void ) { -return __get_cpu_var (cpu_dr7) &DR_GLOBAL_ENABLE_MASK ; +return __this_cpu_read (cpu_dr7) &DR_GLOBAL_ENABLE_MASK ; }extern void aout_dump_debugregs (struct user *dump);
arch/x86/kernel/apic/io_apic.c +2 -2
@@ -2302,7 +2302,7 @@ asmlinkage void smp_irq_move_cleanup_intunsigned int irr;struct irq_desc *desc;struct irq_cfg *cfg; - irq =__get_cpu_var (vector_irq)[vector]; + irq =__this_cpu_read (vector_irq[vector]);if (irq ==-1 )continue ;
@@ -2336,7 +2336,7 @@ asmlinkage void smp_irq_move_cleanup_int apic->send_IPI_self (IRQ_MOVE_CLEANUP_VECTOR ); goto unlock; } -__get_cpu_var (vector_irq)[vector] =-1 ; +__this_cpu_write (vector_irq[vector],-1 ); unlock:raw_spin_unlock (&desc->lock); }
arch/x86/kernel/apic/nmi.c +12 -12
@@ -306,12 +306,12 @@ void acpi_nmi_disable(void)*/ void cpu_nmi_set_wd_enabled (void ) { -__get_cpu_var (wd_enabled) =1 ; +__this_cpu_write (wd_enabled,1 ); }void setup_apic_nmi_watchdog (void *unused) { -if (__get_cpu_var (wd_enabled)) +if (__this_cpu_read (wd_enabled))return ;/* cheap hack to support suspend/resume */
@@ -322,12 +322,12 @@ void setup_apic_nmi_watchdog(void *unuseswitch (nmi_watchdog) {case NMI_LOCAL_APIC :if (lapic_watchdog_init (nmi_hz) <0 ) { -__get_cpu_var (wd_enabled) =0 ; +__this_cpu_write (wd_enabled,0 );return ; }/* FALL THROUGH */ case NMI_IO_APIC : -__get_cpu_var (wd_enabled) =1 ; +__this_cpu_write (wd_enabled,1 );atomic_inc (&nmi_active); } }
@@ -337,13 +337,13 @@ void stop_apic_nmi_watchdog(void *unused/* only support LOCAL and IO APICs for now */ if (!nmi_watchdog_active ())return ; -if (__get_cpu_var (wd_enabled) ==0 ) +if (__this_cpu_read (wd_enabled) ==0 )return ;if (nmi_watchdog ==NMI_LOCAL_APIC )lapic_watchdog_stop ();else __acpi_nmi_disable (NULL ); -__get_cpu_var (wd_enabled) =0 ; +__this_cpu_write (wd_enabled,0 );atomic_dec (&nmi_active); }
@@ -403,8 +403,8 @@ nmi_watchdog_tick(struct pt_regs *regs, sum =get_timer_irqs (cpu); -if (__get_cpu_var (nmi_touch)) { -__get_cpu_var (nmi_touch) =0 ; +if (__this_cpu_read (nmi_touch)) { +__this_cpu_write (nmi_touch,0 ); touched =1 ; }
@@ -427,7 +427,7 @@ nmi_watchdog_tick(struct pt_regs *regs, touched =1 ;/* if the none of the timers isn't firing, this cpu isn't doing much */ -if (!touched &&__get_cpu_var (last_irq_sum) == sum) { +if (!touched &&__this_cpu_read (last_irq_sum) == sum) {/* * Ayiee, looks like this CPU is stuck ... * wait a few IRQs (5 seconds) before doing the oops ...
@@ -440,12 +440,12 @@ nmi_watchdog_tick(struct pt_regs *regs,die_nmi ("BUG: NMI Watchdog detected LOCKUP" , regs, panic_on_timeout); }else { -__get_cpu_var (last_irq_sum) = sum; +__this_cpu_write (last_irq_sum, sum);__this_cpu_write (alert_counter,0 ); }/* see if the nmi watchdog went off */ -if (!__get_cpu_var (wd_enabled)) +if (!__this_cpu_read (wd_enabled))return rc;switch (nmi_watchdog) {case NMI_LOCAL_APIC :
@@ -467,7 +467,7 @@ nmi_watchdog_tick(struct pt_regs *regs,static void enable_ioapic_nmi_watchdog_single (void *unused) { -__get_cpu_var (wd_enabled) =1 ; +__this_cpu_write (wd_enabled,1 );atomic_inc (&nmi_active);__acpi_nmi_enable (NULL ); }
arch/x86/kernel/apic/x2apic_uv_x.c +4 -4
@@ -118,8 +118,8 @@ static int __init uv_acpi_madt_oem_checkelse if (!strcmp (oem_table_id,"UVX" )) uv_system_type =UV_X2APIC ;else if (!strcmp (oem_table_id,"UVH" )) { -__get_cpu_var (x2apic_extra_bits) = - nodeid << (uvh_apicid.s.pnode_shift -1 ); +__this_cpu_write (x2apic_extra_bits, + nodeid << (uvh_apicid.s.pnode_shift -1 )); uv_system_type =UV_NON_UNIQUE_APIC ;uv_set_apicid_hibit ();return 1 ;
@@ -284,7 +284,7 @@ static unsigned int x2apic_get_apic_id(uunsigned int id;WARN_ON (preemptible () &&num_online_cpus () >1 ); - id = x |__get_cpu_var (x2apic_extra_bits); + id = x |__this_cpu_read (x2apic_extra_bits);return id; }
@@ -376,7 +376,7 @@ struct apic __refdata apic_x2apic_uv_x =static __cpuinit void set_x2apic_extra_bits (int pnode) { -__get_cpu_var (x2apic_extra_bits) = (pnode <<6 ); +__this_cpu_write (x2apic_extra_bits, (pnode <<6 )); }/*
arch/x86/kernel/cpu/cpufreq/powernow-k8.c +1 -1
@@ -1377,7 +1377,7 @@ static int __devexit powernowk8_cpu_exitstatic void query_values_on_cpu (void *_err) {int *err = _err; -struct powernow_k8_data *data =__get_cpu_var (powernow_data); +struct powernow_k8_data *data =__this_cpu_read (powernow_data); *err =query_current_values_with_pending_wait (data); }
arch/x86/kernel/cpu/mcheck/mce.c +3 -3
@@ -326,7 +326,7 @@ static void mce_panic(char *msg, structstatic int msr_to_offset (u32 msr) { -unsigned bank =__get_cpu_var (injectm.bank); +unsigned bank =__this_cpu_read (injectm.bank);if (msr == rip_msr)return offsetof(struct mce , ip);
@@ -346,7 +346,7 @@ static u64 mce_rdmsrl(u32 msr) {u64 v; -if (__get_cpu_var (injectm).finished) { +if (__this_cpu_read (injectm.finished)) {int offset =msr_to_offset (msr);if (offset <0 )
@@ -369,7 +369,7 @@ static u64 mce_rdmsrl(u32 msr)static void mce_wrmsrl (u32 msr,u64 v) { -if (__get_cpu_var (injectm).finished) { +if (__this_cpu_read (injectm.finished)) {int offset =msr_to_offset (msr);if (offset >=0 )
arch/x86/kernel/cpu/perf_event.c +11 -16
@@ -968,8 +968,7 @@ x86_perf_event_set_period(struct perf_evstatic void x86_pmu_enable_event (struct perf_event *event) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); -if (cpuc->enabled) +if (__this_cpu_read (cpu_hw_events.enabled))__x86_pmu_enable_event (&event->hw,ARCH_PERFMON_EVENTSEL_ENABLE ); }
@@ -1243,7 +1242,7 @@ perf_event_nmi_handler(struct notifier_bbreak ;case DIE_NMIUNKNOWN : this_nmi =percpu_read (irq_stat.__nmi_count); -if (this_nmi !=__get_cpu_var (pmu_nmi).marked) +if (this_nmi !=__this_cpu_read (pmu_nmi.marked))/* let the kernel handle the unknown nmi */ return NOTIFY_DONE ;/*
@@ -1267,8 +1266,8 @@ perf_event_nmi_handler(struct notifier_b this_nmi =percpu_read (irq_stat.__nmi_count);if ((handled >1 ) ||/* the next nmi could be a back-to-back nmi */ - ((__get_cpu_var (pmu_nmi).marked == this_nmi) && - (__get_cpu_var (pmu_nmi).handled >1 ))) { + ((__this_cpu_read (pmu_nmi.marked) == this_nmi) && + (__this_cpu_read (pmu_nmi.handled) >1 ))) {/* * We could have two subsequent back-to-back nmis: The * first handles more than one counter, the 2nd
@@ -1279,8 +1278,8 @@ perf_event_nmi_handler(struct notifier_b* handling more than one counter. We will mark the * next (3rd) and then drop it if unhandled. */ -__get_cpu_var (pmu_nmi).marked = this_nmi +1 ; -__get_cpu_var (pmu_nmi).handled = handled; +__this_cpu_write (pmu_nmi.marked, this_nmi +1 ); +__this_cpu_write (pmu_nmi.handled, handled); }return NOTIFY_STOP ;
@@ -1454,11 +1453,9 @@ static inline void x86_pmu_read(struct p*/ static void x86_pmu_start_txn (struct pmu *pmu) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); -perf_pmu_disable (pmu); - cpuc->group_flag |=PERF_EVENT_TXN ; - cpuc->n_txn =0 ; +__this_cpu_or (cpu_hw_events.group_flag,PERF_EVENT_TXN ); +__this_cpu_write (cpu_hw_events.n_txn,0 ); }/*
@@ -1468,14 +1465,12 @@ static void x86_pmu_start_txn(struct pmu*/ static void x86_pmu_cancel_txn (struct pmu *pmu) { -struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); - - cpuc->group_flag &= ~PERF_EVENT_TXN ; +__this_cpu_and (cpu_hw_events.group_flag, ~PERF_EVENT_TXN );/* * Truncate the collected events. */ - cpuc->n_added -= cpuc->n_txn; - cpuc->n_events -= cpuc->n_txn; +__this_cpu_sub (cpu_hw_events.n_added,__this_cpu_read (cpu_hw_events.n_txn)); +__this_cpu_sub (cpu_hw_events.n_events,__this_cpu_read (cpu_hw_events.n_txn));perf_pmu_enable (pmu); }
arch/x86/kernel/cpu/perf_event_intel.c +2 -2
@@ -649,7 +649,7 @@ static void intel_pmu_enable_event(strucstruct hw_perf_event *hwc = &event->hw;if (unlikely (hwc->idx ==X86_PMC_IDX_FIXED_BTS )) { -if (!__get_cpu_var (cpu_hw_events).enabled) +if (!__this_cpu_read (cpu_hw_events.enabled))return ;intel_pmu_enable_bts (hwc->config);
@@ -679,7 +679,7 @@ static int intel_pmu_save_and_restart(ststatic void intel_pmu_reset (void ) { -struct debug_store *ds =__get_cpu_var (cpu_hw_events).ds; +struct debug_store *ds =__this_cpu_read (cpu_hw_events.ds);unsigned long flags;int idx;
arch/x86/kernel/ftrace.c +3 -3
@@ -167,9 +167,9 @@ static void ftrace_mod_code(void)void ftrace_nmi_enter (void ) { -__get_cpu_var (save_modifying_code) = modifying_code; +__this_cpu_write (save_modifying_code, modifying_code); -if (!__get_cpu_var (save_modifying_code)) +if (!__this_cpu_read (save_modifying_code))return ;if (atomic_inc_return (&nmi_running) &MOD_CODE_WRITE_FLAG ) {
@@ -183,7 +183,7 @@ void ftrace_nmi_enter(void)void ftrace_nmi_exit (void ) { -if (!__get_cpu_var (save_modifying_code)) +if (!__this_cpu_read (save_modifying_code))return ;/* Finish all executions before clearing nmi_running */
arch/x86/kernel/hw_breakpoint.c +6 -6
@@ -122,7 +122,7 @@ int arch_install_hw_breakpoint(struct pereturn -EBUSY ;set_debugreg (info->address, i); -__get_cpu_var (cpu_debugreg[i]) = info->address; +__this_cpu_write (cpu_debugreg[i], info->address); dr7 = &__get_cpu_var (cpu_dr7); *dr7 |=encode_dr7 (i, info->len, info->type);
@@ -397,12 +397,12 @@ void flush_ptrace_hw_breakpoint(struct tvoid hw_breakpoint_restore (void ) { -set_debugreg (__get_cpu_var (cpu_debugreg[0 ]),0 ); -set_debugreg (__get_cpu_var (cpu_debugreg[1 ]),1 ); -set_debugreg (__get_cpu_var (cpu_debugreg[2 ]),2 ); -set_debugreg (__get_cpu_var (cpu_debugreg[3 ]),3 ); +set_debugreg (__this_cpu_read (cpu_debugreg[0 ]),0 ); +set_debugreg (__this_cpu_read (cpu_debugreg[1 ]),1 ); +set_debugreg (__this_cpu_read (cpu_debugreg[2 ]),2 ); +set_debugreg (__this_cpu_read (cpu_debugreg[3 ]),3 );set_debugreg (current->thread.debugreg6,6 ); -set_debugreg (__get_cpu_var (cpu_dr7),7 ); +set_debugreg (__this_cpu_read (cpu_dr7),7 ); }EXPORT_SYMBOL_GPL (hw_breakpoint_restore);
arch/x86/kernel/irq.c +3 -3
@@ -234,7 +234,7 @@ unsigned int __irq_entry do_IRQ(struct pexit_idle ();irq_enter (); - irq =__get_cpu_var (vector_irq)[vector]; + irq =__this_cpu_read (vector_irq[vector]);if (!handle_irq (irq, regs)) {ack_APIC_irq ();
@@ -350,12 +350,12 @@ void fixup_irqs(void)for (vector =FIRST_EXTERNAL_VECTOR ; vector <NR_VECTORS ; vector++) {unsigned int irr; -if (__get_cpu_var (vector_irq)[vector] <0 ) +if (__this_cpu_read (vector_irq[vector]) <0 )continue ; irr =apic_read (APIC_IRR + (vector /32 *0x10 ));if (irr & (1 << (vector %32 ))) { - irq =__get_cpu_var (vector_irq)[vector]; + irq =__this_cpu_read (vector_irq[vector]); data =irq_get_irq_data (irq);raw_spin_lock (&desc->lock);
arch/x86/kernel/irq_32.c +2 -2
@@ -79,7 +79,7 @@ execute_on_irq_stack(int overflow, strucu32 *isp, arg1, arg2; curctx = (union irq_ctx *)current_thread_info (); - irqctx =__get_cpu_var (hardirq_ctx); + irqctx =__this_cpu_read (hardirq_ctx);/* * this is where we switch to the IRQ stack. However, if we are
@@ -166,7 +166,7 @@ asmlinkage void do_softirq(void)if (local_softirq_pending ()) { curctx =current_thread_info (); - irqctx =__get_cpu_var (softirq_ctx); + irqctx =__this_cpu_read (softirq_ctx); irqctx->tinfo.task = curctx->task; irqctx->tinfo.previous_esp = current_stack_pointer;
arch/x86/kernel/tsc.c +1 -1
@@ -659,7 +659,7 @@ void restore_sched_clock_state(void)local_irq_save (flags); -__get_cpu_var (cyc2ns_offset) =0 ; +__this_cpu_write (cyc2ns_offset,0 ); offset = cyc2ns_suspend -sched_clock ();for_each_possible_cpu (cpu)
arch/x86/kvm/x86.c +4 -4
@@ -981,7 +981,7 @@ static inline u64 nsec_to_cycles(u64 nseif (kvm_tsc_changes_freq ())printk_once (KERN_WARNING "kvm: unreliable cycle conversion on adjustable rate TSC\n" ); - ret = nsec *__get_cpu_var (cpu_tsc_khz); + ret = nsec *__this_cpu_read (cpu_tsc_khz);do_div (ret,USEC_PER_SEC );return ret; }
@@ -1066,7 +1066,7 @@ static int kvm_guest_time_update(structlocal_irq_save (flags);kvm_get_msr (v,MSR_IA32_TSC , &tsc_timestamp); kernel_ns =get_kernel_ns (); - this_tsc_khz =__get_cpu_var (cpu_tsc_khz); + this_tsc_khz =__this_cpu_read (cpu_tsc_khz);if (unlikely (this_tsc_khz ==0 )) {local_irq_restore (flags);
@@ -4432,7 +4432,7 @@ EXPORT_SYMBOL_GPL(kvm_fast_pio_out);static void tsc_bad (void *info) { -__get_cpu_var (cpu_tsc_khz) =0 ; +__this_cpu_write (cpu_tsc_khz,0 ); }static void tsc_khz_changed (void *data)
@@ -4446,7 +4446,7 @@ static void tsc_khz_changed(void *data) khz =cpufreq_quick_get (raw_smp_processor_id ());if (!khz) khz = tsc_khz; -__get_cpu_var (cpu_tsc_khz) = khz; +__this_cpu_write (cpu_tsc_khz, khz); } static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
arch/x86/oprofile/nmi_int.c +1 -1
@@ -143,7 +143,7 @@ static inline int has_mux(void)inline int op_x86_phys_to_virt (int phys) { -return __get_cpu_var (switch_index) + phys; +return __this_cpu_read (switch_index) + phys; }inline int op_x86_virt_to_phys (int virt)
Re: [Use cpuops V1 01/11] percpucounter: Optimize __percpu_counter_add a bit through the use of this_cpu() options.
In reply to Christoph LameterQuoted 2 lines · Christoph Lameter
The this_cpu_* options can be used to optimize
__percpu_counter_adda bit. Avoids some address arithmetic and saves 12 bytes.
...
Quoted 5 lines
Reviewed-by Pekka Enberg <[email protected]>Reviewed-by Tejun Heo <[email protected]>Reviewed-by Mathieu Desnoyers <[email protected]>Signed-off-by Christoph Lameter <[email protected]>
applied to percpu#for-next.
Thanks.
Signature · 1 line
tejun
Re: [Use cpuops V1 02/11] vmstat: Optimize zone counter modifications through the use of this cpu operations
In reply to Christoph LameterQuoted 19 lines · Christoph Lameter
this cpu operations can be used to slightly optimize the function. The changes will avoid some address calculations and replace them with the use of the percpu segment register.
If one would have
this_cpu_inc_returnandthis_cpu_dec_returnthen it would be possible to optimizeinc_zone_page_stateanddec_zone_page_stateeven more.
V1->V2:
- Fix__dec_zone_stateoverflow handling
- Use s8 variables for temporary storage.
V2->V3:
- Put__percpuannotations in correct places.Reviewed-by Pekka Enberg <[email protected]>Signed-off-by Christoph Lameter <[email protected]>(quoted diff) +0
-0@@ -221,16 +223,19 @@ EXPORT_SYMBOL(mod_zone_page_state);
...
mm/vmstat.c EXPORT_SYMBOL(mod_zone_page_state); · line 238
Quoted 1 line
mm/vmstat.c +1
-0+__this_cpu_write (*p, - overstep);
Applied to percpu#for-next with the space after '-' removed from the above line.
Thanks.
Signature · 1 line
tejun
Tejun Heo
maintainerRe: [Use cpuops V1 04/11] x86: Use this_cpu_ops for current_cpu_data accesses
In reply to Christoph LameterQuoted 27 lines · Christoph Lameter
Current_cpu_dataaccesses are per cpu accesses. We can also usethis_cpu_opsif a scalar is retrieved.Cc Yinghai Lu <[email protected]>Cc Ingo Molnar <[email protected]>Signed-off-by Christoph Lameter <[email protected]>--- arch/x86/kernel/cpu/cpufreq/powernow-k8.c | 2 +- arch/x86/kernel/cpu/intel_cacheinfo.c | 4 ++-- arch/x86/kernel/smpboot.c | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-)Index:
linux-2.6/arch/x86/kernel/smpboot.c
===================================================================arch/x86/kernel/smpboot.c +1
-1@@ -430,7 +430,7 @@ void __cpuinit set_cpu_sibling_map(int ccpumask_set_cpu (cpu, c->llc_shared_map); -if (current_cpu_data.x86_max_cores ==1 ) { +if (__this_cpu_read (cpu_info.x86_max_cores) ==1 ) {cpumask_copy (cpu_core_mask (cpu),cpu_sibling_mask (cpu)); c->booted_cores =1 ;return ;
arch/x86/kernel/smpboot.c void play_dead_common(void) · line 1380
Quoted 5 lines
arch/x86/kernel/smpboot.c +1
-1@@ -1377,7 +1377,7 @@ void play_dead_common(void)mb ();/* Ack it */ -__get_cpu_var (cpu_state) =CPU_DEAD ; +__this_cpu_write (cpu_state,CPU_DEAD );
This belongs to the previous patch, right? I'll move it over and apply 03 and 04. I think routing these through percpu is okay but if anyone wants these to go through x86, scream.
Thanks.
Signature · 1 line
tejun
Tejun Heo
maintainerRe: [Use cpuops V1 04/11] x86: Use this_cpu_ops for current_cpu_data accesses
Quoted 29 lines · Tejun Heo
Quoted 28 lines · Christoph Lameter
Current_cpu_dataaccesses are per cpu accesses. We can also usethis_cpu_opsif a scalar is retrieved.Cc Yinghai Lu <[email protected]>Cc Ingo Molnar <[email protected]>Signed-off-by Christoph Lameter <[email protected]>--- arch/x86/kernel/cpu/cpufreq/powernow-k8.c | 2 +- arch/x86/kernel/cpu/intel_cacheinfo.c | 4 ++-- arch/x86/kernel/smpboot.c | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-)Index:
linux-2.6/arch/x86/kernel/smpboot.c
===================================================================arch/x86/kernel/smpboot.c +1
-1@@ -430,7 +430,7 @@ void __cpuinit set_cpu_sibling_map(int ccpumask_set_cpu (cpu, c->llc_shared_map); -if (current_cpu_data.x86_max_cores ==1 ) { +if (__this_cpu_read (cpu_info.x86_max_cores) ==1 ) {cpumask_copy (cpu_core_mask (cpu),cpu_sibling_mask (cpu)); c->booted_cores =1 ;return ;@@ -1377,7 +1377,7 @@ void play_dead_common(void)
Quoted 8 lines
arch/x86/kernel/smpboot.c void play_dead_common(void) · line 1380
Quoted 4 lines
arch/x86/kernel/smpboot.c +1
-1mb ();/* Ack it */ -__get_cpu_var (cpu_state) =CPU_DEAD ; +__this_cpu_write (cpu_state,CPU_DEAD );This belongs to the previous patch, right? I'll move it over and apply 03 and 04. I think routing these through percpu is okay but if anyone wants these to go through x86, scream.
Ooh, was too fast. I think mixing the use of percpu accesses to cpu_info and the wrapper macro current_cpu_data is quite confusing. There aren't too many current_cpu_data users in x86 anyway. Can you please make the conversion complete? I'm moving the above misplaced chunk into 03 and not applying 04 for now.
Thank you.
Signature · 1 line
tejun
Tejun Heo
maintainerRe: [Use cpuops V1 05/11] core: Replace __get_cpu_var with __this_cpu_read if not used for an address.
In reply to Christoph LameterHello,
Quoted 13 lines · Christoph Lameter
__get_cpu_var()can be replaced withthis_cpu_readand will then use a single read instruction with implied address calculation to access the correct per cpu instance.However, the address of a per cpu variable passed to
__this_cpu_read()cannot be determed (since its an implied address conversion through segment prefixes). Therefore apply this only to uses of__get_cpu_varwhere the addres of the variable is not used.Cc Pekka Enberg <[email protected]>Cc Hugh Dickins <[email protected]>Cc Thomas Gleixner <[email protected]>Signed-off-by Christoph Lameter <[email protected]>
...
Quoted 8 lines
kernel/softirq.c +2
-2@@ -802,18 +802,18 @@ static void takeover_tasklets(unsigned i/* Find end, append list for that CPU. */ if (&per_cpu (tasklet_vec, cpu).head !=per_cpu (tasklet_vec, cpu).tail) { - *(__get_cpu_var (tasklet_vec).tail) =per_cpu (tasklet_vec, cpu).head; -__get_cpu_var (tasklet_vec).tail =per_cpu (tasklet_vec, cpu).tail; + *__this_cpu_read (tasklet_vec.tail) =per_cpu (tasklet_vec, cpu).head; +this_cpu_write (tasklet_vec.tail,per_cpu (tasklet_vec, cpu).tail);
kernel/softirq.c static void takeover_tasklets(unsigned i · line 814
Quoted 10 lines
kernel/softirq.c +2
-2per_cpu (tasklet_vec, cpu).head =NULL ;per_cpu (tasklet_vec, cpu).tail = &per_cpu (tasklet_vec, cpu).head; }raise_softirq_irqoff (TASKLET_SOFTIRQ );if (&per_cpu (tasklet_hi_vec, cpu).head !=per_cpu (tasklet_hi_vec, cpu).tail) { - *__get_cpu_var (tasklet_hi_vec).tail =per_cpu (tasklet_hi_vec, cpu).head; -__get_cpu_var (tasklet_hi_vec).tail =per_cpu (tasklet_hi_vec, cpu).tail; + *__this_cpuo_read (tasklet_hi_vec.tail) =per_cpu (tasklet_hi_vec, cpu).head; + __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail;
I don't think __this_cpuo_read() would build.
Signature · 1 line
tejun
Re: [Use cpuops V1 06/11] drivers: Replace __get_cpu_var with __this_cpu_read if not used for an address.
In reply to Christoph LameterQuoted 9 lines · Christoph Lameter
__get_cpu_var()can be replaced withthis_cpu_readand will then use a single read instruction with implied address calculation to access the correct per cpu instance.However, the address of a per cpu variable passed to
__this_cpu_read()cannot be determed (since its an implied address conversion through segment prefixes). Therefore apply this only to uses of__get_cpu_varwhere the addres of the variable is not used.
Quoted 8 lines
V3->V4:
- Move one instance ofthis_cpu_inc_returnto a later patch so that this one can go in without percpu infrastructrure changes.Cc Neil Horman <[email protected]>Cc Martin Schwidefsky <[email protected]>Signed-off-by Christoph Lameter <[email protected]>
applied.
Signature · 1 line
tejun
Re: [Use cpuops V1 04/11] x86: Use this_cpu_ops for current_cpu_data accesses
In reply to Tejun HeoQuoted 9 lines · Tejun Heo
Quoted 3 lines
This belongs to the previous patch, right? I'll move it over and apply 03 and 04. I think routing these through percpu is okay but if anyone wants these to go through x86, scream.
Ooh, was too fast. I think mixing the use of percpu accesses to
cpu_infoand the wrapper macrocurrent_cpu_datais quite confusing. There aren't too manycurrent_cpu_datausers in x86 anyway. Can you please make the conversion complete? I'm moving the above misplaced chunk into 03 and not applying 04 for now.
We cannot make that conversion complete since we cannot obtain the address if we use a this_cpu operation. Then its better to drop this patch.
Re: [Use cpuops V1 05/11] core: Replace __get_cpu_var with __this_cpu_read if not used for an address.
In reply to Tejun HeoQuoted 7 lines · Tejun Heo
Review
Quoted 5 lines
(quoted diff) +2
-2if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) { - *__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head; - __get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail; + *__this_cpuo_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head; + __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail;I don't think
__this_cpuo_read()would build.
A CONFIG_HOTPLUG_CPU section.
Fix up CONFIG_HOTPLUG_CPU compilation
kernel/softirq.c +3 -3
@@ -810,10 +810,10 @@ static void takeover_tasklets(unsigned iraise_softirq_irqoff (TASKLET_SOFTIRQ );if (&per_cpu (tasklet_hi_vec, cpu).head !=per_cpu (tasklet_hi_vec, cpu).tail) { - *__this_cpuo_read (tasklet_hi_vec.tail) =per_cpu (tasklet_hi_vec, cpu).head; - __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail; + *__this_cpu_read (tasklet_hi_vec.tail) =per_cpu (tasklet_hi_vec, cpu).head; +__this_cpu_write (tasklet_hi_vec.tail,per_cpu (tasklet_hi_vec, cpu).tail);per_cpu (tasklet_hi_vec, cpu).head =NULL ; - per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head); +per_cpu (tasklet_hi_vec, cpu).tail = &per_cpu (tasklet_hi_vec, cpu).head; }raise_softirq_irqoff (HI_SOFTIRQ );
Tejun Heo
maintainerRe: [Use cpuops V1 04/11] x86: Use this_cpu_ops for current_cpu_data accesses
In reply to Christoph LameterQuoted 6 lines · Christoph Lameter
Quoted 4 lines · Tejun Heo
Quoted 3 lines
This belongs to the previous patch, right? I'll move it over and apply 03 and 04. I think routing these through percpu is okay but if anyone wants these to go through x86, scream.
Quoted 8 lines
Quoted 5 lines
Ooh, was too fast. I think mixing the use of percpu accesses to
cpu_infoand the wrapper macrocurrent_cpu_datais quite confusing. There aren't too manycurrent_cpu_datausers in x86 anyway. Can you please make the conversion complete? I'm moving the above misplaced chunk into 03 and not applying 04 for now.We cannot make that conversion complete since we cannot obtain the address if we use a
this_cpuoperation. Then its better to drop this patch.
No, I was talking about dropping current_cpu_data macro. We can use __this_cpu_ptr() for addresses. I just find it very confusing to mix current_cpu_data and using direct percpu accessors on cpu_info.
Thanks.
Signature · 1 line
tejun
Re: [Use cpuops V1 07/11] kprobes: Use this_cpu_ops
In reply to Christoph LameterQuoted 5 lines · Christoph Lameter
Use
this_cpuops in various places to optimize per cpu data access.Cc Jason Baron <[email protected]>Cc Namhyung Kim <[email protected]>Signed-off-by Christoph Lameter <[email protected]>
applied
Signature · 1 line
tejun
Re: [Use cpuops V1 04/11] x86: Use this_cpu_ops for current_cpu_data accesses
In reply to Tejun HeoQuoted 6 lines · Tejun Heo
Quoted 6 lines
Quoted 5 lines
Ooh, was too fast. I think mixing the use of percpu accesses to
cpu_infoand the wrapper macrocurrent_cpu_datais quite confusing. There aren't too manycurrent_cpu_datausers in x86 anyway. Can you please make the conversion complete? I'm moving the above misplaced chunk into 03 and not applying 04 for now.
Quoted 6 lines
Quoted 2 lines
We cannot make that conversion complete since we cannot obtain the address if we use a
this_cpuoperation. Then its better to drop this patch.No, I was talking about dropping
current_cpu_datamacro. We can use__this_cpu_ptr()for addresses. I just find it very confusing to mixcurrent_cpu_dataand using direct percpu accessors oncpu_info.
Hmmm... Ok 21 occurrences. So two patches: Conversion and then this_cpu_ops. Or one doing both?
Re: [Use cpuops V1 08/11] Fakekey: Simplify speakup_fake_key_pressed through this_cpu_ops
In reply to Christoph LameterQuoted 6 lines · Christoph Lameter
The whole function can be expressed as a simple
this_cpu_read()operation. The function overhead is now likely multiple times that of the single instruction that is executed in it.Cc William Hubbs <[email protected]>Signed-off-by Christoph Lameter <[email protected]>
applied.
Signature · 1 line
tejun
Tejun Heo
maintainerRe: [Use cpuops V1 09/11] Connector: Use this_cpu operations
In reply to Christoph LameterQuoted 16 lines · Christoph Lameter
get_seqcan benefit fromthis_cpu_operations. Address calculation is avoided and the increment is done using an xadd.Cc Scott James Remnant <[email protected]>Cc Mike Frysinger <[email protected]>Signed-off-by Christoph Lameter <[email protected]>--- drivers/connector/cn_proc.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)Index:
linux-2.6/drivers/connector/cn_proc.c
===================================================================
drivers/connector/cn_proc.c static DEFINE_PER_CPU(__u32, proc_event_ · line 47
Quoted 6 lines
drivers/connector/cn_proc.c +2
-1@@ -43,9 +43,10 @@ static DEFINE_PER_CPU(__u32, proc_event_static inline void get_seq (__u32 *ts,int *cpu) { - *ts =get_cpu_var (proc_event_counts)++; +preempt_disable (); + *ts =__this_cpu_inc (proc_event_counts);
Eh? __this_cpu_inc() evaluates to the incremented value?
Signature · 1 line
tejun
Tejun Heo
maintainerRe: [Use cpuops V1 04/11] x86: Use this_cpu_ops for current_cpu_data accesses
In reply to Christoph LameterQuoted 11 lines · Christoph Lameter
Quoted 9 lines · Tejun Heo
Quoted 8 lines
Quoted 5 lines
Ooh, was too fast. I think mixing the use of percpu accesses to
cpu_infoand the wrapper macrocurrent_cpu_datais quite confusing. There aren't too manycurrent_cpu_datausers in x86 anyway. Can you please make the conversion complete? I'm moving the above misplaced chunk into 03 and not applying 04 for now.We cannot make that conversion complete since we cannot obtain the address if we use a
this_cpuoperation. Then its better to drop this patch.
Quoted 6 lines
Quoted 3 lines
No, I was talking about dropping
current_cpu_datamacro. We can use__this_cpu_ptr()for addresses. I just find it very confusing to mixcurrent_cpu_dataand using direct percpu accessors oncpu_info.Hmmm... Ok 21 occurrences. So two patches: Conversion and then
this_cpu_ops. Or one doing both?
Two steps is better probably. Thank you.
Signature · 1 line
tejun
Tejun Heo
maintainerRe: [Use cpuops V1 10/11] fs: Use this_cpu_xx operations in buffer.c
In reply to Christoph LameterQuoted 19 lines · Christoph Lameter
Optimize various per cpu area operations through the new this cpu operations. These operations avoid address calculations through the use of segment prefixes avoid multiple memory references through RMW instructions etc.
Reduces code size:
Before:
[email protected]$ size fs/buffer.o text data bss dec hex filename 19169 80 28 19277 4b4d fs/buffer.oAfter:
[email protected]$ size fs/buffer.o text data bss dec hex filename 19138 80 28 19246 4b2e fs/buffer.o
Quoted 7 lines
V3->V4:
- Move the use ofthis_cpu_inc_returninto a later patch so that this one can go in without percpu infrastructure changes.Cc Wu Fengguang <[email protected]>Cc Christoph Hellwig <[email protected]>Signed-off-by Christoph Lameter <[email protected]>
applied 10 and 11.
Signature · 1 line
tejun
Tejun Heo
maintainerRe: [Use cpuops V1 05/11] core: Replace __get_cpu_var with __this_cpu_read if not used for an address.
In reply to Christoph LameterHello, again.
Quoted 10 lines · Christoph Lameter
Quoted 7 lines · Tejun Heo
Review
Quoted 5 lines
(quoted diff) +2
-2if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) { - *__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head; - __get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail; + *__this_cpuo_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head; + __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail;I don't think
__this_cpuo_read()would build.
Quoted 5 lines
A
CONFIG_HOTPLUG_CPUsection.Fix up
CONFIG_HOTPLUG_CPUcompilationSigned-off-by Christoph Lameter <[email protected]>
Can you please fold this into the original patch and resend? I skipped 04, 05 and 09. percpu#for-next now contains other patches.
git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu.git for-next
Thanks.
Signature · 1 line
tejun
Re: [Use cpuops V1 09/11] Connector: Use this_cpu operations
In reply to Tejun HeoQuoted 6 lines · Tejun Heo
Quoted 6 lines
Index:
linux-2.6/drivers/connector/cn_proc.c
===================================================================drivers/connector/cn_proc.c +0
-0@@ -43,9 +43,10 @@ static DEFINE_PER_CPU(__u32, proc_event_
Quoted 7 lines
drivers/connector/cn_proc.c static DEFINE_PER_CPU(__u32, proc_event_ · line 47
Quoted 5 lines
drivers/connector/cn_proc.c +2
-1static inline void get_seq (__u32 *ts,int *cpu) { - *ts =get_cpu_var (proc_event_counts)++; +preempt_disable (); + *ts =__this_cpu_inc (proc_event_counts);Eh?
__this_cpu_inc()evaluates to the incremented value?
Err no. Drop this and I will add it to the series that depends on the inc.
Re: [Use cpuops V1 05/11] core: Replace __get_cpu_var with __this_cpu_read if not used for an address.
In reply to Tejun HeoQuoted 6 lines · Tejun Heo
Can you please fold this into the original patch and resend? I skipped 04, 05 and 09. percpu#for-next now contains other patches.
git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu.gitfor-nextThanks.
Subject: core: Replace __get_cpu_var with __this_cpu_read if not used for an address.
__get_cpu_var() can be replaced with this_cpu_read and will then use a single read instruction with implied address calculation to access the correct per cpu instance.
However, the address of a per cpu variable passed to __this_cpu_read() cannot be determed (since its an implied address conversion through segment prefixes). Therefore apply this only to uses of __get_cpu_var where the addres of the variable is not used.
--- include/asm-generic/irq_regs.h | 8 +++---- include/linux/elevator.h | 12 ++--------- include/linux/kernel_stat.h | 2 - kernel/exit.c | 2 - kernel/fork.c | 2 - kernel/hrtimer.c | 2 - kernel/printk.c | 4 +-- kernel/rcutree.c | 4 +-- kernel/softirq.c | 42 ++++++++++++++++++++--------------------- kernel/time/tick-common.c | 2 - kernel/time/tick-oneshot.c | 4 +-- kernel/watchdog.c | 36 +++++++++++++++++------------------ mm/slab.c | 6 ++--- 13 files changed, 60 insertions(+), 66 deletions(-)
mm/slab.c +3 -3
@@ -829,12 +829,12 @@ static void init_reap_node(int cpu)static void next_reap_node (void ) { -int node =__get_cpu_var (slab_reap_node); +int node =__this_cpu_read (slab_reap_node); node =next_node (node, node_online_map);if (unlikely (node >=MAX_NUMNODES )) node =first_node (node_online_map); -__get_cpu_var (slab_reap_node) = node; +__this_cpu_write (slab_reap_node, node); }#else
@@ -1012,7 +1012,7 @@ static void __drain_alien_cache(struct k*/ static void reap_alien (struct kmem_cache *cachep,struct kmem_list3 *l3) { -int node =__get_cpu_var (slab_reap_node); +int node =__this_cpu_read (slab_reap_node);if (l3->alien) {struct array_cache *ac = l3->alien[node];
kernel/rcutree.c +2 -2
@@ -367,8 +367,8 @@ void rcu_irq_exit(void)WARN_ON_ONCE (rdtp->dynticks &0x1 );/* If the interrupt queued a callback, get out of dyntick mode. */ -if (__get_cpu_var (rcu_sched_data).nxtlist || -__get_cpu_var (rcu_bh_data).nxtlist) +if (__this_cpu_read (rcu_sched_data.nxtlist) || +__this_cpu_read (rcu_bh_data.nxtlist))set_need_resched (); }
kernel/softirq.c +21 -21
102 lines not on this page · raw message
kernel/time/tick-common.c +1 -1
@@ -49,7 +49,7 @@ struct tick_device *tick_get_device(int*/ int tick_is_oneshot_available (void ) { -struct clock_event_device *dev =__get_cpu_var (tick_cpu_device).evtdev; +struct clock_event_device *dev =__this_cpu_read (tick_cpu_device.evtdev);return dev && (dev->features &CLOCK_EVT_FEAT_ONESHOT ); }
kernel/time/tick-oneshot.c +2 -2
@@ -95,7 +95,7 @@ int tick_dev_program_event(struct clock_*/ int tick_program_event (ktime_t expires,int force) { -struct clock_event_device *dev =__get_cpu_var (tick_cpu_device).evtdev; +struct clock_event_device *dev =__this_cpu_read (tick_cpu_device.evtdev);return tick_dev_program_event (dev, expires, force); }
@@ -167,7 +167,7 @@ int tick_oneshot_mode_active(void)int ret;local_irq_save (flags); - ret =__get_cpu_var (tick_cpu_device).mode ==TICKDEV_MODE_ONESHOT ; + ret =__this_cpu_read (tick_cpu_device.mode) ==TICKDEV_MODE_ONESHOT ;local_irq_restore (flags);return ret;
kernel/watchdog.c +18 -18
@@ -116,12 +116,12 @@ static void __touch_watchdog(void) {int this_cpu =smp_processor_id (); -__get_cpu_var (watchdog_touch_ts) =get_timestamp (this_cpu); +__this_cpu_write (watchdog_touch_ts,get_timestamp (this_cpu)); }void touch_softlockup_watchdog (void ) { -__raw_get_cpu_var (watchdog_touch_ts) =0 ; +__this_cpu_write (watchdog_touch_ts,0 ); }EXPORT_SYMBOL (touch_softlockup_watchdog);
@@ -165,12 +165,12 @@ void touch_softlockup_watchdog_sync(void/* watchdog detector functions */ static int is_hardlockup (void ) { -unsigned long hrint =__get_cpu_var (hrtimer_interrupts); +unsigned long hrint =__this_cpu_read (hrtimer_interrupts); -if (__get_cpu_var (hrtimer_interrupts_saved) == hrint) +if (__this_cpu_read (hrtimer_interrupts_saved) == hrint)return 1 ; -__get_cpu_var (hrtimer_interrupts_saved) = hrint; +__this_cpu_write (hrtimer_interrupts_saved, hrint);return 0 ; }#endif
@@ -203,8 +203,8 @@ static void watchdog_overflow_callback(s/* Ensure the watchdog never gets throttled */ event->hw.interrupts =0 ; -if (__get_cpu_var (watchdog_nmi_touch) == true) { -__get_cpu_var (watchdog_nmi_touch) = false; +if (__this_cpu_read (watchdog_nmi_touch) == true) { +__this_cpu_write (watchdog_nmi_touch, false);return ; }
@@ -218,7 +218,7 @@ static void watchdog_overflow_callback(sint this_cpu =smp_processor_id ();/* only print hardlockups once */ -if (__get_cpu_var (hard_watchdog_warn) == true) +if (__this_cpu_read (hard_watchdog_warn) == true)return ;if (hardlockup_panic)
@@ -226,16 +226,16 @@ static void watchdog_overflow_callback(s else WARN(1, "Watchdog detected hard LOCKUP on cpu %d", this_cpu); -__get_cpu_var (hard_watchdog_warn) = true; +__this_cpu_write (hard_watchdog_warn, true); return; } -__get_cpu_var (hard_watchdog_warn) = false; +__this_cpu_write (hard_watchdog_warn, false);return ; }static void watchdog_interrupt_count (void ) { -__get_cpu_var (hrtimer_interrupts)++; +__this_cpu_inc (hrtimer_interrupts); }#else static inline void watchdog_interrupt_count (void ) {return ; }
@@ -244,7 +244,7 @@ static inline void watchdog_interrupt_co/* watchdog kicker functions */ static enum hrtimer_restart watchdog_timer_fn (struct hrtimer *hrtimer) { -unsigned long touch_ts =__get_cpu_var (watchdog_touch_ts); +unsigned long touch_ts =__this_cpu_read (watchdog_touch_ts);struct pt_regs *regs =get_irq_regs ();int duration;
@@ -252,18 +252,18 @@ static enum hrtimer_restart watchdog_timwatchdog_interrupt_count ();/* kick the softlockup detector */ -wake_up_process (__get_cpu_var (softlockup_watchdog)); +wake_up_process (__this_cpu_read (softlockup_watchdog));/* .. and repeat */ hrtimer_forward_now (hrtimer,ns_to_ktime (get_sample_period ()));if (touch_ts ==0 ) { -if (unlikely (__get_cpu_var (softlockup_touch_sync))) { +if (unlikely (__this_cpu_read (softlockup_touch_sync))) {/* * If the time stamp was touched atomically * make sure the scheduler tick is up to date. */ -__get_cpu_var (softlockup_touch_sync) = false; +__this_cpu_write (softlockup_touch_sync, false);sched_clock_tick (); }__touch_watchdog ();
@@ -279,7 +279,7 @@ static enum hrtimer_restart watchdog_tim duration =is_softlockup (touch_ts);if (unlikely (duration)) {/* only warn once */ -if (__get_cpu_var (soft_watchdog_warn) == true) +if (__this_cpu_read (soft_watchdog_warn) == true)return HRTIMER_RESTART ; printk(KERN_ERR "BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]\n",
@@ -294,9 +294,9 @@ static enum hrtimer_restart watchdog_timif (softlockup_panic)panic ("softlockup: hung tasks" ); -__get_cpu_var (soft_watchdog_warn) = true; +__this_cpu_write (soft_watchdog_warn, true); }else -__get_cpu_var (soft_watchdog_warn) = false; +__this_cpu_write (soft_watchdog_warn, false);return HRTIMER_RESTART ; }
kernel/printk.c +2 -2
@@ -1074,8 +1074,8 @@ static DEFINE_PER_CPU(int, printk_pendinvoid printk_tick (void ) { -if (__get_cpu_var (printk_pending)) { -__get_cpu_var (printk_pending) =0 ; +if (__this_cpu_read (printk_pending)) { +__this_cpu_write (printk_pending,0 );wake_up_interruptible (&log_wait); } }
include/asm-generic/irq_regs.h +4 -4
@@ -22,15 +22,15 @@ DECLARE_PER_CPU(struct pt_regs *, __irq_static inline struct pt_regs *get_irq_regs (void ) { -return __get_cpu_var (__irq_regs); +return __this_cpu_read (__irq_regs); }static inline struct pt_regs *set_irq_regs (struct pt_regs *new_regs) { -struct pt_regs *old_regs, **pp_regs = &__get_cpu_var (__irq_regs); +struct pt_regs *old_regs; - old_regs = *pp_regs; - *pp_regs = new_regs; + old_regs =__this_cpu_read (__irq_regs); +__this_cpu_write (__irq_regs, new_regs);return old_regs; }
include/linux/elevator.h +3 -9
@@ -195,15 +195,9 @@ enum {/* * io context count accounting */ -#define elv_ioc_count_mod (name, __val) \ - do { \ - preempt_disable(); \ - __get_cpu_var(name) += (__val); \ - preempt_enable(); \ - } while (0) - -#define elv_ioc_count_inc (name) elv_ioc_count_mod(name, 1) -#define elv_ioc_count_dec (name) elv_ioc_count_mod(name, -1) +#define elv_ioc_count_mod (name, __val) this_cpu_add(name, __val) +#define elv_ioc_count_inc (name) this_cpu_inc(name) +#define elv_ioc_count_dec (name) this_cpu_dec(name)#define elv_ioc_count_read (name) \ ({ \
include/linux/kernel_stat.h +1 -1
@@ -47,7 +47,7 @@ extern unsigned long long nr_context_swi#ifndef CONFIG_GENERIC_HARDIRQS #define kstat_irqs_this_cpu (irq) \ - (kstat_this_cpu.irqs[irq]) + (this_cpu_read(kstat.irqs[irq])struct irq_desc ;
kernel/exit.c +1 -1
@@ -69,7 +69,7 @@ static void __unhash_process(struct tasklist_del_rcu (&p->tasks);list_del_init (&p->sibling); -__get_cpu_var (process_counts)--; +__this_cpu_dec (process_counts); }list_del_rcu (&p->thread_group); }
kernel/fork.c +1 -1
@@ -1282,7 +1282,7 @@ static struct task_struct *copy_process(attach_pid (p,PIDTYPE_SID ,task_session (current));list_add_tail (&p->sibling, &p->real_parent->children);list_add_tail_rcu (&p->tasks, &init_task.tasks); -__get_cpu_var (process_counts)++; +__this_cpu_inc (process_counts); }attach_pid (p,PIDTYPE_PID , pid); nr_threads++;
kernel/hrtimer.c +1 -1
@@ -497,7 +497,7 @@ static inline int hrtimer_is_hres_enable*/ static inline int hrtimer_hres_active (void ) { -return __get_cpu_var (hrtimer_bases).hres_active; +return __this_cpu_read (hrtimer_bases.hres_active); }/*
Re: [Use cpuops V1 04/11] x86: Use this_cpu_ops for current_cpu_data accesses
In reply to Tejun HeoQuoted 4 lines · Tejun Heo
Quoted 2 lines
Hmmm... Ok 21 occurrences. So two patches: Conversion and then
this_cpu_ops. Or one doing both?Two steps is better probably. Thank you.
Well the first one would have been difficult without the second one. So lets have it in one step. Compiles and runs under kvm with NUMA and UP.
Subject: x86: Replace uses of current_cpu_data with this_cpu ops
Replace all uses of current_cpu_data with this_cpu operations on the per cpu structure cpu_info.
--- arch/x86/include/asm/processor.h | 3 +-- arch/x86/kernel/apic/apic.c | 2 +- arch/x86/kernel/cpu/amd.c | 2 +- arch/x86/kernel/cpu/cpufreq/powernow-k8.c | 2 +- arch/x86/kernel/cpu/intel_cacheinfo.c | 4 ++-- arch/x86/kernel/cpu/mcheck/mce.c | 14 +++++++------- arch/x86/kernel/cpu/mcheck/mce_intel.c | 2 +- arch/x86/kernel/process.c | 4 ++-- arch/x86/kernel/smpboot.c | 12 ++++++------ arch/x86/oprofile/op_model_ppro.c | 8 ++++---- drivers/staging/lirc/lirc_serial.c | 4 ++-- 11 files changed, 28 insertions(+), 29 deletions(-)
arch/x86/include/asm/processor.h +1 -2
@@ -141,10 +141,9 @@ extern __u32 cpu_caps_set[NCAPINTS];#ifdef CONFIG_SMP DECLARE_PER_CPU_SHARED_ALIGNED(struct cpuinfo_x86, cpu_info);#define cpu_data (cpu) per_cpu(cpu_info, cpu) -#define current_cpu_data __get_cpu_var(cpu_info)#else +#define cpu_info boot_cpu_data#define cpu_data (cpu) boot_cpu_data -#define current_cpu_data boot_cpu_data#endif extern const struct seq_operations cpuinfo_op;
arch/x86/kernel/apic/apic.c +1 -1
@@ -516,7 +516,7 @@ static void __cpuinit setup_APIC_timer(v {struct clock_event_device *levt = &__get_cpu_var (lapic_events); -if (cpu_has (¤t_cpu_data,X86_FEATURE_ARAT )) { +if (cpu_has (__this_cpu_ptr (&cpu_info),X86_FEATURE_ARAT )) { lapic_clockevent.features &= ~CLOCK_EVT_FEAT_C3STOP ;/* Make LAPIC timer preferrable over percpu HPET */ lapic_clockevent.rating =150 ;
arch/x86/kernel/cpu/amd.c +1 -1
@@ -668,7 +668,7 @@ EXPORT_SYMBOL_GPL(amd_erratum_383);bool cpu_has_amd_erratum (const int *erratum) { -struct cpuinfo_x86 *cpu = ¤t_cpu_data; +struct cpuinfo_x86 *cpu =__this_cpu_ptr (&cpu_info);int osvw_id = *erratum++;u32 range;u32 ms;
arch/x86/kernel/cpu/cpufreq/powernow-k8.c +1 -1
@@ -521,7 +521,7 @@ static void check_supported_cpu(void *_r *rc = -ENODEV ; -if (current_cpu_data.x86_vendor !=X86_VENDOR_AMD ) +if (__this_cpu_read (cpu_info.x86_vendor) !=X86_VENDOR_AMD )return ; eax =cpuid_eax (CPUID_PROCESSOR_SIGNATURE );
arch/x86/kernel/cpu/intel_cacheinfo.c +2 -2
@@ -266,7 +266,7 @@ amd_cpuid4(int leaf, union _cpuid4_leaf_ line_size = l2.line_size; lines_per_tag = l2.lines_per_tag;/* cpu_data has errata corrections for K7 applied */ - size_in_kb = current_cpu_data.x86_cache_size; + size_in_kb =__this_cpu_read (cpu_info.x86_cache_size);break ;case 3 :if (!l3.val)
@@ -288,7 +288,7 @@ amd_cpuid4(int leaf, union _cpuid4_leaf_ eax->split.type = types[leaf]; eax->split.level = levels[leaf]; eax->split.num_threads_sharing =0 ; - eax->split.num_cores_on_die = current_cpu_data.x86_max_cores -1 ; + eax->split.num_cores_on_die =__this_cpu_read (cpu_info.x86_max_cores) -1 ;if (assoc ==0xffff )
arch/x86/kernel/cpu/mcheck/mce.c +7 -7
@@ -1159,7 +1159,7 @@ static void mce_start_timer(unsigned lonWARN_ON (smp_processor_id () != data); -if (mce_available (¤t_cpu_data)) { +if (mce_available (__this_cpu_ptr (&cpu_info))) {machine_check_poll (MCP_TIMESTAMP , &__get_cpu_var (mce_poll_banks)); }
@@ -1767,7 +1767,7 @@ static int mce_shutdown(struct sys_devicstatic int mce_resume (struct sys_device *dev) {__mcheck_cpu_init_generic (); -__mcheck_cpu_init_vendor (¤t_cpu_data); +__mcheck_cpu_init_vendor (__this_cpu_ptr (&cpu_info));return 0 ; }
@@ -1775,7 +1775,7 @@ static int mce_resume(struct sys_devicestatic void mce_cpu_restart (void *data) {del_timer_sync (&__get_cpu_var (mce_timer)); -if (!mce_available (¤t_cpu_data)) +if (!mce_available (__this_cpu_ptr (&cpu_info)))return ;__mcheck_cpu_init_generic ();__mcheck_cpu_init_timer ();
@@ -1790,7 +1790,7 @@ static void mce_restart(void)/* Toggle features for corrected errors */ static void mce_disable_ce (void *all) { -if (!mce_available (¤t_cpu_data)) +if (!mce_available (__this_cpu_ptr (&cpu_info)))return ;if (all)del_timer_sync (&__get_cpu_var (mce_timer));
@@ -1799,7 +1799,7 @@ static void mce_disable_ce(void *all)static void mce_enable_ce (void *all) { -if (!mce_available (¤t_cpu_data)) +if (!mce_available (__this_cpu_ptr (&cpu_info)))return ;cmci_reenable ();cmci_recheck ();
@@ -2022,7 +2022,7 @@ static void __cpuinit mce_disable_cpu(vounsigned long action = *(unsigned long *)h;int i; -if (!mce_available (¤t_cpu_data)) +if (!mce_available (__this_cpu_ptr (&cpu_info)))return ;if (!(action &CPU_TASKS_FROZEN ))
@@ -2040,7 +2040,7 @@ static void __cpuinit mce_reenable_cpu(vunsigned long action = *(unsigned long *)h;int i; -if (!mce_available (¤t_cpu_data)) +if (!mce_available (__this_cpu_ptr (&cpu_info)))return ;if (!(action &CPU_TASKS_FROZEN ))
arch/x86/kernel/cpu/mcheck/mce_intel.c +1 -1
@@ -130,7 +130,7 @@ void cmci_recheck(void)unsigned long flags;int banks; -if (!mce_available (¤t_cpu_data) || !cmci_supported (&banks)) +if (!mce_available (__this_cpu_ptr (&cpu_info)) || !cmci_supported (&banks))return ;local_irq_save (flags);machine_check_poll (MCP_TIMESTAMP , &__get_cpu_var (mce_banks_owned));
arch/x86/kernel/process.c +2 -2
@@ -445,7 +445,7 @@ void mwait_idle_with_hints(unsigned long {trace_power_start (POWER_CSTATE , (ax>>4 )+1 ,smp_processor_id ());if (!need_resched ()) { -if (cpu_has (¤t_cpu_data,X86_FEATURE_CLFLUSH_MONITOR )) +if (cpu_has (__this_cpu_ptr (&cpu_info),X86_FEATURE_CLFLUSH_MONITOR ))clflush ((void *)¤t_thread_info ()->flags);__monitor ((void *)¤t_thread_info ()->flags,0 ,0 );
@@ -460,7 +460,7 @@ static void mwait_idle(void) {if (!need_resched ()) {trace_power_start (POWER_CSTATE ,1 ,smp_processor_id ()); -if (cpu_has (¤t_cpu_data,X86_FEATURE_CLFLUSH_MONITOR )) +if (cpu_has (__this_cpu_ptr (&cpu_info),X86_FEATURE_CLFLUSH_MONITOR ))clflush ((void *)¤t_thread_info ()->flags);__monitor ((void *)¤t_thread_info ()->flags,0 ,0 );
arch/x86/kernel/smpboot.c +6 -6
@@ -430,7 +430,7 @@ void __cpuinit set_cpu_sibling_map(int ccpumask_set_cpu (cpu, c->llc_shared_map); -if (current_cpu_data.x86_max_cores ==1 ) { +if (__this_cpu_read (cpu_info.x86_max_cores) ==1 ) {cpumask_copy (cpu_core_mask (cpu),cpu_sibling_mask (cpu)); c->booted_cores =1 ;return ;
@@ -1094,7 +1094,7 @@ void __init native_smp_prepare_cpus(unsipreempt_disable ();smp_cpu_index_default (); - current_cpu_data = boot_cpu_data; +memcpy (__this_cpu_ptr (&cpu_info), &boot_cpu_data,sizeof (cpu_info));cpumask_copy (cpu_callin_mask,cpumask_of (0 ));mb ();/*
@@ -1397,11 +1397,11 @@ static inline void mwait_play_dead(void)int i;void *mwait_ptr; -if (!cpu_has (¤t_cpu_data,X86_FEATURE_MWAIT )) +if (!cpu_has (__this_cpu_ptr (&cpu_info),X86_FEATURE_MWAIT ))return ; -if (!cpu_has (¤t_cpu_data,X86_FEATURE_CLFLSH )) +if (!cpu_has (__this_cpu_ptr (&cpu_info),X86_FEATURE_CLFLSH ))return ; -if (current_cpu_data.cpuid_level <CPUID_MWAIT_LEAF ) +if (__this_cpu_read (cpu_info.cpuid_level) <CPUID_MWAIT_LEAF )return ; eax =CPUID_MWAIT_LEAF ;
@@ -1452,7 +1452,7 @@ static inline void mwait_play_dead(void)static inline void hlt_play_dead (void ) { -if (current_cpu_data.x86 >=4 ) +if (__this_cpu_read (cpu_info.x86) >=4 )wbinvd ();while (1 ) {
arch/x86/oprofile/op_model_ppro.c +4 -4
@@ -95,8 +95,8 @@ static void ppro_setup_ctrs(struct op_x8* counter width: */ if (!(eax.split.version_id ==0 && - current_cpu_data.x86 ==6 && - current_cpu_data.x86_model ==15 )) { +__this_cpu_read (cpu_info.x86) ==6 && +__this_cpu_read (cpu_info.x86_model) ==15 )) {if (counter_width < eax.split.bit_width) counter_width = eax.split.bit_width;
@@ -235,8 +235,8 @@ static void arch_perfmon_setup_counters( eax.full =cpuid_eax (0xa );/* Workaround for BIOS bugs in 6/15. Taken from perfmon2 */ -if (eax.split.version_id ==0 && current_cpu_data.x86 ==6 && - current_cpu_data.x86_model ==15 ) { +if (eax.split.version_id ==0 &&__this_cpu_read (cpu_info.x86) ==6 && +__this_cpu_read (cpu_info.x86_model) ==15 ) { eax.split.version_id =2 ; eax.split.num_counters =2 ; eax.split.bit_width =40 ;
drivers/staging/lirc/lirc_serial.c +2 -2
@@ -377,7 +377,7 @@ static int init_timing_params(unsigned i duty_cycle = new_duty_cycle; freq = new_freq; - loops_per_sec = current_cpu_data.loops_per_jiffy; + loops_per_sec =__this_cpu_read (cpu.info.loops_per_jiffy); loops_per_sec *=HZ ;/* How many clocks in a microsecond?, avoiding long long divide */
@@ -398,7 +398,7 @@ static int init_timing_params(unsigned idprintk ("in init_timing_params, freq=%d, duty_cycle=%d, " "clk/jiffy=%ld, pulse=%ld, space=%ld, " "conv_us_to_clocks=%ld\n" , - freq, duty_cycle, current_cpu_data.loops_per_jiffy, + freq, duty_cycle,__this_cpu_read (cpu_info.loops_per_jiffy), pulse_width, space_width, conv_us_to_clocks);return 0 ; }
Tejun Heo
maintainerPATCH 1/1 x86: Replace uses of current_cpu_data with this_cpu ops
Quoted 1 line
From
9e9b27bff0e2441512ec27d3a90b252ecdc6ec3eMon Sep 17 00:00:00 2001
From: Christoph Lameter <[email protected]>
Date: Wed, 8 Dec 2010 14:11:07 +0100
Replace all uses of current_cpu_data with this_cpu operations on the per cpu structure cpu_info. The scala accesses are replaced with the matching this_cpu ops which results in smaller and more efficient code.
In the long run, it might be a good idea to remove cpu_data() macro too and use per_cpu macro directly.
tj: updated description
---
Comitted with updated description. Thanks.
arch/x86/include/asm/processor.h | 3 +-- arch/x86/kernel/apic/apic.c | 2 +- arch/x86/kernel/cpu/amd.c | 2 +- arch/x86/kernel/cpu/cpufreq/powernow-k8.c | 2 +- arch/x86/kernel/cpu/intel_cacheinfo.c | 4 ++-- arch/x86/kernel/cpu/mcheck/mce.c | 14 +++++++------- arch/x86/kernel/cpu/mcheck/mce_intel.c | 2 +- arch/x86/kernel/process.c | 4 ++-- arch/x86/kernel/smpboot.c | 12 ++++++------ arch/x86/oprofile/op_model_ppro.c | 8 ++++---- drivers/staging/lirc/lirc_serial.c | 4 ++-- 11 files changed, 28 insertions(+), 29 deletions(-)
arch/x86/include/asm/processor.h +1 -2
@@ -141,10 +141,9 @@ extern __u32 cpu_caps_set[NCAPINTS];#ifdef CONFIG_SMP DECLARE_PER_CPU_SHARED_ALIGNED(struct cpuinfo_x86, cpu_info);#define cpu_data (cpu) per_cpu(cpu_info, cpu) -#define current_cpu_data __get_cpu_var(cpu_info)#else +#define cpu_info boot_cpu_data#define cpu_data (cpu) boot_cpu_data -#define current_cpu_data boot_cpu_data#endif extern const struct seq_operations cpuinfo_op;
arch/x86/kernel/apic/apic.c +1 -1
@@ -516,7 +516,7 @@ static void __cpuinit setup_APIC_timer(void) {struct clock_event_device *levt = &__get_cpu_var (lapic_events); -if (cpu_has (¤t_cpu_data,X86_FEATURE_ARAT )) { +if (cpu_has (__this_cpu_ptr (&cpu_info),X86_FEATURE_ARAT )) { lapic_clockevent.features &= ~CLOCK_EVT_FEAT_C3STOP ;/* Make LAPIC timer preferrable over percpu HPET */ lapic_clockevent.rating =150 ;
arch/x86/kernel/cpu/amd.c +1 -1
@@ -668,7 +668,7 @@ EXPORT_SYMBOL_GPL(amd_erratum_383);bool cpu_has_amd_erratum (const int *erratum) { -struct cpuinfo_x86 *cpu = ¤t_cpu_data; +struct cpuinfo_x86 *cpu =__this_cpu_ptr (&cpu_info);int osvw_id = *erratum++;u32 range;u32 ms;
arch/x86/kernel/cpu/cpufreq/powernow-k8.c +1 -1
@@ -521,7 +521,7 @@ static void check_supported_cpu(void *_rc) *rc = -ENODEV ; -if (current_cpu_data.x86_vendor !=X86_VENDOR_AMD ) +if (__this_cpu_read (cpu_info.x86_vendor) !=X86_VENDOR_AMD )return ; eax =cpuid_eax (CPUID_PROCESSOR_SIGNATURE );
arch/x86/kernel/cpu/intel_cacheinfo.c +2 -2
@@ -266,7 +266,7 @@ amd_cpuid4(int leaf, union _cpuid4_leaf_eax *eax, line_size = l2.line_size; lines_per_tag = l2.lines_per_tag;/* cpu_data has errata corrections for K7 applied */ - size_in_kb = current_cpu_data.x86_cache_size; + size_in_kb =__this_cpu_read (cpu_info.x86_cache_size);break ;case 3 :if (!l3.val)
@@ -288,7 +288,7 @@ amd_cpuid4(int leaf, union _cpuid4_leaf_eax *eax, eax->split.type = types[leaf]; eax->split.level = levels[leaf]; eax->split.num_threads_sharing =0 ; - eax->split.num_cores_on_die = current_cpu_data.x86_max_cores -1 ; + eax->split.num_cores_on_die =__this_cpu_read (cpu_info.x86_max_cores) -1 ;if (assoc ==0xffff )
arch/x86/kernel/cpu/mcheck/mce.c +7 -7
@@ -1159,7 +1159,7 @@ static void mce_start_timer(unsigned long data)WARN_ON (smp_processor_id () != data); -if (mce_available (¤t_cpu_data)) { +if (mce_available (__this_cpu_ptr (&cpu_info))) {machine_check_poll (MCP_TIMESTAMP , &__get_cpu_var (mce_poll_banks)); }
@@ -1767,7 +1767,7 @@ static int mce_shutdown(struct sys_device *dev)static int mce_resume (struct sys_device *dev) {__mcheck_cpu_init_generic (); -__mcheck_cpu_init_vendor (¤t_cpu_data); +__mcheck_cpu_init_vendor (__this_cpu_ptr (&cpu_info));return 0 ; }
@@ -1775,7 +1775,7 @@ static int mce_resume(struct sys_device *dev)static void mce_cpu_restart (void *data) {del_timer_sync (&__get_cpu_var (mce_timer)); -if (!mce_available (¤t_cpu_data)) +if (!mce_available (__this_cpu_ptr (&cpu_info)))return ;__mcheck_cpu_init_generic ();__mcheck_cpu_init_timer ();
@@ -1790,7 +1790,7 @@ static void mce_restart(void)/* Toggle features for corrected errors */ static void mce_disable_ce (void *all) { -if (!mce_available (¤t_cpu_data)) +if (!mce_available (__this_cpu_ptr (&cpu_info)))return ;if (all)del_timer_sync (&__get_cpu_var (mce_timer));
@@ -1799,7 +1799,7 @@ static void mce_disable_ce(void *all)static void mce_enable_ce (void *all) { -if (!mce_available (¤t_cpu_data)) +if (!mce_available (__this_cpu_ptr (&cpu_info)))return ;cmci_reenable ();cmci_recheck ();
@@ -2022,7 +2022,7 @@ static void __cpuinit mce_disable_cpu(void *h)unsigned long action = *(unsigned long *)h;int i; -if (!mce_available (¤t_cpu_data)) +if (!mce_available (__this_cpu_ptr (&cpu_info)))return ;if (!(action &CPU_TASKS_FROZEN ))
@@ -2040,7 +2040,7 @@ static void __cpuinit mce_reenable_cpu(void *h)unsigned long action = *(unsigned long *)h;int i; -if (!mce_available (¤t_cpu_data)) +if (!mce_available (__this_cpu_ptr (&cpu_info)))return ;if (!(action &CPU_TASKS_FROZEN ))
arch/x86/kernel/cpu/mcheck/mce_intel.c +1 -1
@@ -130,7 +130,7 @@ void cmci_recheck(void)unsigned long flags;int banks; -if (!mce_available (¤t_cpu_data) || !cmci_supported (&banks)) +if (!mce_available (__this_cpu_ptr (&cpu_info)) || !cmci_supported (&banks))return ;local_irq_save (flags);machine_check_poll (MCP_TIMESTAMP , &__get_cpu_var (mce_banks_owned));
arch/x86/kernel/process.c +2 -2
@@ -445,7 +445,7 @@ void mwait_idle_with_hints(unsigned long ax, unsigned long cx) {trace_power_start (POWER_CSTATE , (ax>>4 )+1 ,smp_processor_id ());if (!need_resched ()) { -if (cpu_has (¤t_cpu_data,X86_FEATURE_CLFLUSH_MONITOR )) +if (cpu_has (__this_cpu_ptr (&cpu_info),X86_FEATURE_CLFLUSH_MONITOR ))clflush ((void *)¤t_thread_info ()->flags);__monitor ((void *)¤t_thread_info ()->flags,0 ,0 );
@@ -460,7 +460,7 @@ static void mwait_idle(void) {if (!need_resched ()) {trace_power_start (POWER_CSTATE ,1 ,smp_processor_id ()); -if (cpu_has (¤t_cpu_data,X86_FEATURE_CLFLUSH_MONITOR )) +if (cpu_has (__this_cpu_ptr (&cpu_info),X86_FEATURE_CLFLUSH_MONITOR ))clflush ((void *)¤t_thread_info ()->flags);__monitor ((void *)¤t_thread_info ()->flags,0 ,0 );
arch/x86/kernel/smpboot.c +6 -6
@@ -430,7 +430,7 @@ void __cpuinit set_cpu_sibling_map(int cpu)cpumask_set_cpu (cpu, c->llc_shared_map); -if (current_cpu_data.x86_max_cores ==1 ) { +if (__this_cpu_read (cpu_info.x86_max_cores) ==1 ) {cpumask_copy (cpu_core_mask (cpu),cpu_sibling_mask (cpu)); c->booted_cores =1 ;return ;
@@ -1094,7 +1094,7 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus)preempt_disable ();smp_cpu_index_default (); - current_cpu_data = boot_cpu_data; +memcpy (__this_cpu_ptr (&cpu_info), &boot_cpu_data,sizeof (cpu_info));cpumask_copy (cpu_callin_mask,cpumask_of (0 ));mb ();/*
@@ -1397,11 +1397,11 @@ static inline void mwait_play_dead(void)int i;void *mwait_ptr; -if (!cpu_has (¤t_cpu_data,X86_FEATURE_MWAIT )) +if (!cpu_has (__this_cpu_ptr (&cpu_info),X86_FEATURE_MWAIT ))return ; -if (!cpu_has (¤t_cpu_data,X86_FEATURE_CLFLSH )) +if (!cpu_has (__this_cpu_ptr (&cpu_info),X86_FEATURE_CLFLSH ))return ; -if (current_cpu_data.cpuid_level <CPUID_MWAIT_LEAF ) +if (__this_cpu_read (cpu_info.cpuid_level) <CPUID_MWAIT_LEAF )return ; eax =CPUID_MWAIT_LEAF ;
@@ -1452,7 +1452,7 @@ static inline void mwait_play_dead(void)static inline void hlt_play_dead (void ) { -if (current_cpu_data.x86 >=4 ) +if (__this_cpu_read (cpu_info.x86) >=4 )wbinvd ();while (1 ) {
arch/x86/oprofile/op_model_ppro.c +4 -4
@@ -95,8 +95,8 @@ static void ppro_setup_ctrs(struct op_x86_model_spec const *model,* counter width: */ if (!(eax.split.version_id ==0 && - current_cpu_data.x86 ==6 && - current_cpu_data.x86_model ==15 )) { +__this_cpu_read (cpu_info.x86) ==6 && +__this_cpu_read (cpu_info.x86_model) ==15 )) {if (counter_width < eax.split.bit_width) counter_width = eax.split.bit_width;
@@ -235,8 +235,8 @@ static void arch_perfmon_setup_counters(void) eax.full =cpuid_eax (0xa );/* Workaround for BIOS bugs in 6/15. Taken from perfmon2 */ -if (eax.split.version_id ==0 && current_cpu_data.x86 ==6 && - current_cpu_data.x86_model ==15 ) { +if (eax.split.version_id ==0 &&__this_cpu_read (cpu_info.x86) ==6 && +__this_cpu_read (cpu_info.x86_model) ==15 ) { eax.split.version_id =2 ; eax.split.num_counters =2 ; eax.split.bit_width =40 ;
drivers/staging/lirc/lirc_serial.c +2 -2
@@ -377,7 +377,7 @@ static int init_timing_params(unsigned int new_duty_cycle, duty_cycle = new_duty_cycle; freq = new_freq; - loops_per_sec = current_cpu_data.loops_per_jiffy; + loops_per_sec =__this_cpu_read (cpu.info.loops_per_jiffy); loops_per_sec *=HZ ;/* How many clocks in a microsecond?, avoiding long long divide */
@@ -398,7 +398,7 @@ static int init_timing_params(unsigned int new_duty_cycle,dprintk ("in init_timing_params, freq=%d, duty_cycle=%d, " "clk/jiffy=%ld, pulse=%ld, space=%ld, " "conv_us_to_clocks=%ld\n" , - freq, duty_cycle, current_cpu_data.loops_per_jiffy, + freq, duty_cycle,__this_cpu_read (cpu_info.loops_per_jiffy), pulse_width, space_width, conv_us_to_clocks);return 0 ; }
Signature · 1 line
1.7.1
Re: [PATCH] x86: Replace uses of current_cpu_data with this_cpu ops
Quoted 4 lines · Tejun Heo
From
9e9b27bff0e2441512ec27d3a90b252ecdc6ec3eMon Sep 17 00:00:00 2001
From: Christoph Lameter <[email protected]>
Date: Wed, 8 Dec 2010 14:11:07 +0100
Quoted 8 lines
Replace all uses of
current_cpu_datawiththis_cpuoperations on the per cpu structurecpu_info. The scala accesses are replaced with the matchingthis_cpuops which results in smaller and more efficient code.In the long run, it might be a good idea to remove
cpu_data()macro too and useper_cpumacro directly.
or introduce this_cpu_has() to remove the adress computation
- if (cpu_has(__this_cpu_ptr(&cpu_info), X86_FEATURE_ARAT)) {
+ if (this_cpu_has(X86_FEATURE_ARAT)) {
Quoted 51 lines
tj: updated description
Cc Yinghai Lu <[email protected]>Cc Ingo Molnar <[email protected]>Signed-off-by Christoph Lameter <[email protected]>Signed-off-by Tejun Heo <[email protected]>---
Comitted with updated description. Thanks.arch/x86/include/asm/processor.h | 3 +-- arch/x86/kernel/apic/apic.c | 2 +- arch/x86/kernel/cpu/amd.c | 2 +- arch/x86/kernel/cpu/cpufreq/powernow-k8.c | 2 +- arch/x86/kernel/cpu/intel_cacheinfo.c | 4 ++-- arch/x86/kernel/cpu/mcheck/mce.c | 14 +++++++------- arch/x86/kernel/cpu/mcheck/mce_intel.c | 2 +- arch/x86/kernel/process.c | 4 ++-- arch/x86/kernel/smpboot.c | 12 ++++++------ arch/x86/oprofile/op_model_ppro.c | 8 ++++---- drivers/staging/lirc/lirc_serial.c | 4 ++-- 11 files changed, 28 insertions(+), 29 deletions(-)arch/x86/include/asm/processor.h +1
-2@@ -141,10 +141,9 @@ extern __u32 cpu_caps_set[NCAPINTS];#ifdef CONFIG_SMP DECLARE_PER_CPU_SHARED_ALIGNED(struct cpuinfo_x86, cpu_info);#define cpu_data (cpu) per_cpu(cpu_info, cpu) -#define current_cpu_data __get_cpu_var(cpu_info)#else +#define cpu_info boot_cpu_data#define cpu_data (cpu) boot_cpu_data -#define current_cpu_data boot_cpu_data#endif extern const struct seq_operations cpuinfo_op;arch/x86/kernel/apic/apic.c +1
-1@@ -516,7 +516,7 @@ static void __cpuinit setup_APIC_timer(void) {struct clock_event_device *levt = &__get_cpu_var (lapic_events); -if (cpu_has (¤t_cpu_data,X86_FEATURE_ARAT )) { +if (cpu_has (__this_cpu_ptr (&cpu_info),X86_FEATURE_ARAT )) { lapic_clockevent.features &= ~CLOCK_EVT_FEAT_C3STOP ;/* Make LAPIC timer preferrable over percpu HPET */ lapic_clockevent.rating =150 ;
...
Re: [Use cpuops V1 05/11] core: Replace __get_cpu_var with __this_cpu_read if not used for an address.
In reply to Christoph LameterQuoted 20 lines · Christoph Lameter
Quoted 6 lines · Tejun Heo
Can you please fold this into the original patch and resend? I skipped 04, 05 and 09. percpu#for-next now contains other patches.
git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu.gitfor-nextThanks.
Subject: core: Replace
__get_cpu_varwith__this_cpu_readif not used for an address.
__get_cpu_var()can be replaced withthis_cpu_readand will then use a single read instruction with implied address calculation to access the correct per cpu instance.However, the address of a per cpu variable passed to
__this_cpu_read()cannot be determed (since its an implied address conversion through segment prefixes). Therefore apply this only to uses of__get_cpu_varwhere the addres of the variable is not used.
Quoted 4 lines
Cc Pekka Enberg <[email protected]>Cc Hugh Dickins <[email protected]>Cc Thomas Gleixner <[email protected]>Signed-off-by Christoph Lameter <[email protected]>
Applied. Thanks.
Signature · 1 line
tejun
Re: [PATCH] x86: Replace uses of current_cpu_data with this_cpu ops
In reply to Eric DumazetReview
Quoted 10 lines · Eric Dumazet
Quoted 3 lines
In the long run, it might be a good idea to remove
cpu_data()macro too and useper_cpumacro directly.or introduce
this_cpu_has()to remove the adress computation(quoted diff) +1
-1- if (cpu_has(__this_cpu_ptr(&cpu_info), X86_FEATURE_ARAT)) { + if (this_cpu_has(X86_FEATURE_ARAT)) {
The fundamental problem is that bitops require memory addresses which does not work with per cpu ops.
Re: [PATCH] x86: Replace uses of current_cpu_data with this_cpu ops
Quoted 8 lines · Christoph Lameter
Quoted 6 lines · Eric Dumazet
Quoted 3 lines
In the long run, it might be a good idea to remove
cpu_data()macro too and useper_cpumacro directly.or introduce
this_cpu_has()to remove the adress computation
Quoted 8 lines
Review
Quoted 4 lines
(quoted diff) +1
-1- if (cpu_has(__this_cpu_ptr(&cpu_info), X86_FEATURE_ARAT)) { + if (this_cpu_has(X86_FEATURE_ARAT)) {The fundamental problem is that bitops require memory addresses which does not work with per cpu ops.
Sure, set() or clear() needs the address, but the read doesnt ?
This can be implemented as
(__this_cpu_read(cpu_info.word[X]) & MASK)
Anyway, even if mapped to cpu_has(__this_cpu_ptr(&cpu_info), XXX), it would be cleaner to use this_cpu_has() helper.
Tejun Heo
maintainerRe: [PATCH] x86: Replace uses of current_cpu_data with this_cpu ops
Hello,
Quoted 18 lines · Eric Dumazet
Quoted 8 lines · Christoph Lameter
Quoted 6 lines · Eric Dumazet
Quoted 3 lines
In the long run, it might be a good idea to remove
cpu_data()macro too and useper_cpumacro directly.or introduce
this_cpu_has()to remove the adress computationQuoted 8 lines
Review
Quoted 4 lines
(quoted diff) +1
-1- if (cpu_has(__this_cpu_ptr(&cpu_info), X86_FEATURE_ARAT)) { + if (this_cpu_has(X86_FEATURE_ARAT)) {The fundamental problem is that bitops require memory addresses which does not work with per cpu ops.
Quoted 8 lines
Sure,
set()orclear()needs the address, but the read doesnt ?This can be implemented as
(
__this_cpu_read(cpu_info.word[X])& MASK)Anyway, even if mapped to
cpu_has(__this_cpu_ptr(&cpu_info), XXX), it would be cleaner to usethis_cpu_has()helper.
Agreed. I'd also like to see cpu_data() gone too. It isn't used too many times and mixed with direct percpu ops only makes things more confusing without much benefit. Does somebody care to send patches for these cleanups?
Thanks.
Signature · 1 line
tejun
Re: [PATCH] x86: Replace uses of current_cpu_data with this_cpu ops
Beginnings of a patch to avoid bitops. If you think this is the right way then I will complete it for all uses of cpu_has.
--- arch/x86/include/asm/cpufeature.h | 15 +++++++++++++++ arch/x86/kernel/apic/apic.c | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-)
arch/x86/include/asm/cpufeature.h +15 -0
@@ -221,6 +221,21 @@ extern const char * const x86_power_flag ? 1 : \ test_cpu_cap(c, bit)) +#define this_cpu_has (bit) \ + (__builtin_constant_p(bit) && \ + ( (((bit)>>5)==0 && (1UL<<((bit)&31) & REQUIRED_MASK0)) || \ + (((bit)>>5)==1 && (1UL<<((bit)&31) & REQUIRED_MASK1)) || \ + (((bit)>>5)==2 && (1UL<<((bit)&31) & REQUIRED_MASK2)) || \ + (((bit)>>5)==3 && (1UL<<((bit)&31) & REQUIRED_MASK3)) || \ + (((bit)>>5)==4 && (1UL<<((bit)&31) & REQUIRED_MASK4)) || \ + (((bit)>>5)==5 && (1UL<<((bit)&31) & REQUIRED_MASK5)) || \ + (((bit)>>5)==6 && (1UL<<((bit)&31) & REQUIRED_MASK6)) || \ + (((bit)>>5)==7 && (1UL<<((bit)&31) & REQUIRED_MASK7)) || \ + (((bit)>>5)==8 && (1UL<<((bit)&31) & REQUIRED_MASK8)) || \ + (((bit)>>5)==9 && (1UL<<((bit)&31) & REQUIRED_MASK9)) ) \ + ? 1 : \ + (this_cpu_read(cpu_info.x86_capabilty) & (1 << bit))) +#define boot_cpu_has (bit) cpu_has(&boot_cpu_data, bit)#define set_cpu_cap (c, bit) set_bit(bit, (unsigned long *)((c)->x86_capability))
arch/x86/kernel/apic/apic.c +1 -1
@@ -516,7 +516,7 @@ static void __cpuinit setup_APIC_timer(v {struct clock_event_device *levt = &__get_cpu_var (lapic_events); -if (cpu_has (__this_cpu_ptr (&cpu_info),X86_FEATURE_ARAT )) { +if (this_cpu_has (X86_FEATURE_ARAT )) { lapic_clockevent.features &= ~CLOCK_EVT_FEAT_C3STOP ;/* Make LAPIC timer preferrable over percpu HPET */ lapic_clockevent.rating =150 ;
Tejun Heo
maintainerRe: [PATCH] x86: Replace uses of current_cpu_data with this_cpu ops
Hello, Christoph.
Quoted 19 lines · Christoph Lameter
Beginnings of a patch to avoid bitops. If you think this is the right way then I will complete it for all uses of
cpu_has.--- arch/x86/include/asm/cpufeature.h | 15 +++++++++++++++ arch/x86/kernel/apic/apic.c | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-)Index:
linux-2.6/arch/x86/include/asm/cpufeature.h
===================================================================arch/x86/include/asm/cpufeature.h +3
-0@@ -221,6 +221,21 @@ extern const char * const x86_power_flag ? 1 : \ test_cpu_cap(c, bit)) +#define this_cpu_has (bit) \ + (__builtin_constant_p(bit) && \ + ( (((bit)>>5)==0 && (1UL<<((bit)&31) & REQUIRED_MASK0)) || \
arch/x86/include/asm/cpufeature.h extern const char * const x86_power_flag · line 236
Quoted 10 lines
arch/x86/include/asm/cpufeature.h +10
-0+ (((bit)>>5 )==1 && (1UL <<((bit)&31 ) &REQUIRED_MASK1 )) || \ + (((bit)>>5 )==2 && (1UL <<((bit)&31 ) &REQUIRED_MASK2 )) || \ + (((bit)>>5 )==3 && (1UL <<((bit)&31 ) &REQUIRED_MASK3 )) || \ + (((bit)>>5 )==4 && (1UL <<((bit)&31 ) &REQUIRED_MASK4 )) || \ + (((bit)>>5 )==5 && (1UL <<((bit)&31 ) &REQUIRED_MASK5 )) || \ + (((bit)>>5 )==6 && (1UL <<((bit)&31 ) &REQUIRED_MASK6 )) || \ + (((bit)>>5 )==7 && (1UL <<((bit)&31 ) &REQUIRED_MASK7 )) || \ + (((bit)>>5 )==8 && (1UL <<((bit)&31 ) &REQUIRED_MASK8 )) || \ + (((bit)>>5)==9 && (1UL<<((bit)&31) & REQUIRED_MASK9)) ) \ + ? 1 : \
It would be nice to factor out the above so that it's not repated for cpu_has() and this_cpu_has().
arch/x86/include/asm/cpufeature.h extern const char * const x86_power_flag · line 237
Quoted 1 line
arch/x86/include/asm/cpufeature.h +1
-0+ (this_cpu_read(cpu_info.x86_capabilty) & (1 << bit)))
bit can go beyond unsigned, so 1 << bit doesn't work. Why not just start with a simple wrapper? ie. sth like
#define this_cpu_has(bit) cpu_has(__this_cpu_ptr(&cpu_info), bit)
We can later optimize in a separate patch along the line of...
#define this_cpu_has(bit) ({ \
__builtin_constant_p(bit) ? \
(cpu_required_bit_test(bit) || \
this_cpu_read(cpu_info.x86_capability + ((bit) >> 5)) & \
(1 << ((bit) & 31))) : \
cpu_has(__this_cpu_ptr(&cpu_info), bit); \
})Thanks.
Signature · 1 line
tejun