Code clean up for percpu_xxx() functions
Subsystems (11)
PATCH 1/1 Code clean up for percpu_xxx() functions
Since percpu_xxx() serial are duplicated with this_cpu_xxx(). I transfer percpu_xxx() to this_cpu_xxx(). This patch is just do code clean up, no function change.
Could you like to review this patch?
Signed-off-by Alex Shi <[email protected]>
--- arch/x86/include/asm/current.h | 2 +- arch/x86/include/asm/hardirq.h | 8 ++-- arch/x86/include/asm/irq_regs.h | 4 +- arch/x86/include/asm/mmu_context.h | 12 ++++---- arch/x86/include/asm/percpu.h | 28 +++++++++--------- arch/x86/include/asm/smp.h | 4 +- arch/x86/include/asm/stackprotector.h | 4 +- arch/x86/include/asm/thread_info.h | 2 +- arch/x86/include/asm/tlbflush.h | 4 +- arch/x86/kernel/cpu/common.c | 2 +- arch/x86/kernel/cpu/mcheck/mce.c | 4 +- arch/x86/kernel/cpu/perf_event.c | 4 +- arch/x86/kernel/paravirt.c | 12 ++++---- arch/x86/kernel/process_32.c | 2 +- arch/x86/kernel/process_64.c | 12 ++++---- arch/x86/kvm/x86.c | 14 ++++---- arch/x86/mm/tlb.c | 10 +++--- arch/x86/xen/enlighten.c | 6 ++-- arch/x86/xen/irq.c | 8 ++-- arch/x86/xen/mmu.c | 20 ++++++------ arch/x86/xen/multicalls.h | 2 +- arch/x86/xen/smp.c | 2 +- include/linux/percpu.h | 53 --------------------------------- include/linux/topology.h | 4 +- net/netfilter/xt_TEE.c | 12 ++++---- net/socket.c | 4 +- 26 files changed, 93 insertions(+), 146 deletions(-)
arch/x86/include/asm/current.h +1 -1
@@ -11,7 +11,7 @@ DECLARE_PER_CPU(struct task_struct *, current_task); static __always_inline struct task_struct *get_current(void) { -return percpu_read_stable (current_task); +return this_cpu_read_stable (current_task); }#define current get_current()
arch/x86/include/asm/hardirq.h +4 -4
@@ -35,14 +35,14 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);#define __ARCH_IRQ_STAT -#define inc_irq_stat (member) percpu_inc(irq_stat.member) +#define inc_irq_stat (member) this_cpu_inc(irq_stat.member) -#define local_softirq_pending () percpu_read(irq_stat.__softirq_pending) +#define local_softirq_pending () this_cpu_read(irq_stat.__softirq_pending)#define __ARCH_SET_SOFTIRQ_PENDING -#define set_softirq_pending (x) percpu_write(irq_stat.__softirq_pending, (x)) -#define or_softirq_pending (x) percpu_or(irq_stat.__softirq_pending, (x)) +#define set_softirq_pending (x) this_cpu_write(irq_stat.__softirq_pending, (x)) +#define or_softirq_pending (x) this_cpu_or(irq_stat.__softirq_pending, (x))extern void ack_bad_irq (unsigned int irq);
arch/x86/include/asm/irq_regs.h +2 -2
@@ -15,7 +15,7 @@ DECLARE_PER_CPU(struct pt_regs *, irq_regs);static inline struct pt_regs *get_irq_regs (void ) { -return percpu_read (irq_regs); +return this_cpu_read (irq_regs); }static inline struct pt_regs *set_irq_regs (struct pt_regs *new_regs)
@@ -23,7 +23,7 @@ static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)struct pt_regs *old_regs; old_regs =get_irq_regs (); -percpu_write (irq_regs, new_regs); +this_cpu_write (irq_regs, new_regs);return old_regs; }
arch/x86/include/asm/mmu_context.h +6 -6
@@ -25,8 +25,8 @@ void destroy_context(struct mm_struct *mm);static inline void enter_lazy_tlb (struct mm_struct *mm,struct task_struct *tsk) {#ifdef CONFIG_SMP -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) -percpu_write (cpu_tlbstate.state,TLBSTATE_LAZY ); +if (this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) +this_cpu_write (cpu_tlbstate.state,TLBSTATE_LAZY );#endif }
@@ -37,8 +37,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,if (likely (prev != next)) {#ifdef CONFIG_SMP -percpu_write (cpu_tlbstate.state,TLBSTATE_OK ); -percpu_write (cpu_tlbstate.active_mm, next); +this_cpu_write (cpu_tlbstate.state,TLBSTATE_OK ); +this_cpu_write (cpu_tlbstate.active_mm, next);#endif cpumask_set_cpu (cpu,mm_cpumask (next));
@@ -56,8 +56,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, }#ifdef CONFIG_SMP else { -percpu_write (cpu_tlbstate.state,TLBSTATE_OK ); -BUG_ON (percpu_read (cpu_tlbstate.active_mm) != next); +this_cpu_write (cpu_tlbstate.state,TLBSTATE_OK ); +BUG_ON (this_cpu_read (cpu_tlbstate.active_mm) != next);if (!cpumask_test_and_set_cpu (cpu,mm_cpumask (next))) {/* We were in lazy tlb mode and leave_mm disabled
arch/x86/include/asm/percpu.h +14 -14
@@ -46,7 +46,7 @@#ifdef CONFIG_SMP #define __percpu_prefix "%%"__stringify(__percpu_seg)":" -#define __my_cpu_offset percpu_read(this_cpu_off) +#define __my_cpu_offset this_cpu_read(this_cpu_off)/* * Compared to the generic __my_cpu_offset version, the following
@@ -351,23 +351,23 @@ do { \ })/* -* percpu_read() makes gcc load the percpu variable every time it is -* accessed while percpu_read_stable() allows the value to be cached. -* percpu_read_stable() is more efficient and can be used if its value +* this_cpu_read() makes gcc load the percpu variable every time it is +* accessed while this_cpu_read_stable() allows the value to be cached. +* this_cpu_read_stable() is more efficient and can be used if its value * is guaranteed to be valid across cpus. The current users include * get_current() and get_thread_info() both of which are actually * per-thread variables implemented as per-cpu variables and thus * stable for the duration of the respective task. */ -#define percpu_read (var) percpu_from_op("mov", var, "m" (var)) -#define percpu_read_stable (var) percpu_from_op("mov", var, "p" (&(var))) -#define percpu_write (var, val) percpu_to_op("mov", var, val) -#define percpu_add (var, val) percpu_add_op(var, val) -#define percpu_sub (var, val) percpu_add_op(var, -(val)) -#define percpu_and (var, val) percpu_to_op("and", var, val) -#define percpu_or (var, val) percpu_to_op("or", var, val) -#define percpu_xor (var, val) percpu_to_op("xor", var, val) -#define percpu_inc (var) percpu_unary_op("inc", var) +#define this_cpu_read (var) percpu_from_op("mov", var, "m" (var)) +#define this_cpu_read_stable (var) percpu_from_op("mov", var, "p" (&(var))) +#define this_cpu_write (var, val) percpu_to_op("mov", var, val) +#define this_cpu_add (var, val) percpu_add_op(var, val) +#define this_cpu_sub (var, val) percpu_add_op(var, -(val)) +#define this_cpu_and (var, val) percpu_to_op("and", var, val) +#define this_cpu_or (var, val) percpu_to_op("or", var, val) +#define this_cpu_xor (var, val) percpu_to_op("xor", var, val) +#define this_cpu_inc (var) percpu_unary_op("inc", var)#define __this_cpu_read_1 (pcp) percpu_from_op("mov", (pcp), "m"(pcp))#define __this_cpu_read_2 (pcp) percpu_from_op("mov", (pcp), "m"(pcp))
@@ -551,7 +551,7 @@ static __always_inline int x86_this_cpu_constant_test_bit(unsigned int nr, {unsigned long __percpu *a = (unsigned long *)addr + nr /BITS_PER_LONG ; -return ((1UL << (nr %BITS_PER_LONG )) &percpu_read (*a)) !=0 ; +return ((1UL << (nr %BITS_PER_LONG )) &this_cpu_read (*a)) !=0 ; } static inline int x86_this_cpu_variable_test_bit(int nr,
arch/x86/include/asm/smp.h +2 -2
@@ -188,11 +188,11 @@ extern unsigned disabled_cpus __cpuinitdata;* from the initial startup. We map APIC_BASE very early in page_setup(), * so this is correct in the x86 case. */ -#define raw_smp_processor_id () (percpu_read(cpu_number)) +#define raw_smp_processor_id () (this_cpu_read(cpu_number))extern int safe_smp_processor_id (void );#elif defined(CONFIG_X86_64_SMP ) -#define raw_smp_processor_id () (percpu_read(cpu_number)) +#define raw_smp_processor_id () (this_cpu_read(cpu_number))#define stack_smp_processor_id () \ ({ \
arch/x86/include/asm/stackprotector.h +2 -2
@@ -76,9 +76,9 @@ static __always_inline void boot_init_stack_canary(void) current->stack_canary = canary;#ifdef CONFIG_X86_64 -percpu_write (irq_stack_union.stack_canary, canary); +this_cpu_write (irq_stack_union.stack_canary, canary);#else -percpu_write (stack_canary.canary, canary); +this_cpu_write (stack_canary.canary, canary);#endif }
arch/x86/include/asm/thread_info.h +1 -1
@@ -219,7 +219,7 @@ DECLARE_PER_CPU(unsigned long, kernel_stack);static inline struct thread_info *current_thread_info (void ) {struct thread_info *ti; - ti = (void *)(percpu_read_stable (kernel_stack) + + ti = (void *)(this_cpu_read_stable (kernel_stack) +KERNEL_STACK_OFFSET -THREAD_SIZE );return ti; }
arch/x86/include/asm/tlbflush.h +2 -2
@@ -156,8 +156,8 @@ DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate);static inline void reset_lazy_tlbstate (void ) { -percpu_write (cpu_tlbstate.state,0 ); -percpu_write (cpu_tlbstate.active_mm, &init_mm); +this_cpu_write (cpu_tlbstate.state,0 ); +this_cpu_write (cpu_tlbstate.active_mm, &init_mm); }#endif /* SMP */
arch/x86/kernel/cpu/common.c +1 -1
@@ -1158,7 +1158,7 @@ void __cpuinit cpu_init(void) oist = &per_cpu (orig_ist, cpu);#ifdef CONFIG_NUMA -if (cpu !=0 &&percpu_read (numa_node) ==0 && +if (cpu !=0 &&this_cpu_read (numa_node) ==0 &&early_cpu_to_node (cpu) !=NUMA_NO_NODE )set_numa_node (early_cpu_to_node (cpu));#endif
arch/x86/kernel/cpu/mcheck/mce.c +2 -2
@@ -512,7 +512,7 @@ void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)struct mce m;int i; -percpu_inc (mce_poll_count); +this_cpu_inc (mce_poll_count);mce_gather_info (&m,NULL );
@@ -906,7 +906,7 @@ void do_machine_check(struct pt_regs *regs, long error_code)atomic_inc (&mce_entry); -percpu_inc (mce_exception_count); +this_cpu_inc (mce_exception_count);if (notify_die (DIE_NMI ,"machine check" , regs, error_code,18 ,SIGKILL ) ==NOTIFY_STOP )
arch/x86/kernel/cpu/perf_event.c +2 -2
@@ -1389,7 +1389,7 @@ perf_event_nmi_handler(struct notifier_block *self,case DIE_NMI :break ;case DIE_NMIUNKNOWN : - this_nmi =percpu_read (irq_stat.__nmi_count); + this_nmi =this_cpu_read (irq_stat.__nmi_count);if (this_nmi !=__this_cpu_read (pmu_nmi.marked))/* let the kernel handle the unknown nmi */ return NOTIFY_DONE ;
@@ -1409,7 +1409,7 @@ perf_event_nmi_handler(struct notifier_block *self,if (!handled)return NOTIFY_DONE ; - this_nmi =percpu_read (irq_stat.__nmi_count); + this_nmi =this_cpu_read (irq_stat.__nmi_count); if ((handled > 1) || /* the next nmi could be a back-to-back nmi */ ((__this_cpu_read(pmu_nmi.marked) == this_nmi) &&
arch/x86/kernel/paravirt.c +6 -6
@@ -239,16 +239,16 @@ static DEFINE_PER_CPU(enum paravirt_lazy_mode, paravirt_lazy_mode) = PARAVIRT_LAstatic inline void enter_lazy (enum paravirt_lazy_mode mode) { -BUG_ON (percpu_read (paravirt_lazy_mode) !=PARAVIRT_LAZY_NONE ); +BUG_ON (this_cpu_read (paravirt_lazy_mode) !=PARAVIRT_LAZY_NONE ); -percpu_write (paravirt_lazy_mode, mode); +this_cpu_write (paravirt_lazy_mode, mode); }static void leave_lazy (enum paravirt_lazy_mode mode) { -BUG_ON (percpu_read (paravirt_lazy_mode) != mode); +BUG_ON (this_cpu_read (paravirt_lazy_mode) != mode); -percpu_write (paravirt_lazy_mode,PARAVIRT_LAZY_NONE ); +this_cpu_write (paravirt_lazy_mode,PARAVIRT_LAZY_NONE ); }void paravirt_enter_lazy_mmu (void )
@@ -265,7 +265,7 @@ void paravirt_start_context_switch(struct task_struct *prev) {BUG_ON (preemptible ()); -if (percpu_read (paravirt_lazy_mode) ==PARAVIRT_LAZY_MMU ) { +if (this_cpu_read (paravirt_lazy_mode) ==PARAVIRT_LAZY_MMU ) {arch_leave_lazy_mmu_mode ();set_ti_thread_flag (task_thread_info (prev),TIF_LAZY_MMU_UPDATES ); }
@@ -287,7 +287,7 @@ enum paravirt_lazy_mode paravirt_get_lazy_mode(void)if (in_interrupt ())return PARAVIRT_LAZY_NONE ; -return percpu_read (paravirt_lazy_mode); +return this_cpu_read (paravirt_lazy_mode); }void arch_flush_lazy_mmu_mode (void )
arch/x86/kernel/process_32.c +1 -1
@@ -373,7 +373,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)if (prev->gs | next->gs)lazy_load_gs (next->gs); -percpu_write (current_task, next_p); +this_cpu_write (current_task, next_p);return prev_p; }
arch/x86/kernel/process_64.c +6 -6
@@ -73,7 +73,7 @@ EXPORT_SYMBOL_GPL(idle_notifier_unregister);void enter_idle (void ) { -percpu_write (is_idle,1 ); +this_cpu_write (is_idle,1 );atomic_notifier_call_chain (&idle_notifier,IDLE_START ,NULL ); }
@@ -336,7 +336,7 @@ start_thread_common(struct pt_regs *regs, unsigned long new_ip,load_gs_index (0 ); regs->ip = new_ip; regs->sp = new_sp; -percpu_write (old_rsp, new_sp); +this_cpu_write (old_rsp, new_sp); regs->cs = _cs; regs->ss = _ss; regs->flags =X86_EFLAGS_IF ;
@@ -470,11 +470,11 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)/* * Switch the PDA and FPU contexts. */ - prev->usersp =percpu_read (old_rsp); -percpu_write (old_rsp, next->usersp); -percpu_write (current_task, next_p); + prev->usersp =this_cpu_read (old_rsp); +this_cpu_write (old_rsp, next->usersp); +this_cpu_write (current_task, next_p); -percpu_write (kernel_stack, +this_cpu_write (kernel_stack, (unsigned long )task_stack_page (next_p) +THREAD_SIZE -KERNEL_STACK_OFFSET );
arch/x86/kvm/x86.c +7 -7
@@ -5024,15 +5024,15 @@ static DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);static int kvm_is_in_guest (void ) { -return percpu_read (current_vcpu) !=NULL ; +return this_cpu_read (current_vcpu) !=NULL ; }static int kvm_is_user_mode (void ) {int user_mode =3 ; -if (percpu_read (current_vcpu)) - user_mode = kvm_x86_ops->get_cpl (percpu_read (current_vcpu)); +if (this_cpu_read (current_vcpu)) + user_mode = kvm_x86_ops->get_cpl (this_cpu_read (current_vcpu));return user_mode !=0 ; }
@@ -5041,8 +5041,8 @@ static unsigned long kvm_get_guest_ip(void) {unsigned long ip =0 ; -if (percpu_read (current_vcpu)) - ip =kvm_rip_read (percpu_read (current_vcpu)); +if (this_cpu_read (current_vcpu)) + ip =kvm_rip_read (this_cpu_read (current_vcpu));return ip; }
@@ -5055,13 +5055,13 @@ static struct perf_guest_info_callbacks kvm_guest_cbs = {void kvm_before_handle_nmi (struct kvm_vcpu *vcpu) { -percpu_write (current_vcpu, vcpu); +this_cpu_write (current_vcpu, vcpu); }EXPORT_SYMBOL_GPL (kvm_before_handle_nmi);void kvm_after_handle_nmi (struct kvm_vcpu *vcpu) { -percpu_write (current_vcpu,NULL ); +this_cpu_write (current_vcpu,NULL ); }EXPORT_SYMBOL_GPL (kvm_after_handle_nmi);
arch/x86/mm/tlb.c +5 -5
@@ -61,10 +61,10 @@ static DEFINE_PER_CPU_READ_MOSTLY(int, tlb_vector_offset);*/ void leave_mm (int cpu) { -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) +if (this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_OK )BUG ();cpumask_clear_cpu (cpu, -mm_cpumask (percpu_read (cpu_tlbstate.active_mm))); +mm_cpumask (this_cpu_read (cpu_tlbstate.active_mm)));load_cr3 (swapper_pg_dir); }EXPORT_SYMBOL_GPL (leave_mm);
@@ -152,8 +152,8 @@ void smp_invalidate_interrupt(struct pt_regs *regs)* BUG(); */ -if (f->flush_mm ==percpu_read (cpu_tlbstate.active_mm)) { -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) { +if (f->flush_mm ==this_cpu_read (cpu_tlbstate.active_mm)) { +if (this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) {if (f->flush_va ==TLB_FLUSH_ALL )local_flush_tlb ();else
@@ -322,7 +322,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)static void do_flush_tlb_all (void *info) {__flush_tlb_all (); -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_LAZY ) +if (this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_LAZY )leave_mm (smp_processor_id ()); }
arch/x86/xen/enlighten.c +3 -3
@@ -776,11 +776,11 @@ static DEFINE_PER_CPU(unsigned long, xen_cr0_value);static unsigned long xen_read_cr0 (void ) { -unsigned long cr0 =percpu_read (xen_cr0_value); +unsigned long cr0 =this_cpu_read (xen_cr0_value);if (unlikely (cr0 ==0 )) { cr0 =native_read_cr0 (); -percpu_write (xen_cr0_value, cr0); +this_cpu_write (xen_cr0_value, cr0); }return cr0;
@@ -790,7 +790,7 @@ static void xen_write_cr0(unsigned long cr0) {struct multicall_space mcs; -percpu_write (xen_cr0_value, cr0); +this_cpu_write (xen_cr0_value, cr0);/* Only pay attention to cr0.TS; everything else is ignored. */
arch/x86/xen/irq.c +4 -4
@@ -26,7 +26,7 @@ static unsigned long xen_save_fl(void)struct vcpu_info *vcpu;unsigned long flags; - vcpu =percpu_read (xen_vcpu); + vcpu =this_cpu_read (xen_vcpu);/* flag has opposite sense of mask */ flags = !vcpu->evtchn_upcall_mask;
@@ -50,7 +50,7 @@ static void xen_restore_fl(unsigned long flags)make sure we're don't switch CPUs between getting the vcpu pointer and updating the mask. */ preempt_disable (); - vcpu =percpu_read (xen_vcpu); + vcpu =this_cpu_read (xen_vcpu); vcpu->evtchn_upcall_mask = flags;preempt_enable_no_resched ();
@@ -72,7 +72,7 @@ static void xen_irq_disable(void)make sure we're don't switch CPUs between getting the vcpu pointer and updating the mask. */ preempt_disable (); -percpu_read (xen_vcpu)->evtchn_upcall_mask =1 ; +this_cpu_read (xen_vcpu)->evtchn_upcall_mask =1 ;preempt_enable_no_resched (); }PV_CALLEE_SAVE_REGS_THUNK (xen_irq_disable);
@@ -86,7 +86,7 @@ static void xen_irq_enable(void)the caller is confused and is trying to re-enable interrupts on an indeterminate processor. */ - vcpu =percpu_read (xen_vcpu); + vcpu =this_cpu_read (xen_vcpu); vcpu->evtchn_upcall_mask =0 ;/* Doesn't matter if we get preempted here, because any
arch/x86/xen/mmu.c +10 -10
@@ -1106,14 +1106,14 @@ static void drop_other_mm_ref(void *info)struct mm_struct *mm = info;struct mm_struct *active_mm; - active_mm =percpu_read (cpu_tlbstate.active_mm); + active_mm =this_cpu_read (cpu_tlbstate.active_mm); -if (active_mm == mm &&percpu_read (cpu_tlbstate.state) !=TLBSTATE_OK ) +if (active_mm == mm &&this_cpu_read (cpu_tlbstate.state) !=TLBSTATE_OK )leave_mm (smp_processor_id ());/* If this cpu still has a stale cr3 reference, then make sure it has been flushed. */ -if (percpu_read (xen_current_cr3) ==__pa (mm->pgd)) +if (this_cpu_read (xen_current_cr3) ==__pa (mm->pgd))load_cr3 (swapper_pg_dir); }
@@ -1220,17 +1220,17 @@ static void __init xen_pagetable_setup_done(pgd_t *base)static void xen_write_cr2 (unsigned long cr2) { -percpu_read (xen_vcpu)->arch.cr2 = cr2; +this_cpu_read (xen_vcpu)->arch.cr2 = cr2; }static unsigned long xen_read_cr2 (void ) { -return percpu_read (xen_vcpu)->arch.cr2; +return this_cpu_read (xen_vcpu)->arch.cr2; }unsigned long xen_read_cr2_direct (void ) { -return percpu_read (xen_vcpu_info.arch.cr2); +return this_cpu_read (xen_vcpu_info.arch.cr2); }static void xen_flush_tlb (void )
@@ -1313,12 +1313,12 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,static unsigned long xen_read_cr3 (void ) { -return percpu_read (xen_cr3); +return this_cpu_read (xen_cr3); }static void set_current_cr3 (void *v) { -percpu_write (xen_current_cr3, (unsigned long )v); +this_cpu_write (xen_current_cr3, (unsigned long )v); }static void __xen_write_cr3 (bool kernel,unsigned long cr3)
@@ -1341,7 +1341,7 @@ static void __xen_write_cr3(bool kernel, unsigned long cr3)xen_extend_mmuext_op (&op);if (kernel) { -percpu_write (xen_cr3, cr3); +this_cpu_write (xen_cr3, cr3);/* Update xen_current_cr3 once the batch has actually been submitted. */
@@ -1357,7 +1357,7 @@ static void xen_write_cr3(unsigned long cr3)/* Update while interrupts are disabled, so its atomic with respect to ipis */ -percpu_write (xen_cr3, cr3); +this_cpu_write (xen_cr3, cr3);__xen_write_cr3 (true, cr3);
arch/x86/xen/multicalls.h +1 -1
@@ -47,7 +47,7 @@ static inline void xen_mc_issue(unsigned mode)xen_mc_flush ();/* restore flags saved in xen_mc_batch */ -local_irq_restore (percpu_read (xen_mc_irq_flags)); +local_irq_restore (this_cpu_read (xen_mc_irq_flags)); }/* Set up a callback to be called when the current batch is flushed */
arch/x86/xen/smp.c +1 -1
@@ -76,7 +76,7 @@ static void __cpuinit cpu_bringup(void)xen_setup_cpu_clockevents ();set_cpu_online (cpu, true); -percpu_write (cpu_state,CPU_ONLINE ); +this_cpu_write (cpu_state,CPU_ONLINE );wmb ();/* We can take interrupts now: we're officially "up". */
include/linux/percpu.h +0 -53
59 lines not on this page · raw message
include/linux/topology.h +2 -2
@@ -235,7 +235,7 @@ static inline int cpu_to_node(int cpu)#ifndef set_numa_nodestatic inline void set_numa_node (int node) { -percpu_write (numa_node, node); +this_cpu_write (numa_node, node); }#endif
@@ -270,7 +270,7 @@ DECLARE_PER_CPU(int, _numa_mem_);#ifndef set_numa_memstatic inline void set_numa_mem (int node) { -percpu_write (_numa_mem_, node); +this_cpu_write (_numa_mem_, node); }#endif
net/netfilter/xt_TEE.c +6 -6
@@ -90,7 +90,7 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)const struct xt_tee_tginfo *info = par->targinfo;struct iphdr *iph; -if (percpu_read (tee_active)) +if (this_cpu_read (tee_active))return XT_CONTINUE ;/* * Copy the skb, and route the copy. Will later return %XT_CONTINUE for
@@ -127,9 +127,9 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)ip_send_check (iph);if (tee_tg_route4 (skb, info)) { -percpu_write (tee_active, true); +this_cpu_write (tee_active, true);ip_local_out (skb); -percpu_write (tee_active, false); +this_cpu_write (tee_active, false); }else {kfree_skb (skb); }
@@ -170,7 +170,7 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par) {const struct xt_tee_tginfo *info = par->targinfo; -if (percpu_read (tee_active)) +if (this_cpu_read (tee_active))return XT_CONTINUE ; skb =pskb_copy (skb,GFP_ATOMIC );if (skb ==NULL )
@@ -188,9 +188,9 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par) --iph->hop_limit; }if (tee_tg_route6 (skb, info)) { -percpu_write (tee_active, true); +this_cpu_write (tee_active, true);ip6_local_out (skb); -percpu_write (tee_active, false); +this_cpu_write (tee_active, false); }else {kfree_skb (skb); }
net/socket.c +2 -2
@@ -479,7 +479,7 @@ static struct socket *sock_alloc(void) inode->i_uid =current_fsuid (); inode->i_gid =current_fsgid (); -percpu_add (sockets_in_use,1 ); +this_cpu_add (sockets_in_use,1 );return sock; }
@@ -522,7 +522,7 @@ void sock_release(struct socket *sock)if (rcu_dereference_protected (sock->wq,1 )->fasync_list)printk (KERN_ERR "sock_release: fasync list not empty!\n" ); -percpu_sub (sockets_in_use,1 ); +this_cpu_sub (sockets_in_use,1 );if (!sock->file) {iput (SOCK_INODE (sock));return ;
Signature · 1 line
1.7.5.1
Christoph Lameter
maintainerarch/x86/include/asm/percpu.h line 49
Quoted 9 lines · Alex,Shi
index 3470c9d..d1de56c 100644
arch/x86/include/asm/percpu.h +1
-1@@ -46,7 +46,7 @@#ifdef CONFIG_SMP #define __percpu_prefix "%%"__stringify(__percpu_seg)":" -#define __my_cpu_offset percpu_read(this_cpu_off) +#define __my_cpu_offset this_cpu_read(this_cpu_off)
Should this not be __this_cpu_read() instead?
Quoted 11 lines
arch/x86/include/asm/percpu.h +0
-8* per-thread variables implemented as per-cpu variables and thus * stable for the duration of the respective task. */ -#define percpu_read (var) percpu_from_op("mov", var, "m" (var)) -#define percpu_read_stable (var) percpu_from_op("mov", var, "p" (&(var))) -#define percpu_write (var, val) percpu_to_op("mov", var, val) -#define percpu_add (var, val) percpu_add_op(var, val) -#define percpu_sub (var, val) percpu_add_op(var, -(val)) -#define percpu_and (var, val) percpu_to_op("and", var, val) -#define percpu_or (var, val) percpu_to_op("or", var, val) -#define percpu_xor (var, val) percpu_to_op("xor", var, val)
arch/x86/include/asm/percpu.h do { \ · line 370
Quoted 10 lines
arch/x86/include/asm/percpu.h +9
-1-#define percpu_inc (var) percpu_unary_op("inc", var) +#define this_cpu_read (var) percpu_from_op("mov", var, "m" (var)) +#define this_cpu_read_stable (var) percpu_from_op("mov", var, "p" (&(var))) +#define this_cpu_write (var, val) percpu_to_op("mov", var, val) +#define this_cpu_add (var, val) percpu_add_op(var, val) +#define this_cpu_sub (var, val) percpu_add_op(var, -(val)) +#define this_cpu_and (var, val) percpu_to_op("and", var, val) +#define this_cpu_or (var, val) percpu_to_op("or", var, val) +#define this_cpu_xor (var, val) percpu_to_op("xor", var, val) +#define this_cpu_inc (var) percpu_unary_op("inc", var)
Why define these operations? They are already defined in the form of this_cpu_add_1/2/4/8 and include/linux/percpu.h branches out to them.
arch/x86/include/asm/percpu.h void sock_release(struct socket *sock) · line 528
Quoted 9 lines
arch/x86/include/asm/percpu.h +1
-1@@ -522,7 +522,7 @@ void sock_release(struct socket *sock)if (rcu_dereference_protected (sock->wq,1 )->fasync_list)printk (KERN_ERR "sock_release: fasync list not empty!\n" ); -percpu_sub (sockets_in_use,1 ); +this_cpu_sub (sockets_in_use,1 );if (!sock->file) {iput (SOCK_INODE (sock));return ;
In general __this_cpu_xx operations can be used instead of this_cpu_xx operations if the context is already preempt safe. There is no change of the code generated for x86 in those cases. But the fallback code on other platforms can generate more efficient code.
Quoted 7 lines · Christoph Lameter
Quoted 5 lines · Alex,Shi
index 3470c9d..d1de56c 100644
arch/x86/include/asm/percpu.h +0
-0@@ -46,7 +46,7 @@
Quoted 6 lines
arch/x86/include/asm/percpu.h line 49
Quoted 4 lines
arch/x86/include/asm/percpu.h +1
-1#ifdef CONFIG_SMP #define __percpu_prefix "%%"__stringify(__percpu_seg)":" -#define __my_cpu_offset percpu_read(this_cpu_off) +#define __my_cpu_offset this_cpu_read(this_cpu_off)Should this not be
__this_cpu_read()instead?
OK.
Quoted 17 lines
Quoted 17 lines
arch/x86/include/asm/percpu.h +4
-9* per-thread variables implemented as per-cpu variables and thus * stable for the duration of the respective task. */ -#define percpu_read (var) percpu_from_op("mov", var, "m" (var)) -#define percpu_read_stable (var) percpu_from_op("mov", var, "p" (&(var))) -#define percpu_write (var, val) percpu_to_op("mov", var, val) -#define percpu_add (var, val) percpu_add_op(var, val) -#define percpu_sub (var, val) percpu_add_op(var, -(val)) -#define percpu_and (var, val) percpu_to_op("and", var, val) -#define percpu_or (var, val) percpu_to_op("or", var, val) -#define percpu_xor (var, val) percpu_to_op("xor", var, val) -#define percpu_inc (var) percpu_unary_op("inc", var) +#define this_cpu_read (var) percpu_from_op("mov", var, "m" (var)) +#define this_cpu_read_stable (var) percpu_from_op("mov", var, "p" (&(var))) +#define this_cpu_write (var, val) percpu_to_op("mov", var, val) +#define this_cpu_add (var, val) percpu_add_op(var, val)
Quoted 8 lines
arch/x86/include/asm/percpu.h
Quoted 5 lines
arch/x86/include/asm/percpu.h +5
-0+#define this_cpu_sub (var, val) percpu_add_op(var, -(val)) +#define this_cpu_and (var, val) percpu_to_op("and", var, val) +#define this_cpu_or (var, val) percpu_to_op("or", var, val) +#define this_cpu_xor (var, val) percpu_to_op("xor", var, val) +#define this_cpu_inc (var) percpu_unary_op("inc", var)Why define these operations? They are already defined in the form of this_cpu_add_1/2/4/8 and
include/linux/percpu.hbranches out to them.
percpu_read() used in asm/percpu.h yet. Anyway, I removed definitions here and do a work around for the function as following:
@@ -551,7 +544,11 @@ static __always_inline int x86_this_cpu_constant_test_bit(unsigned int nr,
{
unsigned long __percpu *a = (unsigned long *)addr + nr / BITS_PER_LONG;- return ((1UL << (nr % BITS_PER_LONG)) & percpu_read(*a)) != 0;
+#ifdef CONFIG_X86_64
+ return ((1UL << (nr % BITS_PER_LONG)) & this_cpu_read_8(*a)) != 0;
+#else
+ return ((1UL << (nr % BITS_PER_LONG)) & this_cpu_read_4(*a)) != 0;
+#endif
Quoted 5 lines
Quoted 5 lines
arch/x86/include/asm/percpu.h +0
-0@@ -522,7 +522,7 @@ void sock_release(struct socket *sock)if (rcu_dereference_protected (sock->wq,1 )->fasync_list)printk (KERN_ERR "sock_release: fasync list not empty!\n" );
Quoted 11 lines
arch/x86/include/asm/percpu.h void sock_release(struct socket *sock) · line 528
Quoted 5 lines
arch/x86/include/asm/percpu.h +1
-1-percpu_sub (sockets_in_use,1 ); +this_cpu_sub (sockets_in_use,1 );if (!sock->file) {iput (SOCK_INODE (sock));return ;In general
__this_cpu_xxoperations can be used instead ofthis_cpu_xxoperations if the context is already preempt safe. There is no change of the code generated for x86 in those cases. But the fallback code on other platforms can generate more efficient code.
Good reminder!
Currently, only x86 has specific percpu_xxx() definition. Other platforms use generic version. And for this function, it is not preempt safe. So, it should be fine here.
=========
Since percpu_xxx() serial functions are duplicate with this_cpu_xxx(). Removing percpu_xxx() definition and replacing them by this_cpu_xxx() in using. This patch is just do code clean up, no function change.
Signed-off-by Alex Shi <[email protected]>
--- arch/x86/include/asm/current.h | 2 +- arch/x86/include/asm/hardirq.h | 8 ++-- arch/x86/include/asm/irq_regs.h | 4 +- arch/x86/include/asm/mmu_context.h | 12 ++++---- arch/x86/include/asm/percpu.h | 25 +++++++--------- arch/x86/include/asm/smp.h | 4 +- arch/x86/include/asm/stackprotector.h | 4 +- arch/x86/include/asm/thread_info.h | 2 +- arch/x86/include/asm/tlbflush.h | 4 +- arch/x86/kernel/cpu/common.c | 2 +- arch/x86/kernel/cpu/mcheck/mce.c | 4 +- arch/x86/kernel/cpu/perf_event.c | 4 +- arch/x86/kernel/paravirt.c | 12 ++++---- arch/x86/kernel/process_32.c | 2 +- arch/x86/kernel/process_64.c | 12 ++++---- arch/x86/kvm/x86.c | 14 ++++---- arch/x86/mm/tlb.c | 10 +++--- arch/x86/xen/enlighten.c | 6 ++-- arch/x86/xen/irq.c | 8 ++-- arch/x86/xen/mmu.c | 20 ++++++------ arch/x86/xen/multicalls.h | 2 +- arch/x86/xen/smp.c | 2 +- include/linux/percpu.h | 53 --------------------------------- include/linux/topology.h | 4 +- net/netfilter/xt_TEE.c | 12 ++++---- net/socket.c | 4 +- 26 files changed, 90 insertions(+), 146 deletions(-)
arch/x86/include/asm/current.h +1 -1
@@ -11,7 +11,7 @@ DECLARE_PER_CPU(struct task_struct *, current_task); static __always_inline struct task_struct *get_current(void) { -return percpu_read_stable (current_task); +return this_cpu_read_stable (current_task); }#define current get_current()
arch/x86/include/asm/hardirq.h +4 -4
@@ -35,14 +35,14 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);#define __ARCH_IRQ_STAT -#define inc_irq_stat (member) percpu_inc(irq_stat.member) +#define inc_irq_stat (member) this_cpu_inc(irq_stat.member) -#define local_softirq_pending () percpu_read(irq_stat.__softirq_pending) +#define local_softirq_pending () this_cpu_read(irq_stat.__softirq_pending)#define __ARCH_SET_SOFTIRQ_PENDING -#define set_softirq_pending (x) percpu_write(irq_stat.__softirq_pending, (x)) -#define or_softirq_pending (x) percpu_or(irq_stat.__softirq_pending, (x)) +#define set_softirq_pending (x) this_cpu_write(irq_stat.__softirq_pending, (x)) +#define or_softirq_pending (x) this_cpu_or(irq_stat.__softirq_pending, (x))extern void ack_bad_irq (unsigned int irq);
arch/x86/include/asm/irq_regs.h +2 -2
@@ -15,7 +15,7 @@ DECLARE_PER_CPU(struct pt_regs *, irq_regs);static inline struct pt_regs *get_irq_regs (void ) { -return percpu_read (irq_regs); +return this_cpu_read (irq_regs); }static inline struct pt_regs *set_irq_regs (struct pt_regs *new_regs)
@@ -23,7 +23,7 @@ static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)struct pt_regs *old_regs; old_regs =get_irq_regs (); -percpu_write (irq_regs, new_regs); +this_cpu_write (irq_regs, new_regs);return old_regs; }
arch/x86/include/asm/mmu_context.h +6 -6
@@ -25,8 +25,8 @@ void destroy_context(struct mm_struct *mm);static inline void enter_lazy_tlb (struct mm_struct *mm,struct task_struct *tsk) {#ifdef CONFIG_SMP -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) -percpu_write (cpu_tlbstate.state,TLBSTATE_LAZY ); +if (this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) +this_cpu_write (cpu_tlbstate.state,TLBSTATE_LAZY );#endif }
@@ -37,8 +37,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,if (likely (prev != next)) {#ifdef CONFIG_SMP -percpu_write (cpu_tlbstate.state,TLBSTATE_OK ); -percpu_write (cpu_tlbstate.active_mm, next); +this_cpu_write (cpu_tlbstate.state,TLBSTATE_OK ); +this_cpu_write (cpu_tlbstate.active_mm, next);#endif cpumask_set_cpu (cpu,mm_cpumask (next));
@@ -56,8 +56,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, }#ifdef CONFIG_SMP else { -percpu_write (cpu_tlbstate.state,TLBSTATE_OK ); -BUG_ON (percpu_read (cpu_tlbstate.active_mm) != next); +this_cpu_write (cpu_tlbstate.state,TLBSTATE_OK ); +BUG_ON (this_cpu_read (cpu_tlbstate.active_mm) != next);if (!cpumask_test_and_set_cpu (cpu,mm_cpumask (next))) {/* We were in lazy tlb mode and leave_mm disabled
arch/x86/include/asm/percpu.h +11 -14
@@ -46,7 +46,7 @@#ifdef CONFIG_SMP #define __percpu_prefix "%%"__stringify(__percpu_seg)":" -#define __my_cpu_offset percpu_read(this_cpu_off) +#define __my_cpu_offset __this_cpu_read(this_cpu_off)/* * Compared to the generic __my_cpu_offset version, the following
@@ -351,23 +351,16 @@ do { \ })/* -* percpu_read() makes gcc load the percpu variable every time it is -* accessed while percpu_read_stable() allows the value to be cached. -* percpu_read_stable() is more efficient and can be used if its value +* this_cpu_read() makes gcc load the percpu variable every time it is +* accessed while this_cpu_read_stable() allows the value to be cached. +* this_cpu_read_stable() is more efficient and can be used if its value * is guaranteed to be valid across cpus. The current users include * get_current() and get_thread_info() both of which are actually * per-thread variables implemented as per-cpu variables and thus * stable for the duration of the respective task. */ -#define percpu_read (var) percpu_from_op("mov", var, "m" (var)) -#define percpu_read_stable (var) percpu_from_op("mov", var, "p" (&(var))) -#define percpu_write (var, val) percpu_to_op("mov", var, val) -#define percpu_add (var, val) percpu_add_op(var, val) -#define percpu_sub (var, val) percpu_add_op(var, -(val)) -#define percpu_and (var, val) percpu_to_op("and", var, val) -#define percpu_or (var, val) percpu_to_op("or", var, val) -#define percpu_xor (var, val) percpu_to_op("xor", var, val) -#define percpu_inc (var) percpu_unary_op("inc", var) +#define this_cpu_read_stable (var) percpu_from_op("mov", var, "p" (&(var))) +#define this_cpu_inc (var) percpu_unary_op("inc", var)#define __this_cpu_read_1 (pcp) percpu_from_op("mov", (pcp), "m"(pcp))#define __this_cpu_read_2 (pcp) percpu_from_op("mov", (pcp), "m"(pcp))
@@ -551,7 +544,11 @@ static __always_inline int x86_this_cpu_constant_test_bit(unsigned int nr, {unsigned long __percpu *a = (unsigned long *)addr + nr /BITS_PER_LONG ; -return ((1UL << (nr %BITS_PER_LONG )) &percpu_read (*a)) !=0 ; +#ifdef CONFIG_X86_64 +return ((1UL << (nr %BITS_PER_LONG )) &this_cpu_read_8 (*a)) !=0 ; +#else +return ((1UL << (nr %BITS_PER_LONG )) &this_cpu_read_4 (*a)) !=0 ; +#endif } static inline int x86_this_cpu_variable_test_bit(int nr,
arch/x86/include/asm/smp.h +2 -2
@@ -188,11 +188,11 @@ extern unsigned disabled_cpus __cpuinitdata;* from the initial startup. We map APIC_BASE very early in page_setup(), * so this is correct in the x86 case. */ -#define raw_smp_processor_id () (percpu_read(cpu_number)) +#define raw_smp_processor_id () (this_cpu_read(cpu_number))extern int safe_smp_processor_id (void );#elif defined(CONFIG_X86_64_SMP ) -#define raw_smp_processor_id () (percpu_read(cpu_number)) +#define raw_smp_processor_id () (this_cpu_read(cpu_number))#define stack_smp_processor_id () \ ({ \
arch/x86/include/asm/stackprotector.h +2 -2
@@ -76,9 +76,9 @@ static __always_inline void boot_init_stack_canary(void) current->stack_canary = canary;#ifdef CONFIG_X86_64 -percpu_write (irq_stack_union.stack_canary, canary); +this_cpu_write (irq_stack_union.stack_canary, canary);#else -percpu_write (stack_canary.canary, canary); +this_cpu_write (stack_canary.canary, canary);#endif }
arch/x86/include/asm/thread_info.h +1 -1
@@ -219,7 +219,7 @@ DECLARE_PER_CPU(unsigned long, kernel_stack);static inline struct thread_info *current_thread_info (void ) {struct thread_info *ti; - ti = (void *)(percpu_read_stable (kernel_stack) + + ti = (void *)(this_cpu_read_stable (kernel_stack) +KERNEL_STACK_OFFSET -THREAD_SIZE );return ti; }
arch/x86/include/asm/tlbflush.h +2 -2
@@ -156,8 +156,8 @@ DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate);static inline void reset_lazy_tlbstate (void ) { -percpu_write (cpu_tlbstate.state,0 ); -percpu_write (cpu_tlbstate.active_mm, &init_mm); +this_cpu_write (cpu_tlbstate.state,0 ); +this_cpu_write (cpu_tlbstate.active_mm, &init_mm); }#endif /* SMP */
arch/x86/kernel/cpu/common.c +1 -1
@@ -1158,7 +1158,7 @@ void __cpuinit cpu_init(void) oist = &per_cpu (orig_ist, cpu);#ifdef CONFIG_NUMA -if (cpu !=0 &&percpu_read (numa_node) ==0 && +if (cpu !=0 &&this_cpu_read (numa_node) ==0 &&early_cpu_to_node (cpu) !=NUMA_NO_NODE )set_numa_node (early_cpu_to_node (cpu));#endif
arch/x86/kernel/cpu/mcheck/mce.c +2 -2
@@ -512,7 +512,7 @@ void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)struct mce m;int i; -percpu_inc (mce_poll_count); +this_cpu_inc (mce_poll_count);mce_gather_info (&m,NULL );
@@ -906,7 +906,7 @@ void do_machine_check(struct pt_regs *regs, long error_code)atomic_inc (&mce_entry); -percpu_inc (mce_exception_count); +this_cpu_inc (mce_exception_count);if (notify_die (DIE_NMI ,"machine check" , regs, error_code,18 ,SIGKILL ) ==NOTIFY_STOP )
arch/x86/kernel/cpu/perf_event.c +2 -2
@@ -1389,7 +1389,7 @@ perf_event_nmi_handler(struct notifier_block *self,case DIE_NMI :break ;case DIE_NMIUNKNOWN : - this_nmi =percpu_read (irq_stat.__nmi_count); + this_nmi =this_cpu_read (irq_stat.__nmi_count);if (this_nmi !=__this_cpu_read (pmu_nmi.marked))/* let the kernel handle the unknown nmi */ return NOTIFY_DONE ;
@@ -1409,7 +1409,7 @@ perf_event_nmi_handler(struct notifier_block *self,if (!handled)return NOTIFY_DONE ; - this_nmi =percpu_read (irq_stat.__nmi_count); + this_nmi =this_cpu_read (irq_stat.__nmi_count); if ((handled > 1) || /* the next nmi could be a back-to-back nmi */ ((__this_cpu_read(pmu_nmi.marked) == this_nmi) &&
arch/x86/kernel/paravirt.c +6 -6
@@ -239,16 +239,16 @@ static DEFINE_PER_CPU(enum paravirt_lazy_mode, paravirt_lazy_mode) = PARAVIRT_LAstatic inline void enter_lazy (enum paravirt_lazy_mode mode) { -BUG_ON (percpu_read (paravirt_lazy_mode) !=PARAVIRT_LAZY_NONE ); +BUG_ON (this_cpu_read (paravirt_lazy_mode) !=PARAVIRT_LAZY_NONE ); -percpu_write (paravirt_lazy_mode, mode); +this_cpu_write (paravirt_lazy_mode, mode); }static void leave_lazy (enum paravirt_lazy_mode mode) { -BUG_ON (percpu_read (paravirt_lazy_mode) != mode); +BUG_ON (this_cpu_read (paravirt_lazy_mode) != mode); -percpu_write (paravirt_lazy_mode,PARAVIRT_LAZY_NONE ); +this_cpu_write (paravirt_lazy_mode,PARAVIRT_LAZY_NONE ); }void paravirt_enter_lazy_mmu (void )
@@ -265,7 +265,7 @@ void paravirt_start_context_switch(struct task_struct *prev) {BUG_ON (preemptible ()); -if (percpu_read (paravirt_lazy_mode) ==PARAVIRT_LAZY_MMU ) { +if (this_cpu_read (paravirt_lazy_mode) ==PARAVIRT_LAZY_MMU ) {arch_leave_lazy_mmu_mode ();set_ti_thread_flag (task_thread_info (prev),TIF_LAZY_MMU_UPDATES ); }
@@ -287,7 +287,7 @@ enum paravirt_lazy_mode paravirt_get_lazy_mode(void)if (in_interrupt ())return PARAVIRT_LAZY_NONE ; -return percpu_read (paravirt_lazy_mode); +return this_cpu_read (paravirt_lazy_mode); }void arch_flush_lazy_mmu_mode (void )
arch/x86/kernel/process_32.c +1 -1
@@ -373,7 +373,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)if (prev->gs | next->gs)lazy_load_gs (next->gs); -percpu_write (current_task, next_p); +this_cpu_write (current_task, next_p);return prev_p; }
arch/x86/kernel/process_64.c +6 -6
@@ -73,7 +73,7 @@ EXPORT_SYMBOL_GPL(idle_notifier_unregister);void enter_idle (void ) { -percpu_write (is_idle,1 ); +this_cpu_write (is_idle,1 );atomic_notifier_call_chain (&idle_notifier,IDLE_START ,NULL ); }
@@ -336,7 +336,7 @@ start_thread_common(struct pt_regs *regs, unsigned long new_ip,load_gs_index (0 ); regs->ip = new_ip; regs->sp = new_sp; -percpu_write (old_rsp, new_sp); +this_cpu_write (old_rsp, new_sp); regs->cs = _cs; regs->ss = _ss; regs->flags =X86_EFLAGS_IF ;
@@ -470,11 +470,11 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)/* * Switch the PDA and FPU contexts. */ - prev->usersp =percpu_read (old_rsp); -percpu_write (old_rsp, next->usersp); -percpu_write (current_task, next_p); + prev->usersp =this_cpu_read (old_rsp); +this_cpu_write (old_rsp, next->usersp); +this_cpu_write (current_task, next_p); -percpu_write (kernel_stack, +this_cpu_write (kernel_stack, (unsigned long )task_stack_page (next_p) +THREAD_SIZE -KERNEL_STACK_OFFSET );
arch/x86/kvm/x86.c +7 -7
@@ -5024,15 +5024,15 @@ static DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);static int kvm_is_in_guest (void ) { -return percpu_read (current_vcpu) !=NULL ; +return this_cpu_read (current_vcpu) !=NULL ; }static int kvm_is_user_mode (void ) {int user_mode =3 ; -if (percpu_read (current_vcpu)) - user_mode = kvm_x86_ops->get_cpl (percpu_read (current_vcpu)); +if (this_cpu_read (current_vcpu)) + user_mode = kvm_x86_ops->get_cpl (this_cpu_read (current_vcpu));return user_mode !=0 ; }
@@ -5041,8 +5041,8 @@ static unsigned long kvm_get_guest_ip(void) {unsigned long ip =0 ; -if (percpu_read (current_vcpu)) - ip =kvm_rip_read (percpu_read (current_vcpu)); +if (this_cpu_read (current_vcpu)) + ip =kvm_rip_read (this_cpu_read (current_vcpu));return ip; }
@@ -5055,13 +5055,13 @@ static struct perf_guest_info_callbacks kvm_guest_cbs = {void kvm_before_handle_nmi (struct kvm_vcpu *vcpu) { -percpu_write (current_vcpu, vcpu); +this_cpu_write (current_vcpu, vcpu); }EXPORT_SYMBOL_GPL (kvm_before_handle_nmi);void kvm_after_handle_nmi (struct kvm_vcpu *vcpu) { -percpu_write (current_vcpu,NULL ); +this_cpu_write (current_vcpu,NULL ); }EXPORT_SYMBOL_GPL (kvm_after_handle_nmi);
arch/x86/mm/tlb.c +5 -5
@@ -61,10 +61,10 @@ static DEFINE_PER_CPU_READ_MOSTLY(int, tlb_vector_offset);*/ void leave_mm (int cpu) { -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) +if (this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_OK )BUG ();cpumask_clear_cpu (cpu, -mm_cpumask (percpu_read (cpu_tlbstate.active_mm))); +mm_cpumask (this_cpu_read (cpu_tlbstate.active_mm)));load_cr3 (swapper_pg_dir); }EXPORT_SYMBOL_GPL (leave_mm);
@@ -152,8 +152,8 @@ void smp_invalidate_interrupt(struct pt_regs *regs)* BUG(); */ -if (f->flush_mm ==percpu_read (cpu_tlbstate.active_mm)) { -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) { +if (f->flush_mm ==this_cpu_read (cpu_tlbstate.active_mm)) { +if (this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) {if (f->flush_va ==TLB_FLUSH_ALL )local_flush_tlb ();else
@@ -322,7 +322,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)static void do_flush_tlb_all (void *info) {__flush_tlb_all (); -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_LAZY ) +if (this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_LAZY )leave_mm (smp_processor_id ()); }
arch/x86/xen/enlighten.c +3 -3
@@ -776,11 +776,11 @@ static DEFINE_PER_CPU(unsigned long, xen_cr0_value);static unsigned long xen_read_cr0 (void ) { -unsigned long cr0 =percpu_read (xen_cr0_value); +unsigned long cr0 =this_cpu_read (xen_cr0_value);if (unlikely (cr0 ==0 )) { cr0 =native_read_cr0 (); -percpu_write (xen_cr0_value, cr0); +this_cpu_write (xen_cr0_value, cr0); }return cr0;
@@ -790,7 +790,7 @@ static void xen_write_cr0(unsigned long cr0) {struct multicall_space mcs; -percpu_write (xen_cr0_value, cr0); +this_cpu_write (xen_cr0_value, cr0);/* Only pay attention to cr0.TS; everything else is ignored. */
arch/x86/xen/irq.c +4 -4
@@ -26,7 +26,7 @@ static unsigned long xen_save_fl(void)struct vcpu_info *vcpu;unsigned long flags; - vcpu =percpu_read (xen_vcpu); + vcpu =this_cpu_read (xen_vcpu);/* flag has opposite sense of mask */ flags = !vcpu->evtchn_upcall_mask;
@@ -50,7 +50,7 @@ static void xen_restore_fl(unsigned long flags)make sure we're don't switch CPUs between getting the vcpu pointer and updating the mask. */ preempt_disable (); - vcpu =percpu_read (xen_vcpu); + vcpu =this_cpu_read (xen_vcpu); vcpu->evtchn_upcall_mask = flags;preempt_enable_no_resched ();
@@ -72,7 +72,7 @@ static void xen_irq_disable(void)make sure we're don't switch CPUs between getting the vcpu pointer and updating the mask. */ preempt_disable (); -percpu_read (xen_vcpu)->evtchn_upcall_mask =1 ; +this_cpu_read (xen_vcpu)->evtchn_upcall_mask =1 ;preempt_enable_no_resched (); }PV_CALLEE_SAVE_REGS_THUNK (xen_irq_disable);
@@ -86,7 +86,7 @@ static void xen_irq_enable(void)the caller is confused and is trying to re-enable interrupts on an indeterminate processor. */ - vcpu =percpu_read (xen_vcpu); + vcpu =this_cpu_read (xen_vcpu); vcpu->evtchn_upcall_mask =0 ;/* Doesn't matter if we get preempted here, because any
arch/x86/xen/mmu.c +10 -10
@@ -1106,14 +1106,14 @@ static void drop_other_mm_ref(void *info)struct mm_struct *mm = info;struct mm_struct *active_mm; - active_mm =percpu_read (cpu_tlbstate.active_mm); + active_mm =this_cpu_read (cpu_tlbstate.active_mm); -if (active_mm == mm &&percpu_read (cpu_tlbstate.state) !=TLBSTATE_OK ) +if (active_mm == mm &&this_cpu_read (cpu_tlbstate.state) !=TLBSTATE_OK )leave_mm (smp_processor_id ());/* If this cpu still has a stale cr3 reference, then make sure it has been flushed. */ -if (percpu_read (xen_current_cr3) ==__pa (mm->pgd)) +if (this_cpu_read (xen_current_cr3) ==__pa (mm->pgd))load_cr3 (swapper_pg_dir); }
@@ -1220,17 +1220,17 @@ static void __init xen_pagetable_setup_done(pgd_t *base)static void xen_write_cr2 (unsigned long cr2) { -percpu_read (xen_vcpu)->arch.cr2 = cr2; +this_cpu_read (xen_vcpu)->arch.cr2 = cr2; }static unsigned long xen_read_cr2 (void ) { -return percpu_read (xen_vcpu)->arch.cr2; +return this_cpu_read (xen_vcpu)->arch.cr2; }unsigned long xen_read_cr2_direct (void ) { -return percpu_read (xen_vcpu_info.arch.cr2); +return this_cpu_read (xen_vcpu_info.arch.cr2); }static void xen_flush_tlb (void )
@@ -1313,12 +1313,12 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,static unsigned long xen_read_cr3 (void ) { -return percpu_read (xen_cr3); +return this_cpu_read (xen_cr3); }static void set_current_cr3 (void *v) { -percpu_write (xen_current_cr3, (unsigned long )v); +this_cpu_write (xen_current_cr3, (unsigned long )v); }static void __xen_write_cr3 (bool kernel,unsigned long cr3)
@@ -1341,7 +1341,7 @@ static void __xen_write_cr3(bool kernel, unsigned long cr3)xen_extend_mmuext_op (&op);if (kernel) { -percpu_write (xen_cr3, cr3); +this_cpu_write (xen_cr3, cr3);/* Update xen_current_cr3 once the batch has actually been submitted. */
@@ -1357,7 +1357,7 @@ static void xen_write_cr3(unsigned long cr3)/* Update while interrupts are disabled, so its atomic with respect to ipis */ -percpu_write (xen_cr3, cr3); +this_cpu_write (xen_cr3, cr3);__xen_write_cr3 (true, cr3);
arch/x86/xen/multicalls.h +1 -1
@@ -47,7 +47,7 @@ static inline void xen_mc_issue(unsigned mode)xen_mc_flush ();/* restore flags saved in xen_mc_batch */ -local_irq_restore (percpu_read (xen_mc_irq_flags)); +local_irq_restore (this_cpu_read (xen_mc_irq_flags)); }/* Set up a callback to be called when the current batch is flushed */
arch/x86/xen/smp.c +1 -1
@@ -76,7 +76,7 @@ static void __cpuinit cpu_bringup(void)xen_setup_cpu_clockevents ();set_cpu_online (cpu, true); -percpu_write (cpu_state,CPU_ONLINE ); +this_cpu_write (cpu_state,CPU_ONLINE );wmb ();/* We can take interrupts now: we're officially "up". */
include/linux/percpu.h +0 -53
59 lines not on this page · raw message
include/linux/topology.h +2 -2
@@ -235,7 +235,7 @@ static inline int cpu_to_node(int cpu)#ifndef set_numa_nodestatic inline void set_numa_node (int node) { -percpu_write (numa_node, node); +this_cpu_write (numa_node, node); }#endif
@@ -270,7 +270,7 @@ DECLARE_PER_CPU(int, _numa_mem_);#ifndef set_numa_memstatic inline void set_numa_mem (int node) { -percpu_write (_numa_mem_, node); +this_cpu_write (_numa_mem_, node); }#endif
net/netfilter/xt_TEE.c +6 -6
@@ -90,7 +90,7 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)const struct xt_tee_tginfo *info = par->targinfo;struct iphdr *iph; -if (percpu_read (tee_active)) +if (this_cpu_read (tee_active))return XT_CONTINUE ;/* * Copy the skb, and route the copy. Will later return %XT_CONTINUE for
@@ -127,9 +127,9 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)ip_send_check (iph);if (tee_tg_route4 (skb, info)) { -percpu_write (tee_active, true); +this_cpu_write (tee_active, true);ip_local_out (skb); -percpu_write (tee_active, false); +this_cpu_write (tee_active, false); }else {kfree_skb (skb); }
@@ -170,7 +170,7 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par) {const struct xt_tee_tginfo *info = par->targinfo; -if (percpu_read (tee_active)) +if (this_cpu_read (tee_active))return XT_CONTINUE ; skb =pskb_copy (skb,GFP_ATOMIC );if (skb ==NULL )
@@ -188,9 +188,9 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par) --iph->hop_limit; }if (tee_tg_route6 (skb, info)) { -percpu_write (tee_active, true); +this_cpu_write (tee_active, true);ip6_local_out (skb); -percpu_write (tee_active, false); +this_cpu_write (tee_active, false); }else {kfree_skb (skb); }
net/socket.c +2 -2
@@ -479,7 +479,7 @@ static struct socket *sock_alloc(void) inode->i_uid =current_fsuid (); inode->i_gid =current_fsgid (); -percpu_add (sockets_in_use,1 ); +this_cpu_add (sockets_in_use,1 );return sock; }
@@ -522,7 +522,7 @@ void sock_release(struct socket *sock)if (rcu_dereference_protected (sock->wq,1 )->fasync_list)printk (KERN_ERR "sock_release: fasync list not empty!\n" ); -percpu_sub (sockets_in_use,1 ); +this_cpu_sub (sockets_in_use,1 );if (!sock->file) {iput (SOCK_INODE (sock));return ;
Signature · 1 line
1.7.5.1
Christoph Lameter
maintainerarch/x86/include/asm/percpu.h static __always_inline int x86_this_cpu_constant_test_bit(unsigned int nr, · line 551
Quoted 13 lines · Alex,Shi
percpu_read()used inasm/percpu.hyet. Anyway, I removed definitions here and do a work around for the function as following:arch/x86/include/asm/percpu.h +5
-1@@ -551,7 +544,11 @@ static __always_inline int x86_this_cpu_constant_test_bit(unsigned int nr, {unsigned long __percpu *a = (unsigned long *)addr + nr /BITS_PER_LONG ; -return ((1UL << (nr %BITS_PER_LONG )) &percpu_read (*a)) !=0 ; +#ifdef CONFIG_X86_64 +return ((1UL << (nr %BITS_PER_LONG )) &this_cpu_read_8 (*a)) !=0 ; +#else +return ((1UL << (nr %BITS_PER_LONG )) &this_cpu_read_4 (*a)) !=0 ; +#endif
OK that looks okay.
Quoted 1 line
Signed-off-by Alex Shi <[email protected]>
Is this a revised patch? I do not see a use of the __this_cpu_xx ops.
arch/x86/include/asm/percpu.h do { \ · line 363
Quoted 9 lines
arch/x86/include/asm/percpu.h +2
-7-#define percpu_write (var, val) percpu_to_op("mov", var, val) -#define percpu_add (var, val) percpu_add_op(var, val) -#define percpu_sub (var, val) percpu_add_op(var, -(val)) -#define percpu_and (var, val) percpu_to_op("and", var, val) -#define percpu_or (var, val) percpu_to_op("or", var, val) -#define percpu_xor (var, val) percpu_to_op("xor", var, val) -#define percpu_inc (var) percpu_unary_op("inc", var) +#define this_cpu_read_stable (var) percpu_from_op("mov", var, "p" (&(var))) +#define this_cpu_inc (var) percpu_unary_op("inc", var)
Why redefine this_cpu_inc? include/linux/percpu.h already defines it.
Quoted 18 lines · Christoph Lameter
Review static __always_inline int x86_this_cpu_constant_test_bit(unsigned int nr, · line 551
Quoted 13 lines · Alex,Shi
percpu_read()used inasm/percpu.hyet. Anyway, I removed definitions here and do a work around for the function as following:(quoted diff) +5
-1@@ -551,7 +544,11 @@ static __always_inline int x86_this_cpu_constant_test_bit(unsigned int nr, { unsigned long __percpu *a = (unsigned long *)addr + nr / BITS_PER_LONG; - return ((1UL << (nr % BITS_PER_LONG)) & percpu_read(*a)) != 0; +#ifdef CONFIG_X86_64 + return ((1UL << (nr % BITS_PER_LONG)) & this_cpu_read_8(*a)) != 0; +#else + return ((1UL << (nr % BITS_PER_LONG)) & this_cpu_read_4(*a)) != 0; +#endifOK that looks okay.
Quoted 3 lines
Quoted 1 line
Signed-off-by Alex Shi <[email protected]>Is this a revised patch? I do not see a use of the
__this_cpu_xxops.
I just changed where you pointed as below at the patch line 146:
#ifdef CONFIG_SMP
#define __percpu_prefix "%%"__stringify(__percpu_seg)":" -#define __my_cpu_offset percpu_read(this_cpu_off) +#define __my_cpu_offset __this_cpu_read(this_cpu_off)
I do not quite understand you here. Do you mean to change every possible this_cpu_xxx() to __this_cpu_xxx()?
For x86 platform code, this_cpu_xxx are equal to __this_cpu_xxx.
For other platform, all of them don't enable specific percpu_xxx, they are using generic percpu_xxx(), and generic percpu_xxx imply preempt_enable and preempt_disable. So, change to this_cpu_xxx do not impact performance on them.
Quoted 12 lines
Review
Quoted 10 lines
(quoted diff) +2
-7-#define percpu_write(var, val) percpu_to_op("mov", var, val) -#define percpu_add(var, val) percpu_add_op(var, val) -#define percpu_sub(var, val) percpu_add_op(var, -(val)) -#define percpu_and(var, val) percpu_to_op("and", var, val) -#define percpu_or(var, val) percpu_to_op("or", var, val) -#define percpu_xor(var, val) percpu_to_op("xor", var, val) -#define percpu_inc(var) percpu_unary_op("inc", var) +#define this_cpu_read_stable(var) percpu_from_op("mov", var, "p" (&(var))) +#define this_cpu_inc(var) percpu_unary_op("inc", var)Why redefine
this_cpu_inc?include/linux/percpu.halready defines it.
Ops, My mistaken, don't need this at all. revise this in the following patch.
===
Quoted 1 line
From
c6f6c18ed79a1cc97cd2b1f86c36f6d5ed3f8665Mon Sep 17 00:00:00 2001
From: Alex Shi <[email protected]>
Date: Wed, 12 Oct 2011 16:55:49 +0800
Subject: [PATCH] Code clean up for percpu_xxx() functions
Since percpu_xxx() serial functions are duplicate with this_cpu_xxx(). Removing percpu_xxx() definition and replacing them by this_cpu_xxx() in using. This patch is just do code clean up, no function change.
Signed-off-by Alex Shi <[email protected]>
--- arch/x86/include/asm/current.h | 2 +- arch/x86/include/asm/hardirq.h | 8 ++-- arch/x86/include/asm/irq_regs.h | 4 +- arch/x86/include/asm/mmu_context.h | 12 ++++---- arch/x86/include/asm/percpu.h | 24 ++++++--------- arch/x86/include/asm/smp.h | 4 +- arch/x86/include/asm/stackprotector.h | 4 +- arch/x86/include/asm/thread_info.h | 2 +- arch/x86/include/asm/tlbflush.h | 4 +- arch/x86/kernel/cpu/common.c | 2 +- arch/x86/kernel/cpu/mcheck/mce.c | 4 +- arch/x86/kernel/cpu/perf_event.c | 4 +- arch/x86/kernel/paravirt.c | 12 ++++---- arch/x86/kernel/process_32.c | 2 +- arch/x86/kernel/process_64.c | 12 ++++---- arch/x86/kvm/x86.c | 14 ++++---- arch/x86/mm/tlb.c | 10 +++--- arch/x86/xen/enlighten.c | 6 ++-- arch/x86/xen/irq.c | 8 ++-- arch/x86/xen/mmu.c | 20 ++++++------ arch/x86/xen/multicalls.h | 2 +- arch/x86/xen/smp.c | 2 +- include/linux/percpu.h | 53 --------------------------------- include/linux/topology.h | 4 +- net/netfilter/xt_TEE.c | 12 ++++---- net/socket.c | 4 +- 26 files changed, 89 insertions(+), 146 deletions(-)
arch/x86/include/asm/current.h +1 -1
@@ -11,7 +11,7 @@ DECLARE_PER_CPU(struct task_struct *, current_task); static __always_inline struct task_struct *get_current(void) { -return percpu_read_stable (current_task); +return this_cpu_read_stable (current_task); }#define current get_current()
arch/x86/include/asm/hardirq.h +4 -4
@@ -35,14 +35,14 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);#define __ARCH_IRQ_STAT -#define inc_irq_stat (member) percpu_inc(irq_stat.member) +#define inc_irq_stat (member) this_cpu_inc(irq_stat.member) -#define local_softirq_pending () percpu_read(irq_stat.__softirq_pending) +#define local_softirq_pending () this_cpu_read(irq_stat.__softirq_pending)#define __ARCH_SET_SOFTIRQ_PENDING -#define set_softirq_pending (x) percpu_write(irq_stat.__softirq_pending, (x)) -#define or_softirq_pending (x) percpu_or(irq_stat.__softirq_pending, (x)) +#define set_softirq_pending (x) this_cpu_write(irq_stat.__softirq_pending, (x)) +#define or_softirq_pending (x) this_cpu_or(irq_stat.__softirq_pending, (x))extern void ack_bad_irq (unsigned int irq);
arch/x86/include/asm/irq_regs.h +2 -2
@@ -15,7 +15,7 @@ DECLARE_PER_CPU(struct pt_regs *, irq_regs);static inline struct pt_regs *get_irq_regs (void ) { -return percpu_read (irq_regs); +return this_cpu_read (irq_regs); }static inline struct pt_regs *set_irq_regs (struct pt_regs *new_regs)
@@ -23,7 +23,7 @@ static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)struct pt_regs *old_regs; old_regs =get_irq_regs (); -percpu_write (irq_regs, new_regs); +this_cpu_write (irq_regs, new_regs);return old_regs; }
arch/x86/include/asm/mmu_context.h +6 -6
@@ -25,8 +25,8 @@ void destroy_context(struct mm_struct *mm);static inline void enter_lazy_tlb (struct mm_struct *mm,struct task_struct *tsk) {#ifdef CONFIG_SMP -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) -percpu_write (cpu_tlbstate.state,TLBSTATE_LAZY ); +if (this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) +this_cpu_write (cpu_tlbstate.state,TLBSTATE_LAZY );#endif }
@@ -37,8 +37,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,if (likely (prev != next)) {#ifdef CONFIG_SMP -percpu_write (cpu_tlbstate.state,TLBSTATE_OK ); -percpu_write (cpu_tlbstate.active_mm, next); +this_cpu_write (cpu_tlbstate.state,TLBSTATE_OK ); +this_cpu_write (cpu_tlbstate.active_mm, next);#endif cpumask_set_cpu (cpu,mm_cpumask (next));
@@ -56,8 +56,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, }#ifdef CONFIG_SMP else { -percpu_write (cpu_tlbstate.state,TLBSTATE_OK ); -BUG_ON (percpu_read (cpu_tlbstate.active_mm) != next); +this_cpu_write (cpu_tlbstate.state,TLBSTATE_OK ); +BUG_ON (this_cpu_read (cpu_tlbstate.active_mm) != next);if (!cpumask_test_and_set_cpu (cpu,mm_cpumask (next))) {/* We were in lazy tlb mode and leave_mm disabled
arch/x86/include/asm/percpu.h +10 -14
@@ -46,7 +46,7 @@#ifdef CONFIG_SMP #define __percpu_prefix "%%"__stringify(__percpu_seg)":" -#define __my_cpu_offset percpu_read(this_cpu_off) +#define __my_cpu_offset __this_cpu_read(this_cpu_off)/* * Compared to the generic __my_cpu_offset version, the following
@@ -351,23 +351,15 @@ do { \ })/* -* percpu_read() makes gcc load the percpu variable every time it is -* accessed while percpu_read_stable() allows the value to be cached. -* percpu_read_stable() is more efficient and can be used if its value +* this_cpu_read() makes gcc load the percpu variable every time it is +* accessed while this_cpu_read_stable() allows the value to be cached. +* this_cpu_read_stable() is more efficient and can be used if its value * is guaranteed to be valid across cpus. The current users include * get_current() and get_thread_info() both of which are actually * per-thread variables implemented as per-cpu variables and thus * stable for the duration of the respective task. */ -#define percpu_read (var) percpu_from_op("mov", var, "m" (var)) -#define percpu_read_stable (var) percpu_from_op("mov", var, "p" (&(var))) -#define percpu_write (var, val) percpu_to_op("mov", var, val) -#define percpu_add (var, val) percpu_add_op(var, val) -#define percpu_sub (var, val) percpu_add_op(var, -(val)) -#define percpu_and (var, val) percpu_to_op("and", var, val) -#define percpu_or (var, val) percpu_to_op("or", var, val) -#define percpu_xor (var, val) percpu_to_op("xor", var, val) -#define percpu_inc (var) percpu_unary_op("inc", var) +#define this_cpu_read_stable (var) percpu_from_op("mov", var, "p" (&(var)))#define __this_cpu_read_1 (pcp) percpu_from_op("mov", (pcp), "m"(pcp))#define __this_cpu_read_2 (pcp) percpu_from_op("mov", (pcp), "m"(pcp))
@@ -551,7 +543,11 @@ static __always_inline int x86_this_cpu_constant_test_bit(unsigned int nr, {unsigned long __percpu *a = (unsigned long *)addr + nr /BITS_PER_LONG ; -return ((1UL << (nr %BITS_PER_LONG )) &percpu_read (*a)) !=0 ; +#ifdef CONFIG_X86_64 +return ((1UL << (nr %BITS_PER_LONG )) &this_cpu_read_8 (*a)) !=0 ; +#else +return ((1UL << (nr %BITS_PER_LONG )) &this_cpu_read_4 (*a)) !=0 ; +#endif } static inline int x86_this_cpu_variable_test_bit(int nr,
arch/x86/include/asm/smp.h +2 -2
@@ -188,11 +188,11 @@ extern unsigned disabled_cpus __cpuinitdata;* from the initial startup. We map APIC_BASE very early in page_setup(), * so this is correct in the x86 case. */ -#define raw_smp_processor_id () (percpu_read(cpu_number)) +#define raw_smp_processor_id () (this_cpu_read(cpu_number))extern int safe_smp_processor_id (void );#elif defined(CONFIG_X86_64_SMP ) -#define raw_smp_processor_id () (percpu_read(cpu_number)) +#define raw_smp_processor_id () (this_cpu_read(cpu_number))#define stack_smp_processor_id () \ ({ \
arch/x86/include/asm/stackprotector.h +2 -2
@@ -76,9 +76,9 @@ static __always_inline void boot_init_stack_canary(void) current->stack_canary = canary;#ifdef CONFIG_X86_64 -percpu_write (irq_stack_union.stack_canary, canary); +this_cpu_write (irq_stack_union.stack_canary, canary);#else -percpu_write (stack_canary.canary, canary); +this_cpu_write (stack_canary.canary, canary);#endif }
arch/x86/include/asm/thread_info.h +1 -1
@@ -219,7 +219,7 @@ DECLARE_PER_CPU(unsigned long, kernel_stack);static inline struct thread_info *current_thread_info (void ) {struct thread_info *ti; - ti = (void *)(percpu_read_stable (kernel_stack) + + ti = (void *)(this_cpu_read_stable (kernel_stack) +KERNEL_STACK_OFFSET -THREAD_SIZE );return ti; }
arch/x86/include/asm/tlbflush.h +2 -2
@@ -156,8 +156,8 @@ DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate);static inline void reset_lazy_tlbstate (void ) { -percpu_write (cpu_tlbstate.state,0 ); -percpu_write (cpu_tlbstate.active_mm, &init_mm); +this_cpu_write (cpu_tlbstate.state,0 ); +this_cpu_write (cpu_tlbstate.active_mm, &init_mm); }#endif /* SMP */
arch/x86/kernel/cpu/common.c +1 -1
@@ -1158,7 +1158,7 @@ void __cpuinit cpu_init(void) oist = &per_cpu (orig_ist, cpu);#ifdef CONFIG_NUMA -if (cpu !=0 &&percpu_read (numa_node) ==0 && +if (cpu !=0 &&this_cpu_read (numa_node) ==0 &&early_cpu_to_node (cpu) !=NUMA_NO_NODE )set_numa_node (early_cpu_to_node (cpu));#endif
arch/x86/kernel/cpu/mcheck/mce.c +2 -2
@@ -512,7 +512,7 @@ void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)struct mce m;int i; -percpu_inc (mce_poll_count); +this_cpu_inc (mce_poll_count);mce_gather_info (&m,NULL );
@@ -906,7 +906,7 @@ void do_machine_check(struct pt_regs *regs, long error_code)atomic_inc (&mce_entry); -percpu_inc (mce_exception_count); +this_cpu_inc (mce_exception_count);if (notify_die (DIE_NMI ,"machine check" , regs, error_code,18 ,SIGKILL ) ==NOTIFY_STOP )
arch/x86/kernel/cpu/perf_event.c +2 -2
@@ -1389,7 +1389,7 @@ perf_event_nmi_handler(struct notifier_block *self,case DIE_NMI :break ;case DIE_NMIUNKNOWN : - this_nmi =percpu_read (irq_stat.__nmi_count); + this_nmi =this_cpu_read (irq_stat.__nmi_count);if (this_nmi !=__this_cpu_read (pmu_nmi.marked))/* let the kernel handle the unknown nmi */ return NOTIFY_DONE ;
@@ -1409,7 +1409,7 @@ perf_event_nmi_handler(struct notifier_block *self,if (!handled)return NOTIFY_DONE ; - this_nmi =percpu_read (irq_stat.__nmi_count); + this_nmi =this_cpu_read (irq_stat.__nmi_count); if ((handled > 1) || /* the next nmi could be a back-to-back nmi */ ((__this_cpu_read(pmu_nmi.marked) == this_nmi) &&
arch/x86/kernel/paravirt.c +6 -6
@@ -239,16 +239,16 @@ static DEFINE_PER_CPU(enum paravirt_lazy_mode, paravirt_lazy_mode) = PARAVIRT_LAstatic inline void enter_lazy (enum paravirt_lazy_mode mode) { -BUG_ON (percpu_read (paravirt_lazy_mode) !=PARAVIRT_LAZY_NONE ); +BUG_ON (this_cpu_read (paravirt_lazy_mode) !=PARAVIRT_LAZY_NONE ); -percpu_write (paravirt_lazy_mode, mode); +this_cpu_write (paravirt_lazy_mode, mode); }static void leave_lazy (enum paravirt_lazy_mode mode) { -BUG_ON (percpu_read (paravirt_lazy_mode) != mode); +BUG_ON (this_cpu_read (paravirt_lazy_mode) != mode); -percpu_write (paravirt_lazy_mode,PARAVIRT_LAZY_NONE ); +this_cpu_write (paravirt_lazy_mode,PARAVIRT_LAZY_NONE ); }void paravirt_enter_lazy_mmu (void )
@@ -265,7 +265,7 @@ void paravirt_start_context_switch(struct task_struct *prev) {BUG_ON (preemptible ()); -if (percpu_read (paravirt_lazy_mode) ==PARAVIRT_LAZY_MMU ) { +if (this_cpu_read (paravirt_lazy_mode) ==PARAVIRT_LAZY_MMU ) {arch_leave_lazy_mmu_mode ();set_ti_thread_flag (task_thread_info (prev),TIF_LAZY_MMU_UPDATES ); }
@@ -287,7 +287,7 @@ enum paravirt_lazy_mode paravirt_get_lazy_mode(void)if (in_interrupt ())return PARAVIRT_LAZY_NONE ; -return percpu_read (paravirt_lazy_mode); +return this_cpu_read (paravirt_lazy_mode); }void arch_flush_lazy_mmu_mode (void )
arch/x86/kernel/process_32.c +1 -1
@@ -373,7 +373,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)if (prev->gs | next->gs)lazy_load_gs (next->gs); -percpu_write (current_task, next_p); +this_cpu_write (current_task, next_p);return prev_p; }
arch/x86/kernel/process_64.c +6 -6
@@ -73,7 +73,7 @@ EXPORT_SYMBOL_GPL(idle_notifier_unregister);void enter_idle (void ) { -percpu_write (is_idle,1 ); +this_cpu_write (is_idle,1 );atomic_notifier_call_chain (&idle_notifier,IDLE_START ,NULL ); }
@@ -336,7 +336,7 @@ start_thread_common(struct pt_regs *regs, unsigned long new_ip,load_gs_index (0 ); regs->ip = new_ip; regs->sp = new_sp; -percpu_write (old_rsp, new_sp); +this_cpu_write (old_rsp, new_sp); regs->cs = _cs; regs->ss = _ss; regs->flags =X86_EFLAGS_IF ;
@@ -470,11 +470,11 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)/* * Switch the PDA and FPU contexts. */ - prev->usersp =percpu_read (old_rsp); -percpu_write (old_rsp, next->usersp); -percpu_write (current_task, next_p); + prev->usersp =this_cpu_read (old_rsp); +this_cpu_write (old_rsp, next->usersp); +this_cpu_write (current_task, next_p); -percpu_write (kernel_stack, +this_cpu_write (kernel_stack, (unsigned long )task_stack_page (next_p) +THREAD_SIZE -KERNEL_STACK_OFFSET );
arch/x86/kvm/x86.c +7 -7
@@ -5024,15 +5024,15 @@ static DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);static int kvm_is_in_guest (void ) { -return percpu_read (current_vcpu) !=NULL ; +return this_cpu_read (current_vcpu) !=NULL ; }static int kvm_is_user_mode (void ) {int user_mode =3 ; -if (percpu_read (current_vcpu)) - user_mode = kvm_x86_ops->get_cpl (percpu_read (current_vcpu)); +if (this_cpu_read (current_vcpu)) + user_mode = kvm_x86_ops->get_cpl (this_cpu_read (current_vcpu));return user_mode !=0 ; }
@@ -5041,8 +5041,8 @@ static unsigned long kvm_get_guest_ip(void) {unsigned long ip =0 ; -if (percpu_read (current_vcpu)) - ip =kvm_rip_read (percpu_read (current_vcpu)); +if (this_cpu_read (current_vcpu)) + ip =kvm_rip_read (this_cpu_read (current_vcpu));return ip; }
@@ -5055,13 +5055,13 @@ static struct perf_guest_info_callbacks kvm_guest_cbs = {void kvm_before_handle_nmi (struct kvm_vcpu *vcpu) { -percpu_write (current_vcpu, vcpu); +this_cpu_write (current_vcpu, vcpu); }EXPORT_SYMBOL_GPL (kvm_before_handle_nmi);void kvm_after_handle_nmi (struct kvm_vcpu *vcpu) { -percpu_write (current_vcpu,NULL ); +this_cpu_write (current_vcpu,NULL ); }EXPORT_SYMBOL_GPL (kvm_after_handle_nmi);
arch/x86/mm/tlb.c +5 -5
@@ -61,10 +61,10 @@ static DEFINE_PER_CPU_READ_MOSTLY(int, tlb_vector_offset);*/ void leave_mm (int cpu) { -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) +if (this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_OK )BUG ();cpumask_clear_cpu (cpu, -mm_cpumask (percpu_read (cpu_tlbstate.active_mm))); +mm_cpumask (this_cpu_read (cpu_tlbstate.active_mm)));load_cr3 (swapper_pg_dir); }EXPORT_SYMBOL_GPL (leave_mm);
@@ -152,8 +152,8 @@ void smp_invalidate_interrupt(struct pt_regs *regs)* BUG(); */ -if (f->flush_mm ==percpu_read (cpu_tlbstate.active_mm)) { -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) { +if (f->flush_mm ==this_cpu_read (cpu_tlbstate.active_mm)) { +if (this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) {if (f->flush_va ==TLB_FLUSH_ALL )local_flush_tlb ();else
@@ -322,7 +322,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)static void do_flush_tlb_all (void *info) {__flush_tlb_all (); -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_LAZY ) +if (this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_LAZY )leave_mm (smp_processor_id ()); }
arch/x86/xen/enlighten.c +3 -3
@@ -776,11 +776,11 @@ static DEFINE_PER_CPU(unsigned long, xen_cr0_value);static unsigned long xen_read_cr0 (void ) { -unsigned long cr0 =percpu_read (xen_cr0_value); +unsigned long cr0 =this_cpu_read (xen_cr0_value);if (unlikely (cr0 ==0 )) { cr0 =native_read_cr0 (); -percpu_write (xen_cr0_value, cr0); +this_cpu_write (xen_cr0_value, cr0); }return cr0;
@@ -790,7 +790,7 @@ static void xen_write_cr0(unsigned long cr0) {struct multicall_space mcs; -percpu_write (xen_cr0_value, cr0); +this_cpu_write (xen_cr0_value, cr0);/* Only pay attention to cr0.TS; everything else is ignored. */
arch/x86/xen/irq.c +4 -4
@@ -26,7 +26,7 @@ static unsigned long xen_save_fl(void)struct vcpu_info *vcpu;unsigned long flags; - vcpu =percpu_read (xen_vcpu); + vcpu =this_cpu_read (xen_vcpu);/* flag has opposite sense of mask */ flags = !vcpu->evtchn_upcall_mask;
@@ -50,7 +50,7 @@ static void xen_restore_fl(unsigned long flags)make sure we're don't switch CPUs between getting the vcpu pointer and updating the mask. */ preempt_disable (); - vcpu =percpu_read (xen_vcpu); + vcpu =this_cpu_read (xen_vcpu); vcpu->evtchn_upcall_mask = flags;preempt_enable_no_resched ();
@@ -72,7 +72,7 @@ static void xen_irq_disable(void)make sure we're don't switch CPUs between getting the vcpu pointer and updating the mask. */ preempt_disable (); -percpu_read (xen_vcpu)->evtchn_upcall_mask =1 ; +this_cpu_read (xen_vcpu)->evtchn_upcall_mask =1 ;preempt_enable_no_resched (); }PV_CALLEE_SAVE_REGS_THUNK (xen_irq_disable);
@@ -86,7 +86,7 @@ static void xen_irq_enable(void)the caller is confused and is trying to re-enable interrupts on an indeterminate processor. */ - vcpu =percpu_read (xen_vcpu); + vcpu =this_cpu_read (xen_vcpu); vcpu->evtchn_upcall_mask =0 ;/* Doesn't matter if we get preempted here, because any
arch/x86/xen/mmu.c +10 -10
@@ -1106,14 +1106,14 @@ static void drop_other_mm_ref(void *info)struct mm_struct *mm = info;struct mm_struct *active_mm; - active_mm =percpu_read (cpu_tlbstate.active_mm); + active_mm =this_cpu_read (cpu_tlbstate.active_mm); -if (active_mm == mm &&percpu_read (cpu_tlbstate.state) !=TLBSTATE_OK ) +if (active_mm == mm &&this_cpu_read (cpu_tlbstate.state) !=TLBSTATE_OK )leave_mm (smp_processor_id ());/* If this cpu still has a stale cr3 reference, then make sure it has been flushed. */ -if (percpu_read (xen_current_cr3) ==__pa (mm->pgd)) +if (this_cpu_read (xen_current_cr3) ==__pa (mm->pgd))load_cr3 (swapper_pg_dir); }
@@ -1220,17 +1220,17 @@ static void __init xen_pagetable_setup_done(pgd_t *base)static void xen_write_cr2 (unsigned long cr2) { -percpu_read (xen_vcpu)->arch.cr2 = cr2; +this_cpu_read (xen_vcpu)->arch.cr2 = cr2; }static unsigned long xen_read_cr2 (void ) { -return percpu_read (xen_vcpu)->arch.cr2; +return this_cpu_read (xen_vcpu)->arch.cr2; }unsigned long xen_read_cr2_direct (void ) { -return percpu_read (xen_vcpu_info.arch.cr2); +return this_cpu_read (xen_vcpu_info.arch.cr2); }static void xen_flush_tlb (void )
@@ -1313,12 +1313,12 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,static unsigned long xen_read_cr3 (void ) { -return percpu_read (xen_cr3); +return this_cpu_read (xen_cr3); }static void set_current_cr3 (void *v) { -percpu_write (xen_current_cr3, (unsigned long )v); +this_cpu_write (xen_current_cr3, (unsigned long )v); }static void __xen_write_cr3 (bool kernel,unsigned long cr3)
@@ -1341,7 +1341,7 @@ static void __xen_write_cr3(bool kernel, unsigned long cr3)xen_extend_mmuext_op (&op);if (kernel) { -percpu_write (xen_cr3, cr3); +this_cpu_write (xen_cr3, cr3);/* Update xen_current_cr3 once the batch has actually been submitted. */
@@ -1357,7 +1357,7 @@ static void xen_write_cr3(unsigned long cr3)/* Update while interrupts are disabled, so its atomic with respect to ipis */ -percpu_write (xen_cr3, cr3); +this_cpu_write (xen_cr3, cr3);__xen_write_cr3 (true, cr3);
arch/x86/xen/multicalls.h +1 -1
@@ -47,7 +47,7 @@ static inline void xen_mc_issue(unsigned mode)xen_mc_flush ();/* restore flags saved in xen_mc_batch */ -local_irq_restore (percpu_read (xen_mc_irq_flags)); +local_irq_restore (this_cpu_read (xen_mc_irq_flags)); }/* Set up a callback to be called when the current batch is flushed */
arch/x86/xen/smp.c +1 -1
@@ -76,7 +76,7 @@ static void __cpuinit cpu_bringup(void)xen_setup_cpu_clockevents ();set_cpu_online (cpu, true); -percpu_write (cpu_state,CPU_ONLINE ); +this_cpu_write (cpu_state,CPU_ONLINE );wmb ();/* We can take interrupts now: we're officially "up". */
include/linux/percpu.h +0 -53
59 lines not on this page · raw message
include/linux/topology.h +2 -2
@@ -235,7 +235,7 @@ static inline int cpu_to_node(int cpu)#ifndef set_numa_nodestatic inline void set_numa_node (int node) { -percpu_write (numa_node, node); +this_cpu_write (numa_node, node); }#endif
@@ -270,7 +270,7 @@ DECLARE_PER_CPU(int, _numa_mem_);#ifndef set_numa_memstatic inline void set_numa_mem (int node) { -percpu_write (_numa_mem_, node); +this_cpu_write (_numa_mem_, node); }#endif
net/netfilter/xt_TEE.c +6 -6
@@ -90,7 +90,7 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)const struct xt_tee_tginfo *info = par->targinfo;struct iphdr *iph; -if (percpu_read (tee_active)) +if (this_cpu_read (tee_active))return XT_CONTINUE ;/* * Copy the skb, and route the copy. Will later return %XT_CONTINUE for
@@ -127,9 +127,9 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)ip_send_check (iph);if (tee_tg_route4 (skb, info)) { -percpu_write (tee_active, true); +this_cpu_write (tee_active, true);ip_local_out (skb); -percpu_write (tee_active, false); +this_cpu_write (tee_active, false); }else {kfree_skb (skb); }
@@ -170,7 +170,7 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par) {const struct xt_tee_tginfo *info = par->targinfo; -if (percpu_read (tee_active)) +if (this_cpu_read (tee_active))return XT_CONTINUE ; skb =pskb_copy (skb,GFP_ATOMIC );if (skb ==NULL )
@@ -188,9 +188,9 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par) --iph->hop_limit; }if (tee_tg_route6 (skb, info)) { -percpu_write (tee_active, true); +this_cpu_write (tee_active, true);ip6_local_out (skb); -percpu_write (tee_active, false); +this_cpu_write (tee_active, false); }else {kfree_skb (skb); }
net/socket.c +2 -2
@@ -479,7 +479,7 @@ static struct socket *sock_alloc(void) inode->i_uid =current_fsuid (); inode->i_gid =current_fsgid (); -percpu_add (sockets_in_use,1 ); +this_cpu_add (sockets_in_use,1 );return sock; }
@@ -522,7 +522,7 @@ void sock_release(struct socket *sock)if (rcu_dereference_protected (sock->wq,1 )->fasync_list)printk (KERN_ERR "sock_release: fasync list not empty!\n" ); -percpu_sub (sockets_in_use,1 ); +this_cpu_sub (sockets_in_use,1 );if (!sock->file) {iput (SOCK_INODE (sock));return ;
Signature · 1 line
1.7.5.1
Christoph Lameter
maintainerQuoted 7 lines · Alex,Shi
#ifdef
CONFIG_SMP
#define__percpu_prefix"%%"__stringify(__percpu_seg)":"arch/x86/include/asm/percpu.h +1
-1-#define __my_cpu_offset percpu_read(this_cpu_off) +#define __my_cpu_offset __this_cpu_read(this_cpu_off)I do not quite understand you here. Do you mean to change every possible
this_cpu_xxx()to__this_cpu_xxx()?
Change only those locations where the kernel thread cannot be preempted.
Quoted 11 lines
Quoted 1 line
Is this a revised patch? I do not see a use of the
__this_cpu_xxops.I just changed where you pointed as below at the patch line 146:
#ifdef
CONFIG_SMP
#define__percpu_prefix"%%"__stringify(__percpu_seg)":"arch/x86/include/asm/percpu.h +1
-1-#define __my_cpu_offset percpu_read(this_cpu_off) +#define __my_cpu_offset __this_cpu_read(this_cpu_off)I do not quite understand you here. Do you mean to change every possible
this_cpu_xxx()to__this_cpu_xxx()?
I thought again of this_cpu_xxx function, yes, some of them are better to be replaced by __this_cpu__xxx if preemption is safe on the scenario. But this change is better to be done in another patch. I did not finished all changed function check today. May try again in next week. What's you pinion of this?
Christoph Lameter
maintainerQuoted 5 lines · Alex,Shi
I thought again of
this_cpu_xxxfunction, yes, some of them are better to be replaced by__this_cpu__xxxif preemption is safe on the scenario. But this change is better to be done in another patch. I did not finished all changed function check today. May try again in next week. What's you pinion of this?
I think its good to have it all in a single patch.
Quoted 10 lines · Christoph Lameter
Quoted 5 lines · Alex,Shi
I thought again of
this_cpu_xxxfunction, yes, some of them are better to be replaced by__this_cpu__xxxif preemption is safe on the scenario. But this change is better to be done in another patch. I did not finished all changed function check today. May try again in next week. What's you pinion of this?I think its good to have it all in a single patch.
I thought over your requirements, but it is difficult for me to know lots of fields in kernel, like for xen/kvm/network etc. So, to avoid introduce bugs here, could we do this blind changes, and then change each of fields and get owner's ack separately.
Anyway if you still want to keep all changes in one patch, the patch may needs long time to wait for each of reviews/acks one by one.
Christoph Lameter
maintainerQuoted 7 lines · Alex,Shi
Quoted 2 lines
I think its good to have it all in a single patch.
I thought over your requirements, but it is difficult for me to know lots of fields in kernel, like for xen/kvm/network etc. So, to avoid introduce bugs here, could we do this blind changes, and then change each of fields and get owner's ack separately.
Well do the obvious (where you can see that interrupts or preemption are disabled in the function or in a calling function). Leave the rest as is and provide separate patches for them?
Quoted 4 lines
Well do the obvious (where you can see that interrupts or preemption are disabled in the function or in a calling function). Leave the rest as is and provide separate patches for them?
Thanks for comments! I initialized the patch as following accordingly, And cc to more maintainers for review. I checked all code except xen/kvm part, totally a idiot for them.
==========
Since percpu_xxx() serial functions are duplicate with this_cpu_xxx(). Removing percpu_xxx() definition and replacing them by this_cpu_xxx() in code.
And further more, as Christoph Lameter's requirement, I try to use __this_cpu_xx to replace this_cpu_xxx if it is in preempt safe scenario. The preempt safe scenarios include:
1, in irq/softirq/nmi handler
2, protected by preempt_disable
3, protected by spin_lock
4, if the code context imply that it is preempt safe, like the code is follows or be followed a preempt safe code.
I left the xen/kvm part code unchanged, since no any idea of them.
BTW, In fact, this_cpu_xxx are same as __this_cpu_xxx since all funcs implement in a single instruction for x86 machine. But it maybe different for other platforms, so, doing this distinguish is helpful for other platforms' performance.
Signed-off-by Alex Shi <[email protected]>
--- arch/x86/include/asm/current.h | 2 +- arch/x86/include/asm/hardirq.h | 8 ++-- arch/x86/include/asm/irq_regs.h | 4 +- arch/x86/include/asm/mmu_context.h | 12 ++++---- arch/x86/include/asm/percpu.h | 24 ++++++--------- arch/x86/include/asm/smp.h | 4 +- arch/x86/include/asm/stackprotector.h | 4 +- arch/x86/include/asm/thread_info.h | 2 +- arch/x86/include/asm/tlbflush.h | 4 +- arch/x86/kernel/cpu/common.c | 2 +- arch/x86/kernel/cpu/mcheck/mce.c | 4 +- arch/x86/kernel/cpu/perf_event.c | 4 +- arch/x86/kernel/paravirt.c | 12 ++++---- arch/x86/kernel/process_32.c | 2 +- arch/x86/kernel/process_64.c | 12 ++++---- arch/x86/kvm/x86.c | 14 ++++---- arch/x86/mm/tlb.c | 10 +++--- arch/x86/xen/enlighten.c | 6 ++-- arch/x86/xen/irq.c | 8 ++-- arch/x86/xen/mmu.c | 20 ++++++------ arch/x86/xen/multicalls.h | 2 +- arch/x86/xen/smp.c | 2 +- include/linux/percpu.h | 53 --------------------------------- include/linux/topology.h | 4 +- net/netfilter/xt_TEE.c | 12 ++++---- net/socket.c | 4 +- 26 files changed, 89 insertions(+), 146 deletions(-)
arch/x86/include/asm/current.h +1 -1
@@ -11,7 +11,7 @@ DECLARE_PER_CPU(struct task_struct *, current_task); static __always_inline struct task_struct *get_current(void) { -return percpu_read_stable (current_task); +return this_cpu_read_stable (current_task); }#define current get_current()
arch/x86/include/asm/hardirq.h +4 -4
@@ -35,14 +35,14 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);#define __ARCH_IRQ_STAT -#define inc_irq_stat (member) percpu_inc(irq_stat.member) +#define inc_irq_stat (member) __this_cpu_inc(irq_stat.member) -#define local_softirq_pending () percpu_read(irq_stat.__softirq_pending) +#define local_softirq_pending () __this_cpu_read(irq_stat.__softirq_pending)#define __ARCH_SET_SOFTIRQ_PENDING -#define set_softirq_pending (x) percpu_write(irq_stat.__softirq_pending, (x)) -#define or_softirq_pending (x) percpu_or(irq_stat.__softirq_pending, (x)) +#define set_softirq_pending (x) __this_cpu_write(irq_stat.__softirq_pending, (x)) +#define or_softirq_pending (x) __this_cpu_or(irq_stat.__softirq_pending, (x))extern void ack_bad_irq (unsigned int irq);
arch/x86/include/asm/irq_regs.h +2 -2
@@ -15,7 +15,7 @@ DECLARE_PER_CPU(struct pt_regs *, irq_regs);static inline struct pt_regs *get_irq_regs (void ) { -return percpu_read (irq_regs); +return __this_cpu_read (irq_regs); }static inline struct pt_regs *set_irq_regs (struct pt_regs *new_regs)
@@ -23,7 +23,7 @@ static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)struct pt_regs *old_regs; old_regs =get_irq_regs (); -percpu_write (irq_regs, new_regs); +__this_cpu_write (irq_regs, new_regs);return old_regs; }
arch/x86/include/asm/mmu_context.h +6 -6
@@ -25,8 +25,8 @@ void destroy_context(struct mm_struct *mm);static inline void enter_lazy_tlb (struct mm_struct *mm,struct task_struct *tsk) {#ifdef CONFIG_SMP -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) -percpu_write (cpu_tlbstate.state,TLBSTATE_LAZY ); +if (__this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) +__this_cpu_write (cpu_tlbstate.state,TLBSTATE_LAZY );#endif }
@@ -37,8 +37,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,if (likely (prev != next)) {#ifdef CONFIG_SMP -percpu_write (cpu_tlbstate.state,TLBSTATE_OK ); -percpu_write (cpu_tlbstate.active_mm, next); +__this_cpu_write (cpu_tlbstate.state,TLBSTATE_OK ); +__this_cpu_write (cpu_tlbstate.active_mm, next);#endif cpumask_set_cpu (cpu,mm_cpumask (next));
@@ -56,8 +56,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, }#ifdef CONFIG_SMP else { -percpu_write (cpu_tlbstate.state,TLBSTATE_OK ); -BUG_ON (percpu_read (cpu_tlbstate.active_mm) != next); +__this_cpu_write (cpu_tlbstate.state,TLBSTATE_OK ); +BUG_ON (__this_cpu_read (cpu_tlbstate.active_mm) != next);if (!cpumask_test_and_set_cpu (cpu,mm_cpumask (next))) {/* We were in lazy tlb mode and leave_mm disabled
arch/x86/include/asm/percpu.h +10 -14
@@ -46,7 +46,7 @@#ifdef CONFIG_SMP #define __percpu_prefix "%%"__stringify(__percpu_seg)":" -#define __my_cpu_offset percpu_read(this_cpu_off) +#define __my_cpu_offset __this_cpu_read(this_cpu_off)/* * Compared to the generic __my_cpu_offset version, the following
@@ -351,23 +351,15 @@ do { \ })/* -* percpu_read() makes gcc load the percpu variable every time it is -* accessed while percpu_read_stable() allows the value to be cached. -* percpu_read_stable() is more efficient and can be used if its value +* this_cpu_read() makes gcc load the percpu variable every time it is +* accessed while this_cpu_read_stable() allows the value to be cached. +* this_cpu_read_stable() is more efficient and can be used if its value * is guaranteed to be valid across cpus. The current users include * get_current() and get_thread_info() both of which are actually * per-thread variables implemented as per-cpu variables and thus * stable for the duration of the respective task. */ -#define percpu_read (var) percpu_from_op("mov", var, "m" (var)) -#define percpu_read_stable (var) percpu_from_op("mov", var, "p" (&(var))) -#define percpu_write (var, val) percpu_to_op("mov", var, val) -#define percpu_add (var, val) percpu_add_op(var, val) -#define percpu_sub (var, val) percpu_add_op(var, -(val)) -#define percpu_and (var, val) percpu_to_op("and", var, val) -#define percpu_or (var, val) percpu_to_op("or", var, val) -#define percpu_xor (var, val) percpu_to_op("xor", var, val) -#define percpu_inc (var) percpu_unary_op("inc", var) +#define this_cpu_read_stable (var) percpu_from_op("mov", var, "p" (&(var)))#define __this_cpu_read_1 (pcp) percpu_from_op("mov", (pcp), "m"(pcp))#define __this_cpu_read_2 (pcp) percpu_from_op("mov", (pcp), "m"(pcp))
@@ -551,7 +543,11 @@ static __always_inline int x86_this_cpu_constant_test_bit(unsigned int nr, {unsigned long __percpu *a = (unsigned long *)addr + nr /BITS_PER_LONG ; -return ((1UL << (nr %BITS_PER_LONG )) &percpu_read (*a)) !=0 ; +#ifdef CONFIG_X86_64 +return ((1UL << (nr %BITS_PER_LONG )) &__this_cpu_read_8 (*a)) !=0 ; +#else +return ((1UL << (nr %BITS_PER_LONG )) &__this_cpu_read_4 (*a)) !=0 ; +#endif } static inline int x86_this_cpu_variable_test_bit(int nr,
arch/x86/include/asm/smp.h +2 -2
@@ -188,11 +188,11 @@ extern unsigned disabled_cpus __cpuinitdata;* from the initial startup. We map APIC_BASE very early in page_setup(), * so this is correct in the x86 case. */ -#define raw_smp_processor_id () (percpu_read(cpu_number)) +#define raw_smp_processor_id () (this_cpu_read(cpu_number))extern int safe_smp_processor_id (void );#elif defined(CONFIG_X86_64_SMP ) -#define raw_smp_processor_id () (percpu_read(cpu_number)) +#define raw_smp_processor_id () (this_cpu_read(cpu_number))#define stack_smp_processor_id () \ ({ \
arch/x86/include/asm/stackprotector.h +2 -2
@@ -76,9 +76,9 @@ static __always_inline void boot_init_stack_canary(void) current->stack_canary = canary;#ifdef CONFIG_X86_64 -percpu_write (irq_stack_union.stack_canary, canary); +__this_cpu_write (irq_stack_union.stack_canary, canary);#else -percpu_write (stack_canary.canary, canary); +__this_cpu_write (stack_canary.canary, canary);#endif }
arch/x86/include/asm/thread_info.h +1 -1
@@ -219,7 +219,7 @@ DECLARE_PER_CPU(unsigned long, kernel_stack);static inline struct thread_info *current_thread_info (void ) {struct thread_info *ti; - ti = (void *)(percpu_read_stable (kernel_stack) + + ti = (void *)(this_cpu_read_stable (kernel_stack) +KERNEL_STACK_OFFSET -THREAD_SIZE );return ti; }
arch/x86/include/asm/tlbflush.h +2 -2
@@ -156,8 +156,8 @@ DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate);static inline void reset_lazy_tlbstate (void ) { -percpu_write (cpu_tlbstate.state,0 ); -percpu_write (cpu_tlbstate.active_mm, &init_mm); +__this_cpu_write (cpu_tlbstate.state,0 ); +__this_cpu_write (cpu_tlbstate.active_mm, &init_mm); }#endif /* SMP */
arch/x86/kernel/cpu/common.c +1 -1
@@ -1158,7 +1158,7 @@ void __cpuinit cpu_init(void) oist = &per_cpu (orig_ist, cpu);#ifdef CONFIG_NUMA -if (cpu !=0 &&percpu_read (numa_node) ==0 && +if (cpu !=0 &&__this_cpu_read (numa_node) ==0 &&early_cpu_to_node (cpu) !=NUMA_NO_NODE )set_numa_node (early_cpu_to_node (cpu));#endif
arch/x86/kernel/cpu/mcheck/mce.c +2 -2
@@ -512,7 +512,7 @@ void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)struct mce m;int i; -percpu_inc (mce_poll_count); +__this_cpu_inc (mce_poll_count);mce_gather_info (&m,NULL );
@@ -906,7 +906,7 @@ void do_machine_check(struct pt_regs *regs, long error_code)atomic_inc (&mce_entry); -percpu_inc (mce_exception_count); +__this_cpu_inc (mce_exception_count);if (notify_die (DIE_NMI ,"machine check" , regs, error_code,18 ,SIGKILL ) ==NOTIFY_STOP )
arch/x86/kernel/cpu/perf_event.c +2 -2
@@ -1389,7 +1389,7 @@ perf_event_nmi_handler(struct notifier_block *self,case DIE_NMI :break ;case DIE_NMIUNKNOWN : - this_nmi =percpu_read (irq_stat.__nmi_count); + this_nmi =__this_cpu_read (irq_stat.__nmi_count);if (this_nmi !=__this_cpu_read (pmu_nmi.marked))/* let the kernel handle the unknown nmi */ return NOTIFY_DONE ;
@@ -1409,7 +1409,7 @@ perf_event_nmi_handler(struct notifier_block *self,if (!handled)return NOTIFY_DONE ; - this_nmi =percpu_read (irq_stat.__nmi_count); + this_nmi =__this_cpu_read (irq_stat.__nmi_count); if ((handled > 1) || /* the next nmi could be a back-to-back nmi */ ((__this_cpu_read(pmu_nmi.marked) == this_nmi) &&
arch/x86/kernel/paravirt.c +6 -6
@@ -239,16 +239,16 @@ static DEFINE_PER_CPU(enum paravirt_lazy_mode, paravirt_lazy_mode) = PARAVIRT_LAstatic inline void enter_lazy (enum paravirt_lazy_mode mode) { -BUG_ON (percpu_read (paravirt_lazy_mode) !=PARAVIRT_LAZY_NONE ); +BUG_ON (__this_cpu_read (paravirt_lazy_mode) !=PARAVIRT_LAZY_NONE ); -percpu_write (paravirt_lazy_mode, mode); +__this_cpu_write (paravirt_lazy_mode, mode); }static void leave_lazy (enum paravirt_lazy_mode mode) { -BUG_ON (percpu_read (paravirt_lazy_mode) != mode); +BUG_ON (__this_cpu_read (paravirt_lazy_mode) != mode); -percpu_write (paravirt_lazy_mode,PARAVIRT_LAZY_NONE ); +__this_cpu_write (paravirt_lazy_mode,PARAVIRT_LAZY_NONE ); }void paravirt_enter_lazy_mmu (void )
@@ -265,7 +265,7 @@ void paravirt_start_context_switch(struct task_struct *prev) {BUG_ON (preemptible ()); -if (percpu_read (paravirt_lazy_mode) ==PARAVIRT_LAZY_MMU ) { +if (__this_cpu_read (paravirt_lazy_mode) ==PARAVIRT_LAZY_MMU ) {arch_leave_lazy_mmu_mode ();set_ti_thread_flag (task_thread_info (prev),TIF_LAZY_MMU_UPDATES ); }
@@ -287,7 +287,7 @@ enum paravirt_lazy_mode paravirt_get_lazy_mode(void)if (in_interrupt ())return PARAVIRT_LAZY_NONE ; -return percpu_read (paravirt_lazy_mode); +return __this_cpu_read (paravirt_lazy_mode); }void arch_flush_lazy_mmu_mode (void )
arch/x86/kernel/process_32.c +1 -1
@@ -373,7 +373,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)if (prev->gs | next->gs)lazy_load_gs (next->gs); -percpu_write (current_task, next_p); +__this_cpu_write (current_task, next_p);return prev_p; }
arch/x86/kernel/process_64.c +6 -6
@@ -73,7 +73,7 @@ EXPORT_SYMBOL_GPL(idle_notifier_unregister);void enter_idle (void ) { -percpu_write (is_idle,1 ); +__this_cpu_write (is_idle,1 );atomic_notifier_call_chain (&idle_notifier,IDLE_START ,NULL ); }
@@ -336,7 +336,7 @@ start_thread_common(struct pt_regs *regs, unsigned long new_ip,load_gs_index (0 ); regs->ip = new_ip; regs->sp = new_sp; -percpu_write (old_rsp, new_sp); +this_cpu_write (old_rsp, new_sp); regs->cs = _cs; regs->ss = _ss; regs->flags =X86_EFLAGS_IF ;
@@ -470,11 +470,11 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)/* * Switch the PDA and FPU contexts. */ - prev->usersp =percpu_read (old_rsp); -percpu_write (old_rsp, next->usersp); -percpu_write (current_task, next_p); + prev->usersp =__this_cpu_read (old_rsp); +__this_cpu_write (old_rsp, next->usersp); +__this_cpu_write (current_task, next_p); -percpu_write (kernel_stack, +__this_cpu_write (kernel_stack, (unsigned long )task_stack_page (next_p) +THREAD_SIZE -KERNEL_STACK_OFFSET );
arch/x86/kvm/x86.c +7 -7
@@ -5024,15 +5024,15 @@ static DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);static int kvm_is_in_guest (void ) { -return percpu_read (current_vcpu) !=NULL ; +return this_cpu_read (current_vcpu) !=NULL ; }static int kvm_is_user_mode (void ) {int user_mode =3 ; -if (percpu_read (current_vcpu)) - user_mode = kvm_x86_ops->get_cpl (percpu_read (current_vcpu)); +if (this_cpu_read (current_vcpu)) + user_mode = kvm_x86_ops->get_cpl (this_cpu_read (current_vcpu));return user_mode !=0 ; }
@@ -5041,8 +5041,8 @@ static unsigned long kvm_get_guest_ip(void) {unsigned long ip =0 ; -if (percpu_read (current_vcpu)) - ip =kvm_rip_read (percpu_read (current_vcpu)); +if (this_cpu_read (current_vcpu)) + ip =kvm_rip_read (this_cpu_read (current_vcpu));return ip; }
@@ -5055,13 +5055,13 @@ static struct perf_guest_info_callbacks kvm_guest_cbs = {void kvm_before_handle_nmi (struct kvm_vcpu *vcpu) { -percpu_write (current_vcpu, vcpu); +__this_cpu_write (current_vcpu, vcpu); }EXPORT_SYMBOL_GPL (kvm_before_handle_nmi);void kvm_after_handle_nmi (struct kvm_vcpu *vcpu) { -percpu_write (current_vcpu,NULL ); +__this_cpu_write (current_vcpu,NULL ); }EXPORT_SYMBOL_GPL (kvm_after_handle_nmi);
arch/x86/mm/tlb.c +5 -5
@@ -61,10 +61,10 @@ static DEFINE_PER_CPU_READ_MOSTLY(int, tlb_vector_offset);*/ void leave_mm (int cpu) { -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) +if (__this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_OK )BUG ();cpumask_clear_cpu (cpu, -mm_cpumask (percpu_read (cpu_tlbstate.active_mm))); +mm_cpumask (__this_cpu_read (cpu_tlbstate.active_mm)));load_cr3 (swapper_pg_dir); }EXPORT_SYMBOL_GPL (leave_mm);
@@ -152,8 +152,8 @@ void smp_invalidate_interrupt(struct pt_regs *regs)* BUG(); */ -if (f->flush_mm ==percpu_read (cpu_tlbstate.active_mm)) { -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) { +if (f->flush_mm ==__this_cpu_read (cpu_tlbstate.active_mm)) { +if (__this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) {if (f->flush_va ==TLB_FLUSH_ALL )local_flush_tlb ();else
@@ -322,7 +322,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)static void do_flush_tlb_all (void *info) {__flush_tlb_all (); -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_LAZY ) +if (__this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_LAZY )leave_mm (smp_processor_id ()); }
arch/x86/xen/enlighten.c +3 -3
@@ -776,11 +776,11 @@ static DEFINE_PER_CPU(unsigned long, xen_cr0_value);static unsigned long xen_read_cr0 (void ) { -unsigned long cr0 =percpu_read (xen_cr0_value); +unsigned long cr0 =this_cpu_read (xen_cr0_value);if (unlikely (cr0 ==0 )) { cr0 =native_read_cr0 (); -percpu_write (xen_cr0_value, cr0); +this_cpu_write (xen_cr0_value, cr0); }return cr0;
@@ -790,7 +790,7 @@ static void xen_write_cr0(unsigned long cr0) {struct multicall_space mcs; -percpu_write (xen_cr0_value, cr0); +this_cpu_write (xen_cr0_value, cr0);/* Only pay attention to cr0.TS; everything else is ignored. */
arch/x86/xen/irq.c +4 -4
@@ -26,7 +26,7 @@ static unsigned long xen_save_fl(void)struct vcpu_info *vcpu;unsigned long flags; - vcpu =percpu_read (xen_vcpu); + vcpu =this_cpu_read (xen_vcpu);/* flag has opposite sense of mask */ flags = !vcpu->evtchn_upcall_mask;
@@ -50,7 +50,7 @@ static void xen_restore_fl(unsigned long flags)make sure we're don't switch CPUs between getting the vcpu pointer and updating the mask. */ preempt_disable (); - vcpu =percpu_read (xen_vcpu); + vcpu =this_cpu_read (xen_vcpu); vcpu->evtchn_upcall_mask = flags;preempt_enable_no_resched ();
@@ -72,7 +72,7 @@ static void xen_irq_disable(void)make sure we're don't switch CPUs between getting the vcpu pointer and updating the mask. */ preempt_disable (); -percpu_read (xen_vcpu)->evtchn_upcall_mask =1 ; +this_cpu_read (xen_vcpu)->evtchn_upcall_mask =1 ;preempt_enable_no_resched (); }PV_CALLEE_SAVE_REGS_THUNK (xen_irq_disable);
@@ -86,7 +86,7 @@ static void xen_irq_enable(void)the caller is confused and is trying to re-enable interrupts on an indeterminate processor. */ - vcpu =percpu_read (xen_vcpu); + vcpu =this_cpu_read (xen_vcpu); vcpu->evtchn_upcall_mask =0 ;/* Doesn't matter if we get preempted here, because any
arch/x86/xen/mmu.c +10 -10
@@ -1106,14 +1106,14 @@ static void drop_other_mm_ref(void *info)struct mm_struct *mm = info;struct mm_struct *active_mm; - active_mm =percpu_read (cpu_tlbstate.active_mm); + active_mm =this_cpu_read (cpu_tlbstate.active_mm); -if (active_mm == mm &&percpu_read (cpu_tlbstate.state) !=TLBSTATE_OK ) +if (active_mm == mm &&this_cpu_read (cpu_tlbstate.state) !=TLBSTATE_OK )leave_mm (smp_processor_id ());/* If this cpu still has a stale cr3 reference, then make sure it has been flushed. */ -if (percpu_read (xen_current_cr3) ==__pa (mm->pgd)) +if (this_cpu_read (xen_current_cr3) ==__pa (mm->pgd))load_cr3 (swapper_pg_dir); }
@@ -1220,17 +1220,17 @@ static void __init xen_pagetable_setup_done(pgd_t *base)static void xen_write_cr2 (unsigned long cr2) { -percpu_read (xen_vcpu)->arch.cr2 = cr2; +this_cpu_read (xen_vcpu)->arch.cr2 = cr2; }static unsigned long xen_read_cr2 (void ) { -return percpu_read (xen_vcpu)->arch.cr2; +return this_cpu_read (xen_vcpu)->arch.cr2; }unsigned long xen_read_cr2_direct (void ) { -return percpu_read (xen_vcpu_info.arch.cr2); +return this_cpu_read (xen_vcpu_info.arch.cr2); }static void xen_flush_tlb (void )
@@ -1313,12 +1313,12 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,static unsigned long xen_read_cr3 (void ) { -return percpu_read (xen_cr3); +return this_cpu_read (xen_cr3); }static void set_current_cr3 (void *v) { -percpu_write (xen_current_cr3, (unsigned long )v); +this_cpu_write (xen_current_cr3, (unsigned long )v); }static void __xen_write_cr3 (bool kernel,unsigned long cr3)
@@ -1341,7 +1341,7 @@ static void __xen_write_cr3(bool kernel, unsigned long cr3)xen_extend_mmuext_op (&op);if (kernel) { -percpu_write (xen_cr3, cr3); +this_cpu_write (xen_cr3, cr3);/* Update xen_current_cr3 once the batch has actually been submitted. */
@@ -1357,7 +1357,7 @@ static void xen_write_cr3(unsigned long cr3)/* Update while interrupts are disabled, so its atomic with respect to ipis */ -percpu_write (xen_cr3, cr3); +this_cpu_write (xen_cr3, cr3);__xen_write_cr3 (true, cr3);
arch/x86/xen/multicalls.h +1 -1
@@ -47,7 +47,7 @@ static inline void xen_mc_issue(unsigned mode)xen_mc_flush ();/* restore flags saved in xen_mc_batch */ -local_irq_restore (percpu_read (xen_mc_irq_flags)); +local_irq_restore (this_cpu_read (xen_mc_irq_flags)); }/* Set up a callback to be called when the current batch is flushed */
arch/x86/xen/smp.c +1 -1
@@ -76,7 +76,7 @@ static void __cpuinit cpu_bringup(void)xen_setup_cpu_clockevents ();set_cpu_online (cpu, true); -percpu_write (cpu_state,CPU_ONLINE ); +this_cpu_write (cpu_state,CPU_ONLINE );wmb ();/* We can take interrupts now: we're officially "up". */
include/linux/percpu.h +0 -53
59 lines not on this page · raw message
include/linux/topology.h +2 -2
@@ -235,7 +235,7 @@ static inline int cpu_to_node(int cpu)#ifndef set_numa_nodestatic inline void set_numa_node (int node) { -percpu_write (numa_node, node); +__this_cpu_write (numa_node, node); }#endif
@@ -270,7 +270,7 @@ DECLARE_PER_CPU(int, _numa_mem_);#ifndef set_numa_memstatic inline void set_numa_mem (int node) { -percpu_write (_numa_mem_, node); +__this_cpu_write (_numa_mem_, node); }#endif
net/netfilter/xt_TEE.c +6 -6
@@ -90,7 +90,7 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)const struct xt_tee_tginfo *info = par->targinfo;struct iphdr *iph; -if (percpu_read (tee_active)) +if (__this_cpu_read (tee_active))return XT_CONTINUE ;/* * Copy the skb, and route the copy. Will later return %XT_CONTINUE for
@@ -127,9 +127,9 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)ip_send_check (iph);if (tee_tg_route4 (skb, info)) { -percpu_write (tee_active, true); +__this_cpu_write (tee_active, true);ip_local_out (skb); -percpu_write (tee_active, false); +__this_cpu_write (tee_active, false); }else {kfree_skb (skb); }
@@ -170,7 +170,7 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par) {const struct xt_tee_tginfo *info = par->targinfo; -if (percpu_read (tee_active)) +if (__this_cpu_read (tee_active))return XT_CONTINUE ; skb =pskb_copy (skb,GFP_ATOMIC );if (skb ==NULL )
@@ -188,9 +188,9 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par) --iph->hop_limit; }if (tee_tg_route6 (skb, info)) { -percpu_write (tee_active, true); +__this_cpu_write (tee_active, true);ip6_local_out (skb); -percpu_write (tee_active, false); +__this_cpu_write (tee_active, false); }else {kfree_skb (skb); }
net/socket.c +2 -2
@@ -479,7 +479,7 @@ static struct socket *sock_alloc(void) inode->i_uid =current_fsuid (); inode->i_gid =current_fsgid (); -percpu_add (sockets_in_use,1 ); +__this_cpu_add (sockets_in_use,1 );return sock; }
@@ -522,7 +522,7 @@ void sock_release(struct socket *sock)if (rcu_dereference_protected (sock->wq,1 )->fasync_list)printk (KERN_ERR "sock_release: fasync list not empty!\n" ); -percpu_sub (sockets_in_use,1 ); +__this_cpu_sub (sockets_in_use,1 );if (!sock->file) {iput (SOCK_INODE (sock));return ;
Signature · 1 line
1.7.5.1
Quoted 25 lines · Alex,Shi
Quoted 3 lines
Well do the obvious (where you can see that interrupts or preemption are disabled in the function or in a calling function). Leave the rest as is and provide separate patches for them?
Thanks for comments! I initialized the patch as following accordingly, And cc to more maintainers for review. I checked all code except xen/kvm part, totally a idiot for them.
==========
Since
percpu_xxx()serial functions are duplicate withthis_cpu_xxx(). Removingpercpu_xxx()definition and replacing them bythis_cpu_xxx()in code.And further more, as Christoph Lameter's requirement, I try to use
__this_cpu_xxto replacethis_cpu_xxxif it is in preempt safe scenario. The preempt safe scenarios include:
1, in irq/softirq/nmi handler
2, protected bypreempt_disable
3, protected byspin_lock
4, if the code context imply that it is preempt safe, like the code is follows or be followed a preempt safe code.
Quoted 3 lines
I left the xen/kvm part code unchanged, since no any idea of them.
All of the kvm usage is in preemption disabled contexts.
Signature · 2 lines
I have a truly marvellous patch that fixes the bug which this signature is too narrow to contain.
Quoted 3 lines
All of the kvm usage is in preemption disabled contexts.
Thanks! So would you like to ack this patch?
===
Quoted 1 line
From
8ca7e6e40d75eab598c4fb4f8d555299b747adfaMon Sep 17 00:00:00 2001
From: Alex Shi <[email protected]>
Date: Thu, 20 Oct 2011 10:31:51 +0800
Subject: [PATCH 2/2] Change this_cpu_xxx to __this_cpu_xxx funcs
According to Avi, all of this_cpu_xx used in preempt disable context, So change them to __this_cpu_xxx that without abundant preempt protection.
Signed-off-by Alex Shi <[email protected]>
--- arch/x86/kvm/x86.c | 10 +++++----- 1 files changed, 5 insertions(+), 5 deletions(-)
arch/x86/kvm/x86.c +5 -5
@@ -5024,15 +5024,15 @@ static DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);static int kvm_is_in_guest (void ) { -return this_cpu_read (current_vcpu) !=NULL ; +return __this_cpu_read (current_vcpu) !=NULL ; }static int kvm_is_user_mode (void ) {int user_mode =3 ; -if (this_cpu_read (current_vcpu)) - user_mode = kvm_x86_ops->get_cpl (this_cpu_read (current_vcpu)); +if (__this_cpu_read (current_vcpu)) + user_mode = kvm_x86_ops->get_cpl (__this_cpu_read (current_vcpu));return user_mode !=0 ; }
@@ -5041,8 +5041,8 @@ static unsigned long kvm_get_guest_ip(void) {unsigned long ip =0 ; -if (this_cpu_read (current_vcpu)) - ip =kvm_rip_read (this_cpu_read (current_vcpu)); +if (__this_cpu_read (current_vcpu)) + ip =kvm_rip_read (__this_cpu_read (current_vcpu));return ip; }
Signature · 1 line
1.6.3.3
PATCH 2/3 kvm: use this_cpu_xxx replace percpu_xxx funcs
In reply to Avi Kivitypercpu_xxx funcs are duplicated with this_cpu_xxx funcs, so replace them for further code clean up.
And in preempt safe scenario, __this_cpu_xxx funcs has a bit better performance since __this_cpu_xxx has no redundant preempt_disable()
Signed-off-by Alex Shi <[email protected]>
--- arch/x86/kvm/x86.c | 14 +++++++------- 1 files changed, 7 insertions(+), 7 deletions(-)
arch/x86/kvm/x86.c +7 -7
@@ -5024,15 +5024,15 @@ static DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);static int kvm_is_in_guest (void ) { -return percpu_read (current_vcpu) !=NULL ; +return __this_cpu_read (current_vcpu) !=NULL ; }static int kvm_is_user_mode (void ) {int user_mode =3 ; -if (percpu_read (current_vcpu)) - user_mode = kvm_x86_ops->get_cpl (percpu_read (current_vcpu)); +if (__this_cpu_read (current_vcpu)) + user_mode = kvm_x86_ops->get_cpl (__this_cpu_read (current_vcpu));return user_mode !=0 ; }
@@ -5041,8 +5041,8 @@ static unsigned long kvm_get_guest_ip(void) {unsigned long ip =0 ; -if (percpu_read (current_vcpu)) - ip =kvm_rip_read (percpu_read (current_vcpu)); +if (__this_cpu_read (current_vcpu)) + ip =kvm_rip_read (__this_cpu_read (current_vcpu));return ip; }
@@ -5055,13 +5055,13 @@ static struct perf_guest_info_callbacks kvm_guest_cbs = {void kvm_before_handle_nmi (struct kvm_vcpu *vcpu) { -percpu_write (current_vcpu, vcpu); +__this_cpu_write (current_vcpu, vcpu); }EXPORT_SYMBOL_GPL (kvm_before_handle_nmi);void kvm_after_handle_nmi (struct kvm_vcpu *vcpu) { -percpu_write (current_vcpu,NULL ); +__this_cpu_write (current_vcpu,NULL ); }EXPORT_SYMBOL_GPL (kvm_after_handle_nmi);
Signature · 1 line
1.6.3.3
Re: [PATCH 2/3] kvm: use this_cpu_xxx replace percpu_xxx funcs
Quoted 6 lines · Alex,Shi
percpu_xxxfuncs are duplicated withthis_cpu_xxxfuncs, so replace them for further code clean up.And in preempt safe scenario,
__this_cpu_xxxfuncs has a bit better performance since__this_cpu_xxxhas no redundantpreempt_disable()
Avi:
Would you like to give some comments of this?
Quoted 55 lines
Signed-off-by Alex Shi <[email protected]>--- arch/x86/kvm/x86.c | 14 +++++++------- 1 files changed, 7 insertions(+), 7 deletions(-)arch/x86/kvm/x86.c +7
-7@@ -5024,15 +5024,15 @@ static DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);static int kvm_is_in_guest (void ) { -return percpu_read (current_vcpu) !=NULL ; +return __this_cpu_read (current_vcpu) !=NULL ; }static int kvm_is_user_mode (void ) {int user_mode =3 ; -if (percpu_read (current_vcpu)) - user_mode = kvm_x86_ops->get_cpl (percpu_read (current_vcpu)); +if (__this_cpu_read (current_vcpu)) + user_mode = kvm_x86_ops->get_cpl (__this_cpu_read (current_vcpu));return user_mode !=0 ; }@@ -5041,8 +5041,8 @@ static unsigned long kvm_get_guest_ip(void) {unsigned long ip =0 ; -if (percpu_read (current_vcpu)) - ip =kvm_rip_read (percpu_read (current_vcpu)); +if (__this_cpu_read (current_vcpu)) + ip =kvm_rip_read (__this_cpu_read (current_vcpu));return ip; }@@ -5055,13 +5055,13 @@ static struct perf_guest_info_callbacks kvm_guest_cbs = {void kvm_before_handle_nmi (struct kvm_vcpu *vcpu) { -percpu_write (current_vcpu, vcpu); +__this_cpu_write (current_vcpu, vcpu); }EXPORT_SYMBOL_GPL (kvm_before_handle_nmi);void kvm_after_handle_nmi (struct kvm_vcpu *vcpu) { -percpu_write (current_vcpu,NULL ); +__this_cpu_write (current_vcpu,NULL ); }EXPORT_SYMBOL_GPL (kvm_after_handle_nmi);
Avi Kivity
Acked-byRe: [PATCH 2/3] kvm: use this_cpu_xxx replace percpu_xxx funcs
Quoted 11 lines · Alex,Shi
Quoted 6 lines · Alex,Shi
percpu_xxxfuncs are duplicated withthis_cpu_xxxfuncs, so replace them for further code clean up.And in preempt safe scenario,
__this_cpu_xxxfuncs has a bit better performance since__this_cpu_xxxhas no redundantpreempt_disable()Avi:
Would you like to give some comments of this?
Sorry, was travelling:
Acked-by Avi Kivity <[email protected]>
Signature · 2 lines
I have a truly marvellous patch that fixes the bug which this signature is too narrow to contain.
Re: [PATCH 2/3] kvm: use this_cpu_xxx replace percpu_xxx funcs
Quoted 9 lines · Avi Kivity
Quoted 8 lines · Alex,Shi
Quoted 6 lines · Alex,Shi
percpu_xxxfuncs are duplicated withthis_cpu_xxxfuncs, so replace them for further code clean up.And in preempt safe scenario,
__this_cpu_xxxfuncs has a bit better performance since__this_cpu_xxxhas no redundantpreempt_disable()
Quoted 8 lines
Quoted 3 lines
Avi:
Would you like to give some comments of this?Sorry, was travelling:
Acked-by Avi Kivity <[email protected]>
And this one, picking up or comments are all appreciated. :)
Re: [PATCH 2/3] kvm: use this_cpu_xxx replace percpu_xxx funcs
Quoted 14 lines · Alex,Shi
Quoted 13 lines · Avi Kivity
Quoted 11 lines · Alex,Shi
Quoted 6 lines · Alex,Shi
percpu_xxxfuncs are duplicated withthis_cpu_xxxfuncs, so replace them for further code clean up.And in preempt safe scenario,
__this_cpu_xxxfuncs has a bit better performance since__this_cpu_xxxhas no redundantpreempt_disable()Avi:
Would you like to give some comments of this?
Quoted 6 lines
Quoted 4 lines
Sorry, was travelling:
Acked-by Avi Kivity <[email protected]>And this one, picking up or comments are all appreciated. :)
Just to be clear, you want this applied in kvm.git?
Signature · 1 line
error compiling committee.c: too many arguments to function
Re: [PATCH 2/3] kvm: use this_cpu_xxx replace percpu_xxx funcs
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 10 lines
Quoted 6 lines
percpu_xxxfuncs are duplicated withthis_cpu_xxxfuncs, so replace them for further code clean up.And in preempt safe scenario,
__this_cpu_xxxfuncs has a bit better performance since__this_cpu_xxxhas no redundantpreempt_disable()Avi:
Would you like to give some comments of this?
Quoted 9 lines
Quoted 6 lines
Quoted 4 lines
Sorry, was travelling:
Acked-by Avi Kivity <[email protected]>And this one, picking up or comments are all appreciated. :)
Just to be clear, you want this applied in kvm.git?
Glad to be there! :)
Re: [PATCH 2/3] kvm: use this_cpu_xxx replace percpu_xxx funcs
In reply to Avi KivityQuoted 8 lines
Quoted 5 lines
Quoted 3 lines
Acked-by Avi Kivity <[email protected]>And this one, picking up or comments are all appreciated. :)
Just to be clear, you want this applied in kvm.git?
Thanks Avi!
I saw it is in your 3.3 submit list.
Quoted 3 lines · Alex,Shi
Thanks for comments! I initialized the patch as following accordingly, And cc to more maintainers for review. I checked all code except xen/kvm part, totally a idiot for them.
Alex,Shi
Acked-by Christoph LameterQuoted 2 lines
Thanks, Christoph!
and resend for code style problem correction.
========
Quoted 1 line
From
b8d595063259a636778b14c43f3cf024e908fd25Mon Sep 17 00:00:00 2001
From: Alex Shi <[email protected]>
Date: Thu, 20 Oct 2011 10:25:50 +0800
Subject: [PATCH 1/2] Code clean up for percpu_xxx() functions
Since percpu_xxx() serial functions are duplicate with this_cpu_xxx(). Removing percpu_xxx() definition and replacing them by this_cpu_xxx() in code.
And further more, as Christoph Lameter's requirement, I try to use __this_cpu_xx to replace this_cpu_xxx if it is in preempt safe scenario. The preempt safe scenarios include:
1, in irq/softirq/nmi handler
2, protected by preempt_disable
3, protected by spin_lock
4, if the code context imply that it is preempt safe, like the code is follows or be followed a preempt safe code.
I left the xen/kvm part code unchanged, since no any idea of them.
BTW, In fact, this_cpu_xxx are same as __this_cpu_xxx since all funcs implement in a single instruction for x86 machine. But it maybe different for other platforms, so, doing this distinguish is helpful for other platforms' performance.
Signed-off-by Alex Shi <[email protected]>
--- arch/x86/include/asm/current.h | 2 +- arch/x86/include/asm/hardirq.h | 9 +++-- arch/x86/include/asm/irq_regs.h | 4 +- arch/x86/include/asm/mmu_context.h | 12 ++++---- arch/x86/include/asm/percpu.h | 24 ++++++--------- arch/x86/include/asm/smp.h | 4 +- arch/x86/include/asm/stackprotector.h | 4 +- arch/x86/include/asm/thread_info.h | 2 +- arch/x86/include/asm/tlbflush.h | 4 +- arch/x86/kernel/cpu/common.c | 2 +- arch/x86/kernel/cpu/mcheck/mce.c | 4 +- arch/x86/kernel/cpu/perf_event.c | 4 +- arch/x86/kernel/paravirt.c | 12 ++++---- arch/x86/kernel/process_32.c | 2 +- arch/x86/kernel/process_64.c | 12 ++++---- arch/x86/kvm/x86.c | 14 ++++---- arch/x86/mm/tlb.c | 10 +++--- arch/x86/xen/enlighten.c | 6 ++-- arch/x86/xen/irq.c | 8 ++-- arch/x86/xen/mmu.c | 20 ++++++------ arch/x86/xen/multicalls.h | 2 +- arch/x86/xen/smp.c | 2 +- include/linux/percpu.h | 53 --------------------------------- include/linux/topology.h | 4 +- net/netfilter/xt_TEE.c | 12 ++++---- net/socket.c | 4 +- 26 files changed, 90 insertions(+), 146 deletions(-)
arch/x86/include/asm/current.h +1 -1
@@ -11,7 +11,7 @@ DECLARE_PER_CPU(struct task_struct *, current_task); static __always_inline struct task_struct *get_current(void) { -return percpu_read_stable (current_task); +return this_cpu_read_stable (current_task); }#define current get_current()
arch/x86/include/asm/hardirq.h +5 -4
@@ -35,14 +35,15 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);#define __ARCH_IRQ_STAT -#define inc_irq_stat (member) percpu_inc(irq_stat.member) +#define inc_irq_stat (member) __this_cpu_inc(irq_stat.member) -#define local_softirq_pending () percpu_read(irq_stat.__softirq_pending) +#define local_softirq_pending () __this_cpu_read(irq_stat.__softirq_pending)#define __ARCH_SET_SOFTIRQ_PENDING -#define set_softirq_pending (x) percpu_write(irq_stat.__softirq_pending, (x)) -#define or_softirq_pending (x) percpu_or(irq_stat.__softirq_pending, (x)) +#define set_softirq_pending (x) \ + __this_cpu_write(irq_stat.__softirq_pending, (x)) +#define or_softirq_pending (x) __this_cpu_or(irq_stat.__softirq_pending, (x))extern void ack_bad_irq (unsigned int irq);
arch/x86/include/asm/irq_regs.h +2 -2
@@ -15,7 +15,7 @@ DECLARE_PER_CPU(struct pt_regs *, irq_regs);static inline struct pt_regs *get_irq_regs (void ) { -return percpu_read (irq_regs); +return __this_cpu_read (irq_regs); }static inline struct pt_regs *set_irq_regs (struct pt_regs *new_regs)
@@ -23,7 +23,7 @@ static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)struct pt_regs *old_regs; old_regs =get_irq_regs (); -percpu_write (irq_regs, new_regs); +__this_cpu_write (irq_regs, new_regs);return old_regs; }
arch/x86/include/asm/mmu_context.h +6 -6
@@ -25,8 +25,8 @@ void destroy_context(struct mm_struct *mm);static inline void enter_lazy_tlb (struct mm_struct *mm,struct task_struct *tsk) {#ifdef CONFIG_SMP -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) -percpu_write (cpu_tlbstate.state,TLBSTATE_LAZY ); +if (__this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) +__this_cpu_write (cpu_tlbstate.state,TLBSTATE_LAZY );#endif }
@@ -37,8 +37,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,if (likely (prev != next)) {#ifdef CONFIG_SMP -percpu_write (cpu_tlbstate.state,TLBSTATE_OK ); -percpu_write (cpu_tlbstate.active_mm, next); +__this_cpu_write (cpu_tlbstate.state,TLBSTATE_OK ); +__this_cpu_write (cpu_tlbstate.active_mm, next);#endif cpumask_set_cpu (cpu,mm_cpumask (next));
@@ -56,8 +56,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, }#ifdef CONFIG_SMP else { -percpu_write (cpu_tlbstate.state,TLBSTATE_OK ); -BUG_ON (percpu_read (cpu_tlbstate.active_mm) != next); +__this_cpu_write (cpu_tlbstate.state,TLBSTATE_OK ); +BUG_ON (__this_cpu_read (cpu_tlbstate.active_mm) != next);if (!cpumask_test_and_set_cpu (cpu,mm_cpumask (next))) {/* We were in lazy tlb mode and leave_mm disabled
arch/x86/include/asm/percpu.h +10 -14
@@ -46,7 +46,7 @@#ifdef CONFIG_SMP #define __percpu_prefix "%%"__stringify(__percpu_seg)":" -#define __my_cpu_offset percpu_read(this_cpu_off) +#define __my_cpu_offset __this_cpu_read(this_cpu_off)/* * Compared to the generic __my_cpu_offset version, the following
@@ -351,23 +351,15 @@ do { \ })/* -* percpu_read() makes gcc load the percpu variable every time it is -* accessed while percpu_read_stable() allows the value to be cached. -* percpu_read_stable() is more efficient and can be used if its value +* this_cpu_read() makes gcc load the percpu variable every time it is +* accessed while this_cpu_read_stable() allows the value to be cached. +* this_cpu_read_stable() is more efficient and can be used if its value * is guaranteed to be valid across cpus. The current users include * get_current() and get_thread_info() both of which are actually * per-thread variables implemented as per-cpu variables and thus * stable for the duration of the respective task. */ -#define percpu_read (var) percpu_from_op("mov", var, "m" (var)) -#define percpu_read_stable (var) percpu_from_op("mov", var, "p" (&(var))) -#define percpu_write (var, val) percpu_to_op("mov", var, val) -#define percpu_add (var, val) percpu_add_op(var, val) -#define percpu_sub (var, val) percpu_add_op(var, -(val)) -#define percpu_and (var, val) percpu_to_op("and", var, val) -#define percpu_or (var, val) percpu_to_op("or", var, val) -#define percpu_xor (var, val) percpu_to_op("xor", var, val) -#define percpu_inc (var) percpu_unary_op("inc", var) +#define this_cpu_read_stable (var) percpu_from_op("mov", var, "p" (&(var)))#define __this_cpu_read_1 (pcp) percpu_from_op("mov", (pcp), "m"(pcp))#define __this_cpu_read_2 (pcp) percpu_from_op("mov", (pcp), "m"(pcp))
@@ -551,7 +543,11 @@ static __always_inline int x86_this_cpu_constant_test_bit(unsigned int nr, {unsigned long __percpu *a = (unsigned long *)addr + nr /BITS_PER_LONG ; -return ((1UL << (nr %BITS_PER_LONG )) &percpu_read (*a)) !=0 ; +#ifdef CONFIG_X86_64 +return ((1UL << (nr %BITS_PER_LONG )) &__this_cpu_read_8 (*a)) !=0 ; +#else +return ((1UL << (nr %BITS_PER_LONG )) &__this_cpu_read_4 (*a)) !=0 ; +#endif } static inline int x86_this_cpu_variable_test_bit(int nr,
arch/x86/include/asm/smp.h +2 -2
@@ -188,11 +188,11 @@ extern unsigned disabled_cpus __cpuinitdata;* from the initial startup. We map APIC_BASE very early in page_setup(), * so this is correct in the x86 case. */ -#define raw_smp_processor_id () (percpu_read(cpu_number)) +#define raw_smp_processor_id () (this_cpu_read(cpu_number))extern int safe_smp_processor_id (void );#elif defined(CONFIG_X86_64_SMP ) -#define raw_smp_processor_id () (percpu_read(cpu_number)) +#define raw_smp_processor_id () (this_cpu_read(cpu_number))#define stack_smp_processor_id () \ ({ \
arch/x86/include/asm/stackprotector.h +2 -2
@@ -76,9 +76,9 @@ static __always_inline void boot_init_stack_canary(void) current->stack_canary = canary;#ifdef CONFIG_X86_64 -percpu_write (irq_stack_union.stack_canary, canary); +__this_cpu_write (irq_stack_union.stack_canary, canary);#else -percpu_write (stack_canary.canary, canary); +__this_cpu_write (stack_canary.canary, canary);#endif }
arch/x86/include/asm/thread_info.h +1 -1
@@ -219,7 +219,7 @@ DECLARE_PER_CPU(unsigned long, kernel_stack);static inline struct thread_info *current_thread_info (void ) {struct thread_info *ti; - ti = (void *)(percpu_read_stable (kernel_stack) + + ti = (void *)(this_cpu_read_stable (kernel_stack) +KERNEL_STACK_OFFSET -THREAD_SIZE );return ti; }
arch/x86/include/asm/tlbflush.h +2 -2
@@ -156,8 +156,8 @@ DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate);static inline void reset_lazy_tlbstate (void ) { -percpu_write (cpu_tlbstate.state,0 ); -percpu_write (cpu_tlbstate.active_mm, &init_mm); +__this_cpu_write (cpu_tlbstate.state,0 ); +__this_cpu_write (cpu_tlbstate.active_mm, &init_mm); }#endif /* SMP */
arch/x86/kernel/cpu/common.c +1 -1
@@ -1158,7 +1158,7 @@ void __cpuinit cpu_init(void) oist = &per_cpu (orig_ist, cpu);#ifdef CONFIG_NUMA -if (cpu !=0 &&percpu_read (numa_node) ==0 && +if (cpu !=0 &&__this_cpu_read (numa_node) ==0 &&early_cpu_to_node (cpu) !=NUMA_NO_NODE )set_numa_node (early_cpu_to_node (cpu));#endif
arch/x86/kernel/cpu/mcheck/mce.c +2 -2
@@ -512,7 +512,7 @@ void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)struct mce m;int i; -percpu_inc (mce_poll_count); +__this_cpu_inc (mce_poll_count);mce_gather_info (&m,NULL );
@@ -906,7 +906,7 @@ void do_machine_check(struct pt_regs *regs, long error_code)atomic_inc (&mce_entry); -percpu_inc (mce_exception_count); +__this_cpu_inc (mce_exception_count);if (notify_die (DIE_NMI ,"machine check" , regs, error_code,18 ,SIGKILL ) ==NOTIFY_STOP )
arch/x86/kernel/cpu/perf_event.c +2 -2
@@ -1389,7 +1389,7 @@ perf_event_nmi_handler(struct notifier_block *self,case DIE_NMI :break ;case DIE_NMIUNKNOWN : - this_nmi =percpu_read (irq_stat.__nmi_count); + this_nmi =__this_cpu_read (irq_stat.__nmi_count);if (this_nmi !=__this_cpu_read (pmu_nmi.marked))/* let the kernel handle the unknown nmi */ return NOTIFY_DONE ;
@@ -1409,7 +1409,7 @@ perf_event_nmi_handler(struct notifier_block *self,if (!handled)return NOTIFY_DONE ; - this_nmi =percpu_read (irq_stat.__nmi_count); + this_nmi =__this_cpu_read (irq_stat.__nmi_count); if ((handled > 1) || /* the next nmi could be a back-to-back nmi */ ((__this_cpu_read(pmu_nmi.marked) == this_nmi) &&
arch/x86/kernel/paravirt.c +6 -6
@@ -239,16 +239,16 @@ static DEFINE_PER_CPU(enum paravirt_lazy_mode, paravirt_lazy_mode) = PARAVIRT_LAstatic inline void enter_lazy (enum paravirt_lazy_mode mode) { -BUG_ON (percpu_read (paravirt_lazy_mode) !=PARAVIRT_LAZY_NONE ); +BUG_ON (__this_cpu_read (paravirt_lazy_mode) !=PARAVIRT_LAZY_NONE ); -percpu_write (paravirt_lazy_mode, mode); +__this_cpu_write (paravirt_lazy_mode, mode); }static void leave_lazy (enum paravirt_lazy_mode mode) { -BUG_ON (percpu_read (paravirt_lazy_mode) != mode); +BUG_ON (__this_cpu_read (paravirt_lazy_mode) != mode); -percpu_write (paravirt_lazy_mode,PARAVIRT_LAZY_NONE ); +__this_cpu_write (paravirt_lazy_mode,PARAVIRT_LAZY_NONE ); }void paravirt_enter_lazy_mmu (void )
@@ -265,7 +265,7 @@ void paravirt_start_context_switch(struct task_struct *prev) {BUG_ON (preemptible ()); -if (percpu_read (paravirt_lazy_mode) ==PARAVIRT_LAZY_MMU ) { +if (__this_cpu_read (paravirt_lazy_mode) ==PARAVIRT_LAZY_MMU ) {arch_leave_lazy_mmu_mode ();set_ti_thread_flag (task_thread_info (prev),TIF_LAZY_MMU_UPDATES ); }
@@ -287,7 +287,7 @@ enum paravirt_lazy_mode paravirt_get_lazy_mode(void)if (in_interrupt ())return PARAVIRT_LAZY_NONE ; -return percpu_read (paravirt_lazy_mode); +return __this_cpu_read (paravirt_lazy_mode); }void arch_flush_lazy_mmu_mode (void )
arch/x86/kernel/process_32.c +1 -1
@@ -373,7 +373,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)if (prev->gs | next->gs)lazy_load_gs (next->gs); -percpu_write (current_task, next_p); +__this_cpu_write (current_task, next_p);return prev_p; }
arch/x86/kernel/process_64.c +6 -6
@@ -73,7 +73,7 @@ EXPORT_SYMBOL_GPL(idle_notifier_unregister);void enter_idle (void ) { -percpu_write (is_idle,1 ); +__this_cpu_write (is_idle,1 );atomic_notifier_call_chain (&idle_notifier,IDLE_START ,NULL ); }
@@ -336,7 +336,7 @@ start_thread_common(struct pt_regs *regs, unsigned long new_ip,load_gs_index (0 ); regs->ip = new_ip; regs->sp = new_sp; -percpu_write (old_rsp, new_sp); +this_cpu_write (old_rsp, new_sp); regs->cs = _cs; regs->ss = _ss; regs->flags =X86_EFLAGS_IF ;
@@ -470,11 +470,11 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)/* * Switch the PDA and FPU contexts. */ - prev->usersp =percpu_read (old_rsp); -percpu_write (old_rsp, next->usersp); -percpu_write (current_task, next_p); + prev->usersp =__this_cpu_read (old_rsp); +__this_cpu_write (old_rsp, next->usersp); +__this_cpu_write (current_task, next_p); -percpu_write (kernel_stack, +__this_cpu_write (kernel_stack, (unsigned long )task_stack_page (next_p) +THREAD_SIZE -KERNEL_STACK_OFFSET );
arch/x86/kvm/x86.c +7 -7
@@ -5024,15 +5024,15 @@ static DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);static int kvm_is_in_guest (void ) { -return percpu_read (current_vcpu) !=NULL ; +return this_cpu_read (current_vcpu) !=NULL ; }static int kvm_is_user_mode (void ) {int user_mode =3 ; -if (percpu_read (current_vcpu)) - user_mode = kvm_x86_ops->get_cpl (percpu_read (current_vcpu)); +if (this_cpu_read (current_vcpu)) + user_mode = kvm_x86_ops->get_cpl (this_cpu_read (current_vcpu));return user_mode !=0 ; }
@@ -5041,8 +5041,8 @@ static unsigned long kvm_get_guest_ip(void) {unsigned long ip =0 ; -if (percpu_read (current_vcpu)) - ip =kvm_rip_read (percpu_read (current_vcpu)); +if (this_cpu_read (current_vcpu)) + ip =kvm_rip_read (this_cpu_read (current_vcpu));return ip; }
@@ -5055,13 +5055,13 @@ static struct perf_guest_info_callbacks kvm_guest_cbs = {void kvm_before_handle_nmi (struct kvm_vcpu *vcpu) { -percpu_write (current_vcpu, vcpu); +__this_cpu_write (current_vcpu, vcpu); }EXPORT_SYMBOL_GPL (kvm_before_handle_nmi);void kvm_after_handle_nmi (struct kvm_vcpu *vcpu) { -percpu_write (current_vcpu,NULL ); +__this_cpu_write (current_vcpu,NULL ); }EXPORT_SYMBOL_GPL (kvm_after_handle_nmi);
arch/x86/mm/tlb.c +5 -5
@@ -61,10 +61,10 @@ static DEFINE_PER_CPU_READ_MOSTLY(int, tlb_vector_offset);*/ void leave_mm (int cpu) { -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) +if (__this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_OK )BUG ();cpumask_clear_cpu (cpu, -mm_cpumask (percpu_read (cpu_tlbstate.active_mm))); +mm_cpumask (__this_cpu_read (cpu_tlbstate.active_mm)));load_cr3 (swapper_pg_dir); }EXPORT_SYMBOL_GPL (leave_mm);
@@ -152,8 +152,8 @@ void smp_invalidate_interrupt(struct pt_regs *regs)* BUG(); */ -if (f->flush_mm ==percpu_read (cpu_tlbstate.active_mm)) { -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) { +if (f->flush_mm ==__this_cpu_read (cpu_tlbstate.active_mm)) { +if (__this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) {if (f->flush_va ==TLB_FLUSH_ALL )local_flush_tlb ();else
@@ -322,7 +322,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)static void do_flush_tlb_all (void *info) {__flush_tlb_all (); -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_LAZY ) +if (__this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_LAZY )leave_mm (smp_processor_id ()); }
arch/x86/xen/enlighten.c +3 -3
@@ -776,11 +776,11 @@ static DEFINE_PER_CPU(unsigned long, xen_cr0_value);static unsigned long xen_read_cr0 (void ) { -unsigned long cr0 =percpu_read (xen_cr0_value); +unsigned long cr0 =this_cpu_read (xen_cr0_value);if (unlikely (cr0 ==0 )) { cr0 =native_read_cr0 (); -percpu_write (xen_cr0_value, cr0); +this_cpu_write (xen_cr0_value, cr0); }return cr0;
@@ -790,7 +790,7 @@ static void xen_write_cr0(unsigned long cr0) {struct multicall_space mcs; -percpu_write (xen_cr0_value, cr0); +this_cpu_write (xen_cr0_value, cr0);/* Only pay attention to cr0.TS; everything else is ignored. */
arch/x86/xen/irq.c +4 -4
@@ -26,7 +26,7 @@ static unsigned long xen_save_fl(void)struct vcpu_info *vcpu;unsigned long flags; - vcpu =percpu_read (xen_vcpu); + vcpu =this_cpu_read (xen_vcpu);/* flag has opposite sense of mask */ flags = !vcpu->evtchn_upcall_mask;
@@ -50,7 +50,7 @@ static void xen_restore_fl(unsigned long flags)make sure we're don't switch CPUs between getting the vcpu pointer and updating the mask. */ preempt_disable (); - vcpu =percpu_read (xen_vcpu); + vcpu =this_cpu_read (xen_vcpu); vcpu->evtchn_upcall_mask = flags;preempt_enable_no_resched ();
@@ -72,7 +72,7 @@ static void xen_irq_disable(void)make sure we're don't switch CPUs between getting the vcpu pointer and updating the mask. */ preempt_disable (); -percpu_read (xen_vcpu)->evtchn_upcall_mask =1 ; +this_cpu_read (xen_vcpu)->evtchn_upcall_mask =1 ;preempt_enable_no_resched (); }PV_CALLEE_SAVE_REGS_THUNK (xen_irq_disable);
@@ -86,7 +86,7 @@ static void xen_irq_enable(void)the caller is confused and is trying to re-enable interrupts on an indeterminate processor. */ - vcpu =percpu_read (xen_vcpu); + vcpu =this_cpu_read (xen_vcpu); vcpu->evtchn_upcall_mask =0 ;/* Doesn't matter if we get preempted here, because any
arch/x86/xen/mmu.c +10 -10
@@ -1106,14 +1106,14 @@ static void drop_other_mm_ref(void *info)struct mm_struct *mm = info;struct mm_struct *active_mm; - active_mm =percpu_read (cpu_tlbstate.active_mm); + active_mm =this_cpu_read (cpu_tlbstate.active_mm); -if (active_mm == mm &&percpu_read (cpu_tlbstate.state) !=TLBSTATE_OK ) +if (active_mm == mm &&this_cpu_read (cpu_tlbstate.state) !=TLBSTATE_OK )leave_mm (smp_processor_id ());/* If this cpu still has a stale cr3 reference, then make sure it has been flushed. */ -if (percpu_read (xen_current_cr3) ==__pa (mm->pgd)) +if (this_cpu_read (xen_current_cr3) ==__pa (mm->pgd))load_cr3 (swapper_pg_dir); }
@@ -1220,17 +1220,17 @@ static void __init xen_pagetable_setup_done(pgd_t *base)static void xen_write_cr2 (unsigned long cr2) { -percpu_read (xen_vcpu)->arch.cr2 = cr2; +this_cpu_read (xen_vcpu)->arch.cr2 = cr2; }static unsigned long xen_read_cr2 (void ) { -return percpu_read (xen_vcpu)->arch.cr2; +return this_cpu_read (xen_vcpu)->arch.cr2; }unsigned long xen_read_cr2_direct (void ) { -return percpu_read (xen_vcpu_info.arch.cr2); +return this_cpu_read (xen_vcpu_info.arch.cr2); }static void xen_flush_tlb (void )
@@ -1313,12 +1313,12 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,static unsigned long xen_read_cr3 (void ) { -return percpu_read (xen_cr3); +return this_cpu_read (xen_cr3); }static void set_current_cr3 (void *v) { -percpu_write (xen_current_cr3, (unsigned long )v); +this_cpu_write (xen_current_cr3, (unsigned long )v); }static void __xen_write_cr3 (bool kernel,unsigned long cr3)
@@ -1341,7 +1341,7 @@ static void __xen_write_cr3(bool kernel, unsigned long cr3)xen_extend_mmuext_op (&op);if (kernel) { -percpu_write (xen_cr3, cr3); +this_cpu_write (xen_cr3, cr3);/* Update xen_current_cr3 once the batch has actually been submitted. */
@@ -1357,7 +1357,7 @@ static void xen_write_cr3(unsigned long cr3)/* Update while interrupts are disabled, so its atomic with respect to ipis */ -percpu_write (xen_cr3, cr3); +this_cpu_write (xen_cr3, cr3);__xen_write_cr3 (true, cr3);
arch/x86/xen/multicalls.h +1 -1
@@ -47,7 +47,7 @@ static inline void xen_mc_issue(unsigned mode)xen_mc_flush ();/* restore flags saved in xen_mc_batch */ -local_irq_restore (percpu_read (xen_mc_irq_flags)); +local_irq_restore (this_cpu_read (xen_mc_irq_flags)); }/* Set up a callback to be called when the current batch is flushed */
arch/x86/xen/smp.c +1 -1
@@ -76,7 +76,7 @@ static void __cpuinit cpu_bringup(void)xen_setup_cpu_clockevents ();set_cpu_online (cpu, true); -percpu_write (cpu_state,CPU_ONLINE ); +this_cpu_write (cpu_state,CPU_ONLINE );wmb ();/* We can take interrupts now: we're officially "up". */
include/linux/percpu.h +0 -53
59 lines not on this page · raw message
include/linux/topology.h +2 -2
@@ -235,7 +235,7 @@ static inline int cpu_to_node(int cpu)#ifndef set_numa_nodestatic inline void set_numa_node (int node) { -percpu_write (numa_node, node); +__this_cpu_write (numa_node, node); }#endif
@@ -270,7 +270,7 @@ DECLARE_PER_CPU(int, _numa_mem_);#ifndef set_numa_memstatic inline void set_numa_mem (int node) { -percpu_write (_numa_mem_, node); +__this_cpu_write (_numa_mem_, node); }#endif
net/netfilter/xt_TEE.c +6 -6
@@ -90,7 +90,7 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)const struct xt_tee_tginfo *info = par->targinfo;struct iphdr *iph; -if (percpu_read (tee_active)) +if (__this_cpu_read (tee_active))return XT_CONTINUE ;/* * Copy the skb, and route the copy. Will later return %XT_CONTINUE for
@@ -127,9 +127,9 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)ip_send_check (iph);if (tee_tg_route4 (skb, info)) { -percpu_write (tee_active, true); +__this_cpu_write (tee_active, true);ip_local_out (skb); -percpu_write (tee_active, false); +__this_cpu_write (tee_active, false); }else {kfree_skb (skb); }
@@ -170,7 +170,7 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par) {const struct xt_tee_tginfo *info = par->targinfo; -if (percpu_read (tee_active)) +if (__this_cpu_read (tee_active))return XT_CONTINUE ; skb =pskb_copy (skb,GFP_ATOMIC );if (skb ==NULL )
@@ -188,9 +188,9 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par) --iph->hop_limit; }if (tee_tg_route6 (skb, info)) { -percpu_write (tee_active, true); +__this_cpu_write (tee_active, true);ip6_local_out (skb); -percpu_write (tee_active, false); +__this_cpu_write (tee_active, false); }else {kfree_skb (skb); }
net/socket.c +2 -2
@@ -479,7 +479,7 @@ static struct socket *sock_alloc(void) inode->i_uid =current_fsuid (); inode->i_gid =current_fsgid (); -percpu_add (sockets_in_use,1 ); +__this_cpu_add (sockets_in_use,1 );return sock; }
@@ -522,7 +522,7 @@ void sock_release(struct socket *sock)if (rcu_dereference_protected (sock->wq,1 )->fasync_list)printk (KERN_ERR "sock_release: fasync list not empty!\n" ); -percpu_sub (sockets_in_use,1 ); +__this_cpu_sub (sockets_in_use,1 );if (!sock->file) {iput (SOCK_INODE (sock));return ;
Signature · 1 line
1.6.3.3
Quoted 5 lines · Alex,Shi
Quoted 1 line
Thanks, Christoph!
and resend for code style problem correction.
Sorry this huge patch brings too many potential bugs.
I ask you a separate patch for the networking part, because I dont want to Ack all the other parts.
net/socket.c static struct socket *sock_alloc(void) · line 482
Quoted 10 lines
net/socket.c +1
-1@@ -479,7 +479,7 @@ static struct socket *sock_alloc(void) inode->i_uid =current_fsuid (); inode->i_gid =current_fsgid (); -percpu_add (sockets_in_use,1 ); +__this_cpu_add (sockets_in_use,1 );
No, we are in process context, you need
this_cpu_add(sockets_in_use, 1);
net/socket.c void sock_release(struct socket *sock) · line 525
Quoted 9 lines
net/socket.c +1
-1return sock; }@@ -522,7 +522,7 @@ void sock_release(struct socket *sock)if (rcu_dereference_protected (sock->wq,1 )->fasync_list)printk (KERN_ERR "sock_release: fasync list not empty!\n" ); -percpu_sub (sockets_in_use,1 ); +__this_cpu_sub (sockets_in_use,1 );
this_cpu_sub();
Quoted 3 lines
net/socket.c +0
-0if (!sock->file) {iput (SOCK_INODE (sock));return ;
Quoted 6 lines
Sorry this huge patch brings too many potential bugs.
I ask you a separate patch for the networking part, because I dont want to Ack all the other parts.
Thanks a lot for review! I split the first patch into 2 according your suggestion. The second one for network here:
Quoted 1 line
From
d5d71127ad22890f6f13fa382736c0df7f5d0231Mon Sep 17 00:00:00 2001
From: Alex Shi <[email protected]>
Date: Thu, 20 Oct 2011 13:44:37 +0800
Subject: [PATCH 2/2] change this_cpu_xxx funcs for preempt safe scenarios
If context is preempt safe, using __this_cpu_xxx to replace this_cpu_xxx have performance benefit, since the later has a abundant preempt_disable instruction.
Signed-off-by Alex Shi <[email protected]>
--- net/netfilter/xt_TEE.c | 12 ++++++------ 1 files changed, 6 insertions(+), 6 deletions(-)
net/netfilter/xt_TEE.c +6 -6
@@ -90,7 +90,7 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)const struct xt_tee_tginfo *info = par->targinfo;struct iphdr *iph; -if (this_cpu_read (tee_active)) +if (__this_cpu_read (tee_active))return XT_CONTINUE ;/* * Copy the skb, and route the copy. Will later return %XT_CONTINUE for
@@ -127,9 +127,9 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)ip_send_check (iph);if (tee_tg_route4 (skb, info)) { -this_cpu_write (tee_active, true); +__this_cpu_write (tee_active, true);ip_local_out (skb); -this_cpu_write (tee_active, false); +__this_cpu_write (tee_active, false); }else {kfree_skb (skb); }
@@ -170,7 +170,7 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par) {const struct xt_tee_tginfo *info = par->targinfo; -if (this_cpu_read (tee_active)) +if (__this_cpu_read (tee_active))return XT_CONTINUE ; skb =pskb_copy (skb,GFP_ATOMIC );if (skb ==NULL )
@@ -188,9 +188,9 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par) --iph->hop_limit; }if (tee_tg_route6 (skb, info)) { -this_cpu_write (tee_active, true); +__this_cpu_write (tee_active, true);ip6_local_out (skb); -this_cpu_write (tee_active, false); +__this_cpu_write (tee_active, false); }else {kfree_skb (skb); }
Signature · 1 line
1.6.3.3
Quoted 8 lines · Alex,Shi
Quoted 5 lines
Sorry this huge patch brings too many potential bugs.
I ask you a separate patch for the networking part, because I dont want to Ack all the other parts.
Thanks a lot for review! I split the first patch into 2 according your suggestion. The second one for network here:
Sorry for too big patch, any other split requirements?
Alex,Shi
Acked-by Christoph LameterQuoted 5 lines
Sorry this huge patch brings too many potential bugs.
I ask you a separate patch for the networking part, because I dont want to Ack all the other parts.
Thanks a lot for review! I split the first patch into 2 according your suggestion. The first one is here:
Quoted 1 line
From
516756ad094c49f8ddc4022157db9a5ce8f86578Mon Sep 17 00:00:00 2001
From: Alex Shi <[email protected]>
Date: Thu, 20 Oct 2011 13:30:11 +0800
Subject: [PATCH 1/2] Code clean up for percpu_xxx() functions
Since percpu_xxx() serial functions are duplicate with this_cpu_xxx(). Removing percpu_xxx() definition and replacing them by this_cpu_xxx() in code.
And further more, as Christoph Lameter's requirement, I try to use __this_cpu_xx to replace this_cpu_xxx if it is in preempt safe scenario. The preempt safe scenarios include:
1, in irq/softirq/nmi handler
2, protected by preempt_disable
3, protected by spin_lock
4, if the code context imply that it is preempt safe, like the code is follows or be followed a preempt safe code.
I left the xen/kvm/network part code unchanged, since no idea of them.
BTW, In fact, this_cpu_xxx are same as __this_cpu_xxx since all funcs implement in a single instruction for x86 machine. But it maybe different for other platforms, so, doing this distinguish is helpful for other platforms' performance.
Signed-off-by Alex Shi <[email protected]>
--- arch/x86/include/asm/current.h | 2 +- arch/x86/include/asm/hardirq.h | 9 +++-- arch/x86/include/asm/irq_regs.h | 4 +- arch/x86/include/asm/mmu_context.h | 12 ++++---- arch/x86/include/asm/percpu.h | 24 ++++++--------- arch/x86/include/asm/smp.h | 4 +- arch/x86/include/asm/stackprotector.h | 4 +- arch/x86/include/asm/thread_info.h | 2 +- arch/x86/include/asm/tlbflush.h | 4 +- arch/x86/kernel/cpu/common.c | 2 +- arch/x86/kernel/cpu/mcheck/mce.c | 4 +- arch/x86/kernel/cpu/perf_event.c | 4 +- arch/x86/kernel/paravirt.c | 12 ++++---- arch/x86/kernel/process_32.c | 2 +- arch/x86/kernel/process_64.c | 12 ++++---- arch/x86/kvm/x86.c | 14 ++++---- arch/x86/mm/tlb.c | 10 +++--- arch/x86/xen/enlighten.c | 6 ++-- arch/x86/xen/irq.c | 8 ++-- arch/x86/xen/mmu.c | 20 ++++++------ arch/x86/xen/multicalls.h | 2 +- arch/x86/xen/smp.c | 2 +- include/linux/percpu.h | 53 --------------------------------- include/linux/topology.h | 4 +- net/netfilter/xt_TEE.c | 12 ++++---- net/socket.c | 4 +- 26 files changed, 90 insertions(+), 146 deletions(-)
arch/x86/include/asm/current.h +1 -1
@@ -11,7 +11,7 @@ DECLARE_PER_CPU(struct task_struct *, current_task); static __always_inline struct task_struct *get_current(void) { -return percpu_read_stable (current_task); +return this_cpu_read_stable (current_task); }#define current get_current()
arch/x86/include/asm/hardirq.h +5 -4
@@ -35,14 +35,15 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);#define __ARCH_IRQ_STAT -#define inc_irq_stat (member) percpu_inc(irq_stat.member) +#define inc_irq_stat (member) __this_cpu_inc(irq_stat.member) -#define local_softirq_pending () percpu_read(irq_stat.__softirq_pending) +#define local_softirq_pending () __this_cpu_read(irq_stat.__softirq_pending)#define __ARCH_SET_SOFTIRQ_PENDING -#define set_softirq_pending (x) percpu_write(irq_stat.__softirq_pending, (x)) -#define or_softirq_pending (x) percpu_or(irq_stat.__softirq_pending, (x)) +#define set_softirq_pending (x) \ + __this_cpu_write(irq_stat.__softirq_pending, (x)) +#define or_softirq_pending (x) __this_cpu_or(irq_stat.__softirq_pending, (x))extern void ack_bad_irq (unsigned int irq);
arch/x86/include/asm/irq_regs.h +2 -2
@@ -15,7 +15,7 @@ DECLARE_PER_CPU(struct pt_regs *, irq_regs);static inline struct pt_regs *get_irq_regs (void ) { -return percpu_read (irq_regs); +return __this_cpu_read (irq_regs); }static inline struct pt_regs *set_irq_regs (struct pt_regs *new_regs)
@@ -23,7 +23,7 @@ static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)struct pt_regs *old_regs; old_regs =get_irq_regs (); -percpu_write (irq_regs, new_regs); +__this_cpu_write (irq_regs, new_regs);return old_regs; }
arch/x86/include/asm/mmu_context.h +6 -6
@@ -25,8 +25,8 @@ void destroy_context(struct mm_struct *mm);static inline void enter_lazy_tlb (struct mm_struct *mm,struct task_struct *tsk) {#ifdef CONFIG_SMP -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) -percpu_write (cpu_tlbstate.state,TLBSTATE_LAZY ); +if (__this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) +__this_cpu_write (cpu_tlbstate.state,TLBSTATE_LAZY );#endif }
@@ -37,8 +37,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,if (likely (prev != next)) {#ifdef CONFIG_SMP -percpu_write (cpu_tlbstate.state,TLBSTATE_OK ); -percpu_write (cpu_tlbstate.active_mm, next); +__this_cpu_write (cpu_tlbstate.state,TLBSTATE_OK ); +__this_cpu_write (cpu_tlbstate.active_mm, next);#endif cpumask_set_cpu (cpu,mm_cpumask (next));
@@ -56,8 +56,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, }#ifdef CONFIG_SMP else { -percpu_write (cpu_tlbstate.state,TLBSTATE_OK ); -BUG_ON (percpu_read (cpu_tlbstate.active_mm) != next); +__this_cpu_write (cpu_tlbstate.state,TLBSTATE_OK ); +BUG_ON (__this_cpu_read (cpu_tlbstate.active_mm) != next);if (!cpumask_test_and_set_cpu (cpu,mm_cpumask (next))) {/* We were in lazy tlb mode and leave_mm disabled
arch/x86/include/asm/percpu.h +10 -14
@@ -46,7 +46,7 @@#ifdef CONFIG_SMP #define __percpu_prefix "%%"__stringify(__percpu_seg)":" -#define __my_cpu_offset percpu_read(this_cpu_off) +#define __my_cpu_offset __this_cpu_read(this_cpu_off)/* * Compared to the generic __my_cpu_offset version, the following
@@ -351,23 +351,15 @@ do { \ })/* -* percpu_read() makes gcc load the percpu variable every time it is -* accessed while percpu_read_stable() allows the value to be cached. -* percpu_read_stable() is more efficient and can be used if its value +* this_cpu_read() makes gcc load the percpu variable every time it is +* accessed while this_cpu_read_stable() allows the value to be cached. +* this_cpu_read_stable() is more efficient and can be used if its value * is guaranteed to be valid across cpus. The current users include * get_current() and get_thread_info() both of which are actually * per-thread variables implemented as per-cpu variables and thus * stable for the duration of the respective task. */ -#define percpu_read (var) percpu_from_op("mov", var, "m" (var)) -#define percpu_read_stable (var) percpu_from_op("mov", var, "p" (&(var))) -#define percpu_write (var, val) percpu_to_op("mov", var, val) -#define percpu_add (var, val) percpu_add_op(var, val) -#define percpu_sub (var, val) percpu_add_op(var, -(val)) -#define percpu_and (var, val) percpu_to_op("and", var, val) -#define percpu_or (var, val) percpu_to_op("or", var, val) -#define percpu_xor (var, val) percpu_to_op("xor", var, val) -#define percpu_inc (var) percpu_unary_op("inc", var) +#define this_cpu_read_stable (var) percpu_from_op("mov", var, "p" (&(var)))#define __this_cpu_read_1 (pcp) percpu_from_op("mov", (pcp), "m"(pcp))#define __this_cpu_read_2 (pcp) percpu_from_op("mov", (pcp), "m"(pcp))
@@ -551,7 +543,11 @@ static __always_inline int x86_this_cpu_constant_test_bit(unsigned int nr, {unsigned long __percpu *a = (unsigned long *)addr + nr /BITS_PER_LONG ; -return ((1UL << (nr %BITS_PER_LONG )) &percpu_read (*a)) !=0 ; +#ifdef CONFIG_X86_64 +return ((1UL << (nr %BITS_PER_LONG )) &__this_cpu_read_8 (*a)) !=0 ; +#else +return ((1UL << (nr %BITS_PER_LONG )) &__this_cpu_read_4 (*a)) !=0 ; +#endif } static inline int x86_this_cpu_variable_test_bit(int nr,
arch/x86/include/asm/smp.h +2 -2
@@ -188,11 +188,11 @@ extern unsigned disabled_cpus __cpuinitdata;* from the initial startup. We map APIC_BASE very early in page_setup(), * so this is correct in the x86 case. */ -#define raw_smp_processor_id () (percpu_read(cpu_number)) +#define raw_smp_processor_id () (this_cpu_read(cpu_number))extern int safe_smp_processor_id (void );#elif defined(CONFIG_X86_64_SMP ) -#define raw_smp_processor_id () (percpu_read(cpu_number)) +#define raw_smp_processor_id () (this_cpu_read(cpu_number))#define stack_smp_processor_id () \ ({ \
arch/x86/include/asm/stackprotector.h +2 -2
@@ -76,9 +76,9 @@ static __always_inline void boot_init_stack_canary(void) current->stack_canary = canary;#ifdef CONFIG_X86_64 -percpu_write (irq_stack_union.stack_canary, canary); +__this_cpu_write (irq_stack_union.stack_canary, canary);#else -percpu_write (stack_canary.canary, canary); +__this_cpu_write (stack_canary.canary, canary);#endif }
arch/x86/include/asm/thread_info.h +1 -1
@@ -219,7 +219,7 @@ DECLARE_PER_CPU(unsigned long, kernel_stack);static inline struct thread_info *current_thread_info (void ) {struct thread_info *ti; - ti = (void *)(percpu_read_stable (kernel_stack) + + ti = (void *)(this_cpu_read_stable (kernel_stack) +KERNEL_STACK_OFFSET -THREAD_SIZE );return ti; }
arch/x86/include/asm/tlbflush.h +2 -2
@@ -156,8 +156,8 @@ DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate);static inline void reset_lazy_tlbstate (void ) { -percpu_write (cpu_tlbstate.state,0 ); -percpu_write (cpu_tlbstate.active_mm, &init_mm); +__this_cpu_write (cpu_tlbstate.state,0 ); +__this_cpu_write (cpu_tlbstate.active_mm, &init_mm); }#endif /* SMP */
arch/x86/kernel/cpu/common.c +1 -1
@@ -1158,7 +1158,7 @@ void __cpuinit cpu_init(void) oist = &per_cpu (orig_ist, cpu);#ifdef CONFIG_NUMA -if (cpu !=0 &&percpu_read (numa_node) ==0 && +if (cpu !=0 &&__this_cpu_read (numa_node) ==0 &&early_cpu_to_node (cpu) !=NUMA_NO_NODE )set_numa_node (early_cpu_to_node (cpu));#endif
arch/x86/kernel/cpu/mcheck/mce.c +2 -2
@@ -512,7 +512,7 @@ void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)struct mce m;int i; -percpu_inc (mce_poll_count); +__this_cpu_inc (mce_poll_count);mce_gather_info (&m,NULL );
@@ -906,7 +906,7 @@ void do_machine_check(struct pt_regs *regs, long error_code)atomic_inc (&mce_entry); -percpu_inc (mce_exception_count); +__this_cpu_inc (mce_exception_count);if (notify_die (DIE_NMI ,"machine check" , regs, error_code,18 ,SIGKILL ) ==NOTIFY_STOP )
arch/x86/kernel/cpu/perf_event.c +2 -2
@@ -1389,7 +1389,7 @@ perf_event_nmi_handler(struct notifier_block *self,case DIE_NMI :break ;case DIE_NMIUNKNOWN : - this_nmi =percpu_read (irq_stat.__nmi_count); + this_nmi =__this_cpu_read (irq_stat.__nmi_count);if (this_nmi !=__this_cpu_read (pmu_nmi.marked))/* let the kernel handle the unknown nmi */ return NOTIFY_DONE ;
@@ -1409,7 +1409,7 @@ perf_event_nmi_handler(struct notifier_block *self,if (!handled)return NOTIFY_DONE ; - this_nmi =percpu_read (irq_stat.__nmi_count); + this_nmi =__this_cpu_read (irq_stat.__nmi_count); if ((handled > 1) || /* the next nmi could be a back-to-back nmi */ ((__this_cpu_read(pmu_nmi.marked) == this_nmi) &&
arch/x86/kernel/paravirt.c +6 -6
@@ -239,16 +239,16 @@ static DEFINE_PER_CPU(enum paravirt_lazy_mode, paravirt_lazy_mode) = PARAVIRT_LAstatic inline void enter_lazy (enum paravirt_lazy_mode mode) { -BUG_ON (percpu_read (paravirt_lazy_mode) !=PARAVIRT_LAZY_NONE ); +BUG_ON (__this_cpu_read (paravirt_lazy_mode) !=PARAVIRT_LAZY_NONE ); -percpu_write (paravirt_lazy_mode, mode); +__this_cpu_write (paravirt_lazy_mode, mode); }static void leave_lazy (enum paravirt_lazy_mode mode) { -BUG_ON (percpu_read (paravirt_lazy_mode) != mode); +BUG_ON (__this_cpu_read (paravirt_lazy_mode) != mode); -percpu_write (paravirt_lazy_mode,PARAVIRT_LAZY_NONE ); +__this_cpu_write (paravirt_lazy_mode,PARAVIRT_LAZY_NONE ); }void paravirt_enter_lazy_mmu (void )
@@ -265,7 +265,7 @@ void paravirt_start_context_switch(struct task_struct *prev) {BUG_ON (preemptible ()); -if (percpu_read (paravirt_lazy_mode) ==PARAVIRT_LAZY_MMU ) { +if (__this_cpu_read (paravirt_lazy_mode) ==PARAVIRT_LAZY_MMU ) {arch_leave_lazy_mmu_mode ();set_ti_thread_flag (task_thread_info (prev),TIF_LAZY_MMU_UPDATES ); }
@@ -287,7 +287,7 @@ enum paravirt_lazy_mode paravirt_get_lazy_mode(void)if (in_interrupt ())return PARAVIRT_LAZY_NONE ; -return percpu_read (paravirt_lazy_mode); +return __this_cpu_read (paravirt_lazy_mode); }void arch_flush_lazy_mmu_mode (void )
arch/x86/kernel/process_32.c +1 -1
@@ -373,7 +373,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)if (prev->gs | next->gs)lazy_load_gs (next->gs); -percpu_write (current_task, next_p); +__this_cpu_write (current_task, next_p);return prev_p; }
arch/x86/kernel/process_64.c +6 -6
@@ -73,7 +73,7 @@ EXPORT_SYMBOL_GPL(idle_notifier_unregister);void enter_idle (void ) { -percpu_write (is_idle,1 ); +__this_cpu_write (is_idle,1 );atomic_notifier_call_chain (&idle_notifier,IDLE_START ,NULL ); }
@@ -336,7 +336,7 @@ start_thread_common(struct pt_regs *regs, unsigned long new_ip,load_gs_index (0 ); regs->ip = new_ip; regs->sp = new_sp; -percpu_write (old_rsp, new_sp); +this_cpu_write (old_rsp, new_sp); regs->cs = _cs; regs->ss = _ss; regs->flags =X86_EFLAGS_IF ;
@@ -470,11 +470,11 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)/* * Switch the PDA and FPU contexts. */ - prev->usersp =percpu_read (old_rsp); -percpu_write (old_rsp, next->usersp); -percpu_write (current_task, next_p); + prev->usersp =__this_cpu_read (old_rsp); +__this_cpu_write (old_rsp, next->usersp); +__this_cpu_write (current_task, next_p); -percpu_write (kernel_stack, +__this_cpu_write (kernel_stack, (unsigned long )task_stack_page (next_p) +THREAD_SIZE -KERNEL_STACK_OFFSET );
arch/x86/kvm/x86.c +7 -7
@@ -5024,15 +5024,15 @@ static DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);static int kvm_is_in_guest (void ) { -return percpu_read (current_vcpu) !=NULL ; +return this_cpu_read (current_vcpu) !=NULL ; }static int kvm_is_user_mode (void ) {int user_mode =3 ; -if (percpu_read (current_vcpu)) - user_mode = kvm_x86_ops->get_cpl (percpu_read (current_vcpu)); +if (this_cpu_read (current_vcpu)) + user_mode = kvm_x86_ops->get_cpl (this_cpu_read (current_vcpu));return user_mode !=0 ; }
@@ -5041,8 +5041,8 @@ static unsigned long kvm_get_guest_ip(void) {unsigned long ip =0 ; -if (percpu_read (current_vcpu)) - ip =kvm_rip_read (percpu_read (current_vcpu)); +if (this_cpu_read (current_vcpu)) + ip =kvm_rip_read (this_cpu_read (current_vcpu));return ip; }
@@ -5055,13 +5055,13 @@ static struct perf_guest_info_callbacks kvm_guest_cbs = {void kvm_before_handle_nmi (struct kvm_vcpu *vcpu) { -percpu_write (current_vcpu, vcpu); +__this_cpu_write (current_vcpu, vcpu); }EXPORT_SYMBOL_GPL (kvm_before_handle_nmi);void kvm_after_handle_nmi (struct kvm_vcpu *vcpu) { -percpu_write (current_vcpu,NULL ); +__this_cpu_write (current_vcpu,NULL ); }EXPORT_SYMBOL_GPL (kvm_after_handle_nmi);
arch/x86/mm/tlb.c +5 -5
@@ -61,10 +61,10 @@ static DEFINE_PER_CPU_READ_MOSTLY(int, tlb_vector_offset);*/ void leave_mm (int cpu) { -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) +if (__this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_OK )BUG ();cpumask_clear_cpu (cpu, -mm_cpumask (percpu_read (cpu_tlbstate.active_mm))); +mm_cpumask (__this_cpu_read (cpu_tlbstate.active_mm)));load_cr3 (swapper_pg_dir); }EXPORT_SYMBOL_GPL (leave_mm);
@@ -152,8 +152,8 @@ void smp_invalidate_interrupt(struct pt_regs *regs)* BUG(); */ -if (f->flush_mm ==percpu_read (cpu_tlbstate.active_mm)) { -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) { +if (f->flush_mm ==__this_cpu_read (cpu_tlbstate.active_mm)) { +if (__this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) {if (f->flush_va ==TLB_FLUSH_ALL )local_flush_tlb ();else
@@ -322,7 +322,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)static void do_flush_tlb_all (void *info) {__flush_tlb_all (); -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_LAZY ) +if (__this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_LAZY )leave_mm (smp_processor_id ()); }
arch/x86/xen/enlighten.c +3 -3
@@ -776,11 +776,11 @@ static DEFINE_PER_CPU(unsigned long, xen_cr0_value);static unsigned long xen_read_cr0 (void ) { -unsigned long cr0 =percpu_read (xen_cr0_value); +unsigned long cr0 =this_cpu_read (xen_cr0_value);if (unlikely (cr0 ==0 )) { cr0 =native_read_cr0 (); -percpu_write (xen_cr0_value, cr0); +this_cpu_write (xen_cr0_value, cr0); }return cr0;
@@ -790,7 +790,7 @@ static void xen_write_cr0(unsigned long cr0) {struct multicall_space mcs; -percpu_write (xen_cr0_value, cr0); +this_cpu_write (xen_cr0_value, cr0);/* Only pay attention to cr0.TS; everything else is ignored. */
arch/x86/xen/irq.c +4 -4
@@ -26,7 +26,7 @@ static unsigned long xen_save_fl(void)struct vcpu_info *vcpu;unsigned long flags; - vcpu =percpu_read (xen_vcpu); + vcpu =this_cpu_read (xen_vcpu);/* flag has opposite sense of mask */ flags = !vcpu->evtchn_upcall_mask;
@@ -50,7 +50,7 @@ static void xen_restore_fl(unsigned long flags)make sure we're don't switch CPUs between getting the vcpu pointer and updating the mask. */ preempt_disable (); - vcpu =percpu_read (xen_vcpu); + vcpu =this_cpu_read (xen_vcpu); vcpu->evtchn_upcall_mask = flags;preempt_enable_no_resched ();
@@ -72,7 +72,7 @@ static void xen_irq_disable(void)make sure we're don't switch CPUs between getting the vcpu pointer and updating the mask. */ preempt_disable (); -percpu_read (xen_vcpu)->evtchn_upcall_mask =1 ; +this_cpu_read (xen_vcpu)->evtchn_upcall_mask =1 ;preempt_enable_no_resched (); }PV_CALLEE_SAVE_REGS_THUNK (xen_irq_disable);
@@ -86,7 +86,7 @@ static void xen_irq_enable(void)the caller is confused and is trying to re-enable interrupts on an indeterminate processor. */ - vcpu =percpu_read (xen_vcpu); + vcpu =this_cpu_read (xen_vcpu); vcpu->evtchn_upcall_mask =0 ;/* Doesn't matter if we get preempted here, because any
arch/x86/xen/mmu.c +10 -10
@@ -1106,14 +1106,14 @@ static void drop_other_mm_ref(void *info)struct mm_struct *mm = info;struct mm_struct *active_mm; - active_mm =percpu_read (cpu_tlbstate.active_mm); + active_mm =this_cpu_read (cpu_tlbstate.active_mm); -if (active_mm == mm &&percpu_read (cpu_tlbstate.state) !=TLBSTATE_OK ) +if (active_mm == mm &&this_cpu_read (cpu_tlbstate.state) !=TLBSTATE_OK )leave_mm (smp_processor_id ());/* If this cpu still has a stale cr3 reference, then make sure it has been flushed. */ -if (percpu_read (xen_current_cr3) ==__pa (mm->pgd)) +if (this_cpu_read (xen_current_cr3) ==__pa (mm->pgd))load_cr3 (swapper_pg_dir); }
@@ -1220,17 +1220,17 @@ static void __init xen_pagetable_setup_done(pgd_t *base)static void xen_write_cr2 (unsigned long cr2) { -percpu_read (xen_vcpu)->arch.cr2 = cr2; +this_cpu_read (xen_vcpu)->arch.cr2 = cr2; }static unsigned long xen_read_cr2 (void ) { -return percpu_read (xen_vcpu)->arch.cr2; +return this_cpu_read (xen_vcpu)->arch.cr2; }unsigned long xen_read_cr2_direct (void ) { -return percpu_read (xen_vcpu_info.arch.cr2); +return this_cpu_read (xen_vcpu_info.arch.cr2); }static void xen_flush_tlb (void )
@@ -1313,12 +1313,12 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,static unsigned long xen_read_cr3 (void ) { -return percpu_read (xen_cr3); +return this_cpu_read (xen_cr3); }static void set_current_cr3 (void *v) { -percpu_write (xen_current_cr3, (unsigned long )v); +this_cpu_write (xen_current_cr3, (unsigned long )v); }static void __xen_write_cr3 (bool kernel,unsigned long cr3)
@@ -1341,7 +1341,7 @@ static void __xen_write_cr3(bool kernel, unsigned long cr3)xen_extend_mmuext_op (&op);if (kernel) { -percpu_write (xen_cr3, cr3); +this_cpu_write (xen_cr3, cr3);/* Update xen_current_cr3 once the batch has actually been submitted. */
@@ -1357,7 +1357,7 @@ static void xen_write_cr3(unsigned long cr3)/* Update while interrupts are disabled, so its atomic with respect to ipis */ -percpu_write (xen_cr3, cr3); +this_cpu_write (xen_cr3, cr3);__xen_write_cr3 (true, cr3);
arch/x86/xen/multicalls.h +1 -1
@@ -47,7 +47,7 @@ static inline void xen_mc_issue(unsigned mode)xen_mc_flush ();/* restore flags saved in xen_mc_batch */ -local_irq_restore (percpu_read (xen_mc_irq_flags)); +local_irq_restore (this_cpu_read (xen_mc_irq_flags)); }/* Set up a callback to be called when the current batch is flushed */
arch/x86/xen/smp.c +1 -1
@@ -76,7 +76,7 @@ static void __cpuinit cpu_bringup(void)xen_setup_cpu_clockevents ();set_cpu_online (cpu, true); -percpu_write (cpu_state,CPU_ONLINE ); +this_cpu_write (cpu_state,CPU_ONLINE );wmb ();/* We can take interrupts now: we're officially "up". */
include/linux/percpu.h +0 -53
59 lines not on this page · raw message
include/linux/topology.h +2 -2
@@ -235,7 +235,7 @@ static inline int cpu_to_node(int cpu)#ifndef set_numa_nodestatic inline void set_numa_node (int node) { -percpu_write (numa_node, node); +__this_cpu_write (numa_node, node); }#endif
@@ -270,7 +270,7 @@ DECLARE_PER_CPU(int, _numa_mem_);#ifndef set_numa_memstatic inline void set_numa_mem (int node) { -percpu_write (_numa_mem_, node); +__this_cpu_write (_numa_mem_, node); }#endif
net/netfilter/xt_TEE.c +6 -6
@@ -90,7 +90,7 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)const struct xt_tee_tginfo *info = par->targinfo;struct iphdr *iph; -if (percpu_read (tee_active)) +if (this_cpu_read (tee_active))return XT_CONTINUE ;/* * Copy the skb, and route the copy. Will later return %XT_CONTINUE for
@@ -127,9 +127,9 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)ip_send_check (iph);if (tee_tg_route4 (skb, info)) { -percpu_write (tee_active, true); +this_cpu_write (tee_active, true);ip_local_out (skb); -percpu_write (tee_active, false); +this_cpu_write (tee_active, false); }else {kfree_skb (skb); }
@@ -170,7 +170,7 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par) {const struct xt_tee_tginfo *info = par->targinfo; -if (percpu_read (tee_active)) +if (this_cpu_read (tee_active))return XT_CONTINUE ; skb =pskb_copy (skb,GFP_ATOMIC );if (skb ==NULL )
@@ -188,9 +188,9 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par) --iph->hop_limit; }if (tee_tg_route6 (skb, info)) { -percpu_write (tee_active, true); +this_cpu_write (tee_active, true);ip6_local_out (skb); -percpu_write (tee_active, false); +this_cpu_write (tee_active, false); }else {kfree_skb (skb); }
net/socket.c +2 -2
@@ -479,7 +479,7 @@ static struct socket *sock_alloc(void) inode->i_uid =current_fsuid (); inode->i_gid =current_fsgid (); -percpu_add (sockets_in_use,1 ); +this_cpu_add (sockets_in_use,1 );return sock; }
@@ -522,7 +522,7 @@ void sock_release(struct socket *sock)if (rcu_dereference_protected (sock->wq,1 )->fasync_list)printk (KERN_ERR "sock_release: fasync list not empty!\n" ); -percpu_sub (sockets_in_use,1 ); +this_cpu_sub (sockets_in_use,1 );if (!sock->file) {iput (SOCK_INODE (sock));return ;
Signature · 1 line
1.6.3.3
cover code clean and optimzed for percpu_xxx serials funcs
In reply to Eric DumazetThink over Eric's requirement, I split my old patches and reorganized them into 3 small patches, include net, kvm and a final clean up patch. Since the clean up contents has no change beside move kvm/net part into seprated patches. I keep Christoph's ack. Hope these are what you want. :)
Regards
Alex
PATCH 1/3 net: use this_cpu_xxx replace percpu_xxx funcs
In reply to Eric Dumazetpercpu_xxx funcs are duplicated with this_cpu_xxx funcs, so replace them for further code clean up.
And in preempt safe scenario, __this_cpu_xxx funcs has a bit better performance since __this_cpu_xxx has no redundant preempt_disable()
Signed-off-by Alex Shi <[email protected]>
--- net/netfilter/xt_TEE.c | 12 ++++++------ net/socket.c | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-)
net/netfilter/xt_TEE.c +6 -6
@@ -90,7 +90,7 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)const struct xt_tee_tginfo *info = par->targinfo;struct iphdr *iph; -if (percpu_read (tee_active)) +if (__this_cpu_read (tee_active))return XT_CONTINUE ;/* * Copy the skb, and route the copy. Will later return %XT_CONTINUE for
@@ -127,9 +127,9 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)ip_send_check (iph);if (tee_tg_route4 (skb, info)) { -percpu_write (tee_active, true); +__this_cpu_write (tee_active, true);ip_local_out (skb); -percpu_write (tee_active, false); +__this_cpu_write (tee_active, false); }else {kfree_skb (skb); }
@@ -170,7 +170,7 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par) {const struct xt_tee_tginfo *info = par->targinfo; -if (percpu_read (tee_active)) +if (__this_cpu_read (tee_active))return XT_CONTINUE ; skb =pskb_copy (skb,GFP_ATOMIC );if (skb ==NULL )
@@ -188,9 +188,9 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par) --iph->hop_limit; }if (tee_tg_route6 (skb, info)) { -percpu_write (tee_active, true); +__this_cpu_write (tee_active, true);ip6_local_out (skb); -percpu_write (tee_active, false); +__this_cpu_write (tee_active, false); }else {kfree_skb (skb); }
net/socket.c +2 -2
@@ -479,7 +479,7 @@ static struct socket *sock_alloc(void) inode->i_uid =current_fsuid (); inode->i_gid =current_fsgid (); -percpu_add (sockets_in_use,1 ); +this_cpu_add (sockets_in_use,1 );return sock; }
@@ -522,7 +522,7 @@ void sock_release(struct socket *sock)if (rcu_dereference_protected (sock->wq,1 )->fasync_list)printk (KERN_ERR "sock_release: fasync list not empty!\n" ); -percpu_sub (sockets_in_use,1 ); +this_cpu_sub (sockets_in_use,1 );if (!sock->file) {iput (SOCK_INODE (sock));return ;
Signature · 1 line
1.6.3.3
Re: [PATCH 1/3] net: use this_cpu_xxx replace percpu_xxx funcs
Quoted 11 lines · Alex,Shi
percpu_xxxfuncs are duplicated withthis_cpu_xxxfuncs, so replace them for further code clean up.And in preempt safe scenario,
__this_cpu_xxxfuncs has a bit better performance since__this_cpu_xxxhas no redundantpreempt_disable()Signed-off-by Alex Shi <[email protected]>--- net/netfilter/xt_TEE.c | 12 ++++++------ net/socket.c | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-)
Acked-by Eric Dumazet <[email protected]>
Thanks !
Re: [PATCH 1/3] net: use this_cpu_xxx replace percpu_xxx funcs
Quoted 13 lines · Eric Dumazet
Quoted 11 lines · Alex,Shi
percpu_xxxfuncs are duplicated withthis_cpu_xxxfuncs, so replace them for further code clean up.And in preempt safe scenario,
__this_cpu_xxxfuncs has a bit better performance since__this_cpu_xxxhas no redundantpreempt_disable()Signed-off-by Alex Shi <[email protected]>--- net/netfilter/xt_TEE.c | 12 ++++++------ net/socket.c | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-)
Quoted 3 lines
Acked-by Eric Dumazet <[email protected]>Thanks !
Anyone like to pick up this patch? or more comments for this?
Quoted 2 lines
Re: [PATCH 1/3] net: use this_cpu_xxx replace percpu_xxx funcs
Quoted 14 lines · Alex,Shi
Quoted 13 lines · Eric Dumazet
Quoted 11 lines · Alex,Shi
percpu_xxxfuncs are duplicated withthis_cpu_xxxfuncs, so replace them for further code clean up.And in preempt safe scenario,
__this_cpu_xxxfuncs has a bit better performance since__this_cpu_xxxhas no redundantpreempt_disable()Signed-off-by Alex Shi <[email protected]>--- net/netfilter/xt_TEE.c | 12 ++++++------ net/socket.c | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-)
Quoted 5 lines
Quoted 3 lines
Acked-by Eric Dumazet <[email protected]>Thanks !
Anyone like to pick up this patch? or more comments for this?
Kaber, David:
I appreciate for your any comments on this. Could you like do me a favor?
Re: [PATCH 1/3] net: use this_cpu_xxx replace percpu_xxx funcs
Quoted 17 lines · Alex,Shi
Quoted 16 lines · Alex,Shi
Quoted 15 lines · Eric Dumazet
Quoted 11 lines · Alex,Shi
percpu_xxxfuncs are duplicated withthis_cpu_xxxfuncs, so replace them for further code clean up.And in preempt safe scenario,
__this_cpu_xxxfuncs has a bit better performance since__this_cpu_xxxhas no redundantpreempt_disable()Signed-off-by Alex Shi <[email protected]>--- net/netfilter/xt_TEE.c | 12 ++++++------ net/socket.c | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-)Acked-by Eric Dumazet <[email protected]>
Quoted 7 lines
Quoted 3 lines
Quoted 1 line
Thanks !
Anyone like to pick up this patch? or more comments for this?
Kaber, David:
I appreciate for your any comments on this. Could you like do me a favor?
No objections from me.
Alex,Shi
Acked-by Eric DumazetRe: [PATCH 1/3] net: use this_cpu_xxx replace percpu_xxx funcs
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 15 lines
Quoted 11 lines
percpu_xxxfuncs are duplicated withthis_cpu_xxxfuncs, so replace them for further code clean up.And in preempt safe scenario,
__this_cpu_xxxfuncs has a bit better performance since__this_cpu_xxxhas no redundantpreempt_disable()Signed-off-by Alex Shi <[email protected]>--- net/netfilter/xt_TEE.c | 12 ++++++------ net/socket.c | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-)Acked-by Eric Dumazet <[email protected]>Thanks !
Quoted 7 lines
Quoted 5 lines
Quoted 1 line
Anyone like to pick up this patch? or more comments for this?
Kaber, David:
I appreciate for your any comments on this. Could you like do me a favor?No objections from me.
rend this patch for 3.2.0 kernel with Eric's Ack.
David, do you have any concerns for this patch? I will very appreciate if it can met 3.3 open window.
---------
Quoted 1 line
From
037bd159fdf52b915e452fac8db2252b1c60297eMon Sep 17 00:00:00 2001
From: Alex Shi <[email protected]>
Date: Thu, 20 Oct 2011 14:52:17 +0800
Subject: [PATCH 1/3] net: use this_cpu_xxx replace percpu_xxx funcs
percpu_xxx funcs are duplicated with this_cpu_xxx funcs, so replace them for further code clean up.
And in preempt safe scenario, __this_cpu_xxx funcs has a bit better performance since __this_cpu_xxx has no redundant preempt_disable()
Signed-off-by Alex Shi <[email protected]>
Acked-by Eric Dumazet <[email protected]>
--- net/netfilter/xt_TEE.c | 12 ++++++------ net/socket.c | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-)
net/netfilter/xt_TEE.c +6 -6
@@ -90,7 +90,7 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)const struct xt_tee_tginfo *info = par->targinfo;struct iphdr *iph; -if (percpu_read (tee_active)) +if (__this_cpu_read (tee_active))return XT_CONTINUE ;/* * Copy the skb, and route the copy. Will later return %XT_CONTINUE for
@@ -127,9 +127,9 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)ip_send_check (iph);if (tee_tg_route4 (skb, info)) { -percpu_write (tee_active, true); +__this_cpu_write (tee_active, true);ip_local_out (skb); -percpu_write (tee_active, false); +__this_cpu_write (tee_active, false); }else {kfree_skb (skb); }
@@ -170,7 +170,7 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par) {const struct xt_tee_tginfo *info = par->targinfo; -if (percpu_read (tee_active)) +if (__this_cpu_read (tee_active))return XT_CONTINUE ; skb =pskb_copy (skb,GFP_ATOMIC );if (skb ==NULL )
@@ -188,9 +188,9 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par) --iph->hop_limit; }if (tee_tg_route6 (skb, info)) { -percpu_write (tee_active, true); +__this_cpu_write (tee_active, true);ip6_local_out (skb); -percpu_write (tee_active, false); +__this_cpu_write (tee_active, false); }else {kfree_skb (skb); }
net/socket.c +2 -2
@@ -479,7 +479,7 @@ static struct socket *sock_alloc(void) inode->i_uid =current_fsuid (); inode->i_gid =current_fsgid (); -percpu_add (sockets_in_use,1 ); +this_cpu_add (sockets_in_use,1 );return sock; }
@@ -522,7 +522,7 @@ void sock_release(struct socket *sock)if (rcu_dereference_protected (sock->wq,1 )->fasync_list)printk (KERN_ERR "sock_release: fasync list not empty!\n" ); -percpu_sub (sockets_in_use,1 ); +this_cpu_sub (sockets_in_use,1 );if (!sock->file) {iput (SOCK_INODE (sock));return ;
Signature · 1 line
1.6.3.3
Re: [PATCH 1/3] net: use this_cpu_xxx replace percpu_xxx funcs
From: "Alex,Shi" <[email protected]>
Date: Wed, 11 Jan 2012 16:45:33 +0800
Quoted 22 lines
Quoted 22 lines
Quoted 21 lines
Quoted 12 lines
Quoted 12 lines
Quoted 11 lines
percpu_xxxfuncs are duplicated withthis_cpu_xxxfuncs, so replace them for further code clean up.And in preempt safe scenario,
__this_cpu_xxxfuncs has a bit better performance since__this_cpu_xxxhas no redundantpreempt_disable()Signed-off-by Alex Shi <[email protected]>--- net/netfilter/xt_TEE.c | 12 ++++++------ net/socket.c | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-)Quoted 5 lines
Quoted 3 lines
Acked-by Eric Dumazet <[email protected]>Thanks !
Anyone like to pick up this patch? or more comments for this?
Kaber, David:
I appreciate for your any comments on this. Could you like do me a favor?
Quoted 6 lines
Quoted 1 line
No objections from me.
rend this patch for 3.2.0 kernel with Eric's Ack.
David, do you have any concerns for this patch? I will very appreciate if it can met 3.3 open window.
Please just submit it directly with the other this_cpu() patches:
Acked-by David S. Miller <[email protected]>
Re: [PATCH 1/3] net: use this_cpu_xxx replace percpu_xxx funcs
Quoted 27 lines · David Miller
From: "Alex,Shi" <[email protected]>
Date: Wed, 11 Jan 2012 16:45:33 +0800Quoted 24 lines
Quoted 23 lines
Quoted 14 lines
Quoted 14 lines
Quoted 14 lines
Quoted 11 lines
percpu_xxxfuncs are duplicated withthis_cpu_xxxfuncs, so replace them for further code clean up.And in preempt safe scenario,
__this_cpu_xxxfuncs has a bit better performance since__this_cpu_xxxhas no redundantpreempt_disable()Signed-off-by Alex Shi <[email protected]>--- net/netfilter/xt_TEE.c | 12 ++++++------ net/socket.c | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-)Acked-by Eric Dumazet <[email protected]>Quoted 7 lines
Quoted 3 lines
Quoted 1 line
Thanks !
Anyone like to pick up this patch? or more comments for this?
Kaber, David:
I appreciate for your any comments on this. Could you like do me a favor?No objections from me.
Quoted 8 lines
Quoted 4 lines
rend this patch for 3.2.0 kernel with Eric's Ack.
David, do you have any concerns for this patch? I will very appreciate if it can met 3.3 open window.
Please just submit it directly with the other
this_cpu()patches:Acked-by David S. Miller <[email protected]>
Thanks a lot! :)
PATCH 3/3 Code clean up for percpu_xxx() functions
In reply to Christoph LameterSince percpu_xxx() serial functions are duplicate with this_cpu_xxx(). Removing percpu_xxx() definition and replacing them by this_cpu_xxx() in code.
And further more, as Christoph Lameter's requirement, I try to use __this_cpu_xx to replace this_cpu_xxx if it is in preempt safe scenario. The preempt safe scenarios include:
1, in irq/softirq/nmi handler
2, protected by preempt_disable
3, protected by spin_lock
4, if the code context imply that it is preempt safe, like the code is follows or be followed a preempt safe code.
I left the xen code unchanged, since no idea of them.
BTW, In fact, this_cpu_xxx are same as __this_cpu_xxx since all funcs implement in a single instruction for x86 machine. But it maybe different for other platforms, so, doing this distinguish is helpful for other platforms' performance.
Signed-off-by Alex Shi <[email protected]>
--- arch/x86/include/asm/current.h | 2 +- arch/x86/include/asm/hardirq.h | 9 +++-- arch/x86/include/asm/irq_regs.h | 4 +- arch/x86/include/asm/mmu_context.h | 12 ++++---- arch/x86/include/asm/percpu.h | 24 ++++++--------- arch/x86/include/asm/smp.h | 4 +- arch/x86/include/asm/stackprotector.h | 4 +- arch/x86/include/asm/thread_info.h | 2 +- arch/x86/include/asm/tlbflush.h | 4 +- arch/x86/kernel/cpu/common.c | 2 +- arch/x86/kernel/cpu/mcheck/mce.c | 4 +- arch/x86/kernel/cpu/perf_event.c | 4 +- arch/x86/kernel/paravirt.c | 12 ++++---- arch/x86/kernel/process_32.c | 2 +- arch/x86/kernel/process_64.c | 12 ++++---- arch/x86/mm/tlb.c | 10 +++--- arch/x86/xen/enlighten.c | 6 ++-- arch/x86/xen/irq.c | 8 ++-- arch/x86/xen/mmu.c | 20 ++++++------ arch/x86/xen/multicalls.h | 2 +- arch/x86/xen/smp.c | 2 +- include/linux/percpu.h | 53 --------------------------------- include/linux/topology.h | 4 +- 23 files changed, 75 insertions(+), 131 deletions(-)
arch/x86/include/asm/current.h +1 -1
@@ -11,7 +11,7 @@ DECLARE_PER_CPU(struct task_struct *, current_task); static __always_inline struct task_struct *get_current(void) { -return percpu_read_stable (current_task); +return this_cpu_read_stable (current_task); }#define current get_current()
arch/x86/include/asm/hardirq.h +5 -4
@@ -35,14 +35,15 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);#define __ARCH_IRQ_STAT -#define inc_irq_stat (member) percpu_inc(irq_stat.member) +#define inc_irq_stat (member) __this_cpu_inc(irq_stat.member) -#define local_softirq_pending () percpu_read(irq_stat.__softirq_pending) +#define local_softirq_pending () __this_cpu_read(irq_stat.__softirq_pending)#define __ARCH_SET_SOFTIRQ_PENDING -#define set_softirq_pending (x) percpu_write(irq_stat.__softirq_pending, (x)) -#define or_softirq_pending (x) percpu_or(irq_stat.__softirq_pending, (x)) +#define set_softirq_pending (x) \ + __this_cpu_write(irq_stat.__softirq_pending, (x)) +#define or_softirq_pending (x) __this_cpu_or(irq_stat.__softirq_pending, (x))extern void ack_bad_irq (unsigned int irq);
arch/x86/include/asm/irq_regs.h +2 -2
@@ -15,7 +15,7 @@ DECLARE_PER_CPU(struct pt_regs *, irq_regs);static inline struct pt_regs *get_irq_regs (void ) { -return percpu_read (irq_regs); +return __this_cpu_read (irq_regs); }static inline struct pt_regs *set_irq_regs (struct pt_regs *new_regs)
@@ -23,7 +23,7 @@ static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)struct pt_regs *old_regs; old_regs =get_irq_regs (); -percpu_write (irq_regs, new_regs); +__this_cpu_write (irq_regs, new_regs);return old_regs; }
arch/x86/include/asm/mmu_context.h +6 -6
@@ -25,8 +25,8 @@ void destroy_context(struct mm_struct *mm);static inline void enter_lazy_tlb (struct mm_struct *mm,struct task_struct *tsk) {#ifdef CONFIG_SMP -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) -percpu_write (cpu_tlbstate.state,TLBSTATE_LAZY ); +if (__this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) +__this_cpu_write (cpu_tlbstate.state,TLBSTATE_LAZY );#endif }
@@ -37,8 +37,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,if (likely (prev != next)) {#ifdef CONFIG_SMP -percpu_write (cpu_tlbstate.state,TLBSTATE_OK ); -percpu_write (cpu_tlbstate.active_mm, next); +__this_cpu_write (cpu_tlbstate.state,TLBSTATE_OK ); +__this_cpu_write (cpu_tlbstate.active_mm, next);#endif cpumask_set_cpu (cpu,mm_cpumask (next));
@@ -56,8 +56,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, }#ifdef CONFIG_SMP else { -percpu_write (cpu_tlbstate.state,TLBSTATE_OK ); -BUG_ON (percpu_read (cpu_tlbstate.active_mm) != next); +__this_cpu_write (cpu_tlbstate.state,TLBSTATE_OK ); +BUG_ON (__this_cpu_read (cpu_tlbstate.active_mm) != next);if (!cpumask_test_and_set_cpu (cpu,mm_cpumask (next))) {/* We were in lazy tlb mode and leave_mm disabled
arch/x86/include/asm/percpu.h +10 -14
@@ -46,7 +46,7 @@#ifdef CONFIG_SMP #define __percpu_prefix "%%"__stringify(__percpu_seg)":" -#define __my_cpu_offset percpu_read(this_cpu_off) +#define __my_cpu_offset __this_cpu_read(this_cpu_off)/* * Compared to the generic __my_cpu_offset version, the following
@@ -351,23 +351,15 @@ do { \ })/* -* percpu_read() makes gcc load the percpu variable every time it is -* accessed while percpu_read_stable() allows the value to be cached. -* percpu_read_stable() is more efficient and can be used if its value +* this_cpu_read() makes gcc load the percpu variable every time it is +* accessed while this_cpu_read_stable() allows the value to be cached. +* this_cpu_read_stable() is more efficient and can be used if its value * is guaranteed to be valid across cpus. The current users include * get_current() and get_thread_info() both of which are actually * per-thread variables implemented as per-cpu variables and thus * stable for the duration of the respective task. */ -#define percpu_read (var) percpu_from_op("mov", var, "m" (var)) -#define percpu_read_stable (var) percpu_from_op("mov", var, "p" (&(var))) -#define percpu_write (var, val) percpu_to_op("mov", var, val) -#define percpu_add (var, val) percpu_add_op(var, val) -#define percpu_sub (var, val) percpu_add_op(var, -(val)) -#define percpu_and (var, val) percpu_to_op("and", var, val) -#define percpu_or (var, val) percpu_to_op("or", var, val) -#define percpu_xor (var, val) percpu_to_op("xor", var, val) -#define percpu_inc (var) percpu_unary_op("inc", var) +#define this_cpu_read_stable (var) percpu_from_op("mov", var, "p" (&(var)))#define __this_cpu_read_1 (pcp) percpu_from_op("mov", (pcp), "m"(pcp))#define __this_cpu_read_2 (pcp) percpu_from_op("mov", (pcp), "m"(pcp))
@@ -551,7 +543,11 @@ static __always_inline int x86_this_cpu_constant_test_bit(unsigned int nr, {unsigned long __percpu *a = (unsigned long *)addr + nr /BITS_PER_LONG ; -return ((1UL << (nr %BITS_PER_LONG )) &percpu_read (*a)) !=0 ; +#ifdef CONFIG_X86_64 +return ((1UL << (nr %BITS_PER_LONG )) &__this_cpu_read_8 (*a)) !=0 ; +#else +return ((1UL << (nr %BITS_PER_LONG )) &__this_cpu_read_4 (*a)) !=0 ; +#endif } static inline int x86_this_cpu_variable_test_bit(int nr,
arch/x86/include/asm/smp.h +2 -2
@@ -188,11 +188,11 @@ extern unsigned disabled_cpus __cpuinitdata;* from the initial startup. We map APIC_BASE very early in page_setup(), * so this is correct in the x86 case. */ -#define raw_smp_processor_id () (percpu_read(cpu_number)) +#define raw_smp_processor_id () (this_cpu_read(cpu_number))extern int safe_smp_processor_id (void );#elif defined(CONFIG_X86_64_SMP ) -#define raw_smp_processor_id () (percpu_read(cpu_number)) +#define raw_smp_processor_id () (this_cpu_read(cpu_number))#define stack_smp_processor_id () \ ({ \
arch/x86/include/asm/stackprotector.h +2 -2
@@ -76,9 +76,9 @@ static __always_inline void boot_init_stack_canary(void) current->stack_canary = canary;#ifdef CONFIG_X86_64 -percpu_write (irq_stack_union.stack_canary, canary); +__this_cpu_write (irq_stack_union.stack_canary, canary);#else -percpu_write (stack_canary.canary, canary); +__this_cpu_write (stack_canary.canary, canary);#endif }
arch/x86/include/asm/thread_info.h +1 -1
@@ -219,7 +219,7 @@ DECLARE_PER_CPU(unsigned long, kernel_stack);static inline struct thread_info *current_thread_info (void ) {struct thread_info *ti; - ti = (void *)(percpu_read_stable (kernel_stack) + + ti = (void *)(this_cpu_read_stable (kernel_stack) +KERNEL_STACK_OFFSET -THREAD_SIZE );return ti; }
arch/x86/include/asm/tlbflush.h +2 -2
@@ -156,8 +156,8 @@ DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate);static inline void reset_lazy_tlbstate (void ) { -percpu_write (cpu_tlbstate.state,0 ); -percpu_write (cpu_tlbstate.active_mm, &init_mm); +__this_cpu_write (cpu_tlbstate.state,0 ); +__this_cpu_write (cpu_tlbstate.active_mm, &init_mm); }#endif /* SMP */
arch/x86/kernel/cpu/common.c +1 -1
@@ -1158,7 +1158,7 @@ void __cpuinit cpu_init(void) oist = &per_cpu (orig_ist, cpu);#ifdef CONFIG_NUMA -if (cpu !=0 &&percpu_read (numa_node) ==0 && +if (cpu !=0 &&__this_cpu_read (numa_node) ==0 &&early_cpu_to_node (cpu) !=NUMA_NO_NODE )set_numa_node (early_cpu_to_node (cpu));#endif
arch/x86/kernel/cpu/mcheck/mce.c +2 -2
@@ -512,7 +512,7 @@ void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)struct mce m;int i; -percpu_inc (mce_poll_count); +__this_cpu_inc (mce_poll_count);mce_gather_info (&m,NULL );
@@ -906,7 +906,7 @@ void do_machine_check(struct pt_regs *regs, long error_code)atomic_inc (&mce_entry); -percpu_inc (mce_exception_count); +__this_cpu_inc (mce_exception_count);if (notify_die (DIE_NMI ,"machine check" , regs, error_code,18 ,SIGKILL ) ==NOTIFY_STOP )
arch/x86/kernel/cpu/perf_event.c +2 -2
@@ -1389,7 +1389,7 @@ perf_event_nmi_handler(struct notifier_block *self,case DIE_NMI :break ;case DIE_NMIUNKNOWN : - this_nmi =percpu_read (irq_stat.__nmi_count); + this_nmi =__this_cpu_read (irq_stat.__nmi_count);if (this_nmi !=__this_cpu_read (pmu_nmi.marked))/* let the kernel handle the unknown nmi */ return NOTIFY_DONE ;
@@ -1409,7 +1409,7 @@ perf_event_nmi_handler(struct notifier_block *self,if (!handled)return NOTIFY_DONE ; - this_nmi =percpu_read (irq_stat.__nmi_count); + this_nmi =__this_cpu_read (irq_stat.__nmi_count); if ((handled > 1) || /* the next nmi could be a back-to-back nmi */ ((__this_cpu_read(pmu_nmi.marked) == this_nmi) &&
arch/x86/kernel/paravirt.c +6 -6
@@ -239,16 +239,16 @@ static DEFINE_PER_CPU(enum paravirt_lazy_mode, paravirt_lazy_mode) = PARAVIRT_LAstatic inline void enter_lazy (enum paravirt_lazy_mode mode) { -BUG_ON (percpu_read (paravirt_lazy_mode) !=PARAVIRT_LAZY_NONE ); +BUG_ON (__this_cpu_read (paravirt_lazy_mode) !=PARAVIRT_LAZY_NONE ); -percpu_write (paravirt_lazy_mode, mode); +__this_cpu_write (paravirt_lazy_mode, mode); }static void leave_lazy (enum paravirt_lazy_mode mode) { -BUG_ON (percpu_read (paravirt_lazy_mode) != mode); +BUG_ON (__this_cpu_read (paravirt_lazy_mode) != mode); -percpu_write (paravirt_lazy_mode,PARAVIRT_LAZY_NONE ); +__this_cpu_write (paravirt_lazy_mode,PARAVIRT_LAZY_NONE ); }void paravirt_enter_lazy_mmu (void )
@@ -265,7 +265,7 @@ void paravirt_start_context_switch(struct task_struct *prev) {BUG_ON (preemptible ()); -if (percpu_read (paravirt_lazy_mode) ==PARAVIRT_LAZY_MMU ) { +if (__this_cpu_read (paravirt_lazy_mode) ==PARAVIRT_LAZY_MMU ) {arch_leave_lazy_mmu_mode ();set_ti_thread_flag (task_thread_info (prev),TIF_LAZY_MMU_UPDATES ); }
@@ -287,7 +287,7 @@ enum paravirt_lazy_mode paravirt_get_lazy_mode(void)if (in_interrupt ())return PARAVIRT_LAZY_NONE ; -return percpu_read (paravirt_lazy_mode); +return __this_cpu_read (paravirt_lazy_mode); }void arch_flush_lazy_mmu_mode (void )
arch/x86/kernel/process_32.c +1 -1
@@ -373,7 +373,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)if (prev->gs | next->gs)lazy_load_gs (next->gs); -percpu_write (current_task, next_p); +__this_cpu_write (current_task, next_p);return prev_p; }
arch/x86/kernel/process_64.c +6 -6
@@ -73,7 +73,7 @@ EXPORT_SYMBOL_GPL(idle_notifier_unregister);void enter_idle (void ) { -percpu_write (is_idle,1 ); +__this_cpu_write (is_idle,1 );atomic_notifier_call_chain (&idle_notifier,IDLE_START ,NULL ); }
@@ -336,7 +336,7 @@ start_thread_common(struct pt_regs *regs, unsigned long new_ip,load_gs_index (0 ); regs->ip = new_ip; regs->sp = new_sp; -percpu_write (old_rsp, new_sp); +this_cpu_write (old_rsp, new_sp); regs->cs = _cs; regs->ss = _ss; regs->flags =X86_EFLAGS_IF ;
@@ -470,11 +470,11 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)/* * Switch the PDA and FPU contexts. */ - prev->usersp =percpu_read (old_rsp); -percpu_write (old_rsp, next->usersp); -percpu_write (current_task, next_p); + prev->usersp =__this_cpu_read (old_rsp); +__this_cpu_write (old_rsp, next->usersp); +__this_cpu_write (current_task, next_p); -percpu_write (kernel_stack, +__this_cpu_write (kernel_stack, (unsigned long )task_stack_page (next_p) +THREAD_SIZE -KERNEL_STACK_OFFSET );
arch/x86/mm/tlb.c +5 -5
@@ -61,10 +61,10 @@ static DEFINE_PER_CPU_READ_MOSTLY(int, tlb_vector_offset);*/ void leave_mm (int cpu) { -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) +if (__this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_OK )BUG ();cpumask_clear_cpu (cpu, -mm_cpumask (percpu_read (cpu_tlbstate.active_mm))); +mm_cpumask (__this_cpu_read (cpu_tlbstate.active_mm)));load_cr3 (swapper_pg_dir); }EXPORT_SYMBOL_GPL (leave_mm);
@@ -152,8 +152,8 @@ void smp_invalidate_interrupt(struct pt_regs *regs)* BUG(); */ -if (f->flush_mm ==percpu_read (cpu_tlbstate.active_mm)) { -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) { +if (f->flush_mm ==__this_cpu_read (cpu_tlbstate.active_mm)) { +if (__this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) {if (f->flush_va ==TLB_FLUSH_ALL )local_flush_tlb ();else
@@ -322,7 +322,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)static void do_flush_tlb_all (void *info) {__flush_tlb_all (); -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_LAZY ) +if (__this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_LAZY )leave_mm (smp_processor_id ()); }
arch/x86/xen/enlighten.c +3 -3
@@ -776,11 +776,11 @@ static DEFINE_PER_CPU(unsigned long, xen_cr0_value);static unsigned long xen_read_cr0 (void ) { -unsigned long cr0 =percpu_read (xen_cr0_value); +unsigned long cr0 =this_cpu_read (xen_cr0_value);if (unlikely (cr0 ==0 )) { cr0 =native_read_cr0 (); -percpu_write (xen_cr0_value, cr0); +this_cpu_write (xen_cr0_value, cr0); }return cr0;
@@ -790,7 +790,7 @@ static void xen_write_cr0(unsigned long cr0) {struct multicall_space mcs; -percpu_write (xen_cr0_value, cr0); +this_cpu_write (xen_cr0_value, cr0);/* Only pay attention to cr0.TS; everything else is ignored. */
arch/x86/xen/irq.c +4 -4
@@ -26,7 +26,7 @@ static unsigned long xen_save_fl(void)struct vcpu_info *vcpu;unsigned long flags; - vcpu =percpu_read (xen_vcpu); + vcpu =this_cpu_read (xen_vcpu);/* flag has opposite sense of mask */ flags = !vcpu->evtchn_upcall_mask;
@@ -50,7 +50,7 @@ static void xen_restore_fl(unsigned long flags)make sure we're don't switch CPUs between getting the vcpu pointer and updating the mask. */ preempt_disable (); - vcpu =percpu_read (xen_vcpu); + vcpu =this_cpu_read (xen_vcpu); vcpu->evtchn_upcall_mask = flags;preempt_enable_no_resched ();
@@ -72,7 +72,7 @@ static void xen_irq_disable(void)make sure we're don't switch CPUs between getting the vcpu pointer and updating the mask. */ preempt_disable (); -percpu_read (xen_vcpu)->evtchn_upcall_mask =1 ; +this_cpu_read (xen_vcpu)->evtchn_upcall_mask =1 ;preempt_enable_no_resched (); }PV_CALLEE_SAVE_REGS_THUNK (xen_irq_disable);
@@ -86,7 +86,7 @@ static void xen_irq_enable(void)the caller is confused and is trying to re-enable interrupts on an indeterminate processor. */ - vcpu =percpu_read (xen_vcpu); + vcpu =this_cpu_read (xen_vcpu); vcpu->evtchn_upcall_mask =0 ;/* Doesn't matter if we get preempted here, because any
arch/x86/xen/mmu.c +10 -10
@@ -1106,14 +1106,14 @@ static void drop_other_mm_ref(void *info)struct mm_struct *mm = info;struct mm_struct *active_mm; - active_mm =percpu_read (cpu_tlbstate.active_mm); + active_mm =this_cpu_read (cpu_tlbstate.active_mm); -if (active_mm == mm &&percpu_read (cpu_tlbstate.state) !=TLBSTATE_OK ) +if (active_mm == mm &&this_cpu_read (cpu_tlbstate.state) !=TLBSTATE_OK )leave_mm (smp_processor_id ());/* If this cpu still has a stale cr3 reference, then make sure it has been flushed. */ -if (percpu_read (xen_current_cr3) ==__pa (mm->pgd)) +if (this_cpu_read (xen_current_cr3) ==__pa (mm->pgd))load_cr3 (swapper_pg_dir); }
@@ -1220,17 +1220,17 @@ static void __init xen_pagetable_setup_done(pgd_t *base)static void xen_write_cr2 (unsigned long cr2) { -percpu_read (xen_vcpu)->arch.cr2 = cr2; +this_cpu_read (xen_vcpu)->arch.cr2 = cr2; }static unsigned long xen_read_cr2 (void ) { -return percpu_read (xen_vcpu)->arch.cr2; +return this_cpu_read (xen_vcpu)->arch.cr2; }unsigned long xen_read_cr2_direct (void ) { -return percpu_read (xen_vcpu_info.arch.cr2); +return this_cpu_read (xen_vcpu_info.arch.cr2); }static void xen_flush_tlb (void )
@@ -1313,12 +1313,12 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,static unsigned long xen_read_cr3 (void ) { -return percpu_read (xen_cr3); +return this_cpu_read (xen_cr3); }static void set_current_cr3 (void *v) { -percpu_write (xen_current_cr3, (unsigned long )v); +this_cpu_write (xen_current_cr3, (unsigned long )v); }static void __xen_write_cr3 (bool kernel,unsigned long cr3)
@@ -1341,7 +1341,7 @@ static void __xen_write_cr3(bool kernel, unsigned long cr3)xen_extend_mmuext_op (&op);if (kernel) { -percpu_write (xen_cr3, cr3); +this_cpu_write (xen_cr3, cr3);/* Update xen_current_cr3 once the batch has actually been submitted. */
@@ -1357,7 +1357,7 @@ static void xen_write_cr3(unsigned long cr3)/* Update while interrupts are disabled, so its atomic with respect to ipis */ -percpu_write (xen_cr3, cr3); +this_cpu_write (xen_cr3, cr3);__xen_write_cr3 (true, cr3);
arch/x86/xen/multicalls.h +1 -1
@@ -47,7 +47,7 @@ static inline void xen_mc_issue(unsigned mode)xen_mc_flush ();/* restore flags saved in xen_mc_batch */ -local_irq_restore (percpu_read (xen_mc_irq_flags)); +local_irq_restore (this_cpu_read (xen_mc_irq_flags)); }/* Set up a callback to be called when the current batch is flushed */
arch/x86/xen/smp.c +1 -1
@@ -76,7 +76,7 @@ static void __cpuinit cpu_bringup(void)xen_setup_cpu_clockevents ();set_cpu_online (cpu, true); -percpu_write (cpu_state,CPU_ONLINE ); +this_cpu_write (cpu_state,CPU_ONLINE );wmb ();/* We can take interrupts now: we're officially "up". */
include/linux/percpu.h +0 -53
59 lines not on this page · raw message
include/linux/topology.h +2 -2
@@ -235,7 +235,7 @@ static inline int cpu_to_node(int cpu)#ifndef set_numa_nodestatic inline void set_numa_node (int node) { -percpu_write (numa_node, node); +__this_cpu_write (numa_node, node); }#endif
@@ -270,7 +270,7 @@ DECLARE_PER_CPU(int, _numa_mem_);#ifndef set_numa_memstatic inline void set_numa_mem (int node) { -percpu_write (_numa_mem_, node); +__this_cpu_write (_numa_mem_, node); }#endif
Signature · 1 line
1.6.3.3
refreshed the patch on latest upstream kernel. Any comments or picking up are appreciated.
===
Quoted 1 line
From
0dce61dc88b8ed2687b4d5c0633aa54d1f66fdc0Mon Sep 17 00:00:00 2001
From: Alex Shi <[email protected]>
Date: Tue, 22 Nov 2011 00:05:37 +0800
Subject: [PATCH 3/3] Code clean up for percpu_xxx() functions
Since percpu_xxx() serial functions are duplicate with this_cpu_xxx(). Removing percpu_xxx() definition and replacing them by this_cpu_xxx() in code.
And further more, as Christoph Lameter's requirement, I try to use __this_cpu_xx to replace this_cpu_xxx if it is in preempt safe scenario. The preempt safe scenarios include:
1, in irq/softirq/nmi handler
2, protected by preempt_disable
3, protected by spin_lock
4, if the code context imply that it is preempt safe, like the code is follows or be followed a preempt safe code.
I left the xen code unchanged, since no idea of them.
BTW, In fact, this_cpu_xxx are same as __this_cpu_xxx since all funcs implement in a single instruction for x86 machine. But it maybe different for other platforms, so, doing this distinguish is helpful for other platforms' performance.
Signed-off-by Alex Shi <[email protected]>
--- arch/x86/include/asm/current.h | 2 +- arch/x86/include/asm/hardirq.h | 9 +++-- arch/x86/include/asm/irq_regs.h | 4 +- arch/x86/include/asm/mmu_context.h | 12 ++++---- arch/x86/include/asm/percpu.h | 24 ++++++--------- arch/x86/include/asm/smp.h | 4 +- arch/x86/include/asm/stackprotector.h | 4 +- arch/x86/include/asm/thread_info.h | 2 +- arch/x86/include/asm/tlbflush.h | 4 +- arch/x86/kernel/cpu/common.c | 2 +- arch/x86/kernel/cpu/mcheck/mce.c | 4 +- arch/x86/kernel/paravirt.c | 12 ++++---- arch/x86/kernel/process_32.c | 2 +- arch/x86/kernel/process_64.c | 12 ++++---- arch/x86/mm/tlb.c | 10 +++--- arch/x86/xen/enlighten.c | 6 ++-- arch/x86/xen/irq.c | 8 ++-- arch/x86/xen/mmu.c | 20 ++++++------ arch/x86/xen/multicalls.h | 2 +- arch/x86/xen/smp.c | 2 +- include/linux/percpu.h | 53 --------------------------------- include/linux/topology.h | 4 +- 22 files changed, 73 insertions(+), 129 deletions(-)
arch/x86/include/asm/current.h +1 -1
@@ -11,7 +11,7 @@ DECLARE_PER_CPU(struct task_struct *, current_task); static __always_inline struct task_struct *get_current(void) { -return percpu_read_stable (current_task); +return this_cpu_read_stable (current_task); }#define current get_current()
arch/x86/include/asm/hardirq.h +5 -4
@@ -35,14 +35,15 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);#define __ARCH_IRQ_STAT -#define inc_irq_stat (member) percpu_inc(irq_stat.member) +#define inc_irq_stat (member) __this_cpu_inc(irq_stat.member) -#define local_softirq_pending () percpu_read(irq_stat.__softirq_pending) +#define local_softirq_pending () __this_cpu_read(irq_stat.__softirq_pending)#define __ARCH_SET_SOFTIRQ_PENDING -#define set_softirq_pending (x) percpu_write(irq_stat.__softirq_pending, (x)) -#define or_softirq_pending (x) percpu_or(irq_stat.__softirq_pending, (x)) +#define set_softirq_pending (x) \ + __this_cpu_write(irq_stat.__softirq_pending, (x)) +#define or_softirq_pending (x) __this_cpu_or(irq_stat.__softirq_pending, (x))extern void ack_bad_irq (unsigned int irq);
arch/x86/include/asm/irq_regs.h +2 -2
@@ -15,7 +15,7 @@ DECLARE_PER_CPU(struct pt_regs *, irq_regs);static inline struct pt_regs *get_irq_regs (void ) { -return percpu_read (irq_regs); +return __this_cpu_read (irq_regs); }static inline struct pt_regs *set_irq_regs (struct pt_regs *new_regs)
@@ -23,7 +23,7 @@ static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)struct pt_regs *old_regs; old_regs =get_irq_regs (); -percpu_write (irq_regs, new_regs); +__this_cpu_write (irq_regs, new_regs);return old_regs; }
arch/x86/include/asm/mmu_context.h +6 -6
@@ -25,8 +25,8 @@ void destroy_context(struct mm_struct *mm);static inline void enter_lazy_tlb (struct mm_struct *mm,struct task_struct *tsk) {#ifdef CONFIG_SMP -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) -percpu_write (cpu_tlbstate.state,TLBSTATE_LAZY ); +if (__this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) +__this_cpu_write (cpu_tlbstate.state,TLBSTATE_LAZY );#endif }
@@ -37,8 +37,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,if (likely (prev != next)) {#ifdef CONFIG_SMP -percpu_write (cpu_tlbstate.state,TLBSTATE_OK ); -percpu_write (cpu_tlbstate.active_mm, next); +__this_cpu_write (cpu_tlbstate.state,TLBSTATE_OK ); +__this_cpu_write (cpu_tlbstate.active_mm, next);#endif cpumask_set_cpu (cpu,mm_cpumask (next));
@@ -56,8 +56,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, }#ifdef CONFIG_SMP else { -percpu_write (cpu_tlbstate.state,TLBSTATE_OK ); -BUG_ON (percpu_read (cpu_tlbstate.active_mm) != next); +__this_cpu_write (cpu_tlbstate.state,TLBSTATE_OK ); +BUG_ON (__this_cpu_read (cpu_tlbstate.active_mm) != next);if (!cpumask_test_and_set_cpu (cpu,mm_cpumask (next))) {/* We were in lazy tlb mode and leave_mm disabled
arch/x86/include/asm/percpu.h +10 -14
@@ -46,7 +46,7 @@#ifdef CONFIG_SMP #define __percpu_prefix "%%"__stringify(__percpu_seg)":" -#define __my_cpu_offset percpu_read(this_cpu_off) +#define __my_cpu_offset __this_cpu_read(this_cpu_off)/* * Compared to the generic __my_cpu_offset version, the following
@@ -351,23 +351,15 @@ do { \ })/* -* percpu_read() makes gcc load the percpu variable every time it is -* accessed while percpu_read_stable() allows the value to be cached. -* percpu_read_stable() is more efficient and can be used if its value +* this_cpu_read() makes gcc load the percpu variable every time it is +* accessed while this_cpu_read_stable() allows the value to be cached. +* this_cpu_read_stable() is more efficient and can be used if its value * is guaranteed to be valid across cpus. The current users include * get_current() and get_thread_info() both of which are actually * per-thread variables implemented as per-cpu variables and thus * stable for the duration of the respective task. */ -#define percpu_read (var) percpu_from_op("mov", var, "m" (var)) -#define percpu_read_stable (var) percpu_from_op("mov", var, "p" (&(var))) -#define percpu_write (var, val) percpu_to_op("mov", var, val) -#define percpu_add (var, val) percpu_add_op(var, val) -#define percpu_sub (var, val) percpu_add_op(var, -(val)) -#define percpu_and (var, val) percpu_to_op("and", var, val) -#define percpu_or (var, val) percpu_to_op("or", var, val) -#define percpu_xor (var, val) percpu_to_op("xor", var, val) -#define percpu_inc (var) percpu_unary_op("inc", var) +#define this_cpu_read_stable (var) percpu_from_op("mov", var, "p" (&(var)))#define __this_cpu_read_1 (pcp) percpu_from_op("mov", (pcp), "m"(pcp))#define __this_cpu_read_2 (pcp) percpu_from_op("mov", (pcp), "m"(pcp))
@@ -551,7 +543,11 @@ static __always_inline int x86_this_cpu_constant_test_bit(unsigned int nr, {unsigned long __percpu *a = (unsigned long *)addr + nr /BITS_PER_LONG ; -return ((1UL << (nr %BITS_PER_LONG )) &percpu_read (*a)) !=0 ; +#ifdef CONFIG_X86_64 +return ((1UL << (nr %BITS_PER_LONG )) &__this_cpu_read_8 (*a)) !=0 ; +#else +return ((1UL << (nr %BITS_PER_LONG )) &__this_cpu_read_4 (*a)) !=0 ; +#endif } static inline int x86_this_cpu_variable_test_bit(int nr,
arch/x86/include/asm/smp.h +2 -2
@@ -188,11 +188,11 @@ extern unsigned disabled_cpus __cpuinitdata;* from the initial startup. We map APIC_BASE very early in page_setup(), * so this is correct in the x86 case. */ -#define raw_smp_processor_id () (percpu_read(cpu_number)) +#define raw_smp_processor_id () (this_cpu_read(cpu_number))extern int safe_smp_processor_id (void );#elif defined(CONFIG_X86_64_SMP ) -#define raw_smp_processor_id () (percpu_read(cpu_number)) +#define raw_smp_processor_id () (this_cpu_read(cpu_number))#define stack_smp_processor_id () \ ({ \
arch/x86/include/asm/stackprotector.h +2 -2
@@ -76,9 +76,9 @@ static __always_inline void boot_init_stack_canary(void) current->stack_canary = canary;#ifdef CONFIG_X86_64 -percpu_write (irq_stack_union.stack_canary, canary); +__this_cpu_write (irq_stack_union.stack_canary, canary);#else -percpu_write (stack_canary.canary, canary); +__this_cpu_write (stack_canary.canary, canary);#endif }
arch/x86/include/asm/thread_info.h +1 -1
@@ -219,7 +219,7 @@ DECLARE_PER_CPU(unsigned long, kernel_stack);static inline struct thread_info *current_thread_info (void ) {struct thread_info *ti; - ti = (void *)(percpu_read_stable (kernel_stack) + + ti = (void *)(this_cpu_read_stable (kernel_stack) +KERNEL_STACK_OFFSET -THREAD_SIZE );return ti; }
arch/x86/include/asm/tlbflush.h +2 -2
@@ -156,8 +156,8 @@ DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate);static inline void reset_lazy_tlbstate (void ) { -percpu_write (cpu_tlbstate.state,0 ); -percpu_write (cpu_tlbstate.active_mm, &init_mm); +__this_cpu_write (cpu_tlbstate.state,0 ); +__this_cpu_write (cpu_tlbstate.active_mm, &init_mm); }#endif /* SMP */
arch/x86/kernel/cpu/common.c +1 -1
@@ -1163,7 +1163,7 @@ void __cpuinit cpu_init(void) oist = &per_cpu (orig_ist, cpu);#ifdef CONFIG_NUMA -if (cpu !=0 &&percpu_read (numa_node) ==0 && +if (cpu !=0 &&__this_cpu_read (numa_node) ==0 &&early_cpu_to_node (cpu) !=NUMA_NO_NODE )set_numa_node (early_cpu_to_node (cpu));#endif
arch/x86/kernel/cpu/mcheck/mce.c +2 -2
@@ -514,7 +514,7 @@ void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)struct mce m;int i; -percpu_inc (mce_poll_count); +__this_cpu_inc (mce_poll_count);mce_gather_info (&m,NULL );
@@ -906,7 +906,7 @@ void do_machine_check(struct pt_regs *regs, long error_code)atomic_inc (&mce_entry); -percpu_inc (mce_exception_count); +__this_cpu_inc (mce_exception_count);if (!banks) goto out;
arch/x86/kernel/paravirt.c +6 -6
@@ -239,16 +239,16 @@ static DEFINE_PER_CPU(enum paravirt_lazy_mode, paravirt_lazy_mode) = PARAVIRT_LAstatic inline void enter_lazy (enum paravirt_lazy_mode mode) { -BUG_ON (percpu_read (paravirt_lazy_mode) !=PARAVIRT_LAZY_NONE ); +BUG_ON (__this_cpu_read (paravirt_lazy_mode) !=PARAVIRT_LAZY_NONE ); -percpu_write (paravirt_lazy_mode, mode); +__this_cpu_write (paravirt_lazy_mode, mode); }static void leave_lazy (enum paravirt_lazy_mode mode) { -BUG_ON (percpu_read (paravirt_lazy_mode) != mode); +BUG_ON (__this_cpu_read (paravirt_lazy_mode) != mode); -percpu_write (paravirt_lazy_mode,PARAVIRT_LAZY_NONE ); +__this_cpu_write (paravirt_lazy_mode,PARAVIRT_LAZY_NONE ); }void paravirt_enter_lazy_mmu (void )
@@ -265,7 +265,7 @@ void paravirt_start_context_switch(struct task_struct *prev) {BUG_ON (preemptible ()); -if (percpu_read (paravirt_lazy_mode) ==PARAVIRT_LAZY_MMU ) { +if (__this_cpu_read (paravirt_lazy_mode) ==PARAVIRT_LAZY_MMU ) {arch_leave_lazy_mmu_mode ();set_ti_thread_flag (task_thread_info (prev),TIF_LAZY_MMU_UPDATES ); }
@@ -287,7 +287,7 @@ enum paravirt_lazy_mode paravirt_get_lazy_mode(void)if (in_interrupt ())return PARAVIRT_LAZY_NONE ; -return percpu_read (paravirt_lazy_mode); +return __this_cpu_read (paravirt_lazy_mode); }void arch_flush_lazy_mmu_mode (void )
arch/x86/kernel/process_32.c +1 -1
@@ -375,7 +375,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)if (prev->gs | next->gs)lazy_load_gs (next->gs); -percpu_write (current_task, next_p); +__this_cpu_write (current_task, next_p);return prev_p; }
arch/x86/kernel/process_64.c +6 -6
@@ -74,7 +74,7 @@ EXPORT_SYMBOL_GPL(idle_notifier_unregister);void enter_idle (void ) { -percpu_write (is_idle,1 ); +__this_cpu_write (is_idle,1 );atomic_notifier_call_chain (&idle_notifier,IDLE_START ,NULL ); }
@@ -338,7 +338,7 @@ start_thread_common(struct pt_regs *regs, unsigned long new_ip,load_gs_index (0 ); regs->ip = new_ip; regs->sp = new_sp; -percpu_write (old_rsp, new_sp); +this_cpu_write (old_rsp, new_sp); regs->cs = _cs; regs->ss = _ss; regs->flags =X86_EFLAGS_IF ;
@@ -472,11 +472,11 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)/* * Switch the PDA and FPU contexts. */ - prev->usersp =percpu_read (old_rsp); -percpu_write (old_rsp, next->usersp); -percpu_write (current_task, next_p); + prev->usersp =__this_cpu_read (old_rsp); +__this_cpu_write (old_rsp, next->usersp); +__this_cpu_write (current_task, next_p); -percpu_write (kernel_stack, +__this_cpu_write (kernel_stack, (unsigned long )task_stack_page (next_p) +THREAD_SIZE -KERNEL_STACK_OFFSET );
arch/x86/mm/tlb.c +5 -5
@@ -61,10 +61,10 @@ static DEFINE_PER_CPU_READ_MOSTLY(int, tlb_vector_offset);*/ void leave_mm (int cpu) { -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) +if (__this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_OK )BUG ();cpumask_clear_cpu (cpu, -mm_cpumask (percpu_read (cpu_tlbstate.active_mm))); +mm_cpumask (__this_cpu_read (cpu_tlbstate.active_mm)));load_cr3 (swapper_pg_dir); }EXPORT_SYMBOL_GPL (leave_mm);
@@ -152,8 +152,8 @@ void smp_invalidate_interrupt(struct pt_regs *regs)* BUG(); */ -if (f->flush_mm ==percpu_read (cpu_tlbstate.active_mm)) { -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) { +if (f->flush_mm ==__this_cpu_read (cpu_tlbstate.active_mm)) { +if (__this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) {if (f->flush_va ==TLB_FLUSH_ALL )local_flush_tlb ();else
@@ -322,7 +322,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)static void do_flush_tlb_all (void *info) {__flush_tlb_all (); -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_LAZY ) +if (__this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_LAZY )leave_mm (smp_processor_id ()); }
arch/x86/xen/enlighten.c +3 -3
@@ -777,11 +777,11 @@ static DEFINE_PER_CPU(unsigned long, xen_cr0_value);static unsigned long xen_read_cr0 (void ) { -unsigned long cr0 =percpu_read (xen_cr0_value); +unsigned long cr0 =this_cpu_read (xen_cr0_value);if (unlikely (cr0 ==0 )) { cr0 =native_read_cr0 (); -percpu_write (xen_cr0_value, cr0); +this_cpu_write (xen_cr0_value, cr0); }return cr0;
@@ -791,7 +791,7 @@ static void xen_write_cr0(unsigned long cr0) {struct multicall_space mcs; -percpu_write (xen_cr0_value, cr0); +this_cpu_write (xen_cr0_value, cr0);/* Only pay attention to cr0.TS; everything else is ignored. */
arch/x86/xen/irq.c +4 -4
@@ -26,7 +26,7 @@ static unsigned long xen_save_fl(void)struct vcpu_info *vcpu;unsigned long flags; - vcpu =percpu_read (xen_vcpu); + vcpu =this_cpu_read (xen_vcpu);/* flag has opposite sense of mask */ flags = !vcpu->evtchn_upcall_mask;
@@ -50,7 +50,7 @@ static void xen_restore_fl(unsigned long flags)make sure we're don't switch CPUs between getting the vcpu pointer and updating the mask. */ preempt_disable (); - vcpu =percpu_read (xen_vcpu); + vcpu =this_cpu_read (xen_vcpu); vcpu->evtchn_upcall_mask = flags;preempt_enable_no_resched ();
@@ -72,7 +72,7 @@ static void xen_irq_disable(void)make sure we're don't switch CPUs between getting the vcpu pointer and updating the mask. */ preempt_disable (); -percpu_read (xen_vcpu)->evtchn_upcall_mask =1 ; +this_cpu_read (xen_vcpu)->evtchn_upcall_mask =1 ;preempt_enable_no_resched (); }PV_CALLEE_SAVE_REGS_THUNK (xen_irq_disable);
@@ -86,7 +86,7 @@ static void xen_irq_enable(void)the caller is confused and is trying to re-enable interrupts on an indeterminate processor. */ - vcpu =percpu_read (xen_vcpu); + vcpu =this_cpu_read (xen_vcpu); vcpu->evtchn_upcall_mask =0 ;/* Doesn't matter if we get preempted here, because any
arch/x86/xen/mmu.c +10 -10
@@ -1071,14 +1071,14 @@ static void drop_other_mm_ref(void *info)struct mm_struct *mm = info;struct mm_struct *active_mm; - active_mm =percpu_read (cpu_tlbstate.active_mm); + active_mm =this_cpu_read (cpu_tlbstate.active_mm); -if (active_mm == mm &&percpu_read (cpu_tlbstate.state) !=TLBSTATE_OK ) +if (active_mm == mm &&this_cpu_read (cpu_tlbstate.state) !=TLBSTATE_OK )leave_mm (smp_processor_id ());/* If this cpu still has a stale cr3 reference, then make sure it has been flushed. */ -if (percpu_read (xen_current_cr3) ==__pa (mm->pgd)) +if (this_cpu_read (xen_current_cr3) ==__pa (mm->pgd))load_cr3 (swapper_pg_dir); }
@@ -1185,17 +1185,17 @@ static void __init xen_pagetable_setup_done(pgd_t *base)static void xen_write_cr2 (unsigned long cr2) { -percpu_read (xen_vcpu)->arch.cr2 = cr2; +this_cpu_read (xen_vcpu)->arch.cr2 = cr2; }static unsigned long xen_read_cr2 (void ) { -return percpu_read (xen_vcpu)->arch.cr2; +return this_cpu_read (xen_vcpu)->arch.cr2; }unsigned long xen_read_cr2_direct (void ) { -return percpu_read (xen_vcpu_info.arch.cr2); +return this_cpu_read (xen_vcpu_info.arch.cr2); }static void xen_flush_tlb (void )
@@ -1278,12 +1278,12 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,static unsigned long xen_read_cr3 (void ) { -return percpu_read (xen_cr3); +return this_cpu_read (xen_cr3); }static void set_current_cr3 (void *v) { -percpu_write (xen_current_cr3, (unsigned long )v); +this_cpu_write (xen_current_cr3, (unsigned long )v); }static void __xen_write_cr3 (bool kernel,unsigned long cr3)
@@ -1306,7 +1306,7 @@ static void __xen_write_cr3(bool kernel, unsigned long cr3)xen_extend_mmuext_op (&op);if (kernel) { -percpu_write (xen_cr3, cr3); +this_cpu_write (xen_cr3, cr3);/* Update xen_current_cr3 once the batch has actually been submitted. */
@@ -1322,7 +1322,7 @@ static void xen_write_cr3(unsigned long cr3)/* Update while interrupts are disabled, so its atomic with respect to ipis */ -percpu_write (xen_cr3, cr3); +this_cpu_write (xen_cr3, cr3);__xen_write_cr3 (true, cr3);
arch/x86/xen/multicalls.h +1 -1
@@ -47,7 +47,7 @@ static inline void xen_mc_issue(unsigned mode)xen_mc_flush ();/* restore flags saved in xen_mc_batch */ -local_irq_restore (percpu_read (xen_mc_irq_flags)); +local_irq_restore (this_cpu_read (xen_mc_irq_flags)); }/* Set up a callback to be called when the current batch is flushed */
arch/x86/xen/smp.c +1 -1
@@ -76,7 +76,7 @@ static void __cpuinit cpu_bringup(void)xen_setup_cpu_clockevents ();set_cpu_online (cpu, true); -percpu_write (cpu_state,CPU_ONLINE ); +this_cpu_write (cpu_state,CPU_ONLINE );wmb ();/* We can take interrupts now: we're officially "up". */
include/linux/percpu.h +0 -53
59 lines not on this page · raw message
include/linux/topology.h +2 -2
@@ -239,7 +239,7 @@ static inline int cpu_to_node(int cpu)#ifndef set_numa_nodestatic inline void set_numa_node (int node) { -percpu_write (numa_node, node); +__this_cpu_write (numa_node, node); }#endif
@@ -274,7 +274,7 @@ DECLARE_PER_CPU(int, _numa_mem_);#ifndef set_numa_memstatic inline void set_numa_mem (int node) { -percpu_write (_numa_mem_, node); +__this_cpu_write (_numa_mem_, node); }#endif
Signature · 1 line
1.7.5.1
(cc'ing hpa and quoting whole body)
Quoted 24 lines · Alex,Shi
refreshed the patch on latest upstream kernel. Any comments or picking up are appreciated.
===
Quoted 1 line
From
0dce61dc88b8ed2687b4d5c0633aa54d1f66fdc0Mon Sep 17 00:00:00 2001From: Alex Shi <[email protected]>
Date: Tue, 22 Nov 2011 00:05:37 +0800
Subject: [PATCH 3/3] Code clean up forpercpu_xxx()functionsSince
percpu_xxx()serial functions are duplicate withthis_cpu_xxx(). Removingpercpu_xxx()definition and replacing them bythis_cpu_xxx()in code.And further more, as Christoph Lameter's requirement, I try to use
__this_cpu_xxto replacethis_cpu_xxxif it is in preempt safe scenario. The preempt safe scenarios include:
1, in irq/softirq/nmi handler
2, protected bypreempt_disable
3, protected byspin_lock
4, if the code context imply that it is preempt safe, like the code is follows or be followed a preempt safe code.I left the xen code unchanged, since no idea of them.
Quoted 7 lines
BTW, In fact,
this_cpu_xxxare same as__this_cpu_xxxsince all funcs implement in a single instruction for x86 machine. But it maybe different for other platforms, so, doing this distinguish is helpful for other platforms' performance.Signed-off-by Alex Shi <[email protected]>
Acked-by Tejun Heo <[email protected]>
hpa, I suppose this should go through x86? The original patch can be accessed at
http://article.gmane.org/gmane.linux.kernel/1218055/raw
Thanks.
Quoted 662 lines
arch/x86/include/asm/current.h | 2 +- arch/x86/include/asm/hardirq.h | 9 +++-- arch/x86/include/asm/irq_regs.h | 4 +- arch/x86/include/asm/mmu_context.h | 12 ++++---- arch/x86/include/asm/percpu.h | 24 ++++++--------- arch/x86/include/asm/smp.h | 4 +- arch/x86/include/asm/stackprotector.h | 4 +- arch/x86/include/asm/thread_info.h | 2 +- arch/x86/include/asm/tlbflush.h | 4 +- arch/x86/kernel/cpu/common.c | 2 +- arch/x86/kernel/cpu/mcheck/mce.c | 4 +- arch/x86/kernel/paravirt.c | 12 ++++---- arch/x86/kernel/process_32.c | 2 +- arch/x86/kernel/process_64.c | 12 ++++---- arch/x86/mm/tlb.c | 10 +++--- arch/x86/xen/enlighten.c | 6 ++-- arch/x86/xen/irq.c | 8 ++-- arch/x86/xen/mmu.c | 20 ++++++------ arch/x86/xen/multicalls.h | 2 +- arch/x86/xen/smp.c | 2 +- include/linux/percpu.h | 53 --------------------------------- include/linux/topology.h | 4 +- 22 files changed, 73 insertions(+), 129 deletions(-)arch/x86/include/asm/current.h +1
-1@@ -11,7 +11,7 @@ DECLARE_PER_CPU(struct task_struct *, current_task); static __always_inline struct task_struct *get_current(void) { -return percpu_read_stable (current_task); +return this_cpu_read_stable (current_task); }#define current get_current()arch/x86/include/asm/hardirq.h +5
-4@@ -35,14 +35,15 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);#define __ARCH_IRQ_STAT -#define inc_irq_stat (member) percpu_inc(irq_stat.member) +#define inc_irq_stat (member) __this_cpu_inc(irq_stat.member) -#define local_softirq_pending () percpu_read(irq_stat.__softirq_pending) +#define local_softirq_pending () __this_cpu_read(irq_stat.__softirq_pending)#define __ARCH_SET_SOFTIRQ_PENDING -#define set_softirq_pending (x) percpu_write(irq_stat.__softirq_pending, (x)) -#define or_softirq_pending (x) percpu_or(irq_stat.__softirq_pending, (x)) +#define set_softirq_pending (x) \ + __this_cpu_write(irq_stat.__softirq_pending, (x)) +#define or_softirq_pending (x) __this_cpu_or(irq_stat.__softirq_pending, (x))extern void ack_bad_irq (unsigned int irq);arch/x86/include/asm/irq_regs.h +2
-2@@ -15,7 +15,7 @@ DECLARE_PER_CPU(struct pt_regs *, irq_regs);static inline struct pt_regs *get_irq_regs (void ) { -return percpu_read (irq_regs); +return __this_cpu_read (irq_regs); }static inline struct pt_regs *set_irq_regs (struct pt_regs *new_regs)@@ -23,7 +23,7 @@ static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)struct pt_regs *old_regs; old_regs =get_irq_regs (); -percpu_write (irq_regs, new_regs); +__this_cpu_write (irq_regs, new_regs);return old_regs; }arch/x86/include/asm/mmu_context.h +6
-6@@ -25,8 +25,8 @@ void destroy_context(struct mm_struct *mm);static inline void enter_lazy_tlb (struct mm_struct *mm,struct task_struct *tsk) {#ifdef CONFIG_SMP -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) -percpu_write (cpu_tlbstate.state,TLBSTATE_LAZY ); +if (__this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) +__this_cpu_write (cpu_tlbstate.state,TLBSTATE_LAZY );#endif }@@ -37,8 +37,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,if (likely (prev != next)) {#ifdef CONFIG_SMP -percpu_write (cpu_tlbstate.state,TLBSTATE_OK ); -percpu_write (cpu_tlbstate.active_mm, next); +__this_cpu_write (cpu_tlbstate.state,TLBSTATE_OK ); +__this_cpu_write (cpu_tlbstate.active_mm, next);#endif cpumask_set_cpu (cpu,mm_cpumask (next));@@ -56,8 +56,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, }#ifdef CONFIG_SMP else { -percpu_write (cpu_tlbstate.state,TLBSTATE_OK ); -BUG_ON (percpu_read (cpu_tlbstate.active_mm) != next); +__this_cpu_write (cpu_tlbstate.state,TLBSTATE_OK ); +BUG_ON (__this_cpu_read (cpu_tlbstate.active_mm) != next);if (!cpumask_test_and_set_cpu (cpu,mm_cpumask (next))) {/* We were in lazy tlb mode and leave_mm disabled arch/x86/include/asm/percpu.h +10
-14@@ -46,7 +46,7 @@#ifdef CONFIG_SMP #define __percpu_prefix "%%"__stringify(__percpu_seg)":" -#define __my_cpu_offset percpu_read(this_cpu_off) +#define __my_cpu_offset __this_cpu_read(this_cpu_off)/* * Compared to the generic __my_cpu_offset version, the following @@ -351,23 +351,15 @@ do { \ })/* -* percpu_read() makes gcc load the percpu variable every time it is -* accessed while percpu_read_stable() allows the value to be cached. -* percpu_read_stable() is more efficient and can be used if its value +* this_cpu_read() makes gcc load the percpu variable every time it is +* accessed while this_cpu_read_stable() allows the value to be cached. +* this_cpu_read_stable() is more efficient and can be used if its value * is guaranteed to be valid across cpus. The current users include * get_current() and get_thread_info() both of which are actually * per-thread variables implemented as per-cpu variables and thus * stable for the duration of the respective task. */ -#define percpu_read (var) percpu_from_op("mov", var, "m" (var)) -#define percpu_read_stable (var) percpu_from_op("mov", var, "p" (&(var))) -#define percpu_write (var, val) percpu_to_op("mov", var, val) -#define percpu_add (var, val) percpu_add_op(var, val) -#define percpu_sub (var, val) percpu_add_op(var, -(val)) -#define percpu_and (var, val) percpu_to_op("and", var, val) -#define percpu_or (var, val) percpu_to_op("or", var, val) -#define percpu_xor (var, val) percpu_to_op("xor", var, val) -#define percpu_inc (var) percpu_unary_op("inc", var) +#define this_cpu_read_stable (var) percpu_from_op("mov", var, "p" (&(var)))#define __this_cpu_read_1 (pcp) percpu_from_op("mov", (pcp), "m"(pcp))#define __this_cpu_read_2 (pcp) percpu_from_op("mov", (pcp), "m"(pcp))@@ -551,7 +543,11 @@ static __always_inline int x86_this_cpu_constant_test_bit(unsigned int nr, {unsigned long __percpu *a = (unsigned long *)addr + nr /BITS_PER_LONG ; -return ((1UL << (nr %BITS_PER_LONG )) &percpu_read (*a)) !=0 ; +#ifdef CONFIG_X86_64 +return ((1UL << (nr %BITS_PER_LONG )) &__this_cpu_read_8 (*a)) !=0 ; +#else +return ((1UL << (nr %BITS_PER_LONG )) &__this_cpu_read_4 (*a)) !=0 ; +#endif } static inline int x86_this_cpu_variable_test_bit(int nr,arch/x86/include/asm/smp.h +2
-2@@ -188,11 +188,11 @@ extern unsigned disabled_cpus __cpuinitdata;* from the initial startup. We map APIC_BASE very early in page_setup(), * so this is correct in the x86 case. */ -#define raw_smp_processor_id () (percpu_read(cpu_number)) +#define raw_smp_processor_id () (this_cpu_read(cpu_number))extern int safe_smp_processor_id (void );#elif defined(CONFIG_X86_64_SMP ) -#define raw_smp_processor_id () (percpu_read(cpu_number)) +#define raw_smp_processor_id () (this_cpu_read(cpu_number))#define stack_smp_processor_id () \ ({ \arch/x86/include/asm/stackprotector.h +2
-2@@ -76,9 +76,9 @@ static __always_inline void boot_init_stack_canary(void) current->stack_canary = canary;#ifdef CONFIG_X86_64 -percpu_write (irq_stack_union.stack_canary, canary); +__this_cpu_write (irq_stack_union.stack_canary, canary);#else -percpu_write (stack_canary.canary, canary); +__this_cpu_write (stack_canary.canary, canary);#endif }arch/x86/include/asm/thread_info.h +1
-1@@ -219,7 +219,7 @@ DECLARE_PER_CPU(unsigned long, kernel_stack);static inline struct thread_info *current_thread_info (void ) {struct thread_info *ti; - ti = (void *)(percpu_read_stable (kernel_stack) + + ti = (void *)(this_cpu_read_stable (kernel_stack) +KERNEL_STACK_OFFSET -THREAD_SIZE );return ti; }arch/x86/include/asm/tlbflush.h +2
-2@@ -156,8 +156,8 @@ DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate);static inline void reset_lazy_tlbstate (void ) { -percpu_write (cpu_tlbstate.state,0 ); -percpu_write (cpu_tlbstate.active_mm, &init_mm); +__this_cpu_write (cpu_tlbstate.state,0 ); +__this_cpu_write (cpu_tlbstate.active_mm, &init_mm); }#endif /* SMP */ arch/x86/kernel/cpu/common.c +1
-1@@ -1163,7 +1163,7 @@ void __cpuinit cpu_init(void) oist = &per_cpu (orig_ist, cpu);#ifdef CONFIG_NUMA -if (cpu !=0 &&percpu_read (numa_node) ==0 && +if (cpu !=0 &&__this_cpu_read (numa_node) ==0 &&early_cpu_to_node (cpu) !=NUMA_NO_NODE )set_numa_node (early_cpu_to_node (cpu));#endif arch/x86/kernel/cpu/mcheck/mce.c +2
-2@@ -514,7 +514,7 @@ void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)struct mce m;int i; -percpu_inc (mce_poll_count); +__this_cpu_inc (mce_poll_count);mce_gather_info (&m,NULL );@@ -906,7 +906,7 @@ void do_machine_check(struct pt_regs *regs, long error_code)atomic_inc (&mce_entry); -percpu_inc (mce_exception_count); +__this_cpu_inc (mce_exception_count);if (!banks) goto out;arch/x86/kernel/paravirt.c +6
-6@@ -239,16 +239,16 @@ static DEFINE_PER_CPU(enum paravirt_lazy_mode, paravirt_lazy_mode) = PARAVIRT_LAstatic inline void enter_lazy (enum paravirt_lazy_mode mode) { -BUG_ON (percpu_read (paravirt_lazy_mode) !=PARAVIRT_LAZY_NONE ); +BUG_ON (__this_cpu_read (paravirt_lazy_mode) !=PARAVIRT_LAZY_NONE ); -percpu_write (paravirt_lazy_mode, mode); +__this_cpu_write (paravirt_lazy_mode, mode); }static void leave_lazy (enum paravirt_lazy_mode mode) { -BUG_ON (percpu_read (paravirt_lazy_mode) != mode); +BUG_ON (__this_cpu_read (paravirt_lazy_mode) != mode); -percpu_write (paravirt_lazy_mode,PARAVIRT_LAZY_NONE ); +__this_cpu_write (paravirt_lazy_mode,PARAVIRT_LAZY_NONE ); }void paravirt_enter_lazy_mmu (void )@@ -265,7 +265,7 @@ void paravirt_start_context_switch(struct task_struct *prev) {BUG_ON (preemptible ()); -if (percpu_read (paravirt_lazy_mode) ==PARAVIRT_LAZY_MMU ) { +if (__this_cpu_read (paravirt_lazy_mode) ==PARAVIRT_LAZY_MMU ) {arch_leave_lazy_mmu_mode ();set_ti_thread_flag (task_thread_info (prev),TIF_LAZY_MMU_UPDATES ); }@@ -287,7 +287,7 @@ enum paravirt_lazy_mode paravirt_get_lazy_mode(void)if (in_interrupt ())return PARAVIRT_LAZY_NONE ; -return percpu_read (paravirt_lazy_mode); +return __this_cpu_read (paravirt_lazy_mode); }void arch_flush_lazy_mmu_mode (void )arch/x86/kernel/process_32.c +1
-1@@ -375,7 +375,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)if (prev->gs | next->gs)lazy_load_gs (next->gs); -percpu_write (current_task, next_p); +__this_cpu_write (current_task, next_p);return prev_p; }arch/x86/kernel/process_64.c +6
-6@@ -74,7 +74,7 @@ EXPORT_SYMBOL_GPL(idle_notifier_unregister);void enter_idle (void ) { -percpu_write (is_idle,1 ); +__this_cpu_write (is_idle,1 );atomic_notifier_call_chain (&idle_notifier,IDLE_START ,NULL ); }@@ -338,7 +338,7 @@ start_thread_common(struct pt_regs *regs, unsigned long new_ip,load_gs_index (0 ); regs->ip = new_ip; regs->sp = new_sp; -percpu_write (old_rsp, new_sp); +this_cpu_write (old_rsp, new_sp); regs->cs = _cs; regs->ss = _ss; regs->flags =X86_EFLAGS_IF ;@@ -472,11 +472,11 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)/* * Switch the PDA and FPU contexts. */ - prev->usersp =percpu_read (old_rsp); -percpu_write (old_rsp, next->usersp); -percpu_write (current_task, next_p); + prev->usersp =__this_cpu_read (old_rsp); +__this_cpu_write (old_rsp, next->usersp); +__this_cpu_write (current_task, next_p); -percpu_write (kernel_stack, +__this_cpu_write (kernel_stack, (unsigned long )task_stack_page (next_p) +THREAD_SIZE -KERNEL_STACK_OFFSET );arch/x86/mm/tlb.c +5
-5@@ -61,10 +61,10 @@ static DEFINE_PER_CPU_READ_MOSTLY(int, tlb_vector_offset);*/ void leave_mm (int cpu) { -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) +if (__this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_OK )BUG ();cpumask_clear_cpu (cpu, -mm_cpumask (percpu_read (cpu_tlbstate.active_mm))); +mm_cpumask (__this_cpu_read (cpu_tlbstate.active_mm)));load_cr3 (swapper_pg_dir); }EXPORT_SYMBOL_GPL (leave_mm);@@ -152,8 +152,8 @@ void smp_invalidate_interrupt(struct pt_regs *regs)* BUG(); */ -if (f->flush_mm ==percpu_read (cpu_tlbstate.active_mm)) { -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) { +if (f->flush_mm ==__this_cpu_read (cpu_tlbstate.active_mm)) { +if (__this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) {if (f->flush_va ==TLB_FLUSH_ALL )local_flush_tlb ();else @@ -322,7 +322,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)static void do_flush_tlb_all (void *info) {__flush_tlb_all (); -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_LAZY ) +if (__this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_LAZY )leave_mm (smp_processor_id ()); }arch/x86/xen/enlighten.c +3
-3@@ -777,11 +777,11 @@ static DEFINE_PER_CPU(unsigned long, xen_cr0_value);static unsigned long xen_read_cr0 (void ) { -unsigned long cr0 =percpu_read (xen_cr0_value); +unsigned long cr0 =this_cpu_read (xen_cr0_value);if (unlikely (cr0 ==0 )) { cr0 =native_read_cr0 (); -percpu_write (xen_cr0_value, cr0); +this_cpu_write (xen_cr0_value, cr0); }return cr0;@@ -791,7 +791,7 @@ static void xen_write_cr0(unsigned long cr0) {struct multicall_space mcs; -percpu_write (xen_cr0_value, cr0); +this_cpu_write (xen_cr0_value, cr0);/* Only pay attention to cr0.TS; everything else is ignored. */ arch/x86/xen/irq.c +4
-4@@ -26,7 +26,7 @@ static unsigned long xen_save_fl(void)struct vcpu_info *vcpu;unsigned long flags; - vcpu =percpu_read (xen_vcpu); + vcpu =this_cpu_read (xen_vcpu);/* flag has opposite sense of mask */ flags = !vcpu->evtchn_upcall_mask;@@ -50,7 +50,7 @@ static void xen_restore_fl(unsigned long flags)make sure we're don't switch CPUs between getting the vcpu pointer and updating the mask. */ preempt_disable (); - vcpu =percpu_read (xen_vcpu); + vcpu =this_cpu_read (xen_vcpu); vcpu->evtchn_upcall_mask = flags;preempt_enable_no_resched ();@@ -72,7 +72,7 @@ static void xen_irq_disable(void)make sure we're don't switch CPUs between getting the vcpu pointer and updating the mask. */ preempt_disable (); -percpu_read (xen_vcpu)->evtchn_upcall_mask =1 ; +this_cpu_read (xen_vcpu)->evtchn_upcall_mask =1 ;preempt_enable_no_resched (); }PV_CALLEE_SAVE_REGS_THUNK (xen_irq_disable);@@ -86,7 +86,7 @@ static void xen_irq_enable(void)the caller is confused and is trying to re-enable interrupts on an indeterminate processor. */ - vcpu =percpu_read (xen_vcpu); + vcpu =this_cpu_read (xen_vcpu); vcpu->evtchn_upcall_mask =0 ;/* Doesn't matter if we get preempted here, because any arch/x86/xen/mmu.c +10
-10@@ -1071,14 +1071,14 @@ static void drop_other_mm_ref(void *info)struct mm_struct *mm = info;struct mm_struct *active_mm; - active_mm =percpu_read (cpu_tlbstate.active_mm); + active_mm =this_cpu_read (cpu_tlbstate.active_mm); -if (active_mm == mm &&percpu_read (cpu_tlbstate.state) !=TLBSTATE_OK ) +if (active_mm == mm &&this_cpu_read (cpu_tlbstate.state) !=TLBSTATE_OK )leave_mm (smp_processor_id ());/* If this cpu still has a stale cr3 reference, then make sure it has been flushed. */ -if (percpu_read (xen_current_cr3) ==__pa (mm->pgd)) +if (this_cpu_read (xen_current_cr3) ==__pa (mm->pgd))load_cr3 (swapper_pg_dir); }@@ -1185,17 +1185,17 @@ static void __init xen_pagetable_setup_done(pgd_t *base)static void xen_write_cr2 (unsigned long cr2) { -percpu_read (xen_vcpu)->arch.cr2 = cr2; +this_cpu_read (xen_vcpu)->arch.cr2 = cr2; }static unsigned long xen_read_cr2 (void ) { -return percpu_read (xen_vcpu)->arch.cr2; +return this_cpu_read (xen_vcpu)->arch.cr2; }unsigned long xen_read_cr2_direct (void ) { -return percpu_read (xen_vcpu_info.arch.cr2); +return this_cpu_read (xen_vcpu_info.arch.cr2); }static void xen_flush_tlb (void )@@ -1278,12 +1278,12 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,static unsigned long xen_read_cr3 (void ) { -return percpu_read (xen_cr3); +return this_cpu_read (xen_cr3); }static void set_current_cr3 (void *v) { -percpu_write (xen_current_cr3, (unsigned long )v); +this_cpu_write (xen_current_cr3, (unsigned long )v); }static void __xen_write_cr3 (bool kernel,unsigned long cr3)@@ -1306,7 +1306,7 @@ static void __xen_write_cr3(bool kernel, unsigned long cr3)xen_extend_mmuext_op (&op);if (kernel) { -percpu_write (xen_cr3, cr3); +this_cpu_write (xen_cr3, cr3);/* Update xen_current_cr3 once the batch has actually been submitted. */ @@ -1322,7 +1322,7 @@ static void xen_write_cr3(unsigned long cr3)/* Update while interrupts are disabled, so its atomic with respect to ipis */ -percpu_write (xen_cr3, cr3); +this_cpu_write (xen_cr3, cr3);__xen_write_cr3 (true, cr3);arch/x86/xen/multicalls.h +1
-1@@ -47,7 +47,7 @@ static inline void xen_mc_issue(unsigned mode)xen_mc_flush ();/* restore flags saved in xen_mc_batch */ -local_irq_restore (percpu_read (xen_mc_irq_flags)); +local_irq_restore (this_cpu_read (xen_mc_irq_flags)); }/* Set up a callback to be called when the current batch is flushed */ arch/x86/xen/smp.c +1
-1@@ -76,7 +76,7 @@ static void __cpuinit cpu_bringup(void)xen_setup_cpu_clockevents ();set_cpu_online (cpu, true); -percpu_write (cpu_state,CPU_ONLINE ); +this_cpu_write (cpu_state,CPU_ONLINE );wmb ();/* We can take interrupts now: we're officially "up". */ include/linux/percpu.h +0
-53@@ -165,59 +165,6 @@ extern phys_addr_t per_cpu_ptr_to_phys(void *addr);#define alloc_percpu (type) \ (typeof(type) __percpu *)__alloc_percpu(sizeof(type), __alignof__(type)) -/* -* Optional methods for optimized non-lvalue per-cpu variable access. -* -* @var can be a percpu variable or a field of it and its size should -* equal char, int or long. percpu_read() evaluates to a lvalue and -* all others to void. -* -* These operations are guaranteed to be atomic w.r.t. preemption. -* The generic versions use plain get/put_cpu_var(). Archs are -* encouraged to implement single-instruction alternatives which don't -* require preemption protection. -*/ -#ifndef percpu_read -# define percpu_read (var) \ - ({ \ - typeof(var) *pr_ptr__ = &(var); \ - typeof(var) pr_ret__; \ - pr_ret__ = get_cpu_var(*pr_ptr__); \ - put_cpu_var(*pr_ptr__); \ - pr_ret__; \ - }) -#endif - -#define __percpu_generic_to_op (var, val, op) \ -do { \ - typeof(var) *pgto_ptr__ = &(var); \ - get_cpu_var(*pgto_ptr__) op val; \ - put_cpu_var(*pgto_ptr__); \ -} while (0) - -#ifndef percpu_write -# define percpu_write (var, val) __percpu_generic_to_op(var, (val), =) -#endif - -#ifndef percpu_add -# define percpu_add (var, val) __percpu_generic_to_op(var, (val), +=) -#endif - -#ifndef percpu_sub -# define percpu_sub (var, val) __percpu_generic_to_op(var, (val), -=) -#endif - -#ifndef percpu_and -# define percpu_and (var, val) __percpu_generic_to_op(var, (val), &=) -#endif - -#ifndef percpu_or -# define percpu_or (var, val) __percpu_generic_to_op(var, (val), |=) -#endif - -#ifndef percpu_xor -# define percpu_xor (var, val) __percpu_generic_to_op(var, (val), ^=) -#endif /* * Branching function to split up a function into a set of functions that include/linux/topology.h +2
-2@@ -239,7 +239,7 @@ static inline int cpu_to_node(int cpu)#ifndef set_numa_nodestatic inline void set_numa_node (int node) { -percpu_write (numa_node, node); +__this_cpu_write (numa_node, node); }#endif @@ -274,7 +274,7 @@ DECLARE_PER_CPU(int, _numa_mem_);#ifndef set_numa_memstatic inline void set_numa_mem (int node) { -percpu_write (_numa_mem_, node); +__this_cpu_write (_numa_mem_, node); }#endif Signature · 1 line
1.7.5.1
Signature · 1 line
tejun
Quoted 11 lines · On Mon, 2011-11-21 at 17:06 -0700, [email protected] wrote:
(cc'ing hpa and quoting whole body)
Quoted 3 lines
Signed-off-by Alex Shi <[email protected]>Acked-by Tejun Heo <[email protected]>hpa, I suppose this should go through x86? The original patch can be accessed at
Rend for 3.2 kernel, no any change needed to apply on latest Linus' tree. :)
Actually, this clean up has no performance or security impact for kernel. On the contrary, removing some potential redundant preempt disable will bring a slight performance benefit to kernel.
This 3rd patch depends on previous 2 patches, the 2nd one kvm code clean up was submitted for 3.3 kernel. but the 2st one net code clean up is waiting for David's comments.
----------
Quoted 1 line
From
0dce61dc88b8ed2687b4d5c0633aa54d1f66fdc0Mon Sep 17 00:00:00 2001
From: Alex Shi <[email protected]>
Date: Tue, 22 Nov 2011 00:05:37 +0800
Subject: [PATCH 3/3] Code clean up for percpu_xxx() functions
Since percpu_xxx() serial functions are duplicate with this_cpu_xxx(). Removing percpu_xxx() definition and replacing them by this_cpu_xxx() in code.
And further more, as Christoph Lameter's requirement, I try to use __this_cpu_xx to replace this_cpu_xxx if it is in preempt safe scenario. The preempt safe scenarios include:
1, in irq/softirq/nmi handler
2, protected by preempt_disable
3, protected by spin_lock
4, if the code context imply that it is preempt safe, like the code is follows or be followed a preempt safe code.
I left the xen code unchanged, since no idea of them.
BTW, In fact, this_cpu_xxx are same as __this_cpu_xxx since all funcs implement in a single instruction for x86 machine. But it maybe different for other platforms, so, doing this distinguish is helpful for other platforms' performance.
Signed-off-by Alex Shi <[email protected]>
Acked-by Tejun Heo <[email protected]>
--- arch/x86/include/asm/current.h | 2 +- arch/x86/include/asm/hardirq.h | 9 +++-- arch/x86/include/asm/irq_regs.h | 4 +- arch/x86/include/asm/mmu_context.h | 12 ++++---- arch/x86/include/asm/percpu.h | 24 ++++++--------- arch/x86/include/asm/smp.h | 4 +- arch/x86/include/asm/stackprotector.h | 4 +- arch/x86/include/asm/thread_info.h | 2 +- arch/x86/include/asm/tlbflush.h | 4 +- arch/x86/kernel/cpu/common.c | 2 +- arch/x86/kernel/cpu/mcheck/mce.c | 4 +- arch/x86/kernel/paravirt.c | 12 ++++---- arch/x86/kernel/process_32.c | 2 +- arch/x86/kernel/process_64.c | 12 ++++---- arch/x86/mm/tlb.c | 10 +++--- arch/x86/xen/enlighten.c | 6 ++-- arch/x86/xen/irq.c | 8 ++-- arch/x86/xen/mmu.c | 20 ++++++------ arch/x86/xen/multicalls.h | 2 +- arch/x86/xen/smp.c | 2 +- include/linux/percpu.h | 53 --------------------------------- include/linux/topology.h | 4 +- 22 files changed, 73 insertions(+), 129 deletions(-)
arch/x86/include/asm/current.h +1 -1
@@ -11,7 +11,7 @@ DECLARE_PER_CPU(struct task_struct *, current_task); static __always_inline struct task_struct *get_current(void) { -return percpu_read_stable (current_task); +return this_cpu_read_stable (current_task); }#define current get_current()
arch/x86/include/asm/hardirq.h +5 -4
@@ -35,14 +35,15 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);#define __ARCH_IRQ_STAT -#define inc_irq_stat (member) percpu_inc(irq_stat.member) +#define inc_irq_stat (member) __this_cpu_inc(irq_stat.member) -#define local_softirq_pending () percpu_read(irq_stat.__softirq_pending) +#define local_softirq_pending () __this_cpu_read(irq_stat.__softirq_pending)#define __ARCH_SET_SOFTIRQ_PENDING -#define set_softirq_pending (x) percpu_write(irq_stat.__softirq_pending, (x)) -#define or_softirq_pending (x) percpu_or(irq_stat.__softirq_pending, (x)) +#define set_softirq_pending (x) \ + __this_cpu_write(irq_stat.__softirq_pending, (x)) +#define or_softirq_pending (x) __this_cpu_or(irq_stat.__softirq_pending, (x))extern void ack_bad_irq (unsigned int irq);
arch/x86/include/asm/irq_regs.h +2 -2
@@ -15,7 +15,7 @@ DECLARE_PER_CPU(struct pt_regs *, irq_regs);static inline struct pt_regs *get_irq_regs (void ) { -return percpu_read (irq_regs); +return __this_cpu_read (irq_regs); }static inline struct pt_regs *set_irq_regs (struct pt_regs *new_regs)
@@ -23,7 +23,7 @@ static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)struct pt_regs *old_regs; old_regs =get_irq_regs (); -percpu_write (irq_regs, new_regs); +__this_cpu_write (irq_regs, new_regs);return old_regs; }
arch/x86/include/asm/mmu_context.h +6 -6
@@ -25,8 +25,8 @@ void destroy_context(struct mm_struct *mm);static inline void enter_lazy_tlb (struct mm_struct *mm,struct task_struct *tsk) {#ifdef CONFIG_SMP -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) -percpu_write (cpu_tlbstate.state,TLBSTATE_LAZY ); +if (__this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) +__this_cpu_write (cpu_tlbstate.state,TLBSTATE_LAZY );#endif }
@@ -37,8 +37,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,if (likely (prev != next)) {#ifdef CONFIG_SMP -percpu_write (cpu_tlbstate.state,TLBSTATE_OK ); -percpu_write (cpu_tlbstate.active_mm, next); +__this_cpu_write (cpu_tlbstate.state,TLBSTATE_OK ); +__this_cpu_write (cpu_tlbstate.active_mm, next);#endif cpumask_set_cpu (cpu,mm_cpumask (next));
@@ -56,8 +56,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, }#ifdef CONFIG_SMP else { -percpu_write (cpu_tlbstate.state,TLBSTATE_OK ); -BUG_ON (percpu_read (cpu_tlbstate.active_mm) != next); +__this_cpu_write (cpu_tlbstate.state,TLBSTATE_OK ); +BUG_ON (__this_cpu_read (cpu_tlbstate.active_mm) != next);if (!cpumask_test_and_set_cpu (cpu,mm_cpumask (next))) {/* We were in lazy tlb mode and leave_mm disabled
arch/x86/include/asm/percpu.h +10 -14
@@ -46,7 +46,7 @@#ifdef CONFIG_SMP #define __percpu_prefix "%%"__stringify(__percpu_seg)":" -#define __my_cpu_offset percpu_read(this_cpu_off) +#define __my_cpu_offset __this_cpu_read(this_cpu_off)/* * Compared to the generic __my_cpu_offset version, the following
@@ -351,23 +351,15 @@ do { \ })/* -* percpu_read() makes gcc load the percpu variable every time it is -* accessed while percpu_read_stable() allows the value to be cached. -* percpu_read_stable() is more efficient and can be used if its value +* this_cpu_read() makes gcc load the percpu variable every time it is +* accessed while this_cpu_read_stable() allows the value to be cached. +* this_cpu_read_stable() is more efficient and can be used if its value * is guaranteed to be valid across cpus. The current users include * get_current() and get_thread_info() both of which are actually * per-thread variables implemented as per-cpu variables and thus * stable for the duration of the respective task. */ -#define percpu_read (var) percpu_from_op("mov", var, "m" (var)) -#define percpu_read_stable (var) percpu_from_op("mov", var, "p" (&(var))) -#define percpu_write (var, val) percpu_to_op("mov", var, val) -#define percpu_add (var, val) percpu_add_op(var, val) -#define percpu_sub (var, val) percpu_add_op(var, -(val)) -#define percpu_and (var, val) percpu_to_op("and", var, val) -#define percpu_or (var, val) percpu_to_op("or", var, val) -#define percpu_xor (var, val) percpu_to_op("xor", var, val) -#define percpu_inc (var) percpu_unary_op("inc", var) +#define this_cpu_read_stable (var) percpu_from_op("mov", var, "p" (&(var)))#define __this_cpu_read_1 (pcp) percpu_from_op("mov", (pcp), "m"(pcp))#define __this_cpu_read_2 (pcp) percpu_from_op("mov", (pcp), "m"(pcp))
@@ -551,7 +543,11 @@ static __always_inline int x86_this_cpu_constant_test_bit(unsigned int nr, {unsigned long __percpu *a = (unsigned long *)addr + nr /BITS_PER_LONG ; -return ((1UL << (nr %BITS_PER_LONG )) &percpu_read (*a)) !=0 ; +#ifdef CONFIG_X86_64 +return ((1UL << (nr %BITS_PER_LONG )) &__this_cpu_read_8 (*a)) !=0 ; +#else +return ((1UL << (nr %BITS_PER_LONG )) &__this_cpu_read_4 (*a)) !=0 ; +#endif } static inline int x86_this_cpu_variable_test_bit(int nr,
arch/x86/include/asm/smp.h +2 -2
@@ -188,11 +188,11 @@ extern unsigned disabled_cpus __cpuinitdata;* from the initial startup. We map APIC_BASE very early in page_setup(), * so this is correct in the x86 case. */ -#define raw_smp_processor_id () (percpu_read(cpu_number)) +#define raw_smp_processor_id () (this_cpu_read(cpu_number))extern int safe_smp_processor_id (void );#elif defined(CONFIG_X86_64_SMP ) -#define raw_smp_processor_id () (percpu_read(cpu_number)) +#define raw_smp_processor_id () (this_cpu_read(cpu_number))#define stack_smp_processor_id () \ ({ \
arch/x86/include/asm/stackprotector.h +2 -2
@@ -76,9 +76,9 @@ static __always_inline void boot_init_stack_canary(void) current->stack_canary = canary;#ifdef CONFIG_X86_64 -percpu_write (irq_stack_union.stack_canary, canary); +__this_cpu_write (irq_stack_union.stack_canary, canary);#else -percpu_write (stack_canary.canary, canary); +__this_cpu_write (stack_canary.canary, canary);#endif }
arch/x86/include/asm/thread_info.h +1 -1
@@ -219,7 +219,7 @@ DECLARE_PER_CPU(unsigned long, kernel_stack);static inline struct thread_info *current_thread_info (void ) {struct thread_info *ti; - ti = (void *)(percpu_read_stable (kernel_stack) + + ti = (void *)(this_cpu_read_stable (kernel_stack) +KERNEL_STACK_OFFSET -THREAD_SIZE );return ti; }
arch/x86/include/asm/tlbflush.h +2 -2
@@ -156,8 +156,8 @@ DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate);static inline void reset_lazy_tlbstate (void ) { -percpu_write (cpu_tlbstate.state,0 ); -percpu_write (cpu_tlbstate.active_mm, &init_mm); +__this_cpu_write (cpu_tlbstate.state,0 ); +__this_cpu_write (cpu_tlbstate.active_mm, &init_mm); }#endif /* SMP */
arch/x86/kernel/cpu/common.c +1 -1
@@ -1163,7 +1163,7 @@ void __cpuinit cpu_init(void) oist = &per_cpu (orig_ist, cpu);#ifdef CONFIG_NUMA -if (cpu !=0 &&percpu_read (numa_node) ==0 && +if (cpu !=0 &&__this_cpu_read (numa_node) ==0 &&early_cpu_to_node (cpu) !=NUMA_NO_NODE )set_numa_node (early_cpu_to_node (cpu));#endif
arch/x86/kernel/cpu/mcheck/mce.c +2 -2
@@ -514,7 +514,7 @@ void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)struct mce m;int i; -percpu_inc (mce_poll_count); +__this_cpu_inc (mce_poll_count);mce_gather_info (&m,NULL );
@@ -906,7 +906,7 @@ void do_machine_check(struct pt_regs *regs, long error_code)atomic_inc (&mce_entry); -percpu_inc (mce_exception_count); +__this_cpu_inc (mce_exception_count);if (!banks) goto out;
arch/x86/kernel/paravirt.c +6 -6
@@ -239,16 +239,16 @@ static DEFINE_PER_CPU(enum paravirt_lazy_mode, paravirt_lazy_mode) = PARAVIRT_LAstatic inline void enter_lazy (enum paravirt_lazy_mode mode) { -BUG_ON (percpu_read (paravirt_lazy_mode) !=PARAVIRT_LAZY_NONE ); +BUG_ON (__this_cpu_read (paravirt_lazy_mode) !=PARAVIRT_LAZY_NONE ); -percpu_write (paravirt_lazy_mode, mode); +__this_cpu_write (paravirt_lazy_mode, mode); }static void leave_lazy (enum paravirt_lazy_mode mode) { -BUG_ON (percpu_read (paravirt_lazy_mode) != mode); +BUG_ON (__this_cpu_read (paravirt_lazy_mode) != mode); -percpu_write (paravirt_lazy_mode,PARAVIRT_LAZY_NONE ); +__this_cpu_write (paravirt_lazy_mode,PARAVIRT_LAZY_NONE ); }void paravirt_enter_lazy_mmu (void )
@@ -265,7 +265,7 @@ void paravirt_start_context_switch(struct task_struct *prev) {BUG_ON (preemptible ()); -if (percpu_read (paravirt_lazy_mode) ==PARAVIRT_LAZY_MMU ) { +if (__this_cpu_read (paravirt_lazy_mode) ==PARAVIRT_LAZY_MMU ) {arch_leave_lazy_mmu_mode ();set_ti_thread_flag (task_thread_info (prev),TIF_LAZY_MMU_UPDATES ); }
@@ -287,7 +287,7 @@ enum paravirt_lazy_mode paravirt_get_lazy_mode(void)if (in_interrupt ())return PARAVIRT_LAZY_NONE ; -return percpu_read (paravirt_lazy_mode); +return __this_cpu_read (paravirt_lazy_mode); }void arch_flush_lazy_mmu_mode (void )
arch/x86/kernel/process_32.c +1 -1
@@ -375,7 +375,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)if (prev->gs | next->gs)lazy_load_gs (next->gs); -percpu_write (current_task, next_p); +__this_cpu_write (current_task, next_p);return prev_p; }
arch/x86/kernel/process_64.c +6 -6
@@ -74,7 +74,7 @@ EXPORT_SYMBOL_GPL(idle_notifier_unregister);void enter_idle (void ) { -percpu_write (is_idle,1 ); +__this_cpu_write (is_idle,1 );atomic_notifier_call_chain (&idle_notifier,IDLE_START ,NULL ); }
@@ -338,7 +338,7 @@ start_thread_common(struct pt_regs *regs, unsigned long new_ip,load_gs_index (0 ); regs->ip = new_ip; regs->sp = new_sp; -percpu_write (old_rsp, new_sp); +this_cpu_write (old_rsp, new_sp); regs->cs = _cs; regs->ss = _ss; regs->flags =X86_EFLAGS_IF ;
@@ -472,11 +472,11 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)/* * Switch the PDA and FPU contexts. */ - prev->usersp =percpu_read (old_rsp); -percpu_write (old_rsp, next->usersp); -percpu_write (current_task, next_p); + prev->usersp =__this_cpu_read (old_rsp); +__this_cpu_write (old_rsp, next->usersp); +__this_cpu_write (current_task, next_p); -percpu_write (kernel_stack, +__this_cpu_write (kernel_stack, (unsigned long )task_stack_page (next_p) +THREAD_SIZE -KERNEL_STACK_OFFSET );
arch/x86/mm/tlb.c +5 -5
@@ -61,10 +61,10 @@ static DEFINE_PER_CPU_READ_MOSTLY(int, tlb_vector_offset);*/ void leave_mm (int cpu) { -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) +if (__this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_OK )BUG ();cpumask_clear_cpu (cpu, -mm_cpumask (percpu_read (cpu_tlbstate.active_mm))); +mm_cpumask (__this_cpu_read (cpu_tlbstate.active_mm)));load_cr3 (swapper_pg_dir); }EXPORT_SYMBOL_GPL (leave_mm);
@@ -152,8 +152,8 @@ void smp_invalidate_interrupt(struct pt_regs *regs)* BUG(); */ -if (f->flush_mm ==percpu_read (cpu_tlbstate.active_mm)) { -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) { +if (f->flush_mm ==__this_cpu_read (cpu_tlbstate.active_mm)) { +if (__this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) {if (f->flush_va ==TLB_FLUSH_ALL )local_flush_tlb ();else
@@ -322,7 +322,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)static void do_flush_tlb_all (void *info) {__flush_tlb_all (); -if (percpu_read (cpu_tlbstate.state) ==TLBSTATE_LAZY ) +if (__this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_LAZY )leave_mm (smp_processor_id ()); }
arch/x86/xen/enlighten.c +3 -3
@@ -777,11 +777,11 @@ static DEFINE_PER_CPU(unsigned long, xen_cr0_value);static unsigned long xen_read_cr0 (void ) { -unsigned long cr0 =percpu_read (xen_cr0_value); +unsigned long cr0 =this_cpu_read (xen_cr0_value);if (unlikely (cr0 ==0 )) { cr0 =native_read_cr0 (); -percpu_write (xen_cr0_value, cr0); +this_cpu_write (xen_cr0_value, cr0); }return cr0;
@@ -791,7 +791,7 @@ static void xen_write_cr0(unsigned long cr0) {struct multicall_space mcs; -percpu_write (xen_cr0_value, cr0); +this_cpu_write (xen_cr0_value, cr0);/* Only pay attention to cr0.TS; everything else is ignored. */
arch/x86/xen/irq.c +4 -4
@@ -26,7 +26,7 @@ static unsigned long xen_save_fl(void)struct vcpu_info *vcpu;unsigned long flags; - vcpu =percpu_read (xen_vcpu); + vcpu =this_cpu_read (xen_vcpu);/* flag has opposite sense of mask */ flags = !vcpu->evtchn_upcall_mask;
@@ -50,7 +50,7 @@ static void xen_restore_fl(unsigned long flags)make sure we're don't switch CPUs between getting the vcpu pointer and updating the mask. */ preempt_disable (); - vcpu =percpu_read (xen_vcpu); + vcpu =this_cpu_read (xen_vcpu); vcpu->evtchn_upcall_mask = flags;preempt_enable_no_resched ();
@@ -72,7 +72,7 @@ static void xen_irq_disable(void)make sure we're don't switch CPUs between getting the vcpu pointer and updating the mask. */ preempt_disable (); -percpu_read (xen_vcpu)->evtchn_upcall_mask =1 ; +this_cpu_read (xen_vcpu)->evtchn_upcall_mask =1 ;preempt_enable_no_resched (); }PV_CALLEE_SAVE_REGS_THUNK (xen_irq_disable);
@@ -86,7 +86,7 @@ static void xen_irq_enable(void)the caller is confused and is trying to re-enable interrupts on an indeterminate processor. */ - vcpu =percpu_read (xen_vcpu); + vcpu =this_cpu_read (xen_vcpu); vcpu->evtchn_upcall_mask =0 ;/* Doesn't matter if we get preempted here, because any
arch/x86/xen/mmu.c +10 -10
@@ -1071,14 +1071,14 @@ static void drop_other_mm_ref(void *info)struct mm_struct *mm = info;struct mm_struct *active_mm; - active_mm =percpu_read (cpu_tlbstate.active_mm); + active_mm =this_cpu_read (cpu_tlbstate.active_mm); -if (active_mm == mm &&percpu_read (cpu_tlbstate.state) !=TLBSTATE_OK ) +if (active_mm == mm &&this_cpu_read (cpu_tlbstate.state) !=TLBSTATE_OK )leave_mm (smp_processor_id ());/* If this cpu still has a stale cr3 reference, then make sure it has been flushed. */ -if (percpu_read (xen_current_cr3) ==__pa (mm->pgd)) +if (this_cpu_read (xen_current_cr3) ==__pa (mm->pgd))load_cr3 (swapper_pg_dir); }
@@ -1185,17 +1185,17 @@ static void __init xen_pagetable_setup_done(pgd_t *base)static void xen_write_cr2 (unsigned long cr2) { -percpu_read (xen_vcpu)->arch.cr2 = cr2; +this_cpu_read (xen_vcpu)->arch.cr2 = cr2; }static unsigned long xen_read_cr2 (void ) { -return percpu_read (xen_vcpu)->arch.cr2; +return this_cpu_read (xen_vcpu)->arch.cr2; }unsigned long xen_read_cr2_direct (void ) { -return percpu_read (xen_vcpu_info.arch.cr2); +return this_cpu_read (xen_vcpu_info.arch.cr2); }static void xen_flush_tlb (void )
@@ -1278,12 +1278,12 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,static unsigned long xen_read_cr3 (void ) { -return percpu_read (xen_cr3); +return this_cpu_read (xen_cr3); }static void set_current_cr3 (void *v) { -percpu_write (xen_current_cr3, (unsigned long )v); +this_cpu_write (xen_current_cr3, (unsigned long )v); }static void __xen_write_cr3 (bool kernel,unsigned long cr3)
@@ -1306,7 +1306,7 @@ static void __xen_write_cr3(bool kernel, unsigned long cr3)xen_extend_mmuext_op (&op);if (kernel) { -percpu_write (xen_cr3, cr3); +this_cpu_write (xen_cr3, cr3);/* Update xen_current_cr3 once the batch has actually been submitted. */
@@ -1322,7 +1322,7 @@ static void xen_write_cr3(unsigned long cr3)/* Update while interrupts are disabled, so its atomic with respect to ipis */ -percpu_write (xen_cr3, cr3); +this_cpu_write (xen_cr3, cr3);__xen_write_cr3 (true, cr3);
arch/x86/xen/multicalls.h +1 -1
@@ -47,7 +47,7 @@ static inline void xen_mc_issue(unsigned mode)xen_mc_flush ();/* restore flags saved in xen_mc_batch */ -local_irq_restore (percpu_read (xen_mc_irq_flags)); +local_irq_restore (this_cpu_read (xen_mc_irq_flags)); }/* Set up a callback to be called when the current batch is flushed */
arch/x86/xen/smp.c +1 -1
@@ -76,7 +76,7 @@ static void __cpuinit cpu_bringup(void)xen_setup_cpu_clockevents ();set_cpu_online (cpu, true); -percpu_write (cpu_state,CPU_ONLINE ); +this_cpu_write (cpu_state,CPU_ONLINE );wmb ();/* We can take interrupts now: we're officially "up". */
include/linux/percpu.h +0 -53
59 lines not on this page · raw message
include/linux/topology.h +2 -2
@@ -239,7 +239,7 @@ static inline int cpu_to_node(int cpu)#ifndef set_numa_nodestatic inline void set_numa_node (int node) { -percpu_write (numa_node, node); +__this_cpu_write (numa_node, node); }#endif
@@ -274,7 +274,7 @@ DECLARE_PER_CPU(int, _numa_mem_);#ifndef set_numa_memstatic inline void set_numa_mem (int node) { -percpu_write (_numa_mem_, node); +__this_cpu_write (_numa_mem_, node); }#endif
Signature · 1 line
1.7.5.1
[email protected]
maintainerQuoted 16 lines · Alex,Shi
Quoted 11 lines · On Mon, 2011-11-21 at 17:06 -0700, [email protected] wrote:
(cc'ing hpa and quoting whole body)
Quoted 3 lines
Signed-off-by Alex Shi <[email protected]>Acked-by Tejun Heo <[email protected]>hpa, I suppose this should go through x86? The original patch can be accessed at
Rend for 3.2 kernel, no any change needed to apply on latest Linus' tree. :)
Quoted 7 lines
Actually, this clean up has no performance or security impact for kernel. On the contrary, removing some potential redundant preempt disable will bring a slight performance benefit to kernel.
This 3rd patch depends on previous 2 patches, the 2nd one kvm code clean up was submitted for 3.3 kernel. but the 2st one net code clean up is waiting for David's comments.
Alex, can you please collect all patches into a single patchset? Please split it such that, usage changes are per-system so that they can be routed through respective subsystems (x86 or net) and updates to percpu proper which can be applied after other changes have been applied. It would probably be best to route these patches separately rather than all through percpu as it touches a lot of different places and is likely to cause conflicts. I *think* the best way would be,
* Submit per-subsystem patches and get them merged to subsystem trees.
* (Optional) Apply a patch to mark unused interface deprecated in percpu tree, so that new usages in linux-next can be detected.
* Towards the end of the next merge window, merge a patch to actually kill the old interface.
Thanks.
Signature · 1 line
tejun
H. Peter Anvin
reviewerQuoted 9 lines · On 01/11/2012 09:19 AM, [email protected] wrote:
Alex, can you please collect all patches into a single patchset? Please split it such that, usage changes are per-system so that they can be routed through respective subsystems (x86 or net) and updates to percpu proper which can be applied after other changes have been applied. It would probably be best to route these patches separately rather than all through percpu as it touches a lot of different places and is likely to cause conflicts. I *think* the best way would be,
Quoted 8 lines
* Submit per-subsystem patches and get them merged to subsystem trees.
* (Optional) Apply a patch to mark unused interface deprecated in percpu tree, so that new usages in linux-next can be detected.
* Towards the end of the next merge window, merge a patch to actually kill the old interface.
That sounds like a good idea.
-hpa
Quoted 12 lines · H. Peter Anvin
Quoted 11 lines · On 01/11/2012 09:19 AM, [email protected] wrote:
Alex, can you please collect all patches into a single patchset? Please split it such that, usage changes are per-system so that they can be routed through respective subsystems (x86 or net) and updates to percpu proper which can be applied after other changes have been applied. It would probably be best to route these patches separately rather than all through percpu as it touches a lot of different places and is likely to cause conflicts. I *think* the best way would be,
* Submit per-subsystem patches and get them merged to subsystem trees.
Quoted 8 lines
Quoted 6 lines
* (Optional) Apply a patch to mark unused interface deprecated in percpu tree, so that new usages in linux-next can be detected.
* Towards the end of the next merge window, merge a patch to actually kill the old interface.
That sounds like a good idea.
I will try to do so. Many thanks for the advices!
Quoted 2 lines
-hpa