irq_work: generic hard-irq context callbacks (resend with ACKs)
Subsystems (11)
PATCH 1/1 irq_work: generic hard-irq context callbacks (resend with ACKs)
From: Peter Zijlstra <[email protected]>
Provide a mechanism that allows running code in IRQ context. It is most useful for NMI code that needs to interact with the rest of the system -- like wakeup a task to drain buffers.
Perf currently has such a mechanism, so extract that and provide it as a generic feature, independent of perf so that others may also benefit.
The IRQ context callback is generated through self-IPIs where possible, or on architectures like powerpc the decrementer (the built-in timer facility) is set to generate an interrupt immediately.
Architectures that don't have anything like this get to do with a callback from the timer tick. These architectures can call irq_work_run() at the tail of any IRQ handlers that might enqueue such work (like the perf IRQ handler) to avoid undue latencies in processing the work.
Huang Ying: some fixes
This patch is only tested on x86 platform.
v6:
- Rewrite the changelog.
v5:
-compile test on all explicitly changed architectures except FRV.
v4:
-rebased on latest -tip tree
--- arch/alpha/Kconfig | 1 arch/alpha/include/asm/perf_event.h | 5 - arch/alpha/kernel/time.c | 30 +++--- arch/arm/Kconfig | 1 arch/arm/include/asm/perf_event.h | 12 -- arch/arm/kernel/perf_event.c | 8 - arch/frv/Kconfig | 1 arch/frv/lib/Makefile | 2 arch/frv/lib/perf_event.c | 19 ---- arch/parisc/Kconfig | 1 arch/parisc/include/asm/perf_event.h | 3 arch/powerpc/Kconfig | 1 arch/powerpc/include/asm/paca.h | 2 arch/powerpc/kernel/time.c | 42 ++++---- arch/s390/Kconfig | 1 arch/s390/include/asm/perf_event.h | 3 arch/sh/Kconfig | 1 arch/sh/include/asm/perf_event.h | 7 - arch/sparc/Kconfig | 2 arch/sparc/include/asm/perf_event.h | 4 arch/sparc/kernel/pcr.c | 8 - arch/x86/Kconfig | 1 arch/x86/include/asm/entry_arch.h | 4 arch/x86/include/asm/hardirq.h | 2 arch/x86/include/asm/hw_irq.h | 2 arch/x86/include/asm/irq_vectors.h | 4 arch/x86/kernel/Makefile | 1 arch/x86/kernel/cpu/perf_event.c | 19 ---- arch/x86/kernel/entry_64.S | 6 - arch/x86/kernel/irq.c | 8 - arch/x86/kernel/irq_work.c | 30 ++++++ arch/x86/kernel/irqinit.c | 6 - include/linux/irq_work.h | 20 ++++ include/linux/perf_event.h | 11 -- init/Kconfig | 8 + kernel/Makefile | 2 kernel/irq_work.c | 164 +++++++++++++++++++++++++++++++++++ kernel/perf_event.c | 104 +--------------------- kernel/timer.c | 7 + 39 files changed, 311 insertions(+), 242 deletions(-) delete mode 100644 arch/frv/lib/perf_event.c create mode 100644 arch/x86/kernel/irq_work.c create mode 100644 include/linux/irq_work.h create mode 100644 kernel/irq_work.c
arch/alpha/Kconfig +1 -0
@@ -9,6 +9,7 @@ config ALPHA select HAVE_IDE select HAVE_OPROFILE select HAVE_SYSCALL_WRAPPERS + select HAVE_IRQ_WORK select HAVE_PERF_EVENTS select HAVE_DMA_ATTRS help
arch/alpha/include/asm/perf_event.h +0 -5
@@ -1,11 +1,6 @@#ifndef __ASM_ALPHA_PERF_EVENT_H#define __ASM_ALPHA_PERF_EVENT_H -/* Alpha only supports software events through this interface. */ -extern void set_perf_event_pending (void ); - -#define PERF_EVENT_INDEX_OFFSET 0 -#ifdef CONFIG_PERF_EVENTS extern void init_hw_perf_events (void );#else
arch/alpha/kernel/time.c +15 -15
@@ -41,7 +41,7 @@#include <linux/init.h> #include <linux/bcd.h> #include <linux/profile.h> -#include <linux/perf_event.h> +#include <linux/irq_work.h> #include <asm/uaccess.h> #include <asm/io.h>
@@ -83,25 +83,25 @@ static struct {unsigned long est_cycle_freq; -#ifdef CONFIG_PERF_EVENTS +#ifdef CONFIG_IRQ_WORK -DEFINE_PER_CPU (u8, perf_event_pending); +DEFINE_PER_CPU (u8, irq_work_pending); -#define set_perf_event_pending_flag () __get_cpu_var(perf_event_pending) = 1 -#define test_perf_event_pending () __get_cpu_var(perf_event_pending) -#define clear_perf_event_pending () __get_cpu_var(perf_event_pending) = 0 +#define set_irq_work_pending_flag () __get_cpu_var(irq_work_pending) = 1 +#define test_irq_work_pending () __get_cpu_var(irq_work_pending) +#define clear_irq_work_pending () __get_cpu_var(irq_work_pending) = 0 -void set_perf_event_pending (void ) +void set_irq_work_pending (void ) { -set_perf_event_pending_flag (); +set_irq_work_pending_flag (); } -#else /* CONFIG_PERF_EVENTS */ +#else /* CONFIG_IRQ_WORK */ -#define test_perf_event_pending () 0 -#define clear_perf_event_pending () +#define test_irq_work_pending () 0 +#define clear_irq_work_pending () -#endif /* CONFIG_PERF_EVENTS */ +#endif /* CONFIG_IRQ_WORK */ static inline __u32 rpcc (void )
@@ -191,9 +191,9 @@ irqreturn_t timer_interrupt(int irq, voiwrite_sequnlock (&xtime_lock); -if (test_perf_event_pending ()) { -clear_perf_event_pending (); -perf_event_do_pending (); +if (test_irq_work_pending ()) { +clear_irq_work_pending (); +irq_work_run (); }#ifndef CONFIG_SMP
arch/arm/Kconfig +1 -0
@@ -23,6 +23,7 @@ config ARM select HAVE_KERNEL_GZIP select HAVE_KERNEL_LZO select HAVE_KERNEL_LZMA + select HAVE_IRQ_WORK select HAVE_PERF_EVENTS select PERF_USE_VMALLOC select HAVE_REGS_AND_STACK_ACCESS_API
arch/arm/include/asm/perf_event.h +0 -12
@@ -12,18 +12,6 @@#ifndef __ARM_PERF_EVENT_H__#define __ARM_PERF_EVENT_H__ -/* -* NOP: on *most* (read: all supported) ARM platforms, the performance -* counter interrupts are regular interrupts and not an NMI. This -* means that when we receive the interrupt we can call -* perf_event_do_pending() that handles all of the work with -* interrupts disabled. -*/ -static inline void -set_perf_event_pending (void ) -{ -} -/* ARM performance counters start from 1 (in the cp15 accesses) so use the * same indexes here for consistency. */ #define PERF_EVENT_INDEX_OFFSET 1
arch/arm/kernel/perf_event.c +4 -4
@@ -1086,7 +1086,7 @@ armv6pmu_handle_irq(int irq_num,* platforms that can have the PMU interrupts raised as an NMI, this * will not work. */ -perf_event_do_pending (); +irq_work_run ();return IRQ_HANDLED ; }
@@ -2062,7 +2062,7 @@ static irqreturn_t armv7pmu_handle_irq(i* platforms that can have the PMU interrupts raised as an NMI, this * will not work. */ -perf_event_do_pending (); +irq_work_run ();return IRQ_HANDLED ; }
@@ -2430,7 +2430,7 @@ xscale1pmu_handle_irq(int irq_num, void armpmu->disable (hwc, idx); } -perf_event_do_pending (); +irq_work_run ();/* * Re-enable the PMU.
@@ -2757,7 +2757,7 @@ xscale2pmu_handle_irq(int irq_num, void armpmu->disable (hwc, idx); } -perf_event_do_pending (); +irq_work_run ();/* * Re-enable the PMU.
arch/frv/Kconfig +1 -0
@@ -7,6 +7,7 @@ config FRV default y select HAVE_IDE select HAVE_ARCH_TRACEHOOK + select HAVE_IRQ_WORK select HAVE_PERF_EVENTS config ZONE_DMA
arch/frv/lib/Makefile +1 -1
@@ -5,4 +5,4 @@ lib-y := \ __ashldi3.o __lshrdi3.o __muldi3.o __ashrdi3.o __negdi2.o __ucmpdi2.o \ checksum.o memcpy.o memset.o atomic-ops.o atomic64-ops.o \ - outsl_ns.o outsl_sw.o insl_ns.o insl_sw.o cache.o perf_event.o + outsl_ns.o outsl_sw.o insl_ns.o insl_sw.o cache.o
arch/frv/lib/perf_event.c deleted +0 -19
@@ -1,19 +0,0 @@ -/* Performance event handling -* -* Copyright (C) 2009 Red Hat, Inc. All Rights Reserved. -* Written by David Howells ([email protected]) -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public Licence -* as published by the Free Software Foundation; either version -* 2 of the Licence, or (at your option) any later version. -*/ - -#include <linux/perf_event.h> - -/* -* mark the performance event as pending -*/ -void set_perf_event_pending (void ) -{ -}
arch/parisc/Kconfig +1 -0
@@ -16,6 +16,7 @@ config PARISC select RTC_DRV_GENERIC select INIT_ALL_POSSIBLE select BUG + select HAVE_IRQ_WORK select HAVE_PERF_EVENTS select GENERIC_ATOMIC64 if !64BIT help
arch/parisc/include/asm/perf_event.h +1 -2
@@ -1,7 +1,6 @@#ifndef __ASM_PARISC_PERF_EVENT_H#define __ASM_PARISC_PERF_EVENT_H -/* parisc only supports software events through this interface. */ -static inline void set_perf_event_pending (void ) { } +/* Empty, just to avoid compiling error */ #endif /* __ASM_PARISC_PERF_EVENT_H */
arch/powerpc/Kconfig +1 -0
@@ -138,6 +138,7 @@ config PPC select HAVE_OPROFILE select HAVE_SYSCALL_WRAPPERS if PPC64 select GENERIC_ATOMIC64 if PPC32 + select HAVE_IRQ_WORK select HAVE_PERF_EVENTS select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_HW_BREAKPOINT if PERF_EVENTS && PPC_BOOK3S_64
arch/powerpc/include/asm/paca.h +1 -1
@@ -129,7 +129,7 @@ struct paca_struct {u8 soft_enabled;/* irq soft-enable flag */ u8 hard_enabled;/* set if irqs are enabled in MSR */ u8 io_sync;/* writel() needs spin_unlock sync */ -u8 perf_event_pending;/* PM interrupt while soft-disabled */ +u8 irq_work_pending;/* IRQ_WORK interrupt while soft-disable */ /* Stuff for accurate time accounting */ u64 user_time;/* accumulated usermode TB ticks */
arch/powerpc/kernel/time.c +21 -21
97 lines not on this page · raw message
arch/s390/Kconfig +1 -0
@@ -95,6 +95,7 @@ config S390 select HAVE_KVM if 64BIT select HAVE_ARCH_TRACEHOOK select INIT_ALL_POSSIBLE + select HAVE_IRQ_WORK select HAVE_PERF_EVENTS select HAVE_KERNEL_GZIP select HAVE_KERNEL_BZIP2
arch/s390/include/asm/perf_event.h +1 -2
@@ -4,7 +4,6 @@* Copyright 2009 Martin Schwidefsky, IBM Corporation. */ -static inline void set_perf_event_pending (void ) {} -static inline void clear_perf_event_pending (void ) {} +/* Empty, just to avoid compiling error */ #define PERF_EVENT_INDEX_OFFSET 0
arch/sh/Kconfig +1 -0
@@ -16,6 +16,7 @@ config SUPERH select HAVE_ARCH_TRACEHOOK select HAVE_DMA_API_DEBUG select HAVE_DMA_ATTRS + select HAVE_IRQ_WORK select HAVE_PERF_EVENTS select PERF_USE_VMALLOC select HAVE_KERNEL_GZIP
arch/sh/include/asm/perf_event.h +0 -7
@@ -26,11 +26,4 @@ extern int register_sh_pmu(struct sh_pmuextern int reserve_pmc_hardware (void );extern void release_pmc_hardware (void ); -static inline void set_perf_event_pending (void ) -{ -/* Nothing to see here, move along. */ -} - -#define PERF_EVENT_INDEX_OFFSET 0 -#endif /* __ASM_SH_PERF_EVENT_H */
arch/sparc/Kconfig +2 -0
@@ -26,6 +26,7 @@ config SPARC select ARCH_WANT_OPTIONAL_GPIOLIB select RTC_CLASS select RTC_DRV_M48T59 + select HAVE_IRQ_WORK select HAVE_PERF_EVENTS select PERF_USE_VMALLOC select HAVE_DMA_ATTRS
@@ -54,6 +55,7 @@ config SPARC64 select RTC_DRV_BQ4802 select RTC_DRV_SUN4V select RTC_DRV_STARFIRE + select HAVE_IRQ_WORK select HAVE_PERF_EVENTS select PERF_USE_VMALLOC
arch/sparc/include/asm/perf_event.h +0 -4
@@ -1,10 +1,6 @@#ifndef __ASM_SPARC_PERF_EVENT_H#define __ASM_SPARC_PERF_EVENT_H -extern void set_perf_event_pending (void ); - -#define PERF_EVENT_INDEX_OFFSET 0 -#ifdef CONFIG_PERF_EVENTS #include <asm/ptrace.h>
arch/sparc/kernel/pcr.c +4 -4
@@ -7,7 +7,7 @@#include <linux/init.h> #include <linux/irq.h> -#include <linux/perf_event.h> +#include <linux/irq_work.h> #include <linux/ftrace.h> #include <asm/pil.h>
@@ -43,14 +43,14 @@ void __irq_entry deferred_pcr_work_irq(i old_regs =set_irq_regs (regs);irq_enter (); -#ifdef CONFIG_PERF_EVENTS -perf_event_do_pending (); +#ifdef CONFIG_IRQ_WORK +irq_work_run ();#endif irq_exit ();set_irq_regs (old_regs); } -void set_perf_event_pending (void ) +void arch_irq_work_raise (void ) {set_softint (1 <<PIL_DEFERRED_PCR_WORK ); }
arch/x86/Kconfig +1 -0
@@ -25,6 +25,7 @@ config X86 select HAVE_IDE select HAVE_OPROFILE select HAVE_PERF_EVENTS if (!M386 && !M486) + select HAVE_IRQ_WORK select HAVE_IOREMAP_PROT select HAVE_KPROBES select HAVE_MEMBLOCK
arch/x86/include/asm/entry_arch.h +2 -2
@@ -49,8 +49,8 @@ BUILD_INTERRUPT(apic_timer_interrupt,LOC BUILD_INTERRUPT(error_interrupt,ERROR_APIC_VECTOR) BUILD_INTERRUPT(spurious_interrupt,SPURIOUS_APIC_VECTOR) -#ifdef CONFIG_PERF_EVENTS -BUILD_INTERRUPT(perf_pending_interrupt, LOCAL_PENDING_VECTOR) +#ifdef CONFIG_IRQ_WORK +BUILD_INTERRUPT(irq_work_interrupt, IRQ_WORK_VECTOR)#endif #ifdef CONFIG_X86_THERMAL_VECTOR
arch/x86/include/asm/hardirq.h +1 -1
@@ -14,7 +14,7 @@ typedef struct {#endif unsigned int x86_platform_ipis;/* arch dependent */ unsigned int apic_perf_irqs; -unsigned int apic_pending_irqs; +unsigned int apic_irq_work_irqs;#ifdef CONFIG_SMP unsigned int irq_resched_count;unsigned int irq_call_count;
arch/x86/include/asm/hw_irq.h +1 -1
@@ -29,7 +29,7 @@extern void apic_timer_interrupt (void );extern void x86_platform_ipi (void );extern void error_interrupt (void ); -extern void perf_pending_interrupt (void ); +extern void irq_work_interrupt (void );extern void spurious_interrupt (void );extern void thermal_interrupt (void );
arch/x86/include/asm/irq_vectors.h +2 -2
@@ -114,9 +114,9 @@#define X86_PLATFORM_IPI_VECTOR 0xed/* -* Performance monitoring pending work vector: +* IRQ work vector: */ -#define LOCAL_PENDING_VECTOR 0xec +#define IRQ_WORK_VECTOR 0xec#define UV_BAU_MESSAGE 0xea
arch/x86/kernel/Makefile +1 -0
@@ -35,6 +35,7 @@ obj-y := process_$(BITS).o signal.o en obj-y += traps.o irq.o irq_$(BITS).o dumpstack_$(BITS).o obj-y += time.o ioport.o ldt.o dumpstack.o obj-y += setup.o x86_init.o i8259.o irqinit.o jump_label.o +obj-$(CONFIG_IRQ_WORK) += irq_work.o obj-$(CONFIG_X86_VISWS) += visws_quirks.o obj-$(CONFIG_X86_32) += probe_roms_32.o obj-$(CONFIG_X86_32) += sys_i386_32.o i386_ksyms_32.o
arch/x86/kernel/cpu/perf_event.c +0 -19
@@ -1196,25 +1196,6 @@ static int x86_pmu_handle_irq(struct pt_return handled; } -void smp_perf_pending_interrupt (struct pt_regs *regs) -{ -irq_enter (); -ack_APIC_irq (); -inc_irq_stat (apic_pending_irqs); -perf_event_do_pending (); -irq_exit (); -} - -void set_perf_event_pending (void ) -{ -#ifdef CONFIG_X86_LOCAL_APIC -if (!x86_pmu.apic || !x86_pmu_initialized ()) -return ; - - apic->send_IPI_self (LOCAL_PENDING_VECTOR ); -#endif -} -void perf_events_lapic_init (void ) {if (!x86_pmu.apic || !x86_pmu_initialized ())
arch/x86/kernel/entry_64.S +3 -3
@@ -1005,9 +1005,9 @@ apicinterrupt ERROR_APIC_VECTOR \ apicinterrupt SPURIOUS_APIC_VECTOR \ spurious_interrupt smp_spurious_interrupt -#ifdef CONFIG_PERF_EVENTS -apicinterrupt LOCAL_PENDING_VECTOR \ - perf_pending_interrupt smp_perf_pending_interrupt +#ifdef CONFIG_IRQ_WORK +apicinterrupt IRQ_WORK_VECTOR \ + irq_work_interrupt smp_irq_work_interrupt #endif /*
arch/x86/kernel/irq.c +4 -4
@@ -67,10 +67,10 @@ static int show_other_interrupts(structfor_each_online_cpu (j)seq_printf (p,"%10u " ,irq_stats (j)->apic_perf_irqs);seq_printf (p," Performance monitoring interrupts\n" ); -seq_printf (p,"%*s: " , prec,"PND" ); +seq_printf (p,"%*s: " , prec,"IWI" );for_each_online_cpu (j) -seq_printf (p,"%10u " ,irq_stats (j)->apic_pending_irqs); -seq_printf (p," Performance pending work\n" ); +seq_printf (p,"%10u " ,irq_stats (j)->apic_irq_work_irqs); +seq_printf (p," IRQ work interrupts\n" );#endif if (x86_platform_ipi_callback) {seq_printf (p,"%*s: " , prec,"PLT" );
@@ -185,7 +185,7 @@ u64 arch_irq_stat_cpu(unsigned int cpu) sum +=irq_stats (cpu)->apic_timer_irqs; sum +=irq_stats (cpu)->irq_spurious_count; sum +=irq_stats (cpu)->apic_perf_irqs; - sum +=irq_stats (cpu)->apic_pending_irqs; + sum +=irq_stats (cpu)->apic_irq_work_irqs;#endif if (x86_platform_ipi_callback) sum +=irq_stats (cpu)->x86_platform_ipis;
arch/x86/kernel/irq_work.c new file +30 -0
@@ -0,0 +1,30 @@ +/* +* x86 specific code for irq_work +* +* Copyright (C) 2010 Red Hat, Inc., Peter Zijlstra <[email protected]> +*/ + +#include <linux/kernel.h> +#include <linux/irq_work.h> +#include <linux/hardirq.h> +#include <asm/apic.h> + +void smp_irq_work_interrupt (struct pt_regs *regs) +{ +irq_enter (); +ack_APIC_irq (); +inc_irq_stat (apic_irq_work_irqs); +irq_work_run (); +irq_exit (); +} + +void arch_irq_work_raise (void ) +{ +#ifdef CONFIG_X86_LOCAL_APIC +if (!cpu_has_apic) +return ; + + apic->send_IPI_self (IRQ_WORK_VECTOR ); +apic_wait_icr_idle (); +#endif +}
arch/x86/kernel/irqinit.c +3 -3
@@ -215,9 +215,9 @@ static void __init apic_intr_init(void)alloc_intr_gate (SPURIOUS_APIC_VECTOR , spurious_interrupt);alloc_intr_gate (ERROR_APIC_VECTOR , error_interrupt); -/* Performance monitoring interrupts: */ -# ifdef CONFIG_PERF_EVENTS -alloc_intr_gate (LOCAL_PENDING_VECTOR , perf_pending_interrupt); +/* IRQ work interrupts: */ +# ifdef CONFIG_IRQ_WORK +alloc_intr_gate (IRQ_WORK_VECTOR , irq_work_interrupt);# endif #endif
include/linux/irq_work.h new file +20 -0
@@ -0,0 +1,20 @@ +#ifndef _LINUX_IRQ_WORK_H +#define _LINUX_IRQ_WORK_H + +struct irq_work { +struct irq_work *next; +void (*func)(struct irq_work *); +}; + +static inline +void init_irq_work (struct irq_work *entry,void (*func)(struct irq_work *)) +{ + entry->next =NULL ; + entry->func = func; +} + +bool irq_work_queue (struct irq_work *entry); +void irq_work_run (void ); +void irq_work_sync (struct irq_work *entry); + +#endif /* _LINUX_IRQ_WORK_H */
include/linux/perf_event.h +2 -9
@@ -486,6 +486,7 @@ struct perf_guest_info_callbacks {#include <linux/workqueue.h> #include <linux/ftrace.h> #include <linux/cpu.h> +#include <linux/irq_work.h> #include <asm/atomic.h> #include <asm/local.h>
@@ -672,11 +673,6 @@ struct perf_buffer {void *data_pages[0 ]; }; -struct perf_pending_entry { -struct perf_pending_entry *next; -void (*func)(struct perf_pending_entry *); -}; -struct perf_sample_data ; typedef void (*perf_overflow_handler_t)(struct perf_event *, int,
@@ -784,7 +780,7 @@ struct perf_event {int pending_wakeup;int pending_kill;int pending_disable; -struct perf_pending_entry pending; +struct irq_work pending;atomic_t event_limit;
@@ -896,8 +892,6 @@ extern int perf_event_init_task(struct textern void perf_event_exit_task (struct task_struct *child);extern void perf_event_free_task (struct task_struct *task);extern void perf_event_delayed_put (struct task_struct *task); -extern void set_perf_event_pending (void ); -extern void perf_event_do_pending (void );extern void perf_event_print_debug (void );extern void perf_pmu_disable (struct pmu *pmu);extern void perf_pmu_enable (struct pmu *pmu);
@@ -1076,7 +1070,6 @@ static inline int perf_event_init_task(sstatic inline void perf_event_exit_task (struct task_struct *child) { }static inline void perf_event_free_task (struct task_struct *task) { }static inline void perf_event_delayed_put (struct task_struct *task) { } -static inline void perf_event_do_pending (void ) { }static inline void perf_event_print_debug (void ) { }static inline int perf_event_task_disable (void ) {return -EINVAL ; }static inline int perf_event_task_enable (void ) {return -EINVAL ; }
init/Kconfig +8 -0
@@ -21,6 +21,13 @@ config CONSTRUCTORS depends on !UML default y +config HAVE_IRQ_WORK + bool + +config IRQ_WORK + bool + depends on HAVE_IRQ_WORK + menu "General setup" config EXPERIMENTAL
@@ -1007,6 +1014,7 @@ config PERF_EVENTS default y if (PROFILING || PERF_COUNTERS) depends on HAVE_PERF_EVENTS select ANON_INODES + select IRQ_WORK help Enable kernel support for various performance events provided by software and hardware.
kernel/Makefile +2 -0
@@ -22,6 +22,7 @@ CFLAGS_REMOVE_rtmutex-debug.o = -pg CFLAGS_REMOVE_cgroup-debug.o = -pg CFLAGS_REMOVE_sched_clock.o = -pg CFLAGS_REMOVE_perf_event.o = -pg +CFLAGS_REMOVE_irq_work.o = -pg endif obj-$(CONFIG_FREEZER) += freezer.o
@@ -100,6 +101,7 @@ obj-$(CONFIG_TRACING) += trace/ obj-$(CONFIG_X86_DS) += trace/ obj-$(CONFIG_RING_BUFFER) += trace/ obj-$(CONFIG_SMP) += sched_cpupri.o +obj-$(CONFIG_IRQ_WORK) += irq_work.o obj-$(CONFIG_PERF_EVENTS) += perf_event.o obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o obj-$(CONFIG_USER_RETURN_NOTIFIER) += user-return-notifier.o
kernel/irq_work.c new file +164 -0
164 lines not on this page · raw message
kernel/perf_event.c +5 -99
144 lines not on this page · raw message
kernel/timer.c +5 -2
@@ -37,7 +37,7 @@#include <linux/delay.h> #include <linux/tick.h> #include <linux/kallsyms.h> -#include <linux/perf_event.h> +#include <linux/irq_work.h> #include <linux/sched.h> #include <linux/slab.h>
@@ -1279,7 +1279,10 @@ void update_process_times(int user_tick)run_local_timers ();rcu_check_callbacks (cpu, user_tick);printk_tick (); -perf_event_do_pending (); +#ifdef CONFIG_IRQ_WORK +if (in_irq ()) +irq_work_run (); +#endif scheduler_tick ();run_posix_cpu_timers (p); }
[tip:perf/core] irq_work: Add generic hardirq context callbacks
Commit-ID: e360adbe29241a0194e10e20595360dd7b98a2b3 Gitweb: http://git.kernel.org/tip/e360adbe29241a0194e10e20595360dd7b98a2b3 Author: Peter Zijlstra <[email protected]> AuthorDate: Thu, 14 Oct 2010 14:01:34 +0800 Committer: Ingo Molnar <[email protected]> CommitDate: Mon, 18 Oct 2010 19:58:50 +0200
irq_work: Add generic hardirq context callbacks
Provide a mechanism that allows running code in IRQ context. It is most useful for NMI code that needs to interact with the rest of the system -- like wakeup a task to drain buffers.
Perf currently has such a mechanism, so extract that and provide it as a generic feature, independent of perf so that others may also benefit.
The IRQ context callback is generated through self-IPIs where possible, or on architectures like powerpc the decrementer (the built-in timer facility) is set to generate an interrupt immediately.
Architectures that don't have anything like this get to do with a callback from the timer tick. These architectures can call irq_work_run() at the tail of any IRQ handlers that might enqueue such work (like the perf IRQ handler) to avoid undue latencies in processing the work.
LKML-Reference: <1287036094.7768.291.camel@yhuang-dev>
--- arch/alpha/Kconfig | 1 + arch/alpha/include/asm/perf_event.h | 5 - arch/alpha/kernel/time.c | 30 +++--- arch/arm/Kconfig | 1 + arch/arm/include/asm/perf_event.h | 12 --- arch/arm/kernel/perf_event.c | 8 +- arch/frv/Kconfig | 1 + arch/frv/lib/Makefile | 2 +- arch/frv/lib/perf_event.c | 19 ---- arch/parisc/Kconfig | 1 + arch/parisc/include/asm/perf_event.h | 3 +- arch/powerpc/Kconfig | 1 + arch/powerpc/include/asm/paca.h | 2 +- arch/powerpc/kernel/time.c | 42 +++++----- arch/s390/Kconfig | 1 + arch/s390/include/asm/perf_event.h | 3 +- arch/sh/Kconfig | 1 + arch/sh/include/asm/perf_event.h | 7 -- arch/sparc/Kconfig | 2 + arch/sparc/include/asm/perf_event.h | 4 - arch/sparc/kernel/pcr.c | 8 +- arch/x86/Kconfig | 1 + arch/x86/include/asm/entry_arch.h | 4 +- arch/x86/include/asm/hardirq.h | 2 +- arch/x86/include/asm/hw_irq.h | 2 +- arch/x86/include/asm/irq_vectors.h | 4 +- arch/x86/kernel/Makefile | 1 + arch/x86/kernel/cpu/perf_event.c | 19 ---- arch/x86/kernel/entry_64.S | 6 +- arch/x86/kernel/irq.c | 8 +- arch/x86/kernel/irq_work.c | 30 ++++++ arch/x86/kernel/irqinit.c | 6 +- include/linux/irq_work.h | 20 ++++ include/linux/perf_event.h | 11 +-- init/Kconfig | 8 ++ kernel/Makefile | 2 + kernel/irq_work.c | 164 ++++++++++++++++++++++++++++++++++ kernel/perf_event.c | 104 +-------------------- kernel/timer.c | 7 +- 39 files changed, 311 insertions(+), 242 deletions(-)
arch/alpha/Kconfig +1 -0
@@ -9,6 +9,7 @@ config ALPHA select HAVE_IDE select HAVE_OPROFILE select HAVE_SYSCALL_WRAPPERS + select HAVE_IRQ_WORK select HAVE_PERF_EVENTS select HAVE_DMA_ATTRS help
arch/alpha/include/asm/perf_event.h +0 -5
@@ -1,11 +1,6 @@#ifndef __ASM_ALPHA_PERF_EVENT_H#define __ASM_ALPHA_PERF_EVENT_H -/* Alpha only supports software events through this interface. */ -extern void set_perf_event_pending (void ); - -#define PERF_EVENT_INDEX_OFFSET 0 -#ifdef CONFIG_PERF_EVENTS extern void init_hw_perf_events (void );#else
arch/alpha/kernel/time.c +15 -15
@@ -41,7 +41,7 @@#include <linux/init.h> #include <linux/bcd.h> #include <linux/profile.h> -#include <linux/perf_event.h> +#include <linux/irq_work.h> #include <asm/uaccess.h> #include <asm/io.h>
@@ -83,25 +83,25 @@ static struct {unsigned long est_cycle_freq; -#ifdef CONFIG_PERF_EVENTS +#ifdef CONFIG_IRQ_WORK -DEFINE_PER_CPU (u8, perf_event_pending); +DEFINE_PER_CPU (u8, irq_work_pending); -#define set_perf_event_pending_flag () __get_cpu_var(perf_event_pending) = 1 -#define test_perf_event_pending () __get_cpu_var(perf_event_pending) -#define clear_perf_event_pending () __get_cpu_var(perf_event_pending) = 0 +#define set_irq_work_pending_flag () __get_cpu_var(irq_work_pending) = 1 +#define test_irq_work_pending () __get_cpu_var(irq_work_pending) +#define clear_irq_work_pending () __get_cpu_var(irq_work_pending) = 0 -void set_perf_event_pending (void ) +void set_irq_work_pending (void ) { -set_perf_event_pending_flag (); +set_irq_work_pending_flag (); } -#else /* CONFIG_PERF_EVENTS */ +#else /* CONFIG_IRQ_WORK */ -#define test_perf_event_pending () 0 -#define clear_perf_event_pending () +#define test_irq_work_pending () 0 +#define clear_irq_work_pending () -#endif /* CONFIG_PERF_EVENTS */ +#endif /* CONFIG_IRQ_WORK */ static inline __u32 rpcc (void )
@@ -191,9 +191,9 @@ irqreturn_t timer_interrupt(int irq, void *dev)write_sequnlock (&xtime_lock); -if (test_perf_event_pending ()) { -clear_perf_event_pending (); -perf_event_do_pending (); +if (test_irq_work_pending ()) { +clear_irq_work_pending (); +irq_work_run (); }#ifndef CONFIG_SMP
arch/arm/Kconfig +1 -0
@@ -23,6 +23,7 @@ config ARM select HAVE_KERNEL_GZIP select HAVE_KERNEL_LZO select HAVE_KERNEL_LZMA + select HAVE_IRQ_WORK select HAVE_PERF_EVENTS select PERF_USE_VMALLOC select HAVE_REGS_AND_STACK_ACCESS_API
arch/arm/include/asm/perf_event.h +0 -12
@@ -12,18 +12,6 @@#ifndef __ARM_PERF_EVENT_H__#define __ARM_PERF_EVENT_H__ -/* -* NOP: on *most* (read: all supported) ARM platforms, the performance -* counter interrupts are regular interrupts and not an NMI. This -* means that when we receive the interrupt we can call -* perf_event_do_pending() that handles all of the work with -* interrupts disabled. -*/ -static inline void -set_perf_event_pending (void ) -{ -} -/* ARM performance counters start from 1 (in the cp15 accesses) so use the * same indexes here for consistency. */ #define PERF_EVENT_INDEX_OFFSET 1
arch/arm/kernel/perf_event.c +4 -4
@@ -1092,7 +1092,7 @@ armv6pmu_handle_irq(int irq_num,* platforms that can have the PMU interrupts raised as an NMI, this * will not work. */ -perf_event_do_pending (); +irq_work_run ();return IRQ_HANDLED ; }
@@ -2068,7 +2068,7 @@ static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev)* platforms that can have the PMU interrupts raised as an NMI, this * will not work. */ -perf_event_do_pending (); +irq_work_run ();return IRQ_HANDLED ; }
@@ -2436,7 +2436,7 @@ xscale1pmu_handle_irq(int irq_num, void *dev) armpmu->disable (hwc, idx); } -perf_event_do_pending (); +irq_work_run ();/* * Re-enable the PMU.
@@ -2763,7 +2763,7 @@ xscale2pmu_handle_irq(int irq_num, void *dev) armpmu->disable (hwc, idx); } -perf_event_do_pending (); +irq_work_run ();/* * Re-enable the PMU.
arch/frv/Kconfig +1 -0
@@ -7,6 +7,7 @@ config FRV default y select HAVE_IDE select HAVE_ARCH_TRACEHOOK + select HAVE_IRQ_WORK select HAVE_PERF_EVENTS config ZONE_DMA
arch/frv/lib/Makefile +1 -1
@@ -5,4 +5,4 @@ lib-y := \ __ashldi3.o __lshrdi3.o __muldi3.o __ashrdi3.o __negdi2.o __ucmpdi2.o \ checksum.o memcpy.o memset.o atomic-ops.o atomic64-ops.o \ - outsl_ns.o outsl_sw.o insl_ns.o insl_sw.o cache.o perf_event.o + outsl_ns.o outsl_sw.o insl_ns.o insl_sw.o cache.o
arch/frv/lib/perf_event.c deleted +0 -19
@@ -1,19 +0,0 @@ -/* Performance event handling -* -* Copyright (C) 2009 Red Hat, Inc. All Rights Reserved. -* Written by David Howells ([email protected]) -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public Licence -* as published by the Free Software Foundation; either version -* 2 of the Licence, or (at your option) any later version. -*/ - -#include <linux/perf_event.h> - -/* -* mark the performance event as pending -*/ -void set_perf_event_pending (void ) -{ -}
arch/parisc/Kconfig +1 -0
@@ -16,6 +16,7 @@ config PARISC select RTC_DRV_GENERIC select INIT_ALL_POSSIBLE select BUG + select HAVE_IRQ_WORK select HAVE_PERF_EVENTS select GENERIC_ATOMIC64 if !64BIT help
arch/parisc/include/asm/perf_event.h +1 -2
@@ -1,7 +1,6 @@#ifndef __ASM_PARISC_PERF_EVENT_H#define __ASM_PARISC_PERF_EVENT_H -/* parisc only supports software events through this interface. */ -static inline void set_perf_event_pending (void ) { } +/* Empty, just to avoid compiling error */ #endif /* __ASM_PARISC_PERF_EVENT_H */
arch/powerpc/Kconfig +1 -0
@@ -138,6 +138,7 @@ config PPC select HAVE_OPROFILE select HAVE_SYSCALL_WRAPPERS if PPC64 select GENERIC_ATOMIC64 if PPC32 + select HAVE_IRQ_WORK select HAVE_PERF_EVENTS select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_HW_BREAKPOINT if PERF_EVENTS && PPC_BOOK3S_64
arch/powerpc/include/asm/paca.h +1 -1
@@ -129,7 +129,7 @@ struct paca_struct {u8 soft_enabled;/* irq soft-enable flag */ u8 hard_enabled;/* set if irqs are enabled in MSR */ u8 io_sync;/* writel() needs spin_unlock sync */ -u8 perf_event_pending;/* PM interrupt while soft-disabled */ +u8 irq_work_pending;/* IRQ_WORK interrupt while soft-disable */ /* Stuff for accurate time accounting */ u64 user_time;/* accumulated usermode TB ticks */
arch/powerpc/kernel/time.c +21 -21
97 lines not on this page · raw message
arch/s390/Kconfig +1 -0
@@ -95,6 +95,7 @@ config S390 select HAVE_KVM if 64BIT select HAVE_ARCH_TRACEHOOK select INIT_ALL_POSSIBLE + select HAVE_IRQ_WORK select HAVE_PERF_EVENTS select HAVE_KERNEL_GZIP select HAVE_KERNEL_BZIP2
arch/s390/include/asm/perf_event.h +1 -2
@@ -4,7 +4,6 @@* Copyright 2009 Martin Schwidefsky, IBM Corporation. */ -static inline void set_perf_event_pending (void ) {} -static inline void clear_perf_event_pending (void ) {} +/* Empty, just to avoid compiling error */ #define PERF_EVENT_INDEX_OFFSET 0
arch/sh/Kconfig +1 -0
@@ -16,6 +16,7 @@ config SUPERH select HAVE_ARCH_TRACEHOOK select HAVE_DMA_API_DEBUG select HAVE_DMA_ATTRS + select HAVE_IRQ_WORK select HAVE_PERF_EVENTS select PERF_USE_VMALLOC select HAVE_KERNEL_GZIP
arch/sh/include/asm/perf_event.h +0 -7
@@ -26,11 +26,4 @@ extern int register_sh_pmu(struct sh_pmu *);extern int reserve_pmc_hardware (void );extern void release_pmc_hardware (void ); -static inline void set_perf_event_pending (void ) -{ -/* Nothing to see here, move along. */ -} - -#define PERF_EVENT_INDEX_OFFSET 0 -#endif /* __ASM_SH_PERF_EVENT_H */
arch/sparc/Kconfig +2 -0
@@ -26,6 +26,7 @@ config SPARC select ARCH_WANT_OPTIONAL_GPIOLIB select RTC_CLASS select RTC_DRV_M48T59 + select HAVE_IRQ_WORK select HAVE_PERF_EVENTS select PERF_USE_VMALLOC select HAVE_DMA_ATTRS
@@ -54,6 +55,7 @@ config SPARC64 select RTC_DRV_BQ4802 select RTC_DRV_SUN4V select RTC_DRV_STARFIRE + select HAVE_IRQ_WORK select HAVE_PERF_EVENTS select PERF_USE_VMALLOC
arch/sparc/include/asm/perf_event.h +0 -4
@@ -1,10 +1,6 @@#ifndef __ASM_SPARC_PERF_EVENT_H#define __ASM_SPARC_PERF_EVENT_H -extern void set_perf_event_pending (void ); - -#define PERF_EVENT_INDEX_OFFSET 0 -#ifdef CONFIG_PERF_EVENTS #include <asm/ptrace.h>
arch/sparc/kernel/pcr.c +4 -4
@@ -7,7 +7,7 @@#include <linux/init.h> #include <linux/irq.h> -#include <linux/perf_event.h> +#include <linux/irq_work.h> #include <linux/ftrace.h> #include <asm/pil.h>
@@ -43,14 +43,14 @@ void __irq_entry deferred_pcr_work_irq(int irq, struct pt_regs *regs) old_regs =set_irq_regs (regs);irq_enter (); -#ifdef CONFIG_PERF_EVENTS -perf_event_do_pending (); +#ifdef CONFIG_IRQ_WORK +irq_work_run ();#endif irq_exit ();set_irq_regs (old_regs); } -void set_perf_event_pending (void ) +void arch_irq_work_raise (void ) {set_softint (1 <<PIL_DEFERRED_PCR_WORK ); }
arch/x86/Kconfig +1 -0
@@ -25,6 +25,7 @@ config X86 select HAVE_IDE select HAVE_OPROFILE select HAVE_PERF_EVENTS if (!M386 && !M486) + select HAVE_IRQ_WORK select HAVE_IOREMAP_PROT select HAVE_KPROBES select ARCH_WANT_OPTIONAL_GPIOLIB
arch/x86/include/asm/entry_arch.h +2 -2
@@ -49,8 +49,8 @@ BUILD_INTERRUPT(apic_timer_interrupt,LOCAL_TIMER_VECTOR) BUILD_INTERRUPT(error_interrupt,ERROR_APIC_VECTOR) BUILD_INTERRUPT(spurious_interrupt,SPURIOUS_APIC_VECTOR) -#ifdef CONFIG_PERF_EVENTS -BUILD_INTERRUPT(perf_pending_interrupt, LOCAL_PENDING_VECTOR) +#ifdef CONFIG_IRQ_WORK +BUILD_INTERRUPT(irq_work_interrupt, IRQ_WORK_VECTOR)#endif #ifdef CONFIG_X86_THERMAL_VECTOR
arch/x86/include/asm/hardirq.h +1 -1
@@ -14,7 +14,7 @@ typedef struct {#endif unsigned int x86_platform_ipis;/* arch dependent */ unsigned int apic_perf_irqs; -unsigned int apic_pending_irqs; +unsigned int apic_irq_work_irqs;#ifdef CONFIG_SMP unsigned int irq_resched_count;unsigned int irq_call_count;
arch/x86/include/asm/hw_irq.h +1 -1
@@ -29,7 +29,7 @@extern void apic_timer_interrupt (void );extern void x86_platform_ipi (void );extern void error_interrupt (void ); -extern void perf_pending_interrupt (void ); +extern void irq_work_interrupt (void );extern void spurious_interrupt (void );extern void thermal_interrupt (void );
arch/x86/include/asm/irq_vectors.h +2 -2
@@ -114,9 +114,9 @@#define X86_PLATFORM_IPI_VECTOR 0xed/* -* Performance monitoring pending work vector: +* IRQ work vector: */ -#define LOCAL_PENDING_VECTOR 0xec +#define IRQ_WORK_VECTOR 0xec#define UV_BAU_MESSAGE 0xea
arch/x86/kernel/Makefile +1 -0
@@ -35,6 +35,7 @@ obj-y := process_$(BITS).o signal.o entry_$(BITS).o obj-y += traps.o irq.o irq_$(BITS).o dumpstack_$(BITS).o obj-y += time.o ioport.o ldt.o dumpstack.o obj-y += setup.o x86_init.o i8259.o irqinit.o jump_label.o +obj-$(CONFIG_IRQ_WORK) += irq_work.o obj-$(CONFIG_X86_VISWS) += visws_quirks.o obj-$(CONFIG_X86_32) += probe_roms_32.o obj-$(CONFIG_X86_32) += sys_i386_32.o i386_ksyms_32.o
arch/x86/kernel/cpu/perf_event.c +0 -19
@@ -1196,25 +1196,6 @@ static int x86_pmu_handle_irq(struct pt_regs *regs)return handled; } -void smp_perf_pending_interrupt (struct pt_regs *regs) -{ -irq_enter (); -ack_APIC_irq (); -inc_irq_stat (apic_pending_irqs); -perf_event_do_pending (); -irq_exit (); -} - -void set_perf_event_pending (void ) -{ -#ifdef CONFIG_X86_LOCAL_APIC -if (!x86_pmu.apic || !x86_pmu_initialized ()) -return ; - - apic->send_IPI_self (LOCAL_PENDING_VECTOR ); -#endif -} -void perf_events_lapic_init (void ) {if (!x86_pmu.apic || !x86_pmu_initialized ())
arch/x86/kernel/entry_64.S +3 -3
@@ -1023,9 +1023,9 @@ apicinterrupt ERROR_APIC_VECTOR \ apicinterrupt SPURIOUS_APIC_VECTOR \ spurious_interrupt smp_spurious_interrupt -#ifdef CONFIG_PERF_EVENTS -apicinterrupt LOCAL_PENDING_VECTOR \ - perf_pending_interrupt smp_perf_pending_interrupt +#ifdef CONFIG_IRQ_WORK +apicinterrupt IRQ_WORK_VECTOR \ + irq_work_interrupt smp_irq_work_interrupt #endif /*
arch/x86/kernel/irq.c +4 -4
@@ -67,10 +67,10 @@ static int show_other_interrupts(struct seq_file *p, int prec)for_each_online_cpu (j)seq_printf (p,"%10u " ,irq_stats (j)->apic_perf_irqs);seq_printf (p," Performance monitoring interrupts\n" ); -seq_printf (p,"%*s: " , prec,"PND" ); +seq_printf (p,"%*s: " , prec,"IWI" );for_each_online_cpu (j) -seq_printf (p,"%10u " ,irq_stats (j)->apic_pending_irqs); -seq_printf (p," Performance pending work\n" ); +seq_printf (p,"%10u " ,irq_stats (j)->apic_irq_work_irqs); +seq_printf (p," IRQ work interrupts\n" );#endif if (x86_platform_ipi_callback) {seq_printf (p,"%*s: " , prec,"PLT" );
@@ -185,7 +185,7 @@ u64 arch_irq_stat_cpu(unsigned int cpu) sum +=irq_stats (cpu)->apic_timer_irqs; sum +=irq_stats (cpu)->irq_spurious_count; sum +=irq_stats (cpu)->apic_perf_irqs; - sum +=irq_stats (cpu)->apic_pending_irqs; + sum +=irq_stats (cpu)->apic_irq_work_irqs;#endif if (x86_platform_ipi_callback) sum +=irq_stats (cpu)->x86_platform_ipis;
arch/x86/kernel/irq_work.c new file +30 -0
@@ -0,0 +1,30 @@ +/* +* x86 specific code for irq_work +* +* Copyright (C) 2010 Red Hat, Inc., Peter Zijlstra <[email protected]> +*/ + +#include <linux/kernel.h> +#include <linux/irq_work.h> +#include <linux/hardirq.h> +#include <asm/apic.h> + +void smp_irq_work_interrupt (struct pt_regs *regs) +{ +irq_enter (); +ack_APIC_irq (); +inc_irq_stat (apic_irq_work_irqs); +irq_work_run (); +irq_exit (); +} + +void arch_irq_work_raise (void ) +{ +#ifdef CONFIG_X86_LOCAL_APIC +if (!cpu_has_apic) +return ; + + apic->send_IPI_self (IRQ_WORK_VECTOR ); +apic_wait_icr_idle (); +#endif +}
arch/x86/kernel/irqinit.c +3 -3
@@ -224,9 +224,9 @@ static void __init apic_intr_init(void)alloc_intr_gate (SPURIOUS_APIC_VECTOR , spurious_interrupt);alloc_intr_gate (ERROR_APIC_VECTOR , error_interrupt); -/* Performance monitoring interrupts: */ -# ifdef CONFIG_PERF_EVENTS -alloc_intr_gate (LOCAL_PENDING_VECTOR , perf_pending_interrupt); +/* IRQ work interrupts: */ +# ifdef CONFIG_IRQ_WORK +alloc_intr_gate (IRQ_WORK_VECTOR , irq_work_interrupt);# endif #endif
include/linux/irq_work.h new file +20 -0
@@ -0,0 +1,20 @@ +#ifndef _LINUX_IRQ_WORK_H +#define _LINUX_IRQ_WORK_H + +struct irq_work { +struct irq_work *next; +void (*func)(struct irq_work *); +}; + +static inline +void init_irq_work (struct irq_work *entry,void (*func)(struct irq_work *)) +{ + entry->next =NULL ; + entry->func = func; +} + +bool irq_work_queue (struct irq_work *entry); +void irq_work_run (void ); +void irq_work_sync (struct irq_work *entry); + +#endif /* _LINUX_IRQ_WORK_H */
include/linux/perf_event.h +2 -9
@@ -486,6 +486,7 @@ struct perf_guest_info_callbacks {#include <linux/workqueue.h> #include <linux/ftrace.h> #include <linux/cpu.h> +#include <linux/irq_work.h> #include <asm/atomic.h> #include <asm/local.h>
@@ -672,11 +673,6 @@ struct perf_buffer {void *data_pages[0 ]; }; -struct perf_pending_entry { -struct perf_pending_entry *next; -void (*func)(struct perf_pending_entry *); -}; -struct perf_sample_data ; typedef void (*perf_overflow_handler_t)(struct perf_event *, int,
@@ -784,7 +780,7 @@ struct perf_event {int pending_wakeup;int pending_kill;int pending_disable; -struct perf_pending_entry pending; +struct irq_work pending;atomic_t event_limit;
@@ -898,8 +894,6 @@ extern int perf_event_init_task(struct task_struct *child);extern void perf_event_exit_task (struct task_struct *child);extern void perf_event_free_task (struct task_struct *task);extern void perf_event_delayed_put (struct task_struct *task); -extern void set_perf_event_pending (void ); -extern void perf_event_do_pending (void );extern void perf_event_print_debug (void );extern void perf_pmu_disable (struct pmu *pmu);extern void perf_pmu_enable (struct pmu *pmu);
@@ -1078,7 +1072,6 @@ static inline int perf_event_init_task(struct task_struct *child) { return 0; }static inline void perf_event_exit_task (struct task_struct *child) { }static inline void perf_event_free_task (struct task_struct *task) { }static inline void perf_event_delayed_put (struct task_struct *task) { } -static inline void perf_event_do_pending (void ) { }static inline void perf_event_print_debug (void ) { }static inline int perf_event_task_disable (void ) {return -EINVAL ; }static inline int perf_event_task_enable (void ) {return -EINVAL ; }
init/Kconfig +8 -0
@@ -21,6 +21,13 @@ config CONSTRUCTORS depends on !UML default y +config HAVE_IRQ_WORK + bool + +config IRQ_WORK + bool + depends on HAVE_IRQ_WORK + menu "General setup" config EXPERIMENTAL
@@ -987,6 +994,7 @@ config PERF_EVENTS default y if (PROFILING || PERF_COUNTERS) depends on HAVE_PERF_EVENTS select ANON_INODES + select IRQ_WORK help Enable kernel support for various performance events provided by software and hardware.
kernel/Makefile +2 -0
@@ -23,6 +23,7 @@ CFLAGS_REMOVE_rtmutex-debug.o = -pg CFLAGS_REMOVE_cgroup-debug.o = -pg CFLAGS_REMOVE_sched_clock.o = -pg CFLAGS_REMOVE_perf_event.o = -pg +CFLAGS_REMOVE_irq_work.o = -pg endif obj-$(CONFIG_FREEZER) += freezer.o
@@ -100,6 +101,7 @@ obj-$(CONFIG_TRACING) += trace/ obj-$(CONFIG_X86_DS) += trace/ obj-$(CONFIG_RING_BUFFER) += trace/ obj-$(CONFIG_SMP) += sched_cpupri.o +obj-$(CONFIG_IRQ_WORK) += irq_work.o obj-$(CONFIG_PERF_EVENTS) += perf_event.o obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o obj-$(CONFIG_USER_RETURN_NOTIFIER) += user-return-notifier.o
kernel/irq_work.c new file +164 -0
164 lines not on this page · raw message
kernel/perf_event.c +5 -99
144 lines not on this page · raw message
kernel/timer.c +5 -2
@@ -37,7 +37,7 @@#include <linux/delay.h> #include <linux/tick.h> #include <linux/kallsyms.h> -#include <linux/perf_event.h> +#include <linux/irq_work.h> #include <linux/sched.h> #include <linux/slab.h>
@@ -1279,7 +1279,10 @@ void update_process_times(int user_tick)run_local_timers ();rcu_check_callbacks (cpu, user_tick);printk_tick (); -perf_event_do_pending (); +#ifdef CONFIG_IRQ_WORK +if (in_irq ()) +irq_work_run (); +#endif scheduler_tick ();run_posix_cpu_timers (p); }