irq_work: generic hard-irq context callbacks
Subsystems (11)
PATCH 1/1 irq_work: generic hard-irq context callbacks
From: Peter Zijlstra <[email protected]>
Provide a mechanism that allows running code in IRQ context. It is most useful for NMI code that needs to interact with the rest of the system -- like wakeup a task to drain buffers.
Perf currently has such a mechanism, so extract that and provide it as a generic feature, independent of perf so that others may also benefit.
The IRQ context callback is generated through self-IPIs where possible, or on architectures like powerpc the decrementer (the built-in timer facility) is set to generate an interrupt immediately.
Architectures that don't have anything like this get to do with a callback from the timer tick. These architectures can call irq_work_run() at the tail of any IRQ handlers that might enqueue such work (like the perf IRQ handler) to avoid undue latencies in processing the work.
Huang Ying: some fixes
This patch is only tested on x86 platform.
v6:
- Rewrite the changelog.
v5:
-compile test on all explicitly changed architectures except FRV.
v4:
-rebased on latest -tip tree
--- arch/alpha/Kconfig | 1 arch/alpha/include/asm/perf_event.h | 5 - arch/alpha/kernel/time.c | 30 +++--- arch/arm/Kconfig | 1 arch/arm/include/asm/perf_event.h | 12 -- arch/arm/kernel/perf_event.c | 8 - arch/frv/Kconfig | 1 arch/frv/lib/Makefile | 2 arch/frv/lib/perf_event.c | 19 ---- arch/parisc/Kconfig | 1 arch/parisc/include/asm/perf_event.h | 3 arch/powerpc/Kconfig | 1 arch/powerpc/include/asm/paca.h | 2 arch/powerpc/kernel/time.c | 42 ++++---- arch/s390/Kconfig | 1 arch/s390/include/asm/perf_event.h | 3 arch/sh/Kconfig | 1 arch/sh/include/asm/perf_event.h | 7 - arch/sparc/Kconfig | 2 arch/sparc/include/asm/perf_event.h | 4 arch/sparc/kernel/pcr.c | 8 - arch/x86/Kconfig | 1 arch/x86/include/asm/entry_arch.h | 4 arch/x86/include/asm/hardirq.h | 2 arch/x86/include/asm/hw_irq.h | 2 arch/x86/include/asm/irq_vectors.h | 4 arch/x86/kernel/Makefile | 1 arch/x86/kernel/cpu/perf_event.c | 19 ---- arch/x86/kernel/entry_64.S | 6 - arch/x86/kernel/irq.c | 8 - arch/x86/kernel/irq_work.c | 30 ++++++ arch/x86/kernel/irqinit.c | 6 - include/linux/irq_work.h | 20 ++++ include/linux/perf_event.h | 11 -- init/Kconfig | 8 + kernel/Makefile | 2 kernel/irq_work.c | 164 +++++++++++++++++++++++++++++++++++ kernel/perf_event.c | 104 +--------------------- kernel/timer.c | 7 + 39 files changed, 311 insertions(+), 242 deletions(-) delete mode 100644 arch/frv/lib/perf_event.c create mode 100644 arch/x86/kernel/irq_work.c create mode 100644 include/linux/irq_work.h create mode 100644 kernel/irq_work.c
arch/alpha/Kconfig +1 -0
@@ -9,6 +9,7 @@ config ALPHA select HAVE_IDE select HAVE_OPROFILE select HAVE_SYSCALL_WRAPPERS + select HAVE_IRQ_WORK select HAVE_PERF_EVENTS select HAVE_DMA_ATTRS help
arch/alpha/include/asm/perf_event.h +0 -5
@@ -1,11 +1,6 @@#ifndef __ASM_ALPHA_PERF_EVENT_H#define __ASM_ALPHA_PERF_EVENT_H -/* Alpha only supports software events through this interface. */ -extern void set_perf_event_pending (void ); - -#define PERF_EVENT_INDEX_OFFSET 0 -#ifdef CONFIG_PERF_EVENTS extern void init_hw_perf_events (void );#else
arch/alpha/kernel/time.c +15 -15
@@ -41,7 +41,7 @@#include <linux/init.h> #include <linux/bcd.h> #include <linux/profile.h> -#include <linux/perf_event.h> +#include <linux/irq_work.h> #include <asm/uaccess.h> #include <asm/io.h>
@@ -83,25 +83,25 @@ static struct {unsigned long est_cycle_freq; -#ifdef CONFIG_PERF_EVENTS +#ifdef CONFIG_IRQ_WORK -DEFINE_PER_CPU (u8, perf_event_pending); +DEFINE_PER_CPU (u8, irq_work_pending); -#define set_perf_event_pending_flag () __get_cpu_var(perf_event_pending) = 1 -#define test_perf_event_pending () __get_cpu_var(perf_event_pending) -#define clear_perf_event_pending () __get_cpu_var(perf_event_pending) = 0 +#define set_irq_work_pending_flag () __get_cpu_var(irq_work_pending) = 1 +#define test_irq_work_pending () __get_cpu_var(irq_work_pending) +#define clear_irq_work_pending () __get_cpu_var(irq_work_pending) = 0 -void set_perf_event_pending (void ) +void set_irq_work_pending (void ) { -set_perf_event_pending_flag (); +set_irq_work_pending_flag (); } -#else /* CONFIG_PERF_EVENTS */ +#else /* CONFIG_IRQ_WORK */ -#define test_perf_event_pending () 0 -#define clear_perf_event_pending () +#define test_irq_work_pending () 0 +#define clear_irq_work_pending () -#endif /* CONFIG_PERF_EVENTS */ +#endif /* CONFIG_IRQ_WORK */ static inline __u32 rpcc (void )
@@ -191,9 +191,9 @@ irqreturn_t timer_interrupt(int irq, voiwrite_sequnlock (&xtime_lock); -if (test_perf_event_pending ()) { -clear_perf_event_pending (); -perf_event_do_pending (); +if (test_irq_work_pending ()) { +clear_irq_work_pending (); +irq_work_run (); }#ifndef CONFIG_SMP
arch/arm/Kconfig +1 -0
@@ -23,6 +23,7 @@ config ARM select HAVE_KERNEL_GZIP select HAVE_KERNEL_LZO select HAVE_KERNEL_LZMA + select HAVE_IRQ_WORK select HAVE_PERF_EVENTS select PERF_USE_VMALLOC select HAVE_REGS_AND_STACK_ACCESS_API
arch/arm/include/asm/perf_event.h +0 -12
@@ -12,18 +12,6 @@#ifndef __ARM_PERF_EVENT_H__#define __ARM_PERF_EVENT_H__ -/* -* NOP: on *most* (read: all supported) ARM platforms, the performance -* counter interrupts are regular interrupts and not an NMI. This -* means that when we receive the interrupt we can call -* perf_event_do_pending() that handles all of the work with -* interrupts disabled. -*/ -static inline void -set_perf_event_pending (void ) -{ -} -/* ARM performance counters start from 1 (in the cp15 accesses) so use the * same indexes here for consistency. */ #define PERF_EVENT_INDEX_OFFSET 1
arch/arm/kernel/perf_event.c +4 -4
@@ -1086,7 +1086,7 @@ armv6pmu_handle_irq(int irq_num,* platforms that can have the PMU interrupts raised as an NMI, this * will not work. */ -perf_event_do_pending (); +irq_work_run ();return IRQ_HANDLED ; }
@@ -2062,7 +2062,7 @@ static irqreturn_t armv7pmu_handle_irq(i* platforms that can have the PMU interrupts raised as an NMI, this * will not work. */ -perf_event_do_pending (); +irq_work_run ();return IRQ_HANDLED ; }
@@ -2430,7 +2430,7 @@ xscale1pmu_handle_irq(int irq_num, void armpmu->disable (hwc, idx); } -perf_event_do_pending (); +irq_work_run ();/* * Re-enable the PMU.
@@ -2757,7 +2757,7 @@ xscale2pmu_handle_irq(int irq_num, void armpmu->disable (hwc, idx); } -perf_event_do_pending (); +irq_work_run ();/* * Re-enable the PMU.
arch/frv/Kconfig +1 -0
@@ -7,6 +7,7 @@ config FRV default y select HAVE_IDE select HAVE_ARCH_TRACEHOOK + select HAVE_IRQ_WORK select HAVE_PERF_EVENTS config ZONE_DMA
arch/frv/lib/Makefile +1 -1
@@ -5,4 +5,4 @@ lib-y := \ __ashldi3.o __lshrdi3.o __muldi3.o __ashrdi3.o __negdi2.o __ucmpdi2.o \ checksum.o memcpy.o memset.o atomic-ops.o atomic64-ops.o \ - outsl_ns.o outsl_sw.o insl_ns.o insl_sw.o cache.o perf_event.o + outsl_ns.o outsl_sw.o insl_ns.o insl_sw.o cache.o
arch/frv/lib/perf_event.c deleted +0 -19
@@ -1,19 +0,0 @@ -/* Performance event handling -* -* Copyright (C) 2009 Red Hat, Inc. All Rights Reserved. -* Written by David Howells ([email protected]) -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public Licence -* as published by the Free Software Foundation; either version -* 2 of the Licence, or (at your option) any later version. -*/ - -#include <linux/perf_event.h> - -/* -* mark the performance event as pending -*/ -void set_perf_event_pending (void ) -{ -}
arch/parisc/Kconfig +1 -0
@@ -16,6 +16,7 @@ config PARISC select RTC_DRV_GENERIC select INIT_ALL_POSSIBLE select BUG + select HAVE_IRQ_WORK select HAVE_PERF_EVENTS select GENERIC_ATOMIC64 if !64BIT help
arch/parisc/include/asm/perf_event.h +1 -2
@@ -1,7 +1,6 @@#ifndef __ASM_PARISC_PERF_EVENT_H#define __ASM_PARISC_PERF_EVENT_H -/* parisc only supports software events through this interface. */ -static inline void set_perf_event_pending (void ) { } +/* Empty, just to avoid compiling error */ #endif /* __ASM_PARISC_PERF_EVENT_H */
arch/powerpc/Kconfig +1 -0
@@ -138,6 +138,7 @@ config PPC select HAVE_OPROFILE select HAVE_SYSCALL_WRAPPERS if PPC64 select GENERIC_ATOMIC64 if PPC32 + select HAVE_IRQ_WORK select HAVE_PERF_EVENTS select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_HW_BREAKPOINT if PERF_EVENTS && PPC_BOOK3S_64
arch/powerpc/include/asm/paca.h +1 -1
@@ -129,7 +129,7 @@ struct paca_struct {u8 soft_enabled;/* irq soft-enable flag */ u8 hard_enabled;/* set if irqs are enabled in MSR */ u8 io_sync;/* writel() needs spin_unlock sync */ -u8 perf_event_pending;/* PM interrupt while soft-disabled */ +u8 irq_work_pending;/* IRQ_WORK interrupt while soft-disable */ /* Stuff for accurate time accounting */ u64 user_time;/* accumulated usermode TB ticks */
arch/powerpc/kernel/time.c +21 -21
97 lines not on this page · raw message
arch/s390/Kconfig +1 -0
@@ -95,6 +95,7 @@ config S390 select HAVE_KVM if 64BIT select HAVE_ARCH_TRACEHOOK select INIT_ALL_POSSIBLE + select HAVE_IRQ_WORK select HAVE_PERF_EVENTS select HAVE_KERNEL_GZIP select HAVE_KERNEL_BZIP2
arch/s390/include/asm/perf_event.h +1 -2
@@ -4,7 +4,6 @@* Copyright 2009 Martin Schwidefsky, IBM Corporation. */ -static inline void set_perf_event_pending (void ) {} -static inline void clear_perf_event_pending (void ) {} +/* Empty, just to avoid compiling error */ #define PERF_EVENT_INDEX_OFFSET 0
arch/sh/Kconfig +1 -0
@@ -16,6 +16,7 @@ config SUPERH select HAVE_ARCH_TRACEHOOK select HAVE_DMA_API_DEBUG select HAVE_DMA_ATTRS + select HAVE_IRQ_WORK select HAVE_PERF_EVENTS select PERF_USE_VMALLOC select HAVE_KERNEL_GZIP
arch/sh/include/asm/perf_event.h +0 -7
@@ -26,11 +26,4 @@ extern int register_sh_pmu(struct sh_pmuextern int reserve_pmc_hardware (void );extern void release_pmc_hardware (void ); -static inline void set_perf_event_pending (void ) -{ -/* Nothing to see here, move along. */ -} - -#define PERF_EVENT_INDEX_OFFSET 0 -#endif /* __ASM_SH_PERF_EVENT_H */
arch/sparc/Kconfig +2 -0
@@ -26,6 +26,7 @@ config SPARC select ARCH_WANT_OPTIONAL_GPIOLIB select RTC_CLASS select RTC_DRV_M48T59 + select HAVE_IRQ_WORK select HAVE_PERF_EVENTS select PERF_USE_VMALLOC select HAVE_DMA_ATTRS
@@ -54,6 +55,7 @@ config SPARC64 select RTC_DRV_BQ4802 select RTC_DRV_SUN4V select RTC_DRV_STARFIRE + select HAVE_IRQ_WORK select HAVE_PERF_EVENTS select PERF_USE_VMALLOC
arch/sparc/include/asm/perf_event.h +0 -4
@@ -1,10 +1,6 @@#ifndef __ASM_SPARC_PERF_EVENT_H#define __ASM_SPARC_PERF_EVENT_H -extern void set_perf_event_pending (void ); - -#define PERF_EVENT_INDEX_OFFSET 0 -#ifdef CONFIG_PERF_EVENTS #include <asm/ptrace.h>
arch/sparc/kernel/pcr.c +4 -4
@@ -7,7 +7,7 @@#include <linux/init.h> #include <linux/irq.h> -#include <linux/perf_event.h> +#include <linux/irq_work.h> #include <linux/ftrace.h> #include <asm/pil.h>
@@ -43,14 +43,14 @@ void __irq_entry deferred_pcr_work_irq(i old_regs =set_irq_regs (regs);irq_enter (); -#ifdef CONFIG_PERF_EVENTS -perf_event_do_pending (); +#ifdef CONFIG_IRQ_WORK +irq_work_run ();#endif irq_exit ();set_irq_regs (old_regs); } -void set_perf_event_pending (void ) +void arch_irq_work_raise (void ) {set_softint (1 <<PIL_DEFERRED_PCR_WORK ); }
arch/x86/Kconfig +1 -0
@@ -25,6 +25,7 @@ config X86 select HAVE_IDE select HAVE_OPROFILE select HAVE_PERF_EVENTS if (!M386 && !M486) + select HAVE_IRQ_WORK select HAVE_IOREMAP_PROT select HAVE_KPROBES select HAVE_MEMBLOCK
arch/x86/include/asm/entry_arch.h +2 -2
@@ -49,8 +49,8 @@ BUILD_INTERRUPT(apic_timer_interrupt,LOC BUILD_INTERRUPT(error_interrupt,ERROR_APIC_VECTOR) BUILD_INTERRUPT(spurious_interrupt,SPURIOUS_APIC_VECTOR) -#ifdef CONFIG_PERF_EVENTS -BUILD_INTERRUPT(perf_pending_interrupt, LOCAL_PENDING_VECTOR) +#ifdef CONFIG_IRQ_WORK +BUILD_INTERRUPT(irq_work_interrupt, IRQ_WORK_VECTOR)#endif #ifdef CONFIG_X86_THERMAL_VECTOR
arch/x86/include/asm/hardirq.h +1 -1
@@ -14,7 +14,7 @@ typedef struct {#endif unsigned int x86_platform_ipis;/* arch dependent */ unsigned int apic_perf_irqs; -unsigned int apic_pending_irqs; +unsigned int apic_irq_work_irqs;#ifdef CONFIG_SMP unsigned int irq_resched_count;unsigned int irq_call_count;
arch/x86/include/asm/hw_irq.h +1 -1
@@ -29,7 +29,7 @@extern void apic_timer_interrupt (void );extern void x86_platform_ipi (void );extern void error_interrupt (void ); -extern void perf_pending_interrupt (void ); +extern void irq_work_interrupt (void );extern void spurious_interrupt (void );extern void thermal_interrupt (void );
arch/x86/include/asm/irq_vectors.h +2 -2
@@ -114,9 +114,9 @@#define X86_PLATFORM_IPI_VECTOR 0xed/* -* Performance monitoring pending work vector: +* IRQ work vector: */ -#define LOCAL_PENDING_VECTOR 0xec +#define IRQ_WORK_VECTOR 0xec#define UV_BAU_MESSAGE 0xea
arch/x86/kernel/Makefile +1 -0
@@ -35,6 +35,7 @@ obj-y := process_$(BITS).o signal.o en obj-y += traps.o irq.o irq_$(BITS).o dumpstack_$(BITS).o obj-y += time.o ioport.o ldt.o dumpstack.o obj-y += setup.o x86_init.o i8259.o irqinit.o jump_label.o +obj-$(CONFIG_IRQ_WORK) += irq_work.o obj-$(CONFIG_X86_VISWS) += visws_quirks.o obj-$(CONFIG_X86_32) += probe_roms_32.o obj-$(CONFIG_X86_32) += sys_i386_32.o i386_ksyms_32.o
arch/x86/kernel/cpu/perf_event.c +0 -19
@@ -1198,25 +1198,6 @@ static int x86_pmu_handle_irq(struct pt_return handled; } -void smp_perf_pending_interrupt (struct pt_regs *regs) -{ -irq_enter (); -ack_APIC_irq (); -inc_irq_stat (apic_pending_irqs); -perf_event_do_pending (); -irq_exit (); -} - -void set_perf_event_pending (void ) -{ -#ifdef CONFIG_X86_LOCAL_APIC -if (!x86_pmu.apic || !x86_pmu_initialized ()) -return ; - - apic->send_IPI_self (LOCAL_PENDING_VECTOR ); -#endif -} -void perf_events_lapic_init (void ) {if (!x86_pmu.apic || !x86_pmu_initialized ())
arch/x86/kernel/entry_64.S +3 -3
@@ -1005,9 +1005,9 @@ apicinterrupt ERROR_APIC_VECTOR \ apicinterrupt SPURIOUS_APIC_VECTOR \ spurious_interrupt smp_spurious_interrupt -#ifdef CONFIG_PERF_EVENTS -apicinterrupt LOCAL_PENDING_VECTOR \ - perf_pending_interrupt smp_perf_pending_interrupt +#ifdef CONFIG_IRQ_WORK +apicinterrupt IRQ_WORK_VECTOR \ + irq_work_interrupt smp_irq_work_interrupt #endif /*
arch/x86/kernel/irq.c +4 -4
@@ -67,10 +67,10 @@ static int show_other_interrupts(structfor_each_online_cpu (j)seq_printf (p,"%10u " ,irq_stats (j)->apic_perf_irqs);seq_printf (p," Performance monitoring interrupts\n" ); -seq_printf (p,"%*s: " , prec,"PND" ); +seq_printf (p,"%*s: " , prec,"IWI" );for_each_online_cpu (j) -seq_printf (p,"%10u " ,irq_stats (j)->apic_pending_irqs); -seq_printf (p," Performance pending work\n" ); +seq_printf (p,"%10u " ,irq_stats (j)->apic_irq_work_irqs); +seq_printf (p," IRQ work interrupts\n" );#endif if (x86_platform_ipi_callback) {seq_printf (p,"%*s: " , prec,"PLT" );
@@ -185,7 +185,7 @@ u64 arch_irq_stat_cpu(unsigned int cpu) sum +=irq_stats (cpu)->apic_timer_irqs; sum +=irq_stats (cpu)->irq_spurious_count; sum +=irq_stats (cpu)->apic_perf_irqs; - sum +=irq_stats (cpu)->apic_pending_irqs; + sum +=irq_stats (cpu)->apic_irq_work_irqs;#endif if (x86_platform_ipi_callback) sum +=irq_stats (cpu)->x86_platform_ipis;
arch/x86/kernel/irq_work.c new file +30 -0
@@ -0,0 +1,30 @@ +/* +* x86 specific code for irq_work +* +* Copyright (C) 2010 Red Hat, Inc., Peter Zijlstra <[email protected]> +*/ + +#include <linux/kernel.h> +#include <linux/irq_work.h> +#include <linux/hardirq.h> +#include <asm/apic.h> + +void smp_irq_work_interrupt (struct pt_regs *regs) +{ +irq_enter (); +ack_APIC_irq (); +inc_irq_stat (apic_irq_work_irqs); +irq_work_run (); +irq_exit (); +} + +void arch_irq_work_raise (void ) +{ +#ifdef CONFIG_X86_LOCAL_APIC +if (!cpu_has_apic) +return ; + + apic->send_IPI_self (IRQ_WORK_VECTOR ); +apic_wait_icr_idle (); +#endif +}
arch/x86/kernel/irqinit.c +3 -3
@@ -224,9 +224,9 @@ static void __init apic_intr_init(void)alloc_intr_gate (SPURIOUS_APIC_VECTOR , spurious_interrupt);alloc_intr_gate (ERROR_APIC_VECTOR , error_interrupt); -/* Performance monitoring interrupts: */ -# ifdef CONFIG_PERF_EVENTS -alloc_intr_gate (LOCAL_PENDING_VECTOR , perf_pending_interrupt); +/* IRQ work interrupts: */ +# ifdef CONFIG_IRQ_WORK +alloc_intr_gate (IRQ_WORK_VECTOR , irq_work_interrupt);# endif #endif
include/linux/irq_work.h new file +20 -0
@@ -0,0 +1,20 @@ +#ifndef _LINUX_IRQ_WORK_H +#define _LINUX_IRQ_WORK_H + +struct irq_work { +struct irq_work *next; +void (*func)(struct irq_work *); +}; + +static inline +void init_irq_work (struct irq_work *entry,void (*func)(struct irq_work *)) +{ + entry->next =NULL ; + entry->func = func; +} + +bool irq_work_queue (struct irq_work *entry); +void irq_work_run (void ); +void irq_work_sync (struct irq_work *entry); + +#endif /* _LINUX_IRQ_WORK_H */
include/linux/perf_event.h +2 -9
@@ -486,6 +486,7 @@ struct perf_guest_info_callbacks {#include <linux/workqueue.h> #include <linux/ftrace.h> #include <linux/cpu.h> +#include <linux/irq_work.h> #include <asm/atomic.h> #include <asm/local.h>
@@ -672,11 +673,6 @@ struct perf_buffer {void *data_pages[0 ]; }; -struct perf_pending_entry { -struct perf_pending_entry *next; -void (*func)(struct perf_pending_entry *); -}; -struct perf_sample_data ; typedef void (*perf_overflow_handler_t)(struct perf_event *, int,
@@ -784,7 +780,7 @@ struct perf_event {int pending_wakeup;int pending_kill;int pending_disable; -struct perf_pending_entry pending; +struct irq_work pending;atomic_t event_limit;
@@ -896,8 +892,6 @@ extern int perf_event_init_task(struct textern void perf_event_exit_task (struct task_struct *child);extern void perf_event_free_task (struct task_struct *task);extern void perf_event_delayed_put (struct task_struct *task); -extern void set_perf_event_pending (void ); -extern void perf_event_do_pending (void );extern void perf_event_print_debug (void );extern void perf_pmu_disable (struct pmu *pmu);extern void perf_pmu_enable (struct pmu *pmu);
@@ -1076,7 +1070,6 @@ static inline int perf_event_init_task(sstatic inline void perf_event_exit_task (struct task_struct *child) { }static inline void perf_event_free_task (struct task_struct *task) { }static inline void perf_event_delayed_put (struct task_struct *task) { } -static inline void perf_event_do_pending (void ) { }static inline void perf_event_print_debug (void ) { }static inline int perf_event_task_disable (void ) {return -EINVAL ; }static inline int perf_event_task_enable (void ) {return -EINVAL ; }
init/Kconfig +8 -0
@@ -21,6 +21,13 @@ config CONSTRUCTORS depends on !UML default y +config HAVE_IRQ_WORK + bool + +config IRQ_WORK + bool + depends on HAVE_IRQ_WORK + menu "General setup" config EXPERIMENTAL
@@ -1005,6 +1012,7 @@ config PERF_EVENTS default y if (PROFILING || PERF_COUNTERS) depends on HAVE_PERF_EVENTS select ANON_INODES + select IRQ_WORK help Enable kernel support for various performance events provided by software and hardware.
kernel/Makefile +2 -0
@@ -22,6 +22,7 @@ CFLAGS_REMOVE_rtmutex-debug.o = -pg CFLAGS_REMOVE_cgroup-debug.o = -pg CFLAGS_REMOVE_sched_clock.o = -pg CFLAGS_REMOVE_perf_event.o = -pg +CFLAGS_REMOVE_irq_work.o = -pg endif obj-$(CONFIG_FREEZER) += freezer.o
@@ -100,6 +101,7 @@ obj-$(CONFIG_TRACING) += trace/ obj-$(CONFIG_X86_DS) += trace/ obj-$(CONFIG_RING_BUFFER) += trace/ obj-$(CONFIG_SMP) += sched_cpupri.o +obj-$(CONFIG_IRQ_WORK) += irq_work.o obj-$(CONFIG_PERF_EVENTS) += perf_event.o obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o obj-$(CONFIG_USER_RETURN_NOTIFIER) += user-return-notifier.o
kernel/irq_work.c new file +164 -0
164 lines not on this page · raw message
kernel/perf_event.c +5 -99
144 lines not on this page · raw message
kernel/timer.c +5 -2
@@ -37,7 +37,7 @@#include <linux/delay.h> #include <linux/tick.h> #include <linux/kallsyms.h> -#include <linux/perf_event.h> +#include <linux/irq_work.h> #include <linux/sched.h> #include <linux/slab.h>
@@ -1279,7 +1279,10 @@ void update_process_times(int user_tick)run_local_timers ();rcu_check_callbacks (cpu, user_tick);printk_tick (); -perf_event_do_pending (); +#ifdef CONFIG_IRQ_WORK +if (in_irq ()) +irq_work_run (); +#endif scheduler_tick ();run_posix_cpu_timers (p); }
Quoted 22 lines · Huang Ying
+++
b/arch/x86/kernel/irq_work.carch/x86/kernel/irq_work.c new file +20
-0@@ -0,0 +1,30 @@ +/* +* x86 specific code for irq_work +* +* Copyright (C) 2010 Red Hat, Inc., Peter Zijlstra <[email protected]> +*/ + +#include <linux/kernel.h> +#include <linux/irq_work.h> +#include <linux/hardirq.h> +#include <asm/apic.h> + +void smp_irq_work_interrupt (struct pt_regs *regs) +{ +irq_enter (); +ack_APIC_irq (); +inc_irq_stat (apic_irq_work_irqs); +irq_work_run (); +irq_exit (); +} +
arch/x86/kernel/irq_work.c line 30
Quoted 10 lines
arch/x86/kernel/irq_work.c new file +10
-0+void arch_irq_work_raise (void ) +{ +#ifdef CONFIG_X86_LOCAL_APIC +if (!cpu_has_apic) +return ; + + apic->send_IPI_self (IRQ_WORK_VECTOR ); +apic_wait_icr_idle (); +#endif +}
So basically, CONFIG_X86_LOCAL_APIC == !HAVE_IRQ_WORK ?
But IIUC, this will fallback to the timer interrupt:
kernel/timer.c void update_process_times(int user_tick) · line 1288
Quoted 12 lines
kernel/timer.c +4
-1@@ -1279,7 +1279,10 @@ void update_process_times(int user_tick)run_local_timers ();rcu_check_callbacks (cpu, user_tick);printk_tick (); -perf_event_do_pending (); +#ifdef CONFIG_IRQ_WORK +if (in_irq ()) +irq_work_run (); +#endif scheduler_tick ();run_posix_cpu_timers (p); }
Then HAVE_IRQ_WORK just means that the arch supports self IPIs. So, CONFIG_IRQ_WORK doesn't need to depend on HAVE_IRQ_WORK because of the timer fallback. But archs that support self IPIs should avoid the above fallback because it bloats the timer interrupt.
Perhaps CONFIG_HAVE_IRQ_WORK should be CONFIG_HAVE_IRQ_WORK_SOURCE or CONFIG_HAVE_IRQ_WORK_TRIGGER to better denote the capability. And then:
config IRQ_WORK_TRIGGER
depends on HAVE_IRQ_WORK_TRIGGER && IRQ_WORK default y
And that would define the right condition to build the fallback in the timer interrupt.
Or I am completely misunderstanding something?
Thanks.
Hi, Frederic,
Quoted 38 lines · Frederic Weisbecker
Quoted 22 lines · Huang Ying
+++
b/arch/x86/kernel/irq_work.carch/x86/kernel/irq_work.c new file +20
-0@@ -0,0 +1,30 @@ +/* +* x86 specific code for irq_work +* +* Copyright (C) 2010 Red Hat, Inc., Peter Zijlstra <[email protected]> +*/ + +#include <linux/kernel.h> +#include <linux/irq_work.h> +#include <linux/hardirq.h> +#include <asm/apic.h> + +void smp_irq_work_interrupt (struct pt_regs *regs) +{ + irq_enter(); + ack_APIC_irq(); + inc_irq_stat(apic_irq_work_irqs); + irq_work_run(); + irq_exit(); +} +arch/x86/kernel/irq_work.c line 30
Quoted 10 lines
arch/x86/kernel/irq_work.c new file +10
-0+void arch_irq_work_raise (void ) +{ +#ifdef CONFIG_X86_LOCAL_APIC + if (!cpu_has_apic) + return; + + apic->send_IPI_self(IRQ_WORK_VECTOR); + apic_wait_icr_idle(); +#endif +}So basically,
CONFIG_X86_LOCAL_APIC== !HAVE_IRQ_WORK ?
Quoted 1 line
But IIUC, this will fallback to the timer interrupt:
If !CONFIG_X86_LOCAL_APIC or !cpu_has_apic, this will fallback to the timer interrupt.
Quoted 6 lines
Quoted 6 lines
kernel/timer.c +1
-1@@ -1279,7 +1279,10 @@ void update_process_times(int user_tick) run_local_timers(); rcu_check_callbacks(cpu, user_tick); printk_tick(); - perf_event_do_pending(); +#ifdef CONFIG_IRQ_WORK
Quoted 8 lines
kernel/timer.c void update_process_times(int user_tick) · line 1288
Quoted 6 lines
kernel/timer.c +3
-0+ if (in_irq()) + irq_work_run(); +#endif scheduler_tick(); run_posix_cpu_timers(p); }Then
HAVE_IRQ_WORKjust means that the arch supports self IPIs.
HAVE_IRQ_WORK means IRQ_WORK is supported on the architecture, it has nothing to do with self IPIs.
Best Regards,
Huang Ying
Quoted 42 lines · huang ying
Hi, Frederic,
Quoted 38 lines · Frederic Weisbecker
Quoted 22 lines · Huang Ying
+++
b/arch/x86/kernel/irq_work.carch/x86/kernel/irq_work.c new file +20
-0@@ -0,0 +1,30 @@ +/* +* x86 specific code for irq_work +* +* Copyright (C) 2010 Red Hat, Inc., Peter Zijlstra <[email protected]> +*/ + +#include <linux/kernel.h> +#include <linux/irq_work.h> +#include <linux/hardirq.h> +#include <asm/apic.h> + +void smp_irq_work_interrupt (struct pt_regs *regs) +{ + irq_enter(); + ack_APIC_irq(); + inc_irq_stat(apic_irq_work_irqs); + irq_work_run(); + irq_exit(); +} +arch/x86/kernel/irq_work.c line 30
Quoted 10 lines
arch/x86/kernel/irq_work.c new file +10
-0+void arch_irq_work_raise (void ) +{ +#ifdef CONFIG_X86_LOCAL_APIC + if (!cpu_has_apic) + return; + + apic->send_IPI_self(IRQ_WORK_VECTOR); + apic_wait_icr_idle(); +#endif +}So basically,
CONFIG_X86_LOCAL_APIC== !HAVE_IRQ_WORK ?
Quoted 4 lines
Quoted 1 line
But IIUC, this will fallback to the timer interrupt:
If !CONFIG_X86_LOCAL_APIC or !cpu_has_apic, this will fallback to the timer interrupt.
Ah right, there is cpu_has_apic, so we may need the dynamic and adaptive fallback. May be cpu_has_apic only appears on very particular configs though, so that we can narrow down this situation statically?
Quoted 15 lines
Quoted 6 lines
Quoted 6 lines
kernel/timer.c +1
-1@@ -1279,7 +1279,10 @@ void update_process_times(int user_tick) run_local_timers(); rcu_check_callbacks(cpu, user_tick); printk_tick(); - perf_event_do_pending(); +#ifdef CONFIG_IRQ_WORK Quoted 8 lines
kernel/timer.c void update_process_times(int user_tick) · line 1288
Quoted 6 lines
kernel/timer.c +3
-0+ if (in_irq()) + irq_work_run(); +#endif scheduler_tick(); run_posix_cpu_timers(p); }Then
HAVE_IRQ_WORKjust means that the arch supports self IPIs.
Quoted 2 lines
HAVE_IRQ_WORKmeansIRQ_WORKis supported on the architecture, it has nothing to do with self IPIs.
Ok, but IRQ_WORK is always supported on the architecture because of the fallback. It seems that archs enable HAVE_IRQ_WORK when they find an optimization (self IPIs in x86, other shortcuts in other archs), but these optimizations never change the core code anyway, right? It would have been nice to avoid building the fallback when the archs can already manage the path by themselves.