Skip to contentSkip to conversation

SGI RTC: add clocksource/clockevent driver and generic timer vector

PATCH Superseded Dimitri Sivanich 2 participants 5 messages Dec 19, 2008 – Dec 21, 2008

Subsystems (5)

v5 of 7 Superseded Latest is v7 →

lore
Share & tools
lore
#0 lore raw

v5 cover SGI RTC: add clocksource/clockevent driver and generic timer vector

The following patches provide a driver for synchronized RTC clocksource and clockevents for SGI systems, as well as a generic timer system interrupt.

With these patches, a module can be installed that registers the system-wide synchronized RTC clocksource and timers as both a clocksource and clockevents device running in high resolution mode.

[PATCH 1/2 v5] SGI RTC: add generic timer system interrupt [PATCH 2/2 v5] SGI RTC: add clocksource driver

#1 lore raw

PATCH v5 1/2 SGI RTC: add generic timer system interrupt

This patch allocates a system interrupt vector for platform specific use in implementing timer interrupts.

---

Refreshed for latest -tip.

 arch/x86/include/asm/hw_irq.h      |    1 
 arch/x86/include/asm/irq.h         |    2 +
 arch/x86/include/asm/irq_vectors.h |    5 +++
 arch/x86/kernel/entry_64.S         |    2 +
 arch/x86/kernel/irqinit_64.c       |   45 +++++++++++++++++++++++++++++++++
 5 files changed, 55 insertions(+)
arch/x86/kernel/entry_64.S +2 -0
Index: linux/arch/x86/kernel/entry_64.S
===================================================================
@@ -985,6 +985,8 @@ apicinterrupt UV_BAU_MESSAGE \
 	uv_bau_message_intr1 uv_bau_message_interrupt
 apicinterrupt LOCAL_TIMER_VECTOR \
 	apic_timer_interrupt smp_apic_timer_interrupt
+apicinterrupt GENERIC_TIMER_VECTOR \
+	generic_timer_interrupt smp_generic_timer_interrupt
 
 #ifdef CONFIG_SMP
 apicinterrupt INVALIDATE_TLB_VECTOR_START+0 \
arch/x86/kernel/irqinit_64.c +45 -0

63 lines not on this page · raw message

arch/x86/include/asm/irq_vectors.h +5 -0
Index: linux/arch/x86/include/asm/irq_vectors.h
===================================================================
@@ -92,6 +92,11 @@
 #define LOCAL_PERF_VECTOR	0xee
 
 /*
+ * Generic timer vector for platform specific use
+ */
+#define GENERIC_TIMER_VECTOR	0xed
+
+/*
  * First APIC vector available to drivers: (vectors 0x30-0xee) we
  * start at 0x31(0x41) to spread out vectors evenly between priority
  * levels. (0x80 is the syscall vector)
arch/x86/include/asm/hw_irq.h +1 -0
Index: linux/arch/x86/include/asm/hw_irq.h
===================================================================
@@ -29,6 +29,7 @@
 
 /* Interrupt handlers registered during init_IRQ */
 extern void apic_timer_interrupt(void);
+extern void generic_timer_interrupt(void);
 extern void error_interrupt(void);
 extern void perf_counter_interrupt(void);
 
arch/x86/include/asm/irq.h +2 -0
Index: linux/arch/x86/include/asm/irq.h
===================================================================
@@ -38,6 +38,8 @@ extern void fixup_irqs(void);
 
 extern unsigned int do_IRQ(struct pt_regs *regs);
 extern void init_IRQ(void);
+extern int register_generic_timer_extension(void (*fn)(void));
+extern void unregister_generic_timer_extension(void);
 extern void native_init_IRQ(void);
 
 /* Interrupt vector management */
#2 lore raw

PATCH v5 2/2 SGI RTC: add clocksource driver

This patch provides a driver for SGI RTC clocks and timers.

This provides a high resolution clock and timer source using the SGI system-wide synchronized RTC clock/timer hardware.

---

Refreshed for latest -tip.

 drivers/clocksource/Makefile |    1 
 drivers/clocksource/rtc_uv.c |  410 +++++++++++++++++++++++++++++++++++++++
 drivers/misc/Kconfig         |    9 
 kernel/time/clockevents.c    |    1 
 kernel/time/clocksource.c    |    1 
 5 files changed, 422 insertions(+)
drivers/clocksource/Makefile +1 -0
Index: linux/drivers/clocksource/Makefile
===================================================================
@@ -2,3 +2,4 @@ obj-$(CONFIG_ATMEL_TCB_CLKSRC)	+= tcb_cl
 obj-$(CONFIG_X86_CYCLONE_TIMER)	+= cyclone.o
 obj-$(CONFIG_X86_PM_TIMER)	+= acpi_pm.o
 obj-$(CONFIG_SCx200HR_TIMER)	+= scx200_hrt.o
+obj-$(CONFIG_SGI_RTC_TIMER)	+= rtc_uv.o
drivers/misc/Kconfig +9 -0
Index: linux/drivers/misc/Kconfig
===================================================================
@@ -498,6 +498,15 @@ config SGI_GRU_DEBUG
 	This option enables addition debugging code for the SGI GRU driver. If
 	you are unsure, say N.
 
+config SGI_RTC_TIMER
+	tristate "SGI RTC driver"
+	depends on GENERIC_CLOCKEVENTS_BUILD
+	depends on (X86_64 || IA64_SGI_UV || IA64_GENERIC) && SMP
+	default m
+	---help---
+	This option enables RTC event handling and adds the systemwide RTC
+	as a high resolution clock source for SGI systems.
+
 source "drivers/misc/c2port/Kconfig"
 
 endif # MISC_DEVICES
drivers/clocksource/rtc_uv.c new file +410 -0

410 lines not on this page · raw message

kernel/time/clockevents.c +1 -0
Index: linux/kernel/time/clockevents.c
===================================================================
@@ -185,6 +185,7 @@ void clockevents_register_device(struct 
 
 	spin_unlock(&clockevents_lock);
 }
+EXPORT_SYMBOL_GPL(clockevents_register_device);
 
 /*
  * Noop handler when we shut down an event device
kernel/time/clocksource.c +1 -0
Index: linux/kernel/time/clocksource.c
===================================================================
@@ -369,6 +369,7 @@ void clocksource_unregister(struct clock
 	next_clocksource = select_clocksource();
 	spin_unlock_irqrestore(&clocksource_lock, flags);
 }
+EXPORT_SYMBOL(clocksource_unregister);
 
 #ifdef CONFIG_SYSFS
 /**
#3 lore raw

Re: [PATCH 2/2 v5] SGI RTC: add clocksource driver

drivers/clocksource/rtc_uv.c line 80

Quoted 5 lines · Dimitri Sivanich
drivers/clocksource/rtc_uv.c new file +5 -0
+
+int cpu_to_pnode(int cpu)
+{
+	return uv_apicid_to_pnode(per_cpu(x86_cpu_to_apicid, cpu));
+}

static please

drivers/clocksource/rtc_uv.c line 88

Quoted 7 lines
drivers/clocksource/rtc_uv.c new file +7 -0
+/*
+ * Hardware interface routines
+ */
+
+/* Send IPIs to another node */
+static void
+uv_rtc_send_IPI(int cpu, int vector)

One line, except the line exceeds 80 chars. All over the place.

drivers/clocksource/rtc_uv.c line 99

Quoted 11 lines
drivers/clocksource/rtc_uv.c new file +11 -0
+{
+	unsigned long val, apicid, lapicid;
+	int pnode;
+
+	apicid = per_cpu(x86_cpu_to_apicid, cpu);
+	lapicid = apicid & 0x3f;
+	pnode = uv_apicid_to_pnode(apicid);
+
+	val = (1UL << UVH_IPI_INT_SEND_SHFT) | (lapicid <<
+						UVH_IPI_INT_APIC_ID_SHFT) |
+		(vector << UVH_IPI_INT_VECTOR_SHFT);

This is horrible to read.

+ val = (1UL << UVH_IPI_INT_SEND_SHFT) |
+ (lapicid << UVH_IPI_INT_APIC_ID_SHFT) |
+ (vector << UVH_IPI_INT_VECTOR_SHFT);

Takes the same amount of lines, but is parseable.

drivers/clocksource/rtc_uv.c line 111

Quoted 12 lines
drivers/clocksource/rtc_uv.c new file +12 -0
+	uv_write_global_mmr64(pnode, UVH_IPI_INT, val);
+}
+
+/* Check for an RTC interrupt pending */
+static int
+uv_intr_pending(int pnode)
+{
+	if (uv_read_global_mmr64(pnode, UVH_EVENT_OCCURRED0) &
+			UVH_EVENT_OCCURRED0_RTC1_MASK)
+		return 1;
+	else
+		return 0;

return uv_read_global_mmr64(pnode, UVH_EVENT_OCCURRED0) &
UVH_EVENT_OCCURRED0_RTC1_MASK;

drivers/clocksource/rtc_uv.c line 153

Quoted 9 lines
drivers/clocksource/rtc_uv.c new file +9 -0
+
+/* Allocate per-node list of cpu timer expiration times. */
+static int
+uv_rtc_allocate_timers(void)
+{
+	int cpu;
+	int max = 0;
+	int pnode = -1;
+	int i = 0;

Please collapse those ints into one line

Quoted 6 lines
drivers/clocksource/rtc_uv.c new file +6 -0
+	struct uv_rtc_timer_head *head = NULL;
+
+	/*
+	 * Determine max possible hyperthreads/pnode for allocation.
+	 *
+	 * Allocate for all cpus that could come up, although we don't

drivers/clocksource/rtc_uv.c line 169

Quoted 10 lines
drivers/clocksource/rtc_uv.c new file +10 -0
+	 * fully support hotplug (yet).
+	 */
+	for_each_possible_cpu(cpu) {
+		if (pnode != cpu_to_pnode(cpu)) {
+			i = 0;
+			pnode = cpu_to_pnode(cpu);
+		}
+		if (max < ++i)
+			max = i;
+	}

Is this worth the trouble whether we allocate a couple of u64s per node or not ?

Also isn't this kind of topology evaluation not done in x86 code already ?

drivers/clocksource/rtc_uv.c line 180

Quoted 10 lines
drivers/clocksource/rtc_uv.c new file +10 -0
+	pnode = -1;
+	for_each_possible_cpu(cpu) {
+		if (pnode != cpu_to_pnode(cpu)) {
+			i = 0;
+			pnode = cpu_to_pnode(cpu);
+			head = kmalloc_node(sizeof(struct uv_rtc_timer_head) +
+					(max * sizeof(u64)),
+					GFP_KERNEL, pnode);
+			if (!head)
+				return -ENOMEM;

Leaks memory for already allocated nodes

Quoted 10 lines
drivers/clocksource/rtc_uv.c new file +10 -0
+			spin_lock_init(&head->lock);
+			head->fcpu = cpu;
+			head->cpus = 0;
+			head->next_cpu = -1;
+		}
+		head->cpus++;
+		head->expires[i++] = ULLONG_MAX;
+		per_cpu(rtc_timer_head, cpu) = head;
+	}
+	return 0;

drivers/clocksource/rtc_uv.c line 200

Quoted 10 lines
drivers/clocksource/rtc_uv.c new file +10 -0
+}
+
+/* Find and set the next expiring timer.  */
+static void
+uv_rtc_find_next_timer(struct uv_rtc_timer_head *head, int pnode)
+{
+	u64 exp;
+	u64 lowest = ULLONG_MAX;
+	int cpu = -1;
+	int c;
  	u64 exp, lowest = ULLONG_MAX;
	int c, cpu = -1;

	please

drivers/clocksource/rtc_uv.c line 209

Quoted 8 lines
drivers/clocksource/rtc_uv.c new file +8 -0
+	head->next_cpu = -1;
+	for (c = 0; c < head->cpus; c++) {
+		exp = head->expires[c];
+		if (exp < lowest) {
+			cpu = c;
+			lowest = exp;
+		}
+	}

How many cpus do we iterate here ? Is this really scaling ?

drivers/clocksource/rtc_uv.c line 216

Quoted 7 lines
drivers/clocksource/rtc_uv.c new file +7 -0
+	if (cpu >= 0) {
+		head->next_cpu = cpu;
+		c = head->fcpu + cpu;
+		uv_setup_intr(c, lowest);
+		/* If we didn't set it up in time, trigger */
+		if (lowest < uv_read_rtc() && !uv_intr_pending(pnode))
+			uv_rtc_send_IPI(c, GENERIC_TIMER_VECTOR);

Please move the expired check into uv_setup_intr() and return the expired status. You need the same check in uv_rtc_set_timer() below.

drivers/clocksource/rtc_uv.c line 343

Quoted 10 lines
drivers/clocksource/rtc_uv.c new file +10 -0
+/*
+ * Local APIC timer broadcast function
+ */
+static void
+uv_rtc_timer_broadcast(cpumask_t mask)
+{
+	int cpu;
+	for_each_cpu_mask(cpu, mask)
+		uv_rtc_send_IPI(cpu, GENERIC_TIMER_VECTOR);
+}

No need to implement broadcast. Your clock event device is not affected from power states or such, so this will never be called.

drivers/clocksource/rtc_uv.c line 352

Quoted 8 lines
drivers/clocksource/rtc_uv.c new file +8 -0
+static void
+uv_rtc_interrupt(void)
+{
+	struct clock_event_device *ced = &__get_cpu_var(cpu_ced);
+	unsigned long flags;
+	int cpu = smp_processor_id();
+
+	local_irq_save(flags);

called with interrupts disabled already.

Quoted 9 lines
drivers/clocksource/rtc_uv.c new file +9 -0
+	if (!ced || !ced->event_handler || !uv_rtc_unset_timer(cpu)) {
+		local_irq_restore(flags);
+		printk(KERN_WARNING
+			"Spurious uv_rtc timer interrupt on cpu %d\n", cpu);
+		return;
+	}
+	local_irq_restore(flags);
+	ced->event_handler(ced);
+}

drivers/clocksource/rtc_uv.c line 371

Quoted 10 lines
drivers/clocksource/rtc_uv.c new file +10 -0
+
+static __init void
+uv_rtc_register_clockevents(void *data)
+{
+	struct clock_event_device *ced = &__get_cpu_var(cpu_ced);
+
+	*ced = clock_event_device_uv;
+	cpus_clear(ced->cpumask);
+	cpu_set(smp_processor_id(), ced->cpumask);
+	clockevents_register_device(ced);

clockevents_register_device can not be called from interrupt context. You use an SMP call for this. Please test your code with lockdep enabled.

Quoted 13 lines
drivers/clocksource/rtc_uv.c new file +13 -0
+}
+
+static __init int
+uv_rtc_setup_clock(void)
+{
+	int rc;
+
+	if (!is_uv_system() ||
+			register_generic_timer_extension(uv_rtc_interrupt))
+		return -ENODEV;
+
+	clocksource_uv.mult = clocksource_hz2mult(sn_rtc_cycles_per_second,
+				clocksource_uv.shift);

drivers/clocksource/rtc_uv.c line 394

Quoted 10 lines
drivers/clocksource/rtc_uv.c new file +10 -0
+
+	rc = clocksource_register(&clocksource_uv);
+	if (rc) {
+		unregister_generic_timer_extension();
+		return rc;
+	}
+
+	/* Setup and register clockevents */
+	rc = uv_rtc_allocate_timers();
+	if (rc) {

The clock source is already registered. So now the module is removed and the timekeeping code explodes.

clocksource_unregister(...) perpaps?

Quoted 21 lines
drivers/clocksource/rtc_uv.c new file +16 -0
+		unregister_generic_timer_extension();
+		return rc;
+	}
+
+	clock_event_device_uv.mult = div_sc(sn_rtc_cycles_per_second,
+				NSEC_PER_SEC, clock_event_device_uv.shift);
+
+	clock_event_device_uv.min_delta_ns = NSEC_PER_SEC /
+						sn_rtc_cycles_per_second;
+	clock_event_device_uv.max_delta_ns = clocksource_uv.mask *
+				(NSEC_PER_SEC / sn_rtc_cycles_per_second);
+
+	on_each_cpu(uv_rtc_register_clockevents, NULL, 0);
+	return 0;
+}
+module_init(uv_rtc_setup_clock);

Index: linux/kernel/time/clockevents.c
===================================================================

kernel/time/clockevents.c void clockevents_register_device(struct · line 188

Quoted 4 lines
kernel/time/clockevents.c +1 -0
@@ -185,6 +185,7 @@ void clockevents_register_device(struct 
 
 	spin_unlock(&clockevents_lock);
 }
+EXPORT_SYMBOL_GPL(clockevents_register_device);

Separate patch please

kernel/time/clocksource.c void clocksource_unregister(struct clock · line 372

Quoted 11 lines
kernel/time/clockevents.c +0 -0
 /*
  * Noop handler when we shut down an event device

Index: linux/kernel/time/clocksource.c
===================================================================

kernel/time/clocksource.c +1 -0
@@ -369,6 +369,7 @@ void clocksource_unregister(struct clock
 	next_clocksource = select_clocksource();
 	spin_unlock_irqrestore(&clocksource_lock, flags);
 }
+EXPORT_SYMBOL(clocksource_unregister);

EXPORT_SYMBOL_GPL and separate patch please.

Thanks,

tglx

#4 lore raw

Re: [PATCH 1/2 v5] SGI RTC: add generic timer system interrupt

In reply to Dimitri Sivanich
Quoted 2 lines · Dimitri Sivanich

This patch allocates a system interrupt vector for platform specific use in implementing timer interrupts.

Why is this restricted to timer interrupts. That's simply a dynamic allocation of an interrupt vector.

arch/x86/kernel/irqinit_64.c void __init init_ISA_irqs(void) · line 102

Quoted 6 lines
arch/x86/kernel/irqinit_64.c +6 -0
+
+/* Function pointer for generic timer interrupt handling */
+static void (*generic_timer_interrupt_extension)(void);
+
+int
+register_generic_timer_extension(void (*fn)(void))

One line. All over the place.

arch/x86/kernel/irqinit_64.c void __init init_ISA_irqs(void) · line 105

Quoted 3 lines
arch/x86/kernel/irqinit_64.c +3 -0
+{
+	if (generic_timer_interrupt_extension)
+		return 1;

-EBUSY perhaps ?

Quoted 15 lines
arch/x86/kernel/irqinit_64.c +15 -0
+	generic_timer_interrupt_extension = fn;
+	return 0;
+}
+EXPORT_SYMBOL_GPL(register_generic_timer_extension);
+
+void
+unregister_generic_timer_extension(void)
+{
+	if (generic_timer_interrupt_extension)
+		generic_timer_interrupt_extension = NULL;
+}
+EXPORT_SYMBOL_GPL(unregister_generic_timer_extension);
+
+void smp_generic_timer_interrupt(struct pt_regs *regs)
+{

arch/x86/kernel/irqinit_64.c void __init init_ISA_irqs(void) · line 131

Quoted 10 lines
arch/x86/kernel/irqinit_64.c +10 -0
+	struct pt_regs *old_regs = set_irq_regs(regs);
+
+	ack_APIC_irq();
+
+	exit_idle();
+
+	irq_enter();
+
+	if (generic_timer_interrupt_extension)
+		generic_timer_interrupt_extension();

interrupt statistics are missing.

arch/x86/kernel/irqinit_64.c void __init init_ISA_irqs(void) · line 137

Quoted 5 lines
arch/x86/kernel/irqinit_64.c +5 -0
+	irq_exit();
+
+	set_irq_regs(old_regs);
+}
+

Thanks,

tglx