Skip to contentSkip to conversation

Direct io on block device has performance regression on 2.6.x kernel

DISCUSSION Chen, Kenneth W 3 participants 6 messages Mar 9, 2005 – Mar 10, 2005

lore
Share & tools
lore
#0 lore raw

Chen, Kenneth W wrote on Wednesday, March 09, 2005 1:59 PM

Quoted 10 lines
Quoted 1 line

Did you generate a kernel profile?

Top 40 kernel hot functions, percentage is normalized to kernel utilization.

_spin_unlock_irqrestore 23.54%
_spin_unlock_irq 19.27%
....

Profile with spin lock inlined, so that it is easier to see functions that has the lock contention, again top 40 hot functions:

Just to clarify here, these data need to be taken at grain of salt. A high count in _spin_unlock_* functions do not automatically points to lock contention. It's one of the blind spot syndrome with timer based profile on ia64. There are some lock contentions in 2.6 kernel that we are staring at. Please do not misinterpret the number here.

- Ken

#1 lore raw
Quoted 6 lines · Kenneth W

Just to clarify here, these data need to be taken at grain of salt. A high count in _spin_unlock_* functions do not automatically points to lock contention. It's one of the blind spot syndrome with timer based profile on ia64. There are some lock contentions in 2.6 kernel that we are staring at. Please do not misinterpret the number here.

Why don't you use oprofileÂ>? It uses NMIs and can profile "inside" interrupt disabled sections.

-Andi

#2 lore raw

Andi Kleen wrote on Wednesday, March 09, 2005 3:23 PM

Quoted 8 lines
Quoted 5 lines

Just to clarify here, these data need to be taken at grain of salt. A high count in _spin_unlock_* functions do not automatically points to lock contention. It's one of the blind spot syndrome with timer based profile on ia64. There are some lock contentions in 2.6 kernel that we are staring at. Please do not misinterpret the number here.

Why don't you use oprofile�>? It uses NMIs and can profile "inside" interrupt disabled sections.

The profile is taken on ia64. we don't have nmi. Oprofile will produce the same result.

- Ken

#3 lore raw
In reply to Andi Kleen
Quoted 9 lines · Andi Kleen
Quoted 5 lines · Kenneth W

Just to clarify here, these data need to be taken at grain of salt. A high count in _spin_unlock_* functions do not automatically points to lock contention. It's one of the blind spot syndrome with timer based profile on ia64. There are some lock contentions in 2.6 kernel that we are staring at. Please do not misinterpret the number here.

Why don't you use oprofileÂ>? It uses NMIs and can profile "inside" interrupt disabled sections.

Oh, and there are other ways of doing interrupt off profiling by using the PMU. q-tools can do this I think.

Jesse

#4 lore raw

Jesse Barnes wrote on Wednesday, March 09, 2005 3:53 PM

Quoted 7 lines
Quoted 7 lines
Quoted 5 lines · Kenneth W

Just to clarify here, these data need to be taken at grain of salt. A high count in _spin_unlock_* functions do not automatically points to lock contention. It's one of the blind spot syndrome with timer based profile on ia64. There are some lock contentions in 2.6 kernel that we are staring at. Please do not misinterpret the number here.

Quoted 5 lines
Quoted 2 lines

Why don't you use oprofile�>? It uses NMIs and can profile "inside" interrupt disabled sections.

Oh, and there are other ways of doing interrupt off profiling by using the PMU. q-tools can do this I think.

Thank you all for the suggestions. I'm well aware of q-tools and been using it on and off. It's just that I don't have any data collected with q-tool for that particular hardware/software benchmark configuration. I posted with whatever data I have.

- Ken

#5 lore raw
In reply to Andi Kleen
Quoted 9 lines · Andi Kleen
Quoted 5 lines · Kenneth W

Just to clarify here, these data need to be taken at grain of salt. A high count in _spin_unlock_* functions do not automatically points to lock contention. It's one of the blind spot syndrome with timer based profile on ia64. There are some lock contentions in 2.6 kernel that we are staring at. Please do not misinterpret the number here.

Why don't you use oprofileÂ>? It uses NMIs and can profile "inside" interrupt disabled sections.

That was oprofile output, but on ia64, 'NMI's are maskable due to the way irq disabling works. Here's a very hackish patch that changes the kernel to use cr.tpr instead of psr.i for interrupt control. Making oprofile use real ia64 NMIs is left as an exercise for the reader :)

Jesse

===== arch/ia64/Kconfig.debug 1.2 vs edited =====

arch/ia64/Kconfig.debug +9 -0
@@ -56,6 +56,15 @@
 	  and restore instructions.  It's useful for tracking down spinlock
 	  problems, but slow!  If you're unsure, select N.
 
+config IA64_ALLOW_NMI
+	bool "Allow non-maskable interrupts"
+	help
+	  The normal ia64 irq enable/disable code prevents even non-maskable
+	  interrupts from occuring, which can be a problem for kernel
+	  debuggers, watchdogs, and profilers.  Say Y here if you're interested
+	  in NMIs and don't mind the small performance penalty this option
+	  imposes.
+
 config SYSVIPC_COMPAT
 	bool
 	depends on COMPAT && SYSVIPC

===== arch/ia64/kernel/head.S 1.31 vs edited =====

arch/ia64/kernel/head.S +10 -2
@@ -59,6 +59,14 @@
 	.save rp, r0		// terminate unwind chain with a NULL rp
 	.body
 
+#ifdef CONFIG_IA64_ALLOW_NMI	// disable interrupts initially (re-enabled in start_kernel())
+	mov r16=1<<16
+	;;
+	mov cr.tpr=r16
+	;;
+	srlz.d
+	;;
+#endif
 	rsm psr.i | psr.ic
 	;;
 	srlz.i
@@ -129,8 +137,8 @@
 	/*
 	 * Switch into virtual mode:
 	 */
-	movl r16=(IA64_PSR_IT|IA64_PSR_IC|IA64_PSR_DT|IA64_PSR_RT|IA64_PSR_DFH|IA64_PSR_BN \
-		  |IA64_PSR_DI)
+	movl r16=(IA64_PSR_IT|IA64_PSR_IC|IA64_PSR_I|IA64_PSR_DT|IA64_PSR_RT|IA64_PSR_DFH|\
+		  IA64_PSR_BN|IA64_PSR_DI)
 	;;
 	mov cr.ipsr=r16
 	movl r17=1f

===== arch/ia64/kernel/irq_ia64.c 1.25 vs edited =====

arch/ia64/kernel/irq_ia64.c +1 -11
@@ -103,8 +103,6 @@
 void
 ia64_handle_irq (ia64_vector vector, struct pt_regs *regs)
 {
-	unsigned long saved_tpr;
-
 #if IRQ_DEBUG
 	{
 		unsigned long bsp, sp;
@@ -135,17 +133,9 @@
 	}
 #endif /* IRQ_DEBUG */
 
-	/*
-	 * Always set TPR to limit maximum interrupt nesting depth to
-	 * 16 (without this, it would be ~240, which could easily lead
-	 * to kernel stack overflows).
-	 */
 	irq_enter();
-	saved_tpr = ia64_getreg(_IA64_REG_CR_TPR);
-	ia64_srlz_d();
 	while (vector != IA64_SPURIOUS_INT_VECTOR) {
 		if (!IS_RESCHEDULE(vector)) {
-			ia64_setreg(_IA64_REG_CR_TPR, vector);
 			ia64_srlz_d();
 
 			__do_IRQ(local_vector_to_irq(vector), regs);
@@ -154,7 +144,6 @@
 			 * Disable interrupts and send EOI:
 			 */
 			local_irq_disable();
-			ia64_setreg(_IA64_REG_CR_TPR, saved_tpr);
 		}
 		ia64_eoi();
 		vector = ia64_get_ivr();
@@ -165,6 +154,7 @@
 	 * come through until ia64_eoi() has been done.
 	 */
 	irq_exit();
+	local_irq_enable();
 }
 
 #ifdef CONFIG_HOTPLUG_CPU

===== include/asm-ia64/hw_irq.h 1.15 vs edited =====

include/asm-ia64/hw_irq.h +4 -0
@@ -36,6 +36,10 @@
 
 #define AUTO_ASSIGN			-1
 
+#define IA64_NMI_VECTOR			0x02	/* NMI (note that this can be
+						   masked if psr.i or psr.ic
+						   are cleared) */
+
 #define IA64_SPURIOUS_INT_VECTOR	0x0f
 
 /*

===== include/asm-ia64/system.h 1.48 vs edited =====

include/asm-ia64/system.h +63 -11

97 lines not on this page · raw message