Optimise root domain stalling (i386/x86_64)

Index: linux-2.6.20/include/linux/ipipe.h
===================================================================
@@ -53,13 +53,6 @@
 #define IPIPE_SPRINTK_FLAG	0	/* Synchronous printk() allowed */
 #define IPIPE_AHEAD_FLAG	1	/* Domain always heads the pipeline */
 
-/* Per-cpu pipeline status */
-#define IPIPE_STALL_FLAG	0	/* Stalls a pipeline stage -- guaranteed at bit #0 */
-#define IPIPE_SYNC_FLAG		1	/* The interrupt syncer is running for the domain */
-#define IPIPE_NOSTACK_FLAG	2	/* Domain currently runs on a foreign stack */
-
-#define IPIPE_SYNC_MASK		(1 << IPIPE_SYNC_FLAG)
-
 /* Interrupt control bits */
 #define IPIPE_HANDLE_FLAG	0
 #define IPIPE_PASS_FLAG		1

@@ -142,10 +135,8 @@ typedef int (*ipipe_event_handler_t)(uns
 				     void *data);
 struct ipipe_domain {
 
-	struct list_head p_link;	/* Link in pipeline */
-
 	struct ipcpudata {
-		unsigned long status;
+		unsigned long status;	/* Must be first in ipipe_domain */
 		unsigned long irq_pending_hi;
 		unsigned long irq_pending_lo[IPIPE_IRQ_IWORDS];
 		struct ipirqcnt {

@@ -162,6 +153,7 @@ struct ipipe_domain {
 		void *cookie;
 	} ____cacheline_aligned irqs[IPIPE_NR_IRQS];
 
+	struct list_head p_link;	/* Link in pipeline */
 	ipipe_event_handler_t evhand[IPIPE_NR_EVENTS]; /* Event handlers. */
 	unsigned long long evself;	/* Self-monitored event bits. */
 	unsigned long flags;

Index: linux-2.6.20/include/linux/ipipe_base.h
===================================================================
@@ -25,19 +25,54 @@
 
 #ifdef CONFIG_IPIPE
 
+/* Per-cpu pipeline status */
+#define IPIPE_STALL_FLAG	0	/* Stalls a pipeline stage -- guaranteed at bit #0 */
+#define IPIPE_SYNC_FLAG		1	/* The interrupt syncer is running for the domain */
+#define IPIPE_NOSTACK_FLAG	2	/* Domain currently runs on a foreign stack */
+
+#define IPIPE_SYNC_MASK		(1 << IPIPE_SYNC_FLAG)
+
+
 extern struct ipipe_domain ipipe_root;
 
 #define ipipe_root_domain (&ipipe_root)
 
 
-void __ipipe_stall_root(void);
+#ifdef CONFIG_SMP
 
-void __ipipe_unstall_root(void);
+void __ipipe_stall_root(void);
 
 unsigned long __ipipe_test_root(void);
 
 unsigned long __ipipe_test_and_stall_root(void);
 
+#else /* !CONFIG_SMP */
+
+/*
+ * Note: This cast relies on cpudata[0].status being the first element in the
+ *       root domain structure (for UP only).
+ */
+#define __ipipe_root_status	(unsigned long *)&ipipe_root
+
+static inline void __ipipe_stall_root(void)
+{
+	set_bit(IPIPE_STALL_FLAG, __ipipe_root_status);
+}
+
+static inline unsigned long __ipipe_test_root(void)
+{
+	return test_bit(IPIPE_STALL_FLAG, __ipipe_root_status);
+}
+
+static inline unsigned long __ipipe_test_and_stall_root(void)
+{
+	return test_and_set_bit(IPIPE_STALL_FLAG, __ipipe_root_status);
+}
+
+#endif /* !CONFIG_SMP */
+
+void __ipipe_unstall_root(void);
+
 void __ipipe_restore_root(unsigned long x);
 
 #endif	/* CONFIG_IPIPE */

Index: linux-2.6.20/kernel/ipipe/core.c
===================================================================
@@ -147,6 +147,7 @@ void __ipipe_cleanup_domain(struct ipipe
 #endif	/* CONFIG_SMP */
 }
 
+#ifdef CONFIG_SMP
 void __ipipe_stall_root(void)
 {
 	ipipe_declare_cpuid;

@@ -157,22 +158,6 @@ void __ipipe_stall_root(void)
 	ipipe_put_cpu(flags);
 }
 
-void __ipipe_unstall_root(void)
-{
-	ipipe_declare_cpuid;
-
-	local_irq_disable_hw();
-
-	ipipe_load_cpuid();
-
-	__clear_bit(IPIPE_STALL_FLAG, &ipipe_root_domain->cpudata[cpuid].status);
-
-	if (unlikely(ipipe_root_domain->cpudata[cpuid].irq_pending_hi != 0))
-		__ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
-
-	local_irq_enable_hw();
-}
-
 unsigned long __ipipe_test_root(void)
 {
 	unsigned long flags, x;

@@ -197,6 +182,23 @@ unsigned long __ipipe_test_and_stall_roo
 
 	return x;
 }
+#endif /* CONFIG_SMP */
+
+void __ipipe_unstall_root(void)
+{
+        ipipe_declare_cpuid;
+
+        local_irq_disable_hw();
+
+        ipipe_load_cpuid();
+
+        __clear_bit(IPIPE_STALL_FLAG, &ipipe_root_domain->cpudata[cpuid].status);
+
+        if (unlikely(ipipe_root_domain->cpudata[cpuid].irq_pending_hi != 0))
+                __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
+
+        local_irq_enable_hw();
+}
 
 void __ipipe_restore_root(unsigned long x)
 {

@@ -1395,10 +1397,12 @@ EXPORT_SYMBOL(ipipe_test_and_unstall_pip
 EXPORT_SYMBOL(ipipe_unstall_pipeline_head);
 EXPORT_SYMBOL(__ipipe_restore_pipeline_head);
 EXPORT_SYMBOL(__ipipe_unstall_root);
-EXPORT_SYMBOL(__ipipe_stall_root);
 EXPORT_SYMBOL(__ipipe_restore_root);
-EXPORT_SYMBOL(__ipipe_test_and_stall_root);
+#ifdef CONFIG_SMP
+EXPORT_SYMBOL(__ipipe_stall_root);
 EXPORT_SYMBOL(__ipipe_test_root);
+EXPORT_SYMBOL(__ipipe_test_and_stall_root);
+#endif /* CONFIG_SMP */
 EXPORT_SYMBOL(__ipipe_pipeline);
 EXPORT_SYMBOL(ipipe_register_domain);
 EXPORT_SYMBOL(ipipe_unregister_domain);

Index: linux-2.6.20/include/asm-i386/irqflags.h
===================================================================
@@ -24,6 +24,7 @@ static inline unsigned long __raw_local_
 
 #ifdef CONFIG_IPIPE
 	flags = (!__ipipe_test_root()) << 9;
+	barrier();
 #else
 	__asm__ __volatile__(
 		"pushfl ; popl %0"

@@ -38,6 +39,7 @@ static inline unsigned long __raw_local_
 static inline void raw_local_irq_restore(unsigned long flags)
 {
 #ifdef CONFIG_IPIPE
+	barrier();
 	__ipipe_restore_root(!(flags & 0x200));
 #else
 	__asm__ __volatile__(

@@ -53,6 +55,7 @@ static inline void raw_local_irq_disable
 {
 #ifdef CONFIG_IPIPE
 	__ipipe_stall_root();
+	barrier();
 #else
 	__asm__ __volatile__("cli" : : : "memory");
 #endif

@@ -61,6 +64,7 @@ static inline void raw_local_irq_disable
 static inline void raw_local_irq_enable(void)
 {
 #ifdef CONFIG_IPIPE
+	barrier();
 	__ipipe_unstall_root();
 #else
 	__asm__ __volatile__("sti" : : : "memory");

@@ -96,17 +100,29 @@ static inline void halt(void)
  */
 static inline unsigned long __raw_local_irq_save(void)
 {
+#ifdef CONFIG_IPIPE
+	unsigned long flags = (!__ipipe_test_and_stall_root()) << 9;
+
+	barrier();
+#else
 	unsigned long flags = __raw_local_save_flags();
 
 	raw_local_irq_disable();
-
+#endif
 	return flags;
 }
 
 #else
 
 #ifdef CONFIG_IPIPE
-#define DISABLE_INTERRUPTS(clobbers)		call __ipipe_stall_root	;  sti
+#ifdef CONFIG_SMP
+#define DISABLE_INTERRUPTS(clobbers)		call __ipipe_stall_root; sti
+#else /* !CONFIG_SMP */
+/*
+ * Disable IRQs == set IPIPE_STALL_FLAG in ipipe_root.cpudata[0].status
+ */
+#define DISABLE_INTERRUPTS(clobbers)		btsl $0,ipipe_root; sti
+#endif /* !CONFIG_SMP */
 #define ENABLE_INTERRUPTS(clobbers)		call __ipipe_unstall_root
 #define ENABLE_INTERRUPTS_HW_COND		sti
 #define DISABLE_INTERRUPTS_HW(clobbers)    	cli

Index: linux-2.6.20-x64/include/asm-x86_64/irqflags.h
===================================================================
@@ -24,6 +24,7 @@ static inline unsigned long __raw_local_
 
 #ifdef CONFIG_IPIPE
 	flags = (!__ipipe_test_root()) << 9;
+	barrier();
 #else
 	__asm__ __volatile__(
 		"# __raw_save_flags\n\t"

@@ -43,6 +44,7 @@ static inline unsigned long __raw_local_
 static inline void raw_local_irq_restore(unsigned long flags)
 {
 #ifdef CONFIG_IPIPE
+	barrier();
 	__ipipe_restore_root(!(flags & 0x200));
 #else
 	__asm__ __volatile__(

@@ -85,6 +87,7 @@ static inline void raw_local_irq_disable
 {
 #ifdef CONFIG_IPIPE
 	__ipipe_stall_root();
+	barrier();
 #else
 	__asm__ __volatile__("cli" : : : "memory");
 #endif

@@ -93,6 +96,7 @@ static inline void raw_local_irq_disable
 static inline void raw_local_irq_enable(void)
 {
 #ifdef CONFIG_IPIPE
+	barrier();
 	__ipipe_unstall_root();
 #else
 	__asm__ __volatile__("sti" : : : "memory");

@@ -112,10 +116,15 @@ static inline int raw_irqs_disabled_flag
 
 static inline unsigned long __raw_local_irq_save(void)
 {
+#ifdef CONFIG_IPIPE
+	unsigned long flags = (!__ipipe_test_and_stall_root()) << 9;
+
+	barrier();
+#else
 	unsigned long flags = __raw_local_save_flags();
 
 	raw_local_irq_disable();
-
+#endif
 	return flags;
 }