Refactor xnlock debugging

PATCH New Jan Kiszka 1 participant 12 messages Mar 2, 2008 1 message not in the archive Generic include/asm header files

lore

Share & tools

lore

Jan Kiszka

author

2008-03-02 10:38 UTC #1 lore raw

PATCH 1/6 Refactor xnlock debugging

In reply to a message not in the archive

Concentrate all xnlock debugging related code in static inline wrappers that can easily be switched on and off without visual harm for the lock services themselves. This does not yet change the debugging logic, it just saves a few byte text by avoiding redundant processor-id accesses.

---
 include/asm-generic/system.h |  296 ++++++++++++++++++++++---------------------
 1 file changed, 154 insertions(+), 142 deletions(-)

include/asm-generic/system.h +154 ~~-142~~

Index: b/include/asm-generic/system.h
===================================================================
@@ -47,6 +47,10 @@
 #define CONFIG_XENO_OPT_DEBUG_NUCLEUS 0
 #endif
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 /* Time base export */
 #define xnarch_declare_tbase(base)		do { } while(0)

@@ -81,7 +85,13 @@ typedef unsigned long spl_t;
 #define spltest()   rthal_local_irq_test()
 #define splget(x)   rthal_local_irq_flags(x)
 
-#if defined(CONFIG_SMP) && defined(CONFIG_XENO_OPT_DEBUG)
+static inline unsigned xnarch_current_cpu(void)
+{
+	return rthal_processor_id();
+}
+
+#if defined(CONFIG_SMP) && XENO_DEBUG(NUCLEUS)
+
 typedef struct {
 
         unsigned long long spin_time;

@@ -114,12 +124,96 @@ typedef struct {
 	0LL,					\
 }
 
-#else /* !(CONFIG_SMP && CONFIG_XENO_OPT_DEBUG) */
+#define XNLOCK_DBG_CONTEXT		, __FILE__, __LINE__, __FUNCTION__
+#define XNLOCK_DBG_CONTEXT_ARGS \
+	, const char *file, int line, const char *function
+#define XNLOCK_DBG_PASS_CONTEXT		, file, line, function
+#define XNLOCK_DBG_MAX_SPINS		10000000
+
+static inline void
+xnlock_dbg_prepare_acquire(unsigned long long *start, unsigned *spin_limit)
+{
+	*start = rthal_rdtsc();
+	*spin_limit = XNLOCK_DBG_MAX_SPINS;
+}
+
+static inline void
+xnlock_dbg_spinning(xnlock_t *lock, int cpu, unsigned int *spin_limit,
+		    const char *file, int line, const char *function)
+{
+	if (--*spin_limit == 0) {
+		rthal_emergency_console();
+		printk(KERN_ERR "Xenomai: stuck on nucleus lock %p\n"
+				"         waiter = %s:%u (%s(), CPU #%d)\n"
+				"         owner  = %s:%u (%s(), CPU #%d)\n",
+		       lock, file, line, function, cpu,
+		       lock->file, lock->line, lock->function, lock->cpu);
+		show_stack(NULL, NULL);
+	}
+}
+
+static inline void
+xnlock_dbg_acquired(xnlock_t *lock, int cpu, unsigned long long *start,
+		    const char *file, int line, const char *function)
+{
+	lock->lock_date = *start;
+	lock->spin_time = rthal_rdtsc() - *start;
+	lock->file = file;
+	lock->function = function;
+	lock->line = line;
+	lock->cpu = cpu;
+}
+
+static inline void xnlock_dbg_release(xnlock_t *lock)
+{
+	extern xnlockinfo_t xnlock_stats[];
+	unsigned long long lock_time = rthal_rdtsc() - lock->lock_date;
+	xnlockinfo_t *stats = &xnlock_stats[xnarch_current_cpu()];
+
+	lock->cpu = -lock->cpu;	/* File that we released it. */
+
+	if (lock_time > stats->lock_time) {
+		stats->lock_time = lock_time;
+		stats->spin_time = lock->spin_time;
+		stats->file = lock->file;
+		stats->function = lock->function;
+		stats->line = lock->line;
+	}
+}
+
+static inline void xnlock_dbg_invalid_release(xnlock_t *lock)
+{
+	rthal_emergency_console();
+	printk(KERN_ERR "Xenomai: unlocking unlocked nucleus lock %p\n"
+			"       owner  = %s:%u (%s(), CPU #%d)\n",
+	       lock, lock->file, lock->line, lock->function, lock->cpu);
+	show_stack(NULL,NULL);
+}
+
+#else /* !(CONFIG_SMP && XENO_DEBUG(NUCLEUS)) */
 
 typedef struct { atomic_t owner; } xnlock_t;
 
 #define XNARCH_LOCK_UNLOCKED (xnlock_t) { { ~0 } }
-#endif /* !(CONFIG_SMP && CONFIG_XENO_OPT_DEBUG) */
+
+#define XNLOCK_DBG_CONTEXT
+#define XNLOCK_DBG_CONTEXT_ARGS
+#define XNLOCK_DBG_PASS_CONTEXT
+
+static inline void
+xnlock_dbg_prepare_acquire(unsigned long long *start, unsigned *spin_limit)
+{ }
+
+static inline void
+xnlock_dbg_spinning(xnlock_t *lock, int cpu, unsigned int *spin_limit)	{ }
+
+static inline void
+xnlock_dbg_acquired(xnlock_t *lock, int cpu, unsigned long long *start)	{ }
+
+static inline void xnlock_dbg_release(xnlock_t *lock)			{ }
+static inline void xnlock_dbg_invalid_release(xnlock_t *lock)		{ }
+
+#endif /* !(CONFIG_SMP && XENO_DEBUG(NUCLEUS)) */
 
 #define XNARCH_NR_CPUS               RTHAL_NR_CPUS

@@ -167,10 +261,6 @@ typedef struct xnarch_heapcb {
 
 } xnarch_heapcb_t;
 
-#ifdef __cplusplus
-extern "C" {
-#endif
-
 unsigned long long xnarch_get_host_time(void);
 
 long long xnarch_tsc_to_ns(long long ts);

@@ -189,11 +279,6 @@ static inline unsigned long long xnarch_
     return RTHAL_CPU_FREQ;
 }
 
-static inline unsigned xnarch_current_cpu(void)
-{
-    return rthal_processor_id();
-}
-
 #define xnarch_halt(emsg) \
 do { \
     rthal_emergency_console(); \

@@ -213,17 +298,11 @@ static inline int xnarch_setimask (int i
 
 #ifdef CONFIG_SMP
 
-#if XENO_DEBUG(NUCLEUS)
-#define xnlock_get(lock) \
-    __xnlock_get(lock, __FILE__, __LINE__,__FUNCTION__)
+#define xnlock_get(lock)		__xnlock_get(lock  XNLOCK_DBG_CONTEXT)
 #define xnlock_get_irqsave(lock,x) \
-    ((x) = __xnlock_get_irqsave(lock, __FILE__, __LINE__,__FUNCTION__))
-#else /* !XENO_DEBUG(NUCLEUS) */
-#define xnlock_get(lock)            __xnlock_get(lock)
-#define xnlock_get_irqsave(lock,x)  ((x) = __xnlock_get_irqsave(lock))
-#endif /* !XENO_DEBUG(NUCLEUS) */
-#define xnlock_clear_irqoff(lock)   xnlock_put_irqrestore(lock,1)
-#define xnlock_clear_irqon(lock)    xnlock_put_irqrestore(lock,0)
+	((x) = __xnlock_get_irqsave(lock  XNLOCK_DBG_CONTEXT))
+#define xnlock_clear_irqoff(lock)	xnlock_put_irqrestore(lock, 1)
+#define xnlock_clear_irqon(lock)	xnlock_put_irqrestore(lock, 0)
 
 static inline void xnlock_init (xnlock_t *lock)
 {

@@ -235,132 +314,65 @@ static inline void xnlock_init (xnlock_t
 #define DEFINE_XNLOCK(lock)		xnlock_t lock = XNARCH_LOCK_UNLOCKED
 #define DEFINE_PRIVATE_XNLOCK(lock)	static DEFINE_XNLOCK(lock)
 
-#if XENO_DEBUG(NUCLEUS)
+static inline int __xnlock_get(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)
+{
+	unsigned long long start;
+	unsigned int spin_limit;
+	int cpu = xnarch_current_cpu();
+
+	if (atomic_read(&lock->owner) == cpu)
+		return 1;
+
+	xnlock_dbg_prepare_acquire(&start, &spin_limit);
+
+	while (atomic_cmpxchg(&lock->owner, ~0, cpu) != ~0)
+		do {
+			cpu_relax();
+			xnlock_dbg_spinning(lock, cpu, &spin_limit /*, */
+					    XNLOCK_DBG_PASS_CONTEXT);
+		} while(atomic_read(&lock->owner) != ~0);
+
+	xnlock_dbg_acquired(lock, cpu, &start /*, */ XNLOCK_DBG_PASS_CONTEXT);
+
+	return 0;
+}
+
+static inline void xnlock_put(xnlock_t *lock)
+{
+	if (likely(atomic_read(&lock->owner) == xnarch_current_cpu())) {
+		xnlock_dbg_release(lock);
+
+		/*
+		 * Make sure all data written inside the lock is visible to
+		 * other CPUs before we release the lock.
+		 */
+		xnarch_memory_barrier();
+
+		atomic_set(&lock->owner, ~0);
+	} else
+		xnlock_dbg_invalid_release(lock);
+}
+
+static inline spl_t
+__xnlock_get_irqsave(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)
+{
+	unsigned long flags;
+
+	rthal_local_irq_save(flags);
 
-#define XNARCH_DEBUG_SPIN_LIMIT 3000000
+	if (__xnlock_get(lock /*, */ XNLOCK_DBG_PASS_CONTEXT))
+		flags |= 2;	/* Recursive acquisition */
 
-static inline int __xnlock_get (xnlock_t *lock,
-				 const char *file,
-				 unsigned line,
-				 const char *function)
-{
-    unsigned spin_count = 0;
-#else /* !XENO_DEBUG(NUCLEUS) */
-static inline int __xnlock_get (xnlock_t *lock)
-{
-#endif /* !XENO_DEBUG(NUCLEUS) */
-    int recursing;
-
-    recursing = (atomic_read(&lock->owner) == rthal_processor_id());
-    if (!recursing) {
-#if XENO_DEBUG(NUCLEUS)
-	    unsigned long long lock_date = rthal_rdtsc();
-#endif /* XENO_DEBUG(NUCLEUS) */
-	    while(atomic_cmpxchg(&lock->owner, ~0, rthal_processor_id()) != ~0)
-		    do {
-			    cpu_relax();
-
-#if XENO_DEBUG(NUCLEUS)
-			    if (++spin_count == XNARCH_DEBUG_SPIN_LIMIT) {
-				    rthal_emergency_console();
-				    printk(KERN_ERR
-					   "Xenomai: stuck on nucleus lock %p\n"
-					   "       waiter = %s:%u (%s(), CPU #%d)\n"
-					   "       owner  = %s:%u (%s(), CPU #%d)\n",
-					   lock,file,line,function,rthal_processor_id(),
-					   lock->file,lock->line,lock->function,lock->cpu);
-				    show_stack(NULL,NULL);
-				    for (;;)
-					    cpu_relax();
-			    }
-#endif /* XENO_DEBUG(NUCLEUS) */
-		    } while(atomic_read(&lock->owner) != ~0);
-
-#if XENO_DEBUG(NUCLEUS)
-	    lock->spin_time = rthal_rdtsc() - lock_date;
-	    lock->lock_date = lock_date;
-	    lock->file = file;
-	    lock->function = function;
-	    lock->line = line;
-	    lock->cpu = rthal_processor_id();
-#endif /* XENO_DEBUG(NUCLEUS) */
-        }
-
-    return recursing;
-}
-
-static inline void xnlock_put (xnlock_t *lock)
-{
-	if (likely(atomic_read(&lock->owner) == rthal_processor_id())) {
-
-#if XENO_DEBUG(NUCLEUS)
-	    extern xnlockinfo_t xnlock_stats[];
-
-	    unsigned long long lock_time = rthal_rdtsc() - lock->lock_date;
-	    int cpu = rthal_processor_id();
-
-	    if (lock_time > xnlock_stats[cpu].lock_time) {
-		    xnlock_stats[cpu].lock_time = lock_time;
-		    xnlock_stats[cpu].spin_time = lock->spin_time;
-		    xnlock_stats[cpu].file = lock->file;
-		    xnlock_stats[cpu].function = lock->function;
-		    xnlock_stats[cpu].line = lock->line;
-	    }
-#endif /* XENO_DEBUG(NUCLEUS) */
-	    /*
-	     * Make sure all data written inside the lock is visible to
-	     * other CPUs before we release the lock.
-	     */
-	    xnarch_memory_barrier();
-
-	    atomic_set(&lock->owner, ~0);
-    }
-#if XENO_DEBUG(NUCLEUS)
-    else {
-	    rthal_emergency_console();
-	    printk(KERN_ERR
-		   "Xenomai: unlocking unlocked nucleus lock %p\n"
-		   "       owner  = %s:%u (%s(), CPU #%d)\n",
-		   lock,lock->file,lock->line,lock->function,lock->cpu);
-	    show_stack(NULL,NULL);
-	    for (;;)
-		    cpu_relax();
-    }
-#endif /* XENO_DEBUG(NUCLEUS) */
-}
-
-#if XENO_DEBUG(NUCLEUS)
-
-static inline spl_t __xnlock_get_irqsave (xnlock_t *lock,
-                                          const char *file,
-                                          unsigned line,
-                                          const char *function)
-{
-#else /* !XENO_DEBUG(NUCLEUS) */
-static inline spl_t __xnlock_get_irqsave (xnlock_t *lock)
-{
-#endif /* !XENO_DEBUG(NUCLEUS) */
-    unsigned long flags;
-
-    rthal_local_irq_save(flags);
-
-#if XENO_DEBUG(NUCLEUS)
-    if (__xnlock_get(lock, file, line, function))
-	    flags |= 2;
-#else /* !XENO_DEBUG(NUCLEUS) */
-    if (__xnlock_get(lock))
-	    flags |= 2;
-#endif /* !XENO_DEBUG(NUCLEUS) */
-	
-    return flags;
+	return flags;
 }
 
-static inline void xnlock_put_irqrestore (xnlock_t *lock, spl_t flags)
+static inline void xnlock_put_irqrestore(xnlock_t *lock, spl_t flags)
 {
-    if (!(flags & 2))
-	    xnlock_put(lock);
+	/* Only release the lock if we didn't take it recursively. */
+	if (!(flags & 2))
+		xnlock_put(lock);
 
-    rthal_local_irq_restore(flags & 1);
+	rthal_local_irq_restore(flags & 1);
 }
 
 static inline int xnarch_send_ipi (xnarch_cpumask_t cpumask)

Jan Kiszka

author

2008-03-02 10:38 UTC #3 lore raw

PATCH 2/6 Whitespace fixes for asm-generic/system.h

In reply to a message not in the archive

No functional changes.

---
 include/asm-generic/system.h |  145 +++++++++++++++++++++----------------------
 1 file changed, 73 insertions(+), 72 deletions(-)

include/asm-generic/system.h +73 ~~-72~~

Index: b/include/asm-generic/system.h
===================================================================
@@ -56,13 +56,13 @@ extern "C" {
 
 /* Tracer interface */
 #define xnarch_trace_max_begin(v)		rthal_trace_max_begin(v)
-#define xnarch_trace_max_end(v)		rthal_trace_max_end(v)
+#define xnarch_trace_max_end(v)			rthal_trace_max_end(v)
 #define xnarch_trace_max_reset()		rthal_trace_max_reset()
 #define xnarch_trace_user_start()		rthal_trace_user_start()
 #define xnarch_trace_user_stop(v)		rthal_trace_user_stop(v)
-#define xnarch_trace_user_freeze(v, once) 	rthal_trace_user_freeze(v, once)
+#define xnarch_trace_user_freeze(v, once)	rthal_trace_user_freeze(v, once)
 #define xnarch_trace_special(id, v)		rthal_trace_special(id, v)
-#define xnarch_trace_special_u64(id, v)	rthal_trace_special_u64(id, v)
+#define xnarch_trace_special_u64(id, v)		rthal_trace_special_u64(id, v)
 #define xnarch_trace_pid(pid, prio)		rthal_trace_pid(pid, prio)
 #define xnarch_trace_panic_freeze()		rthal_trace_panic_freeze()
 #define xnarch_trace_panic_dump()		rthal_trace_panic_dump()

@@ -94,23 +94,23 @@ static inline unsigned xnarch_current_cp
 
 typedef struct {
 
-        unsigned long long spin_time;
-        unsigned long long lock_time;
-        const char *file;
-        const char *function;
-        unsigned line;
+	unsigned long long spin_time;
+	unsigned long long lock_time;
+	const char *file;
+	const char *function;
+	unsigned line;
 
 } xnlockinfo_t;
 
 typedef struct {
 
-    atomic_t owner;
-    const char *file;
-    const char *function;
-    unsigned line;
-    int cpu;
-    unsigned long long spin_time;
-    unsigned long long lock_date;
+	atomic_t owner;
+	const char *file;
+	const char *function;
+	unsigned line;
+	int cpu;
+	unsigned long long spin_time;
+	unsigned long long lock_date;
 
 } xnlock_t;

@@ -194,7 +194,7 @@ static inline void xnlock_dbg_invalid_re
 
 typedef struct { atomic_t owner; } xnlock_t;
 
-#define XNARCH_LOCK_UNLOCKED (xnlock_t) { { ~0 } }
+#define XNARCH_LOCK_UNLOCKED		(xnlock_t) { { ~0 } }
 
 #define XNLOCK_DBG_CONTEXT
 #define XNLOCK_DBG_CONTEXT_ARGS

@@ -215,49 +215,48 @@ static inline void xnlock_dbg_invalid_re
 
 #endif /* !(CONFIG_SMP && XENO_DEBUG(NUCLEUS)) */
 
-#define XNARCH_NR_CPUS               RTHAL_NR_CPUS
+#define XNARCH_NR_CPUS			RTHAL_NR_CPUS
 
-#define XNARCH_NR_IRQS               RTHAL_NR_IRQS
-#define XNARCH_TIMER_IRQ	     RTHAL_TIMER_IRQ
-#define XNARCH_TIMER_DEVICE          RTHAL_TIMER_DEVICE
-#define XNARCH_CLOCK_DEVICE          RTHAL_CLOCK_DEVICE
+#define XNARCH_NR_IRQS			RTHAL_NR_IRQS
+#define XNARCH_TIMER_IRQ		RTHAL_TIMER_IRQ
+#define XNARCH_TIMER_DEVICE		RTHAL_TIMER_DEVICE
+#define XNARCH_CLOCK_DEVICE		RTHAL_CLOCK_DEVICE
 
-#define XNARCH_ROOT_STACKSZ   0	/* Only a placeholder -- no stack */
+#define XNARCH_ROOT_STACKSZ		0 /* Only a placeholder -- no stack */
 
 #define XNARCH_PROMPT "Xenomai: "
-#define xnarch_loginfo(fmt,args...)  printk(KERN_INFO XNARCH_PROMPT fmt , ##args)
-#define xnarch_logwarn(fmt,args...)  printk(KERN_WARNING XNARCH_PROMPT fmt , ##args)
-#define xnarch_logerr(fmt,args...)   printk(KERN_ERR XNARCH_PROMPT fmt , ##args)
-#define xnarch_printf(fmt,args...)   printk(KERN_INFO XNARCH_PROMPT fmt , ##args)
+#define xnarch_loginfo(fmt, args...)	printk(KERN_INFO XNARCH_PROMPT fmt, ##args)
+#define xnarch_logwarn(fmt, args...)	printk(KERN_WARNING XNARCH_PROMPT fmt, ##args)
+#define xnarch_logerr(fmt, args...)	printk(KERN_ERR XNARCH_PROMPT fmt, ##args)
+#define xnarch_printf(fmt, args...)	printk(KERN_INFO XNARCH_PROMPT fmt, ##args)
 
 typedef cpumask_t xnarch_cpumask_t;
 
 #ifdef CONFIG_SMP
-#define xnarch_cpu_online_map            cpu_online_map
+#define xnarch_cpu_online_map			cpu_online_map
 #else
-#define xnarch_cpu_online_map		 cpumask_of_cpu(0)
+#define xnarch_cpu_online_map			cpumask_of_cpu(0)
 #endif
-#define xnarch_num_online_cpus()          num_online_cpus()
-#define xnarch_cpu_set(cpu, mask)         cpu_set(cpu, (mask))
-#define xnarch_cpu_clear(cpu, mask)       cpu_clear(cpu, (mask))
-#define xnarch_cpus_clear(mask)           cpus_clear(mask)
-#define xnarch_cpu_isset(cpu, mask)       cpu_isset(cpu, (mask))
-#define xnarch_cpus_and(dst, src1, src2)  cpus_and((dst), (src1), (src2))
-#define xnarch_cpus_equal(mask1, mask2)   cpus_equal((mask1), (mask2))
-#define xnarch_cpus_empty(mask)           cpus_empty(mask)
-#define xnarch_cpumask_of_cpu(cpu)        cpumask_of_cpu(cpu)
-#define xnarch_cpu_test_and_set(cpu,mask) cpu_test_and_set(cpu, (mask))
-
-#define xnarch_first_cpu(mask)            first_cpu(mask)
-#define XNARCH_CPU_MASK_ALL               CPU_MASK_ALL
+#define xnarch_num_online_cpus()		num_online_cpus()
+#define xnarch_cpu_set(cpu, mask)		cpu_set(cpu, (mask))
+#define xnarch_cpu_clear(cpu, mask)		cpu_clear(cpu, (mask))
+#define xnarch_cpus_clear(mask)			cpus_clear(mask)
+#define xnarch_cpu_isset(cpu, mask)		cpu_isset(cpu, (mask))
+#define xnarch_cpus_and(dst, src1, src2)	cpus_and((dst), (src1), (src2))
+#define xnarch_cpus_equal(mask1, mask2)		cpus_equal((mask1), (mask2))
+#define xnarch_cpus_empty(mask)			cpus_empty(mask)
+#define xnarch_cpumask_of_cpu(cpu)		cpumask_of_cpu(cpu)
+#define xnarch_cpu_test_and_set(cpu, mask)	cpu_test_and_set(cpu, (mask))
+#define xnarch_first_cpu(mask)			first_cpu(mask)
+#define XNARCH_CPU_MASK_ALL			CPU_MASK_ALL
 
 typedef struct xnarch_heapcb {
 
-    atomic_t numaps;	/* # of active user-space mappings. */
+	atomic_t numaps;	/* # of active user-space mappings. */
 
-    int kmflags;	/* Kernel memory flags (0 if vmalloc()). */
+	int kmflags;		/* Kernel memory flags (0 if vmalloc()). */
 
-    void *heapbase;	/* Shared heap memory base. */
+	void *heapbase;		/* Shared heap memory base. */
 
 } xnarch_heapcb_t;

@@ -267,7 +266,7 @@ long long xnarch_tsc_to_ns(long long ts)
 
 static inline long long xnarch_tsc_to_ns_rounded(long long ts)
 {
-    return (xnarch_llimd(ts, 1000000000, RTHAL_CPU_FREQ/2) + 1) / 2;
+	return (xnarch_llimd(ts, 1000000000, RTHAL_CPU_FREQ/2) + 1) / 2;
 }
 
 long long xnarch_ns_to_tsc(long long ns);

@@ -276,24 +275,26 @@ unsigned long long xnarch_get_cpu_time(v
 
 static inline unsigned long long xnarch_get_cpu_freq(void)
 {
-    return RTHAL_CPU_FREQ;
+	return RTHAL_CPU_FREQ;
 }
 
-#define xnarch_halt(emsg) \
-do { \
-    rthal_emergency_console(); \
-    xnarch_logerr("fatal: %s\n",emsg); \
-    show_stack(NULL,NULL);			\
-    xnarch_trace_panic_dump();			\
-    for (;;) cpu_relax();			\
-} while(0)
+#define xnarch_halt(emsg)				\
+	do {						\
+		rthal_emergency_console();		\
+		xnarch_logerr("fatal: %s\n", emsg);	\
+		show_stack(NULL,NULL);			\
+		xnarch_trace_panic_dump();		\
+		for (;;)				\
+			cpu_relax();			\
+	} while(0)
 
 static inline int xnarch_setimask (int imask)
 {
-    spl_t s;
-    splhigh(s);
-    splexit(!!imask);
-    return !!s;
+	spl_t s;
+
+	splhigh(s);
+	splexit(!!imask);
+	return !!s;
 }
 
 #ifdef CONFIG_SMP

@@ -306,7 +307,7 @@ static inline int xnarch_setimask (int i
 
 static inline void xnlock_init (xnlock_t *lock)
 {
-    *lock = XNARCH_LOCK_UNLOCKED;
+	*lock = XNARCH_LOCK_UNLOCKED;
 }
 
 #define DECLARE_XNLOCK(lock)		xnlock_t lock

@@ -375,9 +376,9 @@ static inline void xnlock_put_irqrestore
 	rthal_local_irq_restore(flags & 1);
 }
 
-static inline int xnarch_send_ipi (xnarch_cpumask_t cpumask)
+static inline int xnarch_send_ipi(xnarch_cpumask_t cpumask)
 {
-    return rthal_send_ipi(RTHAL_SERVICE_IPI0, cpumask);
+	return rthal_send_ipi(RTHAL_SERVICE_IPI0, cpumask);
 }
 
 static inline int xnlock_is_owner(xnlock_t *lock)

@@ -387,14 +388,14 @@ static inline int xnlock_is_owner(xnlock
 
 #else /* !CONFIG_SMP */
 
-#define xnlock_init(lock)              do { } while(0)
-#define xnlock_get(lock)               do { } while(0)
-#define xnlock_put(lock)               do { } while(0)
-#define xnlock_get_irqsave(lock,x)     rthal_local_irq_save(x)
-#define xnlock_put_irqrestore(lock,x)  rthal_local_irq_restore(x)
-#define xnlock_clear_irqoff(lock)      rthal_local_irq_disable()
-#define xnlock_clear_irqon(lock)       rthal_local_irq_enable()
-#define xnlock_is_owner(lock)	       1
+#define xnlock_init(lock)		do { } while(0)
+#define xnlock_get(lock)		do { } while(0)
+#define xnlock_put(lock)		do { } while(0)
+#define xnlock_get_irqsave(lock,x)	rthal_local_irq_save(x)
+#define xnlock_put_irqrestore(lock,x)	rthal_local_irq_restore(x)
+#define xnlock_clear_irqoff(lock)	rthal_local_irq_disable()
+#define xnlock_clear_irqon(lock)	rthal_local_irq_enable()
+#define xnlock_is_owner(lock)		1
 
 #define DECLARE_XNLOCK(lock)
 #define DECLARE_EXTERN_XNLOCK(lock)

@@ -403,7 +404,7 @@ static inline int xnlock_is_owner(xnlock
 
 static inline int xnarch_send_ipi (xnarch_cpumask_t cpumask)
 {
-    return 0;
+	return 0;
 }
 
 #endif /* !CONFIG_SMP */

@@ -418,7 +419,7 @@ static inline int xnarch_remap_vm_page(s
 				       unsigned long from,
 				       unsigned long to)
 {
-    return wrap_remap_vm_page(vma,from,to);
+	return wrap_remap_vm_page(vma, from, to);
 }
 
 static inline int xnarch_remap_io_page_range(struct vm_area_struct *vma,

@@ -427,7 +428,7 @@ static inline int xnarch_remap_io_page_r
 					     unsigned long size,
 					     pgprot_t prot)
 {
-    return wrap_remap_io_page_range(vma,from,to,size,prot);
+	return wrap_remap_io_page_range(vma, from, to, size, prot);
 }
 
 #ifndef xnarch_hisyscall_entry

Jan Kiszka

author

2008-03-02 10:38 UTC #5 lore raw

PATCH 3/6 Consolidate xnlock_put debugging

In reply to a message not in the archive

The owner check on xnlock release looks light-weight in first sight, but it isn't. It requires a processor-id lookup + it adds a conditional to the otherwise unconditional release path. Moreover, this code is inlined into every xnlock_put user, thus increases the text size of hot paths.

This patch therefore moves the check under the same switch (XENO_OPT_DEBUG_NUCLEUS) as the rest of xnlock debugging already is.

---
 include/asm-generic/system.h |   52 +++++++++++++++++++++++--------------------
 1 file changed, 28 insertions(+), 24 deletions(-)

include/asm-generic/system.h +28 ~~-24~~

Index: b/include/asm-generic/system.h
===================================================================
@@ -164,11 +164,23 @@ xnlock_dbg_acquired(xnlock_t *lock, int 
 	lock->cpu = cpu;
 }
 
-static inline void xnlock_dbg_release(xnlock_t *lock)
+static inline int xnlock_dbg_release(xnlock_t *lock)
 {
 	extern xnlockinfo_t xnlock_stats[];
 	unsigned long long lock_time = rthal_rdtsc() - lock->lock_date;
-	xnlockinfo_t *stats = &xnlock_stats[xnarch_current_cpu()];
+	int cpu = xnarch_current_cpu();
+	xnlockinfo_t *stats = &xnlock_stats[cpu];
+
+	if (unlikely(atomic_read(&lock->owner) != cpu)) {
+		rthal_emergency_console();
+		printk(KERN_ERR "Xenomai: unlocking unlocked nucleus lock %p"
+				" on CPU #%d\n"
+				"         owner  = %s:%u (%s(), CPU #%d)\n",
+		       lock, cpu, lock->file, lock->line, lock->function,
+		       lock->cpu);
+		show_stack(NULL,NULL);
+		return 1;
+	}
 
 	lock->cpu = -lock->cpu;	/* File that we released it. */

@@ -179,15 +191,7 @@ static inline void xnlock_dbg_release(xn
 		stats->function = lock->function;
 		stats->line = lock->line;
 	}
-}
-
-static inline void xnlock_dbg_invalid_release(xnlock_t *lock)
-{
-	rthal_emergency_console();
-	printk(KERN_ERR "Xenomai: unlocking unlocked nucleus lock %p\n"
-			"       owner  = %s:%u (%s(), CPU #%d)\n",
-	       lock, lock->file, lock->line, lock->function, lock->cpu);
-	show_stack(NULL,NULL);
+	return 0;
 }
 
 #else /* !(CONFIG_SMP && XENO_DEBUG(NUCLEUS)) */

@@ -210,8 +214,10 @@ xnlock_dbg_spinning(xnlock_t *lock, int 
 static inline void
 xnlock_dbg_acquired(xnlock_t *lock, int cpu, unsigned long long *start)	{ }
 
-static inline void xnlock_dbg_release(xnlock_t *lock)			{ }
-static inline void xnlock_dbg_invalid_release(xnlock_t *lock)		{ }
+static inline int xnlock_dbg_release(xnlock_t *lock)
+{
+	return 0;
+}
 
 #endif /* !(CONFIG_SMP && XENO_DEBUG(NUCLEUS)) */

@@ -340,18 +346,16 @@ static inline int __xnlock_get(xnlock_t 
 
 static inline void xnlock_put(xnlock_t *lock)
 {
-	if (likely(atomic_read(&lock->owner) == xnarch_current_cpu())) {
-		xnlock_dbg_release(lock);
+	if (xnlock_dbg_release(lock))
+		return;
+
+	/*
+	 * Make sure all data written inside the lock is visible to
+	 * other CPUs before we release the lock.
+	 */
+	xnarch_memory_barrier();
 
-		/*
-		 * Make sure all data written inside the lock is visible to
-		 * other CPUs before we release the lock.
-		 */
-		xnarch_memory_barrier();
-
-		atomic_set(&lock->owner, ~0);
-	} else
-		xnlock_dbg_invalid_release(lock);
+	atomic_set(&lock->owner, ~0);
 }
 
 static inline spl_t

Jan Kiszka

author

2008-03-02 10:38 UTC #7 lore raw

PATCH 4/6 Move spinning code out-of-line

In reply to a message not in the archive

To save text thus and thus avoid cache misses in the worst-case path, this patch moves the slow path of xnlock_get out-of-line.

---
 include/asm-generic/bits/pod.h |   18 ++++++++++++++++++
 include/asm-generic/system.h   |   29 ++++++++++++++---------------
 2 files changed, 32 insertions(+), 15 deletions(-)

include/asm-generic/bits/pod.h +18 -0

Index: b/include/asm-generic/bits/pod.h
===================================================================
@@ -295,4 +295,22 @@ unsigned long long xnarch_get_cpu_time(v
 
 EXPORT_SYMBOL(xnarch_get_cpu_time);
 
+#ifdef CONFIG_SMP
+void __xnlock_spin(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)
+{
+	unsigned int spin_limit;
+	int cpu = xnarch_current_cpu();
+
+	xnlock_dbg_prepare_spin(&spin_limit);
+
+	while (atomic_cmpxchg(&lock->owner, ~0, cpu) != ~0)
+		do {
+			cpu_relax();
+			xnlock_dbg_spinning(lock, cpu, &spin_limit /*, */
+					    XNLOCK_DBG_PASS_CONTEXT);
+		} while(atomic_read(&lock->owner) != ~0);
+}
+EXPORT_SYMBOL(__xnlock_spin);
+#endif /* CONFIG_SMP */
+
 #endif /* !_XENO_ASM_GENERIC_BITS_POD_H */

include/asm-generic/system.h +14 ~~-15~~

Index: b/include/asm-generic/system.h
===================================================================
@@ -130,10 +130,13 @@ typedef struct {
 #define XNLOCK_DBG_PASS_CONTEXT		, file, line, function
 #define XNLOCK_DBG_MAX_SPINS		10000000
 
-static inline void
-xnlock_dbg_prepare_acquire(unsigned long long *start, unsigned *spin_limit)
+static inline void xnlock_dbg_prepare_acquire(unsigned long long *start)
 {
 	*start = rthal_rdtsc();
+}
+
+static inline void xnlock_dbg_prepare_spin(unsigned *spin_limit)
+{
 	*spin_limit = XNLOCK_DBG_MAX_SPINS;
 }

@@ -204,15 +207,14 @@ typedef struct { atomic_t owner; } xnloc
 #define XNLOCK_DBG_CONTEXT_ARGS
 #define XNLOCK_DBG_PASS_CONTEXT
 
-static inline void
-xnlock_dbg_prepare_acquire(unsigned long long *start, unsigned *spin_limit)
-{ }
+static inline void xnlock_dbg_prepare_acquire(unsigned long long *start) { }
+static inline void xnlock_dbg_prepare_spin(unsigned *spin_limit)	 { }
 
 static inline void
-xnlock_dbg_spinning(xnlock_t *lock, int cpu, unsigned int *spin_limit)	{ }
+xnlock_dbg_spinning(xnlock_t *lock, int cpu, unsigned int *spin_limit)	 { }
 
 static inline void
-xnlock_dbg_acquired(xnlock_t *lock, int cpu, unsigned long long *start)	{ }
+xnlock_dbg_acquired(xnlock_t *lock, int cpu, unsigned long long *start)	 { }
 
 static inline int xnlock_dbg_release(xnlock_t *lock)
 {

@@ -321,23 +323,20 @@ static inline void xnlock_init (xnlock_t
 #define DEFINE_XNLOCK(lock)		xnlock_t lock = XNARCH_LOCK_UNLOCKED
 #define DEFINE_PRIVATE_XNLOCK(lock)	static DEFINE_XNLOCK(lock)
 
+void __xnlock_spin(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS);
+
 static inline int __xnlock_get(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)
 {
 	unsigned long long start;
-	unsigned int spin_limit;
 	int cpu = xnarch_current_cpu();
 
 	if (atomic_read(&lock->owner) == cpu)
 		return 1;
 
-	xnlock_dbg_prepare_acquire(&start, &spin_limit);
+	xnlock_dbg_prepare_acquire(&start);
 
-	while (atomic_cmpxchg(&lock->owner, ~0, cpu) != ~0)
-		do {
-			cpu_relax();
-			xnlock_dbg_spinning(lock, cpu, &spin_limit /*, */
-					    XNLOCK_DBG_PASS_CONTEXT);
-		} while(atomic_read(&lock->owner) != ~0);
+	if (unlikely(atomic_cmpxchg(&lock->owner, ~0, cpu) != ~0))
+		__xnlock_spin(lock /*, */ XNLOCK_DBG_PASS_CONTEXT);
 
 	xnlock_dbg_acquired(lock, cpu, &start /*, */ XNLOCK_DBG_PASS_CONTEXT);

Jan Kiszka

author

2008-03-02 10:38 UTC #9 lore raw

PATCH 5/6 Introduce optional recursive ticket xnlock

In reply to a message not in the archive

The origin of this series: When Nick Piggin posted his first suggestion for ticket spinlocks on LKML, I immediately liked the idea. For details check LWN [1], in a nutshell: This algorithm enforces strict FIFO order for the admission to contended spinlocks, thus it improves the determinism on SMP systems with more than 2 CPUs.

Meanwhile, ticket spinlocks are mainline (2.6.25). But that version has to drawbacks for us: it doesn't support nesting like xnlock does, and it is x86-only so far.

So I designed a version for Xenomai which is both nestable and arch-independent. It is certainly not as optimal as mainline's version, but our code path stresses the locking code differently anyway, and no one denies us to introduce per-arch optimized variants later on (if this lock implementation gains broader relevance).

Survived heavy testing on a 4-way box (15, 20, 25, and 30 us latency instances across the cores).

[1] http://lwn.net/Articles/267331

---
 include/asm-generic/bits/pod.h |    4 -
 include/asm-generic/system.h   |  122 +++++++++++++++++++++++++++++++++++++----
 ksrc/nucleus/Kconfig           |   12 ++++
 3 files changed, 126 insertions(+), 12 deletions(-)

include/asm-generic/system.h +112 ~~-10~~

Index: b/include/asm-generic/system.h
===================================================================
@@ -104,7 +104,12 @@ typedef struct {
 
 typedef struct {
 
+#ifdef CONFIG_XENO_OPT_TICKET_LOCK
+	atomic_t fifo;
+	int owner;
+#else
 	atomic_t owner;
+#endif
 	const char *file;
 	const char *function;
 	unsigned line;

@@ -115,7 +120,7 @@ typedef struct {
 } xnlock_t;
 
 #define XNARCH_LOCK_UNLOCKED (xnlock_t) {	\
-	{ ~0 },					\
+	XNLOCK_INIT_VALUE,			\
 	NULL,					\
 	NULL,					\
 	0,					\

@@ -167,6 +172,8 @@ xnlock_dbg_acquired(xnlock_t *lock, int 
 	lock->cpu = cpu;
 }
 
+static inline int xnlock_owner(xnlock_t *lock);
+
 static inline int xnlock_dbg_release(xnlock_t *lock)
 {
 	extern xnlockinfo_t xnlock_stats[];

@@ -174,7 +181,7 @@ static inline int xnlock_dbg_release(xnl
 	int cpu = xnarch_current_cpu();
 	xnlockinfo_t *stats = &xnlock_stats[cpu];
 
-	if (unlikely(atomic_read(&lock->owner) != cpu)) {
+	if (unlikely(xnlock_owner(lock) != cpu)) {
 		rthal_emergency_console();
 		printk(KERN_ERR "Xenomai: unlocking unlocked nucleus lock %p"
 				" on CPU #%d\n"

@@ -199,9 +206,18 @@ static inline int xnlock_dbg_release(xnl
 
 #else /* !(CONFIG_SMP && XENO_DEBUG(NUCLEUS)) */
 
+#ifdef CONFIG_XENO_OPT_TICKET_LOCK
+typedef struct {
+
+	atomic_t fifo;
+	int owner;
+
+} xnlock_t;
+#else
 typedef struct { atomic_t owner; } xnlock_t;
+#endif
 
-#define XNARCH_LOCK_UNLOCKED		(xnlock_t) { { ~0 } }
+#define XNARCH_LOCK_UNLOCKED		(xnlock_t) { XNLOCK_INIT_VALUE }
 
 #define XNLOCK_DBG_CONTEXT
 #define XNLOCK_DBG_CONTEXT_ARGS

@@ -313,11 +329,6 @@ static inline int xnarch_setimask (int i
 #define xnlock_clear_irqoff(lock)	xnlock_put_irqrestore(lock, 1)
 #define xnlock_clear_irqon(lock)	xnlock_put_irqrestore(lock, 0)
 
-static inline void xnlock_init (xnlock_t *lock)
-{
-	*lock = XNARCH_LOCK_UNLOCKED;
-}
-
 #define DECLARE_XNLOCK(lock)		xnlock_t lock
 #define DECLARE_EXTERN_XNLOCK(lock)	extern xnlock_t lock
 #define DEFINE_XNLOCK(lock)		xnlock_t lock = XNARCH_LOCK_UNLOCKED

@@ -325,12 +336,97 @@ static inline void xnlock_init (xnlock_t
 
 void __xnlock_spin(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS);
 
+#ifdef CONFIG_XENO_OPT_TICKET_LOCK
+/* Note: We assume atomic_t is at least 32-bit wide. */
+#define XNLOCK_HEAD_MASK		0x000000FF
+#define XNLOCK_HEAD_INC			0x00000001
+#define XNLOCK_TAIL_MASK		0xFF000000
+#define XNLOCK_TAIL_SHIFT		24
+#define XNLOCK_TAIL_INC			0x01000000
+#define XNLOCK_TICKET_MASK		(XNLOCK_HEAD_MASK | XNLOCK_TAIL_MASK)
+#define XNLOCK_INIT_VALUE		{ 0x00000001 }, ~0
+
+static inline int xnlock_owner(xnlock_t *lock)
+{
+	return lock->owner;
+}
+
+static inline int __xnlock_get(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)
+{
+	unsigned long long start;
+	int cpu = xnarch_current_cpu();
+	u32 fifo;
+	u8 ticket;
+
+	if (xnlock_owner(lock) == cpu)
+		return 1;
+
+	xnlock_dbg_prepare_acquire(&start);
+
+	fifo = atomic_add_return(XNLOCK_TAIL_INC, &lock->fifo);
+	ticket = fifo >> XNLOCK_TAIL_SHIFT;
+
+	/* First check if we are already the owner (head == tail). */
+	if (unlikely((u8)fifo != ticket)) {
+	 	unsigned int spin_limit;
+
+		xnlock_dbg_prepare_spin(&spin_limit);
+
+		/*
+		 * Spin until head equals our ticket. In this case,
+		 * moving out-of-line does not pay off.
+		 */
+		do {
+			cpu_relax();
+			xnlock_dbg_spinning(lock, cpu, &spin_limit /*, */
+					    XNLOCK_DBG_PASS_CONTEXT);
+		} while ((u8)atomic_read(&lock->fifo) != ticket);
+	}
+	lock->owner = cpu;
+
+	xnlock_dbg_acquired(lock, cpu, &start /*, */ XNLOCK_DBG_PASS_CONTEXT);
+
+	return 0;
+}
+
+static inline void xnlock_put(xnlock_t *lock)
+{
+	u32 fifo = atomic_read(&lock->fifo);
+
+	if (xnlock_dbg_release(lock))
+		return;
+
+	lock->owner = ~0;
+
+	/*
+	 * Make sure all data written inside the lock is visible to
+	 * other CPUs before we release the lock.
+	 */
+	xnarch_memory_barrier();
+
+	/*
+	 * Carefully calculate the addend so that we confine the
+	 * XNLOCK_HEAD_INC just to the first byte of fifo.
+	 */
+	atomic_add(((fifo + XNLOCK_HEAD_INC) & XNLOCK_TICKET_MASK) - fifo,
+		   &lock->fifo);
+}
+
+#else /* ! CONFIG_XENO_OPT_TICKET_LOCK */
+
+#define XNLOCK_INIT_VALUE		{ ~0 }
+
+static inline int xnlock_owner(xnlock_t *lock)
+{
+	return atomic_read(&lock->owner);
+}
+
 static inline int __xnlock_get(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)
 {
 	unsigned long long start;
 	int cpu = xnarch_current_cpu();
 
-	if (atomic_read(&lock->owner) == cpu)
+	if (xnlock_owner(lock) == cpu)
 		return 1;
 
 	xnlock_dbg_prepare_acquire(&start);

@@ -356,6 +452,12 @@ static inline void xnlock_put(xnlock_t *
 
 	atomic_set(&lock->owner, ~0);
 }
+#endif /* ! CONFIG_XENO_OPT_TICKET_LOCK */
+
+static inline void xnlock_init(xnlock_t *lock)
+{
+	*lock = XNARCH_LOCK_UNLOCKED;
+}
 
 static inline spl_t
 __xnlock_get_irqsave(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)

@@ -386,7 +488,7 @@ static inline int xnarch_send_ipi(xnarch
 
 static inline int xnlock_is_owner(xnlock_t *lock)
 {
-	return atomic_read(&lock->owner) == xnarch_current_cpu();
+	return xnlock_owner(lock) == xnarch_current_cpu();
 }
 
 #else /* !CONFIG_SMP */

ksrc/nucleus/Kconfig +12 -0

Index: b/ksrc/nucleus/Kconfig
===================================================================
@@ -340,6 +340,18 @@ config XENO_OPT_TIMER_WHEEL_STEP
 	Set the duration in ns of a timer wheel step. At each step, 
 	the timer wheel use the next hash bucket.
 
+config XENO_OPT_TICKET_LOCK
+	bool "FIFO ticket spinlocks"
+	depends on SMP
+	help
+
+	Uses a ticket spinlock algorithm which enforces strict FIFO
+	admission order on contended locks, improving the determinism
+	of lock acquisitions on large SMP systems. The ticket spinlock
+	has a higher overhead than the unordered algorithm and should
+	therefore only be used if there are more than two CPUs, thus
+	more than one waiter in the worst case.
+
 endmenu
 
 endif

include/asm-generic/bits/pod.h +2 -2

Index: b/include/asm-generic/bits/pod.h
===================================================================
@@ -295,7 +295,7 @@ unsigned long long xnarch_get_cpu_time(v
 
 EXPORT_SYMBOL(xnarch_get_cpu_time);
 
-#ifdef CONFIG_SMP
+#if defined(CONFIG_SMP) && !defined(CONFIG_XENO_OPT_TICKET_LOCK)
 void __xnlock_spin(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)
 {
 	unsigned int spin_limit;

@@ -311,6 +311,6 @@ void __xnlock_spin(xnlock_t *lock /*, */
 		} while(atomic_read(&lock->owner) != ~0);
 }
 EXPORT_SYMBOL(__xnlock_spin);
-#endif /* CONFIG_SMP */
+#endif /* CONFIG_SMP && !CONFIG_XENO_OPT_TICKET_LOCK */
 
 #endif /* !_XENO_ASM_GENERIC_BITS_POD_H */

Jan Kiszka

author

2008-03-02 10:38 UTC #11 lore raw

PATCH 6/6 Allow xnlock debugging on single-CPU systems

In reply to a message not in the archive

After all the refactoring, arming xnlock_dbg services also for !CONFIG_SMP setups was straightforward.

---
 include/asm-generic/bits/pod.h |    4 ++--
 include/asm-generic/system.h   |   31 +++++++++++++++----------------
 ksrc/nucleus/intr.c            |    4 ++--
 ksrc/nucleus/module.c          |   12 ++++++------
 ksrc/nucleus/pod.c             |    4 ++--
 ksrc/skins/posix/signal.c      |    2 +-
 6 files changed, 28 insertions(+), 29 deletions(-)

include/asm-generic/bits/pod.h +2 -2

Index: b/include/asm-generic/bits/pod.h
===================================================================
@@ -295,7 +295,7 @@ unsigned long long xnarch_get_cpu_time(v
 
 EXPORT_SYMBOL(xnarch_get_cpu_time);
 
-#if defined(CONFIG_SMP) && !defined(CONFIG_XENO_OPT_TICKET_LOCK)
+#if (defined(CONFIG_SMP) || XENO_DEBUG(NUCLEUS)) && !defined(CONFIG_XENO_OPT_TICKET_LOCK)
 void __xnlock_spin(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)
 {
 	unsigned int spin_limit;

@@ -311,6 +311,6 @@ void __xnlock_spin(xnlock_t *lock /*, */
 		} while(atomic_read(&lock->owner) != ~0);
 }
 EXPORT_SYMBOL(__xnlock_spin);
-#endif /* CONFIG_SMP && !CONFIG_XENO_OPT_TICKET_LOCK */
+#endif /* (CONFIG_SMP || XENO_DEBUG(NUCLEUS)) && !CONFIG_XENO_OPT_TICKET_LOCK */
 
 #endif /* !_XENO_ASM_GENERIC_BITS_POD_H */

include/asm-generic/system.h +15 ~~-16~~

Index: b/include/asm-generic/system.h
===================================================================
@@ -76,11 +76,11 @@ extern "C" {
 typedef unsigned long spl_t;
 
 #define splhigh(x)  rthal_local_irq_save(x)
-#ifdef CONFIG_SMP
+#if defined(CONFIG_SMP) || XENO_DEBUG(NUCLEUS)
 #define splexit(x)  rthal_local_irq_restore((x) & 1)
-#else /* !CONFIG_SMP */
+#else /* !CONFIG_SMP || XENO_DEBUG(NUCLEUS) */
 #define splexit(x)  rthal_local_irq_restore(x)
-#endif /* !CONFIG_SMP */
+#endif /* !CONFIG_SMP || XENO_DEBUG(NUCLEUS) */
 #define splnone()   rthal_local_irq_enable()
 #define spltest()   rthal_local_irq_test()
 #define splget(x)   rthal_local_irq_flags(x)

@@ -90,7 +90,7 @@ static inline unsigned xnarch_current_cp
 	return rthal_processor_id();
 }
 
-#if defined(CONFIG_SMP) && XENO_DEBUG(NUCLEUS)
+#if XENO_DEBUG(NUCLEUS)
 
 typedef struct {

@@ -204,7 +204,7 @@ static inline int xnlock_dbg_release(xnl
 	return 0;
 }
 
-#else /* !(CONFIG_SMP && XENO_DEBUG(NUCLEUS)) */
+#else /* !XENO_DEBUG(NUCLEUS) */
 
 #ifdef CONFIG_XENO_OPT_TICKET_LOCK
 typedef struct {

@@ -237,7 +237,7 @@ static inline int xnlock_dbg_release(xnl
 	return 0;
 }
 
-#endif /* !(CONFIG_SMP && XENO_DEBUG(NUCLEUS)) */
+#endif /* !XENO_DEBUG(NUCLEUS) */
 
 #define XNARCH_NR_CPUS			RTHAL_NR_CPUS

@@ -321,7 +321,7 @@ static inline int xnarch_setimask (int i
 	return !!s;
 }
 
-#ifdef CONFIG_SMP
+#if defined(CONFIG_SMP) || XENO_DEBUG(NUCLEUS)
 
 #define xnlock_get(lock)		__xnlock_get(lock  XNLOCK_DBG_CONTEXT)
 #define xnlock_get_irqsave(lock,x) \

@@ -481,17 +481,12 @@ static inline void xnlock_put_irqrestore
 	rthal_local_irq_restore(flags & 1);
 }
 
-static inline int xnarch_send_ipi(xnarch_cpumask_t cpumask)
-{
-	return rthal_send_ipi(RTHAL_SERVICE_IPI0, cpumask);
-}
-
 static inline int xnlock_is_owner(xnlock_t *lock)
 {
 	return xnlock_owner(lock) == xnarch_current_cpu();
 }
 
-#else /* !CONFIG_SMP */
+#else /* !(CONFIG_SMP || XENO_DEBUG(NUCLEUS)) */
 
 #define xnlock_init(lock)		do { } while(0)
 #define xnlock_get(lock)		do { } while(0)

@@ -507,12 +502,16 @@ static inline int xnlock_is_owner(xnlock
 #define DEFINE_XNLOCK(lock)
 #define DEFINE_PRIVATE_XNLOCK(lock)
 
-static inline int xnarch_send_ipi (xnarch_cpumask_t cpumask)
+#endif /* !(CONFIG_SMP || XENO_DEBUG(NUCLEUS)) */
+
+static inline int xnarch_send_ipi(xnarch_cpumask_t cpumask)
 {
+#ifdef CONFIG_SMP
+	return rthal_send_ipi(RTHAL_SERVICE_IPI0, cpumask);
+#else /* !CONFIG_SMP */
 	return 0;
-}
-
 #endif /* !CONFIG_SMP */
+}
 
 #define xnlock_sync_irq(lock, x)			\
 	do {						\

ksrc/nucleus/intr.c +2 -2

Index: b/ksrc/nucleus/intr.c
===================================================================
@@ -387,7 +387,7 @@ static inline int xnintr_irq_detach(xnin
 
 #else /* !CONFIG_XENO_OPT_SHIRQ */
 
-#ifdef CONFIG_SMP
+#if defined(CONFIG_SMP) || XENO_DEBUG(NUCLEUS)
 typedef struct xnintr_irq {
 
 	DECLARE_XNLOCK(lock);

@@ -395,7 +395,7 @@ typedef struct xnintr_irq {
 } ____cacheline_aligned_in_smp xnintr_irq_t;
 
 static xnintr_irq_t xnirqs[XNARCH_NR_IRQS];
-#endif /* CONFIG_SMP */
+#endif /* CONFIG_SMP || XENO_DEBUG(NUCLEUS) */
 
 static inline xnintr_t *xnintr_shirq_first(unsigned irq)
 {

ksrc/nucleus/module.c +6 -6

Index: b/ksrc/nucleus/module.c
===================================================================
@@ -664,7 +664,7 @@ void xnpod_discard_tbase_proc(xntbase_t 
 
 #endif /* CONFIG_XENO_OPT_STATS */
 
-#if defined(CONFIG_SMP) && XENO_DEBUG(NUCLEUS)
+#if XENO_DEBUG(NUCLEUS)
 
 xnlockinfo_t xnlock_stats[RTHAL_NR_CPUS];

@@ -712,7 +712,7 @@ static int lock_read_proc(char *page,
 
 EXPORT_SYMBOL(xnlock_stats);
 
-#endif /* CONFIG_SMP && XENO_DEBUG(NUCLEUS) */
+#endif /* XENO_DEBUG(NUCLEUS) */
 
 static int latency_read_proc(char *page,
 			     char **start,

@@ -1033,9 +1033,9 @@ void xnpod_init_proc(void)
 		create_proc_entry("timerstat", S_IFDIR, rthal_proc_root);
 #endif /* CONFIG_XENO_OPT_STATS */
 
-#if defined(CONFIG_SMP) && XENO_DEBUG(NUCLEUS)
+#if XENO_DEBUG(NUCLEUS)
 	add_proc_leaf("lock", &lock_read_proc, NULL, NULL, rthal_proc_root);
-#endif /* CONFIG_SMP && XENO_DEBUG(NUCLEUS) */
+#endif /* XENO_DEBUG(NUCLEUS) */
 
 	add_proc_leaf("latency",
 		      &latency_read_proc,

@@ -1087,9 +1087,9 @@ void xnpod_delete_proc(void)
 	remove_proc_entry("timerstat", rthal_proc_root);
 	remove_proc_entry("stat", rthal_proc_root);
 #endif /* CONFIG_XENO_OPT_STATS */
-#if defined(CONFIG_SMP) && XENO_DEBUG(NUCLEUS)
+#if XENO_DEBUG(NUCLEUS)
 	remove_proc_entry("lock", rthal_proc_root);
-#endif /* CONFIG_SMP && XENO_DEBUG(NUCLEUS) */
+#endif /* XENO_DEBUG(NUCLEUS) */
 }
 
 #ifdef CONFIG_XENO_OPT_PERVASIVE

ksrc/nucleus/pod.c +2 -2

Index: b/ksrc/nucleus/pod.c
===================================================================
@@ -3472,7 +3472,7 @@ EXPORT_SYMBOL(xnpod_welcome_thread);
 
 EXPORT_SYMBOL(nkpod_struct);
 
-#ifdef CONFIG_SMP
+#if defined(CONFIG_SMP) || XENO_DEBUG(NUCLEUS)
 EXPORT_SYMBOL(nklock);
-#endif /* CONFIG_SMP */
+#endif /* CONFIG_SMP || XENO_DEBUG(NUCLEUS) */
 EXPORT_SYMBOL(nklatency);

ksrc/skins/posix/signal.c +1 -1

Index: b/ksrc/skins/posix/signal.c
===================================================================
@@ -64,7 +64,7 @@ typedef void siginfo_handler_t(int, sigi
 #define SIGRTMAX 64
 static struct sigaction actions[SIGRTMAX];
 static pse51_siginfo_t pse51_infos_pool[PSE51_SIGQUEUE_MAX];
-#ifdef CONFIG_SMP
+#if defined(CONFIG_SMP) || XENO_DEBUG(NUCLEUS)
 static xnlock_t pse51_infos_lock = XNARCH_LOCK_UNLOCKED;
 #endif
 static xnpqueue_t pse51_infos_free_list;

Jan Kiszka

author

2008-03-02 11:02 UTC #2 lore raw

PATCH 1/6 Refactor xnlock debugging

In reply to a message not in the archive

---
 include/asm-generic/system.h |  296 ++++++++++++++++++++++---------------------
 1 file changed, 154 insertions(+), 142 deletions(-)

include/asm-generic/system.h +154 ~~-142~~

Index: b/include/asm-generic/system.h
===================================================================
@@ -47,6 +47,10 @@
 #define CONFIG_XENO_OPT_DEBUG_NUCLEUS 0
 #endif
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 /* Time base export */
 #define xnarch_declare_tbase(base)		do { } while(0)

@@ -81,7 +85,13 @@ typedef unsigned long spl_t;
 #define spltest()   rthal_local_irq_test()
 #define splget(x)   rthal_local_irq_flags(x)
 
-#if defined(CONFIG_SMP) && defined(CONFIG_XENO_OPT_DEBUG)
+static inline unsigned xnarch_current_cpu(void)
+{
+	return rthal_processor_id();
+}
+
+#if defined(CONFIG_SMP) && XENO_DEBUG(NUCLEUS)
+
 typedef struct {
 
         unsigned long long spin_time;

@@ -114,12 +124,96 @@ typedef struct {
 	0LL,					\
 }
 
-#else /* !(CONFIG_SMP && CONFIG_XENO_OPT_DEBUG) */
+#define XNLOCK_DBG_CONTEXT		, __FILE__, __LINE__, __FUNCTION__
+#define XNLOCK_DBG_CONTEXT_ARGS \
+	, const char *file, int line, const char *function
+#define XNLOCK_DBG_PASS_CONTEXT		, file, line, function
+#define XNLOCK_DBG_MAX_SPINS		10000000
+
+static inline void
+xnlock_dbg_prepare_acquire(unsigned long long *start, unsigned *spin_limit)
+{
+	*start = rthal_rdtsc();
+	*spin_limit = XNLOCK_DBG_MAX_SPINS;
+}
+
+static inline void
+xnlock_dbg_spinning(xnlock_t *lock, int cpu, unsigned int *spin_limit,
+		    const char *file, int line, const char *function)
+{
+	if (--*spin_limit == 0) {
+		rthal_emergency_console();
+		printk(KERN_ERR "Xenomai: stuck on nucleus lock %p\n"
+				"         waiter = %s:%u (%s(), CPU #%d)\n"
+				"         owner  = %s:%u (%s(), CPU #%d)\n",
+		       lock, file, line, function, cpu,
+		       lock->file, lock->line, lock->function, lock->cpu);
+		show_stack(NULL, NULL);
+	}
+}
+
+static inline void
+xnlock_dbg_acquired(xnlock_t *lock, int cpu, unsigned long long *start,
+		    const char *file, int line, const char *function)
+{
+	lock->lock_date = *start;
+	lock->spin_time = rthal_rdtsc() - *start;
+	lock->file = file;
+	lock->function = function;
+	lock->line = line;
+	lock->cpu = cpu;
+}
+
+static inline void xnlock_dbg_release(xnlock_t *lock)
+{
+	extern xnlockinfo_t xnlock_stats[];
+	unsigned long long lock_time = rthal_rdtsc() - lock->lock_date;
+	xnlockinfo_t *stats = &xnlock_stats[xnarch_current_cpu()];
+
+	lock->cpu = -lock->cpu;	/* File that we released it. */
+
+	if (lock_time > stats->lock_time) {
+		stats->lock_time = lock_time;
+		stats->spin_time = lock->spin_time;
+		stats->file = lock->file;
+		stats->function = lock->function;
+		stats->line = lock->line;
+	}
+}
+
+static inline void xnlock_dbg_invalid_release(xnlock_t *lock)
+{
+	rthal_emergency_console();
+	printk(KERN_ERR "Xenomai: unlocking unlocked nucleus lock %p\n"
+			"       owner  = %s:%u (%s(), CPU #%d)\n",
+	       lock, lock->file, lock->line, lock->function, lock->cpu);
+	show_stack(NULL,NULL);
+}
+
+#else /* !(CONFIG_SMP && XENO_DEBUG(NUCLEUS)) */
 
 typedef struct { atomic_t owner; } xnlock_t;
 
 #define XNARCH_LOCK_UNLOCKED (xnlock_t) { { ~0 } }
-#endif /* !(CONFIG_SMP && CONFIG_XENO_OPT_DEBUG) */
+
+#define XNLOCK_DBG_CONTEXT
+#define XNLOCK_DBG_CONTEXT_ARGS
+#define XNLOCK_DBG_PASS_CONTEXT
+
+static inline void
+xnlock_dbg_prepare_acquire(unsigned long long *start, unsigned *spin_limit)
+{ }
+
+static inline void
+xnlock_dbg_spinning(xnlock_t *lock, int cpu, unsigned int *spin_limit)	{ }
+
+static inline void
+xnlock_dbg_acquired(xnlock_t *lock, int cpu, unsigned long long *start)	{ }
+
+static inline void xnlock_dbg_release(xnlock_t *lock)			{ }
+static inline void xnlock_dbg_invalid_release(xnlock_t *lock)		{ }
+
+#endif /* !(CONFIG_SMP && XENO_DEBUG(NUCLEUS)) */
 
 #define XNARCH_NR_CPUS               RTHAL_NR_CPUS

@@ -167,10 +261,6 @@ typedef struct xnarch_heapcb {
 
 } xnarch_heapcb_t;
 
-#ifdef __cplusplus
-extern "C" {
-#endif
-
 unsigned long long xnarch_get_host_time(void);
 
 long long xnarch_tsc_to_ns(long long ts);

@@ -189,11 +279,6 @@ static inline unsigned long long xnarch_
     return RTHAL_CPU_FREQ;
 }
 
-static inline unsigned xnarch_current_cpu(void)
-{
-    return rthal_processor_id();
-}
-
 #define xnarch_halt(emsg) \
 do { \
     rthal_emergency_console(); \

@@ -213,17 +298,11 @@ static inline int xnarch_setimask (int i
 
 #ifdef CONFIG_SMP
 
-#if XENO_DEBUG(NUCLEUS)
-#define xnlock_get(lock) \
-    __xnlock_get(lock, __FILE__, __LINE__,__FUNCTION__)
+#define xnlock_get(lock)		__xnlock_get(lock  XNLOCK_DBG_CONTEXT)
 #define xnlock_get_irqsave(lock,x) \
-    ((x) = __xnlock_get_irqsave(lock, __FILE__, __LINE__,__FUNCTION__))
-#else /* !XENO_DEBUG(NUCLEUS) */
-#define xnlock_get(lock)            __xnlock_get(lock)
-#define xnlock_get_irqsave(lock,x)  ((x) = __xnlock_get_irqsave(lock))
-#endif /* !XENO_DEBUG(NUCLEUS) */
-#define xnlock_clear_irqoff(lock)   xnlock_put_irqrestore(lock,1)
-#define xnlock_clear_irqon(lock)    xnlock_put_irqrestore(lock,0)
+	((x) = __xnlock_get_irqsave(lock  XNLOCK_DBG_CONTEXT))
+#define xnlock_clear_irqoff(lock)	xnlock_put_irqrestore(lock, 1)
+#define xnlock_clear_irqon(lock)	xnlock_put_irqrestore(lock, 0)
 
 static inline void xnlock_init (xnlock_t *lock)
 {

@@ -235,132 +314,65 @@ static inline void xnlock_init (xnlock_t
 #define DEFINE_XNLOCK(lock)		xnlock_t lock = XNARCH_LOCK_UNLOCKED
 #define DEFINE_PRIVATE_XNLOCK(lock)	static DEFINE_XNLOCK(lock)
 
-#if XENO_DEBUG(NUCLEUS)
+static inline int __xnlock_get(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)
+{
+	unsigned long long start;
+	unsigned int spin_limit;
+	int cpu = xnarch_current_cpu();
+
+	if (atomic_read(&lock->owner) == cpu)
+		return 1;
+
+	xnlock_dbg_prepare_acquire(&start, &spin_limit);
+
+	while (atomic_cmpxchg(&lock->owner, ~0, cpu) != ~0)
+		do {
+			cpu_relax();
+			xnlock_dbg_spinning(lock, cpu, &spin_limit /*, */
+					    XNLOCK_DBG_PASS_CONTEXT);
+		} while(atomic_read(&lock->owner) != ~0);
+
+	xnlock_dbg_acquired(lock, cpu, &start /*, */ XNLOCK_DBG_PASS_CONTEXT);
+
+	return 0;
+}
+
+static inline void xnlock_put(xnlock_t *lock)
+{
+	if (likely(atomic_read(&lock->owner) == xnarch_current_cpu())) {
+		xnlock_dbg_release(lock);
+
+		/*
+		 * Make sure all data written inside the lock is visible to
+		 * other CPUs before we release the lock.
+		 */
+		xnarch_memory_barrier();
+
+		atomic_set(&lock->owner, ~0);
+	} else
+		xnlock_dbg_invalid_release(lock);
+}
+
+static inline spl_t
+__xnlock_get_irqsave(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)
+{
+	unsigned long flags;
+
+	rthal_local_irq_save(flags);
 
-#define XNARCH_DEBUG_SPIN_LIMIT 3000000
+	if (__xnlock_get(lock /*, */ XNLOCK_DBG_PASS_CONTEXT))
+		flags |= 2;	/* Recursive acquisition */
 
-static inline int __xnlock_get (xnlock_t *lock,
-				 const char *file,
-				 unsigned line,
-				 const char *function)
-{
-    unsigned spin_count = 0;
-#else /* !XENO_DEBUG(NUCLEUS) */
-static inline int __xnlock_get (xnlock_t *lock)
-{
-#endif /* !XENO_DEBUG(NUCLEUS) */
-    int recursing;
-
-    recursing = (atomic_read(&lock->owner) == rthal_processor_id());
-    if (!recursing) {
-#if XENO_DEBUG(NUCLEUS)
-	    unsigned long long lock_date = rthal_rdtsc();
-#endif /* XENO_DEBUG(NUCLEUS) */
-	    while(atomic_cmpxchg(&lock->owner, ~0, rthal_processor_id()) != ~0)
-		    do {
-			    cpu_relax();
-
-#if XENO_DEBUG(NUCLEUS)
-			    if (++spin_count == XNARCH_DEBUG_SPIN_LIMIT) {
-				    rthal_emergency_console();
-				    printk(KERN_ERR
-					   "Xenomai: stuck on nucleus lock %p\n"
-					   "       waiter = %s:%u (%s(), CPU #%d)\n"
-					   "       owner  = %s:%u (%s(), CPU #%d)\n",
-					   lock,file,line,function,rthal_processor_id(),
-					   lock->file,lock->line,lock->function,lock->cpu);
-				    show_stack(NULL,NULL);
-				    for (;;)
-					    cpu_relax();
-			    }
-#endif /* XENO_DEBUG(NUCLEUS) */
-		    } while(atomic_read(&lock->owner) != ~0);
-
-#if XENO_DEBUG(NUCLEUS)
-	    lock->spin_time = rthal_rdtsc() - lock_date;
-	    lock->lock_date = lock_date;
-	    lock->file = file;
-	    lock->function = function;
-	    lock->line = line;
-	    lock->cpu = rthal_processor_id();
-#endif /* XENO_DEBUG(NUCLEUS) */
-        }
-
-    return recursing;
-}
-
-static inline void xnlock_put (xnlock_t *lock)
-{
-	if (likely(atomic_read(&lock->owner) == rthal_processor_id())) {
-
-#if XENO_DEBUG(NUCLEUS)
-	    extern xnlockinfo_t xnlock_stats[];
-
-	    unsigned long long lock_time = rthal_rdtsc() - lock->lock_date;
-	    int cpu = rthal_processor_id();
-
-	    if (lock_time > xnlock_stats[cpu].lock_time) {
-		    xnlock_stats[cpu].lock_time = lock_time;
-		    xnlock_stats[cpu].spin_time = lock->spin_time;
-		    xnlock_stats[cpu].file = lock->file;
-		    xnlock_stats[cpu].function = lock->function;
-		    xnlock_stats[cpu].line = lock->line;
-	    }
-#endif /* XENO_DEBUG(NUCLEUS) */
-	    /*
-	     * Make sure all data written inside the lock is visible to
-	     * other CPUs before we release the lock.
-	     */
-	    xnarch_memory_barrier();
-
-	    atomic_set(&lock->owner, ~0);
-    }
-#if XENO_DEBUG(NUCLEUS)
-    else {
-	    rthal_emergency_console();
-	    printk(KERN_ERR
-		   "Xenomai: unlocking unlocked nucleus lock %p\n"
-		   "       owner  = %s:%u (%s(), CPU #%d)\n",
-		   lock,lock->file,lock->line,lock->function,lock->cpu);
-	    show_stack(NULL,NULL);
-	    for (;;)
-		    cpu_relax();
-    }
-#endif /* XENO_DEBUG(NUCLEUS) */
-}
-
-#if XENO_DEBUG(NUCLEUS)
-
-static inline spl_t __xnlock_get_irqsave (xnlock_t *lock,
-                                          const char *file,
-                                          unsigned line,
-                                          const char *function)
-{
-#else /* !XENO_DEBUG(NUCLEUS) */
-static inline spl_t __xnlock_get_irqsave (xnlock_t *lock)
-{
-#endif /* !XENO_DEBUG(NUCLEUS) */
-    unsigned long flags;
-
-    rthal_local_irq_save(flags);
-
-#if XENO_DEBUG(NUCLEUS)
-    if (__xnlock_get(lock, file, line, function))
-	    flags |= 2;
-#else /* !XENO_DEBUG(NUCLEUS) */
-    if (__xnlock_get(lock))
-	    flags |= 2;
-#endif /* !XENO_DEBUG(NUCLEUS) */
-	
-    return flags;
+	return flags;
 }
 
-static inline void xnlock_put_irqrestore (xnlock_t *lock, spl_t flags)
+static inline void xnlock_put_irqrestore(xnlock_t *lock, spl_t flags)
 {
-    if (!(flags & 2))
-	    xnlock_put(lock);
+	/* Only release the lock if we didn't take it recursively. */
+	if (!(flags & 2))
+		xnlock_put(lock);
 
-    rthal_local_irq_restore(flags & 1);
+	rthal_local_irq_restore(flags & 1);
 }
 
 static inline int xnarch_send_ipi (xnarch_cpumask_t cpumask)

Jan Kiszka

author

2008-03-02 11:02 UTC #4 lore raw

PATCH 2/6 Whitespace fixes for asm-generic/system.h

In reply to a message not in the archive

No functional changes.

---
 include/asm-generic/system.h |  145 +++++++++++++++++++++----------------------
 1 file changed, 73 insertions(+), 72 deletions(-)

include/asm-generic/system.h +73 ~~-72~~

Index: b/include/asm-generic/system.h
===================================================================
@@ -56,13 +56,13 @@ extern "C" {
 
 /* Tracer interface */
 #define xnarch_trace_max_begin(v)		rthal_trace_max_begin(v)
-#define xnarch_trace_max_end(v)		rthal_trace_max_end(v)
+#define xnarch_trace_max_end(v)			rthal_trace_max_end(v)
 #define xnarch_trace_max_reset()		rthal_trace_max_reset()
 #define xnarch_trace_user_start()		rthal_trace_user_start()
 #define xnarch_trace_user_stop(v)		rthal_trace_user_stop(v)
-#define xnarch_trace_user_freeze(v, once) 	rthal_trace_user_freeze(v, once)
+#define xnarch_trace_user_freeze(v, once)	rthal_trace_user_freeze(v, once)
 #define xnarch_trace_special(id, v)		rthal_trace_special(id, v)
-#define xnarch_trace_special_u64(id, v)	rthal_trace_special_u64(id, v)
+#define xnarch_trace_special_u64(id, v)		rthal_trace_special_u64(id, v)
 #define xnarch_trace_pid(pid, prio)		rthal_trace_pid(pid, prio)
 #define xnarch_trace_panic_freeze()		rthal_trace_panic_freeze()
 #define xnarch_trace_panic_dump()		rthal_trace_panic_dump()

@@ -94,23 +94,23 @@ static inline unsigned xnarch_current_cp
 
 typedef struct {
 
-        unsigned long long spin_time;
-        unsigned long long lock_time;
-        const char *file;
-        const char *function;
-        unsigned line;
+	unsigned long long spin_time;
+	unsigned long long lock_time;
+	const char *file;
+	const char *function;
+	unsigned line;
 
 } xnlockinfo_t;
 
 typedef struct {
 
-    atomic_t owner;
-    const char *file;
-    const char *function;
-    unsigned line;
-    int cpu;
-    unsigned long long spin_time;
-    unsigned long long lock_date;
+	atomic_t owner;
+	const char *file;
+	const char *function;
+	unsigned line;
+	int cpu;
+	unsigned long long spin_time;
+	unsigned long long lock_date;
 
 } xnlock_t;

@@ -194,7 +194,7 @@ static inline void xnlock_dbg_invalid_re
 
 typedef struct { atomic_t owner; } xnlock_t;
 
-#define XNARCH_LOCK_UNLOCKED (xnlock_t) { { ~0 } }
+#define XNARCH_LOCK_UNLOCKED		(xnlock_t) { { ~0 } }
 
 #define XNLOCK_DBG_CONTEXT
 #define XNLOCK_DBG_CONTEXT_ARGS

@@ -215,49 +215,48 @@ static inline void xnlock_dbg_invalid_re
 
 #endif /* !(CONFIG_SMP && XENO_DEBUG(NUCLEUS)) */
 
-#define XNARCH_NR_CPUS               RTHAL_NR_CPUS
+#define XNARCH_NR_CPUS			RTHAL_NR_CPUS
 
-#define XNARCH_NR_IRQS               RTHAL_NR_IRQS
-#define XNARCH_TIMER_IRQ	     RTHAL_TIMER_IRQ
-#define XNARCH_TIMER_DEVICE          RTHAL_TIMER_DEVICE
-#define XNARCH_CLOCK_DEVICE          RTHAL_CLOCK_DEVICE
+#define XNARCH_NR_IRQS			RTHAL_NR_IRQS
+#define XNARCH_TIMER_IRQ		RTHAL_TIMER_IRQ
+#define XNARCH_TIMER_DEVICE		RTHAL_TIMER_DEVICE
+#define XNARCH_CLOCK_DEVICE		RTHAL_CLOCK_DEVICE
 
-#define XNARCH_ROOT_STACKSZ   0	/* Only a placeholder -- no stack */
+#define XNARCH_ROOT_STACKSZ		0 /* Only a placeholder -- no stack */
 
 #define XNARCH_PROMPT "Xenomai: "
-#define xnarch_loginfo(fmt,args...)  printk(KERN_INFO XNARCH_PROMPT fmt , ##args)
-#define xnarch_logwarn(fmt,args...)  printk(KERN_WARNING XNARCH_PROMPT fmt , ##args)
-#define xnarch_logerr(fmt,args...)   printk(KERN_ERR XNARCH_PROMPT fmt , ##args)
-#define xnarch_printf(fmt,args...)   printk(KERN_INFO XNARCH_PROMPT fmt , ##args)
+#define xnarch_loginfo(fmt, args...)	printk(KERN_INFO XNARCH_PROMPT fmt, ##args)
+#define xnarch_logwarn(fmt, args...)	printk(KERN_WARNING XNARCH_PROMPT fmt, ##args)
+#define xnarch_logerr(fmt, args...)	printk(KERN_ERR XNARCH_PROMPT fmt, ##args)
+#define xnarch_printf(fmt, args...)	printk(KERN_INFO XNARCH_PROMPT fmt, ##args)
 
 typedef cpumask_t xnarch_cpumask_t;
 
 #ifdef CONFIG_SMP
-#define xnarch_cpu_online_map            cpu_online_map
+#define xnarch_cpu_online_map			cpu_online_map
 #else
-#define xnarch_cpu_online_map		 cpumask_of_cpu(0)
+#define xnarch_cpu_online_map			cpumask_of_cpu(0)
 #endif
-#define xnarch_num_online_cpus()          num_online_cpus()
-#define xnarch_cpu_set(cpu, mask)         cpu_set(cpu, (mask))
-#define xnarch_cpu_clear(cpu, mask)       cpu_clear(cpu, (mask))
-#define xnarch_cpus_clear(mask)           cpus_clear(mask)
-#define xnarch_cpu_isset(cpu, mask)       cpu_isset(cpu, (mask))
-#define xnarch_cpus_and(dst, src1, src2)  cpus_and((dst), (src1), (src2))
-#define xnarch_cpus_equal(mask1, mask2)   cpus_equal((mask1), (mask2))
-#define xnarch_cpus_empty(mask)           cpus_empty(mask)
-#define xnarch_cpumask_of_cpu(cpu)        cpumask_of_cpu(cpu)
-#define xnarch_cpu_test_and_set(cpu,mask) cpu_test_and_set(cpu, (mask))
-
-#define xnarch_first_cpu(mask)            first_cpu(mask)
-#define XNARCH_CPU_MASK_ALL               CPU_MASK_ALL
+#define xnarch_num_online_cpus()		num_online_cpus()
+#define xnarch_cpu_set(cpu, mask)		cpu_set(cpu, (mask))
+#define xnarch_cpu_clear(cpu, mask)		cpu_clear(cpu, (mask))
+#define xnarch_cpus_clear(mask)			cpus_clear(mask)
+#define xnarch_cpu_isset(cpu, mask)		cpu_isset(cpu, (mask))
+#define xnarch_cpus_and(dst, src1, src2)	cpus_and((dst), (src1), (src2))
+#define xnarch_cpus_equal(mask1, mask2)		cpus_equal((mask1), (mask2))
+#define xnarch_cpus_empty(mask)			cpus_empty(mask)
+#define xnarch_cpumask_of_cpu(cpu)		cpumask_of_cpu(cpu)
+#define xnarch_cpu_test_and_set(cpu, mask)	cpu_test_and_set(cpu, (mask))
+#define xnarch_first_cpu(mask)			first_cpu(mask)
+#define XNARCH_CPU_MASK_ALL			CPU_MASK_ALL
 
 typedef struct xnarch_heapcb {
 
-    atomic_t numaps;	/* # of active user-space mappings. */
+	atomic_t numaps;	/* # of active user-space mappings. */
 
-    int kmflags;	/* Kernel memory flags (0 if vmalloc()). */
+	int kmflags;		/* Kernel memory flags (0 if vmalloc()). */
 
-    void *heapbase;	/* Shared heap memory base. */
+	void *heapbase;		/* Shared heap memory base. */
 
 } xnarch_heapcb_t;

@@ -267,7 +266,7 @@ long long xnarch_tsc_to_ns(long long ts)
 
 static inline long long xnarch_tsc_to_ns_rounded(long long ts)
 {
-    return (xnarch_llimd(ts, 1000000000, RTHAL_CPU_FREQ/2) + 1) / 2;
+	return (xnarch_llimd(ts, 1000000000, RTHAL_CPU_FREQ/2) + 1) / 2;
 }
 
 long long xnarch_ns_to_tsc(long long ns);

@@ -276,24 +275,26 @@ unsigned long long xnarch_get_cpu_time(v
 
 static inline unsigned long long xnarch_get_cpu_freq(void)
 {
-    return RTHAL_CPU_FREQ;
+	return RTHAL_CPU_FREQ;
 }
 
-#define xnarch_halt(emsg) \
-do { \
-    rthal_emergency_console(); \
-    xnarch_logerr("fatal: %s\n",emsg); \
-    show_stack(NULL,NULL);			\
-    xnarch_trace_panic_dump();			\
-    for (;;) cpu_relax();			\
-} while(0)
+#define xnarch_halt(emsg)				\
+	do {						\
+		rthal_emergency_console();		\
+		xnarch_logerr("fatal: %s\n", emsg);	\
+		show_stack(NULL,NULL);			\
+		xnarch_trace_panic_dump();		\
+		for (;;)				\
+			cpu_relax();			\
+	} while(0)
 
 static inline int xnarch_setimask (int imask)
 {
-    spl_t s;
-    splhigh(s);
-    splexit(!!imask);
-    return !!s;
+	spl_t s;
+
+	splhigh(s);
+	splexit(!!imask);
+	return !!s;
 }
 
 #ifdef CONFIG_SMP

@@ -306,7 +307,7 @@ static inline int xnarch_setimask (int i
 
 static inline void xnlock_init (xnlock_t *lock)
 {
-    *lock = XNARCH_LOCK_UNLOCKED;
+	*lock = XNARCH_LOCK_UNLOCKED;
 }
 
 #define DECLARE_XNLOCK(lock)		xnlock_t lock

@@ -375,9 +376,9 @@ static inline void xnlock_put_irqrestore
 	rthal_local_irq_restore(flags & 1);
 }
 
-static inline int xnarch_send_ipi (xnarch_cpumask_t cpumask)
+static inline int xnarch_send_ipi(xnarch_cpumask_t cpumask)
 {
-    return rthal_send_ipi(RTHAL_SERVICE_IPI0, cpumask);
+	return rthal_send_ipi(RTHAL_SERVICE_IPI0, cpumask);
 }
 
 static inline int xnlock_is_owner(xnlock_t *lock)

@@ -387,14 +388,14 @@ static inline int xnlock_is_owner(xnlock
 
 #else /* !CONFIG_SMP */
 
-#define xnlock_init(lock)              do { } while(0)
-#define xnlock_get(lock)               do { } while(0)
-#define xnlock_put(lock)               do { } while(0)
-#define xnlock_get_irqsave(lock,x)     rthal_local_irq_save(x)
-#define xnlock_put_irqrestore(lock,x)  rthal_local_irq_restore(x)
-#define xnlock_clear_irqoff(lock)      rthal_local_irq_disable()
-#define xnlock_clear_irqon(lock)       rthal_local_irq_enable()
-#define xnlock_is_owner(lock)	       1
+#define xnlock_init(lock)		do { } while(0)
+#define xnlock_get(lock)		do { } while(0)
+#define xnlock_put(lock)		do { } while(0)
+#define xnlock_get_irqsave(lock,x)	rthal_local_irq_save(x)
+#define xnlock_put_irqrestore(lock,x)	rthal_local_irq_restore(x)
+#define xnlock_clear_irqoff(lock)	rthal_local_irq_disable()
+#define xnlock_clear_irqon(lock)	rthal_local_irq_enable()
+#define xnlock_is_owner(lock)		1
 
 #define DECLARE_XNLOCK(lock)
 #define DECLARE_EXTERN_XNLOCK(lock)

@@ -403,7 +404,7 @@ static inline int xnlock_is_owner(xnlock
 
 static inline int xnarch_send_ipi (xnarch_cpumask_t cpumask)
 {
-    return 0;
+	return 0;
 }
 
 #endif /* !CONFIG_SMP */

@@ -418,7 +419,7 @@ static inline int xnarch_remap_vm_page(s
 				       unsigned long from,
 				       unsigned long to)
 {
-    return wrap_remap_vm_page(vma,from,to);
+	return wrap_remap_vm_page(vma, from, to);
 }
 
 static inline int xnarch_remap_io_page_range(struct vm_area_struct *vma,

@@ -427,7 +428,7 @@ static inline int xnarch_remap_io_page_r
 					     unsigned long size,
 					     pgprot_t prot)
 {
-    return wrap_remap_io_page_range(vma,from,to,size,prot);
+	return wrap_remap_io_page_range(vma, from, to, size, prot);
 }
 
 #ifndef xnarch_hisyscall_entry

Jan Kiszka

author

2008-03-02 11:02 UTC #6 lore raw

PATCH 3/6 Consolidate xnlock_put debugging

In reply to a message not in the archive

This patch therefore moves the check under the same switch (XENO_OPT_DEBUG_NUCLEUS) as the rest of xnlock debugging already is.

---
 include/asm-generic/system.h |   52 +++++++++++++++++++++++--------------------
 1 file changed, 28 insertions(+), 24 deletions(-)

include/asm-generic/system.h +28 ~~-24~~

Index: b/include/asm-generic/system.h
===================================================================
@@ -164,11 +164,23 @@ xnlock_dbg_acquired(xnlock_t *lock, int 
 	lock->cpu = cpu;
 }
 
-static inline void xnlock_dbg_release(xnlock_t *lock)
+static inline int xnlock_dbg_release(xnlock_t *lock)
 {
 	extern xnlockinfo_t xnlock_stats[];
 	unsigned long long lock_time = rthal_rdtsc() - lock->lock_date;
-	xnlockinfo_t *stats = &xnlock_stats[xnarch_current_cpu()];
+	int cpu = xnarch_current_cpu();
+	xnlockinfo_t *stats = &xnlock_stats[cpu];
+
+	if (unlikely(atomic_read(&lock->owner) != cpu)) {
+		rthal_emergency_console();
+		printk(KERN_ERR "Xenomai: unlocking unlocked nucleus lock %p"
+				" on CPU #%d\n"
+				"         owner  = %s:%u (%s(), CPU #%d)\n",
+		       lock, cpu, lock->file, lock->line, lock->function,
+		       lock->cpu);
+		show_stack(NULL,NULL);
+		return 1;
+	}
 
 	lock->cpu = -lock->cpu;	/* File that we released it. */

@@ -179,15 +191,7 @@ static inline void xnlock_dbg_release(xn
 		stats->function = lock->function;
 		stats->line = lock->line;
 	}
-}
-
-static inline void xnlock_dbg_invalid_release(xnlock_t *lock)
-{
-	rthal_emergency_console();
-	printk(KERN_ERR "Xenomai: unlocking unlocked nucleus lock %p\n"
-			"       owner  = %s:%u (%s(), CPU #%d)\n",
-	       lock, lock->file, lock->line, lock->function, lock->cpu);
-	show_stack(NULL,NULL);
+	return 0;
 }
 
 #else /* !(CONFIG_SMP && XENO_DEBUG(NUCLEUS)) */

@@ -210,8 +214,10 @@ xnlock_dbg_spinning(xnlock_t *lock, int 
 static inline void
 xnlock_dbg_acquired(xnlock_t *lock, int cpu, unsigned long long *start)	{ }
 
-static inline void xnlock_dbg_release(xnlock_t *lock)			{ }
-static inline void xnlock_dbg_invalid_release(xnlock_t *lock)		{ }
+static inline int xnlock_dbg_release(xnlock_t *lock)
+{
+	return 0;
+}
 
 #endif /* !(CONFIG_SMP && XENO_DEBUG(NUCLEUS)) */

@@ -340,18 +346,16 @@ static inline int __xnlock_get(xnlock_t 
 
 static inline void xnlock_put(xnlock_t *lock)
 {
-	if (likely(atomic_read(&lock->owner) == xnarch_current_cpu())) {
-		xnlock_dbg_release(lock);
+	if (xnlock_dbg_release(lock))
+		return;
+
+	/*
+	 * Make sure all data written inside the lock is visible to
+	 * other CPUs before we release the lock.
+	 */
+	xnarch_memory_barrier();
 
-		/*
-		 * Make sure all data written inside the lock is visible to
-		 * other CPUs before we release the lock.
-		 */
-		xnarch_memory_barrier();
-
-		atomic_set(&lock->owner, ~0);
-	} else
-		xnlock_dbg_invalid_release(lock);
+	atomic_set(&lock->owner, ~0);
 }
 
 static inline spl_t

Jan Kiszka

author

2008-03-02 11:02 UTC #8 lore raw

PATCH 4/6 Move spinning code out-of-line

In reply to a message not in the archive

To save text thus and thus avoid cache misses in the worst-case path, this patch moves the slow path of xnlock_get out-of-line.

---
 include/asm-generic/bits/pod.h |   18 ++++++++++++++++++
 include/asm-generic/system.h   |   29 ++++++++++++++---------------
 2 files changed, 32 insertions(+), 15 deletions(-)

include/asm-generic/bits/pod.h +18 -0

Index: b/include/asm-generic/bits/pod.h
===================================================================
@@ -295,4 +295,22 @@ unsigned long long xnarch_get_cpu_time(v
 
 EXPORT_SYMBOL(xnarch_get_cpu_time);
 
+#ifdef CONFIG_SMP
+void __xnlock_spin(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)
+{
+	unsigned int spin_limit;
+	int cpu = xnarch_current_cpu();
+
+	xnlock_dbg_prepare_spin(&spin_limit);
+
+	while (atomic_cmpxchg(&lock->owner, ~0, cpu) != ~0)
+		do {
+			cpu_relax();
+			xnlock_dbg_spinning(lock, cpu, &spin_limit /*, */
+					    XNLOCK_DBG_PASS_CONTEXT);
+		} while(atomic_read(&lock->owner) != ~0);
+}
+EXPORT_SYMBOL(__xnlock_spin);
+#endif /* CONFIG_SMP */
+
 #endif /* !_XENO_ASM_GENERIC_BITS_POD_H */

include/asm-generic/system.h +14 ~~-15~~

Index: b/include/asm-generic/system.h
===================================================================
@@ -130,10 +130,13 @@ typedef struct {
 #define XNLOCK_DBG_PASS_CONTEXT		, file, line, function
 #define XNLOCK_DBG_MAX_SPINS		10000000
 
-static inline void
-xnlock_dbg_prepare_acquire(unsigned long long *start, unsigned *spin_limit)
+static inline void xnlock_dbg_prepare_acquire(unsigned long long *start)
 {
 	*start = rthal_rdtsc();
+}
+
+static inline void xnlock_dbg_prepare_spin(unsigned *spin_limit)
+{
 	*spin_limit = XNLOCK_DBG_MAX_SPINS;
 }

@@ -204,15 +207,14 @@ typedef struct { atomic_t owner; } xnloc
 #define XNLOCK_DBG_CONTEXT_ARGS
 #define XNLOCK_DBG_PASS_CONTEXT
 
-static inline void
-xnlock_dbg_prepare_acquire(unsigned long long *start, unsigned *spin_limit)
-{ }
+static inline void xnlock_dbg_prepare_acquire(unsigned long long *start) { }
+static inline void xnlock_dbg_prepare_spin(unsigned *spin_limit)	 { }
 
 static inline void
-xnlock_dbg_spinning(xnlock_t *lock, int cpu, unsigned int *spin_limit)	{ }
+xnlock_dbg_spinning(xnlock_t *lock, int cpu, unsigned int *spin_limit)	 { }
 
 static inline void
-xnlock_dbg_acquired(xnlock_t *lock, int cpu, unsigned long long *start)	{ }
+xnlock_dbg_acquired(xnlock_t *lock, int cpu, unsigned long long *start)	 { }
 
 static inline int xnlock_dbg_release(xnlock_t *lock)
 {

@@ -321,23 +323,20 @@ static inline void xnlock_init (xnlock_t
 #define DEFINE_XNLOCK(lock)		xnlock_t lock = XNARCH_LOCK_UNLOCKED
 #define DEFINE_PRIVATE_XNLOCK(lock)	static DEFINE_XNLOCK(lock)
 
+void __xnlock_spin(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS);
+
 static inline int __xnlock_get(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)
 {
 	unsigned long long start;
-	unsigned int spin_limit;
 	int cpu = xnarch_current_cpu();
 
 	if (atomic_read(&lock->owner) == cpu)
 		return 1;
 
-	xnlock_dbg_prepare_acquire(&start, &spin_limit);
+	xnlock_dbg_prepare_acquire(&start);
 
-	while (atomic_cmpxchg(&lock->owner, ~0, cpu) != ~0)
-		do {
-			cpu_relax();
-			xnlock_dbg_spinning(lock, cpu, &spin_limit /*, */
-					    XNLOCK_DBG_PASS_CONTEXT);
-		} while(atomic_read(&lock->owner) != ~0);
+	if (unlikely(atomic_cmpxchg(&lock->owner, ~0, cpu) != ~0))
+		__xnlock_spin(lock /*, */ XNLOCK_DBG_PASS_CONTEXT);
 
 	xnlock_dbg_acquired(lock, cpu, &start /*, */ XNLOCK_DBG_PASS_CONTEXT);

Jan Kiszka

author

2008-03-02 11:02 UTC #10 lore raw

PATCH 5/6 Introduce optional recursive ticket xnlock

In reply to a message not in the archive

Meanwhile, ticket spinlocks are mainline (2.6.25). But that version has to drawbacks for us: it doesn't support nesting like xnlock does, and it is x86-only so far.

Survived heavy testing on a 4-way box (15, 20, 25, and 30 us latency instances across the cores).

[1] http://lwn.net/Articles/267331

---
 include/asm-generic/bits/pod.h |    4 -
 include/asm-generic/system.h   |  122 +++++++++++++++++++++++++++++++++++++----
 ksrc/nucleus/Kconfig           |   12 ++++
 3 files changed, 126 insertions(+), 12 deletions(-)

include/asm-generic/system.h +112 ~~-10~~

Index: b/include/asm-generic/system.h
===================================================================
@@ -104,7 +104,12 @@ typedef struct {
 
 typedef struct {
 
+#ifdef CONFIG_XENO_OPT_TICKET_LOCK
+	atomic_t fifo;
+	int owner;
+#else
 	atomic_t owner;
+#endif
 	const char *file;
 	const char *function;
 	unsigned line;

@@ -115,7 +120,7 @@ typedef struct {
 } xnlock_t;
 
 #define XNARCH_LOCK_UNLOCKED (xnlock_t) {	\
-	{ ~0 },					\
+	XNLOCK_INIT_VALUE,			\
 	NULL,					\
 	NULL,					\
 	0,					\

@@ -167,6 +172,8 @@ xnlock_dbg_acquired(xnlock_t *lock, int 
 	lock->cpu = cpu;
 }
 
+static inline int xnlock_owner(xnlock_t *lock);
+
 static inline int xnlock_dbg_release(xnlock_t *lock)
 {
 	extern xnlockinfo_t xnlock_stats[];

@@ -174,7 +181,7 @@ static inline int xnlock_dbg_release(xnl
 	int cpu = xnarch_current_cpu();
 	xnlockinfo_t *stats = &xnlock_stats[cpu];
 
-	if (unlikely(atomic_read(&lock->owner) != cpu)) {
+	if (unlikely(xnlock_owner(lock) != cpu)) {
 		rthal_emergency_console();
 		printk(KERN_ERR "Xenomai: unlocking unlocked nucleus lock %p"
 				" on CPU #%d\n"

@@ -199,9 +206,18 @@ static inline int xnlock_dbg_release(xnl
 
 #else /* !(CONFIG_SMP && XENO_DEBUG(NUCLEUS)) */
 
+#ifdef CONFIG_XENO_OPT_TICKET_LOCK
+typedef struct {
+
+	atomic_t fifo;
+	int owner;
+
+} xnlock_t;
+#else
 typedef struct { atomic_t owner; } xnlock_t;
+#endif
 
-#define XNARCH_LOCK_UNLOCKED		(xnlock_t) { { ~0 } }
+#define XNARCH_LOCK_UNLOCKED		(xnlock_t) { XNLOCK_INIT_VALUE }
 
 #define XNLOCK_DBG_CONTEXT
 #define XNLOCK_DBG_CONTEXT_ARGS

@@ -313,11 +329,6 @@ static inline int xnarch_setimask (int i
 #define xnlock_clear_irqoff(lock)	xnlock_put_irqrestore(lock, 1)
 #define xnlock_clear_irqon(lock)	xnlock_put_irqrestore(lock, 0)
 
-static inline void xnlock_init (xnlock_t *lock)
-{
-	*lock = XNARCH_LOCK_UNLOCKED;
-}
-
 #define DECLARE_XNLOCK(lock)		xnlock_t lock
 #define DECLARE_EXTERN_XNLOCK(lock)	extern xnlock_t lock
 #define DEFINE_XNLOCK(lock)		xnlock_t lock = XNARCH_LOCK_UNLOCKED

@@ -325,12 +336,97 @@ static inline void xnlock_init (xnlock_t
 
 void __xnlock_spin(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS);
 
+#ifdef CONFIG_XENO_OPT_TICKET_LOCK
+/* Note: We assume atomic_t is at least 32-bit wide. */
+#define XNLOCK_HEAD_MASK		0x000000FF
+#define XNLOCK_HEAD_INC			0x00000001
+#define XNLOCK_TAIL_MASK		0xFF000000
+#define XNLOCK_TAIL_SHIFT		24
+#define XNLOCK_TAIL_INC			0x01000000
+#define XNLOCK_TICKET_MASK		(XNLOCK_HEAD_MASK | XNLOCK_TAIL_MASK)
+#define XNLOCK_INIT_VALUE		{ 0x00000001 }, ~0
+
+static inline int xnlock_owner(xnlock_t *lock)
+{
+	return lock->owner;
+}
+
+static inline int __xnlock_get(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)
+{
+	unsigned long long start;
+	int cpu = xnarch_current_cpu();
+	u32 fifo;
+	u8 ticket;
+
+	if (xnlock_owner(lock) == cpu)
+		return 1;
+
+	xnlock_dbg_prepare_acquire(&start);
+
+	fifo = atomic_add_return(XNLOCK_TAIL_INC, &lock->fifo);
+	ticket = fifo >> XNLOCK_TAIL_SHIFT;
+
+	/* First check if we are already the owner (head == tail). */
+	if (unlikely((u8)fifo != ticket)) {
+	 	unsigned int spin_limit;
+
+		xnlock_dbg_prepare_spin(&spin_limit);
+
+		/*
+		 * Spin until head equals our ticket. In this case,
+		 * moving out-of-line does not pay off.
+		 */
+		do {
+			cpu_relax();
+			xnlock_dbg_spinning(lock, cpu, &spin_limit /*, */
+					    XNLOCK_DBG_PASS_CONTEXT);
+		} while ((u8)atomic_read(&lock->fifo) != ticket);
+	}
+	lock->owner = cpu;
+
+	xnlock_dbg_acquired(lock, cpu, &start /*, */ XNLOCK_DBG_PASS_CONTEXT);
+
+	return 0;
+}
+
+static inline void xnlock_put(xnlock_t *lock)
+{
+	u32 fifo = atomic_read(&lock->fifo);
+
+	if (xnlock_dbg_release(lock))
+		return;
+
+	lock->owner = ~0;
+
+	/*
+	 * Make sure all data written inside the lock is visible to
+	 * other CPUs before we release the lock.
+	 */
+	xnarch_memory_barrier();
+
+	/*
+	 * Carefully calculate the addend so that we confine the
+	 * XNLOCK_HEAD_INC just to the first byte of fifo.
+	 */
+	atomic_add(((fifo + XNLOCK_HEAD_INC) & XNLOCK_TICKET_MASK) - fifo,
+		   &lock->fifo);
+}
+
+#else /* ! CONFIG_XENO_OPT_TICKET_LOCK */
+
+#define XNLOCK_INIT_VALUE		{ ~0 }
+
+static inline int xnlock_owner(xnlock_t *lock)
+{
+	return atomic_read(&lock->owner);
+}
+
 static inline int __xnlock_get(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)
 {
 	unsigned long long start;
 	int cpu = xnarch_current_cpu();
 
-	if (atomic_read(&lock->owner) == cpu)
+	if (xnlock_owner(lock) == cpu)
 		return 1;
 
 	xnlock_dbg_prepare_acquire(&start);

@@ -356,6 +452,12 @@ static inline void xnlock_put(xnlock_t *
 
 	atomic_set(&lock->owner, ~0);
 }
+#endif /* ! CONFIG_XENO_OPT_TICKET_LOCK */
+
+static inline void xnlock_init(xnlock_t *lock)
+{
+	*lock = XNARCH_LOCK_UNLOCKED;
+}
 
 static inline spl_t
 __xnlock_get_irqsave(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)

@@ -386,7 +488,7 @@ static inline int xnarch_send_ipi(xnarch
 
 static inline int xnlock_is_owner(xnlock_t *lock)
 {
-	return atomic_read(&lock->owner) == xnarch_current_cpu();
+	return xnlock_owner(lock) == xnarch_current_cpu();
 }
 
 #else /* !CONFIG_SMP */

ksrc/nucleus/Kconfig +12 -0

Index: b/ksrc/nucleus/Kconfig
===================================================================
@@ -340,6 +340,18 @@ config XENO_OPT_TIMER_WHEEL_STEP
 	Set the duration in ns of a timer wheel step. At each step, 
 	the timer wheel use the next hash bucket.
 
+config XENO_OPT_TICKET_LOCK
+	bool "FIFO ticket spinlocks"
+	depends on SMP
+	help
+
+	Uses a ticket spinlock algorithm which enforces strict FIFO
+	admission order on contended locks, improving the determinism
+	of lock acquisitions on large SMP systems. The ticket spinlock
+	has a higher overhead than the unordered algorithm and should
+	therefore only be used if there are more than two CPUs, thus
+	more than one waiter in the worst case.
+
 endmenu
 
 endif

include/asm-generic/bits/pod.h +2 -2

Index: b/include/asm-generic/bits/pod.h
===================================================================
@@ -295,7 +295,7 @@ unsigned long long xnarch_get_cpu_time(v
 
 EXPORT_SYMBOL(xnarch_get_cpu_time);
 
-#ifdef CONFIG_SMP
+#if defined(CONFIG_SMP) && !defined(CONFIG_XENO_OPT_TICKET_LOCK)
 void __xnlock_spin(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)
 {
 	unsigned int spin_limit;

@@ -311,6 +311,6 @@ void __xnlock_spin(xnlock_t *lock /*, */
 		} while(atomic_read(&lock->owner) != ~0);
 }
 EXPORT_SYMBOL(__xnlock_spin);
-#endif /* CONFIG_SMP */
+#endif /* CONFIG_SMP && !CONFIG_XENO_OPT_TICKET_LOCK */
 
 #endif /* !_XENO_ASM_GENERIC_BITS_POD_H */

Jan Kiszka

author

2008-03-02 11:02 UTC #12 lore raw

PATCH 6/6 Allow xnlock debugging on single-CPU systems

In reply to a message not in the archive

After all the refactoring, arming xnlock_dbg services also for !CONFIG_SMP setups was straightforward.

---
 include/asm-generic/bits/pod.h |    4 ++--
 include/asm-generic/system.h   |   31 +++++++++++++++----------------
 ksrc/nucleus/intr.c            |    4 ++--
 ksrc/nucleus/module.c          |   12 ++++++------
 ksrc/nucleus/pod.c             |    4 ++--
 ksrc/skins/posix/signal.c      |    2 +-
 6 files changed, 28 insertions(+), 29 deletions(-)

include/asm-generic/bits/pod.h +2 -2

Index: b/include/asm-generic/bits/pod.h
===================================================================
@@ -295,7 +295,7 @@ unsigned long long xnarch_get_cpu_time(v
 
 EXPORT_SYMBOL(xnarch_get_cpu_time);
 
-#if defined(CONFIG_SMP) && !defined(CONFIG_XENO_OPT_TICKET_LOCK)
+#if (defined(CONFIG_SMP) || XENO_DEBUG(NUCLEUS)) && !defined(CONFIG_XENO_OPT_TICKET_LOCK)
 void __xnlock_spin(xnlock_t *lock /*, */ XNLOCK_DBG_CONTEXT_ARGS)
 {
 	unsigned int spin_limit;

@@ -311,6 +311,6 @@ void __xnlock_spin(xnlock_t *lock /*, */
 		} while(atomic_read(&lock->owner) != ~0);
 }
 EXPORT_SYMBOL(__xnlock_spin);
-#endif /* CONFIG_SMP && !CONFIG_XENO_OPT_TICKET_LOCK */
+#endif /* (CONFIG_SMP || XENO_DEBUG(NUCLEUS)) && !CONFIG_XENO_OPT_TICKET_LOCK */
 
 #endif /* !_XENO_ASM_GENERIC_BITS_POD_H */

include/asm-generic/system.h +15 ~~-16~~

Index: b/include/asm-generic/system.h
===================================================================
@@ -76,11 +76,11 @@ extern "C" {
 typedef unsigned long spl_t;
 
 #define splhigh(x)  rthal_local_irq_save(x)
-#ifdef CONFIG_SMP
+#if defined(CONFIG_SMP) || XENO_DEBUG(NUCLEUS)
 #define splexit(x)  rthal_local_irq_restore((x) & 1)
-#else /* !CONFIG_SMP */
+#else /* !CONFIG_SMP || XENO_DEBUG(NUCLEUS) */
 #define splexit(x)  rthal_local_irq_restore(x)
-#endif /* !CONFIG_SMP */
+#endif /* !CONFIG_SMP || XENO_DEBUG(NUCLEUS) */
 #define splnone()   rthal_local_irq_enable()
 #define spltest()   rthal_local_irq_test()
 #define splget(x)   rthal_local_irq_flags(x)

@@ -90,7 +90,7 @@ static inline unsigned xnarch_current_cp
 	return rthal_processor_id();
 }
 
-#if defined(CONFIG_SMP) && XENO_DEBUG(NUCLEUS)
+#if XENO_DEBUG(NUCLEUS)
 
 typedef struct {

@@ -204,7 +204,7 @@ static inline int xnlock_dbg_release(xnl
 	return 0;
 }
 
-#else /* !(CONFIG_SMP && XENO_DEBUG(NUCLEUS)) */
+#else /* !XENO_DEBUG(NUCLEUS) */
 
 #ifdef CONFIG_XENO_OPT_TICKET_LOCK
 typedef struct {

@@ -237,7 +237,7 @@ static inline int xnlock_dbg_release(xnl
 	return 0;
 }
 
-#endif /* !(CONFIG_SMP && XENO_DEBUG(NUCLEUS)) */
+#endif /* !XENO_DEBUG(NUCLEUS) */
 
 #define XNARCH_NR_CPUS			RTHAL_NR_CPUS

@@ -321,7 +321,7 @@ static inline int xnarch_setimask (int i
 	return !!s;
 }
 
-#ifdef CONFIG_SMP
+#if defined(CONFIG_SMP) || XENO_DEBUG(NUCLEUS)
 
 #define xnlock_get(lock)		__xnlock_get(lock  XNLOCK_DBG_CONTEXT)
 #define xnlock_get_irqsave(lock,x) \

@@ -481,17 +481,12 @@ static inline void xnlock_put_irqrestore
 	rthal_local_irq_restore(flags & 1);
 }
 
-static inline int xnarch_send_ipi(xnarch_cpumask_t cpumask)
-{
-	return rthal_send_ipi(RTHAL_SERVICE_IPI0, cpumask);
-}
-
 static inline int xnlock_is_owner(xnlock_t *lock)
 {
 	return xnlock_owner(lock) == xnarch_current_cpu();
 }
 
-#else /* !CONFIG_SMP */
+#else /* !(CONFIG_SMP || XENO_DEBUG(NUCLEUS)) */
 
 #define xnlock_init(lock)		do { } while(0)
 #define xnlock_get(lock)		do { } while(0)

@@ -507,12 +502,16 @@ static inline int xnlock_is_owner(xnlock
 #define DEFINE_XNLOCK(lock)
 #define DEFINE_PRIVATE_XNLOCK(lock)
 
-static inline int xnarch_send_ipi (xnarch_cpumask_t cpumask)
+#endif /* !(CONFIG_SMP || XENO_DEBUG(NUCLEUS)) */
+
+static inline int xnarch_send_ipi(xnarch_cpumask_t cpumask)
 {
+#ifdef CONFIG_SMP
+	return rthal_send_ipi(RTHAL_SERVICE_IPI0, cpumask);
+#else /* !CONFIG_SMP */
 	return 0;
-}
-
 #endif /* !CONFIG_SMP */
+}
 
 #define xnlock_sync_irq(lock, x)			\
 	do {						\

ksrc/nucleus/intr.c +2 -2

Index: b/ksrc/nucleus/intr.c
===================================================================
@@ -387,7 +387,7 @@ static inline int xnintr_irq_detach(xnin
 
 #else /* !CONFIG_XENO_OPT_SHIRQ */
 
-#ifdef CONFIG_SMP
+#if defined(CONFIG_SMP) || XENO_DEBUG(NUCLEUS)
 typedef struct xnintr_irq {
 
 	DECLARE_XNLOCK(lock);

@@ -395,7 +395,7 @@ typedef struct xnintr_irq {
 } ____cacheline_aligned_in_smp xnintr_irq_t;
 
 static xnintr_irq_t xnirqs[XNARCH_NR_IRQS];
-#endif /* CONFIG_SMP */
+#endif /* CONFIG_SMP || XENO_DEBUG(NUCLEUS) */
 
 static inline xnintr_t *xnintr_shirq_first(unsigned irq)
 {

ksrc/nucleus/module.c +6 -6

Index: b/ksrc/nucleus/module.c
===================================================================
@@ -664,7 +664,7 @@ void xnpod_discard_tbase_proc(xntbase_t 
 
 #endif /* CONFIG_XENO_OPT_STATS */
 
-#if defined(CONFIG_SMP) && XENO_DEBUG(NUCLEUS)
+#if XENO_DEBUG(NUCLEUS)
 
 xnlockinfo_t xnlock_stats[RTHAL_NR_CPUS];

@@ -712,7 +712,7 @@ static int lock_read_proc(char *page,
 
 EXPORT_SYMBOL(xnlock_stats);
 
-#endif /* CONFIG_SMP && XENO_DEBUG(NUCLEUS) */
+#endif /* XENO_DEBUG(NUCLEUS) */
 
 static int latency_read_proc(char *page,
 			     char **start,

@@ -1033,9 +1033,9 @@ void xnpod_init_proc(void)
 		create_proc_entry("timerstat", S_IFDIR, rthal_proc_root);
 #endif /* CONFIG_XENO_OPT_STATS */
 
-#if defined(CONFIG_SMP) && XENO_DEBUG(NUCLEUS)
+#if XENO_DEBUG(NUCLEUS)
 	add_proc_leaf("lock", &lock_read_proc, NULL, NULL, rthal_proc_root);
-#endif /* CONFIG_SMP && XENO_DEBUG(NUCLEUS) */
+#endif /* XENO_DEBUG(NUCLEUS) */
 
 	add_proc_leaf("latency",
 		      &latency_read_proc,

@@ -1087,9 +1087,9 @@ void xnpod_delete_proc(void)
 	remove_proc_entry("timerstat", rthal_proc_root);
 	remove_proc_entry("stat", rthal_proc_root);
 #endif /* CONFIG_XENO_OPT_STATS */
-#if defined(CONFIG_SMP) && XENO_DEBUG(NUCLEUS)
+#if XENO_DEBUG(NUCLEUS)
 	remove_proc_entry("lock", rthal_proc_root);
-#endif /* CONFIG_SMP && XENO_DEBUG(NUCLEUS) */
+#endif /* XENO_DEBUG(NUCLEUS) */
 }
 
 #ifdef CONFIG_XENO_OPT_PERVASIVE

ksrc/nucleus/pod.c +2 -2

Index: b/ksrc/nucleus/pod.c
===================================================================
@@ -3472,7 +3472,7 @@ EXPORT_SYMBOL(xnpod_welcome_thread);
 
 EXPORT_SYMBOL(nkpod_struct);
 
-#ifdef CONFIG_SMP
+#if defined(CONFIG_SMP) || XENO_DEBUG(NUCLEUS)
 EXPORT_SYMBOL(nklock);
-#endif /* CONFIG_SMP */
+#endif /* CONFIG_SMP || XENO_DEBUG(NUCLEUS) */
 EXPORT_SYMBOL(nklatency);

ksrc/skins/posix/signal.c +1 -1

Index: b/ksrc/skins/posix/signal.c
===================================================================
@@ -64,7 +64,7 @@ typedef void siginfo_handler_t(int, sigi
 #define SIGRTMAX 64
 static struct sigaction actions[SIGRTMAX];
 static pse51_siginfo_t pse51_infos_pool[PSE51_SIGQUEUE_MAX];
-#ifdef CONFIG_SMP
+#if defined(CONFIG_SMP) || XENO_DEBUG(NUCLEUS)
 static xnlock_t pse51_infos_lock = XNARCH_LOCK_UNLOCKED;
 #endif
 static xnpqueue_t pse51_infos_free_list;