semaphore optimisation

@@ -535,6 +535,7 @@
 #define CURRENT_TIME (xtime.tv_sec)
 
 extern void FASTCALL(__wake_up(wait_queue_head_t *q, unsigned int mode, unsigned int wq_mode));
+extern void FASTCALL(____wake_up(wait_queue_head_t *q, unsigned int mode, unsigned int wq_mode));
 extern void FASTCALL(__wake_up_sync(wait_queue_head_t *q, unsigned int mode, unsigned int wq_mode));
 extern void FASTCALL(sleep_on(wait_queue_head_t *q));
 extern long FASTCALL(sleep_on_timeout(wait_queue_head_t *q,

@@ -72,6 +72,7 @@
 # define wq_read_unlock_irqrestore read_unlock_irqrestore
 # define wq_read_unlock read_unlock
 # define wq_write_lock_irq write_lock_irq
+# define wq_write_unlock_irq write_unlock_irq
 # define wq_write_lock_irqsave write_lock_irqsave
 # define wq_write_unlock_irqrestore write_unlock_irqrestore
 # define wq_write_unlock write_unlock

@@ -84,6 +85,7 @@
 # define wq_read_unlock spin_unlock
 # define wq_read_unlock_irqrestore spin_unlock_irqrestore
 # define wq_write_lock_irq spin_lock_irq
+# define wq_write_unlock_irq spin_unlock_irq
 # define wq_write_lock_irqsave spin_lock_irqsave
 # define wq_write_unlock_irqrestore spin_unlock_irqrestore
 # define wq_write_unlock spin_unlock

@@ -53,16 +53,15 @@
 	wake_up(&sem->wait);
 }
 
-static spinlock_t semaphore_lock = SPIN_LOCK_UNLOCKED;
-
 void __down(struct semaphore * sem)
 {
 	struct task_struct *tsk = current;
 	DECLARE_WAITQUEUE(wait, tsk);
-	tsk->state = TASK_UNINTERRUPTIBLE;
-	add_wait_queue_exclusive(&sem->wait, &wait);
 
-	spin_lock_irq(&semaphore_lock);
+	tsk->state = TASK_UNINTERRUPTIBLE;
+	wq_write_lock_irq(&sem->wait.lock);
+	wait.flags = WQ_FLAG_EXCLUSIVE;
+	__add_wait_queue_tail(&sem->wait, &wait);
 	sem->sleepers++;
 	for (;;) {
 		int sleepers = sem->sleepers;

@@ -76,14 +75,14 @@
 			break;
 		}
 		sem->sleepers = 1;	/* us - see -1 above */
-		spin_unlock_irq(&semaphore_lock);
+		wq_write_unlock_irq(&sem->wait.lock);
 
 		schedule();
 		tsk->state = TASK_UNINTERRUPTIBLE;
-		spin_lock_irq(&semaphore_lock);
+		wq_write_lock_irq(&sem->wait.lock);
 	}
-	spin_unlock_irq(&semaphore_lock);
-	remove_wait_queue(&sem->wait, &wait);
+	__remove_wait_queue(&sem->wait, &wait);
+	wq_write_unlock_irq(&sem->wait.lock);
 	tsk->state = TASK_RUNNING;
 	wake_up(&sem->wait);
 }

@@ -93,10 +92,11 @@
 	int retval = 0;
 	struct task_struct *tsk = current;
 	DECLARE_WAITQUEUE(wait, tsk);
-	tsk->state = TASK_INTERRUPTIBLE;
-	add_wait_queue_exclusive(&sem->wait, &wait);
 
-	spin_lock_irq(&semaphore_lock);
+	tsk->state = TASK_INTERRUPTIBLE;
+	wq_write_lock_irq(&sem->wait.lock);
+	wait.flags = WQ_FLAG_EXCLUSIVE;
+	__add_wait_queue_tail(&sem->wait, &wait);
 	sem->sleepers ++;
 	for (;;) {
 		int sleepers = sem->sleepers;

@@ -126,15 +126,15 @@
 			break;
 		}
 		sem->sleepers = 1;	/* us - see -1 above */
-		spin_unlock_irq(&semaphore_lock);
+		wq_write_unlock_irq(&sem->wait.lock);
 
 		schedule();
 		tsk->state = TASK_INTERRUPTIBLE;
-		spin_lock_irq(&semaphore_lock);
+		wq_write_lock_irq(&sem->wait.lock);
 	}
-	spin_unlock_irq(&semaphore_lock);
+	__remove_wait_queue(&sem->wait, &wait);
+	wq_write_unlock_irq(&sem->wait.lock);
 	tsk->state = TASK_RUNNING;
-	remove_wait_queue(&sem->wait, &wait);
 	wake_up(&sem->wait);
 	return retval;
 }

@@ -152,7 +152,7 @@
 	int sleepers;
 	unsigned long flags;
 
-	spin_lock_irqsave(&semaphore_lock, flags);
+	wq_write_lock_irqsave(&sem->wait.lock, flags);
 	sleepers = sem->sleepers + 1;
 	sem->sleepers = 0;
 

@@ -161,9 +161,9 @@
 	 * playing, because we own the spinlock.
 	 */
 	if (!atomic_add_negative(sleepers, &sem->count))
-		wake_up(&sem->wait);
+		____wake_up(&sem->wait, TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE, WQ_FLAG_EXCLUSIVE);
 
-	spin_unlock_irqrestore(&semaphore_lock, flags);
+	wq_write_unlock_irqrestore(&sem->wait.lock, flags);
 	return 1;
 }
 

@@ -700,7 +700,7 @@
 }
 
 static inline void __wake_up_common (wait_queue_head_t *q, unsigned int mode,
-				     unsigned int wq_mode, const int sync)
+				     unsigned int wq_mode, const int sync, const int do_locking)
 {
 	struct list_head *tmp, *head;
 	struct task_struct *p, *best_exclusive;

@@ -713,7 +713,8 @@
 	best_cpu = smp_processor_id();
 	irq = in_interrupt();
 	best_exclusive = NULL;
-	wq_write_lock_irqsave(&q->lock, flags);
+	if (do_locking)
+		wq_write_lock_irqsave(&q->lock, flags);
 
 #if WAITQUEUE_DEBUG
 	CHECK_MAGIC_WQHEAD(q);

@@ -768,19 +769,25 @@
 		else
 			wake_up_process(best_exclusive);
 	}
-	wq_write_unlock_irqrestore(&q->lock, flags);
+	if (do_locking)
+		wq_write_unlock_irqrestore(&q->lock, flags);
 out:
 	return;
 }
 
 void __wake_up(wait_queue_head_t *q, unsigned int mode, unsigned int wq_mode)
 {
-	__wake_up_common(q, mode, wq_mode, 0);
+	__wake_up_common(q, mode, wq_mode, 0, 1);
+}
+
+void ____wake_up(wait_queue_head_t *q, unsigned int mode, unsigned int wq_mode)
+{
+	__wake_up_common(q, mode, wq_mode, 0, 0);
 }
 
 void __wake_up_sync(wait_queue_head_t *q, unsigned int mode, unsigned int wq_mode)
 {
-	__wake_up_common(q, mode, wq_mode, 1);
+	__wake_up_common(q, mode, wq_mode, 1, 1);
 }
 
 #define	SLEEP_ON_VAR				\