syscall latency improvement #1 try 2

diff -uNr linux-2.5.3-pre5/arch/i386/kernel/process.c linux-work-253p5/arch/i386/kernel/process.c
@@ -89,7 +89,7 @@
 
 /*
  * On SMP it's slightly faster (but much more power-consuming!)
- * to poll the ->need_resched flag instead of waiting for the
+ * to poll the ->work.need_resched flag instead of waiting for the
  * cross-CPU IPI to arrive. Use this option with caution.
  */
 static void poll_idle (void)

@@ -102,15 +102,15 @@
 	 * Deal with another CPU just having chosen a thread to
 	 * run here:
 	 */
-	oldval = xchg(&current->need_resched, -1);
+	oldval = xchg(&current->work.need_resched, -1);
 
 	if (!oldval)
 		asm volatile(
 			"2:"
-			"cmpl $-1, %0;"
+			"cmpb $-1, %0;"
 			"rep; nop;"
 			"je 2b;"
-				: :"m" (current->need_resched));
+				: :"m" (current->work.need_resched));
 }
 
 /*

65 lines not on this page · raw message

diff -uNr linux-2.5.3-pre5/arch/i386/kernel/signal.c linux-work-253p5/arch/i386/kernel/signal.c
@@ -28,8 +28,6 @@
 
 #define _BLOCKABLE (~(sigmask(SIGKILL) | sigmask(SIGSTOP)))
 
-int FASTCALL(do_signal(struct pt_regs *regs, sigset_t *oldset));
-
 int copy_siginfo_to_user(siginfo_t *to, siginfo_t *from)
 {
 	if (!access_ok (VERIFY_WRITE, to, sizeof(siginfo_t)))

diff -uNr linux-2.5.3-pre5/arch/i386/kernel/vm86.c linux-work-253p5/arch/i386/kernel/vm86.c
@@ -212,7 +212,7 @@
 	info->regs.__null_ds = 0;
 	info->regs.__null_es = 0;
 
-/* we are clearing fs,gs later just before "jmp ret_from_sys_call",
+/* we are clearing fs,gs later just before "jmp resume_userspace",
  * because starting with Linux 2.1.x they aren't no longer saved/restored
  */
 

@@ -255,7 +255,7 @@
 	__asm__ __volatile__(
 		"xorl %%eax,%%eax; movl %%eax,%%fs; movl %%eax,%%gs\n\t"
 		"movl %0,%%esp\n\t"
-		"jmp ret_from_sys_call"
+		"jmp resume_userspace"
 		: /* no outputs */
 		:"r" (&info->regs), "b" (tsk) : "ax");
 	/* we never return here */

@@ -268,7 +268,7 @@
 	regs32 = save_v86_state(regs16);
 	regs32->eax = retval;
 	__asm__ __volatile__("movl %0,%%esp\n\t"
-		"jmp ret_from_sys_call"
+		"jmp resume_userspace"
 		: : "r" (regs32), "b" (current));
 }
 

diff -uNr linux-2.5.3-pre5/fs/lockd/svc.c linux-work-253p5/fs/lockd/svc.c
@@ -304,7 +304,7 @@
 	 * Wait for the lockd process to exit, but since we're holding
 	 * the lockd semaphore, we can't wait around forever ...
 	 */
-	current->sigpending = 0;
+	current->work.sigpending = 0;
 	interruptible_sleep_on_timeout(&lockd_exit, HZ);
 	if (nlmsvc_pid) {
 		printk(KERN_WARNING 

diff -uNr linux-2.5.3-pre5/fs/nfsd/export.c linux-work-253p5/fs/nfsd/export.c
@@ -468,7 +468,7 @@
 		return 0;
 	}
 
-	current->sigpending = 0;
+	current->work.sigpending = 0;
 	want_lock++;
 	while (hash_count || hash_lock) {
 		interruptible_sleep_on(&hash_wait);

diff -uNr linux-2.5.3-pre5/include/asm-i386/signal.h linux-work-253p5/include/asm-i386/signal.h
@@ -2,6 +2,7 @@
 #define _ASMi386_SIGNAL_H
 
 #include <linux/types.h>
+#include <linux/linkage.h>
 
 /* Avoid too many header ordering problems.  */
 struct siginfo;

@@ -216,6 +217,8 @@
 	return word;
 }
 
+extern int FASTCALL(do_signal(struct pt_regs *regs, sigset_t *oldset));
+
 #endif /* __KERNEL__ */
 
 #endif

diff -uNr linux-2.5.3-pre5/include/linux/init_task.h linux-work-253p5/include/linux/init_task.h
@@ -35,6 +35,14 @@
 	siglock:	SPIN_LOCK_UNLOCKED 		\
 }
 
+#define INIT_TASK_WORK				\
+{						\
+	need_resched:	0,			\
+	syscall_trace:	0,			\
+	sigpending:	0,			\
+	notify_resume:	0,			\
+}
+
 /*
  *  INIT_TASK is used to set up the first task table, touch at
  * your own risk!. Base=0, limit=0x1fffff (=2MB)

@@ -43,7 +51,7 @@
 {									\
     state:		0,						\
     flags:		0,						\
-    sigpending:		0,						\
+    work:		INIT_TASK_WORK,					\
     addr_limit:		KERNEL_DS,					\
     exec_domain:	&default_exec_domain,				\
     lock_depth:		-1,						\

diff -uNr linux-2.5.3-pre5/include/linux/sched.h linux-work-253p5/include/linux/sched.h
@@ -228,19 +228,29 @@
 
 typedef struct prio_array prio_array_t;
 
+/* this struct must occupy one 32-bit chunk so that is can be read in one go */
+struct task_work {
+	__s8	need_resched;
+	__u8	syscall_trace;	/* count of syscall interceptors */
+	__u8	sigpending;
+	__u8	notify_resume;	/* request for notification on
+				   userspace execution resumption */
+} __attribute__((packed));
+
 struct task_struct {
 	/*
 	 * offsets of these are hardcoded elsewhere - touch with care
 	 */
 	volatile long state;	/* -1 unrunnable, 0 runnable, >0 stopped */
 	unsigned long flags;	/* per process flags, defined below */
-	int sigpending;
+	volatile struct task_work work;
+
 	mm_segment_t addr_limit;	/* thread address space:
 					 	0-0xBFFFFFFF for user-thead
 						0-0xFFFFFFFF for kernel-thread
 					 */
 	struct exec_domain *exec_domain;
-	volatile long need_resched;
+	long __pad;
 	unsigned long ptrace;
 
 	int lock_depth;		/* Lock depth */

@@ -381,7 +391,7 @@
  */
 
 #define PT_PTRACED	0x00000001
-#define PT_TRACESYS	0x00000002
+#define PT_SYSCALLTRACE	0x00000002	/* T if syscall_trace is +1 for ptrace() */
 #define PT_DTRACE	0x00000004	/* delayed trace (used on m68k, i386) */
 #define PT_TRACESYSGOOD	0x00000008
 #define PT_PTRACE_CAP	0x00000010	/* ptracer can follow suid-exec */

@@ -564,12 +574,12 @@
 
 static inline int signal_pending(struct task_struct *p)
 {
-	return (p->sigpending != 0);
+	return (p->work.sigpending != 0);
 }
   
 static inline int need_resched(void)
 {
-	return unlikely(current->need_resched != 0);
+	return unlikely(current->work.need_resched != 0);
 }
 
 static inline void cond_resched(void)

@@ -614,7 +624,7 @@
 
 static inline void recalc_sigpending(struct task_struct *t)
 {
-	t->sigpending = has_pending_signals(&t->pending.signal, &t->blocked);
+	t->work.sigpending = has_pending_signals(&t->pending.signal, &t->blocked);
 }
 
 /* True if we are on the alternate signal stack.  */

diff -uNr linux-2.5.3-pre5/kernel/fork.c linux-work-253p5/kernel/fork.c
@@ -631,7 +631,7 @@
 	}
 	spin_lock_init(&p->alloc_lock);
 
-	p->sigpending = 0;
+	p->work.sigpending = 0;
 	init_sigpending(&p->pending);
 
 	p->it_real_value = p->it_virt_value = p->it_prof_value = 0;

@@ -756,7 +756,7 @@
 		 * Let the child process run first, to avoid most of the
 		 * COW overhead when the child exec()s afterwards.
 		 */
-		current->need_resched = 1;
+		current->work.need_resched = 1;
 
 fork_out:
 	return retval;

diff -uNr linux-2.5.3-pre5/kernel/sched.c linux-work-253p5/kernel/sched.c
@@ -176,9 +176,9 @@
 {
 	int need_resched;
 
-	need_resched = p->need_resched;
+	need_resched = p->work.need_resched;
 	wmb();
-	p->need_resched = 1;
+	p->work.need_resched = 1;
 	if (!need_resched && (p->cpu != smp_processor_id()))
 		smp_send_reschedule(p->cpu);
 }

@@ -483,7 +483,7 @@
 	this_rq->nr_running++;
 	enqueue_task(next, this_rq->active);
 	if (next->prio < current->prio)
-		current->need_resched = 1;
+		current->work.need_resched = 1;
 	if (!idle && --imbalance) {
 		if (array == busiest->expired) {
 			array = busiest->active;

@@ -528,7 +528,7 @@
 		return idle_tick();
 	/* Task might have expired already, but not scheduled off yet */
 	if (p->array != rq->active) {
-		p->need_resched = 1;
+		p->work.need_resched = 1;
 		return;
 	}
 	spin_lock(&rq->lock);

@@ -539,7 +539,7 @@
 		 */
 		if ((p->policy == SCHED_RR) && !--p->time_slice) {
 			p->time_slice = NICE_TO_TIMESLICE(p->__nice);
-			p->need_resched = 1;
+			p->work.need_resched = 1;
 
 			/* put it at the end of the queue: */
 			dequeue_task(p, rq->active);

@@ -559,7 +559,7 @@
 		p->sleep_avg--;
 	if (!--p->time_slice) {
 		dequeue_task(p, rq->active);
-		p->need_resched = 1;
+		p->work.need_resched = 1;
 		p->prio = effective_prio(p);
 		p->time_slice = NICE_TO_TIMESLICE(p->__nice);
 		enqueue_task(p, TASK_INTERACTIVE(p) ? rq->active : rq->expired);

@@ -622,7 +622,7 @@
 	next = list_entry(queue->next, task_t, run_list);
 
 switch_tasks:
-	prev->need_resched = 0;
+	prev->work.need_resched = 0;
 
 	if (likely(prev != next)) {
 		rq->nr_switches++;

@@ -1246,7 +1246,7 @@
 	current->prio = MAX_PRIO;
 	current->state = TASK_RUNNING;
 	double_rq_unlock(this_rq, rq);
-	current->need_resched = 1;
+	current->work.need_resched = 1;
 	__restore_flags(flags);
 }
 

diff -uNr linux-2.5.3-pre5/kernel/signal.c linux-work-253p5/kernel/signal.c
@@ -105,7 +105,7 @@
 void
 flush_signals(struct task_struct *t)
 {
-	t->sigpending = 0;
+	t->work.sigpending = 0;
 	flush_sigqueue(&t->pending);
 }
 

@@ -119,7 +119,7 @@
 		if (atomic_dec_and_test(&sig->count))
 			kmem_cache_free(sigact_cachep, sig);
 	}
-	tsk->sigpending = 0;
+	tsk->work.sigpending = 0;
 	flush_sigqueue(&tsk->pending);
 	spin_unlock_irq(&tsk->sigmask_lock);
 }

@@ -246,7 +246,7 @@
 		if (current->notifier) {
 			if (sigismember(current->notifier_mask, sig)) {
 				if (!(current->notifier)(current->notifier_data)) {
-					current->sigpending = 0;
+					current->work.sigpending = 0;
 					return 0;
 				}
 			}

@@ -465,7 +465,7 @@
  */
 static inline void signal_wake_up(struct task_struct *t)
 {
-	t->sigpending = 1;
+	t->work.sigpending = 1;
 
 #ifdef CONFIG_SMP
 	/*

diff -uNr linux-2.5.3-pre5/net/sunrpc/sched.c linux-work-253p5/net/sunrpc/sched.c
@@ -1109,7 +1109,7 @@
 	unsigned long flags;
 
 	while (all_tasks) {
-		current->sigpending = 0;
+		current->work.sigpending = 0;
 		rpc_killall_tasks(NULL);
 		__rpc_schedule();
 		if (all_tasks) {

@@ -1183,7 +1183,7 @@
 	 * Usually rpciod will exit very quickly, so we
 	 * wait briefly before checking the process id.
 	 */
-	current->sigpending = 0;
+	current->work.sigpending = 0;
 	yield();
 	/*
 	 * Display a message if we're going to wait longer.

diff -uNr linux-2.5.3-pre5/net/sunrpc/svc.c linux-work-253p5/net/sunrpc/svc.c
@@ -185,7 +185,7 @@
 		progp->pg_name, proto == IPPROTO_UDP? "udp" : "tcp", port);
 
 	if (!port)
-		current->sigpending = 0;
+		current->work.sigpending = 0;
 
 	for (i = 0; i < progp->pg_nvers; i++) {
 		if (progp->pg_vers[i] == NULL)