Read-Copy Update [new & shiny]

PATCH New Dipankar Sarma 1 participant 1 message Jul 12, 2002

Subsystems (4)

MM core Scheduler exec and binfmt RCU

lore

Share & tools

lore

Dipankar Sarma

author

2002-07-12 20:43 UTC #0 lore raw

PATCH 1/1 Read-Copy Update [new & shiny]

At OLS, we presented some data on latency and overhead for different approaches to RCU. After some discussions with Andrea and Rusty, it seemed that it is a good idea to use this and come out with a patch. So here is a new patch. The main features are -

1. Based on rcu_poll which is already in production kernels (SuSE) and has low latency of grace period.
2. In order to reduce overheads, I added per-CPU queue support to it.
3. Works in preemptive kernel environment by either requiring the use of rcu_read_lock/unlock or using call_rcu_preempt().
4. synchronize_kernel() uses call_rcu_preempt() which requires no disabling of preemption on read-side making it useful for things like CPU hotplug.
5. Added docbook tags in headers.

Future plans are -

1. More overhead and latency measurements and tweaking around if necessary. But for infrequent things like CPU hotplug, this is not important.
2. Cleanups/bugfixes etc.

Comments, suggestions and even flames are welcome.

Thanks

Signature · 2 lines

Dipankar Sarma  <[email protected]> http://lse.sourceforge.net
Linux Technology Center, IBM Software Lab, Bangalore, India.

include/linux/init_task.h +1 -0

diff -urN linux-2.5.24-base/include/linux/init_task.h linux-2.5.24-rcu_poll/include/linux/init_task.h
@@ -79,6 +79,7 @@
     blocked:		{{0}},						\
     alloc_lock:		SPIN_LOCK_UNLOCKED,				\
     journal_info:	NULL,						\
+    cpu_preempt_cntr:	NULL,					\
 }

include/linux/rcupdate.h +85 -0

85 lines not on this page · raw message

include/linux/sched.h +51 -0

81 lines not on this page · raw message

init/main.c +2 -0

diff -urN linux-2.5.24-base/init/main.c linux-2.5.24-rcu_poll/init/main.c
@@ -28,6 +28,7 @@
 #include <linux/bootmem.h>
 #include <linux/tty.h>
 #include <linux/percpu.h>
+#include <linux/rcupdate.h>
 
 #include <asm/io.h>
 #include <asm/bugs.h>

@@ -346,6 +347,7 @@
 	printk("Kernel command line: %s\n", saved_command_line);
 	parse_options(command_line);
 	trap_init();
+	rcu_init();
 	init_IRQ();
 	sched_init();
 	softirq_init();

kernel/Makefile +2 -2

diff -urN linux-2.5.24-base/kernel/Makefile linux-2.5.24-rcu_poll/kernel/Makefile
@@ -10,12 +10,12 @@
 O_TARGET := kernel.o
 
 export-objs = signal.o sys.o kmod.o context.o ksyms.o pm.o exec_domain.o \
-		printk.o platform.o suspend.o
+		printk.o platform.o suspend.o rcupdate.o
 
 obj-y     = sched.o dma.o fork.o exec_domain.o panic.o printk.o \
 	    module.o exit.o itimer.o time.o softirq.o resource.o \
 	    sysctl.o capability.o ptrace.o timer.o user.o \
-	    signal.o sys.o kmod.o context.o futex.o platform.o
+	    signal.o sys.o kmod.o context.o futex.o platform.o rcupdate.o
 
 obj-$(CONFIG_UID16) += uid16.o
 obj-$(CONFIG_MODULES) += ksyms.o

kernel/exit.c +1 -0

diff -urN linux-2.5.24-base/kernel/exit.c linux-2.5.24-rcu_poll/kernel/exit.c
@@ -556,6 +556,7 @@
 
 	tsk->exit_code = code;
 	exit_notify();
+	rcu_preempt_put();
 	schedule();
 	BUG();
 /*

kernel/fork.c +1 -0

diff -urN linux-2.5.24-base/kernel/fork.c linux-2.5.24-rcu_poll/kernel/fork.c
@@ -118,6 +118,7 @@
 	tsk->thread_info = ti;
 	ti->task = tsk;
 	atomic_set(&tsk->usage,1);
+	tsk->cpu_preempt_cntr = NULL;
 
 	return tsk;
 }

kernel/rcupdate.c +293 -0

293 lines not on this page · raw message

kernel/sched.c +40 -0

diff -urN linux-2.5.24-base/kernel/sched.c linux-2.5.24-rcu_poll/kernel/sched.c
@@ -26,6 +26,7 @@
 #include <linux/interrupt.h>
 #include <linux/completion.h>
 #include <linux/kernel_stat.h>
+#include <linux/percpu.h>
 
 /*
  * Convert user-nice values [ -20 ... 0 ... 19 ]

@@ -146,12 +147,30 @@
 
 static struct runqueue runqueues[NR_CPUS] __cacheline_aligned;
 
+long cpu_quiescent __per_cpu_data;
+
 #define cpu_rq(cpu)		(runqueues + (cpu))
 #define this_rq()		cpu_rq(smp_processor_id())
 #define task_rq(p)		cpu_rq(task_cpu(p))
 #define cpu_curr(cpu)		(cpu_rq(cpu)->curr)
 #define rt_task(p)		((p)->prio < MAX_RT_PRIO)
 
+#ifdef CONFIG_PREEMPT
+atomic_t rcu_preempt_cntr[2] __per_cpu_data;
+atomic_t *curr_preempt_cntr __per_cpu_data;
+atomic_t *next_preempt_cntr __per_cpu_data;
+static inline void rcu_init_preempt_cntr(int cpu)
+{
+
+	atomic_set(&per_cpu(rcu_preempt_cntr[0], cpu), 0);
+	atomic_set(&per_cpu(rcu_preempt_cntr[1], cpu), 0);
+	per_cpu(curr_preempt_cntr, cpu) = 
+			&per_cpu(rcu_preempt_cntr[1], cpu);
+	per_cpu(next_preempt_cntr, cpu) = 
+			&per_cpu(rcu_preempt_cntr[0], cpu);
+}
+#endif
+
 /*
  * task_rq_lock - lock the runqueue a given task resides on and disable
  * interrupts.  Note the ordering: we can safely lookup the task_rq without

@@ -725,6 +744,10 @@
 	runqueue_t *rq = this_rq();
 	task_t *p = current;
 
+	if (user_tick ||
+            (!in_softirq() && local_irq_count(cpu) <= 1 && idle_cpu(cpu)))
+		per_cpu(cpu_quiescent, cpu)++;
+
 	if (p == rq->idle) {
 		if (local_bh_count(cpu) || local_irq_count(cpu) > 1)
 			kstat.per_cpu_system[cpu] += system;

@@ -826,6 +849,8 @@
 	 */
 	if (unlikely(preempt_get_count() & PREEMPT_ACTIVE))
 		goto pick_next_task;
+	else
+		rcu_preempt_put();
 
 	switch (prev->state) {
 	case TASK_INTERRUPTIBLE:

@@ -868,6 +893,7 @@
 switch_tasks:
 	prefetch(next);
 	clear_tsk_need_resched(prev);
+	per_cpu(cpu_quiescent, prev->thread_info->cpu)++;
 
 	if (likely(prev != next)) {
 		rq->nr_switches++;

@@ -903,6 +929,7 @@
 
 need_resched:
 	ti->preempt_count = PREEMPT_ACTIVE;
+	rcu_preempt_get();
 	schedule();
 	ti->preempt_count = 0;

@@ -1105,6 +1132,18 @@
 	task_rq_unlock(rq, &flags);
 }
 
+void force_cpu_reschedule(int cpu)
+{
+	struct runqueue *rq;
+
+	rq = cpu_rq(cpu);
+	/* Need to save flags if someday called from irq context */
+	spin_lock_irq(&rq->lock);
+	resched_task(rq->curr);
+	spin_unlock_irq(&rq->lock);
+}
+
+
 #ifndef __alpha__
 
 /*

@@ -1669,6 +1708,7 @@
 		rq->expired = rq->arrays + 1;
 		spin_lock_init(&rq->lock);
 		INIT_LIST_HEAD(&rq->migration_queue);
+		rcu_init_preempt_cntr(i);
 
 		for (j = 0; j < 2; j++) {
 			array = rq->arrays + j;