node affine NUMA scheduler Nod20

diff -Nru a/arch/i386/kernel/smpboot.c b/arch/i386/kernel/smpboot.c
@@ -764,6 +764,8 @@
 
 extern unsigned long cpu_initialized;
 
+static int __initdata nr_lnodes = 0;
+
 static void __init do_boot_cpu (int apicid) 
 /*
  * NOTE - on most systems this is a PHYSICAL apic ID, but on multiquad

@@ -772,11 +774,17 @@
 {
 	struct task_struct *idle;
 	unsigned long boot_error = 0;
-	int timeout, cpu;
+	int timeout, cpu, cell;
 	unsigned long start_eip;
 	unsigned short nmi_high, nmi_low;
 
 	cpu = ++cpucount;
+#ifdef CONFIG_NUMA_SCHED
+	cell = SAPICID_TO_PNODE(apicid);
+	if (pnode_to_lnode[cell] < 0) {
+		pnode_to_lnode[cell] = nr_lnodes++;
+	}
+#endif
 	/*
 	 * We can't use kernel_thread since we must avoid to
 	 * reschedule the child.

@@ -995,6 +1003,10 @@
 	set_bit(0, &cpu_callout_map);
 	boot_cpu_logical_apicid = logical_smp_processor_id();
 	map_cpu_to_boot_apicid(0, boot_cpu_apicid);
+#ifdef CONFIG_NUMA_SCHED
+	pnode_to_lnode[SAPICID_TO_PNODE(boot_cpu_apicid)] = nr_lnodes++;
+	printk("boot_cpu_apicid = %d, nr_lnodes = %d, lnode = %d\n", boot_cpu_apicid, nr_lnodes, pnode_to_lnode[0]);
+#endif
 
 	current_thread_info()->cpu = 0;
 	smp_tune_scheduling();

@@ -1221,4 +1233,9 @@
 void __init smp_cpus_done(unsigned int max_cpus)
 {
 	zap_low_mappings();
+#ifdef CONFIG_NUMA_SCHED
+	pooldata_lock();
+	bld_pools();
+	pooldata_unlock();
+#endif
 }

81 lines not on this page · raw message

diff -Nru a/fs/exec.c b/fs/exec.c
@@ -865,6 +865,11 @@
 	int retval;
 	int i;
 
+#ifdef CONFIG_NODE_AFFINE_SCHED
+	if (current->node_policy == NODPOL_EXEC)
+		sched_balance_exec();
+#endif
+
 	file = open_exec(filename);
 
 	retval = PTR_ERR(file);

diff -Nru a/include/asm-i386/atomic.h b/include/asm-i386/atomic.h
@@ -111,6 +111,18 @@
 }
 
 /**
+ * atomic_inc_return - increment atomic variable and return new value
+ * @v: pointer of type atomic_t
+ *
+ * Atomically increments @v by 1 and return it's new value.  Note that
+ * the guaranteed useful range of an atomic_t is only 24 bits.
+ */
+static inline int atomic_inc_return(atomic_t *v){
+	atomic_inc(v);
+	return v->counter;
+}
+
+/**
  * atomic_dec - decrement atomic variable
  * @v: pointer of type atomic_t
  * 

diff -Nru a/include/asm-i386/smp.h b/include/asm-i386/smp.h
@@ -124,5 +124,11 @@
 
 #define NO_PROC_ID		0xFF		/* No processor magic marker */
 
+#ifdef CONFIG_NUMA_SCHED
+#define NR_NODES               8
+#define cpu_physical_id(cpuid) (cpu_to_physical_apicid(cpuid))
+#define SAPICID_TO_PNODE(hwid)  (hwid >> 4)
+#endif
+
 #endif
 #endif

diff -Nru a/include/asm-ia64/smp.h b/include/asm-ia64/smp.h
@@ -13,6 +13,7 @@
 
 #ifdef CONFIG_SMP
 
+#include <linux/cache.h>
 #include <linux/init.h>
 #include <linux/threads.h>
 #include <linux/kernel.h>

@@ -129,6 +130,32 @@
 extern void __cpu_die (unsigned int cpu);
 extern int __cpu_up (unsigned int cpu);
 extern void __init smp_build_cpu_map(void);
+
+#ifdef CONFIG_NUMA_SCHED
+#ifdef CONFIG_IA64_DIG
+/* sooner or later this should be a configurable parameter [EF] */
+#define NR_NODES 8
+/* 
+ * This is the node ID on the NEC AzusA, 
+ * on LION and BigSur it correctly initializes to node 0
+ */
+#define SAPICID_TO_PNODE(hwid) ((hwid >> 12) & 0xff)
+
+#elif defined(CONFIG_IA64_SGI_SN)
+
+/*
+ * SGI SN1 & SN2 specific macros
+ */
+#define NR_NODES 32
+#define SAPICID_TO_PNODE(hwid) cpuid_to_cnodeid(hwid)
+
+#endif
+
+#else   /* CONFIG_NODE_AFFINE_SCHED */
+#define NR_NODES               1
+#define CPU_TO_NODE(cpu)       0
+#define SAPICID_TO_PNODE(hwid)  0
+#endif  /* CONFIG_NODE_AFFINE_SCHED */
 
 extern void __init init_smp_config (void);
 extern void smp_do_timer (struct pt_regs *regs);

diff -Nru a/include/linux/prctl.h b/include/linux/prctl.h
@@ -34,4 +34,10 @@
 # define PR_FP_EXC_ASYNC	2	/* async recoverable exception mode */
 # define PR_FP_EXC_PRECISE	3	/* precise exception mode */
 
+/* Get/set node for node-affine scheduling */
+#define PR_GET_NODE       16
+#define PR_SET_NODE       17
+#define PR_GET_NODPOL     18
+#define PR_SET_NODPOL     19
+
 #endif /* _LINUX_PRCTL_H */

76 lines not on this page · raw message

diff -Nru a/kernel/fork.c b/kernel/fork.c
@@ -727,6 +727,13 @@
 #ifdef CONFIG_SMP
 	{
 		int i;
+#ifdef CONFIG_NODE_AFFINE_SCHED
+		if (p->node_policy == NODPOL_FORK_ALL ||
+		    (p->node_policy == NODPOL_FORK && !(clone_flags & CLONE_VM)))
+			{
+			sched_balance_fork(p);
+			}
+#endif
 
 		/* ?? should we just memset this ?? */
 		for(i = 0; i < NR_CPUS; i++)

diff -Nru a/kernel/ksyms.c b/kernel/ksyms.c
@@ -601,6 +601,18 @@
 
 EXPORT_SYMBOL(tasklist_lock);
 EXPORT_SYMBOL(pidhash);
+
+#ifdef CONFIG_NUMA_SCHED
+#include <linux/sched.h>
+EXPORT_SYMBOL(runqueues_address);
+EXPORT_SYMBOL(numpools);
+EXPORT_SYMBOL(pool_ptr);
+EXPORT_SYMBOL(pool_cpus);
+EXPORT_SYMBOL(pool_nr_cpus);
+EXPORT_SYMBOL(pool_mask);
+EXPORT_SYMBOL(sched_migrate_task);
+#endif
+
 #if defined(CONFIG_SMP) && defined(__GENERIC_PER_CPU)
 EXPORT_SYMBOL(__per_cpu_offset);
 #endif

921 lines not on this page · raw message

diff -Nru a/kernel/softirq.c b/kernel/softirq.c
@@ -365,6 +365,7 @@
 	set_cpus_allowed(current, 1UL << cpu);
 	if (smp_processor_id() != cpu)
 		BUG();
+	set_task_node(current, CPU_TO_NODE(cpu));
 
 	sprintf(current->comm, "ksoftirqd_CPU%d", cpu);
 

73 lines not on this page · raw message