driverfs multi-node(board) patch [1/2]

Patrick,
Here's the pre-reqs to my patch. I put in a memblk_online_map (just like
cpu_online_map) and associated functions (memblk_online(), etc).
Also there's a include/linux/topology.h that has some wrappers for the
in-kernel macros. Just does bound checking, etc.

This patch sits on top of 2.5.39-mm1 (which includes my in-kernel
topology API). The *real* driverfs additions will sit on top of this patch... Coming momentarily...

Cheers!

-Matt

diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.39-vanilla/arch/i386/kernel/smpboot.c linux-2.5.39-driverfs_pre-reqs/arch/i386/kernel/smpboot.c
@@ -62,6 +62,9 @@
 /* Bitmask of currently online CPUs */
 unsigned long cpu_online_map;
 
+/* Bitmask of currently online memory blocks */
+unsigned long memblk_online_map = 0UL;
+
 static volatile unsigned long cpu_callin_map;
 volatile unsigned long cpu_callout_map;
 static unsigned long smp_commenced_mask;

diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.39-vanilla/drivers/base/core.c linux-2.5.39-driverfs_pre-reqs/drivers/base/core.c
@@ -68,7 +68,7 @@
 }
 
 /**
- * device_attach - try to associated device with a driver
+ * do_device_attach - try to associate device with a driver
  * @drv:	current driver to try
  * @data:	device in disguise
  *

diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.39-vanilla/include/asm-i386/smp.h linux-2.5.39-driverfs_pre-reqs/include/asm-i386/smp.h
@@ -54,6 +54,7 @@
 extern void smp_alloc_memory(void);
 extern unsigned long phys_cpu_present_map;
 extern unsigned long cpu_online_map;
+extern unsigned long memblk_online_map;
 extern volatile unsigned long smp_invalidate_needed;
 extern int pic_mode;
 extern int smp_num_siblings;

@@ -102,6 +103,13 @@
 	return -1;
 }
 
+#define memblk_online(memblk) (memblk_online_map & (1<<(memblk)))
+ 
+extern inline unsigned int num_online_memblks(void)
+{
+	return hweight32(memblk_online_map);
+}
+
 static __inline int hard_smp_processor_id(void)
 {
 	/* we don't want to mark this access volatile - bad code generation */

diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.39-vanilla/include/linux/smp.h linux-2.5.39-driverfs_pre-reqs/include/linux/smp.h
@@ -94,7 +94,10 @@
 #define cpu_online(cpu)				({ BUG_ON((cpu) != 0); 1; })
 #define num_online_cpus()			1
 #define num_booting_cpus()			1
-#define cpu_possible(cpu)				({ BUG_ON((cpu) != 0); 1; })
+#define cpu_possible(cpu)			({ BUG_ON((cpu) != 0); 1; })
+#define memblk_online_map			1
+#define memblk_online(memblk)			({ BUG_ON((memblk) != 0); 1; })
+#define num_online_memblks()			1
 
 struct notifier_block;
 

diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.39-vanilla/mm/numa.c linux-2.5.39-driverfs_pre-reqs/mm/numa.c
@@ -37,6 +37,7 @@
 	}
 	contig_page_data.node_mem_map = pmap;
 	free_area_init_core(&contig_page_data, zones_size, zholes_size);
+	memblk_online_map = 1UL;
 	mem_map = contig_page_data.node_mem_map;
 }
 

@@ -74,6 +75,10 @@
 	}
 	pgdat->node_mem_map = pmap;
 	free_area_init_core(pgdat, zones_size, zholes_size);
+	if (test_and_set_bit(num_online_memblks(), &memblk_online_map)){
+		printk("free_area_init_core: memblk alread counted?!?!\n");
+		BUG();
+	}
 
 	/*
 	 * Get space for the valid bitmap.

Patrick,
Ok.. here are the real changes. I'd really like to get some feedback on
what you (or anyone else) thinks of these proposed changes. This sets up a generic topology initialization routine which should discover all online nodes (boards), CPUs, and Memory Blocks at boot time. It also makes the CPUs and memblks it discovers children of the appropriate nodes.

Cheers!

-Matt

diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.39-mm1+pre_reqs/arch/i386/kernel/cpu/common.c linux-2.5.39-mm1+pre_reqs+additions/arch/i386/kernel/cpu/common.c
@@ -1,7 +1,6 @@
 #include <linux/init.h>
 #include <linux/string.h>
 #include <linux/delay.h>
-#include <linux/cpu.h>
 #include <linux/smp.h>
 #include <asm/semaphore.h>
 #include <asm/processor.h>

@@ -507,37 +506,3 @@
 	current->used_math = 0;
 	stts();
 }
-
-/*
- * Bulk registration of the cpu devices with the system.
- * Some of this stuff could possibly be moved into a shared 
- * location..
- * Also, these devices should be integrated with other CPU data..
- */
-
-static struct cpu cpu_devices[NR_CPUS];
-
-static struct device_driver cpu_driver = {
-	.name		= "cpu",
-	.bus		= &system_bus_type,
-	.devclass	= &cpu_devclass,
-};
-
-static int __init register_cpus(void)
-{
-	int i;
-
-	driver_register(&cpu_driver);
-
-	for (i = 0; i < NR_CPUS; i++) {
-		struct sys_device * sysdev = &cpu_devices[i].sysdev;
-		sysdev->name = "cpu";
-		sysdev->id = i;
-		sysdev->dev.driver = &cpu_driver;
-		if (cpu_possible(i))
-			sys_device_register(sysdev);
-	}
-	return 0;
-}
-
-subsys_initcall(register_cpus);

diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.39-mm1+pre_reqs/drivers/base/Makefile linux-2.5.39-mm1+pre_reqs+additions/drivers/base/Makefile
@@ -2,13 +2,14 @@
 
 obj-y		:= core.o sys.o interface.o power.o bus.o \
 			driver.o class.o intf.o platform.o \
-			cpu.o
+			node.o cpu.o memblk.o topology.o
 
 obj-y		+= fs/
 
 obj-$(CONFIG_HOTPLUG)	+= hotplug.o
 
 export-objs	:= core.o power.o sys.o bus.o driver.o \
-			class.o intf.o platform.o cpu.o 
+			class.o intf.o platform.o node.o \
+			cpu.o memblk.o
 
 include $(TOPDIR)/Rules.make

diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.39-mm1+pre_reqs/drivers/base/cpu.c linux-2.5.39-mm1+pre_reqs+additions/drivers/base/cpu.c
@@ -5,7 +5,6 @@
 #include <linux/device.h>
 #include <linux/module.h>
 #include <linux/init.h>
-#include <linux/cpu.h>
 
 static int cpu_add_device(struct device * dev)
 {

diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.39-mm1+pre_reqs/drivers/base/memblk.c linux-2.5.39-mm1+pre_reqs+additions/drivers/base/memblk.c
@@ -0,0 +1,27 @@
+/*
+ * memblk.c - basic memblk class support
+ */
+
+#include <linux/device.h>
+#include <linux/module.h>
+#include <linux/init.h>
+
+static int memblk_add_device(struct device * dev)
+{
+	return 0;
+}
+
+struct device_class memblk_devclass = {
+	.name		= "memblk",
+	.add_device	= memblk_add_device,
+};
+
+
+static int __init memblk_devclass_init(void)
+{
+	return devclass_register(&memblk_devclass);
+}
+
+postcore_initcall(memblk_devclass_init);
+
+EXPORT_SYMBOL(memblk_devclass);

diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.39-mm1+pre_reqs/drivers/base/node.c linux-2.5.39-mm1+pre_reqs+additions/drivers/base/node.c
@@ -0,0 +1,27 @@
+/*
+ * node.c - basic node class support
+ */
+
+#include <linux/device.h>
+#include <linux/module.h>
+#include <linux/init.h>
+
+static int node_add_device(struct device * dev)
+{
+	return 0;
+}
+
+struct device_class node_devclass = {
+	.name		= "node",
+	.add_device	= node_add_device,
+};
+
+
+static int __init node_devclass_init(void)
+{
+	return devclass_register(&node_devclass);
+}
+
+postcore_initcall(node_devclass_init);
+
+EXPORT_SYMBOL(node_devclass);

diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.39-mm1+pre_reqs/drivers/base/sys.c linux-2.5.39-mm1+pre_reqs+additions/drivers/base/sys.c
@@ -55,6 +55,9 @@
 
 	pr_debug("Registering system board %d\n",root->id);
 
+	if (!root->dev.parent)
+		root->dev.parent = &system_bus;
+
 	error = device_register(&root->dev);
 	if (!error) {
 		strncpy(root->sysdev.bus_id,"sys",BUS_ID_SIZE);

Can you show an example output of what the directory structure now looks like with this patch?

Curious,

greg k-h

Surely, Greg! Something I definitely should have put in the original post... here's the before:

*****************BEFORE**************************** [root@elm3b79 devices]# tree -d bus/system/devices/ bus/system/devices/

|-- cpu0 -> ../../../root/sys/cpu0
|-- cpu1 -> ../../../root/sys/cpu1
|-- cpu2 -> ../../../root/sys/cpu2
|-- cpu3 -> ../../../root/sys/cpu3
|-- cpu4 -> ../../../root/sys/cpu4
|-- cpu5 -> ../../../root/sys/cpu5
|-- cpu6 -> ../../../root/sys/cpu6
|-- cpu7 -> ../../../root/sys/cpu7
|-- pic0 -> ../../../root/sys/pic0

`-- rtc0 -> ../../../root/sys/rtc0

10 directories
[root@elm3b79 devices]# tree -d class/
class/

|-- cpu
|   |-- devices
|   |   |-- 0 -> ../../../root/sys/cpu0
|   |   |-- 1 -> ../../../root/sys/cpu1
|   |   |-- 2 -> ../../../root/sys/cpu2
|   |   |-- 3 -> ../../../root/sys/cpu3
|   |   |-- 4 -> ../../../root/sys/cpu4
|   |   |-- 5 -> ../../../root/sys/cpu5
|   |   |-- 6 -> ../../../root/sys/cpu6
|   |   `-- 7 -> ../../../root/sys/cpu7
|   `-- drivers
|-- disk
|   |-- devices
|   `-- drivers
`-- input
    |-- devices
    `-- drivers

17 directories
[root@elm3b79 devices]# tree -d root/sys/
root/sys/

|-- cpu0
|-- cpu1
|-- cpu2
|-- cpu3
|-- cpu4
|-- cpu5
|-- cpu6
|-- cpu7
|-- pic0

`-- rtc0

10 directories *****************BEFORE****************************

And here is the output after my changes:
******************AFTER**************************** [root@elm3b79 devices]# tree -d bus/system/devices/ bus/system/devices/

|-- cpu0 -> ../../../root/sys/node0/sys/cpu0
|-- cpu1 -> ../../../root/sys/node0/sys/cpu1
|-- cpu2 -> ../../../root/sys/node0/sys/cpu2
|-- cpu3 -> ../../../root/sys/node0/sys/cpu3
|-- cpu4 -> ../../../root/sys/node1/sys/cpu4
|-- cpu5 -> ../../../root/sys/node1/sys/cpu5
|-- cpu6 -> ../../../root/sys/node1/sys/cpu6
|-- cpu7 -> ../../../root/sys/node1/sys/cpu7
|-- memblk0 -> ../../../root/sys/node0/sys/memblk0
|-- memblk1 -> ../../../root/sys/node1/sys/memblk1
|-- node0 -> ../../../root/sys/node0
|-- node1 -> ../../../root/sys/node1
|-- pic0 -> ../../../root/sys/pic0

`-- rtc0 -> ../../../root/sys/rtc0

14 directories
[root@elm3b79 devices]# tree -d class/
class/

|-- cpu
|   |-- devices
|   |   |-- 0 -> ../../../root/sys/node0/sys/cpu0
|   |   |-- 1 -> ../../../root/sys/node0/sys/cpu1
|   |   |-- 2 -> ../../../root/sys/node0/sys/cpu2
|   |   |-- 3 -> ../../../root/sys/node0/sys/cpu3
|   |   |-- 4 -> ../../../root/sys/node1/sys/cpu4
|   |   |-- 5 -> ../../../root/sys/node1/sys/cpu5
|   |   |-- 6 -> ../../../root/sys/node1/sys/cpu6
|   |   `-- 7 -> ../../../root/sys/node1/sys/cpu7
|   `-- drivers
|-- disk
|   |-- devices
|   `-- drivers
|-- input
|   |-- devices
|   `-- drivers
|-- memblk
|   |-- devices
|   |   |-- 0 -> ../../../root/sys/node0/sys/memblk0
|   |   `-- 1 -> ../../../root/sys/node1/sys/memblk1
|   `-- drivers
`-- node
    |-- devices
    |   |-- 0 -> ../../../root/sys/node0
    |   `-- 1 -> ../../../root/sys/node1
    `-- drivers

27 directories
[root@elm3b79 devices]# tree -d root/sys/
root/sys/

|-- node0
|   `-- sys
|       |-- cpu0
|       |-- cpu1
|       |-- cpu2
|       |-- cpu3
|       `-- memblk0
|-- node1
|   `-- sys
|       |-- cpu4
|       |-- cpu5
|       |-- cpu6
|       |-- cpu7
|       `-- memblk1
|-- pic0

`-- rtc0

16 directories ******************AFTER****************************

Basically, the patch just adds nodes and memblks to the topology and nests the cpus/memblks under the nodes. I'd like to add more information to these directories (node-node distances, cpu speeds, memory block sizes/physical page ranges, etc, etc, etc), but this is just a first-pass.

Cheers!

-Matt

Matt,

I have some comments about the structure of the code, but those will come in another email..

Shouldn't nodes (or, erm, boards) be added as children of the root? Aren't all types of devices present on the various boards (PCI, etc)?

-pat

Cool... Any/all comments are definitely welcome...

Um, yes! I don't quite know how, though... I call sys_register_root() for each of the nodes... That call seems to parent them under the root/sys directory... How can I change that?

Yes... I have some patches that I'm working that will put PCI busses and devices into the topology infrastructure (both in-kernel & via driverfs). Again, this is just a first pass of what I'd like to see... ;)

Cheers!

-Matt

You'll have to wait prolly 1 more day, though. Sorry..

Woops. That sounds like my booboo. I'll look into that.

Sweet, that should be cool.

-pat

Ok, I'm finally getting back to you..

You didn't append the patch, which is annoying, but I'll deal..

The main problem I have is the code placement. I put the CPU stuff under arch/ because I anticpate wrapping the cpu structure with an arch-specific one, so you can ascertain arch-specific information via the generic structure. Moving it out of arch/ precludes that from happening (easily).

Ditto for memblks, though I'm not really sure what other info you'd want in the structures.

Ditto+ for nodes, or boards. Those are definitely arch-specific structures, and shouldn't be in drivers/base/. On top of that, I don't think their registration should all be munged together in one file. Maybe they should be, in their own play area (under arch/i386/mach-ccnuma/).

-pat