Skip to contentSkip to conversation

move mmu_gathers into local_cpu_data

PATCH In review Martin Hicks 3 participants 4 messages Apr 22, 2003 – May 15, 2003

Subsystems (2)
Generic include/asm header files MMU gather and TLB invalidation
lore
Share & tools
lore

Martin Hicks

author
#0 lore raw

PATCH 1/1 move mmu_gathers into local_cpu_data

Hi,

This is a patch that moves mmu_gathers into local_cpu_data on IA64 NUMA. I'm not exactly sure where to push this patch, because it does touch include/asm-generic/tlb.h although it has no effect on non ia64 arches.

The main reason for making this change is that the mmu_gathers array gets very large as the number of processors increases.

mh

Signature · 83 lines
Wild Open Source Inc.                  [email protected]


diff -X /home/mort/diffex -uNr linux-2.4.21-pre5.pristine/arch/ia64/kernel/setup.c linux-2.4.21-pre5/arch/ia64/kernel/setup.c
--- linux-2.4.21-pre5.pristine/arch/ia64/kernel/setup.c	Sun Mar 30 17:31:07 2003
+++ linux-2.4.21-pre5/arch/ia64/kernel/setup.c	Tue Apr 22 13:22:48 2003
@@ -566,6 +566,8 @@
 		for (cpu = 1; cpu < NR_CPUS; ++cpu)
 			memcpy(my_cpu_data->cpu_data[cpu]->cpu_data,
 			       my_cpu_data->cpu_data, sizeof(my_cpu_data->cpu_data));
+		my_cpu_data->mmu_gathers = alloc_bootmem_pages_node(BOOT_NODE_DATA(boot_get_local_cnodeid()),
+								    sizeof(mmu_gather_t));
 	} else {
 		order = get_order(sizeof(struct cpuinfo_ia64));
 		my_cpu_data = page_address(alloc_pages_node(numa_node_id(), GFP_KERNEL, order));
@@ -575,6 +577,10 @@
 			     order);
 		for (cpu = 0; cpu < NR_CPUS; ++cpu)
 			boot_cpu_data->cpu_data[cpu]->cpu_data[smp_processor_id()] = my_cpu_data;
+
+		my_cpu_data->mmu_gathers = page_address(boot_alloc_pages_node(boot_get_local_cnodeid(),
+									      GFP_KERNEL,
+									      get_order(sizeof(mmu_gather_t)));
 	}
 #else
 	my_cpu_data = cpu_data(smp_processor_id());
diff -X /home/mort/diffex -uNr linux-2.4.21-pre5.pristine/arch/ia64/mm/init.c linux-2.4.21-pre5/arch/ia64/mm/init.c
--- linux-2.4.21-pre5.pristine/arch/ia64/mm/init.c	Sun Mar 30 17:31:07 2003
+++ linux-2.4.21-pre5/arch/ia64/mm/init.c	Tue Apr 22 13:22:48 2003
@@ -28,7 +28,9 @@
 #include <asm/uaccess.h>
 #include <asm/tlb.h>
 
+#ifndef CONFIG_NUMA
 mmu_gather_t mmu_gathers[NR_CPUS];
+#endif
 
 /* References to section boundaries: */
 extern char _stext, _etext, _edata, __init_begin, __init_end;
diff -X /home/mort/diffex -uNr linux-2.4.21-pre5.pristine/include/asm-generic/tlb.h linux-2.4.21-pre5/include/asm-generic/tlb.h
--- linux-2.4.21-pre5.pristine/include/asm-generic/tlb.h	Fri Aug  2 18:39:45 2002
+++ linux-2.4.21-pre5/include/asm-generic/tlb.h	Tue Apr 22 13:24:36 2003
@@ -31,15 +31,18 @@
 	pte_t	ptes[FREE_PTE_NR];
 } mmu_gather_t;
 
+#ifndef local_mmu_gathers
 /* Users of the generic TLB shootdown code must declare this storage space. */
 extern mmu_gather_t	mmu_gathers[NR_CPUS];
+#define local_mmu_gathers	&mmu_gathers[smp_processor_id()]
+#endif
 
 /* tlb_gather_mmu
  *	Return a pointer to an initialized mmu_gather_t.
  */
 static inline mmu_gather_t *tlb_gather_mmu(struct mm_struct *mm)
 {
-	mmu_gather_t *tlb = &mmu_gathers[smp_processor_id()];
+	mmu_gather_t *tlb = local_mmu_gathers;
 
 	tlb->mm = mm;
 	/* Use fast mode if there is only one user of this mm (this process) */
diff -X /home/mort/diffex -uNr linux-2.4.21-pre5.pristine/include/asm-ia64/processor.h linux-2.4.21-pre5/include/asm-ia64/processor.h
--- linux-2.4.21-pre5.pristine/include/asm-ia64/processor.h	Wed Apr 16 17:41:04 2003
+++ linux-2.4.21-pre5/include/asm-ia64/processor.h	Tue Apr 22 13:38:58 2003
@@ -189,6 +189,7 @@
 	void *node_directory;
 	int numa_node_id;
 	struct cpuinfo_ia64 *cpu_data[NR_CPUS];
+        void *mmu_gathers;
 #endif
 	/* Platform specific word.  MUST BE LAST IN STRUCT */
 	__u64 platform_specific;
diff -X /home/mort/diffex -uNr linux-2.4.21-pre5.pristine/include/asm-ia64/tlb.h linux-2.4.21-pre5/include/asm-ia64/tlb.h
--- linux-2.4.21-pre5.pristine/include/asm-ia64/tlb.h	Fri Apr 11 17:56:37 2003
+++ linux-2.4.21-pre5/include/asm-ia64/tlb.h	Tue Apr 22 13:38:59 2003
@@ -1 +1,6 @@
+
+#ifdef CONFIG_NUMA
+#define local_mmu_gathers local_cpu_data->mmu_gathers
+#endif
+
 #include <asm-generic/tlb.h>
#1 lore raw
Quoted 9 lines · Martin Hicks

Hi,

This is a patch that moves mmu_gathers into local_cpu_data on IA64 NUMA. I'm not exactly sure where to push this patch, because it does touch include/asm-generic/tlb.h although it has no effect on non ia64 arches.

The main reason for making this change is that the mmu_gathers array gets very large as the number of processors increases.

For 2.5 please just make mmu_gathers per-cpu data for all arches.

author
#2 lore raw
In reply to Martin Hicks
Quoted 4 lines · Christoph Hellwig
Quoted 3 lines · Martin Hicks

Hi,

Quoted 9 lines
Quoted 6 lines

This is a patch that moves mmu_gathers into local_cpu_data on IA64 NUMA. I'm not exactly sure where to push this patch, because it does touch include/asm-generic/tlb.h although it has no effect on non ia64 arches.

The main reason for making this change is that the mmu_gathers array gets very large as the number of processors increases.

For 2.5 please just make mmu_gathers per-cpu data for all arches.

That was my intention. For 2.4 my goal was "touch as little as possible, and hopefully it will get accepted".

mh

Signature · 1 line
Wild Open Source Inc.                  [email protected]
#3 lore raw
In reply to Martin Hicks
Quoted 3 lines · Martin Hicks

This is a patch that moves mmu_gathers into local_cpu_data on IA64 NUMA. I'm not exactly sure where to push this patch, because it does touch include/asm-generic/tlb.h although it has no effect on non ia64 arches.

I applied this for 2.4, with a couple minor changes:
- moved the static mmu_gathers[] declaration to setup.c, so all the mmu_gather stuff is together
- moved the cpuinfo change outside CONFIG_NUMA, so we can just use the same local_mmu_gathers #define all the time

I'd actually like to see the dynamic mmu_gather allocation done in the non-NUMA case as well. Non-NUMA SMP kernels are statically allocating 128K (NR_CPUS = 32, 4K per mmu_gather_t), but typical machines will only use 8K or 16K of that.

Here's the actual patch I applied.

#### AUTHOR [email protected]
#### COMMENT START
### Comments for ChangeSet
Move mmu_gathers[] to local_cpu_data on ia64.

This is a patch that moves mmu_gathers into local_cpu_data on IA64 NUMA. I'm not exactly sure where to push this patch, because it does touch include/asm-generic/tlb.h although it has no effect on non ia64 arches.

The main reason for making this change is that the mmu_gathers array gets very large as the number of processors increases.
### Comments for include/asm-generic/tlb.h
(local_mmu_gathers): Use if defined.
### Comments for include/asm-ia64/tlb.h
(local_mmu_gathers): Define.
### Comments for arch/ia64/kernel/setup.c
(mmu_gathers[]): Allocate statically for non-NUMA, per present CPU for NUMA. ### Comments for arch/ia64/mm/init.c
(mmu_gathers[]): move to kernel/setup.c alongside NUMA allocation. ### Comments for include/asm-ia64/processor.h
(mmu_gathers): Add to cpuinfo_ia64.
#### COMMENT END

# This is a BitKeeper generated patch for the following project:
# Project Name: Linux kernel tree
# This patch format is intended for GNU patch command version 2.5 or higher. # This patch includes the following deltas:
# ChangeSet 1.1145 -> 1.1146
# include/asm-generic/tlb.h 1.2 -> 1.3
# arch/ia64/kernel/setup.c 1.11 -> 1.12
# include/asm-ia64/processor.h 1.16 -> 1.17
# include/asm-ia64/tlb.h 1.1 -> 1.2
# arch/ia64/mm/init.c 1.11 -> 1.12
#
# The following is the BitKeeper ChangeSet Log
# --------------------------------------------
# 03/05/15 [email protected] 1.1146
# Move mmu_gathers into per cpu area.
# --------------------------------------------
#

arch/ia64/kernel/setup.c +9 -0
diff -Nru a/arch/ia64/kernel/setup.c b/arch/ia64/kernel/setup.c
@@ -40,6 +40,7 @@
 #include <asm/system.h>
 #include <asm/mca.h>
 #include <asm/smp.h>
+#include <asm/tlb.h>
 
 #ifdef CONFIG_BLK_DEV_RAM
 # include <linux/blk.h>

@@ -58,6 +59,7 @@
  struct cpuinfo_ia64 *boot_cpu_data;
 #else
  struct cpuinfo_ia64 _cpu_data[NR_CPUS] __attribute__ ((section ("__special_page_section")));
+ mmu_gather_t mmu_gathers[NR_CPUS];
 #endif
 
 unsigned long ia64_cycles_per_usec;

@@ -567,6 +569,8 @@
 		for (cpu = 1; cpu < NR_CPUS; ++cpu)
 			memcpy(my_cpu_data->cpu_data[cpu]->cpu_data,
 			       my_cpu_data->cpu_data, sizeof(my_cpu_data->cpu_data));
+		my_cpu_data->mmu_gathers = alloc_bootmem_pages_node(BOOT_NODE_DATA(boot_get_local_cnodeid()),
+								    sizeof(mmu_gather_t));
 	} else {
 		order = get_order(sizeof(struct cpuinfo_ia64));
 		my_cpu_data = page_address(alloc_pages_node(numa_node_id(), GFP_KERNEL, order));

@@ -576,9 +580,14 @@
 			     order);
 		for (cpu = 0; cpu < NR_CPUS; ++cpu)
 			boot_cpu_data->cpu_data[cpu]->cpu_data[smp_processor_id()] = my_cpu_data;
+
+		my_cpu_data->mmu_gathers = page_address(boot_alloc_pages_node(boot_get_local_cnodeid(),
+									      GFP_KERNEL,
+									      get_order(sizeof(mmu_gather_t)));
 	}
 #else
 	my_cpu_data = cpu_data(smp_processor_id());
+	my_cpu_data->mmu_gathers = &mmu_gathers[smp_processor_id()];
 #endif
 
 	/*
arch/ia64/mm/init.c +0 -3
diff -Nru a/arch/ia64/mm/init.c b/arch/ia64/mm/init.c
@@ -26,9 +26,6 @@
 #include <asm/sal.h>
 #include <asm/system.h>
 #include <asm/uaccess.h>
-#include <asm/tlb.h>
-
-mmu_gather_t mmu_gathers[NR_CPUS];
 
 /* References to section boundaries: */
 extern char _stext, _etext, _edata, __init_begin, __init_end;
include/asm-generic/tlb.h +4 -1
diff -Nru a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
@@ -31,15 +31,18 @@
 	pte_t	ptes[FREE_PTE_NR];
 } mmu_gather_t;
 
+#ifndef local_mmu_gathers
 /* Users of the generic TLB shootdown code must declare this storage space. */
 extern mmu_gather_t	mmu_gathers[NR_CPUS];
+#define local_mmu_gathers	&mmu_gathers[smp_processor_id()]
+#endif
 
 /* tlb_gather_mmu
  *	Return a pointer to an initialized mmu_gather_t.
  */
 static inline mmu_gather_t *tlb_gather_mmu(struct mm_struct *mm)
 {
-	mmu_gather_t *tlb = &mmu_gathers[smp_processor_id()];
+	mmu_gather_t *tlb = local_mmu_gathers;
 
 	tlb->mm = mm;
 	/* Use fast mode if there is only one user of this mm (this process) */
include/asm-ia64/processor.h +1 -0
diff -Nru a/include/asm-ia64/processor.h b/include/asm-ia64/processor.h
@@ -168,6 +168,7 @@
 	__u32 ptce_count[2];
 	__u32 ptce_stride[2];
 	struct task_struct *ksoftirqd;	/* kernel softirq daemon for this CPU */
+	void *mmu_gathers;
 # ifdef CONFIG_PERFMON
 	unsigned long pfm_syst_info;
 # endif
include/asm-ia64/tlb.h +2 -0
diff -Nru a/include/asm-ia64/tlb.h b/include/asm-ia64/tlb.h
@@ -1 +1,3 @@
+#define local_mmu_gathers local_cpu_data->mmu_gathers
+
 #include <asm-generic/tlb.h>