Skip to contentSkip to conversation

x86: spread tlb flush vector between nodes

PATCH In review Shaohua Li 4 participants 10 messages Oct 20, 2010

Subsystems (2)
lore
Share & tools
lore

Shaohua Li

author
#0 lore raw

PATCH 2/2 x86: spread tlb flush vector between nodes

Currently flush tlb vector allocation is based on below equation:
sender = smp_processor_id() % 8
This isn't optimal, CPUs from different node can have the same vector, this causes a lot of lock contention. Instead, we can assign the same vectors to CPUs from the same node, while different node has different vectors. This has below advantages:
a. if there is lock contention, the lock contention is between CPUs from one node. This should be much cheaper than the contention between nodes. b. completely avoid lock contention between nodes. This especially benefits kswapd, which is the biggest user of tlb flush, since kswapd sets its affinity to specific node.

In my test, this could reduce > 20% CPU overhead in extreme case.The test machine has 4 nodes and each node has 16 CPUs. I then bind each node's kswapd to the first CPU of the node. I run a workload with 4 sequential mmap file read thread. The files are empty sparse file. This workload will trigger a lot of page reclaim and tlbflush. The kswapd bind is to easy trigger the extreme tlb flush lock contention because otherwise kswapd keeps migrating between CPUs of a node and I can't get stable result. Sure in real workload, we can't always see so big tlb flush lock contention, but it's possible.

---
 arch/x86/mm/tlb.c |   48 +++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 47 insertions(+), 1 deletion(-)
arch/x86/mm/tlb.c +47 -1
Index: linux/arch/x86/mm/tlb.c
===================================================================
@@ -5,6 +5,7 @@
 #include <linux/smp.h>
 #include <linux/interrupt.h>
 #include <linux/module.h>
+#include <linux/cpu.h>
 
 #include <asm/tlbflush.h>
 #include <asm/mmu_context.h>
@@ -52,6 +53,8 @@ union smp_flush_state {
    want false sharing in the per cpu data segment. */
 static union smp_flush_state flush_state[NUM_INVALIDATE_TLB_VECTORS];
 
+static DEFINE_PER_CPU_READ_MOSTLY(int, tlb_vector_offset);
+
 /*
  * We cannot call mmdrop() because we are in interrupt context,
  * instead update mm->cpu_vm_mask.
@@ -173,7 +176,7 @@ static void flush_tlb_others_ipi(const s
 	union smp_flush_state *f;
 
 	/* Caller has disabled preemption */
-	sender = smp_processor_id() % NUM_INVALIDATE_TLB_VECTORS;
+	sender = per_cpu(tlb_vector_offset, smp_processor_id());
 	f = &flush_state[sender];
 
 	/*
@@ -218,6 +221,47 @@ void native_flush_tlb_others(const struc
 	flush_tlb_others_ipi(cpumask, mm, va);
 }
 
+static void __cpuinit calculate_tlb_offset(void)
+{
+	int cpu, node, nr_node_vecs;
+	/*
+	 * we are changing tlb_vector_offset for each CPU in runtime, but this
+	 * will not cause inconsistency, as the write is atomic under X86. we
+	 * might see more lock contentions in a short time, but after all CPU's
+	 * tlb_vector_offset are changed, everything should go normal
+	 *
+	 * Note: if NUM_INVALIDATE_TLB_VECTORS % nr_online_nodes !=0, we might
+	 * waste some vectors.
+	 **/
+	if (nr_online_nodes > NUM_INVALIDATE_TLB_VECTORS)
+		nr_node_vecs = 1;
+	else
+		nr_node_vecs = NUM_INVALIDATE_TLB_VECTORS/nr_online_nodes;
+
+	for_each_online_node(node) {
+		int node_offset = (node % NUM_INVALIDATE_TLB_VECTORS) *
+			nr_node_vecs;
+		int cpu_offset = 0;
+		for_each_cpu(cpu, cpumask_of_node(node)) {
+			per_cpu(tlb_vector_offset, cpu) = node_offset +
+				cpu_offset;
+			cpu_offset++;
+			cpu_offset = cpu_offset % nr_node_vecs;
+		}
+	}
+}
+
+static int tlb_cpuhp_notify(struct notifier_block *n,
+		unsigned long action, void *hcpu)
+{
+	switch (action & 0xf) {
+	case CPU_ONLINE:
+	case CPU_DEAD:
+		calculate_tlb_offset();
+	}
+	return NOTIFY_OK;
+}
+
 static int __cpuinit init_smp_flush(void)
 {
 	int i;
@@ -225,6 +269,8 @@ static int __cpuinit init_smp_flush(void
 	for (i = 0; i < ARRAY_SIZE(flush_state); i++)
 		raw_spin_lock_init(&flush_state[i].tlbstate_lock);
 
+	calculate_tlb_offset();
+	hotcpu_notifier(tlb_cpuhp_notify, 0);
 	return 0;
 }
 core_initcall(init_smp_flush);
#1 lore raw
Quoted 48 lines · Shaohua Li

Currently flush tlb vector allocation is based on below equation:
sender = smp_processor_id() % 8
This isn't optimal, CPUs from different node can have the same vector, this causes a lot of lock contention. Instead, we can assign the same vectors to CPUs from the same node, while different node has different vectors. This has below advantages:
a. if there is lock contention, the lock contention is between CPUs from one node. This should be much cheaper than the contention between nodes. b. completely avoid lock contention between nodes. This especially benefits kswapd, which is the biggest user of tlb flush, since kswapd sets its affinity to specific node.

In my test, this could reduce > 20% CPU overhead in extreme case.The test machine has 4 nodes and each node has 16 CPUs. I then bind each node's kswapd to the first CPU of the node. I run a workload with 4 sequential mmap file read thread. The files are empty sparse file. This workload will trigger a lot of page reclaim and tlbflush. The kswapd bind is to easy trigger the extreme tlb flush lock contention because otherwise kswapd keeps migrating between CPUs of a node and I can't get stable result. Sure in real workload, we can't always see so big tlb flush lock contention, but it's possible.

---
 arch/x86/mm/tlb.c |   48 +++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 47 insertions(+), 1 deletion(-)

Index: linux/arch/x86/mm/tlb.c
===================================================================

arch/x86/mm/tlb.c +3 -0
@@ -5,6 +5,7 @@
 #include <linux/smp.h>
 #include <linux/interrupt.h>
 #include <linux/module.h>
+#include <linux/cpu.h>
 
 #include <asm/tlbflush.h>
 #include <asm/mmu_context.h>
@@ -52,6 +53,8 @@ union smp_flush_state {
    want false sharing in the per cpu data segment. */
 static union smp_flush_state flush_state[NUM_INVALIDATE_TLB_VECTORS];
 
+static DEFINE_PER_CPU_READ_MOSTLY(int, tlb_vector_offset);
+
 /*
  * We cannot call mmdrop() because we are in interrupt context,
  * instead update mm->cpu_vm_mask.

arch/x86/mm/tlb.c static void flush_tlb_others_ipi(const s · line 179

Quoted 5 lines
arch/x86/mm/tlb.c +1 -1
@@ -173,7 +176,7 @@ static void flush_tlb_others_ipi(const s
 	union smp_flush_state *f;
 
 	/* Caller has disabled preemption */
-	sender = smp_processor_id() % NUM_INVALIDATE_TLB_VECTORS;
+	sender = per_cpu(tlb_vector_offset, smp_processor_id());
	sender = this_cpu_read(tlb_vector_offset);

arch/x86/mm/tlb.c void native_flush_tlb_others(const struc · line 223

Quoted 7 lines
arch/x86/mm/tlb.c +0 -0
 	f = &flush_state[sender];
 
 	/*
@@ -218,6 +221,47 @@ void native_flush_tlb_others(const struc
 	flush_tlb_others_ipi(cpumask, mm, va);
 }
 

Thats a pretty good patch, thanks !

Maybe we should have a per_node memory infrastructure, so that we can lower memory needs of currently per_cpu objects.

#2 lore raw
Quoted 2 lines

Maybe we should have a per_node memory infrastructure, so that we can lower memory needs of currently per_cpu objects.

I have been looking at that :- for a lot of things per core data makes sense too.

Really a lot of the per CPU scaling we have today should be per core or per node to avoid explosion.

But for this particular case it doesn't help because you still need a mapping for each CPU.

-Andi

Signature · 1 line
[email protected] -- Speaking for myself only.
maintainer
#3 lore raw
Quoted 2 lines · Andi Kleen

Really a lot of the per CPU scaling we have today should be per core or per node to avoid explosion.

Shouldn't that be per-cache instead of per-core?

#4 lore raw
Quoted 5 lines · Peter Zijlstra
Quoted 2 lines · Andi Kleen

Really a lot of the per CPU scaling we have today should be per core or per node to avoid explosion.

Shouldn't that be per-cache instead of per-core?

That's the same on modern x86:

per core = per L1/L2 cache
per node = per L3 cache

-Andi

Signature · 1 line
[email protected] -- Speaking for myself only.
maintainer
#5 lore raw
Quoted 8 lines · Andi Kleen
Quoted 5 lines · Peter Zijlstra
Quoted 2 lines · Andi Kleen

Really a lot of the per CPU scaling we have today should be per core or per node to avoid explosion.

Shouldn't that be per-cache instead of per-core?

That's the same on modern x86:

Last time I checked there's more than 1 directory in arch/

#6 lore raw
Quoted 11 lines · Peter Zijlstra
Quoted 8 lines · Andi Kleen
Quoted 5 lines · Peter Zijlstra
Quoted 2 lines · Andi Kleen

Really a lot of the per CPU scaling we have today should be per core or per node to avoid explosion.

Shouldn't that be per-cache instead of per-core?

That's the same on modern x86:

Last time I checked there's more than 1 directory in arch/

Not sure what your point is?

I believe non x86 server processors have similar cache layouts as the one I described, occasionally with another cache level, and should do well with a similar setup.

For non server it typically doesn't matter too much because there are not enough cores.

-Andi

Signature · 1 line
[email protected] -- Speaking for myself only.
#7 lore raw
In reply to Shaohua Li

Hi Shaohua,

arch/x86/mm/tlb.c void native_flush_tlb_others(const struc · line 239

Quoted 4 lines
arch/x86/mm/tlb.c +4 -0
+	if (nr_online_nodes > NUM_INVALIDATE_TLB_VECTORS)
+		nr_node_vecs = 1;
+	else
+		nr_node_vecs = NUM_INVALIDATE_TLB_VECTORS/nr_online_nodes;

Does this build without CONFIG_NUMA? AFAIK nr_online_nodes is only defined for a numa kernel.

arch/x86/mm/tlb.c void native_flush_tlb_others(const struc · line 260

Quoted 8 lines
arch/x86/mm/tlb.c +8 -0
+
+static int tlb_cpuhp_notify(struct notifier_block *n,
+		unsigned long action, void *hcpu)
+{
+	switch (action & 0xf) {
+	case CPU_ONLINE:
+	case CPU_DEAD:
+		calculate_tlb_offset();

I still think the notifier is overkill and a static mapping at boot time would be fine.

The rest looks ok to me.

-andi

author
#8 lore raw
Quoted 9 lines · Andi Kleen

Hi Shaohua,

arch/x86/mm/tlb.c void native_flush_tlb_others(const struc · line 239

Quoted 4 lines
arch/x86/mm/tlb.c +4 -0
+	if (nr_online_nodes > NUM_INVALIDATE_TLB_VECTORS)
+		nr_node_vecs = 1;
+	else
+		nr_node_vecs = NUM_INVALIDATE_TLB_VECTORS/nr_online_nodes;

Does this build without CONFIG_NUMA? AFAIK nr_online_nodes is only defined for a numa kernel.

yes it's ok without CONFIG_NUMA

Quoted 11 lines

arch/x86/mm/tlb.c void native_flush_tlb_others(const struc · line 260

Quoted 8 lines
arch/x86/mm/tlb.c +8 -0
+
+static int tlb_cpuhp_notify(struct notifier_block *n,
+		unsigned long action, void *hcpu)
+{
+	switch (action & 0xf) {
+	case CPU_ONLINE:
+	case CPU_DEAD:
+		calculate_tlb_offset();

I still think the notifier is overkill and a static mapping at boot time would be fine.

this hasn't much overhead, so please keep it. I'm afraid static mapping calculation for hotplug cpu will introduce more complex.

#9 lore raw

[tip:x86/mm] x86: Spread tlb flush vector between nodes

In reply to Shaohua Li
Commit-ID:  932967202182743c01a2eee4bdfa2c42697bc586
Gitweb:     http://git.kernel.org/tip/932967202182743c01a2eee4bdfa2c42697bc586
Author:     Shaohua Li <[email protected]>
AuthorDate: Wed, 20 Oct 2010 11:07:03 +0800
Committer:  H. Peter Anvin <[email protected]>
CommitDate: Wed, 20 Oct 2010 14:44:42 -0700

x86: Spread tlb flush vector between nodes

Currently flush tlb vector allocation is based on below equation:
sender = smp_processor_id() % 8
This isn't optimal, CPUs from different node can have the same vector, this causes a lot of lock contention. Instead, we can assign the same vectors to CPUs from the same node, while different node has different vectors. This has below advantages:
a. if there is lock contention, the lock contention is between CPUs from one node. This should be much cheaper than the contention between nodes. b. completely avoid lock contention between nodes. This especially benefits kswapd, which is the biggest user of tlb flush, since kswapd sets its affinity to specific node.

In my test, this could reduce > 20% CPU overhead in extreme case.The test machine has 4 nodes and each node has 16 CPUs. I then bind each node's kswapd to the first CPU of the node. I run a workload with 4 sequential mmap file read thread. The files are empty sparse file. This workload will trigger a lot of page reclaim and tlbflush. The kswapd bind is to easy trigger the extreme tlb flush lock contention because otherwise kswapd keeps migrating between CPUs of a node and I can't get stable result. Sure in real workload, we can't always see so big tlb flush lock contention, but it's possible.

[ hpa: folded in fix from Eric Dumazet to use this_cpu_read() ]

LKML-Reference: <[email protected]>

---
 arch/x86/mm/tlb.c |   48 +++++++++++++++++++++++++++++++++++++++++++++++-
 1 files changed, 47 insertions(+), 1 deletions(-)
arch/x86/mm/tlb.c +47 -1
index c03f14a..4935848 100644
@@ -5,6 +5,7 @@
 #include <linux/smp.h>
 #include <linux/interrupt.h>
 #include <linux/module.h>
+#include <linux/cpu.h>
 
 #include <asm/tlbflush.h>
 #include <asm/mmu_context.h>
@@ -52,6 +53,8 @@ union smp_flush_state {
    want false sharing in the per cpu data segment. */
 static union smp_flush_state flush_state[NUM_INVALIDATE_TLB_VECTORS];
 
+static DEFINE_PER_CPU_READ_MOSTLY(int, tlb_vector_offset);
+
 /*
  * We cannot call mmdrop() because we are in interrupt context,
  * instead update mm->cpu_vm_mask.
@@ -173,7 +176,7 @@ static void flush_tlb_others_ipi(const struct cpumask *cpumask,
 	union smp_flush_state *f;
 
 	/* Caller has disabled preemption */
-	sender = smp_processor_id() % NUM_INVALIDATE_TLB_VECTORS;
+	sender = this_cpu_read(tlb_vector_offset);
 	f = &flush_state[sender];
 
 	/*
@@ -218,6 +221,47 @@ void native_flush_tlb_others(const struct cpumask *cpumask,
 	flush_tlb_others_ipi(cpumask, mm, va);
 }
 
+static void __cpuinit calculate_tlb_offset(void)
+{
+	int cpu, node, nr_node_vecs;
+	/*
+	 * we are changing tlb_vector_offset for each CPU in runtime, but this
+	 * will not cause inconsistency, as the write is atomic under X86. we
+	 * might see more lock contentions in a short time, but after all CPU's
+	 * tlb_vector_offset are changed, everything should go normal
+	 *
+	 * Note: if NUM_INVALIDATE_TLB_VECTORS % nr_online_nodes !=0, we might
+	 * waste some vectors.
+	 **/
+	if (nr_online_nodes > NUM_INVALIDATE_TLB_VECTORS)
+		nr_node_vecs = 1;
+	else
+		nr_node_vecs = NUM_INVALIDATE_TLB_VECTORS/nr_online_nodes;
+
+	for_each_online_node(node) {
+		int node_offset = (node % NUM_INVALIDATE_TLB_VECTORS) *
+			nr_node_vecs;
+		int cpu_offset = 0;
+		for_each_cpu(cpu, cpumask_of_node(node)) {
+			per_cpu(tlb_vector_offset, cpu) = node_offset +
+				cpu_offset;
+			cpu_offset++;
+			cpu_offset = cpu_offset % nr_node_vecs;
+		}
+	}
+}
+
+static int tlb_cpuhp_notify(struct notifier_block *n,
+		unsigned long action, void *hcpu)
+{
+	switch (action & 0xf) {
+	case CPU_ONLINE:
+	case CPU_DEAD:
+		calculate_tlb_offset();
+	}
+	return NOTIFY_OK;
+}
+
 static int __cpuinit init_smp_flush(void)
 {
 	int i;
@@ -225,6 +269,8 @@ static int __cpuinit init_smp_flush(void)
 	for (i = 0; i < ARRAY_SIZE(flush_state); i++)
 		raw_spin_lock_init(&flush_state[i].tlbstate_lock);
 
+	calculate_tlb_offset();
+	hotcpu_notifier(tlb_cpuhp_notify, 0);
 	return 0;
 }
 core_initcall(init_smp_flush);