Allow global purge traslation cache (ptc.g) to be disabled

From: Natalie Protasevich <[email protected]>

This patch allows to disable ptc.g. The code used to be in the kernel, then was removed in 2.4 since the bug that it was fixing has gone away. However, some large system vendors now want this capability available through a means that can be controlled by the platform in the event that there is an issue with either processor or their chipset where global ptc.g is not operational. They want the mechanism for future platforms to work around such issues. It is also needed for platform makers when they deliberately do not use the global cache purge in their chipset implementation. (For such cases, Intel provided a SAL table entry to specify if ptc.g is allowed and how many).

---

 arch/ia64/kernel/smp.c |   49 ++++++++++++++++++++++++++++++++
 arch/ia64/mm/tlb.c     |   75 
++++++++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 123 insertions(+), 1 deletion(-)

diff -puN arch/ia64/mm/tlb.c~ptcg arch/ia64/mm/tlb.c
--- linux-2.6.23-rc4/arch/ia64/mm/tlb.c~ptcg 2007-08-30
11:58:40.000000000 -0700
+++ linux-2.6.23-rc4-nataliep/arch/ia64/mm/tlb.c 2007-08-30
11:58:40.000000000 -0700

(quoted diff) +68 -0

@@ -18,6 +18,7 @@
 #include <linux/smp.h>
 #include <linux/mm.h>
 #include <linux/bootmem.h>
+#include <linux/irq.h>
 
 #include <asm/delay.h>
 #include <asm/mmu_context.h>

@@ -83,6 +84,73 @@ wrap_mmu_context (struct mm_struct *mm)
 	local_flush_tlb_all();
 }
 
+static int noptcg;
+
+static void __init noptcg_setup(char *str)
+{
+	noptcg = 1;
+}
+__setup("noptcg", noptcg_setup);
+
+unsigned long	flush_end, flush_start, flush_nbits, flush_rid;
+atomic_t flush_cpu_count;
+
+/*
+ * flush_tlb_no_ptcg is called with ptcg_lock locked
+ */
+static inline void
+flush_tlb_no_ptcg (unsigned long start, unsigned long end, unsigned long 
 nbits)
+{
+	extern void smp_send_flush_tlb (void);
+	unsigned long saved_tpr = 0;
+	unsigned long flags;
+	int cpus = num_online_cpus();
+
+	/*
+	 * Sometimes this is called with interrupts disabled and causes
+	 * deadlock; to avoid this we enable interrupt and raise the TPR
+	 * to enable ONLY IPI.
+	 */
+	local_save_flags(flags);
+	if (!(flags & IA64_PSR_I)) {
+		saved_tpr = ia64_getreg(_IA64_REG_CR_TPR);
+		ia64_srlz_d();
+		ia64_setreg(_IA64_REG_CR_TPR, saved_tpr);
+		ia64_srlz_d();
+		local_irq_enable();
+	}
+
+	flush_rid = ia64_get_rr(start);
+	ia64_srlz_d();
+	flush_start = start;
+	flush_end = end;
+	flush_nbits = nbits;
+	atomic_set(&flush_cpu_count, cpus - 1);
+	smp_send_flush_tlb();
+	/*
+	 * Purge local TLB entries. ALAT invalidation is done in ia64_leave_kernel.
+	 */
+	do {
+		asm volatile ("ptc.l %0,%1" :: "r"(start), "r"(nbits<<2) : "memory");
+		start += (1UL << nbits);
+	} while (start < end);
+
+	ia64_srlz_i();			/* srlz.i implies srlz.d */
+
+	/*
+	 * Wait for other CPUs to finish purging entries.
+	 */
+	while (atomic_read(&flush_cpu_count)) {
+		/* Nothing */
+	}
+
+	if (!(flags & IA64_PSR_I)) {
+		local_irq_disable();
+	 	ia64_setreg(_IA64_REG_CR_TPR, saved_tpr);
+		ia64_srlz_d();
+	}
+}
+
 void
 ia64_global_tlb_purge (struct mm_struct *mm, unsigned long start,

(quoted diff) +6 -1

 		       unsigned long end, unsigned long nbits)

@@ -96,7 +164,9 @@ ia64_global_tlb_purge (struct mm_struct 
 
 	/* HW requires global serialization of ptc.ga.  */
 	spin_lock(&ptcg_lock);
-	{
+	if (noptcg)
+		flush_tlb_no_ptcg(start, end, nbits);
+	else {
 		do {
 			/*
 			 * Flush ALAT entries also.

@@ -109,6 +179,9 @@ ia64_global_tlb_purge (struct mm_struct 
 	spin_unlock(&ptcg_lock);
 }
 
+unsigned long	flush_end, flush_start, flush_nbits, flush_rid;
+atomic_t flush_cpu_count;
+
 void
 local_flush_tlb_all (void)
 {

diff -puN arch/ia64/kernel/smp.c~ptcg arch/ia64/kernel/smp.c
--- linux-2.6.23-rc4/arch/ia64/kernel/smp.c~ptcg 2007-08-30
11:58:40.000000000 -0700
+++ linux-2.6.23-rc4-nataliep/arch/ia64/kernel/smp.c 2007-08-30
11:58:40.000000000 -0700

(quoted diff) +1 -0

@@ -79,6 +79,7 @@ static volatile struct call_data_struct 
 
 #define IPI_CALL_FUNC		0
 #define IPI_CPU_STOP		1
+#define IPI_FLUSH_TLB		2
 #define IPI_KDUMP_CPU_STOP	3
 
 /* This needs to be cacheline aligned because it is written to by *other* 

CPUs. */

(quoted diff) +48 -0

@@ -174,6 +175,47 @@ handle_IPI (int irq, void *dev_id)
 				unw_init_running(kdump_cpu_freeze, NULL);
 				break;
 #endif
+
+			      case IPI_FLUSH_TLB:
+			      {
+				extern unsigned long flush_start, flush_end, flush_nbits, flush_rid;
+				extern atomic_t flush_cpu_count;
+				unsigned long saved_rid = ia64_get_rr(flush_start);
+				unsigned long end = flush_end;
+				unsigned long start = flush_start;
+				unsigned long nbits = flush_nbits;
+	
+				/*
+				 * Current CPU may be running with different RID so we need to
+				 * reload the RID of flushed address.  Purging the translation
+				 * also needs ALAT invalidation; we do not need "invala" here
+				 * since it is done in ia64_leave_kernel.
+				 */
+				ia64_srlz_d();
+				if (saved_rid != flush_rid) {
+					ia64_set_rr(flush_start, flush_rid);
+					ia64_srlz_d();
+				}
+
+				do {
+					/*
+					 * Purge local TLB entries.
+				 	 */
+					__asm__ __volatile__ ("ptc.l %0,%1" ::
+							      "r"(start), "r"(nbits<<2) : "memory");
+					start += (1UL << nbits);
+				} while (start < end);
+
+				ia64_barrier();
+				ia64_srlz_i();			/* srlz.i implies srlz.d */
+
+				if (saved_rid != flush_rid) {
+					ia64_set_rr(flush_start, saved_rid);
+					ia64_srlz_d();
+				}
+				atomic_dec(&flush_cpu_count);
+				break;
+			      }
 			      default:
 				printk(KERN_CRIT "Unknown IPI on CPU %d: %lu\n", this_cpu, which);
 				break;

@@ -317,6 +359,13 @@ smp_flush_tlb_cpumask(cpumask_t xcpumask
 }
 
 void
+smp_send_flush_tlb (void)
+{
+	send_IPI_allbutself(IPI_FLUSH_TLB);
+}
+
+
+void
 smp_flush_tlb_all (void)
 {
 	on_each_cpu((void (*)(void *))local_flush_tlb_all, NULL, 1, 1);

_
-
To unsubscribe from this list: send the line "unsubscribe linux-ia64" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Please consider the scenario when many cpus are calling ia64_global_tlb_purge at the same time without any lock. Using global variables to indicate tlb flush start and end is not safe here.

Quoted 4 lines

Please consider the scenario when many cpus are calling ia64_global_tlb_purge at the same time without any lock. Using global variables to indicate tlb flush start and end is not safe here.

Isn't everything protected by the

	spin_lock(&ptcg_lock);
	 ...
	spin_unlock(&ptcg_lock);

in ia64_global_tlb_purge()? Though the comment that this lock is required for HW serialization would need to be updated to note that the s/w implementation also requires this serialization.

-Tony

Quoted 11 lines · Tony

Quoted 4 lines

Please consider the scenario when many cpus are calling ia64_global_tlb_purge at the same time without any lock. Using global variables to indicate tlb flush start and end is not safe here.

Isn't everything protected by the

      spin_lock(&ptcg_lock);
       ...
      spin_unlock(&ptcg_lock);

Quoted 5 lines

in ia64_global_tlb_purge()? Though the comment that this lock is required for HW serialization would need to be updated to note that the s/w implementation also requires this serialization.

-Tony

Ah Yes, I missed this line when reading the patch...

So my comments would be
1. avoid declare global variables and functions in .c file.
2. use more precise name for global variable.to e.g ia64_global_tlb_flush_start is better than flush_start.
3. do not use asm in .c file, use intrins instead.

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

"Natalie" = Natalie Protasevich <[email protected]> writes:

From: Natalie Protasevich <[email protected]>

This patch allows to disable ptc.g. The code used to be in the kernel, then was removed in 2.4 since the bug that it was fixing has gone away. However, some large system vendors now want this capability available through a means that can be controlled by the platform in the event that there is an issue with either processor or their chipset where global ptc.g is not operational. They want the mechanism for future platforms to work around such issues. It is also needed for platform makers when they deliberately do not use the global cache purge in their chipset implementation. (For such cases, Intel provided a SAL table entry to specify if ptc.g is allowed and how many).

global cache purge in their chipset implementation. (For such cases, Intel provided a SAL table entry to specify if ptc.g is allowed and how many).

Quoted 2 lines

global cache purge in their chipset implementation. (For such cases, Intel provided a SAL table entry to specify if ptc.g is allowed and how many).

This seems odd. You never use that sal call to initialized noptcg. Is that an oversight?

Quoted 5 lines

Quoted 2 lines

global cache purge in their chipset implementation. (For such cases, Intel provided a SAL table entry to specify if ptc.g is allowed and how many).

This seems odd. You never use that sal call to initialized noptcg. Is that an oversight?

The mechanism is a SAL table entry, not a SAL call. Currently that entry provides no mechanism to specify that ptc.g should not be used at all (the entry provides the count of how many ptc.g can happen in parallel, but the spec says that the value "0" means "1" :-( )

There is an ongoing discussion with DIG64 to use a currently reserved field in the table to specify the value "0". If that change is approved, then we can add code to enable Natalie's code based on the SAL table.

-Tony

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

"Natalie" = Natalie Protasevich <[email protected]> writes:

Natalie> From: Natalie Protasevich <[email protected]>

Natalie> This patch allows to disable ptc.g. The code used to be in Natalie> the kernel, then was removed in 2.4 since the bug that it was Natalie> fixing has gone away. However, some large system vendors now Natalie> want this capability available through a means that can be Natalie> controlled by the platform in the event that there is an Natalie> issue with either processor or their chipset where global Natalie> ptc.g is not operational. They want the mechanism for future Natalie> platforms to work around such issues. It is also needed for Natalie> platform makers when they deliberately do not use the global Natalie> cache purge in their chipset implementation. (For such cases, Natalie> Intel provided a SAL table entry to specify if ptc.g is Natalie> allowed and how many).

If you need a special TLB purge implementation, please use the machine vector as we do on the sn2 architecture - there is no reason to implement a boot time option for this.

Who are these large system vendors who want this patch in? I presume it's not the people running gmail? :-)

Cheers,
Jes

This patch allows to disable ptc.g. The code used to be in the kernel, then was removed in 2.4 since the bug that it was fixing has gone away. However, some large system vendors now want this capability available through a means that can be controlled by the platform in the event that there is an issue with either processor or their chipset where global ptc.g is not operational. They want the mechanism for future platforms to work around such issues. It is also needed for platform makers when they deliberately do not use the global cache purge in their chipset implementation. (For such cases, Intel provided a SAL table entry to specify if ptc.g is allowed and how many).

(quoted diff) +29 -0

+	ia64_global_tlb_flush_start = start;
+	ia64_global_tlb_flush_end = end;
+	ia64_global_tlb_flush_nbits = nbits;
+	atomic_set(&ia64_global_tlb_flush_cpu_count, cpus - 1);
+	smp_send_flush_tlb();
+	/*
+	 * Purge local TLB entries. ALAT invalidation is done in ia64_leave_kernel.
+	 */
+	do {
+		ia64_ptcl(start, nbits<<2);
+		start += (1UL << nbits);
+	} while (start < end);
+
+	ia64_srlz_i();			/* srlz.i implies srlz.d */
+
+	/*
+	 * Wait for other CPUs to finish purging entries.
+	 */
+	while (atomic_read(&ia64_global_tlb_flush_cpu_count)) {
+		/* Nothing */
+	}
+
+	if (!(flags & IA64_PSR_I)) {
+		local_irq_disable();
+	 	ia64_setreg(_IA64_REG_CR_TPR, saved_tpr);
+		ia64_srlz_d();
+	}
+}
+
 void
 ia64_global_tlb_purge (struct mm_struct *mm, unsigned long start,
 		       unsigned long end, unsigned long nbits)

diff -puN arch/ia64/kernel/smp.c~ptcg arch/ia64/kernel/smp.c

arch/ia64/kernel/smp.c renamed from arch/ia64/kernel/smp.c~ptcg +9 -0

@@ -174,6 +175,48 @@ handle_IPI (int irq, void *dev_id)
 				unw_init_running(kdump_cpu_freeze, NULL);
 				break;
 #endif
+
+			      case IPI_FLUSH_TLB:
+			      {
+				extern unsigned long ia64_global_tlb_flush_start, 
+				       ia64_global_tlb_flush_end, ia64_global_tlb_flush_nbits, 
+				       ia64_global_tlb_flush_rid;
+				extern atomic_t ia64_global_tlb_flush_cpu_count;
+				unsigned long saved_rid = ia64_get_rr(ia64_global_tlb_flush_start);
+				unsigned long end = ia64_global_tlb_flush_end;

arch/ia64/kernel/smp.c renamed from arch/ia64/kernel/smp.c~ptcg +5 -0

+				if (saved_rid != ia64_global_tlb_flush_rid) {
+					ia64_set_rr(ia64_global_tlb_flush_start, ia64_global_tlb_flush_rid);
+					ia64_srlz_d();
+				}
+

arch/ia64/kernel/smp.c renamed from arch/ia64/kernel/smp.c~ptcg +7 -0

+				if (saved_rid != ia64_global_tlb_flush_rid) {
+					ia64_set_rr(ia64_global_tlb_flush_start, saved_rid);
+					ia64_srlz_d();
+				}
+				atomic_dec(&ia64_global_tlb_flush_cpu_count);
+				break;
+			      }
 			      default:
 				printk(KERN_CRIT "Unknown IPI on CPU %d: %lu\n", this_cpu, which);
 				break;

Quoted 8 lines · Natalie Protasevich

This patch allows to disable ptc.g. The code used to be in the kernel, then was removed in 2.4 since the bug that it was fixing has gone away. However, some large system vendors now want this capability available through a means that can be controlled by the platform in the event that there is an issue with either processor or their chipset where global ptc.g is not operational. They want the mechanism for future platforms to work around such issues. It is also needed for platform makers when they deliberately do not use the global cache purge in their chipset implementation. (For such cases, Intel provided a SAL table entry to specify if ptc.g is allowed and how many).

This is an area prone to hard-to-reproduce and hard-to-debug problems, and there's a lot of subtle stuff in this patch. So I worry about the fact that we're adding a noptcg path that will be basically untested, compared with the normal path.

Review

Quoted 9 lines

(quoted diff) +8 -0

+static inline void
+flush_tlb_no_ptcg (unsigned long start, unsigned long end,
+                 unsigned long nbits)
+{
+     extern void smp_send_flush_tlb (void);
+     unsigned long saved_tpr = 0;
+     unsigned long flags;
+     int cpus = num_online_cpus();

This isn't safe with respect to CPU hotplug, is it? What if a CPU goes offline between here and the "wait for other CPUs" loop below?

Review

Quoted 6 lines

(quoted diff) +5 -0

+     /*
+      * Sometimes this is called with interrupts disabled and causes
+      * deadlock; to avoid this we enable interrupt and raise the TPR
+      * to enable ONLY IPI.
+      */

I don't think the comment matches the code. With TPR, you can mask interrupts 0x10-0x1f, 0x10-0x2f, 0x10-0x3f, ..., 0x10-0xef, or 0x10-0xff. So you have to leave at least interrupts 0xf0-0xff unmasked, which includes IA64_IPI_VECTOR at 0xfe and 15 others.

Review

Quoted 5 lines

(quoted diff) +4 -0

+     local_save_flags(flags);
+     if (!(flags & IA64_PSR_I)) {
+             saved_tpr = ia64_getreg(_IA64_REG_CR_TPR);
+             ia64_srlz_d();

Why is this srlz.d needed?

Quoted 1 line

+ ia64_setreg(_IA64_REG_CR_TPR, saved_tpr);

This just writes back the same value we read above. It doesn't really do anything, does it?

Review

Quoted 7 lines

(quoted diff) +6 -0

+             ia64_srlz_d();
+             local_irq_enable();
+     }
+
+     ia64_global_tlb_flush_rid = ia64_get_rr(start);
+     ia64_srlz_d();

Why is this srlz.d needed?

Quoted 48 lines

(quoted diff) +29 -0

+     ia64_global_tlb_flush_start = start;
+     ia64_global_tlb_flush_end = end;
+     ia64_global_tlb_flush_nbits = nbits;
+     atomic_set(&ia64_global_tlb_flush_cpu_count, cpus - 1);
+     smp_send_flush_tlb();
+     /*
+      * Purge local TLB entries. ALAT invalidation is done in ia64_leave_kernel.
+      */
+     do {
+             ia64_ptcl(start, nbits<<2);
+             start += (1UL << nbits);
+     } while (start < end);
+
+     ia64_srlz_i();                  /* srlz.i implies srlz.d */
+
+     /*
+      * Wait for other CPUs to finish purging entries.
+      */
+     while (atomic_read(&ia64_global_tlb_flush_cpu_count)) {
+             /* Nothing */
+     }
+
+     if (!(flags & IA64_PSR_I)) {
+             local_irq_disable();
+             ia64_setreg(_IA64_REG_CR_TPR, saved_tpr);
+             ia64_srlz_d();
+     }
+}
+
 void
 ia64_global_tlb_purge (struct mm_struct *mm, unsigned long start,
                     unsigned long end, unsigned long nbits)

diff -puN arch/ia64/kernel/smp.c~ptcg arch/ia64/kernel/smp.c

arch/ia64/kernel/smp.c 2007-09-02 23:59:25.000000000 -0700 renamed from arch/ia64/kernel/smp.c~ptcg 2007-09-02 23:58:54.000000000 -0700 +9 -0

@@ -174,6 +175,48 @@ handle_IPI (int irq, void *dev_id)
                              unw_init_running(kdump_cpu_freeze, NULL);
                              break;
 #endif
+
+                           case IPI_FLUSH_TLB:
+                           {
+                             extern unsigned long ia64_global_tlb_flush_start,
+                                    ia64_global_tlb_flush_end, ia64_global_tlb_flush_nbits,
+                                    ia64_global_tlb_flush_rid;
+                             extern atomic_t ia64_global_tlb_flush_cpu_count;
+                             unsigned long saved_rid = ia64_get_rr(ia64_global_tlb_flush_start);
+                             unsigned long end = ia64_global_tlb_flush_end;

arch/ia64/kernel/smp.c 2007-09-02 23:59:25.000000000 -0700 handle_IPI (int irq, void *dev_id) · line 196

Quoted 10 lines

arch/ia64/kernel/smp.c 2007-09-02 23:59:25.000000000 -0700 renamed from arch/ia64/kernel/smp.c~ptcg 2007-09-02 23:58:54.000000000 -0700 +10 -0

+                             unsigned long start = ia64_global_tlb_flush_start;
+                             unsigned long nbits = ia64_global_tlb_flush_nbits;
+
+                             /*
+                              * Current CPU may be running with different RID so we need to
+                              * reload the RID of flushed address.  Purging the translation
+                              * also needs ALAT invalidation; we do not need "invala" here
+                              * since it is done in ia64_leave_kernel.
+                              */
+                             ia64_srlz_d();

Why is this srlz.d needed?

Quoted 5 lines

arch/ia64/kernel/smp.c 2007-09-02 23:59:25.000000000 -0700 renamed from arch/ia64/kernel/smp.c~ptcg 2007-09-02 23:58:54.000000000 -0700 +5 -0

+                             if (saved_rid != ia64_global_tlb_flush_rid) {
+                                     ia64_set_rr(ia64_global_tlb_flush_start, ia64_global_tlb_flush_rid);
+                                     ia64_srlz_d();
+                             }
+

arch/ia64/kernel/smp.c 2007-09-02 23:59:25.000000000 -0700 handle_IPI (int irq, void *dev_id) · line 211

Quoted 10 lines

arch/ia64/kernel/smp.c 2007-09-02 23:59:25.000000000 -0700 renamed from arch/ia64/kernel/smp.c~ptcg 2007-09-02 23:58:54.000000000 -0700 +10 -0

+                             do {
+                                     /*
+                                      * Purge local TLB entries.
+                                      */
+                                     ia64_ptcl(start, nbits<<2);
+                                     start += (1UL << nbits);
+                             } while (start < end);
+
+                             ia64_barrier();
+                             ia64_srlz_i();                  /* srlz.i implies srlz.d */

Why are these (barrier & srlz.i) needed?

Quoted 10 lines

arch/ia64/kernel/smp.c 2007-09-02 23:59:25.000000000 -0700 renamed from arch/ia64/kernel/smp.c~ptcg 2007-09-02 23:58:54.000000000 -0700 +7 -0

+                             if (saved_rid != ia64_global_tlb_flush_rid) {
+                                     ia64_set_rr(ia64_global_tlb_flush_start, saved_rid);
+                                     ia64_srlz_d();
+                             }
+                             atomic_dec(&ia64_global_tlb_flush_cpu_count);
+                             break;
+                           }
                            default:
                              printk(KERN_CRIT "Unknown IPI on CPU %d: %lu\n", this_cpu, which);
                              break;

On 03 Sep 2007 05:21:41 -0400, Jes Sorensen <[email protected]> wrote:
The boot option is only temporary, until the SAL/PAL mechanism will be shaken up.
I will digest and use the machvec then, how about that? For now I'd like to provide this revived patch as is for it is proven to be working in the past, if you don't mind this first stage.

Quoted 2 lines

Who are these large system vendors who want this patch in? I presume it's not the people running gmail? :-)

No :) those interested are big hardware makers of large scaled out boxes, such as HP, UIS. They are using own asics and are not necessarily being able to keep chipset native capabilities intact. As I said in the preamble, the mechanism has to be there so they can turn the ptc.g off and run the OS.

No :) those interested are big hardware makers of large scaled out boxes, such as HP, UIS. They are using own asics and are not necessarily being able to keep chipset native capabilities intact. As I said in the preamble, the mechanism has to be there so they can turn the ptc.g off and run the OS.

Quoted 5 lines

No :) those interested are big hardware makers of large scaled out boxes, such as HP, UIS. They are using own asics and are not necessarily being able to keep chipset native capabilities intact. As I said in the preamble, the mechanism has to be there so they can turn the ptc.g off and run the OS.

This really sounds like it is best done as a machvec. Currently, Altix (sn2) does not have a ptc.g that spans nodes. We have our own tlb purge mechanism which is very flexible and scalable. Maybe it would be best to follow that example. To repeat Bjorn's comments, this area can be fraught with _VERY_ difficult to diagnose problems. A boot test can hardly be expected to uncover all the races so be patient while we walk through this code.

Quoted 6 lines · Robin Holt

Quoted 5 lines

No :) those interested are big hardware makers of large scaled out boxes, such as HP, UIS. They are using own asics and are not necessarily being able to keep chipset native capabilities intact. As I said in the preamble, the mechanism has to be there so they can turn the ptc.g off and run the OS.

Quoted 8 lines

This really sounds like it is best done as a machvec. Currently, Altix (sn2) does not have a ptc.g that spans nodes. We have our own tlb purge mechanism which is very flexible and scalable. Maybe it would be best to follow that example. To repeat Bjorn's comments, this area can be fraught with _VERY_ difficult to diagnose problems. A boot test can hardly be expected to uncover all the races so be patient while we walk through this code.

All right, all right ... :) I'll do the machvec then. Thanks guys, will peek into the sn2 to get an idea.

Regards,
--Natalie
-
To unsubscribe from this list: send the line "unsubscribe linux-ia64" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

In my opinion, machvec is a bad idea for a temporary workaround. You'll need to create new header files etc etc all just for a short-lived workaround. Just my 2 cents.

--david

Quoted 18 lines · Robin Holt

Quoted 6 lines

Quoted 5 lines

No :) those interested are big hardware makers of large scaled out boxes, such as HP, UIS. They are using own asics and are not necessarily being able to keep chipset native capabilities intact. As I said in the preamble, the mechanism has to be there so they can turn the ptc.g off and run the OS.

Quoted 8 lines

This really sounds like it is best done as a machvec. Currently, Altix (sn2) does not have a ptc.g that spans nodes. We have our own tlb purge mechanism which is very flexible and scalable. Maybe it would be best to follow that example. To repeat Bjorn's comments, this area can be fraught with _VERY_ difficult to diagnose problems. A boot test can hardly be expected to uncover all the races so be patient while we walk through this code.

All right, all right ... :) I'll do the machvec then. Thanks guys, will peek into the sn2 to get an idea.

Quoted 7 lines

Regards,
--Natalie
-
To unsubscribe from this list: send the line "unsubscribe linux-ia64" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

--
Mosberger Consulting LLC, http://www.mosberger-consulting.com/

The boot option is only temporary, until the SAL/PAL mechanism will be shaken up.

Quoted 5 lines · David Mosberger-Tang

In my opinion, machvec is a bad idea for a temporary workaround. You'll need to create new header files etc etc all just for a short-lived workaround. Just my 2 cents.

--david

This is actually a feature. Can be a workaround for platform that didn't manage to make ptcg work, but can be intentionally disabled by design. Big modular scaled out systems may choose to disable it intentionally. I think it's more of feature implementation, which is offered by architecture and has found some users, so I'd say this might stay and not be short-lived. In this case, will machvec be more appropriate?
Thanks,
--Natalie

Quoted 32 lines · Natalie Protasevich

Quoted 18 lines · Robin Holt

Quoted 6 lines

Quoted 5 lines

No :) those interested are big hardware makers of large scaled out boxes, such as HP, UIS. They are using own asics and are not necessarily being able to keep chipset native capabilities intact. As I said in the preamble, the mechanism has to be there so they can turn the ptc.g off and run the OS.

Quoted 8 lines

This really sounds like it is best done as a machvec. Currently, Altix (sn2) does not have a ptc.g that spans nodes. We have our own tlb purge mechanism which is very flexible and scalable. Maybe it would be best to follow that example. To repeat Bjorn's comments, this area can be fraught with _VERY_ difficult to diagnose problems. A boot test can hardly be expected to uncover all the races so be patient while we walk through this code.

All right, all right ... :) I'll do the machvec then. Thanks guys, will peek into the sn2 to get an idea.

Quoted 7 lines

Regards,
--Natalie
-
To unsubscribe from this list: send the line "unsubscribe linux-ia64" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

--
Mosberger Consulting LLC, http://www.mosberger-consulting.com/

Natalie,

Quoted 2 lines · you

The boot option is only temporary, until the SAL/PAL mechanism will be shaken up.

Once the SAL/PAL mechanism exists, there won't be a need for bootoption or a separate machvec anymore, or am I missing something?

The other thing that makes me a bit dubious about using a machvec here is that it doesn't just apply for DIG, I'd think. We wouldn't want to duplicate all the existing machvecs (other than SGI, which never used ptc.g anyhow), or would we?

--david

Quoted 7 lines · Natalie Protasevich

Quoted 5 lines · David Mosberger-Tang

In my opinion, machvec is a bad idea for a temporary workaround. You'll need to create new header files etc etc all just for a short-lived workaround. Just my 2 cents.

--david

Quoted 43 lines

This is actually a feature. Can be a workaround for platform that didn't manage to make ptcg work, but can be intentionally disabled by design. Big modular scaled out systems may choose to disable it intentionally. I think it's more of feature implementation, which is offered by architecture and has found some users, so I'd say this might stay and not be short-lived. In this case, will machvec be more appropriate?
Thanks,
--Natalie

Quoted 32 lines · Natalie Protasevich

Quoted 18 lines · Robin Holt

Quoted 6 lines

Quoted 5 lines

No :) those interested are big hardware makers of large scaled out boxes, such as HP, UIS. They are using own asics and are not necessarily being able to keep chipset native capabilities intact. As I said in the preamble, the mechanism has to be there so they can turn the ptc.g off and run the OS.

Quoted 8 lines

This really sounds like it is best done as a machvec. Currently, Altix (sn2) does not have a ptc.g that spans nodes. We have our own tlb purge mechanism which is very flexible and scalable. Maybe it would be best to follow that example. To repeat Bjorn's comments, this area can be fraught with _VERY_ difficult to diagnose problems. A boot test can hardly be expected to uncover all the races so be patient while we walk through this code.

All right, all right ... :) I'll do the machvec then. Thanks guys, will peek into the sn2 to get an idea.

Quoted 7 lines

Regards,
--Natalie
-
To unsubscribe from this list: send the line "unsubscribe linux-ia64" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

--
Mosberger Consulting LLC, http://www.mosberger-consulting.com/

--
Mosberger Consulting LLC, http://www.mosberger-consulting.com/

So far it appears to me that it will be defined for ia64 somewhere in asm-generic/tlb.h and in asm-ia64/tlb.h as platform_global_tlb_ipi_purge() something, but I'm not sure about that, will have to start digging.

Quoted 4 lines

So far it appears to me that it will be defined for ia64 somewhere in asm-generic/tlb.h and in asm-ia64/tlb.h as platform_global_tlb_ipi_purge() something, but I'm not sure about that, will have to start digging.

I'm voting with David on this one ... machvec doesn't look like the right mechanism for this (all of the non-sn2 machvecs would potentially have to be duplicated to provide ptc.g and !ptc.g variants ... that doesn't sound like a good plan).

The reason why I suggested a machvec is to avoid cluttering up the generic function with something like this. In addition I only see two machvec's that may potentially get copied, the dig one and the hpzx1 one.

If we start adding clutter to the generic functions we will end up with a 2-dimensional mesh of things where some is handled machvec style and then we have local variations within the machvecs. I suspect the Unisys and HP machines will end up having special machvecs for these machines anyway.

Quoted 10 lines · Jes Sorensen

The reason why I suggested a machvec is to avoid cluttering up the generic function with something like this. In addition I only see two machvec's that may potentially get copied, the dig one and the hpzx1 one.

If we start adding clutter to the generic functions we will end up with a 2-dimensional mesh of things where some is handled machvec style and then we have local variations within the machvecs. I suspect the Unisys and HP machines will end up having special machvecs for these machines anyway.

I'm not aware of a current or planned HP machine where ptc.g will not work.

Sometimes ptc.g doesn't work because of defects in early processor or chipset prototypes, but I consider that a bring-up issue that is not significant enough to warrant additional permanent code in the kernel.

So I don't think we should have a special machvec for HP machines.