Skip to contentSkip to conversation

Kexec/Kdump support POWER6

PATCH In review Sachin P. Sant 7 participants 18 messages May 22, 2007 – May 29, 2007

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

cover Kexec/Kdump support POWER6

Here are a set of patches which adds kdump support for Power6.

The first patch reads VRMA page size from device tree.

Second patch adds Kexec/Kdump support for Power6 processor. On Power machines supporting VRMA, Kexec/Kdump does not work. Hypervisor stores VRMA mapping used by the OS, in the hpte hash tables. This patch makes sure these hpte entries are left untouched.

Comments are welcome.

Thanks
-Sachin

---

#1 lore raw

PATCH 1/2 Kexec/Kdump support POWER6

Read supported VRMA page size from device tree.

Thanks
-Sachin

* Read supported VRMA page size from device tree.

---

arch/powerpc/kernel/setup_64.c +4 -0
diff -Naurp linux-2.6.22-rc2/arch/powerpc/kernel/setup_64.c linux-2.6.22-rc2-vrma/arch/powerpc/kernel/setup_64.c
@@ -73,6 +73,10 @@ int have_of = 1;
 int boot_cpuid = 0;
 u64 ppc64_pft_size;
 
+/* For VRMA */
+u64 ppc64_vrma_page_size;
+int have_vrma = 0;
+
 /* Pick defaults since we might want to patch instructions
  * before we've read this from the device tree.
  */
arch/powerpc/mm/hash_utils_64.c +7 -0
diff -Naurp linux-2.6.22-rc2/arch/powerpc/mm/hash_utils_64.c linux-2.6.22-rc2-vrma/arch/powerpc/mm/hash_utils_64.c
@@ -199,6 +199,13 @@ static int __init htab_dt_scan_page_size
 	if (type == NULL || strcmp(type, "cpu") != 0)
 		return 0;
 
+	prop = (u32 *)of_get_flat_dt_prop(node, "ibm,vrma-page-sizes", &size);
+	if (prop != NULL) {
+		DBG("VRMA Page size from device-tree:\n");
+		ppc64_vrma_page_size = prop[0];
+		have_vrma = 1;
+	}
+
 	prop = (u32 *)of_get_flat_dt_prop(node,
 					  "ibm,segment-page-sizes", &size);
 	if (prop != NULL) {
include/asm-powerpc/page_64.h +4 -0
diff -Naurp linux-2.6.22-rc2/include/asm-powerpc/page_64.h linux-2.6.22-rc2-vrma/include/asm-powerpc/page_64.h
@@ -76,6 +76,10 @@ static inline void copy_page(void *to, v
 /* Log 2 of page table size */
 extern u64 ppc64_pft_size;
 
+/* For VRMA */
+extern u64 ppc64_vrma_page_size;
+extern int have_vrma;
+
 /* Large pages size */
 #ifdef CONFIG_HUGETLB_PAGE
 extern unsigned int HPAGE_SHIFT;
#2 lore raw

PATCH 2/2 Kexec/Kdump support POWER6

On Power machines supporting VRMA, Kexec/Kdump does not work. Hypervisor stores VRMA mapping used by the OS, in the hpte hash tables. Make sure these hpte entries are left untouched.

Thanks
-Sachin

* On Power machines supporting VRMA, Kexec/Kdump does not work.
* Hypervisor stores VRMA mapping used by the OS, in the hpte hash tables.
* Make sure these hpte entries are left untouched.

---

arch/powerpc/kernel/machine_kexec_64.c +3 -0
diff -Naurp linux-2.6.22-rc2-vrma/arch/powerpc/kernel/machine_kexec_64.c linux-2.6.22-rc2-p6/arch/powerpc/kernel/machine_kexec_64.c
@@ -279,6 +279,9 @@ void default_machine_kexec(struct kimage
 	kexec_stack.thread_info.task = current_thread_info()->task;
 	kexec_stack.thread_info.flags = 0;
 
+	if (have_vrma)
+		pSeries_find_hpte_vrma();
+
 	/* Some things are best done in assembly.  Finding globals with
 	 * a toc is easier in C, so pass in what we can.
 	 */
arch/powerpc/platforms/pseries/lpar.c +55 -1

69 lines not on this page · raw message

include/asm-powerpc/kexec.h +2 -0
diff -Naurp linux-2.6.22-rc2-vrma/include/asm-powerpc/kexec.h linux-2.6.22-rc2-p6/include/asm-powerpc/kexec.h
@@ -24,6 +24,8 @@
 
 #define KEXEC_CONTROL_CODE_SIZE 4096
 
+extern void pSeries_find_hpte_vrma(void);
+
 /* The native architecture */
 #ifdef __powerpc64__
 #define KEXEC_ARCH KEXEC_ARCH_PPC64
include/asm-powerpc/mmu-hash64.h +5 -0
diff -Naurp linux-2.6.22-rc2-vrma/include/asm-powerpc/mmu-hash64.h linux-2.6.22-rc2-p6/include/asm-powerpc/mmu-hash64.h
@@ -94,6 +94,11 @@ extern char initial_stab[];
 #define HPTE_R_C		ASM_CONST(0x0000000000000080)
 #define HPTE_R_R		ASM_CONST(0x0000000000000100)
 
+#define HPTE_V_RMA_VPN         ASM_CONST(0x001FFFFFF0000000)
+#define HPTE_V_MASK            ASM_CONST(0xc000000000000000)
+#define MAGIC_SKIP_HPTE        ASM_CONST(0x4000000000000000)
+#define HPTE_V_RMA_NUM         16
+
 /* Values for PP (assumes Ks=0, Kp=1) */
 /* pp0 will always be 0 for linux     */
 #define PP_RWXX	0	/* Supervisor read/write, User none */
#3 lore raw

Hi,

Quoted 8 lines · Sachin P. Sant

On Power machines supporting VRMA, Kexec/Kdump does not work. Hypervisor stores VRMA mapping used by the OS, in the hpte hash tables. Make sure these hpte entries are left untouched.

diff -Naurp linux-2.6.22-rc2-vrma/arch/powerpc/kernel/machine_kexec_64.c linux-2.6.22-rc2-p6/arch/powerpc/kernel/machine_kexec_64.c

arch/powerpc/kernel/machine_kexec_64.c void default_machine_kexec(struct kimage · line 284

Quoted 6 lines
arch/powerpc/kernel/machine_kexec_64.c +3 -0
@@ -279,6 +279,9 @@ void default_machine_kexec(struct kimage
 	kexec_stack.thread_info.task = current_thread_info()->task;
 	kexec_stack.thread_info.flags = 0;
 
+	if (have_vrma)
+		pSeries_find_hpte_vrma();
+

This will break kexec builds on non-pseries. It's referring to platform code that might not be built.

Quoted 7 lines
arch/powerpc/kernel/machine_kexec_64.c +0 -0
 	/* Some things are best done in assembly.  Finding globals with
 	 * a toc is easier in C, so pass in what we can.
 	 */

diff -Naurp linux-2.6.22-rc2-vrma/arch/powerpc/platforms/pseries/lpar.c linux-2.6.22-rc2-p6/arch/powerpc/platforms/pseries/lpar.c

arch/powerpc/platforms/pseries/lpar.c static long pSeries_lpar_hpte_remove(uns · line 375

Quoted 7 lines
arch/powerpc/platforms/pseries/lpar.c +4 -0
@@ -369,6 +369,56 @@ static long pSeries_lpar_hpte_remove(uns
 	return -1;
 }
 
+unsigned long hpte_vrma_slots[HPTE_V_RMA_NUM];
+unsigned int num_hpte_vrma_slots = 0;
+
+void pSeries_find_hpte_vrma(void)

Does this function find the vrma, or save it away? Seems like the name is misleading.

arch/powerpc/platforms/pseries/lpar.c static long pSeries_lpar_hpte_remove(uns · line 387

Quoted 12 lines
arch/powerpc/platforms/pseries/lpar.c +12 -0
+{
+	unsigned int step;
+	unsigned long hash, slot, vaddr;
+	unsigned long dword0, dummy1, rma_size;
+	long lpar_rc;
+	int i;
+	
+	/* Get the RMA size */
+	rma_size = lmb.rmo_size;
+	
+	/* Get the VRMA page size */	
+	step = 1 << ppc64_vrma_page_size;

Is ppc64_vrma_page_size really the size, or the shift? Above would indicate that it's really a shift value.

arch/powerpc/platforms/pseries/lpar.c static long pSeries_lpar_hpte_remove(uns · line 393

Quoted 6 lines
arch/powerpc/platforms/pseries/lpar.c +6 -0
+
+	vaddr = HPTE_V_RMA_VPN + rma_size;
+
+	/* Find hpte's with VRMA mappings */
+	for (; vaddr >= HPTE_V_RMA_VPN; vaddr -= step) {
+		hash = hpt_hash(vaddr, mmu_psize_defs[MMU_PAGE_16M].shift);

Why is 16M hardcoded here, when you're taking such great care to read out the pagesize earlier?

arch/powerpc/platforms/pseries/lpar.c static long pSeries_lpar_hpte_remove(uns · line 400

Quoted 7 lines
arch/powerpc/platforms/pseries/lpar.c +7 -0
+		slot = ((hash & htab_hash_mask) * HPTES_PER_GROUP);	
+
+		for (i = 0; i < HPTES_PER_GROUP; i++) {
+			lpar_rc = plpar_pte_read(0, slot, 
+						&dword0, &dummy1);
+			if (!lpar_rc && dword0 &&
+			((dword0 & HPTE_V_MASK) == MAGIC_SKIP_HPTE)) {

Indentation

arch/powerpc/platforms/pseries/lpar.c static long pSeries_lpar_hpte_remove(uns · line 402

Quoted 2 lines
arch/powerpc/platforms/pseries/lpar.c +2 -0
+				/* store the hpte */
+				hpte_vrma_slots[num_hpte_vrma_slots++] = slot;

Here you rely on global exported state (num_hpte_vrma_slots), increasing it without checking for limits. What happens if this function is ever called twice? Should you set it to 0 in the beginning of the function and check it against the size of the hpte_vrma_slots array instead?

Quoted 40 lines
arch/powerpc/platforms/pseries/lpar.c +24 -1
+				break;
+			}
+			slot++;
+		}
+	}
+}
+
+static inline int check_vrma_slot(int slot)
+{
+	int j;
+
+	for (j = 0; j < num_hpte_vrma_slots; j++)
+		if (hpte_vrma_slots[j] == slot) 
+			return 1;
+
+	return 0;
+
+}
+
 static void pSeries_lpar_hptab_clear(void)
 {
 	unsigned long size_bytes = 1UL << ppc64_pft_size;
@@ -377,8 +427,12 @@ static void pSeries_lpar_hptab_clear(voi
 	int i;
 
 	/* TODO: Use bulk call */
-	for (i = 0; i < hpte_count; i++)
+	for (i = 0; i < hpte_count; i++) {
+		if (have_vrma && check_vrma_slot(i))
+			/* You don't want to remove this hpte */
+			continue;
 		plpar_pte_remove_raw(0, i, 0, &dummy1, &dummy2);
+	}
 }
 
 /*

diff -Naurp linux-2.6.22-rc2-vrma/include/asm-powerpc/kexec.h linux-2.6.22-rc2-p6/include/asm-powerpc/kexec.h

include/asm-powerpc/kexec.h line 28

Quoted 5 lines
include/asm-powerpc/kexec.h +2 -0
@@ -24,6 +24,8 @@
 
 #define KEXEC_CONTROL_CODE_SIZE 4096
 
+extern void pSeries_find_hpte_vrma(void);
+

Same comment as above: This isn't a kexec function as much as a pseries function, so it should be defined in some other header instead.

Quoted 7 lines
include/asm-powerpc/kexec.h +0 -0
 /* The native architecture */
 #ifdef __powerpc64__
 #define KEXEC_ARCH KEXEC_ARCH_PPC64

diff -Naurp linux-2.6.22-rc2-vrma/include/asm-powerpc/mmu-hash64.h linux-2.6.22-rc2-p6/include/asm-powerpc/mmu-hash64.h

include/asm-powerpc/mmu-hash64.h extern char initial_stab[]; · line 100

Quoted 7 lines
include/asm-powerpc/mmu-hash64.h +4 -0
@@ -94,6 +94,11 @@ extern char initial_stab[];
 #define HPTE_R_C		ASM_CONST(0x0000000000000080)
 #define HPTE_R_R		ASM_CONST(0x0000000000000100)
 
+#define HPTE_V_RMA_VPN         ASM_CONST(0x001FFFFFF0000000)
+#define HPTE_V_MASK            ASM_CONST(0xc000000000000000)
+#define MAGIC_SKIP_HPTE        ASM_CONST(0x4000000000000000)
+#define HPTE_V_RMA_NUM         16

"MAGIC_SKIP_HPTE"? I'm sure there's a proper name for this field in the PAPR, isn't there? Also, HPTE_V_RMA_NUM isn't a HPTE_V field, it shouldn't have that prefix. It's not a property of the mmu in the first place.

These should maybe be local defines in the pseries lpar code instead, since it's more of a lpar<->phyp interface than mmu programming interface.

-Olof

#4 lore raw

Hi Olaf, thanks for the review.

Quoted 9 lines

arch/powerpc/kernel/machine_kexec_64.c void default_machine_kexec(struct kimage · line 284

Quoted 4 lines
arch/powerpc/kernel/machine_kexec_64.c +3 -0
+	if (have_vrma)
+		pSeries_find_hpte_vrma();
+

This will break kexec builds on non-pseries. It's referring to platform code that might not be built.

Ok. Will call this function from lpar.c instead of default_machine_kexec().

Quoted 4 lines

Does this function find the vrma, or save it away? Seems like the name is misleading.

Well it finds a vrma entry and saves it. I thought of pSeries_find_save_hpte_vrma(), but decided against it. I could change it to pSeries_save_hpte_vrma().

Quoted 4 lines

Is ppc64_vrma_page_size really the size, or the shift? Above would indicate that it's really a shift value.

It is a shift. I will change it to ppc64_vrma_page_shift.

Quoted 4 lines

Why is 16M hardcoded here, when you're taking such great care to read out the pagesize earlier?

Hrmm. Ok will use the vrma_page_shift value.

Quoted 4 lines

arch/powerpc/platforms/pseries/lpar.c static long pSeries_lpar_hpte_remove(uns · line 400

Quoted 2 lines
arch/powerpc/platforms/pseries/lpar.c +1 -0
+			((dword0 & HPTE_V_MASK) == MAGIC_SKIP_HPTE)) {

Indentation

Done.

Quoted 9 lines

arch/powerpc/platforms/pseries/lpar.c static long pSeries_lpar_hpte_remove(uns · line 402

Quoted 3 lines
arch/powerpc/platforms/pseries/lpar.c +2 -0
+				/* store the hpte */
+				hpte_vrma_slots[num_hpte_vrma_slots++] = slot;

Here you rely on global exported state (num_hpte_vrma_slots), increasing it without checking for limits. What happens if this function is ever called twice? Should you set it to 0 in the beginning of the function and check it against the size of the hpte_vrma_slots array instead?

Will add proper checks for num_hpte_vrma_slots variable value.

Quoted 17 lines
Quoted 3 lines

+extern void pSeries_find_hpte_vrma(void);

Same comment as above: This isn't a kexec function as much as a pseries function, so it should be defined in some other header instead.

include/asm-powerpc/mmu-hash64.h extern char initial_stab[]; · line 100

Quoted 6 lines
include/asm-powerpc/mmu-hash64.h +4 -0
+#define HPTE_V_RMA_VPN         ASM_CONST(0x001FFFFFF0000000)
+#define HPTE_V_MASK            ASM_CONST(0xc000000000000000)
+#define MAGIC_SKIP_HPTE        ASM_CONST(0x4000000000000000)
+#define HPTE_V_RMA_NUM         16

"MAGIC_SKIP_HPTE"? I'm sure there's a proper name for this field in the PAPR, isn't there? Also, HPTE_V_RMA_NUM isn't a HPTE_V field, it shouldn't have that prefix. It's not a property of the mmu in the first place.

Quoted 3 lines

These should maybe be local defines in the pseries lpar code instead, since it's more of a lpar<->phyp interface than mmu programming interface.

Will move them to pseries lpar code.
Updated patch on its way.

Thanks
-Sachin

#5 lore raw
In reply to Olof Johansson

Hi Olof, thanks for the review.

Quoted 9 lines

arch/powerpc/kernel/machine_kexec_64.c void default_machine_kexec(struct kimage · line 284

Quoted 4 lines
arch/powerpc/kernel/machine_kexec_64.c +3 -0
+	if (have_vrma)
+		pSeries_find_hpte_vrma();
+

This will break kexec builds on non-pseries. It's referring to platform code that might not be built.

Ok. Will call this function from lpar.c instead of default_machine_kexec().

Quoted 4 lines

Does this function find the vrma, or save it away? Seems like the name is misleading.

Well it finds a vrma entry and saves it. I thought of pSeries_find_save_hpte_vrma(), but decided against it. I could change it to pSeries_save_hpte_vrma().

Quoted 4 lines

Is ppc64_vrma_page_size really the size, or the shift? Above would indicate that it's really a shift value.

It is a shift. I will change it to ppc64_vrma_page_shift.

Quoted 4 lines

Why is 16M hardcoded here, when you're taking such great care to read out the pagesize earlier?

Hrmm. Ok will use the vrma_page_shift value.

Quoted 4 lines

arch/powerpc/platforms/pseries/lpar.c static long pSeries_lpar_hpte_remove(uns · line 400

Quoted 2 lines
arch/powerpc/platforms/pseries/lpar.c +1 -0
+			((dword0 & HPTE_V_MASK) == MAGIC_SKIP_HPTE)) {

Indentation

Done.

Quoted 9 lines

arch/powerpc/platforms/pseries/lpar.c static long pSeries_lpar_hpte_remove(uns · line 402

Quoted 3 lines
arch/powerpc/platforms/pseries/lpar.c +2 -0
+				/* store the hpte */
+				hpte_vrma_slots[num_hpte_vrma_slots++] = slot;

Here you rely on global exported state (num_hpte_vrma_slots), increasing it without checking for limits. What happens if this function is ever called twice? Should you set it to 0 in the beginning of the function and check it against the size of the hpte_vrma_slots array instead?

Will add proper checks for num_hpte_vrma_slots variable value.

Quoted 17 lines
Quoted 3 lines

+extern void pSeries_find_hpte_vrma(void);

Same comment as above: This isn't a kexec function as much as a pseries function, so it should be defined in some other header instead.

include/asm-powerpc/mmu-hash64.h extern char initial_stab[]; · line 100

Quoted 6 lines
include/asm-powerpc/mmu-hash64.h +4 -0
+#define HPTE_V_RMA_VPN         ASM_CONST(0x001FFFFFF0000000)
+#define HPTE_V_MASK            ASM_CONST(0xc000000000000000)
+#define MAGIC_SKIP_HPTE        ASM_CONST(0x4000000000000000)
+#define HPTE_V_RMA_NUM         16

"MAGIC_SKIP_HPTE"? I'm sure there's a proper name for this field in the PAPR, isn't there? Also, HPTE_V_RMA_NUM isn't a HPTE_V field, it shouldn't have that prefix. It's not a property of the mmu in the first place.

Quoted 3 lines

These should maybe be local defines in the pseries lpar code instead, since it's more of a lpar<->phyp interface than mmu programming interface.

Will move them to pseries lpar code.
Updated patch on its way.

Thanks
-Sachin

#6 lore raw

PATCH 2/2 Kexec/Kdump support - POWER6

In reply to Olof Johansson

On Power machines supporting VRMA, Kexec/Kdump does not work. Hypervisor stores VRMA mapping used by the OS, in the hpte hash tables. Make sure these hpte entries are left untouched.

This patch also adds plpar_pte_read_raw() on the lines of plpar_pte_remove_raw().

Thanks
-Sachin

* On Power machines supporting VRMA, Kexec/Kdump does not work.
* Hypervisor stores VRMA mapping used by the OS, in the hpte hash tables.
* Make sure these hpte entries are left untouched.

---

arch/powerpc/platforms/pseries/lpar.c +64 -1
diff -Naurp linux-2.6.22-rc2-vrma/arch/powerpc/platforms/pseries/lpar.c linux-2.6.22-rc2-kexec/arch/powerpc/platforms/pseries/lpar.c
@@ -369,6 +369,62 @@ static long pSeries_lpar_hpte_remove(uns
 	return -1;
 }
 
+#define VRMA_VPN        ASM_CONST(0x001FFFFFF0000000)
+#define VRMA_MASK       ASM_CONST(0xc000000000000000)
+#define VRMA_HPTE_B_1TB ASM_CONST(0x4000000000000000)
+#define VRMA_NUM        16
+
+unsigned long hpte_vrma_slots[VRMA_NUM];
+unsigned int num_hpte_vrma_slots = 0;
+
+static void pSeries_save_hpte_vrma(void)
+{
+	unsigned int step;
+	unsigned long hash, slot, vaddr;
+	unsigned long dword0, dummy1, rma_size;
+	long lpar_rc;
+	int i;
+	
+	/* Get the RMA size */
+	rma_size = lmb.rmo_size;
+	
+	/* Get the VRMA page size */	
+	step = 1 << ppc64_vrma_page_shift;
+
+	vaddr = VRMA_VPN + rma_size;
+
+	/* Find hpte's with VRMA mappings */
+	for (; vaddr >= VRMA_VPN; vaddr -= step) {
+		hash = hpt_hash(vaddr, ppc64_vrma_page_shift);
+		slot = ((hash & htab_hash_mask) * HPTES_PER_GROUP);	
+
+		for (i = 0; i < HPTES_PER_GROUP; i++) {
+			lpar_rc = plpar_pte_read_raw(0, slot, 
+						&dword0, &dummy1);
+			if (!lpar_rc && dword0 &&
+				((dword0 & VRMA_MASK) == VRMA_HPTE_B_1TB) &&
+				(num_hpte_vrma_slots < VRMA_NUM)) {
+				/* store the hpte */
+				hpte_vrma_slots[num_hpte_vrma_slots++] = slot;
+				break;
+			}
+			slot++;
+		}
+	}
+}
+
+static inline int check_vrma_slot(int slot)
+{
+	int j;
+
+	for (j = 0; j < num_hpte_vrma_slots; j++)
+		if (hpte_vrma_slots[j] == slot) 
+			return 1;
+
+	return 0;
+
+}
+
 static void pSeries_lpar_hptab_clear(void)
 {
 	unsigned long size_bytes = 1UL << ppc64_pft_size;
@@ -376,9 +432,16 @@ static void pSeries_lpar_hptab_clear(voi
 	unsigned long dummy1, dummy2;
 	int i;
 
+	if (have_vrma)
+		pSeries_save_hpte_vrma();
+
 	/* TODO: Use bulk call */
-	for (i = 0; i < hpte_count; i++)
+	for (i = 0; i < hpte_count; i++) {
+		if (have_vrma && check_vrma_slot(i))
+			/* You don't want to remove this hpte */
+			continue;
 		plpar_pte_remove_raw(0, i, 0, &dummy1, &dummy2);
+	}
 }
 
 /*
arch/powerpc/platforms/pseries/plpar_wrappers.h +15 -0
diff -Naurp linux-2.6.22-rc2-vrma/arch/powerpc/platforms/pseries/plpar_wrappers.h linux-2.6.22-rc2-kexec/arch/powerpc/platforms/pseries/plpar_wrappers.h
@@ -108,6 +108,21 @@ static inline long plpar_pte_read(unsign
 	return rc;
 }
 
+/* plpar_pte_read_raw can be called in real mode. It calls plpar_hcall_raw */
+static inline long plpar_pte_read_raw(unsigned long flags, unsigned long ptex,
+		unsigned long *old_pteh_ret, unsigned long *old_ptel_ret)
+{
+	long rc;
+	unsigned long retbuf[PLPAR_HCALL_BUFSIZE];
+
+	rc = plpar_hcall_raw(H_READ, retbuf, flags, ptex);
+
+	*old_pteh_ret = retbuf[0];
+	*old_ptel_ret = retbuf[1];
+
+	return rc;
+}
+
 static inline long plpar_pte_protect(unsigned long flags, unsigned long ptex,
 		unsigned long avpn)
 {
#7 lore raw

Re: [Patch 2/2] Kexec/Kdump support - POWER6

Quoted 3 lines · Sachin P. Sant

On Power machines supporting VRMA, Kexec/Kdump does not work. Hypervisor stores VRMA mapping used by the OS, in the hpte hash tables. Make sure these hpte entries are left untouched.

Surely all we need to do is to avoid clearing the VRMA entries. We can do this by not clearing any HPTE where the top 40 bits of the first dword are 0x4001ffffff (B=1 for a 1TB segment and the 0x0001ffffff special VSID). In fact we can avoid having to read each entry by doing the H_REMOVEs with H_ANDCOND and the bolted bit when we clear the hash table, and only reading the HPTEs for which the H_REMOVE returns an error.

Paul.

#8 lore raw

Re: [Patch 2/2] Kexec/Kdump support - POWER6

Quoted 6 lines · Paul Mackerras
Quoted 3 lines · Sachin P. Sant

On Power machines supporting VRMA, Kexec/Kdump does not work. Hypervisor stores VRMA mapping used by the OS, in the hpte hash tables. Make sure these hpte entries are left untouched.

Quoted 8 lines

Surely all we need to do is to avoid clearing the VRMA entries. We can do this by not clearing any HPTE where the top 40 bits of the first dword are 0x4001ffffff (B=1 for a 1TB segment and the 0x0001ffffff special VSID). In fact we can avoid having to read each entry by doing the H_REMOVEs with H_ANDCOND and the bolted bit when we clear the hash table, and only reading the HPTEs for which the H_REMOVE returns an error.

Paul,

Thanks for your suggestion.

But we can not use 0x4001fffff for H_ANDCOND flag since AND'ing 0x4001ffff value with most of the HPTEs results in non-zero and most of the HPTE entries are not removed.

Quoted 1 line

From POWER ISA 2.04 document page 425,

"Programming Note:
Software should specify PTE(B) = 0b01 for all Page Table Entries that map the VRMA in order to be consistent with the values in Figure 14."

So I tried 0x4000000000000000 as AVPN parameter and using that it removes all hpte entries other than VRMA(ie non 1TB segment size PTE entries).

Tested on POWER6/POWER5 machines.

========================
Starting from POWER5+, hypervisor stores VRMA entries in the HPTE tables and these entries are need by OS and they should not be cleared. These VRMA entries will be of segment size 1TB. Using H_ANDCOND flag for H_REMOVE hypervisor call it is made sure that we will not remove any PTE whose size is 1TB(ie VRMA entries)

---
 arch/powerpc/platforms/pseries/lpar.c |    6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)
arch/powerpc/platforms/pseries/lpar.c +5 -1
Index: linux-2.6.21.1/arch/powerpc/platforms/pseries/lpar.c
===================================================================
@@ -369,6 +369,8 @@ static long pSeries_lpar_hpte_remove(uns
 	return -1;
 }
 
+#define VRMA_HPTE_B_1TB ASM_CONST(0x4000000000000000)
+
 static void pSeries_lpar_hptab_clear(void)
 {
 	unsigned long size_bytes = 1UL << ppc64_pft_size;
@@ -378,7 +380,9 @@ static void pSeries_lpar_hptab_clear(voi
 
 	/* TODO: Use bulk call */
 	for (i = 0; i < hpte_count; i++)
-		plpar_pte_remove_raw(0, i, 0, &dummy1, &dummy2);
+		/* dont remove HPTEs of segments size 1TB (VRMA entries) */
+		plpar_pte_remove_raw(H_ANDCOND, i, VRMA_HPTE_B_1TB,
+						&dummy1, &dummy2);
 }
 
 /*
#9 lore raw

Re: [Patch 2/2] Kexec/Kdump support - POWER6

Quoted 15 lines · Mohan Kumar M
Quoted 6 lines · Paul Mackerras
Quoted 3 lines · Sachin P. Sant

On Power machines supporting VRMA, Kexec/Kdump does not work. Hypervisor stores VRMA mapping used by the OS, in the hpte hash tables. Make sure these hpte entries are left untouched.

Quoted 7 lines

Surely all we need to do is to avoid clearing the VRMA entries. We can do this by not clearing any HPTE where the top 40 bits of the first dword are 0x4001ffffff (B=1 for a 1TB segment and the 0x0001ffffff special VSID). In fact we can avoid having to read each entry by doing the H_REMOVEs with H_ANDCOND and the bolted bit when we clear the hash table, and only reading the HPTEs for which the H_REMOVE returns an error.

Quoted 5 lines

So I tried 0x4000000000000000 as AVPN parameter and using that it removes all hpte entries other than VRMA(ie non 1TB segment size PTE entries).

Tested on POWER6/POWER5 machines.

Hi,

As Paul says above, you need to check for failures and compare the VSID and possibly unhash it anyway in case of non-match. Otherwise if the kernel ever starts using 1TB segments for regular use, those pages will never be unhashed. I don't see your code doing that now.

-Olof

#10 lore raw

PATCH 1/1 Kexec/Kdump support - POWER6

Quoted 6 lines · Olof Johansson

As Paul says above, you need to check for failures and compare the VSID and possibly unhash it anyway in case of non-match. Otherwise if the kernel ever starts using 1TB segments for regular use, those pages will never be unhashed. I don't see your code doing that now.

How about the following patch ?

Thanks
-Sachin

---

* On Power machines supporting VRMA, Kexec/Kdump does not work.
* Hypervisor stores VRMA mapping used by the OS, in the hpte hash tables.
* Make sure these hpte entries are left untouched.
*
* This patch also adds plpar_pte_read_raw() on the lines of
* plpar_pte_remove_raw().

---

arch/powerpc/platforms/pseries/lpar.c +17 -3
diff -Naurp a/arch/powerpc/platforms/pseries/lpar.c b/arch/powerpc/platforms/pseries/lpar.c
@@ -369,16 +369,30 @@ static long pSeries_lpar_hpte_remove(uns
 	return -1;
 }
 
+#define VRMA_HPTE_B_1TB ASM_CONST(0x4000000000000000)
+#define VRMA_MASK	ASM_CONST(0x4001ffffff)
+
 static void pSeries_lpar_hptab_clear(void)
 {
 	unsigned long size_bytes = 1UL << ppc64_pft_size;
 	unsigned long hpte_count = size_bytes >> 4;
-	unsigned long dummy1, dummy2;
+	unsigned long dummy1, dummy2, dword0;
+	long lpar_rc;
 	int i;
 
 	/* TODO: Use bulk call */
-	for (i = 0; i < hpte_count; i++)
-		plpar_pte_remove_raw(0, i, 0, &dummy1, &dummy2);
+	for (i = 0; i < hpte_count; i++) {
+		/* dont remove HPTEs with VRMA mappings */
+		lpar_rc = plpar_pte_remove_raw(H_ANDCOND, i, VRMA_HPTE_B_1TB,
+						&dummy1, &dummy2);
+		if (lpar_rc == H_NOT_FOUND) {
+			lpar_rc = plpar_pte_read_raw(0, i, &dword0, &dummy1);
+			if (!lpar_rc && 
+				(((dword0 >> 24) & VRMA_MASK) != VRMA_MASK))
+				/* Can be hpte for 1TB Seg. So remove it */
+				plpar_pte_remove_raw(0, i, 0, &dummy1, &dummy2);
+		}
+	}
 }
 
 /*
arch/powerpc/platforms/pseries/plpar_wrappers.h +15 -0
diff -Naurp a/arch/powerpc/platforms/pseries/plpar_wrappers.h b/arch/powerpc/platforms/pseries/plpar_wrappers.h
@@ -108,6 +108,21 @@ static inline long plpar_pte_read(unsign
 	return rc;
 }
 
+/* plpar_pte_read_raw can be called in real mode. It calls plpar_hcall_raw */
+static inline long plpar_pte_read_raw(unsigned long flags, unsigned long ptex,
+		unsigned long *old_pteh_ret, unsigned long *old_ptel_ret)
+{
+	long rc;
+	unsigned long retbuf[PLPAR_HCALL_BUFSIZE];
+
+	rc = plpar_hcall_raw(H_READ, retbuf, flags, ptex);
+
+	*old_pteh_ret = retbuf[0];
+	*old_ptel_ret = retbuf[1];
+
+	return rc;
+}
+
 static inline long plpar_pte_protect(unsigned long flags, unsigned long ptex,
 		unsigned long avpn)
 {
#11 lore raw

Re: [Patch ] Kexec/Kdump support - POWER6

arch/powerpc/platforms/pseries/lpar.c static long pSeries_lpar_hpte_remove(uns · line 387

Quoted 9 lines
arch/powerpc/platforms/pseries/lpar.c +4 -2
 	/* TODO: Use bulk call */
-	for (i = 0; i < hpte_count; i++)
-		plpar_pte_remove_raw(0, i, 0, &dummy1, &dummy2);
+	for (i = 0; i < hpte_count; i++) {
+		/* dont remove HPTEs with VRMA mappings */
+		lpar_rc = plpar_pte_remove_raw(H_ANDCOND, i, VRMA_HPTE_B_1TB,
+						&dummy1, &dummy2);

If you're going to use the B (1T segment) bit instead of the bolted bit, at least define a proper constant in line with the existing naming of the hash table constants in mmu-hash64.h. I would suggest doing the same with the VRMA_MASK/VALUE thing and calling it HPTE_V_VRMA_MASK or something similar.

arch/powerpc/platforms/pseries/lpar.c static long pSeries_lpar_hpte_remove(uns · line 396

Quoted 9 lines
arch/powerpc/platforms/pseries/lpar.c +8 -0
+		if (lpar_rc == H_NOT_FOUND) {
+			lpar_rc = plpar_pte_read_raw(0, i, &dword0, &dummy1);
+			if (!lpar_rc && 
+				(((dword0 >> 24) & VRMA_MASK) != VRMA_MASK))
+				/* Can be hpte for 1TB Seg. So remove it */
+				plpar_pte_remove_raw(0, i, 0, &dummy1, &dummy2);
+		}
+	}
 }

In addition, I would recommend following Michael's advice and using using the bulk remove Hcall whenever possible.

Cheers,
Ben.

_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec

#12 lore raw

PATCH 1/1 Kexec/Kdump support - POWER6

Quoted 6 lines · Benjamin Herrenschmidt

If you're going to use the B (1T segment) bit instead of the bolted bit, at least define a proper constant in line with the existing naming of the hash table constants in mmu-hash64.h. I would suggest doing the same with the VRMA_MASK/VALUE thing and calling it HPTE_V_VRMA_MASK or something similar.

Well i had used them properly in my previous patches. Don't know why i changed it in this patch :-(

Here is the updated patch.

Quoted 6 lines

Review

Quoted 3 lines
(quoted diff) +1 -0
+	}
 }

In addition, I would recommend following Michael's advice and using using the bulk remove Hcall whenever possible.

Yes will send out a separate patch to use bulk remove Hcall.

Thanks
-Sachin

---

* On Power machines supporting VRMA, Kexec/Kdump does not work.
* Hypervisor stores VRMA mapping used by the OS, in the hpte hash tables.
* Make sure these hpte entries are left untouched.
*
* This patch also adds plpar_pte_read_raw() on the lines of
* plpar_pte_remove_raw().

---

arch/powerpc/platforms/pseries/lpar.c +17 -3
diff -Naurp a/arch/powerpc/platforms/pseries/lpar.c b/arch/powerpc/platforms/pseries/lpar.c
@@ -369,16 +369,30 @@ static long pSeries_lpar_hpte_remove(uns
 	return -1;
 }
 
+#define HPTE_V_1TB_SEG		ASM_CONST(0x4000000000000000)
+#define HPTE_V_VRMA_MASK	ASM_CONST(0x4001ffffff)
+
 static void pSeries_lpar_hptab_clear(void)
 {
 	unsigned long size_bytes = 1UL << ppc64_pft_size;
 	unsigned long hpte_count = size_bytes >> 4;
-	unsigned long dummy1, dummy2;
+	unsigned long dummy1, dummy2, dword0;
+	long lpar_rc;
 	int i;
 
 	/* TODO: Use bulk call */
-	for (i = 0; i < hpte_count; i++)
-		plpar_pte_remove_raw(0, i, 0, &dummy1, &dummy2);
+	for (i = 0; i < hpte_count; i++) {
+		/* dont remove HPTEs with VRMA mappings */
+		lpar_rc = plpar_pte_remove_raw(H_ANDCOND, i, HPTE_V_1TB_SEG,
+						&dummy1, &dummy2);
+		if (lpar_rc == H_NOT_FOUND) {
+			lpar_rc = plpar_pte_read_raw(0, i, &dword0, &dummy1);
+			if (!lpar_rc && (((dword0 >> 24) & HPTE_V_VRMA_MASK) 
+				!= HPTE_V_VRMA_MASK))
+				/* Can be hpte for 1TB Seg. So remove it */
+				plpar_pte_remove_raw(0, i, 0, &dummy1, &dummy2);
+		}
+	}
 }
 
 /*
arch/powerpc/platforms/pseries/plpar_wrappers.h +15 -0
diff -Naurp a/arch/powerpc/platforms/pseries/plpar_wrappers.h b/arch/powerpc/platforms/pseries/plpar_wrappers.h
@@ -108,6 +108,21 @@ static inline long plpar_pte_read(unsign
 	return rc;
 }
 
+/* plpar_pte_read_raw can be called in real mode. It calls plpar_hcall_raw */
+static inline long plpar_pte_read_raw(unsigned long flags, unsigned long ptex,
+		unsigned long *old_pteh_ret, unsigned long *old_ptel_ret)
+{
+	long rc;
+	unsigned long retbuf[PLPAR_HCALL_BUFSIZE];
+
+	rc = plpar_hcall_raw(H_READ, retbuf, flags, ptex);
+
+	*old_pteh_ret = retbuf[0];
+	*old_ptel_ret = retbuf[1];
+
+	return rc;
+}
+
 static inline long plpar_pte_protect(unsigned long flags, unsigned long ptex,
 		unsigned long avpn)
 {
#13 lore raw

Re: [Patch ] Kexec/Kdump support - POWER6

arch/powerpc/platforms/pseries/lpar.c static long pSeries_lpar_hpte_remove(uns · line 373

Quoted 3 lines · Sachin P. Sant
arch/powerpc/platforms/pseries/lpar.c +2 -0
+#define HPTE_V_1TB_SEG         ASM_CONST(0x4000000000000000)
+#define HPTE_V_VRMA_MASK       ASM_CONST(0x4001ffffff)

Move those to mmu-hash64.h along with the other ones. Also, keep the mask aligned to the top bits

Quoted 10 lines

static void pSeries_lpar_hptab_clear(void)

 {
        unsigned long size_bytes = 1UL << ppc64_pft_size;
        unsigned long hpte_count = size_bytes >> 4;
arch/powerpc/platforms/pseries/lpar.c +2 -1
-       unsigned long dummy1, dummy2;
+       unsigned long dummy1, dummy2, dword0;
+       long lpar_rc;
        int i;
 
        /* TODO: Use bulk call */

arch/powerpc/platforms/pseries/lpar.c static long pSeries_lpar_hpte_remove(uns · line 391

Quoted 10 lines
arch/powerpc/platforms/pseries/lpar.c +8 -2
-       for (i = 0; i < hpte_count; i++)
-               plpar_pte_remove_raw(0, i, 0, &dummy1, &dummy2);
+       for (i = 0; i < hpte_count; i++) {
+               /* dont remove HPTEs with VRMA mappings */
+               lpar_rc = plpar_pte_remove_raw(H_ANDCOND, i, HPTE_V_1TB_SEG,
+                                               &dummy1, &dummy2);
+               if (lpar_rc == H_NOT_FOUND) {
+                       lpar_rc = plpar_pte_read_raw(0, i, &dword0, &dummy1);
+                       if (!lpar_rc && (((dword0 >> 24) & HPTE_V_VRMA_MASK) 
+                               != HPTE_V_VRMA_MASK))

No need to shift >> 24, just have the mask left justified in the first place. No need to have a "magic" shift value in there.

arch/powerpc/platforms/pseries/lpar.c static long pSeries_lpar_hpte_remove(uns · line 396

Quoted 5 lines
arch/powerpc/platforms/pseries/lpar.c +4 -0
+                               /* Can be hpte for 1TB Seg. So remove it */
+                               plpar_pte_remove_raw(0, i, 0, &dummy1, &dummy2);
+               }
+       }
 }

Appart from that, looks good. Does it actually work ? :-)

Cheers,
Ben.

_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec

#14 lore raw

Re: [Patch ] Kexec/Kdump support - POWER6

Quoted 3 lines · Benjamin Herrenschmidt

Move those to mmu-hash64.h along with the other ones. Also, keep the mask aligned to the top bits

In previous patches i had these #defines in mmu-hash64.h. But got review comments to move them to lpar.c :-)

Oh well i will move them back to mmu-hash64.h.

Quoted 5 lines
Quoted 2 lines

+ if (!lpar_rc && (((dword0 >> 24) & HPTE_V_VRMA_MASK)

No need to shift >> 24, just have the mask left justified in the first place. No need to have a "magic" shift value in there.

Ok.

Quoted 2 lines

Appart from that, looks good. Does it actually work ? :-)

Yes it does work. It must be my lucky day when i coded the patch :-)

Thanks
-Sachin

---

* On Power machines supporting VRMA, Kexec/Kdump does not work.
* Hypervisor stores VRMA mapping used by the OS, in the hpte hash tables.
* Make sure these hpte entries are left untouched.
*
* This patch also adds plpar_pte_read_raw() on the lines of
* plpar_pte_remove_raw().

---

arch/powerpc/platforms/pseries/lpar.c +14 -3
diff -Naurp a/arch/powerpc/platforms/pseries/lpar.c b/arch/powerpc/platforms/pseries/lpar.c
@@ -373,12 +373,23 @@ static void pSeries_lpar_hptab_clear(voi
 {
 	unsigned long size_bytes = 1UL << ppc64_pft_size;
 	unsigned long hpte_count = size_bytes >> 4;
-	unsigned long dummy1, dummy2;
+	unsigned long dummy1, dummy2, dword0;
+	long lpar_rc;
 	int i;
 
 	/* TODO: Use bulk call */
-	for (i = 0; i < hpte_count; i++)
-		plpar_pte_remove_raw(0, i, 0, &dummy1, &dummy2);
+	for (i = 0; i < hpte_count; i++) {
+		/* dont remove HPTEs with VRMA mappings */
+		lpar_rc = plpar_pte_remove_raw(H_ANDCOND, i, HPTE_V_1TB_SEG,
+						&dummy1, &dummy2);
+		if (lpar_rc == H_NOT_FOUND) {
+			lpar_rc = plpar_pte_read_raw(0, i, &dword0, &dummy1);
+			if (!lpar_rc && ((dword0 & HPTE_V_VRMA_MASK) 
+				!= HPTE_V_VRMA_MASK))
+				/* Can be hpte for 1TB Seg. So remove it */
+				plpar_pte_remove_raw(0, i, 0, &dummy1, &dummy2);
+		}
+	}
 }
 
 /*
arch/powerpc/platforms/pseries/plpar_wrappers.h +15 -0
diff -Naurp a/arch/powerpc/platforms/pseries/plpar_wrappers.h b/arch/powerpc/platforms/pseries/plpar_wrappers.h
@@ -108,6 +108,21 @@ static inline long plpar_pte_read(unsign
 	return rc;
 }
 
+/* plpar_pte_read_raw can be called in real mode. It calls plpar_hcall_raw */
+static inline long plpar_pte_read_raw(unsigned long flags, unsigned long ptex,
+		unsigned long *old_pteh_ret, unsigned long *old_ptel_ret)
+{
+	long rc;
+	unsigned long retbuf[PLPAR_HCALL_BUFSIZE];
+
+	rc = plpar_hcall_raw(H_READ, retbuf, flags, ptex);
+
+	*old_pteh_ret = retbuf[0];
+	*old_ptel_ret = retbuf[1];
+
+	return rc;
+}
+
 static inline long plpar_pte_protect(unsigned long flags, unsigned long ptex,
 		unsigned long avpn)
 {
include/asm-powerpc/mmu-hash64.h +3 -0
diff -Naurp a/include/asm-powerpc/mmu-hash64.h b/include/asm-powerpc/mmu-hash64.h
@@ -94,6 +94,9 @@ extern char initial_stab[];
 #define HPTE_R_C		ASM_CONST(0x0000000000000080)
 #define HPTE_R_R		ASM_CONST(0x0000000000000100)
 
+#define HPTE_V_1TB_SEG          ASM_CONST(0x4000000000000000)
+#define HPTE_V_VRMA_MASK        ASM_CONST(0x4001ffffff000000)
+
 /* Values for PP (assumes Ks=0, Kp=1) */
 /* pp0 will always be 0 for linux     */
 #define PP_RWXX	0	/* Supervisor read/write, User none */
#15 lore raw

Re: [Patch ] Kexec/Kdump support - POWER6

Quoted 3 lines · Sachin P. Sant

In previous patches i had these #defines in mmu-hash64.h. But got review comments to move them to lpar.c :-)

Hrm... weird. Oh well, _I_ personally prefer them in mmu-hash64.h and since I'm writing most of the MMU code lately I'd say my opinion makes rule :-)

Quoted 1 line

Oh well i will move them back to mmu-hash64.h.

Thanks.

Cheers,
Ben.

_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec

#16 lore raw

Re: [Patch ] Kexec/Kdump support - POWER6

In reply to Sachin P. Sant
Quoted 62 lines · Sachin P. Sant
Quoted 6 lines · Benjamin Herrenschmidt

If you're going to use the B (1T segment) bit instead of the bolted bit, at least define a proper constant in line with the existing naming of the hash table constants in mmu-hash64.h. I would suggest doing the same with the VRMA_MASK/VALUE thing and calling it HPTE_V_VRMA_MASK or something similar.

Well i had used them properly in my previous patches. Don't know why i changed it in this patch :-(

Here is the updated patch.

Quoted 6 lines

Review

Quoted 3 lines
(quoted diff) +1 -0
+	}
 }

In addition, I would recommend following Michael's advice and using using the bulk remove Hcall whenever possible.

Yes will send out a separate patch to use bulk remove Hcall.

Thanks
-Sachin

---

plain text document attachment (kexec-kdump-support-on-POWER6)
* On Power machines supporting VRMA, Kexec/Kdump does not work.
* Hypervisor stores VRMA mapping used by the OS, in the hpte hash tables.
* Make sure these hpte entries are left untouched.
*
* This patch also adds plpar_pte_read_raw() on the lines of
* plpar_pte_remove_raw().

---

diff -Naurp a/arch/powerpc/platforms/pseries/lpar.c b/arch/powerpc/platforms/pseries/lpar.c

arch/powerpc/platforms/pseries/lpar.c +5 -1
@@ -369,16 +369,30 @@ static long pSeries_lpar_hpte_remove(uns
 	return -1;
 }
 
+#define HPTE_V_1TB_SEG		ASM_CONST(0x4000000000000000)
+#define HPTE_V_VRMA_MASK	ASM_CONST(0x4001ffffff)
+
 static void pSeries_lpar_hptab_clear(void)
 {
 	unsigned long size_bytes = 1UL << ppc64_pft_size;
 	unsigned long hpte_count = size_bytes >> 4;
-	unsigned long dummy1, dummy2;
+	unsigned long dummy1, dummy2, dword0;
+	long lpar_rc;
 	int i;
 

arch/powerpc/platforms/pseries/lpar.c static long pSeries_lpar_hpte_remove(uns · line 390

Quoted 10 lines
arch/powerpc/platforms/pseries/lpar.c +7 -2
 	/* TODO: Use bulk call */
-	for (i = 0; i < hpte_count; i++)
-		plpar_pte_remove_raw(0, i, 0, &dummy1, &dummy2);
+	for (i = 0; i < hpte_count; i++) {
+		/* dont remove HPTEs with VRMA mappings */
+		lpar_rc = plpar_pte_remove_raw(H_ANDCOND, i, HPTE_V_1TB_SEG,
+						&dummy1, &dummy2);
+		if (lpar_rc == H_NOT_FOUND) {
+			lpar_rc = plpar_pte_read_raw(0, i, &dword0, &dummy1);
+			if (!lpar_rc && (((dword0 >> 24) & HPTE_V_VRMA_MASK) <xxxxxxx

You have trailing whitespace on that line.

Add this to your ~/.vimrc, you do use vim right :)

highlight RedundantWhitespace ctermbg=red guibg=red

cheers

Signature · 8 lines
Michael Ellerman
OzLabs, IBM Australia Development Lab

wwweb: http://michael.ellerman.id.au
phone: +61 2 6212 1183 (tie line 70 21183)

We do not inherit the earth from our ancestors,
we borrow it from our children. - S.M.A.R.T Person
#17 lore raw

PATCH 1/1 Kexec/Kdump support - POWER6

Quoted 4 lines · On Tue, 29 May 2007 20:14:44 +1000 Michael Ellerman <[email protected]>…

Add this to your ~/.vimrc, you do use vim right :)

highlight RedundantWhitespace ctermbg=red guibg=red

You need this as well:

match RedundantWhitespace /\s\+$\| \+\ze\t/

--
Cheers,
Stephen Rothwell sfr at canb.auug.org.au
http://www.canb.auug.org.au/~sfr/
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 189 bytes
Desc: not available
Url : http://lists.infradead.org/pipermail/kexec/attachments/20070529/85b1ef30/attachment.bin