Skip to contentSkip to conversation

Problems using psr.dd

PATCH Keith Owens 4 participants 11 messages Nov 14, 2003 – Nov 21, 2003

lore
Share & tools
lore
#0 lore raw

I am unable to successfully use psr.dd on 2.4 and 2.6 kernels. The patches below plus the debug.c module demonstrate the problem. do_trap29() sets regs->cr_ipsr.dd and returns. The kernel drops through ia64_leave_kernel, loads the modified ipsr and does rfi. However the same debug fault occurs again, as if psr.dd were being ignored.

What am I missing?

# depmod -ae;rmmod debug 2>/dev/null; modprobe debug
init_debug: start
dbr[0]=0xa0000000000b8f90 dbr[1]=0x41fffffffffffffc
setpsrdb: on=1
init_debug: test
trap 29 ifa=0xa0000000000b8f90 iip=0xa0000000000b8430 count=0
trap 29 ifa=0xa0000000000b8f90 iip=0xa0000000000b8430 count=1
trap 29 ifa=0xa0000000000b8f90 iip=0xa0000000000b8430 count=2
trap 29 ifa=0xa0000000000b8f90 iip=0xa0000000000b8430 count=3
db is looping, disabling it
setpsrdb: on=0
init_debug: end
arch/ia64/kernel/Makefile renamed from arch/ia64/kernel/Makefile_1.24 +1 -1
@@ -11,7 +11,7 @@
 
 O_TARGET := kernel.o
 
-export-objs := ia64_ksyms.o sal.o
+export-objs := ia64_ksyms.o sal.o traps.o
 
 obj-y := acpi.o entry.o gate.o efi.o efi_stub.o ia64_ksyms.o irq.o irq_ia64.o irq_lsapic.o ivt.o \
 	 machvec.o pal.o process.o perfmon.o ptrace.o sal.o salinfo.o semaphore.o setup.o	 \
arch/ia64/kernel/entry.S renamed from arch/ia64/kernel/entry.S_1.33 +121 -1

128 lines not on this page · raw message

arch/ia64/kernel/traps.c renamed from arch/ia64/kernel/traps.c_1.27 +6 -0
@@ -452,6 +452,10 @@
 	return rv;
 }
 
+#include <linux/module.h>
+int (*trap29)(unsigned long ifa, struct pt_regs *regs);
+EXPORT_SYMBOL(trap29);
+
 void
 ia64_fault (unsigned long vector, unsigned long isr, unsigned long ifa,
 	    unsigned long iim, unsigned long itir, unsigned long arg5,
@@ -551,6 +555,8 @@
 		break;
 
 	      case 29: /* Debug */
+			if (trap29 && trap29(ifa, regs))
+				return;
 	      case 35: /* Taken Branch Trap */
 	      case 36: /* Single Step Trap */
 		switch (vector) {

== debug.c =
#include <linux/config.h>
#include <linux/slab.h>
#include <linux/kdb.h>
#include <linux/module.h>

MODULE_LICENSE("GPL");

int victim;
static int trap29count;
extern int (*trap29)(unsigned long ifa, struct pt_regs *regs);

static int do_trap29(unsigned long ifa, struct pt_regs *regs)

{
	if (user_mode(regs))
		return 0;
	kdb_printf("trap 29 ifa=0x%016lx iip=0x%016lx count=%d\n", ifa, regs->cr_iip, trap29count);
	if (trap29count >= 3) {
		kdb_printf("db is looping, disabling it\n");
		ia64_psr(regs)->db = 0;
	} else {
		++trap29count;
		ia64_psr(regs)->dd = 1;
	}
	return 1;
}

static void setdbr(unsigned long regnum, unsigned long address, unsigned long mask)

{
	kdb_printf("dbr[%ld]=0x%016lx dbr[%ld]=0x%016lx\n", regnum, address, regnum+1, mask);
	__asm__ __volatile__ ("mov dbr[%0]=%1;;mov dbr[%2]=%3;;"::"r"(regnum),"r"(address),"r"(regnum+1),"r"(mask));
	ia64_srlz_d();
}

static void setpsrdb(int on)

{
	unsigned long tmp;
	kdb_printf("%s: on=%d\n", __FUNCTION__, on);
	if (on)
		__asm__ __volatile__ ("mov %0=psr;;dep %0=-1,%0,24,1;;mov psr.l=%0;;srlz.d;;srlz.i;;":"=r"(tmp)::"memory");
	else
		__asm__ __volatile__ ("mov %0=psr;;dep %0=0,%0,24,1;;mov psr.l=%0;;srlz.d;;srlz.i;;":"=r"(tmp)::"memory");
}

static int __init init_debug(void)

{
	kdb_printf("%s: start\n", __FUNCTION__);
	trap29 = &do_trap29;
	setdbr(0, (unsigned long)&victim, 1UL<<62 | 1UL<<56 | (-4UL & 0xffffffffffffffUL));
	setpsrdb(1);
	kdb_printf("%s: test\n", __FUNCTION__);
	victim = 1;
	setpsrdb(0);
	kdb_printf("%s: end\n", __FUNCTION__);
	return 0;
}

static void __exit exit_debug(void)

{
	kdb_printf("%s: start\n", __FUNCTION__);
	trap29 = NULL;
	setdbr(0, 0, 0);
	kdb_printf("%s: end\n", __FUNCTION__);
}

module_init(init_debug)
module_exit(exit_debug)

#1 lore raw
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line · Keith Owens

Keith> I am unable to successfully use psr.dd on 2.4 and 2.6 Keith> kernels. The patches below plus the debug.c module Keith> demonstrate the problem. do_trap29() sets regs->cr_ipsr.dd Keith> and returns. The kernel drops through ia64_leave_kernel, Keith> loads the modified ipsr and does rfi. However the same debug Keith> fault occurs again, as if psr.dd were being ignored.

Keith> What am I missing?

Did you find the answer already?

I'm wondering if the problem would still occurr if the debug-fault-triggering memory access was in a function with an empty register frame.

	--david
#2 lore raw
In reply to Keith Owens
Quoted 9 lines · David Mosberger
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line · Keith Owens

Keith> I am unable to successfully use psr.dd on 2.4 and 2.6 Keith> kernels. The patches below plus the debug.c module Keith> demonstrate the problem. do_trap29() sets regs->cr_ipsr.dd Keith> and returns. The kernel drops through ia64_leave_kernel, Keith> loads the modified ipsr and does rfi. However the same debug Keith> fault occurs again, as if psr.dd were being ignored.

Quoted 3 lines

Keith> What am I missing?

Did you find the answer already?

No, your response is the only one so far.

Quoted 3 lines

I'm wondering if the problem would still occurr if the debug-fault-triggering memory access was in a function with an empty register frame.

Like this? It makes no difference, trap29 just keeps occuring, even though psr.dd is being set..

static void set_victim(void)

{
        victim = 1;
 300:   0d 78 00 02 00 24       [MFI]       addl r15=0,r1
                        300: GPREL22    victim
 306:   00 00 00 02 00 c0                   nop.f 0x0
 30c:   11 00 00 90                         mov r14=1;;
 310:   1d 00 38 1e 90 11       [MFB]       st4 [r15]=r14
 316:   00 00 00 02 00 80                   nop.f 0x0
 31c:   08 00 84 00                         br.ret.sptk.many b0;;

}

Forgot to mention, this is on a vanilla McKinley box, not a SN2.

# cat /proc/cpuinfo 
processor  : 0
vendor     : GenuineIntel
arch       : IA-64
family     : Itanium 2
model      : 0
revision   : 5
archrev    : 0
features   : branchlong
cpu number : 0
cpu regs   : 4
cpu MHz    : 897.454998
itc MHz    : 897.454998
BogoMIPS   : 1342.17
...
#3 lore raw
In reply to Keith Owens
Quoted 2 lines · Keith Owens

Forgot to mention, this is on a vanilla McKinley box, not a SN2.

I tried your code on an Itanium 1 (using 2.6.0-test9-bk without kdb, s/kdb_printf/printk/). I get exactly the same behaviour.

However if I take out the printk in do_trap29 (reducing the register stack depth?) it seems to work okay.

Matt

#4 lore raw
In reply to Keith Owens

DavidM suggested that I redo the test debug module without kdb_printf() and the problem went away!

It turns out that the problem is not kdb, it is an RSE problem which is highly sensitive to the amount of RSE activity that occurs inside the hardware breakpoint handler. kdb_printf() always prints its messages using polling mode, printk() delays output if the kernel is in interrupt context. The extra work that is done by kdb_printf() as opposed to printk() is enough to trip the RSE problem.

If there is enough RSE activity during the hardware breakpoint handler to flush the registers at the time of the breakpoint then returning with psr.db = 1 and psr.dd = 1 to step over the hardware breakpoint will not work. The RSE activity that occurs as the flushed registers are restored is counted as a valid instruction, psr.dd is cleared, the original instruction is reexecuted and it trips again.

Beware of doing too much activity in the hardware breakpoint handler. I will create a patch against traps.c to detect a return with psr.dd set and issue loadrs to ensure that the RSE problem does not bite us.

The debug.c module below demonstrates the problem. "modprobe debug.o" usually works, but even without flushrs it sometimes trips the RSE problem, depending on system activity. "modprobe debug.o do_flushrs=1" always hits the problem.

== debug.c ==

#include <linux/config.h>
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/console.h>

int victim;
int do_flushrs;
static int trap29count;
extern int (*trap29)(unsigned long ifa, struct pt_regs *regs);

MODULE_LICENSE("GPL");
MODULE_PARM(do_flushrs, "i");

static int do_trap29(unsigned long ifa, struct pt_regs *regs)

{
	if (user_mode(regs))
		return 0;
	printk("trap 29 ifa=0x%016lx iip=0x%016lx count=%d victim=%d\n", ifa, regs->cr_iip, trap29count, victim);
	if (do_flushrs)
		asm volatile (";;flushrs;; " :::"memory");
	if (trap29count >= 3) {
		printk("db is looping, disabling it\n");
		ia64_psr(regs)->db = 0;
	} else {
		++trap29count;
		ia64_psr(regs)->dd = 1;
	}
	return 1;
}

static void setdbr(unsigned long regnum, unsigned long address, unsigned long mask)

{
	__asm__ __volatile__ ("mov dbr[%0]=%1;;mov dbr[%2]=%3;;"::"r"(regnum),"r"(address),"r"(regnum+1),"r"(mask));
	ia64_srlz_d();
}

static void setpsrdb(int on)

{
	unsigned long tmp;
	if (on)
		__asm__ __volatile__ ("mov %0=psr;;dep %0=-1,%0,24,1;;mov psr.l=%0;;srlz.d;;srlz.i;;":"=r"(tmp)::"memory");
	else
		__asm__ __volatile__ ("mov %0=psr;;dep %0=0,%0,24,1;;mov psr.l=%0;;srlz.d;;srlz.i;;":"=r"(tmp)::"memory");
}

static int __init init_debug(void)

{
	trap29 = &do_trap29;
	setdbr(0, (unsigned long)&victim, 1UL<<62 | 1UL<<56 | (-4UL & 0xffffffffffffffUL));
	setpsrdb(1);
	victim = 1;
	barrier();
	victim = 2;
	setpsrdb(0);
	return 0;
}

static void __exit exit_debug(void)

{
	trap29 = NULL;
	setdbr(0, 0, 0);
}

module_init(init_debug)
module_exit(exit_debug)

======

Patch against traps.c to invoke the test trap29() handler in module debug.c.

arch/ia64/kernel/Makefile +1 -1
@@ -11,7 +11,7 @@ all: kernel.o head.o init_task.o
 
 O_TARGET := kernel.o
 
-export-objs := ia64_ksyms.o
+export-objs := ia64_ksyms.o traps.o
 
 obj-y := acpi.o entry.o gate.o efi.o efi_stub.o ia64_ksyms.o irq.o irq_ia64.o irq_lsapic.o ivt.o \
 	 machvec.o pal.o process.o perfmon.o ptrace.o sal.o salinfo.o semaphore.o setup.o	 \
arch/ia64/kernel/traps.c +6 -0
@@ -429,6 +429,10 @@ ia64_illegal_op_fault (unsigned long ec,
 	return rv;
 }
 
+#include <linux/module.h>
+int (*trap29)(unsigned long ifa, struct pt_regs *regs);
+EXPORT_SYMBOL(trap29);
+
 void
 ia64_fault (unsigned long vector, unsigned long isr, unsigned long ifa,
 	    unsigned long iim, unsigned long itir, unsigned long arg5,
@@ -528,6 +532,8 @@ ia64_fault (unsigned long vector, unsign
 		break;
 
 	      case 29: /* Debug */
+			if (trap29 && trap29(ifa, regs))
+				return;
 	      case 35: /* Taken Branch Trap */
 	      case 36: /* Single Step Trap */
 		switch (vector) {
#5 lore raw
In reply to Keith Owens
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line · Keith Owens

Keith> If there is enough RSE activity during the hardware Keith> breakpoint handler to flush the registers at the time of the Keith> breakpoint then returning with psr.db = 1 and psr.dd = 1 to Keith> step over the hardware breakpoint will not work. The RSE Keith> activity that occurs as the flushed registers are restored is Keith> counted as a valid instruction, psr.dd is cleared, the Keith> original instruction is reexecuted and it trips again.

Keith> Beware of doing too much activity in the hardware breakpoint Keith> handler. I will create a patch against traps.c to detect a Keith> return with psr.dd set and issue loadrs to ensure that the Keith> RSE problem does not bite us.

I'm still not sure what's triggering the spurious fault:

(1) The RSE activity generated for restoring the current frame, or
(2) The RSE activity generated by the alloc instruction that eventually follows the instruction to which "rfi" returns to.

Unfortunately, the description of Erratum 11 isn't terribly clear to me. It says:

... The rfi instruction is followed by additional instructions that generate register stack (RSE) activity (alloc, flushrs, br.ret).

I assume "followed" is referring to execution order, not program order (i.e., inserting nop's around the "rfi" isn't going to do us any good). In your (original) case, the "rfi" is followed by a store instruction, br.call, then alloc. From reading the description, I wouldn't have expected this to trigger the Erratum, but perhaps it does. Can someone clarify?

	--david
#6 lore raw
In reply to Keith Owens

Erratum 11 applies to the cases where rfi is followed by RSE generating instruction in static code stream (i.e. in the source). Have not looked at the Keith's changes but hopefully he has not put any instruction like alloc after rfi in the source.

Thanks,
rohit

Quoted 9 lines

-----Original Message-----
From: [email protected] [mailto:linux-ia64- [email protected]] On Behalf Of David Mosberger
Sent: Thursday, November 20, 2003 10:48 AM
To: Keith Owens
Cc: [email protected]
Subject: Re: Problems using psr.dd

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

On Thu, 20 Nov 2003 18:52:33 +1100, Keith Owens <[email protected]>

Quoted 14 lines · said:

Keith> If there is enough RSE activity during the hardware Keith> breakpoint handler to flush the registers at the time of the Keith> breakpoint then returning with psr.db = 1 and psr.dd = 1 to Keith> step over the hardware breakpoint will not work. The RSE Keith> activity that occurs as the flushed registers are restored is Keith> counted as a valid instruction, psr.dd is cleared, the Keith> original instruction is reexecuted and it trips again.

Keith> Beware of doing too much activity in the hardware breakpoint Keith> handler. I will create a patch against traps.c to detect a Keith> return with psr.dd set and issue loadrs to ensure that the Keith> RSE problem does not bite us.

Quoted 4 lines

I'm still not sure what's triggering the spurious fault:

(1) The RSE activity generated for restoring the current frame, or
(2) The RSE activity generated by the alloc instruction that

eventually

Quoted 16 lines

follows the instruction to which "rfi" returns to.

Unfortunately, the description of Erratum 11 isn't terribly clear to me. It says:

... The rfi instruction is followed by additional instructions that generate register stack (RSE) activity (alloc, flushrs, br.ret).

I assume "followed" is referring to execution order, not program order (i.e., inserting nop's around the "rfi" isn't going to do us any good). In your (original) case, the "rfi" is followed by a store instruction, br.call, then alloc. From reading the description, I wouldn't have expected this to trigger the Erratum, but perhaps it does. Can someone clarify?

Quoted 3 lines
	--david

-
To unsubscribe from this list: send the line "unsubscribe linux-ia64"

in

Quoted 2 lines

the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

#7 lore raw
In reply to Keith Owens
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line · Rohit

Rohit> Erratum 11 applies to the cases where rfi is followed by RSE Rohit> generating instruction in static code stream (i.e. in the Rohit> source).

Ah, so Keith's interpretation was correct.

Rohit> Have not looked at the Keith's changes but hopefully he has Rohit> not put any instruction like alloc after rfi in the source.

No, quite the opposite: he put NOPs around it. So clearly this remains an open issue.

	--david
#8 lore raw
In reply to Keith Owens
Quoted 11 lines · David Mosberger
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line · Keith Owens

Keith> Beware of doing too much activity in the hardware breakpoint Keith> handler. I will create a patch against traps.c to detect a Keith> return with psr.dd set and issue loadrs to ensure that the Keith> RSE problem does not bite us.

I'm still not sure what's triggering the spurious fault:

(1) The RSE activity generated for restoring the current frame, or
(2) The RSE activity generated by the alloc instruction that eventually follows the instruction to which "rfi" returns to.

It has to be (1). psr.dd is getting cleared before the st8 is being executed. Just to be sure, I inserted 20 bundles of nop before and after 'victim = 1;', the problem still occurs.

Quoted 7 lines

Unfortunately, the description of Erratum 11 isn't terribly clear to me. It says:

... The rfi instruction is followed by additional instructions that generate register stack (RSE) activity (alloc, flushrs, br.ret).

Quoted 6 lines

I assume "followed" is referring to execution order, not program order (i.e., inserting nop's around the "rfi" isn't going to do us any good). In your (original) case, the "rfi" is followed by a store instruction, br.call, then alloc. From reading the description, I wouldn't have expected this to trigger the Erratum, but perhaps it does. Can someone clarify?

Agreed that it is unclear. Given the workaround for errata 11 (add 4 nop bundles around rfi) I interpret errata 11 as an instruction read ahead problem, where RSE instructions in the bundles after rfi could be read, decoded and trip RSE activity even though they were not executed. In any case, I added 20 nop bundles around the trapping instruction and it made no difference.

Arch volume 2, section 5.3 is quite explicit, "A non-faulting mandatory RSE load will clear PSR.da and PSR.dd". This does not appear to be a problem when returning to user space with these bits set, probably because the context switch does loadrs. AFAICT it is only an issue when using hardware debug registers on the kernel itself. traps.c has to detect that it is returning to kernel state with either of these bits set and issue loadrs to ensure that the rfi does not require RSE loads. Patch in progress.

#9 lore raw
In reply to Keith Owens
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line · Keith Owens

Keith> Arch volume 2, section 5.3 is quite explicit, "A non-faulting Keith> mandatory RSE load will clear PSR.da and PSR.dd".

Ah, I see, that clears it up, then. I had forgotten about this behavior. No mystery then.

Thanks,

	--david
#10 lore raw
In reply to Keith Owens
Quoted 1 line

Arch volume 2, section 5.3 is quite explicit, "A non-faulting

mandatory

Quoted 8 lines

RSE load will clear PSR.da and PSR.dd". This does not appear to be a problem when returning to user space with these bits set, probably because the context switch does loadrs. AFAICT it is only an issue when using hardware debug registers on the kernel itself. traps.c has to detect that it is returning to kernel state with either of these bits set and issue loadrs to ensure that the rfi does not require RSE loads. Patch in progress.

I think you meant the kernel return path (and not context switch) while referring to loadrs above. It is true that loadrs operation is done only while transitioning to user space from kernel and not when you are going back to kernel mode. So for the kernel->kernel case rfi can lead to mandatory RSE activity and thus clearing the da and dd bits.

Quoted 1 line

-