Problems using psr.dd
I am unable to successfully use psr.dd on 2.4 and 2.6 kernels. The patches below plus the debug.c module demonstrate the problem. do_trap29() sets regs->cr_ipsr.dd and returns. The kernel drops through ia64_leave_kernel, loads the modified ipsr and does rfi. However the same debug fault occurs again, as if psr.dd were being ignored.
What am I missing?
# depmod -ae;rmmod debug 2>/dev/null; modprobe debug init_debug: start dbr[0]=0xa0000000000b8f90 dbr[1]=0x41fffffffffffffc setpsrdb: on=1 init_debug: test trap 29 ifa=0xa0000000000b8f90 iip=0xa0000000000b8430 count=0 trap 29 ifa=0xa0000000000b8f90 iip=0xa0000000000b8430 count=1 trap 29 ifa=0xa0000000000b8f90 iip=0xa0000000000b8430 count=2 trap 29 ifa=0xa0000000000b8f90 iip=0xa0000000000b8430 count=3 db is looping, disabling it setpsrdb: on=0 init_debug: end
arch/ia64/kernel/Makefile renamed from arch/ia64/kernel/Makefile_1.24 +1 -1
@@ -11,7 +11,7 @@ O_TARGET := kernel.o -export-objs := ia64_ksyms.o sal.o +export-objs := ia64_ksyms.o sal.o traps.o obj-y := acpi.o entry.o gate.o efi.o efi_stub.o ia64_ksyms.o irq.o irq_ia64.o irq_lsapic.o ivt.o \ machvec.o pal.o process.o perfmon.o ptrace.o sal.o salinfo.o semaphore.o setup.o \
arch/ia64/kernel/entry.S renamed from arch/ia64/kernel/entry.S_1.33 +121 -1
128 lines not on this page · raw message
arch/ia64/kernel/traps.c renamed from arch/ia64/kernel/traps.c_1.27 +6 -0
@@ -452,6 +452,10 @@return rv; } +#include <linux/module.h> +int (*trap29)(unsigned long ifa,struct pt_regs *regs); +EXPORT_SYMBOL (trap29); + void ia64_fault (unsigned long vector, unsigned long isr, unsigned long ifa, unsigned long iim, unsigned long itir, unsigned long arg5,
@@ -551,6 +555,8 @@break ;case 29 :/* Debug */ +if (trap29 &&trap29 (ifa, regs)) +return ;case 35 :/* Taken Branch Trap */ case 36 :/* Single Step Trap */ switch (vector) {
== debug.c =
#include <linux/config.h>
#include <linux/slab.h>
#include <linux/kdb.h>
#include <linux/module.h>
MODULE_LICENSE("GPL");
int victim;
static int trap29count;
extern int (*trap29)(unsigned long ifa, struct pt_regs *regs);
static int do_trap29(unsigned long ifa, struct pt_regs *regs)
{
if (user_mode(regs))
return 0;
kdb_printf("trap 29 ifa=0x%016lx iip=0x%016lx count=%d\n", ifa, regs->cr_iip, trap29count);
if (trap29count >= 3) {
kdb_printf("db is looping, disabling it\n");
ia64_psr(regs)->db = 0;
} else {
++trap29count;
ia64_psr(regs)->dd = 1;
}
return 1;
}static void setdbr(unsigned long regnum, unsigned long address, unsigned long mask)
{
kdb_printf("dbr[%ld]=0x%016lx dbr[%ld]=0x%016lx\n", regnum, address, regnum+1, mask);
__asm__ __volatile__ ("mov dbr[%0]=%1;;mov dbr[%2]=%3;;"::"r"(regnum),"r"(address),"r"(regnum+1),"r"(mask));
ia64_srlz_d();
}static void setpsrdb(int on)
{
unsigned long tmp;
kdb_printf("%s: on=%d\n", __FUNCTION__, on);
if (on)
__asm__ __volatile__ ("mov %0=psr;;dep %0=-1,%0,24,1;;mov psr.l=%0;;srlz.d;;srlz.i;;":"=r"(tmp)::"memory");
else
__asm__ __volatile__ ("mov %0=psr;;dep %0=0,%0,24,1;;mov psr.l=%0;;srlz.d;;srlz.i;;":"=r"(tmp)::"memory");
}static int __init init_debug(void)
{
kdb_printf("%s: start\n", __FUNCTION__);
trap29 = &do_trap29;
setdbr(0, (unsigned long)&victim, 1UL<<62 | 1UL<<56 | (-4UL & 0xffffffffffffffUL));
setpsrdb(1);
kdb_printf("%s: test\n", __FUNCTION__);
victim = 1;
setpsrdb(0);
kdb_printf("%s: end\n", __FUNCTION__);
return 0;
}static void __exit exit_debug(void)
{
kdb_printf("%s: start\n", __FUNCTION__);
trap29 = NULL;
setdbr(0, 0, 0);
kdb_printf("%s: end\n", __FUNCTION__);
}module_init(init_debug)module_exit(exit_debug)
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line · Keith Owens
Keith> I am unable to successfully use psr.dd on 2.4 and 2.6 Keith> kernels. The patches below plus the debug.c module Keith> demonstrate the problem. do_trap29() sets regs->cr_ipsr.dd Keith> and returns. The kernel drops through ia64_leave_kernel, Keith> loads the modified ipsr and does rfi. However the same debug Keith> fault occurs again, as if psr.dd were being ignored.
Keith> What am I missing?
Did you find the answer already?
I'm wondering if the problem would still occurr if the debug-fault-triggering memory access was in a function with an empty register frame.
--david
Quoted 9 lines · David Mosberger
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line · Keith Owens
Keith> I am unable to successfully use psr.dd on 2.4 and 2.6 Keith> kernels. The patches below plus the
debug.cmodule Keith> demonstrate the problem.do_trap29()setsregs->cr_ipsr.ddKeith> and returns. The kernel drops throughia64_leave_kernel, Keith> loads the modified ipsr and does rfi. However the same debug Keith> fault occurs again, as if psr.dd were being ignored.
Quoted 3 lines
Keith> What am I missing?
Did you find the answer already?
No, your response is the only one so far.
Quoted 3 lines
I'm wondering if the problem would still occurr if the debug-fault-triggering memory access was in a function with an empty register frame.
Like this? It makes no difference, trap29 just keeps occuring, even though psr.dd is being set..
static void set_victim(void)
{
victim = 1;
300: 0d 78 00 02 00 24 [MFI] addl r15=0,r1
300: GPREL22 victim
306: 00 00 00 02 00 c0 nop.f 0x0
30c: 11 00 00 90 mov r14=1;;
310: 1d 00 38 1e 90 11 [MFB] st4 [r15]=r14
316: 00 00 00 02 00 80 nop.f 0x0
31c: 08 00 84 00 br.ret.sptk.many b0;;}
Forgot to mention, this is on a vanilla McKinley box, not a SN2.
# cat /proc/cpuinfo processor : 0 vendor : GenuineIntel arch : IA-64 family : Itanium 2 model : 0 revision : 5 archrev : 0 features : branchlong cpu number : 0 cpu regs : 4 cpu MHz : 897.454998 itc MHz : 897.454998 BogoMIPS : 1342.17 ...
Quoted 2 lines · Keith Owens
Forgot to mention, this is on a vanilla McKinley box, not a SN2.
I tried your code on an Itanium 1 (using 2.6.0-test9-bk without kdb, s/kdb_printf/printk/). I get exactly the same behaviour.
However if I take out the printk in do_trap29 (reducing the register stack depth?) it seems to work okay.
Matt
DavidM suggested that I redo the test debug module without kdb_printf() and the problem went away!
It turns out that the problem is not kdb, it is an RSE problem which is highly sensitive to the amount of RSE activity that occurs inside the hardware breakpoint handler. kdb_printf() always prints its messages using polling mode, printk() delays output if the kernel is in interrupt context. The extra work that is done by kdb_printf() as opposed to printk() is enough to trip the RSE problem.
If there is enough RSE activity during the hardware breakpoint handler to flush the registers at the time of the breakpoint then returning with psr.db = 1 and psr.dd = 1 to step over the hardware breakpoint will not work. The RSE activity that occurs as the flushed registers are restored is counted as a valid instruction, psr.dd is cleared, the original instruction is reexecuted and it trips again.
Beware of doing too much activity in the hardware breakpoint handler. I will create a patch against traps.c to detect a return with psr.dd set and issue loadrs to ensure that the RSE problem does not bite us.
The debug.c module below demonstrates the problem. "modprobe debug.o" usually works, but even without flushrs it sometimes trips the RSE problem, depending on system activity. "modprobe debug.o do_flushrs=1" always hits the problem.
== debug.c ==
#include <linux/config.h>
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/console.h>
int victim;
int do_flushrs;
static int trap29count;
extern int (*trap29)(unsigned long ifa, struct pt_regs *regs);
MODULE_LICENSE("GPL");MODULE_PARM(do_flushrs, "i");
static int do_trap29(unsigned long ifa, struct pt_regs *regs)
{
if (user_mode(regs))
return 0;
printk("trap 29 ifa=0x%016lx iip=0x%016lx count=%d victim=%d\n", ifa, regs->cr_iip, trap29count, victim);
if (do_flushrs)
asm volatile (";;flushrs;; " :::"memory");
if (trap29count >= 3) {
printk("db is looping, disabling it\n");
ia64_psr(regs)->db = 0;
} else {
++trap29count;
ia64_psr(regs)->dd = 1;
}
return 1;
}static void setdbr(unsigned long regnum, unsigned long address, unsigned long mask)
{
__asm__ __volatile__ ("mov dbr[%0]=%1;;mov dbr[%2]=%3;;"::"r"(regnum),"r"(address),"r"(regnum+1),"r"(mask));
ia64_srlz_d();
}static void setpsrdb(int on)
{
unsigned long tmp;
if (on)
__asm__ __volatile__ ("mov %0=psr;;dep %0=-1,%0,24,1;;mov psr.l=%0;;srlz.d;;srlz.i;;":"=r"(tmp)::"memory");
else
__asm__ __volatile__ ("mov %0=psr;;dep %0=0,%0,24,1;;mov psr.l=%0;;srlz.d;;srlz.i;;":"=r"(tmp)::"memory");
}static int __init init_debug(void)
{
trap29 = &do_trap29;
setdbr(0, (unsigned long)&victim, 1UL<<62 | 1UL<<56 | (-4UL & 0xffffffffffffffUL));
setpsrdb(1);
victim = 1;
barrier();
victim = 2;
setpsrdb(0);
return 0;
}static void __exit exit_debug(void)
{
trap29 = NULL;
setdbr(0, 0, 0);
}module_init(init_debug)module_exit(exit_debug)
======
Patch against traps.c to invoke the test trap29() handler in module debug.c.
arch/ia64/kernel/Makefile +1 -1
@@ -11,7 +11,7 @@ all: kernel.o head.o init_task.o O_TARGET := kernel.o -export-objs := ia64_ksyms.o +export-objs := ia64_ksyms.o traps.o obj-y := acpi.o entry.o gate.o efi.o efi_stub.o ia64_ksyms.o irq.o irq_ia64.o irq_lsapic.o ivt.o \ machvec.o pal.o process.o perfmon.o ptrace.o sal.o salinfo.o semaphore.o setup.o \
arch/ia64/kernel/traps.c +6 -0
@@ -429,6 +429,10 @@ ia64_illegal_op_fault (unsigned long ec,return rv; } +#include <linux/module.h> +int (*trap29)(unsigned long ifa,struct pt_regs *regs); +EXPORT_SYMBOL (trap29); + void ia64_fault (unsigned long vector, unsigned long isr, unsigned long ifa, unsigned long iim, unsigned long itir, unsigned long arg5,
@@ -528,6 +532,8 @@ ia64_fault (unsigned long vector, unsignbreak ;case 29 :/* Debug */ +if (trap29 &&trap29 (ifa, regs)) +return ;case 35 :/* Taken Branch Trap */ case 36 :/* Single Step Trap */ switch (vector) {
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line · Keith Owens
Keith> If there is enough RSE activity during the hardware Keith> breakpoint handler to flush the registers at the time of the Keith> breakpoint then returning with psr.db = 1 and psr.dd = 1 to Keith> step over the hardware breakpoint will not work. The RSE Keith> activity that occurs as the flushed registers are restored is Keith> counted as a valid instruction, psr.dd is cleared, the Keith> original instruction is reexecuted and it trips again.
Keith> Beware of doing too much activity in the hardware breakpoint Keith> handler. I will create a patch against traps.c to detect a Keith> return with psr.dd set and issue loadrs to ensure that the Keith> RSE problem does not bite us.
I'm still not sure what's triggering the spurious fault:
(1) The RSE activity generated for restoring the current frame, or
(2) The RSE activity generated by the alloc instruction that eventually follows the instruction to which "rfi" returns to.
Unfortunately, the description of Erratum 11 isn't terribly clear to me. It says:
... The rfi instruction is followed by additional instructions that generate register stack (RSE) activity (alloc, flushrs, br.ret).
I assume "followed" is referring to execution order, not program order (i.e., inserting nop's around the "rfi" isn't going to do us any good). In your (original) case, the "rfi" is followed by a store instruction, br.call, then alloc. From reading the description, I wouldn't have expected this to trigger the Erratum, but perhaps it does. Can someone clarify?
--david
Erratum 11 applies to the cases where rfi is followed by RSE generating instruction in static code stream (i.e. in the source). Have not looked at the Keith's changes but hopefully he has not put any instruction like alloc after rfi in the source.
Thanks,
rohit
Quoted 9 lines
-----Original Message-----
From: [email protected] [mailto:linux-ia64- [email protected]] On Behalf Of David Mosberger
Sent: Thursday, November 20, 2003 10:48 AM
To: Keith Owens
Cc: [email protected]
Subject: Re: Problems using psr.ddQuoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
On Thu, 20 Nov 2003 18:52:33 +1100, Keith Owens <[email protected]>
Quoted 14 lines · said:
Keith> If there is enough RSE activity during the hardware Keith> breakpoint handler to flush the registers at the time of the Keith> breakpoint then returning with psr.db = 1 and psr.dd = 1 to Keith> step over the hardware breakpoint will not work. The RSE Keith> activity that occurs as the flushed registers are restored is Keith> counted as a valid instruction, psr.dd is cleared, the Keith> original instruction is reexecuted and it trips again.
Keith> Beware of doing too much activity in the hardware breakpoint Keith> handler. I will create a patch against
traps.cto detect a Keith> return with psr.dd set and issue loadrs to ensure that the Keith> RSE problem does not bite us.
Quoted 4 lines
I'm still not sure what's triggering the spurious fault:
(1) The RSE activity generated for restoring the current frame, or
(2) The RSE activity generated by the alloc instruction that
eventually
Quoted 16 lines
follows the instruction to which "rfi" returns to.
Unfortunately, the description of Erratum 11 isn't terribly clear to me. It says:
... The rfi instruction is followed by additional instructions that generate register stack (RSE) activity (alloc, flushrs, br.ret).
I assume "followed" is referring to execution order, not program order (i.e., inserting nop's around the "rfi" isn't going to do us any good). In your (original) case, the "rfi" is followed by a store instruction, br.call, then alloc. From reading the description, I wouldn't have expected this to trigger the Erratum, but perhaps it does. Can someone clarify?
Quoted 3 lines
--david-
To unsubscribe from this list: send the line "unsubscribe linux-ia64"
in
Quoted 2 lines
the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line · Rohit
Rohit> Erratum 11 applies to the cases where rfi is followed by RSE Rohit> generating instruction in static code stream (i.e. in the Rohit> source).
Ah, so Keith's interpretation was correct.
Rohit> Have not looked at the Keith's changes but hopefully he has Rohit> not put any instruction like alloc after rfi in the source.
No, quite the opposite: he put NOPs around it. So clearly this remains an open issue.
--david
Quoted 11 lines · David Mosberger
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line · Keith Owens
Keith> Beware of doing too much activity in the hardware breakpoint Keith> handler. I will create a patch against
traps.cto detect a Keith> return with psr.dd set and issue loadrs to ensure that the Keith> RSE problem does not bite us.I'm still not sure what's triggering the spurious fault:
(1) The RSE activity generated for restoring the current frame, or
(2) The RSE activity generated by the alloc instruction that eventually follows the instruction to which "rfi" returns to.
It has to be (1). psr.dd is getting cleared before the st8 is being executed. Just to be sure, I inserted 20 bundles of nop before and after 'victim = 1;', the problem still occurs.
Quoted 7 lines
Unfortunately, the description of Erratum 11 isn't terribly clear to me. It says:
... The rfi instruction is followed by additional instructions that generate register stack (RSE) activity (alloc, flushrs, br.ret).
Quoted 6 lines
I assume "followed" is referring to execution order, not program order (i.e., inserting nop's around the "rfi" isn't going to do us any good). In your (original) case, the "rfi" is followed by a store instruction, br.call, then alloc. From reading the description, I wouldn't have expected this to trigger the Erratum, but perhaps it does. Can someone clarify?
Agreed that it is unclear. Given the workaround for errata 11 (add 4 nop bundles around rfi) I interpret errata 11 as an instruction read ahead problem, where RSE instructions in the bundles after rfi could be read, decoded and trip RSE activity even though they were not executed. In any case, I added 20 nop bundles around the trapping instruction and it made no difference.
Arch volume 2, section 5.3 is quite explicit, "A non-faulting mandatory RSE load will clear PSR.da and PSR.dd". This does not appear to be a problem when returning to user space with these bits set, probably because the context switch does loadrs. AFAICT it is only an issue when using hardware debug registers on the kernel itself. traps.c has to detect that it is returning to kernel state with either of these bits set and issue loadrs to ensure that the rfi does not require RSE loads. Patch in progress.
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line · Keith Owens
Keith> Arch volume 2, section 5.3 is quite explicit, "A non-faulting Keith> mandatory RSE load will clear PSR.da and PSR.dd".
Ah, I see, that clears it up, then. I had forgotten about this behavior. No mystery then.
Thanks,
--david
Quoted 1 line
Arch volume 2, section 5.3 is quite explicit, "A non-faulting
mandatory
Quoted 8 lines
RSE load will clear PSR.da and PSR.dd". This does not appear to be a problem when returning to user space with these bits set, probably because the context switch does loadrs. AFAICT it is only an issue when using hardware debug registers on the kernel itself.
traps.chas to detect that it is returning to kernel state with either of these bits set and issue loadrs to ensure that the rfi does not require RSE loads. Patch in progress.
I think you meant the kernel return path (and not context switch) while referring to loadrs above. It is true that loadrs operation is done only while transitioning to user space from kernel and not when you are going back to kernel mode. So for the kernel->kernel case rfi can lead to mandatory RSE activity and thus clearing the da and dd bits.
Quoted 1 line
-