kernel oops due to unaligned access with lswi

Alan,

2.6 has a function parse_header(), its part of gunzip and used for initramfs as example. I got an oops, exception 600, sig 7. I think that means unaligned access.

Why does the compiler optimize this one? I see it with 2.95, 3.2, 3.3 and 3.4 with -O1 and better, not with -O0.

gcc -msoft-float -mmultiple -mstring -O2 -Wall -Wstrict-prototypes \
-Wno-trigraphs -Wno-uninitialized -version -fno-strict-aliasing \
-fno-common -ffixed-r2 -fomit-frame-pointer -o parse_header \
-c parse_header.c -v --save-temps

	lswi 9,31,8
	stswi 9,28,8

s = r31. How can gcc be sure that s aligned?
ppc64 will call memcpy (tested with gcc3.2.3).

	.file	"parse_header.c"
	.section	.init.text,"ax",@progbits
	.align 2
	.type	parse_header,@function
parse_header:
	stwu 1,-112(1)
	mflr 0
	stmw 27,92(1)
	stw 0,116(1)
	li 0,0
	stb 0,64(1)
	li 30,0
	addi 31,3,6
	addi 28,1,56
	addi 27,1,8
.L6:
	lswi 9,31,8
	stswi 9,28,8
	slwi 29,30,2
	mr 3,28
	li 4,0
	li 5,16
	bl simple_strtoul
	stwx 3,29,27
	addi 30,30,1
	addi 31,31,8
	cmpwi 0,30,11
	ble+ 0,.L6
	lwz 0,116(1)
	mtlr 0
	lmw 27,92(1)
	addi 1,1,112
	blr
.Lfe1:
	.size	parse_header,.Lfe1-parse_header
	.section	".text"
	.align 2
	.globl main
	.type	main,@function
main:
	stwu 1,-16(1)
	mflr 0
	stw 0,20(1)
	lis 3,0x1
	ori 3,3,57920
	bl malloc
	bl parse_header
	li 3,0
	lwz 0,20(1)
	mtlr 0
	addi 1,1,16
	blr
.Lfe2:
	.size	main,.Lfe2-main
	.ident	"GCC: (GNU) 3.2.3 (SuSE Linux)"

#include <stdlib.h>
typedef unsigned int __kernel_size_t;
extern void *memcpy(void *, const void *, __kernel_size_t); extern unsigned long simple_strtoul(const char *, char **, unsigned int);

static void __attribute__ ((__section__(".init.text"))) parse_header(char *s)

{
	unsigned long parsed[12];
	char buf[9];
	int i;

	buf[8] = '\0';
	for (i = 0, s += 6; i < 12; i++, s += 8) {
		memcpy(buf, s, 8);
		parsed[i] = simple_strtoul(buf, ((void *) 0), 16);
	}
}

int
main(void)

{
	char *s;
	s = malloc(123456);
	parse_header(s);
	return 0;
}

--
USB is for mice, FireWire is for men!

sUse lINUX ag, nÜRNBERG

** Sent via the linuxppc-dev mail list. See http://lists.linuxppc.org/

This might be a useable workaround.

@@ -100,11 +100,11 @@ static void __init parse_header(char *s)
 {
        unsigned long parsed[12];
        char buf[9];
-       int i;
+       int i, j = 1;
 
        buf[8] = '\0';
        for (i = 0, s += 6; i < 12; i++, s += 8) {
-               memcpy(buf, s, 8);
+               memcpy(buf, s, 7 + j); /* s might be unaligned, gcc will optimized the call to lswi on ppc */
                parsed[i] = simple_strtoul(buf, NULL, 16);
        }
        ino = parsed[0];

--
USB is for mice, FireWire is for men!

sUse lINUX ag, nÜRNBERG

** Sent via the linuxppc-dev mail list. See http://lists.linuxppc.org/

Olaf> 2.6 has a function parse_header(), its part of gunzip and used for Olaf> initramfs as example. I got an oops, exception 600, sig 7. I think that Olaf> means unaligned access.

Olaf> + memcpy(buf, s, 7 + j); /* s might be unaligned, gcc will optimized the call to lswi on ppc */

lswi specifically accepts unaligned addresses.

david

** Sent via the linuxppc-dev mail list. See http://lists.linuxppc.org/

Thanks. Then the bug is somewhere else, because it crashes like this if I use '8' instead of '7 + j' (no memcpy call):

Calibrating delay loop... 89.49 BogoMIPS
Dentry cache hash table entries: 16384 (order: 4, 65536 bytes) Inode-cache hash table entries: 8192 (order: 3, 32768 bytes) Mount-cache hash table entries: 512 (order: 0, 4096 bytes) Serial port locked ON by debugger !
vector: 600 at pc = c025bf14, lr = c025bf20
msr = 9030, sp = c02598c0 [c0259810]
dar = c043cffa, dsisr = a53f
current = c02191c8, pid = 0, comm = swapper
mon> di c025bf14

c025bf14  7d3f44aa      lswi    r9,r31,8
c025bf18  7d3c45aa      stswi   r9,r28,8
c025bf1c  4be62ee1      bl      0xc00bedfc
c025bf20  2c1e000b      cmpwi   r30,11
c025bf24  3bff0008      addi    r31,r31,8
c025bf28  7c7dd92e      stwx    r3,r29,r27
c025bf2c  4081ffd4      ble     0xc025bf00
c025bf30  8001002c      lwz     r0,44(r1)
c025bf34  81210030      lwz     r9,48(r1)
c025bf38  5400a016      rlwinm  r0,r0,20,0,11
c025bf3c  81610008      lwz     r11,8(r1)
c025bf40  7c004b78      or      r0,r0,r9
c025bf44  5409a32e      rlwinm  r9,r0,20,12,23
c025bf48  540a063e      clrlwi  r10,r0,24
c025bf4c  7d4a4b78      or      r10,r10,r9
c025bf50  54006016      rlwinm  r0,r0,12,0,11
mon> r
R00 = 00000008   R01 = c02598c0   R02 = c02191c8   R03 = c02598f8
R04 = 00000000   R05 = 00000010   R06 = c043cfb1   R07 = 00000001
R08 = 00000000   R09 = c02224f4   R10 = ffffffd0   R11 = 00000000
R12 = 00014fac   R13 = deadbeef   R14 = deadbeef   R15 = deadbeef
R16 = deadbeef   R17 = deadbeef   R18 = deadbeef   R19 = deadbeef
R20 = 0000003f   R21 = 000001ff   R22 = c47f1408   R23 = c032e008
R24 = 00000006   R25 = 00000009   R26 = 0000000f   R27 = c02598c8
R28 = c02598f8   R29 = 00000024   R30 = 0000000a   R31 = c043cffa
pc  = c025bf14   msr = 00009030   lr  = c025bf20   cr  = 95000053
ctr = 00000000   xer = c000be6f   trap =  600

mon>

pmac 7200/90 with 601 cpu.

--
USB is for mice, FireWire is for men!

sUse lINUX ag, nÜRNBERG

** Sent via the linuxppc-dev mail list. See http://lists.linuxppc.org/

The asm diff looks like that:

@@ -165,16 +165,18 @@
        addi 28,1,56
        addi 27,1,8
 .L85:
+       mr 4,31
+       li 5,8
+       mr 3,28
        slwi 29,30,2
+       bl memcpy
+       addi 30,30,1
        mr 3,28
        li 4,0
        li 5,16
-       addi 30,30,1
-       lswi 9,31,8
-       stswi 9,28,8
+       addi 31,31,8
        bl simple_strtoul
        cmpwi 0,30,11
-       addi 31,31,8
        stwx 3,29,27
        ble+ 0,.L85
        lwz 0,44(1)

--
USB is for mice, FireWire is for men!

sUse lINUX ag, nÜRNBERG

** Sent via the linuxppc-dev mail list. See http://lists.linuxppc.org/

I didn't mean that lswi cannot take an alignment exception on some
PPC implementations, but that lswi is suppose to be able to handle block loads from addresses with arbitrary alignment.

David

** Sent via the linuxppc-dev mail list. See http://lists.linuxppc.org/

I remember beeing regulary told (I think by Apple while I was still doing MacOS hacking) that those string instructions were evil, deprecated, and should be avoided as they weren't peforming better than the equivalent set of load/store instructions... Is this still true ? In which case we may want to avoid generating them from gcc..

Also, if the 601 effectively gets alignement exceptions on these, it's quite bad to have them implicitely generated by gcc for memcpy's since our OFs seem to not implement the alignement handler for them, thus breaking our boot wrappers.

Finally, the pem32b at least seem to be clear about not encouraging to use these especially on non-aligned accesses. It looks like a weird optimisation to do for memcpy...

Ben.

** Sent via the linuxppc-dev mail list. See http://lists.linuxppc.org/

If Ben's comments are correct simply removing -mstring as an option passed to the build should get the desired behavior.

- kumar

** Sent via the linuxppc-dev mail list. See http://lists.linuxppc.org/

Yes. I can't see any problem with gcc's behaviour here, and I'm surprised that some processor is taking alignment exceptions on lswi. book3 ppcas says lswi will generate an alignment exception when "the operand is in storage that is Write Through Required or Caching Inhibited, or the processor is in Little-Endian mode". It can also happen for operands that cross segment boundaries or page boudaries with different attributes.

--
Alan Modra
IBM OzLabs - Linux Technology Centre

** Sent via the linuxppc-dev mail list. See http://lists.linuxppc.org/

Alan Modra wrote:

| On Sun, Nov 16, 2003 at 11:49:32AM -0600, Kumar Gala wrote:
|
|>If Ben's comments are correct simply removing -mstring as an option
|>passed to the build should get the desired behavior.
|
|
| Yes.  I can't see any problem with gcc's behaviour here, and I'm
| surprised that some processor is taking alignment exceptions on lswi.

The 601 reputedly broke a number of the PowerPC specifications however so I am not surprised if it has this problem - the trouble is you can fix up the alignment exception handler in Linux but not the firmware handlers being used prior to that. Anyway people suggested workarounds.

Jon.

** Sent via the linuxppc-dev mail list. See http://lists.linuxppc.org/

Well, firstly PPCAS didn't exist when the 601 was designed, and secondly the architecture allows implementations to consist of a mix of hardware and software - meaning that hardware can take an exception on any condition it likes and expect software to fix it up (provided hardware gives software enough information to fix it up, etc.).

The original PPC architecture talks about one of the reasons for an alignment interrupt being that "the operand of an elementary string load or store crosses a protection boundary". And the 601 manual says that a string load or store (except lscbx) will only cause an interrupt if it crosses a page boundary and it is not word aligned. Olaf, do we know what the source and destination addresses were?

Paul.

** Sent via the linuxppc-dev mail list. See http://lists.linuxppc.org/

source was c0fe4ffa, I cant reproduce it righht now to get the destination address.

--
USB is for mice, FireWire is for men!

sUse lINUX ag, nÜRNBERG

** Sent via the linuxppc-dev mail list. See http://lists.linuxppc.org/

We surely don't want them on G5 at least as they are microcoded

Ben.

** Sent via the linuxppc-dev mail list. See http://lists.linuxppc.org/

Ben> We surely don't want them on G5 at least as they are microcoded

Again, one cannot approach this as black or white. What about
optimizing for size?

david

** Sent via the linuxppc-dev mail list. See http://lists.linuxppc.org/

Using lswi/stswi is fine when optimizing for size, but I think that using these instructions for the case of an 8 byte move is wrong in most cases because an additional instruction is often (but not always) needed to compute the address.

Moving 8 bytes with lswi/stswi:

	la rx,src # compute source address
	lswi 5,rx,8
	la ry, dst# compute destination address
	stswi 5,ry,8

Moving 8 bytes with standard instructions:

lwz rx,src
lwz ry,src+4
stw rx,dst
stw ry,dst+4

IMHO, lswi/stswi should only be used when the move would be split into 3 or more l[bhw]z/st[bhw] pairs (i.e., for sizes of 7 or 9 and up).

Regards,
Gabriel

** Sent via the linuxppc-dev mail list. See http://lists.linuxppc.org/

Ben> I remember beeing regulary told (I think by Apple while I was still Ben> doing MacOS hacking) that those string instructions were evil, Ben> deprecated, and should be avoided as they weren't peforming better Ben> than the equivalent set of load/store instructions... Is this Ben> still true ? In which case we may want to avoid generating them Ben> from gcc..

The information that you received about lwsi are overly
simplistic. The instructions are neither overly good nor overly bad -- they should not be used for everything, but neither should they be avoided at all cost. They are particularly good for producing compact code and preserving the instruction cache. Remember, programming, including assembly language programming, is an art.

David

** Sent via the linuxppc-dev mail list. See http://lists.linuxppc.org/

In the experiments that I did comparing different memcpy loops, I didn't find any combination of alignment, size of copy and processor implementation where using string loads/stores was as fast as than ordinary loads and stores. So I am inclined to think that the only advantage of the string load/store instructions is in saving a little bit of icache.

Paul.

** Sent via the linuxppc-dev mail list. See http://lists.linuxppc.org/

Back in ye olde dayes, these insns were way better (by a factor of 3x) for doing load/stores to i/o space, cause they could pump out a word every bus cycle as opposed to every 3 cycles (due to pipeline stalls).

I guess things like G5, etc. now have enough load/store units and etc. hardware that this is no longer an issue? So that if I wanted to, I could PIO fast enough to e.g. keep a pci bus saturated? (We used PIO in ye olde days for dynamic data that would have gone stale by the time a dma was set up and run.) Just curious, these insn's used to be freinds, not enemies.

--linas

** Sent via the linuxppc-dev mail list. See http://lists.linuxppc.org/

I think they've always been the enemy of the CPU designers. At least that's how it was explained to me from a Motorolan, and I assume the same is true for IBM. And they're the ones writing the CPU manuals telling people not to use the instructions... :)

--
Hollis Blanchard
IBM Linux Technology Center

** Sent via the linuxppc-dev mail list. See http://lists.linuxppc.org/

Depends on whose CPU designer camp you visit.

DEC Alpha strategy: pump up the clock; minimize number of gate
delays between start of insn cycle and end of cycle. Less gate delay == faster clock.

Ye olde superscalar strategy: do more per clock cycle by deploying
more transistors (even if one must have slower clock as a result.)

The load string insn needs a big fat shift register with oodles of gate delays right in the middle of the load/store path. No other
insn's need or use this register. Getting rid of it allows you
to pump up the clock.

The original POWER cpu designers clearly thought it was a worthwhile tradeoff, otherwise it wouldn't have been in the insn set to begin with. But the alpha camp sure made a clear and ringing point ...

--linas

** Sent via the linuxppc-dev mail list. See http://lists.linuxppc.org/

Olaf,

What machine is this? I looked at the manuals for 750, 7450, POWER4 and they all handle unaligned string ops in hardware. The alignment handler doesn't handle string ops, I believe, although it could. And which arch (ppc32 or ppc64)?

Paul.

** Sent via the linuxppc-dev mail list. See http://lists.linuxppc.org/

Its a 7200/90 with 601 cpu. And I'm afraid, the zlib.c needs also tweaking. I think the gcc built-in memcpy is used in the bootloader. Same issue, 'DEFAULT CATCH!, code=FFF00600' without this change (adds also zlib debugging, but doesnt work for prepboot right now, if enabled).

@@ -1,3 +1,5 @@
+#define DEBUG_ZLIB 1
+#define verbose 1
 /*
  * This file is derived from various .h and .c files from the zlib-0.95
  * distribution by Jean-loup Gailly and Mark Adler, with some additions

@@ -85,11 +87,11 @@ extern char *z_errmsg[]; /* indexed by 1
 
 /* Diagnostic functions */
 #ifdef DEBUG_ZLIB
-#  include <stdio.h>
+#  include <nonstdio.h>
 #  ifndef verbose
 #    define verbose 0
 #  endif
-#  define Assert(cond,msg) {if(!(cond)) z_error(msg);}
+#  define Assert(cond,msg) {if(!(cond)) printf(msg);}
 #  define Trace(x) fprintf x
 #  define Tracev(x) {if (verbose) fprintf x ;}
 #  define Tracevv(x) {if (verbose>1) fprintf x ;}

@@ -884,7 +886,14 @@ local int inflate_blocks(
       t = s->sub.left;
       if (t > n) t = n;
       if (t > m) t = m;
+#if 0
       zmemcpy(q, p, t);
+#else
+      {
+       int i;
+       for(i=0;i <t;i++)q[i]=p[i];
+      }
+#endif
       p += t;  n -= t;
       q += t;  m -= t;
       if ((s->sub.left -= t) != 0)

@@ -1230,7 +1239,7 @@ local uInt cpdext[] = { /* Extra bits fo
 #define N_MAX 288       /* maximum number of codes in any set */
 
 #ifdef DEBUG_ZLIB
-  uInt inflate_hufts;
+  local uInt inflate_hufts;
 #endif
 
 local int huft_build(

--
USB is for mice, FireWire is for men!

sUse lINUX ag, nÜRNBERG

** Sent via the linuxppc-dev mail list. See http://lists.linuxppc.org/

This patch allows zlib debugging, defines a dummy printf for prep, prints all 4 bytes of a pointer in the coff bootloader.

diff -x bin -x ash -x klibc-0.81 -purNX /home/olaf/kernel/kernel_exclude.txt linuxppc-2.5_2.6.0-test9-bk.orig/arch/ppc/boot/common/misc-common.c linuxppc-2.5_2.6.0-test9-bk/arch/ppc/boot/common/misc-common.c
@@ -67,6 +67,8 @@ extern unsigned char serial_getc(unsigne
 extern void serial_putc(unsigned long com_port, unsigned char c);
 #endif
 
+int printf(const char *fmt, ...) { return 0; }
+
 void pause(void)
 {
 	puts("pause\n");

diff -x bin -x ash -x klibc-0.81 -purNX /home/olaf/kernel/kernel_exclude.txt linuxppc-2.5_2.6.0-test9-bk.orig/arch/ppc/boot/openfirmware/coffmain.c linuxppc-2.5_2.6.0-test9-bk/arch/ppc/boot/openfirmware/coffmain.c
@@ -60,7 +60,7 @@ void boot(int a1, int a2, void *prom)
 	a1 = initrd_start;
 	a2 = initrd_size;
 	claim(initrd_start, ram_end - initrd_start, 0);
-	printf("initial ramdisk moving 0x%x <- 0x%p (%x bytes)\n\r",
+	printf("initial ramdisk moving 0x%08x <- 0x%p (%08x bytes)\n\r",
 	       initrd_start, (char *)(&__ramdisk_begin), initrd_size);
 	memcpy((char *)initrd_start, (char *)(&__ramdisk_begin), initrd_size);
 	prog_size = initrd_start - prog_start;

--
USB is for mice, FireWire is for men!

sUse lINUX ag, nÜRNBERG

** Sent via the linuxppc-dev mail list. See http://lists.linuxppc.org/

One of your old friends actually:

olaf> pmac 7200/90 with 601 cpu.

:)

Ben.

** Sent via the linuxppc-dev mail list. See http://lists.linuxppc.org/