Skip to contentSkip to conversation

realmode test fixes for clang

PATCH Superseded Bill Wendling 3 participants 14 messages Oct 13, 2019 – Oct 21, 2019 KVM

lore
Share & tools
lore
#0 lore raw

cover realmode test fixes for clang

The following patches fix some realmode test compatibility issues between gcc and clang.

Please look at the "x86: realmode: use inline asm to get stack pointer" patch closely. I'm fairly certain it's the correct fix, but would like confirmation.

Thanks!

Bill Wendling (2):
x86: realmode: explicitly copy structure to avoid memcpy
x86: realmode: use inline asm to get stack pointer

 x86/realmode.c | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)
Signature · 1 line
2.23.0.700.g56cf767bdb-goog
#1 lore raw

PATCH 1/2 x86: realmode: explicitly copy structure to avoid memcpy

Clang prefers to use a "mempcy" (or equivalent) to copy the "regs" structure. This doesn't work in 16-bit mode, as it will end up copying over half the number of bytes. GCC performs a field-by-field copy of the structure, so force clang to do the same thing.

---
 x86/realmode.c | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)
x86/realmode.c +16 -3
index 303d093..cf45fd6 100644
@@ -117,6 +117,19 @@ struct regs {
 	u32 eip, eflags;
 };
 
+#define COPY_REG(name, dst, src) (dst).name = (src).name
+#define COPY_REGS(dst, src)				\
+	COPY_REG(eax, dst, src);			\
+	COPY_REG(ebx, dst, src);			\
+	COPY_REG(ecx, dst, src);			\
+	COPY_REG(edx, dst, src);			\
+	COPY_REG(esi, dst, src);			\
+	COPY_REG(edi, dst, src);			\
+	COPY_REG(esp, dst, src);			\
+	COPY_REG(ebp, dst, src);			\
+	COPY_REG(eip, dst, src);			\
+	COPY_REG(eflags, dst, src)
+
 struct table_descr {
 	u16 limit;
 	void *base;
@@ -148,11 +161,11 @@ static void exec_in_big_real_mode(struct insn_desc *insn)
 	extern u8 test_insn[], test_insn_end[];
 
 	for (i = 0; i < insn->len; ++i)
-	    test_insn[i] = ((u8 *)(unsigned long)insn->ptr)[i];
+		test_insn[i] = ((u8 *)(unsigned long)insn->ptr)[i];
 	for (; i < test_insn_end - test_insn; ++i)
 		test_insn[i] = 0x90; // nop
 
-	save = inregs;
+	COPY_REGS(save, inregs);
 	asm volatile(
 		"lgdtl %[gdt_descr] \n\t"
 		"mov %%cr0, %[tmp] \n\t"
@@ -196,7 +209,7 @@ static void exec_in_big_real_mode(struct insn_desc *insn)
 		: [gdt_descr]"m"(gdt_descr), [bigseg]"r"((short)16)
 		: "cc", "memory"
 		);
-	outregs = save;
+	COPY_REGS(outregs, save);
 }
 
 #define R_AX 1
Signature · 1 line
2.23.0.700.g56cf767bdb-goog
#2 lore raw

Re: [kvm-unit-tests PATCH 1/2] x86: realmode: explicitly copy structure to avoid memcpy

Quoted 22 lines · Bill Wendling

Clang prefers to use a "mempcy" (or equivalent) to copy the "regs" structure. This doesn't work in 16-bit mode, as it will end up copying over half the number of bytes. GCC performs a field-by-field copy of the structure, so force clang to do the same thing.

---
 x86/realmode.c | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)
x86/realmode.c +3 -0
index 303d093..cf45fd6 100644
@@ -117,6 +117,19 @@ struct regs {
        u32 eip, eflags;
 };
 
+#define COPY_REG(name, dst, src) (dst).name = (src).name
+#define COPY_REGS(dst, src)                            \
+       COPY_REG(eax, dst, src);                        \

x86/realmode.c struct regs { · line 132

Quoted 10 lines
x86/realmode.c +10 -0
+       COPY_REG(ebx, dst, src);                        \
+       COPY_REG(ecx, dst, src);                        \
+       COPY_REG(edx, dst, src);                        \
+       COPY_REG(esi, dst, src);                        \
+       COPY_REG(edi, dst, src);                        \
+       COPY_REG(esp, dst, src);                        \
+       COPY_REG(ebp, dst, src);                        \
+       COPY_REG(eip, dst, src);                        \
+       COPY_REG(eflags, dst, src)
+

This seems very fragile, too. Can we introduce our own address-space-size-independent "memcpy" and use that?

I'm thinking something like:

static void bytecopy(void *dst, void *src, u32 count) {
asm volatile("rep movsb"
: "+D" (dst), "+S" (src), "+c" (count) : : "cc");
}

Quoted 29 lines
x86/realmode.c +3 -4
 struct table_descr {
        u16 limit;
        void *base;
@@ -148,11 +161,11 @@ static void exec_in_big_real_mode(struct insn_desc *insn)
        extern u8 test_insn[], test_insn_end[];
 
        for (i = 0; i < insn->len; ++i)
-           test_insn[i] = ((u8 *)(unsigned long)insn->ptr)[i];
+               test_insn[i] = ((u8 *)(unsigned long)insn->ptr)[i];
        for (; i < test_insn_end - test_insn; ++i)
                test_insn[i] = 0x90; // nop
 
-       save = inregs;
+       COPY_REGS(save, inregs);
        asm volatile(
                "lgdtl %[gdt_descr] \n\t"
                "mov %%cr0, %[tmp] \n\t"
@@ -196,7 +209,7 @@ static void exec_in_big_real_mode(struct insn_desc *insn)
                : [gdt_descr]"m"(gdt_descr), [bigseg]"r"((short)16)
                : "cc", "memory"
                );
-       outregs = save;
+       COPY_REGS(outregs, save);
 }
 
 #define R_AX 1
--

2.23.0.700.g56cf767bdb-goog

maintainer
#3 lore raw

Re: [kvm-unit-tests PATCH 1/2] x86: realmode: explicitly copy structure to avoid memcpy

In reply to Bill Wendling
Quoted 4 lines · Bill Wendling

Clang prefers to use a "mempcy" (or equivalent) to copy the "regs" structure. This doesn't work in 16-bit mode, as it will end up copying over half the number of bytes. GCC performs a field-by-field copy of the structure, so force clang to do the same thing.

Do you mean that:

1) the compiler is compiling a "call memcpy", where the bytes "rep movsl" are interpreted as "rep movsw"

2) the compiler's integrated assembler is including the bytes for "rep movsl", but in 16-bit mode they are "rep movsw"

3) something else

The -m16 or -no-integrated-as flags perhaps can help?

Paolo

Quoted 53 lines
---
 x86/realmode.c | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)
x86/realmode.c +16 -3
index 303d093..cf45fd6 100644
@@ -117,6 +117,19 @@ struct regs {
 	u32 eip, eflags;
 };
 
+#define COPY_REG(name, dst, src) (dst).name = (src).name
+#define COPY_REGS(dst, src)				\
+	COPY_REG(eax, dst, src);			\
+	COPY_REG(ebx, dst, src);			\
+	COPY_REG(ecx, dst, src);			\
+	COPY_REG(edx, dst, src);			\
+	COPY_REG(esi, dst, src);			\
+	COPY_REG(edi, dst, src);			\
+	COPY_REG(esp, dst, src);			\
+	COPY_REG(ebp, dst, src);			\
+	COPY_REG(eip, dst, src);			\
+	COPY_REG(eflags, dst, src)
+
 struct table_descr {
 	u16 limit;
 	void *base;
@@ -148,11 +161,11 @@ static void exec_in_big_real_mode(struct insn_desc *insn)
 	extern u8 test_insn[], test_insn_end[];
 
 	for (i = 0; i < insn->len; ++i)
-	    test_insn[i] = ((u8 *)(unsigned long)insn->ptr)[i];
+		test_insn[i] = ((u8 *)(unsigned long)insn->ptr)[i];
 	for (; i < test_insn_end - test_insn; ++i)
 		test_insn[i] = 0x90; // nop
 
-	save = inregs;
+	COPY_REGS(save, inregs);
 	asm volatile(
 		"lgdtl %[gdt_descr] \n\t"
 		"mov %%cr0, %[tmp] \n\t"
@@ -196,7 +209,7 @@ static void exec_in_big_real_mode(struct insn_desc *insn)
 		: [gdt_descr]"m"(gdt_descr), [bigseg]"r"((short)16)
 		: "cc", "memory"
 		);
-	outregs = save;
+	COPY_REGS(outregs, save);
 }
 
 #define R_AX 1
#4 lore raw

PATCH 2/2 x86: realmode: use inline asm to get stack pointer

In reply to Bill Wendling

It's fragile to try to retrieve the stack pointer by taking the address of a variable on the stack. For instance, clang reserves more stack space than gcc here, indicating that the variable may not be at the start of the stack. Instead of relying upon this to work, retrieve the "%rbp" value, which contains the value of "%rsp" before stack allocation.

---
 x86/realmode.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)
x86/realmode.c +3 -2
index cf45fd6..7c89dd1 100644
@@ -518,11 +518,12 @@ extern void retf_imm(void);
 
 static void test_call(void)
 {
-	u32 esp[16];
 	u32 addr;
 
 	inregs = (struct regs){ 0 };
-	inregs.esp = (u32)esp;
+
+	// At this point the original stack pointer is in %ebp.
+	asm volatile ("mov %%ebp, %0" : "=rm"(inregs.esp));
 
 	MK_INSN(call1, "mov $test_function, %eax \n\t"
 		       "call *%eax\n\t");
Signature · 1 line
2.23.0.700.g56cf767bdb-goog
#5 lore raw

Re: [kvm-unit-tests PATCH 2/2] x86: realmode: use inline asm to get stack pointer

Quoted 19 lines · Bill Wendling

It's fragile to try to retrieve the stack pointer by taking the address of a variable on the stack. For instance, clang reserves more stack space than gcc here, indicating that the variable may not be at the start of the stack. Instead of relying upon this to work, retrieve the "%rbp" value, which contains the value of "%rsp" before stack allocation.

---
 x86/realmode.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)
x86/realmode.c +0 -0
index cf45fd6..7c89dd1 100644
@@ -518,11 +518,12 @@ extern void retf_imm(void);
 

x86/realmode.c extern void retf_imm(void); · line 526

Quoted 10 lines
x86/realmode.c +3 -2
 static void test_call(void)
 {
-       u32 esp[16];
        u32 addr;
 
        inregs = (struct regs){ 0 };
-       inregs.esp = (u32)esp;
+
+       // At this point the original stack pointer is in %ebp.
+       asm volatile ("mov %%ebp, %0" : "=rm"(inregs.esp));

I don't think we should assume the existence of frame pointers. Moreover, I think %esp is actually the value that should be saved here, regardless of how large the current frame is.

Quoted 5 lines
x86/realmode.c +0 -1
        MK_INSN(call1, "mov $test_function, %eax \n\t"
                       "call *%eax\n\t");
--

2.23.0.700.g56cf767bdb-goog

#6 lore raw

Re: [kvm-unit-tests PATCH 2/2] x86: realmode: use inline asm to get stack pointer

Quoted 26 lines · Jim Mattson
Quoted 24 lines · Bill Wendling

It's fragile to try to retrieve the stack pointer by taking the address of a variable on the stack. For instance, clang reserves more stack space than gcc here, indicating that the variable may not be at the start of the stack. Instead of relying upon this to work, retrieve the "%rbp" value, which contains the value of "%rsp" before stack allocation.

---
 x86/realmode.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)
x86/realmode.c +0 -1
index cf45fd6..7c89dd1 100644
@@ -518,11 +518,12 @@ extern void retf_imm(void);
 
 static void test_call(void)
 {
-       u32 esp[16];
        u32 addr;
 
Quoted 9 lines

x86/realmode.c extern void retf_imm(void); · line 526

Quoted 5 lines
x86/realmode.c +3 -1
        inregs = (struct regs){ 0 };
-       inregs.esp = (u32)esp;
+
+       // At this point the original stack pointer is in %ebp.
+       asm volatile ("mov %%ebp, %0" : "=rm"(inregs.esp));

I don't think we should assume the existence of frame pointers. Moreover, I think %esp is actually the value that should be saved here, regardless of how large the current frame is.

Never mind. After taking a closer look, esp[] is meant to provide stack space for the code under test, but inregs.esp should point to the top of this stack rather than the bottom. This is apparently a long-standing bug, similar to the one Avi fixed for test_long_jmp() in commit 4aa22949 ("realmode: fix esp in long jump test").

For consistency with test_long_jmp, I'd suggest changing the inregs.esp assignment to:

       inregs.esp = (u32)(esp+16);

Note that you absolutely must preserve the esp[] array!

Quoted 5 lines
Quoted 5 lines
x86/realmode.c +0 -1
        MK_INSN(call1, "mov $test_function, %eax \n\t"
                       "call *%eax\n\t");
--

2.23.0.700.g56cf767bdb-goog

maintainer
#7 lore raw

Re: [kvm-unit-tests PATCH 2/2] x86: realmode: use inline asm to get stack pointer

Quoted 6 lines · Jim Mattson

Never mind. After taking a closer look, esp[] is meant to provide stack space for the code under test, but inregs.esp should point to the top of this stack rather than the bottom. This is apparently a long-standing bug, similar to the one Avi fixed for test_long_jmp() in commit 4aa22949 ("realmode: fix esp in long jump test").

Quoted 6 lines

For consistency with test_long_jmp, I'd suggest changing the inregs.esp assignment to:

       inregs.esp = (u32)(esp+16);

Note that you absolutely must preserve the esp[] array!

Agreed, thanks Jim for the review.

Paolo

#8 lore raw

v2 cover realmode test fixes for clang

In reply to Bill Wendling

The following patches fix some realmode test compatibility issues between gcc and clang.

This update should address the comments by Jim Mattson.

Re explicitly copying the regs structure, I tried to use the inline asm code you suggested, but wasn't able to get it to work for me. It never seemed to copy the structure correctly. Doing the loop explicitly solved the issue.

Bill Wendling (2):
x86: realmode: explicitly copy regs structure
x86: realmode: fix esp in call test

 x86/realmode.c | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)
Signature · 1 line
2.23.0.700.g56cf767bdb-goog
#9 lore raw

PATCH v2 1/2 x86: realmode: explicitly copy regs structure

Clang prefers to use a "rep;movsl" (or equivalent) to copy the "regs" structure. This doesn't work in 16-bit mode, as it will end up copying over half the number of bytes. Avoid this by copying over the structure a byte at a time.

---
 x86/realmode.c | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)
x86/realmode.c +14 -3
index 303d093..41b8592 100644
@@ -140,6 +140,16 @@ struct insn_desc {
 
 static struct regs inregs, outregs;
 
+static inline void copy_regs(struct regs *dst_regs, struct regs *src_regs)
+{
+	char *dst = (char*)dst_regs;
+	char *src = (char*)src_regs;
+	u32 i;
+
+	for (i = 0; i < sizeof(struct regs); i++)
+		dst[i] = src[i];
+}
+
 static void exec_in_big_real_mode(struct insn_desc *insn)
 {
 	unsigned long tmp;
@@ -148,11 +158,11 @@ static void exec_in_big_real_mode(struct insn_desc *insn)
 	extern u8 test_insn[], test_insn_end[];
 
 	for (i = 0; i < insn->len; ++i)
-	    test_insn[i] = ((u8 *)(unsigned long)insn->ptr)[i];
+		test_insn[i] = ((u8 *)(unsigned long)insn->ptr)[i];
 	for (; i < test_insn_end - test_insn; ++i)
 		test_insn[i] = 0x90; // nop
 
-	save = inregs;
+	copy_regs(&save, &inregs);
 	asm volatile(
 		"lgdtl %[gdt_descr] \n\t"
 		"mov %%cr0, %[tmp] \n\t"
@@ -196,7 +206,8 @@ static void exec_in_big_real_mode(struct insn_desc *insn)
 		: [gdt_descr]"m"(gdt_descr), [bigseg]"r"((short)16)
 		: "cc", "memory"
 		);
-	outregs = save;
+	copy_regs(&outregs, &save);
+
 }
 
 #define R_AX 1
Signature · 1 line
2.23.0.700.g56cf767bdb-goog
#10 lore raw

PATCH v2 2/2 x86: realmode: fix esp in call test

In reply to Bill Wendling

esp needs to point at the end of the stack, or it will corrupt memory.

This is a port of Avi Kivity patch for the long jump test:
4aa229495b0e4159642b4a77e9adfdc81501c095.

---
 x86/realmode.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
x86/realmode.c +1 -1
index 41b8592..f318910 100644
@@ -520,7 +520,7 @@ static void test_call(void)
 	u32 addr;
 
 	inregs = (struct regs){ 0 };
-	inregs.esp = (u32)esp;
+	inregs.esp = (u32)(esp+16);
 
 	MK_INSN(call1, "mov $test_function, %eax \n\t"
 		       "call *%eax\n\t");
Signature · 1 line
2.23.0.700.g56cf767bdb-goog
maintainer
#12 lore raw

Re: [kvm-unit-tests v2 PATCH 2/2] x86: realmode: fix esp in call test

In reply to Bill Wendling

x86/realmode.c static void test_call(void) · line 523

Quoted 10 lines · Bill Wendling
x86/realmode.c +1 -1
index 41b8592..f318910 100644
@@ -520,7 +520,7 @@ static void test_call(void)
 	u32 addr;
 
 	inregs = (struct regs){ 0 };
-	inregs.esp = (u32)esp;
+	inregs.esp = (u32)(esp+16);

Applied with

+ inregs.esp = (u32)&esp[ARRAY_SIZE(esp)];

Paolo

#13 lore raw

Re: [kvm-unit-tests v2 PATCH 2/2] x86: realmode: fix esp in call test

Quoted 15 lines · Paolo Bonzini

x86/realmode.c static void test_call(void) · line 523

Quoted 10 lines · Bill Wendling
x86/realmode.c +1 -1
index 41b8592..f318910 100644
@@ -520,7 +520,7 @@ static void test_call(void)
      u32 addr;
 
      inregs = (struct regs){ 0 };
-     inregs.esp = (u32)esp;
+     inregs.esp = (u32)(esp+16);

Applied with

Quoted 3 lines

+ inregs.esp = (u32)&esp[ARRAY_SIZE(esp)];

Paolo

Would you mind doing the same for test_long_jmp?