Skip to contentSkip to conversation

Avoid additional relocations in trampoline code

PATCH In review Frediano Ziglio 2 participants 5 messages Aug 22, 2024 – Aug 28, 2024

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 Avoid additional relocations in trampoline code

The trampoline could have "manual" relocation entries (created by assembly macros and some code to use them) and (in case of PE) normal executable relocations.
Remove all normal executable ones. In this way we don't have to worry about applying both correctly (they need proper order which is hard to spot looking at the code).

---
 xen/arch/x86/boot/trampoline.S | 4 ++--
 xen/arch/x86/boot/wakeup.S     | 6 +++---
 xen/arch/x86/xen.lds.S         | 7 ++++++-
 3 files changed, 11 insertions(+), 6 deletions(-)
xen/arch/x86/boot/trampoline.S +2 -2
index b8ab0ffdcb..0724c953e2 100644
@@ -73,7 +73,7 @@ trampoline_protmode_entry:
         mov     %ecx,%cr4
 
         /* Load pagetable base register. */
-        mov     $sym_offs(idle_pg_table),%eax
+        mov     $idle_pg_table_pa, %eax
         add     bootsym_rel(trampoline_xen_phys_start,4,%eax)
         mov     %eax,%cr3
 
@@ -113,7 +113,7 @@ trampoline_protmode_entry:
         .code64
 start64:
         /* Jump to high mappings. */
-        movabs  $__high_start, %rdi
+        movabs  $__high_start_pa + __XEN_VIRT_START, %rdi
         jmpq    *%rdi
 
 #include "video.h"
xen/arch/x86/boot/wakeup.S +3 -3
index 08447e1934..ebe72e1fdd 100644
@@ -99,7 +99,7 @@ wakeup_32:
         mov     $bootsym_rel(wakeup_stack, 4, %esp)
 
         # check saved magic again
-        mov     $sym_offs(saved_magic),%eax
+        mov     $saved_magic_pa, %eax
         add     bootsym_rel(trampoline_xen_phys_start, 4, %eax)
         mov     (%eax), %eax
         cmp     $0x9abcdef0, %eax
@@ -112,7 +112,7 @@ wakeup_32:
         mov     %ecx, %cr4
 
         /* Load pagetable base register */
-        mov     $sym_offs(idle_pg_table),%eax
+        mov     $idle_pg_table_pa ,%eax
         add     bootsym_rel(trampoline_xen_phys_start,4,%eax)
         mov     %eax,%cr3
 
@@ -156,7 +156,7 @@ wakeup_32:
         .code64
 wakeup_64:
         /* Jump to high mappings and the higher-level wakeup code. */
-        movabs  $s3_resume, %rbx
+        movabs  $s3_resume_pa + __XEN_VIRT_START, %rbx
         jmp     *%rbx
 
 bogus_saved_magic:
xen/arch/x86/xen.lds.S +6 -1
index 9a1dfe1b34..8e604dde48 100644
@@ -71,7 +71,12 @@ SECTIONS
   __2M_text_start = .;         /* Start of 2M superpages, mapped RX. */
 #endif
 
-  start_pa = ABSOLUTE(start - __XEN_VIRT_START);
+#define DEFINE_PA_ADDRESS(sym) sym ## _pa = ABSOLUTE(sym - __XEN_VIRT_START)
+  DEFINE_PA_ADDRESS(start);
+  DEFINE_PA_ADDRESS(saved_magic);
+  DEFINE_PA_ADDRESS(idle_pg_table);
+  DEFINE_PA_ADDRESS(__high_start);
+  DEFINE_PA_ADDRESS(s3_resume);
 
   . = __XEN_VIRT_START + XEN_IMG_OFFSET;
   _start = .;
Signature · 1 line
2.46.0
#1 lore raw
Quoted 8 lines · Frediano Ziglio

The trampoline could have "manual" relocation entries (created by assembly macros and some code to use them) and (in case of PE) normal executable relocations.
Remove all normal executable ones. In this way we don't have to worry about applying both correctly (they need proper order which is hard to spot looking at the code).

I think this wants splitting into one patch replacing sym_offs() and a 2nd one introducing the hand-crafted __XEN_VIRT_START additions (which may want macro-izing). Plus the justification for the change wants extending, to actually explain what the problem is - after all there's no issue anywhere right now.

Quoted 41 lines
xen/arch/x86/boot/trampoline.S +2 -2
@@ -73,7 +73,7 @@ trampoline_protmode_entry:
         mov     %ecx,%cr4
 
         /* Load pagetable base register. */
-        mov     $sym_offs(idle_pg_table),%eax
+        mov     $idle_pg_table_pa, %eax
         add     bootsym_rel(trampoline_xen_phys_start,4,%eax)
         mov     %eax,%cr3
 
@@ -113,7 +113,7 @@ trampoline_protmode_entry:
         .code64
 start64:
         /* Jump to high mappings. */
-        movabs  $__high_start, %rdi
+        movabs  $__high_start_pa + __XEN_VIRT_START, %rdi
         jmpq    *%rdi
 
 #include "video.h"
xen/arch/x86/boot/wakeup.S +2 -2
@@ -99,7 +99,7 @@ wakeup_32:
         mov     $bootsym_rel(wakeup_stack, 4, %esp)
 
         # check saved magic again
-        mov     $sym_offs(saved_magic),%eax
+        mov     $saved_magic_pa, %eax
         add     bootsym_rel(trampoline_xen_phys_start, 4, %eax)
         mov     (%eax), %eax
         cmp     $0x9abcdef0, %eax
@@ -112,7 +112,7 @@ wakeup_32:
         mov     %ecx, %cr4
 
         /* Load pagetable base register */
-        mov     $sym_offs(idle_pg_table),%eax
+        mov     $idle_pg_table_pa ,%eax
         add     bootsym_rel(trampoline_xen_phys_start,4,%eax)
         mov     %eax,%cr3
 

xen/arch/x86/boot/wakeup.S wakeup_32: · line 162

Quoted 8 lines
xen/arch/x86/boot/wakeup.S +1 -1
@@ -156,7 +156,7 @@ wakeup_32:
         .code64
 wakeup_64:
         /* Jump to high mappings and the higher-level wakeup code. */
-        movabs  $s3_resume, %rbx
+        movabs  $s3_resume_pa + __XEN_VIRT_START, %rbx
         jmp     *%rbx
 
 bogus_saved_magic:

With the sym_offs() uses gone, I think it would be best if the macro was #undef-ed ahead of the inclusion of trampoline.S. Since x86_64.S uses the macro, that'll require careful re-arrangement of #include order.

Quoted 6 lines
xen/arch/x86/xen.lds.S +0 -0
@@ -71,7 +71,12 @@ SECTIONS
   __2M_text_start = .;         /* Start of 2M superpages, mapped RX. */
 #endif
 

xen/arch/x86/xen.lds.S SECTIONS · line 82

Quoted 10 lines
xen/arch/x86/xen.lds.S +6 -1
-  start_pa = ABSOLUTE(start - __XEN_VIRT_START);
+#define DEFINE_PA_ADDRESS(sym) sym ## _pa = ABSOLUTE(sym - __XEN_VIRT_START)
+  DEFINE_PA_ADDRESS(start);
+  DEFINE_PA_ADDRESS(saved_magic);
+  DEFINE_PA_ADDRESS(idle_pg_table);
+  DEFINE_PA_ADDRESS(__high_start);
+  DEFINE_PA_ADDRESS(s3_resume);
 
   . = __XEN_VIRT_START + XEN_IMG_OFFSET;
   _start = .;

For the cases where in assembly code you add __XEN_VIRT_START this is pretty
odd: You subtract the value here just to add it back there. Did you consider a more straightforward approach, like introducing absolute xxx_va symbols?

Also, as a general request: Can you please become used to adding meaningful subject prefixes to your patches?

Jan

#2 lore raw
Quoted 9 lines · Jan Beulich
Quoted 7 lines · Frediano Ziglio

The trampoline could have "manual" relocation entries (created by assembly macros and some code to use them) and (in case of PE) normal executable relocations.
Remove all normal executable ones. In this way we don't have to worry about applying both correctly (they need proper order which is hard to spot looking at the code).

Quoted 8 lines
Quoted 1 line

I think this wants splitting into one patch replacing sym_offs() and a 2nd one introducing the hand-crafted __XEN_VIRT_START additions (which may want macro-izing). Plus the justification for the change wants extending, to actually explain what the problem is - after all there's no issue anywhere right now.

Should I explain the time dependency issue with source code? I suppose I can describe where currently is and why it would be better to have it removed (honestly I though I did but reading the commit message I didn't).
Maybe for search reasons it would be better to define 2 macros like the following?

#define phys_addr(sym) sym ## _pa
#define virt_addr(sym) sym ## _va

This way, for instance, searching for the "__high_start" word (like "grep -rw __high_start") would produce a result.

Quoted 48 lines
Quoted 48 lines
xen/arch/x86/boot/trampoline.S +2 -2
@@ -73,7 +73,7 @@ trampoline_protmode_entry:
         mov     %ecx,%cr4
 
         /* Load pagetable base register. */
-        mov     $sym_offs(idle_pg_table),%eax
+        mov     $idle_pg_table_pa, %eax
         add     bootsym_rel(trampoline_xen_phys_start,4,%eax)
         mov     %eax,%cr3
 
@@ -113,7 +113,7 @@ trampoline_protmode_entry:
         .code64
 start64:
         /* Jump to high mappings. */
-        movabs  $__high_start, %rdi
+        movabs  $__high_start_pa + __XEN_VIRT_START, %rdi
         jmpq    *%rdi
 
 #include "video.h"
xen/arch/x86/boot/wakeup.S +3 -3
@@ -99,7 +99,7 @@ wakeup_32:
         mov     $bootsym_rel(wakeup_stack, 4, %esp)
 
         # check saved magic again
-        mov     $sym_offs(saved_magic),%eax
+        mov     $saved_magic_pa, %eax
         add     bootsym_rel(trampoline_xen_phys_start, 4, %eax)
         mov     (%eax), %eax
         cmp     $0x9abcdef0, %eax
@@ -112,7 +112,7 @@ wakeup_32:
         mov     %ecx, %cr4
 
         /* Load pagetable base register */
-        mov     $sym_offs(idle_pg_table),%eax
+        mov     $idle_pg_table_pa ,%eax
         add     bootsym_rel(trampoline_xen_phys_start,4,%eax)
         mov     %eax,%cr3
 
@@ -156,7 +156,7 @@ wakeup_32:
         .code64
 wakeup_64:
         /* Jump to high mappings and the higher-level wakeup code. */
-        movabs  $s3_resume, %rbx
+        movabs  $s3_resume_pa + __XEN_VIRT_START, %rbx
         jmp     *%rbx
 
Quoted 6 lines

xen/arch/x86/boot/wakeup.S wakeup_32: · line 162

Quoted 1 line
xen/arch/x86/boot/wakeup.S +0 -0
 bogus_saved_magic:

With the sym_offs() uses gone, I think it would be best if the macro was #undef-ed ahead of the inclusion of trampoline.S. Since x86_64.S uses the macro, that'll require careful re-arrangement of #include order.

I think you mean including the trampoline after including x86_64.S in head.S. I can do it.

Quoted 14 lines
Quoted 14 lines
xen/arch/x86/xen.lds.S +6 -1
@@ -71,7 +71,12 @@ SECTIONS
   __2M_text_start = .;         /* Start of 2M superpages, mapped RX. */
 #endif
 
-  start_pa = ABSOLUTE(start - __XEN_VIRT_START);
+#define DEFINE_PA_ADDRESS(sym) sym ## _pa = ABSOLUTE(sym - __XEN_VIRT_START)
+  DEFINE_PA_ADDRESS(start);
+  DEFINE_PA_ADDRESS(saved_magic);
+  DEFINE_PA_ADDRESS(idle_pg_table);
+  DEFINE_PA_ADDRESS(__high_start);
+  DEFINE_PA_ADDRESS(s3_resume);
 
Quoted 7 lines

xen/arch/x86/xen.lds.S SECTIONS · line 82

Quoted 2 lines
xen/arch/x86/xen.lds.S +0 -0
   . = __XEN_VIRT_START + XEN_IMG_OFFSET;
   _start = .;

For the cases where in assembly code you add __XEN_VIRT_START this is pretty
odd: You subtract the value here just to add it back there. Did you consider a more straightforward approach, like introducing absolute xxx_va symbols?

I didn't consider. Would something like

#define DEFINE_ABS_ADDRESSES(sym) \
sym ## _pa = ABSOLUTE(sym - __XEN_VIRT_START); \
sym ## _va = ABSOLUTE(sym)

make sense? Maybe the _pa and _va suffixes are too similar? Maybe _physaddr and _virtaddr? Or use capical letters and macros (as above) to avoid possible clashes?

Quoted 3 lines

Also, as a general request: Can you please become used to adding meaningful subject prefixes to your patches?

Sure

Quoted 1 line

Jan

Frediano

#3 lore raw
Quoted 12 lines · Frediano Ziglio
Quoted 11 lines · Jan Beulich
Quoted 8 lines · Frediano Ziglio

The trampoline could have "manual" relocation entries (created by assembly macros and some code to use them) and (in case of PE) normal executable relocations.
Remove all normal executable ones. In this way we don't have to worry about applying both correctly (they need proper order which is hard to spot looking at the code).

Quoted 7 lines
Quoted 5 lines

I think this wants splitting into one patch replacing sym_offs() and a 2nd one introducing the hand-crafted __XEN_VIRT_START additions (which may want macro-izing). Plus the justification for the change wants extending, to actually explain what the problem is - after all there's no issue anywhere right now.

Should I explain the time dependency issue with source code?

Time dependency? I guess I don't understand ...

Quoted 11 lines

I suppose I can describe where currently is and why it would be better to have it removed (honestly I though I did but reading the commit message I didn't).
Maybe for search reasons it would be better to define 2 macros like the following?

#define phys_addr(sym) sym ## _pa
#define virt_addr(sym) sym ## _va

This way, for instance, searching for the "__high_start" word (like "grep -rw __high_start") would produce a result.

I assume it's best to keep everything there together, so see below.

Quoted 7 lines
Quoted 4 lines

With the sym_offs() uses gone, I think it would be best if the macro was #undef-ed ahead of the inclusion of trampoline.S. Since x86_64.S uses the macro, that'll require careful re-arrangement of #include order.

I think you mean including the trampoline after including x86_64.S in head.S.

Yes.

Quoted 23 lines
Quoted 14 lines
Quoted 14 lines
xen/arch/x86/xen.lds.S +6 -1
@@ -71,7 +71,12 @@ SECTIONS
   __2M_text_start = .;         /* Start of 2M superpages, mapped RX. */
 #endif
 
-  start_pa = ABSOLUTE(start - __XEN_VIRT_START);
+#define DEFINE_PA_ADDRESS(sym) sym ## _pa = ABSOLUTE(sym - __XEN_VIRT_START)
+  DEFINE_PA_ADDRESS(start);
+  DEFINE_PA_ADDRESS(saved_magic);
+  DEFINE_PA_ADDRESS(idle_pg_table);
+  DEFINE_PA_ADDRESS(__high_start);
+  DEFINE_PA_ADDRESS(s3_resume);
 
Quoted 6 lines

xen/arch/x86/xen.lds.S SECTIONS · line 82

Quoted 2 lines
xen/arch/x86/xen.lds.S +0 -0
   . = __XEN_VIRT_START + XEN_IMG_OFFSET;
   _start = .;

For the cases where in assembly code you add __XEN_VIRT_START this is pretty
odd: You subtract the value here just to add it back there. Did you consider a more straightforward approach, like introducing absolute xxx_va symbols?

I didn't consider. Would something like

Quoted 7 lines

#define DEFINE_ABS_ADDRESSES(sym) \
sym ## _pa = ABSOLUTE(sym - __XEN_VIRT_START); \
sym ## _va = ABSOLUTE(sym)

make sense? Maybe the _pa and _va suffixes are too similar? Maybe _physaddr and _virtaddr? Or use capical letters and macros (as above) to avoid possible clashes?

I'd like to ask that we don't introduce symbols we don't actually use. Hence a single macro defining both is probably not going to be overly helpful. As to capital letters: I'm struggling with the "(as above)" - I don't see any use of capital letters in symbol names being generated. But yes, I was going to suggest to consider _VA and _PA tags, precisely to reduce the risk of clashes.

Jan

#4 lore raw
Quoted 20 lines · Jan Beulich
Quoted 18 lines · Frediano Ziglio
Quoted 16 lines · Jan Beulich
Quoted 8 lines · Frediano Ziglio

The trampoline could have "manual" relocation entries (created by assembly macros and some code to use them) and (in case of PE) normal executable relocations.
Remove all normal executable ones. In this way we don't have to worry about applying both correctly (they need proper order which is hard to spot looking at the code).

I think this wants splitting into one patch replacing sym_offs() and a 2nd one introducing the hand-crafted __XEN_VIRT_START additions (which may want macro-izing). Plus the justification for the change wants extending, to actually explain what the problem is - after all there's no issue anywhere right now.

Quoted 4 lines
Quoted 1 line

Should I explain the time dependency issue with source code?

Time dependency? I guess I don't understand ...

You have a time dependency between 2 code path when one has to be executed before/after another.
Some patterns are expected (you don't call a method on a destroyed object) but nasty hidden ones makes code more complicated and less robust, making code changes harder and program fragile.
In that specific case the copy of the trampoline, for EFI path, is done after relocation rollback. In that specific code path, you are executing code which is not meant to be executed at that location (now should be executed at higher locations) hoping that compiler/binary code can cope with it. The code should relocate back and switch to higher locations as quick as possible.

Quoted 19 lines
Quoted 11 lines

I suppose I can describe where currently is and why it would be better to have it removed (honestly I though I did but reading the commit message I didn't).
Maybe for search reasons it would be better to define 2 macros like the following?

#define phys_addr(sym) sym ## _pa
#define virt_addr(sym) sym ## _va

This way, for instance, searching for the "__high_start" word (like "grep -rw __high_start") would produce a result.

I assume it's best to keep everything there together, so see below.

Quoted 5 lines
Quoted 4 lines

With the sym_offs() uses gone, I think it would be best if the macro was #undef-ed ahead of the inclusion of trampoline.S. Since x86_64.S uses the macro, that'll require careful re-arrangement of #include order.

Quoted 5 lines
Quoted 2 lines

I think you mean including the trampoline after including x86_64.S in head.S.

Yes.

That worked like a charm!

Quoted 31 lines
Quoted 23 lines
Quoted 14 lines
Quoted 14 lines
xen/arch/x86/xen.lds.S +6 -1
@@ -71,7 +71,12 @@ SECTIONS
   __2M_text_start = .;         /* Start of 2M superpages, mapped RX. */
 #endif
 
-  start_pa = ABSOLUTE(start - __XEN_VIRT_START);
+#define DEFINE_PA_ADDRESS(sym) sym ## _pa = ABSOLUTE(sym - __XEN_VIRT_START)
+  DEFINE_PA_ADDRESS(start);
+  DEFINE_PA_ADDRESS(saved_magic);
+  DEFINE_PA_ADDRESS(idle_pg_table);
+  DEFINE_PA_ADDRESS(__high_start);
+  DEFINE_PA_ADDRESS(s3_resume);
 
Quoted 6 lines

xen/arch/x86/xen.lds.S SECTIONS · line 82

Quoted 2 lines
xen/arch/x86/xen.lds.S +0 -0
   . = __XEN_VIRT_START + XEN_IMG_OFFSET;
   _start = .;

For the cases where in assembly code you add __XEN_VIRT_START this is pretty
odd: You subtract the value here just to add it back there. Did you consider a more straightforward approach, like introducing absolute xxx_va symbols?

I didn't consider. Would something like

Quoted 7 lines

#define DEFINE_ABS_ADDRESSES(sym) \
sym ## _pa = ABSOLUTE(sym - __XEN_VIRT_START); \
sym ## _va = ABSOLUTE(sym)

make sense? Maybe the _pa and _va suffixes are too similar? Maybe _physaddr and _virtaddr? Or use capical letters and macros (as above) to avoid possible clashes?

Quoted 7 lines

I'd like to ask that we don't introduce symbols we don't actually use. Hence a single macro defining both is probably not going to be overly helpful. As to capital letters: I'm struggling with the "(as above)" - I don't see any use of capital letters in symbol names being generated. But yes, I was going to suggest to consider _VA and _PA tags, precisely to reduce the risk of clashes.

Unfortunately this worked but is producing these warnings which are pretty annoying:

ld: ./.xen.efi.0xffff82d040000000.0: stripping non-representable symbol 'saved_magic_va' (value 0xffff82d040816018)

I tried to remove them somehow, but I didn't find any way (no hidden symbols, not been able to force the linker to strip it and not complain).
I think I'll get back to the single "pa" symbol, this time with _PA suffix, maybe hidden, and adding macros (phys_addr/virt_addr) to make clear the purpose.

Quoted 1 line

Jan

I'll try to split the patch as suggested, although it will be hard to come with a sensible message for the second commit.

Regards,
Frediano