x86/build: Get rid of vmlinux postlink step

From: Ard Biesheuvel <[email protected]>

Some architectures build vmlinux with static relocations preserved, but strip them again from the final vmlinux image. Arch specific tools consume these static relocations in order to construct relocation tables for KASLR.

The fact that vmlinux is created, consumed and subsequently updated goes against the typical, declarative paradigm used by Make, which is based on rules and dependencies. So as a first step towards cleaning this up, introduce a Kconfig symbol to declare that the arch wants to consume the static relocations emitted into vmlinux. This will be wired up further in subsequent patches.

Signed-off-by Ard Biesheuvel <[email protected]>

---
 Makefile            | 4 ++++
 arch/Kconfig        | 7 +++++++
 arch/mips/Kconfig   | 1 +
 arch/mips/Makefile  | 4 ----
 arch/riscv/Kconfig  | 1 +
 arch/riscv/Makefile | 2 +-
 arch/s390/Kconfig   | 1 +
 arch/s390/Makefile  | 2 +-
 arch/x86/Kconfig    | 1 +
 arch/x86/Makefile   | 6 ------
 10 files changed, 17 insertions(+), 12 deletions(-)

Makefile +4 -0

index 30dab4c8b012..a3302dce56de 100644
@@ -1119,6 +1119,10 @@ ifdef CONFIG_LD_ORPHAN_WARN
 LDFLAGS_vmlinux += --orphan-handling=$(CONFIG_LD_ORPHAN_WARN_LEVEL)
 endif
 
+ifneq ($(CONFIG_ARCH_VMLINUX_NEEDS_RELOCS),)
+LDFLAGS_vmlinux        += --emit-relocs --discard-none
+endif
+
 # Align the bit size of userspace programs with the kernel
 KBUILD_USERCFLAGS  += $(filter -m32 -m64 --target=%, $(KBUILD_CPPFLAGS) $(KBUILD_CFLAGS))
 KBUILD_USERLDFLAGS += $(filter -m32 -m64 --target=%, $(KBUILD_CPPFLAGS) $(KBUILD_CFLAGS))

arch/Kconfig +7 -0

index b8a4ff365582..101a13fcde8e 100644
@@ -1695,6 +1695,13 @@ config ARCH_HAS_KERNEL_FPU_SUPPORT
          Architectures that select this option can run floating-point code in
          the kernel, as described in Documentation/core-api/floating-point.rst.
 
+config ARCH_VMLINUX_NEEDS_RELOCS
+       bool
+       help
+         Whether the architecture needs vmlinux to be built with static
+         relocations preserved. This is used by some architectures to
+         construct bespoke relocation tables for KASLR.
+
 source "kernel/gcov/Kconfig"
 
 source "scripts/gcc-plugins/Kconfig"

arch/mips/Kconfig +1 -0

index 1924f2d83932..5aedbd7afadb 100644
@@ -2617,6 +2617,7 @@ config RELOCATABLE
                   CPU_MIPS32_R6 || CPU_MIPS64_R6 || \
                   CPU_P5600 || CAVIUM_OCTEON_SOC || \
                   CPU_LOONGSON64
+       select ARCH_VMLINUX_NEEDS_RELOCS
        help
          This builds a kernel image that retains relocation information
          so it can be loaded someplace besides the default 1MB.

arch/mips/Makefile +0 -4

index be8cb44a89fd..d9057e29bc62 100644
@@ -100,10 +100,6 @@ LDFLAGS_vmlinux                    += -G 0 -static -n -nostdlib
 KBUILD_AFLAGS_MODULE           += -mlong-calls
 KBUILD_CFLAGS_MODULE           += -mlong-calls
 
-ifeq ($(CONFIG_RELOCATABLE),y)
-LDFLAGS_vmlinux                        += --emit-relocs
-endif
-
 cflags-y += -ffreestanding
 
 cflags-$(CONFIG_CPU_BIG_ENDIAN)                += -EB

arch/riscv/Kconfig +1 -0

index 7612c52e9b1e..6f5800114416 100644
@@ -1077,6 +1077,7 @@ config RELOCATABLE
        bool "Build a relocatable kernel"
        depends on MMU && 64BIT && !XIP_KERNEL
        select MODULE_SECTIONS if MODULES
+       select ARCH_VMLINUX_NEEDS_RELOCS
        help
           This builds a kernel as a Position Independent Executable (PIE),
           which retains all relocation metadata required to relocate the

arch/riscv/Makefile +1 -1

index 13fbc0f94238..6ef0d10e0c50 100644
@@ -8,7 +8,7 @@
 
 LDFLAGS_vmlinux := -z norelro
 ifeq ($(CONFIG_RELOCATABLE),y)
-       LDFLAGS_vmlinux += -shared -Bsymbolic -z notext --emit-relocs
+       LDFLAGS_vmlinux += -shared -Bsymbolic -z notext
        KBUILD_CFLAGS += -fPIE
 endif
 ifeq ($(CONFIG_DYNAMIC_FTRACE),y)

arch/s390/Kconfig +1 -0

index 9c9ec08d78c7..ea67b7317138 100644
@@ -630,6 +630,7 @@ endchoice
 
 config RELOCATABLE
        def_bool y
+       select ARCH_VMLINUX_NEEDS_RELOCS
        help
          This builds a kernel image that retains relocation information
          so it can be loaded at an arbitrary address.

arch/s390/Makefile +1 -1

index 5fae311203c2..d5f4be440879 100644
@@ -15,7 +15,7 @@ KBUILD_CFLAGS_MODULE += -fPIC
 KBUILD_AFLAGS  += -m64
 KBUILD_CFLAGS  += -m64
 KBUILD_CFLAGS  += -fPIC
-LDFLAGS_vmlinux        := -no-pie --emit-relocs --discard-none
+LDFLAGS_vmlinux        := -no-pie
 extra_tools    := relocs
 aflags_dwarf   := -Wa,-gdwarf-2
 KBUILD_AFLAGS_DECOMPRESSOR := $(CLANG_FLAGS) -m64 -D__ASSEMBLY__

arch/x86/Kconfig +1 -0

index be2c311f5118..2005d80ff8d1 100644
@@ -2199,6 +2199,7 @@ config RANDOMIZE_BASE
 config X86_NEED_RELOCS
        def_bool y
        depends on RANDOMIZE_BASE || (X86_32 && RELOCATABLE)
+       select ARCH_VMLINUX_NEEDS_RELOCS
 
 config PHYSICAL_ALIGN
        hex "Alignment value to which kernel should be aligned"

arch/x86/Makefile +0 -7

index 5b773b34768d..f65ed6dcd6fb 100644
@@ -251,12 +251,6 @@ endif
 
 KBUILD_LDFLAGS += -m elf_$(UTS_MACHINE)
 
-ifdef CONFIG_X86_NEED_RELOCS
-LDFLAGS_vmlinux := --emit-relocs --discard-none
-else
-LDFLAGS_vmlinux :=
-endif
-
 #
 # The 64-bit kernel must be aligned to 2MB.  Pass -z max-page-size=0x200000 to
 # the linker to force 2MB page size regardless of the default page size used
--

2.48.1.601.g30ceb7b040-goog

From: Ard Biesheuvel <[email protected]>

The imperative paradigm used to build vmlinux, extract some info from it or perform some checks on it, and subsequently modify it again goes against the declarative paradigm that is usually employed for defining make rules.

In particular, the Makefile.postlink files that consume their input via an output rule result in some dodgy logic in the decompressor makefiles for RISC-V and x86, given that the vmlinux.relocs input file needed to generate the arch-specific relocation tables may not exist or be out of date, but cannot be constructed using the ordinary Make dependency based rules, because the info needs to be extracted while vmlinux is in its ephemeral, non-stripped form.

So instead, for architectures that require the static relocations that are emitted into vmlinux when passing --emit-relocs to the linker, and are subsequently stripped out again, introduce an intermediate vmlinux target called vmlinux.unstripped, and organize the reset of the build logic accordingly:

- vmlinux.unstripped is created only once, and not updated again
- build rules under arch/*/boot can depend on vmlinux.unstripped without running the risk of the data disappearing or being out of date
- the final vmlinux generated by the build is not bloated with static relocations that are never needed again after the build completes.

Signed-off-by Ard Biesheuvel <[email protected]>

---

 arch/mips/Makefile.postlink  |  2 +-
 arch/riscv/Makefile.postlink | 11 +--------
 arch/riscv/boot/Makefile     |  5 +---
 arch/s390/Makefile.postlink  |  4 +---
 arch/x86/Makefile.postlink   |  8 +++----
 scripts/Makefile.lib         |  2 +-
 scripts/Makefile.vmlinux     | 25 ++++++++++++++------
 7 files changed, 26 insertions(+), 31 deletions(-)

arch/mips/Makefile.postlink +1 -1

index 6cfdc149d3bc..ea0add7d56b2 100644
@@ -22,7 +22,7 @@ quiet_cmd_relocs = RELOCS  $@
 
 # `@true` prevents complaint when there is nothing to be done
 
-vmlinux: FORCE
+vmlinux vmlinux.unstripped: FORCE
        @true
 ifeq ($(CONFIG_CPU_LOONGSON3_WORKAROUNDS),y)
        $(call if_changed,ls3_llsc)

arch/riscv/Makefile.postlink +1 -10

index 6b0580949b6a..0e4cf8ad2f14 100644
@@ -10,26 +10,17 @@ __archpost:
 
 -include include/config/auto.conf
 include $(srctree)/scripts/Kbuild.include
-include $(srctree)/scripts/Makefile.lib
 
 quiet_cmd_relocs_check = CHKREL  $@
 cmd_relocs_check =                                                     \
        $(CONFIG_SHELL) $(srctree)/arch/riscv/tools/relocs_check.sh "$(OBJDUMP)" "$(NM)" "$@"
 
-ifdef CONFIG_RELOCATABLE
-quiet_cmd_cp_vmlinux_relocs = CPREL   vmlinux.relocs
-cmd_cp_vmlinux_relocs = cp vmlinux vmlinux.relocs
-
-endif
-
 # `@true` prevents complaint when there is nothing to be done
 
-vmlinux: FORCE
+vmlinux vmlinux.unstripped: FORCE
        @true
 ifdef CONFIG_RELOCATABLE
        $(call if_changed,relocs_check)
-       $(call if_changed,cp_vmlinux_relocs)
-       $(call if_changed,strip_relocs)
 endif
 
 clean:

arch/riscv/boot/Makefile +1 -4

index b25d524ce5eb..bfc3d0b75b9b 100644
@@ -32,10 +32,7 @@ $(obj)/xipImage: vmlinux FORCE
 endif
 
 ifdef CONFIG_RELOCATABLE
-vmlinux.relocs: vmlinux
-       @ (! [ -f vmlinux.relocs ] && echo "vmlinux.relocs can't be found, please remove vmlinux and try again") || true
-
-$(obj)/Image: vmlinux.relocs FORCE
+$(obj)/Image: vmlinux.unstripped FORCE
 else
 $(obj)/Image: vmlinux FORCE
 endif

arch/s390/Makefile.postlink +1 -3

index 1ae5478cd6ac..c2b737500a91 100644
@@ -11,7 +11,6 @@ __archpost:
 
 -include include/config/auto.conf
 include $(srctree)/scripts/Kbuild.include
-include $(srctree)/scripts/Makefile.lib
 
 CMD_RELOCS=arch/s390/tools/relocs
 OUT_RELOCS = arch/s390/boot

@@ -20,9 +19,8 @@ quiet_cmd_relocs = RELOCS  $(OUT_RELOCS)/relocs.S
        mkdir -p $(OUT_RELOCS); \
        $(CMD_RELOCS) $@ > $(OUT_RELOCS)/relocs.S
 
-vmlinux: FORCE
+vmlinux.unstripped: FORCE
        $(call cmd,relocs)
-       $(call cmd,strip_relocs)
 
 clean:
        @rm -f $(OUT_RELOCS)/relocs.S

arch/x86/Makefile.postlink +3 -5

index 8b8a68162c94..445fce66630f 100644
@@ -11,23 +11,21 @@ __archpost:
 
 -include include/config/auto.conf
 include $(srctree)/scripts/Kbuild.include
-include $(srctree)/scripts/Makefile.lib
 
 CMD_RELOCS = arch/x86/tools/relocs
 OUT_RELOCS = arch/x86/boot/compressed
-quiet_cmd_relocs = RELOCS  $(OUT_RELOCS)/[email protected]
+quiet_cmd_relocs = RELOCS  $(OUT_RELOCS)/vmlinux.relocs
       cmd_relocs = \
        mkdir -p $(OUT_RELOCS); \
-       $(CMD_RELOCS) $@ > $(OUT_RELOCS)/[email protected]; \
+       $(CMD_RELOCS) $@ > $(OUT_RELOCS)/vmlinux.relocs; \
        $(CMD_RELOCS) --abs-relocs $@
 
 # `@true` prevents complaint when there is nothing to be done
 
-vmlinux: FORCE
+vmlinux vmlinux.unstripped: FORCE
        @true
 ifeq ($(CONFIG_X86_NEED_RELOCS),y)
        $(call cmd,relocs)
-       $(call cmd,strip_relocs)
 endif
 
 clean:

scripts/Makefile.lib +1 -1

index cad20f0e66ee..7a023f17a21d 100644
@@ -377,7 +377,7 @@ quiet_cmd_objcopy = OBJCOPY $@
 cmd_objcopy = $(OBJCOPY) $(OBJCOPYFLAGS) $(OBJCOPYFLAGS_$(@F)) $< $@
 
 quiet_cmd_strip_relocs = RSTRIP  $@
-cmd_strip_relocs = $(OBJCOPY) --remove-section='.rel*' $@
+cmd_strip_relocs = $(OBJCOPY) --remove-section='.rel*' $< $@
 
 # Gzip
 # ---------------------------------------------------------------------------

scripts/Makefile.vmlinux +18 -8

index 3523ce3ce3dc..1e3ea8e4b4b0 100644
@@ -9,6 +9,17 @@ include $(srctree)/scripts/Makefile.lib
 
 targets :=
 
+ifdef CONFIG_ARCH_VMLINUX_NEEDS_RELOCS
+vmlinux-final := vmlinux.unstripped
+
+vmlinux: $(vmlinux-final) FORCE
+       $(call if_changed,strip_relocs)
+
+targets += vmlinux
+else
+vmlinux-final := vmlinux
+endif
+
 %.o: %.c FORCE
        $(call if_changed_rule,cc_o_c)
 

@@ -47,7 +58,7 @@ targets += .builtin-dtbs-list
 
 ifdef CONFIG_GENERIC_BUILTIN_DTB
 targets += .builtin-dtbs.S .builtin-dtbs.o
-vmlinux: .builtin-dtbs.o
+$(vmlinux-final): .builtin-dtbs.o
 endif
 
 # vmlinux

@@ -55,11 +66,11 @@ endif
 
 ifdef CONFIG_MODULES
 targets += .vmlinux.export.o
-vmlinux: .vmlinux.export.o
+$(vmlinux-final): .vmlinux.export.o
 endif
 
 ifdef CONFIG_ARCH_WANTS_PRE_LINK_VMLINUX
-vmlinux: arch/$(SRCARCH)/tools/vmlinux.arch.o
+$(vmlinux-final): arch/$(SRCARCH)/tools/vmlinux.arch.o
 
 arch/$(SRCARCH)/tools/vmlinux.arch.o: vmlinux.o FORCE
        $(Q)$(MAKE) $(build)=arch/$(SRCARCH)/tools $@

@@ -72,11 +83,11 @@ cmd_link_vmlinux =                                                  \
        $< "$(LD)" "$(KBUILD_LDFLAGS)" "$(LDFLAGS_vmlinux)" "$@";       \
        $(if $(ARCH_POSTLINK), $(MAKE) -f $(ARCH_POSTLINK) $@, true)
 
-targets += vmlinux
-vmlinux: scripts/link-vmlinux.sh vmlinux.o $(KBUILD_LDS) FORCE
+targets += $(vmlinux-final)
+$(vmlinux-final): scripts/link-vmlinux.sh vmlinux.o $(KBUILD_LDS) FORCE
        +$(call if_changed_dep,link_vmlinux)
 ifdef CONFIG_DEBUG_INFO_BTF
-vmlinux: $(RESOLVE_BTFIDS)
+$(vmlinux-final): $(RESOLVE_BTFIDS)
 endif
 
 # module.builtin.ranges

@@ -92,7 +103,7 @@ modules.builtin.ranges: $(srctree)/scripts/generate_builtin_ranges.awk \
                        modules.builtin vmlinux.map vmlinux.o.map FORCE
        $(call if_changed,modules_builtin_ranges)
 
-vmlinux.map: vmlinux
+vmlinux.map: $(vmlinux-final)
        @:
 
 endif
--

2.48.1.601.g30ceb7b040-goog

From: Ard Biesheuvel <[email protected]>

Kbuild supports an architecture specific Makefile.postlink file that is invoked for the vmlinux target after it has been built. This Makefile takes 'vmlinux' (which has just been built) as the target, and mangles the file and/or constructs other intermediate artifacts from it.

This violates the general philosophy of Make, which is based on rules and dependencies, and artifacts that are rebuilt only when any of their dependencies have been updated.

Instead, the different incarnations of vmlinux that are consumed by different stages of the build should be emitted as distinct files, where rules and dependencies are used to define one in terms of the other.

This also works around an error observed here [0], where vmlinux is deleted by Make because a subsequent step that consumes it as input throws an error.

So refactor the vmlinux shell scripts and build rules so that architectures that rely on --emit-relocs to construct vmlinux with static relocations preserved will get a separate vmlinux.unstripped file carrying those relocations. This removes the need for an imperative postlink step, given that any rules that depend on the unstripped vmlinux can now simply depend on vmlinux.unstripped, rather than inject a build step into Makefile.postlink

But in terms of justification for upstreaming, the reduction in complexity alone makes it worth it IMO:

  19 files changed, 52 insertions(+), 87 deletions(-)

Quoted 5 lines · Ingo Molnar

But in terms of justification for upstreaming, the reduction in complexity alone makes it worth it IMO:

  19 files changed, 52 insertions(+), 87 deletions(-)

Yeah, absolutely. Our fancy make build rules still have too many of the phony forced targets, but this is a few less of them and makes the build confirm (more) to the usual rules.

I do wonder if we could just get rid of that CONFIG_ARCH_VMLINUX_NEEDS_RELOCS entirely and make it just be how all architectures do it.

Yes, it was apparently "just" riscv/s390/x86/mips that did that 'strip_relocs' hack, but at the same time that whole pass *feels* entirely generic.

Quoted 8 lines

So refactor the vmlinux shell scripts and build rules so that architectures that rely on --emit-relocs to construct vmlinux with static relocations preserved will get a separate vmlinux.unstripped file carrying those relocations. This removes the need for an imperative postlink step, given that any rules that depend on the unstripped vmlinux can now simply depend on vmlinux.unstripped, rather than inject a build step into Makefile.postlink

Nice! Does this also result in any measurable speedup of the build, or are the steps still similar?

From: Ard Biesheuvel <[email protected]>

Kbuild supports an architecture specific Makefile.postlink file that is invoked for the vmlinux target after it has been built. This Makefile takes 'vmlinux' (which has just been built) as the target, and mangles the file and/or constructs other intermediate artifacts from it.

This violates the general philosophy of Make, which is based on rules and dependencies, and artifacts that are rebuilt only when any of their dependencies have been updated.

Instead, the different incarnations of vmlinux that are consumed by different stages of the build should be emitted as distinct files, where rules and dependencies are used to define one in terms of the other.

This also works around an error observed here [0], where vmlinux is deleted by Make because a subsequent step that consumes it as input throws an error.

So refactor the vmlinux shell scripts and build rules so that architectures that rely on --emit-relocs to construct vmlinux with static relocations preserved will get a separate vmlinux.unstripped file carrying those relocations. This removes the need for an imperative postlink step, given that any rules that depend on the unstripped vmlinux can now simply depend on vmlinux.unstripped, rather than inject a build step into Makefile.postlink

S390 should be able to do the same. MIPS and RISC-V perform some post-build checks on vmlinux, which is reasonable in principle for a postlink step, although deleting vmlinux when the check fails is equally unhelpful.

[0] https://lore.kernel.org/all/[email protected]/T/#m731ed0206949fc3f39fcc8a7b82fe348a8fc80c4

Cc Masahiro Yamada <[email protected]>

Cc Ingo Molnar <[email protected]>

Ard Biesheuvel (4):
Kbuild/link-vmlinux.sh: Make output file name configurable
Kbuild: Introduce Kconfig symbol for linking vmlinux with relocations
Kbuild: Create intermediate vmlinux build with relocations preserved
x86: Get rid of Makefile.postlink

Quoted 8 lines · Ard Biesheuvel

From: Ard Biesheuvel <[email protected]>

Kbuild supports an architecture specific Makefile.postlink file that is invoked for the vmlinux target after it has been built. This Makefile takes 'vmlinux' (which has just been built) as the target, and mangles the file and/or constructs other intermediate artifacts from it.

Quoted 7 lines

This violates the general philosophy of Make, which is based on rules and dependencies, and artifacts that are rebuilt only when any of their dependencies have been updated.

Instead, the different incarnations of vmlinux that are consumed by different stages of the build should be emitted as distinct files, where rules and dependencies are used to define one in terms of the other.

Quoted 19 lines

This also works around an error observed here [0], where vmlinux is deleted by Make because a subsequent step that consumes it as input throws an error.

So refactor the vmlinux shell scripts and build rules so that architectures that rely on --emit-relocs to construct vmlinux with static relocations preserved will get a separate vmlinux.unstripped file carrying those relocations. This removes the need for an imperative postlink step, given that any rules that depend on the unstripped vmlinux can now simply depend on vmlinux.unstripped, rather than inject a build step into Makefile.postlink

S390 should be able to do the same. MIPS and RISC-V perform some post-build checks on vmlinux, which is reasonable in principle for a postlink step, although deleting vmlinux when the check fails is equally unhelpful.

[0] https://lore.kernel.org/all/[email protected]/T/#m731ed0206949fc3f39fcc8a7b82fe348a8fc80c4

Quoted 9 lines

Cc Masahiro Yamada <[email protected]>

Cc Ingo Molnar <[email protected]>

Ard Biesheuvel (4):
Kbuild/link-vmlinux.sh: Make output file name configurable
Kbuild: Introduce Kconfig symbol for linking vmlinux with relocations
Kbuild: Create intermediate vmlinux build with relocations preserved
x86: Get rid of Makefile.postlink

Ping?

Quoted 10 lines · Ard Biesheuvel

Quoted 8 lines · Ard Biesheuvel

From: Ard Biesheuvel <[email protected]>

Kbuild supports an architecture specific Makefile.postlink file that is invoked for the vmlinux target after it has been built. This Makefile takes 'vmlinux' (which has just been built) as the target, and mangles the file and/or constructs other intermediate artifacts from it.

Quoted 7 lines

Quoted 7 lines

This violates the general philosophy of Make, which is based on rules and dependencies, and artifacts that are rebuilt only when any of their dependencies have been updated.

Instead, the different incarnations of vmlinux that are consumed by different stages of the build should be emitted as distinct files, where rules and dependencies are used to define one in terms of the other.

In my understanding, the build rule of vmlinux is atomic because vmlinux embeds a timestamp and a build version.

Now, you are splitting it into two stages.

vmlinux.unstripped (this includes timestamp and the build version)
--(cmd_strip_relocs)--> vmlinux

When cmd_strip_relocs is changed, only vmlinux is updated. This changes the content of vmlinux, but its timestamp and build version remain the same.

So, I am not sure if this is the right direction.

You can see more steps for updating vmlinux.
Do you believe the build rule should be further split into more fine-grained stages?

For example,

vmlinux.pre-sort (this includes timestamp and the build version)
--(scripts/sortable)-->

vmlinux.unstripped
--(cmd_strip_relocs)-->
vmlinux

But, again, even when sorttable is changed, the timestamp and the build version remain the same.

Yeah, arch/*/Makefile.postlink is a crap
where arch maintainers build a fence
and start whatever they want to do.

If they completely disappear, I would love it.

However, this seems a partial clean-up
within the scope you are interested in.
(more specifically your motivation is because Linus pointed out a failure in arch/x86/Makefile.postlink deleted vmlinux)

Quoted 34 lines · Masahiro Yamada

Quoted 10 lines · Ard Biesheuvel

Quoted 8 lines · Ard Biesheuvel

From: Ard Biesheuvel <[email protected]>

Kbuild supports an architecture specific Makefile.postlink file that is invoked for the vmlinux target after it has been built. This Makefile takes 'vmlinux' (which has just been built) as the target, and mangles the file and/or constructs other intermediate artifacts from it.

Quoted 7 lines

Quoted 7 lines

This violates the general philosophy of Make, which is based on rules and dependencies, and artifacts that are rebuilt only when any of their dependencies have been updated.

Instead, the different incarnations of vmlinux that are consumed by different stages of the build should be emitted as distinct files, where rules and dependencies are used to define one in terms of the other.

In my understanding, the build rule of vmlinux is atomic because vmlinux embeds a timestamp and a build version.

Now, you are splitting it into two stages.

vmlinux.unstripped (this includes timestamp and the build version)
--(cmd_strip_relocs)--> vmlinux

When cmd_strip_relocs is changed, only vmlinux is updated. This changes the content of vmlinux, but its timestamp and build version remain the same.

Quoted 2 lines

So, I am not sure if this is the right direction.

You are saying that, when we change the contents of the cmd_strip_relocs build rule and rebuild without cleaning, vmlinux will have an older timestamp? I think there are many more cases where the version is not updated, so I don't see this as a problem at all.

Quoted 6 lines

You can see more steps for updating vmlinux.
Do you believe the build rule should be further split into more fine-grained stages?

No.

The problem is that vmlinux.relocs (in arch/x86/boot/compressed) depends on vmlinux, but not the version of vmlinux that is ultimately produced.

It should depend on an artifact that is always suitable to generate the relocation table, not only during the short time between the initial creation of vmlinux and the point during the execution of Makefile.postlink where the relocations that are needed to generate vmlinux.relocs are stripped from vmlinux.

Quoted 4 lines

For example,

vmlinux.pre-sort (this includes timestamp and the build version)
--(scripts/sortable)-->

Why? Which other artifact depends on the unsorted tables, and can no longer be generated correctly after the tables have been sorted?

Quoted 4 lines

vmlinux.unstripped
--(cmd_strip_relocs)-->
vmlinux

Quoted 3 lines

But, again, even when sorttable is changed, the timestamp and the build version remain the same.

Again, there are many other ways in which the final vmlinux can be newer than the internal version fields suggest. I really don't think this is an issue.

Quoted 8 lines

Yeah, arch/*/Makefile.postlink is a crap
where arch maintainers build a fence
and start whatever they want to do.

If they completely disappear, I would love it.

Good.

Quoted 5 lines

However, this seems a partial clean-up
within the scope you are interested in.
(more specifically your motivation is because Linus pointed out a failure in arch/x86/Makefile.postlink deleted vmlinux)

Yes. Makefile.postlink failures propagate back to the build rule that generates vmlinux, and so the file is deleted again.

For sanity checks performed on vmlinux, this is fine. But for generating another file that takes vmlinux as its input, this is completely broken.

I do not think it is broken.
As I mentioned above, I regard vmlinux.relocs as a byproduct of the atomic build rule of vmlinux. This works.

Quoted 1 line · Masahiro Yamada

...

Quoted 3 lines

I do not think it is broken.
As I mentioned above, I regard vmlinux.relocs as a byproduct of the atomic build rule of vmlinux. This works.

There is no make rule for vmlinux.relocs, and so

- if it gets deleted, it cannot be rebuilt and even though the build does not break, the relocation data is missing from the compressed image, and this could potentially break the kaslr startup code,
- it vmlinux.relocs is older than vmlinux for some reason, make will not notice and silently reuse the outdated version,
- when creating vmlinux.relocs from vmlinux and an error occurs, vmlinux is deleted, making it difficult to diagnose the problem.

I think this is badly broken, but if you think this is all working as it should, I am not going to debate this further, and you can consider the patch series withdrawn.

Quoted 6 lines · Ard Biesheuvel

Quoted 1 line · Masahiro Yamada

...

Quoted 3 lines

I do not think it is broken.
As I mentioned above, I regard vmlinux.relocs as a byproduct of the atomic build rule of vmlinux. This works.

Except when it doesn't work, such as when an intermediate linking step fails, and intermediate build products are lost and cannot be recreated easily (or at all without modifying the source)?

And the thing is, there should be no such thing as an 'atomic build rule of vmlinux' if it means lost information when the build is broken at an intermediate step. What purpose does it have?

Quoted 10 lines

There is no make rule for vmlinux.relocs, and so

- if it gets deleted, it cannot be rebuilt and even though the build does not break, the relocation data is missing from the compressed image, and this could potentially break the kaslr startup code,
- it vmlinux.relocs is older than vmlinux for some reason, make will not notice and silently reuse the outdated version,
- when creating vmlinux.relocs from vmlinux and an error occurs, vmlinux is deleted, making it difficult to diagnose the problem.

Quoted 3 lines

I think this is badly broken, but if you think this is all working as it should, I am not going to debate this further, and you can consider the patch series withdrawn.

That's very sad, as both the simplification is substantial:

  19 files changed, 52 insertions(+), 87 deletions(-)

and the increase in debuggability is substantial as well.

Thanks,

Ingo

Quoted 39 lines · Ingo Molnar

Quoted 6 lines · Ard Biesheuvel

Quoted 1 line · Masahiro Yamada

...

Quoted 3 lines

I do not think it is broken.
As I mentioned above, I regard vmlinux.relocs as a byproduct of the atomic build rule of vmlinux. This works.

Except when it doesn't work, such as when an intermediate linking step fails, and intermediate build products are lost and cannot be recreated easily (or at all without modifying the source)?

And the thing is, there should be no such thing as an 'atomic build rule of vmlinux' if it means lost information when the build is broken at an intermediate step. What purpose does it have?

Quoted 10 lines

There is no make rule for vmlinux.relocs, and so

- if it gets deleted, it cannot be rebuilt and even though the build does not break, the relocation data is missing from the compressed image, and this could potentially break the kaslr startup code,
- it vmlinux.relocs is older than vmlinux for some reason, make will not notice and silently reuse the outdated version,
- when creating vmlinux.relocs from vmlinux and an error occurs, vmlinux is deleted, making it difficult to diagnose the problem.

Quoted 3 lines

I think this is badly broken, but if you think this is all working as it should, I am not going to debate this further, and you can consider the patch series withdrawn.

That's very sad, as both the simplification is substantial:

  19 files changed, 52 insertions(+), 87 deletions(-)

and the increase in debuggability is substantial as well.

Quoted 3 lines

Thanks,

Ingo

When a byproduct is accidentally lost
(for example, manually deleted), it is not automatically restored. Running 'rm vmlinux' or 'make clean' is needed.

vmlinux.relocs is one such byproduct.
Another is the map file when CONFIG_VMLINUX_MAP=y is enabled.

I am a bit concerned about having more and more
intermediate vmlinux.* files, but maybe only me.
I hope vmlinux.unstripped is the only/last one.

OK, let's do this.

Ard, please send v2 with .gitignore and 'clean' target updates.

Quoted 41 lines · Masahiro Yamada

Quoted 39 lines · Ingo Molnar

Quoted 6 lines · Ard Biesheuvel

Quoted 1 line · Masahiro Yamada

...

Quoted 3 lines

I do not think it is broken.
As I mentioned above, I regard vmlinux.relocs as a byproduct of the atomic build rule of vmlinux. This works.

Except when it doesn't work, such as when an intermediate linking step fails, and intermediate build products are lost and cannot be recreated easily (or at all without modifying the source)?

And the thing is, there should be no such thing as an 'atomic build rule of vmlinux' if it means lost information when the build is broken at an intermediate step. What purpose does it have?

Quoted 10 lines

There is no make rule for vmlinux.relocs, and so

- if it gets deleted, it cannot be rebuilt and even though the build does not break, the relocation data is missing from the compressed image, and this could potentially break the kaslr startup code,
- it vmlinux.relocs is older than vmlinux for some reason, make will not notice and silently reuse the outdated version,
- when creating vmlinux.relocs from vmlinux and an error occurs, vmlinux is deleted, making it difficult to diagnose the problem.

Quoted 3 lines

I think this is badly broken, but if you think this is all working as it should, I am not going to debate this further, and you can consider the patch series withdrawn.

That's very sad, as both the simplification is substantial:

  19 files changed, 52 insertions(+), 87 deletions(-)

and the increase in debuggability is substantial as well.

Quoted 8 lines

Quoted 3 lines

Thanks,

Ingo

When a byproduct is accidentally lost
(for example, manually deleted), it is not automatically restored. Running 'rm vmlinux' or 'make clean' is needed.

Exactly. Make cannot detect this situation, and so the build breaks in some way.

Quoted 3 lines

vmlinux.relocs is one such byproduct.
Another is the map file when CONFIG_VMLINUX_MAP=y is enabled.

The linker map is not depended upon by other build targets, and is typically for human debug consumption, so while not ideal, it is not as broken as for the unstripped vmlinux.

Quoted 4 lines

I am a bit concerned about having more and more
intermediate vmlinux.* files, but maybe only me.
I hope vmlinux.unstripped is the only/last one.

Maybe we should not strip vmlinux at all, but only remove any static relocations when packaging? E.g., tar-pkg, etc

Quoted 1 line

Quoted 4 lines

OK, let's do this.

Ard, please send v2 with .gitignore and 'clean' target updates.

Thanks, I'll prepare a v2, but in the meantime, can we think about not removing the relocations in the first place?

Quoted 4 lines · Masahiro Yamada

I am a bit concerned about having more and more
intermediate vmlinux.* files, but maybe only me.
I hope vmlinux.unstripped is the only/last one.

Maybe we should not strip vmlinux at all, but only remove any static relocations when packaging? E.g., tar-pkg, etc

Quoted 41 lines · Masahiro Yamada

Quoted 39 lines · Ingo Molnar

Quoted 6 lines · Ard Biesheuvel

Quoted 1 line · Masahiro Yamada

...

Quoted 3 lines

I do not think it is broken.
As I mentioned above, I regard vmlinux.relocs as a byproduct of the atomic build rule of vmlinux. This works.

Except when it doesn't work, such as when an intermediate linking step fails, and intermediate build products are lost and cannot be recreated easily (or at all without modifying the source)?

And the thing is, there should be no such thing as an 'atomic build rule of vmlinux' if it means lost information when the build is broken at an intermediate step. What purpose does it have?

Quoted 10 lines

There is no make rule for vmlinux.relocs, and so

- if it gets deleted, it cannot be rebuilt and even though the build does not break, the relocation data is missing from the compressed image, and this could potentially break the kaslr startup code,
- it vmlinux.relocs is older than vmlinux for some reason, make will not notice and silently reuse the outdated version,
- when creating vmlinux.relocs from vmlinux and an error occurs, vmlinux is deleted, making it difficult to diagnose the problem.

Quoted 3 lines

I think this is badly broken, but if you think this is all working as it should, I am not going to debate this further, and you can consider the patch series withdrawn.

That's very sad, as both the simplification is substantial:

  19 files changed, 52 insertions(+), 87 deletions(-)

and the increase in debuggability is substantial as well.

Quoted 8 lines

Quoted 3 lines

Thanks,

Ingo

When a byproduct is accidentally lost
(for example, manually deleted), it is not automatically restored. Running 'rm vmlinux' or 'make clean' is needed.

Exactly. Make cannot detect this situation, and so the build breaks in some way.

Quoted 3 lines

vmlinux.relocs is one such byproduct.
Another is the map file when CONFIG_VMLINUX_MAP=y is enabled.

The linker map is not depended upon by other build targets, and is typically for human debug consumption, so while not ideal, it is not as broken as for the unstripped vmlinux.